From 21d3a8ac2585e5d7bb000624dec04734d0110fab Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Fri, 24 Apr 2026 14:30:17 -0700 Subject: [PATCH 001/115] Added substrait converter for the fragments (#21348) * Added substrait converter for the fragments Signed-off-by: Vinay Krishna Pudyodu * spotless fix Signed-off-by: Vinay Krishna Pudyodu * fixed pr comments Signed-off-by: Vinay Krishna Pudyodu * fix failing test Signed-off-by: Vinay Krishna Pudyodu --------- Signed-off-by: Vinay Krishna Pudyodu --- .../analytics-backend-datafusion/build.gradle | 29 +- .../licenses/core-0.67.0.jar.sha1 | 1 + .../licenses/core-LICENSE.txt | 8 + .../licenses/core-NOTICE.txt | 1 + .../licenses/flatbuffers-java-2.0.0.jar.sha1 | 1 - .../licenses/isthmus-0.67.0.jar.sha1 | 1 + .../licenses/isthmus-LICENSE.txt | 8 + .../licenses/isthmus-NOTICE.txt | 1 + .../jackson-datatype-jdk8-2.21.2.jar.sha1 | 1 + .../jackson-datatype-jdk8-LICENSE.txt | 8 + .../licenses/jackson-datatype-jdk8-NOTICE.txt | 1 + .../DataFusionAnalyticsBackendPlugin.java | 6 + .../DataFusionFragmentConvertor.java | 141 +++++++++ .../be/datafusion/DataFusionPlugin.java | 21 ++ .../DatafusionSearchExecEngine.java | 7 +- .../DataFusionFragmentConvertorTests.java | 281 ++++++++++++++++++ sandbox/plugins/analytics-engine/build.gradle | 22 +- .../licenses/arrow-format-18.1.0.jar.sha1 | 0 .../licenses/arrow-format-LICENSE.txt | 0 .../licenses/arrow-format-NOTICE.txt | 0 .../licenses/flatbuffers-java-2.0.0.jar.sha1 | 1 + .../licenses/flatbuffers-java-LICENSE.txt | 0 .../licenses/flatbuffers-java-NOTICE.txt | 0 .../jackson-annotations-2.21.jar.sha1 | 1 + .../licenses/jackson-annotations-LICENSE.txt | 8 + .../licenses/jackson-annotations-NOTICE.txt | 1 + .../licenses/jackson-databind-2.21.2.jar.sha1 | 1 + .../licenses/jackson-databind-LICENSE.txt | 8 + .../licenses/jackson-databind-NOTICE.txt | 20 ++ .../exec/AnalyticsSearchService.java | 10 +- .../analytics/exec/DefaultPlanExecutor.java | 8 + .../exec/DefaultPlanExecutorTests.java | 137 ++++----- .../dsl/action/TransportDslExecuteAction.java | 34 ++- .../dsl/executor/DslQueryPlanExecutor.java | 18 ++ .../TransportDslExecuteActionTests.java | 20 +- 35 files changed, 690 insertions(+), 115 deletions(-) create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/core-0.67.0.jar.sha1 create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/core-LICENSE.txt create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/core-NOTICE.txt delete mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/flatbuffers-java-2.0.0.jar.sha1 create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.67.0.jar.sha1 create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-LICENSE.txt create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-NOTICE.txt create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.2.jar.sha1 create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-LICENSE.txt create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-NOTICE.txt create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java rename sandbox/plugins/{analytics-backend-datafusion => analytics-engine}/licenses/arrow-format-18.1.0.jar.sha1 (100%) rename sandbox/plugins/{analytics-backend-datafusion => analytics-engine}/licenses/arrow-format-LICENSE.txt (100%) rename sandbox/plugins/{analytics-backend-datafusion => analytics-engine}/licenses/arrow-format-NOTICE.txt (100%) create mode 100644 sandbox/plugins/analytics-engine/licenses/flatbuffers-java-2.0.0.jar.sha1 rename sandbox/plugins/{analytics-backend-datafusion => analytics-engine}/licenses/flatbuffers-java-LICENSE.txt (100%) rename sandbox/plugins/{analytics-backend-datafusion => analytics-engine}/licenses/flatbuffers-java-NOTICE.txt (100%) create mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-annotations-2.21.jar.sha1 create mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-annotations-LICENSE.txt create mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-annotations-NOTICE.txt create mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.2.jar.sha1 create mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-databind-LICENSE.txt create mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-databind-NOTICE.txt diff --git a/sandbox/plugins/analytics-backend-datafusion/build.gradle b/sandbox/plugins/analytics-backend-datafusion/build.gradle index bd13d2b8137ac..74f0948448d13 100644 --- a/sandbox/plugins/analytics-backend-datafusion/build.gradle +++ b/sandbox/plugins/analytics-backend-datafusion/build.gradle @@ -14,6 +14,14 @@ opensearchplugin { java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } +// Guava is forbidden on compile classpaths by OpenSearch. Calcite's TableScan +// references ImmutableList, so tests need it. Bypass via custom configuration. +configurations { + calciteTestCompile + testCompileClasspath { exclude group: 'com.google.guava' } +} +sourceSets.test.compileClasspath += configurations.calciteTestCompile + dependencies { // Shared native bridge lib (provides the unified .so and FFM SymbolLookup) implementation project(':sandbox:libs:dataformat-native') @@ -32,14 +40,22 @@ dependencies { compileOnly "org.apache.arrow:arrow-memory-core:${versions.arrow}" compileOnly "org.apache.arrow:arrow-memory-unsafe:${versions.arrow}" implementation "org.apache.arrow:arrow-c-data:${versions.arrow}" - implementation "org.apache.arrow:arrow-format:${versions.arrow}" - implementation "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" + // Provided at runtime by parent analytics-engine plugin; compileOnly to avoid jar hell. + compileOnly "org.apache.arrow:arrow-format:${versions.arrow}" + compileOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" - // SLF4J and Jackson — provided at runtime by the analytics-engine parent plugin. compileOnly "org.slf4j:slf4j-api:${versions.slf4j}" compileOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + // Provided at runtime by parent analytics-engine plugin; compileOnly to avoid jar hell. compileOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" compileOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + + // Substrait — Calcite RelNode to Substrait plan conversion for DataFusion native runtime + implementation "io.substrait:isthmus:0.67.0" + implementation "io.substrait:core:0.67.0" + implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}" + + calciteTestCompile "com.google.guava:guava:${versions.guava}" } test { @@ -54,6 +70,13 @@ tasks.withType(JavaCompile).configureEach { options.compilerArgs -= '-Werror' } +tasks.named('thirdPartyAudit').configure { + ignoreMissingClasses( + // Optional Calcite DDL parser — lives in calcite-server module, not needed at runtime + 'org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl' + ) +} + // TODO: Remove once back-end is built out with test suite testingConventions.enabled = false diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/core-0.67.0.jar.sha1 b/sandbox/plugins/analytics-backend-datafusion/licenses/core-0.67.0.jar.sha1 new file mode 100644 index 0000000000000..f0389973b3195 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/core-0.67.0.jar.sha1 @@ -0,0 +1 @@ +e843a3f25fadaf71c5743ccfcf3d2f28587354de \ No newline at end of file diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/core-LICENSE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/core-LICENSE.txt new file mode 100644 index 0000000000000..f5f45d26a49d6 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/core-LICENSE.txt @@ -0,0 +1,8 @@ +This copy of Jackson JSON processor streaming parser/generator is licensed under the +Apache (Software) License, version 2.0 ("the License"). +See the License for details about distribution rights, and the +specific rights regarding derivate works. + +You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/core-NOTICE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/core-NOTICE.txt new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/core-NOTICE.txt @@ -0,0 +1 @@ + diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/flatbuffers-java-2.0.0.jar.sha1 b/sandbox/plugins/analytics-backend-datafusion/licenses/flatbuffers-java-2.0.0.jar.sha1 deleted file mode 100644 index ed9f08036de5a..0000000000000 --- a/sandbox/plugins/analytics-backend-datafusion/licenses/flatbuffers-java-2.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19da0c1d9f585d2c402057f993f8dea2ff382837 diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.67.0.jar.sha1 b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.67.0.jar.sha1 new file mode 100644 index 0000000000000..fd5b6b13576b7 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.67.0.jar.sha1 @@ -0,0 +1 @@ +3deab1e77fa2549a7ce2d35f8961e4c7bbe30644 \ No newline at end of file diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-LICENSE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-LICENSE.txt new file mode 100644 index 0000000000000..f5f45d26a49d6 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-LICENSE.txt @@ -0,0 +1,8 @@ +This copy of Jackson JSON processor streaming parser/generator is licensed under the +Apache (Software) License, version 2.0 ("the License"). +See the License for details about distribution rights, and the +specific rights regarding derivate works. + +You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-NOTICE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-NOTICE.txt new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-NOTICE.txt @@ -0,0 +1 @@ + diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.2.jar.sha1 b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.2.jar.sha1 new file mode 100644 index 0000000000000..ee9b80f634c10 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.2.jar.sha1 @@ -0,0 +1 @@ +0222d829578930f27afa8d9b6842393824e2eb38 \ No newline at end of file diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-LICENSE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-LICENSE.txt new file mode 100644 index 0000000000000..f5f45d26a49d6 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-LICENSE.txt @@ -0,0 +1,8 @@ +This copy of Jackson JSON processor streaming parser/generator is licensed under the +Apache (Software) License, version 2.0 ("the License"). +See the License for details about distribution rights, and the +specific rights regarding derivate works. + +You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-NOTICE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-NOTICE.txt new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-NOTICE.txt @@ -0,0 +1 @@ + diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 2d86c3390d868..0acb0c956a55f 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -16,6 +16,7 @@ import org.opensearch.analytics.spi.FieldType; import org.opensearch.analytics.spi.FilterCapability; import org.opensearch.analytics.spi.FilterOperator; +import org.opensearch.analytics.spi.FragmentConvertor; import org.opensearch.analytics.spi.ScanCapability; import org.opensearch.analytics.spi.SearchExecEngineProvider; import org.opensearch.index.engine.dataformat.DataFormatRegistry; @@ -118,6 +119,11 @@ public Set aggregateCapabilities() { }; } + @Override + public FragmentConvertor getFragmentConvertor() { + return new DataFusionFragmentConvertor(plugin.getSubstraitExtensions()); + } + @Override public SearchExecEngineProvider getSearchExecEngineProvider() { return ctx -> { diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java new file mode 100644 index 0000000000000..6170da33dcd1d --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -0,0 +1,141 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlKind; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.analytics.spi.FragmentConvertor; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import io.substrait.extension.SimpleExtension; +import io.substrait.isthmus.ImmutableFeatureBoard; +import io.substrait.isthmus.SubstraitRelVisitor; +import io.substrait.isthmus.TypeConverter; +import io.substrait.isthmus.expression.AggregateFunctionConverter; +import io.substrait.isthmus.expression.ScalarFunctionConverter; +import io.substrait.isthmus.expression.WindowFunctionConverter; +import io.substrait.plan.Plan; +import io.substrait.plan.PlanProtoConverter; +import io.substrait.relation.NamedScan; +import io.substrait.relation.Rel; +import io.substrait.relation.RelCopyOnWriteVisitor; +import io.substrait.util.EmptyVisitationContext; + +/** + * Converts Calcite RelNode fragments to Substrait protobuf bytes + * for the DataFusion Rust runtime. + * + * @opensearch.internal + */ +public class DataFusionFragmentConvertor implements FragmentConvertor { + + private static final Logger LOGGER = LogManager.getLogger(DataFusionFragmentConvertor.class); + + private final SimpleExtension.ExtensionCollection extensions; + + public DataFusionFragmentConvertor(SimpleExtension.ExtensionCollection extensions) { + this.extensions = extensions; + } + + @Override + public byte[] convertShardScanFragment(String tableName, RelNode fragment) { + LOGGER.debug("Converting shard scan fragment for table [{}]", tableName); + return convertToSubstrait(fragment); + } + + @Override + public byte[] attachPartialAggOnTop(RelNode partialAggFragment, byte[] innerBytes) { + throw new UnsupportedOperationException("Multi-stage partial aggregate not yet implemented"); + } + + @Override + public byte[] convertFinalAggFragment(RelNode fragment) { + throw new UnsupportedOperationException("Multi-stage final aggregate not yet implemented"); + } + + @Override + public byte[] attachFragmentOnTop(RelNode fragment, byte[] innerBytes) { + throw new UnsupportedOperationException("Multi-stage fragment attachment not yet implemented"); + } + + private byte[] convertToSubstrait(RelNode fragment) { + RelRoot root = RelRoot.of(fragment, SqlKind.SELECT); + SubstraitRelVisitor visitor = createVisitor(fragment); + Rel substraitRel = visitor.apply(root.rel); + + List fieldNames = root.fields.stream().map(field -> field.getValue()).toList(); + + Plan.Root substraitRoot = Plan.Root.builder().input(substraitRel).names(fieldNames).build(); + Plan plan = Plan.builder().addRoots(substraitRoot).build(); + + plan = new TableNameModifier().modifyTableNames(plan); + + io.substrait.proto.Plan protoPlan = new PlanProtoConverter().toProto(plan); + byte[] bytes = protoPlan.toByteArray(); + LOGGER.debug("Substrait plan: {} bytes", bytes.length); + return bytes; + } + + private SubstraitRelVisitor createVisitor(RelNode relNode) { + RelDataTypeFactory typeFactory = relNode.getCluster().getTypeFactory(); + TypeConverter typeConverter = TypeConverter.DEFAULT; + + AggregateFunctionConverter aggConverter = new AggregateFunctionConverter(extensions.aggregateFunctions(), typeFactory); + ScalarFunctionConverter scalarConverter = new ScalarFunctionConverter( + extensions.scalarFunctions(), + List.of(), + typeFactory, + typeConverter + ); + WindowFunctionConverter windowConverter = new WindowFunctionConverter(extensions.windowFunctions(), typeFactory); + + return new SubstraitRelVisitor( + typeFactory, + scalarConverter, + aggConverter, + windowConverter, + typeConverter, + ImmutableFeatureBoard.builder().build() + ); + } + + private static class TableNameModifier { + Plan modifyTableNames(Plan plan) { + TableNameVisitor visitor = new TableNameVisitor(); + List modifiedRoots = new ArrayList<>(); + for (Plan.Root root : plan.getRoots()) { + Optional modifiedRel = root.getInput().accept(visitor, null); + if (modifiedRel.isPresent()) { + modifiedRoots.add(Plan.Root.builder().from(root).input(modifiedRel.get()).build()); + } else { + modifiedRoots.add(root); + } + } + return Plan.builder().from(plan).roots(modifiedRoots).build(); + } + + private static class TableNameVisitor extends RelCopyOnWriteVisitor { + @Override + public Optional visit(NamedScan namedScan, EmptyVisitationContext context) { + List names = namedScan.getNames(); + if (names.size() > 1) { + return Optional.of(NamedScan.builder().from(namedScan).names(List.of(names.get(names.size() - 1))).build()); + } + return super.visit(namedScan, context); + } + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java index 8175feb7b9940..dbeb46049ac98 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java @@ -35,6 +35,9 @@ import java.util.List; import java.util.function.Supplier; +import io.substrait.extension.DefaultExtensionCatalog; +import io.substrait.extension.SimpleExtension; + /** * Main plugin class for the DataFusion native engine integration. *

@@ -66,6 +69,7 @@ public class DataFusionPlugin extends Plugin implements SearchBackEndPlugin createComponents( dataFusionService.start(); logger.debug("DataFusion plugin initialized — memory pool {}B, spill limit {}B", memoryPoolLimit, spillMemoryLimit); + this.substraitExtensions = loadSubstraitExtensions(); + return Collections.singletonList(dataFusionService); } + private static SimpleExtension.ExtensionCollection loadSubstraitExtensions() { + Thread t = Thread.currentThread(); + ClassLoader previous = t.getContextClassLoader(); + try { + t.setContextClassLoader(DataFusionPlugin.class.getClassLoader()); + return DefaultExtensionCatalog.DEFAULT_COLLECTION; + } finally { + t.setContextClassLoader(previous); + } + } + + SimpleExtension.ExtensionCollection getSubstraitExtensions() { + return substraitExtensions; + } + DataFormatRegistry getDataFormatRegistry() { return dataFormatRegistry; } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearchExecEngine.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearchExecEngine.java index b906f3ec1c25b..eb3b33848c455 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearchExecEngine.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearchExecEngine.java @@ -9,6 +9,8 @@ package org.opensearch.be.datafusion; import org.apache.arrow.memory.BufferAllocator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.analytics.backend.EngineResultStream; import org.opensearch.analytics.backend.ExecutionContext; import org.opensearch.analytics.backend.SearchExecEngine; @@ -28,6 +30,8 @@ @ExperimentalApi public class DatafusionSearchExecEngine implements SearchExecEngine { + private static final Logger logger = LogManager.getLogger(DatafusionSearchExecEngine.class); + private final DatafusionContext datafusionContext; private final Supplier allocatorFactory; @@ -43,8 +47,7 @@ public DatafusionSearchExecEngine(DatafusionContext datafusionContext, Supplier< @Override public void prepare(ExecutionContext requestContext) { - // TODO: wire Substrait conversion (RelNode → Substrait bytes) - byte[] substraitBytes = null; + byte[] substraitBytes = requestContext.getFragmentBytes(); long contextId = datafusionContext.task() != null ? datafusionContext.task().getId() : 0L; datafusionContext.setDatafusionQuery(new DatafusionQuery(requestContext.getTableName(), substraitBytes, contextId)); } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java new file mode 100644 index 0000000000000..ff1666277fee4 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java @@ -0,0 +1,281 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptSchema; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelDistribution; +import org.apache.calcite.rel.RelDistributions; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelReferentialConstraint; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.schema.ColumnStrategy; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.ImmutableBitSet; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +import io.substrait.extension.DefaultExtensionCatalog; +import io.substrait.extension.SimpleExtension; +import io.substrait.proto.FilterRel; +import io.substrait.proto.Plan; +import io.substrait.proto.PlanRel; +import io.substrait.proto.ReadRel; +import io.substrait.proto.Rel; + +/** + * Tests for {@link DataFusionFragmentConvertor}. + * Builds Calcite RelNode trees in-process, converts them via the convertor, + * then decodes the returned Substrait proto bytes and asserts on proto shape. + * + *

TODO: Add end-to-end tests that feed the generated Substrait bytes into + * the actual DataFusion native runtime (see {@link DataFusionQueryExecutionTests} + * for the pattern) to verify the plans execute correctly, not just that the + * proto structure is well-formed. + */ +public class DataFusionFragmentConvertorTests extends OpenSearchTestCase { + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private RelOptCluster cluster; + private SimpleExtension.ExtensionCollection extensions; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + cluster = RelOptCluster.create(planner, rexBuilder); + Thread t = Thread.currentThread(); + ClassLoader prev = t.getContextClassLoader(); + try { + t.setContextClassLoader(DataFusionFragmentConvertorTests.class.getClassLoader()); + extensions = DefaultExtensionCatalog.DEFAULT_COLLECTION; + } finally { + t.setContextClassLoader(prev); + } + } + + // ── Helpers ───────────────────────────────────────────────────────────────── + + private RelDataType rowType(String... columns) { + RelDataTypeFactory.Builder b = typeFactory.builder(); + for (String c : columns) { + b.add(c, typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.INTEGER), true)); + } + return b.build(); + } + + private RelNode buildTableScan(String tableName, String... columns) { + return new MinimalTableScan(cluster, cluster.traitSet(), tableName, rowType(columns)); + } + + private DataFusionFragmentConvertor newConvertor() { + return new DataFusionFragmentConvertor(extensions); + } + + private Plan decodeSubstrait(byte[] bytes) throws Exception { + assertNotNull(bytes); + assertTrue(bytes.length > 0); + return Plan.parseFrom(bytes); + } + + private Rel rootRel(Plan plan) { + assertFalse(plan.getRelationsList().isEmpty()); + PlanRel planRel = plan.getRelationsList().get(0); + assertTrue(planRel.hasRoot()); + return planRel.getRoot().getInput(); + } + + // ── Tests ────────────────────────────────────────────────────────────────── + + public void testConvertShardScanFragmentTableScan() throws Exception { + RelNode scan = buildTableScan("test_index", "A", "B"); + byte[] bytes = newConvertor().convertShardScanFragment("test_index", scan); + + Plan plan = decodeSubstrait(bytes); + Rel root = rootRel(plan); + assertTrue(root.hasRead()); + ReadRel read = root.getRead(); + assertTrue(read.hasNamedTable()); + assertEquals(List.of("test_index"), read.getNamedTable().getNamesList()); + } + + public void testConvertShardScanFragmentFilterOverScan() throws Exception { + RelNode scan = buildTableScan("test_index", "A", "B"); + RexNode predicate = rexBuilder.makeCall( + SqlStdOperatorTable.GREATER_THAN, + rexBuilder.makeInputRef(scan, 0), + rexBuilder.makeLiteral(10, typeFactory.createSqlType(SqlTypeName.INTEGER), true) + ); + RelNode filter = LogicalFilter.create(scan, predicate); + + byte[] bytes = newConvertor().convertShardScanFragment("test_index", filter); + + Plan plan = decodeSubstrait(bytes); + Rel root = rootRel(plan); + assertTrue(root.hasFilter()); + FilterRel filterRel = root.getFilter(); + assertTrue(filterRel.hasCondition()); + Rel inner = filterRel.getInput(); + assertTrue(inner.hasRead()); + assertEquals(List.of("test_index"), inner.getRead().getNamedTable().getNamesList()); + } + + public void testConvertShardScanFragmentProducesNonEmptyBytes() throws Exception { + RelNode scan = buildTableScan("my_index", "col1"); + byte[] bytes = newConvertor().convertShardScanFragment("my_index", scan); + + assertNotNull(bytes); + assertTrue(bytes.length > 0); + } + + public void testConvertShardScanFragmentStripsTableNamePrefix() throws Exception { + RelNode scan = new MinimalTableScan(cluster, cluster.traitSet(), List.of("opensearch", "schema", "my_table"), rowType("X")); + byte[] bytes = newConvertor().convertShardScanFragment("my_table", scan); + + Plan plan = decodeSubstrait(bytes); + Rel root = rootRel(plan); + assertTrue(root.hasRead()); + assertEquals(List.of("my_table"), root.getRead().getNamedTable().getNamesList()); + } + + public void testAttachPartialAggOnTopThrowsUnsupported() { + RelNode scan = buildTableScan("t", "A"); + expectThrows(UnsupportedOperationException.class, () -> newConvertor().attachPartialAggOnTop(scan, new byte[0])); + } + + public void testConvertFinalAggFragmentThrowsUnsupported() { + RelNode scan = buildTableScan("t", "A"); + expectThrows(UnsupportedOperationException.class, () -> newConvertor().convertFinalAggFragment(scan)); + } + + public void testAttachFragmentOnTopThrowsUnsupported() { + RelNode scan = buildTableScan("t", "A"); + expectThrows(UnsupportedOperationException.class, () -> newConvertor().attachFragmentOnTop(scan, new byte[0])); + } + + public void testExtensionCollectionReuse() throws Exception { + DataFusionFragmentConvertor convertor = newConvertor(); + RelNode scan = buildTableScan("t", "A"); + byte[] first = convertor.convertShardScanFragment("t", scan); + byte[] second = convertor.convertShardScanFragment("t", scan); + assertNotNull(first); + assertNotNull(second); + assertArrayEquals(first, second); + } + + // ── Minimal Calcite stubs for building test RelNodes ───────────────────────── + + static final class MinimalTableScan extends TableScan { + MinimalTableScan(RelOptCluster cluster, RelTraitSet traitSet, String tableName, RelDataType rowType) { + this(cluster, traitSet, List.of(tableName), rowType); + } + + MinimalTableScan(RelOptCluster cluster, RelTraitSet traitSet, List qualifiedName, RelDataType rowType) { + super(cluster, traitSet, List.of(), new MinimalRelOptTable(qualifiedName, rowType)); + } + } + + static final class MinimalRelOptTable implements RelOptTable { + private final List qualifiedName; + private final RelDataType rowType; + + MinimalRelOptTable(List qualifiedName, RelDataType rowType) { + this.qualifiedName = qualifiedName; + this.rowType = rowType; + } + + @Override + public List getQualifiedName() { + return qualifiedName; + } + + @Override + public RelDataType getRowType() { + return rowType; + } + + @Override + public double getRowCount() { + return 100; + } + + @Override + public RelOptSchema getRelOptSchema() { + return null; + } + + @Override + public RelNode toRel(ToRelContext context) { + throw new UnsupportedOperationException(); + } + + @Override + public List getColumnStrategies() { + return List.of(); + } + + @Override + public C unwrap(Class aClass) { + return null; + } + + @Override + public boolean isKey(ImmutableBitSet columns) { + return false; + } + + @Override + public List getKeys() { + return List.of(); + } + + @Override + public List getReferentialConstraints() { + return List.of(); + } + + @Override + public List getCollationList() { + return List.of(); + } + + @Override + public RelDistribution getDistribution() { + return RelDistributions.ANY; + } + + @Override + @SuppressWarnings("rawtypes") + public org.apache.calcite.linq4j.tree.Expression getExpression(Class clazz) { + return null; + } + + @Override + public RelOptTable extend(List extendedFields) { + return this; + } + } +} diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle index 41ff4c9ef9b58..451a3f767064a 100644 --- a/sandbox/plugins/analytics-engine/build.gradle +++ b/sandbox/plugins/analytics-engine/build.gradle @@ -69,12 +69,10 @@ dependencies { transitive = false } - // Arrow Flight types reference these at compile/javadoc time. compileOnly so they - // don't end up in the zip — arrow-flight-rpc provides them at runtime. - compileOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" - compileOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" - compileOnly "org.apache.arrow:arrow-format:${versions.arrow}" - compileOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" + // Required at runtime by arrow-vector (Schema, ArrowType subclasses reference jackson). + // implementation (not compileOnly) so bundlePlugin doesn't strip them. + implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" // Guava — required at compile time because Calcite base classes expose guava types. // Uses custom config to bypass forbidden-dependencies.gradle check on compileClasspath. @@ -101,10 +99,9 @@ dependencies { // discovers via ServiceLoader at runtime. Must be in the parent plugin's classloader // because BaseAllocator (from arrow-memory-core) does the ServiceLoader lookup. runtimeOnly "org.apache.arrow:arrow-memory-unsafe:${versions.arrow}" - // arrow-format + flatbuffers-java satisfy Arrow's IPC Schema serialization path - // that some VectorSchemaRoot operations invoke transitively. - testRuntimeOnly "org.apache.arrow:arrow-format:${versions.arrow}" - testRuntimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" + // Required at runtime by arrow-vector (ArrowType subclasses reference flatbuffers and arrow-format). + runtimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" + runtimeOnly "org.apache.arrow:arrow-format:${versions.arrow}" // SQL Unified Query API for PPL parsing testImplementation("org.opensearch.query:unified-query-api:${sqlUnifiedQueryVersion}") { @@ -120,11 +117,6 @@ dependencies { // Arrow Flight streaming transport for ITs internalClusterTestImplementation project(':plugins:arrow-flight-rpc') - // jackson-annotations — required at runtime by jackson-databind (transitive via Calcite). - // Without this, child plugins that use Arrow's Schema (which triggers ObjectMapper init) - // fail with NoClassDefFoundError for JsonSerializeAs. - runtimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" - // Calcite bytecode references @Immutable from immutables — resolve at test compile time testCompileOnly 'org.immutables:value-annotations:2.8.8' } diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/arrow-format-18.1.0.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/arrow-format-18.1.0.jar.sha1 similarity index 100% rename from sandbox/plugins/analytics-backend-datafusion/licenses/arrow-format-18.1.0.jar.sha1 rename to sandbox/plugins/analytics-engine/licenses/arrow-format-18.1.0.jar.sha1 diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/arrow-format-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/arrow-format-LICENSE.txt similarity index 100% rename from sandbox/plugins/analytics-backend-datafusion/licenses/arrow-format-LICENSE.txt rename to sandbox/plugins/analytics-engine/licenses/arrow-format-LICENSE.txt diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/arrow-format-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/arrow-format-NOTICE.txt similarity index 100% rename from sandbox/plugins/analytics-backend-datafusion/licenses/arrow-format-NOTICE.txt rename to sandbox/plugins/analytics-engine/licenses/arrow-format-NOTICE.txt diff --git a/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-2.0.0.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-2.0.0.jar.sha1 new file mode 100644 index 0000000000000..08a9b7e399e74 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-2.0.0.jar.sha1 @@ -0,0 +1 @@ +19da0c1d9f585d2c402057f993f8dea2ff382837 \ No newline at end of file diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/flatbuffers-java-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-LICENSE.txt similarity index 100% rename from sandbox/plugins/analytics-backend-datafusion/licenses/flatbuffers-java-LICENSE.txt rename to sandbox/plugins/analytics-engine/licenses/flatbuffers-java-LICENSE.txt diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/flatbuffers-java-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-NOTICE.txt similarity index 100% rename from sandbox/plugins/analytics-backend-datafusion/licenses/flatbuffers-java-NOTICE.txt rename to sandbox/plugins/analytics-engine/licenses/flatbuffers-java-NOTICE.txt diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-annotations-2.21.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/jackson-annotations-2.21.jar.sha1 new file mode 100644 index 0000000000000..081172e4c72c3 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/jackson-annotations-2.21.jar.sha1 @@ -0,0 +1 @@ +b1bc1868bf02dc0bd6c7836257a036a331005309 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-annotations-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/jackson-annotations-LICENSE.txt new file mode 100644 index 0000000000000..f5f45d26a49d6 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/jackson-annotations-LICENSE.txt @@ -0,0 +1,8 @@ +This copy of Jackson JSON processor streaming parser/generator is licensed under the +Apache (Software) License, version 2.0 ("the License"). +See the License for details about distribution rights, and the +specific rights regarding derivate works. + +You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-annotations-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/jackson-annotations-NOTICE.txt new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/jackson-annotations-NOTICE.txt @@ -0,0 +1 @@ + diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.2.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.2.jar.sha1 new file mode 100644 index 0000000000000..52686081905c0 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.2.jar.sha1 @@ -0,0 +1 @@ +71ab8ff75b4fd74afdee0004173fdd15de1d6a28 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-databind-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/jackson-databind-LICENSE.txt new file mode 100644 index 0000000000000..f5f45d26a49d6 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/jackson-databind-LICENSE.txt @@ -0,0 +1,8 @@ +This copy of Jackson JSON processor streaming parser/generator is licensed under the +Apache (Software) License, version 2.0 ("the License"). +See the License for details about distribution rights, and the +specific rights regarding derivate works. + +You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-databind-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/jackson-databind-NOTICE.txt new file mode 100644 index 0000000000000..4c976b7b4cc58 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/jackson-databind-NOTICE.txt @@ -0,0 +1,20 @@ +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers, as well as supported +commercially by FasterXML.com. + +## Licensing + +Jackson core and extension components may licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java index 3d6ba2abb7570..43c6828083f06 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java @@ -21,7 +21,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.core.tasks.TaskCancelledException; -import org.opensearch.index.engine.DataFormatAwareEngine; +import org.opensearch.index.engine.exec.IndexReaderProvider; import org.opensearch.index.engine.exec.IndexReaderProvider.Reader; import org.opensearch.index.shard.IndexShard; @@ -76,9 +76,9 @@ public FragmentExecutionResponse executeFragment(FragmentExecutionRequest reques * @return a response containing field names and result rows */ public FragmentExecutionResponse executeFragment(FragmentExecutionRequest request, IndexShard shard, AnalyticsShardTask task) { - DataFormatAwareEngine compositeEngine = shard.getCompositeEngine(); - if (compositeEngine == null) { - throw new IllegalStateException("No CompositeEngine on " + shard.shardId()); + IndexReaderProvider readerProvider = shard.getReaderProvider(); + if (readerProvider == null) { + throw new IllegalStateException("No ReaderProvider on " + shard.shardId()); } // Select the first available plan alternative whose backend is registered on this node. @@ -106,7 +106,7 @@ public FragmentExecutionResponse executeFragment(FragmentExecutionRequest reques listener.onPreFragmentExecution(queryId, stageId, shardIdStr); long startNanos = System.nanoTime(); - try (GatedCloseable gatedReader = compositeEngine.acquireReader()) { + try (GatedCloseable gatedReader = readerProvider.acquireReader()) { SearchShardTask searchShardTask = null; // TODO: real task for cancellation ExecutionContext ctx = new ExecutionContext(request.getShardId().getIndexName(), searchShardTask, gatedReader.get()); ctx.setFragmentBytes(selectedPlan.getFragmentBytes()); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java index 252e9179af6ab..0d3752a976d2c 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java @@ -24,6 +24,8 @@ import org.opensearch.analytics.planner.PlannerContext; import org.opensearch.analytics.planner.PlannerImpl; import org.opensearch.analytics.planner.dag.DAGBuilder; +import org.opensearch.analytics.planner.dag.FragmentConversionDriver; +import org.opensearch.analytics.planner.dag.PlanForker; import org.opensearch.analytics.planner.dag.QueryDAG; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; @@ -93,10 +95,15 @@ public DefaultPlanExecutor( this.scheduler = scheduler; } + // TODO: Extract plan → optimize → fork → convert → DAG into a dedicated component (e.g. QueryDAGBuilder) + // that takes the logical fragment and returns a fully-built DAG ready for scheduling. + // Also add per-step timing (plan, fork, convert, schedule, execute) for observability. @Override public Iterable execute(RelNode logicalFragment, Object context) { RelNode plan = PlannerImpl.createPlan(logicalFragment, new PlannerContext(capabilityRegistry, clusterService.state())); QueryDAG dag = DAGBuilder.build(plan, capabilityRegistry, clusterService); + PlanForker.forkAll(dag, capabilityRegistry); + FragmentConversionDriver.convertAll(dag, capabilityRegistry); logger.info("[DefaultPlanExecutor] QueryDAG:\n{}", dag); // Register coordinator-level query task with TaskManager (like SearchTask). @@ -196,6 +203,7 @@ static Iterable batchesToRows(Iterable batches) { } rows.add(row); } + batch.close(); } return rows; } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java index d77b4691260d3..761b683c54982 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java @@ -57,34 +57,24 @@ public void testBatchesToRowsEmpty() { public void testBatchesToRowsSingleBatchIntegers() { VectorSchemaRoot batch = makeIntBatch("x", 10, 20, 30); - try { - List rows = toList(DefaultPlanExecutor.batchesToRows(List.of(batch))); - assertEquals(3, rows.size()); - assertArrayEquals(new Object[] { 10 }, rows.get(0)); - assertArrayEquals(new Object[] { 20 }, rows.get(1)); - assertArrayEquals(new Object[] { 30 }, rows.get(2)); - } finally { - batch.close(); - } + List rows = toList(DefaultPlanExecutor.batchesToRows(List.of(batch))); + assertEquals(3, rows.size()); + assertArrayEquals(new Object[] { 10 }, rows.get(0)); + assertArrayEquals(new Object[] { 20 }, rows.get(1)); + assertArrayEquals(new Object[] { 30 }, rows.get(2)); } public void testBatchesToRowsMultipleBatchesPreservesOrder() { VectorSchemaRoot batch1 = makeIntBatch("x", 1, 2); VectorSchemaRoot batch2 = makeIntBatch("x", 3); VectorSchemaRoot batch3 = makeIntBatch("x", 4, 5); - try { - List rows = toList(DefaultPlanExecutor.batchesToRows(List.of(batch1, batch2, batch3))); - assertEquals(5, rows.size()); - assertEquals(1, rows.get(0)[0]); - assertEquals(2, rows.get(1)[0]); - assertEquals(3, rows.get(2)[0]); - assertEquals(4, rows.get(3)[0]); - assertEquals(5, rows.get(4)[0]); - } finally { - batch1.close(); - batch2.close(); - batch3.close(); - } + List rows = toList(DefaultPlanExecutor.batchesToRows(List.of(batch1, batch2, batch3))); + assertEquals(5, rows.size()); + assertEquals(1, rows.get(0)[0]); + assertEquals(2, rows.get(1)[0]); + assertEquals(3, rows.get(2)[0]); + assertEquals(4, rows.get(3)[0]); + assertEquals(5, rows.get(4)[0]); } public void testBatchesToRowsMultipleColumns() { @@ -95,25 +85,21 @@ public void testBatchesToRowsMultipleColumns() { ) ); VectorSchemaRoot batch = VectorSchemaRoot.create(schema, allocator); - try { - batch.allocateNew(); - BigIntVector ids = (BigIntVector) batch.getVector(0); - VarCharVector names = (VarCharVector) batch.getVector(1); - ids.setSafe(0, 100L); - ids.setSafe(1, 200L); - names.setSafe(0, "alice".getBytes(StandardCharsets.UTF_8)); - names.setSafe(1, "bob".getBytes(StandardCharsets.UTF_8)); - batch.setRowCount(2); - - List rows = toList(DefaultPlanExecutor.batchesToRows(List.of(batch))); - assertEquals(2, rows.size()); - assertEquals(100L, rows.get(0)[0]); - assertEquals("alice", rows.get(0)[1]); - assertEquals(200L, rows.get(1)[0]); - assertEquals("bob", rows.get(1)[1]); - } finally { - batch.close(); - } + batch.allocateNew(); + BigIntVector ids = (BigIntVector) batch.getVector(0); + VarCharVector names = (VarCharVector) batch.getVector(1); + ids.setSafe(0, 100L); + ids.setSafe(1, 200L); + names.setSafe(0, "alice".getBytes(StandardCharsets.UTF_8)); + names.setSafe(1, "bob".getBytes(StandardCharsets.UTF_8)); + batch.setRowCount(2); + + List rows = toList(DefaultPlanExecutor.batchesToRows(List.of(batch))); + assertEquals(2, rows.size()); + assertEquals(100L, rows.get(0)[0]); + assertEquals("alice", rows.get(0)[1]); + assertEquals(200L, rows.get(1)[0]); + assertEquals("bob", rows.get(1)[1]); } public void testBatchesToRowsHandlesNulls() { @@ -121,22 +107,18 @@ public void testBatchesToRowsHandlesNulls() { new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(32, true)), null))), allocator ); - try { - batch.allocateNew(); - IntVector vec = (IntVector) batch.getVector(0); - vec.setSafe(0, 1); - vec.setNull(1); - vec.setSafe(2, 3); - batch.setRowCount(3); - - List rows = toList(DefaultPlanExecutor.batchesToRows(List.of(batch))); - assertEquals(3, rows.size()); - assertEquals(1, rows.get(0)[0]); - assertNull(rows.get(1)[0]); - assertEquals(3, rows.get(2)[0]); - } finally { - batch.close(); - } + batch.allocateNew(); + IntVector vec = (IntVector) batch.getVector(0); + vec.setSafe(0, 1); + vec.setNull(1); + vec.setSafe(2, 3); + batch.setRowCount(3); + + List rows = toList(DefaultPlanExecutor.batchesToRows(List.of(batch))); + assertEquals(3, rows.size()); + assertEquals(1, rows.get(0)[0]); + assertNull(rows.get(1)[0]); + assertEquals(3, rows.get(2)[0]); } public void testBatchesToRowsVarCharDecodedAsString() { @@ -144,29 +126,38 @@ public void testBatchesToRowsVarCharDecodedAsString() { new Schema(List.of(new Field("s", FieldType.nullable(ArrowType.Utf8.INSTANCE), null))), allocator ); - try { - batch.allocateNew(); - VarCharVector vec = (VarCharVector) batch.getVector(0); - vec.setSafe(0, "hello".getBytes(StandardCharsets.UTF_8)); - vec.setSafe(1, "world".getBytes(StandardCharsets.UTF_8)); - batch.setRowCount(2); - - List rows = toList(DefaultPlanExecutor.batchesToRows(List.of(batch))); - assertEquals("hello", rows.get(0)[0]); - assertEquals("world", rows.get(1)[0]); - // explicit type check — we return String, not the raw Text the underlying getObject returns - assertTrue(rows.get(0)[0] instanceof String); - } finally { - batch.close(); - } + batch.allocateNew(); + VarCharVector vec = (VarCharVector) batch.getVector(0); + vec.setSafe(0, "hello".getBytes(StandardCharsets.UTF_8)); + vec.setSafe(1, "world".getBytes(StandardCharsets.UTF_8)); + batch.setRowCount(2); + + List rows = toList(DefaultPlanExecutor.batchesToRows(List.of(batch))); + assertEquals("hello", rows.get(0)[0]); + assertEquals("world", rows.get(1)[0]); + assertTrue(rows.get(0)[0] instanceof String); + } + + public void testBatchesToRowsClosesBatches() { + BufferAllocator child = allocator.newChildAllocator("test", 0, Long.MAX_VALUE); + VectorSchemaRoot batch = makeIntBatch(child, "x", 1, 2); + long before = child.getAllocatedMemory(); + assertTrue("batch should hold allocated memory", before > 0); + DefaultPlanExecutor.batchesToRows(List.of(batch)); + assertEquals("batch buffers should be released after batchesToRows", 0, child.getAllocatedMemory()); + child.close(); } // ── helpers ────────────────────────────────────────────────────────── private VectorSchemaRoot makeIntBatch(String fieldName, int... values) { + return makeIntBatch(allocator, fieldName, values); + } + + private VectorSchemaRoot makeIntBatch(BufferAllocator alloc, String fieldName, int... values) { Field field = new Field(fieldName, FieldType.nullable(new ArrowType.Int(32, true)), null); Schema schema = new Schema(List.of(field)); - VectorSchemaRoot vsr = VectorSchemaRoot.create(schema, allocator); + VectorSchemaRoot vsr = VectorSchemaRoot.create(schema, alloc); vsr.allocateNew(); IntVector vec = (IntVector) vsr.getVector(0); for (int i = 0; i < values.length; i++) { diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java index 80bbd35852f6e..f3e0cba19b228 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java @@ -28,6 +28,7 @@ import org.opensearch.dsl.result.ExecutionResult; import org.opensearch.dsl.result.SearchResponseBuilder; import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import java.util.List; @@ -47,6 +48,7 @@ public class TransportDslExecuteAction extends HandledTransportAction> executor, ClusterService clusterService, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + ThreadPool threadPool ) { super(DslExecuteAction.NAME, transportService, actionFilters, SearchRequest::new); this.engineContext = engineContext; this.planExecutor = new DslQueryPlanExecutor(executor); this.clusterService = clusterService; this.indexNameExpressionResolver = indexNameExpressionResolver; + this.threadPool = threadPool; } @Override protected void doExecute(Task task, SearchRequest request, ActionListener listener) { - try { - String indexName = resolveToSingleIndex(request); + threadPool.executor(ThreadPool.Names.SEARCH).execute(() -> { + try { + String indexName = resolveToSingleIndex(request); - long convertStart = System.nanoTime(); - SearchSourceConverter converter = new SearchSourceConverter(engineContext.getSchema()); - QueryPlans plans = converter.convert(request.source(), indexName); - long convertTime = System.nanoTime() - convertStart; - List results = planExecutor.execute(plans); - SearchResponse response = SearchResponseBuilder.build(results, convertTime); - listener.onResponse(response); - } catch (Exception e) { - logger.error("DSL execution failed", e); - listener.onFailure(e); - } + long convertStart = System.nanoTime(); + SearchSourceConverter converter = new SearchSourceConverter(engineContext.getSchema()); + QueryPlans plans = converter.convert(request.source(), indexName); + long convertTime = System.nanoTime() - convertStart; + List results = planExecutor.execute(plans); + SearchResponse response = SearchResponseBuilder.build(results, convertTime); + listener.onResponse(response); + } catch (Exception e) { + logger.error("DSL execution failed", e); + listener.onFailure(e); + } + }); } // TODO: Consider delegating index resolution to Analytics Core plugin (e.g. via diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/executor/DslQueryPlanExecutor.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/executor/DslQueryPlanExecutor.java index 92656b3cbfad7..80ffbdeb0fc05 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/executor/DslQueryPlanExecutor.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/executor/DslQueryPlanExecutor.java @@ -15,6 +15,7 @@ import org.opensearch.dsl.result.ExecutionResult; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** @@ -53,12 +54,29 @@ public List execute(QueryPlans plans) { logPlan(relNode); // TODO: context param is null, may carry execution hints Iterable rows = executor.execute(relNode, null); + logRows(rows); results.add(new ExecutionResult(plan, rows)); } return results; } + private static void logRows(Iterable rows) { + if (logger.isInfoEnabled() == false) return; + List list = (rows instanceof List) ? (List) rows : null; + int count = list != null ? list.size() : -1; + logger.info("Query result rowCount={}", count); + if (list != null) { + int preview = Math.min(20, list.size()); + for (int i = 0; i < preview; i++) { + logger.info("row[{}]={}", i, Arrays.toString(list.get(i))); + } + if (list.size() > preview) { + logger.info("... ({} more rows)", list.size() - preview); + } + } + } + // TODO: move plan logging behind a debug flag // invalidateMetadataQuery() and THREAD_PROVIDERS are only needed for explain() output private void logPlan(RelNode relNode) { diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/action/TransportDslExecuteActionTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/action/TransportDslExecuteActionTests.java index d8a40aa7a9f8d..05e4d6d910163 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/action/TransportDslExecuteActionTests.java +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/action/TransportDslExecuteActionTests.java @@ -29,12 +29,15 @@ import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import java.util.Collections; +import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicReference; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -85,7 +88,8 @@ public void testDoExecuteFailsWhenIndexNotInClusterState() { buildEngineContext(), (plan, ctx) -> Collections.emptyList(), clusterService, - resolver + resolver, + mockThreadPool() ); TestListener listener = executeWith(action, "bogus-index"); @@ -117,7 +121,8 @@ private TransportDslExecuteAction createAction(Index... resolvedIndices) { buildEngineContext(), (plan, ctx) -> Collections.emptyList(), clusterService, - resolver + resolver, + mockThreadPool() ); } @@ -147,6 +152,17 @@ public RelDataType getRowType(RelDataTypeFactory tf) { return schema; } + private static ThreadPool mockThreadPool() { + ThreadPool threadPool = mock(ThreadPool.class); + ExecutorService executorService = mock(ExecutorService.class); + when(threadPool.executor(any())).thenReturn(executorService); + doAnswer(invocation -> { + ((Runnable) invocation.getArgument(0)).run(); + return null; + }).when(executorService).execute(any()); + return threadPool; + } + private static class TestListener implements ActionListener { final AtomicReference response = new AtomicReference<>(); final AtomicReference failure = new AtomicReference<>(); From 4943859ddb6104a6b2f65db10c3dad19cf67a771 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Mon, 27 Apr 2026 13:44:12 +0530 Subject: [PATCH 002/115] Fix O(n^2) removeAll in remote translog metadata cleanup (#21350) ArrayList.removeAll(ArrayList) is O(n*m) due to linear contains() checks. Wrap the argument in HashSet for O(1) lookups, reducing the complexity to O(n). This was causing CPU spikes on the remote_purge thread when metadata file counts grew large. Signed-off-by: Gaurav Bafna --- .../RemoteFsTimestampAwareTranslog.java | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index 1832d1e7d035a..32d79439b004e 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -202,12 +202,22 @@ public void onResponse(List blobMetadata) { return; } - logger.debug(() -> "metadataFilesToBeDeleted = " + metadataFilesToBeDeleted); + logger.debug( + () -> "metadataFilesToBeDeleted count = " + + metadataFilesToBeDeleted.size() + + ", metadataFilesToBeDeleted = " + + metadataFilesToBeDeleted + ); // For all the files that we are keeping, fetch min and max generations List metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles); - metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); + metadataFilesNotToBeDeleted.removeAll(new HashSet<>(metadataFilesToBeDeleted)); - logger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted); + logger.debug( + () -> "metadataFilesNotToBeDeleted count = " + + metadataFilesNotToBeDeleted.size() + + ", metadataFilesNotToBeDeleted = " + + metadataFilesNotToBeDeleted + ); Set generationsToBeDeleted = getGenerationsToBeDeleted( metadataFilesNotToBeDeleted, @@ -373,7 +383,7 @@ protected static List getMetadataFilesToBeDeleted( long maxGeneration = TranslogTransferMetadata.getMaxGenerationFromFileName(md); return maxGeneration == -1 || maxGeneration >= minGenerationToKeepInRemote; }).collect(Collectors.toList()); - metadataFilesToBeDeleted.removeAll(metadataFilesContainingMinGenerationToKeep); + metadataFilesToBeDeleted.removeAll(new HashSet<>(metadataFilesContainingMinGenerationToKeep)); logger.trace( "metadataFilesContainingMinGenerationToKeep.size = {}, metadataFilesToBeDeleted based on minGenerationToKeep filtering = {}, minGenerationToKeep = {}", @@ -572,12 +582,22 @@ public void onResponse(List blobMetadata) { staticLogger.debug("No metadata files to delete"); return; } - staticLogger.debug(() -> "metadataFilesToBeDeleted = " + metadataFilesToBeDeleted); + staticLogger.debug( + () -> "metadataFilesToBeDeleted count = " + + metadataFilesToBeDeleted.size() + + ", metadataFilesToBeDeleted = " + + metadataFilesToBeDeleted + ); // For all the files that we are keeping, fetch min and max generations List metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles); - metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); - staticLogger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted); + metadataFilesNotToBeDeleted.removeAll(new HashSet<>(metadataFilesToBeDeleted)); + staticLogger.debug( + () -> "metadataFilesNotToBeDeleted count = " + + metadataFilesNotToBeDeleted.size() + + ", metadataFilesNotToBeDeleted = " + + metadataFilesNotToBeDeleted + ); // Delete stale metadata files translogTransferManager.deleteMetadataFilesAsync(metadataFilesToBeDeleted, () -> {}); From c74ea29a01c06e3aed530afa6cf289bf648884a6 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 27 Apr 2026 07:53:47 -0400 Subject: [PATCH 003/115] Fix Flaky Test Report for Netty4Http3IT (#21360) Signed-off-by: Andriy Redko --- .../opensearch/http/netty4/Netty4Http3IT.java | 51 ++++++++++++++----- .../Netty4CompositeHttpServerTransport.java | 4 ++ 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4Http3IT.java b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4Http3IT.java index b3dbf778890fe..71c892559e951 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4Http3IT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4Http3IT.java @@ -18,6 +18,8 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.http.AbstractHttpServerTransport; +import org.opensearch.http.HttpRequest.HttpVersion; import org.opensearch.http.HttpServerTransport; import org.opensearch.http.HttpTransportSettings; import org.opensearch.http.netty4.http3.Http3Utils; @@ -32,6 +34,7 @@ import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLException; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.IntStream; @@ -47,6 +50,7 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -81,22 +85,25 @@ public void testThatNettyHttpServerSupportsHttp2OrHttp3Get() throws Exception { String[] requests = new String[] { "/", "/_nodes/stats", "/", "/_cluster/state", "/" }; HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); - TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses(); - TransportAddress transportAddress = randomFrom(boundAddresses); + assertThat(httpServerTransport, instanceOf(Netty4CompositeHttpServerTransport.class)); @SuppressWarnings("unchecked") - final Tuple client = randomFrom( - Tuple.tuple(Netty4HttpClient.http3().withLogger(logger), "h2="), - Tuple.tuple(Netty4HttpClient.https().withLogger(logger), "h3=") + final Tuple> client = randomFrom( + Tuple.tuple(Netty4HttpClient.http3().withLogger(logger), Tuple.tuple("h2=", HttpVersion.HTTP_3_0)), + Tuple.tuple(Netty4HttpClient.https().withLogger(logger), Tuple.tuple("h3=", HttpVersion.HTTP_2_0)) ); try (Netty4HttpClient nettyHttpClient = client.v1()) { - Collection responses = nettyHttpClient.get(transportAddress.address(), randomFrom(requests)); + final TransportAddress transportAddress = randomFrom( + (Netty4CompositeHttpServerTransport) httpServerTransport, + client.v2().v2() + ); + final Collection responses = nettyHttpClient.get(transportAddress.address(), randomFrom(requests)); try { assertThat(responses, hasSize(1)); for (HttpResponse response : responses) { - assertThat(response.headers().get("Alt-Svc"), containsString(client.v2())); + assertThat(response.headers().get("Alt-Svc"), containsString(client.v2().v1())); } Collection opaqueIds = Netty4HttpClient.returnOpaqueIds(responses); @@ -115,23 +122,26 @@ public void testThatNettyHttpServerSupportsHttp2OrHttp3Post() throws Exception { final List> requests = List.of(Tuple.tuple("/_search", "{\"query\":{ \"match_all\":{}}}")); HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); - TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses(); - TransportAddress transportAddress = randomFrom(boundAddresses); + assertThat(httpServerTransport, instanceOf(Netty4CompositeHttpServerTransport.class)); @SuppressWarnings("unchecked") - final Tuple client = randomFrom( - Tuple.tuple(Netty4HttpClient.http3().withLogger(logger), "h2="), - Tuple.tuple(Netty4HttpClient.https().withLogger(logger), "h3=") + final Tuple> client = randomFrom( + Tuple.tuple(Netty4HttpClient.http3().withLogger(logger), Tuple.tuple("h2=", HttpVersion.HTTP_3_0)), + Tuple.tuple(Netty4HttpClient.https().withLogger(logger), Tuple.tuple("h3=", HttpVersion.HTTP_2_0)) ); try (Netty4HttpClient nettyHttpClient = client.v1()) { - Collection responses = nettyHttpClient.post(transportAddress.address(), requests); + final TransportAddress transportAddress = randomFrom( + (Netty4CompositeHttpServerTransport) httpServerTransport, + client.v2().v2() + ); + final Collection responses = nettyHttpClient.post(transportAddress.address(), requests); try { assertThat(responses, hasSize(1)); for (FullHttpResponse response : responses) { assertThat(response.status(), equalTo(HttpResponseStatus.OK)); - assertThat(response.headers().get("Alt-Svc"), containsString(client.v2())); + assertThat(response.headers().get("Alt-Svc"), containsString(client.v2().v1())); } Collection opaqueIds = Netty4HttpClient.returnOpaqueIds(responses); @@ -157,6 +167,19 @@ protected Collection> nodePlugins() { return Stream.concat(super.nodePlugins().stream(), Stream.of(SecureSettingsPlugin.class)).toList(); } + private TransportAddress randomFrom(final Netty4CompositeHttpServerTransport transport, HttpVersion protocol) { + final AbstractHttpServerTransport httpServerTransport = Arrays.stream(transport.transports()).filter(t -> { + if (protocol == HttpVersion.HTTP_3_0) { + return t instanceof Netty4Http3ServerTransport; + } else { + return t instanceof Netty4HttpServerTransport; + } + }).findAny().orElseThrow(); + + TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses(); + return randomFrom(boundAddresses); + } + private void assertOpaqueIdsInAnyOrder(int expected, Collection opaqueIds) { // check if opaque ids are present in any order, since for HTTP/2 we use streaming (no head of line blocking) // and responses may come back at any order diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4CompositeHttpServerTransport.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4CompositeHttpServerTransport.java index 4853bdee208ca..dab9a96754c1a 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4CompositeHttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4CompositeHttpServerTransport.java @@ -72,4 +72,8 @@ protected void doClose() throws IOException { IOUtils.closeWhileHandlingException(transport); } } + + AbstractHttpServerTransport[] transports() { + return transports; + } } From 9bcf4f0fb1fa4e089040fde4657c66acde75fa3d Mon Sep 17 00:00:00 2001 From: Divya <117009486+divyaruhil@users.noreply.github.com> Date: Mon, 27 Apr 2026 22:09:55 +0530 Subject: [PATCH 004/115] Fix for RemoteIndexPrimaryRelocationIT Flaky test (#21249) Signed-off-by: Divya Co-authored-by: DIVYA2 Co-authored-by: Divya Co-authored-by: Andrew Ross --- .../indices/recovery/IndexPrimaryRelocationIT.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexPrimaryRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexPrimaryRelocationIT.java index 9decd17d95eab..e5b651d528fb1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexPrimaryRelocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexPrimaryRelocationIT.java @@ -39,6 +39,7 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; @@ -78,12 +79,14 @@ public void testPrimaryRelocationWhileIndexing() throws Exception { }); indexingThread.start(); - ClusterState initialState = client().admin().cluster().prepareState().get().getState(); - DiscoveryNode[] dataNodes = initialState.getNodes().getDataNodes().values().toArray(new DiscoveryNode[0]); - DiscoveryNode relocationSource = initialState.getNodes() - .getDataNodes() - .get(initialState.getRoutingTable().shardRoutingTable("test", 0).primaryShard().currentNodeId()); for (int i = 0; i < RELOCATION_COUNT; i++) { + // Fetch fresh cluster state to get current shard location and available nodes + ClusterState currentState = client().admin().cluster().prepareState().get().getState(); + DiscoveryNode[] dataNodes = currentState.getNodes().getDataNodes().values().toArray(new DiscoveryNode[0]); + + ShardRouting primaryShard = currentState.getRoutingTable().shardRoutingTable("test", 0).primaryShard(); + DiscoveryNode relocationSource = currentState.getNodes().getDataNodes().get(primaryShard.currentNodeId()); + DiscoveryNode relocationTarget = randomFrom(dataNodes); while (relocationTarget.equals(relocationSource)) { relocationTarget = randomFrom(dataNodes); @@ -125,7 +128,6 @@ public void testPrimaryRelocationWhileIndexing() throws Exception { throw new AssertionError("timed out waiting for relocation iteration [" + i + "] "); } logger.info("--> [iteration {}] relocation complete", i); - relocationSource = relocationTarget; // indexing process aborted early, no need for more relocations as test has already failed if (indexingThread.isAlive() == false) { break; From 3cd9aa635320f2371cc3d4595f9968217cf1dce1 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 27 Apr 2026 12:42:02 -0400 Subject: [PATCH 005/115] Update logic in FipsBuildParams.isInFipsApprovedOnlyMode to check for env param instead of org.bouncycastle.fips.approved_only (#21366) Signed-off-by: Craig Perkins --- .../gradle/info/FipsBuildParams.java | 4 +++- .../gradle/info/FipsBuildParamsTests.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/info/FipsBuildParams.java b/buildSrc/src/main/java/org/opensearch/gradle/info/FipsBuildParams.java index e1427466c702e..8b4e27472f4df 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/info/FipsBuildParams.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/info/FipsBuildParams.java @@ -9,6 +9,7 @@ package org.opensearch.gradle.info; import java.util.function.Function; +import java.util.function.Supplier; public class FipsBuildParams { @@ -18,6 +19,7 @@ public class FipsBuildParams { public static final String DEFAULT_FIPS_MODE = "FIPS-140-3"; private static String fipsMode; + static Supplier fipsModeEnvSupplier = () -> System.getenv("OPENSEARCH_FIPS_MODE"); public static void init(Function fipsValue) { var fipsBuildParamForTests = Boolean.parseBoolean((String) fipsValue.apply(FIPS_BUILD_PARAM_FOR_TESTS)); @@ -37,7 +39,7 @@ public static boolean isInFipsMode() { } public static boolean isInFipsApprovedOnlyMode() { - return isInFipsMode() && "true".equals(System.getProperty("org.bouncycastle.fips.approved_only")); + return isInFipsMode() && "true".equalsIgnoreCase(fipsModeEnvSupplier.get()); } public static String getFipsMode() { diff --git a/buildSrc/src/test/java/org/opensearch/gradle/info/FipsBuildParamsTests.java b/buildSrc/src/test/java/org/opensearch/gradle/info/FipsBuildParamsTests.java index 2a25a275ebd0d..8e95d52774023 100644 --- a/buildSrc/src/test/java/org/opensearch/gradle/info/FipsBuildParamsTests.java +++ b/buildSrc/src/test/java/org/opensearch/gradle/info/FipsBuildParamsTests.java @@ -14,6 +14,30 @@ public class FipsBuildParamsTests extends GradleUnitTestCase { + public void testIsInFipsApprovedOnlyMode() { + FipsBuildParams.init(cryptoEntryFnWithStringParam); + + FipsBuildParams.fipsModeEnvSupplier = () -> "true"; + assertTrue(FipsBuildParams.isInFipsApprovedOnlyMode()); + + FipsBuildParams.fipsModeEnvSupplier = () -> "TRUE"; + assertTrue(FipsBuildParams.isInFipsApprovedOnlyMode()); + + FipsBuildParams.fipsModeEnvSupplier = () -> "false"; + assertFalse(FipsBuildParams.isInFipsApprovedOnlyMode()); + + FipsBuildParams.fipsModeEnvSupplier = () -> null; + assertFalse(FipsBuildParams.isInFipsApprovedOnlyMode()); + + // Not in FIPS mode — should always be false regardless of env var + FipsBuildParams.init(param -> null); + FipsBuildParams.fipsModeEnvSupplier = () -> "true"; + assertFalse(FipsBuildParams.isInFipsApprovedOnlyMode()); + + // Reset + FipsBuildParams.fipsModeEnvSupplier = () -> System.getenv("OPENSEARCH_FIPS_MODE"); + } + public void testIsInFipsMode() { FipsBuildParams.init(cryptoEntryFnWithStringParam); assertTrue(FipsBuildParams.isInFipsMode()); From 30920ea7a08aef0b6b388426ec9c586fa0679ebc Mon Sep 17 00:00:00 2001 From: MayankHarsh03 <156461043+MayankHarsh03@users.noreply.github.com> Date: Tue, 28 Apr 2026 07:57:40 +0530 Subject: [PATCH 006/115] Fix BlockTransferManagerTests to extend OpenSearchTestCase instead of LuceneTestCase (#21363) BlockTransferManagerTests was extending LuceneTestCase directly which causes sysout check failures since the test uses loggers that print to console. Changed to extend OpenSearchTestCase which already includes @SuppressSysoutChecks and follows the project convention for all server tests. Signed-off-by: Mayank Harsh Co-authored-by: Mayank Harsh --- .../opensearch/storage/common/BlockTransferManagerTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/opensearch/storage/common/BlockTransferManagerTests.java b/server/src/test/java/org/opensearch/storage/common/BlockTransferManagerTests.java index 0a8e107274d78..a37c0813376a6 100644 --- a/server/src/test/java/org/opensearch/storage/common/BlockTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/storage/common/BlockTransferManagerTests.java @@ -12,7 +12,6 @@ import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.UUIDs; @@ -23,6 +22,7 @@ import org.opensearch.index.store.remote.utils.TransferManager; import org.opensearch.node.Node; import org.opensearch.storage.indexinput.BlockFetchRequest; +import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Assert; @@ -52,7 +52,7 @@ * Tests cover single block downloads, failure scenarios, duplicate handling, and concurrent operations. */ @ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class) -public class BlockTransferManagerTests extends LuceneTestCase { +public class BlockTransferManagerTests extends OpenSearchTestCase { // Node and index configuration constants private static final String TEST_NODE_NAME = "test-node"; From 443253810392025bb2f9daeb4e51ecec1c402f70 Mon Sep 17 00:00:00 2001 From: Sagar Darji <42430138+darjisagar7@users.noreply.github.com> Date: Tue, 28 Apr 2026 09:10:07 +0530 Subject: [PATCH 007/115] Adding CompositeMergeHandler and CompositeMergePolicy (#21128) * Adding CompositeMergeHandler and CompositeMergePolicy Signed-off-by: Sagar Darji # Conflicts: # sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java * Addressing comments Signed-off-by: Sagar Darji * Split the monolithic CompositeMergeHandler into classes with clear responsibilities: Signed-off-by: Bukhtawar Khan * Fix up tests Signed-off-by: Bukhtawar Khan * Addressing commits Signed-off-by: Sagar Darji * Integrating the merge flow with the DataFormatAwareEngine Signed-off-by: Sagar Darji # Conflicts: # server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java # server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java # server/src/test/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManagerTests.java # Conflicts: # server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java * Addressed the comments Signed-off-by: Sagar Darji --------- Signed-off-by: Sagar Darji Signed-off-by: Bukhtawar Khan Co-authored-by: Sagar Darji Co-authored-by: Bukhtawar Khan --- .../index/LuceneIndexingExecutionEngine.java | 4 +- .../composite/CompositeMergeIT.java | 209 ++++++ .../composite/CompositeDataFormat.java | 19 +- .../CompositeIndexingExecutionEngine.java | 5 +- .../merge/CompositeMergeExecutor.java | 90 +++ .../composite/merge/CompositeMerger.java | 106 +++ .../composite/merge/FormatMergeResult.java | 45 ++ .../opensearch/composite/merge/MergePlan.java | 71 ++ .../composite/merge/package-info.java | 17 + .../composite/CompositeDataFormatTests.java | 37 +- ...CompositeIndexingExecutionEngineTests.java | 2 +- .../composite/CompositeTestHelper.java | 3 +- .../composite/merge/CompositeMergerTests.java | 629 ++++++++++++++++++ .../parquet/engine/ParquetIndexingEngine.java | 4 +- .../engine/ParquetIndexingEngineTests.java | 4 +- .../opensearch/OpenSearchServerException.java | 9 + .../index/engine/DataFormatAwareEngine.java | 92 ++- .../OpenSearchConcurrentMergeScheduler.java | 43 +- .../index/engine/dataformat/MergeInput.java | 26 +- .../merge/DataFormatAwareMergePolicy.java | 333 ++++++++++ .../merge/MergeFailedEngineException.java | 44 ++ .../engine/dataformat/merge/MergeHandler.java | 176 ++++- .../dataformat/merge/MergeScheduler.java | 136 +++- .../exec/coord/CatalogSnapshotManager.java | 74 +++ .../index/merge/MergeStatsTracker.java | 90 +++ .../org/opensearch/threadpool/ThreadPool.java | 3 + .../ExceptionSerializationTests.java | 1 + .../dataformat/DataFormatPluginTests.java | 6 +- .../DataFormatAwareMergePolicyTests.java | 435 ++++++++++++ .../MergeFailedEngineExceptionTests.java | 34 + .../engine/dataformat/merge/MergeTests.java | 391 ++++++----- .../coord/CatalogSnapshotManagerTests.java | 106 +++ .../threadpool/ScalingThreadPoolTests.java | 1 + .../dataformat/stub/MockDataFormatPlugin.java | 2 +- .../engine/dataformat/stub/MockMerger.java | 7 +- 35 files changed, 2972 insertions(+), 282 deletions(-) create mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java create mode 100644 sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMergeExecutor.java create mode 100644 sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMerger.java create mode 100644 sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/FormatMergeResult.java create mode 100644 sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/MergePlan.java create mode 100644 sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/package-info.java create mode 100644 sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/merge/CompositeMergerTests.java create mode 100644 server/src/main/java/org/opensearch/index/engine/dataformat/merge/DataFormatAwareMergePolicy.java create mode 100644 server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeFailedEngineException.java create mode 100644 server/src/main/java/org/opensearch/index/merge/MergeStatsTracker.java create mode 100644 server/src/test/java/org/opensearch/index/engine/dataformat/merge/DataFormatAwareMergePolicyTests.java create mode 100644 server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeFailedEngineExceptionTests.java diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngine.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngine.java index 416bfdefdb2a6..d12099e82f083 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngine.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngine.java @@ -26,6 +26,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.MergeResult; import org.opensearch.index.engine.dataformat.Merger; import org.opensearch.index.engine.dataformat.RefreshInput; import org.opensearch.index.engine.dataformat.RefreshResult; @@ -278,7 +279,8 @@ public RefreshResult refresh(RefreshInput refreshInput) throws IOException { /** Returns {@code null} — merge scheduling is not yet implemented for the Lucene format. */ @Override public Merger getMerger() { - return null; + // TODO: Implement merge support as ParquetMerger + return mergeInput -> new MergeResult(Map.of()); } /** diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java new file mode 100644 index 0000000000000..4faa5948d7c4e --- /dev/null +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java @@ -0,0 +1,209 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite; + +import org.opensearch.action.admin.indices.refresh.RefreshResponse; +import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.CommitStats; +import org.opensearch.index.engine.dataformat.DataFormatDescriptor; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.dataformat.ReaderManagerConfig; +import org.opensearch.index.engine.dataformat.stub.MockDataFormat; +import org.opensearch.index.engine.dataformat.stub.MockDataFormatPlugin; +import org.opensearch.index.engine.dataformat.stub.MockReaderManager; +import org.opensearch.index.engine.exec.EngineReaderManager; +import org.opensearch.index.engine.exec.coord.DataformatAwareCatalogSnapshot; +import org.opensearch.index.merge.MergeStats; +import org.opensearch.index.store.PrecomputedChecksumStrategy; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SearchBackEndPlugin; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +/** + * Integration tests for composite merge operations across single and multiple data format engines. + * + * Requires JDK 25 and sandbox enabled. Run with: + * ./gradlew :sandbox:plugins:composite-engine:internalClusterTest \ + * --tests "*.CompositeMergeIT" \ + * -Dsandbox.enabled=true + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1) +public class CompositeMergeIT extends OpenSearchIntegTestCase { + + private static final String INDEX_NAME = "test-composite-merge"; + private static final String MERGE_ENABLED_PROPERTY = "opensearch.pluggable.dataformat.merge.enabled"; + + // ── Mock DataFormatPlugin using test framework stubs ── + + public static class MockParquetDataFormatPlugin extends MockDataFormatPlugin implements SearchBackEndPlugin { + private static final MockDataFormat PARQUET_FORMAT = new MockDataFormat("parquet", 0L, Set.of()); + + public MockParquetDataFormatPlugin() { + super(PARQUET_FORMAT); + } + + @Override + public Map getFormatDescriptors(IndexSettings indexSettings, DataFormatRegistry registry) { + return Map.of("parquet", new DataFormatDescriptor("parquet", new PrecomputedChecksumStrategy())); + } + + @Override + public String name() { + return "mock-parquet-backend"; + } + + @Override + public List getSupportedFormats() { + return List.of("parquet"); + } + + @Override + public EngineReaderManager createReaderManager(ReaderManagerConfig settings) { + return new MockReaderManager("parquet"); + } + } + + // ── Test setup ── + + @Override + public void setUp() throws Exception { + enableMerge(); + super.setUp(); + } + + @Override + public void tearDown() throws Exception { + try { + client().admin().indices().prepareDelete(INDEX_NAME).get(); + } catch (Exception e) { + // index may not exist if test failed before creation + } + super.tearDown(); + disableMerge(); + } + + @SuppressForbidden(reason = "enable pluggable dataformat merge for integration testing") + private static void enableMerge() { + System.setProperty(MERGE_ENABLED_PROPERTY, "true"); + } + + @SuppressForbidden(reason = "restore pluggable dataformat merge property after test") + private static void disableMerge() { + System.clearProperty(MERGE_ENABLED_PROPERTY); + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(MockParquetDataFormatPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .build(); + } + + // ── Tests ── + + /** + * Verifies that background merges are triggered automatically after refresh + * when enough segments accumulate to exceed the TieredMergePolicy threshold. + *

+ * Flow: index docs across many refresh cycles → each refresh calls + * triggerPossibleMerges() → MergeScheduler picks up merge candidates + * asynchronously → segment count decreases. + */ + public void testBackgroundMergeSingleEngine() throws Exception { + createIndex(INDEX_NAME, singleEngineSettings()); + ensureGreen(INDEX_NAME); + + // Create enough segments to exceed TieredMergePolicy's default threshold (~10) + int totalSegmentsCreated = indexDocsAcrossMultipleRefreshes(15, 5); + + // Wait for async background merges to complete + assertBusy(() -> { + flush(INDEX_NAME); + DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); + assertTrue( + "Expected merges to reduce segment count below " + totalSegmentsCreated + ", but got: " + snapshot.getSegments().size(), + snapshot.getSegments().size() < totalSegmentsCreated + ); + }); + + MergeStats mergeStats = getMergeStats(); + assertTrue("Expected at least one merge to have occurred", mergeStats.getTotal() > 0); + + DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); + assertEquals(Set.of("parquet"), snapshot.getDataFormats()); + } + + // ── Helpers ── + + private Settings singleEngineSettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats") + .build(); + } + + private int indexDocsAcrossMultipleRefreshes(int refreshCycles, int docsPerCycle) { + for (int cycle = 0; cycle < refreshCycles; cycle++) { + for (int i = 0; i < docsPerCycle; i++) { + IndexResponse response = client().prepareIndex() + .setIndex(INDEX_NAME) + .setSource("field_text", randomAlphaOfLength(10), "field_number", randomIntBetween(1, 1000)) + .get(); + assertEquals(RestStatus.CREATED, response.status()); + } + RefreshResponse refreshResponse = client().admin().indices().prepareRefresh(INDEX_NAME).get(); + assertEquals(RestStatus.OK, refreshResponse.getStatus()); + } + return refreshCycles; + } + + private DataformatAwareCatalogSnapshot getCatalogSnapshot() throws IOException { + IndicesStatsResponse statsResponse = client().admin().indices().prepareStats(INDEX_NAME).clear().setStore(true).get(); + ShardStats shardStats = statsResponse.getIndex(INDEX_NAME).getShards()[0]; + CommitStats commitStats = shardStats.getCommitStats(); + assertNotNull(commitStats); + assertTrue(commitStats.getUserData().containsKey(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY)); + return DataformatAwareCatalogSnapshot.deserializeFromString( + commitStats.getUserData().get(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY), + Function.identity() + ); + } + + private MergeStats getMergeStats() { + IndicesStatsResponse statsResponse = client().admin().indices().prepareStats(INDEX_NAME).clear().setMerge(true).get(); + return statsResponse.getIndex(INDEX_NAME).getShards()[0].getStats().getMerge(); + } +} diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormat.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormat.java index 2633ad0f30330..b474121550ef7 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormat.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormat.java @@ -26,14 +26,17 @@ @ExperimentalApi public class CompositeDataFormat extends DataFormat { + private final DataFormat primaryDataFormat; private final List dataFormats; /** - * Constructs a CompositeDataFormat from the given list of data formats. + * Constructs a CompositeDataFormat with a designated primary format and a list of all constituent formats. * - * @param dataFormats the constituent data formats + * @param primaryDataFormat the authoritative data format used for merge operations + * @param dataFormats all constituent data formats (including the primary) */ - public CompositeDataFormat(List dataFormats) { + public CompositeDataFormat(DataFormat primaryDataFormat, List dataFormats) { + this.primaryDataFormat = Objects.requireNonNull(primaryDataFormat, "primaryDataFormat must not be null"); this.dataFormats = List.copyOf(Objects.requireNonNull(dataFormats, "dataFormats must not be null")); } @@ -41,6 +44,7 @@ public CompositeDataFormat(List dataFormats) { * Constructs an empty CompositeDataFormat with no constituent formats. */ public CompositeDataFormat() { + this.primaryDataFormat = null; this.dataFormats = List.of(); } @@ -53,6 +57,15 @@ public List getDataFormats() { return dataFormats; } + /** + * Returns the primary data format used for merge operations. + * + * @return the primary data format + */ + public DataFormat getPrimaryDataFormat() { + return primaryDataFormat; + } + @Override public String name() { return "composite"; diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java index a73e9af47e2e4..831ccfb0971c8 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java @@ -13,6 +13,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.composite.merge.CompositeMerger; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.DataFormatPlugin; @@ -129,7 +130,7 @@ public CompositeIndexingExecutionEngine( } this.secondaryEngines = Set.copyOf(secondaries); - this.compositeDataFormat = new CompositeDataFormat(allFormats); + this.compositeDataFormat = new CompositeDataFormat(primaryFormat, allFormats); this.committer = committer; } @@ -181,7 +182,7 @@ public Writer createWriter(long writerGeneration) { /** {@inheritDoc} Delegates to the primary engine's merger. */ @Override public Merger getMerger() { - return primaryEngine.getMerger(); + return new CompositeMerger(this, compositeDataFormat); } /** diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMergeExecutor.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMergeExecutor.java new file mode 100644 index 0000000000000..caf75785175db --- /dev/null +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMergeExecutor.java @@ -0,0 +1,90 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite.merge; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.MergeInput; +import org.opensearch.index.engine.dataformat.MergeResult; +import org.opensearch.index.engine.dataformat.Merger; +import org.opensearch.index.engine.dataformat.RowIdMapping; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Executes a composite merge: primary format first, then secondaries using the + * row-ID mapping from the primary. Stateless — all state comes from the + * {@link MergePlan} and the merger map. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeMergeExecutor { + + private final Map mergers; + + public CompositeMergeExecutor(Map mergers) { + this.mergers = Map.copyOf(mergers); + } + + /** + * Executes the merge described by the plan. + * + * @param plan the pre-validated merge plan + * @return the combined merge result across all formats + */ + public MergeResult execute(MergePlan plan) { + List completed = new ArrayList<>(); + try { + FormatMergeResult primaryResult = mergeFormat(plan, plan.primaryFormat(), null); + completed.add(primaryResult); + + RowIdMapping mapping = plan.hasSecondaries() + ? primaryResult.rowIdMappingOpt() + .orElseThrow(() -> new IllegalStateException("Primary merge did not produce row-ID mapping required by secondaries")) + : null; + + for (DataFormat secondary : plan.secondaryFormats()) { + completed.add(mergeFormat(plan, secondary, mapping)); + } + + return toMergeResult(completed, mapping); + } catch (Exception e) { + completed.forEach(FormatMergeResult::cleanup); + if (e instanceof RuntimeException re) throw re; + throw new UncheckedIOException((IOException) e); + } + } + + private FormatMergeResult mergeFormat(MergePlan plan, DataFormat format, RowIdMapping mapping) throws IOException { + Merger merger = mergers.get(format); + List files = plan.filesFor(format); + List segments = new ArrayList<>(); + for (WriterFileSet wfs : files) { + segments.add(Segment.builder(wfs.writerGeneration()).addSearchableFiles(format, wfs).build()); + } + MergeResult result = merger.merge(new MergeInput(segments, mapping, plan.mergedWriterGeneration())); + return new FormatMergeResult(format, result.getMergedWriterFileSetForDataformat(format), result.rowIdMapping().orElse(null)); + } + + private static MergeResult toMergeResult(List results, RowIdMapping mapping) { + Map merged = new HashMap<>(); + for (FormatMergeResult r : results) { + merged.put(r.format(), r.mergedFiles()); + } + return new MergeResult(merged, mapping); + } +} diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMerger.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMerger.java new file mode 100644 index 0000000000000..b32d50a1368f1 --- /dev/null +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMerger.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite.merge; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.composite.CompositeDataFormat; +import org.opensearch.composite.CompositeIndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.MergeInput; +import org.opensearch.index.engine.dataformat.MergeResult; +import org.opensearch.index.engine.dataformat.Merger; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A {@link Merger} that orchestrates composite merges across primary and secondary + * data formats by delegating to {@link CompositeMergeExecutor}. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeMerger implements Merger { + + private final DataFormat primaryFormat; + private final List secondaryFormats; + private final CompositeMergeExecutor executor; + + public CompositeMerger(CompositeIndexingExecutionEngine engine, CompositeDataFormat compositeDataFormat) { + this.primaryFormat = compositeDataFormat.getPrimaryDataFormat(); + this.secondaryFormats = resolveSecondaryFormats(compositeDataFormat, primaryFormat); + this.executor = new CompositeMergeExecutor(buildMergerMap(engine)); + } + + @Override + public MergeResult merge(MergeInput mergeInput) throws IOException { + Map> filesByFormat = extractFilesByFormat(mergeInput.segments()); + MergePlan plan = new MergePlan(mergeInput.newWriterGeneration(), primaryFormat, secondaryFormats, filesByFormat); + return executor.execute(plan); + } + + private Map> extractFilesByFormat(List segments) { + Set allFormats = new LinkedHashSet<>(); + allFormats.add(primaryFormat); + allFormats.addAll(secondaryFormats); + + Map> filesByFormat = new LinkedHashMap<>(); + for (DataFormat format : allFormats) { + List files = new ArrayList<>(); + for (Segment segment : segments) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get(format.name()); + if (wfs != null) { + files.add(wfs); + } + } + filesByFormat.put(format, List.copyOf(files)); + } + return filesByFormat; + } + + private static List resolveSecondaryFormats(CompositeDataFormat compositeDataFormat, DataFormat primaryFormat) { + List secondaries = new ArrayList<>(); + for (DataFormat format : compositeDataFormat.getDataFormats()) { + if (format.equals(primaryFormat) == false) { + secondaries.add(format); + } + } + return List.copyOf(secondaries); + } + + private static Map buildMergerMap(CompositeIndexingExecutionEngine engine) { + Map map = new HashMap<>(); + + Merger primaryMerger = engine.getPrimaryDelegate().getMerger(); + if (primaryMerger == null) { + throw new IllegalStateException( + "Primary format [" + engine.getPrimaryDelegate().getDataFormat().name() + "] does not provide a Merger" + ); + } + map.put(engine.getPrimaryDelegate().getDataFormat(), primaryMerger); + + for (IndexingExecutionEngine secondary : engine.getSecondaryDelegates()) { + Merger merger = secondary.getMerger(); + if (merger == null) { + throw new IllegalStateException("Secondary format [" + secondary.getDataFormat().name() + "] does not provide a Merger"); + } + map.put(secondary.getDataFormat(), merger); + } + return Map.copyOf(map); + } +} diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/FormatMergeResult.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/FormatMergeResult.java new file mode 100644 index 0000000000000..21b3cd1b4c94c --- /dev/null +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/FormatMergeResult.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite.merge; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.RowIdMapping; +import org.opensearch.index.engine.exec.WriterFileSet; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +/** + * Result of merging a single data format's files. + */ +@ExperimentalApi +public record FormatMergeResult(DataFormat format, WriterFileSet mergedFiles, RowIdMapping rowIdMapping) { + + public Optional rowIdMappingOpt() { + return Optional.ofNullable(rowIdMapping); + } + + /** + * Deletes the merged output files. Called during cleanup on merge failure. + */ + public void cleanup() { + if (mergedFiles == null) return; + for (String file : mergedFiles.files()) { + try { + Path resolved = mergedFiles.directory() != null ? Path.of(mergedFiles.directory(), file) : Path.of(file); + Files.deleteIfExists(resolved); + } catch (IOException ignored) { + // Best-effort cleanup + } + } + } +} diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/MergePlan.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/MergePlan.java new file mode 100644 index 0000000000000..acefbc2fcd53e --- /dev/null +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/MergePlan.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite.merge; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.merge.OneMerge; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Pre-validated merge plan with per-format file lists and primary/secondary distinction. + * Segments that predate a format are skipped (null entries filtered). + * + * @opensearch.experimental + */ +@ExperimentalApi +public record MergePlan(long mergedWriterGeneration, DataFormat primaryFormat, List secondaryFormats, Map< + DataFormat, + List> filesByFormat) { + + public MergePlan { + secondaryFormats = List.copyOf(secondaryFormats); + filesByFormat = Map.copyOf(filesByFormat); + } + + /** Files for a given format, empty list if the format has no files. */ + public List filesFor(DataFormat format) { + return filesByFormat.getOrDefault(format, List.of()); + } + + /** Whether this plan has any secondary formats. */ + public boolean hasSecondaries() { + return secondaryFormats.isEmpty() == false; + } + + /** + * Builds a plan from a merge operation, a primary format, secondary formats, and a generation. + */ + public static MergePlan from(OneMerge oneMerge, DataFormat primaryFormat, List secondaryFormats, long generation) { + Set allFormats = new LinkedHashSet<>(); + allFormats.add(primaryFormat); + allFormats.addAll(secondaryFormats); + + Map> filesByFormat = new LinkedHashMap<>(); + for (DataFormat format : allFormats) { + List files = new ArrayList<>(); + for (Segment segment : oneMerge.getSegmentsToMerge()) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get(format.name()); + if (wfs != null) { + files.add(wfs); + } + } + filesByFormat.put(format, List.copyOf(files)); + } + return new MergePlan(generation, primaryFormat, secondaryFormats, filesByFormat); + } +} diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/package-info.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/package-info.java new file mode 100644 index 0000000000000..4b10dd414f782 --- /dev/null +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/package-info.java @@ -0,0 +1,17 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Merge support for composite data formats. Adapts Lucene merge policies to + * the composite segment model and orchestrates per-format merge execution. + * @opensearch.experimental + */ +@ExperimentalApi +package org.opensearch.composite.merge; + +import org.opensearch.common.annotation.ExperimentalApi; diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatTests.java index b6be1f41767d9..5a8007c3f58fa 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatTests.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatTests.java @@ -21,19 +21,22 @@ public class CompositeDataFormatTests extends OpenSearchTestCase { public void testNameReturnsComposite() { - CompositeDataFormat format = new CompositeDataFormat(List.of(mockFormat("lucene", 1, Set.of()))); + DataFormat primary = mockFormat("lucene", 1, Set.of()); + CompositeDataFormat format = new CompositeDataFormat(primary, List.of(primary)); assertEquals("composite", format.name()); } public void testPriorityReturnsMinValue() { - CompositeDataFormat format = new CompositeDataFormat(List.of(mockFormat("lucene", 1, Set.of()))); + DataFormat primary = mockFormat("lucene", 1, Set.of()); + CompositeDataFormat format = new CompositeDataFormat(primary, List.of(primary)); assertEquals(Long.MIN_VALUE, format.priority()); } - public void testDefaultConstructorReturnsEmptyFormats() { - CompositeDataFormat format = new CompositeDataFormat(); - assertTrue(format.getDataFormats().isEmpty()); - assertEquals(Set.of(), format.supportedFields()); + public void testGetPrimaryDataformatReturnsPrimary() { + DataFormat primary = mockFormat("lucene", 1, Set.of()); + DataFormat secondary = mockFormat("parquet", 2, Set.of()); + CompositeDataFormat composite = new CompositeDataFormat(primary, List.of(primary, secondary)); + assertSame(primary, composite.getPrimaryDataFormat()); } public void testSupportedFieldsDelegatesToFirstFormat() { @@ -42,36 +45,44 @@ public void testSupportedFieldsDelegatesToFirstFormat() { DataFormat primary = mockFormat("lucene", 1, Set.of(cap1)); DataFormat secondary = mockFormat("parquet", 2, Set.of(cap2)); - CompositeDataFormat composite = new CompositeDataFormat(List.of(primary, secondary)); + CompositeDataFormat composite = new CompositeDataFormat(primary, List.of(primary, secondary)); // supportedFields() returns the first format's fields assertEquals(Set.of(cap1), composite.supportedFields()); } public void testSupportedFieldsEmptyWhenNoFormats() { - CompositeDataFormat composite = new CompositeDataFormat(List.of()); + DataFormat primary = mockFormat("lucene", 1, Set.of()); + CompositeDataFormat composite = new CompositeDataFormat(primary, List.of()); assertEquals(Set.of(), composite.supportedFields()); } public void testGetDataFormatsReturnsAllFormats() { DataFormat f1 = mockFormat("lucene", 1, Set.of()); DataFormat f2 = mockFormat("parquet", 2, Set.of()); - CompositeDataFormat composite = new CompositeDataFormat(List.of(f1, f2)); + CompositeDataFormat composite = new CompositeDataFormat(f1, List.of(f1, f2)); assertEquals(2, composite.getDataFormats().size()); assertSame(f1, composite.getDataFormats().get(0)); assertSame(f2, composite.getDataFormats().get(1)); } public void testGetDataFormatsIsUnmodifiable() { - CompositeDataFormat composite = new CompositeDataFormat(List.of(mockFormat("lucene", 1, Set.of()))); + DataFormat primary = mockFormat("lucene", 1, Set.of()); + CompositeDataFormat composite = new CompositeDataFormat(primary, List.of(primary)); expectThrows(UnsupportedOperationException.class, () -> composite.getDataFormats().add(mockFormat("x", 0, Set.of()))); } - public void testConstructorRejectsNull() { - expectThrows(NullPointerException.class, () -> new CompositeDataFormat(null)); + public void testConstructorRejectsNullDataFormats() { + DataFormat primary = mockFormat("lucene", 1, Set.of()); + expectThrows(NullPointerException.class, () -> new CompositeDataFormat(primary, null)); + } + + public void testConstructorRejectsNullPrimaryDataformat() { + expectThrows(NullPointerException.class, () -> new CompositeDataFormat(null, List.of())); } public void testToStringContainsClassName() { - CompositeDataFormat composite = new CompositeDataFormat(List.of(mockFormat("lucene", 1, Set.of()))); + DataFormat primary = mockFormat("lucene", 1, Set.of()); + CompositeDataFormat composite = new CompositeDataFormat(primary, List.of(primary)); String str = composite.toString(); assertTrue(str.contains("CompositeDataFormat")); assertTrue(str.contains("dataFormats=")); diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java index 41c82a6f44979..59bd20c7a231b 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java @@ -167,7 +167,7 @@ public void testCreateWriterReturnsCompositeWriter() throws IOException { public void testGetMergerDelegatesToPrimary() { CompositeIndexingExecutionEngine engine = CompositeTestHelper.createStubEngine("lucene"); - assertNull(engine.getMerger()); + assertNotNull(engine.getMerger()); } public void testGetNativeBytesUsedSumsAllEngines() { diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java index 5ba2882620d40..f15dadd01fc4a 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java @@ -22,6 +22,7 @@ import org.opensearch.index.engine.dataformat.FileInfos; import org.opensearch.index.engine.dataformat.IndexingEngineConfig; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.MergeResult; import org.opensearch.index.engine.dataformat.Merger; import org.opensearch.index.engine.dataformat.RefreshInput; import org.opensearch.index.engine.dataformat.RefreshResult; @@ -164,7 +165,7 @@ public Writer> createWriter(long writerGeneration) { @Override public Merger getMerger() { - return null; + return mergeInput -> new MergeResult(Map.of()); } @Override diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/merge/CompositeMergerTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/merge/CompositeMergerTests.java new file mode 100644 index 0000000000000..b0a655034c2bc --- /dev/null +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/merge/CompositeMergerTests.java @@ -0,0 +1,629 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite.merge; + +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.concurrent.GatedCloseable; +import org.opensearch.common.settings.Settings; +import org.opensearch.composite.CompositeDataFormat; +import org.opensearch.composite.CompositeIndexingExecutionEngine; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; +import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.MergeResult; +import org.opensearch.index.engine.dataformat.Merger; +import org.opensearch.index.engine.dataformat.RowIdMapping; +import org.opensearch.index.engine.dataformat.merge.DataFormatAwareMergePolicy; +import org.opensearch.index.engine.dataformat.merge.MergeHandler; +import org.opensearch.index.engine.dataformat.merge.OneMerge; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; +import org.opensearch.index.engine.exec.coord.CatalogSnapshot; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Tests for {@link CompositeMerger}. + */ +public class CompositeMergerTests extends OpenSearchTestCase { + + private static final ShardId SHARD_ID = new ShardId(new Index("test-index", "uuid"), 0); + private static final RowIdMapping STUB_ROW_ID_MAPPING = (oldId, oldGen) -> oldId; + + private DataFormat primaryFormat; + private DataFormat secondaryFormat; + private Merger primaryMerger; + private Merger secondaryMerger; + private CompositeIndexingExecutionEngine compositeEngine; + private CompositeDataFormat compositeDataFormat; + private Supplier> snapshotSupplier; + + @Override + public void setUp() throws Exception { + super.setUp(); + primaryFormat = stubFormat("lucene"); + secondaryFormat = stubFormat("parquet"); + primaryMerger = mock(Merger.class); + secondaryMerger = mock(Merger.class); + snapshotSupplier = () -> new GatedCloseable<>(null, () -> {}); + + IndexingExecutionEngine primaryEngine = mockEngine(primaryFormat, primaryMerger); + IndexingExecutionEngine secondaryEngine = mockEngine(secondaryFormat, secondaryMerger); + + compositeEngine = mock(CompositeIndexingExecutionEngine.class); + doReturn(primaryEngine).when(compositeEngine).getPrimaryDelegate(); + doReturn(Set.of(secondaryEngine)).when(compositeEngine).getSecondaryDelegates(); + when(compositeEngine.getNextWriterGeneration()).thenReturn(99L); + + compositeDataFormat = new CompositeDataFormat(primaryFormat, List.of(primaryFormat, secondaryFormat)); + } + + // ========== doMerge: successful primary + secondary ========== + + public void testDoMergeSuccessWithPrimaryAndSecondary() throws IOException { + Path tempDir = createTempDir(); + WriterFileSet primaryWfs = wfs(tempDir, 1L, Set.of("p1.dat"), 10); + WriterFileSet secondaryWfs = wfs(tempDir, 1L, Set.of("s1.dat"), 10); + + Segment segment = buildSegment(0L, primaryFormat, primaryWfs, secondaryFormat, secondaryWfs); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + WriterFileSet mergedPrimaryWfs = wfs(tempDir, 99L, Set.of("mp.dat"), 10); + WriterFileSet mergedSecondaryWfs = wfs(tempDir, 99L, Set.of("ms.dat"), 10); + + MergeResult primaryResult = new MergeResult(Map.of(primaryFormat, mergedPrimaryWfs), STUB_ROW_ID_MAPPING); + MergeResult secondaryResult = new MergeResult(Map.of(secondaryFormat, mergedSecondaryWfs)); + + when(primaryMerger.merge(any())).thenReturn(primaryResult); + when(secondaryMerger.merge(any())).thenReturn(secondaryResult); + + MergeHandler handler = createHandler(); + MergeResult result = handler.doMerge(oneMerge); + + assertNotNull(result); + assertEquals(2, result.getMergedWriterFileSet().size()); + assertSame(mergedPrimaryWfs, result.getMergedWriterFileSetForDataformat(primaryFormat)); + assertSame(mergedSecondaryWfs, result.getMergedWriterFileSetForDataformat(secondaryFormat)); + } + + // ========== doMerge: primary only (no secondaries) ========== + + public void testDoMergePrimaryOnlyNoSecondaries() throws IOException { + CompositeIndexingExecutionEngine engineNoSecondary = mock(CompositeIndexingExecutionEngine.class); + IndexingExecutionEngine primaryEngine = mockEngine(primaryFormat, primaryMerger); + doReturn(primaryEngine).when(engineNoSecondary).getPrimaryDelegate(); + doReturn(Set.of()).when(engineNoSecondary).getSecondaryDelegates(); + when(engineNoSecondary.getNextWriterGeneration()).thenReturn(50L); + + CompositeDataFormat primaryOnlyFormat = new CompositeDataFormat(primaryFormat, List.of(primaryFormat)); + + Path tempDir = createTempDir(); + WriterFileSet primaryWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + Segment segment = Segment.builder(0L).addSearchableFiles(primaryFormat, primaryWfs).build(); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + WriterFileSet mergedWfs = wfs(tempDir, 50L, Set.of("merged.dat"), 5); + MergeResult primaryResult = new MergeResult(Map.of(primaryFormat, mergedWfs)); + when(primaryMerger.merge(any())).thenReturn(primaryResult); + + MergeHandler handler = new MergeHandler( + snapshotSupplier, + new CompositeMerger(engineNoSecondary, primaryOnlyFormat), + SHARD_ID, + mock(MergeHandler.MergePolicy.class), + mock(MergeHandler.MergeListener.class) + ); + + MergeResult result = handler.doMerge(oneMerge); + assertNotNull(result); + assertEquals(1, result.getMergedWriterFileSet().size()); + assertSame(mergedWfs, result.getMergedWriterFileSetForDataformat(primaryFormat)); + } + + // ========== doMerge: primary merge throws IOException ========== + + public void testDoMergePrimaryFailureThrowsUncheckedIOException() throws IOException { + Path tempDir = createTempDir(); + WriterFileSet primaryWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + WriterFileSet secondaryWfs = wfs(tempDir, 1L, Set.of("s.dat"), 5); + Segment segment = buildSegment(0L, primaryFormat, primaryWfs, secondaryFormat, secondaryWfs); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + when(primaryMerger.merge(any())).thenThrow(new IOException("primary disk error")); + + MergeHandler handler = createHandler(); + UncheckedIOException ex = expectThrows(UncheckedIOException.class, () -> handler.doMerge(oneMerge)); + assertNotNull(ex.getCause()); + assertEquals("primary disk error", ex.getCause().getMessage()); + } + + // ========== doMerge: single secondary failure ========== + + public void testDoMergeSingleSecondaryFailureThrowsUncheckedIOException() throws IOException { + Path tempDir = createTempDir(); + WriterFileSet primaryWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + WriterFileSet secondaryWfs = wfs(tempDir, 1L, Set.of("s.dat"), 5); + Segment segment = buildSegment(0L, primaryFormat, primaryWfs, secondaryFormat, secondaryWfs); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + WriterFileSet mergedPrimaryWfs = wfs(tempDir, 99L, Set.of("mp.dat"), 5); + MergeResult primaryResult = new MergeResult(Map.of(primaryFormat, mergedPrimaryWfs), STUB_ROW_ID_MAPPING); + when(primaryMerger.merge(any())).thenReturn(primaryResult); + when(secondaryMerger.merge(any())).thenThrow(new IOException("secondary disk error")); + + MergeHandler handler = createHandler(); + UncheckedIOException ex = expectThrows(UncheckedIOException.class, () -> handler.doMerge(oneMerge)); + assertNotNull(ex.getCause()); + assertEquals("secondary disk error", ex.getCause().getMessage()); + } + + // ========== doMerge: multiple secondaries — fails fast on first error ========== + + public void testDoMergeMultipleSecondariesFailsFastOnFirstError() throws IOException { + DataFormat secondaryFormat2 = stubFormat("arrow"); + Merger secondaryMerger2 = mock(Merger.class); + + CompositeIndexingExecutionEngine multiEngine = mock(CompositeIndexingExecutionEngine.class); + IndexingExecutionEngine primaryEngine = mockEngine(primaryFormat, primaryMerger); + doReturn(primaryEngine).when(multiEngine).getPrimaryDelegate(); + doReturn(Set.of(mockEngine(secondaryFormat, secondaryMerger), mockEngine(secondaryFormat2, secondaryMerger2))).when(multiEngine) + .getSecondaryDelegates(); + when(multiEngine.getNextWriterGeneration()).thenReturn(99L); + + CompositeDataFormat multiFormat = new CompositeDataFormat(primaryFormat, List.of(primaryFormat, secondaryFormat, secondaryFormat2)); + + Path tempDir = createTempDir(); + WriterFileSet pWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + WriterFileSet sWfs = wfs(tempDir, 1L, Set.of("s.dat"), 5); + WriterFileSet s2Wfs = wfs(tempDir, 1L, Set.of("s2.dat"), 5); + Segment segment = Segment.builder(0L) + .addSearchableFiles(primaryFormat, pWfs) + .addSearchableFiles(secondaryFormat, sWfs) + .addSearchableFiles(secondaryFormat2, s2Wfs) + .build(); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + WriterFileSet mergedPWfs = wfs(tempDir, 99L, Set.of("mp.dat"), 5); + MergeResult primaryResult = new MergeResult(Map.of(primaryFormat, mergedPWfs), STUB_ROW_ID_MAPPING); + when(primaryMerger.merge(any())).thenReturn(primaryResult); + when(secondaryMerger.merge(any())).thenThrow(new IOException("parquet error")); + when(secondaryMerger2.merge(any())).thenThrow(new IOException("arrow error")); + + MergeHandler handler = new MergeHandler( + snapshotSupplier, + new CompositeMerger(multiEngine, multiFormat), + SHARD_ID, + mock(MergeHandler.MergePolicy.class), + mock(MergeHandler.MergeListener.class) + ); + + UncheckedIOException ex = expectThrows(UncheckedIOException.class, () -> handler.doMerge(oneMerge)); + assertNotNull(ex.getCause()); + // Fail-fast: only the first secondary failure is reported, no suppressed exceptions + assertEquals(0, ex.getCause().getSuppressed().length); + } + + // ========== doMerge: missing rowIdMapping throws IllegalStateException ========== + + public void testDoMergeMissingRowIdMappingThrowsIllegalState() throws IOException { + Path tempDir = createTempDir(); + WriterFileSet primaryWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + WriterFileSet secondaryWfs = wfs(tempDir, 1L, Set.of("s.dat"), 5); + Segment segment = buildSegment(0L, primaryFormat, primaryWfs, secondaryFormat, secondaryWfs); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + WriterFileSet mergedPrimaryWfs = wfs(tempDir, 99L, Set.of("mp.dat"), 5); + // Primary result without rowIdMapping + MergeResult primaryResult = new MergeResult(Map.of(primaryFormat, mergedPrimaryWfs)); + when(primaryMerger.merge(any())).thenReturn(primaryResult); + + MergeHandler handler = createHandler(); + IllegalStateException ex = expectThrows(IllegalStateException.class, () -> handler.doMerge(oneMerge)); + assertTrue(ex.getMessage().contains("row-ID mapping")); + assertTrue(ex.getMessage().contains("secondaries")); + } + + // ========== doMerge: cleanup on failure deletes stale files ========== + + public void testDoMergeCleanupDeletesStaleMergedFilesOnFailure() throws IOException { + Path tempDir = createTempDir(); + + Path staleFile = tempDir.resolve("mp.dat"); + Files.createFile(staleFile); + assertTrue(Files.exists(staleFile)); + + WriterFileSet primaryWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + WriterFileSet secondaryWfs = wfs(tempDir, 1L, Set.of("s.dat"), 5); + Segment segment = buildSegment(0L, primaryFormat, primaryWfs, secondaryFormat, secondaryWfs); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + WriterFileSet mergedPrimaryWfs = wfs(tempDir, 99L, Set.of("mp.dat"), 5); + MergeResult primaryResult = new MergeResult(Map.of(primaryFormat, mergedPrimaryWfs), STUB_ROW_ID_MAPPING); + when(primaryMerger.merge(any())).thenReturn(primaryResult); + when(secondaryMerger.merge(any())).thenThrow(new IOException("secondary fail")); + + MergeHandler handler = createHandler(); + expectThrows(UncheckedIOException.class, () -> handler.doMerge(oneMerge)); + + assertFalse("Stale merged file should be deleted on failure", Files.exists(staleFile)); + } + + // ========== doMerge: cleanup handles non-existent files gracefully ========== + + public void testDoMergeCleanupHandlesNonExistentFilesGracefully() throws IOException { + Path tempDir = createTempDir(); + + WriterFileSet primaryWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + WriterFileSet secondaryWfs = wfs(tempDir, 1L, Set.of("s.dat"), 5); + Segment segment = buildSegment(0L, primaryFormat, primaryWfs, secondaryFormat, secondaryWfs); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + WriterFileSet mergedPrimaryWfs = wfs(tempDir, 99L, Set.of("nonexistent.dat"), 5); + MergeResult primaryResult = new MergeResult(Map.of(primaryFormat, mergedPrimaryWfs), STUB_ROW_ID_MAPPING); + when(primaryMerger.merge(any())).thenReturn(primaryResult); + when(secondaryMerger.merge(any())).thenThrow(new IOException("fail")); + + MergeHandler handler = createHandler(); + // Should not throw during cleanup even though file doesn't exist + expectThrows(UncheckedIOException.class, () -> handler.doMerge(oneMerge)); + } + + // ========== doMerge: no cleanup when mergedWriterFileSet is empty ========== + + public void testDoMergeNoCleanupWhenPrimaryFails() throws IOException { + Path tempDir = createTempDir(); + WriterFileSet primaryWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + WriterFileSet secondaryWfs = wfs(tempDir, 1L, Set.of("s.dat"), 5); + Segment segment = buildSegment(0L, primaryFormat, primaryWfs, secondaryFormat, secondaryWfs); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + when(primaryMerger.merge(any())).thenThrow(new IOException("primary fail")); + + MergeHandler handler = createHandler(); + UncheckedIOException ex = expectThrows(UncheckedIOException.class, () -> handler.doMerge(oneMerge)); + assertEquals("primary fail", ex.getCause().getMessage()); + } + + // ========== doMerge: multiple segments ========== + + public void testDoMergeWithMultipleSegments() throws IOException { + Path tempDir = createTempDir(); + WriterFileSet pWfs1 = wfs(tempDir, 1L, Set.of("p1.dat"), 5); + WriterFileSet sWfs1 = wfs(tempDir, 1L, Set.of("s1.dat"), 5); + WriterFileSet pWfs2 = wfs(tempDir, 2L, Set.of("p2.dat"), 5); + WriterFileSet sWfs2 = wfs(tempDir, 2L, Set.of("s2.dat"), 5); + + Segment seg1 = buildSegment(1L, primaryFormat, pWfs1, secondaryFormat, sWfs1); + Segment seg2 = buildSegment(2L, primaryFormat, pWfs2, secondaryFormat, sWfs2); + OneMerge oneMerge = new OneMerge(List.of(seg1, seg2)); + + WriterFileSet mergedPWfs = wfs(tempDir, 99L, Set.of("mp.dat"), 10); + WriterFileSet mergedSWfs = wfs(tempDir, 99L, Set.of("ms.dat"), 10); + MergeResult primaryResult = new MergeResult(Map.of(primaryFormat, mergedPWfs), STUB_ROW_ID_MAPPING); + MergeResult secondaryResult = new MergeResult(Map.of(secondaryFormat, mergedSWfs)); + + when(primaryMerger.merge(any())).thenReturn(primaryResult); + when(secondaryMerger.merge(any())).thenReturn(secondaryResult); + + MergeHandler handler = createHandler(); + MergeResult result = handler.doMerge(oneMerge); + + assertNotNull(result); + assertEquals(2, result.getMergedWriterFileSet().size()); + verify(primaryMerger, times(1)).merge(any()); + verify(secondaryMerger, times(1)).merge(any()); + } + + // ========== doMerge: secondary format equals primary is skipped ========== + + public void testDoMergeSkipsSecondaryThatEqualsPrimary() throws IOException { + // The duplicate secondary has the same DataFormat as primary, so it should be skipped + // in the secondary loop. We use the same primaryMerger for both to avoid NPE in the + // constructor's dataFormatMergerMap (last-write-wins for same key). + IndexingExecutionEngine primaryEngine = mockEngine(primaryFormat, primaryMerger); + IndexingExecutionEngine duplicateEngine = mockEngine(primaryFormat, primaryMerger); + + CompositeIndexingExecutionEngine dupEngine = mock(CompositeIndexingExecutionEngine.class); + doReturn(primaryEngine).when(dupEngine).getPrimaryDelegate(); + doReturn(Set.of(duplicateEngine)).when(dupEngine).getSecondaryDelegates(); + when(dupEngine.getNextWriterGeneration()).thenReturn(99L); + + CompositeDataFormat dupFormat = new CompositeDataFormat(primaryFormat, List.of(primaryFormat)); + + Path tempDir = createTempDir(); + WriterFileSet pWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + Segment segment = Segment.builder(0L).addSearchableFiles(primaryFormat, pWfs).build(); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + WriterFileSet mergedWfs = wfs(tempDir, 99L, Set.of("mp.dat"), 5); + MergeResult primaryResult = new MergeResult(Map.of(primaryFormat, mergedWfs), STUB_ROW_ID_MAPPING); + when(primaryMerger.merge(any())).thenReturn(primaryResult); + + MergeHandler handler = new MergeHandler( + snapshotSupplier, + new CompositeMerger(dupEngine, dupFormat), + SHARD_ID, + mock(MergeHandler.MergePolicy.class), + mock(MergeHandler.MergeListener.class) + ); + + MergeResult result = handler.doMerge(oneMerge); + assertNotNull(result); + assertEquals(1, result.getMergedWriterFileSet().size()); + } + + // ========== findMerges ========== + + public void testFindMergesReturnsEmptyWhenNoSegments() { + CatalogSnapshot catalogSnapshot = mockCatalogSnapshot(Collections.emptyList()); + snapshotSupplier = () -> new GatedCloseable<>(catalogSnapshot, () -> {}); + + MergeHandler handler = createHandler(); + Collection merges = handler.findMerges(); + assertNotNull(merges); + assertTrue(merges.isEmpty()); + } + + public void testFindMergesThrowsOnSnapshotFailure() { + snapshotSupplier = () -> { throw new RuntimeException("snapshot unavailable"); }; + + MergeHandler handler = createHandler(); + RuntimeException ex = expectThrows(RuntimeException.class, handler::findMerges); + assertTrue(ex.getMessage().contains("snapshot unavailable")); + } + + // ========== findForceMerges ========== + + public void testFindForceMergesReturnsEmptyWhenNoSegments() { + CatalogSnapshot catalogSnapshot = mockCatalogSnapshot(Collections.emptyList()); + snapshotSupplier = () -> new GatedCloseable<>(catalogSnapshot, () -> {}); + + MergeHandler handler = createHandler(); + Collection merges = handler.findForceMerges(1); + assertNotNull(merges); + assertTrue(merges.isEmpty()); + } + + public void testFindForceMergesThrowsOnSnapshotFailure() { + snapshotSupplier = () -> { throw new RuntimeException("snapshot unavailable"); }; + + MergeHandler handler = createHandler(); + RuntimeException ex = expectThrows(RuntimeException.class, () -> handler.findForceMerges(1)); + assertTrue(ex.getMessage().contains("snapshot unavailable")); + } + + // ========== registerMerge / onMergeFinished / onMergeFailure ========== + + public void testRegisterMergeAndOnMergeFinished() { + Path tempDir = createTempDir(); + WriterFileSet pWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + Segment segment = Segment.builder(0L).addSearchableFiles(primaryFormat, pWfs).build(); + + CatalogSnapshot catalogSnapshot = mockCatalogSnapshot(List.of(segment)); + snapshotSupplier = () -> new GatedCloseable<>(catalogSnapshot, () -> {}); + + MergeHandler handler = createHandler(); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + handler.registerMerge(oneMerge); + assertTrue(handler.hasPendingMerges()); + + handler.onMergeFinished(oneMerge); + } + + public void testRegisterMergeAndOnMergeFailure() { + Path tempDir = createTempDir(); + WriterFileSet pWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + Segment segment = Segment.builder(0L).addSearchableFiles(primaryFormat, pWfs).build(); + + CatalogSnapshot catalogSnapshot = mockCatalogSnapshot(List.of(segment)); + snapshotSupplier = () -> new GatedCloseable<>(catalogSnapshot, () -> {}); + + MergeHandler handler = createHandler(); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + handler.registerMerge(oneMerge); + assertTrue(handler.hasPendingMerges()); + + handler.onMergeFailure(oneMerge); + assertFalse(handler.hasPendingMerges()); + } + + public void testGetNextMergeReturnsNullWhenEmpty() { + MergeHandler handler = createHandler(); + assertNull(handler.getNextMerge()); + assertFalse(handler.hasPendingMerges()); + } + + public void testGetNextMergeReturnsMergeAfterRegister() { + Path tempDir = createTempDir(); + WriterFileSet pWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + Segment segment = Segment.builder(0L).addSearchableFiles(primaryFormat, pWfs).build(); + + CatalogSnapshot catalogSnapshot = mockCatalogSnapshot(List.of(segment)); + snapshotSupplier = () -> new GatedCloseable<>(catalogSnapshot, () -> {}); + + MergeHandler handler = createHandler(); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + handler.registerMerge(oneMerge); + OneMerge retrieved = handler.getNextMerge(); + assertNotNull(retrieved); + assertSame(oneMerge, retrieved); + assertFalse(handler.hasPendingMerges()); + } + + // ========== findMerges with merge candidates ========== + + public void testFindMergesReturnsMergeCandidates() throws IOException { + Path tempDir = createTempDir(); + // Create many small segments with real files to trigger TieredMergePolicy + List segments = new java.util.ArrayList<>(); + for (int i = 0; i < 15; i++) { + Path file = tempDir.resolve("seg" + i + ".dat"); + Files.write(file, new byte[100]); + WriterFileSet pWfs = wfs(tempDir, i, Set.of("seg" + i + ".dat"), 10); + segments.add(Segment.builder(i).addSearchableFiles(primaryFormat, pWfs).build()); + } + + CatalogSnapshot catalogSnapshot = mockCatalogSnapshot(segments); + snapshotSupplier = () -> new GatedCloseable<>(catalogSnapshot, () -> {}); + + MergeHandler handler = createHandlerWithRealPolicy(); + Collection merges = handler.findMerges(); + assertNotNull(merges); + // TieredMergePolicy should find merge candidates with 15 small segments + assertFalse("Expected merge candidates from 15 small segments", merges.isEmpty()); + for (OneMerge merge : merges) { + assertFalse(merge.getSegmentsToMerge().isEmpty()); + } + } + + // ========== findForceMerges with merge candidates ========== + + public void testFindForceMergesReturnsMergeCandidates() throws IOException { + Path tempDir = createTempDir(); + List segments = new java.util.ArrayList<>(); + for (int i = 0; i < 5; i++) { + Path file = tempDir.resolve("fseg" + i + ".dat"); + Files.write(file, new byte[100]); + WriterFileSet pWfs = wfs(tempDir, i, Set.of("fseg" + i + ".dat"), 10); + segments.add(Segment.builder(i).addSearchableFiles(primaryFormat, pWfs).build()); + } + + CatalogSnapshot catalogSnapshot = mockCatalogSnapshot(segments); + snapshotSupplier = () -> new GatedCloseable<>(catalogSnapshot, () -> {}); + + MergeHandler handler = createHandlerWithRealPolicy(); + // Force merge down to 1 segment should produce candidates + Collection merges = handler.findForceMerges(1); + assertNotNull(merges); + assertFalse("Expected force merge candidates when targeting 1 segment from 5", merges.isEmpty()); + } + + // ========== cleanup: exception during file deletion is logged but not thrown ========== + + public void testCleanupStaleMergedFilesLogsExceptionOnDeleteFailure() throws IOException { + Path tempDir = createTempDir(); + // Create a directory with the same name as the file to delete — deleteIfExists on a + // non-empty directory throws DirectoryNotEmptyException + Path dirAsFile = tempDir.resolve("mp.dat"); + Files.createDirectory(dirAsFile); + Files.createFile(dirAsFile.resolve("child.txt")); + + WriterFileSet primaryWfs = wfs(tempDir, 1L, Set.of("p.dat"), 5); + WriterFileSet secondaryWfs = wfs(tempDir, 1L, Set.of("s.dat"), 5); + Segment segment = buildSegment(0L, primaryFormat, primaryWfs, secondaryFormat, secondaryWfs); + OneMerge oneMerge = new OneMerge(List.of(segment)); + + // mergedPrimaryWfs points to "mp.dat" which is a non-empty directory + WriterFileSet mergedPrimaryWfs = wfs(tempDir, 99L, Set.of("mp.dat"), 5); + MergeResult primaryResult = new MergeResult(Map.of(primaryFormat, mergedPrimaryWfs), STUB_ROW_ID_MAPPING); + when(primaryMerger.merge(any())).thenReturn(primaryResult); + when(secondaryMerger.merge(any())).thenThrow(new IOException("secondary fail")); + + MergeHandler handler = createHandler(); + // The merge fails due to secondary, cleanup tries to delete "mp.dat" (a non-empty dir) + // which throws DirectoryNotEmptyException — caught and logged, not re-thrown + expectThrows(UncheckedIOException.class, () -> handler.doMerge(oneMerge)); + // The directory should still exist since deleteIfExists fails on non-empty dirs + assertTrue(Files.exists(dirAsFile)); + } + + // ========== Helper methods ========== + + private MergeHandler createHandler() { + return new MergeHandler( + snapshotSupplier, + new CompositeMerger(compositeEngine, compositeDataFormat), + SHARD_ID, + mock(MergeHandler.MergePolicy.class), + mock(MergeHandler.MergeListener.class) + ); + } + + private MergeHandler createHandlerWithRealPolicy() { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .build(); + IndexMetadata indexMetadata = IndexMetadata.builder("test-index").settings(settings).build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(indexSettings.getMergePolicy(true), SHARD_ID); + return new MergeHandler(snapshotSupplier, new CompositeMerger(compositeEngine, compositeDataFormat), SHARD_ID, policy, policy); + } + + private static DataFormat stubFormat(String name) { + return new DataFormat() { + @Override + public String name() { + return name; + } + + @Override + public long priority() { + return 1; + } + + @Override + public Set supportedFields() { + return Set.of(); + } + + @Override + public String toString() { + return "StubFormat{" + name + "}"; + } + }; + } + + @SuppressWarnings("unchecked") + private static IndexingExecutionEngine mockEngine(DataFormat format, Merger merger) { + IndexingExecutionEngine engine = mock(IndexingExecutionEngine.class); + when(engine.getDataFormat()).thenReturn(format); + when(engine.getMerger()).thenReturn(merger); + return engine; + } + + private static WriterFileSet wfs(Path dir, long gen, Set files, long numRows) { + return new WriterFileSet(dir.toString(), gen, files, numRows); + } + + private static Segment buildSegment(long generation, DataFormat fmt1, WriterFileSet wfs1, DataFormat fmt2, WriterFileSet wfs2) { + return Segment.builder(generation).addSearchableFiles(fmt1, wfs1).addSearchableFiles(fmt2, wfs2).build(); + } + + private static CatalogSnapshot mockCatalogSnapshot(List segments) { + CatalogSnapshot snapshot = mock(CatalogSnapshot.class); + when(snapshot.getSegments()).thenReturn(segments); + return snapshot; + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java index b47103b8251fe..1d5c43522e418 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java @@ -14,6 +14,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.MergeResult; import org.opensearch.index.engine.dataformat.Merger; import org.opensearch.index.engine.dataformat.RefreshInput; import org.opensearch.index.engine.dataformat.RefreshResult; @@ -167,7 +168,8 @@ public long getNativeBytesUsed() { @Override public Merger getMerger() { - return null; + // TODO: Implement merge support as ParquetMerger + return mergeInput -> new MergeResult(Map.of()); } @Override diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java index 92504864cf60f..51cc7619adfe7 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java @@ -127,8 +127,8 @@ public void testRefreshWithNullInput() throws Exception { assertTrue(result.refreshedSegments().isEmpty()); } - public void testGetMergerReturnsNull() { - assertNull(engine.getMerger()); + public void testGetMergerReturnsNoopMerger() { + assertNotNull(engine.getMerger()); } public void testGetNextWriterGenerationThrows() { diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index 7e299abd8d943..e593e4fa16537 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -24,6 +24,7 @@ import static org.opensearch.Version.V_2_7_0; import static org.opensearch.Version.V_3_0_0; import static org.opensearch.Version.V_3_2_0; +import static org.opensearch.Version.V_3_7_0; /** * Utility class to register server exceptions @@ -1241,5 +1242,13 @@ public static void registerExceptions() { V_3_2_0 ) ); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.index.engine.dataformat.merge.MergeFailedEngineException.class, + org.opensearch.index.engine.dataformat.merge.MergeFailedEngineException::new, + 178, + V_3_7_0 + ) + ); } } diff --git a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java index 0081c382965e5..bfb919cba8e6d 100644 --- a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.Term; import org.apache.lucene.store.AlreadyClosedException; +import org.opensearch.common.Booleans; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; import org.opensearch.common.annotation.ExperimentalApi; @@ -33,11 +34,17 @@ import org.opensearch.index.engine.dataformat.FileInfos; import org.opensearch.index.engine.dataformat.IndexingEngineConfig; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.MergeResult; import org.opensearch.index.engine.dataformat.ReaderManagerConfig; import org.opensearch.index.engine.dataformat.RefreshInput; import org.opensearch.index.engine.dataformat.RefreshResult; import org.opensearch.index.engine.dataformat.WriteResult; import org.opensearch.index.engine.dataformat.Writer; +import org.opensearch.index.engine.dataformat.merge.DataFormatAwareMergePolicy; +import org.opensearch.index.engine.dataformat.merge.MergeFailedEngineException; +import org.opensearch.index.engine.dataformat.merge.MergeHandler; +import org.opensearch.index.engine.dataformat.merge.MergeScheduler; +import org.opensearch.index.engine.dataformat.merge.OneMerge; import org.opensearch.index.engine.exec.CatalogSnapshotLifecycleListener; import org.opensearch.index.engine.exec.CombinedCatalogSnapshotDeletionPolicy; import org.opensearch.index.engine.exec.EngineReaderManager; @@ -164,6 +171,19 @@ public class DataFormatAwareEngine implements Indexer { // Refresh tracker private final LastRefreshedCheckpointListener lastRefreshedCheckpointListener; + // Merge + private final MergeScheduler mergeScheduler; + + /** + * System property to enable or disable pluggable dataformat merge operations. + * Set to "true" to enable merges (e.g., {@code -Dopensearch.pluggable.dataformat.merge.enabled=true}). + * Defaults to "false" (merges disabled) as the merge implementations are not yet complete + * for all data formats. + *

+ * TODO: Remove this flag once merge implementations are complete for all data formats. + */ + static final String MERGE_ENABLED_PROPERTY = "opensearch.pluggable.dataformat.merge.enabled"; + @Nullable private final String historyUUID; @@ -291,6 +311,28 @@ public DataFormatAwareEngine(EngineConfig engineConfig) { assert indexingExecutionEngine != null : "indexing execution engine must be initialized"; assert committer != null : "committer must be initialized"; assert writerPool != null : "writer pool must be initialized"; + + DataFormatAwareMergePolicy dataFormatAwareMergePolicy = new DataFormatAwareMergePolicy( + engineConfig.getIndexSettings().getMergePolicy(true), + shardId + ); + + // Merge + MergeHandler mergeHandler = new MergeHandler( + this::acquireSnapshot, + indexingExecutionEngine.getMerger(), + shardId, + dataFormatAwareMergePolicy, + dataFormatAwareMergePolicy + ); + this.mergeScheduler = new MergeScheduler( + mergeHandler, + this::applyMergeChanges, + shardId, + engineConfig.getIndexSettings(), + engineConfig.getThreadPool() + ); + success = true; logger.trace("created new DataFormatBasedEngine"); } catch (IOException | TranslogCorruptedException e) { @@ -688,13 +730,8 @@ public void refresh(String source) throws EngineException { + result.refreshedSegments().size(); catalogSnapshotManager.commitNewSnapshot(result.refreshedSegments()); - // TODO: Add other Refresh listeners - // Notify reader managers so they can create readers for the new snapshot try (GatedCloseable newSnapshotRef = catalogSnapshotManager.acquireSnapshot()) { - CatalogSnapshot newSnapshot = newSnapshotRef.get(); - for (EngineReaderManager rm : readerManagers.values()) { - rm.afterRefresh(refreshed, newSnapshot); - } + refreshListeners(refreshed, newSnapshotRef.get()); } } } finally { @@ -702,6 +739,7 @@ public void refresh(String source) throws EngineException { } if (refreshed) { lastRefreshedCheckpointListener.updateRefreshedCheckpoint(localCheckpointBeforeRefresh); + triggerPossibleMerges(); // trigger merges } } } finally { @@ -893,6 +931,9 @@ public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue tran final TranslogDeletionPolicy translogDeletionPolicy = translogManager.getDeletionPolicy(); translogDeletionPolicy.setRetentionAgeInMillis(translogRetentionAge.millis()); translogDeletionPolicy.setRetentionSizeInBytes(translogRetentionSize.getBytes()); + + // This checks if the settings related to merge are changed and based on that updates the local variables in the class + mergeScheduler.refreshConfig(); } /** {@inheritDoc} Always returns {@code true} — a refresh is always considered needed. */ @@ -1056,8 +1097,7 @@ public PollingIngestStats pollingIngestStats() { @Override public MergeStats getMergeStats() { - // TODO: MergeHandler to provide this. - return new MergeStats(); + return mergeScheduler.stats(); } @Override @@ -1252,6 +1292,42 @@ public void close() throws IOException { awaitPendingClose(); } + private void applyMergeChanges(MergeResult mergeResult, OneMerge oneMerge) { + refreshLock.lock(); + try { + catalogSnapshotManager.applyMergeResults(mergeResult, oneMerge); + try (GatedCloseable newSnapshotRef = catalogSnapshotManager.acquireSnapshot()) { + refreshListeners(true, newSnapshotRef.get()); + } + } catch (Exception ex) { + try { + logger.error(() -> new ParameterizedMessage("Merge failed while registering merged files in Snapshot"), ex); + failEngine("Merge failed while registering merged files in Snapshot", ex); + } catch (Exception inner) { + ex.addSuppressed(inner); + } + throw new MergeFailedEngineException(shardId, ex); + } finally { + refreshLock.unlock(); + } + } + + private void refreshListeners(boolean refreshed, CatalogSnapshot catalogSnapshot) throws IOException { + // TODO: Add other Refresh listeners + // Notify reader managers so they can create readers for the new snapshot + for (EngineReaderManager rm : readerManagers.values()) { + rm.afterRefresh(refreshed, catalogSnapshot); + } + } + + private void triggerPossibleMerges() { + if (Booleans.parseBoolean(System.getProperty(MERGE_ENABLED_PROPERTY, Boolean.FALSE.toString())) == false) { + logger.debug("Pluggable dataformat merge is disabled via system property [{}], skipping merge", MERGE_ENABLED_PROPERTY); + return; + } + mergeScheduler.triggerMerges(); + } + private void closeNoLock(String reason) { if (isClosed.compareAndSet(false, true)) { assert rwl.isWriteLockedByCurrentThread() || failEngineLock.isHeldByCurrentThread() diff --git a/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java b/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java index e79ca86daef04..cf313b2e95f9c 100644 --- a/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java +++ b/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java @@ -37,8 +37,6 @@ import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.MergeScheduler; import org.opensearch.common.logging.Loggers; -import org.opensearch.common.metrics.CounterMetric; -import org.opensearch.common.metrics.MeanMetric; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.OpenSearchExecutors; @@ -47,6 +45,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.MergeSchedulerConfig; import org.opensearch.index.merge.MergeStats; +import org.opensearch.index.merge.MergeStatsTracker; import org.opensearch.index.merge.MergedSegmentTransferTracker; import org.opensearch.index.merge.OnGoingMerge; @@ -67,14 +66,7 @@ class OpenSearchConcurrentMergeScheduler extends ConcurrentMergeScheduler { private final IndexSettings indexSettings; private final ShardId shardId; - private final MeanMetric totalMerges = new MeanMetric(); - private final CounterMetric totalMergesNumDocs = new CounterMetric(); - private final CounterMetric totalMergesSizeInBytes = new CounterMetric(); - private final CounterMetric currentMerges = new CounterMetric(); - private final CounterMetric currentMergesNumDocs = new CounterMetric(); - private final CounterMetric currentMergesSizeInBytes = new CounterMetric(); - private final CounterMetric totalMergeStoppedTime = new CounterMetric(); - private final CounterMetric totalMergeThrottledTime = new CounterMetric(); + private final MergeStatsTracker mergeStatsTracker = new MergeStatsTracker(); private final Set onGoingMerges = ConcurrentCollections.newConcurrentSet(); private final Set readOnlyOnGoingMerges = Collections.unmodifiableSet(onGoingMerges); @@ -110,9 +102,7 @@ protected void doMerge(MergeSource mergeSource, MergePolicy.OneMerge merge) thro int totalNumDocs = merge.totalNumDocs(); long totalSizeInBytes = merge.totalBytesSize(); long timeNS = System.nanoTime(); - currentMerges.inc(); - currentMergesNumDocs.inc(totalNumDocs); - currentMergesSizeInBytes.inc(totalSizeInBytes); + mergeStatsTracker.beforeMerge(totalNumDocs, totalSizeInBytes); OnGoingMerge onGoingMerge = new OnGoingMerge(merge); onGoingMerges.add(onGoingMerge); @@ -136,21 +126,16 @@ protected void doMerge(MergeSource mergeSource, MergePolicy.OneMerge merge) thro onGoingMerges.remove(onGoingMerge); afterMerge(onGoingMerge); - currentMerges.dec(); - currentMergesNumDocs.dec(totalNumDocs); - currentMergesSizeInBytes.dec(totalSizeInBytes); + mergeStatsTracker.afterMerge(tookMS, totalNumDocs, totalSizeInBytes); - totalMergesNumDocs.inc(totalNumDocs); - totalMergesSizeInBytes.inc(totalSizeInBytes); - totalMerges.inc(tookMS); long stoppedMS = TimeValue.nsecToMSec( merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.STOPPED) ); long throttledMS = TimeValue.nsecToMSec( merge.getMergeProgress().getPauseTimes().get(MergePolicy.OneMergeProgress.PauseReason.PAUSED) ); - totalMergeStoppedTime.inc(stoppedMS); - totalMergeThrottledTime.inc(throttledMS); + mergeStatsTracker.incStoppedTime(stoppedMS); + mergeStatsTracker.incThrottledTime(throttledMS); String message = String.format( Locale.ROOT, @@ -207,20 +192,10 @@ protected MergeThread getMergeThread(MergeSource mergeSource, MergePolicy.OneMer } MergeStats stats() { - final MergeStats mergeStats = new MergeStats(); - mergeStats.add( - totalMerges.count(), - totalMerges.sum(), - totalMergesNumDocs.count(), - totalMergesSizeInBytes.count(), - currentMerges.count(), - currentMergesNumDocs.count(), - currentMergesSizeInBytes.count(), - totalMergeStoppedTime.count(), - totalMergeThrottledTime.count(), - config.isAutoThrottle() ? getIORateLimitMBPerSec() : Double.POSITIVE_INFINITY, - mergedSegmentTransferTracker.stats() + final MergeStats mergeStats = mergeStatsTracker.toMergeStats( + config.isAutoThrottle() ? getIORateLimitMBPerSec() : Double.POSITIVE_INFINITY ); + mergeStats.add(mergedSegmentTransferTracker.stats()); return mergeStats; } diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/MergeInput.java b/server/src/main/java/org/opensearch/index/engine/dataformat/MergeInput.java index b9b312bc39dcc..9f61869075a83 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/MergeInput.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/MergeInput.java @@ -9,7 +9,7 @@ package org.opensearch.index.engine.dataformat; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.engine.exec.WriterFileSet; +import org.opensearch.index.engine.exec.Segment; import java.util.ArrayList; import java.util.List; @@ -21,14 +21,14 @@ * @opensearch.experimental */ @ExperimentalApi -public record MergeInput(List writerFiles, RowIdMapping rowIdMapping, long newWriterGeneration) { +public record MergeInput(List segments, RowIdMapping rowIdMapping, long newWriterGeneration) { public MergeInput { - writerFiles = List.copyOf(writerFiles); + segments = List.copyOf(segments); } private MergeInput(Builder builder) { - this(new ArrayList<>(builder.fileMetadataList), builder.rowIdMapping, builder.newWriterGeneration); + this(new ArrayList<>(builder.segments), builder.rowIdMapping, builder.newWriterGeneration); } /** @@ -45,31 +45,31 @@ public static Builder builder() { */ @ExperimentalApi public static class Builder { - private List fileMetadataList = new ArrayList<>(); + private List segments = new ArrayList<>(); private RowIdMapping rowIdMapping; private long newWriterGeneration; private Builder() {} /** - * Sets the list of writer file sets to merge. + * Sets the list of segments to merge. * - * @param fileMetadataList the writer file sets + * @param segments the segments to merge * @return this builder */ - public Builder fileMetadataList(List fileMetadataList) { - this.fileMetadataList = new ArrayList<>(fileMetadataList); + public Builder segments(List segments) { + this.segments = new ArrayList<>(segments); return this; } /** - * Adds a writer file set to merge. + * Adds a segment to merge. * - * @param writerFileSet the writer file set to add + * @param segment the segment to add * @return this builder */ - public Builder addFileMetadata(WriterFileSet writerFileSet) { - this.fileMetadataList.add(writerFileSet); + public Builder addSegment(Segment segment) { + this.segments.add(segment); return this; } diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/merge/DataFormatAwareMergePolicy.java b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/DataFormatAwareMergePolicy.java new file mode 100644 index 0000000000000..8b6b64aa72695 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/DataFormatAwareMergePolicy.java @@ -0,0 +1,333 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat.merge; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.index.MergeTrigger; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.logging.Loggers; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Adapts a Lucene {@link org.apache.lucene.index.MergePolicy} to work with the data-format-aware segment model. + *

+ * Converts {@link Segment} instances into Lucene {@link SegmentCommitInfo} + * wrappers so the underlying merge policy can select merge candidates. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class DataFormatAwareMergePolicy implements MergeHandler.MergePolicy, MergeHandler.MergeListener { + private final org.apache.lucene.index.MergePolicy luceneMergePolicy; + private final Logger logger; + private final Directory sharedDirectory; + private final DataFormatMergeContext mergeContext; + + /** + * Constructs a DataFormatAwareMergePolicy. + * + * @param mergePolicy the Lucene merge policy to delegate candidate selection to + * @param shardId the shard ID for logging context + */ + public DataFormatAwareMergePolicy(org.apache.lucene.index.MergePolicy mergePolicy, ShardId shardId) { + this.luceneMergePolicy = mergePolicy; + this.logger = Loggers.getLogger(getClass(), shardId); + this.sharedDirectory = new ByteBuffersDirectory(); + this.mergeContext = new DataFormatMergeContext(logger); + } + + /** + * Finds force-merge candidates from the given segments, targeting the specified maximum segment count. + * + * @param segments the current list of segments + * @param maxSegmentCount the target maximum number of segments after merging + * @return a list of segment groups, each group representing one merge operation + * @throws IOException if an I/O error occurs during candidate selection + */ + @Override + public List> findForceMergeCandidates(List segments, int maxSegmentCount) throws IOException { + Map segmentMap = new HashMap<>(); + SegmentInfos segmentInfos = convertToSegmentInfos(segments, segmentMap); + + Map segmentsToMerge = new HashMap<>(); + segmentInfos.forEach(seg -> segmentsToMerge.put(seg, true)); + + try { + org.apache.lucene.index.MergePolicy.MergeSpecification mergeSpec = luceneMergePolicy.findForcedMerges( + segmentInfos, + maxSegmentCount, + segmentsToMerge, + mergeContext + ); + return convertMergeSpecification(mergeSpec, segmentMap); + } catch (Exception e) { + logger.error("Error finding force merge candidates", e); + throw new RuntimeException("Error finding force merge candidates", e); + } + } + + /** + * Finds merge candidates from the given segments using the configured Lucene merge policy. + * + * @param segments the current list of segments + * @return a list of segment groups, each group representing one merge operation + * @throws IOException if an I/O error occurs during candidate selection + */ + @Override + public List> findMergeCandidates(List segments) throws IOException { + Map segmentMap = new HashMap<>(); + SegmentInfos segmentInfos = convertToSegmentInfos(segments, segmentMap); + + try { + org.apache.lucene.index.MergePolicy.MergeSpecification mergeSpec = luceneMergePolicy.findMerges( + MergeTrigger.COMMIT, + segmentInfos, + mergeContext + ); + return convertMergeSpecification(mergeSpec, segmentMap); + } catch (Exception e) { + logger.error("Error finding merge candidates", e); + throw new RuntimeException("Error finding merge candidates", e); + } + } + + /** + * Registers segments as currently merging so the merge policy excludes them from future candidates. + * + * @param segments the segments being merged + */ + @Override + public void addMergingSegment(Collection segments) { + for (Segment segment : segments) { + mergeContext.addMergingSegment(createWrapper(segment)); + } + } + + /** + * Removes segments from the currently-merging set after a merge completes or fails. + * + * @param segments the segments to remove + */ + @Override + public void removeMergingSegment(Collection segments) { + for (Segment segment : segments) { + mergeContext.removeMergingSegment(createWrapper(segment)); + } + } + + /** + * Creates a {@link SegmentWrapper} for the given segment. + * + * @param segment the segment to wrap + * @return a Lucene-compatible {@link SegmentCommitInfo} wrapper + */ + private SegmentWrapper createWrapper(Segment segment) { + return new SegmentWrapper(sharedDirectory, segment, calculateTotalSize(segment), calculateNumDocs(segment)); + } + + /** + * Converts a list of {@link Segment} instances into a Lucene {@link SegmentInfos} + * and populates the reverse mapping from wrapper to original segment. + * + * @param segments the segments to convert + * @param segmentMap populated with wrapper → original segment mappings + * @return the Lucene segment infos + */ + private SegmentInfos convertToSegmentInfos(List segments, Map segmentMap) { + SegmentInfos segmentInfos = new SegmentInfos(Version.LATEST.major); + + for (Segment segment : segments) { + SegmentWrapper wrapper = createWrapper(segment); + segmentInfos.add(wrapper); + segmentMap.put(wrapper, segment); + } + + return segmentInfos; + } + + /** + * Converts a Lucene {@link org.apache.lucene.index.MergePolicy.MergeSpecification} back into groups of + * {@link Segment} instances using the reverse mapping. + * + * @param mergeSpecification the Lucene merge specification (may be {@code null}) + * @param segmentMap the wrapper → original segment mapping + * @return a list of segment groups, each representing one merge operation + */ + private List> convertMergeSpecification( + org.apache.lucene.index.MergePolicy.MergeSpecification mergeSpecification, + Map segmentMap + ) { + List> merges = new ArrayList<>(); + + if (mergeSpecification != null) { + for (org.apache.lucene.index.MergePolicy.OneMerge merge : mergeSpecification.merges) { + List segmentMerge = new ArrayList<>(); + for (SegmentCommitInfo segment : merge.segments) { + segmentMerge.add(segmentMap.get(segment)); + } + merges.add(segmentMerge); + } + } + + return merges; + } + + private long calculateNumDocs(Segment segment) { + return segment.dfGroupedSearchableFiles().values().stream().mapToLong(WriterFileSet::numRows).sum(); + } + + private long calculateTotalSize(Segment segment) { + return segment.dfGroupedSearchableFiles().values().stream().mapToLong(WriterFileSet::getTotalSize).sum(); + } + + /** + * A {@link org.apache.lucene.index.MergePolicy.MergeContext} implementation that tracks merging segments + * and provides info-stream logging for the Lucene merge policy. + * + * @opensearch.experimental + */ + @ExperimentalApi + public static class DataFormatMergeContext implements org.apache.lucene.index.MergePolicy.MergeContext { + + private final HashSet mergingSegments = new HashSet<>(); + private final InfoStream infoStream; + + public DataFormatMergeContext(Logger logger) { + this.infoStream = new InfoStream() { + @Override + public void message(String component, String message) { + logger.debug(() -> new ParameterizedMessage("[DF_MERGE_POLICY] Merge [{}]: {}", component, message)); + } + + @Override + public boolean isEnabled(String component) { + return logger.isDebugEnabled(); + } + + @Override + public void close() throws IOException {} + }; + } + + @Override + public int numDeletesToMerge(SegmentCommitInfo segmentCommitInfo) throws IOException { + return 0; + } + + @Override + public int numDeletedDocs(SegmentCommitInfo segmentCommitInfo) { + return 0; + } + + @Override + public InfoStream getInfoStream() { + return this.infoStream; + } + + @Override + public synchronized Set getMergingSegments() { + return Set.copyOf(mergingSegments); + } + + synchronized void addMergingSegment(SegmentCommitInfo segment) { + mergingSegments.add(segment); + } + + synchronized void removeMergingSegment(SegmentCommitInfo segment) { + mergingSegments.remove(segment); + } + } + + /** + * Lucene {@link SegmentCommitInfo} wrapper that exposes segment + * size and doc-count information to the underlying merge policy. + *

+ * Identity is based on segment generation so that wrappers created + * from the same {@link Segment} are equal. + */ + private static class SegmentWrapper extends SegmentCommitInfo { + private static final byte[] DUMMY_ID = new byte[16]; + private static final Map EMPTY_DIAGNOSTICS = Map.of(); + private static final Map EMPTY_ATTRIBUTES = Map.of(); + + private final long generation; + private final long totalSizeBytes; + + public SegmentWrapper(Directory directory, Segment segment, long totalSizeBytes, long totalNumDocs) { + super( + new org.apache.lucene.index.SegmentInfo( + directory, + Version.LATEST, + Version.LATEST, + "segment_" + segment.generation(), + (int) Math.min(totalNumDocs, Integer.MAX_VALUE), + false, + false, + Codec.getDefault(), + EMPTY_DIAGNOSTICS, + DUMMY_ID, + EMPTY_ATTRIBUTES, + null + ), + 0, + 0, + 0, + -1, + -1, + DUMMY_ID + ); + this.generation = segment.generation(); + this.totalSizeBytes = totalSizeBytes; + } + + @Override + public long sizeInBytes() { + return totalSizeBytes; + } + + @Override + public int getDelCount() { + return 0; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o instanceof SegmentWrapper other) { + return generation == other.generation; + } + return false; + } + + @Override + public int hashCode() { + return Objects.hashCode(generation); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeFailedEngineException.java b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeFailedEngineException.java new file mode 100644 index 0000000000000..21978ee13b1ab --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeFailedEngineException.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat.merge; + +import org.opensearch.OpenSearchException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.shard.ShardId; + +import java.io.IOException; + +/** + * Exception thrown when a segment merge operation fails within the engine. + * + * @opensearch.experimental + */ +public class MergeFailedEngineException extends OpenSearchException { + + /** + * Constructs a new MergeFailedEngineException. + * + * @param shardId the shard where the merge failed + * @param t the underlying cause of the failure + */ + public MergeFailedEngineException(ShardId shardId, Throwable t) { + super("Merge failed", t); + setShard(shardId); + } + + /** + * Constructs a new MergeFailedEngineException from a {@link StreamInput}. + * + * @param in the stream input to deserialize from + * @throws IOException if an I/O error occurs + */ + public MergeFailedEngineException(StreamInput in) throws IOException { + super(in); + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeHandler.java b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeHandler.java index 7c6b2e3cb657d..da91cb77bad42 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeHandler.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeHandler.java @@ -14,38 +14,62 @@ import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.logging.Loggers; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.engine.dataformat.MergeInput; import org.opensearch.index.engine.dataformat.MergeResult; -import org.opensearch.index.engine.exec.Indexer; +import org.opensearch.index.engine.dataformat.Merger; import org.opensearch.index.engine.exec.Segment; import org.opensearch.index.engine.exec.coord.CatalogSnapshot; +import java.io.IOException; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Collection; import java.util.Deque; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.Supplier; /** - * Abstract handler responsible for managing segment merge operations. + * Manages the segment merge queue, lifecycle callbacks, and merge candidate + * selection via {@link MergePolicy}. *

- * Subclasses define the merge policy by implementing {@link #findMerges()} and - * {@link #findForceMerges(int)}, while this base class manages the pending merge - * queue and lifecycle callbacks. + * Merge execution is delegated to a {@link Merger} provided at construction. + * Per-format plugins (Parquet, Lucene) implement {@link Merger} + * only — they don't know about multi-format orchestration. * * @opensearch.experimental */ @ExperimentalApi -public abstract class MergeHandler { +public class MergeHandler { - private final Deque mergingSegments = new ArrayDeque<>(); + private final Deque pendingMerges = new ArrayDeque<>(); private final Set currentlyMergingSegments = new HashSet<>(); - private final Indexer indexer; + private final Supplier> snapshotSupplier; + private final MergePolicy mergePolicy; + private final MergeListener mergeListener; + private final Merger merger; private final Logger logger; - public MergeHandler(Indexer indexer, ShardId shardId) { + /** + * Creates a new merge handler. + * + * @param snapshotSupplier supplier for acquiring catalog snapshots for segment validation + * @param merger the merger that performs the actual merge operation + * @param shardId the shard this handler is associated with (used for logging) + */ + public MergeHandler( + Supplier> snapshotSupplier, + Merger merger, + ShardId shardId, + MergePolicy mergePolicy, + MergeListener mergeListener + ) { this.logger = Loggers.getLogger(getClass(), shardId); - this.indexer = indexer; + this.snapshotSupplier = snapshotSupplier; + this.mergePolicy = mergePolicy; + this.mergeListener = mergeListener; + this.merger = merger; } /** @@ -53,7 +77,20 @@ public MergeHandler(Indexer indexer, ShardId shardId) { * * @return a collection of merges to execute, or an empty collection if none are needed */ - public abstract Collection findMerges(); + public Collection findMerges() { + List oneMerges = new ArrayList<>(); + try (GatedCloseable catalogSnapshotRef = snapshotSupplier.get()) { + List segmentList = catalogSnapshotRef.get().getSegments(); + List> mergeCandidates = mergePolicy.findMergeCandidates(segmentList); + for (List mergeGroup : mergeCandidates) { + oneMerges.add(new OneMerge(mergeGroup)); + } + } catch (Exception e) { + logger.warn("Failed to acquire snapshots", e); + throw new RuntimeException(e); + } + return oneMerges; + } /** * Finds merges required to reduce the number of segments to at most {@code maxSegmentCount}. @@ -61,13 +98,26 @@ public MergeHandler(Indexer indexer, ShardId shardId) { * @param maxSegmentCount the maximum number of segments allowed after merging * @return a collection of merges to execute */ - public abstract Collection findForceMerges(int maxSegmentCount); + public Collection findForceMerges(int maxSegmentCount) { + List oneMerges = new ArrayList<>(); + try (GatedCloseable catalogSnapshotRef = snapshotSupplier.get()) { + List segmentList = catalogSnapshotRef.get().getSegments(); + List> mergeCandidates = mergePolicy.findForceMergeCandidates(segmentList, maxSegmentCount); + for (List mergeGroup : mergeCandidates) { + oneMerges.add(new OneMerge(mergeGroup)); + } + } catch (Exception e) { + logger.warn("Failed to acquire snapshots", e); + throw new RuntimeException(e); + } + return oneMerges; + } /** * Updates the set of pending merges. Called to refresh the merge queue * when the segment state changes. */ - public synchronized void updatePendingMerges() { + public synchronized void findAndRegisterMerges() { Collection oneMerges = findMerges(); for (OneMerge oneMerge : oneMerges) { boolean isValidMerge = true; @@ -89,9 +139,8 @@ public synchronized void updatePendingMerges() { * @param merge the merge to register */ public synchronized void registerMerge(OneMerge merge) { - try (GatedCloseable catalogSnapshotReleasableRef = indexer.acquireSnapshot()) { - // Validate segments exist in catalog - List catalogSegments = catalogSnapshotReleasableRef.get().getSegments(); + try (GatedCloseable catalogSnapshotRef = snapshotSupplier.get()) { + List catalogSegments = catalogSnapshotRef.get().getSegments(); for (Segment mergeSegment : merge.getSegmentsToMerge()) { if (!catalogSegments.contains(mergeSegment)) { return; @@ -101,9 +150,10 @@ public synchronized void registerMerge(OneMerge merge) { logger.warn("Failed to acquire snapshots", e); throw new RuntimeException(e); } - mergingSegments.add(merge); + pendingMerges.add(merge); currentlyMergingSegments.addAll(merge.getSegmentsToMerge()); - logger.debug(() -> new ParameterizedMessage("Registered merge [{}], mergingSegments: [{}]", merge, mergingSegments)); + mergeListener.addMergingSegment(merge.getSegmentsToMerge()); + logger.debug(() -> new ParameterizedMessage("Registered merge [{}], pendingMerges: [{}]", merge, pendingMerges)); } /** @@ -112,7 +162,7 @@ public synchronized void registerMerge(OneMerge merge) { * @return {@code true} if there are pending merges */ public synchronized boolean hasPendingMerges() { - return !mergingSegments.isEmpty(); + return !pendingMerges.isEmpty(); } /** @@ -121,20 +171,28 @@ public synchronized boolean hasPendingMerges() { * @return the next merge to execute, or {@code null} if the queue is empty */ public synchronized OneMerge getNextMerge() { - if (mergingSegments.isEmpty()) { + if (pendingMerges.isEmpty()) { return null; } - return mergingSegments.removeFirst(); + return pendingMerges.removeFirst(); } /** * Callback invoked when a merge completes successfully. + *

+ * IMPORTANT: The caller MUST apply the merge result to the catalog + * (replacing source segments with the merged segment) BEFORE calling this method. + * This method calls {@link #findAndRegisterMerges()} which reads the catalog to find + * new merge candidates. If the catalog still contains the old source segments, + * they may be incorrectly selected for another merge. * * @param oneMerge the merge that finished + * @see MergeScheduler — the production caller that enforces this ordering via + * {@code applyMergeChanges.accept(mergeResult, oneMerge)} before this call */ public synchronized void onMergeFinished(OneMerge oneMerge) { removeMergingSegments(oneMerge); - updatePendingMerges(); + findAndRegisterMerges(); } /** @@ -148,16 +206,84 @@ public synchronized void onMergeFailure(OneMerge oneMerge) { } /** - * Executes the given merge operation. + * Executes the given merge operation by delegating to the {@link Merger}. * * @param oneMerge the merge to execute * @return the result of the merge + * @throws IOException if the merge operation fails */ - public abstract MergeResult doMerge(OneMerge oneMerge); + public MergeResult doMerge(OneMerge oneMerge) throws IOException { + MergeInput mergeInput = MergeInput.builder().segments(oneMerge.getSegmentsToMerge()).build(); + return merger.merge(mergeInput); + } private synchronized void removeMergingSegments(OneMerge oneMerge) { - mergingSegments.remove(oneMerge); + pendingMerges.remove(oneMerge); oneMerge.getSegmentsToMerge().forEach(currentlyMergingSegments::remove); + mergeListener.removeMergingSegment(oneMerge.getSegmentsToMerge()); + } + + /** + * A policy that determines how segments should be merged together. + *

+ * Implementations define the strategy for selecting which segments to merge + * during both regular background merges and forced merge operations. + * + * @opensearch.experimental + */ + public interface MergePolicy { + + /** + * Finds groups of segments that are candidates for merging. + *

+ * Each inner list represents a set of segments that should be merged together + * into a single new segment. The outer list contains all such merge groups. + * + * @param segments the current list of segments to evaluate for merging + * @return a list of segment groups, where each group is a list of segments to be merged together; + * returns an empty list if no merges are needed + * @throws IOException if an I/O error occurs while evaluating segments + */ + List> findMergeCandidates(List segments) throws IOException; + + /** + * Finds groups of segments that are candidates for a forced merge operation. + *

+ * A forced merge reduces the total number of segments to at most {@code maxSegmentCount}. + * Each inner list represents a set of segments that should be merged together + * into a single new segment. + * + * @param segments the current list of segments to evaluate for merging + * @param maxSegmentCount the maximum number of segments that should remain after all merges complete + * @return a list of segment groups, where each group is a list of segments to be merged together; + * returns an empty list if the segment count is already within the limit + * @throws IOException if an I/O error occurs while evaluating segments + */ + List> findForceMergeCandidates(List segments, int maxSegmentCount) throws IOException; } + /** + * A listener that is notified when segments begin or finish participating in a merge. + *

+ * Implementations can use these callbacks to track which segments are currently + * being merged, for example to exclude them from future merge candidate selection. + * + * @opensearch.experimental + */ + public interface MergeListener { + + /** + * Called when the given segments begin participating in a merge. + * + * @param mergingSegments the segments that are now being merged + */ + void addMergingSegment(Collection mergingSegments); + + /** + * Called when the given segments have finished participating in a merge. + * + * @param mergingSegments the segments that are no longer being merged + */ + void removeMergingSegment(Collection mergingSegments); + } } diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeScheduler.java b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeScheduler.java index ea0250f9f31c3..66cf5734b6b45 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeScheduler.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeScheduler.java @@ -12,17 +12,28 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.logging.Loggers; +import org.opensearch.common.unit.TimeValue; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; import org.opensearch.index.MergeSchedulerConfig; +import org.opensearch.index.engine.dataformat.MergeResult; import org.opensearch.index.merge.MergeStats; +import org.opensearch.index.merge.MergeStatsTracker; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.Collection; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; /** * Schedules and coordinates segment merge operations for a shard. *

* This scheduler delegates merge selection to a {@link MergeHandler} and controls - * concurrency via configurable thread and merge count limits sourced from - * {@link MergeSchedulerConfig}. + * concurrency via configurable merge count limits sourced from + * {@link MergeSchedulerConfig}. Merge tasks are submitted to the OpenSearch + * {@link ThreadPool} using the {@link ThreadPool.Names#FORCE_MERGE} executor. * * @opensearch.experimental */ @@ -30,9 +41,15 @@ public class MergeScheduler { private final Logger logger; + private final MergeHandler mergeHandler; + private final BiConsumer applyMergeChanges; + private final ThreadPool threadPool; + private final AtomicInteger activeMerges = new AtomicInteger(0); + private final AtomicBoolean isShutdown = new AtomicBoolean(false); private volatile int maxConcurrentMerges; private volatile int maxMergeCount; private final MergeSchedulerConfig mergeSchedulerConfig; + private final MergeStatsTracker mergeStatsTracker = new MergeStatsTracker(); /** true if we should rate-limit writes for each merge */ private boolean doAutoIOThrottle = false; @@ -46,11 +63,22 @@ public class MergeScheduler { /** * Creates a new merge scheduler. * - * @param mergeHandler the handler that selects and executes merges - * @param shardId the shard this scheduler is associated with - * @param indexSettings the index settings providing merge scheduler configuration + * @param mergeHandler the handler that selects and executes merges + * @param applyMergeChanges callback to apply merge results (e.g., update the catalog) + * @param shardId the shard this scheduler is associated with + * @param indexSettings the index settings providing merge scheduler configuration + * @param threadPool the OpenSearch thread pool for executing merge tasks */ - public MergeScheduler(MergeHandler mergeHandler, ShardId shardId, IndexSettings indexSettings) { + public MergeScheduler( + MergeHandler mergeHandler, + BiConsumer applyMergeChanges, + ShardId shardId, + IndexSettings indexSettings, + ThreadPool threadPool + ) { + this.mergeHandler = mergeHandler; + this.applyMergeChanges = applyMergeChanges; + this.threadPool = threadPool; logger = Loggers.getLogger(getClass(), shardId); this.mergeSchedulerConfig = indexSettings.getMergeSchedulerConfig(); refreshConfig(); @@ -88,16 +116,41 @@ public synchronized void refreshConfig() { * concurrency limits. */ public void triggerMerges() { + if (isShutdown.get()) { + logger.warn("MergeScheduler is shutdown, ignoring merge trigger"); + return; + } + mergeHandler.findAndRegisterMerges(); + + executeMerge(); } /** * Forces a merge down to at most {@code maxNumSegment} segments. + * Runs synchronously on the calling thread. * * @param maxNumSegment the maximum number of segments after the force merge */ - public void forceMerge(int maxNumSegment) { - + public void forceMerge(int maxNumSegment) throws IOException { + if (activeMerges.get() > 0) { + logger.warn("Cannot force merge while background merges are active"); + throw new IllegalStateException("Cannot force merge while background merges are active"); + } + Collection oneMerges = mergeHandler.findForceMerges(maxNumSegment); + + for (OneMerge oneMerge : oneMerges) { + threadPool.executor(ThreadPool.Names.FORCE_MERGE).execute(() -> { + try { + MergeResult mergeResult = mergeHandler.doMerge(oneMerge); + applyMergeChanges.accept(mergeResult, oneMerge); + mergeHandler.onMergeFinished(oneMerge); + } catch (Exception e) { + logger.error(new ParameterizedMessage("Force merge failed for: {}", oneMerge), e); + mergeHandler.onMergeFailure(oneMerge); + } + }); + } } /** @@ -122,12 +175,77 @@ public synchronized double getIORateLimitMBPerSec() { return Double.POSITIVE_INFINITY; } + /** + * Shuts down this merge scheduler, preventing new merges from being submitted. + */ + public void shutdown() { + isShutdown.set(true); + } + /** * Returns the current merge statistics for this scheduler. * * @return the merge stats */ public MergeStats stats() { - return new MergeStats(); + return mergeStatsTracker.toMergeStats(mergeSchedulerConfig.isAutoThrottle() ? getIORateLimitMBPerSec() : Double.POSITIVE_INFINITY); + } + + /** + * Drains the pending-merge queue up to {@link #maxConcurrentMerges}, + * submitting each merge as a task to the thread pool. + */ + private void executeMerge() { + while (activeMerges.get() < maxConcurrentMerges && mergeHandler.hasPendingMerges()) { + OneMerge oneMerge = mergeHandler.getNextMerge(); + if (oneMerge == null) { + return; + } + try { + submitMergeTask(oneMerge); + } catch (Exception e) { + mergeHandler.onMergeFailure(oneMerge); + } + } + } + + /** + * Submits a merge task to the thread pool's force merge executor. + * + * @param oneMerge the merge to execute + */ + private void submitMergeTask(OneMerge oneMerge) { + activeMerges.incrementAndGet(); + threadPool.executor(ThreadPool.Names.MERGE).execute(() -> { + long totalSizeInBytes = oneMerge.getTotalSizeInBytes(); + long totalNumDocs = oneMerge.getTotalNumDocs(); + long timeNS = System.nanoTime(); + long tookMS = 0; + try { + if (isShutdown.get()) { + logger.debug("MergeScheduler is shutdown, skipping merge"); + return; + } + + mergeStatsTracker.beforeMerge(totalNumDocs, totalSizeInBytes); + + MergeResult mergeResult = mergeHandler.doMerge(oneMerge); + applyMergeChanges.accept(mergeResult, oneMerge); + mergeHandler.onMergeFinished(oneMerge); + + tookMS = TimeValue.nsecToMSec((System.nanoTime() - timeNS)); + logger.info("Merge completed in {}ms", tookMS); + + } catch (Exception e) { + logger.error(new ParameterizedMessage("Unexpected error during merge for: {}", oneMerge), e); + mergeHandler.onMergeFailure(oneMerge); + } finally { + mergeStatsTracker.afterMerge(tookMS, totalNumDocs, totalSizeInBytes); + + activeMerges.decrementAndGet(); + // A completed merge may free up capacity for new merges, so check again. + executeMerge(); + } + }); } } diff --git a/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java b/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java index 8a08667bf5b55..e6a0e54247443 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java @@ -13,18 +13,25 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.concurrent.GatedConditionalCloseable; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.MergeResult; +import org.opensearch.index.engine.dataformat.merge.OneMerge; import org.opensearch.index.engine.exec.CatalogSnapshotDeletionPolicy; import org.opensearch.index.engine.exec.CatalogSnapshotLifecycleListener; import org.opensearch.index.engine.exec.CommitFileManager; import org.opensearch.index.engine.exec.FileDeleter; import org.opensearch.index.engine.exec.FilesListener; import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; import org.opensearch.index.shard.ShardPath; import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -113,6 +120,54 @@ public CatalogSnapshotManager( ); } + /** + * Applies the results of a completed merge to the latest catalog snapshot. + * Replaces the merged segments with the new merged segment and commits a new snapshot. + * + * @param mergeResult the result of the merge containing the merged writer file set + * @param oneMerge the merge specification identifying which segments were merged + * @throws IOException if committing the new snapshot fails + */ + public synchronized void applyMergeResults(MergeResult mergeResult, OneMerge oneMerge) throws IOException { + + List segmentList = new ArrayList<>(latestCatalogSnapshot.getSegments()); + + Segment segmentToAdd = getSegment(mergeResult.getMergedWriterFileSet()); + Set segmentsToRemove = new HashSet<>(oneMerge.getSegmentsToMerge()); + + boolean inserted = false; + int newSegIdx = 0; + for (int segIdx = 0, cnt = segmentList.size(); segIdx < cnt; segIdx++) { + assert segIdx >= newSegIdx; + Segment currSegment = segmentList.get(segIdx); + if (segmentsToRemove.contains(currSegment)) { + if (!inserted) { + segmentList.set(segIdx, segmentToAdd); + inserted = true; + newSegIdx++; + } + } else { + segmentList.set(newSegIdx, currSegment); + newSegIdx++; + } + } + + // the rest of the segments in list are duplicates, so don't remove from map, only list! + segmentList.subList(newSegIdx, segmentList.size()).clear(); + + // Either we found place to insert segment, or, we did + // not, but only because all segments we merged became + // deleted while we are merging, in which case it should + // be the case that the new segment is also all deleted, + // we insert it at the beginning if it should not be dropped: + if (!inserted) { + segmentList.add(0, segmentToAdd); + } + + // Commit new catalog snapshot + commitNewSnapshot(segmentList); + } + // ---- Refresh path ---- /** @@ -264,6 +319,25 @@ private void decRefAndMaybeDelete(CatalogSnapshot snapshot) { } } + /** + * Builds a {@link Segment} from a map of data format to writer file set entries. + * + * @param writerFileSetMap the map of data formats to their corresponding writer file sets + * @return the constructed segment + * @throws IllegalArgumentException if the map is empty + */ + private Segment getSegment(Map writerFileSetMap) { + if (writerFileSetMap.isEmpty()) { + throw new IllegalArgumentException("writerFileSetMap must not be empty"); + } + long generation = writerFileSetMap.values().iterator().next().writerGeneration(); + Segment.Builder segment = Segment.builder(generation); + for (Map.Entry entry : writerFileSetMap.entrySet()) { + segment.addSearchableFiles(entry.getKey(), entry.getValue()); + } + return segment.build(); + } + /** * Closes this manager. Idempotent. DecRefs the current snapshot and cleans up if count reaches zero. */ diff --git a/server/src/main/java/org/opensearch/index/merge/MergeStatsTracker.java b/server/src/main/java/org/opensearch/index/merge/MergeStatsTracker.java new file mode 100644 index 0000000000000..468b7a4f89902 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/merge/MergeStatsTracker.java @@ -0,0 +1,90 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.merge; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.metrics.CounterMetric; +import org.opensearch.common.metrics.MeanMetric; + +/** + * Tracks live merge metrics (in-progress and completed) using thread-safe counters. + * Use {@link #toMergeStats(double)} to produce a serializable {@link MergeStats} snapshot. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class MergeStatsTracker { + + private final MeanMetric totalMerges = new MeanMetric(); + private final CounterMetric totalMergesNumDocs = new CounterMetric(); + private final CounterMetric totalMergesSizeInBytes = new CounterMetric(); + private final CounterMetric currentMerges = new CounterMetric(); + private final CounterMetric currentMergesNumDocs = new CounterMetric(); + private final CounterMetric currentMergesSizeInBytes = new CounterMetric(); + private final CounterMetric totalMergeStoppedTime = new CounterMetric(); + private final CounterMetric totalMergeThrottledTime = new CounterMetric(); + + /** + * Records the start of a merge operation, incrementing current merge counters. + */ + public void beforeMerge(long numDocs, long sizeInBytes) { + currentMerges.inc(); + currentMergesNumDocs.inc(numDocs); + currentMergesSizeInBytes.inc(sizeInBytes); + } + + /** + * Records the completion of a merge operation, decrementing current and incrementing total counters. + * + * @param tookMS time the merge took in milliseconds + * @param numDocs number of documents in the merge + * @param sizeInBytes size of the merge in bytes + */ + public void afterMerge(long tookMS, long numDocs, long sizeInBytes) { + currentMerges.dec(); + currentMergesNumDocs.dec(numDocs); + currentMergesSizeInBytes.dec(sizeInBytes); + + totalMergesNumDocs.inc(numDocs); + totalMergesSizeInBytes.inc(sizeInBytes); + totalMerges.inc(tookMS); + } + + public void incStoppedTime(long timeMillis) { + totalMergeStoppedTime.inc(timeMillis); + } + + public void incThrottledTime(long timeMillis) { + totalMergeThrottledTime.inc(timeMillis); + } + + /** + * Creates a snapshot of the current merge statistics. + * + * @param mbPerSecAutoThrottle the current auto-throttle rate in MB/sec, + * or {@code Double.POSITIVE_INFINITY} if not throttled + * @return a new {@link MergeStats} instance + */ + public MergeStats toMergeStats(double mbPerSecAutoThrottle) { + final MergeStats mergeStats = new MergeStats(); + mergeStats.add( + totalMerges.count(), + totalMerges.sum(), + totalMergesNumDocs.count(), + totalMergesSizeInBytes.count(), + currentMerges.count(), + currentMergesNumDocs.count(), + currentMergesSizeInBytes.count(), + totalMergeStoppedTime.count(), + totalMergeThrottledTime.count(), + mbPerSecAutoThrottle + ); + return mergeStats; + } +} diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 8071f0d134a97..5995efbb44583 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -115,6 +115,7 @@ public static class Names { public static final String SNAPSHOT = "snapshot"; public static final String SNAPSHOT_DELETION = "snapshot_deletion"; public static final String FORCE_MERGE = "force_merge"; + public static final String MERGE = "merge"; public static final String FETCH_SHARD_STARTED = "fetch_shard_started"; public static final String FETCH_SHARD_STORE = "fetch_shard_store"; public static final String SYSTEM_READ = "system_read"; @@ -194,6 +195,7 @@ public static ThreadPoolType fromType(String type) { map.put(Names.SNAPSHOT, ThreadPoolType.SCALING); map.put(Names.SNAPSHOT_DELETION, ThreadPoolType.SCALING); map.put(Names.FORCE_MERGE, ThreadPoolType.FIXED); + map.put(Names.MERGE, ThreadPoolType.SCALING); map.put(Names.FETCH_SHARD_STARTED, ThreadPoolType.SCALING); map.put(Names.FETCH_SHARD_STORE, ThreadPoolType.SCALING); map.put(Names.SEARCH_THROTTLED, ThreadPoolType.RESIZABLE); @@ -300,6 +302,7 @@ public ThreadPool( Names.FORCE_MERGE, new FixedExecutorBuilder(settings, Names.FORCE_MERGE, oneEighthAllocatedProcessors(allocatedProcessors), -1) ); + builders.put(Names.MERGE, new ScalingExecutorBuilder(Names.MERGE, 1, allocatedProcessors, TimeValue.timeValueMinutes(5))); builders.put( Names.FETCH_SHARD_STORE, new ScalingExecutorBuilder(Names.FETCH_SHARD_STORE, 1, 2 * allocatedProcessors, TimeValue.timeValueMinutes(5)) diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index d011826e81af4..1d7e3fd13a887 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -902,6 +902,7 @@ public void testIds() { ids.put(175, ResponseLimitBreachedException.class); ids.put(176, IngestionEngineException.class); ids.put(177, StreamException.class); + ids.put(178, org.opensearch.index.engine.dataformat.merge.MergeFailedEngineException.class); ids.put(10001, IndexCreateBlockException.class); Map, Integer> reverse = new HashMap<>(); diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java index 55c55a5ef7e90..18645af9421ac 100644 --- a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java @@ -133,7 +133,9 @@ public void testFullDataFormatLifecycle() throws IOException { // 5. Merge the two writer file sets Merger merger = engine.getMerger(); - MergeInput mergeInput = MergeInput.builder().fileMetadataList(List.of(fileSet1, fileSet2)).newWriterGeneration(3L).build(); + Segment seg1 = Segment.builder(fileSet1.writerGeneration()).addSearchableFiles(format, fileSet1).build(); + Segment seg2 = Segment.builder(fileSet2.writerGeneration()).addSearchableFiles(format, fileSet2).build(); + MergeInput mergeInput = MergeInput.builder().segments(List.of(seg1, seg2)).newWriterGeneration(3L).build(); MergeResult mergeResult = merger.merge(mergeInput); WriterFileSet merged = mergeResult.getMergedWriterFileSetForDataformat(format); assertNotNull(merged); @@ -148,7 +150,7 @@ public void testFullDataFormatLifecycle() throws IOException { // 6. Merge with an existing RowIdMapping (secondary data format merge) MergeInput secondaryMergeInput = MergeInput.builder() - .fileMetadataList(List.of(fileSet1, fileSet2)) + .segments(List.of(seg1, seg2)) .rowIdMapping(mapping) .newWriterGeneration(4L) .build(); diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/merge/DataFormatAwareMergePolicyTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/merge/DataFormatAwareMergePolicyTests.java new file mode 100644 index 0000000000000..2f9069ac9cbcc --- /dev/null +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/merge/DataFormatAwareMergePolicyTests.java @@ -0,0 +1,435 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat.merge; + +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.MergeTrigger; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.TieredMergePolicy; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.engine.dataformat.stub.MockDataFormat; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicReference; + +import org.mockito.ArgumentCaptor; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Tests for {@link DataFormatAwareMergePolicy}. + */ +public class DataFormatAwareMergePolicyTests extends OpenSearchTestCase { + + private static final ShardId SHARD_ID = new ShardId(new Index("test-index", "uuid"), 0); + + // ========== findMergeCandidates ========== + + public void testFindMergeCandidatesCapturesMergeContext() throws IOException { + Path tempDir = createTempDir(); + MockDataFormat fmt = new MockDataFormat("lucene", 100L, Set.of()); + WriterFileSet wfs = new WriterFileSet(tempDir.toString(), 1L, Set.of(), 10); + Segment seg1 = Segment.builder(1L).addSearchableFiles(fmt, wfs).build(); + Segment seg2 = Segment.builder(2L).addSearchableFiles(fmt, wfs).build(); + + MergePolicy lucenePolicy = mock(MergePolicy.class); + ArgumentCaptor segInfosCaptor = ArgumentCaptor.forClass(SegmentInfos.class); + ArgumentCaptor ctxCaptor = ArgumentCaptor.forClass(MergePolicy.MergeContext.class); + when(lucenePolicy.findMerges(any(MergeTrigger.class), segInfosCaptor.capture(), ctxCaptor.capture())).thenReturn(null); + + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(lucenePolicy, SHARD_ID); + List> result = policy.findMergeCandidates(List.of(seg1, seg2)); + + assertTrue(result.isEmpty()); + + SegmentInfos capturedInfos = segInfosCaptor.getValue(); + assertEquals(2, capturedInfos.size()); + + MergePolicy.MergeContext capturedCtx = ctxCaptor.getValue(); + assertNotNull(capturedCtx.getInfoStream()); + assertTrue(capturedCtx.getMergingSegments().isEmpty()); + assertEquals(0, capturedCtx.numDeletedDocs(mock(SegmentCommitInfo.class))); + assertEquals(0, capturedCtx.numDeletesToMerge(mock(SegmentCommitInfo.class))); + } + + public void testFindMergeCandidatesMergeContextReflectsAddedAndRemovedSegments() throws IOException { + Path tempDir = createTempDir(); + MockDataFormat fmt = new MockDataFormat("lucene", 100L, Set.of()); + WriterFileSet wfs1 = new WriterFileSet(tempDir.toString(), 1L, Set.of(), 10); + WriterFileSet wfs2 = new WriterFileSet(tempDir.toString(), 2L, Set.of(), 20); + Segment seg1 = Segment.builder(1L).addSearchableFiles(fmt, wfs1).build(); + Segment seg2 = Segment.builder(2L).addSearchableFiles(fmt, wfs2).build(); + Segment seg3 = Segment.builder(3L).addSearchableFiles(fmt, wfs1).build(); + + MergePolicy lucenePolicy = mock(MergePolicy.class); + ArgumentCaptor ctxCaptor = ArgumentCaptor.forClass(MergePolicy.MergeContext.class); + when(lucenePolicy.findMerges(any(MergeTrigger.class), any(SegmentInfos.class), ctxCaptor.capture())).thenReturn(null); + + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(lucenePolicy, SHARD_ID); + List allSegments = List.of(seg1, seg2, seg3); + + // Add seg1 as merging — context should show 1 + policy.addMergingSegment(List.of(seg1)); + policy.findMergeCandidates(allSegments); + assertEquals(1, ctxCaptor.getValue().getMergingSegments().size()); + + // Add seg2 as merging — context should show 2 + policy.addMergingSegment(List.of(seg2)); + policy.findMergeCandidates(allSegments); + assertEquals(2, ctxCaptor.getValue().getMergingSegments().size()); + + // Remove seg1 — context should show 1 + policy.removeMergingSegment(List.of(seg1)); + policy.findMergeCandidates(allSegments); + assertEquals(1, ctxCaptor.getValue().getMergingSegments().size()); + + // Remove seg2 — context should be empty + policy.removeMergingSegment(List.of(seg2)); + policy.findMergeCandidates(allSegments); + assertTrue(ctxCaptor.getValue().getMergingSegments().isEmpty()); + } + + public void testFindMergeCandidatesExceptionWrapped() throws IOException { + MergePolicy lucenePolicy = mock(MergePolicy.class); + when(lucenePolicy.findMerges(any(MergeTrigger.class), any(SegmentInfos.class), any(MergePolicy.MergeContext.class))).thenThrow( + new RuntimeException("merge error") + ); + + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(lucenePolicy, SHARD_ID); + RuntimeException ex = expectThrows(RuntimeException.class, () -> policy.findMergeCandidates(Collections.emptyList())); + assertEquals("Error finding merge candidates", ex.getMessage()); + } + + // ========== findForceMergeCandidates ========== + + @SuppressWarnings("unchecked") + public void testFindForceMergeCandidatesCapturesMergeContext() throws IOException { + Path tempDir = createTempDir(); + MockDataFormat fmt = new MockDataFormat("lucene", 100L, Set.of()); + WriterFileSet wfs = new WriterFileSet(tempDir.toString(), 1L, Set.of(), 10); + Segment seg1 = Segment.builder(1L).addSearchableFiles(fmt, wfs).build(); + Segment seg2 = Segment.builder(2L).addSearchableFiles(fmt, wfs).build(); + + MergePolicy lucenePolicy = mock(MergePolicy.class); + ArgumentCaptor segInfosCaptor = ArgumentCaptor.forClass(SegmentInfos.class); + ArgumentCaptor> segmentsToMergeCaptor = ArgumentCaptor.forClass(Map.class); + ArgumentCaptor ctxCaptor = ArgumentCaptor.forClass(MergePolicy.MergeContext.class); + when(lucenePolicy.findForcedMerges(segInfosCaptor.capture(), anyInt(), segmentsToMergeCaptor.capture(), ctxCaptor.capture())) + .thenReturn(null); + + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(lucenePolicy, SHARD_ID); + List> result = policy.findForceMergeCandidates(List.of(seg1, seg2), 1); + + assertTrue(result.isEmpty()); + + SegmentInfos capturedInfos = segInfosCaptor.getValue(); + assertEquals(2, capturedInfos.size()); + + Map capturedSegmentsToMerge = segmentsToMergeCaptor.getValue(); + assertEquals(2, capturedSegmentsToMerge.size()); + assertTrue("All segments should be marked for merge", capturedSegmentsToMerge.values().stream().allMatch(v -> v)); + + MergePolicy.MergeContext capturedCtx = ctxCaptor.getValue(); + assertNotNull(capturedCtx.getInfoStream()); + assertTrue(capturedCtx.getMergingSegments().isEmpty()); + } + + @SuppressWarnings("unchecked") + public void testFindForceMergeCandidatesExceptionWrapped() throws IOException { + MergePolicy lucenePolicy = mock(MergePolicy.class); + when(lucenePolicy.findForcedMerges(any(SegmentInfos.class), anyInt(), any(Map.class), any(MergePolicy.MergeContext.class))) + .thenThrow(new RuntimeException("force merge error")); + + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(lucenePolicy, SHARD_ID); + RuntimeException ex = expectThrows(RuntimeException.class, () -> policy.findForceMergeCandidates(Collections.emptyList(), 1)); + assertEquals("Error finding force merge candidates", ex.getMessage()); + } + + // ========== Complex add/remove/add/remove lifecycle ========== + + public void testMergeContextTracksMultipleAddRemoveCycles() throws IOException { + Path tempDir = createTempDir(); + MockDataFormat fmt = new MockDataFormat("lucene", 100L, Set.of()); + Segment seg1 = Segment.builder(1L).addSearchableFiles(fmt, new WriterFileSet(tempDir.toString(), 1L, Set.of(), 10)).build(); + Segment seg2 = Segment.builder(2L).addSearchableFiles(fmt, new WriterFileSet(tempDir.toString(), 2L, Set.of(), 20)).build(); + Segment seg3 = Segment.builder(3L).addSearchableFiles(fmt, new WriterFileSet(tempDir.toString(), 3L, Set.of(), 30)).build(); + Segment seg4 = Segment.builder(4L).addSearchableFiles(fmt, new WriterFileSet(tempDir.toString(), 4L, Set.of(), 40)).build(); + List allSegments = List.of(seg1, seg2, seg3, seg4); + + MergePolicy lucenePolicy = mock(MergePolicy.class); + ArgumentCaptor ctxCaptor = ArgumentCaptor.forClass(MergePolicy.MergeContext.class); + when(lucenePolicy.findMerges(any(MergeTrigger.class), any(SegmentInfos.class), ctxCaptor.capture())).thenReturn(null); + + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(lucenePolicy, SHARD_ID); + + // Round 1: add seg1, seg2 — expect 2 merging + policy.addMergingSegment(List.of(seg1, seg2)); + policy.findMergeCandidates(allSegments); + assertEquals(2, ctxCaptor.getValue().getMergingSegments().size()); + + // Round 2: remove seg1 — expect 1 merging + policy.removeMergingSegment(List.of(seg1)); + policy.findMergeCandidates(allSegments); + assertEquals(1, ctxCaptor.getValue().getMergingSegments().size()); + + // Round 3: add seg3, seg4 — expect 3 merging (seg2 still there) + policy.addMergingSegment(List.of(seg3, seg4)); + policy.findMergeCandidates(allSegments); + assertEquals(3, ctxCaptor.getValue().getMergingSegments().size()); + + // Round 4: remove seg2, seg3 — expect 1 merging (seg4) + policy.removeMergingSegment(List.of(seg2, seg3)); + policy.findMergeCandidates(allSegments); + assertEquals(1, ctxCaptor.getValue().getMergingSegments().size()); + + // Round 5: re-add seg1 — expect 2 merging (seg4, seg1) + policy.addMergingSegment(List.of(seg1)); + policy.findMergeCandidates(allSegments); + assertEquals(2, ctxCaptor.getValue().getMergingSegments().size()); + + // Round 6: remove all — expect 0 + policy.removeMergingSegment(List.of(seg1, seg4)); + policy.findMergeCandidates(allSegments); + assertTrue(ctxCaptor.getValue().getMergingSegments().isEmpty()); + + // Round 7: remove already-removed segment is a no-op — still 0 + policy.removeMergingSegment(List.of(seg1)); + policy.findMergeCandidates(allSegments); + assertTrue(ctxCaptor.getValue().getMergingSegments().isEmpty()); + + // Round 8: add duplicate — should still be 1 (set semantics) + policy.addMergingSegment(List.of(seg2)); + policy.addMergingSegment(List.of(seg2)); + policy.findMergeCandidates(allSegments); + assertEquals(1, ctxCaptor.getValue().getMergingSegments().size()); + + // Round 9: single remove clears the duplicate — expect 0 + policy.removeMergingSegment(List.of(seg2)); + policy.findMergeCandidates(allSegments); + assertTrue(ctxCaptor.getValue().getMergingSegments().isEmpty()); + } + + // ========== MergeContext immutability ========== + + public void testGetMergingSegmentsIsUnmodifiable() { + DataFormatAwareMergePolicy.DataFormatMergeContext ctx = new DataFormatAwareMergePolicy.DataFormatMergeContext( + org.apache.logging.log4j.LogManager.getLogger(getClass()) + ); + Set mergingSegments = ctx.getMergingSegments(); + expectThrows(UnsupportedOperationException.class, () -> mergingSegments.add(mock(SegmentCommitInfo.class))); + } + + // ========== Edge cases ========== + + public void testSegmentWithMultipleFormatsAggregatesDocCountAndSize() throws IOException { + MergePolicy lucenePolicy = mock(MergePolicy.class); + ArgumentCaptor segInfosCaptor = ArgumentCaptor.forClass(SegmentInfos.class); + when(lucenePolicy.findMerges(any(MergeTrigger.class), segInfosCaptor.capture(), any(MergePolicy.MergeContext.class))).thenReturn( + null + ); + + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(lucenePolicy, SHARD_ID); + + Path tempDir = createTempDir(); + MockDataFormat fmt1 = new MockDataFormat("lucene", 100L, Set.of()); + MockDataFormat fmt2 = new MockDataFormat("columnar", 50L, Set.of()); + WriterFileSet wfs1 = new WriterFileSet(tempDir.toString(), 1L, Set.of(), 10); + WriterFileSet wfs2 = new WriterFileSet(tempDir.toString(), 1L, Set.of(), 20); + Segment seg = Segment.builder(1L).addSearchableFiles(fmt1, wfs1).addSearchableFiles(fmt2, wfs2).build(); + + policy.findMergeCandidates(List.of(seg)); + + SegmentInfos capturedInfos = segInfosCaptor.getValue(); + assertEquals(1, capturedInfos.size()); + } + + public void testSegmentWithNoSearchableFiles() throws IOException { + MergePolicy lucenePolicy = mock(MergePolicy.class); + ArgumentCaptor segInfosCaptor = ArgumentCaptor.forClass(SegmentInfos.class); + when(lucenePolicy.findMerges(any(MergeTrigger.class), segInfosCaptor.capture(), any(MergePolicy.MergeContext.class))).thenReturn( + null + ); + + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(lucenePolicy, SHARD_ID); + Segment seg = Segment.builder(1L).build(); + + policy.findMergeCandidates(List.of(seg)); + + assertEquals(1, segInfosCaptor.getValue().size()); + } + + // ========== Real TieredMergePolicy ========== + + public void testFindMergeCandidatesWithRealPolicyReturnsMerges() throws IOException { + Path tempDir = createTempDir(); + MockDataFormat fmt = new MockDataFormat("lucene", 100L, Set.of()); + + List segments = new ArrayList<>(); + for (int i = 0; i < 15; i++) { + Path file = tempDir.resolve("seg" + i + ".dat"); + Files.write(file, new byte[100]); + WriterFileSet wfs = new WriterFileSet(tempDir.toString(), i, Set.of("seg" + i + ".dat"), 10); + segments.add(Segment.builder(i).addSearchableFiles(fmt, wfs).build()); + } + + TieredMergePolicy tieredPolicy = new TieredMergePolicy(); + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(tieredPolicy, SHARD_ID); + + List> result = policy.findMergeCandidates(segments); + assertNotNull(result); + assertFalse("TieredMergePolicy should find merge candidates with 15 small segments", result.isEmpty()); + for (List group : result) { + assertFalse(group.isEmpty()); + } + } + + public void testFindForceMergeCandidatesWithRealPolicyReturnsMerges() throws IOException { + Path tempDir = createTempDir(); + MockDataFormat fmt = new MockDataFormat("lucene", 100L, Set.of()); + + List segments = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Path file = tempDir.resolve("fseg" + i + ".dat"); + Files.write(file, new byte[100]); + WriterFileSet wfs = new WriterFileSet(tempDir.toString(), i, Set.of("fseg" + i + ".dat"), 10); + segments.add(Segment.builder(i).addSearchableFiles(fmt, wfs).build()); + } + + TieredMergePolicy tieredPolicy = new TieredMergePolicy(); + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(tieredPolicy, SHARD_ID); + + List> result = policy.findForceMergeCandidates(segments, 1); + assertNotNull(result); + assertFalse("Force merge to 1 segment should produce candidates from 5 segments", result.isEmpty()); + } + + // ========== Concurrency ========== + + public void testConcurrentAddRemoveDoesNotThrow() throws Exception { + MergePolicy lucenePolicy = mock(MergePolicy.class); + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(lucenePolicy, SHARD_ID); + + Path tempDir = createTempDir(); + MockDataFormat fmt = new MockDataFormat("lucene", 100L, Set.of()); + + int numSegments = 50; + List segments = new ArrayList<>(); + for (int i = 0; i < numSegments; i++) { + WriterFileSet wfs = new WriterFileSet(tempDir.toString(), i, Set.of(), 10); + segments.add(Segment.builder(i).addSearchableFiles(fmt, wfs).build()); + } + + AtomicReference failure = new AtomicReference<>(); + CyclicBarrier barrier = new CyclicBarrier(2); + CountDownLatch done = new CountDownLatch(2); + + Thread adder = new Thread(() -> { + try { + barrier.await(); + for (int i = 0; i < 100; i++) { + policy.addMergingSegment(List.of(segments.get(i % numSegments))); + } + } catch (Exception e) { + failure.compareAndSet(null, e); + } finally { + done.countDown(); + } + }); + + Thread remover = new Thread(() -> { + try { + barrier.await(); + for (int i = 0; i < 100; i++) { + policy.removeMergingSegment(List.of(segments.get(i % numSegments))); + } + } catch (Exception e) { + failure.compareAndSet(null, e); + } finally { + done.countDown(); + } + }); + + adder.start(); + remover.start(); + done.await(); + + assertNull("Concurrent add/remove should not throw, but got: " + failure.get(), failure.get()); + } + + public void testConcurrentFindMergeCandidatesAndAddMergingSegment() throws Exception { + TieredMergePolicy tieredPolicy = new TieredMergePolicy(); + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(tieredPolicy, SHARD_ID); + + Path tempDir = createTempDir(); + MockDataFormat fmt = new MockDataFormat("lucene", 100L, Set.of()); + + List segments = new ArrayList<>(); + for (int i = 0; i < 15; i++) { + Path file = tempDir.resolve("cseg" + i + ".dat"); + Files.write(file, new byte[100]); + WriterFileSet wfs = new WriterFileSet(tempDir.toString(), i, Set.of("cseg" + i + ".dat"), 10); + segments.add(Segment.builder(i).addSearchableFiles(fmt, wfs).build()); + } + + AtomicReference failure = new AtomicReference<>(); + CyclicBarrier barrier = new CyclicBarrier(2); + CountDownLatch done = new CountDownLatch(2); + + Thread finder = new Thread(() -> { + try { + barrier.await(); + for (int i = 0; i < 50; i++) { + policy.findMergeCandidates(segments); + } + } catch (Exception e) { + failure.compareAndSet(null, e); + } finally { + done.countDown(); + } + }); + + Thread mutator = new Thread(() -> { + try { + barrier.await(); + for (int i = 0; i < 50; i++) { + Segment seg = segments.get(i % segments.size()); + policy.addMergingSegment(List.of(seg)); + policy.removeMergingSegment(List.of(seg)); + } + } catch (Exception e) { + failure.compareAndSet(null, e); + } finally { + done.countDown(); + } + }); + + finder.start(); + mutator.start(); + done.await(); + + assertNull("Concurrent findMergeCandidates and addMergingSegment should not throw, but got: " + failure.get(), failure.get()); + } +} diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeFailedEngineExceptionTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeFailedEngineExceptionTests.java new file mode 100644 index 0000000000000..3c7d829f5d54e --- /dev/null +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeFailedEngineExceptionTests.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat.merge; + +import org.opensearch.OpenSearchException; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +/** + * Tests for {@link MergeFailedEngineException}. + */ +public class MergeFailedEngineExceptionTests extends OpenSearchTestCase { + + public void testExceptionMessageAndCause() { + ShardId shardId = new ShardId(new Index("test-index", "uuid"), 0); + IOException cause = new IOException("disk full"); + + MergeFailedEngineException exception = new MergeFailedEngineException(shardId, cause); + + assertSame(cause, exception.getCause()); + assertTrue(exception.getMessage().contains("Merge failed")); + assertEquals(shardId, exception.getShardId()); + assertTrue(exception instanceof OpenSearchException); + } +} diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeTests.java index 9444d0d6d11f8..a701cbb7d48fb 100644 --- a/server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeTests.java +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeTests.java @@ -8,27 +8,41 @@ package org.opensearch.index.engine.dataformat.merge; +import org.opensearch.common.SuppressForbidden; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.settings.Settings; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; import org.opensearch.index.MergeSchedulerConfig; -import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.MergeResult; +import org.opensearch.index.engine.dataformat.Merger; import org.opensearch.index.engine.dataformat.stub.MockDataFormat; -import org.opensearch.index.engine.exec.Indexer; import org.opensearch.index.engine.exec.Segment; import org.opensearch.index.engine.exec.WriterFileSet; import org.opensearch.index.engine.exec.coord.CatalogSnapshot; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import java.io.IOException; +import java.lang.reflect.Field; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import static org.opensearch.index.IndexSettingsTests.newIndexMeta; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -37,34 +51,94 @@ */ public class MergeTests extends OpenSearchTestCase { - // ---- Helpers ---- + private static final ShardId SHARD_ID = new ShardId("test", "_na_", 0); - private static class TestMergeHandler extends MergeHandler { - private final List merges; + private final List executors = new CopyOnWriteArrayList<>(); - TestMergeHandler(Indexer indexer, ShardId shardId, List merges) { - super(indexer, shardId); - this.merges = merges; - } + private ExecutorService daemonPool() { + ExecutorService pool = Executors.newCachedThreadPool(r -> { + Thread t = new Thread(r); + t.setDaemon(true); + return t; + }); + executors.add(pool); + return pool; + } + + private ThreadPool mockThreadPool() { + ThreadPool tp = mock(ThreadPool.class); + when(tp.executor(eq(ThreadPool.Names.MERGE))).thenReturn(daemonPool()); + when(tp.executor(eq(ThreadPool.Names.FORCE_MERGE))).thenReturn(daemonPool()); + return tp; + } - TestMergeHandler(Indexer indexer, ShardId shardId) { - this(indexer, shardId, Collections.emptyList()); + @Override + public void tearDown() throws Exception { + for (ExecutorService pool : executors) { + pool.shutdownNow(); + pool.awaitTermination(5, TimeUnit.SECONDS); } + executors.clear(); + super.tearDown(); + } + private static final MergeHandler.MergePolicy NOOP_MERGE_POLICY = new MergeHandler.MergePolicy() { @Override - public Collection findMerges() { - return merges; + public List> findMergeCandidates(List segments) { + return List.of(); } @Override - public Collection findForceMerges(int maxSegmentCount) { - return merges; + public List> findForceMergeCandidates(List segments, int maxSegmentCount) { + return List.of(); } + }; + + private static final MergeHandler.MergeListener NOOP_MERGE_LISTENER = new MergeHandler.MergeListener() { + @Override + public void addMergingSegment(Collection mergingSegments) {} @Override - public MergeResult doMerge(OneMerge oneMerge) { - return null; + public void removeMergingSegment(Collection mergingSegments) {} + }; + + private MergeHandler createNoopHandler(Supplier> snapshotSupplier) { + Merger noopMerger = mergeInput -> new MergeResult(Map.of()); + return new MergeHandler(snapshotSupplier, noopMerger, SHARD_ID, NOOP_MERGE_POLICY, NOOP_MERGE_LISTENER); + } + + private MergeHandler createHandlerWithRealPolicy(Supplier> snapshotSupplier, Merger merger) { + DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy( + new IndexSettings(newIndexMeta("test", Settings.EMPTY), Settings.EMPTY).getMergePolicy(true), + SHARD_ID + ); + return new MergeHandler(snapshotSupplier, merger, SHARD_ID, policy, policy); + } + + private static Supplier> snapshotSupplierOf(List segments) { + CatalogSnapshot snap = mock(CatalogSnapshot.class); + when(snap.getSegments()).thenReturn(segments); + return () -> new GatedCloseable<>(snap, () -> {}); + } + + private static Supplier> emptySnapshotSupplier() { + return snapshotSupplierOf(Collections.emptyList()); + } + + private static List createSegments(int count) { + List segments = new ArrayList<>(); + for (int i = 1; i <= count; i++) { + segments.add(Segment.builder(i).build()); } + return segments; + } + + private static IndexSettings mergeSchedulerSettings() { + Settings settings = Settings.builder() + .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1") + .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "6") + .build(); + return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); } private MergeScheduler createMergeScheduler() { @@ -73,8 +147,13 @@ private MergeScheduler createMergeScheduler() { .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "6") .build(); IndexSettings idxSettings = new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); - ShardId shardId = new ShardId("test", "_na_", 0); - return new MergeScheduler(new TestMergeHandler(mock(Indexer.class), shardId), shardId, idxSettings); + return new MergeScheduler( + createNoopHandler(emptySnapshotSupplier()), + (mergeResult, oneMerge) -> {}, + SHARD_ID, + idxSettings, + mockThreadPool() + ); } // ---- OneMerge tests ---- @@ -88,7 +167,7 @@ public void testOneMergeWithEmptySegments() { public void testOneMergeAggregatesDocCounts() { Path dir = createTempDir(); - DataFormat format = new MockDataFormat(); + MockDataFormat format = new MockDataFormat(); WriterFileSet fs1 = new WriterFileSet(dir.toString(), 1L, Set.of(), 10); WriterFileSet fs2 = new WriterFileSet(dir.toString(), 2L, Set.of(), 20); @@ -114,41 +193,16 @@ public void testOneMergeToString() { // ---- MergeHandler tests ---- public void testMergeHandlerInitiallyEmpty() { - MergeHandler handler = new TestMergeHandler(mock(Indexer.class), new ShardId("test", "_na_", 0)); + MergeHandler handler = createNoopHandler(() -> new GatedCloseable<>(null, () -> {})); assertFalse(handler.hasPendingMerges()); assertNull(handler.getNextMerge()); } - public void testMergeHandlerFindMerges() { - OneMerge merge = new OneMerge(List.of(Segment.builder(1L).build())); - TestMergeHandler handler = new TestMergeHandler(mock(Indexer.class), new ShardId("test", "_na_", 0), List.of(merge)); - Collection found = handler.findMerges(); - assertEquals(1, found.size()); - assertSame(merge, found.iterator().next()); - } - - public void testMergeHandlerFindForceMerges() { - OneMerge merge = new OneMerge(List.of(Segment.builder(1L).build())); - TestMergeHandler handler = new TestMergeHandler(mock(Indexer.class), new ShardId("test", "_na_", 0), List.of(merge)); - assertEquals(1, handler.findForceMerges(1).size()); - } - - public void testMergeHandlerDoMergeReturnsNull() { - assertNull( - new TestMergeHandler(mock(Indexer.class), new ShardId("test", "_na_", 0)).doMerge(new OneMerge(Collections.emptyList())) - ); - } - public void testMergeHandlerLifecycleCallbacks() { - Indexer mockIndexer = mock(Indexer.class); - CatalogSnapshot mockSnapshot = mock(CatalogSnapshot.class); - when(mockSnapshot.getSegments()).thenReturn(Collections.emptyList()); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); - - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0)); + MergeHandler handler = createNoopHandler(emptySnapshotSupplier()); OneMerge merge = new OneMerge(Collections.emptyList()); handler.registerMerge(merge); - handler.updatePendingMerges(); + handler.findAndRegisterMerges(); handler.onMergeFinished(merge); handler.onMergeFailure(merge); } @@ -157,12 +211,7 @@ public void testRegisterMergeWithValidSegments() { Segment seg1 = Segment.builder(1L).build(); Segment seg2 = Segment.builder(2L).build(); - Indexer mockIndexer = mock(Indexer.class); - CatalogSnapshot mockSnapshot = mock(CatalogSnapshot.class); - when(mockSnapshot.getSegments()).thenReturn(List.of(seg1, seg2)); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); - - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0)); + MergeHandler handler = createNoopHandler(snapshotSupplierOf(List.of(seg1, seg2))); OneMerge merge = new OneMerge(List.of(seg1, seg2)); handler.registerMerge(merge); @@ -175,117 +224,42 @@ public void testRegisterMergeRejectsSegmentNotInCatalog() { Segment catalogSeg = Segment.builder(1L).build(); Segment unknownSeg = Segment.builder(99L).build(); - Indexer mockIndexer = mock(Indexer.class); - CatalogSnapshot mockSnapshot = mock(CatalogSnapshot.class); - when(mockSnapshot.getSegments()).thenReturn(List.of(catalogSeg)); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); - - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0)); + MergeHandler handler = createNoopHandler(snapshotSupplierOf(List.of(catalogSeg))); handler.registerMerge(new OneMerge(List.of(unknownSeg))); assertFalse(handler.hasPendingMerges()); } public void testRegisterMergeThrowsOnAcquireSnapshotFailure() { - Indexer mockIndexer = mock(Indexer.class); - when(mockIndexer.acquireSnapshot()).thenThrow(new RuntimeException("snapshot unavailable")); + Supplier> failingSupplier = () -> { throw new RuntimeException("snapshot unavailable"); }; - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0)); + MergeHandler handler = createNoopHandler(failingSupplier); expectThrows(RuntimeException.class, () -> handler.registerMerge(new OneMerge(Collections.emptyList()))); assertFalse(handler.hasPendingMerges()); } - public void testUpdatePendingMergesSkipsAlreadyMergingSegments() { - Segment seg = Segment.builder(1L).build(); - - Indexer mockIndexer = mock(Indexer.class); - CatalogSnapshot mockSnapshot = mock(CatalogSnapshot.class); - when(mockSnapshot.getSegments()).thenReturn(List.of(seg)); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); - - OneMerge merge = new OneMerge(List.of(seg)); - // Handler whose findMerges returns a merge containing seg - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0), List.of(merge)); - - // Register the merge directly so seg is in currentlyMergingSegments - handler.registerMerge(merge); - assertTrue(handler.hasPendingMerges()); - - // Now updatePendingMerges calls findMerges which returns the same merge, - // but seg is already in currentlyMergingSegments so isValidMerge=false, skip - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); - handler.updatePendingMerges(); - - // Should still have only the original merge, no duplicate - assertNotNull(handler.getNextMerge()); - assertNull(handler.getNextMerge()); - } - - public void testUpdatePendingMergesWithEmptySegmentsMerge() { - Indexer mockIndexer = mock(Indexer.class); - CatalogSnapshot mockSnapshot = mock(CatalogSnapshot.class); - when(mockSnapshot.getSegments()).thenReturn(Collections.emptyList()); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); - - // findMerges returns a merge with empty segments list — inner for loop doesn't iterate, - // isValidMerge stays true, registerMerge is called - OneMerge emptyMerge = new OneMerge(Collections.emptyList()); - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0), List.of(emptyMerge)); - - handler.updatePendingMerges(); - assertTrue(handler.hasPendingMerges()); - } - - public void testUpdatePendingMergesWithNoMergesFound() { - Indexer mockIndexer = mock(Indexer.class); - // findMerges returns empty — outer for loop doesn't iterate - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0), Collections.emptyList()); - - handler.updatePendingMerges(); - assertFalse(handler.hasPendingMerges()); - } - public void testRegisterMergeWithEmptySegmentsList() { - Indexer mockIndexer = mock(Indexer.class); - CatalogSnapshot mockSnapshot = mock(CatalogSnapshot.class); - when(mockSnapshot.getSegments()).thenReturn(Collections.emptyList()); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); - - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0)); - // Empty segments list — for loop in registerMerge doesn't iterate, merge is registered + MergeHandler handler = createNoopHandler(emptySnapshotSupplier()); handler.registerMerge(new OneMerge(Collections.emptyList())); assertTrue(handler.hasPendingMerges()); } - public void testOnMergeFinishedRemovesSegmentsAndUpdates() { + public void testOnMergeFinishedRemovesSegments() { Segment seg = Segment.builder(1L).build(); - Indexer mockIndexer = mock(Indexer.class); - CatalogSnapshot mockSnapshot = mock(CatalogSnapshot.class); - when(mockSnapshot.getSegments()).thenReturn(List.of(seg)); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); - - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0)); + MergeHandler handler = createNoopHandler(snapshotSupplierOf(List.of(seg))); OneMerge merge = new OneMerge(List.of(seg)); handler.registerMerge(merge); assertTrue(handler.hasPendingMerges()); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); handler.onMergeFinished(merge); - // After onMergeFinished, the merge is removed; updatePendingMerges is called - // but findMerges returns empty list for this handler, so nothing new is added assertFalse(handler.hasPendingMerges()); } public void testOnMergeFailureRemovesSegments() { Segment seg = Segment.builder(1L).build(); - Indexer mockIndexer = mock(Indexer.class); - CatalogSnapshot mockSnapshot = mock(CatalogSnapshot.class); - when(mockSnapshot.getSegments()).thenReturn(List.of(seg)); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); - - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0)); + MergeHandler handler = createNoopHandler(snapshotSupplierOf(List.of(seg))); OneMerge merge = new OneMerge(List.of(seg)); handler.registerMerge(merge); assertTrue(handler.hasPendingMerges()); @@ -298,17 +272,11 @@ public void testGetNextMergeReturnsInOrder() { Segment seg1 = Segment.builder(1L).build(); Segment seg2 = Segment.builder(2L).build(); - Indexer mockIndexer = mock(Indexer.class); - CatalogSnapshot mockSnapshot = mock(CatalogSnapshot.class); - when(mockSnapshot.getSegments()).thenReturn(List.of(seg1, seg2)); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); - - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0)); + MergeHandler handler = createNoopHandler(snapshotSupplierOf(List.of(seg1, seg2))); OneMerge merge1 = new OneMerge(List.of(seg1)); OneMerge merge2 = new OneMerge(List.of(seg2)); handler.registerMerge(merge1); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); handler.registerMerge(merge2); assertTrue(handler.hasPendingMerges()); @@ -321,34 +289,22 @@ public void testRegisterMergeRejectsWhenSecondSegmentNotInCatalog() { Segment catalogSeg = Segment.builder(1L).build(); Segment unknownSeg = Segment.builder(99L).build(); - Indexer mockIndexer = mock(Indexer.class); - CatalogSnapshot mockSnapshot = mock(CatalogSnapshot.class); - when(mockSnapshot.getSegments()).thenReturn(List.of(catalogSeg)); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); - - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0)); - // First segment is in catalog, second is not — covers the loop-continue-then-return branch + MergeHandler handler = createNoopHandler(snapshotSupplierOf(List.of(catalogSeg))); handler.registerMerge(new OneMerge(List.of(catalogSeg, unknownSeg))); assertFalse(handler.hasPendingMerges()); } - public void testUpdatePendingMergesRegistersValidMerges() { - Segment seg = Segment.builder(1L).build(); - - Indexer mockIndexer = mock(Indexer.class); - CatalogSnapshot mockSnapshot = mock(CatalogSnapshot.class); - when(mockSnapshot.getSegments()).thenReturn(List.of(seg)); - when(mockIndexer.acquireSnapshot()).thenReturn(new GatedCloseable<>(mockSnapshot, () -> {})); + // ---- MergeHandler doMerge tests ---- - OneMerge merge = new OneMerge(List.of(seg)); - // Handler whose findMerges returns a merge with a valid segment - MergeHandler handler = new TestMergeHandler(mockIndexer, new ShardId("test", "_na_", 0), List.of(merge)); + public void testDoMergeReturnsResult() throws IOException { + MergeResult expectedResult = new MergeResult(Map.of()); + Merger merger = mergeInput -> expectedResult; - handler.updatePendingMerges(); + MergeHandler handler = new MergeHandler(emptySnapshotSupplier(), merger, SHARD_ID, NOOP_MERGE_POLICY, NOOP_MERGE_LISTENER); + MergeResult result = handler.doMerge(new OneMerge(Collections.emptyList())); - assertTrue(handler.hasPendingMerges()); - assertSame(merge, handler.getNextMerge()); + assertSame(expectedResult, result); } // ---- MergeScheduler tests ---- @@ -369,14 +325,119 @@ public void testSchedulerStatsReturnsNonNull() { public void testSchedulerRefreshConfigIdempotent() { MergeScheduler scheduler = createMergeScheduler(); - // Second call with same config should be a no-op (covers the early return branch) scheduler.refreshConfig(); scheduler.refreshConfig(); } - public void testSchedulerTriggerAndForceMerge() { + public void testSchedulerTriggerAndForceMerge() throws IOException { + MergeScheduler scheduler = createMergeScheduler(); + scheduler.triggerMerges(); + scheduler.forceMerge(1); + } + + @SuppressForbidden(reason = "test needs to set private isShutdown field via reflection") + public void testTriggerMergesAfterShutdown() throws Exception { + MergeScheduler scheduler = createMergeScheduler(); + setShutdownFlag(scheduler, true); + scheduler.triggerMerges(); + } + + public void testTriggerMergesWithNoPendingMerges() { MergeScheduler scheduler = createMergeScheduler(); scheduler.triggerMerges(); + assertEquals(0, scheduler.stats().getCurrent()); + } + + public void testStatsWithAutoThrottleEnabled() { + Settings autoThrottleSettings = Settings.builder() + .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1") + .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "6") + .put(MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey(), "true") + .build(); + IndexSettings idxSettings = new IndexSettings(newIndexMeta("test", autoThrottleSettings), Settings.EMPTY); + MergeScheduler scheduler = new MergeScheduler( + createNoopHandler(emptySnapshotSupplier()), + (mr, om) -> {}, + SHARD_ID, + idxSettings, + mockThreadPool() + ); + scheduler.enableAutoIOThrottle(); + assertNotNull(scheduler.stats()); + } + + // ---- MergeScheduler: integration with real merge execution ---- + + public void testTriggerMergesExecutesMergeThread() throws Exception { + List segments = createSegments(15); + MergeResult mergeResult = new MergeResult(Map.of()); + CountDownLatch latch = new CountDownLatch(1); + + Merger merger = mergeInput -> { + latch.countDown(); + return mergeResult; + }; + MergeHandler handler = createHandlerWithRealPolicy(snapshotSupplierOf(segments), merger); + + AtomicReference captured = new AtomicReference<>(); + MergeScheduler scheduler = new MergeScheduler( + handler, + (mr, om) -> captured.set(mr), + SHARD_ID, + mergeSchedulerSettings(), + mockThreadPool() + ); + + scheduler.triggerMerges(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + Thread.sleep(200); + assertNotNull(captured.get()); + } + + public void testTriggerMergesHandlesMergeFailure() throws Exception { + List segments = createSegments(15); + CountDownLatch latch = new CountDownLatch(1); + + Merger failingMerger = mergeInput -> { + latch.countDown(); + throw new IOException("merge boom"); + }; + MergeHandler handler = createHandlerWithRealPolicy(snapshotSupplierOf(segments), failingMerger); + + MergeScheduler scheduler = new MergeScheduler(handler, (mr, om) -> {}, SHARD_ID, mergeSchedulerSettings(), mockThreadPool()); + + scheduler.triggerMerges(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + Thread.sleep(200); + } + + public void testForceMergeExecutesMerges() throws Exception { + List segments = createSegments(3); + MergeResult mergeResult = new MergeResult(Map.of()); + CountDownLatch latch = new CountDownLatch(1); + + Merger merger = mergeInput -> mergeResult; + MergeHandler handler = createHandlerWithRealPolicy(snapshotSupplierOf(segments), merger); + + AtomicReference captured = new AtomicReference<>(); + MergeScheduler scheduler = new MergeScheduler(handler, (mr, om) -> { + captured.set(mr); + latch.countDown(); + }, SHARD_ID, mergeSchedulerSettings(), mockThreadPool()); + scheduler.forceMerge(1); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertNotNull(captured.get()); + } + + @SuppressForbidden(reason = "helper to set private isShutdown field via reflection for testing") + private static void setShutdownFlag(MergeScheduler scheduler, boolean value) { + try { + Field f = MergeScheduler.class.getDeclaredField("isShutdown"); + f.setAccessible(true); + ((AtomicBoolean) f.get(scheduler)).set(value); + } catch (Exception e) { + throw new RuntimeException(e); + } } } diff --git a/server/src/test/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManagerTests.java b/server/src/test/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManagerTests.java index 7b59cb4455a43..df9641a5b35f1 100644 --- a/server/src/test/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManagerTests.java +++ b/server/src/test/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManagerTests.java @@ -10,6 +10,10 @@ import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.concurrent.GatedConditionalCloseable; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.MergeResult; +import org.opensearch.index.engine.dataformat.merge.OneMerge; +import org.opensearch.index.engine.dataformat.stub.MockDataFormat; import org.opensearch.index.engine.exec.CatalogSnapshotDeletionPolicy; import org.opensearch.index.engine.exec.CombinedCatalogSnapshotDeletionPolicy; import org.opensearch.index.engine.exec.FileDeleter; @@ -270,6 +274,108 @@ public void testCloseInternalNotInvokedWhileRefsHeld() throws Exception { manager.close(); } + public void testApplyMergeResultsReplacesSegments() throws Exception { + DataFormat format = new MockDataFormat(); + WriterFileSet wfs1 = new WriterFileSet("/tmp/dir", 1L, Set.of("a.cfs"), 100); + WriterFileSet wfs2 = new WriterFileSet("/tmp/dir", 2L, Set.of("b.cfs"), 200); + WriterFileSet wfs3 = new WriterFileSet("/tmp/dir", 3L, Set.of("c.cfs"), 300); + WriterFileSet mergedWfs = new WriterFileSet("/tmp/dir", 4L, Set.of("merged.cfs"), 500); + + Segment seg1 = new Segment(1L, Map.of(format.name(), wfs1)); + Segment seg2 = new Segment(2L, Map.of(format.name(), wfs2)); + Segment seg3 = new Segment(3L, Map.of(format.name(), wfs3)); + + CatalogSnapshotManager manager = new CatalogSnapshotManager( + List.of(new DataformatAwareCatalogSnapshot(0, 0, 1, List.of(seg1, seg2, seg3), 0, Map.of())), + CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, + Map.of(), + Map.of(), + List.of(), + null, + null + ); + try { + MergeResult mergeResult = new MergeResult(Map.of(format, mergedWfs)); + OneMerge oneMerge = new OneMerge(List.of(seg1, seg2)); + + manager.applyMergeResults(mergeResult, oneMerge); + + try (GatedCloseable ref = manager.acquireSnapshot()) { + List segments = ref.get().getSegments(); + assertEquals(2, segments.size()); + // merged segment replaces at position of first merged segment + assertEquals(4L, segments.get(0).generation()); + assertEquals(Set.of("merged.cfs"), segments.get(0).dfGroupedSearchableFiles().get(format.name()).files()); + // unmerged segment preserved + assertEquals(seg3, segments.get(1)); + } + } finally { + manager.close(); + } + } + + public void testApplyMergeResultsWhenAllMergedSegmentsRemoved() throws Exception { + DataFormat format = new MockDataFormat(); + WriterFileSet wfs1 = new WriterFileSet("/tmp/dir", 1L, Set.of("a.cfs"), 100); + WriterFileSet wfs2 = new WriterFileSet("/tmp/dir", 2L, Set.of("b.cfs"), 200); + WriterFileSet mergedWfs = new WriterFileSet("/tmp/dir", 3L, Set.of("merged.cfs"), 300); + + Segment seg1 = new Segment(1L, Map.of(format.name(), wfs1)); + Segment seg2 = new Segment(2L, Map.of(format.name(), wfs2)); + Segment unrelatedSeg = new Segment(99L, Map.of(format.name(), wfs1)); + + // Manager has only unrelatedSeg — the segments being merged are not present + CatalogSnapshotManager manager = new CatalogSnapshotManager( + List.of(new DataformatAwareCatalogSnapshot(0, 0, 1, List.of(unrelatedSeg), 0, Map.of())), + CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, + Map.of(), + Map.of(), + List.of(), + null, + null + ); + try { + MergeResult mergeResult = new MergeResult(Map.of(format, mergedWfs)); + OneMerge oneMerge = new OneMerge(List.of(seg1, seg2)); + + manager.applyMergeResults(mergeResult, oneMerge); + + try (GatedCloseable ref = manager.acquireSnapshot()) { + List segments = ref.get().getSegments(); + assertEquals(2, segments.size()); + // merged segment inserted at position 0 + assertEquals(3L, segments.get(0).generation()); + assertEquals(unrelatedSeg, segments.get(1)); + } + } finally { + manager.close(); + } + } + + public void testApplyMergeResultsWithEmptyWriterFileSetMapThrows() throws Exception { + DataFormat format = new MockDataFormat(); + WriterFileSet wfs1 = new WriterFileSet("/tmp/dir", 1L, Set.of("a.cfs"), 100); + Segment seg1 = new Segment(1L, Map.of(format.name(), wfs1)); + + CatalogSnapshotManager manager = new CatalogSnapshotManager( + List.of(new DataformatAwareCatalogSnapshot(0, 0, 1, List.of(seg1), 0, Map.of())), + CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, + Map.of(), + Map.of(), + List.of(), + null, + null + ); + try { + MergeResult mergeResult = new MergeResult(Map.of()); + OneMerge oneMerge = new OneMerge(List.of(seg1)); + + expectThrows(IllegalArgumentException.class, () -> manager.applyMergeResults(mergeResult, oneMerge)); + } finally { + manager.close(); + } + } + // --- File deletion and commit lifecycle tests --- private static Map commitUserData(long maxSeqNo, long localCheckpoint, String translogUUID) { diff --git a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java index 23c21648b1263..9aa808ec66064 100644 --- a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java +++ b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java @@ -156,6 +156,7 @@ private int expectedSize(final String threadPoolName, final int numberOfProcesso sizes.put(ThreadPool.Names.REMOTE_PURGE, ThreadPool::halfAllocatedProcessors); sizes.put(ThreadPool.Names.REMOTE_REFRESH_RETRY, ThreadPool::halfAllocatedProcessors); sizes.put(ThreadPool.Names.REMOTE_RECOVERY, ThreadPool::twiceAllocatedProcessors); + sizes.put(ThreadPool.Names.MERGE, n -> n); return sizes.get(threadPoolName).apply(numberOfProcessors); } diff --git a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockDataFormatPlugin.java b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockDataFormatPlugin.java index 93a04314d5b46..021fef15f9969 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockDataFormatPlugin.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockDataFormatPlugin.java @@ -27,7 +27,7 @@ public MockDataFormatPlugin() { this(new MockDataFormat("", 100L, Set.of())); } - MockDataFormatPlugin(MockDataFormat mockDataFormat) { + protected MockDataFormatPlugin(MockDataFormat mockDataFormat) { this.dataFormat = mockDataFormat; } diff --git a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockMerger.java b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockMerger.java index 68ea15efd3333..bf2f9cfafaf45 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockMerger.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockMerger.java @@ -13,9 +13,11 @@ import org.opensearch.index.engine.dataformat.MergeResult; import org.opensearch.index.engine.dataformat.Merger; import org.opensearch.index.engine.dataformat.RowIdMapping; +import org.opensearch.index.engine.exec.Segment; import org.opensearch.index.engine.exec.WriterFileSet; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,7 +36,10 @@ public MockMerger(DataFormat dataFormat, Path directory) { @Override public MergeResult merge(MergeInput mergeInput) { - List fileMetadataList = mergeInput.writerFiles(); + List fileMetadataList = new ArrayList<>(); + for (Segment segment : mergeInput.segments()) { + fileMetadataList.addAll(segment.dfGroupedSearchableFiles().values()); + } long newWriterGeneration = mergeInput.newWriterGeneration(); RowIdMapping existingMapping = mergeInput.rowIdMapping(); From 4ddd0845950d8e2e58e604f85108b2b7690656ae Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Mon, 27 Apr 2026 21:05:25 -0700 Subject: [PATCH 008/115] Fix incorrect defaults in FieldStorageResolver. (#21408) * Fix incorrect defaults in FieldStorageResolver. Signed-off-by: Marc Handalian * test fixes Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../DataFusionAnalyticsBackendPlugin.java | 1 + .../planner/FieldStorageResolver.java | 8 +-- .../planner/FieldStorageResolverTests.java | 69 +++++++++++++++++++ .../analytics/planner/FilterRuleTests.java | 4 +- 4 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FieldStorageResolverTests.java diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 0acb0c956a55f..ca3338ca8dbf0 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -44,6 +44,7 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP SUPPORTED_FIELD_TYPES.addAll(FieldType.keyword()); SUPPORTED_FIELD_TYPES.addAll(FieldType.date()); SUPPORTED_FIELD_TYPES.add(FieldType.BOOLEAN); + SUPPORTED_FIELD_TYPES.add(FieldType.TEXT); } private static final Set STANDARD_FILTER_OPS = Set.of( diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java index 72cbbb1ddd3c8..c8f964ef92615 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java @@ -92,11 +92,11 @@ public List resolve(List fieldNames) { } private static FieldStorageInfo resolveField(String fieldName, String fieldType, Map fieldProps, String primaryFormat) { - // Doc values: present for all types except text, unless explicitly disabled - boolean hasDocValues = !"text".equals(fieldType) && !Boolean.FALSE.equals(fieldProps.get("doc_values")); + // Doc values: present for all types unless explicitly disabled + boolean hasDocValues = !Boolean.FALSE.equals(fieldProps.get("doc_values")); - // Index: only when explicitly set to true in mapping - boolean isIndexed = Boolean.TRUE.equals(fieldProps.get("index")); + // Index: only when explicitly set to false in mapping - enabled by default. + boolean isIndexed = !Boolean.FALSE.equals(fieldProps.get("index")); // Stored fields: only when explicitly set to true in mapping boolean isStored = Boolean.TRUE.equals(fieldProps.get("store")); diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FieldStorageResolverTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FieldStorageResolverTests.java new file mode 100644 index 0000000000000..c82f6e2b08936 --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FieldStorageResolverTests.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.planner; + +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.Index; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link FieldStorageResolver} field storage resolution. + */ +public class FieldStorageResolverTests extends OpenSearchTestCase { + + public void testTextFieldGetsDocValuesInPrimaryFormat() { + FieldStorageResolver resolver = newResolver("parquet", Map.of("name", Map.of("type", "text"))); + + FieldStorageInfo info = resolver.resolve(List.of("name")).get(0); + + assertEquals("name", info.getFieldName()); + assertEquals(List.of("parquet"), info.getDocValueFormats()); + assertEquals(List.of("lucene"), info.getIndexFormats()); + } + + public void testLongFieldGetsDocValuesInPrimaryFormat() { + FieldStorageResolver resolver = newResolver("parquet", Map.of("age", Map.of("type", "long"))); + + FieldStorageInfo info = resolver.resolve(List.of("age")).get(0); + + assertEquals("age", info.getFieldName()); + assertEquals(List.of("parquet"), info.getDocValueFormats()); + assertEquals(List.of("lucene"), info.getIndexFormats()); + } + + public void testFieldWithAllStorageDisabledHasNoStorage() { + IllegalStateException ex = expectThrows( + IllegalStateException.class, + () -> newResolver("parquet", Map.of("name", Map.of("type", "text", "doc_values", false, "index", false))) + ); + assertTrue("expected 'no storage' error, got: " + ex.getMessage(), ex.getMessage().contains("has no storage in any format")); + } + + private static FieldStorageResolver newResolver(String primaryFormat, Map> fieldMappings) { + Map mappingSource = Map.of("properties", fieldMappings); + + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.sourceAsMap()).thenReturn(mappingSource); + + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.getIndex()).thenReturn(new Index("test_index", "uuid")); + when(indexMetadata.getSettings()).thenReturn(Settings.builder().put("index.composite.primary_data_format", primaryFormat).build()); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + + return new FieldStorageResolver(indexMetadata); + } +} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java index 606e79f33c621..f9a5aec1c22d9 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java @@ -172,7 +172,9 @@ public void testFullTextErrorsWithoutDelegation() { RexNode condition = makeFullTextCall(FilterOperator.MATCH_PHRASE.toSqlFunction(), 0, "hello world"); LogicalFilter filter = LogicalFilter.create(stubScan(table), condition); - PlannerContext context = buildContext("parquet", Map.of("message", Map.of("type", "keyword"))); + // index=false strips the inverted index so no backend can satisfy the full-text predicate + // natively, forcing the "without delegation" code path under test. + PlannerContext context = buildContext("parquet", Map.of("message", Map.of("type", "keyword", "index", false))); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> runPlanner(filter, context)); assertTrue(exception.getMessage().contains("No backend can evaluate filter predicate")); From 0b0eae9d00a247da0ec7fe141302f49e5052f9e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Apr 2026 06:23:20 -0400 Subject: [PATCH 009/115] Bump com.nimbusds:nimbus-jose-jwt in /test/fixtures/hdfs-fixture (#21214) Bumps [com.nimbusds:nimbus-jose-jwt](https://bitbucket.org/connect2id/nimbus-jose-jwt) from 10.8 to 10.9. - [Changelog](https://bitbucket.org/connect2id/nimbus-jose-jwt/src/master/CHANGELOG.txt) - [Commits](https://bitbucket.org/connect2id/nimbus-jose-jwt/branches/compare/10.9..10.8) --- updated-dependencies: - dependency-name: com.nimbusds:nimbus-jose-jwt dependency-version: '10.9' dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig Perkins --- test/fixtures/hdfs-fixture/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index d3a5a994fdeab..e8f1b06bd65e0 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -84,7 +84,7 @@ dependencies { api "org.jboss.xnio:xnio-nio:3.8.17.Final" api 'org.jline:jline:4.0.0' api 'org.apache.commons:commons-configuration2:2.13.0' - api 'com.nimbusds:nimbus-jose-jwt:10.8' + api 'com.nimbusds:nimbus-jose-jwt:10.9' api ('org.apache.kerby:kerb-admin:2.1.1') { exclude group: "org.jboss.xnio" exclude group: "org.jline" From 968438fb0a7998b157d23c7b7303e536519d3a94 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Apr 2026 06:35:37 -0400 Subject: [PATCH 010/115] Bump com.google.protobuf from 0.9.6 to 0.10.0 in /server (#21291) Bumps com.google.protobuf from 0.9.6 to 0.10.0. --- updated-dependencies: - dependency-name: com.google.protobuf dependency-version: 0.10.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sandesh Kumar Co-authored-by: Craig Perkins --- server/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/build.gradle b/server/build.gradle index 09f05853ff680..0e338f901cd5c 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -31,7 +31,7 @@ import groovy.xml.XmlParser plugins { - id('com.google.protobuf') version '0.9.6' + id('com.google.protobuf') version '0.10.0' id('opensearch.build') id('opensearch.publish') id('opensearch.internal-cluster-test') From d89118b7c2f7511b71dfa40e42256676ff04a6fb Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Tue, 28 Apr 2026 10:30:51 -0700 Subject: [PATCH 011/115] Passthrough backend evaluation when no RexCall in Projection (#21407) Signed-off-by: Vinay Krishna Pudyodu --- sandbox/plugins/analytics-engine/build.gradle | 2 ++ .../licenses/httpcore5-5.4.jar.sha1 | 1 + .../licenses/httpcore5-LICENSE.txt | 8 +++++ .../licenses/httpcore5-NOTICE.txt | 1 + .../planner/rules/OpenSearchProjectRule.java | 13 +++++-- .../analytics/planner/ProjectRuleTests.java | 36 +++++++++++++++++++ 6 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 sandbox/plugins/analytics-engine/licenses/httpcore5-5.4.jar.sha1 create mode 100644 sandbox/plugins/analytics-engine/licenses/httpcore5-LICENSE.txt create mode 100644 sandbox/plugins/analytics-engine/licenses/httpcore5-NOTICE.txt diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle index 451a3f767064a..f5993d130d9c5 100644 --- a/sandbox/plugins/analytics-engine/build.gradle +++ b/sandbox/plugins/analytics-engine/build.gradle @@ -102,6 +102,8 @@ dependencies { // Required at runtime by arrow-vector (ArrowType subclasses reference flatbuffers and arrow-format). runtimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" runtimeOnly "org.apache.arrow:arrow-format:${versions.arrow}" + // Required at runtime by avatica-core (BuiltInConnectionProperty. references Timeout). + runtimeOnly "org.apache.httpcomponents.core5:httpcore5:5.4" // SQL Unified Query API for PPL parsing testImplementation("org.opensearch.query:unified-query-api:${sqlUnifiedQueryVersion}") { diff --git a/sandbox/plugins/analytics-engine/licenses/httpcore5-5.4.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/httpcore5-5.4.jar.sha1 new file mode 100644 index 0000000000000..103becc417de6 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/httpcore5-5.4.jar.sha1 @@ -0,0 +1 @@ +e40011ec0dae056466399f8e414ede4772001621 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/httpcore5-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/httpcore5-LICENSE.txt new file mode 100644 index 0000000000000..f5f45d26a49d6 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/httpcore5-LICENSE.txt @@ -0,0 +1,8 @@ +This copy of Jackson JSON processor streaming parser/generator is licensed under the +Apache (Software) License, version 2.0 ("the License"). +See the License for details about distribution rights, and the +specific rights regarding derivate works. + +You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 diff --git a/sandbox/plugins/analytics-engine/licenses/httpcore5-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/httpcore5-NOTICE.txt new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/httpcore5-NOTICE.txt @@ -0,0 +1 @@ + diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java index 80b3d2544ec79..9124b0521e615 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java @@ -65,11 +65,20 @@ public void onMatch(RelOptRuleCall call) { // SqlKind → viable backends map once per onMatch() call, and (b) returning // childViableBackends directly when all candidates pass to avoid allocation. List annotatedExprs = new ArrayList<>(project.getProjects().size()); + boolean requiresBackendCapabilityEvaluation = false; for (RexNode expr : project.getProjects()) { - annotatedExprs.add(annotateExpr(expr, childViableBackends)); + RexNode annotated = annotateExpr(expr, childViableBackends); + annotatedExprs.add(annotated); + if (annotated instanceof AnnotatedProjectExpression) { + requiresBackendCapabilityEvaluation = true; + } } - List viableBackends = computeProjectViableBackends(annotatedExprs, childViableBackends); + // Passthrough projection: no RexCall to evaluate, so any child backend can emit it. + List viableBackends = requiresBackendCapabilityEvaluation + ? computeProjectViableBackends(annotatedExprs, childViableBackends) + : childViableBackends; + if (viableBackends.isEmpty()) { throw new IllegalStateException("No backend can execute all project expressions among " + childViableBackends); } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java index c225543886bcf..155b42b1f416d 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java @@ -77,6 +77,42 @@ public void testSimpleFieldProjection() { } } + public void testPassthroughProjectionSucceedsWithoutProjectCapability() { + // A backend that declares NO ProjectCapability should still execute a passthrough + // projection (only field refs). Verifies the short-circuit in OpenSearchProjectRule.onMatch + // that skips the backend-refinement gate when no RexCall needs evaluation. + OpenSearchProject result = runProject( + MockDataFusionBackend.PARQUET_DATA_FORMAT, + List.of(new MockDataFusionBackend(), LUCENE), + rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 0), + rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1) + ); + assertTrue(result.getViableBackends().contains(MockDataFusionBackend.NAME)); + for (RexNode expr : result.getProjects()) { + assertFalse("Passthrough expressions must not be annotated", expr instanceof AnnotatedProjectExpression); + } + } + + public void testExpressionProjectionStillRequiresCapabilityWithoutDeclaration() { + // Negative guard: the short-circuit must apply only to passthrough. If a RexCall is + // present and the backend declares no matching scalar ProjectCapability, the rule must + // still throw — otherwise a later refactor could silently loosen the gate too much. + RexNode castExpr = rexBuilder.makeCast( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1) + ); + RelOptTable table = mockTable( + "test_index", + new String[] { "name", "value" }, + new SqlTypeName[] { SqlTypeName.VARCHAR, SqlTypeName.INTEGER } + ); + LogicalProject project = LogicalProject.create(stubScan(table), List.of(), List.of(castExpr), List.of("casted")); + PlannerContext context = buildContext("parquet", nameValueFields(), List.of(new MockDataFusionBackend(), LUCENE)); + + IllegalStateException exception = expectThrows(IllegalStateException.class, () -> runPlanner(project, context)); + assertTrue(exception.getMessage().contains("No backend supports scalar function")); + } + // ---- Scalar functions ---- public void testSupportedScalarFunction() { From 66e4e3313dd184c0f4ba5f08510a36c3b792f5a9 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 28 Apr 2026 15:32:38 -0400 Subject: [PATCH 012/115] Onboard issue dedupe workflow (OpenSearch) (#21405) * Onboard issue dedupe workflow (OpenSearch) Signed-off-by: Peter Zhu * Add missing --- Signed-off-by: Peter Zhu --------- Signed-off-by: Peter Zhu --- .github/workflows/issue-dedupe.yml | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/issue-dedupe.yml diff --git a/.github/workflows/issue-dedupe.yml b/.github/workflows/issue-dedupe.yml new file mode 100644 index 0000000000000..c7299ebceb78a --- /dev/null +++ b/.github/workflows/issue-dedupe.yml @@ -0,0 +1,40 @@ +--- +name: Issue Dedupe Main +on: + issues: + types: [opened] + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + inputs: + issue_number: + description: 'Issue number to check for duplicates' + required: true + type: string + +jobs: + detect-issue: + if: >- + (github.event_name == 'workflow_dispatch' && + github.repository == 'opensearch-project/OpenSearch') || + (github.event_name == 'issues' && + github.event.issue.user.type != 'Bot' && + github.repository == 'opensearch-project/OpenSearch') + uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-detect.yml@main + permissions: + contents: read + issues: write + id-token: write + secrets: + BEDROCK_ACCESS_ROLE_ISSUE_DEDUPE: ${{ secrets.BEDROCK_ACCESS_ROLE_ISSUE_DEDUPE }} + with: + issue_number: ${{ inputs.issue_number || '' }} + grace_days: ${{ vars.DUPLICATE_GRACE_DAYS || '7' }} + + auto-close-issue: + if: github.event_name == 'schedule' && github.repository == 'opensearch-project/OpenSearch' + uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-autoclose.yml@main + permissions: + issues: write + with: + grace_days: ${{ vars.DUPLICATE_GRACE_DAYS || '7' }} From d16f1890e52abe97d8710bd7603517d9671bfd6b Mon Sep 17 00:00:00 2001 From: Terry Quigley <77437788+terryquigleysas@users.noreply.github.com> Date: Tue, 28 Apr 2026 20:37:26 +0100 Subject: [PATCH 013/115] Update logic in FIPS bootstrap check (#21415) * Update logic in FIPS bootstrap check Signed-off-by: Terry Quigley --- server/src/main/java/org/opensearch/bootstrap/Bootstrap.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java index d8f1592d7e7a4..70e365025fe07 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java @@ -197,7 +197,9 @@ private void setup(boolean addShutdownHook, Environment environment) throws Boot ); var cryptoStandard = System.getenv("OPENSEARCH_CRYPTO_STANDARD"); - if ("FIPS-140-3".equals(cryptoStandard) || "true".equalsIgnoreCase(System.getProperty("org.bouncycastle.fips.approved_only"))) { + var fipsMode = System.getenv("OPENSEARCH_FIPS_MODE"); + + if ("FIPS-140-3".equals(cryptoStandard) || "true".equalsIgnoreCase(fipsMode)) { LogManager.getLogger(Bootstrap.class).info("running in FIPS-140-3 mode"); SecurityProviderManager.removeNonCompliantFipsProviders(); FipsTrustStoreValidator.validate(); From 96bb22def4dac0900643440ceacfd03642754aad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Apr 2026 16:55:27 -0400 Subject: [PATCH 014/115] Bump org.apache.commons:commons-configuration2 from 2.13.0 to 2.14.0 in /plugins/repository-hdfs (#21213) * Bump org.apache.commons:commons-configuration2 Bumps org.apache.commons:commons-configuration2 from 2.13.0 to 2.14.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-configuration2 dependency-version: 2.14.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig Perkins --- plugins/repository-hdfs/build.gradle | 2 +- .../licenses/commons-configuration2-2.13.0.jar.sha1 | 1 - .../licenses/commons-configuration2-2.14.0.jar.sha1 | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-hdfs/licenses/commons-configuration2-2.13.0.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/commons-configuration2-2.14.0.jar.sha1 diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index ebbf6fac12feb..130ad6390b6e2 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -77,7 +77,7 @@ dependencies { api "commons-codec:commons-codec:${versions.commonscodec}" api 'commons-collections:commons-collections:3.2.2' api "org.apache.commons:commons-compress:${versions.commonscompress}" - api 'org.apache.commons:commons-configuration2:2.13.0' + api 'org.apache.commons:commons-configuration2:2.14.0' api "commons-io:commons-io:${versions.commonsio}" api "org.apache.commons:commons-lang3:${versions.commonslang}" implementation 'com.google.re2j:re2j:1.8' diff --git a/plugins/repository-hdfs/licenses/commons-configuration2-2.13.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-configuration2-2.13.0.jar.sha1 deleted file mode 100644 index f1a1be6428197..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-configuration2-2.13.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc795ed544554745c52d56e0ab5f42529a7cef4e \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-configuration2-2.14.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-configuration2-2.14.0.jar.sha1 new file mode 100644 index 0000000000000..15718a98afbdb --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-configuration2-2.14.0.jar.sha1 @@ -0,0 +1 @@ +dcbaad997c64f3df16d8c75c730ad7aaf0d2d2a3 \ No newline at end of file From 8804f3c29105971c20103d68b20352252028b858 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 29 Apr 2026 08:09:52 -0700 Subject: [PATCH 015/115] update DefaultPlanExecutor to be async. (#21423) * update DefaultPlanExecutor to be async. This change updates DefaultPlanExecutor to accept an actionlistener from front-end plugins. Signed-off-by: Marc Handalian * automated code review fixes Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../analytics/exec/QueryPlanExecutor.java | 13 +-- .../analytics/exec/DefaultPlanExecutor.java | 86 +++++++++++++------ .../exec/action/AnalyticsQueryAction.java | 4 +- .../exec/DefaultPlanExecutorTests.java | 83 ++++++++++++++++++ .../ppl/planner/PushDownPlannerTests.java | 2 +- .../rel/OpenSearchBoundaryTableScan.java | 7 +- .../rel/OpenSearchBoundaryTableScanTests.java | 20 ++--- .../ppl/planner/rules/PushDownRulesTests.java | 2 +- .../dsl/action/TransportDslExecuteAction.java | 30 ++++--- .../dsl/executor/DslQueryPlanExecutor.java | 40 ++++++--- .../TransportDslExecuteActionTests.java | 4 +- .../executor/DslQueryPlanExecutorTests.java | 7 +- 12 files changed, 227 insertions(+), 71 deletions(-) diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/exec/QueryPlanExecutor.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/exec/QueryPlanExecutor.java index 6353f4b749977..6342aada1445d 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/exec/QueryPlanExecutor.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/exec/QueryPlanExecutor.java @@ -8,6 +8,8 @@ package org.opensearch.analytics.exec; +import org.opensearch.core.action.ActionListener; + /** * Executes a logical query plan fragment against the underlying data store. * @@ -17,11 +19,12 @@ public interface QueryPlanExecutor { /** - * Executes the given logical fragment and returns result rows. + * Executes the given logical fragment and delivers the result stream (or a failure) + * to {@code listener}. * - * @param plan the logical subtree to execute - * @param context execution context (opaque Object to avoid server dependency) - * @return rows produced by the engine + * @param plan the logical subtree to execute + * @param context execution context (opaque Object to avoid server dependency) + * @param listener receives the produced stream on success, or the failure cause on error */ - Stream execute(LogicalPlan plan, Object context); + void execute(LogicalPlan plan, Object context, ActionListener listener); } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java index 0d3752a976d2c..b2fa4d08c3649 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java @@ -15,7 +15,6 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.TimeoutTaskCancellationUtility; import org.opensearch.analytics.EngineContext; import org.opensearch.analytics.exec.action.AnalyticsQueryAction; @@ -54,9 +53,10 @@ * so that Guice injects all dependencies ({@link TransportService}, * {@link ClusterService}, {@link ThreadPool}, etc.) automatically. * - *

The SQL plugin resolves this class from the Node's Guice injector and invokes - * {@link #execute(RelNode, Object)} directly. The transport path ({@code doExecute}) - * is reserved for future remote query invocation. + *

Front-end plugins resolve this class from the Node's Guice injector and invoke + * {@link #execute(RelNode, Object, ActionListener)} directly. Execution is asynchronous — + * the listener is fired by the scheduler once the query completes (or fails). The transport + * path ({@code doExecute}) is reserved for future remote query invocation. * * @opensearch.internal */ @@ -99,7 +99,27 @@ public DefaultPlanExecutor( // that takes the logical fragment and returns a fully-built DAG ready for scheduling. // Also add per-step timing (plan, fork, convert, schedule, execute) for observability. @Override - public Iterable execute(RelNode logicalFragment, Object context) { + public void execute(RelNode logicalFragment, Object context, ActionListener> listener) { + // Fork the entire query lifecycle (planning, scheduling, cleanup) onto the SEARCH + // executor so the calling thread — which may be a transport thread — is freed + // immediately. The scheduler then drives execution asynchronously and fires + // {@code listener} once the query terminates; nothing on this path blocks. + searchExecutor.execute(() -> { + try { + executeInternal(logicalFragment, listener); + } catch (Exception e) { + listener.onFailure(e); + } + }); + } + + /** + * Plans, registers the query task, and dispatches to the {@link Scheduler}. Runs on + * the SEARCH thread pool — never on a transport thread. The result (or failure) is + * delivered to {@code listener} by the scheduler; this method returns as soon as the + * scheduler has accepted the query. + */ + private void executeInternal(RelNode logicalFragment, ActionListener> listener) { RelNode plan = PlannerImpl.createPlan(logicalFragment, new PlannerContext(capabilityRegistry, clusterService.state())); QueryDAG dag = DAGBuilder.build(plan, capabilityRegistry, clusterService); PlanForker.forkAll(dag, capabilityRegistry); @@ -108,49 +128,47 @@ public Iterable execute(RelNode logicalFragment, Object context) { // Register coordinator-level query task with TaskManager (like SearchTask). // This gives us a proper unique ID, visibility in _tasks API, and cancellation support. - // TODO: accept a request type from FrontEnd including cancelAfterTimeInterval - its set from cluster settings below, null in req. + // TODO: accept a request type from FrontEnd including cancelAfterTimeInterval — set from cluster settings below, null in req. final AnalyticsQueryTask queryTask = (AnalyticsQueryTask) taskManager.register( "transport", "analytics_query", new AnalyticsQueryTaskRequest(dag.queryId(), null) ); - - // Create per-query context - QueryContext config = new QueryContext(dag, searchExecutor, queryTask); - - PlainActionFuture> future = new PlainActionFuture<>(); + final QueryContext config = new QueryContext(dag, searchExecutor, queryTask); // Per-query cleanup on terminal. Stage-execution cancellation on external // task-cancel/timeout is wired inside the Scheduler — on this path the // walker has already cascaded cancellations by the time we see the failure. // Scheduler yields batches; we materialize rows at the API edge for callers // that still consume Iterable. - ActionListener> listener = ActionListener.wrap(batches -> { - Iterable rows = batchesToRows(batches); - config.closeBufferAllocator(); - taskManager.unregister(queryTask); - future.onResponse(rows); - }, e -> { - config.closeBufferAllocator(); - taskManager.unregister(queryTask); - future.onFailure(e); + ActionListener> batchesListener = buildBatchesListener(listener, () -> { + try { + config.closeBufferAllocator(); + } finally { + taskManager.unregister(queryTask); + } }); TimeValue taskTimeout = queryTask.getCancelAfterTimeInterval(); TimeValue clusterTimeout = clusterService.getClusterSettings().get(SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING); if (taskTimeout != null || SearchService.NO_TIMEOUT.equals(clusterTimeout) == false) { - listener = TimeoutTaskCancellationUtility.wrapWithCancellationListener(client, queryTask, clusterTimeout, listener, e -> {}); + batchesListener = TimeoutTaskCancellationUtility.wrapWithCancellationListener( + client, + queryTask, + clusterTimeout, + batchesListener, + e -> {} + ); } - scheduler.execute(config, listener); - return future.actionGet(); // TODO: single blocking point — Should be async with Front-End passing listener. + scheduler.execute(config, batchesListener); } @Override protected void doExecute(Task task, ActionRequest request, ActionListener listener) { // Transport path — reserved for future remote query invocation. - // Currently, the SQL plugin invokes execute(RelNode, Object) directly. - listener.onFailure(new UnsupportedOperationException("Direct invocation only — use execute(RelNode, Object)")); + // Currently, front-ends invoke execute(RelNode, Object, ActionListener) directly. + listener.onFailure(new UnsupportedOperationException("Direct invocation only — use execute(RelNode, Object, ActionListener)")); } /** @@ -184,6 +202,24 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, } } + /** + * Builds the batches→rows {@link ActionListener} used by {@link #executeInternal}. {@code cleanup} + * runs exactly once before {@code downstream} is notified — on either response or failure paths. + * A cleanup failure on the response path is routed to {@code downstream.onFailure}; on the failure + * path it is attached as a suppressed exception. This eliminates the double-cleanup that the prior + * try/finally pattern produced when an exception in the success path was caught by + * {@link ActionListener#wrap} and re-routed to the failure callback. + * + *

Package-private for unit testing. + */ + static ActionListener> buildBatchesListener( + ActionListener> downstream, + Runnable cleanup + ) { + ActionListener> wrapped = ActionListener.runBefore(downstream, cleanup::run); + return ActionListener.wrap(batches -> wrapped.onResponse(batchesToRows(batches)), wrapped::onFailure); + } + /** * Materializes Arrow batches into row-oriented {@code Object[]}s for the * external query API. The scheduler yields batches (the native wire format); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/AnalyticsQueryAction.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/AnalyticsQueryAction.java index 58bd6906d252f..bb91d4d3c2a72 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/AnalyticsQueryAction.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/AnalyticsQueryAction.java @@ -19,8 +19,8 @@ *

Currently used as a Guice injection vehicle for {@link DefaultPlanExecutor} * — the transport action registration lets Guice construct the executor with all * its dependencies ({@code TransportService}, {@code ClusterService}, etc.). - * The SQL plugin invokes the executor directly via - * {@link QueryPlanExecutor#execute(Object, Object)}, not through transport. + * Front-end plugins invoke the executor directly via + * {@link QueryPlanExecutor#execute}, not through transport. * *

Future: the transport path ({@code doExecute}) will accept query strings * for remote invocation. diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java index 761b683c54982..3d209066229d6 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java @@ -18,12 +18,15 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.core.action.ActionListener; import org.opensearch.test.OpenSearchTestCase; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * Tests for {@link DefaultPlanExecutor}'s row-materialization boundary. @@ -138,6 +141,86 @@ public void testBatchesToRowsVarCharDecodedAsString() { assertTrue(rows.get(0)[0] instanceof String); } + public void testBuildBatchesListenerSuccessRunsCleanupOnce() { + AtomicInteger cleanupCount = new AtomicInteger(0); + AtomicReference> result = new AtomicReference<>(); + AtomicReference failure = new AtomicReference<>(); + ActionListener> downstream = ActionListener.wrap(result::set, failure::set); + + ActionListener> batchesListener = DefaultPlanExecutor.buildBatchesListener( + downstream, + cleanupCount::incrementAndGet + ); + + VectorSchemaRoot batch = makeIntBatch("x", 1, 2); + batchesListener.onResponse(List.of(batch)); + + assertEquals(1, cleanupCount.get()); + assertNotNull(result.get()); + assertEquals(2, toList(result.get()).size()); + assertNull(failure.get()); + } + + public void testBuildBatchesListenerFailureRunsCleanupOnce() { + AtomicInteger cleanupCount = new AtomicInteger(0); + AtomicReference> result = new AtomicReference<>(); + AtomicReference failure = new AtomicReference<>(); + ActionListener> downstream = ActionListener.wrap(result::set, failure::set); + + ActionListener> batchesListener = DefaultPlanExecutor.buildBatchesListener( + downstream, + cleanupCount::incrementAndGet + ); + + Exception cause = new RuntimeException("upstream failure"); + batchesListener.onFailure(cause); + + assertEquals(1, cleanupCount.get()); + assertNull(result.get()); + assertSame(cause, failure.get()); + } + + public void testBuildBatchesListenerConversionFailureRoutesToFailureWithSingleCleanup() { + AtomicInteger cleanupCount = new AtomicInteger(0); + AtomicReference> result = new AtomicReference<>(); + AtomicReference failure = new AtomicReference<>(); + ActionListener> downstream = ActionListener.wrap(result::set, failure::set); + + ActionListener> batchesListener = DefaultPlanExecutor.buildBatchesListener( + downstream, + cleanupCount::incrementAndGet + ); + + Iterable badBatches = () -> { throw new RuntimeException("conversion failed"); }; + batchesListener.onResponse(badBatches); + + assertEquals("cleanup must run exactly once when conversion throws", 1, cleanupCount.get()); + assertNull(result.get()); + assertNotNull(failure.get()); + assertEquals("conversion failed", failure.get().getMessage()); + } + + public void testBuildBatchesListenerCleanupFailureOnSuccessRoutesToFailure() { + AtomicInteger cleanupCount = new AtomicInteger(0); + AtomicReference> result = new AtomicReference<>(); + AtomicReference failure = new AtomicReference<>(); + ActionListener> downstream = ActionListener.wrap(result::set, failure::set); + + Runnable cleanup = () -> { + cleanupCount.incrementAndGet(); + throw new RuntimeException("cleanup failed"); + }; + ActionListener> batchesListener = DefaultPlanExecutor.buildBatchesListener(downstream, cleanup); + + VectorSchemaRoot batch = makeIntBatch("x", 1, 2); + batchesListener.onResponse(List.of(batch)); + + assertEquals("cleanup runs exactly once even when it throws", 1, cleanupCount.get()); + assertNull("downstream onResponse must not fire when cleanup throws on success path", result.get()); + assertNotNull(failure.get()); + assertEquals("cleanup failed", failure.get().getMessage()); + } + public void testBatchesToRowsClosesBatches() { BufferAllocator child = allocator.newChildAllocator("test", 0, Long.MAX_VALUE); VectorSchemaRoot batch = makeIntBatch(child, "x", 1, 2); diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlannerTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlannerTests.java index 406abf0a69543..a73456e5afa39 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlannerTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlannerTests.java @@ -84,7 +84,7 @@ public RelDataType getRowType(RelDataTypeFactory tf) { table = catalogReader.getTable(List.of("test_table")); assertNotNull("Table should be found in catalog", table); - planExecutor = (fragment, ctx) -> Collections.emptyList(); + planExecutor = (fragment, ctx, l) -> l.onResponse(Collections.emptyList()); } /** diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScan.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScan.java index 976fda062956e..14db142f2b4b7 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScan.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScan.java @@ -113,7 +113,12 @@ public Enumerable execute() { @SuppressWarnings("unchecked") public Enumerable bind(DataContext dataContext) { try { - Iterable result = (Iterable) planExecutor.execute(logicalFragment, dataContext); + // Calcite's bind() is synchronous (driven by the EnumerableInterpreter), so we + // block here on the async planExecutor until the engine completes. + org.opensearch.action.support.PlainActionFuture> future = + new org.opensearch.action.support.PlainActionFuture<>(); + planExecutor.execute(logicalFragment, dataContext, future); + Iterable result = future.actionGet(); return Linq4j.asEnumerable(result); } catch (Exception e) { throw new RuntimeException( diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScanTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScanTests.java index 3c74ef9431d05..b9c152b084d12 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScanTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScanTests.java @@ -84,7 +84,7 @@ public RelDataType getRowType(RelDataTypeFactory tf) { public void testExtendsTableScanNotLogicalTableScan() { LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx) -> Linq4j.emptyEnumerable(); + QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); @@ -95,7 +95,7 @@ public void testExtendsTableScanNotLogicalTableScan() { public void testImplementsEnumerableRel() { LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx) -> Linq4j.emptyEnumerable(); + QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); @@ -112,10 +112,10 @@ public void testBindCallsEngineExecutorWithLogicalFragment() { final RelNode[] capturedFragment = new RelNode[1]; final Object[] capturedContext = new Object[1]; Object[][] rows = { new Object[] { 1, "a", 1.0 } }; - QueryPlanExecutor> executor = (fragment, ctx) -> { + QueryPlanExecutor> executor = (fragment, ctx, l) -> { capturedFragment[0] = fragment; capturedContext[0] = ctx; - return Linq4j.asEnumerable(rows); + l.onResponse(Linq4j.asEnumerable(rows)); }; OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); @@ -134,9 +134,9 @@ public void testBindPassesFilterFragmentToExecutor() { RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); final RelNode[] capturedFragment = new RelNode[1]; - QueryPlanExecutor> executor = (fragment, ctx) -> { + QueryPlanExecutor> executor = (fragment, ctx, l) -> { capturedFragment[0] = fragment; - return Linq4j.emptyEnumerable(); + l.onResponse(Linq4j.emptyEnumerable()); }; OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, filter, executor); @@ -151,7 +151,7 @@ public void testBindPassesFilterFragmentToExecutor() { public void testCopyPreservesLogicalFragment() { LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx) -> Linq4j.emptyEnumerable(); + QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); @@ -165,7 +165,7 @@ public void testCopyPreservesLogicalFragment() { public void testCopyPreservesTable() { LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx) -> Linq4j.emptyEnumerable(); + QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); @@ -180,7 +180,7 @@ public void testCopyPreservesTable() { public void testGetLogicalFragmentReturnsScanSubtree() { LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx) -> Linq4j.emptyEnumerable(); + QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); @@ -192,7 +192,7 @@ public void testGetLogicalFragmentReturnsFilterSubtree() { RexNode condition = rexBuilder.makeLiteral(true); LogicalFilter filter = LogicalFilter.create(scan, condition); RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx) -> Linq4j.emptyEnumerable(); + QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, filter, executor); diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/PushDownRulesTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/PushDownRulesTests.java index bd8e109846c72..7fcdd83eec35a 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/PushDownRulesTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/PushDownRulesTests.java @@ -86,7 +86,7 @@ public RelDataType getRowType(RelDataTypeFactory tf) { table = catalogReader.getTable(List.of("test_table")); assertNotNull("Table should be found in catalog", table); - planExecutor = (fragment, ctx) -> Linq4j.emptyEnumerable(); + planExecutor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); } // --- BoundaryTableScanRule tests (ConverterRule, uses VolcanoPlanner) --- diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java index f3e0cba19b228..25b150e7cefd9 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java @@ -25,14 +25,11 @@ import org.opensearch.dsl.converter.SearchSourceConverter; import org.opensearch.dsl.executor.DslQueryPlanExecutor; import org.opensearch.dsl.executor.QueryPlans; -import org.opensearch.dsl.result.ExecutionResult; import org.opensearch.dsl.result.SearchResponseBuilder; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; -import java.util.List; - /** * Coordinates DSL query execution: converts SearchSourceBuilder to Calcite RelNode plans, * executes them via the analytics engine, and builds a SearchResponse. @@ -81,20 +78,33 @@ public TransportDslExecuteAction( @Override protected void doExecute(Task task, SearchRequest request, ActionListener listener) { threadPool.executor(ThreadPool.Names.SEARCH).execute(() -> { + final QueryPlans plans; + final long convertTime; try { String indexName = resolveToSingleIndex(request); - long convertStart = System.nanoTime(); SearchSourceConverter converter = new SearchSourceConverter(engineContext.getSchema()); - QueryPlans plans = converter.convert(request.source(), indexName); - long convertTime = System.nanoTime() - convertStart; - List results = planExecutor.execute(plans); - SearchResponse response = SearchResponseBuilder.build(results, convertTime); - listener.onResponse(response); + plans = converter.convert(request.source(), indexName); + convertTime = System.nanoTime() - convertStart; } catch (Exception e) { - logger.error("DSL execution failed", e); + logger.error("DSL conversion failed", e); listener.onFailure(e); + return; } + planExecutor.execute(plans, ActionListener.wrap(results -> { + final SearchResponse response; + try { + response = SearchResponseBuilder.build(results, convertTime); + } catch (Exception buildEx) { + logger.error("DSL response building failed", buildEx); + listener.onFailure(buildEx); + return; + } + listener.onResponse(response); + }, e -> { + logger.error("DSL execution failed", e); + listener.onFailure(e); + })); }); } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/executor/DslQueryPlanExecutor.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/executor/DslQueryPlanExecutor.java index 80ffbdeb0fc05..ac962c984c7fd 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/executor/DslQueryPlanExecutor.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/executor/DslQueryPlanExecutor.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.analytics.exec.QueryPlanExecutor; +import org.opensearch.core.action.ActionListener; import org.opensearch.dsl.result.ExecutionResult; import java.util.ArrayList; @@ -40,25 +41,40 @@ public DslQueryPlanExecutor(QueryPlanExecutor> execu // TODO: add per-plan error handling so a failure in one plan // doesn't prevent returning partial results from other plans (e.g. HITS) /** - * Executes all plans and returns results in plan order. + * Executes all plans sequentially and delivers results, in plan order, to the listener. * - * @param plans the query plans to execute - * @return execution results, one per plan + *

Plans run one-at-a-time: plan {@code N+1} is dispatched only after plan {@code N} + * completes successfully. The first failure aborts the chain — the listener fires + * {@code onFailure} with that error and remaining plans do not run. + * + * @param plans the query plans to execute + * @param listener receives the ordered list of results on success, or the first failure */ - public List execute(QueryPlans plans) { + public void execute(QueryPlans plans, ActionListener> listener) { List queryPlans = plans.getAll(); List results = new ArrayList<>(queryPlans.size()); + executeNext(queryPlans, 0, results, listener); + } - for (QueryPlans.QueryPlan plan : queryPlans) { - RelNode relNode = plan.relNode(); - logPlan(relNode); - // TODO: context param is null, may carry execution hints - Iterable rows = executor.execute(relNode, null); + private void executeNext( + List queryPlans, + int index, + List results, + ActionListener> outer + ) { + if (index >= queryPlans.size()) { + outer.onResponse(results); + return; + } + QueryPlans.QueryPlan plan = queryPlans.get(index); + RelNode relNode = plan.relNode(); + logPlan(relNode); + // TODO: context param is null, may carry execution hints + executor.execute(relNode, null, ActionListener.wrap(rows -> { logRows(rows); results.add(new ExecutionResult(plan, rows)); - } - - return results; + executeNext(queryPlans, index + 1, results, outer); + }, outer::onFailure)); } private static void logRows(Iterable rows) { diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/action/TransportDslExecuteActionTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/action/TransportDslExecuteActionTests.java index 05e4d6d910163..0679cd4e8b1ae 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/action/TransportDslExecuteActionTests.java +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/action/TransportDslExecuteActionTests.java @@ -86,7 +86,7 @@ public void testDoExecuteFailsWhenIndexNotInClusterState() { mock(TransportService.class), new ActionFilters(Collections.emptySet()), buildEngineContext(), - (plan, ctx) -> Collections.emptyList(), + (plan, ctx, l) -> l.onResponse(Collections.emptyList()), clusterService, resolver, mockThreadPool() @@ -119,7 +119,7 @@ private TransportDslExecuteAction createAction(Index... resolvedIndices) { mock(TransportService.class), new ActionFilters(Collections.emptySet()), buildEngineContext(), - (plan, ctx) -> Collections.emptyList(), + (plan, ctx, l) -> l.onResponse(Collections.emptyList()), clusterService, resolver, mockThreadPool() diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/executor/DslQueryPlanExecutorTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/executor/DslQueryPlanExecutorTests.java index d135d45de1fe5..3dbd2acfd1ef8 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/executor/DslQueryPlanExecutorTests.java +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/executor/DslQueryPlanExecutorTests.java @@ -9,6 +9,7 @@ package org.opensearch.dsl.executor; import org.apache.calcite.rel.logical.LogicalTableScan; +import org.opensearch.action.support.PlainActionFuture; import org.opensearch.dsl.TestUtils; import org.opensearch.dsl.result.ExecutionResult; import org.opensearch.test.OpenSearchTestCase; @@ -28,10 +29,12 @@ public void setUp() throws Exception { public void testExecuteDelegatesEachPlanToExecutor() { List expectedRows = List.of(new Object[] { "laptop", 1200 }); - DslQueryPlanExecutor executor = new DslQueryPlanExecutor((plan, ctx) -> expectedRows); + DslQueryPlanExecutor executor = new DslQueryPlanExecutor((plan, ctx, listener) -> listener.onResponse(expectedRows)); QueryPlans plans = new QueryPlans.Builder().add(new QueryPlans.QueryPlan(QueryPlans.Type.HITS, scan)).build(); - List results = executor.execute(plans); + PlainActionFuture> future = new PlainActionFuture<>(); + executor.execute(plans, future); + List results = future.actionGet(); assertEquals(1, results.size()); ExecutionResult result = results.get(0); From 0f24ad7e1629b415fe5087cb4df8b122893b7d68 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Wed, 29 Apr 2026 09:57:06 -0700 Subject: [PATCH 016/115] Added queryTimeout to the IndexSearcher to ensure that queries that uses this timeout during search exit when queries timesout (#21316) Signed-off-by: Navneet Verma --- .../search/internal/ContextIndexSearcher.java | 47 +++++++++- .../internal/ContextIndexSearcherTests.java | 90 +++++++++++++++++++ 2 files changed, 136 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java index 755c70111ae75..4ebc0838c27a9 100644 --- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java @@ -37,6 +37,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.QueryTimeout; import org.apache.lucene.index.Term; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.CollectionStatistics; @@ -69,6 +70,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; +import org.opensearch.core.tasks.TaskCancelledException; import org.opensearch.lucene.util.CombinedBitSet; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchHits; @@ -157,6 +159,13 @@ private ContextIndexSearcher( setQueryCachingPolicy(queryCachingPolicy); this.cancellable = cancellable; this.searchContext = searchContext; + // Set the timeout on the IndexSearcher so that Lucene-native timeout-aware components + // (e.g. TimeLimitingKnnCollectorManager used by AbstractKnnVectorQuery) can enforce + // the query timeout. Without this, searcher.getTimeout() returns null and KNN vector + // searches ignore the configured query timeout entirely. + if (cancellable != null) { + setTimeout(cancellable); + } } public void setProfiler(QueryProfiler profiler) { @@ -604,7 +613,23 @@ public DirectoryReader getDirectoryReader() { return (DirectoryReader) reader; } - private static class MutableQueryTimeout implements ExitableDirectoryReader.QueryCancellation { + /** + * A mutable timeout implementation that bridges OpenSearch's cancellation mechanism with Lucene's + * {@link QueryTimeout} interface. + *

+ * This class implements both {@link ExitableDirectoryReader.QueryCancellation} (used by OpenSearch's + * {@link ExitableDirectoryReader} to check for cancellation while iterating terms, points, and stored fields) + * and {@link QueryTimeout} (used by Lucene's {@link org.apache.lucene.search.IndexSearcher} to enforce + * timeouts in components like {@link org.apache.lucene.search.TimeLimitingKnnCollectorManager} for KNN + * vector queries). + *

+ * Cancellation runnables are added/removed dynamically via {@link #add} and {@link #remove}. When any + * runnable throws a {@link RuntimeException} (e.g. {@link org.opensearch.search.query.QueryPhase.TimeExceededException}), + * it signals that the query should be terminated. + * + * @opensearch.internal + */ + private static class MutableQueryTimeout implements ExitableDirectoryReader.QueryCancellation, QueryTimeout { private final Set runnables = new HashSet<>(); @@ -632,6 +657,26 @@ public boolean isEnabled() { return runnables.isEmpty() == false; } + /** + * Implements {@link QueryTimeout#shouldExit()} by delegating to {@link #checkCancelled()}. + * Returns {@code true} if a registered cancellation runnable throws a + * {@link org.opensearch.search.query.QueryPhase.TimeExceededException} (timeout) or + * {@link org.opensearch.core.tasks.TaskCancelledException} (task cancellation), + * indicating that the query should be terminated early. + *

+ * This is called by Lucene's {@link org.apache.lucene.search.TimeLimitingKnnCollectorManager} + * during KNN vector search to check whether the search should be terminated early. + */ + @Override + public boolean shouldExit() { + try { + checkCancelled(); + } catch (QueryPhase.TimeExceededException | TaskCancelledException e) { + return true; + } + return false; + } + public void clear() { runnables.clear(); } diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java index 6ea54e619c277..fc94e715435b3 100644 --- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java @@ -46,6 +46,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.QueryTimeout; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BoostQuery; @@ -72,11 +73,13 @@ import org.apache.lucene.util.SparseFixedBitSet; import org.opensearch.ExceptionsHelper; import org.opensearch.action.support.StreamSearchChannelListener; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.lucene.index.SequentialStoredFieldsLeafReader; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.tasks.TaskCancelledException; import org.opensearch.index.IndexSettings; import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.shard.IndexShard; @@ -88,6 +91,7 @@ import org.opensearch.search.aggregations.metrics.InternalSum; import org.opensearch.search.fetch.FetchSearchResult; import org.opensearch.search.fetch.QueryFetchSearchResult; +import org.opensearch.search.query.QueryPhase; import org.opensearch.search.query.QuerySearchResult; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; @@ -586,6 +590,92 @@ public void visit(QueryVisitor visitor) { } } + public void testTimeoutIsSetOnSearcher() throws Exception { + withContextIndexSearcher(searcher -> { + QueryTimeout timeout = searcher.getTimeout(); + assertNotNull("setTimeout should have been called with MutableQueryTimeout", timeout); + }); + } + + public void testTimeoutShouldExitReturnsFalseWhenNoCancellations() throws Exception { + withContextIndexSearcher(searcher -> { + assertFalse("shouldExit should return false when no cancellations are registered", searcher.getTimeout().shouldExit()); + }); + } + + public void testTimeoutShouldExitReturnsFalseWhenCancellationDoesNotThrow() throws Exception { + withContextIndexSearcher(searcher -> { + searcher.addQueryCancellation(() -> {}); + assertFalse("shouldExit should return false when cancellation does not throw", searcher.getTimeout().shouldExit()); + }); + } + + public void testTimeoutShouldExitReturnsTrueWhenTimeoutExceeded() throws Exception { + withContextIndexSearcher(searcher -> { + searcher.addQueryCancellation(() -> { throw new QueryPhase.TimeExceededException(); }); + assertTrue("shouldExit should return true on TimeExceededException", searcher.getTimeout().shouldExit()); + }); + } + + public void testTimeoutShouldExitReturnsTrueWhenTaskCancelled() throws Exception { + withContextIndexSearcher(searcher -> { + searcher.addQueryCancellation(() -> { throw new TaskCancelledException("cancelled"); }); + assertTrue("shouldExit should return true on TaskCancelledException", searcher.getTimeout().shouldExit()); + }); + } + + public void testTimeoutShouldExitDoesNotCatchUnrelatedExceptions() throws Exception { + withContextIndexSearcher(searcher -> { + searcher.addQueryCancellation(() -> { throw new NullPointerException("unrelated"); }); + expectThrows(NullPointerException.class, () -> searcher.getTimeout().shouldExit()); + }); + } + + public void testTimeoutShouldExitReflectsRemoval() throws Exception { + withContextIndexSearcher(searcher -> { + Runnable cancellation = searcher.addQueryCancellation(() -> { throw new QueryPhase.TimeExceededException(); }); + assertTrue("shouldExit should return true while cancellation is active", searcher.getTimeout().shouldExit()); + + searcher.removeQueryCancellation(cancellation); + assertFalse("shouldExit should return false after cancellation is removed", searcher.getTimeout().shouldExit()); + }); + } + + /** + * Helper that creates a {@link ContextIndexSearcher} backed by a single-doc index and a mocked + * {@link SearchContext}, then passes it to the provided consumer. All resources are closed + * automatically. + */ + private void withContextIndexSearcher(CheckedConsumer test) throws Exception { + try ( + Directory directory = newDirectory(); + IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new StandardAnalyzer())) + ) { + Document doc = new Document(); + doc.add(new StringField("field", "value", Field.Store.NO)); + writer.addDocument(doc); + writer.commit(); + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + SearchContext searchContext = mock(SearchContext.class); + IndexShard indexShard = mock(IndexShard.class); + when(searchContext.indexShard()).thenReturn(indexShard); + when(searchContext.bucketCollectorProcessor()).thenReturn(SearchContext.NO_OP_BUCKET_COLLECTOR_PROCESSOR); + + ContextIndexSearcher searcher = new ContextIndexSearcher( + reader, + IndexSearcher.getDefaultSimilarity(), + IndexSearcher.getDefaultQueryCache(), + IndexSearcher.getDefaultQueryCachingPolicy(), + true, + null, + searchContext + ); + test.accept(searcher); + } + } + } + public void testSendBatchWithSingleAggregation() throws Exception { try ( Directory directory = newDirectory(); From 613c16661d1d530f3ef7908d7dcdc5ec12eb760b Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 29 Apr 2026 10:02:45 -0700 Subject: [PATCH 017/115] Coordinator side DataFusion reduce (#21356) * coord-reduce: end-to-end streaming sink + IT - DatafusionReduceSink: native streaming ExchangeSink via Substrait + Arrow C-Data - Wire engine phases (PlanForker, FragmentConversionDriver) in DefaultPlanExecutor - Convertor: stage-input rewrite, partial-agg/final-agg methods - LocalStageExecution: outputSource wired to downstream; no premature close - CoordinatorReduceIT: 2-shard parquet-backed composite index smoke test Signed-off-by: Marc Handalian * add memtable implementation Signed-off-by: Marc Handalian * checkpoint Signed-off-by: Marc Handalian * move test-ppl-plugin to a top lvl plugin to be reused across qa and analytics-engine ITs. This also removes PushDownPlanner, which is no longer used. Signed-off-by: Marc Handalian * unwanted changes Signed-off-by: Marc Handalian * reduce duplication between sink implementations. Signed-off-by: Marc Handalian * Minor improvements to coordinator reduce Signed-off-by: Bukhtawar Khan * Fix up Signed-off-by: Bukhtawar Khan * Fix up Signed-off-by: Bukhtawar Khan * Fix up Signed-off-by: Bukhtawar Khan * add explicit drain thread and ensure cpu runtime is used for query execution Signed-off-by: Marc Handalian * spotless Signed-off-by: Marc Handalian * fix: seed Calcite THREAD_PROVIDERS in DefaultPlanExecutor.executeInternal to prevent NPE on SEARCH-pool thread Signed-off-by: Marc Handalian * fix merge conflict Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian Signed-off-by: Bukhtawar Khan Co-authored-by: Bukhtawar Khan Co-authored-by: Bukhtawar Khan --- gradle/run.gradle | 9 + sandbox/libs/analytics-framework/build.gradle | 72 +-- .../analytics/spi/ExchangeSinkContext.java | 44 ++ .../analytics/spi/ExchangeSinkProvider.java | 10 +- .../nativebridge/spi/NativeCall.java | 15 + .../analytics-backend-datafusion/build.gradle | 126 +++- .../licenses/core-LICENSE.txt | 206 ++++++- .../licenses/core-NOTICE.txt | 6 + .../licenses/isthmus-LICENSE.txt | 206 ++++++- .../licenses/isthmus-NOTICE.txt | 6 + .../jackson-datatype-jdk8-LICENSE.txt | 4 +- .../licenses/jackson-datatype-jdk8-NOTICE.txt | 16 + .../rust/src/api.rs | 244 +++++++- .../rust/src/executor.rs | 2 +- .../rust/src/ffm.rs | 127 ++++ .../rust/src/lib.rs | 2 + .../rust/src/local_executor.rs | 328 ++++++++++ .../rust/src/partition_stream.rs | 307 ++++++++++ .../rust/tests/local_exec_test.rs | 400 +++++++++++++ .../be/datafusion/CoordinatorReduceIT.java | 163 +++++ .../CoordinatorReduceMemtableIT.java | 135 +++++ .../AbstractDatafusionReduceSink.java | 203 +++++++ .../be/datafusion/ArrowSchemaIpc.java | 42 ++ .../DataFusionAnalyticsBackendPlugin.java | 18 + .../DataFusionFragmentConvertor.java | 276 ++++++++- .../be/datafusion/DataFusionPlugin.java | 40 ++ .../be/datafusion/DatafusionLocalSession.java | 38 ++ .../DatafusionMemtableReduceSink.java | 120 ++++ .../datafusion/DatafusionPartitionSender.java | 36 ++ .../be/datafusion/DatafusionReduceSink.java | 203 +++++++ .../be/datafusion/nativelib/NativeBridge.java | 165 ++++++ .../DataFusionFragmentConvertorTests.java | 298 +++++----- .../DatafusionMemtableReduceSinkTests.java | 149 +++++ .../datafusion/DatafusionReduceSinkTests.java | 300 ++++++++++ .../NativeBridgeLocalSessionTests.java | 169 ++++++ sandbox/plugins/analytics-engine/build.gradle | 32 + .../licenses/commons-math3-3.6.1.jar.sha1 | 1 + .../licenses/commons-math3-LICENSE.txt | 202 +++++++ .../licenses/commons-math3-NOTICE.txt | 5 + .../licenses/httpclient5-5.6.jar.sha1 | 1 + .../licenses/httpclient5-LICENSE.txt | 558 ++++++++++++++++++ .../licenses/httpclient5-NOTICE.txt | 6 + .../licenses/httpcore5-h2-5.4.jar.sha1 | 1 + .../licenses/httpcore5-h2-LICENSE.txt | 558 ++++++++++++++++++ .../licenses/httpcore5-h2-NOTICE.txt | 6 + .../unified/ClickBenchUnifiedPipelineIT.java | 243 -------- .../analytics/exec/DefaultPlanExecutor.java | 36 +- .../analytics/exec/RowProducingSink.java | 44 +- .../exec/stage/LocalStageExecution.java | 21 +- .../exec/stage/LocalStageScheduler.java | 38 +- .../exec/stage/LocalStageExecutionTests.java | 15 +- .../planner/MockDataFusionBackend.java | 2 +- .../ppl/action/UnifiedQueryService.java | 107 ---- .../ppl/action/UnifiedQueryServiceTests.java | 364 ------------ .../ppl/compiler/OpenSearchQueryCompiler.java | 181 ------ .../ppl/planner/PushDownPlanner.java | 104 ---- .../ppl/planner/PushDownPlannerTests.java | 184 ------ .../rel/OpenSearchBoundaryTableScan.java | 143 ----- .../rel/OpenSearchBoundaryTableScanTests.java | 201 ------- .../planner/rules/AbsorbAggregateRule.java | 52 -- .../ppl/planner/rules/AbsorbFilterRule.java | 65 -- .../ppl/planner/rules/AbsorbProjectRule.java | 74 --- .../ppl/planner/rules/AbsorbRuleUtils.java | 127 ---- .../ppl/planner/rules/AbsorbSortRule.java | 44 -- .../planner/rules/BoundaryTableScanRule.java | 62 -- .../ppl/planner/rules/PushDownRulesTests.java | 214 ------- .../plugins/test-ppl-frontend/build.gradle | 165 ++++++ .../org/opensearch/ppl/TestPPLPlugin.java | 23 + .../org/opensearch/ppl/action/PPLRequest.java | 0 .../opensearch/ppl/action/PPLResponse.java | 26 +- .../ppl/action/RestPPLQueryAction.java | 66 +++ .../ppl/action/TestPPLTransportAction.java | 13 +- .../ppl/action/UnifiedPPLExecuteAction.java | 0 .../ppl/action/UnifiedQueryService.java | 120 ++++ .../ppl/action/PPLResponseTests.java | 49 ++ .../ppl/action/RestPPLQueryActionTests.java | 40 ++ .../action/TestPPLTransportActionTests.java | 0 77 files changed, 6210 insertions(+), 2468 deletions(-) create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/partition_stream.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/tests/local_exec_test.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceIT.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceMemtableIT.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrowSchemaIpc.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionLocalSession.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSink.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionPartitionSender.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSinkTests.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReduceSinkTests.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/NativeBridgeLocalSessionTests.java create mode 100644 sandbox/plugins/analytics-engine/licenses/commons-math3-3.6.1.jar.sha1 create mode 100644 sandbox/plugins/analytics-engine/licenses/commons-math3-LICENSE.txt create mode 100644 sandbox/plugins/analytics-engine/licenses/commons-math3-NOTICE.txt create mode 100644 sandbox/plugins/analytics-engine/licenses/httpclient5-5.6.jar.sha1 create mode 100644 sandbox/plugins/analytics-engine/licenses/httpclient5-LICENSE.txt create mode 100644 sandbox/plugins/analytics-engine/licenses/httpclient5-NOTICE.txt create mode 100644 sandbox/plugins/analytics-engine/licenses/httpcore5-h2-5.4.jar.sha1 create mode 100644 sandbox/plugins/analytics-engine/licenses/httpcore5-h2-LICENSE.txt create mode 100644 sandbox/plugins/analytics-engine/licenses/httpcore5-h2-NOTICE.txt delete mode 100644 sandbox/plugins/analytics-engine/src/internalClusterTest/java/org/opensearch/fe/planner/unified/ClickBenchUnifiedPipelineIT.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/UnifiedQueryService.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/UnifiedQueryServiceTests.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/compiler/OpenSearchQueryCompiler.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlanner.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlannerTests.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScan.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScanTests.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbAggregateRule.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbFilterRule.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbProjectRule.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbRuleUtils.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbSortRule.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/BoundaryTableScanRule.java delete mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/PushDownRulesTests.java create mode 100644 sandbox/plugins/test-ppl-frontend/build.gradle rename sandbox/plugins/{analytics-engine/src/test => test-ppl-frontend/src/main}/java/org/opensearch/ppl/TestPPLPlugin.java (53%) rename sandbox/plugins/{analytics-engine/src/test => test-ppl-frontend/src/main}/java/org/opensearch/ppl/action/PPLRequest.java (100%) rename sandbox/plugins/{analytics-engine/src/test => test-ppl-frontend/src/main}/java/org/opensearch/ppl/action/PPLResponse.java (67%) create mode 100644 sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/RestPPLQueryAction.java rename sandbox/plugins/{analytics-engine/src/test => test-ppl-frontend/src/main}/java/org/opensearch/ppl/action/TestPPLTransportAction.java (80%) rename sandbox/plugins/{analytics-engine/src/test => test-ppl-frontend/src/main}/java/org/opensearch/ppl/action/UnifiedPPLExecuteAction.java (100%) create mode 100644 sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/UnifiedQueryService.java create mode 100644 sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/PPLResponseTests.java create mode 100644 sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/RestPPLQueryActionTests.java rename sandbox/plugins/{analytics-engine => test-ppl-frontend}/src/test/java/org/opensearch/ppl/action/TestPPLTransportActionTests.java (100%) diff --git a/gradle/run.gradle b/gradle/run.gradle index 3a5478848ed72..b342d8c5251a1 100644 --- a/gradle/run.gradle +++ b/gradle/run.gradle @@ -90,6 +90,15 @@ testClusters { systemProperty 'io.netty.tryUnsafe', 'true' systemProperty 'io.netty.tryReflectionSetAccessible', 'true' } + if (p.equals("parquet-data-format") || p.equals("analytics-backend-datafusion")) { + // Composite engine / DataFusion requires pluggable dataformat feature flag + systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true' + // Native lib path for DataFusion FFM bridge + def nativeLibDir = new File(project(':sandbox:libs:dataformat-native').projectDir, 'rust/target/release').absolutePath + systemProperty 'java.library.path', nativeLibDir + jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' + jvmArgs '--enable-native-access=ALL-UNNAMED' + } } } } diff --git a/sandbox/libs/analytics-framework/build.gradle b/sandbox/libs/analytics-framework/build.gradle index da1a533401c98..4463c14064947 100644 --- a/sandbox/libs/analytics-framework/build.gradle +++ b/sandbox/libs/analytics-framework/build.gradle @@ -68,6 +68,11 @@ dependencies { // consumers' compile/javadoc classpath without becoming a runtime dep. compileOnlyApi 'org.apiguardian:apiguardian-api:1.1.2' compileOnlyApi 'org.checkerframework:checker-qual:3.43.0' + + // Arrow's Schema.class carries @JsonInclude / @JsonTypeInfo / @JsonProperty annotations. + // Needed on the javadoc/compile classpath so the references resolve; compileOnly keeps + // it out of the runtime bundle (runtime Jackson is provided by server). + compileOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" } testingConventions.enabled = false @@ -90,73 +95,6 @@ tasks.named('forbiddenApisMain').configure { // Split into multiple calls to stay under the JVM method parameter limit. tasks.named('thirdPartyAudit').configure { ignoreMissingClasses( - // Jackson annotations (transitive dep of jackson-databind, provided by server at runtime) - 'com.fasterxml.jackson.annotation.JacksonAnnotationsInside', - 'com.fasterxml.jackson.annotation.JacksonInject', - 'com.fasterxml.jackson.annotation.JacksonInject$Value', - 'com.fasterxml.jackson.annotation.JsonAlias', - 'com.fasterxml.jackson.annotation.JsonAnyGetter', - 'com.fasterxml.jackson.annotation.JsonAnySetter', - 'com.fasterxml.jackson.annotation.JsonAutoDetect', - 'com.fasterxml.jackson.annotation.JsonAutoDetect$Value', - 'com.fasterxml.jackson.annotation.JsonAutoDetect$Visibility', - 'com.fasterxml.jackson.annotation.JsonBackReference', - 'com.fasterxml.jackson.annotation.JsonClassDescription', - 'com.fasterxml.jackson.annotation.JsonCreator', - 'com.fasterxml.jackson.annotation.JsonCreator$Mode', - 'com.fasterxml.jackson.annotation.JsonDeserializeAs', - 'com.fasterxml.jackson.annotation.JsonEnumDefaultValue', - 'com.fasterxml.jackson.annotation.JsonFilter', - 'com.fasterxml.jackson.annotation.JsonFormat', - 'com.fasterxml.jackson.annotation.JsonFormat$Feature', - 'com.fasterxml.jackson.annotation.JsonFormat$Shape', - 'com.fasterxml.jackson.annotation.JsonFormat$Value', - 'com.fasterxml.jackson.annotation.JsonGetter', - 'com.fasterxml.jackson.annotation.JsonIdentityInfo', - 'com.fasterxml.jackson.annotation.JsonIdentityReference', - 'com.fasterxml.jackson.annotation.JsonIgnore', - 'com.fasterxml.jackson.annotation.JsonIgnoreProperties', - 'com.fasterxml.jackson.annotation.JsonIgnoreProperties$Value', - 'com.fasterxml.jackson.annotation.JsonIgnoreType', - 'com.fasterxml.jackson.annotation.JsonInclude', - 'com.fasterxml.jackson.annotation.JsonInclude$Include', - 'com.fasterxml.jackson.annotation.JsonInclude$Value', - 'com.fasterxml.jackson.annotation.JsonIncludeProperties', - 'com.fasterxml.jackson.annotation.JsonIncludeProperties$Value', - 'com.fasterxml.jackson.annotation.JsonKey', - 'com.fasterxml.jackson.annotation.JsonManagedReference', - 'com.fasterxml.jackson.annotation.JsonMerge', - 'com.fasterxml.jackson.annotation.JsonProperty', - 'com.fasterxml.jackson.annotation.JsonProperty$Access', - 'com.fasterxml.jackson.annotation.JsonPropertyDescription', - 'com.fasterxml.jackson.annotation.JsonPropertyOrder', - 'com.fasterxml.jackson.annotation.JsonRawValue', - 'com.fasterxml.jackson.annotation.JsonRootName', - 'com.fasterxml.jackson.annotation.JsonSerializeAs', - 'com.fasterxml.jackson.annotation.JsonSetter', - 'com.fasterxml.jackson.annotation.JsonSetter$Value', - 'com.fasterxml.jackson.annotation.JsonSubTypes', - 'com.fasterxml.jackson.annotation.JsonSubTypes$Type', - 'com.fasterxml.jackson.annotation.JsonTypeId', - 'com.fasterxml.jackson.annotation.JsonTypeInfo', - 'com.fasterxml.jackson.annotation.JsonTypeInfo$As', - 'com.fasterxml.jackson.annotation.JsonTypeInfo$Id', - 'com.fasterxml.jackson.annotation.JsonTypeInfo$None', - 'com.fasterxml.jackson.annotation.JsonTypeInfo$Value', - 'com.fasterxml.jackson.annotation.JsonTypeName', - 'com.fasterxml.jackson.annotation.JsonUnwrapped', - 'com.fasterxml.jackson.annotation.JsonValue', - 'com.fasterxml.jackson.annotation.JsonView', - 'com.fasterxml.jackson.annotation.Nulls', - 'com.fasterxml.jackson.annotation.ObjectIdGenerator', - 'com.fasterxml.jackson.annotation.ObjectIdGenerator$IdKey', - 'com.fasterxml.jackson.annotation.ObjectIdGenerators$None', - 'com.fasterxml.jackson.annotation.ObjectIdGenerators$PropertyGenerator', - 'com.fasterxml.jackson.annotation.ObjectIdResolver', - 'com.fasterxml.jackson.annotation.OptBoolean', - 'com.fasterxml.jackson.annotation.PropertyAccessor', - 'com.fasterxml.jackson.annotation.SimpleObjectIdResolver', - // Gson (optional json-path provider) 'com.google.gson.Gson', 'com.google.gson.JsonArray', diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java new file mode 100644 index 0000000000000..dcf2be02d52d9 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.pojo.Schema; + +/** + * Context passed to {@link ExchangeSinkProvider#createSink} when a + * coordinator-reduce stage is being set up. Carries everything the backend + * needs to build an {@link ExchangeSink}: serialized plan, buffer allocator, + * input schema, and the downstream sink the backend writes results to. + * + *

Fields: + *

    + *
  • {@code queryId} / {@code stageId} — correlation ids for backend logs + * and metrics.
    • + *
  • {@code fragmentBytes} — backend-specific serialized plan (e.g. + * Substrait) the backend will execute over the fed batches.
    • + *
  • {@code allocator} — the parent buffer allocator the backend should + * derive its own child allocators from. Sharing the allocator tree + * keeps output batches within the query's memory accounting.
    • + *
  • {@code inputSchema} — Arrow schema of batches fed into the sink + * (derived from the single child stage's fragment rowtype).
    • + *
  • {@code downstream} — sink the backend drains its reduced output + * into. The backend owns {@code downstream}'s lifecycle: it must + * feed every produced batch and close it when draining is complete.
    • + *
+ * + *

Single-sink simplification: this context assumes exactly one input + * stream. Per-child routing (e.g., joins with multiple inputs of different + * schemas) will require a richer context and is not modeled yet. + * + * @opensearch.internal + */ +public record ExchangeSinkContext(String queryId, int stageId, byte[] fragmentBytes, BufferAllocator allocator, Schema inputSchema, + ExchangeSink downstream) { +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkProvider.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkProvider.java index f1eba97c976db..001b318578f9b 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkProvider.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkProvider.java @@ -23,10 +23,10 @@ public interface ExchangeSinkProvider { /** - * Creates a sink for coordinator-side execution using the serialized coordinator - * fragment produced by {@link FragmentConvertor#convertFinalAggFragment}. - * - * @param coordinatorFragmentBytes backend-specific serialized coordinator fragment + * Creates a sink for coordinator-side execution. The backend implementation + * uses {@link ExchangeSinkContext#fragmentBytes()} as the serialized plan + * (produced by {@link FragmentConvertor#convertFinalAggFragment}) and + * writes its reduced output into {@link ExchangeSinkContext#downstream()}. */ - ExchangeSink createSink(byte[] coordinatorFragmentBytes); + ExchangeSink createSink(ExchangeSinkContext context); } diff --git a/sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeCall.java b/sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeCall.java index e5a2de8e92f1c..ca52c3bc5e643 100644 --- a/sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeCall.java +++ b/sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeCall.java @@ -204,6 +204,21 @@ public MemorySegment bytes(byte[] data) { return arena.allocateFrom(ValueLayout.JAVA_BYTE, data); } + /** + * Allocate a segment from a long array. Returns an empty (zero-byte) segment if the array + * is empty so callers can pass it as a non-null pointer with count zero. + */ + public MemorySegment longs(long[] data) { + ensureOpen(); + if (data == null) { + throw new NullPointerException("Cannot marshal null long array to native"); + } + if (data.length == 0) { + return arena.allocate(0); + } + return arena.allocateFrom(ValueLayout.JAVA_LONG, data); + } + // ---- Invocation ---- /** diff --git a/sandbox/plugins/analytics-backend-datafusion/build.gradle b/sandbox/plugins/analytics-backend-datafusion/build.gradle index 74f0948448d13..8f2c902595684 100644 --- a/sandbox/plugins/analytics-backend-datafusion/build.gradle +++ b/sandbox/plugins/analytics-backend-datafusion/build.gradle @@ -6,20 +6,39 @@ * compatible open source license. */ +apply plugin: 'opensearch.internal-cluster-test' + +// SQL Unified Query API version (aligned with OpenSearch build version) — required by +// the PPL transport plugin used in CoordinatorReduceIT. +def sqlUnifiedQueryVersion = '3.6.0.0-SNAPSHOT' + opensearchplugin { description = 'DataFusion native execution engine plugin for the query engine.' classname = 'org.opensearch.be.datafusion.DataFusionPlugin' extendedPlugins = ['analytics-engine'] } +repositories { + maven { + name = 'OpenSearch Snapshots' + url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/' + } +} + java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } -// Guava is forbidden on compile classpaths by OpenSearch. Calcite's TableScan -// references ImmutableList, so tests need it. Bypass via custom configuration. +// Guava is forbidden on compile classpaths by OpenSearch. The fragment convertor's +// StageInputTableScan extends Calcite TableScan (which leaks ImmutableList in its +// constructor signature), and tests use Calcite types directly. Bypass via custom +// configurations on both main and test compileClasspath while keeping Guava off the +// plugin's runtime bundle (provided by analytics-engine). configurations { + calciteCompile calciteTestCompile + compileClasspath { exclude group: 'com.google.guava' } testCompileClasspath { exclude group: 'com.google.guava' } } +sourceSets.main.compileClasspath += configurations.calciteCompile sourceSets.test.compileClasspath += configurations.calciteTestCompile dependencies { @@ -28,19 +47,22 @@ dependencies { // Provided at runtime by the parent analytics-engine plugin; compile-only to avoid jar hell. compileOnly project(':sandbox:libs:analytics-framework') + // analytics-engine's RelNode types (OpenSearchStageInputScan) are referenced by the + // fragment convertor for pre-isthmus rewrite. compileOnly — provided at runtime by + // the parent analytics-engine plugin (extendedPlugins above). + compileOnly project(':sandbox:plugins:analytics-engine') compileOnly "org.apache.logging.log4j:log4j-api:${versions.log4j}" compileOnly "org.apache.logging.log4j:log4j-core:${versions.log4j}" // Apache Arrow dependencies. - // arrow-vector + arrow-memory-core are provided at runtime by the parent analytics-engine - // plugin (we extend it via extendedPlugins); compile-only here to avoid duplicate bundling + // arrow-vector + arrow-memory-core + arrow-format + flatbuffers are provided at runtime + // by the parent analytics-engine plugin; compile-only here to avoid duplicate bundling // and license files. compileOnly "org.apache.arrow:arrow-vector:${versions.arrow}" compileOnly "org.apache.arrow:arrow-memory-core:${versions.arrow}" compileOnly "org.apache.arrow:arrow-memory-unsafe:${versions.arrow}" implementation "org.apache.arrow:arrow-c-data:${versions.arrow}" - // Provided at runtime by parent analytics-engine plugin; compileOnly to avoid jar hell. compileOnly "org.apache.arrow:arrow-format:${versions.arrow}" compileOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" @@ -53,9 +75,32 @@ dependencies { // Substrait — Calcite RelNode to Substrait plan conversion for DataFusion native runtime implementation "io.substrait:isthmus:0.67.0" implementation "io.substrait:core:0.67.0" + // Jackson datatype module required by Substrait's DefaultExtensionCatalog (not transitively + // bundled by the analytics-engine parent plugin). implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}" + calciteCompile "com.google.guava:guava:${versions.guava}" calciteTestCompile "com.google.guava:guava:${versions.guava}" + + // ── internalClusterTest: end-to-end coordinator-reduce IT ─────────────────── + // Pulls in every sibling plugin needed to construct a parquet-backed composite + // index, dispatch a multi-shard PPL aggregate, and exercise DatafusionReduceSink. + internalClusterTestImplementation project(':sandbox:plugins:analytics-engine') + internalClusterTestImplementation project(':sandbox:plugins:parquet-data-format') + internalClusterTestImplementation project(':sandbox:plugins:composite-engine') + internalClusterTestImplementation project(':sandbox:plugins:analytics-backend-lucene') + internalClusterTestImplementation project(':plugins:arrow-flight-rpc') + internalClusterTestImplementation("org.opensearch.query:unified-query-api:${sqlUnifiedQueryVersion}") { + exclude group: 'org.opensearch' + } + internalClusterTestImplementation("org.opensearch.query:unified-query-core:${sqlUnifiedQueryVersion}") { + exclude group: 'org.opensearch' + } + internalClusterTestImplementation("org.opensearch.query:unified-query-ppl:${sqlUnifiedQueryVersion}") { + exclude group: 'org.opensearch' + } + // PPL front-end plugin — provides UnifiedPPLExecuteAction transport action used by the IT. + internalClusterTestImplementation project(':sandbox:plugins:test-ppl-frontend') } test { @@ -65,18 +110,67 @@ test { dependsOn ':sandbox:libs:dataformat-native:buildRustLibrary' } +internalClusterTest { + jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' + jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED' + jvmArgs '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED' + jvmArgs '--enable-native-access=ALL-UNNAMED' + jvmArgs '-Darrow.memory.debug.allocator=false' + jvmArgs += ["--add-opens", "java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED"] + systemProperty 'io.netty.allocator.numDirectArenas', '1' + systemProperty 'io.netty.noUnsafe', 'false' + systemProperty 'io.netty.tryUnsafe', 'true' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' + systemProperty 'native.lib.path', project(':sandbox:libs:dataformat-native').ext.nativeLibPath.absolutePath + dependsOn ':sandbox:libs:dataformat-native:buildRustLibrary' +} + +configurations.all { + // okhttp-aws-signer is a transitive dep of unified-query-common (via unified-query-core), + // only published on JitPack, not needed for PPL parsing/planning + exclude group: 'com.github.babbel', module: 'okhttp-aws-signer' + + resolutionStrategy { + // Align transitive versions with OpenSearch's managed versions — required because the + // unified-query-* artifacts pull in older versions of common libs that conflict with + // OpenSearch's enforced versions on internalClusterTest classpath. + force 'com.google.guava:guava:33.4.0-jre' + force 'com.google.guava:failureaccess:1.0.2' + force 'com.google.errorprone:error_prone_annotations:2.36.0' + force 'org.checkerframework:checker-qual:3.43.0' + force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + force "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" + force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" + force "org.slf4j:slf4j-api:${versions.slf4j}" + force "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" + force "org.locationtech.jts:jts-core:${versions.jts}" + force "commons-codec:commons-codec:${versions.commonscodec}" + force "joda-time:joda-time:2.12.7" + force "org.yaml:snakeyaml:2.4" + force "org.codehaus.janino:janino:3.1.12" + force "org.codehaus.janino:commons-compiler:3.1.12" + force "commons-io:commons-io:${versions.commonsio}" + force "org.apache.commons:commons-lang3:3.18.0" + force "org.apache.commons:commons-text:1.11.0" + force "commons-logging:commons-logging:1.3.5" + force "net.minidev:json-smart:2.5.2" + force "org.apache.httpcomponents.client5:httpclient5:5.6" + force "org.apache.httpcomponents.core5:httpcore5:5.4" + force "com.squareup.okhttp3:okhttp:4.12.0" + force "org.jetbrains.kotlin:kotlin-stdlib:1.8.21" + force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.8.21" + force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.8.21" + force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10" + } +} + tasks.withType(JavaCompile).configureEach { // Arrow references Jackson annotations not on classpath — harmless warnings options.compilerArgs -= '-Werror' } -tasks.named('thirdPartyAudit').configure { - ignoreMissingClasses( - // Optional Calcite DDL parser — lives in calcite-server module, not needed at runtime - 'org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl' - ) -} - // TODO: Remove once back-end is built out with test suite testingConventions.enabled = false @@ -87,3 +181,11 @@ tasks.named('forbiddenPatterns').configure { tasks.matching { it.name == 'missingJavadoc' }.configureEach { enabled = false } + +tasks.named('thirdPartyAudit').configure { + ignoreMissingClasses( + // SqlDdlParserImpl is generated by Calcite at build time and not bundled in the + // calcite-core jar; substrait-isthmus references it through reflection in optional code paths. + 'org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl' + ) +} diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/core-LICENSE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/core-LICENSE.txt index f5f45d26a49d6..d645695673349 100644 --- a/sandbox/plugins/analytics-backend-datafusion/licenses/core-LICENSE.txt +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/core-LICENSE.txt @@ -1,8 +1,202 @@ -This copy of Jackson JSON processor streaming parser/generator is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. -You may obtain a copy of the License at: + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/core-NOTICE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/core-NOTICE.txt index 8b137891791fe..acb3b6e0c4770 100644 --- a/sandbox/plugins/analytics-backend-datafusion/licenses/core-NOTICE.txt +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/core-NOTICE.txt @@ -1 +1,7 @@ +Substrait Java +Copyright The Substrait Authors +This product includes software developed by The Substrait Authors +(https://github.com/substrait-io/substrait-java). + +Licensed under the Apache License, Version 2.0. diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-LICENSE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-LICENSE.txt index f5f45d26a49d6..d645695673349 100644 --- a/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-LICENSE.txt +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-LICENSE.txt @@ -1,8 +1,202 @@ -This copy of Jackson JSON processor streaming parser/generator is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. -You may obtain a copy of the License at: + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-NOTICE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-NOTICE.txt index 8b137891791fe..acb3b6e0c4770 100644 --- a/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-NOTICE.txt +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-NOTICE.txt @@ -1 +1,7 @@ +Substrait Java +Copyright The Substrait Authors +This product includes software developed by The Substrait Authors +(https://github.com/substrait-io/substrait-java). + +Licensed under the Apache License, Version 2.0. diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-LICENSE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-LICENSE.txt index f5f45d26a49d6..227e33f960898 100644 --- a/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-LICENSE.txt +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-LICENSE.txt @@ -1,7 +1,7 @@ -This copy of Jackson JSON processor streaming parser/generator is licensed under the +This copy of Jackson JSON processor Java 8 Modules is licensed under the Apache (Software) License, version 2.0 ("the License"). See the License for details about distribution rights, and the -specific rights regarding derivate works. +specific rights regarding derivative works. You may obtain a copy of the License at: diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-NOTICE.txt b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-NOTICE.txt index 8b137891791fe..d55c59a0d506f 100644 --- a/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-NOTICE.txt +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-NOTICE.txt @@ -1 +1,17 @@ +# Jackson JSON processor +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers. + +## Licensing + +Jackson components are licensed under Apache (Software) License, version 2.0, +as per accompanying LICENSE file. + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs index af0d9378aeaa1..973b50f11fac9 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs @@ -146,25 +146,30 @@ //! // } //! ``` +use std::io::Cursor; use std::num::NonZeroUsize; use std::path::PathBuf; use std::sync::Arc; +use arrow::ipc::reader::StreamReader; use arrow_array::{Array, StructArray}; use arrow_array::ffi::FFI_ArrowArray; +use arrow_array::RecordBatch; use arrow_schema::ffi::FFI_ArrowSchema; use datafusion::common::DataFusionError; use datafusion::datasource::listing::ListingTableUrl; use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; use datafusion::execution::memory_pool::{GreedyMemoryPool, TrackConsumersPool}; use datafusion::execution::runtime_env::RuntimeEnvBuilder; -use datafusion::execution::{SessionState, SessionStateBuilder}; +use datafusion::execution::SessionStateBuilder; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::execution::RecordBatchStream; use datafusion::prelude::SessionConfig; use futures::TryStreamExt; use crate::cross_rt_stream::CrossRtStream; +use crate::local_executor::LocalSession; +use crate::partition_stream::PartitionStreamSender; use crate::query_memory_pool_tracker::QueryTrackingContext; use crate::runtime_manager::RuntimeManager; @@ -460,3 +465,240 @@ pub unsafe fn sql_to_substrait( Ok(buf) }) } + +// --------------------------------------------------------------------------- +// Coordinator-reduce local execution API +// +// Mirrors the shard-scan path: a `LocalSession` pointer is created once per +// reduce stage, streaming inputs are registered under synthetic names, a +// Substrait plan is executed against those inputs, and the output stream is +// drained via the existing `stream_next` / `stream_close` exports (because +// `execute_local_plan` hands back a `QueryStreamHandle` of the same shape +// `execute_query` returns). +// --------------------------------------------------------------------------- + +/// Creates a `LocalSession` bound to the given runtime's [`RuntimeEnv`] +/// (memory pool, disk manager, and caches are shared). +/// +/// Returns a heap-allocated pointer (as i64) to `LocalSession`. Caller must +/// call `close_local_session` exactly once to free it. +/// +/// # Safety +/// `runtime_ptr` must be a valid, non-zero pointer returned by +/// `create_global_runtime`. +pub unsafe fn create_local_session(runtime_ptr: i64) -> Result { + let runtime = &*(runtime_ptr as *const DataFusionRuntime); + let session = LocalSession::new(&runtime.runtime_env); + Ok(Box::into_raw(Box::new(session)) as i64) +} + +/// Closes a `LocalSession`. Safe to call with 0 (no-op). +/// +/// # Safety +/// `ptr` must be 0 or a valid pointer returned by `create_local_session`. +pub unsafe fn close_local_session(ptr: i64) { + if ptr != 0 { + let _ = Box::from_raw(ptr as *mut LocalSession); + } +} + +/// Registers a streaming input on the session under `input_id`, using the +/// Arrow schema decoded from the IPC stream bytes. +/// +/// The IPC bytes are expected to be a single schema message produced by +/// Arrow's streaming IPC writer (e.g. Java's `MessageSerializer.serializeMetadata` +/// or an `ArrowStreamWriter` flush of just the schema). Only the schema is +/// read — any payload in the buffer is ignored. +/// +/// Returns a heap-allocated pointer (as i64) to a [`PartitionStreamSender`]. +/// Caller must call `sender_close` exactly once to free it (closing the +/// sender signals EOF to the receiver side, so the native execute driver +/// naturally completes). +/// +/// # Safety +/// `session_ptr` must be a valid, non-zero pointer returned by +/// `create_local_session`. +pub unsafe fn register_partition_stream( + session_ptr: i64, + input_id: &str, + schema_ipc: &[u8], +) -> Result { + let session = &mut *(session_ptr as *mut LocalSession); + let mut cursor = Cursor::new(schema_ipc); + let reader = StreamReader::try_new(&mut cursor, None).map_err(|e| { + DataFusionError::Execution(format!( + "Failed to decode Arrow IPC schema for '{}': {}", + input_id, e + )) + })?; + let schema = reader.schema(); + let sender = session.register_partition(input_id, schema)?; + Ok(Box::into_raw(Box::new(sender)) as i64) +} + +/// Executes a Substrait plan against a `LocalSession` and returns a +/// `QueryStreamHandle` pointer whose output can be drained via the existing +/// `stream_next` / `stream_close` exports. +/// +/// The returned stream wraps the DataFusion output in the same +/// `CrossRtStream` + `RecordBatchStreamAdapter` shape as `execute_query`, +/// so the session produces batches on the CPU executor while `stream_next` +/// consumes them on the I/O runtime. +/// +/// This is an async function — the bridge layer decides how to run it +/// (`block_on` for synchronous FFM entry, `spawn` for async delivery). +/// +/// # Safety +/// `session_ptr` must be a valid, non-zero pointer returned by +/// `create_local_session`. +pub async unsafe fn execute_local_plan( + session_ptr: i64, + substrait_bytes: &[u8], + manager: &RuntimeManager, + context_id: i64, +) -> Result { + let session = &*(session_ptr as *const LocalSession); + + // Per-query memory tracking — wraps the session's global pool. A + // `context_id` of 0 disables tracking (pool is not consulted). + let query_context = QueryTrackingContext::new(context_id, session.memory_pool()); + + let df_stream = session.execute_substrait(substrait_bytes).await?; + + // Wrap the output in the same CrossRtStream + RecordBatchStreamAdapter + // shape as `execute_query`, so existing `stream_next` / `stream_close` + // drain this handle unchanged. + let cross_rt_stream = + CrossRtStream::new_with_df_error_stream(df_stream, manager.cpu_executor()); + let wrapped = RecordBatchStreamAdapter::new(cross_rt_stream.schema(), cross_rt_stream); + + let handle = QueryStreamHandle::new(wrapped, query_context); + Ok(Box::into_raw(Box::new(handle)) as i64) +} + +/// Imports an Arrow C Data batch and pushes it through the partition +/// stream's mpsc. The Rust side takes ownership of the +/// `FFI_ArrowArray` / `FFI_ArrowSchema` structs on success — the Java side +/// must not release them after a successful send. On error ownership is +/// released back to Rust's drop impls (the imported structs go out of scope +/// without being forgotten). +/// +/// The `io_handle` is the Tokio handle used to drive the blocking send; +/// typically the `io_runtime` handle from the global `RuntimeManager`. +/// +/// # Safety +/// - `sender_ptr` must be a valid, non-zero pointer returned by +/// `register_partition_stream`. +/// - `array_ptr` must point to a populated `FFI_ArrowArray` struct owned by +/// the caller; ownership transfers to Rust on success. +/// - `schema_ptr` must point to a populated `FFI_ArrowSchema` struct owned +/// by the caller; ownership transfers to Rust on success. +pub unsafe fn sender_send( + sender_ptr: i64, + array_ptr: i64, + schema_ptr: i64, + io_handle: &tokio::runtime::Handle, +) -> Result<(), DataFusionError> { + let sender = &*(sender_ptr as *const PartitionStreamSender); + + // Take ownership of the Java-allocated FFI structs. `from_raw` reads + // the struct contents into Rust-owned values; the original memory is + // now Rust's responsibility to drop. + let ffi_array = FFI_ArrowArray::from_raw(array_ptr as *mut FFI_ArrowArray); + let ffi_schema = FFI_ArrowSchema::from_raw(schema_ptr as *mut FFI_ArrowSchema); + + // `from_ffi` takes the array by value (consumes it) and the schema by + // reference (it is still dropped when `ffi_schema` goes out of scope). + let array_data = arrow_array::ffi::from_ffi(ffi_array, &ffi_schema).map_err(|e| { + DataFusionError::Execution(format!("Failed to import Arrow C Data array: {}", e)) + })?; + + let struct_array = StructArray::from(array_data); + let batch = RecordBatch::from(struct_array); + + sender.send_blocking(Ok(batch), io_handle) +} + +/// Closes a partition stream sender. Dropping the sender closes the mpsc, +/// which the receiver side (DataFusion's streaming table) interprets as +/// end-of-input. +/// +/// Safe to call with 0 (no-op). +/// +/// # Safety +/// `sender_ptr` must be 0 or a valid pointer returned by +/// `register_partition_stream`. +pub unsafe fn sender_close(sender_ptr: i64) { + if sender_ptr != 0 { + let _ = Box::from_raw(sender_ptr as *mut PartitionStreamSender); + } +} + +/// Imports a batch of Arrow C Data structures into a [`Vec`] and +/// registers them as an in-memory table on the given session under `input_id`. +/// +/// The Java side has accumulated all shard responses, exported each +/// `VectorSchemaRoot` to a paired `FFI_ArrowArray` / `FFI_ArrowSchema`, and +/// passed the raw pointers as two parallel slices. Rust takes ownership of +/// the FFI structs on success. +/// +/// On error ownership is released back to Rust's drop impls (the imported +/// structs go out of scope without being forgotten). +/// +/// # Safety +/// - `session_ptr` must be a valid, non-zero pointer returned by +/// `create_local_session`. +/// - `array_ptrs` and `schema_ptrs` must point to populated FFI structs owned +/// by the caller; ownership transfers to Rust on success. +pub unsafe fn register_memtable( + session_ptr: i64, + input_id: &str, + schema_ipc: &[u8], + array_ptrs: &[i64], + schema_ptrs: &[i64], +) -> Result<(), DataFusionError> { + if array_ptrs.len() != schema_ptrs.len() { + return Err(DataFusionError::Execution(format!( + "register_memtable: array_ptrs.len()={} != schema_ptrs.len()={}", + array_ptrs.len(), + schema_ptrs.len() + ))); + } + let session = &mut *(session_ptr as *mut LocalSession); + + let mut cursor = Cursor::new(schema_ipc); + let reader = StreamReader::try_new(&mut cursor, None).map_err(|e| { + DataFusionError::Execution(format!( + "Failed to decode Arrow IPC schema for '{}': {}", + input_id, e + )) + })?; + let table_schema = reader.schema(); + + // The IPC schema is what the substrait plan was compiled against — same as the streaming + // sink registers. The exported VSRs may arrive with batch-level schemas that differ in + // nullability/metadata/field-naming details; the streaming sink tolerates this because + // DataFusion's streaming source addresses columns by index. `MemTable::try_new` instead + // checks each batch's schema against the table schema. To stay compatible with both + // shapes, rebuild each imported batch with `table_schema` — the column data is reused + // verbatim, but the schema header is the planner's. + let mut batches = Vec::with_capacity(array_ptrs.len()); + for (&array_ptr, &schema_ptr) in array_ptrs.iter().zip(schema_ptrs.iter()) { + let ffi_array = FFI_ArrowArray::from_raw(array_ptr as *mut FFI_ArrowArray); + let ffi_schema = FFI_ArrowSchema::from_raw(schema_ptr as *mut FFI_ArrowSchema); + let array_data = arrow_array::ffi::from_ffi(ffi_array, &ffi_schema).map_err(|e| { + DataFusionError::Execution(format!("Failed to import Arrow C Data array: {}", e)) + })?; + let struct_array = StructArray::from(array_data); + let raw = RecordBatch::from(struct_array); + let aligned = RecordBatch::try_new(Arc::clone(&table_schema), raw.columns().to_vec()).map_err(|e| { + DataFusionError::Execution(format!( + "Failed to align imported batch to registered schema for '{}': {}", + input_id, e + )) + })?; + batches.push(aligned); + } + + session.register_memtable(input_id, table_schema, batches) +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs index c26912cc88bc6..2b6b53426e179 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs @@ -8,7 +8,7 @@ use futures::{future::BoxFuture, Future, FutureExt, TryFutureExt}; use parking_lot::RwLock; -use std::sync::{Arc, OnceLock}; +use std::sync::Arc; use std::time::Duration; use tokio::{ runtime::Handle, diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index aac42083c6f0e..6a531a7839bfa 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -165,3 +165,130 @@ pub unsafe extern "C" fn df_sql_to_substrait( } Ok(0) } + +// --------------------------------------------------------------------------- +// Coordinator-reduce local execution exports +// +// Mirror the shard-scan exports above: fallible entry points use `#[ffm_safe]` +// so `Err(String)` returns are converted into a negated heap-allocated error +// string pointer that `NativeCall.invoke` reads and frees on the Java side. +// Close functions are infallible and do not use the macro. The output stream +// returned by `df_execute_local_plan` is the same `QueryStreamHandle` shape +// as `df_execute_query`, so it drains through the existing `df_stream_next` / +// `df_stream_close` paths unchanged. +// --------------------------------------------------------------------------- + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_create_local_session(runtime_ptr: i64) -> i64 { + api::create_local_session(runtime_ptr).map_err(|e| e.to_string()) +} + +#[no_mangle] +pub unsafe extern "C" fn df_close_local_session(ptr: i64) { + api::close_local_session(ptr); +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_register_partition_stream( + session_ptr: i64, + input_id_ptr: *const u8, + input_id_len: i64, + schema_ipc_ptr: *const u8, + schema_ipc_len: i64, +) -> i64 { + let input_id = str_from_raw(input_id_ptr, input_id_len) + .map_err(|e| format!("df_register_partition_stream: input_id: {}", e))?; + let schema_ipc = slice::from_raw_parts(schema_ipc_ptr, schema_ipc_len as usize); + api::register_partition_stream(session_ptr, input_id, schema_ipc).map_err(|e| e.to_string()) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_execute_local_plan( + session_ptr: i64, + substrait_ptr: *const u8, + substrait_len: i64, +) -> i64 { + let mgr = get_rt_manager()?; + // Copy substrait bytes into an owned Vec so the spawned future can move them + // (cpu_executor.spawn requires 'static). Clone the manager Arc twice — once for + // the inner future to access the runtime env / etc., once for the outer block_on + // closure to call `cpu_executor().spawn`. + let bytes_vec = slice::from_raw_parts(substrait_ptr, substrait_len as usize).to_vec(); + let mgr_for_inner = Arc::clone(&mgr); + let mgr_for_spawn = Arc::clone(&mgr); + // Wrap plan setup in cpu_executor.spawn so internal DataFusion spawns + // (RepartitionExec drain, CoalescePartitionsExec, etc.) inherit the CPU executor + // instead of the IO runtime. Without this, operator hash work runs on IO workers. + // The IO runtime still drives the outer block_on (bridging the synchronous FFI + // call to the async spawn handle). + mgr.io_runtime + .block_on(async move { + let inner_fut = async move { + unsafe { api::execute_local_plan(session_ptr, &bytes_vec, &mgr_for_inner, 0).await } + }; + match mgr_for_spawn.cpu_executor().spawn(inner_fut).await { + Ok(inner_result) => inner_result, + Err(e) => Err(datafusion::error::DataFusionError::Execution(format!( + "execute_local_plan: CPU spawn failed: {e:?}" + ))), + } + }) + .map_err(|e| e.to_string()) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_sender_send(sender_ptr: i64, array_ptr: i64, schema_ptr: i64) -> i64 { + let mgr = get_rt_manager()?; + api::sender_send(sender_ptr, array_ptr, schema_ptr, mgr.io_runtime.handle()) + .map(|_| 0) + .map_err(|e| e.to_string()) +} + +#[no_mangle] +pub unsafe extern "C" fn df_sender_close(sender_ptr: i64) { + api::sender_close(sender_ptr); +} + +/// Memtable variant of `df_register_partition_stream`: instead of returning a +/// sender that streams batches one at a time, the caller hands across `n` +/// already-exported Arrow C Data batches in two parallel pointer arrays and +/// the native side constructs a [`MemTable`] in one shot. +/// +/// `array_ptrs` and `schema_ptrs` must each point to an `n`-element array of +/// `i64`s, where each pair `(array_ptrs[i], schema_ptrs[i])` is a populated +/// `FFI_ArrowArray` / `FFI_ArrowSchema` pair owned by the caller. On success +/// Rust takes ownership; on error the structs are dropped on the Rust side. +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_register_memtable( + session_ptr: i64, + input_id_ptr: *const u8, + input_id_len: i64, + schema_ipc_ptr: *const u8, + schema_ipc_len: i64, + array_ptrs: *const i64, + schema_ptrs: *const i64, + n_batches: i64, +) -> i64 { + let input_id = str_from_raw(input_id_ptr, input_id_len) + .map_err(|e| format!("df_register_memtable: input_id: {}", e))?; + let schema_ipc = slice::from_raw_parts(schema_ipc_ptr, schema_ipc_len as usize); + let n = n_batches as usize; + let array_slice: &[i64] = if n == 0 { + &[] + } else { + slice::from_raw_parts(array_ptrs, n) + }; + let schema_slice: &[i64] = if n == 0 { + &[] + } else { + slice::from_raw_parts(schema_ptrs, n) + }; + api::register_memtable(session_ptr, input_id, schema_ipc, array_slice, schema_slice) + .map(|_| 0) + .map_err(|e| e.to_string()) +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs index 7c3cde3e82fe2..c72a15134347e 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs @@ -16,6 +16,8 @@ pub mod cross_rt_stream; pub mod executor; pub mod ffm; pub mod io; +pub mod local_executor; +pub mod partition_stream; pub mod query_executor; pub mod query_memory_pool_tracker; pub mod runtime_manager; diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs new file mode 100644 index 0000000000000..78fb22b455bc9 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs @@ -0,0 +1,328 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Coordinator-reduce local execution. +//! +//! A [`LocalSession`] holds a DataFusion [`SessionContext`] configured to share +//! the caller-supplied [`RuntimeEnv`] (and therefore its memory pool) with the +//! rest of the node. The session is the Rust-side counterpart of +//! `DatafusionReduceSink` on the Java side: +//! +//! 1. For each declared stage input, [`LocalSession::register_partition`] +//! creates a [`PartitionStreamSender`] / [`PartitionStreamReceiver`] pair, +//! wraps the receiver in a [`SingleReceiverPartition`], and registers it as +//! a [`StreamingTable`] on the session under the input id. +//! 2. [`LocalSession::execute_substrait`] decodes a Substrait plan against the +//! session (its table references resolve to the streaming tables) and hands +//! back a [`SendableRecordBatchStream`] the bridge layer can drain. +//! +//! The session has no knowledge of the FFM bridge; it is exposed to Java via a +//! raw `Box::into_raw` pointer managed in `api.rs`, matching the lifecycle +//! model used by `DataFusionRuntime` / `ShardView` / `QueryStreamHandle`. + +use std::sync::Arc; + +use arrow_array::RecordBatch; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::catalog::streaming::StreamingTable; +use datafusion::common::DataFusionError; +use datafusion::datasource::MemTable; +use datafusion::execution::memory_pool::MemoryPool; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::execution::{SendableRecordBatchStream, SessionStateBuilder}; +use datafusion::physical_plan::streaming::PartitionStream; +use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion_substrait::logical_plan::consumer::from_substrait_plan; +use prost::Message; +use substrait::proto::Plan; + +use crate::partition_stream::{channel, PartitionStreamSender, SingleReceiverPartition}; + +/// Coordinator-reduce DataFusion session. +/// +/// Owns a [`SessionContext`] that reuses the caller's [`RuntimeEnv`] so memory +/// accounting shares the node-wide pool. One session corresponds to one reduce +/// stage; it holds the streaming inputs registered by +/// [`Self::register_partition`] and is drained exactly once via +/// [`Self::execute_substrait`]. +pub struct LocalSession { + ctx: SessionContext, +} + +impl LocalSession { + /// Builds a session whose `SessionContext` reuses the given [`RuntimeEnv`]. + /// + /// The runtime's memory pool, disk manager, and caches are inherited — + /// every batch consumed or produced by this session counts against the + /// same limits as the shard-scan path. + pub fn new(runtime_env: &RuntimeEnv) -> Self { + // Cheaply clone the env so the session owns a handle independent of + // the caller. `RuntimeEnv` internally holds `Arc`s — this is a + // lightweight clone, not a deep copy of the pool or disk manager. + let runtime_env = Arc::new(runtime_env.clone()); + let state = SessionStateBuilder::new() + .with_config(SessionConfig::new()) + .with_runtime_env(runtime_env) + .with_default_features() + .build(); + Self { + ctx: SessionContext::new_with_state(state), + } + } + + /// Registers a streaming input on the session under `name` and returns the + /// producer side of the channel. + /// + /// The receiver is wrapped in a [`SingleReceiverPartition`] and registered + /// as a [`StreamingTable`]; Substrait plans executed through + /// [`Self::execute_substrait`] resolve table references named `name` to + /// this streaming table. The caller pushes `RecordBatch`es into the + /// returned [`PartitionStreamSender`] via + /// [`PartitionStreamSender::send_blocking`]. + pub fn register_partition( + &mut self, + name: &str, + schema: SchemaRef, + ) -> Result { + let (sender, receiver) = channel(Arc::clone(&schema)); + let partition: Arc = + Arc::new(SingleReceiverPartition::new(receiver)); + let table = StreamingTable::try_new(schema, vec![partition])?; + self.ctx + .register_table(name, Arc::new(table)) + .map_err(|e| { + DataFusionError::Execution(format!( + "Failed to register streaming table '{}': {}", + name, e + )) + })?; + Ok(sender) + } + + /// Registers an in-memory input on the session under `name`, holding all + /// `batches` in a single [`MemTable`] partition. + /// + /// Unlike [`Self::register_partition`], this method does not return a + /// channel sender — the batches are fully materialized in the table. Used + /// by the memtable variant of the coordinator-reduce sink, which buffers + /// shard responses in Java and hands them across in one call. + pub fn register_memtable( + &mut self, + name: &str, + schema: SchemaRef, + batches: Vec, + ) -> Result<(), DataFusionError> { + let table = MemTable::try_new(schema, vec![batches])?; + self.ctx + .register_table(name, Arc::new(table)) + .map_err(|e| { + DataFusionError::Execution(format!( + "Failed to register memtable '{}': {}", + name, e + )) + })?; + Ok(()) + } + + /// Decodes a Substrait plan against the session and returns the resulting + /// stream. + /// + /// Table references in the plan resolve through the session's registered + /// streaming tables, so input batches pushed into + /// [`PartitionStreamSender`]s flow naturally into the DataFusion physical + /// plan. The returned stream is hot — polling it drives both the reduce + /// computation and the consumption of the streaming inputs. + pub async fn execute_substrait( + &self, + bytes: &[u8], + ) -> Result { + let plan = Plan::decode(bytes).map_err(|e| { + DataFusionError::Execution(format!("Failed to decode Substrait plan: {}", e)) + })?; + let logical_plan = from_substrait_plan(&self.ctx.state(), &plan).await?; + self.ctx + .execute_logical_plan(logical_plan) + .await? + .execute_stream() + .await + } + + /// Returns the memory pool the session's `RuntimeEnv` was built with. + /// + /// Used by the bridge layer to seed a per-query tracking context so + /// reduce-stage allocations count against the same pool as the shard-scan + /// path. + pub fn memory_pool(&self) -> Arc { + Arc::clone(&self.ctx.runtime_env().memory_pool) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use arrow_array::{Int64Array, RecordBatch}; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::execution::runtime_env::RuntimeEnvBuilder; + use datafusion_substrait::logical_plan::producer::to_substrait_plan; + use futures::StreamExt; + use tokio::runtime::Handle; + + fn test_runtime_env() -> RuntimeEnv { + RuntimeEnvBuilder::new() + .build() + .expect("runtime env builds") + } + + fn i64_schema(column: &str) -> SchemaRef { + Arc::new(Schema::new(vec![Field::new(column, DataType::Int64, false)])) + } + + fn i64_batch(schema: &SchemaRef, values: &[i64]) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(schema), + vec![Arc::new(Int64Array::from(values.to_vec()))], + ) + .expect("batch builds") + } + + #[tokio::test] + async fn register_partition_makes_table_resolvable() { + let env = test_runtime_env(); + let mut session = LocalSession::new(&env); + let schema = i64_schema("x"); + let _sender = session + .register_partition("input-0", Arc::clone(&schema)) + .expect("register succeeds"); + + // A trivial `SELECT * FROM "input-0"` proves the table resolves. + let df = session.ctx.sql("SELECT x FROM \"input-0\"").await.expect("sql parses"); + assert_eq!(df.schema().fields().len(), 1); + } + + #[tokio::test] + async fn execute_substrait_sums_streaming_input() { + let env = test_runtime_env(); + let mut session = LocalSession::new(&env); + let schema = i64_schema("x"); + let sender = session + .register_partition("input-0", Arc::clone(&schema)) + .expect("register succeeds"); + + // Build the Substrait bytes from a SQL-built logical plan against a + // matching session — the plan only references `input-0`, so it is + // portable onto our real session. + let substrait_bytes = { + let env = test_runtime_env(); + let mut producer = LocalSession::new(&env); + let _unused = producer + .register_partition("input-0", Arc::clone(&schema)) + .expect("producer register"); + let df = producer + .ctx + .sql("SELECT SUM(x) AS total FROM \"input-0\"") + .await + .expect("sum parses"); + let plan = df.logical_plan().clone(); + let substrait = to_substrait_plan(&plan, &producer.ctx.state()) + .expect("to_substrait"); + let mut buf = Vec::new(); + substrait.encode(&mut buf).expect("encode"); + buf + }; + + // Push three batches totaling 45 = 1+2+3+4+5+6+7+8+9, then close. + let producer_schema = Arc::clone(&schema); + let handle = Handle::current(); + let producer = std::thread::spawn(move || { + for chunk in &[vec![1i64, 2, 3], vec![4, 5, 6], vec![7, 8, 9]] { + sender + .send_blocking(Ok(i64_batch(&producer_schema, chunk)), &handle) + .expect("send"); + } + drop(sender); // EOF + }); + + let mut stream = session + .execute_substrait(&substrait_bytes) + .await + .expect("execute"); + + let mut total: i64 = 0; + while let Some(batch) = stream.next().await { + let batch = batch.expect("batch ok"); + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("i64 col"); + for i in 0..col.len() { + total += col.value(i); + } + } + producer.join().expect("producer thread"); + assert_eq!(total, 45); + } + + #[tokio::test] + async fn execute_substrait_sums_memtable_input() { + let env = test_runtime_env(); + let mut session = LocalSession::new(&env); + let schema = i64_schema("x"); + + let batches = vec![ + i64_batch(&schema, &[1, 2, 3]), + i64_batch(&schema, &[4, 5, 6]), + i64_batch(&schema, &[7, 8, 9]), + ]; + session + .register_memtable("input-0", Arc::clone(&schema), batches) + .expect("register memtable"); + + // Build the Substrait bytes from a SQL-built logical plan against a + // matching session — the plan only references `input-0`, so it is + // portable onto our real session. + let substrait_bytes = { + let env = test_runtime_env(); + let mut producer = LocalSession::new(&env); + producer + .register_memtable("input-0", Arc::clone(&schema), vec![]) + .expect("producer register"); + let df = producer + .ctx + .sql("SELECT SUM(x) AS total FROM \"input-0\"") + .await + .expect("sum parses"); + let plan = df.logical_plan().clone(); + let substrait = to_substrait_plan(&plan, &producer.ctx.state()) + .expect("to_substrait"); + let mut buf = Vec::new(); + substrait.encode(&mut buf).expect("encode"); + buf + }; + + let mut stream = session + .execute_substrait(&substrait_bytes) + .await + .expect("execute"); + + let mut total: i64 = 0; + while let Some(batch) = stream.next().await { + let batch = batch.expect("batch ok"); + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("i64 col"); + for i in 0..col.len() { + total += col.value(i); + } + } + assert_eq!(total, 45); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/partition_stream.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/partition_stream.rs new file mode 100644 index 0000000000000..fdc196efc4622 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/partition_stream.rs @@ -0,0 +1,307 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Streaming input for coordinator-reduce execution. +//! +//! A [`channel`] produces a paired [`PartitionStreamSender`] / +//! [`PartitionStreamReceiver`]. The sender is exposed through the FFM bridge so +//! Java can push Arrow `RecordBatch`es synchronously via +//! [`PartitionStreamSender::send_blocking`]. The receiver implements DataFusion's +//! [`RecordBatchStream`] and is wrapped in a [`SingleReceiverPartition`] so it +//! can be registered on a `SessionContext` as a `StreamingTable`. +//! +//! # Backpressure +//! +//! The underlying mpsc is bounded (capacity 4) — chosen small so the sender +//! back-pressures when the DataFusion execute side falls behind. Under load the +//! Java feeder thread blocks on `send_blocking`, which naturally stalls the +//! shard response pipeline. +//! +//! # Single-consumer contract +//! +//! [`SingleReceiverPartition::execute`] hands out the receiver exactly once. Any +//! subsequent `execute()` call on the same partition returns an already-closed +//! empty stream rather than panicking — matches the "take once, then empty" +//! contract expected by DataFusion's `StreamingTable` when a partition is +//! re-executed. + +use std::fmt; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::common::DataFusionError; +use datafusion::execution::{RecordBatchStream, TaskContext}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::streaming::PartitionStream; +use datafusion::physical_plan::SendableRecordBatchStream; +use futures::{stream, Stream}; +use tokio::runtime::Handle; +use tokio::sync::mpsc; + +/// Bounded channel capacity. Small by design — producers back-pressure when the +/// DataFusion execute side falls behind. +const CHANNEL_CAPACITY: usize = 4; + +/// Producer side of a partition stream. +/// +/// Owned by the FFM bridge via `Box::into_raw`; dropping the sender (e.g. via +/// `df_sender_close`) closes the channel, which signals EOF to the DataFusion +/// receiver side. +pub struct PartitionStreamSender { + tx: mpsc::Sender>, + schema: SchemaRef, +} + +impl PartitionStreamSender { + /// Returns the schema this sender was created with. + pub fn schema(&self) -> &SchemaRef { + &self.schema + } + + /// Push a batch into the channel from a synchronous (non-async) context. + /// + /// The provided `handle` is used to drive the async send — typically the + /// `io_runtime` handle from the global `RuntimeManager`. This lets the FFM + /// bridge push without being async itself and without requiring the calling + /// thread to be a Tokio worker. + /// + /// Blocks while the channel is full (natural backpressure). Returns an + /// error only if the receiver has been dropped. + pub fn send_blocking( + &self, + batch: Result, + handle: &Handle, + ) -> Result<(), DataFusionError> { + handle.block_on(self.tx.send(batch)).map_err(|_| { + DataFusionError::Execution( + "partition stream receiver dropped before send".to_string(), + ) + }) + } +} + +impl fmt::Debug for PartitionStreamSender { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PartitionStreamSender") + .field("schema", &self.schema) + .finish() + } +} + +/// Consumer side of a partition stream. +/// +/// Implements [`Stream`] + [`RecordBatchStream`] so DataFusion can poll it +/// directly. Typically handed to [`SingleReceiverPartition`] and registered on +/// a `SessionContext` as a `StreamingTable`. +pub struct PartitionStreamReceiver { + rx: mpsc::Receiver>, + schema: SchemaRef, +} + +impl fmt::Debug for PartitionStreamReceiver { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PartitionStreamReceiver") + .field("schema", &self.schema) + .finish() + } +} + +impl Stream for PartitionStreamReceiver { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.rx.poll_recv(cx) + } +} + +impl RecordBatchStream for PartitionStreamReceiver { + fn schema(&self) -> SchemaRef { + Arc::clone(&self.schema) + } +} + +/// Creates a paired sender/receiver over a bounded mpsc (capacity +/// [`CHANNEL_CAPACITY`]). +/// +/// Both halves share the provided [`SchemaRef`]. Dropping the sender closes the +/// channel — the receiver's `poll_next` then yields `Ready(None)` once any +/// buffered batches are drained, which DataFusion interprets as end-of-input. +pub fn channel(schema: SchemaRef) -> (PartitionStreamSender, PartitionStreamReceiver) { + let (tx, rx) = mpsc::channel(CHANNEL_CAPACITY); + let sender = PartitionStreamSender { + tx, + schema: Arc::clone(&schema), + }; + let receiver = PartitionStreamReceiver { rx, schema }; + (sender, receiver) +} + +/// Wraps a [`PartitionStreamReceiver`] so it can be registered as a DataFusion +/// `StreamingTable` partition. +/// +/// DataFusion's [`PartitionStream::execute`] contract may invoke `execute` more +/// than once across the life of a plan. The receiver can only be consumed once, +/// so the first `execute` takes it and subsequent calls return an empty stream +/// (zero batches, end-of-stream immediately) rather than panicking. +pub(crate) struct SingleReceiverPartition { + schema: SchemaRef, + receiver: Mutex>, +} + +impl SingleReceiverPartition { + pub(crate) fn new(receiver: PartitionStreamReceiver) -> Self { + Self { + schema: Arc::clone(&receiver.schema), + receiver: Mutex::new(Some(receiver)), + } + } +} + +impl fmt::Debug for SingleReceiverPartition { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SingleReceiverPartition") + .field("schema", &self.schema) + .finish() + } +} + +impl PartitionStream for SingleReceiverPartition { + fn schema(&self) -> &SchemaRef { + &self.schema + } + + fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { + let taken = self.receiver.lock().expect("partition mutex poisoned").take(); + match taken { + Some(receiver) => Box::pin(receiver), + None => { + // Second+ execute: hand back an already-closed empty stream so + // DataFusion sees the partition as drained. + Box::pin(RecordBatchStreamAdapter::new( + Arc::clone(&self.schema), + stream::empty(), + )) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::Int64Array; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use futures::StreamExt; + + fn test_schema() -> SchemaRef { + Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, false)])) + } + + fn test_batch(schema: &SchemaRef, values: &[i64]) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(schema), + vec![Arc::new(Int64Array::from(values.to_vec()))], + ) + .unwrap() + } + + #[tokio::test] + async fn channel_preserves_schema() { + let schema = test_schema(); + let (sender, receiver) = channel(Arc::clone(&schema)); + assert_eq!(sender.schema(), &schema); + assert_eq!(RecordBatchStream::schema(&receiver), schema); + } + + #[tokio::test] + async fn receiver_yields_sent_batches_then_eof() { + let schema = test_schema(); + let (sender, mut receiver) = channel(Arc::clone(&schema)); + + let producer_schema = Arc::clone(&schema); + let producer = tokio::spawn(async move { + sender.tx.send(Ok(test_batch(&producer_schema, &[1, 2]))).await.unwrap(); + sender.tx.send(Ok(test_batch(&producer_schema, &[3]))).await.unwrap(); + drop(sender); + }); + + let first = receiver.next().await.unwrap().unwrap(); + assert_eq!(first.num_rows(), 2); + let second = receiver.next().await.unwrap().unwrap(); + assert_eq!(second.num_rows(), 1); + assert!(receiver.next().await.is_none()); + producer.await.unwrap(); + } + + #[tokio::test] + async fn send_blocking_pushes_through_handle() { + let schema = test_schema(); + let (sender, mut receiver) = channel(Arc::clone(&schema)); + let handle = Handle::current(); + + let sender_schema = Arc::clone(&schema); + let producer = std::thread::spawn(move || { + sender + .send_blocking(Ok(test_batch(&sender_schema, &[7, 8, 9])), &handle) + .unwrap(); + drop(sender); + }); + + let batch = receiver.next().await.unwrap().unwrap(); + assert_eq!(batch.num_rows(), 3); + assert!(receiver.next().await.is_none()); + producer.join().unwrap(); + } + + #[test] + fn send_blocking_reports_receiver_dropped() { + let rt = tokio::runtime::Runtime::new().expect("runtime builds"); + let handle = rt.handle().clone(); + + let schema = test_schema(); + let (sender, receiver) = channel(Arc::clone(&schema)); + drop(receiver); + + let err = std::thread::spawn(move || { + sender + .send_blocking(Ok(test_batch(&schema, &[1])), &handle) + .unwrap_err() + }) + .join() + .unwrap(); + assert!(err.to_string().contains("receiver dropped")); + } + + #[tokio::test] + async fn single_receiver_partition_executes_once_then_empty() { + let schema = test_schema(); + let (sender, receiver) = channel(Arc::clone(&schema)); + let partition = SingleReceiverPartition::new(receiver); + assert_eq!(partition.schema(), &schema); + + let producer_schema = Arc::clone(&schema); + let producer = tokio::spawn(async move { + sender.tx.send(Ok(test_batch(&producer_schema, &[42]))).await.unwrap(); + drop(sender); + }); + + let ctx = Arc::new(TaskContext::default()); + let mut first = partition.execute(Arc::clone(&ctx)); + let batch = first.next().await.unwrap().unwrap(); + assert_eq!(batch.num_rows(), 1); + assert!(first.next().await.is_none()); + producer.await.unwrap(); + + // Second execute() must not panic and must yield an empty stream. + let mut second = partition.execute(ctx); + assert!(second.next().await.is_none()); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/tests/local_exec_test.rs b/sandbox/plugins/analytics-backend-datafusion/rust/tests/local_exec_test.rs new file mode 100644 index 0000000000000..a1672089a952a --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/tests/local_exec_test.rs @@ -0,0 +1,400 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Integration tests for the coordinator-reduce FFM exports. +//! +//! These tests drive the `df_*` C ABI entry points directly (same symbols +//! the Java side calls through `NativeBridge`) and validate: +//! +//! - `df_create_local_session` / `df_close_local_session` lifecycle +//! - `df_register_partition_stream` exposes the input as a DataFusion table +//! - `df_sender_send` → execute → `df_stream_next` drains a `SUM` aggregate +//! - Error path: `df_sender_send` on a sender whose receiver is gone returns +//! a negative rc and the heap-allocated error string decodes cleanly +//! - `df_close_local_session` drops registered senders (receiver side of the +//! mpsc closes), so a subsequent `df_sender_send` fails +//! +//! The runtime manager (`df_init_runtime_manager`) is a process-global +//! singleton, so we initialize it exactly once across all tests via a +//! `OnceLock` guard. + +use std::ffi::CString; +use std::os::raw::c_char; +use std::sync::{Arc, OnceLock}; +use std::thread; +use std::time::Duration; + +use arrow::ipc::writer::StreamWriter; +use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use arrow_array::{Array, Int64Array, RecordBatch, StructArray}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use datafusion::datasource::MemTable; +use datafusion::prelude::SessionContext; +use datafusion_substrait::logical_plan::producer::to_substrait_plan; +use prost::Message; +use tempfile::TempDir; + +use opensearch_datafusion::ffm::{ + df_close_global_runtime, df_close_local_session, df_create_global_runtime, + df_create_local_session, df_execute_local_plan, df_init_runtime_manager, + df_register_partition_stream, df_sender_close, df_sender_send, df_stream_close, + df_stream_next, +}; + +// --------------------------------------------------------------------------- +// One-time setup +// --------------------------------------------------------------------------- + +static RT_INIT: OnceLock<()> = OnceLock::new(); + +fn ensure_runtime_manager() { + RT_INIT.get_or_init(|| { + df_init_runtime_manager(1); + }); +} + +/// Holds a DataFusionRuntime pointer and its backing temp dir so the spill +/// directory is removed when the test finishes. +struct RuntimeGuard { + ptr: i64, + _spill_dir: TempDir, +} + +impl RuntimeGuard { + fn new() -> Self { + ensure_runtime_manager(); + let spill_dir = TempDir::new().expect("tempdir"); + let spill_path = spill_dir + .path() + .to_str() + .expect("utf-8 spill path") + .to_string(); + let spill_bytes = spill_path.as_bytes(); + // 128 MiB pool, 64 MiB spill cap — arbitrary; these tests push a + // handful of small batches. + let rc = unsafe { + df_create_global_runtime( + 128 * 1024 * 1024, + spill_bytes.as_ptr(), + spill_bytes.len() as i64, + 64 * 1024 * 1024, + ) + }; + assert!(rc > 0, "df_create_global_runtime returned {}", rc); + Self { + ptr: rc, + _spill_dir: spill_dir, + } + } +} + +impl Drop for RuntimeGuard { + fn drop(&mut self) { + unsafe { df_close_global_runtime(self.ptr) }; + } +} + +// --------------------------------------------------------------------------- +// Helpers: IPC schema bytes, Arrow C Data export, error decoding +// --------------------------------------------------------------------------- + +fn i64_schema(column: &str) -> SchemaRef { + Arc::new(Schema::new(vec![Field::new(column, DataType::Int64, false)])) +} + +fn i64_batch(schema: &SchemaRef, values: &[i64]) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(schema), + vec![Arc::new(Int64Array::from(values.to_vec()))], + ) + .expect("batch builds") +} + +/// Serialize a `Schema` to Arrow IPC stream bytes — the shape +/// `df_register_partition_stream` expects. +fn schema_to_ipc_bytes(schema: &Schema) -> Vec { + let mut buf: Vec = Vec::new(); + { + let mut writer = + StreamWriter::try_new(&mut buf, schema).expect("stream writer builds"); + writer.finish().expect("stream writer finishes"); + } + buf +} + +/// Export a `RecordBatch` as (FFI_ArrowArray*, FFI_ArrowSchema*) and transfer +/// ownership to the caller — mirrors what `DatafusionReduceSink.feed` will do +/// on the Java side. +/// +/// Returns `(array_ptr, schema_ptr)` as `i64` addresses. On successful +/// `df_sender_send` the Rust side consumes both pointers via `from_raw`. +fn export_batch_ptrs(batch: RecordBatch) -> (i64, i64) { + let schema = batch.schema(); + let ffi_schema = + FFI_ArrowSchema::try_from(schema.as_ref()).expect("schema export"); + + let struct_array: StructArray = batch.into(); + let array_data = struct_array.into_data(); + let ffi_array = FFI_ArrowArray::new(&array_data); + + let array_ptr = Box::into_raw(Box::new(ffi_array)) as i64; + let schema_ptr = Box::into_raw(Box::new(ffi_schema)) as i64; + (array_ptr, schema_ptr) +} + +/// Decode (and free) the heap-allocated error string referenced by a +/// negative FFM return code. Convention is defined in +/// `sandbox/libs/dataformat-native/rust/common/src/error.rs`: the error +/// pointer is the positive value of the negated return code, and the +/// string is a `CString` that must be freed via `CString::from_raw`. +fn decode_error(rc: i64) -> String { + assert!(rc < 0, "expected negative rc, got {}", rc); + let ptr = (-rc) as *mut c_char; + // SAFETY: the FFM layer produces this string via `CString::into_raw`; + // taking ownership back via `from_raw` both reads and frees it. + let cstring = unsafe { CString::from_raw(ptr) }; + cstring.to_string_lossy().into_owned() +} + +/// Build a Substrait plan for `SELECT SUM(x) AS total FROM "input-0"` using a +/// throwaway session that only knows the schema — the plan is portable onto +/// any session where `"input-0"` has the same schema. +async fn build_sum_substrait(schema: SchemaRef) -> Vec { + let ctx = SessionContext::new(); + let empty = MemTable::try_new(Arc::clone(&schema), vec![vec![]]).expect("mem table"); + ctx.register_table("input-0", Arc::new(empty)) + .expect("register input-0"); + let plan = ctx + .sql("SELECT SUM(x) AS total FROM \"input-0\"") + .await + .expect("sql parses") + .logical_plan() + .clone(); + let substrait = to_substrait_plan(&plan, &ctx.state()).expect("to_substrait"); + let mut buf = Vec::new(); + substrait.encode(&mut buf).expect("encode"); + buf +} + +fn register_input(session_ptr: i64, input_id: &str, schema: &Schema) -> i64 { + let ipc = schema_to_ipc_bytes(schema); + let id_bytes = input_id.as_bytes(); + let rc = unsafe { + df_register_partition_stream( + session_ptr, + id_bytes.as_ptr(), + id_bytes.len() as i64, + ipc.as_ptr(), + ipc.len() as i64, + ) + }; + assert!(rc > 0, "df_register_partition_stream rc={}", rc); + rc +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[test] +fn test_create_session_returns_nonzero_ptr() { + let runtime = RuntimeGuard::new(); + let session_ptr = unsafe { df_create_local_session(runtime.ptr) }; + assert!( + session_ptr > 0, + "df_create_local_session returned {}", + session_ptr + ); + unsafe { df_close_local_session(session_ptr) }; +} + +#[test] +fn test_register_partition_exposes_streaming_table() { + let runtime = RuntimeGuard::new(); + let session_ptr = unsafe { df_create_local_session(runtime.ptr) }; + assert!(session_ptr > 0); + + let schema_a = Schema::new(vec![Field::new("x", DataType::Int64, false)]); + let schema_b = Schema::new(vec![Field::new("y", DataType::Int64, true)]); + + let sender_a = register_input(session_ptr, "input-0", &schema_a); + let sender_b = register_input(session_ptr, "input-1", &schema_b); + assert_ne!(sender_a, sender_b); + + // Drop the senders first so the mpsc receivers in the registered + // StreamingTables see EOF when the session is later dropped. + unsafe { df_sender_close(sender_a) }; + unsafe { df_sender_close(sender_b) }; + unsafe { df_close_local_session(session_ptr) }; +} + +#[test] +fn test_execute_sum_substrait() { + let runtime = RuntimeGuard::new(); + let session_ptr = unsafe { df_create_local_session(runtime.ptr) }; + assert!(session_ptr > 0); + + let schema = i64_schema("x"); + let sender_ptr = register_input(session_ptr, "input-0", schema.as_ref()); + + // Build the Substrait plan off-thread — we need it before calling + // `df_execute_local_plan` which runs `block_on` on the caller. + let substrait_bytes = { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build tokio rt"); + rt.block_on(build_sum_substrait(Arc::clone(&schema))) + }; + + // Producer must start BEFORE `df_execute_local_plan`: the execute path + // blocks on `block_on(execute_substrait)` which may poll the streaming + // input during logical → physical planning. The channel capacity (4) is + // wide enough to buffer all three batches, so the producer finishes + // without waiting on a consumer. + let producer_schema = Arc::clone(&schema); + let producer = thread::spawn(move || { + for chunk in [ + vec![1i64, 2, 3], + vec![4i64, 5, 6], + vec![7i64, 8, 9], + ] { + let batch = i64_batch(&producer_schema, &chunk); + let (arr_ptr, sch_ptr) = export_batch_ptrs(batch); + let rc = unsafe { df_sender_send(sender_ptr, arr_ptr, sch_ptr) }; + assert_eq!(rc, 0, "df_sender_send rc={}", rc); + } + // EOF — releases the sender, which closes the mpsc. + unsafe { df_sender_close(sender_ptr) }; + }); + + // Give the producer a moment to start pushing so the first batches are + // in the channel before the physical plan pulls from it. Not required + // for correctness (mpsc is bounded and the producer drives its own + // block_on), but reduces interleaving surprises. + thread::sleep(Duration::from_millis(10)); + + let stream_ptr = unsafe { + df_execute_local_plan( + session_ptr, + substrait_bytes.as_ptr(), + substrait_bytes.len() as i64, + ) + }; + assert!(stream_ptr > 0, "df_execute_local_plan rc={}", stream_ptr); + + // Drain the output stream. Each `df_stream_next` returns either a + // pointer to a heap-allocated `FFI_ArrowArray` (caller owns it and + // must drop it) or 0 for EOS. + let mut total: i64 = 0; + loop { + let rc = unsafe { df_stream_next(stream_ptr) }; + assert!(rc >= 0, "df_stream_next rc={}", rc); + if rc == 0 { + break; + } + // SAFETY: `rc` is a `Box::into_raw(Box::new(FFI_ArrowArray))` ptr. + let ffi_array = unsafe { Box::from_raw(rc as *mut FFI_ArrowArray) }; + let result_schema = Arc::new(Schema::new(vec![Field::new( + "total", + DataType::Int64, + true, + )])); + let ffi_schema = FFI_ArrowSchema::try_from(result_schema.as_ref()) + .expect("result schema export"); + let array_data = unsafe { + arrow_array::ffi::from_ffi(*ffi_array, &ffi_schema).expect("import") + }; + let struct_array = StructArray::from(array_data); + let batch = RecordBatch::from(struct_array); + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("i64 column"); + for i in 0..col.len() { + if !col.is_null(i) { + total += col.value(i); + } + } + } + unsafe { df_stream_close(stream_ptr) }; + unsafe { df_close_local_session(session_ptr) }; + producer.join().expect("producer thread"); + + assert_eq!(total, 45, "SUM(1..=9) should be 45, got {}", total); +} + +#[test] +fn test_sender_send_error_path() { + let runtime = RuntimeGuard::new(); + let session_ptr = unsafe { df_create_local_session(runtime.ptr) }; + assert!(session_ptr > 0); + + let schema = i64_schema("x"); + let sender_ptr = register_input(session_ptr, "input-0", schema.as_ref()); + + // Drop the session: its registered StreamingTable drops the + // SingleReceiverPartition, which drops the mpsc receiver. The sender's + // channel is now closed. + unsafe { df_close_local_session(session_ptr) }; + + // Attempting to send now fails — `send_blocking` reports "receiver + // dropped before send". + let batch = i64_batch(&schema, &[1, 2, 3]); + let (arr_ptr, sch_ptr) = export_batch_ptrs(batch); + let rc = unsafe { df_sender_send(sender_ptr, arr_ptr, sch_ptr) }; + assert!(rc < 0, "expected error, got rc={}", rc); + + let msg = decode_error(rc); + assert!( + msg.contains("receiver dropped") || msg.contains("receiver"), + "unexpected error message: {}", + msg + ); + + unsafe { df_sender_close(sender_ptr) }; +} + +#[test] +fn test_close_session_drops_registered_senders() { + let runtime = RuntimeGuard::new(); + let session_ptr = unsafe { df_create_local_session(runtime.ptr) }; + assert!(session_ptr > 0); + + let schema = i64_schema("x"); + let sender_ptr = register_input(session_ptr, "input-0", schema.as_ref()); + + // Sanity check: before session close, sending succeeds (the channel has + // capacity 4 so one send does not block). + let batch_ok = i64_batch(&schema, &[10]); + let (a0, s0) = export_batch_ptrs(batch_ok); + let rc_ok = unsafe { df_sender_send(sender_ptr, a0, s0) }; + assert_eq!(rc_ok, 0, "pre-close send should succeed, rc={}", rc_ok); + + // Close the session. The surviving sender's mpsc is now orphaned. + unsafe { df_close_local_session(session_ptr) }; + + // Subsequent `df_sender_send` on the still-live sender pointer fails. + let batch_fail = i64_batch(&schema, &[20]); + let (a1, s1) = export_batch_ptrs(batch_fail); + let rc_err = unsafe { df_sender_send(sender_ptr, a1, s1) }; + assert!( + rc_err < 0, + "post-close send should fail, got rc={}", + rc_err + ); + let msg = decode_error(rc_err); + assert!( + msg.contains("receiver"), + "expected receiver-dropped error, got: {}", + msg + ); + + unsafe { df_sender_close(sender_ptr) }; +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceIT.java new file mode 100644 index 0000000000000..717f3054cb97a --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceIT.java @@ -0,0 +1,163 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.Version; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.analytics.AnalyticsPlugin; +import org.opensearch.arrow.flight.transport.FlightStreamPlugin; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.composite.CompositeDataFormatPlugin; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.PluginInfo; +import org.opensearch.ppl.TestPPLPlugin; +import org.opensearch.ppl.action.PPLRequest; +import org.opensearch.ppl.action.PPLResponse; +import org.opensearch.ppl.action.UnifiedPPLExecuteAction; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * End-to-end smoke test for the streaming coordinator-reduce path: + * + * 

+ *   PPL → planner → multi-shard SHARD_FRAGMENT dispatch → DataFusion shard scan
+ *       → ExchangeSink.feed → DatafusionReduceSink (Substrait SUM via convertFinalAggFragment)
+ *       → drain → downstream → assembled PPLResponse
+ *
+ * + *

Builds a parquet-backed composite index with two shards, indexes a small + * deterministic dataset, then runs a {@code stats sum(value) as total} aggregate. + * The total is a function of the indexed values × shard count; any drift in + * shard fan-out, sink wiring, or final-agg merge will show up as a mismatch. + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 2) +public class CoordinatorReduceIT extends OpenSearchIntegTestCase { + + private static final String INDEX = "coord_reduce_e2e"; + private static final int NUM_SHARDS = 2; + private static final int DOCS_PER_SHARD = 10; + /** Constant `value` for every doc — picks a deterministic SUM independent of shard routing. */ + private static final int VALUE = 7; + + @Override + protected Collection> nodePlugins() { + // Plugins with no extendedPlugins requirement go here. Plugins that need + // explicit extendedPlugins (so SPI ExtensionLoader walks the right parent + // classloader) are declared in additionalNodePlugins() below. + return List.of(TestPPLPlugin.class, FlightStreamPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class); + } + + @Override + protected Collection additionalNodePlugins() { + // OpenSearchIntegTestCase's nodePlugins() builds PluginInfo with empty + // extendedPlugins, which breaks ExtensiblePlugin.loadExtensions(...) for + // plugins like DataFusionPlugin that ride on AnalyticsPlugin's SPI. Use + // additionalNodePlugins() to declare the parent relationships explicitly. + return List.of( + classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) + ); + } + + private static PluginInfo classpathPlugin(Class pluginClass, List extendedPlugins) { + return new PluginInfo( + pluginClass.getName(), + "classpath plugin", + "NA", + Version.CURRENT, + "1.8", + pluginClass.getName(), + null, + extendedPlugins, + false + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + // STREAM_TRANSPORT (Arrow Flight RPC for shard→coordinator response streaming) + // is intentionally NOT enabled here. With it on, AnalyticsSearchTransportService + // routes all sendChildRequest calls through StreamTransportService whose connection + // profile only carries stream channels, breaking the non-stream fragment dispatch + // request. The non-stream path is enough for this IT's small-result SUM aggregate. + .build(); + } + + /** + * {@code source = T | stats sum(value) as total} on a 2-shard parquet-backed index + * → coordinator-reduce path runs the final SUM via {@link DatafusionReduceSink} + * and returns the deterministic total. + */ + public void testScalarSumAcrossShards() throws Exception { + createParquetBackedIndex(); + indexDeterministicDocs(); + + PPLResponse response = executePPL("source = " + INDEX + " | stats sum(value) as total"); + + assertNotNull("PPLResponse must not be null", response); + assertTrue("columns must contain 'total', got " + response.getColumns(), response.getColumns().contains("total")); + assertEquals("scalar agg must return exactly 1 row", 1, response.getRows().size()); + + int idx = response.getColumns().indexOf("total"); + Object cell = response.getRows().get(0)[idx]; + assertNotNull("SUM(value) cell must not be null — coordinator-reduce returned no value", cell); + long actual = ((Number) cell).longValue(); + long expected = (long) VALUE * NUM_SHARDS * DOCS_PER_SHARD; + assertEquals( + "SUM(value) across " + NUM_SHARDS + " shards × " + DOCS_PER_SHARD + " docs × value=" + VALUE + " = " + expected, + expected, + actual + ); + } + + private void createParquetBackedIndex() { + Settings indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats") + .build(); + + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(INDEX) + .setSettings(indexSettings) + .setMapping("value", "type=integer") + .get(); + assertTrue("index creation must be acknowledged", response.isAcknowledged()); + ensureGreen(INDEX); + } + + private void indexDeterministicDocs() { + int total = NUM_SHARDS * DOCS_PER_SHARD; + for (int i = 0; i < total; i++) { + client().prepareIndex(INDEX).setId(String.valueOf(i)).setSource("value", VALUE).get(); + } + client().admin().indices().prepareRefresh(INDEX).get(); + client().admin().indices().prepareFlush(INDEX).get(); + } + + private PPLResponse executePPL(String ppl) { + return client().execute(UnifiedPPLExecuteAction.INSTANCE, new PPLRequest(ppl)).actionGet(); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceMemtableIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceMemtableIT.java new file mode 100644 index 0000000000000..438742158d990 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceMemtableIT.java @@ -0,0 +1,135 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.Version; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.analytics.AnalyticsPlugin; +import org.opensearch.arrow.flight.transport.FlightStreamPlugin; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.composite.CompositeDataFormatPlugin; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.PluginInfo; +import org.opensearch.ppl.TestPPLPlugin; +import org.opensearch.ppl.action.PPLRequest; +import org.opensearch.ppl.action.PPLResponse; +import org.opensearch.ppl.action.UnifiedPPLExecuteAction; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * Memtable variant of {@link CoordinatorReduceIT}. Identical query and assertion, but the cluster + * starts with {@code datafusion.reduce.input_mode=memtable} so the coordinator-reduce path uses + * {@link DatafusionMemtableReduceSink} instead of the streaming sink. Verifies the sink dispatch + * wiring in {@link DataFusionAnalyticsBackendPlugin#getExchangeSinkProvider} and the buffered + * memtable handoff against a real multi-shard scan. + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 2) +public class CoordinatorReduceMemtableIT extends OpenSearchIntegTestCase { + + private static final String INDEX = "coord_reduce_memtable_e2e"; + private static final int NUM_SHARDS = 2; + private static final int DOCS_PER_SHARD = 10; + private static final int VALUE = 7; + + @Override + protected Collection> nodePlugins() { + return List.of(TestPPLPlugin.class, FlightStreamPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class); + } + + @Override + protected Collection additionalNodePlugins() { + return List.of( + classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) + ); + } + + private static PluginInfo classpathPlugin(Class pluginClass, List extendedPlugins) { + return new PluginInfo( + pluginClass.getName(), + "classpath plugin", + "NA", + Version.CURRENT, + "1.8", + pluginClass.getName(), + null, + extendedPlugins, + false + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .put(DataFusionPlugin.DATAFUSION_REDUCE_INPUT_MODE.getKey(), "memtable") + .build(); + } + + public void testScalarSumAcrossShardsViaMemtable() throws Exception { + createParquetBackedIndex(); + indexDeterministicDocs(); + + PPLResponse response = executePPL("source = " + INDEX + " | stats sum(value) as total"); + + assertNotNull("PPLResponse must not be null", response); + assertTrue("columns must contain 'total', got " + response.getColumns(), response.getColumns().contains("total")); + assertEquals("scalar agg must return exactly 1 row", 1, response.getRows().size()); + + int idx = response.getColumns().indexOf("total"); + Object cell = response.getRows().get(0)[idx]; + assertNotNull("SUM(value) cell must not be null — memtable coordinator-reduce returned no value", cell); + long actual = ((Number) cell).longValue(); + long expected = (long) VALUE * NUM_SHARDS * DOCS_PER_SHARD; + assertEquals("SUM(value) memtable path must match streaming path", expected, actual); + } + + private void createParquetBackedIndex() { + Settings indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats") + .build(); + + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(INDEX) + .setSettings(indexSettings) + .setMapping("value", "type=integer") + .get(); + assertTrue("index creation must be acknowledged", response.isAcknowledged()); + ensureGreen(INDEX); + } + + private void indexDeterministicDocs() { + int total = NUM_SHARDS * DOCS_PER_SHARD; + for (int i = 0; i < total; i++) { + client().prepareIndex(INDEX).setId(String.valueOf(i)).setSource("value", VALUE).get(); + } + client().admin().indices().prepareRefresh(INDEX).get(); + client().admin().indices().prepareFlush(INDEX).get(); + } + + private PPLResponse executePPL(String ppl) { + return client().execute(UnifiedPPLExecuteAction.INSTANCE, new PPLRequest(ppl)).actionGet(); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java new file mode 100644 index 0000000000000..f40bf225dda50 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java @@ -0,0 +1,203 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.CDataDictionaryProvider; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.analytics.spi.ExchangeSink; +import org.opensearch.analytics.spi.ExchangeSinkContext; +import org.opensearch.be.datafusion.nativelib.NativeBridge; +import org.opensearch.be.datafusion.nativelib.StreamHandle; +import org.opensearch.core.action.ActionListener; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.function.Consumer; + +import static org.apache.arrow.c.Data.importField; + +/** + * Shared lifecycle skeleton for coordinator-side {@link ExchangeSink}s backed by a native + * DataFusion local session. Subclasses customise per-batch handling and the close-time + * native handoff via {@link #feedBatchUnderLock} and {@link #closeUnderLock}. + * + *

Lifecycle invariants enforced by this base: + *

    + *
  • {@link #feed} synchronises on {@link #feedLock}, short-circuits when {@link #closed}, + * and always closes the supplied {@link VectorSchemaRoot} in {@code finally} regardless + * of whether {@link #feedBatchUnderLock} succeeds.
    • + *
  • {@link #close} flips {@link #closed} once under {@link #feedLock}, runs the + * subclass-specific {@link #closeUnderLock} hook, and unconditionally closes + * {@link #session} in {@code finally}, accumulating any failures and rethrowing.
    • + *
  • The downstream from {@link ExchangeSinkContext#downstream()} is intentionally NOT + * closed here — it accumulates drained results consumed by the walker after the + * sink is done.
    • + *
+ * + * @opensearch.internal + */ +abstract class AbstractDatafusionReduceSink implements ExchangeSink { + + /** Substrait/DataFusion table name for the single registered input partition. */ + // TODO: This will change to represent child ID and taken as input in context + // for now we have a single partition. + static final String INPUT_ID = "input-0"; + + protected final ExchangeSinkContext ctx; + protected final NativeRuntimeHandle runtimeHandle; + protected final DatafusionLocalSession session; + protected final byte[] schemaIpc; + + /** Guards {@link #closed} and serialises {@link #feed}/{@link #close} against producers. */ + protected final Object feedLock = new Object(); + + /** Set once in {@link #close} under {@link #feedLock}. Visible to all threads via volatile. */ + protected volatile boolean closed; + + protected AbstractDatafusionReduceSink(ExchangeSinkContext ctx, NativeRuntimeHandle runtimeHandle) { + this.ctx = ctx; + this.runtimeHandle = runtimeHandle; + this.schemaIpc = ArrowSchemaIpc.toBytes(ctx.inputSchema()); + this.session = new DatafusionLocalSession(runtimeHandle.get()); + } + + @Override + public void feed(VectorSchemaRoot batch) { + synchronized (feedLock) { + if (closed) { + batch.close(); + return; + } + try { + feedBatchUnderLock(batch); + } finally { + batch.close(); + } + } + } + + @Override + public final void close() { + synchronized (feedLock) { + if (closed) { + return; + } + closed = true; + } + Throwable failure = null; + try { + failure = closeUnderLock(); + } catch (Throwable t) { + failure = accumulate(failure, t); + } finally { + try { + session.close(); + } catch (Throwable t) { + failure = accumulate(failure, t); + } + } + rethrow(failure); + } + + /** + * Per-batch hook. Called inside {@code synchronized(feedLock)} after {@code closed} is + * verified false. Implementations export and hand off (or buffer) {@code batch} via the + * native bridge. Implementations MUST NOT close {@code batch} — the base class does that + * in {@code finally}. + */ + protected abstract void feedBatchUnderLock(VectorSchemaRoot batch); + + /** + * Subclass-specific shutdown. Runs after {@link #closed} is set and before + * {@link #session} is closed. Implementations should close their owned native resources + * (sender, output stream, accumulated FFI structs, …) and drain any pending output. + * + * @return the first failure encountered (use {@link #accumulate(Throwable, Throwable)} + * when multiple steps may fail), or {@code null} on clean shutdown. + */ + protected abstract Throwable closeUnderLock(); + + /** + * Drains a native output stream into {@link ExchangeSinkContext#downstream()}, importing + * each {@link ArrowArray} into a fresh {@link VectorSchemaRoot} on the Java side. + */ + protected final void drainOutputIntoDownstream(StreamHandle outStream) { + BufferAllocator alloc = ctx.allocator(); + try (CDataDictionaryProvider dictProvider = new CDataDictionaryProvider()) { + long schemaAddr = asyncCall(listener -> NativeBridge.streamGetSchema(outStream.getPointer(), listener)); + Schema outSchema; + try (ArrowSchema arrowSchema = ArrowSchema.wrap(schemaAddr)) { + Field structField = importField(alloc, arrowSchema, dictProvider); + outSchema = new Schema(structField.getChildren(), structField.getMetadata()); + } + while (true) { + long arrayAddr = asyncCall(listener -> NativeBridge.streamNext(runtimeHandle.get(), outStream.getPointer(), listener)); + if (arrayAddr == 0) { + break; + } + VectorSchemaRoot vsr = VectorSchemaRoot.create(outSchema, alloc); + try (ArrowArray arrowArray = ArrowArray.wrap(arrayAddr)) { + Data.importIntoVectorSchemaRoot(alloc, arrowArray, vsr, dictProvider); + } + ctx.downstream().feed(vsr); + } + } + } + + /** + * Synchronously awaits the result of an async native call expressed as a + * {@code Consumer>}. Restores interrupt state on + * {@link InterruptedException} and unwraps {@link ExecutionException} to surface the + * original cause. + */ + protected static long asyncCall(Consumer> call) { + CompletableFuture future = new CompletableFuture<>(); + call.accept(ActionListener.wrap(future::complete, future::completeExceptionally)); + try { + return future.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException re) { + throw re; + } + throw new RuntimeException(cause); + } + } + + /** Returns {@code t} if {@code acc} is null; otherwise adds {@code t} as a suppressed of {@code acc}. */ + protected static Throwable accumulate(Throwable acc, Throwable t) { + if (acc == null) { + return t; + } + acc.addSuppressed(t); + return acc; + } + + private static void rethrow(Throwable failure) { + if (failure == null) { + return; + } + if (failure instanceof RuntimeException re) { + throw re; + } + if (failure instanceof Error err) { + throw err; + } + throw new RuntimeException(failure); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrowSchemaIpc.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrowSchemaIpc.java new file mode 100644 index 0000000000000..1e8cee72d8c4b --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrowSchemaIpc.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.arrow.vector.ipc.WriteChannel; +import org.apache.arrow.vector.ipc.message.MessageSerializer; +import org.apache.arrow.vector.types.pojo.Schema; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.channels.Channels; + +/** + * Helper for serializing an Arrow {@link Schema} to the IPC stream bytes expected by the + * Rust-side {@code df_register_partition_stream} export. + */ +public final class ArrowSchemaIpc { + + private ArrowSchemaIpc() {} + + /** + * Encodes the schema as a single Arrow IPC stream message containing the schema header. + * + * @param schema the Arrow schema + * @return a heap byte array safe to hand to FFM + */ + public static byte[] toBytes(Schema schema) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (WriteChannel channel = new WriteChannel(Channels.newChannel(baos))) { + MessageSerializer.serialize(channel, schema); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialize Arrow schema to IPC bytes", e); + } + return baos.toByteArray(); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index ca3338ca8dbf0..24fa3415bed51 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -13,6 +13,7 @@ import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; import org.opensearch.analytics.spi.BackendCapabilityProvider; import org.opensearch.analytics.spi.EngineCapability; +import org.opensearch.analytics.spi.ExchangeSinkProvider; import org.opensearch.analytics.spi.FieldType; import org.opensearch.analytics.spi.FilterCapability; import org.opensearch.analytics.spi.FilterOperator; @@ -154,4 +155,21 @@ public SearchExecEngineProvider getSearchExecEngineProvider() { return engine; }; } + + @Override + public ExchangeSinkProvider getExchangeSinkProvider() { + return ctx -> { + DataFusionService svc = plugin.getDataFusionService(); + if (svc == null) { + throw new IllegalStateException("DataFusionService not initialized"); + } + String mode = plugin.getClusterService() != null + ? plugin.getClusterService().getClusterSettings().get(DataFusionPlugin.DATAFUSION_REDUCE_INPUT_MODE) + : "streaming"; + if ("memtable".equals(mode)) { + return new DatafusionMemtableReduceSink(ctx, svc.getNativeRuntime()); + } + return new DatafusionReduceSink(ctx, svc.getNativeRuntime()); + }; + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index 6170da33dcd1d..1af21101c1fe4 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -8,18 +8,35 @@ package org.opensearch.be.datafusion; +import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptSchema; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelDistribution; +import org.apache.calcite.rel.RelDistributions; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelReferentialConstraint; import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.schema.ColumnStrategy; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.util.ImmutableBitSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.analytics.planner.rel.OpenSearchStageInputScan; import org.opensearch.analytics.spi.FragmentConvertor; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import io.substrait.expression.AggregateFunctionInvocation; +import io.substrait.expression.Expression; import io.substrait.extension.SimpleExtension; import io.substrait.isthmus.ImmutableFeatureBoard; import io.substrait.isthmus.SubstraitRelVisitor; @@ -29,15 +46,31 @@ import io.substrait.isthmus.expression.WindowFunctionConverter; import io.substrait.plan.Plan; import io.substrait.plan.PlanProtoConverter; +import io.substrait.plan.ProtoPlanConverter; +import io.substrait.relation.Aggregate; +import io.substrait.relation.Filter; import io.substrait.relation.NamedScan; +import io.substrait.relation.Project; import io.substrait.relation.Rel; import io.substrait.relation.RelCopyOnWriteVisitor; +import io.substrait.relation.Sort; import io.substrait.util.EmptyVisitationContext; /** * Converts Calcite RelNode fragments to Substrait protobuf bytes * for the DataFusion Rust runtime. * + *

Dispatch summary: + *

    + *
  • {@link #convertShardScanFragment(String, RelNode)} and + * {@link #convertFinalAggFragment(RelNode)} — full-fragment conversions via + * {@link #convertToSubstrait(RelNode)}.
    • + *
  • {@link #attachPartialAggOnTop(RelNode, byte[])} and + * {@link #attachFragmentOnTop(RelNode, byte[])} — convert the wrapping + * operator standalone, then rewire its input to the decoded inner plan's + * root via {@link #rewire(Plan, Rel)}.
    • + *
+ * * @opensearch.internal */ public class DataFusionFragmentConvertor implements FragmentConvertor { @@ -58,19 +91,33 @@ public byte[] convertShardScanFragment(String tableName, RelNode fragment) { @Override public byte[] attachPartialAggOnTop(RelNode partialAggFragment, byte[] innerBytes) { - throw new UnsupportedOperationException("Multi-stage partial aggregate not yet implemented"); + LOGGER.debug("Attaching partial aggregate on top of {} inner bytes", innerBytes.length); + Plan inner = decodePlan(innerBytes); + Rel wrapper = convertStandalone(partialAggFragment); + Plan rewired = rewire(inner, withAggregationPhase(wrapper, Expression.AggregationPhase.INITIAL_TO_INTERMEDIATE)); + return serializePlan(rewired); } @Override public byte[] convertFinalAggFragment(RelNode fragment) { - throw new UnsupportedOperationException("Multi-stage final aggregate not yet implemented"); + LOGGER.debug("Converting final-aggregate fragment"); + // Rewrite any OpenSearchStageInputScan leaves to plain TableScan nodes so the + // isthmus visitor (which only knows about Calcite core / Logical RelNodes) + // emits a ReadRel with the stage-input-id as the named table. + RelNode rewritten = rewriteStageInputScans(fragment); + return convertToSubstrait(rewritten); } @Override public byte[] attachFragmentOnTop(RelNode fragment, byte[] innerBytes) { - throw new UnsupportedOperationException("Multi-stage fragment attachment not yet implemented"); + LOGGER.debug("Attaching generic fragment [{}] on top of {} inner bytes", fragment.getClass().getSimpleName(), innerBytes.length); + Plan inner = decodePlan(innerBytes); + Rel wrapper = convertStandalone(fragment); + return serializePlan(rewire(inner, wrapper)); } + // ── Core conversion helpers ───────────────────────────────────────────────── + private byte[] convertToSubstrait(RelNode fragment) { RelRoot root = RelRoot.of(fragment, SqlKind.SELECT); SubstraitRelVisitor visitor = createVisitor(fragment); @@ -89,6 +136,108 @@ private byte[] convertToSubstrait(RelNode fragment) { return bytes; } + /** + * Converts a single operator into a Substrait {@link Rel}. The operator may carry + * children (e.g. the {@code attachPartialAggOnTop} caller passes a + * {@code LogicalAggregate} whose input is the already-stripped inner tree); we + * deliberately discard those children by taking only the outermost rel of the + * conversion and rewiring its input during {@link #rewire(Plan, Rel)}. + */ + private Rel convertStandalone(RelNode operator) { + SubstraitRelVisitor visitor = createVisitor(operator); + return visitor.apply(operator); + } + + /** + * Rewires the Substrait {@code wrapper} rel to sit above the root relation of + * {@code inner}. Returns a new {@link Plan} whose single root is + * {@code wrapper(inner.root)}. Supports the known single-input wrappers emitted + * by our four SPI methods ({@link Aggregate}, {@link Sort}, {@link Filter}, + * {@link Project}). + */ + static Plan rewire(Plan inner, Rel wrapper) { + if (inner.getRoots().isEmpty()) { + throw new IllegalArgumentException("Inner Substrait plan has no root relation to rewire under wrapper"); + } + Plan.Root innerRoot = inner.getRoots().get(0); + Rel innerRel = innerRoot.getInput(); + Rel rewired = replaceInput(wrapper, innerRel); + return Plan.builder().addRoots(Plan.Root.builder().input(rewired).names(innerRoot.getNames()).build()).build(); + } + + private static Rel replaceInput(Rel wrapper, Rel newInput) { + if (wrapper instanceof Aggregate agg) { + return Aggregate.builder().from(agg).input(newInput).build(); + } + if (wrapper instanceof Sort sort) { + return Sort.builder().from(sort).input(newInput).build(); + } + if (wrapper instanceof Filter filter) { + return Filter.builder().from(filter).input(newInput).build(); + } + if (wrapper instanceof Project project) { + return Project.builder().from(project).input(newInput).build(); + } + throw new UnsupportedOperationException( + "Cannot attach-on-top a Substrait Rel of type " + wrapper.getClass().getSimpleName() + " — no single-input rewire defined" + ); + } + + /** + * Overrides the {@link Expression.AggregationPhase} on every {@link Aggregate.Measure} + * inside an {@link Aggregate} wrapper. No-op for non-aggregate wrappers. + * + *

Isthmus hardcodes {@code INITIAL_TO_RESULT} on every aggregate-function + * invocation. For the partial-agg-attach-on-shard path we want + * {@code INITIAL_TO_INTERMEDIATE}; the final-agg path stays at + * {@code INITIAL_TO_RESULT} (isthmus's default) which the DataFusion + * substrait deserialiser treats as the single-stage/final form. + */ + private static Rel withAggregationPhase(Rel rel, Expression.AggregationPhase phase) { + if (!(rel instanceof Aggregate agg)) { + return rel; + } + List newMeasures = new ArrayList<>(agg.getMeasures().size()); + for (Aggregate.Measure m : agg.getMeasures()) { + AggregateFunctionInvocation fn = m.getFunction(); + AggregateFunctionInvocation rephased = AggregateFunctionInvocation.builder().from(fn).aggregationPhase(phase).build(); + newMeasures.add(Aggregate.Measure.builder().from(m).function(rephased).build()); + } + return Aggregate.builder().from(agg).measures(newMeasures).build(); + } + + /** + * Rewrites every {@link OpenSearchStageInputScan} in the RelNode tree to a plain + * Calcite {@link TableScan} whose qualified name matches what + * {@link DatafusionReduceSink#INPUT_ID} registers on the native session. + * The isthmus visitor then emits a {@link NamedScan} with that name. + * + *

Single-input simplification: every {@link OpenSearchStageInputScan} maps to + * the same {@code "input-0"} table id. Multi-input / join shapes will require + * carrying per-input ids through {@link io.substrait.relation.NamedScan} once + * implemented. + */ + private static RelNode rewriteStageInputScans(RelNode node) { + if (node instanceof OpenSearchStageInputScan scan) { + return new StageInputTableScan(scan.getCluster(), scan.getTraitSet(), DatafusionReduceSink.INPUT_ID, scan.getRowType()); + } + List newInputs = new ArrayList<>(node.getInputs().size()); + boolean changed = false; + for (RelNode input : node.getInputs()) { + RelNode rewritten = rewriteStageInputScans(input); + newInputs.add(rewritten); + if (rewritten != input) { + changed = true; + } + } + if (changed) { + return node.copy(node.getTraitSet(), newInputs); + } + return node; + } + + // ── Visitor wiring ────────────────────────────────────────────────────────── + private SubstraitRelVisitor createVisitor(RelNode relNode) { RelDataTypeFactory typeFactory = relNode.getCluster().getTypeFactory(); TypeConverter typeConverter = TypeConverter.DEFAULT; @@ -112,6 +261,29 @@ private SubstraitRelVisitor createVisitor(RelNode relNode) { ); } + // ── Plan serde helpers ────────────────────────────────────────────────────── + + /** Decodes serialized Substrait bytes into a model-level {@link Plan}. */ + private Plan decodePlan(byte[] bytes) { + try { + io.substrait.proto.Plan proto = io.substrait.proto.Plan.parseFrom(bytes); + return new ProtoPlanConverter(extensions).from(proto); + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException("Failed to decode Substrait plan bytes", e); + } + } + + /** Serializes a model-level {@link Plan} to proto bytes. */ + private static byte[] serializePlan(Plan plan) { + return new PlanProtoConverter().toProto(plan).toByteArray(); + } + + // ── NamedScan prefix stripper ─────────────────────────────────────────────── + + /** + * Strips Calcite catalog prefixes (e.g. "opensearch") from {@link NamedScan} table + * names — DataFusion expects the bare index / stage-input id. + */ private static class TableNameModifier { Plan modifyTableNames(Plan plan) { TableNameVisitor visitor = new TableNameVisitor(); @@ -138,4 +310,102 @@ public Optional visit(NamedScan namedScan, EmptyVisitationContext context) } } } + + // ── Calcite TableScan wrappers for OpenSearchStageInputScan rewrite ───────── + + /** + * Minimal {@link TableScan} representing a stage-input source. The backing + * {@link StageInputRelOptTable} reports the stage-input id as its single qualified + * name; isthmus converts this to a {@link NamedScan} with that one-element name. + */ + static final class StageInputTableScan extends TableScan { + StageInputTableScan(RelOptCluster cluster, RelTraitSet traitSet, String stageInputId, RelDataType rowType) { + super(cluster, traitSet, List.of(), new StageInputRelOptTable(stageInputId, rowType)); + } + } + + /** + * Minimal {@link RelOptTable} implementation — only {@code getQualifiedName()} and + * {@code getRowType()} are consulted by the isthmus visitor. + */ + static final class StageInputRelOptTable implements RelOptTable { + private final List qualifiedName; + private final RelDataType rowType; + + StageInputRelOptTable(String stageInputId, RelDataType rowType) { + this.qualifiedName = List.of(stageInputId); + this.rowType = rowType; + } + + @Override + public List getQualifiedName() { + return qualifiedName; + } + + @Override + public RelDataType getRowType() { + return rowType; + } + + @Override + public double getRowCount() { + return 100; + } + + @Override + public RelOptSchema getRelOptSchema() { + return null; + } + + @Override + public RelNode toRel(ToRelContext context) { + throw new UnsupportedOperationException("StageInputRelOptTable.toRel not supported"); + } + + @Override + public List getColumnStrategies() { + return List.of(); + } + + @Override + public C unwrap(Class aClass) { + return null; + } + + @Override + public boolean isKey(ImmutableBitSet columns) { + return false; + } + + @Override + public List getKeys() { + return List.of(); + } + + @Override + public List getReferentialConstraints() { + return List.of(); + } + + @Override + public List getCollationList() { + return List.of(); + } + + @Override + public RelDistribution getDistribution() { + return RelDistributions.ANY; + } + + @Override + @SuppressWarnings("rawtypes") + public org.apache.calcite.linq4j.tree.Expression getExpression(Class clazz) { + return null; + } + + @Override + public RelOptTable extend(List extendedFields) { + return this; + } + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java index dbeb46049ac98..d70aa344a7df6 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java @@ -65,11 +65,34 @@ public class DataFusionPlugin extends Plugin implements SearchBackEndPlugin + *

  • {@code streaming} (default) — use {@link DatafusionReduceSink}: each batch is pushed + * through a tokio mpsc, the native plan polls inputs as it executes.
    • + *
  • {@code memtable} — use {@link DatafusionMemtableReduceSink}: all batches are buffered + * in Java and handed across in one call as a {@code MemTable}. Trades memory for a + * simpler input lifecycle with no cross-runtime spawn or oneshot machinery.
    • + * + */ + public static final Setting DATAFUSION_REDUCE_INPUT_MODE = Setting.simpleString( + "datafusion.reduce.input_mode", + "streaming", + v -> { + if (!"streaming".equals(v) && !"memtable".equals(v)) { + throw new IllegalArgumentException("datafusion.reduce.input_mode must be 'streaming' or 'memtable', got: " + v); + } + }, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private static final String SUPPORTED_FORMAT = "parquet"; private volatile DataFusionService dataFusionService; private volatile DataFormatRegistry dataFormatRegistry; private volatile SimpleExtension.ExtensionCollection substraitExtensions; + private volatile ClusterService clusterService; /** * Creates the DataFusion plugin. @@ -92,6 +115,7 @@ public Collection createComponents( DataFormatRegistry dataFormatRegistry ) { this.dataFormatRegistry = dataFormatRegistry; + this.clusterService = clusterService; Settings settings = environment.settings(); long memoryPoolLimit = DATAFUSION_MEMORY_POOL_LIMIT.get(settings); long spillMemoryLimit = DATAFUSION_SPILL_MEMORY_LIMIT.get(settings); @@ -110,6 +134,13 @@ public Collection createComponents( return Collections.singletonList(dataFusionService); } + /** + * Loads the Substrait default extension catalog with the plugin's classloader as the + * thread context classloader. Jackson polymorphic deserialization (used by Substrait + * to load its {@code SimpleExtension} subclasses) consults the TCCL; in an OpenSearch + * plugin context the TCCL is typically the server classloader, which cannot see the + * plugin-local Substrait classes. + */ private static SimpleExtension.ExtensionCollection loadSubstraitExtensions() { Thread t = Thread.currentThread(); ClassLoader previous = t.getContextClassLoader(); @@ -133,6 +164,15 @@ DataFusionService getDataFusionService() { return dataFusionService; } + ClusterService getClusterService() { + return clusterService; + } + + @Override + public List> getSettings() { + return List.of(DATAFUSION_MEMORY_POOL_LIMIT, DATAFUSION_SPILL_MEMORY_LIMIT, DATAFUSION_REDUCE_INPUT_MODE); + } + @Override public String name() { return "datafusion"; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionLocalSession.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionLocalSession.java new file mode 100644 index 0000000000000..9457220da9ea1 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionLocalSession.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.analytics.backend.jni.NativeHandle; +import org.opensearch.be.datafusion.nativelib.NativeBridge; + +/** + * Type-safe wrapper around a native DataFusion {@code LocalSession} pointer, used by the + * coordinator-reduce path ({@link DatafusionReduceSink}). + * + *

    The session holds a DataFusion {@code SessionContext} bound to the node-global runtime's + * memory pool and disk manager. It owns any input partition streams registered via + * {@link NativeBridge#registerPartitionStream(long, String, byte[])} and drops them when the + * session itself is closed. + */ +public final class DatafusionLocalSession extends NativeHandle { + + /** + * Creates a new local session tied to the given global runtime pointer. + * + * @param runtimePtr pointer returned by {@link NativeBridge#createGlobalRuntime} + */ + public DatafusionLocalSession(long runtimePtr) { + super(NativeBridge.createLocalSession(runtimePtr)); + } + + @Override + protected void doClose() { + NativeBridge.closeLocalSession(ptr); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSink.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSink.java new file mode 100644 index 0000000000000..52354a6ad77c7 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSink.java @@ -0,0 +1,120 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.analytics.spi.ExchangeSinkContext; +import org.opensearch.be.datafusion.nativelib.NativeBridge; +import org.opensearch.be.datafusion.nativelib.StreamHandle; + +import java.util.ArrayList; +import java.util.List; + +/** + * Memtable variant of {@link DatafusionReduceSink}: instead of opening a streaming partition + * and pushing each shard response through it, this sink buffers every fed + * {@link VectorSchemaRoot} as an exported Arrow C Data pair and on {@link #close()} hands the + * full set across in one native call. The native side builds a {@code MemTable}, registers it, + * and runs the Substrait plan against the materialized input. + * + *

    Trade-offs: + *

      + *
    • + No tokio mpsc, no cross-runtime spawn machinery in the input path. The single-shot + * handoff is simpler to reason about and matches the lifecycle already used for the + * output stream.
      • + *
    • − All input batches live in memory until {@code close()}. Use the streaming sink when + * the working set is too large to retain.
      • + *
    + * + *

    Lifecycle invariants and {@code feed}/{@code close} skeleton are implemented in + * {@link AbstractDatafusionReduceSink}. This subclass owns the buffered FFI structs and the + * close-time {@code registerMemtable + executeLocalPlan + drain} sequence. + */ +public final class DatafusionMemtableReduceSink extends AbstractDatafusionReduceSink { + + private final List arrays = new ArrayList<>(); + private final List schemas = new ArrayList<>(); + + public DatafusionMemtableReduceSink(ExchangeSinkContext ctx, NativeRuntimeHandle runtimeHandle) { + super(ctx, runtimeHandle); + } + + @Override + protected void feedBatchUnderLock(VectorSchemaRoot batch) { + BufferAllocator alloc = ctx.allocator(); + ArrowArray array = ArrowArray.allocateNew(alloc); + ArrowSchema arrowSchema = ArrowSchema.allocateNew(alloc); + try { + Data.exportVectorSchemaRoot(alloc, batch, null, array, arrowSchema); + arrays.add(array); + schemas.add(arrowSchema); + array = null; + arrowSchema = null; + } finally { + if (array != null) { + array.close(); + } + if (arrowSchema != null) { + arrowSchema.close(); + } + } + } + + @Override + protected Throwable closeUnderLock() { + Throwable failure = null; + long streamPtr = 0; + try { + long[] arrayPtrs = new long[arrays.size()]; + long[] schemaPtrs = new long[schemas.size()]; + for (int i = 0; i < arrays.size(); i++) { + arrayPtrs[i] = arrays.get(i).memoryAddress(); + schemaPtrs[i] = schemas.get(i).memoryAddress(); + } + NativeBridge.registerMemtable(session.getPointer(), INPUT_ID, schemaIpc, arrayPtrs, schemaPtrs); + + streamPtr = NativeBridge.executeLocalPlan(session.getPointer(), ctx.fragmentBytes()); + try (StreamHandle outStream = new StreamHandle(streamPtr, runtimeHandle)) { + streamPtr = 0; + drainOutputIntoDownstream(outStream); + } + } catch (Throwable t) { + failure = accumulate(failure, t); + } finally { + // The Arrow Java wrappers must always be closed. On the success path Rust has + // consumed the underlying FFI structs (release callback nulled), so close is a + // no-op for the data. On the failure-before-handoff path close releases the + // exported data buffers back to the Java allocator. + for (ArrowArray a : arrays) { + try { + a.close(); + } catch (Throwable t) { + failure = accumulate(failure, t); + } + } + for (ArrowSchema s : schemas) { + try { + s.close(); + } catch (Throwable t) { + failure = accumulate(failure, t); + } + } + arrays.clear(); + schemas.clear(); + if (streamPtr != 0) { + NativeBridge.streamClose(streamPtr); + } + } + return failure; + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionPartitionSender.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionPartitionSender.java new file mode 100644 index 0000000000000..d59f82f4b04af --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionPartitionSender.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.analytics.backend.jni.NativeHandle; +import org.opensearch.be.datafusion.nativelib.NativeBridge; + +/** + * Type-safe wrapper around a native {@code PartitionStreamSender} pointer. + * + *

    Produced by {@link NativeBridge#registerPartitionStream(long, String, byte[])} and used + * by {@link DatafusionReduceSink#feed} to push Arrow C Data batches into the reduce input + * stream. Closing the sender signals EOF to the DataFusion receiver side. + */ +public final class DatafusionPartitionSender extends NativeHandle { + + /** + * Wraps the given sender pointer. + * + * @param senderPtr pointer returned by {@link NativeBridge#registerPartitionStream} + */ + public DatafusionPartitionSender(long senderPtr) { + super(senderPtr); + } + + @Override + protected void doClose() { + NativeBridge.senderClose(ptr); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java new file mode 100644 index 0000000000000..538f8ab9358c0 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java @@ -0,0 +1,203 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.analytics.spi.ExchangeSinkContext; +import org.opensearch.be.datafusion.nativelib.NativeBridge; +import org.opensearch.be.datafusion.nativelib.StreamHandle; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Streaming coordinator-side reduce sink: opens a native partition stream up front, + * pushes each fed batch through a tokio mpsc-backed sender, and on close drains the native + * output stream into {@link ExchangeSinkContext#downstream()}. + * + *

    Overrides the base class's {@code synchronized(feedLock)} with a lock-free + * implementation. Multiple shard response handlers call {@link #feed} concurrently; + * backpressure comes from the native Rust mpsc channel (bounded, capacity 4). + * The send-after-close race is handled by catching the native error when the + * receiver has been dropped. + * + *

    Lifecycle: + *

      + *
    1. Constructor registers the input partition stream and kicks off native execution.
      2. + *
    2. {@link #feed} exports each batch via Arrow C Data and sends it lock-free.
      3. + *
    3. {@link #close} closes the sender (EOF), drains output, releases native resources.
      4. + *
    + */ +public final class DatafusionReduceSink extends AbstractDatafusionReduceSink { + + private static final Logger logger = LogManager.getLogger(DatafusionReduceSink.class); + + private final DatafusionPartitionSender sender; + private final StreamHandle outStream; + /** Cumulative batches fed into the native sender. */ + private final AtomicLong feedCount = new AtomicLong(); + /** + * Background thread that drains {@link #outStream} into the downstream sink as soon + * as the FINAL plan emits batches — running concurrently with feeds. + * + *

    Without this thread, the FINAL plan's downstream side is not polled until + * {@code close()} runs {@link #drainOutputIntoDownstream}. That polling chain is + * what causes DataFusion's input operators to pull from our partition stream's + * receiver. Without a concurrent puller, producers wedge past the input mpsc + * capacity (verified empirically with target_partitions=1; without RepartitionExec + * or this drain thread, the 2nd send_blocking parks indefinitely). + * + *

    The thread starts polling immediately at construction. It exits naturally + * when the FINAL plan reaches EOF (after {@link #sender}.close() signals input + * EOF and DataFusion completes the last aggregation). + */ + private final Thread drainThread; + /** Captures any throwable from the drain thread for surfacing during close(). */ + private final AtomicReference drainFailure = new AtomicReference<>(); + + public DatafusionReduceSink(ExchangeSinkContext ctx, NativeRuntimeHandle runtimeHandle) { + super(ctx, runtimeHandle); + long senderPtr = 0; + long streamPtr = 0; + try { + senderPtr = NativeBridge.registerPartitionStream(session.getPointer(), INPUT_ID, schemaIpc); + this.sender = new DatafusionPartitionSender(senderPtr); + streamPtr = NativeBridge.executeLocalPlan(session.getPointer(), ctx.fragmentBytes()); + this.outStream = new StreamHandle(streamPtr, runtimeHandle); + } catch (RuntimeException e) { + if (streamPtr != 0) { + NativeBridge.streamClose(streamPtr); + } + if (senderPtr != 0) { + NativeBridge.senderClose(senderPtr); + } + session.close(); + throw e; + } + // Spawn the drain thread AFTER the native handles are constructed so the catch-block + // doesn't have to deal with thread teardown on construction failure. + this.drainThread = new Thread(this::drainLoop, "df-reduce-drain-q" + ctx.queryId() + "-s" + ctx.stageId()); + this.drainThread.setDaemon(true); + this.drainThread.start(); + } + + /** + * Drain loop body. Runs on {@link #drainThread} from sink construction until the + * FINAL plan reaches EOF (which only happens after {@code sender.close()} is called + * by {@link #closeUnderLock}). + * + *

    Polls {@link #outStream} via {@code streamNext} and forwards each emitted batch + * to {@code ctx.downstream()}. Any throwable is captured in {@link #drainFailure} + * and re-surfaced from {@link #closeUnderLock} via the existing accumulate pattern. + */ + private void drainLoop() { + try { + drainOutputIntoDownstream(outStream); + } catch (Throwable t) { + drainFailure.set(t); + logger.warn("[ReduceSink] drain thread terminated with error", t); + } + } + + /** + * Lock-free feed: overrides the base class's synchronized feed. + * Arrow C Data export and native send happen without any Java-side lock. + * Backpressure comes from the Rust mpsc channel; the send-after-close + * race is caught via the native error. + */ + @Override + public void feed(VectorSchemaRoot batch) { + // Best-effort fast path — skip export work if already closed. + if (closed) { + batch.close(); + return; + } + // Export Arrow C Data outside any lock. The allocator is thread-safe; + // multiple shard handlers can export concurrently. + BufferAllocator alloc = ctx.allocator(); + ArrowArray array = ArrowArray.allocateNew(alloc); + ArrowSchema arrowSchema = ArrowSchema.allocateNew(alloc); + try { + Data.exportVectorSchemaRoot(alloc, batch, null, array, arrowSchema); + } catch (Throwable t) { + array.close(); + arrowSchema.close(); + batch.close(); + throw t; + } finally { + batch.close(); + } + // No lock — send directly. The Rust mpsc::Sender is thread-safe. + // If close() raced and already called senderClose(), the native side + // returns an error ("receiver dropped") which we catch and discard. + try { + NativeBridge.senderSend(sender.getPointer(), array.memoryAddress(), arrowSchema.memoryAddress()); + feedCount.incrementAndGet(); + } catch (RuntimeException e) { + if (closed) { + logger.debug("[ReduceSink] send-after-close race caught, discarding batch"); + return; + } + throw e; + } finally { + array.close(); + arrowSchema.close(); + } + } + + /** + * Not used — feed() is overridden directly. Required by the abstract class contract. + */ + @Override + protected void feedBatchUnderLock(VectorSchemaRoot batch) { + throw new UnsupportedOperationException("DatafusionReduceSink overrides feed() directly"); + } + + @Override + protected Throwable closeUnderLock() { + Throwable failure = null; + // 1. Signal EOF on input. The drain thread, which is already polling the output + // stream, will receive the final batches and then EOF, then exit cleanly. + try { + sender.close(); + } catch (Throwable t) { + failure = accumulate(failure, t); + } + // 2. Wait for the drain thread to finish processing remaining output. + try { + drainThread.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + failure = accumulate(failure, e); + } + // 3. Surface any error captured by the drain thread. + Throwable drainErr = drainFailure.get(); + if (drainErr != null) { + failure = accumulate(failure, drainErr); + } + // 4. Close native resources. + try { + outStream.close(); + } catch (Throwable t) { + failure = accumulate(failure, t); + } + return failure; + } + + /** Returns the cumulative number of batches fed into the native sender. */ + public long feedCount() { + return feedCount.get(); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index dfc37008908fa..059e34b6747f8 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -51,6 +51,13 @@ public final class NativeBridge { private static final MethodHandle STREAM_NEXT; private static final MethodHandle STREAM_CLOSE; private static final MethodHandle SQL_TO_SUBSTRAIT; + private static final MethodHandle CREATE_LOCAL_SESSION; + private static final MethodHandle CLOSE_LOCAL_SESSION; + private static final MethodHandle REGISTER_PARTITION_STREAM; + private static final MethodHandle EXECUTE_LOCAL_PLAN; + private static final MethodHandle SENDER_SEND; + private static final MethodHandle SENDER_CLOSE; + private static final MethodHandle REGISTER_MEMTABLE; static { SymbolLookup lib = NativeLibraryLoader.symbolLookup(); @@ -138,6 +145,64 @@ public final class NativeBridge { ValueLayout.ADDRESS ) ); + + // ── Coordinator-reduce bindings ── + // i64 df_create_local_session(runtime_ptr) + CREATE_LOCAL_SESSION = linker.downcallHandle( + lib.find("df_create_local_session").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + + // void df_close_local_session(session_ptr) + CLOSE_LOCAL_SESSION = linker.downcallHandle( + lib.find("df_close_local_session").orElseThrow(), + FunctionDescriptor.ofVoid(ValueLayout.JAVA_LONG) + ); + + // i64 df_register_partition_stream(session_ptr, input_id_ptr, input_id_len, schema_ipc_ptr, schema_ipc_len) + REGISTER_PARTITION_STREAM = linker.downcallHandle( + lib.find("df_register_partition_stream").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG + ) + ); + + // i64 df_execute_local_plan(session_ptr, substrait_ptr, substrait_len) + EXECUTE_LOCAL_PLAN = linker.downcallHandle( + lib.find("df_execute_local_plan").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) + ); + + // i64 df_sender_send(sender_ptr, array_ptr, schema_ptr) + SENDER_SEND = linker.downcallHandle( + lib.find("df_sender_send").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + + // void df_sender_close(sender_ptr) + SENDER_CLOSE = linker.downcallHandle(lib.find("df_sender_close").orElseThrow(), FunctionDescriptor.ofVoid(ValueLayout.JAVA_LONG)); + + // i64 df_register_memtable(session_ptr, input_id_ptr, input_id_len, schema_ipc_ptr, schema_ipc_len, + // array_ptrs, schema_ptrs, n_batches) + REGISTER_MEMTABLE = linker.downcallHandle( + lib.find("df_register_memtable").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG + ) + ); } private NativeBridge() {} @@ -275,6 +340,106 @@ public static byte[] sqlToSubstrait(long readerPtr, String tableName, String sql } } + // ---- Coordinator-reduce exports ---- + + /** + * Creates a local DataFusion session tied to the given global runtime. Returns an opaque + * native pointer freed by {@link #closeLocalSession}. + */ + public static long createLocalSession(long runtimePtr) { + NativeHandle.validatePointer(runtimePtr, "runtime"); + try (var call = new NativeCall()) { + return call.invoke(CREATE_LOCAL_SESSION, runtimePtr); + } + } + + /** Frees the native local session. Tolerates a zero pointer for idempotent close. */ + public static void closeLocalSession(long sessionPtr) { + NativeCall.invokeVoid(CLOSE_LOCAL_SESSION, sessionPtr); + } + + /** + * Registers an input partition stream on the session under {@code inputId}, with the given + * Arrow IPC-encoded schema. Returns an opaque sender pointer freed by {@link #senderClose}. + */ + public static long registerPartitionStream(long sessionPtr, String inputId, byte[] schemaIpc) { + NativeHandle.validatePointer(sessionPtr, "session"); + try (var call = new NativeCall()) { + var id = call.str(inputId); + return call.invoke( + REGISTER_PARTITION_STREAM, + sessionPtr, + id.segment(), + id.len(), + call.bytes(schemaIpc), + (long) schemaIpc.length + ); + } + } + + /** + * Executes a Substrait plan on the session, returning an opaque stream pointer. The stream is + * drained via {@link #streamNext} and freed by {@link #streamClose}. + */ + public static long executeLocalPlan(long sessionPtr, byte[] substrait) { + NativeHandle.validatePointer(sessionPtr, "session"); + try (var call = new NativeCall()) { + return call.invoke(EXECUTE_LOCAL_PLAN, sessionPtr, call.bytes(substrait), (long) substrait.length); + } + } + + /** + * Pushes one Arrow C Data-exported batch (array + schema addresses) into the sender. The + * native side takes ownership of both FFI structs. + */ + public static long senderSend(long senderPtr, long arrayPtr, long schemaPtr) { + NativeHandle.validatePointer(senderPtr, "sender"); + // arrayPtr/schemaPtr come from Arrow Java's C Data export (ArrowArray.memoryAddress()), + // NOT from our NativeHandle lifecycle — validate as non-zero rather than live-handle. + if (arrayPtr == 0) { + throw new IllegalArgumentException("arrayPtr must be non-zero"); + } + if (schemaPtr == 0) { + throw new IllegalArgumentException("schemaPtr must be non-zero"); + } + try (var call = new NativeCall()) { + return call.invoke(SENDER_SEND, senderPtr, arrayPtr, schemaPtr); + } + } + + /** Closes the sender, signalling end-of-input. Tolerates a zero pointer. */ + public static void senderClose(long senderPtr) { + NativeCall.invokeVoid(SENDER_CLOSE, senderPtr); + } + + /** + * Memtable variant of {@link #registerPartitionStream}: hands across a list of + * already-exported Arrow C Data batches in two parallel pointer arrays so the native side can + * build a {@code MemTable} in one shot. Native takes ownership of all FFI structs on success. + */ + public static long registerMemtable(long sessionPtr, String inputId, byte[] schemaIpc, long[] arrayPtrs, long[] schemaPtrs) { + NativeHandle.validatePointer(sessionPtr, "session"); + if (arrayPtrs.length != schemaPtrs.length) { + throw new IllegalArgumentException( + "arrayPtrs.length (" + arrayPtrs.length + ") != schemaPtrs.length (" + schemaPtrs.length + ")" + ); + } + try (var call = new NativeCall()) { + var id = call.str(inputId); + return call.invoke( + REGISTER_MEMTABLE, + sessionPtr, + id.segment(), + id.len(), + call.bytes(schemaIpc), + (long) schemaIpc.length, + call.longs(arrayPtrs), + call.longs(schemaPtrs), + (long) arrayPtrs.length + ); + } + } + public static void cacheManagerAddFiles(long runtimePtr, String[] filePaths) {} public static void cacheManagerRemoveFiles(long runtimePtr, String[] filePaths) {} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java index ff1666277fee4..3d156927e45f8 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java @@ -10,48 +10,44 @@ import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptSchema; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.plan.hep.HepPlanner; import org.apache.calcite.plan.hep.HepProgramBuilder; -import org.apache.calcite.rel.RelCollation; -import org.apache.calcite.rel.RelDistribution; -import org.apache.calcite.rel.RelDistributions; +import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.RelReferentialConstraint; -import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.schema.ColumnStrategy; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ImmutableBitSet; +import org.opensearch.analytics.planner.rel.OpenSearchStageInputScan; import org.opensearch.test.OpenSearchTestCase; import java.util.List; import io.substrait.extension.DefaultExtensionCatalog; import io.substrait.extension.SimpleExtension; +import io.substrait.proto.AggregateFunction; +import io.substrait.proto.AggregateRel; +import io.substrait.proto.AggregationPhase; import io.substrait.proto.FilterRel; import io.substrait.proto.Plan; import io.substrait.proto.PlanRel; import io.substrait.proto.ReadRel; import io.substrait.proto.Rel; +import io.substrait.proto.SortRel; /** - * Tests for {@link DataFusionFragmentConvertor}. - * Builds Calcite RelNode trees in-process, converts them via the convertor, - * then decodes the returned Substrait proto bytes and asserts on proto shape. + * Tests for {@link DataFusionFragmentConvertor}. Each conversion method is + * exercised independently against a Calcite RelNode constructed in-process, + * the returned Substrait proto bytes are decoded back into proto structures, + * and assertions are made on proto shape — not serialized string content. * - *

    TODO: Add end-to-end tests that feed the generated Substrait bytes into - * the actual DataFusion native runtime (see {@link DataFusionQueryExecutionTests} - * for the pattern) to verify the plans execute correctly, not just that the - * proto structure is well-formed. */ public class DataFusionFragmentConvertorTests extends OpenSearchTestCase { @@ -67,6 +63,8 @@ public void setUp() throws Exception { rexBuilder = new RexBuilder(typeFactory); HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); cluster = RelOptCluster.create(planner, rexBuilder); + // Load the Substrait extension catalog with the test classloader as TCCL — + // mirrors the swap performed by DataFusionPlugin#loadSubstraitExtensions. Thread t = Thread.currentThread(); ClassLoader prev = t.getContextClassLoader(); try { @@ -77,8 +75,13 @@ public void setUp() throws Exception { } } - // ── Helpers ───────────────────────────────────────────────────────────────── + private DataFusionFragmentConvertor newConvertor() { + return new DataFusionFragmentConvertor(extensions); + } + + // ── Helpers ──────────────────────────────────────────────────────────────── + /** Builds a nullable row type with integer columns named "A", "B", ... */ private RelDataType rowType(String... columns) { RelDataTypeFactory.Builder b = typeFactory.builder(); for (String c : columns) { @@ -87,42 +90,65 @@ private RelDataType rowType(String... columns) { return b.build(); } - private RelNode buildTableScan(String tableName, String... columns) { - return new MinimalTableScan(cluster, cluster.traitSet(), tableName, rowType(columns)); - } - - private DataFusionFragmentConvertor newConvertor() { - return new DataFusionFragmentConvertor(extensions); - } - + /** Decodes Substrait proto bytes into a {@link Plan}. */ private Plan decodeSubstrait(byte[] bytes) throws Exception { - assertNotNull(bytes); - assertTrue(bytes.length > 0); + assertNotNull("convertor bytes must not be null", bytes); + assertTrue("convertor bytes must not be empty", bytes.length > 0); return Plan.parseFrom(bytes); } + /** Extracts the single root {@link Rel} of a Substrait {@link Plan}. */ private Rel rootRel(Plan plan) { - assertFalse(plan.getRelationsList().isEmpty()); + assertFalse("plan must contain at least one relation", plan.getRelationsList().isEmpty()); PlanRel planRel = plan.getRelationsList().get(0); - assertTrue(planRel.hasRoot()); + assertTrue("plan relation must carry a root", planRel.hasRoot()); return planRel.getRoot().getInput(); } + /** + * Builds a Calcite {@code LogicalTableScan} via the convertor's own + * {@link DataFusionFragmentConvertor.StageInputTableScan} — a minimal TableScan + * subclass that the isthmus visitor emits as a {@link ReadRel} with a + * one-element named-table reference. + */ + private RelNode buildTableScan(String tableName, String... columns) { + return new DataFusionFragmentConvertor.StageInputTableScan(cluster, cluster.traitSet(), tableName, rowType(columns)); + } + + private LogicalAggregate buildSumAggregate(RelNode input, int columnIndex) { + AggregateCall sumCall = AggregateCall.create( + SqlStdOperatorTable.SUM, + false, + List.of(columnIndex), + -1, + typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.INTEGER), true), + "sum_col" + ); + return LogicalAggregate.create(input, List.of(), ImmutableBitSet.of(), null, List.of(sumCall)); + } + // ── Tests ────────────────────────────────────────────────────────────────── - public void testConvertShardScanFragmentTableScan() throws Exception { + /** + * A bare table scan converts to a {@code ReadRel} whose named table carries + * the supplied tableName (no catalog prefix). + */ + public void testConvertShardScanFragment_TableScan() throws Exception { RelNode scan = buildTableScan("test_index", "A", "B"); byte[] bytes = newConvertor().convertShardScanFragment("test_index", scan); Plan plan = decodeSubstrait(bytes); Rel root = rootRel(plan); - assertTrue(root.hasRead()); + assertTrue("root must be a ReadRel", root.hasRead()); ReadRel read = root.getRead(); - assertTrue(read.hasNamedTable()); + assertTrue("ReadRel must reference a named table", read.hasNamedTable()); assertEquals(List.of("test_index"), read.getNamedTable().getNamesList()); } - public void testConvertShardScanFragmentFilterOverScan() throws Exception { + /** + * A {@code Filter(Scan)} fragment converts to {@code FilterRel(ReadRel)}. + */ + public void testConvertShardScanFragment_FilterOverScan() throws Exception { RelNode scan = buildTableScan("test_index", "A", "B"); RexNode predicate = rexBuilder.makeCall( SqlStdOperatorTable.GREATER_THAN, @@ -135,147 +161,107 @@ public void testConvertShardScanFragmentFilterOverScan() throws Exception { Plan plan = decodeSubstrait(bytes); Rel root = rootRel(plan); - assertTrue(root.hasFilter()); + assertTrue("root must be a FilterRel", root.hasFilter()); FilterRel filterRel = root.getFilter(); - assertTrue(filterRel.hasCondition()); + assertTrue("FilterRel must carry a condition", filterRel.hasCondition()); Rel inner = filterRel.getInput(); - assertTrue(inner.hasRead()); + assertTrue("Filter input must be a ReadRel", inner.hasRead()); assertEquals(List.of("test_index"), inner.getRead().getNamedTable().getNamesList()); } - public void testConvertShardScanFragmentProducesNonEmptyBytes() throws Exception { - RelNode scan = buildTableScan("my_index", "col1"); - byte[] bytes = newConvertor().convertShardScanFragment("my_index", scan); + /** + * Attaching a partial aggregate on top of inner bytes yields an + * {@code AggregateRel(readRel)} with phase INITIAL_TO_INTERMEDIATE. + */ + public void testAttachPartialAggOnTop_WrapsInner() throws Exception { + DataFusionFragmentConvertor convertor = newConvertor(); - assertNotNull(bytes); - assertTrue(bytes.length > 0); - } + // Inner bytes from a shard-scan conversion. + RelNode scan = buildTableScan("test_index", "A"); + byte[] innerBytes = convertor.convertShardScanFragment("test_index", scan); - public void testConvertShardScanFragmentStripsTableNamePrefix() throws Exception { - RelNode scan = new MinimalTableScan(cluster, cluster.traitSet(), List.of("opensearch", "schema", "my_table"), rowType("X")); - byte[] bytes = newConvertor().convertShardScanFragment("my_table", scan); + // Build a bare partial-agg fragment whose input matches the inner's rowType. + LogicalAggregate partialAgg = buildSumAggregate(scan, 0); - Plan plan = decodeSubstrait(bytes); + byte[] combined = convertor.attachPartialAggOnTop(partialAgg, innerBytes); + + Plan plan = decodeSubstrait(combined); Rel root = rootRel(plan); - assertTrue(root.hasRead()); - assertEquals(List.of("my_table"), root.getRead().getNamedTable().getNamesList()); + assertTrue("root must be an AggregateRel", root.hasAggregate()); + AggregateRel agg = root.getAggregate(); + assertFalse("aggregate must have at least one measure", agg.getMeasuresList().isEmpty()); + AggregateFunction fn = agg.getMeasures(0).getMeasure(); + assertEquals( + "partial-agg phase must be INITIAL_TO_INTERMEDIATE", + AggregationPhase.AGGREGATION_PHASE_INITIAL_TO_INTERMEDIATE, + fn.getPhase() + ); + // Aggregate is rewired over the inner plan's root ReadRel. + Rel inner = agg.getInput(); + assertTrue("Aggregate input must be a ReadRel", inner.hasRead()); + assertEquals(List.of("test_index"), inner.getRead().getNamedTable().getNamesList()); } - public void testAttachPartialAggOnTopThrowsUnsupported() { - RelNode scan = buildTableScan("t", "A"); - expectThrows(UnsupportedOperationException.class, () -> newConvertor().attachPartialAggOnTop(scan, new byte[0])); - } + /** + * A final-agg fragment whose leaf is an {@link OpenSearchStageInputScan} + * converts to {@code AggregateRel(ReadRel(namedTable=[__stage__input__]))}. + */ + public void testConvertFinalAggFragment_WithStageInputScanLeaf() throws Exception { + RelDataType stageRowType = rowType("A"); + RelNode stageInput = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 7, stageRowType, List.of("datafusion")); + LogicalAggregate finalAgg = buildSumAggregate(stageInput, 0); - public void testConvertFinalAggFragmentThrowsUnsupported() { - RelNode scan = buildTableScan("t", "A"); - expectThrows(UnsupportedOperationException.class, () -> newConvertor().convertFinalAggFragment(scan)); - } + byte[] bytes = newConvertor().convertFinalAggFragment(finalAgg); - public void testAttachFragmentOnTopThrowsUnsupported() { - RelNode scan = buildTableScan("t", "A"); - expectThrows(UnsupportedOperationException.class, () -> newConvertor().attachFragmentOnTop(scan, new byte[0])); + Plan plan = decodeSubstrait(bytes); + Rel root = rootRel(plan); + assertTrue("root must be an AggregateRel", root.hasAggregate()); + AggregateRel agg = root.getAggregate(); + assertFalse("aggregate must have at least one measure", agg.getMeasuresList().isEmpty()); + // Isthmus defaults final-mode aggregates to INITIAL_TO_RESULT. + AggregateFunction fn = agg.getMeasures(0).getMeasure(); + assertEquals("final-agg phase must be INITIAL_TO_RESULT", AggregationPhase.AGGREGATION_PHASE_INITIAL_TO_RESULT, fn.getPhase()); + Rel inner = agg.getInput(); + assertTrue("Aggregate input must be a ReadRel", inner.hasRead()); + assertEquals( + "StageInputScan must be emitted as a ReadRel with the stage-input id", + List.of(DatafusionReduceSink.INPUT_ID), + inner.getRead().getNamedTable().getNamesList() + ); } - public void testExtensionCollectionReuse() throws Exception { + /** + * Attaching a {@link LogicalSort} on top of inner bytes yields + * {@code SortRel()}. + */ + public void testAttachFragmentOnTop_Sort() throws Exception { DataFusionFragmentConvertor convertor = newConvertor(); - RelNode scan = buildTableScan("t", "A"); - byte[] first = convertor.convertShardScanFragment("t", scan); - byte[] second = convertor.convertShardScanFragment("t", scan); - assertNotNull(first); - assertNotNull(second); - assertArrayEquals(first, second); - } - - // ── Minimal Calcite stubs for building test RelNodes ───────────────────────── - - static final class MinimalTableScan extends TableScan { - MinimalTableScan(RelOptCluster cluster, RelTraitSet traitSet, String tableName, RelDataType rowType) { - this(cluster, traitSet, List.of(tableName), rowType); - } - MinimalTableScan(RelOptCluster cluster, RelTraitSet traitSet, List qualifiedName, RelDataType rowType) { - super(cluster, traitSet, List.of(), new MinimalRelOptTable(qualifiedName, rowType)); - } - } - - static final class MinimalRelOptTable implements RelOptTable { - private final List qualifiedName; - private final RelDataType rowType; - - MinimalRelOptTable(List qualifiedName, RelDataType rowType) { - this.qualifiedName = qualifiedName; - this.rowType = rowType; - } - - @Override - public List getQualifiedName() { - return qualifiedName; - } - - @Override - public RelDataType getRowType() { - return rowType; - } - - @Override - public double getRowCount() { - return 100; - } - - @Override - public RelOptSchema getRelOptSchema() { - return null; - } - - @Override - public RelNode toRel(ToRelContext context) { - throw new UnsupportedOperationException(); - } - - @Override - public List getColumnStrategies() { - return List.of(); - } - - @Override - public C unwrap(Class aClass) { - return null; - } - - @Override - public boolean isKey(ImmutableBitSet columns) { - return false; - } - - @Override - public List getKeys() { - return List.of(); - } - - @Override - public List getReferentialConstraints() { - return List.of(); - } - - @Override - public List getCollationList() { - return List.of(); - } + // Inner: final-agg over stage-input. + RelDataType stageRowType = rowType("A"); + RelNode stageInput = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 3, stageRowType, List.of("datafusion")); + LogicalAggregate finalAgg = buildSumAggregate(stageInput, 0); + byte[] innerBytes = convertor.convertFinalAggFragment(finalAgg); - @Override - public RelDistribution getDistribution() { - return RelDistributions.ANY; - } + // Contract: attachFragmentOnTop receives a childless operator. Sort requires an + // input for row-type validation in the isthmus visitor; give it a bare placeholder + // with the same output row type as the inner agg. The placeholder is discarded + // during rewire (replaced with the inner plan's root). + RelNode placeholderInput = buildTableScan("__placeholder__", "sum_col"); + LogicalSort sort = LogicalSort.create(placeholderInput, RelCollations.of(0), null, null); - @Override - @SuppressWarnings("rawtypes") - public org.apache.calcite.linq4j.tree.Expression getExpression(Class clazz) { - return null; - } + byte[] combined = convertor.attachFragmentOnTop(sort, innerBytes); - @Override - public RelOptTable extend(List extendedFields) { - return this; - } + Plan plan = decodeSubstrait(combined); + Rel root = rootRel(plan); + assertTrue("root must be a SortRel", root.hasSort()); + SortRel sortRel = root.getSort(); + // Sort is rewired over the inner agg. + Rel inner = sortRel.getInput(); + assertTrue("Sort input must be an AggregateRel", inner.hasAggregate()); + Rel aggInput = inner.getAggregate().getInput(); + assertTrue("Agg input must be a ReadRel", aggInput.hasRead()); + assertEquals(List.of(DatafusionReduceSink.INPUT_ID), aggInput.getRead().getNamedTable().getNamesList()); } + } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSinkTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSinkTests.java new file mode 100644 index 0000000000000..f7acdb2c2ed8f --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSinkTests.java @@ -0,0 +1,149 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.ImmutableBitSet; +import org.opensearch.analytics.spi.ExchangeSink; +import org.opensearch.analytics.spi.ExchangeSinkContext; +import org.opensearch.be.datafusion.nativelib.NativeBridge; +import org.opensearch.test.OpenSearchTestCase; + +import java.nio.file.Path; +import java.util.List; + +import io.substrait.extension.DefaultExtensionCatalog; +import io.substrait.extension.SimpleExtension; + +/** + * Mirror of {@link DatafusionReduceSinkTests} for the memtable variant. Same Substrait plan, same + * batches, same downstream assertion — exercises the buffered-batch handoff path instead of the + * streaming sender path. + */ +public class DatafusionMemtableReduceSinkTests extends OpenSearchTestCase { + + public void testInputIdConstantMatchesDesign() { + assertEquals("Single-input reduce uses the synthetic id 'input-0'", "input-0", DatafusionMemtableReduceSink.INPUT_ID); + } + + public void testFeedDrainsSumToDownstream() throws Exception { + NativeBridge.initTokioRuntimeManager(2); + Path spillDir = createTempDir("datafusion-spill"); + long runtimePtr = NativeBridge.createGlobalRuntime(64 * 1024 * 1024, 0L, spillDir.toString(), 32 * 1024 * 1024); + assertTrue("runtime ptr non-zero", runtimePtr != 0); + NativeRuntimeHandle runtimeHandle = new NativeRuntimeHandle(runtimePtr); + + try (RootAllocator alloc = new RootAllocator(Long.MAX_VALUE)) { + Schema inputSchema = new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(64, true)), null))); + byte[] substrait = buildSumSubstraitBytes(DatafusionMemtableReduceSink.INPUT_ID); + + CapturingSink downstream = new CapturingSink(); + ExchangeSinkContext ctx = new ExchangeSinkContext("q-1", 0, substrait, alloc, inputSchema, downstream); + + DatafusionMemtableReduceSink sink = new DatafusionMemtableReduceSink(ctx, runtimeHandle); + try { + sink.feed(makeBatch(alloc, inputSchema, new long[] { 1L, 2L, 3L })); + sink.feed(makeBatch(alloc, inputSchema, new long[] { 4L, 5L, 6L })); + sink.feed(makeBatch(alloc, inputSchema, new long[] { 7L, 8L, 9L })); + } finally { + sink.close(); + } + + assertFalse("downstream must NOT be closed by the reduce sink", downstream.closed); + assertTrue("downstream should receive at least one row, got " + downstream.totalRows, downstream.totalRows >= 1); + assertEquals("SUM(1..9) should be 45", 45L, downstream.total); + } finally { + runtimeHandle.close(); + } + } + + private static byte[] buildSumSubstraitBytes(String inputId) { + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + RexBuilder rexBuilder = new RexBuilder(typeFactory); + HepPlanner hepPlanner = new HepPlanner(new HepProgramBuilder().build()); + RelOptCluster cluster = RelOptCluster.create(hepPlanner, rexBuilder); + + RelDataType bigintNullable = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BIGINT), true); + RelDataType rowType = typeFactory.builder().add("x", bigintNullable).build(); + + RelNode scan = new DataFusionFragmentConvertor.StageInputTableScan(cluster, cluster.traitSet(), inputId, rowType); + + AggregateCall sumCall = AggregateCall.create(SqlStdOperatorTable.SUM, false, List.of(0), -1, bigintNullable, "total"); + LogicalAggregate agg = LogicalAggregate.create(scan, List.of(), ImmutableBitSet.of(), null, List.of(sumCall)); + + return new DataFusionFragmentConvertor(loadExtensions()).convertFinalAggFragment(agg); + } + + private static SimpleExtension.ExtensionCollection loadExtensions() { + Thread t = Thread.currentThread(); + ClassLoader prev = t.getContextClassLoader(); + try { + t.setContextClassLoader(DatafusionMemtableReduceSinkTests.class.getClassLoader()); + return DefaultExtensionCatalog.DEFAULT_COLLECTION; + } finally { + t.setContextClassLoader(prev); + } + } + + private static VectorSchemaRoot makeBatch(BufferAllocator alloc, Schema schema, long[] values) { + VectorSchemaRoot root = VectorSchemaRoot.create(schema, alloc); + root.allocateNew(); + BigIntVector col = (BigIntVector) root.getVector(0); + for (int i = 0; i < values.length; i++) { + col.setSafe(i, values[i]); + } + col.setValueCount(values.length); + root.setRowCount(values.length); + return root; + } + + private static final class CapturingSink implements ExchangeSink { + long total; + int totalRows; + boolean closed; + + @Override + public synchronized void feed(VectorSchemaRoot batch) { + try { + BigIntVector col = (BigIntVector) batch.getVector(0); + int rows = batch.getRowCount(); + totalRows += rows; + for (int i = 0; i < rows; i++) { + total += col.getDataBuffer().getLong((long) i * BigIntVector.TYPE_WIDTH); + } + } finally { + batch.close(); + } + } + + @Override + public synchronized void close() { + closed = true; + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReduceSinkTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReduceSinkTests.java new file mode 100644 index 0000000000000..63046074286bc --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReduceSinkTests.java @@ -0,0 +1,300 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.ImmutableBitSet; +import org.opensearch.analytics.spi.ExchangeSink; +import org.opensearch.analytics.spi.ExchangeSinkContext; +import org.opensearch.be.datafusion.nativelib.NativeBridge; +import org.opensearch.test.OpenSearchTestCase; + +import java.nio.file.Path; +import java.util.List; + +import io.substrait.extension.DefaultExtensionCatalog; +import io.substrait.extension.SimpleExtension; + +/** + * Unit tests for {@link DatafusionReduceSink}. + * + *

    The sink is exercised at two levels: + *

      + *
    • Lightweight assertions that don't touch the native library (encoding helper, + * fixed input-id constant).
      • + *
    • A real end-to-end feed/drain round trip against a live native runtime: + * build Substrait bytes via {@link DataFusionFragmentConvertor}, construct the + * sink, feed Arrow batches, close, and assert the downstream sink received the + * reduced result.
      • + *
    + */ +public class DatafusionReduceSinkTests extends OpenSearchTestCase { + + public void testArrowSchemaIpcEncodesSchema() { + Schema schema = new Schema(List.of(new Field("message", FieldType.notNullable(new ArrowType.Int(64, true)), null))); + byte[] ipc = ArrowSchemaIpc.toBytes(schema); + assertNotNull("ipc bytes should be non-null", ipc); + assertTrue("ipc bytes should be non-empty", ipc.length > 0); + } + + public void testInputIdConstantMatchesDesign() { + assertEquals("Single-input reduce uses the synthetic id 'input-0'", "input-0", DatafusionReduceSink.INPUT_ID); + } + + /** + * End-to-end feed + drain: feeds three Arrow batches (values 1..9) into a real + * {@link DatafusionReduceSink} running a {@code SELECT SUM(x) FROM "input-0"} + * Substrait plan, then asserts the downstream sink received a single-row batch + * containing 45. + * + *

    Mirrors the Rust integration test {@code test_execute_sum_substrait}; the + * Java side proves the FFI ownership + drain wiring works against the same plan. + */ + public void testFeedDrainsSumToDownstream() throws Exception { + NativeBridge.initTokioRuntimeManager(2); + Path spillDir = createTempDir("datafusion-spill"); + long runtimePtr = NativeBridge.createGlobalRuntime(64 * 1024 * 1024, 0L, spillDir.toString(), 32 * 1024 * 1024); + assertTrue("runtime ptr non-zero", runtimePtr != 0); + // Wrap in NativeRuntimeHandle so the pointer is registered in the + // NativeHandle live-set that validatePointer consults. + NativeRuntimeHandle runtimeHandle = new NativeRuntimeHandle(runtimePtr); + + try (RootAllocator alloc = new RootAllocator(Long.MAX_VALUE)) { + Schema inputSchema = new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(64, true)), null))); + byte[] substrait = buildSumSubstraitBytes(DatafusionReduceSink.INPUT_ID); + + CapturingSink downstream = new CapturingSink(); + ExchangeSinkContext ctx = new ExchangeSinkContext("q-1", 0, substrait, alloc, inputSchema, downstream); + + DatafusionReduceSink sink = new DatafusionReduceSink(ctx, runtimeHandle); + try { + sink.feed(makeBatch(alloc, inputSchema, new long[] { 1L, 2L, 3L })); + sink.feed(makeBatch(alloc, inputSchema, new long[] { 4L, 5L, 6L })); + sink.feed(makeBatch(alloc, inputSchema, new long[] { 7L, 8L, 9L })); + } finally { + sink.close(); + } + + // Downstream is NOT closed by the reduce sink — its lifecycle is owned by + // the walker/orchestrator, which reads buffered batches after the sink drains. + assertFalse("downstream must NOT be closed by the reduce sink", downstream.closed); + assertTrue("downstream should receive at least one row, got " + downstream.totalRows, downstream.totalRows >= 1); + assertEquals("SUM(1..9) should be 45", 45L, downstream.total); + } finally { + runtimeHandle.close(); + } + } + + /** + * Demonstrates that producers wedge past the input mpsc capacity (4) when no + * consumer is draining — and proves that no consumer IS draining during the + * feed phase, because the CPU executor's spawned task only fires on the first + * poll of the output stream, which only happens inside {@code close()} via + * {@code drainOutputIntoDownstream → streamNext}. + * + *

    Expected log signature when this test runs: + * 

    +     *   [partition_stream] send_blocking enter — channel capacity remaining: 4
+     *   [partition_stream] send_blocking returned ok=true
+     *   [partition_stream] send_blocking enter — channel capacity remaining: 3
+     *   [partition_stream] send_blocking returned ok=true
+     *   ... 4 successful sends ...
+     *   [partition_stream] send_blocking enter — channel capacity remaining: 0
+     *   (no return — parked)
+     *   (no [cross_rt_stream] driver polled message before close — proves CPU never started)
+     *   ...test asserts producer parked at 4 feeds...
+     *   ...test calls close()...
+     *   [cross_rt_stream] driver polled for first time — submitting CPU spawn
+     *   [cross_rt_stream] CPU task started — beginning to pull from input stream
+     *
    + * + *

    The logs prove: producers are blocked, CPU executor hasn't spawned yet, + * and the spawn only fires when close() drains. Run with + * {@code -Dtests.logger.level=DEBUG} to see partition_stream logs. + */ + public void testProducersDoNotWedgePastCapacity() throws Exception { + NativeBridge.initTokioRuntimeManager(2); + Path spillDir = createTempDir("datafusion-spill"); + long runtimePtr = NativeBridge.createGlobalRuntime(64 * 1024 * 1024, 0L, spillDir.toString(), 32 * 1024 * 1024); + NativeRuntimeHandle runtimeHandle = new NativeRuntimeHandle(runtimePtr); + + try (RootAllocator alloc = new RootAllocator(Long.MAX_VALUE)) { + Schema inputSchema = new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(64, true)), null))); + byte[] substrait = buildSumSubstraitBytes(DatafusionReduceSink.INPUT_ID); + + CapturingSink downstream = new CapturingSink(); + ExchangeSinkContext ctx = new ExchangeSinkContext("q-wedge", 0, substrait, alloc, inputSchema, downstream); + + DatafusionReduceSink sink = new DatafusionReduceSink(ctx, runtimeHandle); + + final int totalBatches = 12; // intentionally > capacity (4) + java.util.concurrent.atomic.AtomicInteger attempts = new java.util.concurrent.atomic.AtomicInteger(); + Thread producer = new Thread(() -> { + for (int i = 0; i < totalBatches; i++) { + attempts.incrementAndGet(); + sink.feed(makeBatch(alloc, inputSchema, new long[] { (long) i })); + } + }, "test-producer-wedge"); + producer.setDaemon(true); + producer.start(); + + // Give the producer plenty of wall-clock time to push every batch if it weren't blocked. + // 4 should land in the mpsc immediately; the 5th will park indefinitely. + Thread.sleep(1500); + + long completed = sink.feedCount(); + int attempted = attempts.get(); + Thread.State state = producer.getState(); + logger.info("After 1500ms wait: completed={}, attempted={}, producerState={}", completed, attempted, state); + + // Channel capacity is 1 (intentionally reduced for diagnostic clarity). If no + // consumer is draining concurrently with feeds, we'd expect: + // completed = 1 (first push lands), attempted = 2 (second push parked), + // state = WAITING/TIMED_WAITING. + // If a consumer IS draining concurrently (e.g. RepartitionExec spawned a + // task during DataFusion plan setup), we'd expect: + // completed = totalBatches, state = TERMINATED. + // The actual outcome tells us which mental model is correct. + // After Part 1 (drain thread) is in place, the drain thread polls the output + // stream which cascades down to our partition stream's receiver — so even + // without RepartitionExec (target_partitions=1), there's a concurrent consumer. + // EXPECTATION: completed == totalBatches, producer terminated. + // + // Without the drain thread (and without RepartitionExec), we'd see: + // completed == 1, attempted == 2, state in {RUNNABLE (FFI-blocked), WAITING}. + // Note: a Java thread blocked inside an FFI call shows up as RUNNABLE in + // Thread.getState() because the JVM doesn't see Rust-level parking — the + // thread is "running native code" from the JVM's perspective. + assertEquals( + "with the drain thread, all " + totalBatches + " feeds should complete; got " + completed, + totalBatches, + completed + ); + assertEquals("producer thread should be TERMINATED after completing all feeds; got " + state, Thread.State.TERMINATED, state); + assertEquals("attempted should equal completed", completed, attempted); + + // Cleanup: close() drops the sender, which fails the parked tx.send futures with + // "receiver dropped". The producer thread errors out of senderSend; the lock-free + // feed catches the runtime exception when closed=true. close() then drains the + // (now empty) output stream and tears down. Producer thread becomes joinable. + sink.close(); + producer.join(5_000); + assertFalse("producer thread should have exited after sink.close()", producer.isAlive()); + + // Final accounting: feedCount reflects only the feeds that actually deposited + // before the parked one was unblocked-by-error. Anywhere from 4..5 inclusive. + logger.info("After close: feedCount={}, downstream rows={}", sink.feedCount(), downstream.totalRows); + } finally { + runtimeHandle.close(); + } + } + + // ── Helpers ────────────────────────────────────────────────────────────── + + /** + * Builds Substrait bytes for {@code SELECT SUM(x) FROM "input-0"} using the + * production {@link DataFusionFragmentConvertor} path — the same conversion + * {@code FragmentConversionDriver} invokes for a coordinator-reduce stage at + * runtime. + */ + private static byte[] buildSumSubstraitBytes(String inputId) { + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + RexBuilder rexBuilder = new RexBuilder(typeFactory); + HepPlanner hepPlanner = new HepPlanner(new HepProgramBuilder().build()); + RelOptCluster cluster = RelOptCluster.create(hepPlanner, rexBuilder); + + RelDataType bigintNullable = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BIGINT), true); + RelDataType rowType = typeFactory.builder().add("x", bigintNullable).build(); + + RelNode scan = new DataFusionFragmentConvertor.StageInputTableScan(cluster, cluster.traitSet(), inputId, rowType); + + AggregateCall sumCall = AggregateCall.create(SqlStdOperatorTable.SUM, false, List.of(0), -1, bigintNullable, "total"); + LogicalAggregate agg = LogicalAggregate.create(scan, List.of(), ImmutableBitSet.of(), null, List.of(sumCall)); + + return new DataFusionFragmentConvertor(loadExtensions()).convertFinalAggFragment(agg); + } + + /** + * Loads the Substrait extension catalog with the test classloader as TCCL — + * mirrors the swap performed by {@code DataFusionPlugin#loadSubstraitExtensions} + * so Jackson polymorphic deserialization can resolve plugin-local Substrait classes. + */ + private static SimpleExtension.ExtensionCollection loadExtensions() { + Thread t = Thread.currentThread(); + ClassLoader prev = t.getContextClassLoader(); + try { + t.setContextClassLoader(DatafusionReduceSinkTests.class.getClassLoader()); + return DefaultExtensionCatalog.DEFAULT_COLLECTION; + } finally { + t.setContextClassLoader(prev); + } + } + + private static VectorSchemaRoot makeBatch(BufferAllocator alloc, Schema schema, long[] values) { + VectorSchemaRoot root = VectorSchemaRoot.create(schema, alloc); + root.allocateNew(); + BigIntVector col = (BigIntVector) root.getVector(0); + for (int i = 0; i < values.length; i++) { + col.setSafe(i, values[i]); + } + col.setValueCount(values.length); + root.setRowCount(values.length); + return root; + } + + /** + * Reads each fed batch's single BIGINT column into {@link #total} + closes the batch. + * Values are extracted synchronously during {@code feed} so the test can assert on + * {@link #total} after {@code close()} has released all Arrow buffers. + */ + private static final class CapturingSink implements ExchangeSink { + long total; + int totalRows; + boolean closed; + + @Override + public synchronized void feed(VectorSchemaRoot batch) { + try { + BigIntVector col = (BigIntVector) batch.getVector(0); + int rows = batch.getRowCount(); + totalRows += rows; + // DataFusion may omit the validity buffer when there are no nulls; read raw. + for (int i = 0; i < rows; i++) { + total += col.getDataBuffer().getLong((long) i * BigIntVector.TYPE_WIDTH); + } + } finally { + batch.close(); + } + } + + @Override + public synchronized void close() { + closed = true; + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/NativeBridgeLocalSessionTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/NativeBridgeLocalSessionTests.java new file mode 100644 index 0000000000000..c2b4d8120fdfc --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/NativeBridgeLocalSessionTests.java @@ -0,0 +1,169 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.WriteChannel; +import org.apache.arrow.vector.ipc.message.MessageSerializer; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.be.datafusion.nativelib.NativeBridge; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.ByteArrayOutputStream; +import java.nio.channels.Channels; +import java.nio.file.Path; +import java.util.List; + +/** + * Smoke test for the coordinator-reduce FFM wrappers added by the datafusion-coordinator-reduce spec. + * + *

    Exercises each new {@link NativeBridge} wrapper against a real native library + global + * runtime. Mirrors the lifecycle pattern used by {@link DataFusionNativeBridgeTests} — each test + * creates its own per-test runtime and closes it at the end. + * + *

    Pointer handling follows the plugin convention: raw pointers returned by {@link NativeBridge} + * are wrapped in {@link org.opensearch.analytics.backend.jni.NativeHandle} subclasses + * ({@link NativeRuntimeHandle}, {@link DatafusionLocalSession}) so they are registered in the + * live-handle set that {@link NativeBridge}'s {@code validatePointer} guards check. + */ +public class NativeBridgeLocalSessionTests extends OpenSearchTestCase { + + private NativeRuntimeHandle createRuntime() { + NativeBridge.initTokioRuntimeManager(2); + Path spillDir = createTempDir("datafusion-spill"); + long runtimePtr = NativeBridge.createGlobalRuntime(64 * 1024 * 1024, 0L, spillDir.toString(), 32 * 1024 * 1024); + assertTrue("runtime ptr non-zero", runtimePtr != 0); + return new NativeRuntimeHandle(runtimePtr); + } + + private static byte[] schemaIpc(Schema schema) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (WriteChannel channel = new WriteChannel(Channels.newChannel(baos))) { + MessageSerializer.serialize(channel, schema); + } + return baos.toByteArray(); + } + + public void testCreateLocalSessionReturnsNonZeroPtr() { + NativeRuntimeHandle runtimeHandle = createRuntime(); + try { + DatafusionLocalSession session = new DatafusionLocalSession(runtimeHandle.get()); + assertTrue("session ptr non-zero", session.getPointer() != 0); + session.close(); + } finally { + runtimeHandle.close(); + } + } + + public void testCloseLocalSessionToleratesZero() { + // Must not throw. + NativeBridge.closeLocalSession(0L); + } + + public void testSenderCloseToleratesZero() { + NativeBridge.senderClose(0L); + } + + public void testRegisterPartitionStreamAndSenderClose() throws Exception { + NativeRuntimeHandle runtimeHandle = createRuntime(); + try { + DatafusionLocalSession session = new DatafusionLocalSession(runtimeHandle.get()); + try { + Schema schema = new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(64, true)), null))); + long senderPtr = NativeBridge.registerPartitionStream(session.getPointer(), "input-0", schemaIpc(schema)); + assertTrue("sender ptr non-zero", senderPtr != 0); + NativeBridge.senderClose(senderPtr); + } finally { + session.close(); + } + } finally { + runtimeHandle.close(); + } + } + + public void testRegisterMemtableAcceptsZeroBatches() throws Exception { + NativeRuntimeHandle runtimeHandle = createRuntime(); + try { + DatafusionLocalSession session = new DatafusionLocalSession(runtimeHandle.get()); + try { + Schema schema = new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(64, true)), null))); + NativeBridge.registerMemtable(session.getPointer(), "input-0", schemaIpc(schema), new long[0], new long[0]); + } finally { + session.close(); + } + } finally { + runtimeHandle.close(); + } + } + + public void testRegisterMemtableImportsBatch() throws Exception { + NativeRuntimeHandle runtimeHandle = createRuntime(); + try (RootAllocator alloc = new RootAllocator(Long.MAX_VALUE)) { + DatafusionLocalSession session = new DatafusionLocalSession(runtimeHandle.get()); + try { + Schema schema = new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(64, true)), null))); + VectorSchemaRoot vsr = VectorSchemaRoot.create(schema, alloc); + vsr.allocateNew(); + BigIntVector col = (BigIntVector) vsr.getVector(0); + col.setSafe(0, 1L); + col.setSafe(1, 2L); + col.setValueCount(2); + vsr.setRowCount(2); + try (ArrowArray array = ArrowArray.allocateNew(alloc); ArrowSchema arrowSchema = ArrowSchema.allocateNew(alloc)) { + Data.exportVectorSchemaRoot(alloc, vsr, null, array, arrowSchema); + NativeBridge.registerMemtable( + session.getPointer(), + "input-0", + schemaIpc(schema), + new long[] { array.memoryAddress() }, + new long[] { arrowSchema.memoryAddress() } + ); + } finally { + vsr.close(); + } + } finally { + session.close(); + } + } finally { + runtimeHandle.close(); + } + } + + public void testRegisterMemtableRejectsLengthMismatch() throws Exception { + NativeRuntimeHandle runtimeHandle = createRuntime(); + try { + DatafusionLocalSession session = new DatafusionLocalSession(runtimeHandle.get()); + try { + Schema schema = new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(64, true)), null))); + expectThrows( + IllegalArgumentException.class, + () -> NativeBridge.registerMemtable( + session.getPointer(), + "input-0", + schemaIpc(schema), + new long[] { 1L, 2L }, + new long[] { 1L } + ) + ); + } finally { + session.close(); + } + } finally { + runtimeHandle.close(); + } + } +} diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle index f5993d130d9c5..ced6ff13ba083 100644 --- a/sandbox/plugins/analytics-engine/build.gradle +++ b/sandbox/plugins/analytics-engine/build.gradle @@ -95,6 +95,25 @@ dependencies { testRuntimeOnly "org.codehaus.janino:janino:3.1.12" testRuntimeOnly "org.codehaus.janino:commons-compiler:3.1.12" + // jackson-annotations — required at runtime by jackson-databind (transitive via Calcite). + // Without this, child plugins that use Arrow's Schema (which triggers ObjectMapper init) + // fail with NoClassDefFoundError for JsonSerializeAs. + runtimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" + + // commons-math3 — Calcite's TimeFrames. references + // org.apache.commons.math3.fraction.BigFraction. Must live in this parent plugin's + // classloader because calcite-core is loaded here; child plugins (opensearch-sql) + // can't satisfy classloads triggered by parent-loaded classes. + runtimeOnly "org.apache.commons:commons-math3:3.6.1" + + // httpcore5 — Avatica's BuiltInConnectionProperty static initializer references + // org.apache.hc.core5.util.Timeout. Must live in this parent plugin's classloader + // because calcite-core/avatica-core are loaded here; child plugins (test-ppl-frontend) + // can't satisfy classloads triggered by parent-loaded classes. + runtimeOnly "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" + runtimeOnly "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" + runtimeOnly "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" + // arrow-memory-unsafe provides the DefaultAllocationManager that arrow-memory-core // discovers via ServiceLoader at runtime. Must be in the parent plugin's classloader // because BaseAllocator (from arrow-memory-core) does the ServiceLoader lookup. @@ -148,6 +167,19 @@ tasks.named('thirdPartyAudit').configure { 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1' ) + + ignoreMissingClasses( + // Optional brotli compression support pulled in by httpclient5 — not used by analytics + 'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status', + 'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper', + 'com.aayushatharva.brotli4j.encoder.Encoder$Mode', + 'com.aayushatharva.brotli4j.encoder.EncoderJNI$Operation', + 'com.aayushatharva.brotli4j.encoder.EncoderJNI$Wrapper', + // Optional Apache Commons Compress reference — gated by runtime classpath probe + 'org.apache.commons.compress.compressors.CompressorStreamFactory', + // Optional Conscrypt provider — TLS support fallback path + 'org.conscrypt.Conscrypt' + ) } configurations.all { diff --git a/sandbox/plugins/analytics-engine/licenses/commons-math3-3.6.1.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/commons-math3-3.6.1.jar.sha1 new file mode 100644 index 0000000000000..72975be4c8851 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/commons-math3-3.6.1.jar.sha1 @@ -0,0 +1 @@ +e4ba98f1d4b3c80ec46392f25e094a6a2e58fcbf \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/commons-math3-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/commons-math3-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/commons-math3-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/sandbox/plugins/analytics-engine/licenses/commons-math3-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/commons-math3-NOTICE.txt new file mode 100644 index 0000000000000..d3d6e140ce4f3 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/commons-math3-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons Logging +Copyright 2003-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/sandbox/plugins/analytics-engine/licenses/httpclient5-5.6.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/httpclient5-5.6.jar.sha1 new file mode 100644 index 0000000000000..f6c5a64d1e4ee --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/httpclient5-5.6.jar.sha1 @@ -0,0 +1 @@ +f502ee00ba82d44a6a29bda06a18f5b959808e09 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/httpclient5-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/httpclient5-LICENSE.txt new file mode 100644 index 0000000000000..32f01eda18fe9 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/httpclient5-LICENSE.txt @@ -0,0 +1,558 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +========================================================================= + +This project includes Public Suffix List copied from + +licensed under the terms of the Mozilla Public License, v. 2.0 + +Full license text: + +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/sandbox/plugins/analytics-engine/licenses/httpclient5-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/httpclient5-NOTICE.txt new file mode 100644 index 0000000000000..afee7c6e6880b --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/httpclient5-NOTICE.txt @@ -0,0 +1,6 @@ +Apache HttpComponents Client +Copyright 1999-2022 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + diff --git a/sandbox/plugins/analytics-engine/licenses/httpcore5-h2-5.4.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/httpcore5-h2-5.4.jar.sha1 new file mode 100644 index 0000000000000..dee91c553000e --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/httpcore5-h2-5.4.jar.sha1 @@ -0,0 +1 @@ +83cdd62ef3140664f46be59c2c2727141d1c5a32 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/httpcore5-h2-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/httpcore5-h2-LICENSE.txt new file mode 100644 index 0000000000000..32f01eda18fe9 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/httpcore5-h2-LICENSE.txt @@ -0,0 +1,558 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +========================================================================= + +This project includes Public Suffix List copied from + +licensed under the terms of the Mozilla Public License, v. 2.0 + +Full license text: + +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/sandbox/plugins/analytics-engine/licenses/httpcore5-h2-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/httpcore5-h2-NOTICE.txt new file mode 100644 index 0000000000000..afee7c6e6880b --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/httpcore5-h2-NOTICE.txt @@ -0,0 +1,6 @@ +Apache HttpComponents Client +Copyright 1999-2022 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + diff --git a/sandbox/plugins/analytics-engine/src/internalClusterTest/java/org/opensearch/fe/planner/unified/ClickBenchUnifiedPipelineIT.java b/sandbox/plugins/analytics-engine/src/internalClusterTest/java/org/opensearch/fe/planner/unified/ClickBenchUnifiedPipelineIT.java deleted file mode 100644 index 3a3720121e308..0000000000000 --- a/sandbox/plugins/analytics-engine/src/internalClusterTest/java/org/opensearch/fe/planner/unified/ClickBenchUnifiedPipelineIT.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.fe.planner.unified; - -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix; -import org.opensearch.analytics.AnalyticsPlugin; -import org.opensearch.arrow.flight.transport.FlightStreamPlugin; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.plugins.Plugin; -import org.opensearch.ppl.TestPPLPlugin; -import org.opensearch.ppl.action.PPLRequest; -import org.opensearch.ppl.action.PPLResponse; -import org.opensearch.ppl.action.UnifiedPPLExecuteAction; -import org.opensearch.test.OpenSearchIntegTestCase; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Internal cluster integration tests that run the ClickBench PPL workload - * through the full unified pipeline against a real OpenSearch cluster. - * - *

    Spawns a real cluster with PPLFrontEndPlugin + the real AnalyticsPlugin - * from sandbox/modules/query-engine, creates the ClickBench 'hits' index - * with the full mapping, and issues each PPL query via the transport action - * using client().execute(). - */ -@AwaitsFix(bugUrl = "analytics engine pipeline not E2E complete: fragment conversion + shard execution + Arrow Flight drain not yet wired") -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 1) -public class ClickBenchUnifiedPipelineIT extends OpenSearchIntegTestCase { - - private static final Logger logger = LogManager.getLogger(ClickBenchUnifiedPipelineIT.class); - private static final String HITS_INDEX = "hits"; - - private final String queryId; - - public ClickBenchUnifiedPipelineIT(String queryId) { - this.queryId = queryId; - } - - @ParametersFactory - public static Collection parameters() { - List params = new ArrayList<>(); - for (int i = 1; i <= 43; i++) { - params.add(new Object[] { "q" + i }); - } - return params; - } - - @Override - protected Collection> nodePlugins() { - return List.of(TestPPLPlugin.class, FlightStreamPlugin.class, AnalyticsPlugin.class); - } - - @Override - public void setUp() throws Exception { - super.setUp(); - createHitsIndex(); - ensureGreen(); - } - - private void createHitsIndex() throws Exception { - if (indexExists(HITS_INDEX)) { - return; - } - XContentBuilder mapping = XContentFactory.jsonBuilder(); - mapping.startObject(); - mapping.startObject("properties"); - addField(mapping, "AdvEngineID", "short"); - addField(mapping, "Age", "short"); - addField(mapping, "BrowserCountry", "keyword"); - addField(mapping, "BrowserLanguage", "keyword"); - addField(mapping, "CLID", "integer"); - addDateField(mapping, "ClientEventTime"); - addField(mapping, "ClientIP", "integer"); - addField(mapping, "ClientTimeZone", "short"); - addField(mapping, "CodeVersion", "integer"); - addField(mapping, "ConnectTiming", "integer"); - addField(mapping, "CookieEnable", "short"); - addField(mapping, "CounterClass", "short"); - addField(mapping, "CounterID", "integer"); - addField(mapping, "DNSTiming", "integer"); - addField(mapping, "DontCountHits", "short"); - addDateField(mapping, "EventDate"); - addDateField(mapping, "EventTime"); - addField(mapping, "FUniqID", "long"); - addField(mapping, "FetchTiming", "integer"); - addField(mapping, "FlashMajor", "short"); - addField(mapping, "FlashMinor", "short"); - addField(mapping, "FlashMinor2", "short"); - addField(mapping, "FromTag", "keyword"); - addField(mapping, "GoodEvent", "short"); - addField(mapping, "HID", "integer"); - addField(mapping, "HTTPError", "short"); - addField(mapping, "HasGCLID", "short"); - addField(mapping, "HistoryLength", "short"); - addField(mapping, "HitColor", "keyword"); - addField(mapping, "IPNetworkID", "integer"); - addField(mapping, "Income", "short"); - addField(mapping, "Interests", "short"); - addField(mapping, "IsArtifical", "short"); - addField(mapping, "IsDownload", "short"); - addField(mapping, "IsEvent", "short"); - addField(mapping, "IsLink", "short"); - addField(mapping, "IsMobile", "short"); - addField(mapping, "IsNotBounce", "short"); - addField(mapping, "IsOldCounter", "short"); - addField(mapping, "IsParameter", "short"); - addField(mapping, "IsRefresh", "short"); - addField(mapping, "JavaEnable", "short"); - addField(mapping, "JavascriptEnable", "short"); - addDateField(mapping, "LocalEventTime"); - addField(mapping, "MobilePhone", "short"); - addField(mapping, "MobilePhoneModel", "keyword"); - addField(mapping, "NetMajor", "short"); - addField(mapping, "NetMinor", "short"); - addField(mapping, "OS", "short"); - addField(mapping, "OpenerName", "integer"); - addField(mapping, "OpenstatAdID", "keyword"); - addField(mapping, "OpenstatCampaignID", "keyword"); - addField(mapping, "OpenstatServiceName", "keyword"); - addField(mapping, "OpenstatSourceID", "keyword"); - addField(mapping, "OriginalURL", "keyword"); - addField(mapping, "PageCharset", "keyword"); - addField(mapping, "ParamCurrency", "keyword"); - addField(mapping, "ParamCurrencyID", "short"); - addField(mapping, "ParamOrderID", "keyword"); - addField(mapping, "ParamPrice", "long"); - addField(mapping, "Params", "keyword"); - addField(mapping, "Referer", "keyword"); - addField(mapping, "RefererCategoryID", "short"); - addField(mapping, "RefererHash", "long"); - addField(mapping, "RefererRegionID", "integer"); - addField(mapping, "RegionID", "integer"); - addField(mapping, "RemoteIP", "integer"); - addField(mapping, "ResolutionDepth", "short"); - addField(mapping, "ResolutionHeight", "short"); - addField(mapping, "ResolutionWidth", "short"); - addField(mapping, "ResponseEndTiming", "integer"); - addField(mapping, "ResponseStartTiming", "integer"); - addField(mapping, "Robotness", "short"); - addField(mapping, "SearchEngineID", "short"); - addField(mapping, "SearchPhrase", "keyword"); - addField(mapping, "SendTiming", "integer"); - addField(mapping, "Sex", "short"); - addField(mapping, "SilverlightVersion1", "short"); - addField(mapping, "SilverlightVersion2", "short"); - addField(mapping, "SilverlightVersion3", "integer"); - addField(mapping, "SilverlightVersion4", "short"); - addField(mapping, "SocialSourceNetworkID", "short"); - addField(mapping, "SocialSourcePage", "keyword"); - addField(mapping, "Title", "keyword"); - addField(mapping, "TraficSourceID", "short"); - addField(mapping, "URL", "keyword"); - addField(mapping, "URLCategoryID", "short"); - addField(mapping, "URLHash", "long"); - addField(mapping, "URLRegionID", "integer"); - addField(mapping, "UTMCampaign", "keyword"); - addField(mapping, "UTMContent", "keyword"); - addField(mapping, "UTMMedium", "keyword"); - addField(mapping, "UTMSource", "keyword"); - addField(mapping, "UTMTerm", "keyword"); - addField(mapping, "UserAgent", "short"); - addField(mapping, "UserAgentMajor", "short"); - addField(mapping, "UserAgentMinor", "keyword"); - addField(mapping, "UserID", "long"); - addField(mapping, "WatchID", "long"); - addField(mapping, "WindowClientHeight", "short"); - addField(mapping, "WindowClientWidth", "short"); - addField(mapping, "WindowName", "integer"); - addField(mapping, "WithHash", "short"); - mapping.endObject(); // properties - mapping.endObject(); - - prepareCreate(HITS_INDEX).setSettings( - Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build() - ).setMapping(mapping).get(); - } - - private static void addField(XContentBuilder builder, String name, String type) throws Exception { - builder.startObject(name).field("type", type).endObject(); - } - - private static void addDateField(XContentBuilder builder, String name) throws Exception { - builder.startObject(name) - .field("type", "date") - .field("format", "yyyy-MM-dd HH:mm:ss||strict_date_optional_time||epoch_millis") - .endObject(); - } - - public void testClickBenchQuery() throws Exception { - runClickBenchQuery(queryId); - } - - private void runClickBenchQuery(String queryId) throws Exception { - String rawPpl = loadQuery(queryId); - String ppl = rawPpl.replace("source=hits", "source=opensearch.hits") - .replace("source =hits", "source =opensearch.hits") - .replace("source= hits", "source= opensearch.hits") - .replace("source = hits", "source = opensearch.hits"); - - logger.info("=== ClickBench {} (Unified Pipeline IT) ===\nPPL: {}", queryId, ppl); - - PPLRequest request = new PPLRequest(ppl); - PPLResponse response = client().execute(UnifiedPPLExecuteAction.INSTANCE, request).actionGet(); - assertNotNull("Response should not be null for " + queryId, response); - assertNotNull("Columns should not be null for " + queryId, response.getColumns()); - assertFalse("Columns should not be empty for " + queryId, response.getColumns().isEmpty()); - logger.info("SUCCESS {}: {} columns, {} rows", queryId, response.getColumns().size(), response.getRows().size()); - } - - private String loadQuery(String queryId) throws Exception { - String resourcePath = "clickbench/queries/" + queryId + ".ppl"; - try (InputStream is = getClass().getClassLoader().getResourceAsStream(resourcePath)) { - assertNotNull("Resource not found: " + resourcePath, is); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { - String content = reader.lines().collect(Collectors.joining("\n")); - content = content.replaceAll("/\\*[\\s\\S]*?\\*/", ""); - content = content.replaceAll("\\n", " ").replaceAll("\\s+", " ").trim(); - return content; - } - } - } - -} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java index b2fa4d08c3649..3d8eedc56c44a 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java @@ -10,6 +10,8 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataQueryBase; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionRequest; @@ -95,9 +97,6 @@ public DefaultPlanExecutor( this.scheduler = scheduler; } - // TODO: Extract plan → optimize → fork → convert → DAG into a dedicated component (e.g. QueryDAGBuilder) - // that takes the logical fragment and returns a fully-built DAG ready for scheduling. - // Also add per-step timing (plan, fork, convert, schedule, execute) for observability. @Override public void execute(RelNode logicalFragment, Object context, ActionListener> listener) { // Fork the entire query lifecycle (planning, scheduling, cleanup) onto the SEARCH @@ -120,11 +119,19 @@ public void execute(RelNode logicalFragment, Object context, ActionListener> listener) { + // Calcite's RelMetadataQuery reads its handler provider from a ThreadLocal + // (RelMetadataQueryBase.THREAD_PROVIDERS). The frontend seeds it on its own + // thread, but execute() hops to the SEARCH executor where the ThreadLocal is + // unset — RelOptUtil.toString / RelNode.explain inside PlannerImpl would then + // NPE on a null metadataHandlerProvider. Re-seed from the inbound cluster. + RelMetadataQueryBase.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(logicalFragment.getCluster().getMetadataProvider())); + logicalFragment.getCluster().invalidateMetadataQuery(); + RelNode plan = PlannerImpl.createPlan(logicalFragment, new PlannerContext(capabilityRegistry, clusterService.state())); QueryDAG dag = DAGBuilder.build(plan, capabilityRegistry, clusterService); PlanForker.forkAll(dag, capabilityRegistry); FragmentConversionDriver.convertAll(dag, capabilityRegistry); - logger.info("[DefaultPlanExecutor] QueryDAG:\n{}", dag); + logger.debug("[DefaultPlanExecutor] QueryDAG:\n{}", dag); // Register coordinator-level query task with TaskManager (like SearchTask). // This gives us a proper unique ID, visibility in _tasks API, and cancellation support. @@ -230,16 +237,21 @@ static ActionListener> buildBatchesListener( static Iterable batchesToRows(Iterable batches) { List rows = new ArrayList<>(); for (VectorSchemaRoot batch : batches) { - int colCount = batch.getFieldVectors().size(); - int rowCount = batch.getRowCount(); - for (int r = 0; r < rowCount; r++) { - Object[] row = new Object[colCount]; - for (int c = 0; c < colCount; c++) { - row[c] = ArrowValues.toJavaValue(batch.getVector(c), r); + try { + int colCount = batch.getFieldVectors().size(); + int rowCount = batch.getRowCount(); + for (int r = 0; r < rowCount; r++) { + Object[] row = new Object[colCount]; + for (int c = 0; c < colCount; c++) { + row[c] = ArrowValues.toJavaValue(batch.getVector(c), r); + } + rows.add(row); } - rows.add(row); + } finally { + // Release the Arrow buffers back to the query allocator. Without this the + // query teardown's allocator.close() detects a leak and fails the query. + batch.close(); } - batch.close(); } return rows; } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/RowProducingSink.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/RowProducingSink.java index 90ef50d551c4d..3d9a9608ae36e 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/RowProducingSink.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/RowProducingSink.java @@ -12,6 +12,7 @@ import org.apache.arrow.vector.types.pojo.Field; import org.opensearch.analytics.backend.ExchangeSource; import org.opensearch.analytics.spi.ExchangeSink; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import java.util.ArrayList; import java.util.List; @@ -26,6 +27,11 @@ * the {@link ExchangeSink} view to child stages and the walker reads * results via the {@link ExchangeSource} view. * + *

    A configurable row count limit ({@link #maxRows}) acts as a guardrail + * against unbounded result accumulation. When exceeded, {@link #feed} + * throws {@link OpenSearchRejectedExecutionException} which propagates to the stage + * execution and transitions it to FAILED. + * *

    Thread safety: {@link #feed} may be called concurrently from * multiple shard response handlers on the SEARCH thread pool. All mutating * and observing methods are synchronized on {@code this} to serialize @@ -36,8 +42,29 @@ */ public class RowProducingSink implements ExchangeSink, ExchangeSource { + /** + * Default maximum number of rows this sink will accept before rejecting + * further batches. Analogous to {@code index.max_result_window} (10k) + * in the core search path, but set higher for analytics workloads. + * + *

    TODO: make configurable via cluster setting. + */ + static final long DEFAULT_MAX_ROWS = 1_000_000L; + private final List batches = new ArrayList<>(); private final List fieldNames = new ArrayList<>(); + private final long maxRows; + private long totalRows; + + /** Creates a sink with the default row limit. */ + public RowProducingSink() { + this(DEFAULT_MAX_ROWS); + } + + /** Creates a sink with a custom row limit. Use {@code Long.MAX_VALUE} to disable. */ + public RowProducingSink(long maxRows) { + this.maxRows = maxRows; + } @Override public synchronized void feed(VectorSchemaRoot batch) { @@ -46,6 +73,17 @@ public synchronized void feed(VectorSchemaRoot batch) { fieldNames.add(f.getName()); } } + long incoming = batch.getRowCount(); + if (totalRows + incoming > maxRows) { + batch.close(); + throw new OpenSearchRejectedExecutionException( + "Analytics query result exceeded maximum row limit of " + + maxRows + + " rows. " + + "Consider adding filters or aggregations to reduce the result set." + ); + } + totalRows += incoming; batches.add(batch); } @@ -68,11 +106,7 @@ public synchronized Iterable readResult() { @Override public synchronized long getRowCount() { - long total = 0; - for (VectorSchemaRoot batch : batches) { - total += batch.getRowCount(); - } - return total; + return totalRows; } /** diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageExecution.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageExecution.java index ac392aff83a92..e10f778535139 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageExecution.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageExecution.java @@ -20,10 +20,6 @@ * backend-provided {@link ExchangeSink} (from {@link org.opensearch.analytics.spi.ExchangeSinkProvider}) * and routes all child stage output into it via {@link #inputSink(int)}. * - *

    This is a placeholder shape: the backend sink accepts batches but there is - * no contract yet for draining its output downstream. The drain/output contract - * will be re-introduced when a real backend implementation lands. - * *

    Lifecycle: * {@code CREATED → RUNNING → (SUCCEEDED | FAILED | CANCELLED)} * @@ -49,11 +45,21 @@ public ExchangeSink inputSink(int childStageId) { return backendSink; } - // No output drain contract yet. Will be reintroduced when a real backend - // implementation is wired up. + /** + * Returns the downstream sink as an {@link ExchangeSource}. The backend sink's + * {@code close()} drains native batches into this same downstream as the + * last step of {@link #start()}, so by the time the walker reads via + * {@code outputSource().readResult()} every result batch is already buffered + * here. + */ @Override public ExchangeSource outputSource() { - throw new UnsupportedOperationException("LocalStageExecution has no output source yet — backend drain contract pending"); + if (downstream instanceof ExchangeSource source) { + return source; + } + throw new UnsupportedOperationException( + "downstream sink " + downstream.getClass().getSimpleName() + " does not implement ExchangeSource" + ); } @Override @@ -62,7 +68,6 @@ public void start() { logger.info("[LocalStage] start() stageId={}", stage.getStageId()); try { backendSink.close(); - downstream.close(); if (transitionTo(State.SUCCEEDED)) { logger.info("[LocalStage] SUCCEEDED stageId={}", stage.getStageId()); } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java index a9a4db19a67e7..cac4a396cbdd8 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java @@ -8,18 +8,26 @@ package org.opensearch.analytics.exec.stage; +import org.apache.arrow.vector.types.pojo.Schema; import org.opensearch.analytics.exec.QueryContext; import org.opensearch.analytics.planner.dag.Stage; import org.opensearch.analytics.planner.dag.StageExecutionType; import org.opensearch.analytics.spi.ExchangeSink; +import org.opensearch.analytics.spi.ExchangeSinkContext; import org.opensearch.analytics.spi.ExchangeSinkProvider; +import java.util.List; + /** * Builds executions for {@link StageExecutionType#COORDINATOR_REDUCE} stages — * those that run at the coordinator with a backend-provided {@link ExchangeSink}. - * Creates the sink via {@link Stage#getExchangeSinkProvider()} using the chosen - * plan alternative's serialized bytes and hands it to {@link LocalStageExecution} - * along with the parent-provided downstream sink. + * Creates the sink via {@link Stage#getExchangeSinkProvider()} using an + * {@link ExchangeSinkContext} carrying the plan bytes, allocator, input + * schema (derived from the single child stage), and downstream sink. Hands + * the resulting sink to {@link LocalStageExecution}. + * + *

    Single-sink simplification: assumes exactly one child stage. Multi-child + * (joins, set ops) will require per-child sink routing in a follow-up. * * @opensearch.internal */ @@ -28,9 +36,17 @@ final class LocalStageScheduler implements StageScheduler { @Override public StageExecution createExecution(Stage stage, ExchangeSink sink, QueryContext config) { ExchangeSinkProvider provider = stage.getExchangeSinkProvider(); + ExchangeSinkContext context = new ExchangeSinkContext( + config.queryId(), + stage.getStageId(), + chosenBytes(stage), + config.bufferAllocator(), + deriveInputSchema(stage), + sink + ); ExchangeSink backendSink; try { - backendSink = provider.createSink(chosenBytes(stage)); + backendSink = provider.createSink(context); } catch (Exception e) { throw new RuntimeException("Failed to create exchange sink for stageId=" + stage.getStageId(), e); } @@ -45,4 +61,18 @@ private static byte[] chosenBytes(Stage stage) { + stage.getPlanAlternatives().size(); return stage.getPlanAlternatives().getFirst().convertedBytes(); } + + /** + * Derives the backend's input Arrow schema from the single child stage's + * fragment rowtype. Multi-child support (joins, set ops with heterogeneous + * inputs) is deferred. + */ + private static Schema deriveInputSchema(Stage stage) { + List children = stage.getChildStages(); + assert children.size() == 1 : "COORDINATOR_REDUCE stage " + + stage.getStageId() + + " expected exactly one child stage, got " + + children.size(); + return ArrowSchemaFromCalcite.arrowSchemaFromRowType(children.getFirst().getFragment().getRowType()); + } } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/LocalStageExecutionTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/LocalStageExecutionTests.java index 7207cca7b4743..0b8583886cc8c 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/LocalStageExecutionTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/LocalStageExecutionTests.java @@ -11,6 +11,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.analytics.exec.RowProducingSink; import org.opensearch.analytics.planner.dag.Stage; import org.opensearch.analytics.spi.ExchangeSink; import org.opensearch.test.OpenSearchTestCase; @@ -42,7 +43,7 @@ public void tearDown() throws Exception { super.tearDown(); } - public void testStartClosesSinksAndTransitionsToSucceeded() { + public void testStartClosesBackendSinkAndTransitionsToSucceeded() { CapturingSink backend = new CapturingSink(); CapturingSink downstream = new CapturingSink(); LocalStageExecution exec = new LocalStageExecution(stageWithId(0), backend, downstream); @@ -50,7 +51,9 @@ public void testStartClosesSinksAndTransitionsToSucceeded() { exec.start(); assertTrue("backend sink closed", backend.closed); - assertTrue("downstream sink closed", downstream.closed); + // Downstream is NOT closed by start() — its lifecycle is owned by the walker, + // which still needs to read the buffered batches via outputSource().readResult(). + assertFalse("downstream must not be closed by LocalStageExecution.start()", downstream.closed); assertEquals(StageExecution.State.SUCCEEDED, exec.getState()); } @@ -63,7 +66,13 @@ public void testInputSinkReturnsBackendSinkForAnyChildId() { assertSame(backend, exec.inputSink(42)); } - public void testOutputSourceThrowsUnsupported() { + public void testOutputSourceReturnsDownstreamWhenItImplementsExchangeSource() { + RowProducingSink downstream = new RowProducingSink(); + LocalStageExecution exec = new LocalStageExecution(stageWithId(0), new CapturingSink(), downstream); + assertSame(downstream, exec.outputSource()); + } + + public void testOutputSourceThrowsWhenDownstreamDoesNotImplementExchangeSource() { LocalStageExecution exec = new LocalStageExecution(stageWithId(0), new CapturingSink(), new CapturingSink()); expectThrows(UnsupportedOperationException.class, exec::outputSource); } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java index 8951a901c5f59..3ef18a626809a 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java @@ -100,7 +100,7 @@ public String name() { @Override public ExchangeSinkProvider getExchangeSinkProvider() { // Stub — real implementation provided by DataFusion backend - return bytes -> new ExchangeSink() { + return context -> new ExchangeSink() { @Override public void feed(VectorSchemaRoot batch) {} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/UnifiedQueryService.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/UnifiedQueryService.java deleted file mode 100644 index b68f43d5700bc..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/UnifiedQueryService.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.action; - -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.schema.SchemaPlus; -import org.opensearch.analytics.EngineContext; -import org.opensearch.ppl.compiler.OpenSearchQueryCompiler; -import org.opensearch.ppl.planner.PushDownPlanner; -import org.opensearch.sql.api.UnifiedQueryContext; -import org.opensearch.sql.api.UnifiedQueryPlanner; -import org.opensearch.sql.executor.QueryType; - -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.util.ArrayList; -import java.util.List; - -/** - * Core orchestrator that ties together PushDownPlanner - * and OpenSearchQueryCompiler into a single execution pipeline. - * - *

    Pipeline: PPL text → RelNode → push-down optimization → compile → execute → response. - */ -public class UnifiedQueryService { - - private static final String DEFAULT_CATALOG = "opensearch"; - - private final PushDownPlanner pushDownPlanner; - private final EngineContext engineContext; - - public UnifiedQueryService(PushDownPlanner pushDownPlanner, EngineContext engineContext) { - this.pushDownPlanner = pushDownPlanner; - this.engineContext = engineContext; - } - - /** - * Executes a PPL query through the full pipeline. - * - * @param pplText the PPL query text - * @return a PPLResponse containing column names and result rows - */ - public PPLResponse execute(String pplText) { - SchemaPlus schemaPlus = engineContext.getSchema(); - - UnifiedQueryContext context = UnifiedQueryContext.builder() - .language(QueryType.PPL) - .catalog(DEFAULT_CATALOG, schemaPlus) - .defaultNamespace(DEFAULT_CATALOG) - .build(); - - try { - UnifiedQueryPlanner planner = new UnifiedQueryPlanner(context); - RelNode logicalPlan = planner.plan(pplText); - RelNode mixedPlan = pushDownPlanner.plan(logicalPlan); - - PreparedStatement statement = compileAndPrepare(context, mixedPlan); - try (statement) { - ResultSet rs = statement.executeQuery(); - - ResultSetMetaData metaData = rs.getMetaData(); - int columnCount = metaData.getColumnCount(); - List columns = new ArrayList<>(); - for (int i = 1; i <= columnCount; i++) { - columns.add(metaData.getColumnName(i)); - } - - List rows = new ArrayList<>(); - while (rs.next()) { - Object[] row = new Object[columnCount]; - for (int i = 1; i <= columnCount; i++) { - row[i - 1] = rs.getObject(i); - } - rows.add(row); - } - - return new PPLResponse(columns, rows); - } - } catch (Exception e) { - if (e instanceof RuntimeException) { - throw (RuntimeException) e; - } - throw new RuntimeException("Failed to execute PPL query: " + e.getMessage(), e); - } finally { - try { - context.close(); - } catch (Exception ignored) { - // best-effort cleanup - } - } - } - - /** - * Compiles the mixed plan into a PreparedStatement. Protected for testability. - */ - protected PreparedStatement compileAndPrepare(UnifiedQueryContext context, RelNode mixedPlan) throws Exception { - OpenSearchQueryCompiler compiler = new OpenSearchQueryCompiler(context); - return compiler.compile(mixedPlan); - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/UnifiedQueryServiceTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/UnifiedQueryServiceTests.java deleted file mode 100644 index 9ae1578e084e5..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/UnifiedQueryServiceTests.java +++ /dev/null @@ -1,364 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.action; - -import org.apache.calcite.jdbc.CalciteSchema; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.impl.AbstractTable; -import org.apache.calcite.sql.SqlOperatorTable; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.type.SqlTypeName; -import org.opensearch.Version; -import org.opensearch.analytics.EngineContext; -import org.opensearch.cluster.ClusterName; -import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.metadata.MappingMetadata; -import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.ppl.planner.PushDownPlanner; -import org.opensearch.sql.api.UnifiedQueryContext; -import org.opensearch.test.OpenSearchTestCase; - -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Unit tests for {@link UnifiedQueryService}. - */ -public class UnifiedQueryServiceTests extends OpenSearchTestCase { - - private PushDownPlanner mockPlanner; - private RelNode mockLogicalPlan; - private RelNode mockMixedPlan; - private EngineContext engineContext; - - @Override - public void setUp() throws Exception { - super.setUp(); - mockPlanner = mock(PushDownPlanner.class); - mockLogicalPlan = mock(RelNode.class); - mockMixedPlan = mock(RelNode.class); - engineContext = buildTestEngineContext(); - - when(mockPlanner.plan(any(RelNode.class))).thenReturn(mockMixedPlan); - } - - /** - * Test full pipeline: PPL → RelNode → optimize → compile → execute → response. - */ - public void testFullPipelineReturnsCorrectResponse() throws Exception { - PreparedStatement mockStatement = createMockStatement( - new String[] { "host", "status" }, - new Object[][] { { "server-1", 200 }, { "server-2", 404 } } - ); - - UnifiedQueryService service = createTestService(mockStatement); - PPLResponse response = service.execute("source=logs"); - - assertEquals(2, response.getColumns().size()); - assertEquals("host", response.getColumns().get(0)); - assertEquals("status", response.getColumns().get(1)); - assertEquals(2, response.getRows().size()); - assertArrayEquals(new Object[] { "server-1", 200 }, response.getRows().get(0)); - assertArrayEquals(new Object[] { "server-2", 404 }, response.getRows().get(1)); - - verify(mockPlanner).plan(any(RelNode.class)); - } - - /** - * Test that results are correctly extracted from a mock ResultSet with various data types. - */ - public void testResultExtractionWithVariousDataTypes() throws Exception { - PreparedStatement mockStatement = createMockStatement( - new String[] { "name", "value", "active" }, - new Object[][] { { "test", 3.14, true } } - ); - - UnifiedQueryService service = createTestService(mockStatement); - PPLResponse response = service.execute("source=data"); - - assertEquals(3, response.getColumns().size()); - assertEquals(1, response.getRows().size()); - assertArrayEquals(new Object[] { "test", 3.14, true }, response.getRows().get(0)); - } - - /** - * Test resource cleanup on success path: statement is closed via try-with-resources. - */ - public void testResourceCleanupOnSuccess() throws Exception { - PreparedStatement mockStatement = createMockStatement(new String[] { "col" }, new Object[0][]); - AtomicBoolean contextClosed = new AtomicBoolean(false); - - UnifiedQueryService service = createTestServiceWithContextTracking(mockStatement, contextClosed); - service.execute("source=test"); - - verify(mockStatement).close(); - assertTrue("UnifiedQueryContext should be closed on success", contextClosed.get()); - } - - /** - * Test resource cleanup on failure path: context is closed even when exception thrown. - */ - public void testResourceCleanupOnFailure() throws Exception { - PreparedStatement mockStatement = mock(PreparedStatement.class); - when(mockStatement.executeQuery()).thenThrow(new SQLException("execution failed")); - AtomicBoolean contextClosed = new AtomicBoolean(false); - - UnifiedQueryService service = createTestServiceWithContextTracking(mockStatement, contextClosed); - - expectThrows(RuntimeException.class, () -> service.execute("source=test")); - verify(mockStatement).close(); - assertTrue("UnifiedQueryContext should be closed on failure", contextClosed.get()); - } - - /** - * Test empty result set returns response with columns but no rows. - */ - public void testEmptyResultSet() throws Exception { - PreparedStatement mockStatement = createMockStatement(new String[] { "a", "b" }, new Object[0][]); - - UnifiedQueryService service = createTestService(mockStatement); - PPLResponse response = service.execute("source=empty"); - - assertEquals(2, response.getColumns().size()); - assertTrue(response.getRows().isEmpty()); - } - - // --- helpers --- - - /** - * Creates a mock PreparedStatement that returns a ResultSet with the given columns and rows. - */ - private PreparedStatement createMockStatement(String[] columnNames, Object[][] rowData) throws Exception { - PreparedStatement mockStatement = mock(PreparedStatement.class); - ResultSet mockRs = mock(ResultSet.class); - ResultSetMetaData mockMetaData = mock(ResultSetMetaData.class); - - when(mockStatement.executeQuery()).thenReturn(mockRs); - when(mockRs.getMetaData()).thenReturn(mockMetaData); - when(mockMetaData.getColumnCount()).thenReturn(columnNames.length); - for (int i = 0; i < columnNames.length; i++) { - when(mockMetaData.getColumnName(i + 1)).thenReturn(columnNames[i]); - } - - // Set up rs.next() to return true for each row, then false - Boolean[] nextResults = new Boolean[rowData.length + 1]; - for (int i = 0; i < rowData.length; i++) { - nextResults[i] = true; - } - nextResults[rowData.length] = false; - if (nextResults.length == 1) { - when(mockRs.next()).thenReturn(false); - } else { - Boolean first = nextResults[0]; - Boolean[] rest = new Boolean[nextResults.length - 1]; - System.arraycopy(nextResults, 1, rest, 0, rest.length); - when(mockRs.next()).thenReturn(first, rest); - } - - // Set up rs.getObject() for each column across rows - for (int col = 0; col < columnNames.length; col++) { - if (rowData.length == 0) continue; - if (rowData.length == 1) { - when(mockRs.getObject(col + 1)).thenReturn(rowData[0][col]); - } else { - Object first = rowData[0][col]; - Object[] rest = new Object[rowData.length - 1]; - for (int row = 1; row < rowData.length; row++) { - rest[row - 1] = rowData[row][col]; - } - when(mockRs.getObject(col + 1)).thenReturn(first, rest); - } - } - - return mockStatement; - } - - private UnifiedQueryService createTestService(PreparedStatement mockStatement) { - return new UnifiedQueryService(mockPlanner, engineContext) { - @Override - protected PreparedStatement compileAndPrepare(UnifiedQueryContext context, RelNode mixedPlan) { - return mockStatement; - } - }; - } - - private UnifiedQueryService createTestServiceWithContextTracking(PreparedStatement mockStatement, AtomicBoolean contextClosed) { - return new UnifiedQueryService(mockPlanner, engineContext) { - @Override - protected PreparedStatement compileAndPrepare(UnifiedQueryContext context, RelNode mixedPlan) { - return mockStatement; - } - - @Override - public PPLResponse execute(String pplText) { - // Replicate the real execute logic but track context cleanup - RelNode mixed = mockPlanner.plan(mockLogicalPlan); - - try { - try (PreparedStatement statement = mockStatement) { - ResultSet rs = statement.executeQuery(); - ResultSetMetaData metaData = rs.getMetaData(); - int columnCount = metaData.getColumnCount(); - List columns = new ArrayList<>(); - for (int i = 1; i <= columnCount; i++) { - columns.add(metaData.getColumnName(i)); - } - List rows = new ArrayList<>(); - while (rs.next()) { - Object[] row = new Object[columnCount]; - for (int i = 1; i <= columnCount; i++) { - row[i - 1] = rs.getObject(i); - } - rows.add(row); - } - return new PPLResponse(columns, rows); - } - } catch (Exception e) { - if (e instanceof RuntimeException) throw (RuntimeException) e; - throw new RuntimeException(e.getMessage(), e); - } finally { - contextClosed.set(true); - } - } - }; - } - - /** - * Builds a test EngineContext with schema derived from a test ClusterState. - */ - @SuppressWarnings("unchecked") - private EngineContext buildTestEngineContext() { - ClusterState clusterState = buildClusterState(); - SchemaPlus schema = buildSchemaFromClusterState(clusterState); - return new EngineContext() { - @Override - public SchemaPlus getSchema() { - return schema; - } - - @Override - public SqlOperatorTable operatorTable() { - return SqlStdOperatorTable.instance(); - } - }; - } - - @SuppressWarnings("unchecked") - private SchemaPlus buildSchemaFromClusterState(ClusterState state) { - CalciteSchema rootSchema = CalciteSchema.createRootSchema(true); - SchemaPlus schemaPlus = rootSchema.plus(); - for (Map.Entry entry : state.metadata().indices().entrySet()) { - String indexName = entry.getKey(); - MappingMetadata mapping = entry.getValue().mapping(); - if (mapping == null) continue; - Map properties = (Map) mapping.sourceAsMap().get("properties"); - if (properties == null) continue; - schemaPlus.add(indexName, new AbstractTable() { - @Override - public RelDataType getRowType(RelDataTypeFactory typeFactory) { - RelDataTypeFactory.Builder builder = typeFactory.builder(); - for (Map.Entry f : properties.entrySet()) { - Map fp = (Map) f.getValue(); - String ft = (String) fp.get("type"); - if (ft == null || "nested".equals(ft) || "object".equals(ft)) continue; - SqlTypeName sqlType; - switch (ft) { - case "keyword": - case "text": - case "ip": - sqlType = SqlTypeName.VARCHAR; - break; - case "long": - sqlType = SqlTypeName.BIGINT; - break; - case "integer": - sqlType = SqlTypeName.INTEGER; - break; - case "double": - sqlType = SqlTypeName.DOUBLE; - break; - case "float": - sqlType = SqlTypeName.FLOAT; - break; - case "boolean": - sqlType = SqlTypeName.BOOLEAN; - break; - case "date": - sqlType = SqlTypeName.TIMESTAMP; - break; - default: - sqlType = SqlTypeName.VARCHAR; - break; - } - builder.add(f.getKey(), typeFactory.createTypeWithNullability(typeFactory.createSqlType(sqlType), true)); - } - return builder.build(); - } - }); - } - return schemaPlus; - } - - private ClusterState buildClusterState() { - try { - IndexMetadata logsIndex = IndexMetadata.builder("logs") - .settings(settings(Version.CURRENT)) - .numberOfShards(1) - .numberOfReplicas(0) - .putMapping("{\"properties\":{\"host\":{\"type\":\"keyword\"},\"status\":{\"type\":\"integer\"}}}") - .build(); - - IndexMetadata dataIndex = IndexMetadata.builder("data") - .settings(settings(Version.CURRENT)) - .numberOfShards(1) - .numberOfReplicas(0) - .putMapping( - "{\"properties\":{\"name\":{\"type\":\"keyword\"},\"value\":{\"type\":\"double\"},\"active\":{\"type\":\"boolean\"}}}" - ) - .build(); - - IndexMetadata emptyIndex = IndexMetadata.builder("empty") - .settings(settings(Version.CURRENT)) - .numberOfShards(1) - .numberOfReplicas(0) - .putMapping("{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}}}") - .build(); - - IndexMetadata testIndex = IndexMetadata.builder("test") - .settings(settings(Version.CURRENT)) - .numberOfShards(1) - .numberOfReplicas(0) - .putMapping("{\"properties\":{\"col\":{\"type\":\"keyword\"}}}") - .build(); - - return ClusterState.builder(new ClusterName("test")) - .metadata( - Metadata.builder().put(logsIndex, false).put(dataIndex, false).put(emptyIndex, false).put(testIndex, false).build() - ) - .build(); - } catch (Exception e) { - throw new RuntimeException("Failed to build test ClusterState", e); - } - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/compiler/OpenSearchQueryCompiler.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/compiler/OpenSearchQueryCompiler.java deleted file mode 100644 index 7a5a590f03525..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/compiler/OpenSearchQueryCompiler.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.compiler; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.interpreter.Bindables; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.ConventionTraitDef; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.plan.volcano.VolcanoPlanner; -import org.apache.calcite.rel.RelCollationTraitDef; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.calcite.rel.logical.LogicalAggregate; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rel.logical.LogicalSort; -import org.apache.calcite.rel.logical.LogicalTableScan; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.tools.RelRunner; -import org.opensearch.ppl.planner.rel.OpenSearchBoundaryTableScan; -import org.opensearch.sql.api.UnifiedQueryContext; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.util.ArrayList; -import java.util.List; - -/** - * Compiles Calcite {@link RelNode} plans into executable {@link PreparedStatement}s. - * - *

    Rebuilds the plan tree in a fresh {@link RelOptCluster} with - * {@link Convention#NONE} traits before calling {@code prepareStatement()}. - * This is necessary because the plan from {@code PushDownPlanner} uses a - * planner that already has nodes registered, and re-registering causes - * assertions in Calcite's Volcano planner. - */ -public class OpenSearchQueryCompiler { - - private final UnifiedQueryContext context; - - public OpenSearchQueryCompiler(UnifiedQueryContext context) { - this.context = context; - } - - /** - * Compiles a plan into an executable {@link PreparedStatement}. - */ - public PreparedStatement compile(RelNode plan) { - if (plan == null) { - throw new IllegalArgumentException("RelNode plan must not be null"); - } - try { - RelNode detached = detachFromPlanner(plan); - Connection connection = context.getPlanContext().connection; - RelRunner runner = connection.unwrap(RelRunner.class); - return runner.prepareStatement(detached); - } catch (Exception e) { - throw new IllegalStateException("Failed to compile logical plan", e); - } - } - - /** - * Rebuilds the plan tree in a fresh {@link RelOptCluster} with - * {@link Convention#NONE} traits and a fully-configured {@link VolcanoPlanner}. - */ - private static RelNode detachFromPlanner(RelNode root) { - VolcanoPlanner freshPlanner = new VolcanoPlanner(); - freshPlanner.addRelTraitDef(ConventionTraitDef.INSTANCE); - freshPlanner.addRelTraitDef(RelCollationTraitDef.INSTANCE); - RelOptUtil.registerDefaultRules(freshPlanner, false, false); - freshPlanner.addRule(BoundaryToEnumerableRule.INSTANCE); - - RexBuilder rexBuilder = root.getCluster().getRexBuilder(); - RelOptCluster freshCluster = RelOptCluster.create(freshPlanner, rexBuilder); - freshCluster.setMetadataProvider(root.getCluster().getMetadataProvider()); - freshCluster.setMetadataQuerySupplier(root.getCluster().getMetadataQuerySupplier()); - - return rebuild(root, freshCluster); - } - - /** - * Recursively rebuilds a RelNode tree in a fresh cluster with - * {@link Convention#NONE} traits. Uses {@code copy()} for generic - * handling of all RelNode types instead of per-type factory methods. - */ - private static RelNode rebuild(RelNode node, RelOptCluster freshCluster) { - // Leaf: OpenSearchBoundaryTableScan — rebuild with NONE convention - if (node instanceof OpenSearchBoundaryTableScan) { - OpenSearchBoundaryTableScan boundary = (OpenSearchBoundaryTableScan) node; - RelTraitSet noneTraits = freshCluster.traitSetOf(Convention.NONE); - return new OpenSearchBoundaryTableScan( - freshCluster, - noneTraits, - boundary.getTable(), - boundary.getLogicalFragment(), - boundary.getEngineExecutor() - ); - } - - // Leaf: LogicalTableScan → BindableTableScan when possible - if (node instanceof LogicalTableScan) { - RelOptTable table = node.getTable(); - if (Bindables.BindableTableScan.canHandle(table)) { - return Bindables.BindableTableScan.create(freshCluster, table); - } - return LogicalTableScan.create(freshCluster, table, List.of()); - } - - // Non-leaf: rebuild children, then reconstruct node using factory methods - // Factory methods derive cluster from inputs, avoiding "belongs to a different planner" errors - List inputs = node.getInputs(); - if (inputs.isEmpty()) { - return node.copy(node.getTraitSet().replace(Convention.NONE), inputs); - } - - List newInputs = new ArrayList<>(inputs.size()); - for (RelNode input : inputs) { - newInputs.add(rebuild(input, freshCluster)); - } - - if (node instanceof LogicalFilter) { - return LogicalFilter.create(newInputs.get(0), ((LogicalFilter) node).getCondition()); - } - if (node instanceof LogicalProject) { - LogicalProject p = (LogicalProject) node; - return LogicalProject.create(newInputs.get(0), p.getHints(), p.getProjects(), p.getRowType()); - } - if (node instanceof LogicalAggregate) { - LogicalAggregate a = (LogicalAggregate) node; - return LogicalAggregate.create(newInputs.get(0), a.getHints(), a.getGroupSet(), a.getGroupSets(), a.getAggCallList()); - } - if (node instanceof LogicalSort) { - LogicalSort s = (LogicalSort) node; - return LogicalSort.create(newInputs.get(0), s.getCollation(), s.offset, s.fetch); - } - return node.copy(node.getTraitSet().replace(Convention.NONE), newInputs); - } - - /** - * Converter rule: {@link OpenSearchBoundaryTableScan} from - * {@link Convention#NONE} to {@link EnumerableConvention}. - */ - private static class BoundaryToEnumerableRule extends ConverterRule { - - static final Config DEFAULT_CONFIG = Config.INSTANCE.withConversion( - OpenSearchBoundaryTableScan.class, - Convention.NONE, - EnumerableConvention.INSTANCE, - "BoundaryToEnumerableRule" - ).withRuleFactory(BoundaryToEnumerableRule::new); - - static final BoundaryToEnumerableRule INSTANCE = new BoundaryToEnumerableRule(DEFAULT_CONFIG); - - protected BoundaryToEnumerableRule(Config config) { - super(config); - } - - @Override - public RelNode convert(RelNode rel) { - OpenSearchBoundaryTableScan scan = (OpenSearchBoundaryTableScan) rel; - RelTraitSet newTraits = scan.getTraitSet().replace(EnumerableConvention.INSTANCE); - return new OpenSearchBoundaryTableScan( - scan.getCluster(), - newTraits, - scan.getTable(), - scan.getLogicalFragment(), - scan.getEngineExecutor() - ); - } - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlanner.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlanner.java deleted file mode 100644 index 59a3edef8f36c..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlanner.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.planner; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.plan.hep.HepPlanner; -import org.apache.calcite.plan.hep.HepProgramBuilder; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.RelShuttleImpl; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.logical.LogicalTableScan; -import org.apache.calcite.sql.SqlOperatorTable; -import org.opensearch.analytics.exec.QueryPlanExecutor; -import org.opensearch.ppl.planner.rel.OpenSearchBoundaryTableScan; -import org.opensearch.ppl.planner.rules.AbsorbAggregateRule; -import org.opensearch.ppl.planner.rules.AbsorbFilterRule; -import org.opensearch.ppl.planner.rules.AbsorbProjectRule; -import org.opensearch.ppl.planner.rules.AbsorbSortRule; - -/** - * Produces a mixed plan where supported operators are absorbed into an - * {@link OpenSearchBoundaryTableScan} and unsupported operators remain as - * Calcite logical nodes. - * - *

    Phase 1 (BoundaryTableScanShuttle): Replaces every - * {@code LogicalTableScan} with an {@code OpenSearchBoundaryTableScan} - * carrying the scan as its initial logical fragment. - * - *

    Phase 2 (HepPlanner): Runs absorb rules to push supported - * operators into the boundary node's logical fragment. Unsupported operators - * (e.g., projects containing functions not in the back-end's - * {@link SqlOperatorTable}) remain above the boundary node and execute - * in-process via Janino bytecode. - */ -public class PushDownPlanner { - - private final SqlOperatorTable operatorTable; - private final QueryPlanExecutor> planExecutor; - - /** - * @param operatorTable supported functions from the back-end engines - * @param planExecutor engine executor passed to boundary nodes for bind-time execution - */ - public PushDownPlanner(SqlOperatorTable operatorTable, QueryPlanExecutor> planExecutor) { - this.operatorTable = operatorTable; - this.planExecutor = planExecutor; - } - - /** - * Optimizes the input RelNode by pushing supported operators into a boundary node. - * - *

      - *
    1. Phase 1: Replace LogicalTableScan → OpenSearchBoundaryTableScan
      2. - *
    2. Phase 2: HepPlanner absorbs supported filter/project/aggregate/sort into boundary node
      3. - *
    - * - * @param input the logical RelNode produced by PPLToRelNodeService - * @return a mixed plan with boundary nodes carrying the OPENSEARCH convention - */ - public RelNode plan(RelNode input) { - // Phase 1: Replace scans with boundary nodes - RelNode withBoundary = input.accept(new BoundaryTableScanShuttle(planExecutor)); - - // Phase 2: Absorb supported operators into boundary nodes - HepProgramBuilder programBuilder = new HepProgramBuilder(); - programBuilder.addRuleInstance(AbsorbFilterRule.create(operatorTable)); - programBuilder.addRuleInstance(AbsorbProjectRule.create(operatorTable)); - programBuilder.addRuleInstance(AbsorbAggregateRule.create(operatorTable)); - programBuilder.addRuleInstance(AbsorbSortRule.create()); - - HepPlanner hepPlanner = new HepPlanner(programBuilder.build()); - hepPlanner.setRoot(withBoundary); - return hepPlanner.findBestExp(); - } - - /** - * Shuttle that replaces every {@link LogicalTableScan} with an - * {@link OpenSearchBoundaryTableScan} carrying the scan as its initial - * logical fragment. - */ - private static class BoundaryTableScanShuttle extends RelShuttleImpl { - private final QueryPlanExecutor> planExecutor; - - BoundaryTableScanShuttle(QueryPlanExecutor> planExecutor) { - this.planExecutor = planExecutor; - } - - @Override - public RelNode visit(TableScan scan) { - if (scan instanceof LogicalTableScan) { - RelTraitSet traitSet = scan.getCluster().traitSetOf(EnumerableConvention.INSTANCE); - return new OpenSearchBoundaryTableScan(scan.getCluster(), traitSet, scan.getTable(), scan, planExecutor); - } - return scan; - } - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlannerTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlannerTests.java deleted file mode 100644 index a73456e5afa39..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/PushDownPlannerTests.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.planner; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.config.CalciteConnectionConfig; -import org.apache.calcite.config.CalciteConnectionConfigImpl; -import org.apache.calcite.jdbc.CalciteSchema; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.plan.ConventionTraitDef; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.volcano.VolcanoPlanner; -import org.apache.calcite.prepare.CalciteCatalogReader; -import org.apache.calcite.rel.RelCollationTraitDef; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rel.logical.LogicalTableScan; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.impl.AbstractTable; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlOperatorTable; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql.util.ListSqlOperatorTable; -import org.opensearch.analytics.exec.QueryPlanExecutor; -import org.opensearch.ppl.planner.rel.OpenSearchBoundaryTableScan; -import org.opensearch.test.OpenSearchTestCase; - -import java.util.Collections; -import java.util.List; -import java.util.Properties; - -/** - * Tests for {@link PushDownPlanner}. - */ -public class PushDownPlannerTests extends OpenSearchTestCase { - - private RelOptCluster cluster; - private RexBuilder rexBuilder; - private RelOptTable table; - private QueryPlanExecutor> planExecutor; - private JavaTypeFactoryImpl typeFactory; - - @Override - public void setUp() throws Exception { - super.setUp(); - - typeFactory = new JavaTypeFactoryImpl(); - rexBuilder = new RexBuilder(typeFactory); - - VolcanoPlanner volcanoPlanner = new VolcanoPlanner(); - volcanoPlanner.addRelTraitDef(ConventionTraitDef.INSTANCE); - volcanoPlanner.addRelTraitDef(RelCollationTraitDef.INSTANCE); - cluster = RelOptCluster.create(volcanoPlanner, rexBuilder); - - CalciteSchema rootSchema = CalciteSchema.createRootSchema(true); - SchemaPlus schemaPlus = rootSchema.plus(); - schemaPlus.add("test_table", new AbstractTable() { - @Override - public RelDataType getRowType(RelDataTypeFactory tf) { - return tf.builder() - .add("id", tf.createSqlType(SqlTypeName.INTEGER)) - .add("name", tf.createSqlType(SqlTypeName.VARCHAR)) - .add("value", tf.createSqlType(SqlTypeName.DOUBLE)) - .build(); - } - }); - - Properties props = new Properties(); - CalciteConnectionConfig config = new CalciteConnectionConfigImpl(props); - CalciteCatalogReader catalogReader = new CalciteCatalogReader(rootSchema, Collections.singletonList(""), typeFactory, config); - table = catalogReader.getTable(List.of("test_table")); - assertNotNull("Table should be found in catalog", table); - - planExecutor = (fragment, ctx, l) -> l.onResponse(Collections.emptyList()); - } - - /** - * Test scan-only query: the boundary node should absorb just the scan. - */ - public void testScanOnlyQueryProducesBoundaryNodeWithScanFragment() { - SqlOperatorTable operatorTable = SqlStdOperatorTable.instance(); - PushDownPlanner planner = new PushDownPlanner(operatorTable, planExecutor); - - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - - RelNode result = planner.plan(scan); - - assertTrue("Result should be an OpenSearchBoundaryTableScan", result instanceof OpenSearchBoundaryTableScan); - OpenSearchBoundaryTableScan boundary = (OpenSearchBoundaryTableScan) result; - assertEquals("Convention should be BINDABLE", EnumerableConvention.INSTANCE, boundary.getConvention()); - - RelNode fragment = boundary.getLogicalFragment(); - assertTrue("Logical fragment should be a LogicalTableScan", fragment instanceof LogicalTableScan); - } - - /** - * Test scan+filter query: the boundary node should absorb both scan and filter. - */ - public void testScanFilterQueryProducesBoundaryNodeWithFilterFragment() { - SqlOperatorTable operatorTable = SqlStdOperatorTable.instance(); - PushDownPlanner planner = new PushDownPlanner(operatorTable, planExecutor); - - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - - // Build: value > 10 (supported condition) - RexNode valueRef = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.DOUBLE), 2); - RexNode literal10 = rexBuilder.makeLiteral(10.0, typeFactory.createSqlType(SqlTypeName.DOUBLE), true); - RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN, valueRef, literal10); - LogicalFilter filter = LogicalFilter.create(scan, condition); - - RelNode result = planner.plan(filter); - - assertTrue("Result should be an OpenSearchBoundaryTableScan", result instanceof OpenSearchBoundaryTableScan); - OpenSearchBoundaryTableScan boundary = (OpenSearchBoundaryTableScan) result; - assertEquals("Convention should be BINDABLE", EnumerableConvention.INSTANCE, boundary.getConvention()); - - RelNode fragment = boundary.getLogicalFragment(); - assertTrue("Logical fragment should be a LogicalFilter (scan+filter absorbed)", fragment instanceof LogicalFilter); - LogicalFilter absorbedFilter = (LogicalFilter) fragment; - assertTrue("Absorbed filter's input should be a LogicalTableScan", absorbedFilter.getInput() instanceof LogicalTableScan); - } - - /** - * Test mixed query: scan+filter are absorbed, unsupported project stays above. - * - * Uses a restricted operator table that does NOT include PLUS, so the project - * containing value + 1 cannot be absorbed and remains above the boundary. - */ - public void testMixedQueryKeepsUnsupportedProjectAboveBoundary() { - // Restricted operator table: supports comparison but NOT PLUS - List ops = List.of( - SqlStdOperatorTable.EQUALS, - SqlStdOperatorTable.GREATER_THAN, - SqlStdOperatorTable.LESS_THAN, - SqlStdOperatorTable.AND, - SqlStdOperatorTable.OR - ); - SqlOperatorTable operatorTable = new ListSqlOperatorTable(ops); - PushDownPlanner planner = new PushDownPlanner(operatorTable, planExecutor); - - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - - // Build filter: value > 10 (supported) - RexNode valueRef = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.DOUBLE), 2); - RexNode literal10 = rexBuilder.makeLiteral(10.0, typeFactory.createSqlType(SqlTypeName.DOUBLE), true); - RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN, valueRef, literal10); - LogicalFilter filter = LogicalFilter.create(scan, condition); - - // Build project: value + 1 (PLUS is unsupported in restricted table) - RexNode filterValueRef = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.DOUBLE), 2); - RexNode literal1 = rexBuilder.makeLiteral(1.0, typeFactory.createSqlType(SqlTypeName.DOUBLE), true); - RexNode plusExpr = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, filterValueRef, literal1); - LogicalProject project = LogicalProject.create(filter, List.of(), List.of(plusExpr), List.of("result")); - - RelNode result = planner.plan(project); - - // The top-level node should NOT be a boundary node — the project stays above - assertFalse("Top-level result should NOT be an OpenSearchBoundaryTableScan", result instanceof OpenSearchBoundaryTableScan); - - // Find the boundary node in the tree (should be the input of the project) - RelNode child = result.getInput(0); - assertTrue("Child of the project should be an OpenSearchBoundaryTableScan", child instanceof OpenSearchBoundaryTableScan); - - OpenSearchBoundaryTableScan boundary = (OpenSearchBoundaryTableScan) child; - RelNode fragment = boundary.getLogicalFragment(); - assertTrue("Boundary's logical fragment should be a LogicalFilter (scan+filter absorbed)", fragment instanceof LogicalFilter); - LogicalFilter absorbedFilter = (LogicalFilter) fragment; - assertTrue("Absorbed filter's input should be a LogicalTableScan", absorbedFilter.getInput() instanceof LogicalTableScan); - } - -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScan.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScan.java deleted file mode 100644 index 14db142f2b4b7..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScan.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.planner.rel; - -import org.apache.calcite.DataContext; -import org.apache.calcite.adapter.enumerable.EnumerableRel; -import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; -import org.apache.calcite.adapter.enumerable.PhysType; -import org.apache.calcite.adapter.enumerable.PhysTypeImpl; -import org.apache.calcite.linq4j.Enumerable; -import org.apache.calcite.linq4j.Linq4j; -import org.apache.calcite.linq4j.tree.Blocks; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Expressions; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.analytics.exec.QueryPlanExecutor; - -import java.util.List; - -/** - * Boundary node that absorbs supported logical operators into a single scan. - * - *

    Extends {@link TableScan} (NOT {@code LogicalTableScan}) so that - * {@code UnifiedQueryCompiler}'s inner RelShuttle — which only matches - * {@code LogicalTableScan} — skips this node. Implements {@link EnumerableRel} - * so Calcite's Janino code-generation path calls {@link #execute()} at - * execution time via the stash pattern. - * - *

    The {@code logicalFragment} field holds the absorbed logical subtree - * (e.g., {@code LogicalFilter → LogicalTableScan}). At execution time, - * {@code execute()} passes the fragment to the {@link QueryPlanExecutor}, which - * returns the result rows. - */ -public class OpenSearchBoundaryTableScan extends TableScan implements EnumerableRel { - - private final RelNode logicalFragment; - @SuppressWarnings("rawtypes") - private final QueryPlanExecutor planExecutor; - - @SuppressWarnings("rawtypes") - public OpenSearchBoundaryTableScan( - RelOptCluster cluster, - RelTraitSet traitSet, - RelOptTable table, - RelNode logicalFragment, - QueryPlanExecutor planExecutor - ) { - super(cluster, traitSet, List.of(), table); - this.logicalFragment = logicalFragment; - this.planExecutor = planExecutor; - } - - /** Returns the absorbed logical subtree passed to the engine at execution time. */ - public RelNode getLogicalFragment() { - return logicalFragment; - } - - /** - * Derives the row type from the logical fragment rather than the table. - * This ensures that after absorbing operators like aggregate or project, - * the boundary node's row type matches the absorbed operator's output type. - */ - @Override - public RelDataType deriveRowType() { - return logicalFragment.getRowType(); - } - - /** Returns the engine executor used for execution. */ - @SuppressWarnings("rawtypes") - public QueryPlanExecutor getEngineExecutor() { - return planExecutor; - } - - /** - * Implements the EnumerableRel interface using the stash pattern. - * Generated Janino code calls {@link #execute()} on the stashed reference. - */ - @Override - public Result implement(EnumerableRelImplementor implementor, Prefer pref) { - PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - - Expression stashedRef = implementor.stash(this, OpenSearchBoundaryTableScan.class); - return implementor.result(physType, Blocks.toBlock(Expressions.call(stashedRef, "execute"))); - } - - /** - * Called by generated Janino code at execution time. - * Delegates to {@link #bind(DataContext)} with a null DataContext. - * - * @return result rows as an Enumerable - */ - public Enumerable execute() { - return bind(null); - } - - /** - * Executes the logical fragment via the {@link QueryPlanExecutor}. - * - * @param dataContext the Calcite data context (may be null) - * @return result rows as an Enumerable - */ - @SuppressWarnings("unchecked") - public Enumerable bind(DataContext dataContext) { - try { - // Calcite's bind() is synchronous (driven by the EnumerableInterpreter), so we - // block here on the async planExecutor until the engine completes. - org.opensearch.action.support.PlainActionFuture> future = - new org.opensearch.action.support.PlainActionFuture<>(); - planExecutor.execute(logicalFragment, dataContext, future); - Iterable result = future.actionGet(); - return Linq4j.asEnumerable(result); - } catch (Exception e) { - throw new RuntimeException( - "Engine execution failed for table [" - + getTable().getQualifiedName() - + "] with logical fragment: " - + logicalFragment.explain(), - e - ); - } - } - - @Override - public RelNode copy(RelTraitSet traitSet, List inputs) { - return new OpenSearchBoundaryTableScan(getCluster(), traitSet, getTable(), logicalFragment, planExecutor); - } - - @Override - public org.apache.calcite.rel.RelWriter explainTerms(org.apache.calcite.rel.RelWriter pw) { - return super.explainTerms(pw).item("fragment", logicalFragment.explain()); - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScanTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScanTests.java deleted file mode 100644 index b9c152b084d12..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rel/OpenSearchBoundaryTableScanTests.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.planner.rel; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.adapter.enumerable.EnumerableRel; -import org.apache.calcite.config.CalciteConnectionConfig; -import org.apache.calcite.config.CalciteConnectionConfigImpl; -import org.apache.calcite.jdbc.CalciteSchema; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.linq4j.Enumerable; -import org.apache.calcite.linq4j.Linq4j; -import org.apache.calcite.plan.ConventionTraitDef; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.plan.volcano.VolcanoPlanner; -import org.apache.calcite.prepare.CalciteCatalogReader; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.calcite.rel.logical.LogicalTableScan; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.impl.AbstractTable; -import org.apache.calcite.sql.type.SqlTypeName; -import org.opensearch.analytics.exec.QueryPlanExecutor; -import org.opensearch.test.OpenSearchTestCase; - -import java.util.Collections; -import java.util.List; -import java.util.Properties; - -/** - * Tests for {@link OpenSearchBoundaryTableScan}. - */ -public class OpenSearchBoundaryTableScanTests extends OpenSearchTestCase { - - private RelOptCluster cluster; - private RexBuilder rexBuilder; - private RelOptTable table; - - @Override - public void setUp() throws Exception { - super.setUp(); - - JavaTypeFactoryImpl typeFactory = new JavaTypeFactoryImpl(); - rexBuilder = new RexBuilder(typeFactory); - VolcanoPlanner planner = new VolcanoPlanner(); - planner.addRelTraitDef(ConventionTraitDef.INSTANCE); - cluster = RelOptCluster.create(planner, rexBuilder); - - CalciteSchema rootSchema = CalciteSchema.createRootSchema(true); - SchemaPlus schemaPlus = rootSchema.plus(); - schemaPlus.add("test_table", new AbstractTable() { - @Override - public RelDataType getRowType(RelDataTypeFactory tf) { - return tf.builder() - .add("id", tf.createSqlType(SqlTypeName.INTEGER)) - .add("name", tf.createSqlType(SqlTypeName.VARCHAR)) - .add("value", tf.createSqlType(SqlTypeName.DOUBLE)) - .build(); - } - }); - - Properties props = new Properties(); - CalciteConnectionConfig config = new CalciteConnectionConfigImpl(props); - CalciteCatalogReader catalogReader = new CalciteCatalogReader(rootSchema, Collections.singletonList(""), typeFactory, config); - table = catalogReader.getTable(List.of("test_table")); - assertNotNull("Table should be found in catalog", table); - } - - // --- Inheritance tests --- - - public void testExtendsTableScanNotLogicalTableScan() { - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); - - OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); - - assertTrue("Should extend TableScan", TableScan.class.isAssignableFrom(OpenSearchBoundaryTableScan.class)); - assertFalse("Should NOT extend LogicalTableScan", LogicalTableScan.class.isAssignableFrom(OpenSearchBoundaryTableScan.class)); - } - - public void testImplementsEnumerableRel() { - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); - - OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); - - assertTrue("Should implement EnumerableRel", boundary instanceof EnumerableRel); - } - - // --- bind() tests --- - - public void testBindCallsEngineExecutorWithLogicalFragment() { - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - - // Track what the executor receives - final RelNode[] capturedFragment = new RelNode[1]; - final Object[] capturedContext = new Object[1]; - Object[][] rows = { new Object[] { 1, "a", 1.0 } }; - QueryPlanExecutor> executor = (fragment, ctx, l) -> { - capturedFragment[0] = fragment; - capturedContext[0] = ctx; - l.onResponse(Linq4j.asEnumerable(rows)); - }; - - OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); - - Enumerable result = boundary.bind(null); - - assertSame("bind() should pass the logical fragment to the executor", scan, capturedFragment[0]); - assertNull("bind() should pass the DataContext to the executor", capturedContext[0]); - assertNotNull("bind() should return a non-null Enumerable", result); - } - - public void testBindPassesFilterFragmentToExecutor() { - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - RexNode condition = rexBuilder.makeLiteral(true); - LogicalFilter filter = LogicalFilter.create(scan, condition); - RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - - final RelNode[] capturedFragment = new RelNode[1]; - QueryPlanExecutor> executor = (fragment, ctx, l) -> { - capturedFragment[0] = fragment; - l.onResponse(Linq4j.emptyEnumerable()); - }; - - OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, filter, executor); - - boundary.bind(null); - - assertSame("bind() should pass the filter fragment to the executor", filter, capturedFragment[0]); - } - - // --- copy() tests --- - - public void testCopyPreservesLogicalFragment() { - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); - - OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); - - RelNode copied = boundary.copy(traitSet, List.of()); - - assertTrue("copy() should return an OpenSearchBoundaryTableScan", copied instanceof OpenSearchBoundaryTableScan); - OpenSearchBoundaryTableScan copiedBoundary = (OpenSearchBoundaryTableScan) copied; - assertSame("copy() should preserve the logical fragment", scan, copiedBoundary.getLogicalFragment()); - } - - public void testCopyPreservesTable() { - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); - - OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); - - RelNode copied = boundary.copy(traitSet, List.of()); - OpenSearchBoundaryTableScan copiedBoundary = (OpenSearchBoundaryTableScan) copied; - - assertSame("copy() should preserve the table reference", table, copiedBoundary.getTable()); - } - - // --- getLogicalFragment() tests --- - - public void testGetLogicalFragmentReturnsScanSubtree() { - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); - - OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, executor); - - assertSame("getLogicalFragment() should return the absorbed subtree", scan, boundary.getLogicalFragment()); - } - - public void testGetLogicalFragmentReturnsFilterSubtree() { - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - RexNode condition = rexBuilder.makeLiteral(true); - LogicalFilter filter = LogicalFilter.create(scan, condition); - RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - QueryPlanExecutor> executor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); - - OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, filter, executor); - - assertSame("getLogicalFragment() should return the filter subtree", filter, boundary.getLogicalFragment()); - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbAggregateRule.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbAggregateRule.java deleted file mode 100644 index 789822274ccc5..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbAggregateRule.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.planner.rules; - -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.logical.LogicalAggregate; -import org.apache.calcite.sql.SqlOperatorTable; -import org.opensearch.ppl.planner.rel.OpenSearchBoundaryTableScan; - -/** - * Absorbs a {@link LogicalAggregate} (and any intermediate nodes between it - * and the boundary) into an {@link OpenSearchBoundaryTableScan}. - * - *

    Checks that all aggregate functions are supported by the back-end's - * {@link SqlOperatorTable} before absorbing. - */ -public class AbsorbAggregateRule extends RelOptRule { - - private final SqlOperatorTable operatorTable; - - public static AbsorbAggregateRule create(SqlOperatorTable operatorTable) { - return new AbsorbAggregateRule(operatorTable); - } - - private AbsorbAggregateRule(SqlOperatorTable operatorTable) { - super(operand(LogicalAggregate.class, any()), "AbsorbAggregateRule"); - this.operatorTable = operatorTable; - } - - @Override - public void onMatch(RelOptRuleCall call) { - LogicalAggregate aggregate = call.rel(0); - - if (!AbsorbRuleUtils.allAggFunctionsSupported(aggregate.getAggCallList(), operatorTable)) { - return; - } - - OpenSearchBoundaryTableScan boundary = AbsorbRuleUtils.findBoundary(aggregate); - if (boundary == null) { - return; - } - - call.transformTo(AbsorbRuleUtils.absorbIntoBoundary(aggregate, boundary)); - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbFilterRule.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbFilterRule.java deleted file mode 100644 index 1360e6f691f9e..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbFilterRule.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.planner.rules; - -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.calcite.sql.SqlOperatorTable; -import org.opensearch.ppl.planner.rel.OpenSearchBoundaryTableScan; - -/** - * RelOptRule that absorbs a {@link LogicalFilter} into an {@link OpenSearchBoundaryTableScan}. - * - *

    Pattern: {@code LogicalFilter} on top of {@code OpenSearchBoundaryTableScan}. - * - *

    When the rule matches, it checks whether all functions in the filter condition - * are supported by the back-end's {@link SqlOperatorTable}. If supported, the filter - * is absorbed into the boundary node's logical fragment. - * - *

    This is NOT a ConverterRule — it transforms an already-converted boundary node - * by growing its internal logical fragment. - */ -public class AbsorbFilterRule extends RelOptRule { - - private final SqlOperatorTable operatorTable; - - public static AbsorbFilterRule create(SqlOperatorTable operatorTable) { - return new AbsorbFilterRule(operatorTable); - } - - private AbsorbFilterRule(SqlOperatorTable operatorTable) { - super(operand(LogicalFilter.class, operand(OpenSearchBoundaryTableScan.class, none())), "AbsorbFilterRule"); - this.operatorTable = operatorTable; - } - - @Override - public void onMatch(RelOptRuleCall call) { - LogicalFilter filter = call.rel(0); - OpenSearchBoundaryTableScan boundary = call.rel(1); - - if (!AbsorbRuleUtils.allFunctionsSupported(filter.getCondition(), operatorTable)) { - return; - } - - // Wrap the existing logical fragment with the filter to build the new absorbed subtree - LogicalFilter absorbedFilter = filter.copy(filter.getTraitSet(), boundary.getLogicalFragment(), filter.getCondition()); - - // Create a new boundary node with the expanded logical fragment - OpenSearchBoundaryTableScan newBoundary = new OpenSearchBoundaryTableScan( - boundary.getCluster(), - boundary.getTraitSet(), - boundary.getTable(), - absorbedFilter, - boundary.getEngineExecutor() - ); - - call.transformTo(newBoundary); - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbProjectRule.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbProjectRule.java deleted file mode 100644 index fda82ddb90d0e..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbProjectRule.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.planner.rules; - -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.SqlOperatorTable; -import org.opensearch.ppl.planner.rel.OpenSearchBoundaryTableScan; - -/** - * RelOptRule that absorbs a {@link LogicalProject} into an {@link OpenSearchBoundaryTableScan}. - * - *

    Pattern: {@code LogicalProject} on top of {@code OpenSearchBoundaryTableScan}. - * - *

    When the rule matches, it checks whether all functions in the project expressions - * are supported by the back-end's {@link SqlOperatorTable}. If supported, the project - * is absorbed into the boundary node's logical fragment. - * - *

    This is NOT a ConverterRule — it transforms an already-converted boundary node - * by growing its internal logical fragment. - */ -public class AbsorbProjectRule extends RelOptRule { - - private final SqlOperatorTable operatorTable; - - public static AbsorbProjectRule create(SqlOperatorTable operatorTable) { - return new AbsorbProjectRule(operatorTable); - } - - private AbsorbProjectRule(SqlOperatorTable operatorTable) { - super(operand(LogicalProject.class, operand(OpenSearchBoundaryTableScan.class, none())), "AbsorbProjectRule"); - this.operatorTable = operatorTable; - } - - @Override - public void onMatch(RelOptRuleCall call) { - LogicalProject project = call.rel(0); - OpenSearchBoundaryTableScan boundary = call.rel(1); - - // Check that all functions in every project expression are supported - for (RexNode expr : project.getProjects()) { - if (!AbsorbRuleUtils.allFunctionsSupported(expr, operatorTable)) { - return; - } - } - - // Wrap the existing logical fragment with the project to build the new absorbed subtree - LogicalProject absorbedProject = project.copy( - project.getTraitSet(), - boundary.getLogicalFragment(), - project.getProjects(), - project.getRowType() - ); - - // Create a new boundary node with the expanded logical fragment - OpenSearchBoundaryTableScan newBoundary = new OpenSearchBoundaryTableScan( - boundary.getCluster(), - boundary.getTraitSet(), - boundary.getTable(), - absorbedProject, - boundary.getEngineExecutor() - ); - - call.transformTo(newBoundary); - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbRuleUtils.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbRuleUtils.java deleted file mode 100644 index 570f3ba5ae7c4..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbRuleUtils.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.planner.rules; - -import org.apache.calcite.plan.hep.HepRelVertex; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.AggregateCall; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexVisitorImpl; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlOperatorTable; -import org.opensearch.ppl.planner.rel.OpenSearchBoundaryTableScan; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * Shared utilities for absorb rules that need to walk down a subtree - * to find an {@link OpenSearchBoundaryTableScan} and replace it with - * its logical fragment. - */ -final class AbsorbRuleUtils { - - private AbsorbRuleUtils() {} - - /** Unwraps HepRelVertex wrappers that HepPlanner uses internally. */ - static RelNode unwrap(RelNode node) { - if (node instanceof HepRelVertex) { - return ((HepRelVertex) node).getCurrentRel(); - } - return node; - } - - /** - * Walks down single-input chains to find an OpenSearchBoundaryTableScan. - */ - static OpenSearchBoundaryTableScan findBoundary(RelNode node) { - for (RelNode rawInput : node.getInputs()) { - RelNode input = unwrap(rawInput); - if (input instanceof OpenSearchBoundaryTableScan) { - return (OpenSearchBoundaryTableScan) input; - } - if (input.getInputs().size() == 1) { - OpenSearchBoundaryTableScan found = findBoundary(input); - if (found != null) { - return found; - } - } - } - return null; - } - - /** - * Recursively copies the subtree from {@code node} down, replacing any - * boundary node with its logical fragment. - */ - static RelNode replaceWithFragment(RelNode node) { - List newInputs = new ArrayList<>(); - for (RelNode rawInput : node.getInputs()) { - RelNode input = unwrap(rawInput); - if (input instanceof OpenSearchBoundaryTableScan) { - newInputs.add(((OpenSearchBoundaryTableScan) input).getLogicalFragment()); - } else { - newInputs.add(replaceWithFragment(input)); - } - } - return node.copy(node.getTraitSet(), newInputs); - } - - /** - * Absorbs the operator (and all intermediate nodes) into the boundary, - * returning a new boundary node with the expanded fragment. - */ - static OpenSearchBoundaryTableScan absorbIntoBoundary(RelNode operator, OpenSearchBoundaryTableScan boundary) { - RelNode absorbed = replaceWithFragment(operator); - return new OpenSearchBoundaryTableScan( - boundary.getCluster(), - boundary.getTraitSet(), - boundary.getTable(), - absorbed, - boundary.getEngineExecutor() - ); - } - - /** - * Checks whether all functions in a RexNode expression tree are present - * in the given operator table. - */ - static boolean allFunctionsSupported(RexNode expression, SqlOperatorTable operatorTable) { - if (expression == null) return true; - Set supported = new HashSet<>(operatorTable.getOperatorList()); - Boolean result = expression.accept(new RexVisitorImpl(true) { - @Override - public Boolean visitCall(RexCall call) { - if (!supported.contains(call.getOperator())) return false; - for (RexNode operand : call.getOperands()) { - Boolean childResult = operand.accept(this); - if (childResult != null && !childResult) return false; - } - return true; - } - }); - return result == null || result; - } - - /** - * Checks whether all aggregate functions in the given list are present - * in the given operator table. - */ - static boolean allAggFunctionsSupported(List aggCalls, SqlOperatorTable operatorTable) { - if (aggCalls == null || aggCalls.isEmpty()) return true; - Set supported = new HashSet<>(operatorTable.getOperatorList()); - for (AggregateCall aggCall : aggCalls) { - if (!supported.contains(aggCall.getAggregation())) return false; - } - return true; - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbSortRule.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbSortRule.java deleted file mode 100644 index d9ac89e765c70..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/AbsorbSortRule.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.planner.rules; - -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.logical.LogicalSort; -import org.opensearch.ppl.planner.rel.OpenSearchBoundaryTableScan; - -/** - * Absorbs a {@link LogicalSort} (and any intermediate nodes between it - * and the boundary) into an {@link OpenSearchBoundaryTableScan}. - * - *

    Sort collations are field references and directions — no expression-level - * capability checks are needed. Sort always absorbs if a boundary exists. - */ -public class AbsorbSortRule extends RelOptRule { - - public static AbsorbSortRule create() { - return new AbsorbSortRule(); - } - - private AbsorbSortRule() { - super(operand(LogicalSort.class, any()), "AbsorbSortRule"); - } - - @Override - public void onMatch(RelOptRuleCall call) { - LogicalSort sort = call.rel(0); - - OpenSearchBoundaryTableScan boundary = AbsorbRuleUtils.findBoundary(sort); - if (boundary == null) { - return; - } - - call.transformTo(AbsorbRuleUtils.absorbIntoBoundary(sort, boundary)); - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/BoundaryTableScanRule.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/BoundaryTableScanRule.java deleted file mode 100644 index faf77480e0644..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/BoundaryTableScanRule.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.calcite.rel.logical.LogicalTableScan; -import org.opensearch.analytics.exec.QueryPlanExecutor; -import org.opensearch.ppl.planner.rel.OpenSearchBoundaryTableScan; - -/** - * ConverterRule: LogicalTableScan (Convention.NONE) → OpenSearchBoundaryTableScan (OPENSEARCH). - * - *

    Converts a {@link LogicalTableScan} into an {@link OpenSearchBoundaryTableScan} with the - * scan itself as the initial logical fragment. The boundary node carries an {@link QueryPlanExecutor} - * so it can delegate execution at {@code bind()} time. - */ -public class BoundaryTableScanRule extends ConverterRule { - - @SuppressWarnings("rawtypes") - private final QueryPlanExecutor queryPlanExecutor; - - /** - * Create a rule instance that converts LogicalTableScan to OpenSearchBoundaryTableScan. - * - * @param QueryPlanExecutor the engine executor passed to the boundary node for bind-time execution - * @return a new BoundaryTableScanRule - */ - @SuppressWarnings("rawtypes") - public static BoundaryTableScanRule create(QueryPlanExecutor QueryPlanExecutor) { - return new BoundaryTableScanRule( - Config.INSTANCE.withConversion(LogicalTableScan.class, Convention.NONE, EnumerableConvention.INSTANCE, "BoundaryTableScanRule"), - QueryPlanExecutor - ); - } - - @SuppressWarnings("rawtypes") - private BoundaryTableScanRule(Config config, QueryPlanExecutor queryPlanExecutor) { - super(config); - this.queryPlanExecutor = queryPlanExecutor; - } - - @Override - public RelNode convert(RelNode rel) { - LogicalTableScan scan = (LogicalTableScan) rel; - return new OpenSearchBoundaryTableScan( - scan.getCluster(), - scan.getTraitSet().replace(EnumerableConvention.INSTANCE), - scan.getTable(), - scan, - queryPlanExecutor - ); - } -} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/PushDownRulesTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/PushDownRulesTests.java deleted file mode 100644 index 7fcdd83eec35a..0000000000000 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/planner/rules/PushDownRulesTests.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ppl.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.config.CalciteConnectionConfig; -import org.apache.calcite.config.CalciteConnectionConfigImpl; -import org.apache.calcite.jdbc.CalciteSchema; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.linq4j.Enumerable; -import org.apache.calcite.linq4j.Linq4j; -import org.apache.calcite.plan.ConventionTraitDef; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.plan.hep.HepPlanner; -import org.apache.calcite.plan.hep.HepProgramBuilder; -import org.apache.calcite.plan.volcano.VolcanoPlanner; -import org.apache.calcite.prepare.CalciteCatalogReader; -import org.apache.calcite.rel.RelCollationTraitDef; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.calcite.rel.logical.LogicalTableScan; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.impl.AbstractTable; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlOperatorTable; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql.util.ListSqlOperatorTable; -import org.opensearch.analytics.exec.QueryPlanExecutor; -import org.opensearch.ppl.planner.rel.OpenSearchBoundaryTableScan; -import org.opensearch.test.OpenSearchTestCase; - -import java.util.Collections; -import java.util.List; -import java.util.Properties; - -/** - * Tests for push-down rules: {@link BoundaryTableScanRule}, {@link AbsorbFilterRule}. - */ -public class PushDownRulesTests extends OpenSearchTestCase { - - private RelOptCluster cluster; - private RexBuilder rexBuilder; - private RelOptTable table; - private QueryPlanExecutor> planExecutor; - - @Override - public void setUp() throws Exception { - super.setUp(); - - JavaTypeFactoryImpl typeFactory = new JavaTypeFactoryImpl(); - rexBuilder = new RexBuilder(typeFactory); - VolcanoPlanner volcanoPlanner = new VolcanoPlanner(); - volcanoPlanner.addRelTraitDef(ConventionTraitDef.INSTANCE); - volcanoPlanner.addRelTraitDef(RelCollationTraitDef.INSTANCE); - cluster = RelOptCluster.create(volcanoPlanner, rexBuilder); - - CalciteSchema rootSchema = CalciteSchema.createRootSchema(true); - SchemaPlus schemaPlus = rootSchema.plus(); - schemaPlus.add("test_table", new AbstractTable() { - @Override - public RelDataType getRowType(RelDataTypeFactory tf) { - return tf.builder() - .add("id", tf.createSqlType(SqlTypeName.INTEGER)) - .add("name", tf.createSqlType(SqlTypeName.VARCHAR)) - .add("value", tf.createSqlType(SqlTypeName.DOUBLE)) - .build(); - } - }); - - Properties props = new Properties(); - CalciteConnectionConfig config = new CalciteConnectionConfigImpl(props); - CalciteCatalogReader catalogReader = new CalciteCatalogReader(rootSchema, Collections.singletonList(""), typeFactory, config); - table = catalogReader.getTable(List.of("test_table")); - assertNotNull("Table should be found in catalog", table); - - planExecutor = (fragment, ctx, l) -> l.onResponse(Linq4j.emptyEnumerable()); - } - - // --- BoundaryTableScanRule tests (ConverterRule, uses VolcanoPlanner) --- - - public void testBoundaryTableScanRuleConvertsLogicalTableScan() { - VolcanoPlanner volcanoPlanner = new VolcanoPlanner(); - volcanoPlanner.addRelTraitDef(ConventionTraitDef.INSTANCE); - volcanoPlanner.addRelTraitDef(RelCollationTraitDef.INSTANCE); - RelOptCluster volcanoCluster = RelOptCluster.create(volcanoPlanner, rexBuilder); - - LogicalTableScan scan = LogicalTableScan.create(volcanoCluster, table, List.of()); - - volcanoPlanner.addRule(BoundaryTableScanRule.create(planExecutor)); - volcanoPlanner.setRoot(volcanoPlanner.changeTraits(scan, scan.getTraitSet().replace(EnumerableConvention.INSTANCE))); - - RelNode result = volcanoPlanner.findBestExp(); - - assertTrue("Result should be an OpenSearchBoundaryTableScan", result instanceof OpenSearchBoundaryTableScan); - OpenSearchBoundaryTableScan boundary = (OpenSearchBoundaryTableScan) result; - assertEquals("Convention should be BINDABLE", EnumerableConvention.INSTANCE, boundary.getConvention()); - } - - public void testBoundaryTableScanRulePreservesLogicalFragmentAsScan() { - VolcanoPlanner volcanoPlanner = new VolcanoPlanner(); - volcanoPlanner.addRelTraitDef(ConventionTraitDef.INSTANCE); - volcanoPlanner.addRelTraitDef(RelCollationTraitDef.INSTANCE); - RelOptCluster volcanoCluster = RelOptCluster.create(volcanoPlanner, rexBuilder); - - LogicalTableScan scan = LogicalTableScan.create(volcanoCluster, table, List.of()); - - volcanoPlanner.addRule(BoundaryTableScanRule.create(planExecutor)); - volcanoPlanner.setRoot(volcanoPlanner.changeTraits(scan, scan.getTraitSet().replace(EnumerableConvention.INSTANCE))); - - RelNode result = volcanoPlanner.findBestExp(); - - assertTrue("Result should be an OpenSearchBoundaryTableScan", result instanceof OpenSearchBoundaryTableScan); - OpenSearchBoundaryTableScan boundary = (OpenSearchBoundaryTableScan) result; - RelNode fragment = boundary.getLogicalFragment(); - assertTrue("Logical fragment should be a LogicalTableScan", fragment instanceof LogicalTableScan); - } - - // --- AbsorbFilterRule tests (RelOptRule, uses HepPlanner for rule application) --- - - /** - * Tests that AbsorbFilterRule absorbs a supported filter into the boundary node. - */ - public void testAbsorbFilterRuleAbsorbsSupportedFilter() { - SqlOperatorTable operatorTable = SqlStdOperatorTable.instance(); - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - - // Create a boundary node wrapping the scan - RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, planExecutor); - - // Build: value > 10 (supported condition) - JavaTypeFactoryImpl typeFactory = new JavaTypeFactoryImpl(); - RexNode valueRef = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.DOUBLE), 2); - RexNode literal10 = rexBuilder.makeLiteral(10.0, typeFactory.createSqlType(SqlTypeName.DOUBLE), true); - RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN, valueRef, literal10); - LogicalFilter filter = LogicalFilter.create(boundary, condition); - - // Run AbsorbFilterRule via HepPlanner - HepProgramBuilder programBuilder = new HepProgramBuilder(); - programBuilder.addRuleInstance(AbsorbFilterRule.create(operatorTable)); - HepPlanner hepPlanner = new HepPlanner(programBuilder.build()); - hepPlanner.setRoot(filter); - RelNode result = hepPlanner.findBestExp(); - - // The filter should be absorbed: result is a new boundary node with filter in fragment - assertTrue("Result should be an OpenSearchBoundaryTableScan (filter absorbed)", result instanceof OpenSearchBoundaryTableScan); - OpenSearchBoundaryTableScan resultBoundary = (OpenSearchBoundaryTableScan) result; - - RelNode fragment = resultBoundary.getLogicalFragment(); - assertTrue("Logical fragment should be a LogicalFilter (absorbed)", fragment instanceof LogicalFilter); - LogicalFilter absorbedFilter = (LogicalFilter) fragment; - assertTrue("Absorbed filter's input should be a LogicalTableScan", absorbedFilter.getInput() instanceof LogicalTableScan); - } - - /** - * Tests that AbsorbFilterRule does NOT absorb a filter when the condition - * contains unsupported functions (e.g. PLUS). - */ - public void testAbsorbFilterRuleDoesNotAbsorbUnsupportedFunctions() { - // Use restricted operator table where PLUS is not supported - List ops = List.of(SqlStdOperatorTable.EQUALS, SqlStdOperatorTable.GREATER_THAN); - SqlOperatorTable operatorTable = new ListSqlOperatorTable(ops); - - LogicalTableScan scan = LogicalTableScan.create(cluster, table, List.of()); - - // Create a boundary node wrapping the scan - RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - OpenSearchBoundaryTableScan boundary = new OpenSearchBoundaryTableScan(cluster, traitSet, table, scan, planExecutor); - - // Build: (value + 1) > 10 — PLUS is not in the restricted operator table - JavaTypeFactoryImpl typeFactory = new JavaTypeFactoryImpl(); - RexNode valueRef = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.DOUBLE), 2); - RexNode literal1 = rexBuilder.makeLiteral(1.0, typeFactory.createSqlType(SqlTypeName.DOUBLE), true); - RexNode literal10 = rexBuilder.makeLiteral(10.0, typeFactory.createSqlType(SqlTypeName.DOUBLE), true); - RexNode plus = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, valueRef, literal1); - RexNode unsupportedCondition = rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN, plus, literal10); - LogicalFilter filter = LogicalFilter.create(boundary, unsupportedCondition); - - // Run AbsorbFilterRule via HepPlanner - HepProgramBuilder programBuilder = new HepProgramBuilder(); - programBuilder.addRuleInstance(AbsorbFilterRule.create(operatorTable)); - HepPlanner hepPlanner = new HepPlanner(programBuilder.build()); - hepPlanner.setRoot(filter); - RelNode result = hepPlanner.findBestExp(); - - // The filter should NOT be absorbed — result should still be a LogicalFilter - assertTrue("Result should still be a LogicalFilter (not absorbed)", result instanceof LogicalFilter); - LogicalFilter resultFilter = (LogicalFilter) result; - assertTrue( - "Filter's input should still be an OpenSearchBoundaryTableScan", - resultFilter.getInput() instanceof OpenSearchBoundaryTableScan - ); - - // The boundary node's fragment should still be just the scan - OpenSearchBoundaryTableScan resultBoundary = (OpenSearchBoundaryTableScan) resultFilter.getInput(); - assertTrue( - "Boundary fragment should still be LogicalTableScan (filter not absorbed)", - resultBoundary.getLogicalFragment() instanceof LogicalTableScan - ); - } -} diff --git a/sandbox/plugins/test-ppl-frontend/build.gradle b/sandbox/plugins/test-ppl-frontend/build.gradle new file mode 100644 index 0000000000000..acb7a94a6c000 --- /dev/null +++ b/sandbox/plugins/test-ppl-frontend/build.gradle @@ -0,0 +1,165 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Test PPL front-end plugin. Provides a REST endpoint (POST /_analytics/ppl) + * backed by the unified PPL pipeline (parse → plan → push-down → compile → execute). + * Extends analytics-engine so that EngineContext and QueryPlanExecutor are + * injected via Guice from the hub plugin. + */ + +apply plugin: 'opensearch.opensearchplugin' + +// SQL Unified Query API version (aligned with OpenSearch build version) +def sqlUnifiedQueryVersion = '3.6.0.0-SNAPSHOT' + +opensearchplugin { + description = 'Test PPL front-end: REST endpoint backed by the unified PPL pipeline.' + classname = 'org.opensearch.ppl.TestPPLPlugin' + extendedPlugins = ['analytics-engine;optional=true'] +} + +repositories { + maven { + name = 'OpenSearch Snapshots' + url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/' + } +} + +// Guava comes transitively from calcite-core and unified-query — forbidden on +// main compile classpaths by OpenSearch. The PPL pipeline code needs it +// (Calcite API exposes ImmutableList, Predicate). Bypass via custom config. +configurations { + calciteCompile + compileClasspath { exclude group: 'com.google.guava' } + testCompileClasspath { exclude group: 'com.google.guava' } +} +sourceSets.main.compileClasspath += configurations.calciteCompile +sourceSets.test.compileClasspath += configurations.calciteCompile + +dependencies { + // Analytics framework + Calcite provided at runtime by analytics-engine (parent classloader via extendedPlugins) + compileOnly project(':sandbox:libs:analytics-framework') + + // Guava for compilation — Calcite API exposes guava types + calciteCompile "com.google.guava:guava:${versions.guava}" + + // Janino + commons-codec provided by analytics-engine parent plugin at runtime + + // SQL Unified Query API for PPL parsing + api("org.opensearch.query:unified-query-api:${sqlUnifiedQueryVersion}") { + exclude group: 'org.opensearch' + } + api("org.opensearch.query:unified-query-core:${sqlUnifiedQueryVersion}") { + exclude group: 'org.opensearch' + } + api("org.opensearch.query:unified-query-ppl:${sqlUnifiedQueryVersion}") { + exclude group: 'org.opensearch' + } + + // Calcite bytecode references @Immutable from immutables — resolve at compile time + compileOnly 'org.immutables:value-annotations:2.8.8' +} + +// Exclude jars provided by analytics-engine plugin (shared via extendedPlugins classloader). +// These are bundled in analytics-engine's ZIP and loaded by its classloader, which is +// the parent classloader for this plugin. +// Exclude jars already in the analytics-engine parent plugin ZIP (via analytics-framework runtimeOnly). +// Everything else must be bundled — plugins have isolated classloaders. +bundlePlugin { + exclude 'analytics-framework-*.jar' + exclude 'calcite-core-*.jar' + exclude 'calcite-linq4j-*.jar' + exclude 'avatica-core-*.jar' + exclude 'avatica-metrics-*.jar' + exclude 'guava-*.jar' + exclude 'failureaccess-*.jar' + exclude 'slf4j-api-*.jar' + exclude 'commons-codec-*.jar' + exclude 'janino-*.jar' + exclude 'commons-compiler-*.jar' + exclude 'joou-java-6-*.jar' + exclude 'jackson-core-*.jar' + exclude 'jackson-databind-*.jar' + exclude 'jackson-annotations-*.jar' + exclude 'commons-lang3-*.jar' + exclude 'commons-text-*.jar' + exclude 'commons-math3-*.jar' + exclude 'value-annotations-*.jar' + exclude 'json-path-*.jar' + exclude 'json-smart-*.jar' + exclude 'accessors-smart-*.jar' + exclude 'asm-*.jar' + exclude 'jts-core-*.jar' + exclude 'jts-io-common-*.jar' + exclude 'proj4j-*.jar' + exclude 'uzaygezen-core-*.jar' + exclude 'sketches-core-*.jar' + exclude 'memory-*.jar' + exclude 'httpcore5-*.jar' + exclude 'httpcore5-h2-*.jar' + exclude 'httpclient5-*.jar' + exclude 'jts-core-*.jar' + exclude 'jackson-core-*.jar' + exclude 'checker-qual-*.jar' + exclude 'error_prone_annotations-*.jar' +} + +// This is a test plugin — package-level javadocs are not required, and the +// bundled unified-query SNAPSHOT jars are internal OpenSearch artifacts not published with +// the LICENSE/NOTICE layout `dependencyLicenses` enforces. +tasks.matching { it.name == 'missingJavadoc' }.configureEach { + enabled = false +} +tasks.matching { it.name == 'dependencyLicenses' }.configureEach { + enabled = false +} +tasks.matching { it.name == 'thirdPartyAudit' }.configureEach { + enabled = false +} + +configurations.all { + // okhttp-aws-signer is a transitive dep of unified-query-common (via unified-query-core), + // only published on JitPack, not needed for PPL parsing/planning + exclude group: 'com.github.babbel', module: 'okhttp-aws-signer' + + resolutionStrategy { + // Align transitive versions with OpenSearch's managed versions + force "com.google.guava:guava:${versions.guava}" + force 'com.google.guava:failureaccess:1.0.2' + force 'com.google.errorprone:error_prone_annotations:2.36.0' + force 'org.checkerframework:checker-qual:3.43.0' + force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + force "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" + force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" + force "org.apache.logging.log4j:log4j-api:${versions.log4j}" + force "org.apache.logging.log4j:log4j-core:${versions.log4j}" + force "org.slf4j:slf4j-api:${versions.slf4j}" + force "org.locationtech.jts:jts-core:${versions.jts}" + force "commons-codec:commons-codec:${versions.commonscodec}" + force "joda-time:joda-time:${versions.joda}" + force "org.yaml:snakeyaml:${versions.snakeyaml}" + force "org.apache.commons:commons-lang3:${versions.commonslang}" + force "org.apache.commons:commons-text:1.11.0" + force "commons-logging:commons-logging:${versions.commonslogging}" + force "net.minidev:json-smart:${versions.json_smart}" + force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" + force "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" + force "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" + force "com.squareup.okhttp3:okhttp:4.12.0" + force "org.jetbrains.kotlin:kotlin-stdlib:${versions.kotlin}" + force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:${versions.kotlin}" + force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${versions.kotlin}" + force "org.jetbrains.kotlin:kotlin-stdlib-common:${versions.kotlin}" + force "commons-io:commons-io:${versions.commonsio}" + force "org.codehaus.janino:janino:3.1.12" + force "org.codehaus.janino:commons-compiler:3.1.12" + } +} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/TestPPLPlugin.java b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/TestPPLPlugin.java similarity index 53% rename from sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/TestPPLPlugin.java rename to sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/TestPPLPlugin.java index ba3c37224db2c..090529fe66ee2 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/TestPPLPlugin.java +++ b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/TestPPLPlugin.java @@ -9,14 +9,24 @@ package org.opensearch.ppl; import org.opensearch.action.ActionRequest; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; import org.opensearch.core.action.ActionResponse; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.ExtensiblePlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.ppl.action.RestPPLQueryAction; import org.opensearch.ppl.action.TestPPLTransportAction; import org.opensearch.ppl.action.UnifiedPPLExecuteAction; +import org.opensearch.rest.RestController; +import org.opensearch.rest.RestHandler; import java.util.List; +import java.util.function.Supplier; /** * Example front-end plugin using analytics-engine. @@ -29,4 +39,17 @@ public class TestPPLPlugin extends Plugin implements ActionPlugin, ExtensiblePlu public List> getActions() { return List.of(new ActionHandler<>(UnifiedPPLExecuteAction.INSTANCE, TestPPLTransportAction.class)); } + + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + return List.of(new RestPPLQueryAction()); + } } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/PPLRequest.java b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/PPLRequest.java similarity index 100% rename from sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/PPLRequest.java rename to sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/PPLRequest.java diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/PPLResponse.java b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/PPLResponse.java similarity index 67% rename from sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/PPLResponse.java rename to sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/PPLResponse.java index f89b7ed98c6f3..4434e220a9620 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/PPLResponse.java +++ b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/PPLResponse.java @@ -11,6 +11,9 @@ import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; @@ -20,7 +23,7 @@ * Transport-layer response carrying column names and result rows * from the unified PPL query execution pipeline. */ -public class PPLResponse extends ActionResponse { +public class PPLResponse extends ActionResponse implements ToXContentObject { private final List columns; private final List rows; @@ -64,4 +67,25 @@ public List getColumns() { public List getRows() { return rows; } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.startArray("columns"); + for (String col : columns) { + builder.value(col); + } + builder.endArray(); + builder.startArray("rows"); + for (Object[] row : rows) { + builder.startArray(); + for (Object val : row) { + builder.value(val); + } + builder.endArray(); + } + builder.endArray(); + builder.endObject(); + return builder; + } } diff --git a/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/RestPPLQueryAction.java b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/RestPPLQueryAction.java new file mode 100644 index 0000000000000..0a31958223af3 --- /dev/null +++ b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/RestPPLQueryAction.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.ppl.action; + +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.transport.client.node.NodeClient; + +import java.io.IOException; +import java.util.List; + +import static org.opensearch.rest.RestRequest.Method.POST; + +/** + * REST handler for PPL queries: {@code POST /_analytics/ppl}. + * Parses {@code {"query": ""}} from the request body and + * delegates to the transport action. + */ +public class RestPPLQueryAction extends BaseRestHandler { + + @Override + public String getName() { + return "analytics_ppl_query"; + } + + @Override + public List routes() { + return List.of(new Route(POST, "/_analytics/ppl")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String queryText; + try (XContentParser parser = request.contentParser()) { + queryText = parseQueryText(parser); + } + PPLRequest pplRequest = new PPLRequest(queryText); + return channel -> client.execute(UnifiedPPLExecuteAction.INSTANCE, pplRequest, new RestToXContentListener<>(channel)); + } + + private String parseQueryText(XContentParser parser) throws IOException { + String query = null; + parser.nextToken(); // START_OBJECT + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = parser.currentName(); + parser.nextToken(); + if ("query".equals(fieldName)) { + query = parser.text(); + } else { + parser.skipChildren(); + } + } + if (query == null || query.isEmpty()) { + throw new IllegalArgumentException("Request body must contain a 'query' field"); + } + return query; + } +} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/TestPPLTransportAction.java b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/TestPPLTransportAction.java similarity index 80% rename from sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/TestPPLTransportAction.java rename to sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/TestPPLTransportAction.java index 4381dcfa058b3..e82c2972c5505 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/TestPPLTransportAction.java +++ b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/TestPPLTransportAction.java @@ -17,19 +17,14 @@ import org.opensearch.analytics.exec.QueryPlanExecutor; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; -import org.opensearch.ppl.planner.PushDownPlanner; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; /** * Transport action that coordinates PPL query execution. * - *

    Receives {@link EngineContext} and {@link QueryPlanExecutor} via Guice injection. - * The engine context provides both the schema (from cluster state) and the aggregated - * operator table from all back-end engines. - * - *

    On success, calls {@code listener.onResponse()} with the {@link PPLResponse}. - * On failure, calls {@code listener.onFailure()} with the exception. + *

    Receives {@link EngineContext} and {@link QueryPlanExecutor} from the analytics-engine + * plugin via Guice injection (enabled by {@code extendedPlugins = ['analytics-engine']}). */ public class TestPPLTransportAction extends HandledTransportAction { @@ -45,9 +40,7 @@ public TestPPLTransportAction( QueryPlanExecutor> executor ) { super(UnifiedPPLExecuteAction.NAME, transportService, actionFilters, PPLRequest::new); - - PushDownPlanner pushDownPlanner = new PushDownPlanner(engineContext.operatorTable(), executor); - this.unifiedQueryService = new UnifiedQueryService(pushDownPlanner, engineContext); + this.unifiedQueryService = new UnifiedQueryService(executor, engineContext); } /** Test-only constructor that accepts a pre-built {@link UnifiedQueryService}. */ diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/UnifiedPPLExecuteAction.java b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/UnifiedPPLExecuteAction.java similarity index 100% rename from sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/UnifiedPPLExecuteAction.java rename to sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/UnifiedPPLExecuteAction.java diff --git a/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/UnifiedQueryService.java b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/UnifiedQueryService.java new file mode 100644 index 0000000000000..aad9e5b4c19bb --- /dev/null +++ b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/UnifiedQueryService.java @@ -0,0 +1,120 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.ppl.action; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.analytics.EngineContext; +import org.opensearch.analytics.exec.QueryPlanExecutor; +import org.opensearch.sql.api.UnifiedQueryContext; +import org.opensearch.sql.api.UnifiedQueryPlanner; +import org.opensearch.sql.executor.QueryType; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Core orchestrator: PPL text → RelNode → QueryPlanExecutor → PPLResponse. + * + *

    Passes the logical RelNode directly to the back-end engine (e.g. DataFusion) + * which handles optimization and execution natively via Substrait. No Janino + * code generation needed. + */ +public class UnifiedQueryService { + + private static final Logger logger = LogManager.getLogger(UnifiedQueryService.class); + private static final String DEFAULT_CATALOG = "opensearch"; + + private final QueryPlanExecutor> planExecutor; + private final EngineContext engineContext; + + public UnifiedQueryService(QueryPlanExecutor> planExecutor, EngineContext engineContext) { + this.planExecutor = planExecutor; + this.engineContext = engineContext; + } + + /** + * Executes a PPL query through the simplified pipeline: + * PPL text → RelNode → planExecutor.execute() → PPLResponse. + */ + public PPLResponse execute(String pplText) { + // Extract tables from the SchemaPlus into a plain AbstractSchema. + // SchemaPlus wraps CalciteSchema — passing it to catalog() causes double-nesting + // where tables become inaccessible. A plain Schema avoids this. + SchemaPlus schemaPlus = engineContext.getSchema(); + Map tableMap = new HashMap<>(); + for (String tableName : schemaPlus.getTableNames()) { + tableMap.put(tableName, schemaPlus.getTable(tableName)); + } + AbstractSchema flatSchema = new AbstractSchema() { + @Override + protected Map getTableMap() { + return tableMap; + } + }; + + logger.info( + "[UnifiedQueryService] schemaPlus class: {}, tableNames: {}, tableMap: {}, engineContext class: {}", + schemaPlus.getClass().getName(), + schemaPlus.getTableNames(), + tableMap.keySet(), + engineContext.getClass().getName() + ); + + try ( + UnifiedQueryContext context = UnifiedQueryContext.builder() + .language(QueryType.PPL) + .catalog(DEFAULT_CATALOG, flatSchema) + .defaultNamespace(DEFAULT_CATALOG) + .build() + ) { + + // Log what the context's root schema looks like + logger.info("[UnifiedQueryService] Context built, planning PPL: {}", pplText); + UnifiedQueryPlanner planner = new UnifiedQueryPlanner(context); + RelNode logicalPlan = planner.plan(pplText); + + // Execute directly via the back-end engine — no Janino compilation needed. + // The executor API is async; this test frontend keeps a sync surface, so we bridge + // via PlainActionFuture. The block happens off the transport thread (the executor + // forks to SEARCH internally), so this is safe for test/IT use. + PlainActionFuture> future = new PlainActionFuture<>(); + planExecutor.execute(logicalPlan, null, future); + Iterable results = future.actionGet(); + + // Extract column names from the RelNode's row type + List fields = logicalPlan.getRowType().getFieldList(); + List columns = new ArrayList<>(fields.size()); + for (RelDataTypeField field : fields) { + columns.add(field.getName()); + } + + // Collect result rows + List rows = new ArrayList<>(); + for (Object[] row : results) { + rows.add(row); + } + + return new PPLResponse(columns, rows); + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } + throw new RuntimeException("Failed to execute PPL query: " + e.getMessage(), e); + } + } +} diff --git a/sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/PPLResponseTests.java b/sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/PPLResponseTests.java new file mode 100644 index 0000000000000..0e10821798435 --- /dev/null +++ b/sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/PPLResponseTests.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.ppl.action; + +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.List; + +public class PPLResponseTests extends OpenSearchTestCase { + + public void testToXContentEmptyResponse() throws IOException { + PPLResponse response = new PPLResponse(List.of(), List.of()); + XContentBuilder builder = XContentFactory.jsonBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + String json = builder.toString(); + assertEquals("{\"columns\":[],\"rows\":[]}", json); + } + + public void testToXContentWithData() throws IOException { + List columns = List.of("name", "age"); + List rows = List.of(new Object[] { "Alice", 30 }, new Object[] { "Bob", 25 }); + PPLResponse response = new PPLResponse(columns, rows); + XContentBuilder builder = XContentFactory.jsonBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + String json = builder.toString(); + assertEquals("{\"columns\":[\"name\",\"age\"],\"rows\":[[\"Alice\",30],[\"Bob\",25]]}", json); + } + + public void testToXContentWithNullValues() throws IOException { + List columns = List.of("col1"); + List rows = new java.util.ArrayList<>(); + rows.add(new Object[] { null }); + PPLResponse response = new PPLResponse(columns, rows); + XContentBuilder builder = XContentFactory.jsonBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + String json = builder.toString(); + assertEquals("{\"columns\":[\"col1\"],\"rows\":[[null]]}", json); + } +} diff --git a/sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/RestPPLQueryActionTests.java b/sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/RestPPLQueryActionTests.java new file mode 100644 index 0000000000000..87b6a5d527f52 --- /dev/null +++ b/sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/RestPPLQueryActionTests.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.ppl.action; + +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.rest.FakeRestRequest; + +public class RestPPLQueryActionTests extends OpenSearchTestCase { + + private final RestPPLQueryAction action = new RestPPLQueryAction(); + + public void testName() { + assertEquals("analytics_ppl_query", action.getName()); + } + + public void testRoutes() { + assertEquals(1, action.routes().size()); + assertEquals(RestRequest.Method.POST, action.routes().get(0).getMethod()); + assertEquals("/_analytics/ppl", action.routes().get(0).getPath()); + } + + public void testPrepareRequestMissingQuery() { + FakeRestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST) + .withPath("/_analytics/ppl") + .withContent(new BytesArray("{\"other\":\"value\"}"), XContentType.JSON) + .build(); + + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> action.prepareRequest(request, null)); + assertTrue(ex.getMessage().contains("query")); + } +} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/TestPPLTransportActionTests.java b/sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/TestPPLTransportActionTests.java similarity index 100% rename from sandbox/plugins/analytics-engine/src/test/java/org/opensearch/ppl/action/TestPPLTransportActionTests.java rename to sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/TestPPLTransportActionTests.java From ac0f565e05a70d9bca7d99dc36791696f025ec50 Mon Sep 17 00:00:00 2001 From: Varun Bharadwaj Date: Wed, 29 Apr 2026 21:37:10 -0700 Subject: [PATCH 018/115] Update the Kafka metadata timeout value to be configurable (#21425) * support configurable kafka metadata read timeout value --------- Signed-off-by: Varun Bharadwaj --- .../plugin/kafka/IngestFromKafkaIT.java | 1 + .../plugin/kafka/KafkaPartitionConsumer.java | 14 +++-- .../plugin/kafka/KafkaSourceConfig.java | 23 +++++++++ .../kafka/KafkaPartitionConsumerTests.java | 16 ++++++ .../plugin/kafka/KafkaSourceConfigTests.java | 51 +++++++++++++++++++ .../pollingingest/DefaultStreamPoller.java | 4 ++ 6 files changed, 105 insertions(+), 4 deletions(-) diff --git a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java index 57f7fcb6b4d0d..e794602d6a3a0 100644 --- a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java +++ b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java @@ -135,6 +135,7 @@ public void testKafkaIngestion_RewindByOffset() { .put("ingestion_source.param.topic", "test") .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "latest") + .put("ingestion_source.param.topic_metadata_fetch_timeout_ms", 5000) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" diff --git a/plugins/ingestion-kafka/src/main/java/org/opensearch/plugin/kafka/KafkaPartitionConsumer.java b/plugins/ingestion-kafka/src/main/java/org/opensearch/plugin/kafka/KafkaPartitionConsumer.java index 25a793d65c171..afa8ff0e50a21 100644 --- a/plugins/ingestion-kafka/src/main/java/org/opensearch/plugin/kafka/KafkaPartitionConsumer.java +++ b/plugins/ingestion-kafka/src/main/java/org/opensearch/plugin/kafka/KafkaPartitionConsumer.java @@ -45,8 +45,6 @@ public class KafkaPartitionConsumer implements IngestionShardConsumer consumer; - // TODO: make this configurable - private final int timeoutMillis = 1000; private long lastFetchedOffset = -1; final String clientId; @@ -76,7 +74,10 @@ protected KafkaPartitionConsumer(String clientId, KafkaSourceConfig config, int this.config = config; String topic = config.getTopic(); List partitionInfos = AccessController.doPrivileged( - (PrivilegedAction>) () -> consumer.partitionsFor(topic, Duration.ofMillis(timeoutMillis)) + (PrivilegedAction>) () -> consumer.partitionsFor( + topic, + Duration.ofMillis(config.getTopicMetadataFetchTimeoutMs()) + ) ); if (partitionInfos == null) { throw new IllegalArgumentException("Topic " + topic + " does not exist"); @@ -86,7 +87,12 @@ protected KafkaPartitionConsumer(String clientId, KafkaSourceConfig config, int } topicPartition = new TopicPartition(topic, partitionId); consumer.assign(Collections.singletonList(topicPartition)); - logger.info("Kafka consumer created for topic {} partition {}", topic, partitionId); + logger.info( + "Kafka consumer created for topic {} partition {} with topic metadata fetch timeout {}ms", + topic, + partitionId, + config.getTopicMetadataFetchTimeoutMs() + ); } /** diff --git a/plugins/ingestion-kafka/src/main/java/org/opensearch/plugin/kafka/KafkaSourceConfig.java b/plugins/ingestion-kafka/src/main/java/org/opensearch/plugin/kafka/KafkaSourceConfig.java index 5082739926731..b94e061c42090 100644 --- a/plugins/ingestion-kafka/src/main/java/org/opensearch/plugin/kafka/KafkaSourceConfig.java +++ b/plugins/ingestion-kafka/src/main/java/org/opensearch/plugin/kafka/KafkaSourceConfig.java @@ -20,11 +20,14 @@ public class KafkaSourceConfig { private final String PROP_TOPIC = "topic"; private final String PROP_BOOTSTRAP_SERVERS = "bootstrap_servers"; + private static final String PROP_TOPIC_METADATA_FETCH_TIMEOUT_MS = "topic_metadata_fetch_timeout_ms"; + private static final int DEFAULT_TOPIC_METADATA_FETCH_TIMEOUT_MS = 1000; private final String topic; private final String bootstrapServers; private final String autoOffsetResetConfig; private final int maxPollRecords; + private final int topicMetadataFetchTimeoutMs; private final Map consumerConfigsMap; @@ -46,9 +49,21 @@ public KafkaSourceConfig(int maxPollSize, Map params) { // maxPollSize will be used instead. this.maxPollRecords = ConfigurationUtils.readIntProperty(params, ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollSize); + this.topicMetadataFetchTimeoutMs = ConfigurationUtils.readIntProperty( + params, + PROP_TOPIC_METADATA_FETCH_TIMEOUT_MS, + DEFAULT_TOPIC_METADATA_FETCH_TIMEOUT_MS + ); + if (this.topicMetadataFetchTimeoutMs <= 0) { + throw new IllegalArgumentException( + "topic_metadata_fetch_timeout_ms must be positive, got: " + this.topicMetadataFetchTimeoutMs + ); + } + // remove metadata configurations consumerConfigsMap.remove(PROP_TOPIC); consumerConfigsMap.remove(PROP_BOOTSTRAP_SERVERS); + consumerConfigsMap.remove(PROP_TOPIC_METADATA_FETCH_TIMEOUT_MS); // add or overwrite required configurations with defaults if not present consumerConfigsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetResetConfig); @@ -84,4 +99,12 @@ public String getAutoOffsetResetConfig() { public Map getConsumerConfigurations() { return consumerConfigsMap; } + + /** + * Get the topic metadata fetch timeout in milliseconds + * @return the topic metadata fetch timeout in milliseconds + */ + public int getTopicMetadataFetchTimeoutMs() { + return topicMetadataFetchTimeoutMs; + } } diff --git a/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaPartitionConsumerTests.java b/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaPartitionConsumerTests.java index 096e2df8a7fe2..34d241b31db1c 100644 --- a/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaPartitionConsumerTests.java +++ b/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaPartitionConsumerTests.java @@ -28,6 +28,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class KafkaPartitionConsumerTests extends OpenSearchTestCase { @@ -213,4 +214,19 @@ public void testGetPointerBasedLagHandlesException() { // Should return -1 on exception assertEquals(-1, lag); } + + public void testTopicMetadataFetchTimeoutUsedFromConfig() { + Map params = new HashMap<>(); + params.put("topic", "test-topic"); + params.put("bootstrap_servers", "localhost:9092"); + params.put("topic_metadata_fetch_timeout_ms", 5000); + + KafkaSourceConfig customConfig = new KafkaSourceConfig(1000, params); + PartitionInfo partitionInfo = new PartitionInfo("test-topic", 0, null, null, null); + when(mockConsumer.partitionsFor(eq("test-topic"), any(Duration.class))).thenReturn(Collections.singletonList(partitionInfo)); + + new KafkaPartitionConsumer("client1", customConfig, 0, mockConsumer); + + verify(mockConsumer).partitionsFor(eq("test-topic"), eq(Duration.ofMillis(5000))); + } } diff --git a/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSourceConfigTests.java b/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSourceConfigTests.java index d7252fbdb688e..df340b14b3e92 100644 --- a/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSourceConfigTests.java +++ b/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSourceConfigTests.java @@ -40,4 +40,55 @@ public void testKafkaSourceConfig() { ); Assert.assertEquals("Incorrect max.poll.records", 100, config.getConsumerConfigurations().get("max.poll.records")); } + + public void testTopicMetadataFetchTimeoutMsDefault() { + Map params = new HashMap<>(); + params.put("topic", "topic"); + params.put("bootstrap_servers", "bootstrap"); + + KafkaSourceConfig config = new KafkaSourceConfig(100, params); + + Assert.assertEquals("Default topic metadata fetch timeout should be 1000ms", 1000, config.getTopicMetadataFetchTimeoutMs()); + Assert.assertFalse( + "topic_metadata_fetch_timeout_ms should not be in consumer configurations", + config.getConsumerConfigurations().containsKey("topic_metadata_fetch_timeout_ms") + ); + } + + public void testTopicMetadataFetchTimeoutMsCustom() { + Map params = new HashMap<>(); + params.put("topic", "topic"); + params.put("bootstrap_servers", "bootstrap"); + params.put("topic_metadata_fetch_timeout_ms", 5000); + + KafkaSourceConfig config = new KafkaSourceConfig(100, params); + + Assert.assertEquals("Custom topic metadata fetch timeout should be respected", 5000, config.getTopicMetadataFetchTimeoutMs()); + Assert.assertFalse( + "topic_metadata_fetch_timeout_ms should not be in consumer configurations", + config.getConsumerConfigurations().containsKey("topic_metadata_fetch_timeout_ms") + ); + } + + public void testTopicMetadataFetchTimeoutMsInvalid() { + Map params = new HashMap<>(); + params.put("topic", "topic"); + params.put("bootstrap_servers", "bootstrap"); + params.put("topic_metadata_fetch_timeout_ms", 0); + + try { + new KafkaSourceConfig(100, params); + fail("Expected IllegalArgumentException for non-positive timeout"); + } catch (IllegalArgumentException e) { + Assert.assertEquals("topic_metadata_fetch_timeout_ms must be positive, got: 0", e.getMessage()); + } + + params.put("topic_metadata_fetch_timeout_ms", -1); + try { + new KafkaSourceConfig(100, params); + fail("Expected IllegalArgumentException for non-positive timeout"); + } catch (IllegalArgumentException e) { + Assert.assertEquals("topic_metadata_fetch_timeout_ms must be positive, got: -1", e.getMessage()); + } + } } diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java index e14cb5092b251..3a8d04405d3ff 100644 --- a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java +++ b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java @@ -696,6 +696,10 @@ private void handleConsumerInitialization() { blockingQueueContainer.clearAllQueues(); initializeConsumer(); + if (this.consumer == null) { + return; + } + // Handle consumer offset reset the first time an index is created. The reset offset takes precedence if available. IngestionShardPointer resetShardPointer = getResetShardPointer(); if (resetShardPointer != null) { From 6f3f36725f5b8f90e0b71b3b7aa879e6ce232947 Mon Sep 17 00:00:00 2001 From: MayankHarsh03 <156461043+MayankHarsh03@users.noreply.github.com> Date: Thu, 30 Apr 2026 12:36:04 +0530 Subject: [PATCH 019/115] Add slow logs, per-query metrics, and TierActionMetrics for WritableWarm tiered storage (#21332) - TieredStorageSearchSlowLog: SearchOperationListener for slow query/fetch logging on warm data - TieredStoragePerQueryMetricImpl: per-query cache hit/miss, prefetch, read-ahead tracking - TieredStorageQueryMetricService: singleton service managing metric collectors across threads - PrefetchStats: completed with StreamInput constructor, writeTo, getters, toXContent - TierActionMetrics: migration success/rejection/latency metrics Modified existing files: - OnDemandPrefetchBlockSnapshotIndexInput: replaced metric TODOs with actual recording calls - StoredFieldsPrefetch: added recordStoredFieldsPrefetch calls - IndexScopedSettings: registered 10 slow log index-scoped settings Signed-off-by: Mayank Harsh --- .../common/settings/IndexScopedSettings.java | 13 + ...DemandPrefetchBlockSnapshotIndexInput.java | 20 +- .../storage/metrics/TierActionMetrics.java | 105 ++++ .../storage/metrics/package-info.java | 12 + .../prefetch/StoredFieldsPrefetch.java | 3 +- .../storage/slowlogs/PrefetchStats.java | 125 ++++ .../slowlogs/TieredStoragePerQueryMetric.java | 55 ++ .../TieredStoragePerQueryMetricImpl.java | 379 ++++++++++++ .../TieredStorageQueryMetricService.java | 303 +++++++++ .../slowlogs/TieredStorageSearchSlowLog.java | 577 ++++++++++++++++++ .../storage/slowlogs/package-info.java | 12 + ...redStoragePerQueryMetricImplJsonTests.java | 81 +++ .../TieredStoragePerQueryMetricImplTests.java | 330 ++++++++++ .../TieredStorageQueryMetricServiceTests.java | 362 +++++++++++ .../TieredStorageSearchSlowLogTests.java | 510 ++++++++++++++++ 15 files changed, 2882 insertions(+), 5 deletions(-) create mode 100644 server/src/main/java/org/opensearch/storage/metrics/TierActionMetrics.java create mode 100644 server/src/main/java/org/opensearch/storage/metrics/package-info.java create mode 100644 server/src/main/java/org/opensearch/storage/slowlogs/PrefetchStats.java create mode 100644 server/src/main/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetric.java create mode 100644 server/src/main/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImpl.java create mode 100644 server/src/main/java/org/opensearch/storage/slowlogs/TieredStorageQueryMetricService.java create mode 100644 server/src/main/java/org/opensearch/storage/slowlogs/TieredStorageSearchSlowLog.java create mode 100644 server/src/main/java/org/opensearch/storage/slowlogs/package-info.java create mode 100644 server/src/test/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImplJsonTests.java create mode 100644 server/src/test/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImplTests.java create mode 100644 server/src/test/java/org/opensearch/storage/slowlogs/TieredStorageQueryMetricServiceTests.java create mode 100644 server/src/test/java/org/opensearch/storage/slowlogs/TieredStorageSearchSlowLogTests.java diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index e65f87713363f..d909aa89b42ec 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -61,6 +61,7 @@ import org.opensearch.indices.IndicesBitsetFilterCache; import org.opensearch.indices.IndicesRequestCache; import org.opensearch.search.streaming.FlushModeResolver; +import org.opensearch.storage.slowlogs.TieredStorageSearchSlowLog; import java.util.Arrays; import java.util.Collections; @@ -313,6 +314,18 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexModule.INDEX_TIERING_STATE, IndexModule.IS_WARM_INDEX_SETTING, + // Tiered storage search slow log settings + TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_ENABLED, + TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING, + TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING, + TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING, + TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING, + TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING, + TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING, + TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING, + TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING, + TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL, + // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { Map groups = s.getAsGroups(); diff --git a/server/src/main/java/org/opensearch/storage/indexinput/OnDemandPrefetchBlockSnapshotIndexInput.java b/server/src/main/java/org/opensearch/storage/indexinput/OnDemandPrefetchBlockSnapshotIndexInput.java index 745fe6d1cda45..b4ee418d7cf57 100644 --- a/server/src/main/java/org/opensearch/storage/indexinput/OnDemandPrefetchBlockSnapshotIndexInput.java +++ b/server/src/main/java/org/opensearch/storage/indexinput/OnDemandPrefetchBlockSnapshotIndexInput.java @@ -19,6 +19,8 @@ import org.opensearch.index.store.remote.utils.BlobFetchRequest; import org.opensearch.index.store.remote.utils.TransferManager; import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings; +import org.opensearch.storage.slowlogs.TieredStoragePerQueryMetric; +import org.opensearch.storage.slowlogs.TieredStorageQueryMetricService; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -60,7 +62,12 @@ public OnDemandPrefetchBlockSnapshotIndexInput( @Override protected IndexInput fetchBlock(int blockId) throws IOException { - // TODO: Metric recording will be added when TieredStorageQueryMetricService is available + // Record cache access attempt and track hit/miss + String blockFileName = fileName + "_block_" + blockId; + boolean cacheHit = checkCacheHit(blockId); + final TieredStoragePerQueryMetric metricCollector = TieredStorageQueryMetricService.getInstance() + .getMetricCollector(Thread.currentThread().threadId()); + metricCollector.recordFileAccess(blockFileName, cacheHit); fetchNextNBlocks(blockId); return super.fetchBlock(blockId); } @@ -114,7 +121,7 @@ protected void fetchNextNBlocks(int blockId) { } logger.trace("Prefetching Read Ahead Block Count: {} from Block ID: {} for File: {}", readAheadBlockCount, blockId, fileName); downloadBlocksAsync(blockId + 1, blockId + readAheadBlockCount, true); - // TODO: Metric recording will be added when TieredStorageQueryMetricService is available + TieredStorageQueryMetricService.getInstance().recordDocValuesPrefetch(true); } @Override @@ -134,6 +141,8 @@ public void prefetch(long offset, long length) throws IOException { } protected void downloadBlocksAsync(int startBlock, int endBlock, boolean isReadAhead) { + final TieredStoragePerQueryMetric metricCollector = TieredStorageQueryMetricService.getInstance() + .getMetricCollector(Thread.currentThread().threadId()); for (int nextBlockId = startBlock; nextBlockId <= endBlock; nextBlockId++) { String blockFileName = fileName + "_block_" + nextBlockId; long blockStart = getBlockStart(nextBlockId); @@ -146,7 +155,11 @@ protected void downloadBlocksAsync(int startBlock, int endBlock, boolean isReadA blockEnd, originalFileSize ); - // TODO: Metric recording will be added when TieredStorageQueryMetricService is available + if (isReadAhead) { + metricCollector.recordReadAhead(fileName, nextBlockId); + } else { + metricCollector.recordPrefetch(fileName, nextBlockId); + } // Block may be present on multiple chunks of a file, so we need // to fetch each chunk/blob part separately to fetch an entire block. BlobFetchRequest blobFetchRequest = BlobFetchRequest.builder() @@ -204,7 +217,6 @@ protected int getTotalBlocks() { /** * Checks if a block file exists in the file cache. * This method determines cache hit/miss status for transfer manager operations. - * TODO: Will be used by TieredStorageQueryMetricService for recording per-query cache metrics. * * @param blockId the id of the block to check * @return true if the block exists in cache (cache hit), false otherwise (cache miss) diff --git a/server/src/main/java/org/opensearch/storage/metrics/TierActionMetrics.java b/server/src/main/java/org/opensearch/storage/metrics/TierActionMetrics.java new file mode 100644 index 0000000000000..2f2172e4d8372 --- /dev/null +++ b/server/src/main/java/org/opensearch/storage/metrics/TierActionMetrics.java @@ -0,0 +1,105 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.metrics; + +import org.opensearch.telemetry.metrics.Counter; +import org.opensearch.telemetry.metrics.Histogram; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.metrics.tags.Tags; + +/** + * Metrics for tracking tier migration operations including successful migrations, + * rejections, and latency. + * + * @opensearch.experimental + */ +public final class TierActionMetrics { + + private static final String LATENCY_METRIC_UNIT_MS = "ms"; + private static final String COUNTER_METRICS_UNIT = "1"; + + /** Tag key for node ID. */ + public static final String NODE_ID = "node_id"; + /** Tag key for index name. */ + public static final String INDEX_NAME = "index_name"; + /** Tag key for tier type. */ + public static final String TIER_TYPE = "tier_type"; + /** Tag key for rejection reason. */ + public static final String REJECTION_REASON = "rejection_reason"; + + /** Counter for successful tier migrations. */ + public final Counter successfulMigrations; + /** Counter for rejected tier migrations. */ + public final Counter rejectionReason; + /** Histogram for tracking end-to-end migration time. */ + public final Histogram migrationLatency; + + /** + * Creates a new TierActionMetrics instance. + * @param metricsRegistry the metrics registry to create counters and histograms + */ + public TierActionMetrics(MetricsRegistry metricsRegistry) { + successfulMigrations = metricsRegistry.createCounter( + "migration_successful", + "Counter for successful tier migrations", + COUNTER_METRICS_UNIT + ); + + rejectionReason = metricsRegistry.createCounter( + "migration_rejection_reason", + "Counter for rejected tier migrations with their reasons", + COUNTER_METRICS_UNIT + ); + + migrationLatency = metricsRegistry.createHistogram( + "migration_latency", + "Histogram for tracking end-to-end migration time", + LATENCY_METRIC_UNIT_MS + ); + } + + /** + * Records migration latency. + * @param value the latency value in milliseconds + * @param nodeId the node ID + * @param indexName the index name + * @param tierType the tier type + */ + public void recordMigrationLatency(Double value, String nodeId, String indexName, String tierType) { + Tags tags = createBaseTags(nodeId, indexName, tierType); + migrationLatency.record(value, tags); + } + + /** + * Records a successful migration. + * @param nodeId the node ID + * @param indexName the index name + * @param tierType the tier type + */ + public void recordSuccessfulMigration(String nodeId, String indexName, String tierType) { + Tags tags = createBaseTags(nodeId, indexName, tierType); + successfulMigrations.add(1.0, tags); + } + + /** + * Records a rejected migration. + * @param nodeId the node ID + * @param indexName the index name + * @param tierType the tier type + * @param reason the rejection reason + */ + public void recordRejectedMigration(String nodeId, String indexName, String tierType, String reason) { + Tags tags = createBaseTags(nodeId, indexName, tierType).addTag(REJECTION_REASON, reason); + rejectionReason.add(1.0, tags); + } + + private Tags createBaseTags(String nodeId, String indexName, String tierType) { + return Tags.create().addTag(NODE_ID, nodeId).addTag(INDEX_NAME, indexName).addTag(TIER_TYPE, tierType); + } +} diff --git a/server/src/main/java/org/opensearch/storage/metrics/package-info.java b/server/src/main/java/org/opensearch/storage/metrics/package-info.java new file mode 100644 index 0000000000000..2ba170e637dc6 --- /dev/null +++ b/server/src/main/java/org/opensearch/storage/metrics/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Metrics for tiered storage operations including migration tracking. + */ +package org.opensearch.storage.metrics; diff --git a/server/src/main/java/org/opensearch/storage/prefetch/StoredFieldsPrefetch.java b/server/src/main/java/org/opensearch/storage/prefetch/StoredFieldsPrefetch.java index b759477073672..59f5b87fd69d0 100644 --- a/server/src/main/java/org/opensearch/storage/prefetch/StoredFieldsPrefetch.java +++ b/server/src/main/java/org/opensearch/storage/prefetch/StoredFieldsPrefetch.java @@ -21,6 +21,7 @@ import org.opensearch.common.lucene.search.Queries; import org.opensearch.index.shard.SearchOperationListener; import org.opensearch.search.internal.SearchContext; +import org.opensearch.storage.slowlogs.TieredStorageQueryMetricService; import java.io.IOException; import java.util.function.Supplier; @@ -50,7 +51,7 @@ public void onPreFetchPhase(SearchContext searchContext) { // Based on cluster settings if (checkIfStoredFieldsPrefetchEnabled()) { executePrefetch(searchContext); - // TODO: Metric recording will be added when TieredStorageQueryMetricService is available + TieredStorageQueryMetricService.getInstance().recordStoredFieldsPrefetch(true); } } diff --git a/server/src/main/java/org/opensearch/storage/slowlogs/PrefetchStats.java b/server/src/main/java/org/opensearch/storage/slowlogs/PrefetchStats.java new file mode 100644 index 0000000000000..9c9615c8563c4 --- /dev/null +++ b/server/src/main/java/org/opensearch/storage/slowlogs/PrefetchStats.java @@ -0,0 +1,125 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.slowlogs; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Stats for prefetch operations on tiered storage. + * + * @opensearch.experimental + */ +public class PrefetchStats implements Writeable, ToXContentFragment { + + private final long storedFieldsPrefetchSuccess; + private final long storedFieldsPrefetchFailure; + private final long docValuesPrefetchSuccess; + private final long docValuesPrefetchFailure; + + /** + * Creates a new PrefetchStats instance. + * @param storedFieldsPrefetchSuccess count of successful stored fields prefetches + * @param storedFieldsPrefetchFailure count of failed stored fields prefetches + * @param docValuesPrefetchSuccess count of successful doc values prefetches + * @param docValuesPrefetchFailure count of failed doc values prefetches + */ + public PrefetchStats( + long storedFieldsPrefetchSuccess, + long storedFieldsPrefetchFailure, + long docValuesPrefetchSuccess, + long docValuesPrefetchFailure + ) { + this.storedFieldsPrefetchSuccess = storedFieldsPrefetchSuccess; + this.storedFieldsPrefetchFailure = storedFieldsPrefetchFailure; + this.docValuesPrefetchSuccess = docValuesPrefetchSuccess; + this.docValuesPrefetchFailure = docValuesPrefetchFailure; + } + + /** + * Creates a new PrefetchStats instance from a stream. + * @param in the stream input + * @throws IOException if an I/O error occurs + */ + public PrefetchStats(StreamInput in) throws IOException { + storedFieldsPrefetchSuccess = in.readVLong(); + storedFieldsPrefetchFailure = in.readVLong(); + docValuesPrefetchSuccess = in.readVLong(); + docValuesPrefetchFailure = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(storedFieldsPrefetchSuccess); + out.writeVLong(storedFieldsPrefetchFailure); + out.writeVLong(docValuesPrefetchSuccess); + out.writeVLong(docValuesPrefetchFailure); + } + + /** + * Returns the count of successful stored fields prefetches. + * @return the success count + */ + public long getStoredFieldsPrefetchSuccess() { + return storedFieldsPrefetchSuccess; + } + + /** + * Returns the count of failed stored fields prefetches. + * @return the failure count + */ + public long getStoredFieldsPrefetchFailure() { + return storedFieldsPrefetchFailure; + } + + /** + * Returns the count of successful doc values prefetches. + * @return the success count + */ + public long getDocValuesPrefetchSuccess() { + return docValuesPrefetchSuccess; + } + + /** + * Returns the count of failed doc values prefetches. + * @return the failure count + */ + public long getDocValuesPrefetchFailure() { + return docValuesPrefetchFailure; + } + + /** + * Field names for XContent serialization. + * + * @opensearch.experimental + */ + static final class Fields { + static final String PREFETCH_STATS = "prefetch_stats"; + static final String STORED_FIELDS_PREFETCH_SUCCESS = "stored_fields_prefetch_success_count"; + static final String STORED_FIELDS_PREFETCH_FAILURE = "stored_fields_prefetch_failure_count"; + static final String DOC_VALUES_PREFETCH_SUCCESS = "doc_values_prefetch_success_count"; + static final String DOC_VALUES_PREFETCH_FAILURE = "doc_values_prefetch_failure_count"; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(Fields.PREFETCH_STATS); + builder.field(Fields.STORED_FIELDS_PREFETCH_SUCCESS, getStoredFieldsPrefetchSuccess()); + builder.field(Fields.STORED_FIELDS_PREFETCH_FAILURE, getStoredFieldsPrefetchFailure()); + builder.field(Fields.DOC_VALUES_PREFETCH_SUCCESS, getDocValuesPrefetchSuccess()); + builder.field(Fields.DOC_VALUES_PREFETCH_FAILURE, getDocValuesPrefetchFailure()); + builder.endObject(); + return builder; + } +} diff --git a/server/src/main/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetric.java b/server/src/main/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetric.java new file mode 100644 index 0000000000000..a43fa7d28e52a --- /dev/null +++ b/server/src/main/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetric.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.slowlogs; + +import org.apache.lucene.util.Accountable; + +/** + * Interface that needs to be implemented by any per query metric collector. + * + * @opensearch.experimental + */ +public interface TieredStoragePerQueryMetric extends Accountable { + + /** + * Records a file access event. + * @param blockFileName the block file name + * @param hit whether the access was a cache hit + */ + void recordFileAccess(String blockFileName, boolean hit); + + /** + * Records a prefetch event. + * @param fileName the file name + * @param blockId the block id + */ + void recordPrefetch(String fileName, int blockId); + + /** + * Records a read-ahead event. + * @param fileName the file name + * @param blockId the block id + */ + void recordReadAhead(String fileName, int blockId); + + /** Records the end time of the metric collection. */ + void recordEndTime(); + + /** + * Returns the parent task id. + * @return the parent task id + */ + String getParentTaskId(); + + /** + * Returns the shard id. + * @return the shard id + */ + String getShardId(); +} diff --git a/server/src/main/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImpl.java b/server/src/main/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImpl.java new file mode 100644 index 0000000000000..48bc8979b8165 --- /dev/null +++ b/server/src/main/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImpl.java @@ -0,0 +1,379 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.slowlogs; + +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Implementation for collecting tiered storage metrics at per query level. + * Tracks cache hits/misses, prefetch operations, and read-ahead operations + * for each file accessed during a query. + * + * @opensearch.experimental + */ +public class TieredStoragePerQueryMetricImpl implements TieredStoragePerQueryMetric, ToXContentObject { + + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(TieredStoragePerQueryMetricImpl.class); + private static final long FC_BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FileCacheStat.class); + private static final long PREFETCH_BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(PrefetchStat.class); + private static final long READ_AHEAD_BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ReadAheadStat.class); + + // File Cache stats will include hit/miss for both block and full file + protected final Map fileCacheStats; + + /** Prefetch stats per file. */ + protected final Map prefetchStats; + /** Prefetch file timestamps. */ + protected final Map prefetchFiles; + /** Read-ahead stats per file. */ + protected final Map readAheadStats; + /** Read-ahead file timestamps. */ + protected final Map readAheadFiles; + + /** Effective bytes transferred. */ + protected long effectiveBytes; + /** Total cache hits. */ + protected long hits; + /** Total cache misses. */ + protected long miss; + private final String parentTaskId; + private final String shardId; + private final long startTime; + private long endTime; + + /** + * Creates a new per-query metric collector. + * @param parentTaskId the parent task id + * @param shardId the shard id + */ + public TieredStoragePerQueryMetricImpl(String parentTaskId, String shardId) { + this.parentTaskId = parentTaskId; + this.shardId = shardId; + this.fileCacheStats = new HashMap<>(); + this.prefetchStats = new HashMap<>(); + this.prefetchFiles = new HashMap<>(); + this.readAheadStats = new HashMap<>(); + this.readAheadFiles = new HashMap<>(); + this.effectiveBytes = 0L; + this.hits = 0L; + this.miss = 0L; + this.startTime = System.currentTimeMillis(); + this.endTime = 0L; + } + + private FileBlock getFileBlock(String blockFileName) { + String[] fileParts = blockFileName.split("[.]", -1); + String fileName = fileParts[0]; + String[] blocks = fileParts[1].split("_", -1); + fileName = fileName + blocks[0]; + if (fileParts.length == 2 && blocks.length == 3) { + // ignore the 4th part which is the block extension + return new FileBlock(fileName, Integer.parseInt(blocks[2])); + } else { + assert false : "getFileBlock called with invalid block name, possibly without the extension"; + return new FileBlock(blockFileName, -1); + } + } + + @Override + public void recordFileAccess(String blockFileName, boolean hit) { + final FileBlock fileBlock = getFileBlock(blockFileName); + FileCacheStat fileCacheStat = this.fileCacheStats.get(fileBlock.fileName); + if (fileCacheStat == null) { + fileCacheStat = new FileCacheStat(); + this.fileCacheStats.put(fileBlock.fileName, fileCacheStat); + } + if (hit) { + fileCacheStat.hits++; + this.hits++; + fileCacheStat.hitBlocks.add(fileBlock.blockId); + } else { + fileCacheStat.miss++; + this.miss++; + fileCacheStat.missBlocks.add(fileBlock.blockId); + } + } + + @Override + public void recordPrefetch(String fileName, int blockId) { + if (!this.prefetchFiles.containsKey(fileName)) { + this.prefetchFiles.put(fileName, System.currentTimeMillis()); + this.prefetchStats.put(fileName, new PrefetchStat()); + } + this.prefetchStats.get(fileName).prefetchBlocks.add(blockId); + } + + @Override + public void recordReadAhead(String fileName, int blockId) { + if (!this.readAheadFiles.containsKey(fileName)) { + this.readAheadFiles.put(fileName, System.currentTimeMillis()); + this.readAheadStats.put(fileName, new ReadAheadStat()); + } + this.readAheadStats.get(fileName).readAheadBlocks.add(blockId); + } + + @Override + public long ramBytesUsed() { + long size = BASE_RAM_BYTES_USED; + // While this is not completely accurate, it serves as + // good approximation for tracking any memory leaks + size += RamUsageEstimator.sizeOf(fileCacheStats.values().toArray(new FileCacheStat[0])); + return size; + } + + @Override + public void recordEndTime() { + this.endTime = System.currentTimeMillis(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("parentTask", parentTaskId); + builder.field("shardId", shardId); + + // Summary section + builder.startObject("summary"); + builder.field("fileCache", String.format(Locale.ROOT, "%d hits out of %d total", this.hits, this.hits + this.miss)); + builder.field("prefetchFiles", this.prefetchFiles); + builder.field("readAheadFiles", this.readAheadFiles); + builder.endObject(); + + // Details section + builder.startObject("details"); + + // File cache details + builder.startObject("fileCache"); + for (Map.Entry entry : this.fileCacheStats.entrySet()) { + builder.startObject(entry.getKey()); + entry.getValue().toXContent(builder, params); + builder.endObject(); + } + builder.endObject(); + + // Prefetch details + // Prefetch details + builder.startObject("prefetch"); + for (Map.Entry entry : this.prefetchStats.entrySet()) { + builder.startObject(entry.getKey()); + entry.getValue().toXContent(builder, params); + builder.endObject(); + } + builder.endObject(); + + // ReadAhead details + builder.startObject("readAhead"); + for (Map.Entry entry : this.readAheadStats.entrySet()) { + builder.startObject(entry.getKey()); + entry.getValue().toXContent(builder, params); + builder.endObject(); + } + builder.endObject(); + + builder.endObject(); // end details + + // Timestamps section + builder.startObject("timestamps"); + builder.field("startTime", this.startTime); + builder.field("endTime", this.endTime); + builder.endObject(); + + builder.endObject(); + return builder; + } + + @Override + public String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder.toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public String getParentTaskId() { + return parentTaskId; + } + + @Override + public String getShardId() { + return shardId; + } + + private long getSetSize(Set set) { + // While this is not completely accurate, it serves as + // good approximation for tracking any memory leaks + long size = RamUsageEstimator.shallowSizeOf(set); + size += set.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF; + size += set.size() * Integer.BYTES; + return size; + } + + private class FileBlock { + final String fileName; + final int blockId; + + FileBlock(String fileName, int blockId) { + this.fileName = fileName; + this.blockId = blockId; + } + } + + /** + * Tracks file cache hit/miss statistics per file. + * + * @opensearch.experimental + */ + protected class FileCacheStat implements Accountable, ToXContent { + /** Number of cache hits. */ + public long hits; + /** Number of cache misses. */ + public long miss; + /** Set of block IDs that were cache hits. */ + public Set hitBlocks; + /** Set of block IDs that were cache misses. */ + public Set missBlocks; + + /** Creates a new FileCacheStat instance. */ + public FileCacheStat() { + this.hits = 0L; + this.miss = 0L; + this.hitBlocks = new HashSet<>(); + this.missBlocks = new HashSet<>(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("hits", this.hits); + builder.field("miss", this.miss); + builder.field("total", this.hits + this.miss); + + if (!hitBlocks.isEmpty() || !missBlocks.isEmpty()) { + builder.startObject("blockDetails"); + builder.field("hitBlockCount", this.hitBlocks.size()); + builder.field("hitBlocks", this.hitBlocks); + builder.field("missBlockCount", this.missBlocks.size()); + builder.field("missBlocks", this.missBlocks); + builder.endObject(); + } + + return builder; + } + + @Override + public String toString() { + // Full file case + if (hitBlocks.isEmpty() && missBlocks.isEmpty()) { + return String.format(Locale.ROOT, "%d hits out of %d total", this.hits, this.hits + this.miss); + } else { + return String.format( + Locale.ROOT, + "%d hits out of %d total, %d distinct hit blocks - %s, %d distinct miss blocks - %s", + this.hits, + this.hits + this.miss, + this.hitBlocks.size(), + this.hitBlocks, + this.missBlocks.size(), + this.missBlocks + ); + } + } + + @Override + public long ramBytesUsed() { + long size = FC_BASE_RAM_BYTES_USED; + size += getSetSize(hitBlocks); + size += getSetSize(missBlocks); + return size; + } + } + + /** + * Tracks read-ahead statistics per file. + * + * @opensearch.experimental + */ + protected class ReadAheadStat implements Accountable, ToXContent { + /** Set of block IDs that were read ahead. */ + public Set readAheadBlocks; + + /** Creates a new ReadAheadStat instance. */ + public ReadAheadStat() { + this.readAheadBlocks = new HashSet<>(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("blockCount", this.readAheadBlocks.size()); + builder.field("blocks", this.readAheadBlocks); + return builder; + } + + @Override + public String toString() { + return String.format(Locale.ROOT, "%d distinct submitted blocks - %s,", this.readAheadBlocks.size(), this.readAheadBlocks); + } + + @Override + public long ramBytesUsed() { + long size = READ_AHEAD_BASE_RAM_BYTES_USED; + size += getSetSize(readAheadBlocks); + return size; + } + } + + /** + * Tracks prefetch statistics per file. + * + * @opensearch.experimental + */ + protected class PrefetchStat implements Accountable, ToXContent { + /** Set of block IDs that were prefetched. */ + public Set prefetchBlocks; + + /** Creates a new PrefetchStat instance. */ + public PrefetchStat() { + this.prefetchBlocks = new HashSet<>(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("blockCount", this.prefetchBlocks.size()); + builder.field("blocks", this.prefetchBlocks); + return builder; + } + + @Override + public String toString() { + return String.format(Locale.ROOT, "%d distinct submitted blocks - %s", this.prefetchBlocks.size(), this.prefetchBlocks); + } + + @Override + public long ramBytesUsed() { + long size = PREFETCH_BASE_RAM_BYTES_USED; + size += getSetSize(prefetchBlocks); + return size; + } + } +} diff --git a/server/src/main/java/org/opensearch/storage/slowlogs/TieredStorageQueryMetricService.java b/server/src/main/java/org/opensearch/storage/slowlogs/TieredStorageQueryMetricService.java new file mode 100644 index 0000000000000..a239e9d67d295 --- /dev/null +++ b/server/src/main/java/org/opensearch/storage/slowlogs/TieredStorageQueryMetricService.java @@ -0,0 +1,303 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.slowlogs; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.RamUsageEstimator; +import org.opensearch.common.metrics.CounterMetric; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Singleton service for maintaining per-query metric collectors across threads. + * Provides thread-safe access to metric collectors during query and fetch phases. + * + * @opensearch.experimental + */ +public class TieredStorageQueryMetricService { + + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(TieredStorageQueryMetricService.class); + + private static final Logger logger = LogManager.getLogger(TieredStorageQueryMetricService.class); + + private static final TieredStorageQueryMetricService INSTANCE = new TieredStorageQueryMetricService(); + + /** + * Map of thread ID to active collector. Only one collector is active per thread at a time. + */ + protected final ConcurrentMap metricCollectors = new ConcurrentHashMap<>(); + + /** + * Map of task id + shard id to set of collectors for query phase. + * Multiple threads can work on the same shard concurrently during concurrent segment search. + */ + protected final ConcurrentMap> taskIdToQueryPhaseCollectorMap = new ConcurrentHashMap<>(); + + /** + * Map of task id + shard id to set of collectors for fetch phase. + */ + protected final ConcurrentMap> taskIdToFetchPhaseCollectorMap = new ConcurrentHashMap<>(); + + private final PrefetchStatsHolder prefetchStats = new PrefetchStatsHolder(); + + private static final int MAX_PER_QUERY_COLLECTOR_SIZE = 1000; + + private TieredStorageQueryMetricService() {} + + /** + * Returns the singleton instance. + * @return the singleton instance + */ + public static TieredStorageQueryMetricService getInstance() { + return INSTANCE; + } + + /** + * Returns the per-query metric collector for the given thread. + * Returns a no-op dummy collector if none exists. + * @param threadId the thread id + * @return the metric collector for the thread + */ + public TieredStoragePerQueryMetric getMetricCollector(final long threadId) { + return metricCollectors.getOrDefault(threadId, TieredStoragePerQueryMetricDummy.getInstance()); + } + + /** + * Adds a metric collector for the given thread. Enforces a hard limit on the + * number of collectors to prevent excessive memory consumption. + * @param threadId the thread id + * @param metricCollector the metric collector + * @param isQueryPhase true if this is for the query phase, false for fetch phase + */ + public void addMetricCollector(final long threadId, final TieredStoragePerQueryMetric metricCollector, boolean isQueryPhase) { + // TODO if possible add thread id in collector + if (metricCollectors.size() >= MAX_PER_QUERY_COLLECTOR_SIZE + || taskIdToQueryPhaseCollectorMap.values().stream().mapToInt(Set::size).sum() >= MAX_PER_QUERY_COLLECTOR_SIZE + || taskIdToFetchPhaseCollectorMap.values().stream().mapToInt(Set::size).sum() >= MAX_PER_QUERY_COLLECTOR_SIZE) { + logger.error( + "Number of metric collectors already equals maximum size of " + + MAX_PER_QUERY_COLLECTOR_SIZE + + ". Skipping. Current sizes - metricCollectors: " + + metricCollectors.size() + + ", queryPhaseCollectors: " + + taskIdToQueryPhaseCollectorMap.values().stream().mapToInt(Set::size).sum() + + ", fetchPhaseCollectors: " + + taskIdToFetchPhaseCollectorMap.values().stream().mapToInt(Set::size).sum() + ); + } else { + // The same threadId will not be used concurrently, so below is safe + metricCollectors.put(threadId, metricCollector); + // Multiple threads can be working on the same shard at the same time though, so below needs to be atomic + if (isQueryPhase) { + taskIdToQueryPhaseCollectorMap.compute( + metricCollector.getParentTaskId() + metricCollector.getShardId(), + (id, collectors) -> { + Set newCollectors = (collectors == null) ? new HashSet<>() : collectors; + newCollectors.add(metricCollector); + return newCollectors; + } + ); + } else { + taskIdToFetchPhaseCollectorMap.compute( + metricCollector.getParentTaskId() + metricCollector.getShardId(), + (id, collectors) -> { + Set newCollectors = (collectors == null) ? new HashSet<>() : collectors; + newCollectors.add(metricCollector); + return newCollectors; + } + ); + } + } + } + + /** + * Removes the metric collector for the given thread and records its end time. + * @param threadId the thread id + * @return the removed metric collector, or null if none existed + */ + public TieredStoragePerQueryMetric removeMetricCollector(final long threadId) { + // Do not update taskIdToCollectorMap here as the query may not be complete + // For safety, use getOrDefault here + metricCollectors.getOrDefault(threadId, TieredStoragePerQueryMetricDummy.getInstance()).recordEndTime(); + return metricCollectors.remove(threadId); + } + + /** + * Removes all metric collectors for the given task and shard combination. + * @param parentTaskId the parent task id + * @param shardId the shard id + * @param isQueryPhase true for query phase collectors, false for fetch phase + * @return the set of removed collectors + */ + public Set removeMetricCollectors(String parentTaskId, String shardId, boolean isQueryPhase) { + final Set collectors; + if (isQueryPhase) { + collectors = taskIdToQueryPhaseCollectorMap.remove(parentTaskId + shardId); + } else { + collectors = taskIdToFetchPhaseCollectorMap.remove(parentTaskId + shardId); + } + if (collectors == null) { + // Slice Execution hooks will not be triggered in the case of a cache hit, however query phase hooks will always be triggered + return Collections.emptySet(); + } + return collectors; + } + + /** + * Returns the task-to-collector map for testing. + * @param isQueryPhase true for query phase map, false for fetch phase + * @return the task-to-collector map + */ + Map> getTaskIdToCollectorMap(boolean isQueryPhase) { + return isQueryPhase ? taskIdToQueryPhaseCollectorMap : taskIdToFetchPhaseCollectorMap; + } + + /** + * Returns the metric collectors map for testing. + * @return the metric collectors map + */ + Map getMetricCollectors() { + return metricCollectors; + } + + /** + * Returns estimated memory consumption of the metric service. + * @return ram bytes usage + */ + public long ramBytesUsed() { + long size = BASE_RAM_BYTES_USED; + // While this is not completely accurate, it serves as good approximation for tracking any memory leaks + // Each collector in metricCollectors will also be referenced in taskIdToCollectorMap, however the opposite is not true. + // Therefore, we use taskIdToCollectorMap to estimate ram usage. + for (Set collectors : taskIdToQueryPhaseCollectorMap.values()) { + size += RamUsageEstimator.sizeOf(collectors.toArray(new TieredStoragePerQueryMetric[0])); + } + for (Set collectors : taskIdToFetchPhaseCollectorMap.values()) { + size += RamUsageEstimator.sizeOf(collectors.toArray(new TieredStoragePerQueryMetric[0])); + } + return size; + } + + /** + * Records a stored fields prefetch event. + * @param success true if the prefetch was successful + */ + public void recordStoredFieldsPrefetch(boolean success) { + if (success) { + prefetchStats.storedFieldsPrefetchSuccess.inc(); + } else { + prefetchStats.storedFieldsPrefetchFailure.inc(); + } + } + + /** + * Records a doc values prefetch event. + * @param success true if the prefetch was successful + */ + public void recordDocValuesPrefetch(boolean success) { + if (success) { + prefetchStats.docValuesPrefetchSuccess.inc(); + } else { + prefetchStats.docValuesPrefetchFailure.inc(); + } + } + + /** + * Returns the current prefetch stats. + * @return the prefetch stats + */ + // TODO has to emit as part of node stats + public PrefetchStats getPrefetchStats() { + return this.prefetchStats.getStats(); + } + + /** + * No-op dummy metric collector to avoid null checks throughout the codebase. + * + * @opensearch.experimental + */ + static class TieredStoragePerQueryMetricDummy implements TieredStoragePerQueryMetric { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(TieredStoragePerQueryMetricDummy.class); + private static final TieredStoragePerQueryMetricDummy INSTANCE = new TieredStoragePerQueryMetricDummy(); + + /** + * Returns the singleton dummy instance. + * @return the dummy instance + */ + public static TieredStoragePerQueryMetricDummy getInstance() { + return INSTANCE; + } + + private TieredStoragePerQueryMetricDummy() {} + + @Override + public void recordFileAccess(String blockFileName, boolean hit) { + // Do nothing + } + + @Override + public void recordEndTime() {} + + @Override + public void recordPrefetch(String fileName, int blockId) {} + + @Override + public void recordReadAhead(String fileName, int blockId) {} + + @Override + public String getParentTaskId() { + return "DummyParentTaskId"; + } + + @Override + public String getShardId() { + return "DummyShardId"; + } + + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED; + } + } + + /** + * Holder for prefetch statistics counters. + * + * @opensearch.experimental + */ + public static final class PrefetchStatsHolder { + /** Counter for successful stored fields prefetches. */ + final CounterMetric storedFieldsPrefetchSuccess = new CounterMetric(); + /** Counter for failed stored fields prefetches. */ + final CounterMetric storedFieldsPrefetchFailure = new CounterMetric(); + /** Counter for successful doc values prefetches. */ + final CounterMetric docValuesPrefetchSuccess = new CounterMetric(); + /** Counter for failed doc values prefetches. */ + final CounterMetric docValuesPrefetchFailure = new CounterMetric(); + + /** + * Returns the current prefetch stats snapshot. + * @return the prefetch stats + */ + PrefetchStats getStats() { + return new PrefetchStats( + storedFieldsPrefetchSuccess.count(), + storedFieldsPrefetchFailure.count(), + docValuesPrefetchSuccess.count(), + docValuesPrefetchFailure.count() + ); + } + } +} diff --git a/server/src/main/java/org/opensearch/storage/slowlogs/TieredStorageSearchSlowLog.java b/server/src/main/java/org/opensearch/storage/slowlogs/TieredStorageSearchSlowLog.java new file mode 100644 index 0000000000000..0c0711d44b06f --- /dev/null +++ b/server/src/main/java/org/opensearch/storage/slowlogs/TieredStorageSearchSlowLog.java @@ -0,0 +1,577 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.slowlogs; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchShardTask; +import org.opensearch.common.logging.Loggers; +import org.opensearch.common.logging.SlowLogLevel; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Setting.Property; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.shard.SearchOperationListener; +import org.opensearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Search slow log implementation for tiered storage (warm data). + * Logs slow queries and fetches with per-query cache and prefetch metrics. + * + * @opensearch.experimental + */ +public final class TieredStorageSearchSlowLog implements SearchOperationListener { + + private volatile boolean tieredStorageSlowlogEnabled; + private volatile long queryWarnThreshold; + private volatile long queryInfoThreshold; + private volatile long queryDebugThreshold; + private volatile long queryTraceThreshold; + + private volatile long fetchWarnThreshold; + private volatile long fetchInfoThreshold; + private volatile long fetchDebugThreshold; + private volatile long fetchTraceThreshold; + + private SlowLogLevel level; + + private final Logger queryLogger; + private final Logger fetchLogger; + + /** Settings prefix for tiered storage search slow log. */ + public static final String TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX = "index.tiered.storage.slowlog"; + + /** Setting to enable or disable tiered storage search slow log. */ + public static final Setting TIERED_STORAGE_SEARCH_SLOWLOG_ENABLED = Setting.boolSetting( + TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".enabled", + false, + Property.Dynamic, + Property.IndexScope + ); + + /** Query warn threshold setting. */ + public static final Setting INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING = Setting.timeSetting( + TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".threshold.query.warn", + TimeValue.timeValueMillis(10000), + TimeValue.timeValueMillis(-1), + Property.Dynamic, + Property.IndexScope + ); + + /** Query info threshold setting. */ + public static final Setting INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING = Setting.timeSetting( + TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".threshold.query.info", + TimeValue.timeValueMillis(5000), + TimeValue.timeValueMillis(-1), + Property.Dynamic, + Property.IndexScope + ); + + /** Query debug threshold setting. */ + public static final Setting INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING = Setting.timeSetting( + TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".threshold.query.debug", + TimeValue.timeValueMillis(2000), + TimeValue.timeValueMillis(-1), + Property.Dynamic, + Property.IndexScope + ); + + /** Query trace threshold setting. */ + public static final Setting INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING = Setting.timeSetting( + TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".threshold.query.trace", + TimeValue.timeValueMillis(500), + TimeValue.timeValueMillis(-1), + Property.Dynamic, + Property.IndexScope + ); + + /** Fetch warn threshold setting. */ + public static final Setting INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING = Setting.timeSetting( + TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.warn", + TimeValue.timeValueMillis(10000), + TimeValue.timeValueMillis(-1), + Property.Dynamic, + Property.IndexScope + ); + + /** Fetch info threshold setting. */ + public static final Setting INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING = Setting.timeSetting( + TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.info", + TimeValue.timeValueMillis(5000), + TimeValue.timeValueMillis(-1), + Property.Dynamic, + Property.IndexScope + ); + + /** Fetch debug threshold setting. */ + public static final Setting INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING = Setting.timeSetting( + TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.debug", + TimeValue.timeValueMillis(2000), + TimeValue.timeValueMillis(-1), + Property.Dynamic, + Property.IndexScope + ); + + /** Fetch trace threshold setting. */ + public static final Setting INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING = Setting.timeSetting( + TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.trace", + TimeValue.timeValueMillis(500), + TimeValue.timeValueMillis(-1), + Property.Dynamic, + Property.IndexScope + ); + + /** Slow log level setting. */ + public static final Setting INDEX_SEARCH_SLOWLOG_LEVEL = new Setting<>( + TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".level", + SlowLogLevel.TRACE.name(), + SlowLogLevel::parse, + Property.Dynamic, + Property.IndexScope + ); + + /** Map of all tiered storage search slow log settings keyed by setting name. */ + public static final Map> TIERED_STORAGE_SEARCH_SLOWLOG_SETTINGS_MAP = Collections.unmodifiableMap(new HashMap<>() { + { + put(TIERED_STORAGE_SEARCH_SLOWLOG_ENABLED.getKey(), TIERED_STORAGE_SEARCH_SLOWLOG_ENABLED); + put(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING.getKey(), INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING); + put(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING.getKey(), INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING); + put(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING.getKey(), INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING); + put(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING.getKey(), INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING); + put(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING.getKey(), INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING); + put(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING.getKey(), INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING); + put(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING.getKey(), INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING); + put(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING.getKey(), INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING); + put(INDEX_SEARCH_SLOWLOG_LEVEL.getKey(), INDEX_SEARCH_SLOWLOG_LEVEL); + } + }); + + /** Set of all tiered storage search slow log settings. */ + public static final Set> TIERED_STORAGE_SEARCH_SLOWLOG_SETTINGS = Collections.unmodifiableSet( + new HashSet<>(TIERED_STORAGE_SEARCH_SLOWLOG_SETTINGS_MAP.values()) + ); + + private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false")); + + /** + * Creates a new TieredStorageSearchSlowLog instance. + * @param indexSettings the index settings + */ + public TieredStorageSearchSlowLog(IndexSettings indexSettings) { + this.queryLogger = LogManager.getLogger(TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".query"); + this.fetchLogger = LogManager.getLogger(TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".fetch"); + + indexSettings.getScopedSettings() + .addSettingsUpdateConsumer(TIERED_STORAGE_SEARCH_SLOWLOG_ENABLED, this::setTieredStorageSlowlogEnabled); + setTieredStorageSlowlogEnabled(indexSettings.getValue(TIERED_STORAGE_SEARCH_SLOWLOG_ENABLED)); + + indexSettings.getScopedSettings() + .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING, this::setQueryWarnThreshold); + setQueryWarnThreshold(indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING)); + indexSettings.getScopedSettings() + .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING, this::setQueryInfoThreshold); + setQueryInfoThreshold(indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING)); + indexSettings.getScopedSettings() + .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING, this::setQueryDebugThreshold); + setQueryDebugThreshold(indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING)); + indexSettings.getScopedSettings() + .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING, this::setQueryTraceThreshold); + setQueryTraceThreshold(indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING)); + + indexSettings.getScopedSettings() + .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING, this::setFetchWarnThreshold); + setFetchWarnThreshold(indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING)); + indexSettings.getScopedSettings() + .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING, this::setFetchInfoThreshold); + setFetchInfoThreshold(indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING)); + indexSettings.getScopedSettings() + .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING, this::setFetchDebugThreshold); + setFetchDebugThreshold(indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING)); + indexSettings.getScopedSettings() + .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING, this::setFetchTraceThreshold); + setFetchTraceThreshold(indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING)); + + indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_LEVEL, this::setLevel); + setLevel(indexSettings.getValue(INDEX_SEARCH_SLOWLOG_LEVEL)); + } + + private void setLevel(SlowLogLevel level) { + this.level = level; + Loggers.setLevel(queryLogger, level.name()); + Loggers.setLevel(fetchLogger, level.name()); + } + + private TieredStoragePerQueryMetric removeMetricCollector() { + return TieredStorageQueryMetricService.getInstance().removeMetricCollector(Thread.currentThread().threadId()); + } + + private Set removeMetricCollectors(String parentTaskId, String shardId, boolean isQueryPhase) { + return TieredStorageQueryMetricService.getInstance().removeMetricCollectors(parentTaskId, shardId, isQueryPhase); + } + + private void setMetricCollector(SearchContext searchContext, boolean isQueryPhase) { + final SearchShardTask searchTask = searchContext.getTask(); + final Logger log = isQueryPhase ? queryLogger : fetchLogger; + if (searchTask == null) { + log.error("Warm Slow Log: Search Task not expected to be null"); + } + TieredStorageQueryMetricService.getInstance() + .addMetricCollector( + Thread.currentThread().threadId(), + new TieredStoragePerQueryMetricImpl( + searchTask == null ? null : searchTask.getParentTaskId().toString(), + searchContext.shardTarget().getShardId().toString() + ), + isQueryPhase + ); + } + + @Override + public void onPreQueryPhase(SearchContext searchContext) { + // The same search thread can pick up multiple slice executions post https://github.com/apache/lucene/pull/13472 + // so we initialize collectors only in onPreSliceExecution + } + + @Override + public void onFailedQueryPhase(SearchContext searchContext) { + // Only clean up if we were collecting metrics + if (tieredStorageSlowlogEnabled) { + removeMetricCollector(); + removeMetricCollectors( + searchContext.getTask().getParentTaskId().toString(), + searchContext.shardTarget().getShardId().toString(), + true + ); + } + } + + @Override + public void onQueryPhase(SearchContext context, long tookInNanos) { + // Get all collectors associated with the task/shard + final List metricCollectors = new ArrayList<>( + removeMetricCollectors(context.getTask().getParentTaskId().toString(), context.shardTarget().getShardId().toString(), true) + ); + + // No need to call removeMetricCollector() here as that will be handled in onSliceExecution in both + // concurrent search and non-concurrent search cases + + // Only log if tiered storage slow log is enabled + if (tieredStorageSlowlogEnabled) { + printSlowLog( + context, + tookInNanos, + metricCollectors, + queryWarnThreshold, + queryLogger, + queryInfoThreshold, + queryDebugThreshold, + queryTraceThreshold + ); + } + } + + private void printSlowLog( + SearchContext context, + long tookInNanos, + List metricCollectors, + long warnThreshold, + Logger log, + long infoThreshold, + long debugThreshold, + long traceThreshold + ) { + if (warnThreshold >= 0 && tookInNanos > warnThreshold) { + log.warn("{}", new TieredStorageSlowLogPrinter(context, tookInNanos, metricCollectors)); + } else if (infoThreshold >= 0 && tookInNanos > infoThreshold) { + log.info("{}", new TieredStorageSlowLogPrinter(context, tookInNanos, metricCollectors)); + } else if (debugThreshold >= 0 && tookInNanos > debugThreshold) { + log.debug("{}", new TieredStorageSlowLogPrinter(context, tookInNanos, metricCollectors)); + } else if (traceThreshold >= 0 && tookInNanos > traceThreshold) { + log.trace("{}", new TieredStorageSlowLogPrinter(context, tookInNanos, metricCollectors)); + } + } + + @Override + public void onPreSliceExecution(SearchContext searchContext) { + // Only collect metrics if tiered storage slow log is enabled + if (tieredStorageSlowlogEnabled) { + setMetricCollector(searchContext, true); + } + } + + @Override + public void onFailedSliceExecution(SearchContext searchContext) { + // Only clean up if we were collecting metrics + if (tieredStorageSlowlogEnabled) { + removeMetricCollector(); + } + } + + @Override + public void onSliceExecution(SearchContext searchContext) { + // Only clean up if we were collecting metrics + if (tieredStorageSlowlogEnabled) { + removeMetricCollector(); + } + } + + @Override + public void onPreFetchPhase(SearchContext searchContext) { + // Fetch phase execution is starting. Add new metric collector only if enabled + if (tieredStorageSlowlogEnabled) { + setMetricCollector(searchContext, false); + } + } + + @Override + public void onFailedFetchPhase(SearchContext searchContext) { + // Only clean up if we were collecting metrics + if (tieredStorageSlowlogEnabled) { + removeMetricCollector(); + removeMetricCollectors( + searchContext.getTask().getParentTaskId().toString(), + searchContext.shardTarget().getShardId().toString(), + false + ); + } + } + + @Override + public void onFetchPhase(SearchContext context, long tookInNanos) { + // Only clean up and log if we were collecting metrics + if (tieredStorageSlowlogEnabled) { + removeMetricCollector(); + // Although fetch phase is single threaded today, we will use the same map implementation for posterity. + // It's also much cleaner than propagating the fetch boolean to TieredStorageQueryMetricService + final List metricCollectors = new ArrayList<>( + removeMetricCollectors(context.getTask().getParentTaskId().toString(), context.shardTarget().getShardId().toString(), false) + ); + assert metricCollectors.size() == 1 : "Fetch phase is expected to be single threaded, so we should only have 1 collector"; + + printSlowLog( + context, + tookInNanos, + metricCollectors, + fetchWarnThreshold, + fetchLogger, + fetchInfoThreshold, + fetchDebugThreshold, + fetchTraceThreshold + ); + } + } + + /** + * Formats slow log output as JSON with warm storage metrics. + * + * @opensearch.experimental + */ + static final class TieredStorageSlowLogPrinter { + private final SearchContext context; + private final long tookInNanos; + private final List metricCollectors; + private final Logger logger = LogManager.getLogger(TieredStorageSlowLogPrinter.class); + + /** + * Creates a new slow log printer. + * @param context the search context + * @param tookInNanos the time taken in nanoseconds + * @param metricCollectors the per-query metric collectors + */ + TieredStorageSlowLogPrinter(SearchContext context, long tookInNanos, List metricCollectors) { + this.context = context; + this.tookInNanos = tookInNanos; + this.metricCollectors = metricCollectors; + } + + @Override + public String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + toXContent(builder, FORMAT_PARAMS); + return builder.toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + + // warm_stats array + builder.startArray("warm_stats"); + if (metricCollectors != null && !metricCollectors.isEmpty()) { + for (TieredStoragePerQueryMetric collector : metricCollectors) { + builder.value(collector.toString()); + } + } + builder.endArray(); + + // took and took_millis + builder.field("took", TimeValue.timeValueNanos(tookInNanos).toString()); + builder.field("took_millis", TimeUnit.NANOSECONDS.toMillis(tookInNanos)); + + // stats array + builder.startArray("stats"); + List stats = new ArrayList<>(); + if (context.groupStats() != null) { + if (metricCollectors != null) { + stats.addAll(metricCollectors); + } + stats.addAll(Objects.requireNonNull(context.groupStats())); + } + if (!stats.isEmpty()) { + for (Object stat : stats) { + builder.value(stat.toString()); + } + } + builder.endArray(); + + // search_type, total_shards, and source + builder.field("search_type", context.searchType().toString()); + builder.field("total_shards", context.numberOfShards()); + + if (context.request().source() != null) { + builder.field("source", context.request().source().toString(params)); + } else { + builder.nullField("source"); + } + + builder.endObject(); + return builder; + } + } + + /** + * Sets whether tiered storage slow log is enabled. + * @param tieredStorageSlowlogEnabled true to enable + */ + public void setTieredStorageSlowlogEnabled(boolean tieredStorageSlowlogEnabled) { + this.tieredStorageSlowlogEnabled = tieredStorageSlowlogEnabled; + } + + private void setQueryWarnThreshold(TimeValue warnThreshold) { + this.queryWarnThreshold = warnThreshold.nanos(); + } + + private void setQueryInfoThreshold(TimeValue infoThreshold) { + this.queryInfoThreshold = infoThreshold.nanos(); + } + + private void setQueryDebugThreshold(TimeValue debugThreshold) { + this.queryDebugThreshold = debugThreshold.nanos(); + } + + private void setQueryTraceThreshold(TimeValue traceThreshold) { + this.queryTraceThreshold = traceThreshold.nanos(); + } + + private void setFetchWarnThreshold(TimeValue warnThreshold) { + this.fetchWarnThreshold = warnThreshold.nanos(); + } + + private void setFetchInfoThreshold(TimeValue infoThreshold) { + this.fetchInfoThreshold = infoThreshold.nanos(); + } + + private void setFetchDebugThreshold(TimeValue debugThreshold) { + this.fetchDebugThreshold = debugThreshold.nanos(); + } + + private void setFetchTraceThreshold(TimeValue traceThreshold) { + this.fetchTraceThreshold = traceThreshold.nanos(); + } + + /** + * Returns the query warn threshold in nanoseconds. + * @return the threshold + */ + long getQueryWarnThreshold() { + return queryWarnThreshold; + } + + /** + * Returns the query info threshold in nanoseconds. + * @return the threshold + */ + long getQueryInfoThreshold() { + return queryInfoThreshold; + } + + /** + * Returns the query debug threshold in nanoseconds. + * @return the threshold + */ + long getQueryDebugThreshold() { + return queryDebugThreshold; + } + + /** + * Returns the query trace threshold in nanoseconds. + * @return the threshold + */ + long getQueryTraceThreshold() { + return queryTraceThreshold; + } + + /** + * Returns the fetch warn threshold in nanoseconds. + * @return the threshold + */ + long getFetchWarnThreshold() { + return fetchWarnThreshold; + } + + /** + * Returns the fetch info threshold in nanoseconds. + * @return the threshold + */ + long getFetchInfoThreshold() { + return fetchInfoThreshold; + } + + /** + * Returns the fetch debug threshold in nanoseconds. + * @return the threshold + */ + long getFetchDebugThreshold() { + return fetchDebugThreshold; + } + + /** + * Returns the fetch trace threshold in nanoseconds. + * @return the threshold + */ + long getFetchTraceThreshold() { + return fetchTraceThreshold; + } + + /** + * Returns the current slow log level. + * @return the slow log level + */ + // TODO check this level + SlowLogLevel getLevel() { + return level; + } +} diff --git a/server/src/main/java/org/opensearch/storage/slowlogs/package-info.java b/server/src/main/java/org/opensearch/storage/slowlogs/package-info.java new file mode 100644 index 0000000000000..c9a8d26f4a384 --- /dev/null +++ b/server/src/main/java/org/opensearch/storage/slowlogs/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Slow logs and per-query metrics for tiered storage search operations. + */ +package org.opensearch.storage.slowlogs; diff --git a/server/src/test/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImplJsonTests.java b/server/src/test/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImplJsonTests.java new file mode 100644 index 0000000000000..94e3f83cd3373 --- /dev/null +++ b/server/src/test/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImplJsonTests.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.slowlogs; + +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +/** + * Test class to verify JSON serialization of TieredStoragePerQueryMetricImpl. + */ +public class TieredStoragePerQueryMetricImplJsonTests extends OpenSearchTestCase { + + public void testToXContentBasic() throws IOException { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-123", "shard-0"); + + // Record some sample data + metric.recordFileAccess("file1.block_0_1", true); // hit + metric.recordFileAccess("file1.block_0_2", false); // miss + metric.recordPrefetch("file2", 1); + metric.recordReadAhead("file3", 2); + metric.recordEndTime(); + + // Test XContentBuilder serialization + XContentBuilder builder = XContentFactory.jsonBuilder(); + metric.toXContent(builder, ToXContent.EMPTY_PARAMS); + String json = builder.toString(); + + // Verify JSON contains expected fields + assertNotNull(json); + assertTrue(json.contains("\"parentTask\":\"task-123\"")); + assertTrue(json.contains("\"shardId\":\"shard-0\"")); + assertTrue(json.contains("\"summary\"")); + assertTrue(json.contains("\"details\"")); + assertTrue(json.contains("\"timestamps\"")); + assertTrue(json.contains("\"fileCache\"")); + assertTrue(json.contains("\"prefetch\"")); + assertTrue(json.contains("\"readAhead\"")); + } + + public void testToStringUsesXContentBuilder() throws IOException { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-456", "shard-1"); + + // Record some sample data + metric.recordFileAccess("test.block_0_5", true); + metric.recordPrefetch("prefetch-file", 10); + metric.recordEndTime(); + + // Test toString method (which should use XContentBuilder internally) + String jsonString = metric.toString(); + + // Verify the toString output is valid JSON + assertNotNull(jsonString); + assertTrue(jsonString.contains("\"parentTask\":\"task-456\"")); + assertTrue(jsonString.contains("\"shardId\":\"shard-1\"")); + } + + public void testEmptyMetricSerialization() throws IOException { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("empty-task", "empty-shard"); + metric.recordEndTime(); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + metric.toXContent(builder, ToXContent.EMPTY_PARAMS); + String json = builder.toString(); + + // Should still contain basic structure even with no data + assertTrue(json.contains("\"parentTask\":\"empty-task\"")); + assertTrue(json.contains("\"shardId\":\"empty-shard\"")); + assertTrue(json.contains("\"summary\"")); + assertTrue(json.contains("\"details\"")); + } +} diff --git a/server/src/test/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImplTests.java b/server/src/test/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImplTests.java new file mode 100644 index 0000000000000..8fe18c0cbcc34 --- /dev/null +++ b/server/src/test/java/org/opensearch/storage/slowlogs/TieredStoragePerQueryMetricImplTests.java @@ -0,0 +1,330 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.slowlogs; + +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +/** + * Comprehensive unit tests for TieredStoragePerQueryMetricImpl. + */ +public class TieredStoragePerQueryMetricImplTests extends OpenSearchTestCase { + + public void testConstructor() { + String parentTaskId = "task-123"; + String shardId = "shard-0"; + + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl(parentTaskId, shardId); + + assertEquals(parentTaskId, metric.getParentTaskId()); + assertEquals(shardId, metric.getShardId()); + assertTrue(metric.ramBytesUsed() > 0); + } + + public void testRecordFileAccessHit() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Test block file access hit + metric.recordFileAccess("file1.block_0_1", true); + + // Verify internal state through XContent + String json = metric.toString(); + assertTrue(json.contains("\"hits\":1")); + assertTrue(json.contains("\"miss\":0")); + assertTrue(json.contains("\"total\":1")); + } + + public void testRecordFileAccessMiss() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Test block file access miss + metric.recordFileAccess("file1.block_0_1", false); + + // Verify internal state through XContent + String json = metric.toString(); + assertTrue(json.contains("\"hits\":0")); + assertTrue(json.contains("\"miss\":1")); + assertTrue(json.contains("\"total\":1")); + } + + public void testRecordFileAccessMultipleFiles() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Test multiple files with different blocks + metric.recordFileAccess("file1.block_0_1", true); // file1 hit + metric.recordFileAccess("file1.block_0_2", false); // file1 miss + metric.recordFileAccess("file2.block_0_1", true); // file2 hit + metric.recordFileAccess("file2.block_0_3", true); // file2 hit + + String json = metric.toString(); + + // Should have entries for both files + assertTrue(json.contains("file1block")); + assertTrue(json.contains("file2block")); + + // Overall stats should be aggregated + assertTrue(json.contains("\"fileCache\":\"3 hits out of 4 total\"")); + } + + public void testRecordFileAccessSameBlockMultipleTimes() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Record same block multiple times + metric.recordFileAccess("file1.block_0_1", true); + metric.recordFileAccess("file1.block_0_1", true); + metric.recordFileAccess("file1.block_0_1", false); + + String json = metric.toString(); + + // Should have 2 hits and 1 miss for total of 3 + assertTrue(json.contains("\"hits\":2")); + assertTrue(json.contains("\"miss\":1")); + assertTrue(json.contains("\"total\":3")); + + // But only 1 unique hit block and 1 unique miss block + assertTrue(json.contains("\"hitBlockCount\":1")); + assertTrue(json.contains("\"missBlockCount\":1")); + } + + public void testRecordPrefetch() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Test prefetch recording + metric.recordPrefetch("file1", 1); + metric.recordPrefetch("file1", 2); + metric.recordPrefetch("file2", 5); + + String json = metric.toString(); + + // Should have prefetch entries + assertTrue(json.contains("\"prefetch\"")); + assertTrue(json.contains("file1")); + assertTrue(json.contains("file2")); + assertTrue(json.contains("\"blockCount\":2")); // file1 has 2 blocks + assertTrue(json.contains("\"blockCount\":1")); // file2 has 1 block + } + + public void testRecordPrefetchSameBlockMultipleTimes() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Record same block multiple times - should only count once + metric.recordPrefetch("file1", 1); + metric.recordPrefetch("file1", 1); + metric.recordPrefetch("file1", 1); + + String json = metric.toString(); + + // Should only have 1 unique block + assertTrue(json.contains("\"blockCount\":1")); + assertTrue(json.contains("[1]")); + } + + public void testRecordReadAhead() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Test read ahead recording + metric.recordReadAhead("file1", 10); + metric.recordReadAhead("file1", 11); + metric.recordReadAhead("file2", 20); + + String json = metric.toString(); + + // Should have read ahead entries + assertTrue(json.contains("\"readAhead\"")); + assertTrue(json.contains("file1")); + assertTrue(json.contains("file2")); + assertTrue(json.contains("\"blockCount\":2")); // file1 has 2 blocks + assertTrue(json.contains("\"blockCount\":1")); // file2 has 1 block + } + + public void testRecordReadAheadSameBlockMultipleTimes() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Record same block multiple times - should only count once + metric.recordReadAhead("file1", 5); + metric.recordReadAhead("file1", 5); + metric.recordReadAhead("file1", 5); + + String json = metric.toString(); + + // Should only have 1 unique block + assertTrue(json.contains("\"blockCount\":1")); + assertTrue(json.contains("[5]")); + } + + public void testRecordEndTime() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + long startTime = System.currentTimeMillis(); + metric.recordEndTime(); + long endTime = System.currentTimeMillis(); + + String json = metric.toString(); + + // Should have timestamps + assertTrue(json.contains("\"timestamps\"")); + assertTrue(json.contains("\"startTime\"")); + assertTrue(json.contains("\"endTime\"")); + + // End time should be after start time and before current time + assertTrue(json.contains("\"endTime\":") && !json.contains("\"endTime\":0")); + } + + public void testGetFileBlockParsing() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Test various block file name formats - use proper format: filename.extension_blockId_blockNumber + // Format: filename.extension_blockId_blockNumber where blockNumber is numeric + metric.recordFileAccess("segments_1.block_0_123", true); + metric.recordFileAccess("_0.cfs_456_789", false); + metric.recordFileAccess("test.dat_0_999", true); + + String json = metric.toString(); + + // Should parse file names correctly - filename + first part of extension + assertTrue(json.contains("segments_1block")); + assertTrue(json.contains("_0cfs")); + assertTrue(json.contains("testdat")); + } + + public void testRamBytesUsed() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + long initialRam = metric.ramBytesUsed(); + assertTrue(initialRam > 0); + + // Add some data and verify RAM usage increases + metric.recordFileAccess("file1.block_0_1", true); + metric.recordFileAccess("file1.block_0_2", false); + metric.recordPrefetch("file2", 1); + metric.recordReadAhead("file3", 1); + + long finalRam = metric.ramBytesUsed(); + assertTrue(finalRam >= initialRam); + } + + public void testToXContentStructure() throws IOException { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-123", "shard-456"); + + // Add comprehensive test data + metric.recordFileAccess("file1.block_0_1", true); + metric.recordFileAccess("file1.block_0_2", false); + metric.recordPrefetch("prefetch-file", 10); + metric.recordReadAhead("readahead-file", 20); + metric.recordEndTime(); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + metric.toXContent(builder, ToXContent.EMPTY_PARAMS); + String json = builder.toString(); + + // Verify complete structure + assertTrue(json.contains("\"parentTask\":\"task-123\"")); + assertTrue(json.contains("\"shardId\":\"shard-456\"")); + + // Summary section + assertTrue(json.contains("\"summary\"")); + assertTrue(json.contains("\"fileCache\":\"1 hits out of 2 total\"")); + assertTrue(json.contains("\"prefetchFiles\"")); + assertTrue(json.contains("\"readAheadFiles\"")); + + // Details section + assertTrue(json.contains("\"details\"")); + assertTrue(json.contains("\"fileCache\"")); + assertTrue(json.contains("\"prefetch\"")); + assertTrue(json.contains("\"readAhead\"")); + + // Timestamps section + assertTrue(json.contains("\"timestamps\"")); + assertTrue(json.contains("\"startTime\"")); + assertTrue(json.contains("\"endTime\"")); + } + + public void testToStringHandlesIOException() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // toString should not throw exception even if there are issues + String result = metric.toString(); + assertNotNull(result); + assertTrue(result.length() > 0); + } + + public void testFileCacheStatToXContent() throws IOException { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Record data to create FileCacheStat + metric.recordFileAccess("file1.block_0_1", true); + metric.recordFileAccess("file1.block_0_2", false); + metric.recordFileAccess("file1.block_0_3", true); + + String json = metric.toString(); + + // Verify FileCacheStat XContent structure + assertTrue(json.contains("\"hits\":2")); + assertTrue(json.contains("\"miss\":1")); + assertTrue(json.contains("\"total\":3")); + assertTrue(json.contains("\"blockDetails\"")); + assertTrue(json.contains("\"hitBlockCount\":2")); + assertTrue(json.contains("\"hitBlocks\":[1,3]")); + assertTrue(json.contains("\"missBlockCount\":1")); + assertTrue(json.contains("\"missBlocks\":[2]")); + } + + public void testPrefetchStatToXContent() throws IOException { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Record prefetch data + metric.recordPrefetch("file1", 5); + metric.recordPrefetch("file1", 10); + metric.recordPrefetch("file1", 15); + + String json = metric.toString(); + + // Verify PrefetchStat XContent structure + assertTrue(json.contains("\"blockCount\":3")); + assertTrue(json.contains("\"blocks\":[5,10,15]")); + } + + public void testReadAheadStatToXContent() throws IOException { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Record read ahead data + metric.recordReadAhead("file1", 25); + metric.recordReadAhead("file1", 30); + + String json = metric.toString(); + + // Verify ReadAheadStat XContent structure + assertTrue(json.contains("\"blockCount\":2")); + assertTrue(json.contains("\"blocks\":[25,30]")); + } + + public void testInnerClassRamBytesUsed() { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + // Add data to create inner class instances + metric.recordFileAccess("file1.block_0_1", true); + metric.recordPrefetch("file2", 1); + metric.recordReadAhead("file3", 1); + + // Verify RAM usage calculation includes inner classes + long ramUsage = metric.ramBytesUsed(); + assertTrue(ramUsage > 0); + + // Add more data and verify RAM increases + metric.recordFileAccess("file1.block_0_2", false); + metric.recordFileAccess("file1.block_0_3", true); + + long newRamUsage = metric.ramBytesUsed(); + assertTrue(newRamUsage >= ramUsage); + } +} diff --git a/server/src/test/java/org/opensearch/storage/slowlogs/TieredStorageQueryMetricServiceTests.java b/server/src/test/java/org/opensearch/storage/slowlogs/TieredStorageQueryMetricServiceTests.java new file mode 100644 index 0000000000000..876f1fea8b0cb --- /dev/null +++ b/server/src/test/java/org/opensearch/storage/slowlogs/TieredStorageQueryMetricServiceTests.java @@ -0,0 +1,362 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.slowlogs; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Map; +import java.util.Set; + +/** + * Comprehensive unit tests for TieredStorageQueryMetricService + */ +public class TieredStorageQueryMetricServiceTests extends OpenSearchTestCase { + + private TieredStorageQueryMetricService service; + + @Override + public void setUp() throws Exception { + super.setUp(); + service = TieredStorageQueryMetricService.getInstance(); + + // Clear any existing state + service.getMetricCollectors().clear(); + service.getTaskIdToCollectorMap(true).clear(); + service.getTaskIdToCollectorMap(false).clear(); + } + + public void testGetInstance() { + TieredStorageQueryMetricService instance1 = TieredStorageQueryMetricService.getInstance(); + TieredStorageQueryMetricService instance2 = TieredStorageQueryMetricService.getInstance(); + + // Should return the same singleton instance + assertSame(instance1, instance2); + } + + public void testGetMetricCollectorWhenNotExists() { + long threadId = Thread.currentThread().threadId(); + + TieredStoragePerQueryMetric collector = service.getMetricCollector(threadId); + + // Should return dummy collector when no collector exists + assertNotNull(collector); + assertTrue(collector instanceof TieredStorageQueryMetricService.TieredStoragePerQueryMetricDummy); + assertEquals("DummyParentTaskId", collector.getParentTaskId()); + assertEquals("DummyShardId", collector.getShardId()); + } + + public void testAddAndGetMetricCollectorQueryPhase() { + long threadId = Thread.currentThread().threadId(); + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + service.addMetricCollector(threadId, metric, true); + + TieredStoragePerQueryMetric retrieved = service.getMetricCollector(threadId); + assertSame(metric, retrieved); + + // Verify it's added to query phase map + Map> queryMap = service.getTaskIdToCollectorMap(true); + assertTrue(queryMap.containsKey("task-1shard-1")); + assertTrue(queryMap.get("task-1shard-1").contains(metric)); + + // Should not be in fetch phase map + Map> fetchMap = service.getTaskIdToCollectorMap(false); + assertFalse(fetchMap.containsKey("task-1shard-1")); + } + + public void testAddAndGetMetricCollectorFetchPhase() { + long threadId = Thread.currentThread().threadId(); + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-2", "shard-2"); + + service.addMetricCollector(threadId, metric, false); + + TieredStoragePerQueryMetric retrieved = service.getMetricCollector(threadId); + assertSame(metric, retrieved); + + // Verify it's added to fetch phase map + Map> fetchMap = service.getTaskIdToCollectorMap(false); + assertTrue(fetchMap.containsKey("task-2shard-2")); + assertTrue(fetchMap.get("task-2shard-2").contains(metric)); + + // Should not be in query phase map + Map> queryMap = service.getTaskIdToCollectorMap(true); + assertFalse(queryMap.containsKey("task-2shard-2")); + } + + public void testAddMultipleCollectorsForSameTaskShard() { + long threadId1 = Thread.currentThread().threadId(); + long threadId2 = threadId1 + 1; // Simulate different thread + + TieredStoragePerQueryMetricImpl metric1 = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + TieredStoragePerQueryMetricImpl metric2 = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + service.addMetricCollector(threadId1, metric1, true); + service.addMetricCollector(threadId2, metric2, true); + + // Both should be in the task-shard map + Map> queryMap = service.getTaskIdToCollectorMap(true); + Set collectors = queryMap.get("task-1shard-1"); + assertEquals(2, collectors.size()); + assertTrue(collectors.contains(metric1)); + assertTrue(collectors.contains(metric2)); + } + + public void testRemoveMetricCollector() { + long threadId = Thread.currentThread().threadId(); + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + service.addMetricCollector(threadId, metric, true); + + TieredStoragePerQueryMetric removed = service.removeMetricCollector(threadId); + assertSame(metric, removed); + + // Should return dummy collector after removal + TieredStoragePerQueryMetric afterRemoval = service.getMetricCollector(threadId); + assertTrue(afterRemoval instanceof TieredStorageQueryMetricService.TieredStoragePerQueryMetricDummy); + + // Should still be in task-shard map (not removed by removeMetricCollector) + Map> queryMap = service.getTaskIdToCollectorMap(true); + assertTrue(queryMap.containsKey("task-1shard-1")); + } + + public void testRemoveMetricCollectorWhenNotExists() { + long threadId = Thread.currentThread().threadId(); + + TieredStoragePerQueryMetric removed = service.removeMetricCollector(threadId); + + // Should return null when no collector exists + assertNull(removed); + } + + public void testRemoveMetricCollectorsQueryPhase() { + long threadId1 = Thread.currentThread().threadId(); + long threadId2 = threadId1 + 1; + + TieredStoragePerQueryMetricImpl metric1 = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + TieredStoragePerQueryMetricImpl metric2 = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + + service.addMetricCollector(threadId1, metric1, true); + service.addMetricCollector(threadId2, metric2, true); + + Set removed = service.removeMetricCollectors("task-1", "shard-1", true); + + assertEquals(2, removed.size()); + assertTrue(removed.contains(metric1)); + assertTrue(removed.contains(metric2)); + + // Should be removed from task-shard map + Map> queryMap = service.getTaskIdToCollectorMap(true); + assertFalse(queryMap.containsKey("task-1shard-1")); + } + + public void testRemoveMetricCollectorsFetchPhase() { + long threadId = Thread.currentThread().threadId(); + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-2", "shard-2"); + + service.addMetricCollector(threadId, metric, false); + + Set removed = service.removeMetricCollectors("task-2", "shard-2", false); + + assertEquals(1, removed.size()); + assertTrue(removed.contains(metric)); + + // Should be removed from task-shard map + Map> fetchMap = service.getTaskIdToCollectorMap(false); + assertFalse(fetchMap.containsKey("task-2shard-2")); + } + + public void testRemoveMetricCollectorsWhenNotExists() { + Set removed = service.removeMetricCollectors("nonexistent", "shard", true); + + assertTrue(removed.isEmpty()); + } + + public void testRamBytesUsed() { + long initialRam = service.ramBytesUsed(); + assertTrue(initialRam > 0); + + // Add some collectors + TieredStoragePerQueryMetricImpl metric1 = new TieredStoragePerQueryMetricImpl("task-1", "shard-1"); + TieredStoragePerQueryMetricImpl metric2 = new TieredStoragePerQueryMetricImpl("task-2", "shard-2"); + + service.addMetricCollector(1L, metric1, true); + service.addMetricCollector(2L, metric2, false); + + long finalRam = service.ramBytesUsed(); + assertTrue(finalRam >= initialRam); + } + + public void testRecordStoredFieldsPrefetchSuccess() { + PrefetchStats initialStats = service.getPrefetchStats(); + long initialSuccess = initialStats.getStoredFieldsPrefetchSuccess(); + + service.recordStoredFieldsPrefetch(true); + + PrefetchStats finalStats = service.getPrefetchStats(); + assertEquals(initialSuccess + 1, finalStats.getStoredFieldsPrefetchSuccess()); + } + + public void testRecordStoredFieldsPrefetchFailure() { + PrefetchStats initialStats = service.getPrefetchStats(); + long initialFailure = initialStats.getStoredFieldsPrefetchFailure(); + + service.recordStoredFieldsPrefetch(false); + + PrefetchStats finalStats = service.getPrefetchStats(); + assertEquals(initialFailure + 1, finalStats.getStoredFieldsPrefetchFailure()); + } + + public void testRecordDocValuesPrefetchSuccess() { + PrefetchStats initialStats = service.getPrefetchStats(); + long initialSuccess = initialStats.getDocValuesPrefetchSuccess(); + + service.recordDocValuesPrefetch(true); + + PrefetchStats finalStats = service.getPrefetchStats(); + assertEquals(initialSuccess + 1, finalStats.getDocValuesPrefetchSuccess()); + } + + public void testRecordDocValuesPrefetchFailure() { + PrefetchStats initialStats = service.getPrefetchStats(); + long initialFailure = initialStats.getDocValuesPrefetchFailure(); + + service.recordDocValuesPrefetch(false); + + PrefetchStats finalStats = service.getPrefetchStats(); + assertEquals(initialFailure + 1, finalStats.getDocValuesPrefetchFailure()); + } + + public void testGetPrefetchStats() { + PrefetchStats stats = service.getPrefetchStats(); + + assertNotNull(stats); + assertTrue(stats.getStoredFieldsPrefetchSuccess() >= 0); + assertTrue(stats.getStoredFieldsPrefetchFailure() >= 0); + assertTrue(stats.getDocValuesPrefetchSuccess() >= 0); + assertTrue(stats.getDocValuesPrefetchFailure() >= 0); + } + + public void testTieredStoragePerQueryMetricDummyGetInstance() { + TieredStorageQueryMetricService.TieredStoragePerQueryMetricDummy dummy1 = + TieredStorageQueryMetricService.TieredStoragePerQueryMetricDummy.getInstance(); + TieredStorageQueryMetricService.TieredStoragePerQueryMetricDummy dummy2 = + TieredStorageQueryMetricService.TieredStoragePerQueryMetricDummy.getInstance(); + + // Should return the same singleton instance + assertSame(dummy1, dummy2); + } + + public void testTieredStoragePerQueryMetricDummyMethods() { + TieredStorageQueryMetricService.TieredStoragePerQueryMetricDummy dummy = + TieredStorageQueryMetricService.TieredStoragePerQueryMetricDummy.getInstance(); + + // All methods should be no-op and not throw exceptions + dummy.recordFileAccess("test.block_0_1", true); + dummy.recordPrefetch("test", 1); + dummy.recordReadAhead("test", 1); + dummy.recordEndTime(); + + assertEquals("DummyParentTaskId", dummy.getParentTaskId()); + assertEquals("DummyShardId", dummy.getShardId()); + assertTrue(dummy.ramBytesUsed() > 0); + } + + public void testMaxCollectorSizeLimit() { + // This test would be difficult to run in practice due to the high limit (1000) + // but we can verify the logic by checking that the service handles the limit gracefully + + // Add a reasonable number of collectors to verify normal operation + for (int i = 0; i < 10; i++) { + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-" + i, "shard-" + i); + service.addMetricCollector((long) i, metric, true); + } + + // Verify all were added + assertEquals(10, service.getMetricCollectors().size()); + assertEquals(10, service.getTaskIdToCollectorMap(true).size()); + } + + public void testConcurrentAccess() { + // Test that the service can handle concurrent access + String taskId = "concurrent-task"; + String shardId = "concurrent-shard"; + + TieredStoragePerQueryMetricImpl metric1 = new TieredStoragePerQueryMetricImpl(taskId, shardId); + TieredStoragePerQueryMetricImpl metric2 = new TieredStoragePerQueryMetricImpl(taskId, shardId); + + // Add collectors for the same task-shard from different threads + service.addMetricCollector(100L, metric1, true); + service.addMetricCollector(200L, metric2, true); + + // Both should be in the same task-shard set + Set collectors = service.getTaskIdToCollectorMap(true).get(taskId + shardId); + assertEquals(2, collectors.size()); + assertTrue(collectors.contains(metric1)); + assertTrue(collectors.contains(metric2)); + } + + public void testPrefetchStatsHolder() { + TieredStorageQueryMetricService.PrefetchStatsHolder holder = new TieredStorageQueryMetricService.PrefetchStatsHolder(); + + // Initial stats should be zero + PrefetchStats initialStats = holder.getStats(); + assertEquals(0, initialStats.getStoredFieldsPrefetchSuccess()); + assertEquals(0, initialStats.getStoredFieldsPrefetchFailure()); + assertEquals(0, initialStats.getDocValuesPrefetchSuccess()); + assertEquals(0, initialStats.getDocValuesPrefetchFailure()); + + // Increment counters + holder.storedFieldsPrefetchSuccess.inc(); + holder.storedFieldsPrefetchFailure.inc(); + holder.docValuesPrefetchSuccess.inc(); + holder.docValuesPrefetchFailure.inc(); + + // Verify increments + PrefetchStats finalStats = holder.getStats(); + assertEquals(1, finalStats.getStoredFieldsPrefetchSuccess()); + assertEquals(1, finalStats.getStoredFieldsPrefetchFailure()); + assertEquals(1, finalStats.getDocValuesPrefetchSuccess()); + assertEquals(1, finalStats.getDocValuesPrefetchFailure()); + } + + public void testMixedQueryAndFetchPhaseCollectors() { + String taskId = "mixed-task"; + String shardId = "mixed-shard"; + + TieredStoragePerQueryMetricImpl queryMetric = new TieredStoragePerQueryMetricImpl(taskId, shardId); + TieredStoragePerQueryMetricImpl fetchMetric = new TieredStoragePerQueryMetricImpl(taskId, shardId); + + service.addMetricCollector(100L, queryMetric, true); + service.addMetricCollector(200L, fetchMetric, false); + + // Should be in separate maps + assertTrue(service.getTaskIdToCollectorMap(true).containsKey(taskId + shardId)); + assertTrue(service.getTaskIdToCollectorMap(false).containsKey(taskId + shardId)); + + assertEquals(1, service.getTaskIdToCollectorMap(true).get(taskId + shardId).size()); + assertEquals(1, service.getTaskIdToCollectorMap(false).get(taskId + shardId).size()); + + // Remove query phase collectors + Set queryCollectors = service.removeMetricCollectors(taskId, shardId, true); + assertEquals(1, queryCollectors.size()); + assertTrue(queryCollectors.contains(queryMetric)); + + // Fetch phase collectors should still be there + assertTrue(service.getTaskIdToCollectorMap(false).containsKey(taskId + shardId)); + + // Remove fetch phase collectors + Set fetchCollectors = service.removeMetricCollectors(taskId, shardId, false); + assertEquals(1, fetchCollectors.size()); + assertTrue(fetchCollectors.contains(fetchMetric)); + + // Both maps should be empty for this task-shard now + assertFalse(service.getTaskIdToCollectorMap(true).containsKey(taskId + shardId)); + assertFalse(service.getTaskIdToCollectorMap(false).containsKey(taskId + shardId)); + } +} diff --git a/server/src/test/java/org/opensearch/storage/slowlogs/TieredStorageSearchSlowLogTests.java b/server/src/test/java/org/opensearch/storage/slowlogs/TieredStorageSearchSlowLogTests.java new file mode 100644 index 0000000000000..9ffafe5c733eb --- /dev/null +++ b/server/src/test/java/org/opensearch/storage/slowlogs/TieredStorageSearchSlowLogTests.java @@ -0,0 +1,510 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.slowlogs; + +import org.opensearch.Version; +import org.opensearch.action.search.SearchShardTask; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.logging.SlowLogLevel; +import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.tasks.TaskId; +import org.opensearch.index.IndexSettings; +import org.opensearch.search.SearchShardTarget; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.internal.ShardSearchRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Comprehensive unit tests for TieredStorageSearchSlowLog + */ +public class TieredStorageSearchSlowLogTests extends OpenSearchTestCase { + + private TieredStorageSearchSlowLog slowLog; + private IndexSettings indexSettings; + private SearchContext searchContext; + private SearchShardTask searchTask; + private TieredStorageQueryMetricService metricService; + + @Before + public void setUp() throws Exception { + super.setUp(); + + // Create mock objects + indexSettings = createMockIndexSettings(); + searchContext = mock(SearchContext.class); + searchTask = mock(SearchShardTask.class); + + // Mock search context setup + when(searchContext.getTask()).thenReturn(searchTask); + SearchShardTarget shardTarget = new SearchShardTarget("testNode", mock(ShardId.class), null, null); + when(searchContext.shardTarget()).thenReturn(shardTarget); + when(searchContext.numberOfShards()).thenReturn(1); + when(searchContext.searchType()).thenReturn(org.opensearch.action.search.SearchType.QUERY_THEN_FETCH); + when(searchContext.request()).thenReturn(mock(ShardSearchRequest.class)); + when(searchContext.request().source()).thenReturn(null); + + // Mock search task - use string directly to avoid TaskId class issues + when(searchTask.getParentTaskId()).thenReturn(TaskId.EMPTY_TASK_ID); + + // Create slow log instance + slowLog = new TieredStorageSearchSlowLog(indexSettings); + + // Mock metric service + metricService = mock(TieredStorageQueryMetricService.class); + } + + private IndexSettings createMockIndexSettings() { + Set> settingSet = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS); + settingSet.add(TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_ENABLED); + settingSet.add(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL); + settingSet.add(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING); + settingSet.add(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING); + settingSet.add(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING); + settingSet.add(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING); + settingSet.add(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING); + settingSet.add(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING); + settingSet.add(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING); + settingSet.add(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING); + + Settings settings = Settings.builder() + .put(TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_ENABLED.getKey(), true) + .put(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING.getKey(), "1s") + .put(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING.getKey(), "500ms") + .put(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING.getKey(), "100ms") + .put(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING.getKey(), "10ms") + .put(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING.getKey(), "1s") + .put(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING.getKey(), "500ms") + .put(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING.getKey(), "100ms") + .put(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING.getKey(), "10ms") + .put(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL.getKey(), "TRACE") + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + + IndexMetadata metadata = IndexMetadata.builder("Index").settings(settings).build(); + return new IndexSettings(metadata, settings, new IndexScopedSettings(settings, settingSet)); + } + + public void testConstructorInitializesSettings() { + // Verify that constructor properly initializes all settings + assertTrue(slowLog.getQueryWarnThreshold() > 0); + assertTrue(slowLog.getQueryInfoThreshold() > 0); + assertTrue(slowLog.getQueryDebugThreshold() > 0); + assertTrue(slowLog.getQueryTraceThreshold() > 0); + + assertTrue(slowLog.getFetchWarnThreshold() > 0); + assertTrue(slowLog.getFetchInfoThreshold() > 0); + assertTrue(slowLog.getFetchDebugThreshold() > 0); + assertTrue(slowLog.getFetchTraceThreshold() > 0); + + assertEquals(SlowLogLevel.TRACE, slowLog.getLevel()); + } + + public void testSetTieredStorageSlowlogEnabled() { + // Test enabling/disabling slow log + slowLog.setTieredStorageSlowlogEnabled(true); + // No direct way to verify, but should not throw exception + + slowLog.setTieredStorageSlowlogEnabled(false); + // No direct way to verify, but should not throw exception + } + + public void testOnPreQueryPhase() { + // onPreQueryPhase should not do anything as per the implementation + // Just verify it doesn't throw exception + slowLog.onPreQueryPhase(searchContext); + } + + public void testOnPreSliceExecutionWhenEnabled() { + // Enable slow log + slowLog.setTieredStorageSlowlogEnabled(true); + + // Should call setMetricCollector when enabled + slowLog.onPreSliceExecution(searchContext); + + // Verify no exception is thrown + } + + public void testOnPreSliceExecutionWhenDisabled() { + // Disable slow log + slowLog.setTieredStorageSlowlogEnabled(false); + + // Should not call setMetricCollector when disabled + slowLog.onPreSliceExecution(searchContext); + + // Verify no exception is thrown + } + + public void testOnSliceExecutionWhenEnabled() { + // Enable slow log + slowLog.setTieredStorageSlowlogEnabled(true); + + // Should call removeMetricCollector when enabled + slowLog.onSliceExecution(searchContext); + + // Verify no exception is thrown + } + + public void testOnSliceExecutionWhenDisabled() { + // Disable slow log + slowLog.setTieredStorageSlowlogEnabled(false); + + // Should not call removeMetricCollector when disabled + slowLog.onSliceExecution(searchContext); + + // Verify no exception is thrown + } + + public void testOnFailedSliceExecutionWhenEnabled() { + // Enable slow log + slowLog.setTieredStorageSlowlogEnabled(true); + + // Should call removeMetricCollector when enabled + slowLog.onFailedSliceExecution(searchContext); + + // Verify no exception is thrown + } + + public void testOnFailedSliceExecutionWhenDisabled() { + // Disable slow log + slowLog.setTieredStorageSlowlogEnabled(false); + + // Should not call removeMetricCollector when disabled + slowLog.onFailedSliceExecution(searchContext); + + // Verify no exception is thrown + } + + public void testOnQueryPhaseWhenEnabled() { + // Enable slow log + slowLog.setTieredStorageSlowlogEnabled(true); + + // Test with time above trace threshold + long tookInNanos = TimeUnit.MILLISECONDS.toNanos(50); // Above 10ms trace threshold + + slowLog.onQueryPhase(searchContext, tookInNanos); + + // Verify no exception is thrown + } + + public void testOnQueryPhaseWhenDisabled() { + // Disable slow log + slowLog.setTieredStorageSlowlogEnabled(false); + + long tookInNanos = TimeUnit.MILLISECONDS.toNanos(50); + + slowLog.onQueryPhase(searchContext, tookInNanos); + + // Verify no exception is thrown + } + + public void testOnFailedQueryPhaseWhenEnabled() { + // Enable slow log + slowLog.setTieredStorageSlowlogEnabled(true); + + slowLog.onFailedQueryPhase(searchContext); + + // Verify no exception is thrown + } + + public void testOnFailedQueryPhaseWhenDisabled() { + // Disable slow log + slowLog.setTieredStorageSlowlogEnabled(false); + + slowLog.onFailedQueryPhase(searchContext); + + // Verify no exception is thrown + } + + public void testOnPreFetchPhaseWhenEnabled() { + // Enable slow log + slowLog.setTieredStorageSlowlogEnabled(true); + + slowLog.onPreFetchPhase(searchContext); + + // Verify no exception is thrown + } + + public void testOnPreFetchPhaseWhenDisabled() { + // Disable slow log + slowLog.setTieredStorageSlowlogEnabled(false); + + slowLog.onPreFetchPhase(searchContext); + + // Verify no exception is thrown + } + + public void testOnFetchPhaseWhenEnabled() { + // Enable slow log + slowLog.setTieredStorageSlowlogEnabled(true); + + slowLog.onPreFetchPhase(searchContext); + + long tookInNanos = TimeUnit.MILLISECONDS.toNanos(50); + + slowLog.onFetchPhase(searchContext, tookInNanos); + + // Verify no exception is thrown + } + + public void testOnFetchPhaseWhenDisabled() { + // Disable slow log + slowLog.setTieredStorageSlowlogEnabled(false); + + long tookInNanos = TimeUnit.MILLISECONDS.toNanos(50); + + slowLog.onFetchPhase(searchContext, tookInNanos); + + // Verify no exception is thrown + } + + public void testOnFailedFetchPhaseWhenEnabled() { + // Enable slow log + slowLog.setTieredStorageSlowlogEnabled(true); + + slowLog.onFailedFetchPhase(searchContext); + + // Verify no exception is thrown + } + + public void testOnFailedFetchPhaseWhenDisabled() { + // Disable slow log + slowLog.setTieredStorageSlowlogEnabled(false); + + slowLog.onFailedFetchPhase(searchContext); + + // Verify no exception is thrown + } + + public void testThresholdGetters() { + // Test all threshold getters return expected values + assertTrue(slowLog.getQueryWarnThreshold() >= 0); + assertTrue(slowLog.getQueryInfoThreshold() >= 0); + assertTrue(slowLog.getQueryDebugThreshold() >= 0); + assertTrue(slowLog.getQueryTraceThreshold() >= 0); + + assertTrue(slowLog.getFetchWarnThreshold() >= 0); + assertTrue(slowLog.getFetchInfoThreshold() >= 0); + assertTrue(slowLog.getFetchDebugThreshold() >= 0); + assertTrue(slowLog.getFetchTraceThreshold() >= 0); + } + + public void testSlowLogSettings() { + // Test that all settings are properly defined + assertNotNull(TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_ENABLED); + assertNotNull(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING); + assertNotNull(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING); + assertNotNull(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING); + assertNotNull(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING); + + assertNotNull(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING); + assertNotNull(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING); + assertNotNull(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING); + assertNotNull(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING); + + assertNotNull(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL); + } + + public void testSlowLogSettingsMap() { + // Test that settings map contains all expected settings + assertFalse(TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_SETTINGS_MAP.isEmpty()); + assertTrue( + TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_SETTINGS_MAP.containsKey( + TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".enabled" + ) + ); + assertTrue( + TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_SETTINGS_MAP.containsKey( + TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".threshold.query.warn" + ) + ); + assertTrue( + TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_SETTINGS_MAP.containsKey( + TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX + ".level" + ) + ); + } + + public void testSlowLogSettingsSet() { + // Test that settings set contains all expected settings + assertFalse(TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_SETTINGS.isEmpty()); + assertEquals(10, TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_SETTINGS.size()); + } + + public void testTieredStorageSlowLogPrinterConstructor() { + TieredStorageSearchSlowLog.TieredStorageSlowLogPrinter printer = new TieredStorageSearchSlowLog.TieredStorageSlowLogPrinter( + searchContext, + TimeUnit.MILLISECONDS.toNanos(100), + java.util.Collections.emptyList() + ); + + assertNotNull(printer); + } + + public void testTieredStorageSlowLogPrinterToString() { + TieredStorageSearchSlowLog.TieredStorageSlowLogPrinter printer = new TieredStorageSearchSlowLog.TieredStorageSlowLogPrinter( + searchContext, + TimeUnit.MILLISECONDS.toNanos(100), + java.util.Collections.emptyList() + ); + + String result = printer.toString(); + assertNotNull(result); + assertTrue(result.length() > 0); + + // Should contain expected JSON structure + assertTrue(result.contains("warm_stats")); + assertTrue(result.contains("took")); + assertTrue(result.contains("took_millis")); + assertTrue(result.contains("stats")); + assertTrue(result.contains("search_type")); + assertTrue(result.contains("total_shards")); + } + + public void testTieredStorageSlowLogPrinterWithMetrics() { + // Create a metric collector + TieredStoragePerQueryMetricImpl metric = new TieredStoragePerQueryMetricImpl("task-1", "shard-0"); + metric.recordFileAccess("file1.block_0_1", true); + metric.recordPrefetch("file2", 1); + metric.recordEndTime(); + + java.util.List metrics = java.util.Arrays.asList(metric); + + TieredStorageSearchSlowLog.TieredStorageSlowLogPrinter printer = new TieredStorageSearchSlowLog.TieredStorageSlowLogPrinter( + searchContext, + TimeUnit.MILLISECONDS.toNanos(100), + metrics + ); + + String result = printer.toString(); + assertNotNull(result); + assertTrue(result.length() > 0); + + // Should contain metric data in warm_stats + assertTrue(result.contains("warm_stats")); + assertTrue(result.contains("parentTask")); + assertTrue(result.contains("task-1")); + } + + public void testSearchContextWithNullTask() { + // Test behavior when search task is null + when(searchContext.getTask()).thenReturn(null); + + slowLog.setTieredStorageSlowlogEnabled(true); + + // Should handle null task gracefully + slowLog.onPreSliceExecution(searchContext); + slowLog.onSliceExecution(searchContext); + slowLog.onPreFetchPhase(searchContext); + + // Verify no exceptions are thrown + } + + public void testDifferentLogLevels() { + slowLog.setTieredStorageSlowlogEnabled(true); + + // Test different time thresholds for different log levels + + // Test TRACE level (10ms threshold) + long traceTime = TimeUnit.MILLISECONDS.toNanos(15); + slowLog.onQueryPhase(searchContext, traceTime); + + // Test DEBUG level (100ms threshold) + long debugTime = TimeUnit.MILLISECONDS.toNanos(150); + slowLog.onQueryPhase(searchContext, debugTime); + + // Test INFO level (500ms threshold) + long infoTime = TimeUnit.MILLISECONDS.toNanos(600); + slowLog.onQueryPhase(searchContext, infoTime); + + // Test WARN level (1s threshold) + long warnTime = TimeUnit.MILLISECONDS.toNanos(1100); + slowLog.onQueryPhase(searchContext, warnTime); + + // All should complete without exceptions + } + + public void testFetchPhaseLogging() { + slowLog.setTieredStorageSlowlogEnabled(true); + + slowLog.onPreFetchPhase(searchContext); + // Test fetch phase with different thresholds + long fetchTime = TimeUnit.MILLISECONDS.toNanos(600); // Above info threshold + + slowLog.onFetchPhase(searchContext, fetchTime); + + // Should complete without exceptions + } + + public void testSettingsPrefix() { + // Verify the settings prefix is correct + String expectedPrefix = TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_PREFIX; + + assertTrue(TieredStorageSearchSlowLog.TIERED_STORAGE_SEARCH_SLOWLOG_ENABLED.getKey().startsWith(expectedPrefix)); + assertTrue(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING.getKey().startsWith(expectedPrefix)); + assertTrue(TieredStorageSearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL.getKey().startsWith(expectedPrefix)); + } + + public void testTimeValueConversion() { + // Test that time values are properly converted to nanoseconds + assertTrue(slowLog.getQueryWarnThreshold() > 0); + assertTrue(slowLog.getQueryInfoThreshold() > 0); + assertTrue(slowLog.getQueryDebugThreshold() > 0); + assertTrue(slowLog.getQueryTraceThreshold() > 0); + + // Verify hierarchy: warn > info > debug > trace + assertTrue(slowLog.getQueryWarnThreshold() >= slowLog.getQueryInfoThreshold()); + assertTrue(slowLog.getQueryInfoThreshold() >= slowLog.getQueryDebugThreshold()); + assertTrue(slowLog.getQueryDebugThreshold() >= slowLog.getQueryTraceThreshold()); + } + + public void testSlowLogPrinterWithNullSource() { + // Test printer when search request source is null + when(searchContext.request().source()).thenReturn(null); + + TieredStorageSearchSlowLog.TieredStorageSlowLogPrinter printer = new TieredStorageSearchSlowLog.TieredStorageSlowLogPrinter( + searchContext, + TimeUnit.MILLISECONDS.toNanos(100), + java.util.Collections.emptyList() + ); + + String result = printer.toString(); + assertNotNull(result); + assertTrue(result.contains("\"source\":null")); + } + + public void testSlowLogPrinterWithGroupStats() { + // Mock group stats - use List to match expected type + java.util.List groupStats = java.util.Arrays.asList("stat1", "stat2"); + when(searchContext.groupStats()).thenReturn(groupStats); + + TieredStorageSearchSlowLog.TieredStorageSlowLogPrinter printer = new TieredStorageSearchSlowLog.TieredStorageSlowLogPrinter( + searchContext, + TimeUnit.MILLISECONDS.toNanos(100), + java.util.Collections.emptyList() + ); + + String result = printer.toString(); + assertNotNull(result); + assertTrue(result.contains("stats")); + } +} From e902536c2acc6ef3554e4fbf20cc147c79250bfd Mon Sep 17 00:00:00 2001 From: expani1729 <110471048+expani@users.noreply.github.com> Date: Thu, 30 Apr 2026 04:17:50 -0700 Subject: [PATCH 020/115] Add BackendPlanAdapter SPI and Unify FilterOperator into ScalarFunction (#21438) * Unify FilterOperator into ScalarFunction and add BackendPlanAdapter SPI Signed-off-by: expani * Added tests based on review comments and add more docs Signed-off-by: expani * Added a catch-all for Category and removed unused function Signed-off-by: expani * Ensured Annotations are intact after adaption Signed-off-by: expani --------- Signed-off-by: expani --- .../spi/BackendCapabilityProvider.java | 11 + .../analytics/spi}/FieldStorageInfo.java | 3 +- .../analytics/spi/FilterCapability.java | 4 +- .../analytics/spi/FilterOperator.java | 112 ------ .../analytics/spi/ScalarFunction.java | 126 ++++--- .../analytics/spi/ScalarFunctionAdapter.java | 48 +++ .../DataFusionAnalyticsBackendPlugin.java | 26 +- .../analytics/exec/DefaultPlanExecutor.java | 2 + .../analytics/planner/CapabilityRegistry.java | 25 +- .../planner/FieldStorageResolver.java | 1 + .../planner/dag/BackendPlanAdapter.java | 208 +++++++++++ .../planner/rel/AggregateCallAnnotation.java | 6 + .../planner/rel/AnnotatedPredicate.java | 5 + .../rel/AnnotatedProjectExpression.java | 5 + .../planner/rel/OpenSearchAggregate.java | 2 +- .../rel/OpenSearchExchangeReducer.java | 2 +- .../planner/rel/OpenSearchFilter.java | 2 +- .../planner/rel/OpenSearchProject.java | 2 +- .../planner/rel/OpenSearchRelNode.java | 2 +- .../analytics/planner/rel/OpenSearchSort.java | 2 +- .../planner/rel/OpenSearchStageInputScan.java | 2 +- .../planner/rel/OpenSearchTableScan.java | 2 +- .../planner/rel/OperatorAnnotation.java | 3 + .../rules/OpenSearchAggregateRule.java | 2 +- .../planner/rules/OpenSearchFilterRule.java | 16 +- .../planner/rules/OpenSearchProjectRule.java | 3 + .../rules/OpenSearchTableScanRule.java | 2 +- .../planner/BasePlannerRulesTests.java | 1 + .../planner/FieldStorageResolverTests.java | 1 + .../analytics/planner/FilterRuleTests.java | 32 +- .../analytics/planner/MockBackend.java | 12 + .../planner/MockDataFusionBackend.java | 26 +- .../analytics/planner/MockLuceneBackend.java | 40 +-- .../planner/dag/BackendPlanAdapterTests.java | 325 ++++++++++++++++++ 34 files changed, 817 insertions(+), 244 deletions(-) rename sandbox/{plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner => libs/analytics-framework/src/main/java/org/opensearch/analytics/spi}/FieldStorageInfo.java (97%) delete mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterOperator.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunctionAdapter.java create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/BackendPlanAdapter.java create mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/BackendPlanAdapterTests.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendCapabilityProvider.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendCapabilityProvider.java index 418e4821225f0..bba6d511d99d5 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendCapabilityProvider.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendCapabilityProvider.java @@ -8,6 +8,7 @@ package org.opensearch.analytics.spi; +import java.util.Map; import java.util.Set; /** @@ -61,4 +62,14 @@ default Set supportedDelegations() { default Set acceptedDelegations() { return Set.of(); } + + /** + * Per-function adapters for transforming backend-agnostic scalar function RexCalls + * into backend-compatible forms before fragment conversion. Keyed by {@link ScalarFunction}. + * Applied regardless of operator context (filter, project, aggregate expression). + * Empty map means no adaptation needed. + */ + default Map scalarFunctionAdapters() { + return Map.of(); + } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageInfo.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FieldStorageInfo.java similarity index 97% rename from sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageInfo.java rename to sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FieldStorageInfo.java index 304a2b49dfd0d..9fd96c235a15b 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageInfo.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FieldStorageInfo.java @@ -6,10 +6,9 @@ * compatible open source license. */ -package org.opensearch.analytics.planner; +package org.opensearch.analytics.spi; import org.apache.calcite.sql.type.SqlTypeName; -import org.opensearch.analytics.spi.FieldType; import java.util.List; diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterCapability.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterCapability.java index 134fef7c7beb7..3065b001a4013 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterCapability.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterCapability.java @@ -24,11 +24,11 @@ public sealed interface FilterCapability { /** Standard comparison filter (EQUALS, GT, IN, LIKE, etc.) on field types in given formats. */ - record Standard(FilterOperator operator, Set fieldTypes, Set formats) implements FilterCapability { + record Standard(ScalarFunction function, Set fieldTypes, Set formats) implements FilterCapability { } /** Full-text filter (MATCH, MATCH_PHRASE, FUZZY, etc.) with supported query parameters. */ - record FullText(FilterOperator operator, FieldType fieldType, Set formats, Set supportedParams) + record FullText(ScalarFunction function, FieldType fieldType, Set formats, Set supportedParams) implements FilterCapability { } diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterOperator.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterOperator.java deleted file mode 100644 index 9a19c801e6771..0000000000000 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterOperator.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.analytics.spi; - -import org.apache.calcite.sql.SqlFunction; -import org.apache.calcite.sql.SqlFunctionCategory; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.type.OperandTypes; -import org.apache.calcite.sql.type.ReturnTypes; - -/** - * All filter operations a backend may support, covering standard comparisons - * and full-text search. - * - *

    Each operator carries a {@link Type} indicating its category and whether - * it supports parameters (e.g., full-text operators accept analyzer, slop, etc.). - * - * @opensearch.internal - */ -public enum FilterOperator { - - // Standard comparison - EQUALS(Type.STANDARD, SqlKind.EQUALS), - NOT_EQUALS(Type.STANDARD, SqlKind.NOT_EQUALS), - GREATER_THAN(Type.STANDARD, SqlKind.GREATER_THAN), - GREATER_THAN_OR_EQUAL(Type.STANDARD, SqlKind.GREATER_THAN_OR_EQUAL), - LESS_THAN(Type.STANDARD, SqlKind.LESS_THAN), - LESS_THAN_OR_EQUAL(Type.STANDARD, SqlKind.LESS_THAN_OR_EQUAL), - IS_NULL(Type.STANDARD, SqlKind.IS_NULL), - IS_NOT_NULL(Type.STANDARD, SqlKind.IS_NOT_NULL), - IN(Type.STANDARD, SqlKind.IN), - LIKE(Type.STANDARD, SqlKind.LIKE), - PREFIX(Type.STANDARD, SqlKind.OTHER), - - // Full-text search - MATCH(Type.FULL_TEXT, SqlKind.OTHER), - MATCH_PHRASE(Type.FULL_TEXT, SqlKind.OTHER), - FUZZY(Type.FULL_TEXT, SqlKind.OTHER), - WILDCARD(Type.FULL_TEXT, SqlKind.OTHER), - REGEXP(Type.FULL_TEXT, SqlKind.OTHER); - - /** - * Category of filter operator. - */ - public enum Type { - STANDARD(false), - FULL_TEXT(true); - - private final boolean supportsParams; - - Type(boolean supportsParams) { - this.supportsParams = supportsParams; - } - - public boolean supportsParams() { - return supportsParams; - } - } - - private final Type type; - private final SqlKind sqlKind; - - FilterOperator(Type type, SqlKind sqlKind) { - this.type = type; - this.sqlKind = sqlKind; - } - - public Type getType() { - return type; - } - - /** - * Returns a Calcite {@link SqlFunction} for this full-text operator. - * Only valid for operators of type {@link Type#FULL_TEXT}. - */ - public SqlFunction toSqlFunction() { - return new SqlFunction( - name(), - SqlKind.OTHER_FUNCTION, - ReturnTypes.BOOLEAN, - null, - OperandTypes.ANY, - SqlFunctionCategory.USER_DEFINED_FUNCTION - ); - } - - /** Maps a Calcite SqlKind to a standard FilterOperator, or null if not recognized. */ - public static FilterOperator fromSqlKind(SqlKind kind) { - for (FilterOperator op : values()) { - if (op.type == Type.STANDARD && op.sqlKind == kind && op.sqlKind != SqlKind.OTHER) { - return op; - } - } - return null; - } - - /** Maps a Calcite SqlFunction to a FULL_TEXT FilterOperator by name, or null if not recognized. */ - public static FilterOperator fromSqlFunction(SqlFunction function) { - try { - FilterOperator op = FilterOperator.valueOf(function.getName().toUpperCase(java.util.Locale.ROOT)); - return op.type == Type.FULL_TEXT ? op : null; - } catch (IllegalArgumentException ignored) { - return null; - } - } -} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java index 9f69a74579bbe..2a1d4f8129187 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java @@ -8,56 +8,104 @@ package org.opensearch.analytics.spi; +import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlKind; +import java.util.Locale; + /** - * Scalar functions that a backend may support in projections and expressions. - * Used by the project rule to verify the backend can evaluate each expression - * in the SELECT clause. + * All scalar functions a backend may support — comparisons, full-text search, + * math, string, conditional, date/time, and cast operations. Used across filter, + * project, and aggregate expression capability declarations. + * + *

    Each function carries a {@link Category} indicating its type and whether + * it supports parameters (e.g., full-text operators accept analyzer, slop, etc.). * * @opensearch.internal */ public enum ScalarFunction { - // String - UPPER(SqlKind.OTHER), - LOWER(SqlKind.OTHER), - TRIM(SqlKind.TRIM), - SUBSTRING(SqlKind.OTHER), - CONCAT(SqlKind.OTHER), - CHAR_LENGTH(SqlKind.OTHER), - - // Math - PLUS(SqlKind.PLUS), - MINUS(SqlKind.MINUS), - TIMES(SqlKind.TIMES), - DIVIDE(SqlKind.DIVIDE), - MOD(SqlKind.MOD), - ABS(SqlKind.OTHER), - CEIL(SqlKind.CEIL), - FLOOR(SqlKind.FLOOR), - - // Cast / type - CAST(SqlKind.CAST), - - // Conditional - CASE(SqlKind.CASE), - COALESCE(SqlKind.COALESCE), - NULLIF(SqlKind.NULLIF), - - // Date/time - EXTRACT(SqlKind.EXTRACT); + // ── Comparisons ────────────────────────────────────────────────── + EQUALS(Category.COMPARISON, SqlKind.EQUALS), + NOT_EQUALS(Category.COMPARISON, SqlKind.NOT_EQUALS), + GREATER_THAN(Category.COMPARISON, SqlKind.GREATER_THAN), + GREATER_THAN_OR_EQUAL(Category.COMPARISON, SqlKind.GREATER_THAN_OR_EQUAL), + LESS_THAN(Category.COMPARISON, SqlKind.LESS_THAN), + LESS_THAN_OR_EQUAL(Category.COMPARISON, SqlKind.LESS_THAN_OR_EQUAL), + IS_NULL(Category.COMPARISON, SqlKind.IS_NULL), + IS_NOT_NULL(Category.COMPARISON, SqlKind.IS_NOT_NULL), + IN(Category.COMPARISON, SqlKind.IN), + LIKE(Category.COMPARISON, SqlKind.LIKE), + PREFIX(Category.COMPARISON, SqlKind.OTHER), + + // ── Full-text search ───────────────────────────────────────────── + MATCH(Category.FULL_TEXT, SqlKind.OTHER), + MATCH_PHRASE(Category.FULL_TEXT, SqlKind.OTHER), + FUZZY(Category.FULL_TEXT, SqlKind.OTHER), + WILDCARD(Category.FULL_TEXT, SqlKind.OTHER), + REGEXP(Category.FULL_TEXT, SqlKind.OTHER), + + // ── String ─────────────────────────────────────────────────────── + UPPER(Category.STRING, SqlKind.OTHER), + LOWER(Category.STRING, SqlKind.OTHER), + TRIM(Category.STRING, SqlKind.TRIM), + SUBSTRING(Category.STRING, SqlKind.OTHER), + CONCAT(Category.STRING, SqlKind.OTHER), + CHAR_LENGTH(Category.STRING, SqlKind.OTHER), + + // ── Math ───────────────────────────────────────────────────────── + PLUS(Category.MATH, SqlKind.PLUS), + MINUS(Category.MATH, SqlKind.MINUS), + TIMES(Category.MATH, SqlKind.TIMES), + DIVIDE(Category.MATH, SqlKind.DIVIDE), + MOD(Category.MATH, SqlKind.MOD), + ABS(Category.MATH, SqlKind.OTHER), + SIN(Category.MATH, SqlKind.OTHER), + CEIL(Category.MATH, SqlKind.CEIL), + FLOOR(Category.MATH, SqlKind.FLOOR), + + // ── Cast / type ────────────────────────────────────────────────── + CAST(Category.SCALAR, SqlKind.CAST), + + // ── Conditional ────────────────────────────────────────────────── + CASE(Category.SCALAR, SqlKind.CASE), + COALESCE(Category.SCALAR, SqlKind.COALESCE), + NULLIF(Category.SCALAR, SqlKind.NULLIF), + + EXTRACT(Category.SCALAR, SqlKind.EXTRACT); + + /** + * Category of scalar function. + */ + public enum Category { + COMPARISON, + FULL_TEXT, + STRING, + MATH, + /** Catch-all for functions that don't fit other categories (CAST, CASE, COALESCE, EXTRACT, etc.). */ + SCALAR + } + + private final Category category; private final SqlKind sqlKind; - ScalarFunction(SqlKind sqlKind) { + ScalarFunction(Category category, SqlKind sqlKind) { + this.category = category; this.sqlKind = sqlKind; } + public Category getCategory() { + return category; + } + public SqlKind getSqlKind() { return sqlKind; } - /** Maps a Calcite SqlKind to a ScalarFunction, or null if not recognized. Skips OTHER. */ + /** + * Maps a Calcite SqlKind to a ScalarFunction, or null if not recognized. + * Skips OTHER to avoid ambiguity (multiple functions share OTHER). + */ public static ScalarFunction fromSqlKind(SqlKind kind) { for (ScalarFunction func : values()) { if (func.sqlKind == kind && func.sqlKind != SqlKind.OTHER) { @@ -67,12 +115,10 @@ public static ScalarFunction fromSqlKind(SqlKind kind) { return null; } - /** Maps a function name to a ScalarFunction. Throws if not recognized. */ - public static ScalarFunction fromNameOrError(String name) { - try { - return valueOf(name); - } catch (IllegalArgumentException e) { - throw new IllegalStateException("Unrecognized scalar function [" + name + "]", e); - } + /** Maps a Calcite SqlFunction to a ScalarFunction by name, or throws if not recognized. */ + public static ScalarFunction fromSqlFunction(SqlFunction function) { + // TODO: Add an explicit functionName field per enum constant instead of relying on + // valueOf(toUpperCase). This couples enum constant naming to SQL function naming convention. + return ScalarFunction.valueOf(function.getName().toUpperCase(Locale.ROOT)); } } diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunctionAdapter.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunctionAdapter.java new file mode 100644 index 0000000000000..4ebd89580b405 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunctionAdapter.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; + +import java.util.List; + +/** + * Per-function adapter that transforms a backend-agnostic scalar function + * {@link RexCall} into a backend-compatible form. Registered by backends + * alongside their capability declarations, keyed by {@link ScalarFunction}. + * + *

    Example: {@code SIN(BIGINT)} → {@code SIN(CAST(BIGINT → DOUBLE))} because + * Substrait only declares {@code sin(fp32)} and {@code sin(fp64)}. + * + * @opensearch.internal + */ +@FunctionalInterface +public interface ScalarFunctionAdapter { + + /** + * Adapt the given expression for backend compatibility. Returns the adapted + * expression, or the original unchanged if no adaptation is needed. + * + *

    For type-conversion decisions (e.g., inserting CAST), use the Calcite type + * on the operand ({@code operand.getType().getSqlTypeName()}) — Substrait + * compatibility depends on the Calcite logical type, not the OpenSearch storage + * type. Use {@code fieldStorage} for decisions that depend on OpenSearch-specific + * type distinctions that Calcite cannot express (e.g., keyword vs text — both + * {@code VARCHAR} in Calcite but different storage semantics in OpenSearch). + * + * @param original the backend-agnostic expression to adapt + * @param fieldStorage positional field storage info from the operator's child, + * indexed by {@link org.apache.calcite.rex.RexInputRef#getIndex()} + * @param cluster provides {@code getRexBuilder()} and {@code getTypeFactory()} + * for constructing new RexNodes + */ + RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster); +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 24fa3415bed51..b77bd4aef0954 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -16,8 +16,8 @@ import org.opensearch.analytics.spi.ExchangeSinkProvider; import org.opensearch.analytics.spi.FieldType; import org.opensearch.analytics.spi.FilterCapability; -import org.opensearch.analytics.spi.FilterOperator; import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.ScalarFunction; import org.opensearch.analytics.spi.ScanCapability; import org.opensearch.analytics.spi.SearchExecEngineProvider; import org.opensearch.index.engine.dataformat.DataFormatRegistry; @@ -48,17 +48,17 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP SUPPORTED_FIELD_TYPES.add(FieldType.TEXT); } - private static final Set STANDARD_FILTER_OPS = Set.of( - FilterOperator.EQUALS, - FilterOperator.NOT_EQUALS, - FilterOperator.GREATER_THAN, - FilterOperator.GREATER_THAN_OR_EQUAL, - FilterOperator.LESS_THAN, - FilterOperator.LESS_THAN_OR_EQUAL, - FilterOperator.IS_NULL, - FilterOperator.IS_NOT_NULL, - FilterOperator.IN, - FilterOperator.LIKE + private static final Set STANDARD_FILTER_OPS = Set.of( + ScalarFunction.EQUALS, + ScalarFunction.NOT_EQUALS, + ScalarFunction.GREATER_THAN, + ScalarFunction.GREATER_THAN_OR_EQUAL, + ScalarFunction.LESS_THAN, + ScalarFunction.LESS_THAN_OR_EQUAL, + ScalarFunction.IS_NULL, + ScalarFunction.IS_NOT_NULL, + ScalarFunction.IN, + ScalarFunction.LIKE ); private static final Set AGG_FUNCTIONS = Set.of( @@ -99,7 +99,7 @@ public Set scanCapabilities() { public Set filterCapabilities() { Set formats = Set.copyOf(plugin.getSupportedFormats()); Set caps = new HashSet<>(); - for (FilterOperator op : STANDARD_FILTER_OPS) { + for (ScalarFunction op : STANDARD_FILTER_OPS) { for (FieldType type : SUPPORTED_FIELD_TYPES) { caps.add(new FilterCapability.Standard(op, Set.of(type), formats)); } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java index 3d8eedc56c44a..39b816202a346 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java @@ -24,6 +24,7 @@ import org.opensearch.analytics.planner.CapabilityRegistry; import org.opensearch.analytics.planner.PlannerContext; import org.opensearch.analytics.planner.PlannerImpl; +import org.opensearch.analytics.planner.dag.BackendPlanAdapter; import org.opensearch.analytics.planner.dag.DAGBuilder; import org.opensearch.analytics.planner.dag.FragmentConversionDriver; import org.opensearch.analytics.planner.dag.PlanForker; @@ -130,6 +131,7 @@ private void executeInternal(RelNode logicalFragment, ActionListener>> private final Map>> scanIndex = new HashMap<>(); - private final Map>> filterIndex = new HashMap<>(); + private final Map>> filterIndex = new HashMap<>(); private final Map>> aggregateIndex = new HashMap<>(); private final Map>> scalarIndex = new HashMap<>(); // Backends that declared supportsLiteralEvaluation=true for a (function, fieldType) @@ -100,13 +100,13 @@ public CapabilityRegistry( switch (cap) { case FilterCapability.Standard standard -> { for (FieldType fieldType : standard.fieldTypes()) { - addToFormatMap(filterIndex, new FilterKey(standard.operator(), fieldType), standard.formats(), name); + addToFormatMap(filterIndex, new ScalarKey(standard.function(), fieldType), standard.formats(), name); } } case FilterCapability.FullText fullText -> { - addToFormatMap(filterIndex, new FilterKey(fullText.operator(), fullText.fieldType()), fullText.formats(), name); + addToFormatMap(filterIndex, new ScalarKey(fullText.function(), fullText.fieldType()), fullText.formats(), name); fullTextParamIndex.put( - new FullTextParamKey(fullText.operator(), fullText.fieldType(), name), + new FullTextParamKey(fullText.function(), fullText.fieldType(), name), fullText.supportedParams() ); } @@ -182,8 +182,8 @@ public List scanBackends(Class kind, FieldType // ---- Single-format lookups ---- - public List filterBackends(FilterOperator operator, FieldType fieldType, String format) { - return filterIndex.getOrDefault(new FilterKey(operator, fieldType), Map.of()).getOrDefault(format, List.of()); + public List filterBackends(ScalarFunction function, FieldType fieldType, String format) { + return filterIndex.getOrDefault(new ScalarKey(function, fieldType), Map.of()).getOrDefault(format, List.of()); } public List aggregateBackends(AggregateFunction function, FieldType fieldType, String format) { @@ -197,14 +197,14 @@ public boolean isOpaqueOperation(String name) { // ---- Field-level lookups (iterates all formats a field has) ---- /** All backends that can filter on this field across all its storage formats. */ - public List filterBackendsForField(FilterOperator operator, FieldStorageInfo field) { + public List filterBackendsForField(ScalarFunction function, FieldStorageInfo field) { FieldType fieldType = field.getFieldType(); List result = new ArrayList<>(); for (String format : field.getDocValueFormats()) { - result.addAll(filterBackends(operator, fieldType, format)); + result.addAll(filterBackends(function, fieldType, format)); } for (String format : field.getIndexFormats()) { - result.addAll(filterBackends(operator, fieldType, format)); + result.addAll(filterBackends(function, fieldType, format)); } return result; } @@ -301,15 +301,12 @@ private static void addToFormatMap(Map>> index, private record ScanKey(Class kind, FieldType fieldType) { } - private record FilterKey(FilterOperator operator, FieldType fieldType) { - } - private record AggregateKey(AggregateFunction function, FieldType fieldType) { } private record ScalarKey(ScalarFunction function, FieldType fieldType) { } - private record FullTextParamKey(FilterOperator operator, FieldType fieldType, String backendName) { + private record FullTextParamKey(ScalarFunction function, FieldType fieldType, String backendName) { } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java index c8f964ef92615..5aa11784743b0 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java @@ -8,6 +8,7 @@ package org.opensearch.analytics.planner; +import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.analytics.spi.FieldType; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/BackendPlanAdapter.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/BackendPlanAdapter.java new file mode 100644 index 0000000000000..a3bd02fa811dd --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/BackendPlanAdapter.java @@ -0,0 +1,208 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.planner.dag; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.analytics.planner.CapabilityRegistry; +import org.opensearch.analytics.planner.RelNodeUtils; +import org.opensearch.analytics.planner.rel.OpenSearchFilter; +import org.opensearch.analytics.planner.rel.OpenSearchProject; +import org.opensearch.analytics.planner.rel.OpenSearchRelNode; +import org.opensearch.analytics.planner.rel.OperatorAnnotation; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Walks a resolved plan and applies per-function {@link ScalarFunctionAdapter}s + * provided by the driving backend. Runs between plan forking and fragment conversion. + * + *

    Each backend declares adapters keyed by {@link ScalarFunction} via + * {@link org.opensearch.analytics.spi.BackendCapabilityProvider#scalarFunctionAdapters()}. + * This component looks up the adapter for each scalar function RexCall in the plan + * and applies it if present. + * + * @opensearch.internal + */ +public class BackendPlanAdapter { + + private static final Logger LOGGER = LogManager.getLogger(BackendPlanAdapter.class); + + private BackendPlanAdapter() {} + + /** + * Adapt all plan alternatives in the DAG using each alternative's driving backend's adapters. + */ + public static void adaptAll(QueryDAG dag, CapabilityRegistry registry) { + adaptStage(dag.rootStage(), registry); + } + + private static void adaptStage(Stage stage, CapabilityRegistry registry) { + for (Stage child : stage.getChildStages()) { + adaptStage(child, registry); + } + List adapted = new ArrayList<>(stage.getPlanAlternatives().size()); + for (StagePlan plan : stage.getPlanAlternatives()) { + Map adapters = registry.getBackend(plan.backendId()) + .getCapabilityProvider() + .scalarFunctionAdapters(); + if (adapters.isEmpty()) { + adapted.add(plan); + } else { + LOGGER.debug("Before adaptation [{}]:\n{}", plan.backendId(), RelOptUtil.toString(plan.resolvedFragment())); + RelNode adaptedFragment = adaptNode(plan.resolvedFragment(), adapters); + LOGGER.debug("After adaptation [{}]:\n{}", plan.backendId(), RelOptUtil.toString(adaptedFragment)); + adapted.add(new StagePlan(adaptedFragment, plan.backendId())); + } + } + stage.setPlanAlternatives(adapted); + } + + private static RelNode adaptNode(RelNode node, Map adapters) { + List adaptedChildren = new ArrayList<>(node.getInputs().size()); + boolean childrenChanged = false; + for (RelNode child : node.getInputs()) { + RelNode adaptedChild = adaptNode(child, adapters); + adaptedChildren.add(adaptedChild); + if (adaptedChild != child) childrenChanged = true; + } + + if (node instanceof OpenSearchFilter filter) { + return adaptFilter(filter, adapters, adaptedChildren, childrenChanged); + } + if (node instanceof OpenSearchProject project) { + return adaptProject(project, adapters, adaptedChildren, childrenChanged); + } + + return childrenChanged ? node.copy(node.getTraitSet(), adaptedChildren) : node; + } + + private static RelNode adaptFilter( + OpenSearchFilter filter, + Map adapters, + List adaptedChildren, + boolean childrenChanged + ) { + List fieldStorage = filter.getOutputFieldStorage(); + RexNode adaptedCondition = adaptRex(filter.getCondition(), adapters, fieldStorage, filter.getCluster()); + if (adaptedCondition != filter.getCondition() || childrenChanged) { + return new OpenSearchFilter( + filter.getCluster(), + filter.getTraitSet(), + childrenChanged ? adaptedChildren.getFirst() : filter.getInput(), + adaptedCondition, + filter.getViableBackends() + ); + } + return filter; + } + + private static RelNode adaptProject( + OpenSearchProject project, + Map adapters, + List adaptedChildren, + boolean childrenChanged + ) { + // RexInputRef in project expressions references the input's row type + OpenSearchRelNode inputNode = (OpenSearchRelNode) RelNodeUtils.unwrapHep(project.getInput()); + List fieldStorage = inputNode.getOutputFieldStorage(); + List adaptedProjects = new ArrayList<>(project.getProjects().size()); + boolean projectsChanged = false; + for (RexNode projectExpr : project.getProjects()) { + RexNode adapted = adaptRex(projectExpr, adapters, fieldStorage, project.getCluster()); + adaptedProjects.add(adapted); + if (adapted != projectExpr) projectsChanged = true; + } + if (projectsChanged || childrenChanged) { + return new OpenSearchProject( + project.getCluster(), + project.getTraitSet(), + childrenChanged ? adaptedChildren.getFirst() : project.getInput(), + adaptedProjects, + project.getRowType(), + project.getViableBackends() + ); + } + return project; + } + + /** + * Adapts RexNodes bottom-up: operands are adapted before the call itself. + * + *

    This means a parent adapter receives already-adapted operands. This is safe + * because adapters only inspect their direct operands via + * {@code operand instanceof RexInputRef} to resolve field storage. If a child + * adapter wraps an operand in CAST, the parent sees a {@code RexCall} (not + * {@code RexInputRef}) and skips adaptation — no double-CAST occurs. + * + *

    This ordering is validated by {@code testNestedAdaptedFunctionsProduceSingleCast} + * which confirms {@code SIN(ABS($0))} with both adapted produces one CAST at the leaf. + */ + private static RexNode adaptRex( + RexNode node, + Map adapters, + List fieldStorage, + RelOptCluster cluster + ) { + if (!(node instanceof RexCall call)) { + return node; + } + + // Annotation wrappers: adapt the inner expression and re-wrap with same metadata. + // Plain RexCall.clone() would drop the annotation subclass, breaking later stripping. + if (node instanceof OperatorAnnotation annotation && annotation.unwrap() != null) { + RexNode adaptedInner = adaptRex(annotation.unwrap(), adapters, fieldStorage, cluster); + return adaptedInner == annotation.unwrap() ? node : annotation.withAdaptedOriginal(adaptedInner); + } + + // Recurse into operands first + List adaptedOperands = new ArrayList<>(call.getOperands().size()); + boolean operandsChanged = false; + for (RexNode operand : call.getOperands()) { + RexNode adapted = adaptRex(operand, adapters, fieldStorage, cluster); + adaptedOperands.add(adapted); + if (adapted != operand) operandsChanged = true; + } + + RexCall current = operandsChanged ? call.clone(call.getType(), adaptedOperands) : call; + + // Look up adapter for this function + ScalarFunction function = resolveFunction(current); + if (function != null) { + ScalarFunctionAdapter adapter = adapters.get(function); + if (adapter != null) { + return adapter.adapt(current, fieldStorage, cluster); + } + } + + return current; + } + + private static ScalarFunction resolveFunction(RexCall call) { + if (call.getOperator() instanceof SqlFunction sqlFunction) { + try { + return ScalarFunction.fromSqlFunction(sqlFunction); + } catch (IllegalArgumentException ignored) { + // Not in our enum — fall through to SqlKind resolution + } + } + return ScalarFunction.fromSqlKind(call.getKind()); + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AggregateCallAnnotation.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AggregateCallAnnotation.java index f9b58fdef7485..4dc584e5e954b 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AggregateCallAnnotation.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AggregateCallAnnotation.java @@ -79,6 +79,12 @@ public RexNode unwrap() { return null; } + @Override + public RexNode withAdaptedOriginal(RexNode adaptedOriginal) { + // AggregateCallAnnotation is a marker, not a wrapper — adaptation does not apply. + return this; + } + /** Extracts the annotation from an AggregateCall's rexList, or null if absent. * *

    TODO: window function aggregate calls may have ORDER BY expressions in rexList diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java index a52af4adf8c06..2ef04d8cb9f7e 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java @@ -77,6 +77,11 @@ public RexNode unwrap() { return original; } + @Override + public RexNode withAdaptedOriginal(RexNode adaptedOriginal) { + return new AnnotatedPredicate(type, adaptedOriginal, viableBackends, annotationId); + } + @Override protected String computeDigest(boolean withType) { return "ANNOTATED_PREDICATE(id=" + annotationId + ", backends=" + viableBackends + ", " + original + ")"; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedProjectExpression.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedProjectExpression.java index 41423436b2b26..e24a368b49a1d 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedProjectExpression.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedProjectExpression.java @@ -80,6 +80,11 @@ public RexNode unwrap() { return original; } + @Override + public RexNode withAdaptedOriginal(RexNode adaptedOriginal) { + return new AnnotatedProjectExpression(type, adaptedOriginal, viableBackends, annotationId); + } + @Override protected String computeDigest(boolean withType) { return "ANNOTATED_PROJECT_EXPR(id=" + annotationId + ", backends=" + viableBackends + ", " + original + ")"; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchAggregate.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchAggregate.java index 81e98bf4a2474..03e146d53ad93 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchAggregate.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchAggregate.java @@ -17,8 +17,8 @@ import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.ImmutableBitSet; -import org.opensearch.analytics.planner.FieldStorageInfo; import org.opensearch.analytics.planner.RelNodeUtils; +import org.opensearch.analytics.spi.FieldStorageInfo; import java.util.ArrayList; import java.util.List; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchExchangeReducer.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchExchangeReducer.java index 1e8bd47e730b0..5efe01f297c24 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchExchangeReducer.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchExchangeReducer.java @@ -16,8 +16,8 @@ import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.SingleRel; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.opensearch.analytics.planner.FieldStorageInfo; import org.opensearch.analytics.planner.RelNodeUtils; +import org.opensearch.analytics.spi.FieldStorageInfo; import java.util.List; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchFilter.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchFilter.java index b7b346d63a59f..8503b7e286ccb 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchFilter.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchFilter.java @@ -19,8 +19,8 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; -import org.opensearch.analytics.planner.FieldStorageInfo; import org.opensearch.analytics.planner.RelNodeUtils; +import org.opensearch.analytics.spi.FieldStorageInfo; import java.util.ArrayList; import java.util.List; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java index 024bc397d11ff..89a94d157fe89 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java @@ -20,8 +20,8 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; -import org.opensearch.analytics.planner.FieldStorageInfo; import org.opensearch.analytics.planner.RelNodeUtils; +import org.opensearch.analytics.spi.FieldStorageInfo; import java.util.ArrayList; import java.util.List; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchRelNode.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchRelNode.java index 98b87a911c41c..130b9ea33a96c 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchRelNode.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchRelNode.java @@ -9,7 +9,7 @@ package org.opensearch.analytics.planner.rel; import org.apache.calcite.rel.RelNode; -import org.opensearch.analytics.planner.FieldStorageInfo; +import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.analytics.spi.FragmentConvertor; import java.util.List; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchSort.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchSort.java index 248213e5dad45..b2f13e6405470 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchSort.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchSort.java @@ -16,8 +16,8 @@ import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rex.RexNode; -import org.opensearch.analytics.planner.FieldStorageInfo; import org.opensearch.analytics.planner.RelNodeUtils; +import org.opensearch.analytics.spi.FieldStorageInfo; import java.util.List; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchStageInputScan.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchStageInputScan.java index 42f8ecd986ee8..d8c5e68df0a6f 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchStageInputScan.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchStageInputScan.java @@ -17,7 +17,7 @@ import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.analytics.planner.FieldStorageInfo; +import org.opensearch.analytics.spi.FieldStorageInfo; import java.util.List; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchTableScan.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchTableScan.java index 65f87e0e8a170..0988347c498bc 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchTableScan.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchTableScan.java @@ -17,7 +17,7 @@ import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.logical.LogicalTableScan; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.opensearch.analytics.planner.FieldStorageInfo; +import org.opensearch.analytics.spi.FieldStorageInfo; import java.util.List; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OperatorAnnotation.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OperatorAnnotation.java index fcd592233fe5d..832c25afb0253 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OperatorAnnotation.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OperatorAnnotation.java @@ -30,4 +30,7 @@ public interface OperatorAnnotation { /** Returns the original unwrapped expression with annotation removed. */ RexNode unwrap(); + + /** Returns a copy of this annotation wrapping a different (adapted) inner expression. */ + RexNode withAdaptedOriginal(RexNode adaptedOriginal); } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateRule.java index e2458ba594ada..bd9b58fa0e501 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateRule.java @@ -17,7 +17,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.analytics.planner.CapabilityRegistry; -import org.opensearch.analytics.planner.FieldStorageInfo; import org.opensearch.analytics.planner.PlannerContext; import org.opensearch.analytics.planner.RelNodeUtils; import org.opensearch.analytics.planner.rel.AggregateCallAnnotation; @@ -26,6 +25,7 @@ import org.opensearch.analytics.planner.rel.OpenSearchRelNode; import org.opensearch.analytics.spi.AggregateFunction; import org.opensearch.analytics.spi.DelegationType; +import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.analytics.spi.FieldType; import java.util.ArrayList; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java index cf00865a211b7..af3b5e48fd1d0 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java @@ -20,15 +20,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.analytics.planner.CapabilityRegistry; -import org.opensearch.analytics.planner.FieldStorageInfo; import org.opensearch.analytics.planner.PlannerContext; import org.opensearch.analytics.planner.RelNodeUtils; import org.opensearch.analytics.planner.rel.AnnotatedPredicate; import org.opensearch.analytics.planner.rel.OpenSearchFilter; import org.opensearch.analytics.planner.rel.OpenSearchRelNode; import org.opensearch.analytics.spi.DelegationType; +import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.analytics.spi.FieldType; -import org.opensearch.analytics.spi.FilterOperator; +import org.opensearch.analytics.spi.ScalarFunction; import java.util.ArrayList; import java.util.HashSet; @@ -153,14 +153,14 @@ private List resolveViableBackends( ); } - FilterOperator operator = null; + ScalarFunction function = null; if (predicate.getOperator() instanceof SqlFunction sqlFunction) { - operator = FilterOperator.fromSqlFunction(sqlFunction); + function = ScalarFunction.fromSqlFunction(sqlFunction); } - if (operator == null) { - operator = FilterOperator.fromSqlKind(predicate.getKind()); + if (function == null) { + function = ScalarFunction.fromSqlKind(predicate.getKind()); } - if (operator == null) { + if (function == null) { throw new IllegalStateException("Unrecognized filter operator [" + predicate.getKind() + "]"); } @@ -186,7 +186,7 @@ private List resolveViableBackends( // A backend is viable only if it has the field in its own storage formats — ensuring // delegation targets are also field-storage-aware (e.g. Lucene is viable for a keyword // field only when the field has indexFormats=[lucene] set in the mapping). - Set fieldViable = new HashSet<>(registry.filterBackendsForField(operator, storageInfo)); + Set fieldViable = new HashSet<>(registry.filterBackendsForField(function, storageInfo)); viableSet.retainAll(fieldViable); } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java index 9124b0521e615..8d98096e21557 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java @@ -159,6 +159,9 @@ private List resolveOpaqueViableBackends(String funcName, List c private List resolveScalarViableBackends(RexCall rexCall, List childViableBackends) { ScalarFunction scalarFunc = ScalarFunction.fromSqlKind(rexCall.getKind()); + if (scalarFunc == null && rexCall.getOperator() instanceof SqlFunction sqlFunction) { + scalarFunc = ScalarFunction.fromSqlFunction(sqlFunction); + } if (scalarFunc == null) { return List.of(); } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTableScanRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTableScanRule.java index 2fcfc1b795ee2..014ac7f074b74 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTableScanRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTableScanRule.java @@ -13,11 +13,11 @@ import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.type.RelDataTypeField; import org.opensearch.analytics.planner.CapabilityRegistry; -import org.opensearch.analytics.planner.FieldStorageInfo; import org.opensearch.analytics.planner.FieldStorageResolver; import org.opensearch.analytics.planner.PlannerContext; import org.opensearch.analytics.planner.rel.OpenSearchTableScan; import org.opensearch.analytics.spi.DelegationType; +import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.cluster.metadata.IndexMetadata; import java.util.ArrayList; diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/BasePlannerRulesTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/BasePlannerRulesTests.java index 948ef3cf5d665..2af531a491ae3 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/BasePlannerRulesTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/BasePlannerRulesTests.java @@ -36,6 +36,7 @@ import org.opensearch.analytics.spi.AggregateCapability; import org.opensearch.analytics.spi.AggregateFunction; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; +import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.analytics.spi.FieldType; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FieldStorageResolverTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FieldStorageResolverTests.java index c82f6e2b08936..d216577b2afa0 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FieldStorageResolverTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FieldStorageResolverTests.java @@ -8,6 +8,7 @@ package org.opensearch.analytics.planner; +import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.settings.Settings; diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java index f9a5aec1c22d9..5c0f5c78021e0 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java @@ -16,7 +16,12 @@ import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ImmutableBitSet; import org.opensearch.analytics.planner.rel.AnnotatedPredicate; @@ -24,7 +29,6 @@ import org.opensearch.analytics.planner.rel.OpenSearchTableScan; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; import org.opensearch.analytics.spi.DelegationType; -import org.opensearch.analytics.spi.FilterOperator; import java.util.List; import java.util.Map; @@ -36,6 +40,17 @@ */ public class FilterRuleTests extends BasePlannerRulesTests { + private static SqlFunction fullTextSqlFunction(String name) { + return new SqlFunction( + name, + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + } + // ---- Per-predicate annotation tests ---- /** Integer equality — both backends can evaluate natively. */ @@ -79,7 +94,7 @@ public void testFullTextViableWithDelegation() { Map.of("message", Map.of("type", "keyword", "index", true)), new String[] { "message" }, new SqlTypeName[] { SqlTypeName.VARCHAR }, - makeFullTextCall(FilterOperator.MATCH_PHRASE.toSqlFunction(), 0, "hello world") + makeFullTextCall(fullTextSqlFunction("MATCH_PHRASE"), 0, "hello world") ); // DF is viable at operator level (has doc values in parquet) @@ -99,10 +114,7 @@ public void testAndWithDelegationBothViable() { Map.of("status", Map.of("type", "integer", "index", true), "message", Map.of("type", "keyword", "index", true)), new String[] { "status", "message" }, new SqlTypeName[] { SqlTypeName.INTEGER, SqlTypeName.VARCHAR }, - makeAnd( - makeEquals(0, SqlTypeName.INTEGER, 200), - makeFullTextCall(FilterOperator.MATCH_PHRASE.toSqlFunction(), 1, "timeout error") - ) + makeAnd(makeEquals(0, SqlTypeName.INTEGER, 200), makeFullTextCall(fullTextSqlFunction("MATCH_PHRASE"), 1, "timeout error")) ); assertTrue(result.getViableBackends().contains(MockDataFusionBackend.NAME)); @@ -125,7 +137,7 @@ public void testOrAcrossBackendsWithDelegation() { makeCall( SqlStdOperatorTable.OR, makeEquals(0, SqlTypeName.INTEGER, 200), - makeFullTextCall(FilterOperator.MATCH.toSqlFunction(), 1, "error") + makeFullTextCall(fullTextSqlFunction("MATCH"), 1, "error") ) ); @@ -148,8 +160,8 @@ public void testMultipleFullTextOrWithDelegation() { new SqlTypeName[] { SqlTypeName.VARCHAR, SqlTypeName.VARCHAR }, makeCall( SqlStdOperatorTable.OR, - makeFullTextCall(FilterOperator.MATCH.toSqlFunction(), 0, "hello"), - makeFullTextCall(FilterOperator.MATCH_PHRASE.toSqlFunction(), 1, "world") + makeFullTextCall(fullTextSqlFunction("MATCH"), 0, "hello"), + makeFullTextCall(fullTextSqlFunction("MATCH_PHRASE"), 1, "world") ) ); @@ -169,7 +181,7 @@ public void testMultipleFullTextOrWithDelegation() { /** Full-text without delegation — errors. */ public void testFullTextErrorsWithoutDelegation() { RelOptTable table = mockTable("test_index", new String[] { "message" }, new SqlTypeName[] { SqlTypeName.VARCHAR }); - RexNode condition = makeFullTextCall(FilterOperator.MATCH_PHRASE.toSqlFunction(), 0, "hello world"); + RexNode condition = makeFullTextCall(fullTextSqlFunction("MATCH_PHRASE"), 0, "hello world"); LogicalFilter filter = LogicalFilter.create(stubScan(table), condition); // index=false strips the inverted index so no backend can satisfy the full-text predicate diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java index 21c266468c70a..804c96cdf1547 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java @@ -15,8 +15,11 @@ import org.opensearch.analytics.spi.EngineCapability; import org.opensearch.analytics.spi.FilterCapability; import org.opensearch.analytics.spi.ProjectCapability; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; import org.opensearch.analytics.spi.ScanCapability; +import java.util.Map; import java.util.Set; /** @@ -66,6 +69,11 @@ public Set supportedDelegations() { public Set acceptedDelegations() { return self.acceptedDelegations(); } + + @Override + public Map scalarFunctionAdapters() { + return self.scalarFunctionAdapters(); + } }; } @@ -97,4 +105,8 @@ protected Set supportedDelegations() { protected Set acceptedDelegations() { return Set.of(); } + + protected Map scalarFunctionAdapters() { + return Map.of(); + } } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java index 3ef18a626809a..5dba4fc2d3085 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java @@ -16,7 +16,7 @@ import org.opensearch.analytics.spi.ExchangeSinkProvider; import org.opensearch.analytics.spi.FieldType; import org.opensearch.analytics.spi.FilterCapability; -import org.opensearch.analytics.spi.FilterOperator; +import org.opensearch.analytics.spi.ScalarFunction; import org.opensearch.analytics.spi.ScanCapability; import org.opensearch.index.engine.dataformat.ReaderManagerConfig; import org.opensearch.index.engine.exec.EngineReaderManager; @@ -50,17 +50,17 @@ public class MockDataFusionBackend extends MockBackend implements SearchBackEndP SUPPORTED_TYPES.add(FieldType.BOOLEAN); } - private static final Set STANDARD_OPS = Set.of( - FilterOperator.EQUALS, - FilterOperator.NOT_EQUALS, - FilterOperator.GREATER_THAN, - FilterOperator.GREATER_THAN_OR_EQUAL, - FilterOperator.LESS_THAN, - FilterOperator.LESS_THAN_OR_EQUAL, - FilterOperator.IS_NULL, - FilterOperator.IS_NOT_NULL, - FilterOperator.IN, - FilterOperator.LIKE + private static final Set STANDARD_OPS = Set.of( + ScalarFunction.EQUALS, + ScalarFunction.NOT_EQUALS, + ScalarFunction.GREATER_THAN, + ScalarFunction.GREATER_THAN_OR_EQUAL, + ScalarFunction.LESS_THAN, + ScalarFunction.LESS_THAN_OR_EQUAL, + ScalarFunction.IS_NULL, + ScalarFunction.IS_NOT_NULL, + ScalarFunction.IN, + ScalarFunction.LIKE ); private static final Set AGG_FUNCTIONS = Set.of( @@ -75,7 +75,7 @@ public class MockDataFusionBackend extends MockBackend implements SearchBackEndP private static final Set FILTER_CAPS; static { Set caps = new HashSet<>(); - for (FilterOperator op : STANDARD_OPS) { + for (ScalarFunction op : STANDARD_OPS) { caps.add(new FilterCapability.Standard(op, SUPPORTED_TYPES, DATAFUSION_FORMATS)); } FILTER_CAPS = caps; diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockLuceneBackend.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockLuceneBackend.java index ea471c99e7dc6..af87dd277f924 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockLuceneBackend.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockLuceneBackend.java @@ -10,7 +10,7 @@ import org.opensearch.analytics.spi.FieldType; import org.opensearch.analytics.spi.FilterCapability; -import org.opensearch.analytics.spi.FilterOperator; +import org.opensearch.analytics.spi.ScalarFunction; import org.opensearch.index.engine.dataformat.ReaderManagerConfig; import org.opensearch.index.engine.exec.EngineReaderManager; import org.opensearch.plugins.SearchBackEndPlugin; @@ -33,25 +33,25 @@ public class MockLuceneBackend extends MockBackend implements SearchBackEndPlugi public static final String LUCENE_DATA_FORMAT = "lucene"; private static final Set LUCENE_FORMATS = Set.of(LUCENE_DATA_FORMAT); - private static final Set STANDARD_OPS = Set.of( - FilterOperator.EQUALS, - FilterOperator.NOT_EQUALS, - FilterOperator.GREATER_THAN, - FilterOperator.GREATER_THAN_OR_EQUAL, - FilterOperator.LESS_THAN, - FilterOperator.LESS_THAN_OR_EQUAL, - FilterOperator.IS_NULL, - FilterOperator.IS_NOT_NULL, - FilterOperator.IN, - FilterOperator.LIKE + private static final Set STANDARD_OPS = Set.of( + ScalarFunction.EQUALS, + ScalarFunction.NOT_EQUALS, + ScalarFunction.GREATER_THAN, + ScalarFunction.GREATER_THAN_OR_EQUAL, + ScalarFunction.LESS_THAN, + ScalarFunction.LESS_THAN_OR_EQUAL, + ScalarFunction.IS_NULL, + ScalarFunction.IS_NOT_NULL, + ScalarFunction.IN, + ScalarFunction.LIKE ); - private static final Set FULL_TEXT_OPS = Set.of( - FilterOperator.MATCH, - FilterOperator.MATCH_PHRASE, - FilterOperator.FUZZY, - FilterOperator.WILDCARD, - FilterOperator.REGEXP + private static final Set FULL_TEXT_OPS = Set.of( + ScalarFunction.MATCH, + ScalarFunction.MATCH_PHRASE, + ScalarFunction.FUZZY, + ScalarFunction.WILDCARD, + ScalarFunction.REGEXP ); private static final Set STANDARD_TYPES = new HashSet<>(); @@ -72,10 +72,10 @@ public class MockLuceneBackend extends MockBackend implements SearchBackEndPlugi private static final Set FILTER_CAPS; static { Set caps = new HashSet<>(); - for (FilterOperator op : STANDARD_OPS) { + for (ScalarFunction op : STANDARD_OPS) { caps.add(new FilterCapability.Standard(op, STANDARD_TYPES, LUCENE_FORMATS)); } - for (FilterOperator op : FULL_TEXT_OPS) { + for (ScalarFunction op : FULL_TEXT_OPS) { for (FieldType type : FULL_TEXT_TYPES) { caps.add(new FilterCapability.FullText(op, type, LUCENE_FORMATS, Set.of())); } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/BackendPlanAdapterTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/BackendPlanAdapterTests.java new file mode 100644 index 0000000000000..b7072555be7dc --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/BackendPlanAdapterTests.java @@ -0,0 +1,325 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.planner.dag; + +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.analytics.planner.BasePlannerRulesTests; +import org.opensearch.analytics.planner.MockDataFusionBackend; +import org.opensearch.analytics.planner.PlannerContext; +import org.opensearch.analytics.planner.rel.AnnotatedPredicate; +import org.opensearch.analytics.planner.rel.OpenSearchFilter; +import org.opensearch.analytics.planner.rel.OperatorAnnotation; +import org.opensearch.analytics.spi.FieldType; +import org.opensearch.analytics.spi.ProjectCapability; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Tests for {@link BackendPlanAdapter} — verifies per-function adapters are applied + * correctly between plan forking and fragment conversion. + */ +public class BackendPlanAdapterTests extends BasePlannerRulesTests { + + private static final Logger LOGGER = LogManager.getLogger(BackendPlanAdapterTests.class); + + private static final SqlFunction SIN_FUNCTION = new SqlFunction( + "SIN", + SqlKind.OTHER_FUNCTION, + ReturnTypes.DOUBLE, + null, + OperandTypes.NUMERIC, + SqlFunctionCategory.NUMERIC + ); + + private final ScalarFunctionAdapter sinCastAdapter = (call, fieldStorage, cluster) -> { + List adaptedOperands = new ArrayList<>(call.getOperands().size()); + boolean changed = false; + for (RexNode operand : call.getOperands()) { + if (operand instanceof RexInputRef) { + SqlTypeName typeName = operand.getType().getSqlTypeName(); + if (typeName == SqlTypeName.INTEGER || typeName == SqlTypeName.BIGINT) { + adaptedOperands.add( + cluster.getRexBuilder().makeCast(cluster.getTypeFactory().createSqlType(SqlTypeName.DOUBLE), operand) + ); + changed = true; + continue; + } + } + adaptedOperands.add(operand); + } + return changed ? call.clone(call.getType(), adaptedOperands) : call; + }; + + private RexCall adaptSinFilter(SqlTypeName operandType, Map> fields) { + return adaptSinFilter(operandType, fields, fields.keySet().toArray(String[]::new), null); + } + + private RexCall adaptSinFilter( + SqlTypeName operandType, + Map> fields, + String[] fieldNames, + SqlTypeName[] fieldTypes + ) { + MockDataFusionBackend dfWithAdapter = new MockDataFusionBackend() { + @Override + protected Map scalarFunctionAdapters() { + return Map.of(ScalarFunction.SIN, sinCastAdapter); + } + }; + + PlannerContext context = buildContext("parquet", 1, fields, List.of(dfWithAdapter)); + + RexNode sinCall = rexBuilder.makeCall(SIN_FUNCTION, rexBuilder.makeInputRef(typeFactory.createSqlType(operandType), 0)); + RexNode condition = rexBuilder.makeCall( + SqlStdOperatorTable.GREATER_THAN, + sinCall, + rexBuilder.makeLiteral(0.5, typeFactory.createSqlType(SqlTypeName.DOUBLE), true) + ); + RelOptTable table = fieldTypes != null ? mockTable("test_index", fieldNames, fieldTypes) : mockTable("test_index", fieldNames); + LogicalFilter filter = LogicalFilter.create(stubScan(table), condition); + + RelNode marked = runPlanner(filter, context); + LOGGER.debug("Marked:\n{}", RelOptUtil.toString(marked)); + + QueryDAG dag = DAGBuilder.build(marked, context.getCapabilityRegistry(), mockClusterService()); + PlanForker.forkAll(dag, context.getCapabilityRegistry()); + BackendPlanAdapter.adaptAll(dag, context.getCapabilityRegistry()); + + StagePlan plan = dag.rootStage().getPlanAlternatives().getFirst(); + OpenSearchFilter adaptedFilter = (OpenSearchFilter) plan.resolvedFragment(); + assertTrue("Annotations must survive adaptation", containsAnnotation(adaptedFilter.getCondition())); + return findCallByName(adaptedFilter.getCondition(), "SIN"); + } + + private static boolean containsAnnotation(RexNode node) { + if (node instanceof OperatorAnnotation) return true; + if (node instanceof RexCall call) { + for (RexNode operand : call.getOperands()) { + if (containsAnnotation(operand)) return true; + } + } + return false; + } + + /** SIN(integer_column) should be adapted to SIN(CAST(integer_column AS DOUBLE)). */ + public void testSinAdapterInsertsCastForIntegerField() { + RexCall sinCall = adaptSinFilter(SqlTypeName.INTEGER, intFields()); + assertNotNull("SIN call should exist in adapted condition", sinCall); + assertEquals("SIN operand should be CAST after adaptation", SqlKind.CAST, sinCall.getOperands().getFirst().getKind()); + } + + /** SIN(double_column) should NOT be adapted — no CAST needed. */ + public void testSinAdapterNoOpForDoubleField() { + Map> doubleFields = Map.of("price", Map.of("type", "double"), "amount", Map.of("type", "double")); + RexCall sinCall = adaptSinFilter( + SqlTypeName.DOUBLE, + doubleFields, + new String[] { "price", "amount" }, + new SqlTypeName[] { SqlTypeName.DOUBLE, SqlTypeName.DOUBLE } + ); + assertNotNull("SIN call should exist in adapted condition", sinCall); + assertNotSame("SIN operand should NOT be CAST for double field", SqlKind.CAST, sinCall.getOperands().getFirst().getKind()); + } + + /** SIN(integer_column) in a project should also get CAST inserted. */ + public void testSinAdapterInProjectInsertsCastForIntegerField() { + MockDataFusionBackend dfWithAdapter = new MockDataFusionBackend() { + @Override + protected Map scalarFunctionAdapters() { + return Map.of(ScalarFunction.SIN, sinCastAdapter); + } + + @Override + protected Set projectCapabilities() { + return Set.of( + new ProjectCapability.Scalar( + ScalarFunction.SIN, + Set.of(FieldType.INTEGER, FieldType.DOUBLE), + Set.of(MockDataFusionBackend.PARQUET_DATA_FORMAT), + false + ) + ); + } + }; + + PlannerContext context = buildContext("parquet", 1, intFields(), List.of(dfWithAdapter)); + + RexNode sinExpr = rexBuilder.makeCall(SIN_FUNCTION, rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 0)); + RelNode scan = stubScan(mockTable("test_index", "status", "size")); + LogicalProject project = LogicalProject.create(scan, List.of(), List.of(sinExpr), List.of("sin_status")); + + RelNode marked = runPlanner(project, context); + LOGGER.info("Marked project:\n{}", RelOptUtil.toString(marked)); + + QueryDAG dag = DAGBuilder.build(marked, context.getCapabilityRegistry(), mockClusterService()); + PlanForker.forkAll(dag, context.getCapabilityRegistry()); + BackendPlanAdapter.adaptAll(dag, context.getCapabilityRegistry()); + + StagePlan plan = dag.rootStage().getPlanAlternatives().getFirst(); + // Find SIN call in the project expressions + RexCall sinCall = null; + if (plan.resolvedFragment() instanceof org.opensearch.analytics.planner.rel.OpenSearchProject adaptedProject) { + for (RexNode expr : adaptedProject.getProjects()) { + assertTrue("Project annotations must survive adaptation", containsAnnotation(expr)); + sinCall = findCallByName(expr, "SIN"); + if (sinCall != null) break; + } + } + assertNotNull("SIN call should exist in adapted project", sinCall); + assertEquals("SIN operand should be CAST after adaptation in project", SqlKind.CAST, sinCall.getOperands().getFirst().getKind()); + } + + /** Filter with SIN (adapted) AND ABS (no adapter) — SIN gets CAST, ABS unchanged. */ + public void testMixedAdaptedAndNonAdaptedFunctions() { + MockDataFusionBackend dfWithSinAdapterOnly = new MockDataFusionBackend() { + @Override + protected Map scalarFunctionAdapters() { + return Map.of(ScalarFunction.SIN, sinCastAdapter); + } + }; + + PlannerContext context = buildContext("parquet", 1, intFields(), List.of(dfWithSinAdapterOnly)); + + RexNode sinGt = rexBuilder.makeCall( + SqlStdOperatorTable.GREATER_THAN, + rexBuilder.makeCall(SIN_FUNCTION, rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 0)), + rexBuilder.makeLiteral(0.5, typeFactory.createSqlType(SqlTypeName.DOUBLE), true) + ); + RexNode absGt = rexBuilder.makeCall( + SqlStdOperatorTable.GREATER_THAN, + rexBuilder.makeCall(SqlStdOperatorTable.ABS, rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1)), + rexBuilder.makeLiteral(10, typeFactory.createSqlType(SqlTypeName.INTEGER), true) + ); + RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.AND, sinGt, absGt); + LogicalFilter filter = LogicalFilter.create(stubScan(mockTable("test_index", "status", "size")), condition); + + RelNode marked = runPlanner(filter, context); + QueryDAG dag = DAGBuilder.build(marked, context.getCapabilityRegistry(), mockClusterService()); + PlanForker.forkAll(dag, context.getCapabilityRegistry()); + BackendPlanAdapter.adaptAll(dag, context.getCapabilityRegistry()); + + StagePlan plan = dag.rootStage().getPlanAlternatives().getFirst(); + OpenSearchFilter adaptedFilter = (OpenSearchFilter) plan.resolvedFragment(); + assertTrue("Annotations must survive mixed adaptation", containsAnnotation(adaptedFilter.getCondition())); + RexCall sinCall = findCallByName(adaptedFilter.getCondition(), "SIN"); + RexCall absCall = findCallByName(adaptedFilter.getCondition(), "ABS"); + assertNotNull("SIN call should exist in adapted condition", sinCall); + assertNotNull("ABS call should exist in adapted condition", absCall); + assertEquals("SIN operand should be CAST after adaptation", SqlKind.CAST, sinCall.getOperands().getFirst().getKind()); + assertEquals("ABS operand should remain INPUT_REF without adapter", SqlKind.INPUT_REF, absCall.getOperands().getFirst().getKind()); + } + + /** No adapters registered — plan should pass through completely unchanged. */ + public void testNoAdaptersRegisteredLeavesEverythingUnchanged() { + PlannerContext context = buildContext("parquet", 1, intFields()); + + RexNode condition = rexBuilder.makeCall( + SqlStdOperatorTable.GREATER_THAN, + rexBuilder.makeCall(SIN_FUNCTION, rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 0)), + rexBuilder.makeLiteral(0.5, typeFactory.createSqlType(SqlTypeName.DOUBLE), true) + ); + LogicalFilter filter = LogicalFilter.create(stubScan(mockTable("test_index", "status", "size")), condition); + + RelNode marked = runPlanner(filter, context); + QueryDAG dag = DAGBuilder.build(marked, context.getCapabilityRegistry(), mockClusterService()); + PlanForker.forkAll(dag, context.getCapabilityRegistry()); + BackendPlanAdapter.adaptAll(dag, context.getCapabilityRegistry()); + + StagePlan plan = dag.rootStage().getPlanAlternatives().getFirst(); + OpenSearchFilter adaptedFilter = (OpenSearchFilter) plan.resolvedFragment(); + assertTrue("Annotations must survive when no adapters registered", containsAnnotation(adaptedFilter.getCondition())); + RexCall sinCall = findCallByName(adaptedFilter.getCondition(), "SIN"); + assertNotNull("SIN call should exist in condition", sinCall); + assertEquals( + "SIN operand should remain INPUT_REF with no adapters registered", + SqlKind.INPUT_REF, + sinCall.getOperands().getFirst().getKind() + ); + } + + /** Nested SIN(ABS($0)) — both have adapters, only one CAST at the leaf. */ + public void testNestedAdaptedFunctionsProduceSingleCast() { + ScalarFunctionAdapter castAdapter = sinCastAdapter; // same logic works for ABS + MockDataFusionBackend dfWithBothAdapters = new MockDataFusionBackend() { + @Override + protected Map scalarFunctionAdapters() { + return Map.of(ScalarFunction.SIN, castAdapter, ScalarFunction.ABS, castAdapter); + } + }; + + PlannerContext context = buildContext("parquet", 1, intFields(), List.of(dfWithBothAdapters)); + + RexNode absCall = rexBuilder.makeCall( + SqlStdOperatorTable.ABS, + rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 0) + ); + RexNode sinAbsCall = rexBuilder.makeCall(SIN_FUNCTION, absCall); + RexNode condition = rexBuilder.makeCall( + SqlStdOperatorTable.GREATER_THAN, + sinAbsCall, + rexBuilder.makeLiteral(0.5, typeFactory.createSqlType(SqlTypeName.DOUBLE), true) + ); + LogicalFilter filter = LogicalFilter.create(stubScan(mockTable("test_index", "status", "size")), condition); + + RelNode marked = runPlanner(filter, context); + QueryDAG dag = DAGBuilder.build(marked, context.getCapabilityRegistry(), mockClusterService()); + PlanForker.forkAll(dag, context.getCapabilityRegistry()); + BackendPlanAdapter.adaptAll(dag, context.getCapabilityRegistry()); + + StagePlan plan = dag.rootStage().getPlanAlternatives().getFirst(); + OpenSearchFilter adaptedFilter = (OpenSearchFilter) plan.resolvedFragment(); + + // ABS should have CAST on its direct RexInputRef operand + RexCall absResult = findCallByName(adaptedFilter.getCondition(), "ABS"); + assertNotNull("ABS call should exist", absResult); + assertEquals("ABS operand should be CAST", SqlKind.CAST, absResult.getOperands().getFirst().getKind()); + + // SIN's operand is ABS (a RexCall, not RexInputRef) — adapter should NOT insert CAST + RexCall sinResult = findCallByName(adaptedFilter.getCondition(), "SIN"); + assertNotNull("SIN call should exist", sinResult); + assertEquals( + "SIN operand should be ABS (no double-CAST)", + "ABS", + ((RexCall) sinResult.getOperands().getFirst()).getOperator().getName() + ); + } + + private static RexCall findCallByName(RexNode node, String name) { + if (node instanceof AnnotatedPredicate annotated) return findCallByName(annotated.getOriginal(), name); + if (node instanceof RexCall call) { + if (call.getOperator().getName().equalsIgnoreCase(name)) return call; + for (RexNode operand : call.getOperands()) { + RexCall found = findCallByName(operand, name); + if (found != null) return found; + } + } + return null; + } +} From e36163741c309210a0a764c3aad8294ed1866bb9 Mon Sep 17 00:00:00 2001 From: Ralph Ursprung <39383228+rursprung@users.noreply.github.com> Date: Thu, 30 Apr 2026 14:20:39 +0200 Subject: [PATCH 021/115] update `httpclient5` to 5.6.1 (#21441) this resolves CVE-2026-40542 Signed-off-by: Ralph Ursprung --- client/rest/licenses/httpclient5-5.6.1.jar.sha1 | 1 + client/rest/licenses/httpclient5-5.6.jar.sha1 | 1 - client/sniffer/licenses/httpclient5-5.6.1.jar.sha1 | 1 + client/sniffer/licenses/httpclient5-5.6.jar.sha1 | 1 - gradle/libs.versions.toml | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 client/rest/licenses/httpclient5-5.6.1.jar.sha1 delete mode 100644 client/rest/licenses/httpclient5-5.6.jar.sha1 create mode 100644 client/sniffer/licenses/httpclient5-5.6.1.jar.sha1 delete mode 100644 client/sniffer/licenses/httpclient5-5.6.jar.sha1 diff --git a/client/rest/licenses/httpclient5-5.6.1.jar.sha1 b/client/rest/licenses/httpclient5-5.6.1.jar.sha1 new file mode 100644 index 0000000000000..8c78044ffe7e2 --- /dev/null +++ b/client/rest/licenses/httpclient5-5.6.1.jar.sha1 @@ -0,0 +1 @@ +b418ba210ace28adf920f1decf64d673953d07cf \ No newline at end of file diff --git a/client/rest/licenses/httpclient5-5.6.jar.sha1 b/client/rest/licenses/httpclient5-5.6.jar.sha1 deleted file mode 100644 index f6c5a64d1e4ee..0000000000000 --- a/client/rest/licenses/httpclient5-5.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f502ee00ba82d44a6a29bda06a18f5b959808e09 \ No newline at end of file diff --git a/client/sniffer/licenses/httpclient5-5.6.1.jar.sha1 b/client/sniffer/licenses/httpclient5-5.6.1.jar.sha1 new file mode 100644 index 0000000000000..8c78044ffe7e2 --- /dev/null +++ b/client/sniffer/licenses/httpclient5-5.6.1.jar.sha1 @@ -0,0 +1 @@ +b418ba210ace28adf920f1decf64d673953d07cf \ No newline at end of file diff --git a/client/sniffer/licenses/httpclient5-5.6.jar.sha1 b/client/sniffer/licenses/httpclient5-5.6.jar.sha1 deleted file mode 100644 index f6c5a64d1e4ee..0000000000000 --- a/client/sniffer/licenses/httpclient5-5.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f502ee00ba82d44a6a29bda06a18f5b959808e09 \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f4dcbb4c26f8a..1acb20727ca15 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -49,7 +49,7 @@ reactor_netty = "1.3.5" reactor = "3.8.5" # client dependencies -httpclient5 = "5.6" +httpclient5 = "5.6.1" httpcore5 = "5.4" httpclient = "4.5.14" httpcore = "4.4.16" From e6d708d5c41414b161063b83a4beebabc092c690 Mon Sep 17 00:00:00 2001 From: A S K Kamal Nayan Date: Thu, 30 Apr 2026 19:37:55 +0530 Subject: [PATCH 022/115] Share single FormatChecksumStrategy instance per shard between engine and store (#21232) * Share single FormatChecksumStrategy instance per shard, between engine and directory Signed-off-by: Kamal Nayan * Minor refactoring Signed-off-by: Kamal Nayan * Added Tests for checkum strategy sharing Signed-off-by: Kamal Nayan * Add checksum read-back assertion to shared strategy test Signed-off-by: Kamal Nayan * Fix indexingEngine call in CompositeIndexingExecutionEngineTests for updated API Signed-off-by: Kamal Nayan * Fix compilation after rebase with upstream Lucene engine plugin Signed-off-by: Kamal Nayan * Updated DataFormatRegistry tests to increase test coverage Signed-off-by: Kamal Nayan * updated the code to use supplier for DataFormatDescriptors for lazy initialization Signed-off-by: Kamal Nayan --------- Signed-off-by: Kamal Nayan Co-authored-by: Kamal Nayan --- .../opensearch/be/lucene/LucenePlugin.java | 7 +- .../be/lucene/index/LuceneCommitterTests.java | 1 + .../LuceneIndexingExecutionEngineTests.java | 1 + .../composite/CompositeMergeIT.java | 5 +- .../composite/CompositeDataFormatPlugin.java | 22 +- .../CompositeIndexingExecutionEngine.java | 9 +- .../CompositeDataFormatPluginTests.java | 23 +- ...CompositeIndexingExecutionEngineTests.java | 2 +- .../composite/CompositeTestHelper.java | 7 +- .../parquet/ParquetDataFormatPlugin.java | 16 +- .../opensearch/index/shard/IndexShardIT.java | 1 + .../org/opensearch/index/IndexService.java | 16 +- .../index/engine/DataFormatAwareEngine.java | 3 +- .../opensearch/index/engine/EngineConfig.java | 15 ++ .../index/engine/EngineConfigFactory.java | 6 +- .../dataformat/DataFormatDescriptor.java | 2 +- .../engine/dataformat/DataFormatPlugin.java | 22 +- .../engine/dataformat/DataFormatRegistry.java | 59 ++++- .../dataformat/IndexingEngineConfig.java | 5 +- .../opensearch/index/shard/IndexShard.java | 12 +- .../store/DataFormatAwareStoreDirectory.java | 43 +--- .../DataFormatAwareStoreDirectoryFactory.java | 6 +- ...tDataFormatAwareStoreDirectoryFactory.java | 15 +- .../engine/EngineConfigFactoryTests.java | 2 + .../dataformat/DataFormatPluginTests.java | 6 +- .../dataformat/DataFormatRegistryTests.java | 33 ++- .../FormatChecksumStrategySharingTests.java | 224 ++++++++++++++++++ .../DataFormatAwareStoreDirectoryTests.java | 27 +-- ...FormatAwareStoreDirectoryFactoryTests.java | 32 +-- .../DataFormatAwareRemoteDirectoryTests.java | 3 +- .../dataformat/stub/MockDataFormatPlugin.java | 3 +- .../index/shard/IndexShardTestCase.java | 1 + 32 files changed, 446 insertions(+), 183 deletions(-) create mode 100644 server/src/test/java/org/opensearch/index/engine/dataformat/FormatChecksumStrategySharingTests.java diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LucenePlugin.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LucenePlugin.java index 3c2de857d9449..88a3c569f53ae 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LucenePlugin.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LucenePlugin.java @@ -22,7 +22,6 @@ import org.opensearch.index.engine.exec.EngineReaderManager; import org.opensearch.index.engine.exec.commit.Committer; import org.opensearch.index.engine.exec.commit.CommitterFactory; -import org.opensearch.index.store.FormatChecksumStrategy; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchBackEndPlugin; @@ -66,15 +65,11 @@ public DataFormat getDataFormat() { * Requires the committer to be a {@link LuceneCommitter}. * * @param indexingEngineConfig the engine configuration containing committer, mapper service, and store - * @param checksumStrategy the checksum strategy for the format (unused by Lucene) * @return a new Lucene indexing execution engine * @throws IllegalStateException if the committer is not a {@link LuceneCommitter} */ @Override - public IndexingExecutionEngine indexingEngine( - IndexingEngineConfig indexingEngineConfig, - FormatChecksumStrategy checksumStrategy - ) { + public IndexingExecutionEngine indexingEngine(IndexingEngineConfig indexingEngineConfig) { Committer committer = indexingEngineConfig.committer(); if (committer instanceof LuceneCommitter luceneCommitter) { return new LuceneIndexingExecutionEngine( diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterTests.java index 6a7d6c0844afd..05bdca5c79170 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterTests.java @@ -94,6 +94,7 @@ private CommitterConfig createCommitterConfig() throws IOException { null, null, null, + null, null ); return new CommitterConfig(engineConfig); diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java index 08b6c6027b855..a84d1c3b782b9 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java @@ -123,6 +123,7 @@ private LuceneCommitter createCommitter() throws IOException { null, null, null, + null, null ); CommitterConfig settings = new CommitterConfig(engineConfig); diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java index 4faa5948d7c4e..3ceda94a96d06 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java @@ -41,6 +41,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.function.Supplier; /** * Integration tests for composite merge operations across single and multiple data format engines. @@ -66,8 +67,8 @@ public MockParquetDataFormatPlugin() { } @Override - public Map getFormatDescriptors(IndexSettings indexSettings, DataFormatRegistry registry) { - return Map.of("parquet", new DataFormatDescriptor("parquet", new PrecomputedChecksumStrategy())); + public Map> getFormatDescriptors(IndexSettings indexSettings, DataFormatRegistry registry) { + return Map.of("parquet", () -> new DataFormatDescriptor("parquet", new PrecomputedChecksumStrategy())); } @Override diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java index d1dc6463b396c..7ec61ceded267 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java @@ -20,7 +20,6 @@ import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.engine.dataformat.IndexingEngineConfig; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; -import org.opensearch.index.store.FormatChecksumStrategy; import org.opensearch.plugins.ExtensiblePlugin; import org.opensearch.plugins.Plugin; @@ -28,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * Sandbox plugin that provides a {@link CompositeIndexingExecutionEngine} for @@ -92,35 +92,33 @@ public DataFormat getDataFormat() { } @Override - public IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings, FormatChecksumStrategy checksumStrategy) { - Map strategies = new HashMap<>(); - for (Map.Entry entry : getFormatDescriptors(settings.indexSettings(), settings.registry()) - .entrySet()) { - strategies.put(entry.getKey(), entry.getValue().getChecksumStrategy()); - } + public IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings) { return new CompositeIndexingExecutionEngine( settings.indexSettings(), settings.mapperService(), settings.committer(), settings.registry(), settings.store(), - strategies + settings.checksumStrategies() ); } @Override - public Map getFormatDescriptors(IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry) { + public Map> getFormatDescriptors( + IndexSettings indexSettings, + DataFormatRegistry dataFormatRegistry + ) { Settings settings = indexSettings.getSettings(); String primaryFormatName = PRIMARY_DATA_FORMAT.get(settings); List secondaryFormatNames = SECONDARY_DATA_FORMATS.get(settings); - Map descriptors = new HashMap<>(); + Map> descriptors = new HashMap<>(); if (primaryFormatName != null) { - descriptors.putAll(dataFormatRegistry.getFormatDescriptors(indexSettings)); + descriptors.putAll(dataFormatRegistry.getFormatDescriptors(indexSettings, dataFormatRegistry.format(primaryFormatName))); } for (String secondaryName : secondaryFormatNames) { if (secondaryName != null) { - descriptors.putAll(dataFormatRegistry.getFormatDescriptors(indexSettings)); + descriptors.putAll(dataFormatRegistry.getFormatDescriptors(indexSettings, dataFormatRegistry.format(secondaryName))); } } return Map.copyOf(descriptors); diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java index 831ccfb0971c8..4dc8b3f8165b5 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java @@ -115,7 +115,14 @@ public CompositeIndexingExecutionEngine( validateFormatsRegistered(dataFormatRegistry, primaryFormatName, secondaryFormatNames); Map strategies = checksumStrategies != null ? checksumStrategies : Map.of(); - IndexingEngineConfig engineSettings = new IndexingEngineConfig(committer, mapperService, indexSettings, store, dataFormatRegistry); + IndexingEngineConfig engineSettings = new IndexingEngineConfig( + committer, + mapperService, + indexSettings, + store, + dataFormatRegistry, + strategies + ); List allFormats = new ArrayList<>(); DataFormat primaryFormat = dataFormatRegistry.format(primaryFormatName); diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java index 4e7dd4cdcea75..c48306473197e 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java @@ -11,11 +11,13 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.test.OpenSearchTestCase; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -60,24 +62,27 @@ public void testGetFormatDescriptorsDelegatestoPlugins() { IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); DataFormatRegistry registry = mock(DataFormatRegistry.class); - when(registry.format("parquet")).thenReturn(CompositeTestHelper.stubFormat("parquet", 2, java.util.Set.of())); - when(registry.getFormatDescriptors(indexSettings)).thenReturn( + DataFormat parquetFormat = CompositeTestHelper.stubFormat("parquet", 2, java.util.Set.of()); + when(registry.format("parquet")).thenReturn(parquetFormat); + when(registry.format("lucene")).thenReturn(CompositeTestHelper.stubFormat("lucene", 1, java.util.Set.of())); + when(registry.getFormatDescriptors(indexSettings, parquetFormat)).thenReturn( Map.of( "parquet", - new org.opensearch.index.engine.dataformat.DataFormatDescriptor( - "parquet", - new org.opensearch.index.store.checksum.GenericCRC32ChecksumHandler() - ) + (Supplier< + org.opensearch.index.engine.dataformat.DataFormatDescriptor>) () -> new org.opensearch.index.engine.dataformat.DataFormatDescriptor( + "parquet", + new org.opensearch.index.store.checksum.GenericCRC32ChecksumHandler() + ) ) ); - Map descriptors = plugin.getFormatDescriptors( + Map> descriptors = plugin.getFormatDescriptors( indexSettings, registry ); assertEquals(1, descriptors.size()); assertTrue(descriptors.containsKey("parquet")); - assertEquals("parquet", descriptors.get("parquet").getFormatName()); + assertEquals("parquet", descriptors.get("parquet").get().getFormatName()); } public void testGetFormatDescriptorsEmptyWhenNoPluginsMatch() { @@ -94,7 +99,7 @@ public void testGetFormatDescriptorsEmptyWhenNoPluginsMatch() { .build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - Map descriptors = plugin.getFormatDescriptors( + Map> descriptors = plugin.getFormatDescriptors( indexSettings, registry ); diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java index 59bd20c7a231b..fc6263f6f8b25 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java @@ -76,7 +76,7 @@ public void testConstructorThrowsWhenSecondaryFormatNotRegistered() { when(registry.getRegisteredFormats()).thenReturn(Set.of(CompositeTestHelper.stubFormat("lucene", 1, Set.of()))); when(registry.getIndexingEngine(any(), any())).thenAnswer(invocation -> { DataFormatPlugin plugin = CompositeTestHelper.stubPlugin("lucene", 1); - return plugin.indexingEngine(null, null); + return plugin.indexingEngine(null); }); Settings settings = Settings.builder() diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java index f15dadd01fc4a..236e52739bc66 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java @@ -31,7 +31,6 @@ import org.opensearch.index.engine.exec.commit.Committer; import org.opensearch.index.engine.exec.commit.IndexStoreProvider; import org.opensearch.index.engine.exec.coord.CatalogSnapshot; -import org.opensearch.index.store.FormatChecksumStrategy; import java.util.Collection; import java.util.Collections; @@ -72,7 +71,7 @@ static CompositeIndexingExecutionEngine createStubEngine(String primaryName, Str when(registry.getIndexingEngine(any(), any())).thenAnswer(invocation -> { DataFormat format = invocation.getArgument(1); DataFormatPlugin plugin = plugins.get(format.name()); - return plugin.indexingEngine(null, null); + return plugin.indexingEngine(null); }); Settings.Builder settingsBuilder = Settings.builder() @@ -101,7 +100,7 @@ public DataFormat getDataFormat() { } @Override - public IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings, FormatChecksumStrategy checksumStrategy) { + public IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings) { return new StubIndexingExecutionEngine(format); } }; @@ -116,7 +115,7 @@ public DataFormat getDataFormat() { } @Override - public IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings, FormatChecksumStrategy checksumStrategy) { + public IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings) { return new StubIndexingExecutionEngine(format); } }; diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java index fc5da5742adf6..33bb655257a78 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java @@ -24,7 +24,6 @@ import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.engine.dataformat.IndexingEngineConfig; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; -import org.opensearch.index.store.FormatChecksumStrategy; import org.opensearch.index.store.PrecomputedChecksumStrategy; import org.opensearch.parquet.engine.ParquetDataFormat; import org.opensearch.parquet.engine.ParquetIndexingEngine; @@ -52,10 +51,9 @@ * {@link #createComponents} and passes them to the per-shard * {@link ParquetIndexingEngine} instances created in {@link #indexingEngine}. * - *

    The descriptor provides a {@link PrecomputedChecksumStrategy} that the directory - * holds at construction time. The {@link ParquetIndexingEngine} receives the same - * strategy instance from the directory via - * {@link org.opensearch.index.store.DataFormatAwareStoreDirectory#getChecksumStrategy}, + *

    The descriptor provides a {@link PrecomputedChecksumStrategy} that is created once + * per shard during initialization. The same strategy instance is shared between the + * directory and the {@link ParquetIndexingEngine} via the checksum strategies map, * so pre-computed CRC32 values registered during write are directly visible to the * upload path — no post-construction wiring needed. * @@ -99,7 +97,7 @@ public DataFormat getDataFormat() { } @Override - public IndexingExecutionEngine indexingEngine(IndexingEngineConfig engineConfig, FormatChecksumStrategy checksumStrategy) { + public IndexingExecutionEngine indexingEngine(IndexingEngineConfig engineConfig) { return new ParquetIndexingEngine( settings, dataFormat, @@ -107,15 +105,15 @@ public DataFormat getDataFormat() { () -> ArrowSchemaBuilder.getSchema(engineConfig.mapperService()), engineConfig.indexSettings(), threadPool, - checksumStrategy + engineConfig.checksumStrategies().get(ParquetDataFormat.PARQUET_DATA_FORMAT_NAME) ); } @Override - public Map getFormatDescriptors(IndexSettings indexSettings, DataFormatRegistry registry) { + public Map> getFormatDescriptors(IndexSettings indexSettings, DataFormatRegistry registry) { return Map.of( ParquetDataFormat.PARQUET_DATA_FORMAT_NAME, - new DataFormatDescriptor(ParquetDataFormat.PARQUET_DATA_FORMAT_NAME, new PrecomputedChecksumStrategy()) + () -> new DataFormatDescriptor(ParquetDataFormat.PARQUET_DATA_FORMAT_NAME, new PrecomputedChecksumStrategy()) ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java index 82a812cb4bb56..ea63e14cfcb3f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java @@ -734,6 +734,7 @@ public static final IndexShard newIndexShard( clusterService.getClusterApplierService(), MergedSegmentPublisher.EMPTY, ReferencedSegmentsPublisher.EMPTY, + Collections.emptyMap(), null // TODO ); } diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 65bcfdcc565c5..12c611bcb2b15 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -100,6 +100,7 @@ import org.opensearch.index.similarity.SimilarityService; import org.opensearch.index.store.DataFormatAwareStoreDirectory; import org.opensearch.index.store.DataFormatAwareStoreDirectoryFactory; +import org.opensearch.index.store.FormatChecksumStrategy; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.Store; import org.opensearch.index.store.remote.filecache.FileCache; @@ -773,6 +774,10 @@ protected void closeInternal() { } Directory directory = null; + Map checksumStrategies = Collections.emptyMap(); + if (this.indexSettings.isPluggableDataFormatEnabled() && dataFormatRegistry != null) { + checksumStrategies = dataFormatRegistry.createChecksumStrategies(this.indexSettings); + } if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_SETTING) && // TODO : Need to remove this check after support for hot indices is added in Composite Directory this.indexSettings.isWarmIndex()) { @@ -788,7 +793,7 @@ protected void closeInternal() { directory = directoryFactory.newDirectory(this.indexSettings, path); } else { // Will be enabled in case of formatAware indices. - directory = createDataFormatAwareStoreDirectory(shardId, path); + directory = createDataFormatAwareStoreDirectory(shardId, path, checksumStrategies); } store = storeFactory.newStore( shardId, @@ -839,6 +844,7 @@ protected void closeInternal() { clusterService.getClusterApplierService(), this.indexSettings.isSegRepEnabledOrRemoteNode() ? mergedSegmentPublisher : null, this.indexSettings.isSegRepEnabledOrRemoteNode() ? referencedSegmentsPublisher : null, + checksumStrategies, dataFormatRegistry ); eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); @@ -1344,7 +1350,11 @@ public boolean isForceExecution() { * Creates DataFormatAwareStoreDirectory using the factory if available, otherwise fallback to Store's internal creation. * This method centralizes the directory creation logic and enables plugin-based format discovery. */ - private DataFormatAwareStoreDirectory createDataFormatAwareStoreDirectory(ShardId shardId, ShardPath shardPath) throws IOException { + private DataFormatAwareStoreDirectory createDataFormatAwareStoreDirectory( + ShardId shardId, + ShardPath shardPath, + Map checksumStrategies + ) throws IOException { if (dataFormatAwareStoreDirectoryFactory != null) { logger.debug("Using DataFormatAwareStoreDirectoryFactory to create directory for shard path: {}", shardPath); return dataFormatAwareStoreDirectoryFactory.newDataFormatAwareStoreDirectory( @@ -1352,7 +1362,7 @@ private DataFormatAwareStoreDirectory createDataFormatAwareStoreDirectory(ShardI shardId, shardPath, directoryFactory, - dataFormatRegistry + checksumStrategies ); } diff --git a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java index bfb919cba8e6d..0906953e68477 100644 --- a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java @@ -238,7 +238,8 @@ public DataFormatAwareEngine(EngineConfig engineConfig) { config().getMapperService(), config().getIndexSettings(), config().getStore(), - registry + registry, + config().getChecksumStrategies() ), registry.format(config().getIndexSettings().pluggableDataFormat()) ); diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 6bf341852bfa1..78e319bfafc3b 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -62,6 +62,7 @@ import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.index.merge.MergedSegmentTransferTracker; import org.opensearch.index.seqno.RetentionLeases; +import org.opensearch.index.store.FormatChecksumStrategy; import org.opensearch.index.store.Store; import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.TranslogConfig; @@ -70,8 +71,10 @@ import org.opensearch.indices.IndexingMemoryController; import org.opensearch.threadpool.ThreadPool; +import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.BooleanSupplier; @@ -123,6 +126,7 @@ public final class EngineConfig { private final DataFormatRegistry dataFormatRegistry; private final MapperService mapperService; private final CommitterFactory committerFactory; + private final Map checksumStrategies; /** * A supplier of the outstanding retention leases. This is used during merged operations to determine which operations that have been @@ -316,6 +320,7 @@ private EngineConfig(Builder builder) { this.dataFormatRegistry = builder.dataFormatRegistry; this.mapperService = builder.mapperService; this.committerFactory = builder.committerFactory; + this.checksumStrategies = builder.checksumStrategies; } /** @@ -655,6 +660,10 @@ public CommitterFactory getCommitterFactory() { return this.committerFactory; } + public Map getChecksumStrategies() { + return this.checksumStrategies; + } + /** * Builder for EngineConfig class * @@ -696,6 +705,7 @@ public static class Builder { private DataFormatRegistry dataFormatRegistry; private MapperService mapperService; private CommitterFactory committerFactory; + private Map checksumStrategies = Collections.emptyMap(); public Builder shardId(ShardId shardId) { this.shardId = shardId; @@ -867,6 +877,11 @@ public Builder committerFactory(CommitterFactory committerFactory) { return this; } + public Builder checksumStrategies(Map checksumStrategies) { + this.checksumStrategies = checksumStrategies; + return this; + } + public EngineConfig build() { return new EngineConfig(this); } diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java index adbeee8ab29c6..b9d5be2ed5f2c 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java @@ -34,6 +34,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.merge.MergedSegmentTransferTracker; import org.opensearch.index.seqno.RetentionLeases; +import org.opensearch.index.store.FormatChecksumStrategy; import org.opensearch.index.store.Store; import org.opensearch.index.translog.TranslogConfig; import org.opensearch.index.translog.TranslogDeletionPolicyFactory; @@ -47,6 +48,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.BooleanSupplier; import java.util.function.LongSupplier; @@ -184,7 +186,8 @@ public EngineConfig newEngineConfig( ClusterApplierService clusterApplierService, MergedSegmentTransferTracker mergedSegmentTransferTracker, DataFormatRegistry dataFormatRegistry, - MapperService mapperService + MapperService mapperService, + Map checksumStrategies ) { CodecService codecServiceToUse = codecService; if (codecService == null && this.codecServiceFactory != null) { @@ -225,6 +228,7 @@ public EngineConfig newEngineConfig( .dataFormatRegistry(dataFormatRegistry) .mapperService(mapperService) .committerFactory(committerFactory) + .checksumStrategies(checksumStrategies) .build(); } diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatDescriptor.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatDescriptor.java index 0df1498a23b41..b88be06567401 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatDescriptor.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatDescriptor.java @@ -19,7 +19,7 @@ *

    The checksum strategy here is the default fallback — a full-file scan. * At runtime, the {@link IndexingExecutionEngine} may override this with a more * efficient strategy (e.g., {@link org.opensearch.index.store.PrecomputedChecksumStrategy}) - * via {@link org.opensearch.index.store.DataFormatAwareStoreDirectory#registerChecksumStrategy}. + * via the shared checksum strategies map created during shard initialization. * * @opensearch.experimental */ diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java index ac34836f97e67..00835289bf122 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java @@ -10,9 +10,9 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.IndexSettings; -import org.opensearch.index.store.FormatChecksumStrategy; import java.util.Map; +import java.util.function.Supplier; /** * Plugin interface for providing custom data format implementations. @@ -35,23 +35,23 @@ public interface DataFormatPlugin { * Creates the indexing engine for the data format. This should be instantiated per shard. * * @param settings the engine initialization settings - * @param checksumStrategy the checksum strategy owned by the directory for this format, - * or null if not available. Engines that pre-compute checksums - * during write should register into this instance so the upload - * path can retrieve them in O(1). * @return the indexing execution engine instance */ - IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings, FormatChecksumStrategy checksumStrategy); + IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings); /** - * Returns format descriptors for this plugin, filtered by the given index settings. - * Each entry maps a format name to its {@link DataFormatDescriptor} containing the - * default checksum strategy and format name. + * Returns format descriptor suppliers for this plugin, filtered by the given index settings. + * Each entry maps a format name to a {@link Supplier} of its {@link DataFormatDescriptor}, + * deferring descriptor object creation until the descriptor is actually needed. + * Callers that only need format names can use {@code keySet()} without triggering creation. * * @param indexSettings the index settings used to determine active formats - * @return map of format name to descriptor + * @return map of format name to descriptor supplier */ - default Map getFormatDescriptors(IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry) { + default Map> getFormatDescriptors( + IndexSettings indexSettings, + DataFormatRegistry dataFormatRegistry + ) { return Map.of(); } } diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java index 5a6254b0ce5ed..f61b47bc39b5c 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java @@ -19,11 +19,13 @@ import org.opensearch.plugins.SearchBackEndPlugin; import java.io.IOException; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -35,9 +37,6 @@ @ExperimentalApi public class DataFormatRegistry { - /** Index setting name that specifies the active pluggable data format. */ - public static final String PLUGGABLE_DATAFORMAT_SETTING = "pluggable_dataformat"; - /** Map from data format to the plugin that provides its indexing engine. */ private final Map dataFormatPluginRegistry; @@ -97,10 +96,7 @@ public DataFormatRegistry(PluginsService pluginsService) { if (plugin == null) { throw new IllegalArgumentException("No plugin registered for DataFormat [" + format.name() + "]"); } - Map descriptors = plugin.getFormatDescriptors(settings.indexSettings(), this); - DataFormatDescriptor descriptor = descriptors.get(format.name()); - FormatChecksumStrategy checksumStrategy = descriptor != null ? descriptor.getChecksumStrategy() : null; - return plugin.indexingEngine(settings, checksumStrategy); + return plugin.indexingEngine(settings); } public DataFormat format(String name) { @@ -140,16 +136,17 @@ public Set getRegisteredFormats() { } /** - * Returns format descriptors for the active data format of the given index. + * Returns format descriptor suppliers for the active data format of the given index. * Resolves the data format from index settings via the {@code pluggable_dataformat} setting, * then delegates to {@link DataFormatPlugin#getFormatDescriptors(IndexSettings, DataFormatRegistry)}. + * Callers that only need format names can use {@code keySet()} without triggering descriptor creation. * * @param indexSettings the index settings used to determine the active data format - * @return unmodifiable map of format name to descriptor, or empty map if no pluggable data format is configured + * @return map of format name to descriptor supplier, or empty map if no pluggable data format is configured */ - public Map getFormatDescriptors(IndexSettings indexSettings) { - String dataformatName = indexSettings.getSettings().get(PLUGGABLE_DATAFORMAT_SETTING); - if (dataformatName != null) { + public Map> getFormatDescriptors(IndexSettings indexSettings) { + String dataformatName = indexSettings.pluggableDataFormat(); + if (dataformatName != null && dataformatName.isEmpty() == false) { DataFormat format = dataFormats.get(dataformatName); if (format != null) { DataFormatPlugin plugin = dataFormatPluginRegistry.get(format); @@ -161,6 +158,44 @@ public Map getFormatDescriptors(IndexSettings inde return Map.of(); } + /** + * Returns format descriptor suppliers for a specific data format, bypassing the + * {@code pluggable_dataformat} index setting lookup. This is used by composite + * plugins to resolve child format descriptors without recursion. + * + * @param indexSettings the index settings + * @param dataFormat the specific data format to get descriptors for + * @return map of format name to descriptor supplier, or empty map if the format is not registered + */ + public Map> getFormatDescriptors(IndexSettings indexSettings, DataFormat dataFormat) { + DataFormatPlugin plugin = dataFormatPluginRegistry.get(dataFormat); + if (plugin == null) { + return Map.of(); + } + return plugin.getFormatDescriptors(indexSettings, this); + } + + /** + * Creates checksum strategies for all formats of the given index, intended to be called + * once per shard during initialization. The returned map should be shared between the + * directory and the engine so that pre-computed checksums registered during write are + * visible to the upload path. + * + * @param indexSettings the index settings used to determine the active data format + * @return unmodifiable map of format name to checksum strategy + */ + public Map createChecksumStrategies(IndexSettings indexSettings) { + Map> descriptors = getFormatDescriptors(indexSettings); + Map strategies = new HashMap<>(); + for (Map.Entry> entry : descriptors.entrySet()) { + FormatChecksumStrategy strategy = entry.getValue().get().getChecksumStrategy(); + if (strategy != null) { + strategies.put(entry.getKey(), strategy); + } + } + return Collections.unmodifiableMap(strategies); + } + /** * Creates {@link EngineReaderManager} instances for all applicable data formats based on index settings/mappings. * Each reader manager is instantiated by applying the store provider and shard path to the factory registered diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingEngineConfig.java b/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingEngineConfig.java index 0e417d9b5c3e7..e5cb8e58fe0e1 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingEngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingEngineConfig.java @@ -12,8 +12,11 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.exec.commit.Committer; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.store.FormatChecksumStrategy; import org.opensearch.index.store.Store; +import java.util.Map; + /** * Initialization parameters for creating an {@link IndexingExecutionEngine} via * {@link DataFormatPlugin#indexingEngine}. Bundling parameters in a record avoids @@ -29,5 +32,5 @@ */ @ExperimentalApi public record IndexingEngineConfig(Committer committer, MapperService mapperService, IndexSettings indexSettings, Store store, - DataFormatRegistry registry) { + DataFormatRegistry registry, Map checksumStrategies) { } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 042cdb0aba013..74b4b9f6d18be 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -184,6 +184,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask; import org.opensearch.index.similarity.SimilarityService; +import org.opensearch.index.store.FormatChecksumStrategy; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.RemoteSegmentStoreDirectory.UploadedSegmentMetadata; import org.opensearch.index.store.RemoteStoreFileDownloader; @@ -416,6 +417,8 @@ Runnable getGlobalCheckpointSyncer() { private final DataFormatRegistry dataFormatRegistry; + private final Map checksumStrategies; + @InternalApi public IndexShard( final ShardRouting shardRouting, @@ -456,6 +459,7 @@ public IndexShard( final ClusterApplierService clusterApplierService, @Nullable final MergedSegmentPublisher mergedSegmentPublisher, @Nullable final ReferencedSegmentsPublisher referencedSegmentsPublisher, + final Map checksumStrategies, @Nullable final DataFormatRegistry dataFormatRegistry ) throws IOException { super(shardRouting.shardId(), indexSettings); @@ -611,6 +615,7 @@ public boolean shouldCache(Query query) { } } this.dataFormatRegistry = dataFormatRegistry; + this.checksumStrategies = checksumStrategies; } /** @@ -634,6 +639,10 @@ public Store store() { return this.store; } + public Map getChecksumStrategies() { + return checksumStrategies; + } + public boolean isMigratingToRemote() { // set it true only if shard is remote, but index setting doesn't say so return shardMigrationState == REMOTE_MIGRATING_UNSEEDED || shardMigrationState == REMOTE_MIGRATING_SEEDED; @@ -4523,7 +4532,8 @@ public void afterRefresh(boolean didRefresh) { clusterApplierService, mergedSegmentTransferTracker, dataFormatRegistry, - mapperService + mapperService, + checksumStrategies ); } diff --git a/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectory.java index 24065799c537e..6413e84933b35 100644 --- a/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectory.java @@ -16,9 +16,6 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.engine.dataformat.DataFormatDescriptor; -import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.checksum.GenericCRC32ChecksumHandler; import org.opensearch.index.store.checksum.LuceneChecksumHandler; @@ -81,32 +78,22 @@ public class DataFormatAwareStoreDirectory extends FilterDirectory { private static final FormatChecksumStrategy DEFAULT_CHECKSUM_STRATEGY = new GenericCRC32ChecksumHandler(); /** - * Constructs a DataFormatAwareStoreDirectory with a {@link DataFormatRegistry} for format-aware - * checksum calculation and other format-specific operations. + * Constructs a DataFormatAwareStoreDirectory with pre-built checksum strategies for + * format-aware checksum calculation and other format-specific operations. * * @param delegate the underlying FSDirectory (typically for <shard>/index/) * @param shardPath the shard path for resolving subdirectories - * @param dataFormatRegistry registry providing format-specific checksum handlers + * @param checksumStrategies pre-built checksum strategies keyed by format name */ - public DataFormatAwareStoreDirectory( - IndexSettings indexSettings, - Directory delegate, - ShardPath shardPath, - DataFormatRegistry dataFormatRegistry - ) { + public DataFormatAwareStoreDirectory(Directory delegate, ShardPath shardPath, Map checksumStrategies) { super(new SubdirectoryAwareDirectory(delegate, shardPath)); this.shardPath = shardPath; - Map descriptors = dataFormatRegistry.getFormatDescriptors(indexSettings); - this.checksumStrategies = new HashMap<>(); - for (Map.Entry entry : descriptors.entrySet()) { - this.checksumStrategies.put(entry.getKey(), entry.getValue().getChecksumStrategy()); - } + this.checksumStrategies = new HashMap<>(checksumStrategies); this.checksumStrategies.put(DEFAULT_FORMAT, new LuceneChecksumHandler()); - logger.debug( "Created DataFormatAwareStoreDirectory for shard {} with checksum strategies for formats: {}", shardPath.getShardId(), - checksumStrategies.keySet() + this.checksumStrategies.keySet() ); } @@ -246,24 +233,6 @@ public String calculateUploadChecksum(String name) throws IOException { return Long.toString(calculateChecksum(name)); } - /** - * Registers a {@link FormatChecksumStrategy} for a data format. - * Overrides any existing strategy - * - *

    Use this to register strategies that support pre-computed checksums (e.g., - * {@link PrecomputedChecksumStrategy} for Parquet files whose CRC32 is computed - * during write by the Rust writer). - * - * @param format the data format name (e.g., "parquet") - * @param strategy the checksum strategy to use for this format - */ - public void registerChecksumStrategy(String format, FormatChecksumStrategy strategy) { - if (format != null && strategy != null) { - checksumStrategies.put(format, strategy); - logger.debug("Registered FormatChecksumStrategy for format [{}]", format); - } - } - /** * Returns the checksum strategy for the given format, or {@code null} if none is registered. * Engines use this to share the directory's strategy instance so that pre-computed diff --git a/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryFactory.java index b633a00ca67eb..9c519e8001d93 100644 --- a/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryFactory.java @@ -11,11 +11,11 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; -import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.shard.ShardPath; import org.opensearch.plugins.IndexStorePlugin; import java.io.IOException; +import java.util.Map; /** * Factory interface for creating DataFormatAwareStoreDirectory instances. @@ -46,7 +46,7 @@ public interface DataFormatAwareStoreDirectoryFactory { * @param shardId the shard identifier * @param shardPath the path the shard is using for file storage * @param localDirectoryFactory the factory for creating the underlying local directory, respecting index store type configuration - * @param dataFormatRegistry registry of available data format plugins + * @param checksumStrategies pre-built checksum strategies keyed by format name * @return a new DataFormatAwareStoreDirectory instance supporting all discovered formats * @throws IOException if directory creation fails or resources cannot be allocated */ @@ -55,6 +55,6 @@ DataFormatAwareStoreDirectory newDataFormatAwareStoreDirectory( ShardId shardId, ShardPath shardPath, IndexStorePlugin.DirectoryFactory localDirectoryFactory, - DataFormatRegistry dataFormatRegistry + Map checksumStrategies ) throws IOException; } diff --git a/server/src/main/java/org/opensearch/index/store/DefaultDataFormatAwareStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/DefaultDataFormatAwareStoreDirectoryFactory.java index 8e32942f5676d..8a53dfe696835 100644 --- a/server/src/main/java/org/opensearch/index/store/DefaultDataFormatAwareStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/DefaultDataFormatAwareStoreDirectoryFactory.java @@ -14,12 +14,12 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; -import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.shard.ShardPath; import org.opensearch.plugins.IndexStorePlugin; import java.io.IOException; import java.util.Locale; +import java.util.Map; /** * Default implementation of DataFormatAwareStoreDirectoryFactory that provides @@ -42,7 +42,7 @@ public class DefaultDataFormatAwareStoreDirectoryFactory implements DataFormatAw * @param shardId the shard identifier * @param shardPath the path the shard is using * @param localDirectoryFactory the factory for creating the underlying local directory - * @param dataFormatRegistry registry of available data format plugins + * @param checksumStrategies pre-built checksum strategies keyed by format name * @return a new DataFormatAwareStoreDirectory instance * @throws IOException if directory creation fails */ @@ -52,7 +52,7 @@ public DataFormatAwareStoreDirectory newDataFormatAwareStoreDirectory( ShardId shardId, ShardPath shardPath, IndexStorePlugin.DirectoryFactory localDirectoryFactory, - DataFormatRegistry dataFormatRegistry + Map checksumStrategies ) throws IOException { if (logger.isDebugEnabled()) { @@ -67,18 +67,13 @@ public DataFormatAwareStoreDirectory newDataFormatAwareStoreDirectory( // Delegate local directory creation to the configured DirectoryFactory Directory delegate = localDirectoryFactory.newDirectory(indexSettings, shardPath); - DataFormatAwareStoreDirectory directory = new DataFormatAwareStoreDirectory( - indexSettings, - delegate, - shardPath, - dataFormatRegistry - ); + DataFormatAwareStoreDirectory directory = new DataFormatAwareStoreDirectory(delegate, shardPath, checksumStrategies); if (logger.isDebugEnabled()) { logger.debug( "Successfully created DataFormatAwareStoreDirectory for shard: {} with registered formats: {}", shardPath.getShardId(), - dataFormatRegistry.getRegisteredFormats() + checksumStrategies.keySet() ); } diff --git a/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java b/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java index 3ec29f1c30841..28d7eccf9e64d 100644 --- a/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java @@ -84,6 +84,7 @@ public void testCreateEngineConfigFromFactory() { null, null, null, + null, null ); @@ -197,6 +198,7 @@ public void testCreateCodecServiceFromFactory() { null, null, null, + null, null ); assertNotNull(config.getCodec()); diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java index 18645af9421ac..69b2b220ec85d 100644 --- a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java @@ -87,9 +87,9 @@ public void testFullDataFormatLifecycle() throws IOException { mock(MapperService.class), new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings), null, - null - ), - null + null, + Map.of() + ) ); assertEquals(format, engine.getDataFormat()); diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java index 94ca8d727c56a..e9a144f78534e 100644 --- a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -146,7 +147,7 @@ public void testGetIndexingEngine() { DataFormatRegistry registry = new DataFormatRegistry(pluginsService); IndexingExecutionEngine engine = registry.getIndexingEngine( - new IndexingEngineConfig(null, mapperService, indexSettings, null, null), + new IndexingEngineConfig(null, mapperService, indexSettings, null, null, Map.of()), format ); assertNotNull(engine); @@ -162,7 +163,10 @@ public void testGetIndexingEngineForUnregisteredFormatThrows() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> registry.getIndexingEngine(new IndexingEngineConfig(null, mapperService, indexSettings, null, null), unregistered) + () -> registry.getIndexingEngine( + new IndexingEngineConfig(null, mapperService, indexSettings, null, null, Map.of()), + unregistered + ) ); assertTrue(e.getMessage().contains("unknown")); } @@ -286,4 +290,29 @@ public void testGetRegisteredFormatsIsUnmodifiable() { expectThrows(UnsupportedOperationException.class, () -> formats.add(new MockDataFormat("new", 1L, Set.of()))); } + + public void testGetFormatDescriptorsByDataFormatReturnsDescriptors() { + MockDataFormat format = new MockDataFormat("columnar", 100L, Set.of()); + MockDataFormatPlugin plugin = MockDataFormatPlugin.of(format); + MockSearchBackEndPlugin backEnd = new MockSearchBackEndPlugin(List.of("columnar")); + + when(pluginsService.filterPlugins(DataFormatPlugin.class)).thenReturn(List.of(plugin)); + when(pluginsService.filterPlugins(SearchBackEndPlugin.class)).thenReturn(List.of(backEnd)); + + DataFormatRegistry registry = new DataFormatRegistry(pluginsService); + + Map> descriptors = registry.getFormatDescriptors(indexSettings, format); + assertNotNull(descriptors); + } + + public void testGetFormatDescriptorsByDataFormatReturnsEmptyForUnregisteredFormat() { + when(pluginsService.filterPlugins(DataFormatPlugin.class)).thenReturn(List.of()); + when(pluginsService.filterPlugins(SearchBackEndPlugin.class)).thenReturn(List.of()); + + DataFormatRegistry registry = new DataFormatRegistry(pluginsService); + MockDataFormat unregistered = new MockDataFormat("unknown", 1L, Set.of()); + + Map> descriptors = registry.getFormatDescriptors(indexSettings, unregistered); + assertTrue(descriptors.isEmpty()); + } } diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/FormatChecksumStrategySharingTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/FormatChecksumStrategySharingTests.java new file mode 100644 index 0000000000000..4377cbaefa439 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/FormatChecksumStrategySharingTests.java @@ -0,0 +1,224 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat; + +import org.apache.lucene.store.FSDirectory; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.stub.MockDataFormat; +import org.opensearch.index.engine.dataformat.stub.MockSearchBackEndPlugin; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.DataFormatAwareStoreDirectory; +import org.opensearch.index.store.FormatChecksumStrategy; +import org.opensearch.index.store.PrecomputedChecksumStrategy; +import org.opensearch.plugins.PluginsService; +import org.opensearch.plugins.SearchBackEndPlugin; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Tests that validate the FormatChecksumStrategy single-instance fix: + * strategies are created once per shard and shared between directory and engine. + */ +public class FormatChecksumStrategySharingTests extends OpenSearchTestCase { + + private static final String FORMAT_NAME = "test_format"; + + /** + * A DataFormatPlugin that returns a new PrecomputedChecksumStrategy on every + * getFormatDescriptors() call — reproducing the original bug pattern. + */ + private static class StrategyCreatingPlugin extends org.opensearch.plugins.Plugin implements DataFormatPlugin { + private final MockDataFormat format; + + StrategyCreatingPlugin(MockDataFormat format) { + this.format = format; + } + + @Override + public DataFormat getDataFormat() { + return format; + } + + @Override + public IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings) { + return null; + } + + @Override + public Map> getFormatDescriptors(IndexSettings indexSettings, DataFormatRegistry registry) { + // Creates a NEW PrecomputedChecksumStrategy every call — this is the bug pattern + return Map.of(FORMAT_NAME, () -> new DataFormatDescriptor(FORMAT_NAME, new PrecomputedChecksumStrategy())); + } + } + + private DataFormatRegistry createRegistry(MockDataFormat format) { + StrategyCreatingPlugin plugin = new StrategyCreatingPlugin(format); + MockSearchBackEndPlugin backEnd = new MockSearchBackEndPlugin(List.of(format.name())); + PluginsService pluginsService = mock(PluginsService.class); + when(pluginsService.filterPlugins(DataFormatPlugin.class)).thenReturn(List.of(plugin)); + when(pluginsService.filterPlugins(SearchBackEndPlugin.class)).thenReturn(List.of(backEnd)); + return new DataFormatRegistry(pluginsService); + } + + private IndexSettings createIndexSettings(String indexName) { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT) + .put(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), FORMAT_NAME) + .build(); + return new IndexSettings(IndexMetadata.builder(indexName).settings(settings).build(), settings); + } + + /** + * Verifies that createChecksumStrategies() returns the same strategy instance + * that both the directory and engine would share. + */ + public void testCreateChecksumStrategiesReturnsSameInstance() { + MockDataFormat format = new MockDataFormat(FORMAT_NAME, 100L, Set.of()); + DataFormatRegistry registry = createRegistry(format); + IndexSettings indexSettings = createIndexSettings("test_index"); + + Map strategies = registry.createChecksumStrategies(indexSettings); + + assertNotNull(strategies.get(FORMAT_NAME)); + assertTrue(strategies.get(FORMAT_NAME) instanceof PrecomputedChecksumStrategy); + } + + /** + * Verifies that calling createChecksumStrategies() twice returns DIFFERENT + * instances (since getFormatDescriptors creates new ones each call). + * This confirms the fix must call it only once per shard. + */ + public void testMultipleCallsCreateDifferentInstances() { + MockDataFormat format = new MockDataFormat(FORMAT_NAME, 100L, Set.of()); + DataFormatRegistry registry = createRegistry(format); + IndexSettings indexSettings = createIndexSettings("test_index"); + + Map first = registry.createChecksumStrategies(indexSettings); + Map second = registry.createChecksumStrategies(indexSettings); + + // Different calls produce different instances — this is WHY we must call it only once + assertNotSame(first.get(FORMAT_NAME), second.get(FORMAT_NAME)); + } + + /** + * Core test: checksum registered via the engine's strategy reference is visible + * from the directory's strategy reference when they share the same instance. + * This is the exact bug scenario that was broken before the fix. + */ + public void testChecksumVisibleAcrossSharedStrategy() throws IOException { + MockDataFormat format = new MockDataFormat(FORMAT_NAME, 100L, Set.of()); + DataFormatRegistry registry = createRegistry(format); + IndexSettings indexSettings = createIndexSettings("test_index"); + + // Single call — same map shared by directory and engine + Map strategies = registry.createChecksumStrategies(indexSettings); + FormatChecksumStrategy sharedStrategy = strategies.get(FORMAT_NAME); + + long expectedChecksum = 3847291056L; + // Simulate engine registering a checksum during write + sharedStrategy.registerChecksum("_0_1.parquet", expectedChecksum, 1L); + + // Simulate directory reading the checksum during upload + Path tempDir = createTempDir(); + Path shardDataPath = tempDir.resolve("uuid").resolve("0"); + Files.createDirectories(shardDataPath.resolve(ShardPath.INDEX_FOLDER_NAME)); + ShardPath shardPath = new ShardPath(false, shardDataPath, shardDataPath, new ShardId("index", "uuid", 0)); + FSDirectory fsDir = FSDirectory.open(shardDataPath.resolve(ShardPath.INDEX_FOLDER_NAME)); + + DataFormatAwareStoreDirectory directory = new DataFormatAwareStoreDirectory(fsDir, shardPath, strategies); + + // The directory's strategy IS the same instance + FormatChecksumStrategy directoryStrategy = directory.getChecksumStrategy(FORMAT_NAME); + assertSame("Directory and engine must share the same strategy instance", sharedStrategy, directoryStrategy); + + // Verify the checksum registered by the engine is readable from the directory's strategy (O(1) lookup) + long actualChecksum = directoryStrategy.computeChecksum(fsDir, "_0_1.parquet"); + assertEquals("Checksum registered by engine must be visible via directory strategy", expectedChecksum, actualChecksum); + + directory.close(); + } + + /** + * Verifies that concurrent shard creation for different indices produces + * isolated strategy instances — no cross-index contamination. + */ + public void testDifferentIndicesGetIsolatedStrategies() { + MockDataFormat format = new MockDataFormat(FORMAT_NAME, 100L, Set.of()); + DataFormatRegistry registry = createRegistry(format); + + IndexSettings indexSettingsA = createIndexSettings("index_a"); + IndexSettings indexSettingsB = createIndexSettings("index_b"); + + Map strategiesA = registry.createChecksumStrategies(indexSettingsA); + Map strategiesB = registry.createChecksumStrategies(indexSettingsB); + + // Different indices get different strategy instances + assertNotSame(strategiesA.get(FORMAT_NAME), strategiesB.get(FORMAT_NAME)); + + // Register checksum in index A's strategy + strategiesA.get(FORMAT_NAME).registerChecksum("_0.parquet", 12345L, 1L); + + // Index B's strategy should NOT see it + PrecomputedChecksumStrategy stratB = (PrecomputedChecksumStrategy) strategiesB.get(FORMAT_NAME); + // computeChecksum would fall back to file scan if not cached — but we can verify + // the cache is empty by checking that a different checksum isn't magically present + PrecomputedChecksumStrategy stratA = (PrecomputedChecksumStrategy) strategiesA.get(FORMAT_NAME); + assertNotSame(stratA, stratB); + } + + /** + * Verifies that the strategies map returned by createChecksumStrategies is unmodifiable. + */ + public void testCreateChecksumStrategiesReturnsUnmodifiableMap() { + MockDataFormat format = new MockDataFormat(FORMAT_NAME, 100L, Set.of()); + DataFormatRegistry registry = createRegistry(format); + IndexSettings indexSettings = createIndexSettings("test_index"); + + Map strategies = registry.createChecksumStrategies(indexSettings); + + expectThrows(UnsupportedOperationException.class, () -> strategies.put("new_format", new PrecomputedChecksumStrategy())); + } + + /** + * Verifies that createChecksumStrategies returns empty map when no pluggable + * data format is configured. + */ + public void testCreateChecksumStrategiesEmptyWhenNoFormat() { + MockDataFormat format = new MockDataFormat(FORMAT_NAME, 100L, Set.of()); + DataFormatRegistry registry = createRegistry(format); + + // Index settings WITHOUT pluggable_dataformat setting + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("plain_index").settings(settings).build(), settings); + + Map strategies = registry.createChecksumStrategies(indexSettings); + + assertTrue(strategies.isEmpty()); + } +} diff --git a/server/src/test/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryTests.java index ba795396451b5..c8566ecbb7715 100644 --- a/server/src/test/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryTests.java @@ -13,17 +13,9 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; -import org.opensearch.Version; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.engine.dataformat.DataFormatPlugin; -import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.shard.ShardPath; -import org.opensearch.plugins.PluginsService; -import org.opensearch.plugins.SearchBackEndPlugin; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; @@ -34,12 +26,10 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.zip.CRC32; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class DataFormatAwareStoreDirectoryTests extends OpenSearchTestCase { private Path tempDir; @@ -64,20 +54,7 @@ public void setUp() throws Exception { ShardId sid = new ShardId(new Index("test-index", indexUUID), shardId); shardPath = new ShardPath(false, shardDataPath, shardDataPath, sid); - PluginsService pluginsService = mock(PluginsService.class); - when(pluginsService.filterPlugins(DataFormatPlugin.class)).thenReturn(List.of()); - when(pluginsService.filterPlugins(SearchBackEndPlugin.class)).thenReturn(List.of()); - DataFormatRegistry dataFormatRegistry = new DataFormatRegistry(pluginsService); - - // Create real IndexSettings (IndexSettings is final, cannot be mocked) - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_INDEX_UUID, indexUUID) - .build(); - IndexMetadata metadata = IndexMetadata.builder("test-index").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); - IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); - - dataFormatAwareStoreDirectory = new DataFormatAwareStoreDirectory(indexSettings, fsDirectory, shardPath, dataFormatRegistry); + dataFormatAwareStoreDirectory = new DataFormatAwareStoreDirectory(fsDirectory, shardPath, Map.of()); } @After diff --git a/server/src/test/java/org/opensearch/index/store/DefaultDataFormatAwareStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/DefaultDataFormatAwareStoreDirectoryFactoryTests.java index d1a47e9710661..b76ac1f3b3511 100644 --- a/server/src/test/java/org/opensearch/index/store/DefaultDataFormatAwareStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/DefaultDataFormatAwareStoreDirectoryFactoryTests.java @@ -14,22 +14,16 @@ import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; -import org.opensearch.index.engine.dataformat.DataFormatPlugin; -import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.shard.ShardPath; import org.opensearch.plugins.IndexStorePlugin; -import org.opensearch.plugins.PluginsService; -import org.opensearch.plugins.SearchBackEndPlugin; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; +import java.util.Map; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Unit tests for {@link DefaultDataFormatAwareStoreDirectoryFactory}. @@ -56,13 +50,6 @@ private IndexSettings createIndexSettings() { return new IndexSettings(metadata, Settings.EMPTY); } - private DataFormatRegistry createEmptyDataFormatRegistry() { - PluginsService pluginsService = mock(PluginsService.class); - when(pluginsService.filterPlugins(DataFormatPlugin.class)).thenReturn(List.of()); - when(pluginsService.filterPlugins(SearchBackEndPlugin.class)).thenReturn(List.of()); - return new DataFormatRegistry(pluginsService); - } - private IndexStorePlugin.DirectoryFactory createFsDirectoryFactory() { return new IndexStorePlugin.DirectoryFactory() { @Override @@ -86,7 +73,6 @@ public org.apache.lucene.store.Directory newFSDirectory( // ═══════════════════════════════════════════════════════════════ public void testNewDataFormatAwareStoreDirectory_CreatesSuccessfully() throws IOException { - DataFormatRegistry registry = createEmptyDataFormatRegistry(); DefaultDataFormatAwareStoreDirectoryFactory factory = new DefaultDataFormatAwareStoreDirectoryFactory(); Path tempDir = createTempDir(); ShardPath shardPath = createShardPath(tempDir); @@ -97,14 +83,13 @@ public void testNewDataFormatAwareStoreDirectory_CreatesSuccessfully() throws IO shardPath.getShardId(), shardPath, createFsDirectoryFactory(), - registry + Map.of() ); assertNotNull("Factory should create a non-null DataFormatAwareStoreDirectory", directory); } public void testNewDataFormatAwareStoreDirectory_HasCorrectShardPath() throws IOException { - DataFormatRegistry registry = createEmptyDataFormatRegistry(); DefaultDataFormatAwareStoreDirectoryFactory factory = new DefaultDataFormatAwareStoreDirectoryFactory(); Path tempDir = createTempDir(); ShardPath shardPath = createShardPath(tempDir); @@ -115,14 +100,13 @@ public void testNewDataFormatAwareStoreDirectory_HasCorrectShardPath() throws IO shardPath.getShardId(), shardPath, createFsDirectoryFactory(), - registry + Map.of() ); assertEquals(shardPath, directory.getShardPath()); } public void testNewDataFormatAwareStoreDirectory_CanListFiles() throws IOException { - DataFormatRegistry registry = createEmptyDataFormatRegistry(); DefaultDataFormatAwareStoreDirectoryFactory factory = new DefaultDataFormatAwareStoreDirectoryFactory(); Path tempDir = createTempDir(); ShardPath shardPath = createShardPath(tempDir); @@ -133,7 +117,7 @@ public void testNewDataFormatAwareStoreDirectory_CanListFiles() throws IOExcepti shardPath.getShardId(), shardPath, createFsDirectoryFactory(), - registry + Map.of() ); // Should not throw @@ -142,7 +126,6 @@ public void testNewDataFormatAwareStoreDirectory_CanListFiles() throws IOExcepti } public void testNewDataFormatAwareStoreDirectory_MultipleCalls_CreatesSeparateInstances() throws IOException { - DataFormatRegistry registry = createEmptyDataFormatRegistry(); DefaultDataFormatAwareStoreDirectoryFactory factory = new DefaultDataFormatAwareStoreDirectoryFactory(); Path tempDir1 = createTempDir(); Path tempDir2 = createTempDir(); @@ -155,14 +138,14 @@ public void testNewDataFormatAwareStoreDirectory_MultipleCalls_CreatesSeparateIn shardPath1.getShardId(), shardPath1, createFsDirectoryFactory(), - registry + Map.of() ); DataFormatAwareStoreDirectory dir2 = factory.newDataFormatAwareStoreDirectory( indexSettings, shardPath2.getShardId(), shardPath2, createFsDirectoryFactory(), - registry + Map.of() ); assertNotNull(dir1); @@ -171,7 +154,6 @@ public void testNewDataFormatAwareStoreDirectory_MultipleCalls_CreatesSeparateIn } public void testNewDataFormatAwareStoreDirectory_InvalidPath_ThrowsIOException() throws IOException { - DataFormatRegistry registry = createEmptyDataFormatRegistry(); DefaultDataFormatAwareStoreDirectoryFactory factory = new DefaultDataFormatAwareStoreDirectoryFactory(); IndexSettings indexSettings = createIndexSettings(); @@ -197,7 +179,7 @@ public void testNewDataFormatAwareStoreDirectory_InvalidPath_ThrowsIOException() invalidShardPath.getShardId(), invalidShardPath, createFsDirectoryFactory(), - registry + Map.of() ) ); assertTrue( diff --git a/server/src/test/java/org/opensearch/index/store/remote/DataFormatAwareRemoteDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/remote/DataFormatAwareRemoteDirectoryTests.java index 62a571aab9a41..ec1a12db8504c 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/DataFormatAwareRemoteDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/DataFormatAwareRemoteDirectoryTests.java @@ -50,6 +50,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import org.mockito.Mockito; @@ -107,7 +108,7 @@ public void setUp() throws Exception { .build(); IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); when(mockRegistry.getFormatDescriptors(any(IndexSettings.class))).thenReturn( - Map.of("parquet", new DataFormatDescriptor("parquet", new GenericCRC32ChecksumHandler())) + Map.of("parquet", (Supplier) () -> new DataFormatDescriptor("parquet", new GenericCRC32ChecksumHandler())) ); directory = new DataFormatAwareRemoteDirectory( diff --git a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockDataFormatPlugin.java b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockDataFormatPlugin.java index 021fef15f9969..82d5a7d929682 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockDataFormatPlugin.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockDataFormatPlugin.java @@ -12,7 +12,6 @@ import org.opensearch.index.engine.dataformat.DataFormatPlugin; import org.opensearch.index.engine.dataformat.IndexingEngineConfig; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; -import org.opensearch.index.store.FormatChecksumStrategy; import org.opensearch.plugins.Plugin; import java.util.Set; @@ -41,7 +40,7 @@ public DataFormat getDataFormat() { } @Override - public IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings, FormatChecksumStrategy checksumStrategy) { + public IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings) { return new MockIndexingExecutionEngine(dataFormat); } } diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index b5408b3709e70..68e14e34624b8 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -807,6 +807,7 @@ protected IndexShard newShard( clusterService.getClusterApplierService(), mergedSegmentPublisher, ReferencedSegmentsPublisher.EMPTY, + Collections.emptyMap(), null // TODO ); indexShard.addShardFailureCallback(DEFAULT_SHARD_FAILURE_HANDLER); From f892dbd04583ae587fe3c6747c31251406bf86da Mon Sep 17 00:00:00 2001 From: Ralph Ursprung <39383228+rursprung@users.noreply.github.com> Date: Thu, 30 Apr 2026 16:27:32 +0200 Subject: [PATCH 023/115] update log4j2 to 2.25.4 (#21416) update log4j2 to 2.25.4 Signed-off-by: Ralph Ursprung --- gradle/libs.versions.toml | 2 +- libs/core/licenses/log4j-api-2.25.3.jar.sha1 | 1 - libs/core/licenses/log4j-api-2.25.4.jar.sha1 | 1 + plugins/crypto-kms/licenses/log4j-1.2-api-2.25.3.jar.sha1 | 1 - plugins/crypto-kms/licenses/log4j-1.2-api-2.25.4.jar.sha1 | 1 + .../licenses/log4j-1.2-api-2.25.3.jar.sha1 | 1 - .../licenses/log4j-1.2-api-2.25.4.jar.sha1 | 1 + plugins/discovery-ec2/licenses/log4j-1.2-api-2.25.3.jar.sha1 | 1 - plugins/discovery-ec2/licenses/log4j-1.2-api-2.25.4.jar.sha1 | 1 + plugins/discovery-gce/licenses/log4j-1.2-api-2.25.3.jar.sha1 | 1 - plugins/discovery-gce/licenses/log4j-1.2-api-2.25.4.jar.sha1 | 1 + .../licenses/log4j-slf4j2-impl-2.25.3.jar.sha1 | 1 - .../licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 | 1 + plugins/repository-s3/licenses/log4j-1.2-api-2.25.3.jar.sha1 | 1 - plugins/repository-s3/licenses/log4j-1.2-api-2.25.4.jar.sha1 | 1 + .../repository-s3/licenses/log4j-slf4j2-impl-2.25.3.jar.sha1 | 1 - .../repository-s3/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 | 1 + .../libs/dataformat-native/licenses/log4j-api-2.25.3.jar.sha1 | 1 - .../libs/dataformat-native/licenses/log4j-api-2.25.4.jar.sha1 | 1 + sandbox/plugins/analytics-backend-datafusion/build.gradle | 2 ++ sandbox/plugins/analytics-engine/build.gradle | 2 ++ server/build.gradle | 4 ++++ server/licenses/log4j-api-2.25.3.jar.sha1 | 1 - server/licenses/log4j-api-2.25.4.jar.sha1 | 1 + server/licenses/log4j-core-2.25.3.jar.sha1 | 1 - server/licenses/log4j-core-2.25.4.jar.sha1 | 1 + server/licenses/log4j-jul-2.25.3.jar.sha1 | 1 - server/licenses/log4j-jul-2.25.4.jar.sha1 | 1 + 28 files changed, 21 insertions(+), 13 deletions(-) delete mode 100644 libs/core/licenses/log4j-api-2.25.3.jar.sha1 create mode 100644 libs/core/licenses/log4j-api-2.25.4.jar.sha1 delete mode 100644 plugins/crypto-kms/licenses/log4j-1.2-api-2.25.3.jar.sha1 create mode 100644 plugins/crypto-kms/licenses/log4j-1.2-api-2.25.4.jar.sha1 delete mode 100644 plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.25.3.jar.sha1 create mode 100644 plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.25.4.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/log4j-1.2-api-2.25.3.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/log4j-1.2-api-2.25.4.jar.sha1 delete mode 100644 plugins/discovery-gce/licenses/log4j-1.2-api-2.25.3.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/log4j-1.2-api-2.25.4.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/log4j-slf4j2-impl-2.25.3.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/log4j-1.2-api-2.25.3.jar.sha1 create mode 100644 plugins/repository-s3/licenses/log4j-1.2-api-2.25.4.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/log4j-slf4j2-impl-2.25.3.jar.sha1 create mode 100644 plugins/repository-s3/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 delete mode 100644 sandbox/libs/dataformat-native/licenses/log4j-api-2.25.3.jar.sha1 create mode 100644 sandbox/libs/dataformat-native/licenses/log4j-api-2.25.4.jar.sha1 delete mode 100644 server/licenses/log4j-api-2.25.3.jar.sha1 create mode 100644 server/licenses/log4j-api-2.25.4.jar.sha1 delete mode 100644 server/licenses/log4j-core-2.25.3.jar.sha1 create mode 100644 server/licenses/log4j-core-2.25.4.jar.sha1 delete mode 100644 server/licenses/log4j-jul-2.25.3.jar.sha1 create mode 100644 server/licenses/log4j-jul-2.25.4.jar.sha1 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1acb20727ca15..cfd2c350e7dda 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,7 +17,7 @@ snakeyaml = "2.6" snakeyaml_engine = "3.0.1" icu4j = "77.1" supercsv = "2.4.0" -log4j = "2.25.3" +log4j = "2.25.4" error_prone_annotations = "2.45.0" slf4j = "2.0.17" asm = "9.9.1" diff --git a/libs/core/licenses/log4j-api-2.25.3.jar.sha1 b/libs/core/licenses/log4j-api-2.25.3.jar.sha1 deleted file mode 100644 index 97dc53d973766..0000000000000 --- a/libs/core/licenses/log4j-api-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fb385330d89c2d61058ef649403f214633569205 \ No newline at end of file diff --git a/libs/core/licenses/log4j-api-2.25.4.jar.sha1 b/libs/core/licenses/log4j-api-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..2f492821ebca6 --- /dev/null +++ b/libs/core/licenses/log4j-api-2.25.4.jar.sha1 @@ -0,0 +1 @@ +89ff2217b193fb187b134aa6ebcbfa8a28b018a9 \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/log4j-1.2-api-2.25.3.jar.sha1 b/plugins/crypto-kms/licenses/log4j-1.2-api-2.25.3.jar.sha1 deleted file mode 100644 index ffa0736153da7..0000000000000 --- a/plugins/crypto-kms/licenses/log4j-1.2-api-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a7e550e638a5e534fd944616c5ae665a67e9501e \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/log4j-1.2-api-2.25.4.jar.sha1 b/plugins/crypto-kms/licenses/log4j-1.2-api-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..cf65c0331d0bd --- /dev/null +++ b/plugins/crypto-kms/licenses/log4j-1.2-api-2.25.4.jar.sha1 @@ -0,0 +1 @@ +351888743c1d0f7c9ec97a909ff2f7901f77df63 \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.25.3.jar.sha1 b/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.25.3.jar.sha1 deleted file mode 100644 index ffa0736153da7..0000000000000 --- a/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a7e550e638a5e534fd944616c5ae665a67e9501e \ No newline at end of file diff --git a/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.25.4.jar.sha1 b/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..cf65c0331d0bd --- /dev/null +++ b/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.25.4.jar.sha1 @@ -0,0 +1 @@ +351888743c1d0f7c9ec97a909ff2f7901f77df63 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/log4j-1.2-api-2.25.3.jar.sha1 b/plugins/discovery-ec2/licenses/log4j-1.2-api-2.25.3.jar.sha1 deleted file mode 100644 index ffa0736153da7..0000000000000 --- a/plugins/discovery-ec2/licenses/log4j-1.2-api-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a7e550e638a5e534fd944616c5ae665a67e9501e \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/log4j-1.2-api-2.25.4.jar.sha1 b/plugins/discovery-ec2/licenses/log4j-1.2-api-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..cf65c0331d0bd --- /dev/null +++ b/plugins/discovery-ec2/licenses/log4j-1.2-api-2.25.4.jar.sha1 @@ -0,0 +1 @@ +351888743c1d0f7c9ec97a909ff2f7901f77df63 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/log4j-1.2-api-2.25.3.jar.sha1 b/plugins/discovery-gce/licenses/log4j-1.2-api-2.25.3.jar.sha1 deleted file mode 100644 index ffa0736153da7..0000000000000 --- a/plugins/discovery-gce/licenses/log4j-1.2-api-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a7e550e638a5e534fd944616c5ae665a67e9501e \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/log4j-1.2-api-2.25.4.jar.sha1 b/plugins/discovery-gce/licenses/log4j-1.2-api-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..cf65c0331d0bd --- /dev/null +++ b/plugins/discovery-gce/licenses/log4j-1.2-api-2.25.4.jar.sha1 @@ -0,0 +1 @@ +351888743c1d0f7c9ec97a909ff2f7901f77df63 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/log4j-slf4j2-impl-2.25.3.jar.sha1 b/plugins/repository-hdfs/licenses/log4j-slf4j2-impl-2.25.3.jar.sha1 deleted file mode 100644 index 9b67ab85d8523..0000000000000 --- a/plugins/repository-hdfs/licenses/log4j-slf4j2-impl-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8d1aca17817adb4a15720e64b98caf9cb3b2cc51 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 b/plugins/repository-hdfs/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..f018d071914e4 --- /dev/null +++ b/plugins/repository-hdfs/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 @@ -0,0 +1 @@ +052a8e43b29eee3b9d6cd9bad696f5d2284d7053 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/log4j-1.2-api-2.25.3.jar.sha1 b/plugins/repository-s3/licenses/log4j-1.2-api-2.25.3.jar.sha1 deleted file mode 100644 index ffa0736153da7..0000000000000 --- a/plugins/repository-s3/licenses/log4j-1.2-api-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a7e550e638a5e534fd944616c5ae665a67e9501e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/log4j-1.2-api-2.25.4.jar.sha1 b/plugins/repository-s3/licenses/log4j-1.2-api-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..cf65c0331d0bd --- /dev/null +++ b/plugins/repository-s3/licenses/log4j-1.2-api-2.25.4.jar.sha1 @@ -0,0 +1 @@ +351888743c1d0f7c9ec97a909ff2f7901f77df63 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/log4j-slf4j2-impl-2.25.3.jar.sha1 b/plugins/repository-s3/licenses/log4j-slf4j2-impl-2.25.3.jar.sha1 deleted file mode 100644 index 9b67ab85d8523..0000000000000 --- a/plugins/repository-s3/licenses/log4j-slf4j2-impl-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8d1aca17817adb4a15720e64b98caf9cb3b2cc51 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 b/plugins/repository-s3/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..f018d071914e4 --- /dev/null +++ b/plugins/repository-s3/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 @@ -0,0 +1 @@ +052a8e43b29eee3b9d6cd9bad696f5d2284d7053 \ No newline at end of file diff --git a/sandbox/libs/dataformat-native/licenses/log4j-api-2.25.3.jar.sha1 b/sandbox/libs/dataformat-native/licenses/log4j-api-2.25.3.jar.sha1 deleted file mode 100644 index 97dc53d973766..0000000000000 --- a/sandbox/libs/dataformat-native/licenses/log4j-api-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fb385330d89c2d61058ef649403f214633569205 \ No newline at end of file diff --git a/sandbox/libs/dataformat-native/licenses/log4j-api-2.25.4.jar.sha1 b/sandbox/libs/dataformat-native/licenses/log4j-api-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..2f492821ebca6 --- /dev/null +++ b/sandbox/libs/dataformat-native/licenses/log4j-api-2.25.4.jar.sha1 @@ -0,0 +1 @@ +89ff2217b193fb187b134aa6ebcbfa8a28b018a9 \ No newline at end of file diff --git a/sandbox/plugins/analytics-backend-datafusion/build.gradle b/sandbox/plugins/analytics-backend-datafusion/build.gradle index 8f2c902595684..f638dd7a412af 100644 --- a/sandbox/plugins/analytics-backend-datafusion/build.gradle +++ b/sandbox/plugins/analytics-backend-datafusion/build.gradle @@ -163,6 +163,8 @@ configurations.all { force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.8.21" force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.8.21" force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10" + force "org.apache.logging.log4j:log4j-api:${versions.log4j}" + force "org.apache.logging.log4j:log4j-core:${versions.log4j}" } } diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle index ced6ff13ba083..c0bec17f95bc1 100644 --- a/sandbox/plugins/analytics-engine/build.gradle +++ b/sandbox/plugins/analytics-engine/build.gradle @@ -218,6 +218,8 @@ configurations.all { force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.8.21" force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.8.21" force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10" + force "org.apache.logging.log4j:log4j-api:${versions.log4j}" + force "org.apache.logging.log4j:log4j-core:${versions.log4j}" } } diff --git a/server/build.gradle b/server/build.gradle index 0e338f901cd5c..5deedc4aa725b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -272,7 +272,9 @@ tasks.named("thirdPartyAudit").configure { 'com.fasterxml.jackson.databind.ser.std.StdScalarSerializer', 'com.fasterxml.jackson.databind.ser.std.StdSerializer', 'com.fasterxml.jackson.dataformat.xml.JacksonXmlModule', + 'com.fasterxml.jackson.dataformat.xml.XmlFactory', 'com.fasterxml.jackson.dataformat.xml.XmlMapper', + 'com.fasterxml.jackson.dataformat.xml.XmlNameProcessor', 'com.fasterxml.jackson.dataformat.xml.util.DefaultXmlPrettyPrinter', 'com.fasterxml.jackson.databind.node.ObjectNode', 'io.micrometer.context.ContextAccessor', @@ -340,6 +342,8 @@ tasks.named("thirdPartyAudit").configure { 'org.apache.kafka.clients.producer.RecordMetadata', 'org.apache.kafka.common.serialization.ByteArraySerializer', 'org.codehaus.stax2.XMLStreamWriter2', + 'org.codehaus.stax2.ri.Stax2WriterAdapter', + 'org.codehaus.stax2.util.StreamWriter2Delegate', 'org.jctools.queues.MpscArrayQueue', 'org.osgi.framework.Bundle', 'org.osgi.framework.BundleActivator', diff --git a/server/licenses/log4j-api-2.25.3.jar.sha1 b/server/licenses/log4j-api-2.25.3.jar.sha1 deleted file mode 100644 index 97dc53d973766..0000000000000 --- a/server/licenses/log4j-api-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fb385330d89c2d61058ef649403f214633569205 \ No newline at end of file diff --git a/server/licenses/log4j-api-2.25.4.jar.sha1 b/server/licenses/log4j-api-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..2f492821ebca6 --- /dev/null +++ b/server/licenses/log4j-api-2.25.4.jar.sha1 @@ -0,0 +1 @@ +89ff2217b193fb187b134aa6ebcbfa8a28b018a9 \ No newline at end of file diff --git a/server/licenses/log4j-core-2.25.3.jar.sha1 b/server/licenses/log4j-core-2.25.3.jar.sha1 deleted file mode 100644 index f04606f9c6047..0000000000000 --- a/server/licenses/log4j-core-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dd9c8ecba5c8dc5e1574804d0bfdc1ef155ad9ea \ No newline at end of file diff --git a/server/licenses/log4j-core-2.25.4.jar.sha1 b/server/licenses/log4j-core-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..3c075c4216500 --- /dev/null +++ b/server/licenses/log4j-core-2.25.4.jar.sha1 @@ -0,0 +1 @@ +b963c3d6bfdf05c61ad47a74e9f9295131607df2 \ No newline at end of file diff --git a/server/licenses/log4j-jul-2.25.3.jar.sha1 b/server/licenses/log4j-jul-2.25.3.jar.sha1 deleted file mode 100644 index 3a73dca2a65ab..0000000000000 --- a/server/licenses/log4j-jul-2.25.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -08bba6d5a56f11931c218c016c96b740e238abbc \ No newline at end of file diff --git a/server/licenses/log4j-jul-2.25.4.jar.sha1 b/server/licenses/log4j-jul-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..3d45a88c063a4 --- /dev/null +++ b/server/licenses/log4j-jul-2.25.4.jar.sha1 @@ -0,0 +1 @@ +72f452618404960dd1a67b6f144fabba5a5093d9 \ No newline at end of file From 417185a9aec3e50e88ace1ff67714291ac2d3ca0 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 30 Apr 2026 11:22:44 -0400 Subject: [PATCH 024/115] Update bundled JDK to JDK 25.0.3 (#21353) Signed-off-by: Andriy Redko --- .../java/org/opensearch/gradle/test/DistroTestPlugin.java | 4 ++-- gradle/libs.versions.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java index 521bdfde1e9a6..d2ed84147ae72 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java @@ -77,9 +77,9 @@ import java.util.stream.Stream; public class DistroTestPlugin implements Plugin { - private static final String SYSTEM_JDK_VERSION = "25.0.2+10"; + private static final String SYSTEM_JDK_VERSION = "25.0.3+9"; private static final String SYSTEM_JDK_VENDOR = "adoptium"; - private static final String GRADLE_JDK_VERSION = "25.0.2+10"; + private static final String GRADLE_JDK_VERSION = "25.0.3+9"; private static final String GRADLE_JDK_VENDOR = "adoptium"; // all distributions used by distro tests. this is temporary until tests are per distribution diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index cfd2c350e7dda..2355ca5f790e1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,7 +3,7 @@ opensearch = "3.7.0" lucene = "10.4.0" bundled_jdk_vendor = "adoptium" -bundled_jdk = "25.0.2+10" +bundled_jdk = "25.0.3+9" # optional dependencies spatial4j = "0.7" From 81ec182b33e3bf894f305c339abb1fb5da0e5b51 Mon Sep 17 00:00:00 2001 From: Shailesh Singh Date: Fri, 1 May 2026 02:51:51 +0530 Subject: [PATCH 025/115] Add merge support for Parquet data format plugin (#21079) * add parquet merge support through a K way merge streaming merge sort Signed-off-by: Shailesh-Kumar-Singh * add tests Signed-off-by: Shailesh-Kumar-Singh * add ColumnMapping optimization Signed-off-by: Shailesh-Kumar-Singh * fix sync_to_disk test Signed-off-by: Shailesh-Kumar-Singh * refractor change, add ParquetSortConfig class Signed-off-by: Shailesh-Kumar-Singh * add InvokeIO in RustBridge Signed-off-by: Shailesh-Kumar-Singh * address comments and refractor changes Signed-off-by: Shailesh-Kumar-Singh * run spotlessApply Signed-off-by: Shailesh-Kumar-Singh * do spotlessApply Signed-off-by: Shailesh-Kumar-Singh * add IntegTests, CRC in merge and address comments Signed-off-by: Shailesh-Kumar-Singh --------- Signed-off-by: Shailesh-Kumar-Singh --- .../libs/dataformat-native/rust/Cargo.toml | 1 + .../composite/CompositeMergeIT.java | 278 +++++++--- .../benchmark/VSRRotationBenchmark.java | 9 +- .../opensearch/parquet/ParquetSettings.java | 154 +++++- .../parquet/bridge/NativeParquetWriter.java | 8 +- .../parquet/bridge/NativeSettings.java | 212 ++++++++ .../parquet/bridge/ParquetSortConfig.java | 51 ++ .../opensearch/parquet/bridge/RustBridge.java | 168 +++++- .../parquet/engine/ParquetIndexingEngine.java | 57 +- .../parquet/fields/ArrowSchemaBuilder.java | 3 +- .../merge/NativeParquetMergeStrategy.java | 100 ++++ .../parquet/merge/ParquetMergeExecutor.java | 30 ++ .../parquet/merge/ParquetMergeStrategy.java | 24 + .../opensearch/parquet/vsr/VSRManager.java | 31 +- .../parquet/writer/ParquetWriter.java | 17 +- .../src/main/rust/Cargo.toml | 10 +- .../src/main/rust/src/crc_writer.rs | 49 ++ .../src/main/rust/src/ffm.rs | 278 +++++++++- .../src/main/rust/src/field_config.rs | 45 ++ .../src/main/rust/src/lib.rs | 10 + .../src/main/rust/src/merge/context.rs | 257 +++++++++ .../src/main/rust/src/merge/cursor.rs | 225 ++++++++ .../src/main/rust/src/merge/error.rs | 56 ++ .../src/main/rust/src/merge/heap.rs | 189 +++++++ .../src/main/rust/src/merge/io_task.rs | 192 +++++++ .../src/main/rust/src/merge/mod.rs | 20 + .../src/main/rust/src/merge/schema.rs | 142 +++++ .../src/main/rust/src/merge/sorted.rs | 225 ++++++++ .../src/main/rust/src/merge/unsorted.rs | 105 ++++ .../src/main/rust/src/native_settings.rs | 150 ++++++ .../src/main/rust/src/rate_limited_writer.rs | 213 ++++++++ .../src/main/rust/src/test_utils.rs | 39 +- .../src/main/rust/src/tests/mod.rs | 380 +++++++++---- .../src/main/rust/src/writer.rs | 407 +++++++++++--- .../rust/src/writer_properties_builder.rs | 199 +++++++ .../rust/tests/merge_integration_tests.rs | 185 +++++++ .../src/main/rust/tests/sort_types_tests.rs | 497 ++++++++++++++++++ .../rust/tests/writer_integration_tests.rs | 9 +- .../bridge/NativeParquetWriterTests.java | 11 +- .../bridge/ParquetMergeIntegrationTests.java | 159 ++++++ .../engine/ParquetIndexingEngineTests.java | 14 +- .../parquet/vsr/VSRManagerTests.java | 35 +- .../parquet/writer/ParquetWriterTests.java | 78 +-- .../index/engine/dataformat/MergeInput.java | 12 + 44 files changed, 4948 insertions(+), 386 deletions(-) create mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeSettings.java create mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/ParquetSortConfig.java create mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java create mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/ParquetMergeExecutor.java create mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/ParquetMergeStrategy.java create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/crc_writer.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/field_config.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/merge/context.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/merge/cursor.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/merge/error.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/merge/heap.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/merge/io_task.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/merge/mod.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/merge/schema.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/native_settings.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/rate_limited_writer.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/writer_properties_builder.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/tests/merge_integration_tests.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/tests/sort_types_tests.rs create mode 100644 sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java diff --git a/sandbox/libs/dataformat-native/rust/Cargo.toml b/sandbox/libs/dataformat-native/rust/Cargo.toml index edc0578d4b09f..25e2ee18e52e2 100644 --- a/sandbox/libs/dataformat-native/rust/Cargo.toml +++ b/sandbox/libs/dataformat-native/rust/Cargo.toml @@ -58,6 +58,7 @@ once_cell = "1.21.3" crc32fast = "1.4" parking_lot = "0.12.5" lazy_static = "1.4.0" +rayon = "1.10" thiserror = "1.0" async-trait = "0.1" bytes = "1" diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java index 3ceda94a96d06..d965df923473d 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java @@ -12,44 +12,47 @@ import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.action.admin.indices.stats.ShardStats; import org.opensearch.action.index.IndexResponse; +import org.opensearch.be.datafusion.DataFusionPlugin; import org.opensearch.be.lucene.LucenePlugin; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.rest.RestStatus; -import org.opensearch.index.IndexSettings; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexService; import org.opensearch.index.engine.CommitStats; -import org.opensearch.index.engine.dataformat.DataFormatDescriptor; -import org.opensearch.index.engine.dataformat.DataFormatRegistry; -import org.opensearch.index.engine.dataformat.ReaderManagerConfig; -import org.opensearch.index.engine.dataformat.stub.MockDataFormat; -import org.opensearch.index.engine.dataformat.stub.MockDataFormatPlugin; -import org.opensearch.index.engine.dataformat.stub.MockReaderManager; -import org.opensearch.index.engine.exec.EngineReaderManager; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; import org.opensearch.index.engine.exec.coord.DataformatAwareCatalogSnapshot; import org.opensearch.index.merge.MergeStats; -import org.opensearch.index.store.PrecomputedChecksumStrategy; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.IndicesService; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.parquet.bridge.ParquetFileMetadata; +import org.opensearch.parquet.bridge.RustBridge; import org.opensearch.plugins.Plugin; -import org.opensearch.plugins.SearchBackEndPlugin; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; -import java.util.function.Supplier; /** - * Integration tests for composite merge operations across single and multiple data format engines. + * Integration tests for composite merge with real Parquet backend. * - * Requires JDK 25 and sandbox enabled. Run with: + * Run with: * ./gradlew :sandbox:plugins:composite-engine:internalClusterTest \ - * --tests "*.CompositeMergeIT" \ - * -Dsandbox.enabled=true + * --tests "*.CompositeMergeIT" -Dsandbox.enabled=true */ @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1) public class CompositeMergeIT extends OpenSearchIntegTestCase { @@ -57,38 +60,6 @@ public class CompositeMergeIT extends OpenSearchIntegTestCase { private static final String INDEX_NAME = "test-composite-merge"; private static final String MERGE_ENABLED_PROPERTY = "opensearch.pluggable.dataformat.merge.enabled"; - // ── Mock DataFormatPlugin using test framework stubs ── - - public static class MockParquetDataFormatPlugin extends MockDataFormatPlugin implements SearchBackEndPlugin { - private static final MockDataFormat PARQUET_FORMAT = new MockDataFormat("parquet", 0L, Set.of()); - - public MockParquetDataFormatPlugin() { - super(PARQUET_FORMAT); - } - - @Override - public Map> getFormatDescriptors(IndexSettings indexSettings, DataFormatRegistry registry) { - return Map.of("parquet", () -> new DataFormatDescriptor("parquet", new PrecomputedChecksumStrategy())); - } - - @Override - public String name() { - return "mock-parquet-backend"; - } - - @Override - public List getSupportedFormats() { - return List.of("parquet"); - } - - @Override - public EngineReaderManager createReaderManager(ReaderManagerConfig settings) { - return new MockReaderManager("parquet"); - } - } - - // ── Test setup ── - @Override public void setUp() throws Exception { enableMerge(); @@ -118,7 +89,7 @@ private static void disableMerge() { @Override protected Collection> nodePlugins() { - return Arrays.asList(MockParquetDataFormatPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class); + return Arrays.asList(ParquetDataFormatPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class, DataFusionPlugin.class); } @Override @@ -129,30 +100,65 @@ protected Settings nodeSettings(int nodeOrdinal) { .build(); } - // ── Tests ── + /** + * Verifies background merge produces a valid merged parquet file + * with correct row count and source files cleaned up. + */ + public void testBackgroundMerge() throws Exception { + client().admin() + .indices() + .prepareCreate(INDEX_NAME) + .setSettings(unsortedSettings()) + .setMapping("name", "type=keyword", "age", "type=integer") + .get(); + ensureGreen(INDEX_NAME); + + int docsPerCycle = 5; + int refreshCycles = 15; + indexDocsAcrossMultipleRefreshes(refreshCycles, docsPerCycle); + int totalDocs = refreshCycles * docsPerCycle; + + assertBusy(() -> { + flush(INDEX_NAME); + DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); + assertTrue( + "Expected merges to reduce segment count below " + refreshCycles + ", but got: " + snapshot.getSegments().size(), + snapshot.getSegments().size() < refreshCycles + ); + }); + + MergeStats mergeStats = getMergeStats(); + assertTrue("Expected at least one merge to have occurred", mergeStats.getTotal() > 0); + + DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); + assertEquals(Set.of("parquet"), snapshot.getDataFormats()); + + verifyRowCount(snapshot, totalDocs); + } /** - * Verifies that background merges are triggered automatically after refresh - * when enough segments accumulate to exceed the TieredMergePolicy threshold. - *

    - * Flow: index docs across many refresh cycles → each refresh calls - * triggerPossibleMerges() → MergeScheduler picks up merge candidates - * asynchronously → segment count decreases. + * Verifies sorted merge with age DESC (nulls first), name ASC (nulls last). */ - public void testBackgroundMergeSingleEngine() throws Exception { - createIndex(INDEX_NAME, singleEngineSettings()); + public void testSortedMerge() throws Exception { + client().admin() + .indices() + .prepareCreate(INDEX_NAME) + .setSettings(sortedSettings()) + .setMapping("name", "type=keyword", "age", "type=integer") + .get(); ensureGreen(INDEX_NAME); - // Create enough segments to exceed TieredMergePolicy's default threshold (~10) - int totalSegmentsCreated = indexDocsAcrossMultipleRefreshes(15, 5); + int docsPerCycle = 10; + int refreshCycles = 15; + indexDocsWithNullsAcrossRefreshes(refreshCycles, docsPerCycle); + int totalDocs = refreshCycles * docsPerCycle; - // Wait for async background merges to complete assertBusy(() -> { flush(INDEX_NAME); DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); assertTrue( - "Expected merges to reduce segment count below " + totalSegmentsCreated + ", but got: " + snapshot.getSegments().size(), - snapshot.getSegments().size() < totalSegmentsCreated + "Expected merges to reduce segment count below " + refreshCycles + ", but got: " + snapshot.getSegments().size(), + snapshot.getSegments().size() < refreshCycles ); }); @@ -161,14 +167,18 @@ public void testBackgroundMergeSingleEngine() throws Exception { DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot(); assertEquals(Set.of("parquet"), snapshot.getDataFormats()); + + verifyRowCount(snapshot, totalDocs); + verifySortOrder(snapshot); } - // ── Helpers ── + // ── Settings ── - private Settings singleEngineSettings() { + private Settings unsortedSettings() { return Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.refresh_interval", "-1") .put("index.pluggable.dataformat.enabled", true) .put("index.pluggable.dataformat", "composite") .put("index.composite.primary_data_format", "parquet") @@ -176,19 +186,153 @@ private Settings singleEngineSettings() { .build(); } - private int indexDocsAcrossMultipleRefreshes(int refreshCycles, int docsPerCycle) { + private Settings sortedSettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.refresh_interval", "-1") + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats") + .putList("index.sort.field", "age", "name") + .putList("index.sort.order", "desc", "asc") + .putList("index.sort.missing", "_first", "_last") + .build(); + } + + // ── Indexing ── + + private void indexDocsAcrossMultipleRefreshes(int refreshCycles, int docsPerCycle) { for (int cycle = 0; cycle < refreshCycles; cycle++) { for (int i = 0; i < docsPerCycle; i++) { IndexResponse response = client().prepareIndex() .setIndex(INDEX_NAME) - .setSource("field_text", randomAlphaOfLength(10), "field_number", randomIntBetween(1, 1000)) + .setSource("name", randomAlphaOfLength(10), "age", randomIntBetween(1, 1000)) .get(); assertEquals(RestStatus.CREATED, response.status()); } RefreshResponse refreshResponse = client().admin().indices().prepareRefresh(INDEX_NAME).get(); assertEquals(RestStatus.OK, refreshResponse.getStatus()); } - return refreshCycles; + } + + private void indexDocsWithNullsAcrossRefreshes(int refreshCycles, int docsPerCycle) { + for (int cycle = 0; cycle < refreshCycles; cycle++) { + for (int i = 0; i < docsPerCycle; i++) { + IndexResponse response; + if (i % 5 == 0) { + response = client().prepareIndex().setIndex(INDEX_NAME).setSource("name", randomAlphaOfLength(10)).get(); + } else { + response = client().prepareIndex() + .setIndex(INDEX_NAME) + .setSource("name", randomAlphaOfLength(10), "age", randomIntBetween(0, 100)) + .get(); + } + assertEquals(RestStatus.CREATED, response.status()); + } + RefreshResponse refreshResponse = client().admin().indices().prepareRefresh(INDEX_NAME).get(); + assertEquals(RestStatus.OK, refreshResponse.getStatus()); + } + } + + // ── Verification ── + + private void verifyRowCount(DataformatAwareCatalogSnapshot snapshot, int expectedTotalDocs) throws IOException { + Path parquetDir = getParquetDir(); + long totalRows = 0; + for (Segment segment : snapshot.getSegments()) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); + assertNotNull("Segment should have parquet files", wfs); + for (String file : wfs.files()) { + Path filePath = parquetDir.resolve(file); + assertTrue("Parquet file should exist: " + filePath, Files.exists(filePath)); + ParquetFileMetadata metadata = RustBridge.getFileMetadata(filePath.toString()); + totalRows += metadata.numRows(); + } + } + assertEquals("Total rows across all segments should match ingested docs", expectedTotalDocs, totalRows); + } + + /** + * Verifies that merged parquet files have age in DESC order with nulls first, + * and within same age, name in ASC order with nulls last. + */ + @SuppressForbidden(reason = "JSON parsing for test verification of parquet output") + private void verifySortOrder(DataformatAwareCatalogSnapshot snapshot) throws Exception { + Path parquetDir = getParquetDir(); + for (Segment segment : snapshot.getSegments()) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); + for (String file : wfs.files()) { + Path filePath = parquetDir.resolve(file); + String json = RustBridge.readAsJson(filePath.toString()); + List> rows; + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + json + ) + ) { + rows = parser.list().stream().map(o -> { + @SuppressWarnings("unchecked") + Map m = (Map) o; + return m; + }).toList(); + } + if (rows.size() <= 1) continue; + + for (int i = 1; i < rows.size(); i++) { + Object prevAge = rows.get(i - 1).get("age"); + Object currAge = rows.get(i).get("age"); + + // nulls first for age + if (prevAge == null && currAge == null) continue; + if (prevAge == null) continue; // null before non-null is correct + if (currAge == null) { + fail("age null should come before non-null, but found non-null at " + (i - 1) + " and null at " + i); + } + + int prevAgeVal = ((Number) prevAge).intValue(); + int currAgeVal = ((Number) currAge).intValue(); + + assertTrue( + "age should be DESC but found " + prevAgeVal + " before " + currAgeVal + " at row " + i, + prevAgeVal >= currAgeVal + ); + + // When age is equal, verify name ASC (nulls last) + if (prevAgeVal == currAgeVal) { + Object prevName = rows.get(i - 1).get("name"); + Object currName = rows.get(i).get("name"); + + if (prevName != null && currName == null) continue; // non-null before null is correct for nulls last + if (prevName == null && currName != null) { + fail("name nulls should be last, but found null at " + (i - 1) + " and non-null at " + i); + } + if (prevName != null && currName != null) { + assertTrue( + "name should be ASC but found '" + prevName + "' before '" + currName + "' at row " + i, + ((String) prevName).compareTo((String) currName) <= 0 + ); + } + } + } + } + } + } + + private Path getParquetDir() { + IndexShard shard = getPrimaryShard(); + return shard.shardPath().getDataPath().resolve("parquet"); + } + + private IndexShard getPrimaryShard() { + String nodeName = getClusterState().routingTable().index(INDEX_NAME).shard(0).primaryShard().currentNodeId(); + String nodeNameResolved = getClusterState().nodes().get(nodeName).getName(); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeNameResolved); + IndexService indexService = indicesService.indexServiceSafe(resolveIndex(INDEX_NAME)); + return indexService.getShard(0); } private DataformatAwareCatalogSnapshot getCatalogSnapshot() throws IOException { diff --git a/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java b/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java index aa47e2f44b287..aee6483e81efd 100644 --- a/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java +++ b/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java @@ -10,7 +10,10 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; @@ -80,6 +83,7 @@ public class VSRRotationBenchmark { private List fieldTypes; private VSRManager vsrManager; private String filePath; + private IndexSettings indexSettings; @Setup(Level.Trial) public void setupTrial() { @@ -123,7 +127,10 @@ public void setupTrial() { public void setup() throws IOException { bufferPool = new ArrowBufferPool(Settings.EMPTY); filePath = Path.of(System.getProperty("java.io.tmpdir"), "benchmark_vsr_" + System.nanoTime() + ".parquet").toString(); - vsrManager = new VSRManager(filePath, schema, bufferPool, maxRowsPerVSR, threadPool, runAsync); + Settings idxSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); + IndexMetadata indexMetadata = IndexMetadata.builder("benchmark-index").settings(idxSettings).build(); + indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + vsrManager = new VSRManager(filePath, indexSettings, schema, bufferPool, maxRowsPerVSR, threadPool, runAsync); } @Benchmark diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetSettings.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetSettings.java index 71e57fb0542fa..ab58d0bfdf11c 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetSettings.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetSettings.java @@ -9,40 +9,95 @@ package org.opensearch.parquet; import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import java.util.List; /** - * Node-scoped settings for the Parquet data format plugin. - * - *

    All settings are registered with OpenSearch via - * {@link ParquetDataFormatPlugin#getSettings()} and can be configured in - * {@code opensearch.yml} or via cluster settings API. - * - *

      - *
    • {@link #MAX_NATIVE_ALLOCATION} — Maximum native memory allocation for Arrow buffers, - * expressed as a percentage of available non-heap system memory (default {@code "10%"}).
      • - *
    • {@link #MAX_ROWS_PER_VSR} — Row count threshold that triggers VectorSchemaRoot rotation - * during document ingestion (default {@code 50000}).
      • - *
    + * Settings for Parquet data format. */ public final class ParquetSettings { private ParquetSettings() {} - /** Default maximum native memory allocation as a percentage of available non-heap memory. */ public static final String DEFAULT_MAX_NATIVE_ALLOCATION = "10%"; - /** Default maximum number of rows per VectorSchemaRoot before rotation. */ public static final int DEFAULT_MAX_ROWS_PER_VSR = 50000; - /** Maximum native memory allocation for Arrow buffers, as a percentage of non-heap memory. */ + /** Group setting prefix for all Parquet settings. */ + public static final Setting PARQUET_SETTINGS = Setting.groupSetting("index.parquet.", Setting.Property.IndexScope); + + /** Data page size limit in bytes (default 1MB). */ + public static final Setting PAGE_SIZE_BYTES = Setting.byteSizeSetting( + "index.parquet.page_size_bytes", + new ByteSizeValue(1, ByteSizeUnit.MB), + Setting.Property.IndexScope + ); + + /** Maximum number of rows per data page (default 20000). */ + public static final Setting PAGE_ROW_LIMIT = Setting.intSetting( + "index.parquet.page_row_limit", + 20000, + 1, + Setting.Property.IndexScope + ); + + /** Dictionary page size limit in bytes (default 2MB). */ + public static final Setting DICT_SIZE_BYTES = Setting.byteSizeSetting( + "index.parquet.dict_size_bytes", + new ByteSizeValue(2, ByteSizeUnit.MB), + Setting.Property.IndexScope + ); + + /** Compression codec for Parquet files, e.g. ZSTD, SNAPPY, LZ4_RAW (default LZ4_RAW). */ + public static final Setting COMPRESSION_TYPE = Setting.simpleString( + "index.parquet.compression_type", + "LZ4_RAW", + Setting.Property.IndexScope + ); + + /** Compression level for the chosen codec (default 2, range 1–9). */ + public static final Setting COMPRESSION_LEVEL = Setting.intSetting( + "index.parquet.compression_level", + 2, + 1, + 9, + Setting.Property.IndexScope + ); + + /** Whether bloom filters are enabled for Parquet columns (default true). */ + public static final Setting BLOOM_FILTER_ENABLED = Setting.boolSetting( + "index.parquet.bloom_filter_enabled", + true, + Setting.Property.IndexScope + ); + + /** Bloom filter false positive probability (default 0.1). */ + public static final Setting BLOOM_FILTER_FPP = Setting.doubleSetting( + "index.parquet.bloom_filter_fpp", + 0.1, + 0.0, + 1.0, + Setting.Property.IndexScope + ); + + /** Bloom filter number of distinct values hint (default 100000). */ + public static final Setting BLOOM_FILTER_NDV = Setting.longSetting( + "index.parquet.bloom_filter_ndv", + 100_000L, + 1L, + Setting.Property.IndexScope + ); + + /** Maximum native memory allocation for Arrow buffers, as a percentage of non-heap memory (default 10%). */ public static final Setting MAX_NATIVE_ALLOCATION = Setting.simpleString( "parquet.max_native_allocation", DEFAULT_MAX_NATIVE_ALLOCATION, Setting.Property.NodeScope ); - /** Maximum number of rows per VectorSchemaRoot before rotation is triggered. */ + /** Maximum rows per VectorSchemaRoot before rotation is triggered (default 50000). */ public static final Setting MAX_ROWS_PER_VSR = Setting.intSetting( "parquet.max_rows_per_vsr", DEFAULT_MAX_ROWS_PER_VSR, @@ -50,8 +105,73 @@ private ParquetSettings() {} Setting.Property.NodeScope ); + /** File size threshold for in-memory sort vs streaming merge sort (default 32MB). */ + public static final Setting SORT_IN_MEMORY_THRESHOLD = Setting.byteSizeSetting( + "index.parquet.sort_in_memory_threshold", + new ByteSizeValue(32, ByteSizeUnit.MB), + Setting.Property.IndexScope + ); + + /** Batch size for streaming merge sort (default 8192 rows). */ + public static final Setting SORT_BATCH_SIZE = Setting.intSetting( + "index.parquet.sort_batch_size", + 8192, + 1, + Setting.Property.IndexScope + ); + + /** Maximum number of rows per row group (default 1000000). */ + public static final Setting ROW_GROUP_MAX_ROWS = Setting.intSetting( + "index.parquet.row_group_max_rows", + 1_000_000, + 1, + Setting.Property.IndexScope + ); + + /** Batch size for reading records during merge (default 100000 rows). */ + public static final Setting MERGE_BATCH_SIZE = Setting.intSetting( + "index.parquet.merge_batch_size", + 100_000, + 1, + Setting.Property.IndexScope + ); + + /** Number of Rayon threads for parallel column encoding during merge (default num_cores/8, min 1). */ + public static final Setting MERGE_RAYON_THREADS = Setting.intSetting( + "parquet.merge_rayon_threads", + Math.max(1, Runtime.getRuntime().availableProcessors() / 8), + 1, + Setting.Property.NodeScope + ); + + /** Number of Tokio IO threads for async disk writes during merge (default num_cores/8, min 1). */ + public static final Setting MERGE_IO_THREADS = Setting.intSetting( + "parquet.merge_io_threads", + Math.max(1, Runtime.getRuntime().availableProcessors() / 8), + 1, + Setting.Property.NodeScope + ); + /** Returns all settings defined by the Parquet plugin. */ public static List> getSettings() { - return List.of(MAX_NATIVE_ALLOCATION, MAX_ROWS_PER_VSR); + return List.of( + PARQUET_SETTINGS, + PAGE_SIZE_BYTES, + PAGE_ROW_LIMIT, + DICT_SIZE_BYTES, + COMPRESSION_TYPE, + COMPRESSION_LEVEL, + BLOOM_FILTER_ENABLED, + BLOOM_FILTER_FPP, + BLOOM_FILTER_NDV, + MAX_NATIVE_ALLOCATION, + MAX_ROWS_PER_VSR, + SORT_IN_MEMORY_THRESHOLD, + SORT_BATCH_SIZE, + ROW_GROUP_MAX_ROWS, + MERGE_BATCH_SIZE, + MERGE_RAYON_THREADS, + MERGE_IO_THREADS + ); } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeParquetWriter.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeParquetWriter.java index 34b41d635d41a..7230c2e79c752 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeParquetWriter.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeParquetWriter.java @@ -18,7 +18,7 @@ * *

    Wraps the stateless JNI methods in {@link RustBridge} with a file-scoped lifecycle: *

      - *
    1. {@code new NativeParquetWriter(filePath, schemaAddress)} — creates the native writer
      2. + *
    2. {@code new NativeParquetWriter(filePath, indexName, schemaAddress, sortConfig)} — creates the native writer
      3. *
    3. {@link #write(long, long)} — sends one or more Arrow batches (repeatable)
      4. *
    4. {@link #flush()} — finalizes the Parquet file and returns metadata
      5. *
    5. {@link #sync()} — fsyncs the file to durable storage (calls flush if needed)
      6. @@ -37,12 +37,14 @@ public class NativeParquetWriter { * Creates a new NativeParquetWriter. * * @param filePath the path to the Parquet file to write + * @param indexName the index name for settings lookup * @param schemaAddress the native memory address of the Arrow schema + * @param sortConfig the sort configuration for the Parquet file * @throws IOException if the native writer creation fails */ - public NativeParquetWriter(String filePath, long schemaAddress) throws IOException { + public NativeParquetWriter(String filePath, String indexName, long schemaAddress, ParquetSortConfig sortConfig) throws IOException { this.filePath = filePath; - RustBridge.createWriter(filePath, schemaAddress); + RustBridge.createWriter(filePath, indexName, schemaAddress, sortConfig); } /** diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeSettings.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeSettings.java new file mode 100644 index 0000000000000..db940828424d3 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeSettings.java @@ -0,0 +1,212 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.bridge; + +/** + * Immutable settings passed to the native Rust writer via JNI. + * The Rust side reads values through the getter methods. + * All fields are nullable; the native side falls back to defaults when null. + */ +public class NativeSettings { + + private final String indexName; + private final String compressionType; + private final Integer compressionLevel; + private final Long pageSizeBytes; + private final Integer pageRowLimit; + private final Long dictSizeBytes; + private final Boolean bloomFilterEnabled; + private final Double bloomFilterFpp; + private final Long bloomFilterNdv; + private final Long sortInMemoryThresholdBytes; + private final Integer sortBatchSize; + private final Integer rowGroupMaxRows; + private final Integer mergeBatchSize; + private final Integer mergeRayonThreads; + private final Integer mergeIoThreads; + + private NativeSettings(Builder builder) { + this.indexName = builder.indexName; + this.compressionType = builder.compressionType; + this.compressionLevel = builder.compressionLevel; + this.pageSizeBytes = builder.pageSizeBytes; + this.pageRowLimit = builder.pageRowLimit; + this.dictSizeBytes = builder.dictSizeBytes; + this.bloomFilterEnabled = builder.bloomFilterEnabled; + this.bloomFilterFpp = builder.bloomFilterFpp; + this.bloomFilterNdv = builder.bloomFilterNdv; + this.sortInMemoryThresholdBytes = builder.sortInMemoryThresholdBytes; + this.sortBatchSize = builder.sortBatchSize; + this.rowGroupMaxRows = builder.rowGroupMaxRows; + this.mergeBatchSize = builder.mergeBatchSize; + this.mergeRayonThreads = builder.mergeRayonThreads; + this.mergeIoThreads = builder.mergeIoThreads; + } + + public String getIndexName() { + return indexName; + } + + public String getCompressionType() { + return compressionType; + } + + public Integer getCompressionLevel() { + return compressionLevel; + } + + public Long getPageSizeBytes() { + return pageSizeBytes; + } + + public Integer getPageRowLimit() { + return pageRowLimit; + } + + public Long getDictSizeBytes() { + return dictSizeBytes; + } + + public Boolean getBloomFilterEnabled() { + return bloomFilterEnabled; + } + + public Double getBloomFilterFpp() { + return bloomFilterFpp; + } + + public Long getBloomFilterNdv() { + return bloomFilterNdv; + } + + public Long getSortInMemoryThresholdBytes() { + return sortInMemoryThresholdBytes; + } + + public Integer getSortBatchSize() { + return sortBatchSize; + } + + public Integer getRowGroupMaxRows() { + return rowGroupMaxRows; + } + + public Integer getMergeBatchSize() { + return mergeBatchSize; + } + + public Integer getMergeRayonThreads() { + return mergeRayonThreads; + } + + public Integer getMergeIoThreads() { + return mergeIoThreads; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String indexName; + private String compressionType; + private Integer compressionLevel; + private Long pageSizeBytes; + private Integer pageRowLimit; + private Long dictSizeBytes; + private Boolean bloomFilterEnabled; + private Double bloomFilterFpp; + private Long bloomFilterNdv; + private Long sortInMemoryThresholdBytes; + private Integer sortBatchSize; + private Integer rowGroupMaxRows; + private Integer mergeBatchSize; + private Integer mergeRayonThreads; + private Integer mergeIoThreads; + + public Builder indexName(String v) { + this.indexName = v; + return this; + } + + public Builder compressionType(String v) { + this.compressionType = v; + return this; + } + + public Builder compressionLevel(Integer v) { + this.compressionLevel = v; + return this; + } + + public Builder pageSizeBytes(Long v) { + this.pageSizeBytes = v; + return this; + } + + public Builder pageRowLimit(Integer v) { + this.pageRowLimit = v; + return this; + } + + public Builder dictSizeBytes(Long v) { + this.dictSizeBytes = v; + return this; + } + + public Builder bloomFilterEnabled(Boolean v) { + this.bloomFilterEnabled = v; + return this; + } + + public Builder bloomFilterFpp(Double v) { + this.bloomFilterFpp = v; + return this; + } + + public Builder bloomFilterNdv(Long v) { + this.bloomFilterNdv = v; + return this; + } + + public Builder sortInMemoryThresholdBytes(Long v) { + this.sortInMemoryThresholdBytes = v; + return this; + } + + public Builder sortBatchSize(Integer v) { + this.sortBatchSize = v; + return this; + } + + public Builder rowGroupMaxRows(Integer v) { + this.rowGroupMaxRows = v; + return this; + } + + public Builder mergeBatchSize(Integer v) { + this.mergeBatchSize = v; + return this; + } + + public Builder mergeRayonThreads(Integer v) { + this.mergeRayonThreads = v; + return this; + } + + public Builder mergeIoThreads(Integer v) { + this.mergeIoThreads = v; + return this; + } + + public NativeSettings build() { + return new NativeSettings(this); + } + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/ParquetSortConfig.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/ParquetSortConfig.java new file mode 100644 index 0000000000000..7d86ac3365f04 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/ParquetSortConfig.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.bridge; + +import org.opensearch.index.IndexSettings; +import org.opensearch.index.IndexSortConfig; +import org.opensearch.search.sort.SortOrder; + +import java.util.Collections; +import java.util.List; + +/** + * Encapsulates index sort configuration for the native Parquet writer. + * + *

      Extracts sort columns, sort orders, and null-handling preferences from + * {@link IndexSettings} and exposes them as typed lists ready for the native bridge. + */ +public record ParquetSortConfig(List sortColumns, List reverseSorts, List nullsFirst) { + + private static final ParquetSortConfig EMPTY = new ParquetSortConfig( + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList() + ); + + /** + * Creates a sort config from index settings. + * + * @param indexSettings the index settings to extract sort configuration from + */ + public ParquetSortConfig(IndexSettings indexSettings) { + this( + IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexSettings.getSettings()), + IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(indexSettings.getSettings()).stream().map(o -> o == SortOrder.DESC).toList(), + IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(indexSettings.getSettings()).stream().map("_first"::equals).toList() + ); + } + + /** + * Returns an empty sort config (no sorting). + */ + public static ParquetSortConfig empty() { + return EMPTY; + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java index c9086cfe4e8e6..ca4ffa0c9c7d9 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java @@ -12,12 +12,15 @@ import org.opensearch.nativebridge.spi.NativeLibraryLoader; import java.io.IOException; +import java.io.UncheckedIOException; import java.lang.foreign.FunctionDescriptor; import java.lang.foreign.Linker; import java.lang.foreign.SymbolLookup; import java.lang.foreign.ValueLayout; import java.lang.invoke.MethodHandle; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; public class RustBridge { @@ -27,13 +30,31 @@ public class RustBridge { private static final MethodHandle SYNC_TO_DISK; private static final MethodHandle GET_FILE_METADATA; private static final MethodHandle GET_FILTERED_BYTES; + private static final MethodHandle ON_SETTINGS_UPDATE; + private static final MethodHandle REMOVE_SETTINGS; + private static final MethodHandle MERGE_FILES; + private static final MethodHandle READ_AS_JSON; static { SymbolLookup lib = NativeLibraryLoader.symbolLookup(); Linker linker = Linker.nativeLinker(); CREATE_WRITER = linker.downcallHandle( lib.find("parquet_create_writer").orElseThrow(), - FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, // file + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, // index_name + ValueLayout.JAVA_LONG, // schema_address + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, // sort_columns (ptrs, lens, count) + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, // reverse_sorts (vals, count) + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG // nulls_first (vals, count) + ) ); WRITE = linker.downcallHandle( lib.find("parquet_write").orElseThrow(), @@ -80,14 +101,83 @@ public class RustBridge { lib.find("parquet_get_filtered_native_bytes_used").orElseThrow(), FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) ); + ON_SETTINGS_UPDATE = linker.downcallHandle( + lib.find("parquet_on_settings_update").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, // index_name + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, // compression_type + ValueLayout.JAVA_LONG, // compression_level + ValueLayout.JAVA_LONG, // page_size_bytes + ValueLayout.JAVA_LONG, // page_row_limit + ValueLayout.JAVA_LONG, // dict_size_bytes + ValueLayout.JAVA_LONG, // bloom_filter_enabled + ValueLayout.JAVA_DOUBLE, // bloom_filter_fpp + ValueLayout.JAVA_LONG, // bloom_filter_ndv + ValueLayout.JAVA_LONG, // sort_in_memory_threshold_bytes + ValueLayout.JAVA_LONG, // sort_batch_size + ValueLayout.JAVA_LONG, // row_group_max_rows + ValueLayout.JAVA_LONG, // merge_batch_size + ValueLayout.JAVA_LONG, // merge_rayon_threads + ValueLayout.JAVA_LONG // merge_io_threads + ) + ); + REMOVE_SETTINGS = linker.downcallHandle( + lib.find("parquet_remove_settings").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) + ); + MERGE_FILES = linker.downcallHandle( + lib.find("parquet_merge_files").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, // input files (ptrs, lens, count) + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, // output file + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG // index_name + ) + ); + READ_AS_JSON = linker.downcallHandle( + lib.find("parquet_read_as_json").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, // file + ValueLayout.ADDRESS, // out_buf + ValueLayout.JAVA_LONG, // buf_capacity + ValueLayout.ADDRESS // out_len + ) + ); } public static void initLogger() {} - static void createWriter(String file, long schemaAddress) throws IOException { + static void createWriter(String file, String indexName, long schemaAddress, ParquetSortConfig sortConfig) throws IOException { try (var call = new NativeCall()) { var f = call.str(file); - call.invokeIO(CREATE_WRITER, f.segment(), f.len(), schemaAddress); + var idx = call.str(indexName); + var sorts = call.strArray(sortConfig.sortColumns().toArray(new String[0])); + var reverseArray = marshalBoolList(call, sortConfig.reverseSorts()); + var nullsFirstArray = marshalBoolList(call, sortConfig.nullsFirst()); + call.invokeIO( + CREATE_WRITER, + f.segment(), + f.len(), + idx.segment(), + idx.len(), + schemaAddress, + sorts.ptrs(), + sorts.lens(), + sorts.count(), + reverseArray, + (long) sortConfig.reverseSorts().size(), + nullsFirstArray, + (long) sortConfig.nullsFirst().size() + ); } } @@ -162,5 +252,77 @@ public static long getFilteredNativeBytesUsed(String pathPrefix) { } } + public static void onSettingsUpdate(NativeSettings nativeSettings) throws IOException { + try (var call = new NativeCall()) { + var idx = call.str(nativeSettings.getIndexName()); + var ct = nativeSettings.getCompressionType() != null ? call.str(nativeSettings.getCompressionType()) : null; + call.invokeIO( + ON_SETTINGS_UPDATE, + idx.segment(), + idx.len(), + ct != null ? ct.segment() : java.lang.foreign.MemorySegment.NULL, + ct != null ? ct.len() : -1L, + nativeSettings.getCompressionLevel() != null ? (long) nativeSettings.getCompressionLevel() : -1L, + nativeSettings.getPageSizeBytes() != null ? nativeSettings.getPageSizeBytes() : -1L, + nativeSettings.getPageRowLimit() != null ? (long) nativeSettings.getPageRowLimit() : -1L, + nativeSettings.getDictSizeBytes() != null ? nativeSettings.getDictSizeBytes() : -1L, + nativeSettings.getBloomFilterEnabled() != null ? (nativeSettings.getBloomFilterEnabled() ? 1L : 0L) : -1L, + nativeSettings.getBloomFilterFpp() != null ? nativeSettings.getBloomFilterFpp() : -1.0, + nativeSettings.getBloomFilterNdv() != null ? nativeSettings.getBloomFilterNdv() : -1L, + nativeSettings.getSortInMemoryThresholdBytes() != null ? nativeSettings.getSortInMemoryThresholdBytes() : -1L, + nativeSettings.getSortBatchSize() != null ? (long) nativeSettings.getSortBatchSize() : -1L, + nativeSettings.getRowGroupMaxRows() != null ? (long) nativeSettings.getRowGroupMaxRows() : -1L, + nativeSettings.getMergeBatchSize() != null ? (long) nativeSettings.getMergeBatchSize() : -1L, + nativeSettings.getMergeRayonThreads() != null ? (long) nativeSettings.getMergeRayonThreads() : -1L, + nativeSettings.getMergeIoThreads() != null ? (long) nativeSettings.getMergeIoThreads() : -1L + ); + } + } + + public static void removeSettings(String indexName) { + try (var call = new NativeCall()) { + var idx = call.str(indexName); + call.invoke(REMOVE_SETTINGS, idx.segment(), idx.len()); + } + } + + public static void mergeParquetFilesInRust(List inputFiles, String outputFile, String indexName) { + String[] paths = inputFiles.stream().map(Path::toString).toArray(String[]::new); + try (var call = new NativeCall()) { + var inputs = call.strArray(paths); + var out = call.str(outputFile); + var idx = call.str(indexName); + call.invokeIO(MERGE_FILES, inputs.ptrs(), inputs.lens(), inputs.count(), out.segment(), out.len(), idx.segment(), idx.len()); + } catch (IOException e) { + throw new UncheckedIOException("Native merge failed", e); + } + } + + private static java.lang.foreign.MemorySegment marshalBoolList(NativeCall call, List bools) { + if (bools == null || bools.isEmpty()) { + return java.lang.foreign.MemorySegment.NULL; + } + var seg = call.buf(bools.size() * 8); + for (int i = 0; i < bools.size(); i++) { + seg.setAtIndex(ValueLayout.JAVA_LONG, i, bools.get(i) ? 1L : 0L); + } + return seg; + } + + /** + * Reads a parquet file and returns its contents as a JSON string. + */ + public static String readAsJson(String file) throws IOException { + try (var call = new NativeCall()) { + var f = call.str(file); + int bufSize = 10 * 1024 * 1024; // 10MB + var outBuf = call.buf(bufSize); + var outLen = call.longOut(); + call.invokeIO(READ_AS_JSON, f.segment(), f.len(), outBuf, (long) bufSize, outLen); + int len = (int) outLen.get(ValueLayout.JAVA_LONG, 0); + return new String(outBuf.asSlice(0, len).toArray(ValueLayout.JAVA_BYTE), StandardCharsets.UTF_8); + } + } + private RustBridge() {} } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java index 1d5c43522e418..0b4cb025e8463 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java @@ -14,7 +14,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; -import org.opensearch.index.engine.dataformat.MergeResult; import org.opensearch.index.engine.dataformat.Merger; import org.opensearch.index.engine.dataformat.RefreshInput; import org.opensearch.index.engine.dataformat.RefreshResult; @@ -24,13 +23,18 @@ import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.FormatChecksumStrategy; import org.opensearch.index.store.PrecomputedChecksumStrategy; +import org.opensearch.parquet.ParquetSettings; +import org.opensearch.parquet.bridge.NativeSettings; import org.opensearch.parquet.bridge.RustBridge; import org.opensearch.parquet.memory.ArrowBufferPool; +import org.opensearch.parquet.merge.NativeParquetMergeStrategy; +import org.opensearch.parquet.merge.ParquetMergeExecutor; import org.opensearch.parquet.writer.ParquetDocumentInput; import org.opensearch.parquet.writer.ParquetWriter; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; @@ -68,9 +72,11 @@ public class ParquetIndexingEngine implements IndexingExecutionEngine schemaSupplier; private final ArrowBufferPool bufferPool; - private final Settings settings; + private final IndexSettings indexSettings; + private final Settings nodeSettings; private final ThreadPool threadPool; private final FormatChecksumStrategy checksumStrategy; + private final Merger parquetMerger; /** * Creates a new ParquetIndexingEngine. @@ -121,7 +127,8 @@ public ParquetIndexingEngine( this.shardPath = shardPath; this.schemaSupplier = schemaSupplier; this.bufferPool = new ArrowBufferPool(settings); - this.settings = settings; + this.indexSettings = indexSettings; + this.nodeSettings = settings; this.threadPool = threadPool; this.checksumStrategy = checksumStrategy; try { @@ -131,6 +138,10 @@ public ParquetIndexingEngine( } catch (IOException e) { throw new RuntimeException(e); } + this.parquetMerger = new ParquetMergeExecutor( + new NativeParquetMergeStrategy(dataFormat, indexSettings.getIndex().getName(), shardPath.getDataPath()) + ); + pushSettingsToRust(); } /** @@ -142,6 +153,32 @@ public FormatChecksumStrategy getChecksumStrategy() { return checksumStrategy; } + private void pushSettingsToRust() { + Settings settings = indexSettings.getSettings(); + NativeSettings config = NativeSettings.builder() + .indexName(indexSettings.getIndex().getName()) + .compressionType(ParquetSettings.COMPRESSION_TYPE.get(settings)) + .compressionLevel(ParquetSettings.COMPRESSION_LEVEL.get(settings)) + .pageSizeBytes(ParquetSettings.PAGE_SIZE_BYTES.get(settings).getBytes()) + .pageRowLimit(ParquetSettings.PAGE_ROW_LIMIT.get(settings)) + .dictSizeBytes(ParquetSettings.DICT_SIZE_BYTES.get(settings).getBytes()) + .bloomFilterEnabled(ParquetSettings.BLOOM_FILTER_ENABLED.get(settings)) + .bloomFilterFpp(ParquetSettings.BLOOM_FILTER_FPP.get(settings)) + .bloomFilterNdv(ParquetSettings.BLOOM_FILTER_NDV.get(settings)) + .sortInMemoryThresholdBytes(ParquetSettings.SORT_IN_MEMORY_THRESHOLD.get(settings).getBytes()) + .sortBatchSize(ParquetSettings.SORT_BATCH_SIZE.get(settings)) + .rowGroupMaxRows(ParquetSettings.ROW_GROUP_MAX_ROWS.get(settings)) + .mergeBatchSize(ParquetSettings.MERGE_BATCH_SIZE.get(settings)) + .mergeRayonThreads(ParquetSettings.MERGE_RAYON_THREADS.get(nodeSettings)) + .mergeIoThreads(ParquetSettings.MERGE_IO_THREADS.get(nodeSettings)) + .build(); + try { + RustBridge.onSettingsUpdate(config); + } catch (IOException e) { + throw new UncheckedIOException("Failed to push Parquet settings to Rust store", e); + } + } + @Override public Writer createWriter(long writerGeneration) { Path filePath = Path.of( @@ -155,7 +192,7 @@ public Writer createWriter(long writerGeneration) { dataFormat, schemaSupplier.get(), bufferPool, - settings, + indexSettings, threadPool, checksumStrategy ); @@ -168,8 +205,7 @@ public long getNativeBytesUsed() { @Override public Merger getMerger() { - // TODO: Implement merge support as ParquetMerger - return mergeInput -> new MergeResult(Map.of()); + return parquetMerger; } @Override @@ -223,6 +259,15 @@ public IndexStoreProvider getProvider() { @Override public void close() throws IOException { + try { + RustBridge.removeSettings(indexSettings.getIndex().getName()); + } catch (Exception e) { + logger.warn( + "Failed to remove Parquet settings from Rust store for index [{}]: {}", + indexSettings.getIndex().getName(), + e.getMessage() + ); + } bufferPool.close(); } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowSchemaBuilder.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowSchemaBuilder.java index 49c1d86b5742d..84b2b21712fa3 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowSchemaBuilder.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ArrowSchemaBuilder.java @@ -12,6 +12,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.FieldNamesFieldMapper; import org.opensearch.index.mapper.IndexFieldMapper; import org.opensearch.index.mapper.Mapper; @@ -57,7 +58,7 @@ public static Schema getSchema(MapperService mapperService) { } // Add row ID field (long) LongParquetField longField = new LongParquetField(); - fields.add(new Field("_row_id", longField.getFieldType(), null)); + fields.add(new Field(DocumentInput.ROW_ID_FIELD, longField.getFieldType(), null)); return new Schema(fields); } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java new file mode 100644 index 0000000000000..a3bbdee35c6f8 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java @@ -0,0 +1,100 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.merge; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.MergeInput; +import org.opensearch.index.engine.dataformat.MergeResult; +import org.opensearch.index.engine.exec.WriterFileSet; +import org.opensearch.parquet.bridge.RustBridge; +import org.opensearch.parquet.engine.ParquetIndexingEngine; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Implements merging of Parquet files. + */ +public class NativeParquetMergeStrategy implements ParquetMergeStrategy { + + private static final Logger logger = LogManager.getLogger(NativeParquetMergeStrategy.class); + + private final DataFormat dataFormat; + private final String indexName; + private final Path shardDataPath; + + public NativeParquetMergeStrategy(DataFormat dataFormat, String indexName, Path shardDataPath) { + this.dataFormat = dataFormat; + this.indexName = indexName; + this.shardDataPath = shardDataPath; + } + + @Override + public MergeResult mergeParquetFiles(MergeInput mergeInput) { + + List files = mergeInput.getFilesForFormat(dataFormat.name()); + long writerGeneration = mergeInput.newWriterGeneration(); + if (files.isEmpty()) { + throw new IllegalArgumentException("No files to merge"); + } + + List filePaths = new ArrayList<>(); + files.forEach( + writerFileSet -> writerFileSet.files() + .forEach(file -> filePaths.add(shardDataPath.resolve(writerFileSet.directory()).resolve(file))) + ); + + String outputDirectory = shardDataPath.resolve(files.getFirst().directory()).toString(); + String mergedFilePath = getMergedFilePath(writerGeneration, outputDirectory); + String mergedFileName = getMergedFileName(writerGeneration); + + try { + // Merge files in Rust + RustBridge.mergeParquetFilesInRust(filePaths, mergedFilePath, indexName); + + WriterFileSet mergedWriterFileSet = WriterFileSet.builder() + .directory(Path.of(files.getFirst().directory())) + .addFile(mergedFileName) + .writerGeneration(writerGeneration) + .build(); + + Map mergedWriterFileSetMap = Collections.singletonMap(dataFormat, mergedWriterFileSet); + + return new MergeResult(mergedWriterFileSetMap); + + } catch (Exception exception) { + logger.error(() -> new ParameterizedMessage("Merge failed while creating merged file [{}]", mergedFilePath), exception); + try { + Files.deleteIfExists(Path.of(mergedFilePath)); + logger.info("Stale Merged File Deleted at : [{}]", mergedFilePath); + } catch (Exception innerException) { + logger.error(() -> new ParameterizedMessage("Failed to delete stale merged file [{}]", mergedFilePath), innerException); + + } + throw exception; + } + + } + + private String getMergedFileName(long generation) { + // TODO: For debugging we have added extra "merged" in file name, later we can remove and keep same as writer + return ParquetIndexingEngine.FILE_NAME_PREFIX + "_merged_" + generation + ParquetIndexingEngine.FILE_NAME_EXT; + } + + private String getMergedFilePath(long generation, String outputDirectory) { + return Path.of(outputDirectory, getMergedFileName(generation)).toString(); + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/ParquetMergeExecutor.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/ParquetMergeExecutor.java new file mode 100644 index 0000000000000..98a2269e7e4fa --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/ParquetMergeExecutor.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.merge; + +import org.opensearch.index.engine.dataformat.MergeInput; +import org.opensearch.index.engine.dataformat.MergeResult; +import org.opensearch.index.engine.dataformat.Merger; + +/** + * Executes Parquet merge operations using a pluggable {@link ParquetMergeStrategy}. + */ +public class ParquetMergeExecutor implements Merger { + + private final ParquetMergeStrategy strategy; + + public ParquetMergeExecutor(ParquetMergeStrategy strategy) { + this.strategy = strategy; + } + + @Override + public MergeResult merge(MergeInput mergeInput) { + return strategy.mergeParquetFiles(mergeInput); + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/ParquetMergeStrategy.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/ParquetMergeStrategy.java new file mode 100644 index 0000000000000..fe3c13c61e94d --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/ParquetMergeStrategy.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.merge; + +import org.opensearch.index.engine.dataformat.MergeInput; +import org.opensearch.index.engine.dataformat.MergeResult; + +/** + * Interface defining a Parquet merge strategy. + */ +public interface ParquetMergeStrategy { + + /** + * Performs the actual Parquet merge. + */ + MergeResult mergeParquetFiles(MergeInput mergeInput); + +} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java index 5038bf8feb36c..668d142d6aae1 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java @@ -13,11 +13,14 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.nativebridge.spi.ArrowExport; import org.opensearch.parquet.ParquetDataFormatPlugin; import org.opensearch.parquet.bridge.NativeParquetWriter; import org.opensearch.parquet.bridge.ParquetFileMetadata; +import org.opensearch.parquet.bridge.ParquetSortConfig; import org.opensearch.parquet.fields.ArrowFieldRegistry; import org.opensearch.parquet.fields.ParquetField; import org.opensearch.parquet.memory.ArrowBufferPool; @@ -57,6 +60,7 @@ public class VSRManager implements AutoCloseable { private final AtomicReference managedVSR = new AtomicReference<>(); private final String fileName; + private final IndexSettings indexSettings; private final VSRPool vsrPool; private final ThreadPool threadPool; private final String vsrRotationThread; @@ -66,21 +70,23 @@ public class VSRManager implements AutoCloseable { /** * Creates a new VSRManager with asynchronous background writes (production default). - * - * @param fileName output Parquet file path - * @param schema Arrow schema for vector creation - * @param bufferPool shared Arrow buffer pool - * @param maxRowsPerVSR row threshold triggering VSR rotation - * @param threadPool the thread pool for background native writes */ - public VSRManager(String fileName, Schema schema, ArrowBufferPool bufferPool, int maxRowsPerVSR, ThreadPool threadPool) { - this(fileName, schema, bufferPool, maxRowsPerVSR, threadPool, true); + public VSRManager( + String fileName, + IndexSettings indexSettings, + Schema schema, + ArrowBufferPool bufferPool, + int maxRowsPerVSR, + ThreadPool threadPool + ) { + this(fileName, indexSettings, schema, bufferPool, maxRowsPerVSR, threadPool, true); } /** * Creates a new VSRManager. * * @param fileName output Parquet file path + * @param indexSettings the index settings (sort config is read from here) * @param schema Arrow schema for vector creation * @param bufferPool shared Arrow buffer pool * @param maxRowsPerVSR row threshold triggering VSR rotation @@ -90,6 +96,7 @@ public VSRManager(String fileName, Schema schema, ArrowBufferPool bufferPool, in */ public VSRManager( String fileName, + IndexSettings indexSettings, Schema schema, ArrowBufferPool bufferPool, int maxRowsPerVSR, @@ -97,6 +104,7 @@ public VSRManager( boolean runAsync ) { this.fileName = fileName; + this.indexSettings = indexSettings; this.vsrPool = new VSRPool("pool-" + fileName, schema, bufferPool, maxRowsPerVSR); this.threadPool = threadPool; this.vsrRotationThread = runAsync ? ParquetDataFormatPlugin.PARQUET_THREAD_POOL_NAME : ThreadPool.Names.SAME; @@ -123,7 +131,7 @@ public void addDocument(ParquetDocumentInput doc) throws IOException { parquetField.createField(fieldType, activeVSR, pair.getValue()); } int rowIndex = activeVSR.getRowCount(); - BigIntVector rowIdVector = (BigIntVector) activeVSR.getVector("_row_id"); + BigIntVector rowIdVector = (BigIntVector) activeVSR.getVector(DocumentInput.ROW_ID_FIELD); if (rowIdVector != null) { rowIdVector.setSafe(rowIndex, doc.getRowId()); } @@ -210,9 +218,12 @@ public void close() { } private void initializeWriter() { + ParquetSortConfig sortConfig = new ParquetSortConfig(indexSettings); + String indexName = indexSettings.getIndex().getName(); + ArrowSchema arrowSchema = managedVSR.get().exportSchema(); try { - writer = new NativeParquetWriter(fileName, arrowSchema.memoryAddress()); + writer = new NativeParquetWriter(fileName, indexName, arrowSchema.memoryAddress(), sortConfig); } catch (Exception e) { throw new RuntimeException("Failed to initialize Parquet writer: " + e.getMessage(), e); } finally { diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java index f02c4893a702b..09d02c7340567 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java @@ -9,7 +9,7 @@ package org.opensearch.parquet.writer; import org.apache.arrow.vector.types.pojo.Schema; -import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.dataformat.FileInfos; import org.opensearch.index.engine.dataformat.WriteResult; import org.opensearch.index.engine.dataformat.Writer; @@ -33,7 +33,7 @@ * by the {@link VSRManager}, and flushed to a Parquet file via the native Rust writer. * *

      Writer-level settings (e.g., {@code parquet.max_rows_per_vsr}) are extracted from - * the {@link Settings} passed at construction time and propagated to the VSR layer. + * the {@link IndexSettings} passed at construction time and propagated to the VSR layer. * *

      The returned {@link FileInfos} from {@link #flush()} contains the file path, writer * generation, and row count for downstream commit tracking. @@ -54,7 +54,7 @@ public class ParquetWriter implements Writer { * @param dataFormat the Parquet data format instance * @param schema Arrow schema for vector creation * @param bufferPool shared Arrow buffer pool - * @param settings node settings for writer configuration + * @param indexSettings index settings for writer configuration * @param threadPool the thread pool for background native writes * @param checksumStrategy strategy to register pre-computed checksums on */ @@ -64,15 +64,22 @@ public ParquetWriter( ParquetDataFormat dataFormat, Schema schema, ArrowBufferPool bufferPool, - Settings settings, + IndexSettings indexSettings, ThreadPool threadPool, FormatChecksumStrategy checksumStrategy ) { this.file = file; this.writerGeneration = writerGeneration; this.dataFormat = dataFormat; - this.vsrManager = new VSRManager(file, schema, bufferPool, ParquetSettings.MAX_ROWS_PER_VSR.get(settings), threadPool); this.checksumStrategy = checksumStrategy; + this.vsrManager = new VSRManager( + file, + indexSettings, + schema, + bufferPool, + ParquetSettings.MAX_ROWS_PER_VSR.get(indexSettings.getSettings()), + threadPool + ); } @Override diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/Cargo.toml b/sandbox/plugins/parquet-data-format/src/main/rust/Cargo.toml index 22466d27a3d60..379ec6aea4149 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/Cargo.toml +++ b/sandbox/plugins/parquet-data-format/src/main/rust/Cargo.toml @@ -14,18 +14,16 @@ crate-type = ["rlib"] [dependencies] arrow = { workspace = true } -arrow-array = { workspace = true } -arrow-schema = { workspace = true } -arrow-buffer = { workspace = true } -log = { workspace = true } parquet = { workspace = true } lazy_static = { workspace = true } dashmap = { workspace = true } -chrono = { workspace = true } -mimalloc = { workspace = true } tempfile = { workspace = true } native-bridge-common = { workspace = true } +rayon = { workspace = true } +tokio = { workspace = true } crc32fast = { workspace = true } +serde_json = { workspace = true } [dev-dependencies] opensearch-parquet-format = { path = ".", features = ["test-utils"] } + diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/crc_writer.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/crc_writer.rs new file mode 100644 index 0000000000000..7ae7c436e9477 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/crc_writer.rs @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::io::{Result, Write}; +use std::sync::{Arc, Mutex}; + +/// Shared CRC32 handle that can be cloned and read independently of the writer. +#[derive(Clone)] +pub struct CrcHandle { + hasher: Arc>, +} + +impl CrcHandle { + pub fn crc32(&self) -> u32 { + self.hasher.lock().unwrap().clone().finalize() + } +} + +/// A writer wrapper that computes CRC32 incrementally on every write. +/// The CRC can be read via a `CrcHandle` without consuming the writer. +pub struct CrcWriter { + inner: W, + hasher: Arc>, +} + +impl CrcWriter { + pub fn new(inner: W) -> (Self, CrcHandle) { + let hasher = Arc::new(Mutex::new(crc32fast::Hasher::new())); + let handle = CrcHandle { hasher: hasher.clone() }; + (Self { inner, hasher }, handle) + } +} + +impl Write for CrcWriter { + fn write(&mut self, buf: &[u8]) -> Result { + let n = self.inner.write(buf)?; + self.hasher.lock().unwrap().update(&buf[..n]); + Ok(n) + } + + fn flush(&mut self) -> Result<()> { + self.inner.flush() + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs index f015a49110ec3..233c7aef36923 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs @@ -16,7 +16,9 @@ use std::str; use native_bridge_common::ffm_safe; -use crate::writer::NativeParquetWriter; +use crate::native_settings::NativeSettings; +use crate::merge; +use crate::writer::{NativeParquetWriter, SETTINGS_STORE}; unsafe fn str_from_raw<'a>(ptr: *const u8, len: i64) -> Result<&'a str, String> { if ptr.is_null() { @@ -29,15 +31,70 @@ unsafe fn str_from_raw<'a>(ptr: *const u8, len: i64) -> Result<&'a str, String> str::from_utf8(bytes).map_err(|e| format!("invalid UTF-8: {}", e)) } +/// Decode a parallel (pointers, lengths, count) triple into `Vec`. +unsafe fn str_array_from_raw( + ptrs: *const *const u8, + lens: *const i64, + count: i64, +) -> Result, String> { + if count == 0 { + return Ok(vec![]); + } + if ptrs.is_null() || lens.is_null() { + return Err("null string array pointer".to_string()); + } + let n = count as usize; + let mut out = Vec::with_capacity(n); + for i in 0..n { + let p = *ptrs.add(i); + let l = *lens.add(i); + out.push(str_from_raw(p, l)?.to_string()); + } + Ok(out) +} + +/// Decode a parallel (pointers, count) array of i64 values interpreted as booleans (0 = false). +unsafe fn bool_array_from_raw( + vals: *const i64, + count: i64, +) -> Vec { + if count == 0 || vals.is_null() { + return vec![]; + } + let n = count as usize; + (0..n).map(|i| *vals.add(i) != 0).collect() +} + +// --------------------------------------------------------------------------- +// Writer lifecycle +// --------------------------------------------------------------------------- + #[ffm_safe] #[no_mangle] pub unsafe extern "C" fn parquet_create_writer( file_ptr: *const u8, file_len: i64, + index_name_ptr: *const u8, + index_name_len: i64, schema_address: i64, + sort_ptrs: *const *const u8, + sort_lens: *const i64, + sort_count: i64, + reverse_vals: *const i64, + reverse_count: i64, + nulls_first_vals: *const i64, + nulls_first_count: i64, ) -> i64 { - let filename = str_from_raw(file_ptr, file_len).map_err(|e| format!("parquet_create_writer: {}", e))?.to_string(); - NativeParquetWriter::create_writer(filename, schema_address) + let filename = str_from_raw(file_ptr, file_len) + .map_err(|e| format!("parquet_create_writer file: {}", e))?.to_string(); + let index_name = str_from_raw(index_name_ptr, index_name_len) + .map_err(|e| format!("parquet_create_writer index_name: {}", e))?.to_string(); + let sort_columns = str_array_from_raw(sort_ptrs, sort_lens, sort_count) + .map_err(|e| format!("parquet_create_writer sort_columns: {}", e))?; + let reverse_sorts = bool_array_from_raw(reverse_vals, reverse_count); + let nulls_first = bool_array_from_raw(nulls_first_vals, nulls_first_count); + + NativeParquetWriter::create_writer(filename, index_name, schema_address, sort_columns, reverse_sorts, nulls_first) .map(|_| 0) .map_err(|e| e.to_string()) } @@ -141,3 +198,218 @@ pub unsafe extern "C" fn parquet_get_filtered_native_bytes_used( let prefix = str_from_raw(prefix_ptr, prefix_len).unwrap_or("").to_string(); NativeParquetWriter::get_filtered_writer_memory_usage(prefix).unwrap_or(0) as i64 } + +// --------------------------------------------------------------------------- +// Settings management +// --------------------------------------------------------------------------- + +/// Update native settings for an index. Nullable fields use sentinel -1 for "not set". +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn parquet_on_settings_update( + index_name_ptr: *const u8, + index_name_len: i64, + compression_type_ptr: *const u8, + compression_type_len: i64, + compression_level: i64, + page_size_bytes: i64, + page_row_limit: i64, + dict_size_bytes: i64, + bloom_filter_enabled: i64, + bloom_filter_fpp: f64, + bloom_filter_ndv: i64, + sort_in_memory_threshold_bytes: i64, + sort_batch_size: i64, + row_group_max_rows: i64, + merge_batch_size: i64, + merge_rayon_threads: i64, + merge_io_threads: i64, +) -> i64 { + let index_name = str_from_raw(index_name_ptr, index_name_len) + .map_err(|e| format!("parquet_on_settings_update index_name: {}", e))?.to_string(); + + let compression_type = if compression_type_ptr.is_null() || compression_type_len < 0 { + None + } else { + Some(str_from_raw(compression_type_ptr, compression_type_len) + .map_err(|e| format!("parquet_on_settings_update compression_type: {}", e))?.to_string()) + }; + + fn opt_i32(v: i64) -> Option { if v < 0 { None } else { Some(v as i32) } } + fn opt_usize(v: i64) -> Option { if v < 0 { None } else { Some(v as usize) } } + fn opt_bool(v: i64) -> Option { if v < 0 { None } else { Some(v != 0) } } + fn opt_f64(v: f64) -> Option { if v < 0.0 { None } else { Some(v) } } + fn opt_u64(v: i64) -> Option { if v < 0 { None } else { Some(v as u64) } } + + let config = NativeSettings { + index_name: Some(index_name.clone()), + compression_type, + compression_level: opt_i32(compression_level), + page_size_bytes: opt_usize(page_size_bytes), + page_row_limit: opt_usize(page_row_limit), + dict_size_bytes: opt_usize(dict_size_bytes), + bloom_filter_enabled: opt_bool(bloom_filter_enabled), + bloom_filter_fpp: opt_f64(bloom_filter_fpp), + bloom_filter_ndv: opt_u64(bloom_filter_ndv), + sort_in_memory_threshold_bytes: opt_u64(sort_in_memory_threshold_bytes), + sort_batch_size: opt_usize(sort_batch_size), + row_group_max_rows: opt_usize(row_group_max_rows), + merge_batch_size: opt_usize(merge_batch_size), + merge_rayon_threads: opt_usize(merge_rayon_threads), + merge_io_threads: opt_usize(merge_io_threads), + ..Default::default() + }; + + SETTINGS_STORE.insert(index_name, config); + Ok(0) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn parquet_remove_settings( + index_name_ptr: *const u8, + index_name_len: i64, +) -> i64 { + let index_name = str_from_raw(index_name_ptr, index_name_len) + .map_err(|e| format!("parquet_remove_settings: {}", e))?.to_string(); + SETTINGS_STORE.remove(&index_name); + Ok(0) +} + +// --------------------------------------------------------------------------- +// Merge +// --------------------------------------------------------------------------- + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn parquet_merge_files( + input_ptrs: *const *const u8, + input_lens: *const i64, + input_count: i64, + output_ptr: *const u8, + output_len: i64, + index_name_ptr: *const u8, + index_name_len: i64, +) -> i64 { + let input_files = str_array_from_raw(input_ptrs, input_lens, input_count) + .map_err(|e| format!("parquet_merge_files inputs: {}", e))?; + let output_path = str_from_raw(output_ptr, output_len) + .map_err(|e| format!("parquet_merge_files output: {}", e))?; + let index_name = str_from_raw(index_name_ptr, index_name_len) + .map_err(|e| format!("parquet_merge_files index_name: {}", e))?; + + let (sort_cols, reverse_flags, nulls_first_flags) = match SETTINGS_STORE.get(index_name) { + Some(s) => { + let sc = s.sort_columns.clone(); + let rf = s.reverse_sorts.clone(); + let nf = s.nulls_first.clone(); + if !sc.is_empty() && rf.is_empty() { + crate::log_info!("parquet_merge_files: sort columns present but reverse_sorts is empty for index '{}', defaulting to ascending", index_name); + } + if !sc.is_empty() && nf.is_empty() { + crate::log_info!("parquet_merge_files: sort columns present but nulls_first is empty for index '{}', defaulting to nulls last", index_name); + } + (sc, rf, nf) + } + None => { + crate::log_info!("parquet_merge_files: no settings found for index '{}', proceeding with unsorted merge", index_name); + (vec![], vec![], vec![]) + } + }; + + if sort_cols.is_empty() { + merge::merge_unsorted(&input_files, output_path, index_name) + } else { + merge::merge_sorted( + &input_files, + output_path, + index_name, + &sort_cols, + &reverse_flags, + &nulls_first_flags, + ) + } + .map(|_| 0) + .map_err(|e| format!("{}", e)) +} + +// --------------------------------------------------------------------------- +// Parquet reader (for test verification) +// --------------------------------------------------------------------------- + +/// Reads a parquet file and returns its contents as a JSON string. +/// Each row is a JSON object. The result is a JSON array of objects. +/// The JSON bytes are written into `out_buf`, actual length into `out_len`. +/// Returns 0 on success. +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn parquet_read_as_json( + file_ptr: *const u8, + file_len: i64, + out_buf: *mut u8, + buf_capacity: i64, + out_len: *mut i64, +) -> i64 { + use arrow::array::Array; + + let filename = str_from_raw(file_ptr, file_len) + .map_err(|e| format!("parquet_read_as_json: {}", e))?.to_string(); + + let file = std::fs::File::open(&filename) + .map_err(|e| format!("Failed to open {}: {}", filename, e))?; + let builder = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new(file) + .map_err(|e| format!("Failed to read parquet: {}", e))?; + let reader = builder.with_batch_size(8192).build() + .map_err(|e| format!("Failed to build reader: {}", e))?; + + let mut rows: Vec = Vec::new(); + for batch_result in reader { + let batch = batch_result.map_err(|e| format!("Read error: {}", e))?; + let schema = batch.schema(); + for row_idx in 0..batch.num_rows() { + let mut obj = serde_json::Map::new(); + for (col_idx, field) in schema.fields().iter().enumerate() { + let col = batch.column(col_idx); + let val = if col.is_null(row_idx) { + serde_json::Value::Null + } else { + match col.data_type() { + arrow::datatypes::DataType::Int32 => { + let arr = col.as_any().downcast_ref::().unwrap(); + serde_json::Value::Number(arr.value(row_idx).into()) + } + arrow::datatypes::DataType::Int64 => { + let arr = col.as_any().downcast_ref::().unwrap(); + serde_json::Value::Number(arr.value(row_idx).into()) + } + arrow::datatypes::DataType::Utf8 => { + let arr = col.as_any().downcast_ref::().unwrap(); + serde_json::Value::String(arr.value(row_idx).to_string()) + } + arrow::datatypes::DataType::Boolean => { + let arr = col.as_any().downcast_ref::().unwrap(); + serde_json::Value::Bool(arr.value(row_idx)) + } + arrow::datatypes::DataType::Float64 => { + let arr = col.as_any().downcast_ref::().unwrap(); + serde_json::json!(arr.value(row_idx)) + } + _ => serde_json::Value::String(format!("", col.data_type())), + } + }; + obj.insert(field.name().clone(), val); + } + rows.push(serde_json::Value::Object(obj)); + } + } + + let json_str = serde_json::to_string(&rows) + .map_err(|e| format!("JSON serialization failed: {}", e))?; + let bytes = json_str.as_bytes(); + if bytes.len() > buf_capacity as usize { + return Err(format!("JSON output ({} bytes) exceeds buffer capacity ({})", bytes.len(), buf_capacity)); + } + std::ptr::copy_nonoverlapping(bytes.as_ptr(), out_buf, bytes.len()); + *out_len = bytes.len() as i64; + Ok(0) +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/field_config.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/field_config.rs new file mode 100644 index 0000000000000..a13b904e3f8d3 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/field_config.rs @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +#[derive(Debug, Clone, Default)] +pub struct FieldConfig { + pub compression_type: Option, + pub compression_level: Option, +} + +impl FieldConfig { + pub fn new() -> Self { + Self::default() + } + + pub fn is_empty(&self) -> bool { + self.compression_type.is_none() && self.compression_level.is_none() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_field_config_default() { + let config = FieldConfig::default(); + assert!(config.is_empty()); + } + + #[test] + fn test_field_config_construction() { + let config = FieldConfig { + compression_type: Some("SNAPPY".to_string()), + compression_level: Some(1), + }; + assert_eq!(config.compression_type, Some("SNAPPY".to_string())); + assert_eq!(config.compression_level, Some(1)); + assert!(!config.is_empty()); + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/lib.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/lib.rs index c13fd3e8b5f10..2ce15506f12c4 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/lib.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/lib.rs @@ -14,5 +14,15 @@ mod tests; pub mod writer; pub mod ffm; +pub mod native_settings; +pub mod field_config; +pub mod writer_properties_builder; +pub mod rate_limited_writer; +pub mod crc_writer; +pub mod merge; +pub use native_settings::NativeSettings; +pub use field_config::FieldConfig; +pub use writer_properties_builder::WriterPropertiesBuilder; +pub use writer::SETTINGS_STORE; pub use native_bridge_common::{log_info, log_error, log_debug}; diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/context.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/context.rs new file mode 100644 index 0000000000000..e2a07c2efffeb --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/context.rs @@ -0,0 +1,257 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::fs::File; +use std::path::Path; +use std::sync::Arc; + +use arrow::array::RecordBatch; +use arrow::compute::concat_batches; +use arrow::datatypes::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; +use parquet::arrow::arrow_writer::{ArrowRowGroupWriterFactory, compute_leaves}; +use parquet::file::writer::SerializedFileWriter; +use parquet::schema::types::SchemaDescriptor; +use rayon::prelude::*; +use tokio::sync::{mpsc as tokio_mpsc, oneshot}; + +use crate::crc_writer::CrcWriter; +use crate::rate_limited_writer::RateLimitedWriter; +use crate::writer_properties_builder::WriterPropertiesBuilder; +use crate::{log_debug, SETTINGS_STORE}; + +use super::error::{MergeError, MergeResult}; +use super::io_task::{ + get_merge_pool, spawn_io_task, IoCommand, RATE_LIMIT_MB_PER_SEC, +}; +use super::schema::{append_row_id, build_parquet_root_schema, ROW_ID_COLUMN_NAME}; + +/// Owns all shared state for a merge operation: schemas, writer factory, +/// IO channel, buffered batches, and counters. Used by both sorted and +/// unsorted merge paths. +pub struct MergeContext { + data_schema: Arc, + output_schema: Arc, + rg_writer_factory: ArrowRowGroupWriterFactory, + io_tx: tokio_mpsc::Sender, + output_chunks: Vec, + output_row_count: usize, + output_flush_rows: usize, + row_group_index: usize, + next_row_id: i64, + total_rows_written: usize, + rayon_threads: Option, +} + +impl MergeContext { + /// Creates a new merge context: builds union schemas, opens the output + /// writer, and spawns the background IO task. + pub fn new( + arrow_schemas: Vec, + parquet_descriptors: &[SchemaDescriptor], + output_path: &str, + index_name: &str, + output_flush_rows: usize, + rayon_threads: Option, + io_threads: Option, + ) -> MergeResult { + if let Some(parent) = Path::new(output_path).parent() { + if !parent.exists() { + return Err(MergeError::Logic(format!( + "Output directory '{}' does not exist.", + parent.display() + ))); + } + } + + let union_data_schema = ArrowSchema::try_merge(arrow_schemas).map_err(|e| { + MergeError::Logic(format!( + "Failed to compute union schema across input files: {}", + e + )) + })?; + let data_schema = Arc::new(union_data_schema); + + let mut output_fields: Vec = data_schema + .fields() + .iter() + .map(|f| f.as_ref().clone()) + .collect(); + output_fields.push(ArrowField::new( + ROW_ID_COLUMN_NAME, + ArrowDataType::Int64, + false, + )); + let output_schema = Arc::new(ArrowSchema::new(output_fields)); + + let parquet_root = build_parquet_root_schema(parquet_descriptors)?; + + let output_file = File::create(output_path)?; + let throttled_writer = + RateLimitedWriter::new(output_file, RATE_LIMIT_MB_PER_SEC).map_err(MergeError::Io)?; + + let (crc_writer, crc_handle) = CrcWriter::new(throttled_writer); + + let config = SETTINGS_STORE + .get(index_name) + .map(|r| r.clone()) + .unwrap_or_default(); + let writer_props = Arc::new(WriterPropertiesBuilder::build(&config)); + + let writer = SerializedFileWriter::new(crc_writer, parquet_root, writer_props)?; + let rg_writer_factory = ArrowRowGroupWriterFactory::new(&writer, output_schema.clone()); + let io_tx = spawn_io_task(writer, crc_handle, io_threads); + + Ok(Self { + data_schema, + output_schema, + rg_writer_factory, + io_tx, + output_chunks: Vec::new(), + output_row_count: 0, + output_flush_rows, + row_group_index: 0, + next_row_id: 0, + total_rows_written: 0, + rayon_threads, + }) + } + + pub fn data_schema(&self) -> &Arc { + &self.data_schema + } + + /// Buffers a batch (already padded to data_schema) and auto-flushes when + /// the row count threshold is reached. + pub fn push_batch(&mut self, batch: RecordBatch) -> MergeResult<()> { + self.output_row_count += batch.num_rows(); + self.output_chunks.push(batch); + if self.output_row_count >= self.output_flush_rows { + self.flush()?; + } + Ok(()) + } + + /// Concat buffered batches, append row IDs, encode columns in parallel, + /// and send the encoded row group to the IO task. + pub fn flush(&mut self) -> MergeResult<()> { + if self.output_chunks.is_empty() { + return Ok(()); + } + + let merged = if self.output_chunks.len() == 1 { + self.output_chunks.pop().unwrap() + } else { + let m = concat_batches(&self.data_schema, self.output_chunks.as_slice())?; + self.output_chunks.clear(); + m + }; + let n = merged.num_rows(); + + let with_id = append_row_id(&merged, self.next_row_id, &self.output_schema)?; + drop(merged); + + let col_writers = self + .rg_writer_factory + .create_column_writers(self.row_group_index)?; + + let leaves_and_writers = match Self::pair_leaves_with_writers(&with_id, &self.output_schema, col_writers) { + Ok(paired) => paired, + Err((err, remaining)) => { + for w in remaining { + let _ = w.close(); + } + return Err(err); + } + }; + + let chunk_results: Vec< + Result, + > = get_merge_pool(self.rayon_threads).install(|| { + leaves_and_writers + .into_par_iter() + .map(|(leaf, mut col_writer)| { + col_writer.write(&leaf)?; + col_writer.close() + }) + .collect() + }); + + let mut encoded_chunks = Vec::with_capacity(chunk_results.len()); + for r in chunk_results { + encoded_chunks.push(r?); + } + + self.io_tx + .blocking_send(IoCommand::WriteRowGroup(encoded_chunks)) + .map_err(|_| MergeError::Logic("IO task terminated unexpectedly".into()))?; + + self.row_group_index += 1; + self.next_row_id += n as i64; + self.total_rows_written += n; + self.output_row_count = 0; + + log_debug!( + "[RUST] Flushed row group {}: {} rows (total: {})", + self.row_group_index - 1, + n, + self.total_rows_written + ); + + Ok(()) + } + + /// Pairs leaf arrays with column writers, returning unconsumed writers on error + /// so the caller can close them. + fn pair_leaves_with_writers( + batch: &RecordBatch, + schema: &Arc, + col_writers: Vec, + ) -> Result< + Vec<(parquet::arrow::arrow_writer::ArrowLeafColumn, parquet::arrow::arrow_writer::ArrowColumnWriter)>, + (MergeError, Vec), + > { + let mut writer_iter = col_writers.into_iter(); + let mut paired = Vec::new(); + for (arr, field) in batch.columns().iter().zip(schema.fields()) { + let leaves = match compute_leaves(field, arr) { + Ok(l) => l, + Err(e) => return Err((e.into(), writer_iter.collect())), + }; + for leaf in leaves { + match writer_iter.next() { + Some(w) => paired.push((leaf, w)), + None => { + return Err(( + MergeError::Logic("Fewer column writers than leaf columns".into()), + Vec::new(), + )) + } + } + } + } + Ok(paired) + } + + /// Final flush + close the IO task. Returns Parquet metadata and CRC32. + pub fn finish(mut self) -> MergeResult<(parquet::file::metadata::ParquetMetaData, u32)> { + self.flush()?; + + let (reply_tx, reply_rx) = + oneshot::channel::>(); + + self.io_tx + .blocking_send(IoCommand::Close(reply_tx)) + .map_err(|_| MergeError::Logic("IO task terminated before close".into()))?; + + drop(self.io_tx); + + reply_rx + .blocking_recv() + .map_err(|_| MergeError::Logic("IO task terminated during close".into()))? + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/cursor.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/cursor.rs new file mode 100644 index 0000000000000..b530820ef3c98 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/cursor.rs @@ -0,0 +1,225 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::fs::File; +use std::sync::{Arc, Mutex}; + +use arrow::array::RecordBatch; +use arrow::datatypes::{DataType as ArrowDataType, Schema as ArrowSchema}; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use parquet::schema::types::SchemaDescriptor; + +use super::error::{MergeError, MergeResult}; +use super::heap::{get_sort_values, SortKey}; +use super::io_task::get_merge_pool; +use super::schema::projection_indices_excluding_row_id; + +/// A cursor over a single sorted Parquet input file. +/// +/// Each cursor reads batches sequentially and prefetches the next batch on the +/// shared Rayon pool to overlap IO with merge computation. +pub struct FileCursor { + reader: Arc>, + prefetch_rx: std::sync::mpsc::Receiver>>, + prefetch_tx: std::sync::mpsc::SyncSender>>, + prefetch_pending: bool, + pub current_batch: Option, + pub row_idx: usize, + pub file_id: usize, + pub sort_col_indices: Vec, + pub sort_col_types: Vec, + pub nulls_first: Vec, +} + +impl FileCursor { + /// Opens a Parquet file and creates a cursor positioned at the first row. + /// + /// Returns `(cursor, projected_arrow_schema, parquet_schema_descriptor)` + /// so the caller can build union schemas without re-opening the file. + pub fn new( + path: &str, + file_id: usize, + sort_columns: &[String], + nulls_first: &[bool], + batch_size: usize, + ) -> MergeResult<(Self, Arc, SchemaDescriptor)> { + let file = File::open(path)?; + let builder = ParquetRecordBatchReaderBuilder::try_new(file)?; + let schema = builder.schema().clone(); + + let mut sort_col_types = Vec::with_capacity(sort_columns.len()); + for col_name in sort_columns { + let dt = schema + .fields() + .iter() + .find(|f| f.name() == col_name.as_str()) + .map(|f| f.data_type().clone()) + .ok_or_else(|| { + MergeError::Logic(format!( + "Sort column '{}' not found in file '{}' (cursor {})", + col_name, path, file_id + )) + })?; + sort_col_types.push(dt); + } + + let parquet_schema_descr = builder.parquet_schema().clone(); + let projection_indices = projection_indices_excluding_row_id(&schema); + + let projection = + parquet::arrow::ProjectionMask::roots(&parquet_schema_descr, projection_indices); + + let mut reader = builder + .with_batch_size(batch_size) + .with_projection(projection) + .build()?; + + let first_batch = match reader.next() { + Some(Ok(b)) if b.num_rows() > 0 => b, + Some(Err(e)) => return Err(e.into()), + _ => { + return Err(MergeError::Logic(format!( + "File '{}' (cursor {}) yielded no rows despite passing validation", + path, file_id + ))); + } + }; + + let projected_schema = first_batch.schema(); + + let mut sort_col_indices = Vec::with_capacity(sort_columns.len()); + for col_name in sort_columns { + let idx = projected_schema + .fields() + .iter() + .position(|f| f.name() == col_name.as_str()) + .ok_or_else(|| { + MergeError::Logic(format!( + "Sort column '{}' not found after projection in file '{}'", + col_name, path + )) + })?; + sort_col_indices.push(idx); + } + + let (prefetch_tx, prefetch_rx) = + std::sync::mpsc::sync_channel::>>(1); + + let reader = Arc::new(Mutex::new(reader)); + + let mut cursor = Self { + reader, + prefetch_rx, + prefetch_tx, + prefetch_pending: false, + current_batch: Some(first_batch), + row_idx: 0, + file_id, + sort_col_indices, + sort_col_types, + nulls_first: nulls_first.to_vec(), + }; + + cursor.start_prefetch(); + + Ok((cursor, projected_schema, parquet_schema_descr)) + } + + fn start_prefetch(&mut self) { + if self.prefetch_pending { + return; + } + self.prefetch_pending = true; + + let reader = Arc::clone(&self.reader); + let tx = self.prefetch_tx.clone(); + + get_merge_pool(None).spawn(move || { + let mut reader = reader.lock().unwrap(); + let result = match reader.next() { + Some(Ok(batch)) if batch.num_rows() > 0 => Some(Ok(batch)), + Some(Err(e)) => Some(Err(MergeError::Arrow(e))), + _ => None, + }; + let _ = tx.send(result); + }); + } + + pub fn load_next_batch(&mut self) -> MergeResult { + self.current_batch = None; + + match self.prefetch_rx.recv() { + Ok(Some(Ok(batch))) => { + self.current_batch = Some(batch); + self.row_idx = 0; + self.prefetch_pending = false; + self.start_prefetch(); + Ok(true) + } + Ok(Some(Err(e))) => { + self.prefetch_pending = false; + Err(e) + } + Ok(None) | Err(_) => { + self.prefetch_pending = false; + Ok(false) + } + } + } + + #[inline] + pub fn current_sort_values(&self) -> MergeResult> { + let batch = self + .current_batch + .as_ref() + .ok_or_else(|| MergeError::Logic("Cursor exhausted".into()))?; + get_sort_values(batch, self.row_idx, &self.sort_col_indices, &self.sort_col_types, &self.nulls_first) + } + + #[inline] + pub fn last_sort_values(&self) -> MergeResult> { + let batch = self + .current_batch + .as_ref() + .ok_or_else(|| MergeError::Logic("Cursor exhausted".into()))?; + get_sort_values( + batch, + batch.num_rows() - 1, + &self.sort_col_indices, + &self.sort_col_types, + &self.nulls_first, + ) + } + + #[inline] + pub fn batch_height(&self) -> usize { + self.current_batch.as_ref().map_or(0, |b| b.num_rows()) + } + + #[inline] + pub fn take_slice(&self, start: usize, len: usize) -> RecordBatch { + self.current_batch.as_ref().unwrap().slice(start, len) + } + + pub fn advance(&mut self) -> MergeResult { + if self.current_batch.is_none() { + return Ok(false); + } + self.row_idx += 1; + if self.row_idx >= self.current_batch.as_ref().unwrap().num_rows() { + self.current_batch = None; + return self.load_next_batch(); + } + Ok(true) + } + + pub fn advance_past_batch(&mut self) -> MergeResult { + self.current_batch = None; + self.load_next_batch() + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/error.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/error.rs new file mode 100644 index 0000000000000..3913604276a41 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/error.rs @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::error::Error; + +/// Result type alias for merge operations. +pub type MergeResult = Result; + +/// Unified error type for all merge failures. +#[derive(Debug)] +pub enum MergeError { + /// Error from the Arrow compute or array layer. + Arrow(arrow::error::ArrowError), + /// Error from the Parquet reader or writer. + Parquet(parquet::errors::ParquetError), + /// Filesystem or network IO error. + Io(std::io::Error), + /// Logic or invariant violation within the merge algorithm. + Logic(String), +} + +impl std::fmt::Display for MergeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MergeError::Arrow(e) => write!(f, "Arrow error: {e}"), + MergeError::Parquet(e) => write!(f, "Parquet error: {e}"), + MergeError::Io(e) => write!(f, "IO error: {e}"), + MergeError::Logic(s) => write!(f, "{s}"), + } + } +} + +impl Error for MergeError {} + +impl From for MergeError { + fn from(e: arrow::error::ArrowError) -> Self { + MergeError::Arrow(e) + } +} + +impl From for MergeError { + fn from(e: parquet::errors::ParquetError) -> Self { + MergeError::Parquet(e) + } +} + +impl From for MergeError { + fn from(e: std::io::Error) -> Self { + MergeError::Io(e) + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/heap.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/heap.rs new file mode 100644 index 0000000000000..45062c27c59aa --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/heap.rs @@ -0,0 +1,189 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::cmp::Ordering; +use std::sync::Arc; + +use arrow::array::{AsArray, RecordBatch}; +use arrow::datatypes::{ + DataType as ArrowDataType, Date32Type, Date64Type, DurationMicrosecondType, + DurationMillisecondType, DurationNanosecondType, DurationSecondType, Float32Type, Float64Type, + Int16Type, Int32Type, Int64Type, Int8Type, TimestampMicrosecondType, + TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt32Type, +}; + +use super::error::{MergeError, MergeResult}; + +// ============================================================================= +// SortKey — typed sort value with null ordering baked in +// ============================================================================= + +#[derive(Debug, Clone)] +pub enum SortKey { + NullFirst, + NullLast, + Int(i64), + Float(f64), + Bytes(Vec), +} + +impl Eq for SortKey {} + +impl PartialEq for SortKey { + fn eq(&self, other: &Self) -> bool { + self.cmp(other) == Ordering::Equal + } +} + +impl Ord for SortKey { + fn cmp(&self, other: &Self) -> Ordering { + match (self, other) { + (SortKey::NullFirst, SortKey::NullFirst) => Ordering::Equal, + (SortKey::NullFirst, _) => Ordering::Less, + (_, SortKey::NullFirst) => Ordering::Greater, + (SortKey::NullLast, SortKey::NullLast) => Ordering::Equal, + (SortKey::NullLast, _) => Ordering::Greater, + (_, SortKey::NullLast) => Ordering::Less, + (SortKey::Int(a), SortKey::Int(b)) => a.cmp(b), + (SortKey::Float(a), SortKey::Float(b)) => a.total_cmp(b), + (SortKey::Bytes(a), SortKey::Bytes(b)) => a.cmp(b), + // Same column always produces the same variant; cross-variant is unreachable. + _ => Ordering::Equal, + } + } +} + +impl PartialOrd for SortKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +// ============================================================================= +// Sort-direction helpers +// ============================================================================= + +/// Lexicographic comparison of two sort-key tuples, respecting per-column +/// sort direction. Returns `Ordering::Equal` when all values match. +#[inline(always)] +pub fn cmp_sort_values(a: &[SortKey], b: &[SortKey], reverse_sorts: &[bool]) -> Ordering { + for (i, (av, bv)) in a.iter().zip(b.iter()).enumerate() { + let ord = av.cmp(bv); + if ord != Ordering::Equal { + let reverse = reverse_sorts.get(i).copied().unwrap_or(false); + let is_null_cmp = matches!(av, SortKey::NullFirst | SortKey::NullLast) + || matches!(bv, SortKey::NullFirst | SortKey::NullLast); + return if reverse && !is_null_cmp { ord.reverse() } else { ord }; + } + } + Ordering::Equal +} + +// ============================================================================= +// HeapItem for k-way merge +// ============================================================================= + +#[derive(Debug)] +pub struct HeapItem { + pub sort_values: Vec, + pub file_id: usize, + pub reverse_sorts: Arc>, +} + +impl Eq for HeapItem {} + +impl PartialEq for HeapItem { + fn eq(&self, other: &Self) -> bool { + self.sort_values == other.sort_values + } +} + +impl Ord for HeapItem { + fn cmp(&self, other: &Self) -> Ordering { + // Swap other/self so max-heap behaves as min-heap. + cmp_sort_values(&other.sort_values, &self.sort_values, &self.reverse_sorts) + } +} + +impl PartialOrd for HeapItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +// ============================================================================= +// Sort value extraction +// ============================================================================= + +#[inline] +pub fn get_sort_value( + batch: &RecordBatch, + row: usize, + col_idx: usize, + dtype: &ArrowDataType, + null_first: bool, +) -> MergeResult { + let col = batch.column(col_idx); + if col.is_null(row) { + return Ok(if null_first { SortKey::NullFirst } else { SortKey::NullLast }); + } + let key = match dtype { + // Integer types → SortKey::Int + ArrowDataType::Int64 => SortKey::Int(col.as_primitive::().value(row)), + ArrowDataType::Int32 => SortKey::Int(col.as_primitive::().value(row) as i64), + ArrowDataType::Int16 => SortKey::Int(col.as_primitive::().value(row) as i64), + ArrowDataType::Int8 => SortKey::Int(col.as_primitive::().value(row) as i64), + ArrowDataType::UInt32 => SortKey::Int(col.as_primitive::().value(row) as i64), + ArrowDataType::Date32 => SortKey::Int(col.as_primitive::().value(row) as i64), + ArrowDataType::Date64 => SortKey::Int(col.as_primitive::().value(row)), + ArrowDataType::Timestamp(unit, _) => SortKey::Int(match unit { + arrow::datatypes::TimeUnit::Second => col.as_primitive::().value(row), + arrow::datatypes::TimeUnit::Millisecond => col.as_primitive::().value(row), + arrow::datatypes::TimeUnit::Microsecond => col.as_primitive::().value(row), + arrow::datatypes::TimeUnit::Nanosecond => col.as_primitive::().value(row), + }), + ArrowDataType::Duration(unit) => SortKey::Int(match unit { + arrow::datatypes::TimeUnit::Second => col.as_primitive::().value(row), + arrow::datatypes::TimeUnit::Millisecond => col.as_primitive::().value(row), + arrow::datatypes::TimeUnit::Microsecond => col.as_primitive::().value(row), + arrow::datatypes::TimeUnit::Nanosecond => col.as_primitive::().value(row), + }), + + // Float types → SortKey::Float + ArrowDataType::Float64 => SortKey::Float(col.as_primitive::().value(row)), + ArrowDataType::Float32 => SortKey::Float(col.as_primitive::().value(row) as f64), + + // String types → SortKey::Bytes + ArrowDataType::Utf8 => SortKey::Bytes(col.as_string::().value(row).as_bytes().to_vec()), + ArrowDataType::LargeUtf8 => SortKey::Bytes(col.as_string::().value(row).as_bytes().to_vec()), + + other => { + return Err(MergeError::Logic(format!( + "Unsupported sort column type: {:?}", + other + ))); + } + }; + Ok(key) +} + +#[inline] +pub fn get_sort_values( + batch: &RecordBatch, + row: usize, + col_indices: &[usize], + dtypes: &[ArrowDataType], + nulls_first: &[bool], +) -> MergeResult> { + let mut values = Vec::with_capacity(col_indices.len()); + for (i, (col_idx, dtype)) in col_indices.iter().zip(dtypes.iter()).enumerate() { + let nf = nulls_first.get(i).copied().unwrap_or(false); + values.push(get_sort_value(batch, row, *col_idx, dtype, nf)?); + } + Ok(values) +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/io_task.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/io_task.rs new file mode 100644 index 0000000000000..1cbd4f41aa4ca --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/io_task.rs @@ -0,0 +1,192 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::fs::File; +use std::sync::OnceLock; + +use parquet::file::metadata::ParquetMetaData; +use parquet::file::writer::SerializedFileWriter; + +use rayon::ThreadPool; + +use tokio::runtime::Runtime; +use tokio::sync::{mpsc as tokio_mpsc, oneshot}; +use tokio::task::JoinHandle; + +use crate::crc_writer::CrcWriter; +use crate::rate_limited_writer::RateLimitedWriter; +use crate::log_error; + +use super::error::{MergeError, MergeResult}; +// ============================================================================= +// Constants +// ============================================================================= + +/// Disk write rate limit in MB/s. +pub const RATE_LIMIT_MB_PER_SEC: f64 = 20.0; + +/// Default thread count for merge pools: max(1, num_cpus / 8). +fn default_merge_threads() -> usize { + std::thread::available_parallelism() + .map(|n| n.get() / 8) + .unwrap_or(1) + .max(1) +} + +/// Bounded channel capacity between the merge loop and the IO task. +const IO_CHANNEL_BUFFER: usize = 2; + +// ============================================================================= +// Process-wide shared Rayon thread pool +// ============================================================================= + +static MERGE_POOL: OnceLock = OnceLock::new(); + +pub fn get_merge_pool(num_threads: Option) -> &'static ThreadPool { + MERGE_POOL.get_or_init(|| { + let n = num_threads.unwrap_or_else(default_merge_threads); + rayon::ThreadPoolBuilder::new() + .num_threads(n) + .thread_name(|idx| format!("parquet-merge-{}", idx)) + .build() + .expect("Failed to build parquet-merge Rayon thread pool") + }) +} + +// ============================================================================= +// Process-wide shared Tokio runtime for async IO +// ============================================================================= + +static IO_RUNTIME: OnceLock = OnceLock::new(); + +fn get_io_runtime(num_threads: Option) -> &'static Runtime { + IO_RUNTIME.get_or_init(|| { + let n = num_threads.unwrap_or_else(default_merge_threads); + tokio::runtime::Builder::new_multi_thread() + .worker_threads(n) + .thread_name("parquet-io") + .enable_all() + .build() + .expect("Failed to build tokio IO runtime") + }) +} + +// ============================================================================= +// IO task protocol +// ============================================================================= + +/// Writer type used by the IO task: CRC → rate-limit → file. +pub type MergeWriter = CrcWriter>; + +/// Commands sent from the merge loop to the background IO task. +pub enum IoCommand { + WriteRowGroup(Vec), + Close(oneshot::Sender>), +} + +async fn drain_on_error(rx: &mut tokio_mpsc::Receiver, msg: &str) { + while let Some(cmd) = rx.recv().await { + if let IoCommand::Close(reply) = cmd { + let _ = reply.send(Err(MergeError::Logic( + format!("Prior IO write failed: {msg}"), + ))); + } + } +} + +/// Spawns the background IO task on the shared Tokio runtime. +/// +/// The IO task owns the `SerializedFileWriter` and receives encoded row groups +/// over a bounded channel. Each disk write is dispatched to `spawn_blocking` +/// but is **not** awaited immediately — this allows the merge loop to prepare +/// the next row group while the current one is still being flushed to disk. +pub fn spawn_io_task( + writer: SerializedFileWriter, + crc_handle: crate::crc_writer::CrcHandle, + io_threads: Option, +) -> tokio_mpsc::Sender { + let (tx, mut rx) = tokio_mpsc::channel::(IO_CHANNEL_BUFFER); + + get_io_runtime(io_threads).spawn(async move { + let mut writer: Option> = Some(writer); + let mut in_flight: Option< + JoinHandle>>, + > = None; + + while let Some(cmd) = rx.recv().await { + match cmd { + IoCommand::WriteRowGroup(chunks) => { + if let Some(handle) = in_flight.take() { + match handle.await { + Ok(Ok(w)) => writer = Some(w), + Ok(Err(e)) => { + let msg = format!("{e}"); + log_error!("[RUST] IO write error during merge: {}", e); + drain_on_error(&mut rx, &msg).await; + return; + } + Err(e) => { + let msg = format!("{e}"); + log_error!("[RUST] IO spawn_blocking panicked during merge: {}", e); + drain_on_error(&mut rx, &msg).await; + return; + } + } + } + + let w = writer.take().unwrap(); + in_flight = Some(tokio::task::spawn_blocking(move || { + let mut w = w; + let mut rg_writer = w.next_row_group()?; + for chunk in chunks { + chunk.append_to_row_group(&mut rg_writer)?; + } + rg_writer.close()?; + Ok(w) + })); + } + + IoCommand::Close(reply) => { + if let Some(handle) = in_flight.take() { + match handle.await { + Ok(Ok(w)) => writer = Some(w), + Ok(Err(e)) => { + let _ = reply.send(Err(e)); + return; + } + Err(e) => { + let _ = reply.send(Err(MergeError::Logic( + format!("IO panic during final write: {e}"), + ))); + return; + } + } + } + + let w = writer.take().unwrap(); + let crc = crc_handle.clone(); + let result = tokio::task::spawn_blocking(move || { + let metadata = w.close().map_err(MergeError::from)?; + Ok((metadata, crc.crc32())) + }) + .await; + + let _ = match result { + Ok(r) => reply.send(r), + Err(e) => reply.send(Err(MergeError::Logic( + format!("Close panicked: {e}"), + ))), + }; + return; + } + } + } + }); + + tx +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/mod.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/mod.rs new file mode 100644 index 0000000000000..ee76348c99b7b --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/mod.rs @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +mod context; +mod cursor; +pub mod error; +pub mod heap; +pub mod io_task; +pub mod schema; +mod sorted; +mod unsorted; + +pub use error::{MergeError, MergeResult}; +pub use sorted::merge_sorted; +pub use unsorted::merge_unsorted; diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/schema.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/schema.rs new file mode 100644 index 0000000000000..9cfbb10fd8c7d --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/schema.rs @@ -0,0 +1,142 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::collections::HashSet; +use std::sync::Arc; + +use arrow::array::{ArrayRef, Int64Array, RecordBatch}; +use arrow::datatypes::Schema as ArrowSchema; +use parquet::basic::Repetition; +use parquet::schema::types::Type; + +use super::error::MergeResult; + +/// Reserved column name for the synthetic row identifier added during merge. +pub const ROW_ID_COLUMN_NAME: &str = "__row_id__"; + +/// Builds the output Parquet schema as the union of pre-read schema descriptors. +/// +/// The output schema contains every column seen across all inputs, except: +/// - Any existing `__row_id__` column is removed. +/// - A fresh `__row_id__` INT64 REQUIRED column is appended at the end. +pub fn build_parquet_root_schema( + schema_descriptors: &[parquet::schema::types::SchemaDescriptor], +) -> MergeResult> { + let mut seen_names: HashSet = HashSet::new(); + let mut parquet_fields: Vec> = Vec::new(); + + for descr in schema_descriptors { + let root = descr.root_schema(); + for field in root.get_fields() { + if field.name() != ROW_ID_COLUMN_NAME + && seen_names.insert(field.name().to_string()) + { + parquet_fields.push(Arc::new(field.as_ref().clone())); + } + } + } + + let row_id_type = + Type::primitive_type_builder(ROW_ID_COLUMN_NAME, parquet::basic::Type::INT64) + .with_repetition(Repetition::REQUIRED) + .build()?; + parquet_fields.push(Arc::new(row_id_type)); + + let parquet_root = Type::group_type_builder("schema") + .with_fields(parquet_fields) + .build()?; + + Ok(Arc::new(parquet_root)) +} + +/// Returns column indices that exclude `__row_id__`, for use as a projection mask. +pub fn projection_indices_excluding_row_id(schema: &ArrowSchema) -> Vec { + schema + .fields() + .iter() + .enumerate() + .filter(|(_, f)| f.name() != ROW_ID_COLUMN_NAME) + .map(|(i, _)| i) + .collect() +} + + +/// Appends a `__row_id__` column with sequential values `[start_id, start_id + N)` +/// to the given batch, producing a new batch with the output schema. +pub fn append_row_id( + batch: &RecordBatch, + start_id: i64, + output_schema: &Arc, +) -> MergeResult { + let n = batch.num_rows() as i64; + let row_ids = Int64Array::from_iter_values(start_id..start_id + n); + let mut columns: Vec = batch.columns().to_vec(); + columns.push(Arc::new(row_ids)); + let result = RecordBatch::try_new(output_schema.clone(), columns)?; + Ok(result) +} + +// ============================================================================= +// ColumnMapping — precomputed source→target index mapping +// ============================================================================= + +/// Precomputed mapping from target schema field positions to source batch +/// column indices. Built once per cursor, reused for every batch from that cursor. +/// +/// Replaces per-batch `schema.index_of(field.name())` name lookups with O(1) +/// indexed access. +pub struct ColumnMapping { + mapping: Vec>, + target_schema: Arc, + is_identity: bool, +} + +impl ColumnMapping { + /// Build a mapping from `source_schema` → `target_schema`. + pub fn new(source_schema: &ArrowSchema, target_schema: &Arc) -> Self { + let mut mapping = Vec::with_capacity(target_schema.fields().len()); + let mut is_identity = source_schema.fields().len() == target_schema.fields().len(); + + for (target_idx, field) in target_schema.fields().iter().enumerate() { + match source_schema.index_of(field.name()) { + Ok(src_idx) => { + if is_identity && src_idx != target_idx { + is_identity = false; + } + mapping.push(Some(src_idx)); + } + Err(_) => { + is_identity = false; + mapping.push(None); + } + } + } + + Self { mapping, target_schema: target_schema.clone(), is_identity } + } + + /// Remap a batch using the precomputed mapping. Zero-copy when schemas match. + #[inline] + pub fn pad_batch(&self, batch: &RecordBatch) -> MergeResult { + if self.is_identity { + return Ok(batch.clone()); + } + let num_rows = batch.num_rows(); + let mut columns: Vec = Vec::with_capacity(self.mapping.len()); + for (i, entry) in self.mapping.iter().enumerate() { + match entry { + Some(src_idx) => columns.push(batch.column(*src_idx).clone()), + None => { + let field = &self.target_schema.fields()[i]; + columns.push(arrow::array::new_null_array(field.data_type(), num_rows)); + } + } + } + Ok(RecordBatch::try_new(self.target_schema.clone(), columns)?) + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs new file mode 100644 index 0000000000000..23f9121cb35c3 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs @@ -0,0 +1,225 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::cmp::Ordering; +use std::collections::BinaryHeap; +use std::sync::Arc; + +use arrow::datatypes::Schema as ArrowSchema; +use parquet::schema::types::SchemaDescriptor; + +use crate::{log_debug, log_info}; + +use super::context::MergeContext; +use super::cursor::FileCursor; +use super::heap::{cmp_sort_values, get_sort_values, HeapItem}; +use super::io_task::get_merge_pool; +use super::schema::ColumnMapping; + +/// Performs a streaming k-way merge with an explicit sort direction per column. +pub fn merge_sorted( + input_files: &[String], + output_path: &str, + index_name: &str, + sort_columns: &[String], + reverse_sorts: &[bool], + nulls_first: &[bool], +) -> super::MergeResult { + let config = crate::writer::SETTINGS_STORE + .get(index_name) + .map(|r| r.clone()) + .unwrap_or_default(); + let batch_size = config.get_merge_batch_size(); + let output_flush_rows = config.get_row_group_max_rows(); + let rayon_threads = config.get_merge_rayon_threads(); + let io_threads = config.get_merge_io_threads(); + if input_files.is_empty() { + return Ok(0); + } + + if sort_columns.is_empty() { + return Err(super::MergeError::Logic( + "merge_sorted called with empty sort_columns; use merge_unsorted instead".into(), + )); + } + + let pool = get_merge_pool(rayon_threads); + let direction_label = if reverse_sorts.iter().all(|&r| !r) { + "ascending" + } else if reverse_sorts.iter().all(|&r| r) { + "descending" + } else { + "mixed" + }; + + log_debug!( + "[RUST] Starting streaming merge ({}): {} input files, sort_columns={:?}, \ + batch_size={}, flush_rows={}, merge_threads={}, output='{}'", + direction_label, + input_files.len(), + sort_columns, + batch_size, + output_flush_rows, + pool.current_num_threads(), + output_path + ); + + // ── Phase 1: Initialize cursors and collect schemas ───────────────── + let mut cursors: Vec = Vec::with_capacity(input_files.len()); + let mut arrow_schemas: Vec = Vec::with_capacity(input_files.len()); + let mut parquet_descriptors: Vec = Vec::with_capacity(input_files.len()); + + for (file_id, path) in input_files.iter().enumerate() { + log_debug!("[RUST] Opening cursor {} for file: {}", file_id, path); + let (cursor, projected_schema, parquet_descr) = + FileCursor::new(path, file_id, sort_columns, nulls_first, batch_size)?; + cursors.push(cursor); + arrow_schemas.push(projected_schema.as_ref().clone()); + parquet_descriptors.push(parquet_descr); + } + + let num_cursors = cursors.len(); + + // ── Phase 2: Create MergeContext (union schemas, writer, IO task) ─── + let mut ctx = MergeContext::new( + arrow_schemas.clone(), + &parquet_descriptors, + output_path, + index_name, + output_flush_rows, + rayon_threads, + io_threads, + )?; + + // Precompute column mappings per cursor (avoids per-batch name lookups) + let col_mappings: Vec = arrow_schemas.iter() + .map(|s| ColumnMapping::new(s, ctx.data_schema())) + .collect(); + + log_debug!( + "[RUST] Merge initialized ({}): {} cursors", + direction_label, + num_cursors + ); + + // ── Phase 3: Seed the heap ────────────────────────────────────────── + let reverse_sorts_arc = Arc::new(reverse_sorts.to_vec()); + let mut heap: BinaryHeap = BinaryHeap::with_capacity(num_cursors); + for cursor in &cursors { + let sv = cursor.current_sort_values()?; + heap.push(HeapItem { + sort_values: sv, + file_id: cursor.file_id, + reverse_sorts: Arc::clone(&reverse_sorts_arc), + }); + } + + // ── Phase 4: K-way merge loop — three-tier cascade ────────────────── + while let Some(item) = heap.pop() { + let file_id = item.file_id; + + // TIER 1: Single cursor remaining — drain it + if heap.is_empty() { + let cursor = &mut cursors[file_id]; + let mapping = &col_mappings[file_id]; + loop { + let remaining = cursor.batch_height() - cursor.row_idx; + if remaining > 0 { + let slice = cursor.take_slice(cursor.row_idx, remaining); + ctx.push_batch(mapping.pad_batch(&slice)?)?; + } + if !cursor.advance_past_batch()? { + break; + } + } + break; + } + + // TIER 2 & 3: Multiple cursors active + let cursor = &mut cursors[file_id]; + let mapping = &col_mappings[file_id]; + + loop { + let heap_top = &heap.peek().unwrap().sort_values; + + // TIER 2: Entire remaining batch fits before heap top + let last_val = cursor.last_sort_values()?; + if cmp_sort_values(&last_val, heap_top, reverse_sorts) != Ordering::Greater { + let remaining = cursor.batch_height() - cursor.row_idx; + let slice = cursor.take_slice(cursor.row_idx, remaining); + ctx.push_batch(mapping.pad_batch(&slice)?)?; + + if !cursor.advance_past_batch()? { + break; + } + continue; + } + + // TIER 3: Binary search for the exact boundary + let run_start = cursor.row_idx; + let batch_h = cursor.batch_height(); + let batch = cursor.current_batch.as_ref().unwrap(); + + let mut lo = run_start; + let mut hi = batch_h - 1; + + while lo + 1 < hi { + let mid = lo + (hi - lo) / 2; + let mid_val = get_sort_values( + batch, + mid, + &cursor.sort_col_indices, + &cursor.sort_col_types, + &cursor.nulls_first, + )?; + + if cmp_sort_values(&mid_val, heap_top, reverse_sorts) != Ordering::Greater { + lo = mid; + } else { + hi = mid; + } + } + let run_end = lo; + + let run_len = run_end - run_start + 1; + if run_len > 0 { + let slice = cursor.take_slice(run_start, run_len); + ctx.push_batch(mapping.pad_batch(&slice)?)?; + } + + cursor.row_idx = run_end; + if !cursor.advance()? { + break; + } + + let next_val = cursor.current_sort_values()?; + if cmp_sort_values(&next_val, heap_top, reverse_sorts) == Ordering::Greater { + heap.push(HeapItem { + sort_values: next_val, + file_id, + reverse_sorts: Arc::clone(&reverse_sorts_arc), + }); + break; + } + } + } + + // ── Phase 5: Close ────────────────────────────────────────────────── + let (_metadata, crc32) = ctx.finish()?; + + log_debug!( + "[RUST] Merge complete ({}): {} total rows written to '{}' in {} row groups, crc32={:#010x}", + direction_label, + _metadata.file_metadata().num_rows(), + output_path, + _metadata.num_row_groups(), + crc32 + ); + + Ok(crc32) +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs new file mode 100644 index 0000000000000..6618cb4477cad --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs @@ -0,0 +1,105 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::fs::File; + +use arrow::array::RecordBatchReader; +use arrow::datatypes::Schema as ArrowSchema; +use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder}; +use parquet::schema::types::SchemaDescriptor; + +use crate::{log_debug, log_info}; + +use super::context::MergeContext; +use super::error::MergeResult; +use super::schema::{projection_indices_excluding_row_id, ColumnMapping}; + +/// Unsorted merge: reads each input file sequentially, pads to union schema, +/// rewrites `__row_id__` with globally sequential values. No sorting performed. +pub fn merge_unsorted( + input_files: &[String], + output_path: &str, + index_name: &str, +) -> MergeResult { + let config = crate::writer::SETTINGS_STORE + .get(index_name) + .map(|r| r.clone()) + .unwrap_or_default(); + let batch_size = config.get_merge_batch_size(); + let output_flush_rows = config.get_row_group_max_rows(); + let rayon_threads = config.get_merge_rayon_threads(); + let io_threads = config.get_merge_io_threads(); + log_debug!( + "[RUST] Starting unsorted merge: {} input files, output='{}'", + input_files.len(), + output_path + ); + + // Single pass: collect schemas and build readers. + let mut arrow_schemas: Vec = Vec::with_capacity(input_files.len()); + let mut parquet_descriptors: Vec = Vec::with_capacity(input_files.len()); + let mut readers: Vec = Vec::with_capacity(input_files.len()); + + for path in input_files { + let file = File::open(path)?; + let builder = ParquetRecordBatchReaderBuilder::try_new(file)?; + let schema = builder.schema().clone(); + let parquet_descr = builder.parquet_schema().clone(); + + let projection_indices = projection_indices_excluding_row_id(&schema); + let projection = parquet::arrow::ProjectionMask::roots(&parquet_descr, projection_indices); + let reader = builder.with_batch_size(batch_size).with_projection(projection).build()?; + + // The reader's schema is the projected schema (__row_id__ excluded). + arrow_schemas.push(reader.schema().as_ref().clone()); + parquet_descriptors.push(parquet_descr); + readers.push(reader); + } + + let mut ctx = MergeContext::new( + arrow_schemas.clone(), + &parquet_descriptors, + output_path, + index_name, + output_flush_rows, + rayon_threads, + io_threads, + )?; + + // Precompute column mappings per reader + let col_mappings: Vec = arrow_schemas.iter() + .map(|s| ColumnMapping::new(s, ctx.data_schema())) + .collect(); + + // Iterate readers for data. + for (file_idx, reader) in readers.into_iter().enumerate() { + log_debug!( + "[RUST] Unsorted merge: processing file {} of {}", + file_idx + 1, + input_files.len() + ); + + let mapping = &col_mappings[file_idx]; + for batch_result in reader { + let batch = batch_result?; + ctx.push_batch(mapping.pad_batch(&batch)?)?; + } + } + + let (_metadata, crc32) = ctx.finish()?; + + log_debug!( + "[RUST] Unsorted merge complete: {} total rows written to '{}' in {} row groups, crc32={:#010x}", + _metadata.file_metadata().num_rows(), + output_path, + _metadata.num_row_groups(), + crc32 + ); + + Ok(crc32) +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/native_settings.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/native_settings.rs new file mode 100644 index 0000000000000..49e68b58437dc --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/native_settings.rs @@ -0,0 +1,150 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::collections::HashMap; + +use crate::field_config::FieldConfig; + +#[derive(Debug, Clone, Default)] +pub struct NativeSettings { + pub index_name: Option, + pub compression_level: Option, + pub compression_type: Option, + pub page_size_bytes: Option, + pub page_row_limit: Option, + pub dict_size_bytes: Option, + pub field_configs: Option>, + pub custom_settings: Option>, + pub bloom_filter_enabled: Option, + pub bloom_filter_fpp: Option, + pub bloom_filter_ndv: Option, + pub sort_columns: Vec, + pub reverse_sorts: Vec, + pub nulls_first: Vec, + pub sort_in_memory_threshold_bytes: Option, + pub sort_batch_size: Option, + pub merge_batch_size: Option, + pub row_group_max_rows: Option, + pub merge_rayon_threads: Option, + pub merge_io_threads: Option, +} + +impl NativeSettings { + pub fn new() -> Self { + Self::default() + } + + pub fn get_compression_type(&self) -> &str { + self.compression_type.as_deref().unwrap_or("LZ4_RAW") + } + + pub fn get_compression_level(&self) -> i32 { + self.compression_level.unwrap_or(2) + } + + pub fn get_page_size_bytes(&self) -> usize { + self.page_size_bytes.unwrap_or(1024 * 1024) + } + + pub fn get_page_row_limit(&self) -> usize { + self.page_row_limit.unwrap_or(20000) + } + + pub fn get_dict_size_bytes(&self) -> usize { + self.dict_size_bytes.unwrap_or(2 * 1024 * 1024) + } + + pub fn get_bloom_filter_enabled(&self) -> bool { + self.bloom_filter_enabled.unwrap_or(true) + } + + pub fn get_bloom_filter_fpp(&self) -> f64 { + self.bloom_filter_fpp.unwrap_or(0.1) + } + + pub fn get_bloom_filter_ndv(&self) -> u64 { + self.bloom_filter_ndv.unwrap_or(100_000) + } + + pub fn get_field_config(&self, field_name: &str) -> Option<&FieldConfig> { + self.field_configs.as_ref()?.get(field_name) + } + + pub fn has_field_configs(&self) -> bool { + self.field_configs.as_ref().map_or(false, |configs| !configs.is_empty()) + } + + pub fn get_sort_in_memory_threshold_bytes(&self) -> u64 { + self.sort_in_memory_threshold_bytes.unwrap_or(32 * 1024 * 1024) + } + + pub fn get_sort_batch_size(&self) -> usize { + self.sort_batch_size.unwrap_or(8192) + } + + pub fn get_merge_batch_size(&self) -> usize { + self.merge_batch_size.unwrap_or(100_000) + } + + pub fn get_row_group_max_rows(&self) -> usize { + self.row_group_max_rows.unwrap_or(1_000_000) + } + + pub fn get_merge_rayon_threads(&self) -> Option { + self.merge_rayon_threads + } + + pub fn get_merge_io_threads(&self) -> Option { + self.merge_io_threads + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_defaults() { + let config = NativeSettings::default(); + assert_eq!(config.get_compression_type(), "LZ4_RAW"); + assert_eq!(config.get_compression_level(), 2); + assert_eq!(config.get_page_row_limit(), 20000); + assert_eq!(config.get_dict_size_bytes(), 2 * 1024 * 1024); + } + + #[test] + fn test_struct_construction() { + let config = NativeSettings { + compression_type: Some("SNAPPY".to_string()), + compression_level: Some(1), + ..Default::default() + }; + assert_eq!(config.get_compression_type(), "SNAPPY"); + assert_eq!(config.get_compression_level(), 1); + } + + #[test] + fn test_field_configs() { + use crate::field_config::FieldConfig; + use std::collections::HashMap; + + let mut field_configs = HashMap::new(); + field_configs.insert("timestamp".to_string(), FieldConfig { + compression_type: Some("SNAPPY".to_string()), + compression_level: None, + }); + let config = NativeSettings { + compression_type: Some("ZSTD".to_string()), + field_configs: Some(field_configs), + ..Default::default() + }; + assert!(config.has_field_configs()); + let fc = config.get_field_config("timestamp").unwrap(); + assert_eq!(fc.compression_type, Some("SNAPPY".to_string())); + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/rate_limited_writer.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/rate_limited_writer.rs new file mode 100644 index 0000000000000..32826276b0fd7 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/rate_limited_writer.rs @@ -0,0 +1,213 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::io::{Result, Write}; +use std::sync::{Arc, RwLock}; +use std::thread::sleep; +use std::time::{Duration, Instant}; + +// TODO: Make this value dynamic based on resource availability (e.g., adjust ±x% based on IOPS pressure) +const MIN_PAUSE_CHECK_MSEC: f64 = 20.0; +const BYTES_PER_MB: f64 = 1024.0 * 1024.0; +const MAX_MIN_PAUSE_CHECK_BYTES: usize = 1024 * 1024; // 1 MB +const MSEC_TO_SEC: f64 = 1000.0; + +/// Configuration for rate limiting behavior. +struct RateLimiterConfig { + /// Maximum throughput in megabytes per second + mb_per_sec: f64, + /// Minimum bytes to write before checking if pause is needed + min_pause_check_bytes: usize, +} + +/// A writer that rate-limits write operations to a specified throughput. +/// +/// This writer wraps another writer and ensures that data is written at a maximum +/// rate specified in megabytes per second. It uses periodic pauses to maintain +/// the target rate, checking after a minimum number of bytes have been written. +/// +/// # Rate Limiting Strategy +/// +/// The rate limiter works by: +/// 1. Tracking bytes written since the last pause +/// 2. Periodically checking if enough time has elapsed for the bytes written +/// 3. Sleeping if the write rate exceeds the configured limit +/// +/// The minimum pause check interval is calculated to avoid excessive overhead +/// from frequent time checks, defaulting to 25ms worth of data or 1MB, whichever +/// is smaller. +/// +/// # Thread Safety +/// +/// The rate limit can be updated dynamically via `set_mb_per_sec()`. The configuration +/// is protected by a `RwLock`, allowing concurrent reads while ensuring safe updates. +/// If the lock becomes poisoned (due to a panic in another thread), the writer will +/// gracefully degrade by skipping rate limiting rather than propagating the panic. +/// +/// +/// # Special Cases +/// +/// - Setting `mb_per_sec` to `0.0` disables rate limiting entirely +/// - Negative values are rejected with an error +/// - Lock poisoning is handled gracefully by skipping rate limiting +pub struct RateLimitedWriter { + inner: W, + rate_limiter_config: Arc>, + bytes_since_last_pause: usize, + last_pause_time: Instant, +} + +impl RateLimitedWriter { + /// Creates a new rate-limited writer with the specified throughput limit. + /// + /// # Arguments + /// + /// * `inner` - The underlying writer to wrap + /// * `mb_per_sec` - Maximum write rate in megabytes per second (must be non-negative) + /// + /// # Returns + /// + /// Returns `Ok(RateLimitedWriter)` on success, or an error if `mb_per_sec` is negative. + /// + /// + /// # Errors + /// + /// Returns `Err` with `ErrorKind::InvalidInput` if `mb_per_sec` is negative. + pub fn new(inner: W, mb_per_sec: f64) -> Result { + if mb_per_sec < 0.0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("mbPerSec must be non-negative: got: {}", mb_per_sec), + )); + } + + let min_pause_check_bytes = Self::calculate_min_pause_check_bytes(mb_per_sec); + Ok(Self { + inner, + rate_limiter_config: Arc::new(RwLock::new(RateLimiterConfig { + mb_per_sec, + min_pause_check_bytes, + })), + bytes_since_last_pause: 0, + last_pause_time: Instant::now(), + }) + } + + /// Updates the rate limit dynamically. + /// + /// This method allows changing the throughput limit while the writer is in use. + /// The new rate takes effect immediately for subsequent write operations. + /// + /// # Arguments + /// + /// * `mb_per_sec` - New maximum write rate in megabytes per second (must be non-negative) + /// + /// # Returns + /// + /// Returns `Ok(())` on success, or an error if the rate is invalid or the lock is poisoned. + /// + /// + /// # Errors + /// + /// Returns `Err` with: + /// - `ErrorKind::InvalidInput` if `mb_per_sec` is negative + /// - `ErrorKind::Other` if the internal lock is poisoned + pub fn set_mb_per_sec(&mut self, mb_per_sec: f64) -> Result<()> { + if mb_per_sec < 0.0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("mbPerSec must be non-negative: got: {}", mb_per_sec), + )); + } + + let min_pause_check_bytes = Self::calculate_min_pause_check_bytes(mb_per_sec); + + let mut config = self.rate_limiter_config.write().map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::Other, + format!("Failed to acquire write lock: {}", e), + ) + })?; + + config.mb_per_sec = mb_per_sec; + config.min_pause_check_bytes = min_pause_check_bytes; + + Ok(()) + } + + /// Calculates the minimum number of bytes to write before checking if a pause is needed. + /// + /// This is based on the configured rate and a minimum pause check interval to avoid + /// excessive overhead from frequent time checks. The result is capped at 1MB. + fn calculate_min_pause_check_bytes(mb_per_sec: f64) -> usize { + let bytes = (MIN_PAUSE_CHECK_MSEC / MSEC_TO_SEC) * mb_per_sec * BYTES_PER_MB; + std::cmp::min(MAX_MIN_PAUSE_CHECK_BYTES, bytes as usize) + } + + /// Pauses execution if the write rate exceeds the configured limit. + /// + /// Calculates the target time for writing the given number of bytes based on + /// the configured rate, and sleeps if insufficient time has elapsed since the + /// last pause. If the lock is poisoned, rate limiting is skipped. + /// + /// # Arguments + /// + /// * `bytes` - Number of bytes written since the last pause + fn pause(&mut self, bytes: usize) { + let config = match self.rate_limiter_config.read() { + Ok(config) => config, + Err(_) => { + // Lock is poisoned, skip rate limiting this time + return; + } + }; + + if config.mb_per_sec == 0.0 { + return; + } + + let elapsed = self.last_pause_time.elapsed().as_secs_f64(); + let target_time = bytes as f64 / (config.mb_per_sec * BYTES_PER_MB); + + if target_time > elapsed { + let sleep_time = Duration::from_secs_f64(target_time - elapsed); + sleep(sleep_time); + } + + self.last_pause_time = Instant::now(); + } +} + +impl Write for RateLimitedWriter { + fn write(&mut self, buf: &[u8]) -> Result { + let n = self.inner.write(buf)?; + self.bytes_since_last_pause += n; + + let current_min_pause_check_bytes = { + match self.rate_limiter_config.read() { + Ok(config) => config.min_pause_check_bytes, + Err(_) => { + // Lock is poisoned, use a safe default + MAX_MIN_PAUSE_CHECK_BYTES + } + } + }; + + if self.bytes_since_last_pause > current_min_pause_check_bytes { + self.pause(self.bytes_since_last_pause); + self.bytes_since_last_pause = 0; + } + Ok(n) + } + + fn flush(&mut self) -> Result<()> { + self.inner.flush() + } +} + + diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/test_utils.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/test_utils.rs index 2a80157518ec8..7b46092b09ddc 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/test_utils.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/test_utils.rs @@ -7,10 +7,13 @@ */ use arrow::array::{Int32Array, StringArray, StructArray}; +use arrow::compute::concat_batches; use arrow::datatypes::{DataType, Field, Schema}; use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; use arrow::record_batch::RecordBatch; -use arrow_array::Array; +use arrow::array::Array; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use std::fs::File; use std::sync::Arc; use tempfile::tempdir; @@ -33,12 +36,16 @@ pub fn cleanup_ffi_schema(schema_ptr: i64) { } pub fn create_test_ffi_data() -> Result<(i64, i64), Box> { + create_test_ffi_data_with_ids(vec![1, 2, 3], vec![Some("Alice"), Some("Bob"), None]) +} + +pub fn create_test_ffi_data_with_ids(ids: Vec, names: Vec>) -> Result<(i64, i64), Box> { let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Int32, false), Field::new("name", DataType::Utf8, true), ])); - let id_array = Arc::new(Int32Array::from(vec![1, 2, 3])); - let name_array = Arc::new(StringArray::from(vec![Some("Alice"), Some("Bob"), None])); + let id_array = Arc::new(Int32Array::from(ids)); + let name_array = Arc::new(StringArray::from(names)); let record_batch = RecordBatch::try_new(schema.clone(), vec![id_array, name_array])?; let struct_array = StructArray::from(record_batch); let array_data = struct_array.into_data(); @@ -65,7 +72,16 @@ pub fn get_temp_file_path(name: &str) -> (tempfile::TempDir, String) { pub fn create_writer_and_assert_success(filename: &str) -> (Arc, i64) { let (schema, schema_ptr) = create_test_ffi_schema(); - let result = NativeParquetWriter::create_writer(filename.to_string(), schema_ptr); + let result = NativeParquetWriter::create_writer(filename.to_string(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![]); + assert!(result.is_ok()); + (schema, schema_ptr) +} + +pub fn create_sorted_writer_and_assert_success(filename: &str, sort_column: &str, reverse: bool) -> (Arc, i64) { + let (schema, schema_ptr) = create_test_ffi_schema(); + let result = NativeParquetWriter::create_writer( + filename.to_string(), "test-index".to_string(), schema_ptr, vec![sort_column.to_string()], vec![reverse], vec![false] + ); assert!(result.is_ok()); (schema, schema_ptr) } @@ -105,3 +121,18 @@ pub fn close_writer_and_get_metadata(filename: &str, schema_ptr: i64) -> crate:: cleanup_ffi_schema(schema_ptr); result.unwrap().unwrap() } + +pub fn read_parquet_file(filename: &str) -> Vec { + let file = File::open(filename).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap(); + let reader = builder.build().unwrap(); + reader.collect::, _>>().unwrap() +} + +pub fn read_parquet_file_sorted_ids(filename: &str) -> Vec { + let batches = read_parquet_file(filename); + let combined = concat_batches(&batches[0].schema(), &batches).unwrap(); + let id_col = combined.column(0) + .as_any().downcast_ref::().unwrap(); + (0..id_col.len()).map(|i| id_col.value(i)).collect() +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs index 9efcc961be225..25c4e375a7264 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs @@ -6,10 +6,15 @@ * compatible open source license. */ +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::thread; +use tempfile::tempdir; + use crate::test_utils::*; -use crate::writer::{NativeParquetWriter, WRITER_MANAGER, FILE_MANAGER}; +use crate::writer::NativeParquetWriter; -use parquet::file::reader::FileReader; use std::fs::File; use std::io::Read; @@ -17,8 +22,7 @@ use std::io::Read; fn test_create_writer_success() { let (_temp_dir, filename) = get_temp_file_path("test.parquet"); let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); - assert!(WRITER_MANAGER.contains_key(&filename)); - assert!(FILE_MANAGER.contains_key(&filename)); + assert!(NativeParquetWriter::has_writer(&filename)); close_writer_and_cleanup_schema(&filename, schema_ptr); } @@ -26,16 +30,15 @@ fn test_create_writer_success() { fn test_create_writer_invalid_path() { let invalid_path = "/invalid/path/that/does/not/exist/test.parquet"; let (_schema, schema_ptr) = create_test_ffi_schema(); - let result = NativeParquetWriter::create_writer(invalid_path.to_string(), schema_ptr); + let result = NativeParquetWriter::create_writer(invalid_path.to_string(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![]); assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("No such file or directory")); cleanup_ffi_schema(schema_ptr); } #[test] fn test_create_writer_invalid_schema_pointer() { let (_temp_dir, filename) = get_temp_file_path("invalid_schema.parquet"); - let result = NativeParquetWriter::create_writer(filename, 0); + let result = NativeParquetWriter::create_writer(filename, "test-index".to_string(), 0, vec![], vec![], vec![]); assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("Invalid schema address")); } @@ -44,9 +47,11 @@ fn test_create_writer_invalid_schema_pointer() { fn test_create_writer_multiple_times_same_file() { let (_temp_dir, filename) = get_temp_file_path("duplicate.parquet"); let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); - let result2 = NativeParquetWriter::create_writer(filename.clone(), schema_ptr); + let (_, schema_ptr2) = create_test_ffi_schema(); + let result2 = NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr2, vec![], vec![], vec![]); assert!(result2.is_err()); assert!(result2.unwrap_err().to_string().contains("Writer already exists")); + cleanup_ffi_schema(schema_ptr2); close_writer_and_cleanup_schema(&filename, schema_ptr); } @@ -70,6 +75,17 @@ fn test_write_data_no_writer() { cleanup_ffi_data(array_ptr, schema_ptr); } +#[test] +fn test_write_data_multiple_batches() { + let (_temp_dir, filename) = get_temp_file_path("multi_write_ffi.parquet"); + let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); + for _ in 0..3 { + let (array_ptr, data_schema_ptr) = write_ffi_data_to_writer(&filename); + cleanup_ffi_data(array_ptr, data_schema_ptr); + } + close_writer_and_cleanup_schema(&filename, schema_ptr); +} + #[test] fn test_write_data_invalid_pointers() { let (_temp_dir, filename) = get_temp_file_path("invalid_ffi.parquet"); @@ -98,16 +114,11 @@ fn test_write_data_incompatible_schema() { fn test_finalize_writer_success() { let (_temp_dir, filename) = get_temp_file_path("test_close.parquet"); let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); + let (array_ptr, data_schema_ptr) = write_ffi_data_to_writer(&filename); + cleanup_ffi_data(array_ptr, data_schema_ptr); let result = NativeParquetWriter::finalize_writer(filename.clone()); assert!(result.is_ok()); - let finalize_result = result.unwrap(); - assert!(finalize_result.is_some()); - let finalize_result = finalize_result.unwrap(); - assert_eq!(finalize_result.metadata.file_metadata().num_rows(), 0); - assert!(finalize_result.metadata.file_metadata().version() > 0); - assert!(!WRITER_MANAGER.contains_key(&filename)); - assert!(FILE_MANAGER.contains_key(&filename)); - FILE_MANAGER.remove(&filename); + assert!(Path::new(&filename).exists()); cleanup_ffi_schema(schema_ptr); } @@ -125,9 +136,7 @@ fn test_finalize_writer_with_data_returns_correct_metadata() { let metadata = result.unwrap().unwrap(); assert_eq!(metadata.metadata.file_metadata().num_rows(), 6); assert!(metadata.metadata.file_metadata().version() > 0); - assert_eq!(metadata.metadata.file_metadata().schema_descr().num_columns(), 3); // root + 2 fields (id, name) assert_ne!(metadata.crc32, 0, "CRC32 should be non-zero for a file with data"); - FILE_MANAGER.remove(&filename); cleanup_ffi_schema(schema_ptr); } @@ -142,17 +151,13 @@ fn test_close_nonexistent_writer() { fn test_close_multiple_times_same_file() { let (_temp_dir, filename) = get_temp_file_path("test.parquet"); let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); + let (array_ptr, data_schema_ptr) = write_ffi_data_to_writer(&filename); + cleanup_ffi_data(array_ptr, data_schema_ptr); let result1 = NativeParquetWriter::finalize_writer(filename.clone()); assert!(result1.is_ok()); - let metadata = result1.unwrap(); - assert!(metadata.is_some()); - assert_eq!(metadata.unwrap().metadata.num_rows, 0); - assert!(!WRITER_MANAGER.contains_key(&filename)); - assert!(FILE_MANAGER.contains_key(&filename)); - let result2 = NativeParquetWriter::finalize_writer(filename.clone()); + let result2 = NativeParquetWriter::finalize_writer(filename); assert!(result2.is_err()); assert!(result2.unwrap_err().to_string().contains("Writer not found")); - FILE_MANAGER.remove(&filename); cleanup_ffi_schema(schema_ptr); } @@ -160,18 +165,115 @@ fn test_close_multiple_times_same_file() { fn test_sync_to_disk_success() { let (_temp_dir, filename) = get_temp_file_path("test_flush.parquet"); let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); - assert!(FILE_MANAGER.contains_key(&filename)); - let result = NativeParquetWriter::sync_to_disk(filename.clone()); + let (array_ptr, data_schema_ptr) = write_ffi_data_to_writer(&filename); + cleanup_ffi_data(array_ptr, data_schema_ptr); + let _ = NativeParquetWriter::finalize_writer(filename.clone()); + let result = NativeParquetWriter::sync_to_disk(filename); assert!(result.is_ok()); - assert!(!FILE_MANAGER.contains_key(&filename)); - close_writer_and_cleanup_schema(&filename, schema_ptr); + cleanup_ffi_schema(schema_ptr); } #[test] fn test_flush_nonexistent_file() { let result = NativeParquetWriter::sync_to_disk("nonexistent.parquet".to_string()); assert!(result.is_err()); - assert_eq!(result.unwrap_err().to_string(), "File not found"); +} + +#[test] +fn test_complete_writer_lifecycle() { + let (_temp_dir, filename) = get_temp_file_path("complete_workflow.parquet"); + let file_path = Path::new(&filename); + let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); + + for _ in 0..3 { + let (array_ptr, data_schema_ptr) = write_ffi_data_to_writer(&filename); + cleanup_ffi_data(array_ptr, data_schema_ptr); + } + + let close_result = NativeParquetWriter::finalize_writer(filename.clone()); + assert!(close_result.is_ok()); + assert!(close_result.unwrap().is_some()); + + assert!(NativeParquetWriter::sync_to_disk(filename.clone()).is_ok()); + assert!(file_path.exists()); + assert!(file_path.metadata().unwrap().len() > 0); + + cleanup_ffi_schema(schema_ptr); +} + +#[test] +fn test_sorted_writer_ascending() { + let (_temp_dir, filename) = get_temp_file_path("sorted_asc.parquet"); + let (_schema, schema_ptr) = create_sorted_writer_and_assert_success(&filename, "id", false); + + let (ap1, sp1) = create_test_ffi_data_with_ids( + vec![30, 10, 50], vec![Some("C"), Some("A"), Some("E")] + ).unwrap(); + NativeParquetWriter::write_data(filename.clone(), ap1, sp1).unwrap(); + cleanup_ffi_data(ap1, sp1); + + let (ap2, sp2) = create_test_ffi_data_with_ids( + vec![20, 40], vec![Some("B"), Some("D")] + ).unwrap(); + NativeParquetWriter::write_data(filename.clone(), ap2, sp2).unwrap(); + cleanup_ffi_data(ap2, sp2); + + NativeParquetWriter::finalize_writer(filename.clone()).unwrap(); + + let ids = read_parquet_file_sorted_ids(&filename); + assert_eq!(ids, vec![10, 20, 30, 40, 50], "Data should be sorted ascending by id"); + + cleanup_ffi_schema(schema_ptr); +} + +#[test] +fn test_sorted_writer_descending() { + let (_temp_dir, filename) = get_temp_file_path("sorted_desc.parquet"); + let (_schema, schema_ptr) = create_sorted_writer_and_assert_success(&filename, "id", true); + + let (ap1, sp1) = create_test_ffi_data_with_ids( + vec![30, 10, 50], vec![Some("C"), Some("A"), Some("E")] + ).unwrap(); + NativeParquetWriter::write_data(filename.clone(), ap1, sp1).unwrap(); + cleanup_ffi_data(ap1, sp1); + + let (ap2, sp2) = create_test_ffi_data_with_ids( + vec![20, 40], vec![Some("B"), Some("D")] + ).unwrap(); + NativeParquetWriter::write_data(filename.clone(), ap2, sp2).unwrap(); + cleanup_ffi_data(ap2, sp2); + + NativeParquetWriter::finalize_writer(filename.clone()).unwrap(); + + let ids = read_parquet_file_sorted_ids(&filename); + assert_eq!(ids, vec![50, 40, 30, 20, 10], "Data should be sorted descending by id"); + + cleanup_ffi_schema(schema_ptr); +} + +#[test] +fn test_unsorted_writer_preserves_insertion_order() { + let (_temp_dir, filename) = get_temp_file_path("unsorted.parquet"); + let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); + + let (ap1, sp1) = create_test_ffi_data_with_ids( + vec![30, 10, 50], vec![Some("C"), Some("A"), Some("E")] + ).unwrap(); + NativeParquetWriter::write_data(filename.clone(), ap1, sp1).unwrap(); + cleanup_ffi_data(ap1, sp1); + + let (ap2, sp2) = create_test_ffi_data_with_ids( + vec![20, 40], vec![Some("B"), Some("D")] + ).unwrap(); + NativeParquetWriter::write_data(filename.clone(), ap2, sp2).unwrap(); + cleanup_ffi_data(ap2, sp2); + + NativeParquetWriter::finalize_writer(filename.clone()).unwrap(); + + let ids = read_parquet_file_sorted_ids(&filename); + assert_eq!(ids, vec![30, 10, 50, 20, 40], "Data should preserve insertion order"); + + cleanup_ffi_schema(schema_ptr); } #[test] @@ -183,68 +285,30 @@ fn test_get_filtered_writer_memory_usage_with_writers() { let (_schema2, schema_ptr2) = create_writer_and_assert_success(&filename2); let result = NativeParquetWriter::get_filtered_writer_memory_usage(prefix); assert!(result.is_ok()); - let _memory_usage = result.unwrap(); - assert!(_memory_usage >= 0); + assert!(result.unwrap() >= 0); close_writer_and_cleanup_schema(&filename1, schema_ptr1); close_writer_and_cleanup_schema(&filename2, schema_ptr2); } +// CRC32 tests -/// Computes CRC32 of a file by reading it from disk in chunks. -/// This is the "re-read" baseline that the streaming checksum must match. fn compute_file_crc32(path: &str) -> u32 { let mut file = File::open(path).unwrap(); let mut hasher = crc32fast::Hasher::new(); let mut buf = [0u8; 64 * 1024]; loop { let n = file.read(&mut buf).unwrap(); - if n == 0 { - break; - } + if n == 0 { break; } hasher.update(&buf[..n]); } hasher.finalize() } -/// Verifies that the streaming CRC32 computed during write (via Crc32Writer) -/// exactly matches a CRC32 computed by re-reading the finalized file from disk. -/// -/// This proves the streaming approach is correct and eliminates the need for -/// a second I/O pass over the file. -#[test] -fn test_streaming_crc32_matches_reread_crc32_empty_file() { - let (_temp_dir, filename) = get_temp_file_path("crc32_empty.parquet"); - let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); - - // Finalize with zero rows — still writes the Parquet magic bytes + footer - let result = NativeParquetWriter::finalize_writer(filename.clone()); - assert!(result.is_ok()); - let finalize_result = result.unwrap().unwrap(); - let streaming_crc32 = finalize_result.crc32; - - // Re-read the file and compute CRC32 independently - let reread_crc32 = compute_file_crc32(&filename); - - assert_eq!( - streaming_crc32, reread_crc32, - "Streaming CRC32 ({:#010x}) must match re-read CRC32 ({:#010x}) for empty Parquet file", - streaming_crc32, reread_crc32 - ); - assert_ne!(streaming_crc32, 0, "CRC32 should be non-zero even for an empty Parquet file (magic bytes + footer)"); - - FILE_MANAGER.remove(&filename); - cleanup_ffi_schema(schema_ptr); -} - -/// Verifies streaming CRC32 matches re-read CRC32 for a file with actual data. -/// Writes multiple batches to exercise the full write path (row groups, column -/// chunks, compression, bloom filters, footer). #[test] -fn test_streaming_crc32_matches_reread_crc32_with_data() { +fn test_crc32_matches_reread_with_data() { let (_temp_dir, filename) = get_temp_file_path("crc32_with_data.parquet"); let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); - // Write 3 batches (9 rows total) to exercise multiple write() calls for _ in 0..3 { let (array_ptr, data_schema_ptr) = create_test_ffi_data().unwrap(); NativeParquetWriter::write_data(filename.clone(), array_ptr, data_schema_ptr).unwrap(); @@ -256,41 +320,23 @@ fn test_streaming_crc32_matches_reread_crc32_with_data() { let finalize_result = result.unwrap().unwrap(); let streaming_crc32 = finalize_result.crc32; - // Verify metadata is correct assert_eq!(finalize_result.metadata.file_metadata().num_rows(), 9); - // Re-read the file and compute CRC32 independently let reread_crc32 = compute_file_crc32(&filename); + assert_eq!(streaming_crc32, reread_crc32); + assert_ne!(streaming_crc32, 0); - assert_eq!( - streaming_crc32, reread_crc32, - "Streaming CRC32 ({:#010x}) must match re-read CRC32 ({:#010x}) for Parquet file with {} rows", - streaming_crc32, reread_crc32, finalize_result.metadata.file_metadata().num_rows() - ); - assert_ne!(streaming_crc32, 0, "CRC32 should be non-zero for a file with data"); - - // Verify the file is a valid Parquet file by reading it back - let file = File::open(&filename).unwrap(); - let reader = parquet::file::reader::SerializedFileReader::new(file).unwrap(); - assert_eq!(reader.metadata().file_metadata().num_rows(), 9); - - FILE_MANAGER.remove(&filename); cleanup_ffi_schema(schema_ptr); } -/// Verifies that two different files produce different CRC32 values, -/// confirming the checksum is content-dependent and not a constant. #[test] -fn test_streaming_crc32_differs_for_different_content() { - // File 1: empty +fn test_crc32_differs_for_different_content() { let (_temp_dir1, filename1) = get_temp_file_path("crc32_diff_a.parquet"); let (_schema1, schema_ptr1) = create_writer_and_assert_success(&filename1); let result1 = NativeParquetWriter::finalize_writer(filename1.clone()); let crc32_empty = result1.unwrap().unwrap().crc32; - FILE_MANAGER.remove(&filename1); cleanup_ffi_schema(schema_ptr1); - // File 2: with data let (_temp_dir2, filename2) = get_temp_file_path("crc32_diff_b.parquet"); let (_schema2, schema_ptr2) = create_writer_and_assert_success(&filename2); let (array_ptr, data_schema_ptr) = create_test_ffi_data().unwrap(); @@ -298,12 +344,148 @@ fn test_streaming_crc32_differs_for_different_content() { cleanup_ffi_data(array_ptr, data_schema_ptr); let result2 = NativeParquetWriter::finalize_writer(filename2.clone()); let crc32_with_data = result2.unwrap().unwrap().crc32; - FILE_MANAGER.remove(&filename2); cleanup_ffi_schema(schema_ptr2); - assert_ne!( - crc32_empty, crc32_with_data, - "Empty file CRC32 ({:#010x}) should differ from file-with-data CRC32 ({:#010x})", - crc32_empty, crc32_with_data - ); + assert_ne!(crc32_empty, crc32_with_data); +} + +// Concurrency tests + +#[test] +fn test_concurrent_writer_creation() { + let temp_dir = tempdir().unwrap(); + let success_count = Arc::new(AtomicUsize::new(0)); + let mut handles = vec![]; + + for i in 0..10 { + let temp_dir_path = temp_dir.path().to_path_buf(); + let success_count = Arc::clone(&success_count); + + let handle = thread::spawn(move || { + let file_path = temp_dir_path.join(format!("concurrent_{}.parquet", i)); + let filename = file_path.to_string_lossy().to_string(); + let (_schema, schema_ptr) = create_test_ffi_schema(); + + if NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![]).is_ok() { + success_count.fetch_add(1, Ordering::SeqCst); + let (ap, sp) = create_test_ffi_data().unwrap(); + let _ = NativeParquetWriter::write_data(filename.clone(), ap, sp); + cleanup_ffi_data(ap, sp); + let _ = NativeParquetWriter::finalize_writer(filename); + } + cleanup_ffi_schema(schema_ptr); + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!(success_count.load(Ordering::SeqCst), 10); +} + +#[test] +fn test_concurrent_close_operations_same_file() { + let (_temp_dir, filename) = get_temp_file_path("close_race.parquet"); + let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); + + let (array_ptr, data_schema_ptr) = write_ffi_data_to_writer(&filename); + cleanup_ffi_data(array_ptr, data_schema_ptr); + + let success_count = Arc::new(AtomicUsize::new(0)); + let mut handles = vec![]; + + for _ in 0..3 { + let filename = filename.clone(); + let success_count = Arc::clone(&success_count); + + let handle = thread::spawn(move || { + if NativeParquetWriter::finalize_writer(filename).is_ok() { + success_count.fetch_add(1, Ordering::SeqCst); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!(success_count.load(Ordering::SeqCst), 1); + cleanup_ffi_schema(schema_ptr); +} + +#[test] +fn test_concurrent_writes_same_file() { + let (_temp_dir, filename) = get_temp_file_path("concurrent_write_ffi.parquet"); + let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); + + let success_count = Arc::new(AtomicUsize::new(0)); + let mut handles = vec![]; + + for _ in 0..5 { + let filename = filename.clone(); + let success_count = Arc::clone(&success_count); + + let handle = thread::spawn(move || { + let (array_ptr, data_schema_ptr) = create_test_ffi_data().unwrap(); + if NativeParquetWriter::write_data(filename, array_ptr, data_schema_ptr).is_ok() { + success_count.fetch_add(1, Ordering::SeqCst); + } + cleanup_ffi_data(array_ptr, data_schema_ptr); + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!(success_count.load(Ordering::SeqCst), 5); + close_writer_and_cleanup_schema(&filename, schema_ptr); +} + +#[test] +fn test_concurrent_writes_different_files() { + let temp_dir = tempdir().unwrap(); + let file_count = 8; + let success_count = Arc::new(AtomicUsize::new(0)); + let mut handles = vec![]; + let mut filenames = vec![]; + let mut schema_ptrs = vec![]; + + for i in 0..file_count { + let file_path = temp_dir.path().join(format!("concurrent_write_{}.parquet", i)); + let filename = file_path.to_string_lossy().to_string(); + let (_schema, schema_ptr) = create_writer_and_assert_success(&filename); + filenames.push(filename); + schema_ptrs.push(schema_ptr); + } + + for i in 0..file_count { + let filename = filenames[i].clone(); + let success_count = Arc::clone(&success_count); + + let handle = thread::spawn(move || { + for _ in 0..2 { + let (array_ptr, data_schema_ptr) = create_test_ffi_data().unwrap(); + if NativeParquetWriter::write_data(filename.clone(), array_ptr, data_schema_ptr).is_ok() { + success_count.fetch_add(1, Ordering::SeqCst); + } + cleanup_ffi_data(array_ptr, data_schema_ptr); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!(success_count.load(Ordering::SeqCst), file_count * 2); + + for (i, filename) in filenames.iter().enumerate() { + close_writer_and_cleanup_schema(filename, schema_ptrs[i]); + } } diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs index 36bb2fe795d7d..d244e306adec6 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs @@ -8,74 +8,84 @@ use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; use arrow::record_batch::RecordBatch; +use arrow::compute::{lexsort_to_indices, take, SortColumn}; use dashmap::DashMap; use lazy_static::lazy_static; -use parquet::arrow::ArrowWriter; -use parquet::basic::Compression; -use parquet::file::properties::WriterProperties; +use parquet::arrow::{arrow_reader::ParquetRecordBatchReaderBuilder, ArrowWriter}; use parquet::file::reader::{FileReader, SerializedFileReader}; use std::fs::File; -use std::io::Write; +use std::path::Path; use std::sync::{Arc, Mutex}; use crate::{log_error, log_debug}; - -/// A write wrapper that computes CRC32 as bytes flow through. -/// Wraps a File and tracks the running checksum without buffering. -pub struct Crc32Writer { - inner: File, - hasher: crc32fast::Hasher, -} - -impl Crc32Writer { - fn new(file: File) -> Self { - Self { - inner: file, - hasher: crc32fast::Hasher::new(), - } - } - - /// Finalizes and returns the CRC32 checksum of all bytes written. - fn checksum(&self) -> u32 { - self.hasher.clone().finalize() - } -} - -impl Write for Crc32Writer { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - let n = self.inner.write(buf)?; - self.hasher.update(&buf[..n]); - Ok(n) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.inner.flush() - } -} +use crate::crc_writer::CrcWriter; +use crate::merge::{merge_sorted, schema::ROW_ID_COLUMN_NAME}; +use crate::native_settings::NativeSettings; +use crate::writer_properties_builder::WriterPropertiesBuilder; /// Result from finalizing a writer: Parquet metadata + whole-file CRC32. +#[derive(Debug)] pub struct FinalizeResult { pub metadata: parquet::file::metadata::ParquetMetaData, pub crc32: u32, } +/// Bundles all per-writer resources so a single `DashMap::remove` atomically +/// drops the writer, closes the file handle, and cleans up sort config. +struct WriterState { + writer: Arc>>>, + settings: NativeSettings, + crc_handle: crate::crc_writer::CrcHandle, +} + lazy_static! { - pub static ref WRITER_MANAGER: DashMap>>> = DashMap::new(); - pub static ref FILE_MANAGER: DashMap = DashMap::new(); + /// Unified per-writer registry. Keyed by temp filename. + static ref WRITERS: DashMap = DashMap::new(); + pub static ref SETTINGS_STORE: DashMap = DashMap::new(); + /// Holds file handles for finalized files pending fsync. Removed after sync. + static ref FILE_MANAGER: DashMap = DashMap::new(); } pub struct NativeParquetWriter; impl NativeParquetWriter { - pub fn create_writer(filename: String, schema_address: i64) -> Result<(), Box> { - log_debug!("create_writer called for file: {}, schema_address: {}", filename, schema_address); + /// Returns true if a writer is currently open for the given filename. + pub fn has_writer(filename: &str) -> bool { + let temp_filename = Self::temp_filename(filename); + WRITERS.contains_key(&temp_filename) + } + /// Build the temp filename by prepending "temp-" to the basename. + fn temp_filename(filename: &str) -> String { + let path = Path::new(filename); + path.parent() + .unwrap_or_else(|| Path::new("")) + .join(format!("temp-{}", path.file_name().unwrap().to_str().unwrap())) + .to_string_lossy() + .to_string() + } + + pub fn create_writer( + filename: String, + index_name: String, + schema_address: i64, + sort_columns: Vec, + reverse_sorts: Vec, + nulls_first: Vec, + ) -> Result<(), Box> { + log_debug!( + "create_writer called for file: {}, index: {}, schema_address: {}, sort_columns: {:?}, reverse_sorts: {:?}, nulls_first: {:?}", + filename, index_name, schema_address, sort_columns, reverse_sorts, nulls_first + ); if (schema_address as *mut u8).is_null() { log_error!("ERROR: Invalid schema address (null pointer) for file: {}", filename); return Err("Invalid schema address".into()); } - if WRITER_MANAGER.contains_key(&filename) { - log_error!("ERROR: Writer already exists for file: {}", filename); + + let temp_filename = Self::temp_filename(&filename); + + if WRITERS.contains_key(&temp_filename) { + log_error!("ERROR: Writer already exists for file: {}", temp_filename); return Err("Writer already exists for this file".into()); } @@ -83,27 +93,39 @@ impl NativeParquetWriter { let schema = Arc::new(arrow::datatypes::Schema::try_from(&arrow_schema)?); log_debug!("Schema created with {} fields", schema.fields().len()); - let file = File::create(&filename)?; - let file_clone = file.try_clone()?; - FILE_MANAGER.insert(filename.clone(), file_clone); - - let props = WriterProperties::builder() - .set_compression(Compression::LZ4_RAW) - .set_bloom_filter_enabled(true) - .set_bloom_filter_fpp(0.1) - .set_bloom_filter_ndv(100000) - .build(); - let crc_writer = Crc32Writer::new(file); - let writer = ArrowWriter::try_new(crc_writer, schema, Some(props))?; - WRITER_MANAGER.insert(filename, Arc::new(Mutex::new(writer))); + let file = File::create(&temp_filename)?; + let (crc_file, crc_handle) = CrcWriter::new(file); + + let mut settings: NativeSettings = SETTINGS_STORE + .get(&index_name) + .map(|r| r.clone()) + .unwrap_or_default(); + settings.index_name = Some(index_name.clone()); + settings.sort_columns = sort_columns; + settings.reverse_sorts = reverse_sorts; + settings.nulls_first = nulls_first; + + let props = WriterPropertiesBuilder::build(&settings); + + SETTINGS_STORE.insert(index_name, settings.clone()); + + let writer = ArrowWriter::try_new(crc_file, schema, Some(props))?; + + WRITERS.insert(temp_filename, WriterState { + writer: Arc::new(Mutex::new(writer)), + settings, + crc_handle, + }); + Ok(()) } pub fn write_data(filename: String, array_address: i64, schema_address: i64) -> Result<(), Box> { - log_debug!("write_data called for file: {}", filename); + let temp_filename = Self::temp_filename(&filename); + log_debug!("write_data called for file: {} (temp: {})", filename, temp_filename); if (array_address as *mut u8).is_null() || (schema_address as *mut u8).is_null() { - log_error!("ERROR: Invalid FFI addresses for file: {}", filename); + log_error!("ERROR: Invalid FFI addresses for file: {}", temp_filename); return Err("Invalid FFI addresses (null pointers)".into()); } @@ -118,12 +140,12 @@ impl NativeParquetWriter { let record_batch = RecordBatch::try_new(schema, struct_array.columns().to_vec())?; log_debug!("Created RecordBatch with {} rows and {} columns", record_batch.num_rows(), record_batch.num_columns()); - if let Some(writer_arc) = WRITER_MANAGER.get(&filename) { - let mut writer = writer_arc.lock().unwrap(); + if let Some(state) = WRITERS.get(&temp_filename) { + let mut writer = state.writer.lock().unwrap(); writer.write(&record_batch)?; Ok(()) } else { - log_error!("ERROR: No writer found for file: {}", filename); + log_error!("ERROR: No writer found for temp file: {}", temp_filename); Err("Writer not found".into()) } } else { @@ -134,30 +156,267 @@ impl NativeParquetWriter { } pub fn finalize_writer(filename: String) -> Result, Box> { - log_debug!("finalize_writer called for file: {}", filename); + let temp_filename = Self::temp_filename(&filename); + log_debug!("finalize_writer called for file: {} (temp: {})", filename, temp_filename); - if let Some((_, writer_arc)) = WRITER_MANAGER.remove(&filename) { + if let Some((_, state)) = WRITERS.remove(&temp_filename) { + let WriterState { writer: writer_arc, settings, crc_handle } = state; + let index_name = settings.index_name.as_deref().unwrap_or(""); match Arc::try_unwrap(writer_arc) { Ok(mutex) => { - let mut writer = mutex.into_inner().unwrap(); - let parquet_metadata = writer.finish()?; - let file_metadata = parquet_metadata.file_metadata(); - log_debug!("Successfully finalized writer for file: {}, num_rows={}", filename, file_metadata.num_rows()); - let crc32 = writer.inner().checksum(); - log_debug!("CRC32 for file {}: {:#010x}", filename, crc32); - Ok(Some(FinalizeResult { metadata: parquet_metadata, crc32 })) + let writer = mutex.into_inner().unwrap(); + match writer.close() { + Ok(_) => { + let temp_crc32 = crc_handle.crc32(); + log_debug!("Successfully closed temp writer for: {}", temp_filename); + + let crc32 = Self::sort_and_rewrite_parquet(&temp_filename, &filename, index_name, &settings.sort_columns, &settings.reverse_sorts, &settings.nulls_first, temp_crc32)?; + + if Path::new(&temp_filename).exists() { + if let Err(e) = std::fs::remove_file(&temp_filename) { + log_error!("Failed to remove temp file {}: {}", temp_filename, e); + } + } + + log_debug!("CRC32 for file {}: {:#010x}", filename, crc32); + + // Keep a handle for sync_to_disk + let file_for_sync = File::open(&filename)?; + FILE_MANAGER.insert(filename.clone(), file_for_sync); + + // Read full ParquetMetaData from the final file + let file = File::open(&filename)?; + let reader = SerializedFileReader::new(file)?; + let parquet_metadata = reader.metadata().clone(); + + Ok(Some(FinalizeResult { metadata: parquet_metadata, crc32 })) + } + Err(e) => { + log_error!("ERROR: Failed to close writer for temp file: {}", temp_filename); + Err(e.into()) + } + } } Err(_) => { - log_error!("ERROR: Writer still in use for file: {}", filename); + log_error!("ERROR: Writer still in use for temp file: {}", temp_filename); Err("Writer still in use".into()) } } } else { - log_error!("ERROR: Writer not found for file: {}", filename); + log_error!("ERROR: Writer not found for temp file: {}", temp_filename); Err("Writer not found".into()) } } + + fn sort_and_rewrite_parquet( + temp_filename: &str, + output_filename: &str, + index_name: &str, + sort_columns: &[String], + reverse_sorts: &[bool], + nulls_first: &[bool], + temp_crc32: u32, + ) -> Result> { + log_debug!( + "sort_and_rewrite_parquet: temp={}, output={}, sort_columns={:?}, reverse_sorts={:?}, nulls_first={:?}", + temp_filename, output_filename, sort_columns, reverse_sorts, nulls_first + ); + + if sort_columns.is_empty() { + log_debug!("No sort columns specified, renaming temp file to final"); + std::fs::rename(temp_filename, output_filename)?; + return Ok(temp_crc32); + } + + let config = SETTINGS_STORE + .get(index_name) + .map(|r| r.clone()) + .unwrap_or_default(); + + let file_size = std::fs::metadata(temp_filename)?.len(); + + if file_size <= config.get_sort_in_memory_threshold_bytes() { + Self::sort_small_file(temp_filename, output_filename, index_name, sort_columns, reverse_sorts, nulls_first) + } else { + Self::sort_large_file(temp_filename, output_filename, index_name, sort_columns, reverse_sorts, nulls_first, config.get_sort_batch_size()) + } + } + + /// In-memory sort for small files: read all batches, concat, sort, rewrite row IDs, write. + fn sort_small_file( + temp_filename: &str, + output_filename: &str, + index_name: &str, + sort_columns: &[String], + reverse_sorts: &[bool], + nulls_first: &[bool], + ) -> Result> { + log_debug!("Using in-memory sort for small file: {}", temp_filename); + + let file = File::open(temp_filename)?; + let builder = ParquetRecordBatchReaderBuilder::try_new(file)?; + let row_count = builder.metadata().file_metadata().num_rows() as usize; + let arrow_reader = builder.with_batch_size(row_count).build()?; + + let batch = match arrow_reader.into_iter().next() { + Some(Ok(b)) if b.num_rows() > 0 => b, + Some(Err(e)) => return Err(e.into()), + _ => { + log_debug!("No data to sort in file: {}", temp_filename); + std::fs::rename(temp_filename, output_filename)?; + return Ok(0); + } + }; + + let schema = batch.schema(); + let sorted_batch = Self::sort_batch(&batch, sort_columns, reverse_sorts, nulls_first)?; + let final_batch = Self::rewrite_row_ids(&sorted_batch, &schema)?; + + let crc32 = Self::write_final_file(output_filename, index_name, &final_batch, schema)?; + Ok(crc32) + } + + /// For large files: read in batches, sort each batch individually, write each + /// as a separate sorted chunk file, then use the streaming k-way merge to + /// produce the final globally-sorted output. + fn sort_large_file( + temp_filename: &str, + output_filename: &str, + index_name: &str, + sort_columns: &[String], + reverse_sorts: &[bool], + nulls_first: &[bool], + batch_size: usize, + ) -> Result> { + log_debug!("Using streaming merge sort for large file: {}", temp_filename); + + let file = File::open(temp_filename)?; + let builder = ParquetRecordBatchReaderBuilder::try_new(file)?; + let arrow_reader = builder.with_batch_size(batch_size).build()?; + + let mut chunk_paths: Vec = Vec::new(); + let mut batch_count = 0; + let chunk_dir = Path::new(output_filename).parent().unwrap_or_else(|| Path::new(".")); + + for batch_result in arrow_reader { + let batch = batch_result?; + let schema = batch.schema(); + let sorted_batch = Self::sort_batch(&batch, sort_columns, reverse_sorts, nulls_first)?; + + let chunk_filename = chunk_dir + .join(format!("temp_sort_chunk_{}_{}.parquet", batch_count, std::process::id())) + .to_string_lossy() + .to_string(); + // CRC for temp chunks is not needed, discard it + Self::write_final_file(&chunk_filename, index_name, &sorted_batch, schema)?; + + chunk_paths.push(chunk_filename); + batch_count += 1; + } + + if chunk_paths.is_empty() { + log_debug!("No data to sort in file: {}", temp_filename); + std::fs::rename(temp_filename, output_filename)?; + return Ok(0); + } + + log_debug!("Created {} sorted chunks, merging via streaming k-way merge", batch_count); + + // merge_sorted returns CRC32 of the final merged output + let crc32 = merge_sorted( + &chunk_paths, + output_filename, + index_name, + sort_columns, + reverse_sorts, + nulls_first, + ).map_err(|e| -> Box { format!("Streaming merge failed: {}", e).into() })?; + + // Clean up temp chunk files + for path in &chunk_paths { + let _ = std::fs::remove_file(path); + } + + Ok(crc32) + } + + fn sort_batch( + batch: &RecordBatch, + sort_columns: &[String], + reverse_sorts: &[bool], + nulls_first: &[bool], + ) -> Result> { + let columns: Vec = sort_columns + .iter() + .enumerate() + .map(|(i, col_name)| { + let reverse = reverse_sorts.get(i).copied().unwrap_or(false); + let nf = nulls_first.get(i).copied().unwrap_or(false); + let options = arrow::compute::SortOptions { + descending: reverse, + nulls_first: nf, + }; + let col_index = batch.schema().index_of(col_name) + .map_err(|_| format!("Sort column '{}' not found in schema", col_name))?; + Ok(SortColumn { + values: batch.column(col_index).clone(), + options: Some(options), + }) + }) + .collect::, Box>>()?; + + let indices = lexsort_to_indices(&columns, None)?; + let sorted_columns: Result, _> = batch + .columns() + .iter() + .map(|col| take(col.as_ref(), &indices, None)) + .collect(); + + Ok(RecordBatch::try_new(batch.schema(), sorted_columns?)?) + } + + /// If a __row_id__ column exists, rewrite it with sequential values 0..N. + fn rewrite_row_ids( + batch: &RecordBatch, + schema: &Arc, + ) -> Result> { + use arrow::array::Int64Array; + + if let Some(row_id_idx) = schema.fields().iter().position(|f| f.name() == ROW_ID_COLUMN_NAME) { + log_debug!("Rewriting __row_id__ column with sequential values 0..{}", batch.num_rows()); + let sequential_ids = Int64Array::from_iter_values( + (0..batch.num_rows() as u64).map(|x| x as i64) + ); + let mut new_columns = batch.columns().to_vec(); + new_columns[row_id_idx] = Arc::new(sequential_ids); + Ok(RecordBatch::try_new(schema.clone(), new_columns)?) + } else { + Ok(batch.clone()) + } + } + + fn write_final_file( + output_filename: &str, + index_name: &str, + batch: &RecordBatch, + schema: Arc, + ) -> Result> { + let config = SETTINGS_STORE + .get(index_name) + .map(|r| r.clone()) + .unwrap_or_default(); + let props = WriterPropertiesBuilder::build(&config); + let file = File::create(output_filename)?; + let (crc_file, crc_handle) = CrcWriter::new(file); + let mut writer = ArrowWriter::try_new(crc_file, schema, Some(props))?; + writer.write(batch)?; + writer.close()?; + let crc32 = crc_handle.crc32(); + log_debug!("Successfully wrote final file: {} (crc32={:#010x})", output_filename, crc32); + Ok(crc32) + } + pub fn sync_to_disk(filename: String) -> Result<(), Box> { log_debug!("sync_to_disk called for file: {}", filename); @@ -175,9 +434,9 @@ impl NativeParquetWriter { pub fn get_filtered_writer_memory_usage(path_prefix: String) -> Result> { let mut total_memory = 0; - for entry in WRITER_MANAGER.iter() { + for entry in WRITERS.iter() { if entry.key().starts_with(&path_prefix) { - if let Ok(writer) = entry.value().lock() { + if let Ok(writer) = entry.value().writer.lock() { total_memory += writer.memory_size(); } } diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/writer_properties_builder.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer_properties_builder.rs new file mode 100644 index 0000000000000..d5676a33c9e0f --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer_properties_builder.rs @@ -0,0 +1,199 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use parquet::basic::{Compression, ZstdLevel, GzipLevel, BrotliLevel}; +use parquet::file::properties::WriterProperties; + +use crate::native_settings::NativeSettings; + +/// Builder for converting NativeSettings into Parquet WriterProperties. +/// +/// This struct follows the Single Responsibility Principle by focusing +/// solely on the conversion logic from configuration to Parquet properties. +/// +/// # Design Principles +/// +/// - **Single Responsibility**: Only handles WriterProperties construction +/// - **Open/Closed**: Can be extended with new compression types without modification +/// - **Dependency Inversion**: Depends on NativeSettings abstraction +pub struct WriterPropertiesBuilder; + +impl WriterPropertiesBuilder { + /// Builds WriterProperties from a NativeSettings. + /// + /// This method applies both index-level and field-level configurations + /// to create a complete WriterProperties instance for Parquet writing. + /// + /// # Arguments + /// + /// * `config` - The native settings to convert + /// + /// # Returns + /// + /// A fully configured WriterProperties instance + pub fn build(config: &NativeSettings) -> WriterProperties { + let mut builder = WriterProperties::builder(); + + // Apply compression settings + builder = Self::apply_compression_settings(builder, config); + + // Apply page settings + builder = Self::apply_page_settings(builder, config); + + // Apply row group settings + builder = Self::apply_row_group_settings(builder, config); + + // Apply dictionary settings + builder = Self::apply_dictionary_settings(builder, config); + + // Apply bloom filter settings + builder = Self::apply_bloom_filter_settings(builder, config); + + // Apply field-level configurations + builder = Self::apply_field_configs(builder, config); + + builder.build() + } + + /// Applies compression settings to the builder. + fn apply_compression_settings( + mut builder: parquet::file::properties::WriterPropertiesBuilder, + config: &NativeSettings + ) -> parquet::file::properties::WriterPropertiesBuilder { + let compression = Self::parse_compression_type( + config.get_compression_type(), + config.get_compression_level() + ); + builder = builder.set_compression(compression); + builder + } + + /// Applies page size and row limit settings. + fn apply_page_settings( + mut builder: parquet::file::properties::WriterPropertiesBuilder, + config: &NativeSettings + ) -> parquet::file::properties::WriterPropertiesBuilder { + builder = builder.set_data_page_size_limit(config.get_page_size_bytes()); + builder = builder.set_data_page_row_count_limit(config.get_page_row_limit()); + builder + } + + /// Applies row group row count limit. + /// In parquet-rs 57.x, `set_max_row_group_size` is a row count limit (not bytes). + fn apply_row_group_settings( + builder: parquet::file::properties::WriterPropertiesBuilder, + config: &NativeSettings + ) -> parquet::file::properties::WriterPropertiesBuilder { + builder + .set_max_row_group_size(config.get_row_group_max_rows()) + } + + /// Applies dictionary encoding settings. + fn apply_dictionary_settings( + mut builder: parquet::file::properties::WriterPropertiesBuilder, + config: &NativeSettings + ) -> parquet::file::properties::WriterPropertiesBuilder { + builder = builder.set_dictionary_page_size_limit(config.get_dict_size_bytes()); + builder + } + + /// Applies bloom filter settings. + fn apply_bloom_filter_settings( + mut builder: parquet::file::properties::WriterPropertiesBuilder, + config: &NativeSettings + ) -> parquet::file::properties::WriterPropertiesBuilder { + builder = builder.set_bloom_filter_enabled(config.get_bloom_filter_enabled()); + builder = builder.set_bloom_filter_fpp(config.get_bloom_filter_fpp()); + builder = builder.set_bloom_filter_ndv(config.get_bloom_filter_ndv()); + builder + } + + /// Applies field-level configurations. + fn apply_field_configs( + mut builder: parquet::file::properties::WriterPropertiesBuilder, + config: &NativeSettings + ) -> parquet::file::properties::WriterPropertiesBuilder { + if let Some(field_configs) = &config.field_configs { + for (field_name, field_config) in field_configs { + if let Some(compression_type) = &field_config.compression_type { + let compression = Self::parse_compression_type( + compression_type, + field_config.compression_level.unwrap_or(3) + ); + builder = builder.set_column_compression(field_name.clone().into(), compression); + } + } + } + builder + } + + /// Parses compression type string to Parquet Compression enum. + /// + /// # Arguments + /// + /// * `compression_type` - String representation of compression type + /// * `level` - Compression level (algorithm-dependent) + /// + /// # Returns + /// + /// Appropriate Compression enum variant + fn parse_compression_type(compression_type: &str, level: i32) -> Compression { + match compression_type.to_uppercase().as_str() { + "ZSTD" => Compression::ZSTD( + ZstdLevel::try_new(level).unwrap_or(ZstdLevel::default()) + ), + "SNAPPY" => Compression::SNAPPY, + "GZIP" => Compression::GZIP( + GzipLevel::try_new(level as u32).unwrap_or_default() + ), + "LZ4" => Compression::LZ4, + "BROTLI" => Compression::BROTLI( + BrotliLevel::try_new(level as u32).unwrap_or_default() + ), + "LZ4_RAW" => Compression::LZ4_RAW, + "UNCOMPRESSED" => Compression::UNCOMPRESSED, + _ => Compression::ZSTD(ZstdLevel::try_new(level).unwrap_or(ZstdLevel::default())) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::native_settings::NativeSettings; + + #[test] + fn test_build_with_compression() { + let config = NativeSettings { + compression_type: Some("ZSTD".to_string()), + compression_level: Some(5), + ..Default::default() + }; + + let props = WriterPropertiesBuilder::build(&config); + assert_ne!(props.compression(&parquet::schema::types::ColumnPath::from("test")), Compression::UNCOMPRESSED); + } + + #[test] + fn test_parse_compression_types() { + assert!(matches!( + WriterPropertiesBuilder::parse_compression_type("ZSTD", 3), + Compression::ZSTD(_) + )); + + assert!(matches!( + WriterPropertiesBuilder::parse_compression_type("SNAPPY", 0), + Compression::SNAPPY + )); + + assert!(matches!( + WriterPropertiesBuilder::parse_compression_type("GZIP", 6), + Compression::GZIP(_) + )); + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/tests/merge_integration_tests.rs b/sandbox/plugins/parquet-data-format/src/main/rust/tests/merge_integration_tests.rs new file mode 100644 index 0000000000000..c05f865381991 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/tests/merge_integration_tests.rs @@ -0,0 +1,185 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use arrow::array::{Array, PrimitiveArray}; +use arrow::array::types::TimestampMillisecondType; +use opensearch_parquet_format::merge::{merge_sorted, merge_unsorted}; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use parquet::file::reader::{FileReader, SerializedFileReader}; +use std::fs::File; +use std::path::Path; +use tempfile::tempdir; + +/// Helper: collect all parquet files in a directory (sorted by name). +fn list_parquet_files(dir: &str) -> Vec { + let mut files: Vec = std::fs::read_dir(dir) + .expect("cannot read directory") + .filter_map(|e| { + let p = e.ok()?.path(); + if p.extension().and_then(|s| s.to_str()) == Some("parquet") + && !p.file_name()?.to_str()?.starts_with("merged") + { + Some(p.to_string_lossy().to_string()) + } else { + None + } + }) + .collect(); + files.sort(); + files +} + +/// Helper: count total rows across input files. +fn count_rows_in_files(files: &[String]) -> i64 { + files + .iter() + .map(|f| { + let reader = SerializedFileReader::new(File::open(f).unwrap()).unwrap(); + reader.metadata().file_metadata().num_rows() + }) + .sum() +} + +/// Helper: count rows in a single parquet file. +fn count_rows(path: &str) -> i64 { + let reader = SerializedFileReader::new(File::open(path).unwrap()).unwrap(); + reader.metadata().file_metadata().num_rows() +} + +fn input_dir() -> Option { + std::env::var("PARQUET_TEST_INPUT_DIR").ok() +} + +#[test] +fn test_unsorted_merge_real_files() { + let Some(input_dir) = input_dir() else { + eprintln!("Skipping: PARQUET_TEST_INPUT_DIR not set"); + return; + }; + if !Path::new(&input_dir).exists() { + eprintln!("Skipping: {} not found", input_dir); + return; + } + + let files = list_parquet_files(&input_dir); + assert!(!files.is_empty(), "No parquet files found in {}", input_dir); + println!("Found {} input files", files.len()); + + let expected_rows = count_rows_in_files(&files); + println!("Total input rows: {}", expected_rows); + + let tmp = tempdir().unwrap(); + let output = tmp.path().join("merged_unsorted.parquet"); + let output_str = output.to_string_lossy().to_string(); + + // Empty sort columns → unsorted merge + merge_unsorted(&files, &output_str, "test-index").unwrap(); + + assert!(output.exists(), "Output file was not created"); + let actual_rows = count_rows(&output_str); + println!("Output rows: {}", actual_rows); + assert_eq!(actual_rows, expected_rows, "Row count mismatch"); +} + +/// Verify that __row_id__ in the output is monotonically increasing (0, 1, 2, ...). +fn verify_row_id_order(path: &str) { + let file = File::open(path).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap(); + let schema = builder.schema().clone(); + let col_idx = schema.index_of("__row_id__").expect("__row_id__ not in output"); + let reader = builder.build().unwrap(); + + let mut expected: i64 = 0; + for batch in reader { + let batch = batch.unwrap(); + let col = batch.column(col_idx).as_any() + .downcast_ref::() + .expect("__row_id__ should be Int64"); + for i in 0..col.len() { + assert!(!col.is_null(i), "__row_id__ should never be null"); + assert_eq!(col.value(i), expected, "__row_id__ gap at row {}", expected); + expected += 1; + } + } + println!("Verified __row_id__ is sequential 0..{}", expected); +} + + +#[test] +fn test_sorted_merge_real_files() { + let Some(input_dir) = input_dir() else { + eprintln!("Skipping: PARQUET_TEST_INPUT_DIR not set"); + return; + }; + if !Path::new(&input_dir).exists() { + eprintln!("Skipping: {} not found", input_dir); + return; + } + + let files = list_parquet_files(&input_dir); + assert!(!files.is_empty(), "No parquet files found in {}", input_dir); + + let expected_rows = count_rows_in_files(&files); + println!("Total input rows: {}", expected_rows); + + let tmp = tempdir().unwrap(); + let output = tmp.path().join("merged_sorted.parquet"); + let output_str = output.to_string_lossy().to_string(); + + // Sort by EventDate ascending (each input file is pre-sorted by EventDate) + let sort_cols = vec!["EventDate".to_string()]; + let reverse = vec![false]; + let nulls_first = vec![false]; + + merge_sorted(&files, &output_str, "test-index", &sort_cols, &reverse, &nulls_first) + .unwrap(); + + assert!(output.exists(), "Output file was not created"); + let actual_rows = count_rows(&output_str); + println!("Output rows: {}", actual_rows); + assert_eq!(actual_rows, expected_rows, "Row count mismatch"); + + // Verify __row_id__ is sequential 0..N + verify_row_id_order(&output_str); + + // Verify EventDate is non-decreasing in the merged output + let file = File::open(&output_str).unwrap(); + let builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap(); + let out_schema = builder.schema().clone(); + let col_idx = out_schema.index_of("EventDate").unwrap(); + let reader = builder.build().unwrap(); + + let mut prev: Option = None; + let mut rows_checked: i64 = 0; + let mut out_of_order: i64 = 0; + + for batch in reader { + let batch = batch.unwrap(); + let col = batch.column(col_idx).as_any() + .downcast_ref::>() + .unwrap(); + for i in 0..col.len() { + if col.is_null(i) { continue; } + let val = col.value(i); + if let Some(p) = prev { + if val < p { + out_of_order += 1; + if out_of_order <= 5 { + eprintln!("Out of order at row {}: prev={}, cur={}", rows_checked, p, val); + } + } + } + prev = Some(val); + rows_checked += 1; + } + } + + println!("Verified EventDate sort order across {} non-null rows", rows_checked); + assert_eq!(out_of_order, 0, "Found {} out-of-order rows in EventDate", out_of_order); +} + diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/tests/sort_types_tests.rs b/sandbox/plugins/parquet-data-format/src/main/rust/tests/sort_types_tests.rs new file mode 100644 index 0000000000000..d1ec9f3527821 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/rust/tests/sort_types_tests.rs @@ -0,0 +1,497 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Tests for merge_sorted across all supported sort column types: +//! Int64, Int32, Float64, Float32, Utf8, and multi-column combinations. + +use std::fs::File; +use std::sync::Arc; + +use arrow::array::*; +use arrow::datatypes::{DataType, Field, Schema}; +use opensearch_parquet_format::merge::merge_sorted; +use parquet::arrow::ArrowWriter; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use tempfile::tempdir; + +/// Write a single RecordBatch to a new Parquet file. +fn write_parquet(path: &str, batch: &RecordBatch) { + let file = File::create(path).unwrap(); + let mut writer = ArrowWriter::try_new(file, batch.schema(), None).unwrap(); + writer.write(batch).unwrap(); + writer.close().unwrap(); +} + +/// Read all values of a typed primitive column from a Parquet file. +fn read_primitive_col( + path: &str, + col_name: &str, +) -> Vec> { + let file = File::open(path).unwrap(); + let reader = ParquetRecordBatchReaderBuilder::try_new(file) + .unwrap() + .build() + .unwrap(); + let mut vals = Vec::new(); + for batch in reader { + let batch = batch.unwrap(); + let idx = batch.schema().index_of(col_name).unwrap(); + let col = batch.column(idx).as_primitive::(); + for i in 0..col.len() { + if col.is_null(i) { + vals.push(None); + } else { + vals.push(Some(col.value(i))); + } + } + } + vals +} + +/// Read all string values from a Utf8 column. +fn read_string_col(path: &str, col_name: &str) -> Vec> { + let file = File::open(path).unwrap(); + let reader = ParquetRecordBatchReaderBuilder::try_new(file) + .unwrap() + .build() + .unwrap(); + let mut vals = Vec::new(); + for batch in reader { + let batch = batch.unwrap(); + let idx = batch.schema().index_of(col_name).unwrap(); + let col = batch.column(idx).as_string::(); + for i in 0..col.len() { + if col.is_null(i) { + vals.push(None); + } else { + vals.push(Some(col.value(i).to_string())); + } + } + } + vals +} + +/// Count rows in a Parquet file. +fn count_rows(path: &str) -> usize { + let file = File::open(path).unwrap(); + let reader = ParquetRecordBatchReaderBuilder::try_new(file) + .unwrap() + .build() + .unwrap(); + reader.map(|b| b.unwrap().num_rows()).sum() +} + +// ─── Int64 ────────────────────────────────────────────────────────────────── + +#[test] +fn test_merge_sort_by_int64() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Int64, false), + ])); + + // File A: [1, 3, 5] File B: [2, 4, 6] File C: [0, 7, 8] + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![1, 3, 5]))]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![2, 4, 6]))]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![0, 7, 8]))]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[false], &[false]).unwrap(); + + let vals = read_primitive_col::(&output, "val"); + let vals: Vec = vals.into_iter().map(|v| v.unwrap()).collect(); + assert_eq!(vals, vec![0, 1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(count_rows(&output), 9); +} + +// ─── Int64 with nulls ─────────────────────────────────────────────────────── + +#[test] +fn test_merge_sort_by_int64_with_nulls() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Int64, true), + ])); + + // Each file pre-sorted: nulls last, then ascending + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![Some(1), Some(5), None]))]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![Some(2), Some(4), None]))]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[false], &[false]).unwrap(); + + let vals = read_primitive_col::(&output, "val"); + assert_eq!(vals, vec![Some(1), Some(2), Some(4), Some(5), None, None]); +} + +// ─── Int32 ────────────────────────────────────────────────────────────────── + +#[test] +fn test_merge_sort_by_int32() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Int32, false), + ])); + + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![10, 30]))]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![20, 40]))]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[false], &[false]).unwrap(); + + let vals = read_primitive_col::(&output, "val"); + let vals: Vec = vals.into_iter().map(|v| v.unwrap()).collect(); + assert_eq!(vals, vec![10, 20, 30, 40]); +} + +// ─── Float64 ──────────────────────────────────────────────────────────────── + +#[test] +fn test_merge_sort_by_float64() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Float64, false), + ])); + + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![Arc::new(Float64Array::from(vec![1.1, 3.3, 5.5]))]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![Arc::new(Float64Array::from(vec![2.2, 4.4, 6.6]))]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[false], &[false]).unwrap(); + + let vals = read_primitive_col::(&output, "val"); + let vals: Vec = vals.into_iter().map(|v| v.unwrap()).collect(); + assert_eq!(vals, vec![1.1, 2.2, 3.3, 4.4, 5.5, 6.6]); +} + +// ─── Float64 with nulls ───────────────────────────────────────────────────── + +#[test] +fn test_merge_sort_by_float64_with_nulls() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Float64, true), + ])); + + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![Arc::new(Float64Array::from(vec![None, Some(1.5), Some(4.0)]))]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![Arc::new(Float64Array::from(vec![None, Some(2.5), Some(3.0)]))]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[false], &[true]).unwrap(); + + let vals = read_primitive_col::(&output, "val"); + assert_eq!(vals, vec![None, None, Some(1.5), Some(2.5), Some(3.0), Some(4.0)]); +} + +// ─── Float32 ──────────────────────────────────────────────────────────────── + +#[test] +fn test_merge_sort_by_float32() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Float32, false), + ])); + + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![Arc::new(Float32Array::from(vec![1.0f32, 3.0]))]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![Arc::new(Float32Array::from(vec![2.0f32, 4.0]))]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[false], &[false]).unwrap(); + + let vals = read_primitive_col::(&output, "val"); + let vals: Vec = vals.into_iter().map(|v| v.unwrap()).collect(); + assert_eq!(vals, vec![1.0, 2.0, 3.0, 4.0]); +} + +// ─── Float32 with nulls ───────────────────────────────────────────────────── + +#[test] +fn test_merge_sort_by_float32_with_nulls() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Float32, true), + ])); + + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![Arc::new(Float32Array::from(vec![Some(1.0f32), Some(3.0), None]))]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![Arc::new(Float32Array::from(vec![Some(2.0f32), None, None]))]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[false], &[false]).unwrap(); + + let vals = read_primitive_col::(&output, "val"); + assert_eq!(vals, vec![Some(1.0), Some(2.0), Some(3.0), None, None, None]); +} + +// ─── Utf8 (String / keyword) ─────────────────────────────────────────────── + +#[test] +fn test_merge_sort_by_string() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Utf8, false), + ])); + + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![Arc::new(StringArray::from(vec!["apple", "cherry", "fig"]))]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![Arc::new(StringArray::from(vec!["banana", "date", "grape"]))]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[false], &[false]).unwrap(); + + let vals = read_string_col(&output, "val"); + let vals: Vec = vals.into_iter().map(|v| v.unwrap()).collect(); + assert_eq!(vals, vec!["apple", "banana", "cherry", "date", "fig", "grape"]); +} + +// ─── Utf8 with nulls ──────────────────────────────────────────────────────── + +#[test] +fn test_merge_sort_by_string_with_nulls() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Utf8, true), + ])); + + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![ + Arc::new(StringArray::from(vec![None, Some("banana"), Some("fig")])), + ]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![ + Arc::new(StringArray::from(vec![None, Some("apple"), Some("cherry")])), + ]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[false], &[true]).unwrap(); + + let vals = read_string_col(&output, "val"); + assert_eq!(vals, vec![None, None, Some("apple".into()), Some("banana".into()), Some("cherry".into()), Some("fig".into())]); +} + +// ─── Descending sort ──────────────────────────────────────────────────────── + +#[test] +fn test_merge_sort_descending() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Int64, false), + ])); + + // Each file sorted descending + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![8, 5, 2]))]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![7, 4, 1]))]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![9, 6, 3]))]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[true], &[false]).unwrap(); + + let vals = read_primitive_col::(&output, "val"); + let vals: Vec = vals.into_iter().map(|v| v.unwrap()).collect(); + assert_eq!(vals, vec![9, 8, 7, 6, 5, 4, 3, 2, 1]); +} + +// ─── Multi-column: String + Int64 ────────────────────────────────────────── + +#[test] +fn test_merge_sort_multi_column_string_and_int() { + let schema = Arc::new(Schema::new(vec![ + Field::new("category", DataType::Utf8, false), + Field::new("priority", DataType::Int64, false), + ])); + + // File A: (alpha,1), (alpha,3), (beta,1) + // File B: (alpha,2), (beta,2), (beta,3) + // Sorted by (category ASC, priority ASC) + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![ + Arc::new(StringArray::from(vec!["alpha", "alpha", "beta"])), + Arc::new(Int64Array::from(vec![1, 3, 1])), + ]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![ + Arc::new(StringArray::from(vec!["alpha", "beta", "beta"])), + Arc::new(Int64Array::from(vec![2, 2, 3])), + ]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted( + &files, &output, "test", + &["category".into(), "priority".into()], + &[false, false], + &[false, false], + ).unwrap(); + + let cats = read_string_col(&output, "category"); + let cats: Vec = cats.into_iter().map(|v| v.unwrap()).collect(); + let pris = read_primitive_col::(&output, "priority"); + let pris: Vec = pris.into_iter().map(|v| v.unwrap()).collect(); + + assert_eq!(cats, vec!["alpha", "alpha", "alpha", "beta", "beta", "beta"]); + assert_eq!(pris, vec![1, 2, 3, 1, 2, 3]); +} + +// ─── Nulls ────────────────────────────────────────────────────────────────── + +#[test] +fn test_merge_sort_with_nulls_first() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Int64, true), + ])); + + // Each file pre-sorted with nulls first, then ascending + // File A: [null, 2, 5] File B: [null, 1, 4] + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![ + Arc::new(Int64Array::from(vec![None, Some(2), Some(5)])), + ]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![ + Arc::new(Int64Array::from(vec![None, Some(1), Some(4)])), + ]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[false], &[true]).unwrap(); + + let vals = read_primitive_col::(&output, "val"); + // nulls_first=true → nulls come first, then ascending + assert_eq!(vals, vec![None, None, Some(1), Some(2), Some(4), Some(5)]); +} + +#[test] +fn test_merge_sort_with_nulls_last() { + let schema = Arc::new(Schema::new(vec![ + Field::new("val", DataType::Int64, true), + ])); + + let batches = vec![ + RecordBatch::try_new(schema.clone(), vec![ + Arc::new(Int64Array::from(vec![Some(1), Some(3), None])), + ]).unwrap(), + RecordBatch::try_new(schema.clone(), vec![ + Arc::new(Int64Array::from(vec![Some(2), None, None])), + ]).unwrap(), + ]; + + let tmp = tempdir().unwrap(); + let files: Vec = batches.iter().enumerate().map(|(i, b)| { + let p = tmp.path().join(format!("input_{}.parquet", i)); + let s = p.to_string_lossy().to_string(); + write_parquet(&s, b); + s + }).collect(); + + let output = tmp.path().join("merged.parquet").to_string_lossy().to_string(); + merge_sorted(&files, &output, "test", &["val".into()], &[false], &[false]).unwrap(); + + let vals = read_primitive_col::(&output, "val"); + // nulls_first=false → values ascending, then nulls + assert_eq!(vals, vec![Some(1), Some(2), Some(3), None, None, None]); +} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/tests/writer_integration_tests.rs b/sandbox/plugins/parquet-data-format/src/main/rust/tests/writer_integration_tests.rs index 8a0bc1c6c8778..2a67511ba9c8f 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/tests/writer_integration_tests.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/tests/writer_integration_tests.rs @@ -6,8 +6,8 @@ * compatible open source license. */ -use parquet_dataformat_jni::test_utils::*; -use parquet_dataformat_jni::writer::NativeParquetWriter; +use opensearch_parquet_format::test_utils::*; +use opensearch_parquet_format::writer::NativeParquetWriter; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::thread; @@ -27,7 +27,6 @@ fn test_complete_writer_lifecycle() { let metadata = close_writer_and_get_metadata(&filename, schema_ptr); assert_eq!(metadata.metadata.file_metadata().num_rows(), 9); // 3 batches × 3 rows assert!(metadata.metadata.file_metadata().version() > 0); - assert_eq!(metadata.metadata.file_metadata().schema_descr().num_columns(), 3); // root + 2 fields assert!(NativeParquetWriter::sync_to_disk(filename.clone()).is_ok()); assert!(file_path.exists()); @@ -51,7 +50,7 @@ fn test_concurrent_writer_creation() { let file_path = temp_dir_path.join(format!("concurrent_{}.parquet", i)); let filename = file_path.to_string_lossy().to_string(); let (_schema, schema_ptr) = create_test_ffi_schema(); - if NativeParquetWriter::create_writer(filename.clone(), schema_ptr).is_ok() { + if NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![]).is_ok() { success_count.fetch_add(1, Ordering::SeqCst); let _ = NativeParquetWriter::finalize_writer(filename); } @@ -174,7 +173,7 @@ fn test_concurrent_complete_writer_lifecycle() { let filename = file_path.to_string_lossy().to_string(); let (_schema, schema_ptr) = create_test_ffi_schema(); - if NativeParquetWriter::create_writer(filename.clone(), schema_ptr).is_ok() { + if NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![]).is_ok() { let (array_ptr, data_schema_ptr) = create_test_ffi_data().unwrap(); let write_ok = NativeParquetWriter::write_data(filename.clone(), array_ptr, data_schema_ptr).is_ok(); cleanup_ffi_data(array_ptr, data_schema_ptr); diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/NativeParquetWriterTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/NativeParquetWriterTests.java index 693f35a846a44..e0795008dde03 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/NativeParquetWriterTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/NativeParquetWriterTests.java @@ -144,14 +144,19 @@ public void testWriteAfterFlushThrows() throws Exception { public void testCreateWriterWithNonExistentDirectory() { expectThrows(IOException.class, () -> { try (ArrowExport export = exportSchema()) { - new NativeParquetWriter("/nonexistent/dir/file.parquet", export.getSchemaAddress()); + new NativeParquetWriter( + "/nonexistent/dir/file.parquet", + "test-index", + export.getSchemaAddress(), + ParquetSortConfig.empty() + ); } }); } public void testCreateWriterWithInvalidSchemaAddress() { String filePath = createTempDir().resolve("bad-schema.parquet").toString(); - expectThrows(Exception.class, () -> new NativeParquetWriter(filePath, 0L)); + expectThrows(Exception.class, () -> new NativeParquetWriter(filePath, "test-index", 0L, ParquetSortConfig.empty())); } public void testWriteWithSchemaMismatch() throws Exception { @@ -235,7 +240,7 @@ public void testWriteWithNullAddresses() throws Exception { private NativeParquetWriter createWriter(String filePath) throws Exception { try (ArrowExport export = exportSchema()) { - return new NativeParquetWriter(filePath, export.getSchemaAddress()); + return new NativeParquetWriter(filePath, "test-index", export.getSchemaAddress(), ParquetSortConfig.empty()); } } diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java new file mode 100644 index 0000000000000..cef9bd6f786f0 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java @@ -0,0 +1,159 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.bridge; + +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.nativebridge.spi.ArrowExport; +import org.opensearch.test.OpenSearchTestCase; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; + +public class ParquetMergeIntegrationTests extends OpenSearchTestCase { + + private static final String INDEX_NAME = "merge-test-index"; + private BufferAllocator allocator; + private Schema schema; + + @Override + public void setUp() throws Exception { + super.setUp(); + RustBridge.initLogger(); + allocator = new RootAllocator(); + schema = new Schema( + List.of( + new Field("timestamp", FieldType.nullable(new ArrowType.Int(64, true)), null), + new Field("message", FieldType.nullable(new ArrowType.Utf8()), null) + ) + ); + } + + @Override + public void tearDown() throws Exception { + allocator.close(); + super.tearDown(); + } + + public void testMergeSortedFiles() throws Exception { + // 1. Push settings + NativeSettings settings = NativeSettings.builder().indexName(INDEX_NAME).compressionType("LZ4_RAW").compressionLevel(2).build(); + RustBridge.onSettingsUpdate(settings); + + Path tempDir = createTempDir(); + + // 2. Create 3 sorted files with non-overlapping timestamp ranges + String file1 = createSortedFile(tempDir, "f1.parquet", new long[] { 100, 200, 300 }, new String[] { "a", "b", "c" }); + String file2 = createSortedFile(tempDir, "f2.parquet", new long[] { 400, 500, 600 }, new String[] { "d", "e", "f" }); + String file3 = createSortedFile(tempDir, "f3.parquet", new long[] { 700, 800, 900 }, new String[] { "g", "h", "i" }); + + // Verify individual files + assertEquals(3, RustBridge.getFileMetadata(file1).numRows()); + assertEquals(3, RustBridge.getFileMetadata(file2).numRows()); + assertEquals(3, RustBridge.getFileMetadata(file3).numRows()); + + // 3. Merge + String mergedFile = tempDir.resolve("merged.parquet").toString(); + RustBridge.mergeParquetFilesInRust(List.of(Path.of(file1), Path.of(file2), Path.of(file3)), mergedFile, INDEX_NAME); + + // 4. Verify merged output + ParquetFileMetadata mergedMeta = RustBridge.getFileMetadata(mergedFile); + assertEquals(9, mergedMeta.numRows()); + + // 5. Cleanup + RustBridge.removeSettings(INDEX_NAME); + } + + public void testMergeWithInterleavedTimestamps() throws Exception { + NativeSettings settings = NativeSettings.builder().indexName(INDEX_NAME).compressionType("LZ4_RAW").build(); + RustBridge.onSettingsUpdate(settings); + + Path tempDir = createTempDir(); + + // Interleaved ranges — merge must sort globally + String file1 = createSortedFile(tempDir, "f1.parquet", new long[] { 100, 300, 500 }, new String[] { "a", "c", "e" }); + String file2 = createSortedFile(tempDir, "f2.parquet", new long[] { 200, 400, 600 }, new String[] { "b", "d", "f" }); + + String mergedFile = tempDir.resolve("merged.parquet").toString(); + RustBridge.mergeParquetFilesInRust(List.of(Path.of(file1), Path.of(file2)), mergedFile, INDEX_NAME); + + assertEquals(6, RustBridge.getFileMetadata(mergedFile).numRows()); + + RustBridge.removeSettings(INDEX_NAME); + } + + public void testMergeSingleFile() throws Exception { + NativeSettings settings = NativeSettings.builder().indexName(INDEX_NAME).compressionType("LZ4_RAW").build(); + RustBridge.onSettingsUpdate(settings); + + Path tempDir = createTempDir(); + String file1 = createSortedFile(tempDir, "f1.parquet", new long[] { 10, 20, 30 }, new String[] { "x", "y", "z" }); + + String mergedFile = tempDir.resolve("merged.parquet").toString(); + RustBridge.mergeParquetFilesInRust(List.of(Path.of(file1)), mergedFile, INDEX_NAME); + + assertEquals(3, RustBridge.getFileMetadata(mergedFile).numRows()); + + RustBridge.removeSettings(INDEX_NAME); + } + + /** + * Creates a sorted Parquet file via the full Rust writer pipeline: + * createWriter (with sort config) → write → finalizeWriter. + */ + private String createSortedFile(Path dir, String name, long[] timestamps, String[] messages) throws Exception { + String filePath = dir.resolve(name).toString(); + ParquetSortConfig sortConfig = new ParquetSortConfig(List.of("timestamp"), List.of(false), List.of(false)); + + try (ArrowExport schemaExport = exportSchema()) { + NativeParquetWriter writer = new NativeParquetWriter(filePath, INDEX_NAME, schemaExport.getSchemaAddress(), sortConfig); + + try (ArrowExport dataExport = exportData(timestamps, messages)) { + writer.write(dataExport.getArrayAddress(), dataExport.getSchemaAddress()); + } + + writer.flush(); + } + return filePath; + } + + private ArrowExport exportSchema() { + ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); + Data.exportSchema(allocator, schema, null, arrowSchema); + return new ArrowExport(null, arrowSchema); + } + + private ArrowExport exportData(long[] timestamps, String[] messages) { + try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + BigIntVector tsVec = (BigIntVector) root.getVector("timestamp"); + VarCharVector msgVec = (VarCharVector) root.getVector("message"); + for (int i = 0; i < timestamps.length; i++) { + tsVec.setSafe(i, timestamps[i]); + msgVec.setSafe(i, messages[i].getBytes(StandardCharsets.UTF_8)); + } + root.setRowCount(timestamps.length); + + ArrowArray array = ArrowArray.allocateNew(allocator); + ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); + Data.exportVectorSchemaRoot(allocator, root, null, array, arrowSchema); + return new ArrowExport(array, arrowSchema); + } + } +} diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java index 51cc7619adfe7..2061d614a86c4 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/engine/ParquetIndexingEngineTests.java @@ -10,8 +10,11 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.dataformat.FileInfos; import org.opensearch.index.engine.dataformat.RefreshInput; import org.opensearch.index.engine.dataformat.RefreshResult; @@ -127,7 +130,7 @@ public void testRefreshWithNullInput() throws Exception { assertTrue(result.refreshedSegments().isEmpty()); } - public void testGetMergerReturnsNoopMerger() { + public void testGetMergerReturnsNonNull() { assertNotNull(engine.getMerger()); } @@ -164,7 +167,14 @@ private ParquetIndexingEngine createEngine() { Path dataPath = tempDir.resolve(indexUUID).resolve("0"); Files.createDirectories(dataPath.resolve("parquet")); ShardPath shardPath = new ShardPath(false, dataPath, dataPath, shardId); - return new ParquetIndexingEngine(Settings.EMPTY, new ParquetDataFormat(), shardPath, () -> schema, null, threadPool); + Settings indexSettingsBuilder = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + IndexMetadata indexMetadata = IndexMetadata.builder("test_index").settings(indexSettingsBuilder).build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + return new ParquetIndexingEngine(Settings.EMPTY, new ParquetDataFormat(), shardPath, () -> schema, indexSettings, threadPool); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java index 6ea57eadd03ed..4d289e7c3ea1f 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java @@ -13,7 +13,10 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.parquet.ParquetDataFormatPlugin; import org.opensearch.parquet.bridge.ParquetFileMetadata; @@ -31,6 +34,7 @@ public class VSRManagerTests extends OpenSearchTestCase { private ArrowBufferPool bufferPool; private Schema schema; private ThreadPool threadPool; + private IndexSettings indexSettings; @Override public void setUp() throws Exception { @@ -38,6 +42,13 @@ public void setUp() throws Exception { RustBridge.initLogger(); bufferPool = new ArrowBufferPool(Settings.EMPTY); schema = new Schema(List.of(new Field("val", FieldType.nullable(new ArrowType.Int(32, true)), null))); + Settings indexSettingsBuilder = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + IndexMetadata indexMetadata = IndexMetadata.builder("test-index").settings(indexSettingsBuilder).build(); + indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); Settings settings = Settings.builder().put("node.name", "vsrmanager-test").build(); threadPool = new ThreadPool( settings, @@ -60,7 +71,7 @@ public void tearDown() throws Exception { public void testConstructionInitializesActiveVSR() throws Exception { String filePath = createTempDir().resolve("init.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 50000, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool); assertNotNull(manager.getActiveManagedVSR()); assertEquals(VSRState.ACTIVE, manager.getActiveManagedVSR().getState()); // flush handles freeze + close internally @@ -69,7 +80,7 @@ public void testConstructionInitializesActiveVSR() throws Exception { public void testFlushWithNoDataReturnsMetadata() throws Exception { String filePath = createTempDir().resolve("empty.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 50000, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool); ParquetFileMetadata metadata = manager.flush(); assertNotNull(metadata); assertEquals(0, metadata.numRows()); @@ -77,7 +88,7 @@ public void testFlushWithNoDataReturnsMetadata() throws Exception { public void testFlushWithData() throws Exception { String filePath = createTempDir().resolve("data.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 50000, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool); ManagedVSR active = manager.getActiveManagedVSR(); IntVector vec = (IntVector) active.getVector("val"); @@ -93,7 +104,7 @@ public void testFlushWithData() throws Exception { public void testAddDocument() throws Exception { String filePath = createTempDir().resolve("add-doc.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 50000, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool); NumberFieldMapper.NumberFieldType valField = new NumberFieldMapper.NumberFieldType("val", NumberFieldMapper.NumberType.INTEGER); ParquetDocumentInput doc = new ParquetDocumentInput(); @@ -109,7 +120,7 @@ public void testAddDocument() throws Exception { public void testSyncAfterFlush() throws Exception { String filePath = createTempDir().resolve("sync.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 50000, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool); ManagedVSR active = manager.getActiveManagedVSR(); IntVector vec = (IntVector) active.getVector("val"); @@ -123,7 +134,7 @@ public void testSyncAfterFlush() throws Exception { public void testMaybeRotateNoOpBelowThreshold() throws Exception { String filePath = createTempDir().resolve("norotate.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 50000, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool); ManagedVSR original = manager.getActiveManagedVSR(); original.setRowCount(100); manager.maybeRotateActiveVSR(); @@ -133,7 +144,7 @@ public void testMaybeRotateNoOpBelowThreshold() throws Exception { public void testMaybeRotateAtThreshold() throws Exception { String filePath = createTempDir().resolve("rotate.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 50000, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool); ManagedVSR original = manager.getActiveManagedVSR(); original.setRowCount(50000); @@ -147,7 +158,7 @@ public void testMaybeRotateAtThreshold() throws Exception { public void testFlushAfterRotation() throws Exception { String filePath = createTempDir().resolve("rotate-flush.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 50000, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool); // Fill first VSR to trigger rotation ManagedVSR first = manager.getActiveManagedVSR(); @@ -171,7 +182,7 @@ public void testFlushAfterRotation() throws Exception { public void testRotationAwaitsWhenFrozenSlotOccupied() throws Exception { String filePath = createTempDir().resolve("double-rotate.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 100, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool); // Fill first VSR to trigger rotation (async write submitted) ManagedVSR first = manager.getActiveManagedVSR(); @@ -206,7 +217,7 @@ public void testRotationAwaitsWhenFrozenSlotOccupied() throws Exception { public void testRotationWritesHappenOnBackgroundThread() throws Exception { String filePath = createTempDir().resolve("bg-thread.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 100, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool); // Fill and rotate ManagedVSR first = manager.getActiveManagedVSR(); @@ -235,7 +246,7 @@ public void testRotationWritesHappenOnBackgroundThread() throws Exception { public void testFlushAwaitsBackgroundWrite() throws Exception { String filePath = createTempDir().resolve("flush-await.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 100, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool); // Fill and rotate to trigger background write ManagedVSR first = manager.getActiveManagedVSR(); @@ -260,7 +271,7 @@ public void testFlushAwaitsBackgroundWrite() throws Exception { public void testCloseAwaitsBackgroundWrite() throws Exception { String filePath = createTempDir().resolve("close-await.parquet").toString(); - VSRManager manager = new VSRManager(filePath, schema, bufferPool, 100, threadPool); + VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool); // Fill and rotate to trigger background write ManagedVSR first = manager.getActiveManagedVSR(); diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetWriterTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetWriterTests.java index 7fa90cf358ed5..d61ec4936c475 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetWriterTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/writer/ParquetWriterTests.java @@ -10,9 +10,11 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.dataformat.FileInfos; -import org.opensearch.index.engine.dataformat.WriteResult; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; @@ -39,6 +41,7 @@ public class ParquetWriterTests extends OpenSearchTestCase { private MappedFieldType scoreField; private Schema schema; private ThreadPool threadPool; + private IndexSettings indexSettings; @Override public void setUp() throws Exception { @@ -49,6 +52,13 @@ public void setUp() throws Exception { nameField = new KeywordFieldMapper.KeywordFieldType("name"); scoreField = new NumberFieldMapper.NumberFieldType("score", NumberFieldMapper.NumberType.LONG); schema = buildSchema(List.of(idField, nameField, scoreField)); + Settings indexSettingsBuilder = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + IndexMetadata indexMetadata = IndexMetadata.builder("test-index").settings(indexSettingsBuilder).build(); + indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); Settings settings = Settings.builder().put("node.name", "parquetwriter-test").build(); threadPool = new ThreadPool( settings, @@ -77,54 +87,7 @@ public void testAddDocReturnsSuccess() throws Exception { new ParquetDataFormat(), schema, bufferPool, - Settings.EMPTY, - threadPool, - null - ); - - ParquetDocumentInput doc = new ParquetDocumentInput(); - doc.addField(idField, 1); - doc.addField(nameField, "alice"); - doc.addField(scoreField, 100L); - WriteResult result = writer.addDoc(doc); - assertTrue(result instanceof WriteResult.Success); - doc.close(); - writer.flush(); - } - - public void testSingleDocumentFlush() throws Exception { - String filePath = createTempDir().resolve("single.parquet").toString(); - ParquetWriter writer = new ParquetWriter( - filePath, - 1L, - new ParquetDataFormat(), - schema, - bufferPool, - Settings.EMPTY, - threadPool, - null - ); - - ParquetDocumentInput doc = new ParquetDocumentInput(); - doc.addField(idField, 42); - doc.addField(nameField, "bob"); - doc.addField(scoreField, 500L); - writer.addDoc(doc); - doc.close(); - - writer.flush(); - assertEquals(1, RustBridge.getFileMetadata(filePath).numRows()); - } - - public void testMultipleDocumentsFlush() throws Exception { - String filePath = createTempDir().resolve("multi.parquet").toString(); - ParquetWriter writer = new ParquetWriter( - filePath, - 1L, - new ParquetDataFormat(), - schema, - bufferPool, - Settings.EMPTY, + indexSettings, threadPool, null ); @@ -152,22 +115,7 @@ public void testFlushWithNoDocuments() throws Exception { new ParquetDataFormat(), schema, bufferPool, - Settings.EMPTY, - threadPool, - null - ); - assertEquals(FileInfos.empty(), writer.flush()); - } - - public void testSyncAfterFlush() throws Exception { - String filePath = createTempDir().resolve("sync.parquet").toString(); - ParquetWriter writer = new ParquetWriter( - filePath, - 1L, - new ParquetDataFormat(), - schema, - bufferPool, - Settings.EMPTY, + indexSettings, threadPool, null ); diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/MergeInput.java b/server/src/main/java/org/opensearch/index/engine/dataformat/MergeInput.java index 9f61869075a83..961b532d2ea1d 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/MergeInput.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/MergeInput.java @@ -10,9 +10,11 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * input data for a merge operation. @@ -31,6 +33,16 @@ private MergeInput(Builder builder) { this(new ArrayList<>(builder.segments), builder.rowIdMapping, builder.newWriterGeneration); } + /** + * Returns the {@link WriterFileSet} for the given data format from each segment. + * + * @param formatName the data format name (e.g. "parquet") + * @return list of writer file sets for the format across all segments + */ + public List getFilesForFormat(String formatName) { + return segments.stream().map(seg -> seg.dfGroupedSearchableFiles().get(formatName)).filter(Objects::nonNull).toList(); + } + /** * Returns a new builder for constructing {@link MergeInput} instances. * From e089d0697a168ec0a548d4e9dc969fc4e1f493c3 Mon Sep 17 00:00:00 2001 From: Thy Tran <58045538+ThyTran1402@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:52:56 -0400 Subject: [PATCH 026/115] Sum file sizes when multiple files in a segment share the same extension (#21000) Previously, we would only track the size of the last file with a given extension. --------- Signed-off-by: Thy Tran <58045538+ThyTran1402@users.noreply.github.com> --- .../org/opensearch/index/engine/Engine.java | 4 +- .../index/engine/InternalEngineTests.java | 123 ++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/engine/Engine.java b/server/src/main/java/org/opensearch/index/engine/Engine.java index 8863ea4166e6e..c1a49a560ec54 100644 --- a/server/src/main/java/org/opensearch/index/engine/Engine.java +++ b/server/src/main/java/org/opensearch/index/engine/Engine.java @@ -1059,10 +1059,10 @@ private Map getSegmentFileSizes(SegmentReader segmentReader) { final Directory finalDirectory = directory; logger.warn(() -> new ParameterizedMessage("Error when trying to query fileLength [{}] [{}]", finalDirectory, file), e); } - if (length == 0L) { + if (length == 0L || extension == null) { continue; } - map.put(extension, length); + map.merge(extension, length, Long::sum); } if (useCompoundFile) { diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index c5c6ed0397c84..7f2d818676f05 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -50,6 +50,7 @@ import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -64,6 +65,7 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PointValues; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.SoftDeletesRetentionMergePolicy; import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; @@ -138,6 +140,7 @@ import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.DocumentMapperForType; import org.opensearch.index.mapper.IdFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ParseContext; import org.opensearch.index.mapper.ParseContext.Document; @@ -243,6 +246,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -9352,4 +9356,123 @@ private ParsedDocument createDocumentWithNestedField(String id, String contactNa return testParsedDocument(id, null, testDocumentWithTextField(), source, null); } + /** + * Verifies that {@code getSegmentFileSizes} correctly accumulates sizes for all files in a + * segment, including multiple files that share the same extension. + * + *

      When fuzzy-set-for-doc-ID is enabled, {@link + * org.opensearch.index.codec.PerFieldMappingPostingFormatCodec} assigns + * {@code FuzzyFilterPostingsFormat} to the {@code _id} field and the standard Lucene format to + * all other text fields. Because these are two distinct {@code PostingsFormat} implementations, + * Lucene's {@code PerFieldPostingsFormat} writes a separate file group for each, producing + * multiple files with the same extension in one segment (e.g. two {@code .tim} files, two + * {@code .doc} files, etc.). + * + *

      The bug was that {@code getSegmentFileSizes} used {@code Map.put(extension, length)}, + * which silently overwrote earlier entries, causing the reported {@code file_sizes} total to be + * less than the actual on-disk size. The fix replaces {@code put} with + * {@code map.merge(extension, length, Long::sum)} so every file's bytes are counted. + */ + public void testSegmentFileSizesAccumulatesAllFilesIncludingDuplicateExtensions() throws Exception { + // Disable compound file so each Lucene file is a separate entry in the directory, + // making it straightforward to compare the expected total (sum of file lengths from the + // directory) against the actual total reported by segmentsStats. + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( + "test_file_sizes", + Settings.builder().put(defaultSettings.getSettings()).put(EngineConfig.INDEX_USE_COMPOUND_FILE.getKey(), false).build() + ); + // Enable fuzzy set for doc ID so that _id uses FuzzyFilterPostingsFormat while other + // text fields use the standard Lucene format. Two distinct PostingsFormat instances + // in one segment cause PerFieldPostingsFormat to write two file groups that share + // extensions, which is exactly the condition that exposed the map.put() bug. + indexSettings.setEnableFuzzySetForDocId(true); + + // Mock MapperService so that PerFieldMappingPostingFormatCodec sees a non-null field + // type for _id (required for the FuzzyFilter branch to be reached). + MappedFieldType idFieldType = mock(MappedFieldType.class); + when(idFieldType.unwrap()).thenReturn(idFieldType); + MapperService mapperService = mock(MapperService.class); + when(mapperService.fieldType(any())).thenReturn(null); + when(mapperService.fieldType(IdFieldMapper.NAME)).thenReturn(idFieldType); + when(mapperService.getIndexSettings()).thenReturn(indexSettings); + when(mapperService.isCompositeIndexPresent()).thenReturn(false); + + CodecService codecService = new CodecService(mapperService, indexSettings, logger, List.of()); + + try (Store store = createStore()) { + Path translogPath = createTempDir(); + // Build a base config then rebuild it with our custom CodecService. + EngineConfig base = config(indexSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, null); + EngineConfig engineConfig = new EngineConfig.Builder().shardId(base.getShardId()) + .threadPool(base.getThreadPool()) + .indexSettings(indexSettings) + .warmer(base.getWarmer()) + .store(store) + .mergePolicy(NoMergePolicy.INSTANCE) + .analyzer(base.getAnalyzer()) + .similarity(base.getSimilarity()) + .codecService(codecService) + .eventListener(base.getEventListener()) + .queryCache(base.getQueryCache()) + .queryCachingPolicy(base.getQueryCachingPolicy()) + .translogConfig(base.getTranslogConfig()) + .flushMergesAfter(base.getFlushMergesAfter()) + .externalRefreshListener(base.getExternalRefreshListener()) + .internalRefreshListener(base.getInternalRefreshListener()) + .indexSort(base.getIndexSort()) + .circuitBreakerService(base.getCircuitBreakerService()) + .globalCheckpointSupplier(base.getGlobalCheckpointSupplier()) + .retentionLeasesSupplier(base.retentionLeasesSupplier()) + .primaryTermSupplier(base.getPrimaryTermSupplier()) + .tombstoneDocSupplier(base.getTombstoneDocSupplier()) + .build(); + + try (InternalEngine engine = createEngine(engineConfig)) { + // Index one document. The _id field goes through FuzzyFilterPostingsFormat; + // the "value" text field goes through the standard format. Both end up in the + // same segment, guaranteeing multiple files per extension. + ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), SOURCE, null); + engine.index(indexForDoc(doc)); + engine.flush(true, true); + engine.refresh("test"); + + // Compute the expected total: sum of the actual on-disk lengths of every file + // that belongs to the flushed segment. + long expectedTotal = 0; + try (Engine.Searcher searcher = engine.acquireSearcher("test")) { + for (LeafReaderContext ctx : searcher.getIndexReader().getContext().leaves()) { + SegmentReader segmentReader = Lucene.segmentReader(ctx.reader()); + for (String file : segmentReader.getSegmentInfo().files()) { + if (IndexFileNames.getExtension(file) == null) { + continue; + } + long len = store.directory().fileLength(file); + if (len == 0L) { + continue; + } + expectedTotal += len; + } + } + } + assertThat("expected at least one segment file after flush", expectedTotal, greaterThan(0L)); + + // Compute the actual total reported by segmentsStats with file sizes enabled. + SegmentsStats stats = engine.segmentsStats(true, false); + Map fileSizes = stats.getFileSizes(); + assertFalse("file_sizes must not be empty when include_segment_file_sizes=true", fileSizes.isEmpty()); + long actualTotal = fileSizes.values().stream().mapToLong(Long::longValue).sum(); + + // With the old map.put() bug, actualTotal < expectedTotal whenever the codec + // writes more than one file per extension (as the FuzzyFilter + standard + // postings formats do). With the fix (map.merge(Long::sum)) all bytes are counted. + assertEquals( + "file_sizes total must equal the actual sum of all segment file sizes; " + + "a mismatch means some files were silently dropped due to duplicate extensions", + expectedTotal, + actualTotal + ); + } + } + } + } From f44c6d296f4614d5519ca9d5e961cf346ea8570f Mon Sep 17 00:00:00 2001 From: bowenlan Date: Fri, 1 May 2026 12:35:03 -0700 Subject: [PATCH 027/115] Transfer Flight batch to a response-owned root before returning (#21437) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Transfer Flight batch to a response-owned root before returning FlightTransportResponse.nextResponse() returned a reference to the shared root held by Arrow's FlightStream. FlightStream reuses one root across batches — the next call to next() clears it and re-loads, and stream.close() releases its vectors. Transfer the shared root's vectors into a response-owned VectorSchemaRoot before returning. Signed-off-by: bowenlan-amzn * Reset source row count to 0 after transferRoot TransferPair.transfer() clears each source vector's valueCount, but VectorSchemaRoot.rowCount is a separate scalar on the root — the per-vector transfers don't touch it. Left alone, the source is inconsistent: the root reports N rows while every field vector has valueCount=0. Reset source.setRowCount(0) after the transfer loop so the source is fully empty. Tighten testTransferToMovesBuffers to assert the root-level row count, not just the vector-level value count. Signed-off-by: bowenlan-amzn * update test comment Signed-off-by: bowenlan-amzn --------- Signed-off-by: bowenlan-amzn --- .../arrow/flight/NativeArrowTransportIT.java | 83 +++++++++++++++++++ .../flight/transport/ArrowBatchResponse.java | 11 +-- .../transport/FlightTransportResponse.java | 22 ++++- .../arrow/flight/transport/FlightUtils.java | 19 +++++ .../transport/ArrowBatchResponseTests.java | 32 ++++++- 5 files changed, 153 insertions(+), 14 deletions(-) diff --git a/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/NativeArrowTransportIT.java b/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/NativeArrowTransportIT.java index 849c63a594e3b..287a57f244d92 100644 --- a/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/NativeArrowTransportIT.java +++ b/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/NativeArrowTransportIT.java @@ -124,6 +124,89 @@ public void testMultipleBatchesSerialNativeArrow() throws Exception { } } + /** + * Collects every batch without reading vector data, fully drains and closes the stream, then + * verifies each retained batch still holds its data. Mirrors an async consumer that defers + * reading until after the stream has advanced or been closed. + */ + @LockFeatureFlag(STREAM_TRANSPORT) + public void testBatchesSurviveStreamAdvanceAndClose() throws Exception { + DiscoveryNode node = getClusterState().nodes().iterator().next(); + StreamTransportService sts = internalCluster().getInstance(StreamTransportService.class); + List retained = new ArrayList<>(); + CountDownLatch latch = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + + int batchCount = 3; + int rowsPerBatch = 4; + sts.sendRequest( + node, + TestArrowAction.NAME, + new TestArrowRequest(batchCount, rowsPerBatch, 1), + TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STREAM).build(), + new StreamTransportResponseHandler() { + @Override + public void handleStreamResponse(StreamTransportResponse streamResponse) { + try { + TestArrowResponse response; + // Collect references WITHOUT reading vector data — defer that until after close. + while ((response = streamResponse.nextResponse()) != null) { + retained.add(response); + } + streamResponse.close(); + } catch (Exception e) { + failure.set(e); + streamResponse.cancel("Test error", e); + } finally { + latch.countDown(); + } + } + + @Override + public void handleException(TransportException exp) { + failure.set(exp); + latch.countDown(); + } + + @Override + public String executor() { + return ThreadPool.Names.GENERIC; + } + + @Override + public TestArrowResponse read(StreamInput in) throws IOException { + return new TestArrowResponse(in); + } + } + ); + + assertTrue("Stream should complete within 30s", latch.await(30, TimeUnit.SECONDS)); + assertNull("No exception expected: " + failure.get(), failure.get()); + assertEquals(batchCount, retained.size()); + + try { + // Every retained batch must still have its data intact even though the stream has + // advanced and closed. + for (int batchIdx = 0; batchIdx < retained.size(); batchIdx++) { + VectorSchemaRoot root = retained.get(batchIdx).getRoot(); + assertEquals("row count must survive stream close", rowsPerBatch, root.getRowCount()); + IntVector batchIdVec = (IntVector) root.getVector("batch_id"); + VarCharVector nameVec = (VarCharVector) root.getVector("name"); + IntVector valueVec = (IntVector) root.getVector("value"); + assertEquals("valueCount must survive stream close", rowsPerBatch, batchIdVec.getValueCount()); + for (int row = 0; row < rowsPerBatch; row++) { + assertEquals("batch_id survives", batchIdx, batchIdVec.get(row)); + assertEquals("name survives", "row-" + batchIdx + "-" + row, new String(nameVec.get(row), StandardCharsets.UTF_8)); + assertEquals("value survives", batchIdx * 1000 + row, valueVec.get(row)); + } + } + } finally { + for (TestArrowResponse r : retained) { + r.getRoot().close(); + } + } + } + @LockFeatureFlag(STREAM_TRANSPORT) public void testParallelBatchProduction() throws Exception { // 100 batches, 10 rows each, produced by 5 parallel threads. diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponse.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponse.java index 2e3c0939f0467..c5091007b6f91 100644 --- a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponse.java +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponse.java @@ -8,16 +8,13 @@ package org.opensearch.arrow.flight.transport; -import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.util.TransferPair; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; -import java.util.List; /** * Base class for transport responses carrying native Arrow data. @@ -96,13 +93,7 @@ public VectorSchemaRoot getRoot() { * @param target the channel's shared root (bound to the Flight stream via start()) */ void transferTo(VectorSchemaRoot target) { - List sourceVectors = producerRoot.getFieldVectors(); - List targetVectors = target.getFieldVectors(); - for (int i = 0; i < sourceVectors.size(); i++) { - TransferPair transfer = sourceVectors.get(i).makeTransferPair(targetVectors.get(i)); - transfer.transfer(); - } - target.setRowCount(producerRoot.getRowCount()); + FlightUtils.transferRoot(producerRoot, target); } @Override diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransportResponse.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransportResponse.java index 1047faab274b3..9a1588a882326 100644 --- a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransportResponse.java +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransportResponse.java @@ -121,9 +121,10 @@ public T nextResponse() { boolean hasNext = firstBatchConsumed ? flightStream.next() : (firstBatchConsumed = true); if (!hasNext) return null; - VectorSchemaRoot root = flightStream.getRoot(); - currentBatchSize = FlightUtils.calculateVectorSchemaRootSize(root); - try (VectorStreamInput input = new VectorStreamInput(root, namedWriteableRegistry)) { + VectorSchemaRoot sharedRoot = flightStream.getRoot(); + currentBatchSize = FlightUtils.calculateVectorSchemaRootSize(sharedRoot); + VectorSchemaRoot ownedRoot = transferToOwnedRoot(sharedRoot); + try (VectorStreamInput input = new VectorStreamInput(ownedRoot, namedWriteableRegistry)) { input.setVersion(initialHeader.getVersion()); return handler.read(input); } @@ -144,6 +145,21 @@ long getCurrentBatchSize() { return currentBatchSize; } + /** + * Transfers the shared root's vectors into a response-owned root so the returned response + * is independent of FlightStream's lifecycle. The next call to {@code flightStream.next()} + * clears the shared root, and {@code stream.close()} releases its vectors — both would wipe + * the data out from under an async listener if we handed back a reference to the shared root. + */ + private static VectorSchemaRoot transferToOwnedRoot(VectorSchemaRoot sharedRoot) { + VectorSchemaRoot ownedRoot = VectorSchemaRoot.create( + sharedRoot.getSchema(), + sharedRoot.getFieldVectors().getFirst().getAllocator() + ); + FlightUtils.transferRoot(sharedRoot, ownedRoot); + return ownedRoot; + } + @Override public void cancel(String reason, Throwable cause) { if (closed) return; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightUtils.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightUtils.java index 57853eed247cd..728df88ce1b12 100644 --- a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightUtils.java +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightUtils.java @@ -8,7 +8,11 @@ package org.opensearch.arrow.flight.transport; +import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.util.TransferPair; + +import java.util.List; class FlightUtils { @@ -27,4 +31,19 @@ static long calculateVectorSchemaRootSize(VectorSchemaRoot root) { } return totalSize; } + + /** + * Zero-copy transfers every vector from {@code source} into {@code target}. After this call, + * the target owns the buffers and holds the row count; the source is empty with row count 0. + */ + static void transferRoot(VectorSchemaRoot source, VectorSchemaRoot target) { + List sources = source.getFieldVectors(); + List targets = target.getFieldVectors(); + for (int i = 0; i < sources.size(); i++) { + TransferPair tp = sources.get(i).makeTransferPair(targets.get(i)); + tp.transfer(); + } + target.setRowCount(source.getRowCount()); + source.setRowCount(0); + } } diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseTests.java index fcc2947cce2b0..03582398f600e 100644 --- a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseTests.java +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseTests.java @@ -92,13 +92,43 @@ public void testTransferToMovesBuffers() { assertEquals(42, dstVec.get(0)); assertEquals(99, dstVec.get(1)); - // Source should be empty after transfer + // Source should be empty after transfer — both at vector and root level assertEquals(0, srcVec.getValueCount()); + assertEquals(0, src.getRowCount()); src.close(); dst.close(); } + /** + * After transfer, closing the source must not affect the destination — the destination owns + * its buffers. This is the invariant FlightTransportResponse relies on to decouple the + * returned response from FlightStream's shared, reused root. + */ + public void testDestinationSurvivesSourceClose() { + VectorSchemaRoot src = VectorSchemaRoot.create(schema, allocator); + IntVector srcVec = (IntVector) src.getVector("val"); + srcVec.allocateNew(); + srcVec.setSafe(0, 7); + srcVec.setSafe(1, 13); + srcVec.setValueCount(2); + src.setRowCount(2); + + VectorSchemaRoot dst = VectorSchemaRoot.create(schema, allocator); + new TestResponse(src).transferTo(dst); + + // Close the source — simulates FlightStream clearing/closing its shared root. + src.close(); + + assertEquals(2, dst.getRowCount()); + IntVector dstVec = (IntVector) dst.getVector("val"); + assertEquals(2, dstVec.getValueCount()); + assertEquals(7, dstVec.get(0)); + assertEquals(13, dstVec.get(1)); + + dst.close(); + } + public void testTransferToWithMultipleVectors() { Schema multiSchema = new Schema( List.of( From 1a021ded4dffa1b158a85797ec79ab681e74f0c9 Mon Sep 17 00:00:00 2001 From: Sandesh Kumar Date: Fri, 1 May 2026 19:02:19 -0700 Subject: [PATCH 028/115] Add sandbox QA module for analytics engine REST integration tests (#21328) --- .../ppl/action/TestPPLTransportAction.java | 36 +- .../action/TestPPLTransportActionTests.java | 73 +++- sandbox/qa/analytics-engine-rest/README.md | 135 ++++++++ sandbox/qa/analytics-engine-rest/build.gradle | 73 ++++ .../analytics/qa/AnalyticsRestTestCase.java | 75 ++++ .../analytics/qa/ClickBenchTestHelper.java | 24 ++ .../org/opensearch/analytics/qa/Dataset.java | 54 +++ .../analytics/qa/DatasetProvisioner.java | 112 ++++++ .../analytics/qa/DatasetQueryRunner.java | 138 ++++++++ .../analytics/qa/DslClickBenchIT.java | 73 ++++ .../analytics/qa/ParquetDataFusionIT.java | 123 +++++++ .../analytics/qa/PplClickBenchIT.java | 73 ++++ .../resources/datasets/clickbench/bulk.json | 201 +++++++++++ .../resources/datasets/clickbench/dsl/q1.json | 10 + .../datasets/clickbench/dsl/q10.json | 46 +++ .../datasets/clickbench/dsl/q11.json | 55 +++ .../datasets/clickbench/dsl/q12.json | 84 +++++ .../datasets/clickbench/dsl/q13.json | 48 +++ .../datasets/clickbench/dsl/q14.json | 55 +++ .../datasets/clickbench/dsl/q15.json | 76 +++++ .../datasets/clickbench/dsl/q16.json | 24 ++ .../datasets/clickbench/dsl/q17.json | 31 ++ .../datasets/clickbench/dsl/q18.json | 32 ++ .../datasets/clickbench/dsl/q19.json | 49 +++ .../resources/datasets/clickbench/dsl/q2.json | 36 ++ .../datasets/clickbench/dsl/q20.json | 19 ++ .../datasets/clickbench/dsl/q21.json | 20 ++ .../datasets/clickbench/dsl/q22.json | 71 ++++ .../datasets/clickbench/dsl/q23.json | 101 ++++++ .../datasets/clickbench/dsl/q24.json | 131 +++++++ .../datasets/clickbench/dsl/q25.json | 43 +++ .../datasets/clickbench/dsl/q26.json | 43 +++ .../datasets/clickbench/dsl/q27.json | 49 +++ .../datasets/clickbench/dsl/q28.json | 87 +++++ .../datasets/clickbench/dsl/q29.json | 109 ++++++ .../resources/datasets/clickbench/dsl/q3.json | 22 ++ .../datasets/clickbench/dsl/q30.json | 17 + .../datasets/clickbench/dsl/q31.json | 102 ++++++ .../datasets/clickbench/dsl/q32.json | 102 ++++++ .../datasets/clickbench/dsl/q33.json | 48 +++ .../datasets/clickbench/dsl/q34.json | 24 ++ .../datasets/clickbench/dsl/q35.json | 24 ++ .../datasets/clickbench/dsl/q36.json | 30 ++ .../datasets/clickbench/dsl/q37.json | 102 ++++++ .../datasets/clickbench/dsl/q38.json | 102 ++++++ .../datasets/clickbench/dsl/q39.json | 109 ++++++ .../resources/datasets/clickbench/dsl/q4.json | 12 + .../datasets/clickbench/dsl/q40.json | 111 ++++++ .../datasets/clickbench/dsl/q41.json | 102 ++++++ .../datasets/clickbench/dsl/q42.json | 107 ++++++ .../datasets/clickbench/dsl/q43.json | 84 +++++ .../resources/datasets/clickbench/dsl/q5.json | 18 + .../resources/datasets/clickbench/dsl/q6.json | 18 + .../resources/datasets/clickbench/dsl/q7.json | 17 + .../resources/datasets/clickbench/dsl/q8.json | 48 +++ .../resources/datasets/clickbench/dsl/q9.json | 31 ++ .../datasets/clickbench/mapping.json | 323 ++++++++++++++++++ .../resources/datasets/clickbench/ppl/q1.ppl | 1 + .../resources/datasets/clickbench/ppl/q10.ppl | 1 + .../resources/datasets/clickbench/ppl/q11.ppl | 1 + .../resources/datasets/clickbench/ppl/q12.ppl | 1 + .../resources/datasets/clickbench/ppl/q13.ppl | 1 + .../resources/datasets/clickbench/ppl/q14.ppl | 1 + .../resources/datasets/clickbench/ppl/q15.ppl | 1 + .../resources/datasets/clickbench/ppl/q16.ppl | 1 + .../resources/datasets/clickbench/ppl/q17.ppl | 1 + .../resources/datasets/clickbench/ppl/q18.ppl | 1 + .../resources/datasets/clickbench/ppl/q19.ppl | 1 + .../resources/datasets/clickbench/ppl/q2.ppl | 1 + .../resources/datasets/clickbench/ppl/q20.ppl | 1 + .../resources/datasets/clickbench/ppl/q21.ppl | 1 + .../resources/datasets/clickbench/ppl/q22.ppl | 1 + .../resources/datasets/clickbench/ppl/q23.ppl | 1 + .../resources/datasets/clickbench/ppl/q24.ppl | 1 + .../resources/datasets/clickbench/ppl/q25.ppl | 1 + .../resources/datasets/clickbench/ppl/q26.ppl | 1 + .../resources/datasets/clickbench/ppl/q27.ppl | 1 + .../resources/datasets/clickbench/ppl/q28.ppl | 1 + .../resources/datasets/clickbench/ppl/q29.ppl | 1 + .../resources/datasets/clickbench/ppl/q3.ppl | 1 + .../resources/datasets/clickbench/ppl/q30.ppl | 1 + .../resources/datasets/clickbench/ppl/q31.ppl | 1 + .../resources/datasets/clickbench/ppl/q32.ppl | 1 + .../resources/datasets/clickbench/ppl/q33.ppl | 1 + .../resources/datasets/clickbench/ppl/q34.ppl | 1 + .../resources/datasets/clickbench/ppl/q35.ppl | 1 + .../resources/datasets/clickbench/ppl/q36.ppl | 1 + .../resources/datasets/clickbench/ppl/q37.ppl | 1 + .../resources/datasets/clickbench/ppl/q38.ppl | 1 + .../resources/datasets/clickbench/ppl/q39.ppl | 1 + .../resources/datasets/clickbench/ppl/q4.ppl | 1 + .../resources/datasets/clickbench/ppl/q40.ppl | 1 + .../resources/datasets/clickbench/ppl/q41.ppl | 1 + .../resources/datasets/clickbench/ppl/q42.ppl | 1 + .../resources/datasets/clickbench/ppl/q43.ppl | 1 + .../resources/datasets/clickbench/ppl/q5.ppl | 1 + .../resources/datasets/clickbench/ppl/q6.ppl | 1 + .../resources/datasets/clickbench/ppl/q7.ppl | 1 + .../resources/datasets/clickbench/ppl/q8.ppl | 1 + .../resources/datasets/clickbench/ppl/q9.ppl | 1 + sandbox/qa/build.gradle | 7 + 101 files changed, 3990 insertions(+), 22 deletions(-) create mode 100644 sandbox/qa/analytics-engine-rest/README.md create mode 100644 sandbox/qa/analytics-engine-rest/build.gradle create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AnalyticsRestTestCase.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ClickBenchTestHelper.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/Dataset.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DatasetProvisioner.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DatasetQueryRunner.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DslClickBenchIT.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ParquetDataFusionIT.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/PplClickBenchIT.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/bulk.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q1.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q10.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q11.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q12.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q13.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q14.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q15.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q16.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q17.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q18.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q19.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q2.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q20.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q21.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q22.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q23.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q24.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q25.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q26.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q27.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q28.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q29.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q3.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q30.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q31.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q32.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q33.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q34.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q35.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q36.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q37.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q38.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q39.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q4.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q40.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q41.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q42.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q43.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q5.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q6.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q7.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q8.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q9.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/mapping.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q1.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q10.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q11.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q12.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q13.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q14.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q15.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q16.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q17.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q18.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q19.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q2.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q20.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q21.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q22.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q23.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q24.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q25.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q26.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q27.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q28.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q29.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q3.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q30.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q31.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q32.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q33.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q34.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q35.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q36.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q37.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q38.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q39.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q4.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q40.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q41.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q42.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q43.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q5.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q6.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q7.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q8.ppl create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q9.ppl create mode 100644 sandbox/qa/build.gradle diff --git a/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/TestPPLTransportAction.java b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/TestPPLTransportAction.java index e82c2972c5505..d81017403abc3 100644 --- a/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/TestPPLTransportAction.java +++ b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/TestPPLTransportAction.java @@ -18,6 +18,7 @@ import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; /** @@ -25,38 +26,55 @@ * *

      Receives {@link EngineContext} and {@link QueryPlanExecutor} from the analytics-engine * plugin via Guice injection (enabled by {@code extendedPlugins = ['analytics-engine']}). + * + *

      Execution is forked to the {@link ThreadPool.Names#SEARCH} thread pool to avoid + * blocking the transport thread (DefaultPlanExecutor uses a blocking future internally). */ public class TestPPLTransportAction extends HandledTransportAction { private static final Logger logger = LogManager.getLogger(TestPPLTransportAction.class); private final UnifiedQueryService unifiedQueryService; + private final ThreadPool threadPool; @Inject public TestPPLTransportAction( TransportService transportService, ActionFilters actionFilters, EngineContext engineContext, - QueryPlanExecutor> executor + QueryPlanExecutor> executor, + ThreadPool threadPool ) { super(UnifiedPPLExecuteAction.NAME, transportService, actionFilters, PPLRequest::new); this.unifiedQueryService = new UnifiedQueryService(executor, engineContext); + this.threadPool = threadPool; } /** Test-only constructor that accepts a pre-built {@link UnifiedQueryService}. */ - public TestPPLTransportAction(TransportService transportService, ActionFilters actionFilters, UnifiedQueryService unifiedQueryService) { + public TestPPLTransportAction( + TransportService transportService, + ActionFilters actionFilters, + UnifiedQueryService unifiedQueryService, + ThreadPool threadPool + ) { super(UnifiedPPLExecuteAction.NAME, transportService, actionFilters, PPLRequest::new); this.unifiedQueryService = unifiedQueryService; + this.threadPool = threadPool; } @Override protected void doExecute(Task task, PPLRequest request, ActionListener listener) { - try { - PPLResponse response = unifiedQueryService.execute(request.getPplText()); - listener.onResponse(response); - } catch (Exception e) { - logger.error("[UNIFIED_PPL] execution failed", e); - listener.onFailure(e); - } + // Fork to SEARCH thread pool — DefaultPlanExecutor.execute() blocks on a future + // internally, which is forbidden on the transport thread. + // TODO: update UnifiedQueryService to consume a listener that DefaultPlanExecutor does to avoid threadpool fork + threadPool.executor(ThreadPool.Names.SEARCH).execute(() -> { + try { + PPLResponse response = unifiedQueryService.execute(request.getPplText()); + listener.onResponse(response); + } catch (Exception e) { + logger.error("[UNIFIED_PPL] execution failed", e); + listener.onFailure(e); + } + }); } } diff --git a/sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/TestPPLTransportActionTests.java b/sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/TestPPLTransportActionTests.java index 54e47b969f8b1..9d8368e854ecb 100644 --- a/sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/TestPPLTransportActionTests.java +++ b/sandbox/plugins/test-ppl-frontend/src/test/java/org/opensearch/ppl/action/TestPPLTransportActionTests.java @@ -11,6 +11,8 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.core.action.ActionListener; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import java.util.ArrayList; @@ -36,33 +38,54 @@ public class TestPPLTransportActionTests extends OpenSearchTestCase { private UnifiedQueryService mockUnifiedQueryService; private TestPPLTransportAction action; + private ThreadPool threadPool; @Override public void setUp() throws Exception { super.setUp(); mockUnifiedQueryService = mock(UnifiedQueryService.class); + threadPool = new TestThreadPool(getTestName()); action = new TestPPLTransportAction( mock(TransportService.class), new ActionFilters(Collections.emptySet()), - mockUnifiedQueryService + mockUnifiedQueryService, + threadPool ); } + @Override + public void tearDown() throws Exception { + ThreadPool.terminate(threadPool, 10, java.util.concurrent.TimeUnit.SECONDS); + super.tearDown(); + } + /** * Success path: {@code unifiedQueryService.execute()} returns a response → * {@code listener.onResponse()} is called with that response. */ - public void testSuccessPathCallsOnResponse() { + public void testSuccessPathCallsOnResponse() throws Exception { List rows = new ArrayList<>(); rows.add(new Object[] { "server-1", 200 }); PPLResponse expectedResponse = new PPLResponse(List.of("host", "status"), rows); when(mockUnifiedQueryService.execute("source=logs")).thenReturn(expectedResponse); - ActionListener listener = mock(ActionListener.class); + AtomicReference captured = new AtomicReference<>(); + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(PPLResponse r) { + captured.set(r); + } + + @Override + public void onFailure(Exception e) { + fail("Should not fail: " + e.getMessage()); + } + }; action.execute(null, new PPLRequest("source=logs"), listener); - verify(listener).onResponse(expectedResponse); + assertBusy(() -> assertNotNull("onResponse should be called", captured.get())); + assertSame(expectedResponse, captured.get()); verify(mockUnifiedQueryService).execute("source=logs"); } @@ -70,21 +93,33 @@ public void testSuccessPathCallsOnResponse() { * Failure path: {@code unifiedQueryService.execute()} throws → * {@code listener.onFailure()} is called with the exception. */ - public void testFailurePathCallsOnFailure() { + public void testFailurePathCallsOnFailure() throws Exception { RuntimeException expectedException = new RuntimeException("PPL execution failed"); when(mockUnifiedQueryService.execute(any(String.class))).thenThrow(expectedException); - ActionListener listener = mock(ActionListener.class); + AtomicReference captured = new AtomicReference<>(); + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(PPLResponse r) { + fail("Should not succeed"); + } + + @Override + public void onFailure(Exception e) { + captured.set(e); + } + }; action.execute(null, new PPLRequest("invalid query"), listener); - verify(listener).onFailure(expectedException); + assertBusy(() -> assertNotNull("onFailure should be called", captured.get())); + assertSame(expectedException, captured.get()); verify(mockUnifiedQueryService).execute("invalid query"); } /** * Exactly-one-callback on success: only {@code onResponse} is called, never {@code onFailure}. */ - public void testExactlyOneCallbackOnSuccess() { + public void testExactlyOneCallbackOnSuccess() throws Exception { PPLResponse response = new PPLResponse(Collections.emptyList(), Collections.emptyList()); when(mockUnifiedQueryService.execute(any(String.class))).thenReturn(response); @@ -105,14 +140,14 @@ public void onFailure(Exception e) { action.execute(null, new PPLRequest("source=test"), listener); - assertEquals("onResponse should be called exactly once", 1, responseCount.get()); + assertBusy(() -> assertEquals("onResponse should be called exactly once", 1, responseCount.get())); assertEquals("onFailure should not be called", 0, failureCount.get()); } /** * Exactly-one-callback on failure: only {@code onFailure} is called, never {@code onResponse}. */ - public void testExactlyOneCallbackOnFailure() { + public void testExactlyOneCallbackOnFailure() throws Exception { when(mockUnifiedQueryService.execute(any(String.class))).thenThrow(new RuntimeException("fail")); AtomicInteger responseCount = new AtomicInteger(0); @@ -134,8 +169,8 @@ public void onFailure(Exception e) { action.execute(null, new PPLRequest("source=test"), listener); + assertBusy(() -> assertEquals("onFailure should be called exactly once", 1, failureCount.get())); assertEquals("onResponse should not be called", 0, responseCount.get()); - assertEquals("onFailure should be called exactly once", 1, failureCount.get()); assertNotNull("Exception should be captured", capturedError.get()); } @@ -143,13 +178,25 @@ public void onFailure(Exception e) { * Verify that the correct PPL text is forwarded to * {@code unifiedQueryService.execute()}. */ - public void testCorrectArgumentsPassedToUnifiedQueryService() { + public void testCorrectArgumentsPassedToUnifiedQueryService() throws Exception { PPLResponse response = new PPLResponse(Collections.emptyList(), Collections.emptyList()); when(mockUnifiedQueryService.execute(any(String.class))).thenReturn(response); - ActionListener listener = mock(ActionListener.class); + AtomicReference captured = new AtomicReference<>(); + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(PPLResponse r) { + captured.set(r); + } + + @Override + public void onFailure(Exception e) { + fail("Should not fail"); + } + }; action.execute(null, new PPLRequest("source=metrics | where status=500"), listener); + assertBusy(() -> assertNotNull(captured.get())); verify(mockUnifiedQueryService).execute("source=metrics | where status=500"); verifyNoMoreInteractions(mockUnifiedQueryService); } diff --git a/sandbox/qa/analytics-engine-rest/README.md b/sandbox/qa/analytics-engine-rest/README.md new file mode 100644 index 0000000000000..46e2db11b3d2c --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/README.md @@ -0,0 +1,135 @@ +# Analytics Engine REST Integration Tests + +REST-based integration tests for the analytics engine, running against a live OpenSearch cluster with sandbox plugins installed. + +## Architecture + +``` +AnalyticsRestTestCase ← abstract base (cluster config, helpers) +├── ParquetDataFusionIT ← parquet indexing sanity + index settings validation +├── DslClickBenchIT ← DSL queries via _search → DataFusion +└── PplClickBenchIT ← PPL queries via /_analytics/ppl → DataFusion + +Dataset ← descriptor for a test dataset (mapping, bulk data, queries) +DatasetProvisioner ← provisions any dataset into a parquet-backed index +DatasetQueryRunner ← auto-discovers queries and runs them against a cluster +ClickBenchTestHelper ← ClickBench dataset constants +``` + +- `AnalyticsRestTestCase` — handles cluster preservation, resource loading, JSON escaping, and assertion helpers. Extend this for any new integration test. +- `Dataset` / `DatasetProvisioner` / `DatasetQueryRunner` — generic test infrastructure. Any new dataset can plug in by adding a directory under `resources/datasets/{name}/`. +- `ClickBenchTestHelper` — thin wrapper that declares the ClickBench dataset descriptor. + +## Adding a New Dataset + +To add a new dataset, create a directory under `src/test/resources/datasets/{name}/` with this structure: + +``` +datasets/ + {name}/ + mapping.json # index mapping + settings + bulk.json # bulk-indexable documents (NDJSON) + dsl/q1.json ... qN.json # DSL queries (auto-discovered) + dsl/expected/q1.json ... # expected responses (optional) + ppl/q1.ppl ... qN.ppl # PPL queries (auto-discovered) + ppl/expected/q1.json ... # expected responses (optional) +``` + +Then declare the dataset in Java: + +```java +Dataset myDataset = new Dataset("myDatasetName", "my_index_name"); +``` + +`DatasetProvisioner.provision(client, myDataset)` creates the index with parquet data format and ingests the bulk data. `DatasetQueryRunner.discoverQueryNumbers(myDataset, "dsl")` auto-discovers all query files. + +## Test Classes + +| Test | Description | +|------|-------------| +| `ParquetDataFusionIT` | Sanity check: creates a parquet-format index, validates settings are persisted, ingests docs, runs a simple search | +| `DslClickBenchIT` | Runs ClickBench DSL queries via `_search` → dsl-query-executor → Calcite → Substrait → DataFusion | +| `PplClickBenchIT` | Runs ClickBench PPL queries via `/_analytics/ppl` → test-ppl-frontend → analytics-engine → Calcite → Substrait → DataFusion | + +## Prerequisites + +### JDK 25+ + +The sandbox requires JDK 25 or newer: + +```bash +export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk-25.jdk/Contents/Home # macOS example +``` + +### Rust toolchain (native library) + +The DataFusion backend requires a native Rust library. Build it once (re-run after Rust code changes): + +```bash +./gradlew :sandbox:libs:dataformat-native:buildRustLibrary -Dsandbox.enabled=true +``` + +## Running Tests + +### Managed testClusters (integTest) — auto-provisioned + +The `integTest` task auto-starts a single-node cluster with all required plugins and runs the tests: + +```bash +./gradlew :sandbox:qa:analytics-engine-rest:integTest -Dsandbox.enabled=true +``` + +The cluster configuration (plugins, feature flag, native library path) is defined in `build.gradle` — no manual setup needed. + +### External cluster (restTest) — manually provisioned + +Start a cluster manually (see below), then run tests against it: + +```bash +# Default: localhost:9200 +./gradlew :sandbox:qa:analytics-engine-rest:restTest -Dsandbox.enabled=true + +# Custom cluster +./gradlew :sandbox:qa:analytics-engine-rest:restTest -Dsandbox.enabled=true -PrestCluster=host:port +``` + +### Starting a cluster manually + +```bash +./gradlew publishToMavenLocal -Dsandbox.enabled=true -x test -x javadoc + +NATIVE_LIB_DIR=$(pwd)/sandbox/libs/dataformat-native/rust/target/release + +./gradlew run -Dsandbox.enabled=true \ + -PinstalledPlugins="['analytics-engine', 'parquet-data-format', 'analytics-backend-datafusion', 'analytics-backend-lucene', 'dsl-query-executor', 'composite-engine', 'test-ppl-frontend']" \ + -Dtests.jvm.argline="-Djava.library.path=$NATIVE_LIB_DIR -Dopensearch.experimental.feature.pluggable.dataformat.enabled=true" \ + -x javadoc -x test -x missingJavadoc +``` + +Note: PPL tests via `/_analytics/ppl` require the `test-ppl-frontend` plugin. It is included in the `integTest` cluster config and can also be added to `./gradlew run` via `-PinstalledPlugins`. + +### Running individual tests + +```bash +# Parquet sanity +./gradlew :sandbox:qa:analytics-engine-rest:integTest -Dsandbox.enabled=true \ + --tests "org.opensearch.analytics.qa.ParquetDataFusionIT" + +# DSL ClickBench +./gradlew :sandbox:qa:analytics-engine-rest:integTest -Dsandbox.enabled=true \ + --tests "org.opensearch.analytics.qa.DslClickBenchIT" + +# PPL ClickBench +./gradlew :sandbox:qa:analytics-engine-rest:integTest -Dsandbox.enabled=true \ + --tests "org.opensearch.analytics.qa.PplClickBenchIT" +``` + +## Notes + +- Parquet indexing uses the composite data format framework: `index.composite.primary_data_format = parquet` +- The `pluggable.dataformat.enabled` feature flag must be set at cluster startup (already configured for `integTest`) +- DSL path: `_search` → dsl-query-executor → Calcite planning → Substrait → DataFusion +- PPL path: `/_analytics/ppl` → test-ppl-frontend → analytics-engine → Calcite → Substrait → DataFusion +- Expected response validation (via `{language}/expected/q{N}.json`) is planned for future iterations — currently the runner only validates that responses are non-empty +- `DslClickBenchIT` runs ClickBench Q1. Auto-discovery of all 43 DSL queries is commented out in the test (see class javadoc) because several queries exercise unsupported translators/planner rules and destabilize the shared cluster. Re-enable as support expands. +- `PplClickBenchIT` runs ClickBench Q1 via the test-ppl-frontend plugin. Auto-discovery is commented out for the same reason as DSL. diff --git a/sandbox/qa/analytics-engine-rest/build.gradle b/sandbox/qa/analytics-engine-rest/build.gradle new file mode 100644 index 0000000000000..e96065e65a88a --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/build.gradle @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +apply plugin: 'opensearch.testclusters' +apply plugin: 'opensearch.standalone-rest-test' +apply plugin: 'opensearch.rest-test' + +// analytics-backend-datafusion targets JDK 25; match it here for dependency resolution. +java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } + +repositories { + maven { + name = 'OpenSearch Snapshots' + url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/' + } +} + +dependencies { + testImplementation project(':sandbox:plugins:analytics-engine') + testImplementation project(':sandbox:plugins:analytics-backend-datafusion') + testImplementation project(':sandbox:plugins:analytics-backend-lucene') + testImplementation project(':sandbox:plugins:dsl-query-executor') + testImplementation project(':sandbox:plugins:composite-engine') + testImplementation project(':sandbox:plugins:parquet-data-format') + testImplementation project(':sandbox:plugins:test-ppl-frontend') +} + +testClusters.integTest { + plugin ':sandbox:plugins:analytics-engine' + plugin ':sandbox:plugins:analytics-backend-datafusion' + plugin ':sandbox:plugins:analytics-backend-lucene' + plugin ':sandbox:plugins:dsl-query-executor' + plugin ':sandbox:plugins:composite-engine' + plugin ':sandbox:plugins:parquet-data-format' + plugin ':sandbox:plugins:test-ppl-frontend' + + // Arrow/Flight JVM flags for DataFusion native library + jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' + jvmArgs '--enable-native-access=ALL-UNNAMED' + + // Native library path for DataFusion + systemProperty 'java.library.path', "${project(':sandbox:libs:dataformat-native').ext.nativeLibPath.parent}" + + // Enable pluggable dataformat feature flag + systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true' +} + +integTest { + systemProperty 'tests.security.manager', 'false' +} + +// Run against an external cluster (no testClusters lifecycle): +// ./gradlew :sandbox:qa:analytics-engine-rest:restTest +// ./gradlew :sandbox:qa:analytics-engine-rest:restTest -PrestCluster=host:port +tasks.register('restTest', Test) { + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + include '**/*IT.class' + def cluster = findProperty('restCluster') ?: 'localhost:9200' + def clusterName = findProperty('restClusterName') ?: 'runTask' + systemProperty 'tests.rest.cluster', cluster + systemProperty 'tests.cluster', cluster + systemProperty 'tests.clustername', clusterName + systemProperty 'tests.security.manager', 'false' + systemProperty 'tests.rest.load_packaged', 'false' + // Inherit OpenSearch test base properties + systemProperty 'tests.artifact', 'analytics-engine-rest' +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AnalyticsRestTestCase.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AnalyticsRestTestCase.java new file mode 100644 index 0000000000000..31d607edf51ba --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AnalyticsRestTestCase.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Response; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Abstract base class for all analytics REST integration tests in the sandbox QA package. + *

      + * Handles cluster-level concerns: preserving cluster/indices across test methods, + * loading classpath resources, JSON escaping, and common assertion helpers. + *

      + * Test data provisioning is handled separately by dataset-specific helpers + * (e.g. {@link ClickBenchTestHelper}) to keep cluster config orthogonal to test data. + */ +public abstract class AnalyticsRestTestCase extends OpenSearchRestTestCase { + + protected static final Logger logger = LogManager.getLogger(AnalyticsRestTestCase.class); + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } + + /** + * Load a classpath resource as a UTF-8 string. + * Fails with an assertion error if the resource does not exist. + */ + protected String loadResource(String path) throws IOException { + try (InputStream is = getClass().getClassLoader().getResourceAsStream(path)) { + assertNotNull("Resource not found: " + path, is); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")); + } + } + } + + /** + * Escape backslashes and double quotes for safe embedding in JSON string values. + */ + protected static String escapeJson(String text) { + return text.replace("\\", "\\\\").replace("\"", "\\\""); + } + + /** + * Assert that the response has HTTP 200 status and return the body as a parsed Map. + * The {@code context} string is included in failure messages for easier debugging. + */ + protected Map assertOkAndParse(Response response, String context) throws IOException { + assertEquals(context + ": expected HTTP 200", 200, response.getStatusLine().getStatusCode()); + return entityAsMap(response); + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ClickBenchTestHelper.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ClickBenchTestHelper.java new file mode 100644 index 0000000000000..7383c42145069 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ClickBenchTestHelper.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +/** + * Helper constants for the ClickBench dataset. + *

      + * Provisioned via {@link DatasetProvisioner} using resources from {@code datasets/clickbench/}. + */ +public final class ClickBenchTestHelper { + + /** ClickBench dataset descriptor. */ + public static final Dataset DATASET = new Dataset("clickbench", "parquet_hits"); + + private ClickBenchTestHelper() { + // utility class + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/Dataset.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/Dataset.java new file mode 100644 index 0000000000000..ea454cbed6d49 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/Dataset.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +/** + * Descriptor for a test dataset loaded from {@code resources/datasets/{name}/}. + *

      + * A dataset consists of: + *

        + *
      • {@code mapping.json} — index mapping and settings
        • + *
      • {@code bulk.json} — bulk-indexable documents (NDJSON)
        • + *
      • {@code {language}/q{N}.{ext}} — query files by language
        • + *
      • {@code {language}/expected/q{N}.json} — expected responses (optional)
        • + *
      + */ +public final class Dataset { + + /** The dataset name, used as the directory under {@code resources/datasets/}. */ + public final String name; + + /** The index name to provision the dataset into. */ + public final String indexName; + + public Dataset(String name, String indexName) { + this.name = name; + this.indexName = indexName; + } + + /** Path to the mapping resource. */ + public String mappingResourcePath() { + return "datasets/" + name + "/mapping.json"; + } + + /** Path to the bulk data resource. */ + public String bulkResourcePath() { + return "datasets/" + name + "/bulk.json"; + } + + /** Path to a query resource for the given language and query number. */ + public String queryResourcePath(String language, String extension, int queryNumber) { + return "datasets/" + name + "/" + language + "/q" + queryNumber + "." + extension; + } + + /** Path to the expected response resource for the given language and query number. */ + public String expectedResponseResourcePath(String language, int queryNumber) { + return "datasets/" + name + "/" + language + "/expected/q" + queryNumber + ".json"; + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DatasetProvisioner.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DatasetProvisioner.java new file mode 100644 index 0000000000000..33178f5cf3624 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DatasetProvisioner.java @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.RestClient; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Generic provisioner that creates an index from a {@link Dataset} descriptor. + *

      + * Reads {@code mapping.json} and {@code bulk.json} from the dataset's resource + * directory and ingests them into the cluster. Idempotent — deletes the index + * first if it already exists. + *

      + * Applies parquet data format settings so the dataset is queryable via the + * DataFusion backend. + */ +public final class DatasetProvisioner { + + private static final Logger logger = LogManager.getLogger(DatasetProvisioner.class); + + private DatasetProvisioner() { + // utility class + } + + /** + * Provision the dataset into the cluster with parquet as the primary data format. + */ + public static void provision(RestClient client, Dataset dataset) throws IOException { + // Delete if exists + try { + client.performRequest(new Request("DELETE", "/" + dataset.indexName)); + } catch (Exception e) { + // index may not exist — ignore + } + + // Load mapping, inject parquet settings, create index + String mapping = loadResource(dataset.mappingResourcePath()); + String indexBody = injectParquetSettings(mapping); + Request createIndex = new Request("PUT", "/" + dataset.indexName); + createIndex.setJsonEntity(indexBody); + client.performRequest(createIndex); + + // Bulk ingest + String bulkBody = loadResource(dataset.bulkResourcePath()); + Request bulkRequest = new Request("POST", "/" + dataset.indexName + "/_bulk"); + bulkRequest.setJsonEntity(bulkBody); + bulkRequest.addParameter("refresh", "true"); + bulkRequest.setOptions( + bulkRequest.getOptions().toBuilder().addHeader("Content-Type", "application/x-ndjson").build() + ); + Response bulkResponse = client.performRequest(bulkRequest); + assertEquals("Bulk insert failed", 200, bulkResponse.getStatusLine().getStatusCode()); + + // Flush to commit parquet files to disk + Request flushRequest = new Request("POST", "/" + dataset.indexName + "/_flush"); + flushRequest.addParameter("force", "true"); + client.performRequest(flushRequest); + + // Wait for index health + Request healthRequest = new Request("GET", "/_cluster/health/" + dataset.indexName); + healthRequest.addParameter("wait_for_status", "yellow"); + healthRequest.addParameter("timeout", "60s"); + client.performRequest(healthRequest); + + logger.info("Dataset [{}] provisioned into index [{}]", dataset.name, dataset.indexName); + } + + /** + * Inject parquet data format settings into the existing settings block. + */ + private static String injectParquetSettings(String mappingBody) { + return mappingBody.replace( + "\"number_of_shards\"", + "\"index.pluggable.dataformat.enabled\": true, " + + "\"index.pluggable.dataformat\": \"composite\", " + + "\"index.composite.primary_data_format\": \"parquet\", " + + "\"number_of_shards\"" + ); + } + + /** + * Load a classpath resource as a UTF-8 string. + */ + public static String loadResource(String path) throws IOException { + try (InputStream is = DatasetProvisioner.class.getClassLoader().getResourceAsStream(path)) { + assertNotNull("Resource not found: " + path, is); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")); + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DatasetQueryRunner.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DatasetQueryRunner.java new file mode 100644 index 0000000000000..880fd4f717fa5 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DatasetQueryRunner.java @@ -0,0 +1,138 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.RestClient; +import org.opensearch.common.io.PathUtils; + +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Locale; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** + * Generic runner that discovers queries from a dataset's resource directory and + * executes them against a live cluster. + *

      + * For a dataset at {@code resources/datasets/{name}/}, queries are auto-discovered + * from {@code {language}/} and executed via the provided {@link QueryExecutor}. + */ +public final class DatasetQueryRunner { + + private static final Logger logger = LogManager.getLogger(DatasetQueryRunner.class); + private static final Pattern QUERY_FILE_PATTERN = Pattern.compile("q(\\d+)\\.\\w+"); + + /** Executes a single query against a live cluster and returns the response body as a Map. */ + @FunctionalInterface + public interface QueryExecutor { + Map execute(RestClient client, Dataset dataset, String queryBody) throws IOException; + } + + private DatasetQueryRunner() { + // utility class + } + + /** + * Discover all query numbers available for the given dataset and language. + * Returns a sorted list of query numbers N such that {@code {language}/q{N}.{ext}} exists. + */ + public static List discoverQueryNumbers(Dataset dataset, String language) throws IOException { + String resourceDir = "datasets/" + dataset.name + "/" + language; + URL url = DatasetQueryRunner.class.getClassLoader().getResource(resourceDir); + if (url == null) { + return Collections.emptyList(); + } + + List numbers = new ArrayList<>(); + FileSystem fs = null; + try { + URI uri = url.toURI(); + Path path; + if ("jar".equals(uri.getScheme())) { + fs = FileSystems.newFileSystem(uri, Collections.emptyMap()); + path = fs.getPath(resourceDir); + } else { + path = PathUtils.get(uri); + } + try (Stream stream = Files.list(path)) { + stream.forEach(p -> { + String fileName = p.getFileName().toString(); + Matcher m = QUERY_FILE_PATTERN.matcher(fileName); + if (m.matches()) { + numbers.add(Integer.parseInt(m.group(1))); + } + }); + } + } catch (Exception e) { + throw new IOException("Failed to discover queries for dataset [" + dataset.name + "] language [" + language + "]", e); + } finally { + if (fs != null) { + fs.close(); + } + } + + Collections.sort(numbers); + return numbers; + } + + /** + * Run the given query numbers against the cluster using the supplied executor. + * Collects failures and returns them as a list — does not fail-fast so all queries are attempted. + * + * @param client the REST client + * @param dataset the dataset descriptor + * @param language the query language directory (e.g. "dsl", "ppl") + * @param extension the query file extension (e.g. "json", "ppl") + * @param queryNumbers the query numbers to run + * @param executor the executor that sends the query to the cluster + * @return list of failure messages (empty if all queries succeeded) + */ + public static List runQueries( + RestClient client, + Dataset dataset, + String language, + String extension, + List queryNumbers, + QueryExecutor executor + ) { + List failures = new ArrayList<>(); + for (int queryNum : queryNumbers) { + String queryId = language.toUpperCase(Locale.ROOT) + " Q" + queryNum; + try { + String queryBody = DatasetProvisioner.loadResource(dataset.queryResourcePath(language, extension, queryNum)); + logger.info("=== {} ===\n{}", queryId, queryBody); + + Map response = executor.execute(client, dataset, queryBody); + logger.info("{} response: {}", queryId, response); + + if (response == null || response.isEmpty()) { + failures.add(queryId + ": empty response"); + } + } catch (Exception e) { + String msg = queryId + " failed: " + e.getMessage(); + logger.error(msg, e); + failures.add(msg); + } + } + return failures; + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DslClickBenchIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DslClickBenchIT.java new file mode 100644 index 0000000000000..d8e9acf2dc74d --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DslClickBenchIT.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.util.List; + +/** + * ClickBench DSL integration test. Runs DSL queries against a parquet-backed ClickBench index. + *

      + * Query path: {@code POST /{index}/_search} → dsl-query-executor → Calcite → Substrait → DataFusion + *

      + * Currently restricted to Q1 to keep CI green. Auto-discovery of all 43 ClickBench queries is + * temporarily disabled because several queries exercise unsupported aggregation translators + * (e.g. ValueCount, Cardinality, MultiTerms) or planner rules, and in some cases crash the + * cluster, which cascades into the PPL suite as well. Re-enable auto-discovery once the + * analytics-engine adds support for those paths. + */ +public class DslClickBenchIT extends AnalyticsRestTestCase { + + /** + * ClickBench DSL query numbers to run. Q1 validates the DSL → DataFusion path end-to-end. + * Additional queries can be added here as the analytics engine adds support for more + * aggregation translators and planner rules. + */ + private static final List QUERY_NUMBERS = List.of(1); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws Exception { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), ClickBenchTestHelper.DATASET); + dataProvisioned = true; + } + } + + public void testClickBenchDslQueries() throws Exception { + ensureDataProvisioned(); + + // Auto-discovery disabled until all ClickBench queries pass. See class javadoc. + // List queryNumbers = DatasetQueryRunner.discoverQueryNumbers(ClickBenchTestHelper.DATASET, "dsl"); + // assertFalse("No DSL queries discovered", queryNumbers.isEmpty()); + // logger.info("Discovered {} DSL queries: {}", queryNumbers.size(), queryNumbers); + List queryNumbers = QUERY_NUMBERS; + logger.info("Running {} DSL queries: {}", queryNumbers.size(), queryNumbers); + + List failures = DatasetQueryRunner.runQueries( + client(), + ClickBenchTestHelper.DATASET, + "dsl", + "json", + queryNumbers, + (client, dataset, queryBody) -> { + Request request = new Request("POST", "/" + dataset.indexName + "/_search"); + request.setJsonEntity(queryBody); + Response response = client.performRequest(request); + return assertOkAndParse(response, "DSL query"); + } + ); + + if (failures.isEmpty() == false) { + fail("DSL query failures (" + failures.size() + " of " + queryNumbers.size() + "):\n" + String.join("\n", failures)); + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ParquetDataFusionIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ParquetDataFusionIT.java new file mode 100644 index 0000000000000..630fb18453193 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ParquetDataFusionIT.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.util.Map; + +/** + * End-to-end integration test for pure Parquet indexing with DataFusion. + *

      + * Validates that a composite index with parquet as primary data format can be + * created, documents can be ingested, and the index settings are correctly persisted. + *

      + * Requires plugins: analytics-engine, analytics-backend-datafusion, analytics-backend-lucene, + * dsl-query-executor, composite-engine, parquet-data-format. + *

      + * Requires feature flag: {@code opensearch.experimental.feature.pluggable.dataformat.enabled=true} + */ +public class ParquetDataFusionIT extends AnalyticsRestTestCase { + + private static final String INDEX_NAME = "parquet_e2e_test"; + + /** + * Creates a parquet-format index, verifies settings are persisted correctly, + * ingests documents, and runs a simple search to confirm the index is functional. + */ + public void testParquetIndexCreationAndIngestion() throws Exception { + // Clean up if exists from a previous run + try { + client().performRequest(new Request("DELETE", "/" + INDEX_NAME)); + } catch (Exception e) { + // index may not exist + } + + // Create index with parquet as primary data format + String body = "{" + + "\"settings\": {" + + " \"number_of_shards\": 1," + + " \"number_of_replicas\": 0," + + " \"index.pluggable.dataformat.enabled\": true," + + " \"index.pluggable.dataformat\": \"composite\"," + + " \"index.composite.primary_data_format\": \"parquet\"" + + "}," + + "\"mappings\": {" + + " \"properties\": {" + + " \"name\": { \"type\": \"keyword\" }," + + " \"age\": { \"type\": \"integer\" }," + + " \"score\": { \"type\": \"double\" }," + + " \"city\": { \"type\": \"keyword\" }" + + " }" + + "}" + + "}"; + + Request createIndex = new Request("PUT", "/" + INDEX_NAME); + createIndex.setJsonEntity(body); + Map createResponse = assertOkAndParse(client().performRequest(createIndex), "Create parquet index"); + assertEquals("Index creation should be acknowledged", true, createResponse.get("acknowledged")); + logger.info("Created parquet index [{}]", INDEX_NAME); + + // Wait for green health + Request healthRequest = new Request("GET", "/_cluster/health/" + INDEX_NAME); + healthRequest.addParameter("wait_for_status", "green"); + healthRequest.addParameter("timeout", "30s"); + client().performRequest(healthRequest); + + // Verify index settings + Response settingsResponse = client().performRequest(new Request("GET", "/" + INDEX_NAME + "/_settings")); + Map settingsMap = assertOkAndParse(settingsResponse, "Get index settings"); + + @SuppressWarnings("unchecked") + Map indexSettings = (Map) settingsMap.get(INDEX_NAME); + assertNotNull("Settings response should contain index", indexSettings); + + @SuppressWarnings("unchecked") + Map settings = (Map) indexSettings.get("settings"); + @SuppressWarnings("unchecked") + Map index = (Map) settings.get("index"); + @SuppressWarnings("unchecked") + Map composite = (Map) index.get("composite"); + + assertEquals("Primary data format should be parquet", "parquet", composite.get("primary_data_format")); + logger.info("Verified index settings: primary_data_format = parquet"); + + // Bulk index 5 documents + StringBuilder bulk = new StringBuilder(); + bulk.append("{\"index\": {}}\n"); + bulk.append("{\"name\": \"alice\", \"age\": 30, \"score\": 95.5, \"city\": \"seattle\"}\n"); + bulk.append("{\"index\": {}}\n"); + bulk.append("{\"name\": \"bob\", \"age\": 25, \"score\": 88.0, \"city\": \"portland\"}\n"); + bulk.append("{\"index\": {}}\n"); + bulk.append("{\"name\": \"carol\", \"age\": 35, \"score\": 92.3, \"city\": \"seattle\"}\n"); + bulk.append("{\"index\": {}}\n"); + bulk.append("{\"name\": \"dave\", \"age\": 28, \"score\": 76.8, \"city\": \"portland\"}\n"); + bulk.append("{\"index\": {}}\n"); + bulk.append("{\"name\": \"eve\", \"age\": 32, \"score\": 91.0, \"city\": \"seattle\"}\n"); + + Request bulkRequest = new Request("POST", "/" + INDEX_NAME + "/_bulk"); + bulkRequest.setJsonEntity(bulk.toString()); + bulkRequest.addParameter("refresh", "true"); + bulkRequest.setOptions( + bulkRequest.getOptions().toBuilder().addHeader("Content-Type", "application/x-ndjson").build() + ); + Map bulkResponse = assertOkAndParse(client().performRequest(bulkRequest), "Bulk index"); + assertEquals("Bulk indexing should have no errors", false, bulkResponse.get("errors")); + logger.info("Indexed 5 documents into parquet index [{}]", INDEX_NAME); + + // Simple search to verify index is functional + Request searchRequest = new Request("POST", "/" + INDEX_NAME + "/_search"); + searchRequest.setJsonEntity("{\"size\": 0, \"track_total_hits\": true}"); + Response searchResponse = client().performRequest(searchRequest); + Map searchMap = assertOkAndParse(searchResponse, "Simple search"); + assertNotNull("Search response should contain hits", searchMap.get("hits")); + logger.info("Simple search completed successfully on parquet index [{}]", INDEX_NAME); + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/PplClickBenchIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/PplClickBenchIT.java new file mode 100644 index 0000000000000..e2ba440dd826d --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/PplClickBenchIT.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.util.List; + +/** + * ClickBench PPL integration test. Runs PPL queries against a parquet-backed ClickBench index. + *

      + * Query path: {@code POST /_analytics/ppl} → test-ppl-frontend → analytics-engine → Calcite → Substrait → DataFusion + *

      + * Currently restricted to Q1 to keep CI green. Auto-discovery of all 43 ClickBench queries is + * temporarily disabled because several queries exercise unsupported translators/planner rules + * and the broader DSL run destabilizes the shared test cluster. Re-enable auto-discovery once + * the analytics-engine adds support for those paths. + */ +public class PplClickBenchIT extends AnalyticsRestTestCase { + + /** + * ClickBench PPL query numbers to run. Q1 validates the PPL → DataFusion path end-to-end. + * Additional queries can be added here as the analytics engine adds support for more + * aggregation translators and planner rules. + */ + private static final List QUERY_NUMBERS = List.of(1); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws Exception { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), ClickBenchTestHelper.DATASET); + dataProvisioned = true; + } + } + + public void testClickBenchPplQueries() throws Exception { + ensureDataProvisioned(); + + // Auto-discovery disabled until all ClickBench queries pass. See class javadoc. + // List queryNumbers = DatasetQueryRunner.discoverQueryNumbers(ClickBenchTestHelper.DATASET, "ppl"); + // assertFalse("No PPL queries discovered", queryNumbers.isEmpty()); + // logger.info("Discovered {} PPL queries: {}", queryNumbers.size(), queryNumbers); + List queryNumbers = QUERY_NUMBERS; + logger.info("Running {} PPL queries: {}", queryNumbers.size(), queryNumbers); + + List failures = DatasetQueryRunner.runQueries( + client(), + ClickBenchTestHelper.DATASET, + "ppl", + "ppl", + queryNumbers, + (client, dataset, queryBody) -> { + String ppl = queryBody.trim().replace("clickbench", dataset.indexName); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client.performRequest(request); + return assertOkAndParse(response, "PPL query"); + } + ); + + if (failures.isEmpty() == false) { + fail("PPL query failures (" + failures.size() + " of " + queryNumbers.size() + "):\n" + String.join("\n", failures)); + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/bulk.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/bulk.json new file mode 100644 index 0000000000000..32e3d2d6213af --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/bulk.json @@ -0,0 +1,201 @@ +{"index":{}} +{"AdvEngineID":10,"Age":30,"BrowserCountry":"IN","BrowserLanguage":"pt","CLID":703,"ClientEventTime":1379750317504,"ClientIP":1835982476,"ClientTimeZone":-12,"CodeVersion":108,"ConnectTiming":51,"CookieEnable":1,"CounterClass":3,"CounterID":85301,"DNSTiming":64,"DontCountHits":0,"EventDate":1381794967396,"EventTime":1401805406823,"FUniqID":6462023907320545241,"FetchTiming":285,"FlashMajor":7,"FlashMinor":2,"FlashMinor2":5,"FromTag":"","GoodEvent":1,"HID":37911257,"HTTPError":0,"HasGCLID":0,"HistoryLength":3,"HitColor":"D","IPNetworkID":18084,"Income":2,"Interests":529,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1375367160271,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":2,"NetMinor":1,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":16,"RefererHash":8133067322914968248,"RefererRegionID":200,"RegionID":128,"RemoteIP":1613872863,"ResolutionDepth":24,"ResolutionHeight":1031,"ResolutionWidth":2028,"ResponseEndTiming":629,"ResponseStartTiming":297,"Robotness":0,"SearchEngineID":19,"SearchPhrase":"","SendTiming":307,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":1,"URL":"https://test.org/home","URLCategoryID":3,"URLHash":8209337701740256096,"URLRegionID":8,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":52,"UserAgentMinor":"72","UserID":7076057925964094100,"WatchID":271656813891023187,"WindowClientHeight":829,"WindowClientWidth":852,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":0,"Age":43,"BrowserCountry":"US","BrowserLanguage":"ja","CLID":22,"ClientEventTime":1390088415291,"ClientIP":2094639260,"ClientTimeZone":-8,"CodeVersion":298,"ConnectTiming":183,"CookieEnable":1,"CounterClass":3,"CounterID":25578,"DNSTiming":50,"DontCountHits":1,"EventDate":1403151850316,"EventTime":1404450998335,"FUniqID":279750900140691670,"FetchTiming":633,"FlashMajor":2,"FlashMinor":1,"FlashMinor2":7,"FromTag":"","GoodEvent":1,"HID":1789235228,"HTTPError":0,"HasGCLID":1,"HistoryLength":3,"HitColor":"S","IPNetworkID":25321,"Income":2,"Interests":814,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1377823666329,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":0,"NetMinor":8,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":12,"RefererHash":5935949476987109840,"RefererRegionID":223,"RegionID":125,"RemoteIP":124734221,"ResolutionDepth":24,"ResolutionHeight":1306,"ResolutionWidth":2137,"ResponseEndTiming":1900,"ResponseStartTiming":433,"Robotness":0,"SearchEngineID":0,"SearchPhrase":"","SendTiming":443,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":1,"URL":"","URLCategoryID":12,"URLHash":1354385786534450042,"URLRegionID":279,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":97,"UserAgentMinor":"45","UserID":4286985234138737462,"WatchID":5518463129470474332,"WindowClientHeight":1106,"WindowClientWidth":1116,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":18,"Age":6,"BrowserCountry":"DE","BrowserLanguage":"ru","CLID":553,"ClientEventTime":1379649602247,"ClientIP":212432663,"ClientTimeZone":9,"CodeVersion":740,"ConnectTiming":455,"CookieEnable":1,"CounterClass":4,"CounterID":98846,"DNSTiming":175,"DontCountHits":0,"EventDate":1381600177851,"EventTime":1377069021105,"FUniqID":6883698060872852611,"FetchTiming":253,"FlashMajor":15,"FlashMinor":0,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":2020459606,"HTTPError":0,"HasGCLID":0,"HistoryLength":4,"HitColor":"S","IPNetworkID":4638,"Income":4,"Interests":854,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1389874926980,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":6,"NetMinor":5,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":16,"RefererHash":929299212018149194,"RefererRegionID":253,"RegionID":34,"RemoteIP":204563378,"ResolutionDepth":24,"ResolutionHeight":1173,"ResolutionWidth":1197,"ResponseEndTiming":377,"ResponseStartTiming":132,"Robotness":0,"SearchEngineID":23,"SearchPhrase":"","SendTiming":61,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":8,"URL":"https://example.com/page2","URLCategoryID":15,"URLHash":8537232695499613353,"URLRegionID":157,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":81,"UserAgentMinor":"35","UserID":3036134858013145160,"WatchID":7407100882636225418,"WindowClientHeight":896,"WindowClientWidth":1609,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":0,"Age":7,"BrowserCountry":"GB","BrowserLanguage":"es","CLID":757,"ClientEventTime":1390178013386,"ClientIP":535866448,"ClientTimeZone":11,"CodeVersion":208,"ConnectTiming":1,"CookieEnable":1,"CounterClass":0,"CounterID":84233,"DNSTiming":156,"DontCountHits":1,"EventDate":1391596495810,"EventTime":1378737587273,"FUniqID":7971403476100292777,"FetchTiming":145,"FlashMajor":0,"FlashMinor":5,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":830971110,"HTTPError":0,"HasGCLID":0,"HistoryLength":6,"HitColor":"F","IPNetworkID":83890,"Income":4,"Interests":819,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1400781477923,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":0,"NetMinor":9,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":10,"RefererHash":363159986948335095,"RefererRegionID":186,"RegionID":60,"RemoteIP":1107530605,"ResolutionDepth":24,"ResolutionHeight":1179,"ResolutionWidth":2051,"ResponseEndTiming":897,"ResponseStartTiming":20,"Robotness":0,"SearchEngineID":22,"SearchPhrase":"","SendTiming":284,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":6,"URL":"https://example.com/page2","URLCategoryID":18,"URLHash":4712336353078827593,"URLRegionID":155,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":79,"UserAgentMinor":"39","UserID":3672301077964001559,"WatchID":6521427429222255901,"WindowClientHeight":776,"WindowClientWidth":1571,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":22,"Age":77,"BrowserCountry":"KR","BrowserLanguage":"ru","CLID":242,"ClientEventTime":1386186235054,"ClientIP":521933193,"ClientTimeZone":-10,"CodeVersion":392,"ConnectTiming":36,"CookieEnable":0,"CounterClass":3,"CounterID":3074,"DNSTiming":91,"DontCountHits":1,"EventDate":1382420907592,"EventTime":1392915859934,"FUniqID":8058296567601765543,"FetchTiming":594,"FlashMajor":4,"FlashMinor":7,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":966092951,"HTTPError":0,"HasGCLID":0,"HistoryLength":6,"HitColor":"F","IPNetworkID":61445,"Income":3,"Interests":921,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1394248882578,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":4,"NetMinor":3,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":7,"RefererHash":3761685867542427511,"RefererRegionID":244,"RegionID":276,"RemoteIP":1706095501,"ResolutionDepth":24,"ResolutionHeight":689,"ResolutionWidth":1862,"ResponseEndTiming":922,"ResponseStartTiming":119,"Robotness":0,"SearchEngineID":11,"SearchPhrase":"","SendTiming":259,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":5,"URL":"https://test.org/home","URLCategoryID":2,"URLHash":4775790706408642788,"URLRegionID":295,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":59,"UserAgentMinor":"91","UserID":4013083712155191581,"WatchID":7950875850776744518,"WindowClientHeight":1184,"WindowClientWidth":1796,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":2,"Age":61,"BrowserCountry":"JP","BrowserLanguage":"ko","CLID":109,"ClientEventTime":1388679635214,"ClientIP":1427889864,"ClientTimeZone":5,"CodeVersion":977,"ConnectTiming":147,"CookieEnable":0,"CounterClass":2,"CounterID":39435,"DNSTiming":146,"DontCountHits":0,"EventDate":1405103939300,"EventTime":1381118741422,"FUniqID":8391292063251479400,"FetchTiming":443,"FlashMajor":4,"FlashMinor":6,"FlashMinor2":2,"FromTag":"","GoodEvent":1,"HID":26377839,"HTTPError":0,"HasGCLID":0,"HistoryLength":9,"HitColor":"F","IPNetworkID":35286,"Income":0,"Interests":453,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1376107714299,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":2,"NetMinor":4,"OS":5,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":9,"RefererHash":6515678428679980985,"RefererRegionID":216,"RegionID":217,"RemoteIP":1081143489,"ResolutionDepth":24,"ResolutionHeight":1278,"ResolutionWidth":1281,"ResponseEndTiming":1594,"ResponseStartTiming":488,"Robotness":0,"SearchEngineID":24,"SearchPhrase":"","SendTiming":163,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":7,"URL":"https://example.com/page1","URLCategoryID":13,"URLHash":5375557560319626612,"URLRegionID":11,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":11,"UserAgentMinor":"15","UserID":276628673459579515,"WatchID":3582921367521951721,"WindowClientHeight":401,"WindowClientWidth":1706,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":29,"Age":0,"BrowserCountry":"CN","BrowserLanguage":"zh","CLID":689,"ClientEventTime":1398793093257,"ClientIP":2020334517,"ClientTimeZone":-6,"CodeVersion":579,"ConnectTiming":275,"CookieEnable":1,"CounterClass":2,"CounterID":92308,"DNSTiming":78,"DontCountHits":1,"EventDate":1388605538012,"EventTime":1394159833212,"FUniqID":2146811678844114879,"FetchTiming":536,"FlashMajor":7,"FlashMinor":5,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":1345325860,"HTTPError":0,"HasGCLID":1,"HistoryLength":19,"HitColor":"F","IPNetworkID":20505,"Income":3,"Interests":220,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1399276044071,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":1,"NetMinor":4,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":5,"RefererHash":5532299434458103057,"RefererRegionID":129,"RegionID":210,"RemoteIP":1639787889,"ResolutionDepth":24,"ResolutionHeight":800,"ResolutionWidth":2149,"ResponseEndTiming":1120,"ResponseStartTiming":149,"Robotness":0,"SearchEngineID":15,"SearchPhrase":"","SendTiming":141,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":7,"URL":"https://shop.io/product","URLCategoryID":7,"URLHash":1628510678228279300,"URLRegionID":145,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":39,"UserAgentMinor":"47","UserID":7186273305202321071,"WatchID":4246717943548105697,"WindowClientHeight":570,"WindowClientWidth":1087,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":27,"Age":65,"BrowserCountry":"CN","BrowserLanguage":"zh","CLID":144,"ClientEventTime":1398833975348,"ClientIP":1089158308,"ClientTimeZone":-5,"CodeVersion":642,"ConnectTiming":394,"CookieEnable":1,"CounterClass":2,"CounterID":11037,"DNSTiming":188,"DontCountHits":1,"EventDate":1402542265559,"EventTime":1402628124783,"FUniqID":797007263018087889,"FetchTiming":400,"FlashMajor":4,"FlashMinor":7,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":1059091810,"HTTPError":0,"HasGCLID":0,"HistoryLength":3,"HitColor":"F","IPNetworkID":56778,"Income":3,"Interests":475,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1393953151878,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":4,"NetMinor":6,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":0,"RefererHash":8686775061687841194,"RefererRegionID":230,"RegionID":284,"RemoteIP":283284006,"ResolutionDepth":24,"ResolutionHeight":614,"ResolutionWidth":2299,"ResponseEndTiming":1004,"ResponseStartTiming":469,"Robotness":0,"SearchEngineID":0,"SearchPhrase":"","SendTiming":44,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":3,"URL":"https://news.net/article","URLCategoryID":8,"URLHash":1130530367291705449,"URLRegionID":122,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":54,"UserAgentMinor":"5","UserID":4834888972533225111,"WatchID":2869802889882812341,"WindowClientHeight":924,"WindowClientWidth":1183,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":5,"Age":3,"BrowserCountry":"US","BrowserLanguage":"ko","CLID":749,"ClientEventTime":1374247548108,"ClientIP":1695239397,"ClientTimeZone":11,"CodeVersion":921,"ConnectTiming":203,"CookieEnable":1,"CounterClass":4,"CounterID":51523,"DNSTiming":85,"DontCountHits":1,"EventDate":1386589634613,"EventTime":1394817643692,"FUniqID":5204228363253016168,"FetchTiming":520,"FlashMajor":6,"FlashMinor":2,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":210162438,"HTTPError":0,"HasGCLID":0,"HistoryLength":9,"HitColor":"T","IPNetworkID":45996,"Income":4,"Interests":94,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1395868669397,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":9,"NetMinor":5,"OS":5,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":6,"RefererHash":3981781431659979154,"RefererRegionID":112,"RegionID":127,"RemoteIP":115431714,"ResolutionDepth":24,"ResolutionHeight":1014,"ResolutionWidth":2419,"ResponseEndTiming":991,"ResponseStartTiming":362,"Robotness":0,"SearchEngineID":13,"SearchPhrase":"","SendTiming":8,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":6,"URL":"","URLCategoryID":8,"URLHash":4291024672244884972,"URLRegionID":133,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":86,"UserAgentMinor":"12","UserID":897131065585982232,"WatchID":540731202969557281,"WindowClientHeight":1121,"WindowClientWidth":845,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":21,"Age":55,"BrowserCountry":"BR","BrowserLanguage":"fr","CLID":961,"ClientEventTime":1398124803947,"ClientIP":2130195362,"ClientTimeZone":-2,"CodeVersion":741,"ConnectTiming":302,"CookieEnable":0,"CounterClass":2,"CounterID":6614,"DNSTiming":73,"DontCountHits":1,"EventDate":1384733521168,"EventTime":1376281212089,"FUniqID":2107254466060500961,"FetchTiming":791,"FlashMajor":12,"FlashMinor":8,"FlashMinor2":3,"FromTag":"","GoodEvent":1,"HID":292840933,"HTTPError":0,"HasGCLID":1,"HistoryLength":0,"HitColor":"S","IPNetworkID":87051,"Income":1,"Interests":998,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1386586951994,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":5,"NetMinor":9,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":11,"RefererHash":7445947595674360646,"RefererRegionID":3,"RegionID":114,"RemoteIP":1351921656,"ResolutionDepth":24,"ResolutionHeight":687,"ResolutionWidth":1162,"ResponseEndTiming":1832,"ResponseStartTiming":54,"Robotness":0,"SearchEngineID":5,"SearchPhrase":"","SendTiming":372,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":4,"URL":"https://shop.io/product","URLCategoryID":5,"URLHash":6834016704114307107,"URLRegionID":274,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":12,"UserAgentMinor":"85","UserID":104423547781479193,"WatchID":2032270572279535667,"WindowClientHeight":1126,"WindowClientWidth":1279,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":21,"Age":47,"BrowserCountry":"DE","BrowserLanguage":"es","CLID":177,"ClientEventTime":1394141302383,"ClientIP":299505898,"ClientTimeZone":-11,"CodeVersion":939,"ConnectTiming":77,"CookieEnable":0,"CounterClass":3,"CounterID":95303,"DNSTiming":88,"DontCountHits":0,"EventDate":1396112239171,"EventTime":1390517411774,"FUniqID":5778807896494678976,"FetchTiming":946,"FlashMajor":14,"FlashMinor":9,"FlashMinor2":7,"FromTag":"","GoodEvent":1,"HID":1406920328,"HTTPError":0,"HasGCLID":0,"HistoryLength":18,"HitColor":"T","IPNetworkID":934,"Income":0,"Interests":948,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1387888427138,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":6,"NetMinor":8,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":1,"RefererHash":5766156719992022382,"RefererRegionID":244,"RegionID":153,"RemoteIP":714858692,"ResolutionDepth":24,"ResolutionHeight":1035,"ResolutionWidth":1727,"ResponseEndTiming":1891,"ResponseStartTiming":336,"Robotness":0,"SearchEngineID":5,"SearchPhrase":"","SendTiming":376,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Product List","TraficSourceID":3,"URL":"https://test.org/home","URLCategoryID":18,"URLHash":4541805941726908621,"URLRegionID":90,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":31,"UserAgentMinor":"20","UserID":4148412588417578652,"WatchID":4057775555270226711,"WindowClientHeight":989,"WindowClientWidth":646,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":9,"Age":37,"BrowserCountry":"RU","BrowserLanguage":"zh","CLID":397,"ClientEventTime":1373913343444,"ClientIP":2105211974,"ClientTimeZone":-9,"CodeVersion":538,"ConnectTiming":422,"CookieEnable":0,"CounterClass":0,"CounterID":96994,"DNSTiming":135,"DontCountHits":1,"EventDate":1386108393823,"EventTime":1398244844677,"FUniqID":8002949440771858634,"FetchTiming":371,"FlashMajor":15,"FlashMinor":4,"FlashMinor2":5,"FromTag":"","GoodEvent":1,"HID":965563715,"HTTPError":0,"HasGCLID":1,"HistoryLength":3,"HitColor":"D","IPNetworkID":39123,"Income":2,"Interests":961,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1383958114093,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":6,"NetMinor":2,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":18,"RefererHash":2185981060553613677,"RefererRegionID":246,"RegionID":111,"RemoteIP":1466070535,"ResolutionDepth":24,"ResolutionHeight":684,"ResolutionWidth":824,"ResponseEndTiming":1483,"ResponseStartTiming":438,"Robotness":0,"SearchEngineID":10,"SearchPhrase":"","SendTiming":345,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":3,"URL":"https://example.com/page2","URLCategoryID":15,"URLHash":8404935053291054283,"URLRegionID":37,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":25,"UserAgentMinor":"84","UserID":190585386646912833,"WatchID":4638436463835387329,"WindowClientHeight":627,"WindowClientWidth":1333,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":15,"Age":68,"BrowserCountry":"KR","BrowserLanguage":"ja","CLID":231,"ClientEventTime":1393098465957,"ClientIP":1440955967,"ClientTimeZone":-12,"CodeVersion":853,"ConnectTiming":106,"CookieEnable":1,"CounterClass":1,"CounterID":95451,"DNSTiming":175,"DontCountHits":1,"EventDate":1386416977256,"EventTime":1390039323054,"FUniqID":4896266720662695112,"FetchTiming":612,"FlashMajor":2,"FlashMinor":7,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":1605850834,"HTTPError":0,"HasGCLID":0,"HistoryLength":11,"HitColor":"F","IPNetworkID":52218,"Income":1,"Interests":282,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1377484244867,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":0,"NetMinor":1,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":9,"RefererHash":4756015516738067280,"RefererRegionID":213,"RegionID":199,"RemoteIP":2134076018,"ResolutionDepth":24,"ResolutionHeight":618,"ResolutionWidth":1871,"ResponseEndTiming":1398,"ResponseStartTiming":341,"Robotness":0,"SearchEngineID":9,"SearchPhrase":"","SendTiming":305,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":2,"URL":"https://example.com/page1","URLCategoryID":4,"URLHash":9073908784508075690,"URLRegionID":74,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":37,"UserAgentMinor":"3","UserID":9086222670251571902,"WatchID":4003948095766148942,"WindowClientHeight":1142,"WindowClientWidth":1652,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":24,"Age":76,"BrowserCountry":"FR","BrowserLanguage":"de","CLID":520,"ClientEventTime":1382803085670,"ClientIP":232408932,"ClientTimeZone":-4,"CodeVersion":380,"ConnectTiming":20,"CookieEnable":1,"CounterClass":1,"CounterID":51668,"DNSTiming":152,"DontCountHits":1,"EventDate":1374808928793,"EventTime":1389054125822,"FUniqID":845846848424090030,"FetchTiming":282,"FlashMajor":12,"FlashMinor":7,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":426409396,"HTTPError":0,"HasGCLID":1,"HistoryLength":13,"HitColor":"F","IPNetworkID":56297,"Income":2,"Interests":506,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1397590392099,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":4,"NetMinor":4,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":7,"RefererHash":6497054386970123033,"RefererRegionID":171,"RegionID":245,"RemoteIP":594557929,"ResolutionDepth":24,"ResolutionHeight":931,"ResolutionWidth":1271,"ResponseEndTiming":661,"ResponseStartTiming":59,"Robotness":0,"SearchEngineID":22,"SearchPhrase":"","SendTiming":331,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":8,"URL":"https://shop.io/product","URLCategoryID":6,"URLHash":8845406007434994612,"URLRegionID":129,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":88,"UserAgentMinor":"72","UserID":9191536579038913609,"WatchID":6274026602209775618,"WindowClientHeight":1155,"WindowClientWidth":1743,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":17,"Age":6,"BrowserCountry":"RU","BrowserLanguage":"fr","CLID":261,"ClientEventTime":1393087598370,"ClientIP":1037397229,"ClientTimeZone":6,"CodeVersion":622,"ConnectTiming":190,"CookieEnable":0,"CounterClass":3,"CounterID":39150,"DNSTiming":188,"DontCountHits":0,"EventDate":1383516756099,"EventTime":1388540035460,"FUniqID":6204225272551327308,"FetchTiming":839,"FlashMajor":2,"FlashMinor":8,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":240469618,"HTTPError":0,"HasGCLID":1,"HistoryLength":14,"HitColor":"S","IPNetworkID":76516,"Income":2,"Interests":494,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1388606921398,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":5,"NetMinor":4,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":4,"RefererHash":714113377297720358,"RefererRegionID":230,"RegionID":97,"RemoteIP":1871091557,"ResolutionDepth":24,"ResolutionHeight":664,"ResolutionWidth":2111,"ResponseEndTiming":352,"ResponseStartTiming":418,"Robotness":0,"SearchEngineID":16,"SearchPhrase":"","SendTiming":284,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":1,"URL":"https://example.com/page1","URLCategoryID":9,"URLHash":1916762399920928349,"URLRegionID":74,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":95,"UserAgentMinor":"58","UserID":6898622773827835944,"WatchID":8198661416696709212,"WindowClientHeight":1132,"WindowClientWidth":737,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":24,"Age":2,"BrowserCountry":"KR","BrowserLanguage":"zh","CLID":648,"ClientEventTime":1405026416958,"ClientIP":1063751808,"ClientTimeZone":5,"CodeVersion":926,"ConnectTiming":94,"CookieEnable":1,"CounterClass":3,"CounterID":98142,"DNSTiming":70,"DontCountHits":0,"EventDate":1389403644525,"EventTime":1397033067488,"FUniqID":5021623205943914292,"FetchTiming":863,"FlashMajor":11,"FlashMinor":6,"FlashMinor2":6,"FromTag":"","GoodEvent":1,"HID":1780085656,"HTTPError":0,"HasGCLID":0,"HistoryLength":4,"HitColor":"D","IPNetworkID":61795,"Income":3,"Interests":786,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1395017412252,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":4,"NetMinor":6,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":9,"RefererHash":4901969939742256504,"RefererRegionID":67,"RegionID":179,"RemoteIP":905910479,"ResolutionDepth":24,"ResolutionHeight":1317,"ResolutionWidth":1824,"ResponseEndTiming":1365,"ResponseStartTiming":172,"Robotness":0,"SearchEngineID":13,"SearchPhrase":"","SendTiming":359,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":7,"URL":"https://news.net/article","URLCategoryID":6,"URLHash":893850460274847160,"URLRegionID":51,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":96,"UserAgentMinor":"0","UserID":1920939874720921606,"WatchID":3409462176131876904,"WindowClientHeight":488,"WindowClientWidth":1815,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":10,"Age":8,"BrowserCountry":"FR","BrowserLanguage":"ru","CLID":307,"ClientEventTime":1400756584177,"ClientIP":1243954057,"ClientTimeZone":9,"CodeVersion":305,"ConnectTiming":382,"CookieEnable":1,"CounterClass":2,"CounterID":17627,"DNSTiming":56,"DontCountHits":1,"EventDate":1374068765501,"EventTime":1377532058090,"FUniqID":8321602121857972373,"FetchTiming":852,"FlashMajor":4,"FlashMinor":6,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":229470491,"HTTPError":0,"HasGCLID":0,"HistoryLength":6,"HitColor":"D","IPNetworkID":61764,"Income":1,"Interests":577,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1378183596522,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":6,"NetMinor":1,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":13,"RefererHash":4551762086533404499,"RefererRegionID":85,"RegionID":127,"RemoteIP":102551279,"ResolutionDepth":24,"ResolutionHeight":724,"ResolutionWidth":1041,"ResponseEndTiming":742,"ResponseStartTiming":64,"Robotness":0,"SearchEngineID":9,"SearchPhrase":"","SendTiming":274,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":8,"URL":"https://news.net/article","URLCategoryID":17,"URLHash":9165379557963187267,"URLRegionID":201,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":60,"UserAgentMinor":"65","UserID":9025091091862156214,"WatchID":4079570585950762208,"WindowClientHeight":1068,"WindowClientWidth":1095,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":23,"Age":27,"BrowserCountry":"GB","BrowserLanguage":"ja","CLID":796,"ClientEventTime":1400952064100,"ClientIP":482008357,"ClientTimeZone":3,"CodeVersion":986,"ConnectTiming":357,"CookieEnable":1,"CounterClass":2,"CounterID":17085,"DNSTiming":11,"DontCountHits":1,"EventDate":1384745493450,"EventTime":1393735305257,"FUniqID":4882091986345612813,"FetchTiming":533,"FlashMajor":2,"FlashMinor":4,"FlashMinor2":3,"FromTag":"","GoodEvent":1,"HID":1372620308,"HTTPError":0,"HasGCLID":1,"HistoryLength":18,"HitColor":"F","IPNetworkID":39152,"Income":3,"Interests":748,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1375712928834,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":3,"NetMinor":0,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":2,"RefererHash":3724731393295304485,"RefererRegionID":163,"RegionID":289,"RemoteIP":1288852088,"ResolutionDepth":24,"ResolutionHeight":1142,"ResolutionWidth":1042,"ResponseEndTiming":880,"ResponseStartTiming":14,"Robotness":0,"SearchEngineID":1,"SearchPhrase":"","SendTiming":145,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":2,"URL":"https://shop.io/product","URLCategoryID":3,"URLHash":5063292407566964434,"URLRegionID":72,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":9,"UserAgentMinor":"62","UserID":8212055674049822063,"WatchID":6900957721977215638,"WindowClientHeight":606,"WindowClientWidth":1252,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":10,"Age":66,"BrowserCountry":"DE","BrowserLanguage":"pt","CLID":118,"ClientEventTime":1403007104075,"ClientIP":1496196495,"ClientTimeZone":6,"CodeVersion":638,"ConnectTiming":498,"CookieEnable":0,"CounterClass":4,"CounterID":50472,"DNSTiming":0,"DontCountHits":0,"EventDate":1403264018286,"EventTime":1380461223187,"FUniqID":3594940686544014769,"FetchTiming":614,"FlashMajor":13,"FlashMinor":0,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":1243922213,"HTTPError":0,"HasGCLID":0,"HistoryLength":14,"HitColor":"F","IPNetworkID":85103,"Income":4,"Interests":992,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1373855099767,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":7,"NetMinor":6,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":10,"RefererHash":8140845368405682970,"RefererRegionID":76,"RegionID":286,"RemoteIP":421791961,"ResolutionDepth":24,"ResolutionHeight":975,"ResolutionWidth":1425,"ResponseEndTiming":1554,"ResponseStartTiming":392,"Robotness":0,"SearchEngineID":8,"SearchPhrase":"","SendTiming":337,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":9,"URL":"","URLCategoryID":9,"URLHash":153793992025540488,"URLRegionID":127,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":8,"UserAgentMinor":"7","UserID":8151193876498539643,"WatchID":5887258638221471733,"WindowClientHeight":786,"WindowClientWidth":674,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":8,"Age":31,"BrowserCountry":"KR","BrowserLanguage":"en","CLID":401,"ClientEventTime":1375521060546,"ClientIP":1518144083,"ClientTimeZone":0,"CodeVersion":676,"ConnectTiming":359,"CookieEnable":1,"CounterClass":0,"CounterID":68105,"DNSTiming":159,"DontCountHits":0,"EventDate":1397920158636,"EventTime":1388745487415,"FUniqID":897717110520824994,"FetchTiming":98,"FlashMajor":5,"FlashMinor":9,"FlashMinor2":1,"FromTag":"","GoodEvent":1,"HID":1426002190,"HTTPError":0,"HasGCLID":0,"HistoryLength":17,"HitColor":"T","IPNetworkID":34017,"Income":1,"Interests":12,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1391855408303,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":2,"NetMinor":2,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":0,"RefererHash":7881451833134605771,"RefererRegionID":266,"RegionID":200,"RemoteIP":76888671,"ResolutionDepth":24,"ResolutionHeight":956,"ResolutionWidth":959,"ResponseEndTiming":1448,"ResponseStartTiming":282,"Robotness":0,"SearchEngineID":2,"SearchPhrase":"","SendTiming":479,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Product List","TraficSourceID":8,"URL":"https://shop.io/product","URLCategoryID":16,"URLHash":1140052875053807077,"URLRegionID":71,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":65,"UserAgentMinor":"16","UserID":8915422680402849462,"WatchID":8038567599249304186,"WindowClientHeight":460,"WindowClientWidth":1435,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":0,"Age":31,"BrowserCountry":"DE","BrowserLanguage":"ja","CLID":727,"ClientEventTime":1377440670531,"ClientIP":195952743,"ClientTimeZone":1,"CodeVersion":487,"ConnectTiming":161,"CookieEnable":1,"CounterClass":1,"CounterID":12277,"DNSTiming":56,"DontCountHits":1,"EventDate":1375933541207,"EventTime":1397398058296,"FUniqID":1365675354874034429,"FetchTiming":497,"FlashMajor":5,"FlashMinor":1,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":1677222611,"HTTPError":0,"HasGCLID":1,"HistoryLength":2,"HitColor":"T","IPNetworkID":97418,"Income":1,"Interests":880,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1389297451449,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":1,"NetMinor":4,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":14,"RefererHash":5724054108827458949,"RefererRegionID":69,"RegionID":231,"RemoteIP":785981319,"ResolutionDepth":24,"ResolutionHeight":1130,"ResolutionWidth":984,"ResponseEndTiming":1303,"ResponseStartTiming":216,"Robotness":0,"SearchEngineID":18,"SearchPhrase":"","SendTiming":459,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":3,"URL":"","URLCategoryID":0,"URLHash":5012683307141321002,"URLRegionID":285,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":58,"UserAgentMinor":"65","UserID":5079818060474351176,"WatchID":6648949939986999277,"WindowClientHeight":639,"WindowClientWidth":1444,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":6,"Age":66,"BrowserCountry":"GB","BrowserLanguage":"es","CLID":310,"ClientEventTime":1382512687791,"ClientIP":1173921930,"ClientTimeZone":-8,"CodeVersion":854,"ConnectTiming":134,"CookieEnable":1,"CounterClass":0,"CounterID":38728,"DNSTiming":32,"DontCountHits":1,"EventDate":1381164411244,"EventTime":1379021761596,"FUniqID":3348879159190126143,"FetchTiming":436,"FlashMajor":1,"FlashMinor":8,"FlashMinor2":3,"FromTag":"","GoodEvent":1,"HID":606613605,"HTTPError":0,"HasGCLID":0,"HistoryLength":18,"HitColor":"T","IPNetworkID":96665,"Income":1,"Interests":25,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1387178861865,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":1,"NetMinor":1,"OS":7,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":15,"RefererHash":8972517152577337198,"RefererRegionID":189,"RegionID":8,"RemoteIP":1500009764,"ResolutionDepth":24,"ResolutionHeight":998,"ResolutionWidth":1753,"ResponseEndTiming":1856,"ResponseStartTiming":67,"Robotness":0,"SearchEngineID":16,"SearchPhrase":"","SendTiming":288,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":1,"URL":"https://test.org/home","URLCategoryID":3,"URLHash":6961047394484301144,"URLRegionID":149,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":69,"UserAgentMinor":"74","UserID":5792838220922727037,"WatchID":5584789738526061037,"WindowClientHeight":564,"WindowClientWidth":906,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":8,"Age":76,"BrowserCountry":"CN","BrowserLanguage":"zh","CLID":149,"ClientEventTime":1382904819498,"ClientIP":817765875,"ClientTimeZone":-3,"CodeVersion":119,"ConnectTiming":4,"CookieEnable":0,"CounterClass":2,"CounterID":93991,"DNSTiming":145,"DontCountHits":1,"EventDate":1389444946057,"EventTime":1399330938213,"FUniqID":4287277630361177463,"FetchTiming":283,"FlashMajor":10,"FlashMinor":8,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":985825063,"HTTPError":0,"HasGCLID":0,"HistoryLength":11,"HitColor":"D","IPNetworkID":76204,"Income":3,"Interests":273,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1373995622587,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":1,"NetMinor":4,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":17,"RefererHash":6107637940473077772,"RefererRegionID":202,"RegionID":265,"RemoteIP":1196128783,"ResolutionDepth":24,"ResolutionHeight":1183,"ResolutionWidth":1764,"ResponseEndTiming":618,"ResponseStartTiming":445,"Robotness":0,"SearchEngineID":16,"SearchPhrase":"","SendTiming":184,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":5,"URL":"","URLCategoryID":3,"URLHash":4011953716569823015,"URLRegionID":51,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":22,"UserAgentMinor":"73","UserID":7699849358132991719,"WatchID":1146020741482869863,"WindowClientHeight":739,"WindowClientWidth":1683,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":2,"Age":35,"BrowserCountry":"US","BrowserLanguage":"ja","CLID":140,"ClientEventTime":1385503389176,"ClientIP":1774540481,"ClientTimeZone":9,"CodeVersion":117,"ConnectTiming":115,"CookieEnable":0,"CounterClass":4,"CounterID":34799,"DNSTiming":198,"DontCountHits":1,"EventDate":1397346333023,"EventTime":1398255230637,"FUniqID":548121723997076496,"FetchTiming":358,"FlashMajor":1,"FlashMinor":1,"FlashMinor2":3,"FromTag":"","GoodEvent":1,"HID":51352635,"HTTPError":0,"HasGCLID":0,"HistoryLength":2,"HitColor":"F","IPNetworkID":32004,"Income":1,"Interests":457,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1403301208517,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":4,"NetMinor":8,"OS":7,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":13,"RefererHash":9005482525937662512,"RefererRegionID":121,"RegionID":68,"RemoteIP":916678657,"ResolutionDepth":24,"ResolutionHeight":1180,"ResolutionWidth":957,"ResponseEndTiming":948,"ResponseStartTiming":448,"Robotness":0,"SearchEngineID":29,"SearchPhrase":"","SendTiming":36,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":1,"URL":"https://example.com/page2","URLCategoryID":2,"URLHash":3781771520443464054,"URLRegionID":150,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":83,"UserAgentMinor":"96","UserID":5462686348938851039,"WatchID":5161026189233275925,"WindowClientHeight":792,"WindowClientWidth":1387,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":24,"Age":76,"BrowserCountry":"US","BrowserLanguage":"es","CLID":185,"ClientEventTime":1387405477764,"ClientIP":516001859,"ClientTimeZone":-8,"CodeVersion":178,"ConnectTiming":404,"CookieEnable":0,"CounterClass":4,"CounterID":48965,"DNSTiming":20,"DontCountHits":0,"EventDate":1377948642067,"EventTime":1388120135519,"FUniqID":6862832545616035506,"FetchTiming":495,"FlashMajor":1,"FlashMinor":0,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":1456579777,"HTTPError":0,"HasGCLID":0,"HistoryLength":18,"HitColor":"S","IPNetworkID":36760,"Income":2,"Interests":418,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1397153506306,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":6,"NetMinor":9,"OS":5,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":3,"RefererHash":4640227194587661489,"RefererRegionID":48,"RegionID":142,"RemoteIP":1546071629,"ResolutionDepth":24,"ResolutionHeight":791,"ResolutionWidth":1460,"ResponseEndTiming":1884,"ResponseStartTiming":268,"Robotness":0,"SearchEngineID":14,"SearchPhrase":"","SendTiming":202,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":5,"URL":"https://example.com/page2","URLCategoryID":1,"URLHash":8843973769795447537,"URLRegionID":11,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":71,"UserAgentMinor":"53","UserID":5632276311989930026,"WatchID":4651140254526360601,"WindowClientHeight":1148,"WindowClientWidth":1798,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":0,"Age":74,"BrowserCountry":"FR","BrowserLanguage":"zh","CLID":238,"ClientEventTime":1403855455571,"ClientIP":1279373144,"ClientTimeZone":-12,"CodeVersion":792,"ConnectTiming":326,"CookieEnable":1,"CounterClass":0,"CounterID":81162,"DNSTiming":68,"DontCountHits":1,"EventDate":1404925663971,"EventTime":1379403966966,"FUniqID":1985006531838525108,"FetchTiming":692,"FlashMajor":10,"FlashMinor":2,"FlashMinor2":7,"FromTag":"","GoodEvent":1,"HID":1620878608,"HTTPError":0,"HasGCLID":1,"HistoryLength":11,"HitColor":"F","IPNetworkID":89442,"Income":1,"Interests":895,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1379566898122,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":7,"NetMinor":3,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":14,"RefererHash":3109268238480190705,"RefererRegionID":188,"RegionID":48,"RemoteIP":1435622621,"ResolutionDepth":24,"ResolutionHeight":907,"ResolutionWidth":1311,"ResponseEndTiming":1177,"ResponseStartTiming":269,"Robotness":0,"SearchEngineID":28,"SearchPhrase":"","SendTiming":499,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":7,"URL":"https://example.com/page1","URLCategoryID":19,"URLHash":3916928153552661482,"URLRegionID":231,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":73,"UserAgentMinor":"61","UserID":2889759834062751568,"WatchID":3288480944005696756,"WindowClientHeight":659,"WindowClientWidth":1424,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":19,"Age":68,"BrowserCountry":"BR","BrowserLanguage":"ja","CLID":849,"ClientEventTime":1382467079466,"ClientIP":740000073,"ClientTimeZone":10,"CodeVersion":544,"ConnectTiming":131,"CookieEnable":1,"CounterClass":0,"CounterID":84649,"DNSTiming":132,"DontCountHits":1,"EventDate":1387031590352,"EventTime":1374876113724,"FUniqID":2924590451684272260,"FetchTiming":516,"FlashMajor":14,"FlashMinor":7,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":1688109819,"HTTPError":0,"HasGCLID":1,"HistoryLength":14,"HitColor":"S","IPNetworkID":26004,"Income":1,"Interests":557,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1399747977673,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":0,"NetMinor":3,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":6,"RefererHash":4647242157024860673,"RefererRegionID":297,"RegionID":17,"RemoteIP":482428004,"ResolutionDepth":24,"ResolutionHeight":1175,"ResolutionWidth":1207,"ResponseEndTiming":1348,"ResponseStartTiming":301,"Robotness":0,"SearchEngineID":0,"SearchPhrase":"","SendTiming":180,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":8,"URL":"https://example.com/page2","URLCategoryID":3,"URLHash":8917236974964166935,"URLRegionID":286,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":38,"UserAgentMinor":"99","UserID":4500539201403282534,"WatchID":4580832337090644420,"WindowClientHeight":960,"WindowClientWidth":1702,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":13,"Age":74,"BrowserCountry":"FR","BrowserLanguage":"ko","CLID":566,"ClientEventTime":1374159306274,"ClientIP":338685559,"ClientTimeZone":9,"CodeVersion":715,"ConnectTiming":35,"CookieEnable":0,"CounterClass":3,"CounterID":71219,"DNSTiming":105,"DontCountHits":0,"EventDate":1387444591704,"EventTime":1378263033995,"FUniqID":3584892481840266253,"FetchTiming":84,"FlashMajor":2,"FlashMinor":6,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":227658931,"HTTPError":0,"HasGCLID":1,"HistoryLength":0,"HitColor":"D","IPNetworkID":19895,"Income":2,"Interests":720,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1403097665908,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":6,"NetMinor":0,"OS":7,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":0,"RefererHash":2773013080098087590,"RefererRegionID":230,"RegionID":181,"RemoteIP":1641285514,"ResolutionDepth":24,"ResolutionHeight":1297,"ResolutionWidth":2256,"ResponseEndTiming":33,"ResponseStartTiming":111,"Robotness":0,"SearchEngineID":7,"SearchPhrase":"","SendTiming":184,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":0,"URL":"https://news.net/article","URLCategoryID":15,"URLHash":4622710465990672776,"URLRegionID":22,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":37,"UserAgentMinor":"35","UserID":6254024187273422820,"WatchID":2578859653808999647,"WindowClientHeight":436,"WindowClientWidth":1594,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":5,"Age":23,"BrowserCountry":"KR","BrowserLanguage":"zh","CLID":562,"ClientEventTime":1379243848321,"ClientIP":432758434,"ClientTimeZone":-10,"CodeVersion":851,"ConnectTiming":140,"CookieEnable":1,"CounterClass":4,"CounterID":50519,"DNSTiming":67,"DontCountHits":1,"EventDate":1389253481580,"EventTime":1404463765254,"FUniqID":2700837420210103268,"FetchTiming":531,"FlashMajor":19,"FlashMinor":1,"FlashMinor2":3,"FromTag":"","GoodEvent":1,"HID":1767984303,"HTTPError":0,"HasGCLID":0,"HistoryLength":19,"HitColor":"S","IPNetworkID":31468,"Income":4,"Interests":54,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1398406867834,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":5,"NetMinor":6,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":15,"RefererHash":1697493775418531840,"RefererRegionID":63,"RegionID":76,"RemoteIP":1617179119,"ResolutionDepth":24,"ResolutionHeight":933,"ResolutionWidth":1001,"ResponseEndTiming":870,"ResponseStartTiming":453,"Robotness":0,"SearchEngineID":4,"SearchPhrase":"","SendTiming":304,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":6,"URL":"https://example.com/page2","URLCategoryID":4,"URLHash":7821931004583057894,"URLRegionID":265,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":80,"UserAgentMinor":"96","UserID":8913494403709732146,"WatchID":1706082036840605211,"WindowClientHeight":770,"WindowClientWidth":1623,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":7,"Age":8,"BrowserCountry":"JP","BrowserLanguage":"ko","CLID":838,"ClientEventTime":1383654853389,"ClientIP":1749429235,"ClientTimeZone":-1,"CodeVersion":416,"ConnectTiming":132,"CookieEnable":1,"CounterClass":2,"CounterID":21021,"DNSTiming":96,"DontCountHits":0,"EventDate":1383916077776,"EventTime":1402478775683,"FUniqID":7819983789319590928,"FetchTiming":723,"FlashMajor":17,"FlashMinor":6,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":2087575927,"HTTPError":0,"HasGCLID":0,"HistoryLength":14,"HitColor":"D","IPNetworkID":95622,"Income":3,"Interests":981,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1395988018857,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":5,"NetMinor":8,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":15,"RefererHash":1520226324398796728,"RefererRegionID":264,"RegionID":5,"RemoteIP":1381706562,"ResolutionDepth":24,"ResolutionHeight":741,"ResolutionWidth":1897,"ResponseEndTiming":499,"ResponseStartTiming":14,"Robotness":0,"SearchEngineID":14,"SearchPhrase":"","SendTiming":2,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":-1,"URL":"https://test.org/home","URLCategoryID":0,"URLHash":6020785011283995796,"URLRegionID":14,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":1,"UserAgentMinor":"46","UserID":6297508625564670780,"WatchID":1888216070885881215,"WindowClientHeight":625,"WindowClientWidth":1899,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":14,"Age":71,"BrowserCountry":"IN","BrowserLanguage":"en","CLID":651,"ClientEventTime":1385224464081,"ClientIP":280404022,"ClientTimeZone":3,"CodeVersion":931,"ConnectTiming":306,"CookieEnable":1,"CounterClass":1,"CounterID":59646,"DNSTiming":14,"DontCountHits":0,"EventDate":1377909050413,"EventTime":1385364769322,"FUniqID":1223976079163491243,"FetchTiming":941,"FlashMajor":3,"FlashMinor":7,"FlashMinor2":0,"FromTag":"","GoodEvent":1,"HID":484236066,"HTTPError":0,"HasGCLID":0,"HistoryLength":1,"HitColor":"D","IPNetworkID":67363,"Income":3,"Interests":747,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1378474533322,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":3,"NetMinor":8,"OS":7,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":9,"RefererHash":2908881481585554313,"RefererRegionID":144,"RegionID":243,"RemoteIP":871976977,"ResolutionDepth":24,"ResolutionHeight":900,"ResolutionWidth":1850,"ResponseEndTiming":1516,"ResponseStartTiming":12,"Robotness":0,"SearchEngineID":7,"SearchPhrase":"","SendTiming":31,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":0,"URL":"","URLCategoryID":7,"URLHash":5065581118374500587,"URLRegionID":33,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":20,"UserAgentMinor":"93","UserID":4126780196955745411,"WatchID":4265432423609845947,"WindowClientHeight":810,"WindowClientWidth":1598,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":4,"Age":12,"BrowserCountry":"CN","BrowserLanguage":"fr","CLID":856,"ClientEventTime":1401199469283,"ClientIP":1158997898,"ClientTimeZone":8,"CodeVersion":509,"ConnectTiming":337,"CookieEnable":1,"CounterClass":0,"CounterID":48132,"DNSTiming":57,"DontCountHits":0,"EventDate":1377246421162,"EventTime":1404856914068,"FUniqID":5027419995967709266,"FetchTiming":362,"FlashMajor":13,"FlashMinor":0,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":787512277,"HTTPError":0,"HasGCLID":1,"HistoryLength":19,"HitColor":"D","IPNetworkID":38643,"Income":1,"Interests":334,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1399201270389,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":6,"NetMinor":6,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":13,"RefererHash":1844112208908886459,"RefererRegionID":236,"RegionID":84,"RemoteIP":180034489,"ResolutionDepth":24,"ResolutionHeight":795,"ResolutionWidth":1818,"ResponseEndTiming":873,"ResponseStartTiming":127,"Robotness":0,"SearchEngineID":3,"SearchPhrase":"","SendTiming":453,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":4,"URL":"https://shop.io/product","URLCategoryID":17,"URLHash":738023836634194840,"URLRegionID":99,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":95,"UserAgentMinor":"12","UserID":7144565679965978013,"WatchID":638723060752033380,"WindowClientHeight":1006,"WindowClientWidth":1165,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":14,"Age":73,"BrowserCountry":"BR","BrowserLanguage":"ko","CLID":851,"ClientEventTime":1387917282278,"ClientIP":1547656796,"ClientTimeZone":-8,"CodeVersion":445,"ConnectTiming":476,"CookieEnable":1,"CounterClass":3,"CounterID":66760,"DNSTiming":83,"DontCountHits":0,"EventDate":1386426355477,"EventTime":1377772754311,"FUniqID":3434171946548516801,"FetchTiming":37,"FlashMajor":3,"FlashMinor":7,"FlashMinor2":7,"FromTag":"","GoodEvent":1,"HID":1403395388,"HTTPError":0,"HasGCLID":1,"HistoryLength":0,"HitColor":"F","IPNetworkID":59478,"Income":1,"Interests":657,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1386590189796,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":8,"NetMinor":1,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":4,"RefererHash":7375082655671175409,"RefererRegionID":238,"RegionID":76,"RemoteIP":5789242,"ResolutionDepth":24,"ResolutionHeight":772,"ResolutionWidth":2381,"ResponseEndTiming":543,"ResponseStartTiming":165,"Robotness":0,"SearchEngineID":0,"SearchPhrase":"","SendTiming":165,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":-1,"URL":"https://example.com/page1","URLCategoryID":1,"URLHash":2643726455277610418,"URLRegionID":235,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":89,"UserAgentMinor":"98","UserID":1802971169683895416,"WatchID":305050023625964908,"WindowClientHeight":433,"WindowClientWidth":1354,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":12,"Age":48,"BrowserCountry":"FR","BrowserLanguage":"es","CLID":516,"ClientEventTime":1382340585341,"ClientIP":1561567013,"ClientTimeZone":-10,"CodeVersion":417,"ConnectTiming":393,"CookieEnable":1,"CounterClass":2,"CounterID":44849,"DNSTiming":76,"DontCountHits":0,"EventDate":1403149256163,"EventTime":1385165336415,"FUniqID":8459571395945485455,"FetchTiming":239,"FlashMajor":9,"FlashMinor":5,"FlashMinor2":2,"FromTag":"","GoodEvent":1,"HID":61889576,"HTTPError":0,"HasGCLID":1,"HistoryLength":19,"HitColor":"D","IPNetworkID":82471,"Income":1,"Interests":562,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1391426093861,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":1,"NetMinor":5,"OS":5,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":3,"RefererHash":937763963651984839,"RefererRegionID":184,"RegionID":244,"RemoteIP":1275628796,"ResolutionDepth":24,"ResolutionHeight":1362,"ResolutionWidth":955,"ResponseEndTiming":249,"ResponseStartTiming":304,"Robotness":0,"SearchEngineID":19,"SearchPhrase":"","SendTiming":311,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":6,"URL":"https://news.net/article","URLCategoryID":3,"URLHash":1047241051579661543,"URLRegionID":236,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":72,"UserAgentMinor":"5","UserID":7249452110336903899,"WatchID":5785797990260432211,"WindowClientHeight":741,"WindowClientWidth":954,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":4,"Age":59,"BrowserCountry":"US","BrowserLanguage":"it","CLID":602,"ClientEventTime":1391245010827,"ClientIP":231280336,"ClientTimeZone":-4,"CodeVersion":327,"ConnectTiming":417,"CookieEnable":1,"CounterClass":3,"CounterID":7676,"DNSTiming":154,"DontCountHits":0,"EventDate":1383933883250,"EventTime":1400673468892,"FUniqID":3410492401938862921,"FetchTiming":224,"FlashMajor":9,"FlashMinor":2,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":507281737,"HTTPError":0,"HasGCLID":1,"HistoryLength":12,"HitColor":"F","IPNetworkID":72401,"Income":1,"Interests":660,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1404848279239,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":6,"NetMinor":6,"OS":7,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":12,"RefererHash":3882459607235037873,"RefererRegionID":249,"RegionID":135,"RemoteIP":1716185860,"ResolutionDepth":24,"ResolutionHeight":750,"ResolutionWidth":2556,"ResponseEndTiming":635,"ResponseStartTiming":304,"Robotness":0,"SearchEngineID":26,"SearchPhrase":"","SendTiming":289,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Product List","TraficSourceID":3,"URL":"","URLCategoryID":6,"URLHash":3502698957483076351,"URLRegionID":205,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":95,"UserAgentMinor":"7","UserID":3461120202554639621,"WatchID":2827752548999545379,"WindowClientHeight":425,"WindowClientWidth":1607,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":13,"Age":64,"BrowserCountry":"RU","BrowserLanguage":"ko","CLID":781,"ClientEventTime":1396588797163,"ClientIP":895384800,"ClientTimeZone":4,"CodeVersion":865,"ConnectTiming":112,"CookieEnable":0,"CounterClass":2,"CounterID":37459,"DNSTiming":136,"DontCountHits":0,"EventDate":1381886607295,"EventTime":1377949072117,"FUniqID":6113687743022505658,"FetchTiming":133,"FlashMajor":17,"FlashMinor":4,"FlashMinor2":1,"FromTag":"","GoodEvent":1,"HID":1979377558,"HTTPError":0,"HasGCLID":1,"HistoryLength":13,"HitColor":"S","IPNetworkID":33183,"Income":0,"Interests":990,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1398030513996,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":9,"NetMinor":2,"OS":7,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":4,"RefererHash":6690087850133728431,"RefererRegionID":34,"RegionID":169,"RemoteIP":1441179548,"ResolutionDepth":24,"ResolutionHeight":1169,"ResolutionWidth":1252,"ResponseEndTiming":415,"ResponseStartTiming":409,"Robotness":0,"SearchEngineID":0,"SearchPhrase":"","SendTiming":460,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Product List","TraficSourceID":9,"URL":"https://news.net/article","URLCategoryID":15,"URLHash":1519708745694378689,"URLRegionID":175,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":60,"UserAgentMinor":"65","UserID":7893193854004508482,"WatchID":7214271753465250470,"WindowClientHeight":735,"WindowClientWidth":1362,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":11,"Age":70,"BrowserCountry":"IN","BrowserLanguage":"it","CLID":824,"ClientEventTime":1397664868737,"ClientIP":1074317957,"ClientTimeZone":0,"CodeVersion":581,"ConnectTiming":124,"CookieEnable":1,"CounterClass":2,"CounterID":99001,"DNSTiming":192,"DontCountHits":0,"EventDate":1377581750872,"EventTime":1397486001722,"FUniqID":9166695144789270361,"FetchTiming":119,"FlashMajor":6,"FlashMinor":9,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":1910136775,"HTTPError":0,"HasGCLID":0,"HistoryLength":1,"HitColor":"S","IPNetworkID":63962,"Income":2,"Interests":285,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1385074904134,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":7,"NetMinor":6,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":17,"RefererHash":7855251378649018409,"RefererRegionID":105,"RegionID":221,"RemoteIP":1237512568,"ResolutionDepth":24,"ResolutionHeight":1042,"ResolutionWidth":1082,"ResponseEndTiming":11,"ResponseStartTiming":375,"Robotness":0,"SearchEngineID":1,"SearchPhrase":"","SendTiming":10,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":8,"URL":"https://example.com/page1","URLCategoryID":0,"URLHash":2770286403909690352,"URLRegionID":101,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":56,"UserAgentMinor":"94","UserID":8508357561349351387,"WatchID":8924795206654726229,"WindowClientHeight":887,"WindowClientWidth":848,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":0,"Age":76,"BrowserCountry":"RU","BrowserLanguage":"pt","CLID":646,"ClientEventTime":1404303025657,"ClientIP":1505087744,"ClientTimeZone":11,"CodeVersion":768,"ConnectTiming":493,"CookieEnable":0,"CounterClass":4,"CounterID":15562,"DNSTiming":77,"DontCountHits":0,"EventDate":1397074797873,"EventTime":1375571923989,"FUniqID":1024427216941111615,"FetchTiming":113,"FlashMajor":5,"FlashMinor":8,"FlashMinor2":0,"FromTag":"","GoodEvent":1,"HID":420714733,"HTTPError":0,"HasGCLID":0,"HistoryLength":2,"HitColor":"T","IPNetworkID":45261,"Income":3,"Interests":409,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1404416640895,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":9,"NetMinor":5,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":19,"RefererHash":5759928592064236873,"RefererRegionID":257,"RegionID":51,"RemoteIP":959378353,"ResolutionDepth":24,"ResolutionHeight":1235,"ResolutionWidth":1535,"ResponseEndTiming":542,"ResponseStartTiming":122,"Robotness":0,"SearchEngineID":7,"SearchPhrase":"","SendTiming":90,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":5,"URL":"https://news.net/article","URLCategoryID":4,"URLHash":8622296006317340585,"URLRegionID":252,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":34,"UserAgentMinor":"13","UserID":5438941971592115913,"WatchID":66535906249598527,"WindowClientHeight":898,"WindowClientWidth":1763,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":4,"Age":57,"BrowserCountry":"RU","BrowserLanguage":"pt","CLID":766,"ClientEventTime":1387060884852,"ClientIP":1789788539,"ClientTimeZone":6,"CodeVersion":700,"ConnectTiming":39,"CookieEnable":0,"CounterClass":0,"CounterID":1863,"DNSTiming":196,"DontCountHits":1,"EventDate":1403850195673,"EventTime":1384825570349,"FUniqID":6798284096726795024,"FetchTiming":321,"FlashMajor":7,"FlashMinor":8,"FlashMinor2":7,"FromTag":"","GoodEvent":1,"HID":1844717714,"HTTPError":0,"HasGCLID":1,"HistoryLength":0,"HitColor":"S","IPNetworkID":57212,"Income":0,"Interests":403,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1374275732006,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":5,"NetMinor":1,"OS":5,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":14,"RefererHash":5235360545935277474,"RefererRegionID":284,"RegionID":107,"RemoteIP":511615579,"ResolutionDepth":24,"ResolutionHeight":1046,"ResolutionWidth":2227,"ResponseEndTiming":263,"ResponseStartTiming":494,"Robotness":0,"SearchEngineID":1,"SearchPhrase":"","SendTiming":398,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":5,"URL":"https://news.net/article","URLCategoryID":12,"URLHash":747099783916377659,"URLRegionID":290,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":14,"UserAgentMinor":"10","UserID":5023665901164810674,"WatchID":7596215580981308817,"WindowClientHeight":924,"WindowClientWidth":1161,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":15,"Age":42,"BrowserCountry":"BR","BrowserLanguage":"zh","CLID":34,"ClientEventTime":1390615674374,"ClientIP":2062909606,"ClientTimeZone":9,"CodeVersion":724,"ConnectTiming":344,"CookieEnable":0,"CounterClass":4,"CounterID":49201,"DNSTiming":160,"DontCountHits":1,"EventDate":1396825107259,"EventTime":1378621882246,"FUniqID":2747301475101919800,"FetchTiming":265,"FlashMajor":6,"FlashMinor":8,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":209969018,"HTTPError":0,"HasGCLID":1,"HistoryLength":4,"HitColor":"D","IPNetworkID":91643,"Income":4,"Interests":131,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1389005555605,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":4,"NetMinor":1,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":8,"RefererHash":461382276892080127,"RefererRegionID":74,"RegionID":185,"RemoteIP":1264767641,"ResolutionDepth":24,"ResolutionHeight":1395,"ResolutionWidth":884,"ResponseEndTiming":981,"ResponseStartTiming":405,"Robotness":0,"SearchEngineID":21,"SearchPhrase":"","SendTiming":438,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":0,"URL":"https://news.net/article","URLCategoryID":5,"URLHash":7579409960198065432,"URLRegionID":105,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":55,"UserAgentMinor":"54","UserID":360248176629955237,"WatchID":2792177944391667944,"WindowClientHeight":788,"WindowClientWidth":953,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":9,"Age":41,"BrowserCountry":"DE","BrowserLanguage":"fr","CLID":440,"ClientEventTime":1405296945301,"ClientIP":2041299217,"ClientTimeZone":-10,"CodeVersion":486,"ConnectTiming":276,"CookieEnable":1,"CounterClass":1,"CounterID":19724,"DNSTiming":106,"DontCountHits":1,"EventDate":1381843236917,"EventTime":1385531848760,"FUniqID":6636501792343295536,"FetchTiming":434,"FlashMajor":15,"FlashMinor":5,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":1551860513,"HTTPError":0,"HasGCLID":1,"HistoryLength":16,"HitColor":"S","IPNetworkID":70095,"Income":3,"Interests":928,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1374559986362,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":8,"NetMinor":9,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":6,"RefererHash":1080132318498396096,"RefererRegionID":147,"RegionID":222,"RemoteIP":1728941352,"ResolutionDepth":24,"ResolutionHeight":667,"ResolutionWidth":2488,"ResponseEndTiming":1497,"ResponseStartTiming":177,"Robotness":0,"SearchEngineID":9,"SearchPhrase":"","SendTiming":103,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":8,"URL":"","URLCategoryID":4,"URLHash":2596354197023096698,"URLRegionID":118,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":42,"UserAgentMinor":"18","UserID":1222875912596395178,"WatchID":2883558330520047267,"WindowClientHeight":498,"WindowClientWidth":780,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":26,"Age":6,"BrowserCountry":"RU","BrowserLanguage":"en","CLID":254,"ClientEventTime":1396929376782,"ClientIP":1062054722,"ClientTimeZone":-9,"CodeVersion":649,"ConnectTiming":43,"CookieEnable":0,"CounterClass":3,"CounterID":44581,"DNSTiming":99,"DontCountHits":0,"EventDate":1380744743456,"EventTime":1391290853283,"FUniqID":1704455679811338255,"FetchTiming":940,"FlashMajor":14,"FlashMinor":0,"FlashMinor2":6,"FromTag":"","GoodEvent":1,"HID":99625347,"HTTPError":0,"HasGCLID":1,"HistoryLength":1,"HitColor":"F","IPNetworkID":91164,"Income":2,"Interests":794,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1390243535720,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":7,"NetMinor":8,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":6,"RefererHash":2004069716512638222,"RefererRegionID":95,"RegionID":190,"RemoteIP":1645668982,"ResolutionDepth":24,"ResolutionHeight":1027,"ResolutionWidth":1000,"ResponseEndTiming":1389,"ResponseStartTiming":186,"Robotness":0,"SearchEngineID":18,"SearchPhrase":"","SendTiming":74,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":1,"URL":"https://example.com/page2","URLCategoryID":1,"URLHash":6972480686331783483,"URLRegionID":30,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":76,"UserAgentMinor":"93","UserID":4990448610460060087,"WatchID":6563455890189765790,"WindowClientHeight":493,"WindowClientWidth":675,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":22,"Age":54,"BrowserCountry":"FR","BrowserLanguage":"it","CLID":230,"ClientEventTime":1381481371836,"ClientIP":1889417464,"ClientTimeZone":9,"CodeVersion":348,"ConnectTiming":348,"CookieEnable":1,"CounterClass":2,"CounterID":46366,"DNSTiming":176,"DontCountHits":0,"EventDate":1392687899957,"EventTime":1380226236362,"FUniqID":8511167865423304162,"FetchTiming":203,"FlashMajor":12,"FlashMinor":9,"FlashMinor2":2,"FromTag":"","GoodEvent":1,"HID":1696899687,"HTTPError":0,"HasGCLID":0,"HistoryLength":13,"HitColor":"D","IPNetworkID":71711,"Income":2,"Interests":384,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1396647831174,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":2,"NetMinor":0,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":7,"RefererHash":2263190215315425916,"RefererRegionID":217,"RegionID":115,"RemoteIP":1991925263,"ResolutionDepth":24,"ResolutionHeight":881,"ResolutionWidth":2451,"ResponseEndTiming":430,"ResponseStartTiming":105,"Robotness":0,"SearchEngineID":22,"SearchPhrase":"","SendTiming":449,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":0,"URL":"","URLCategoryID":6,"URLHash":8291773555900613672,"URLRegionID":230,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":34,"UserAgentMinor":"72","UserID":698507841730920047,"WatchID":2914139897015031974,"WindowClientHeight":1007,"WindowClientWidth":1838,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":4,"Age":52,"BrowserCountry":"BR","BrowserLanguage":"zh","CLID":473,"ClientEventTime":1400818876100,"ClientIP":798934755,"ClientTimeZone":-12,"CodeVersion":289,"ConnectTiming":320,"CookieEnable":0,"CounterClass":1,"CounterID":75101,"DNSTiming":73,"DontCountHits":1,"EventDate":1375833910507,"EventTime":1402250047761,"FUniqID":1247205651827871409,"FetchTiming":726,"FlashMajor":6,"FlashMinor":7,"FlashMinor2":7,"FromTag":"","GoodEvent":1,"HID":851482550,"HTTPError":0,"HasGCLID":1,"HistoryLength":0,"HitColor":"S","IPNetworkID":42284,"Income":1,"Interests":180,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1385117913039,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":8,"NetMinor":8,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":5,"RefererHash":5221036294617523381,"RefererRegionID":197,"RegionID":222,"RemoteIP":1580477190,"ResolutionDepth":24,"ResolutionHeight":686,"ResolutionWidth":2037,"ResponseEndTiming":1266,"ResponseStartTiming":89,"Robotness":0,"SearchEngineID":4,"SearchPhrase":"","SendTiming":236,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":6,"URL":"https://example.com/page2","URLCategoryID":10,"URLHash":7654445653466882603,"URLRegionID":220,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":4,"UserAgentMinor":"76","UserID":4156827405997518020,"WatchID":263975601830173657,"WindowClientHeight":970,"WindowClientWidth":1243,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":19,"Age":7,"BrowserCountry":"BR","BrowserLanguage":"en","CLID":590,"ClientEventTime":1377571448003,"ClientIP":2042510245,"ClientTimeZone":-7,"CodeVersion":251,"ConnectTiming":225,"CookieEnable":0,"CounterClass":3,"CounterID":32158,"DNSTiming":77,"DontCountHits":0,"EventDate":1386061457141,"EventTime":1377701345245,"FUniqID":4386148682688164942,"FetchTiming":529,"FlashMajor":7,"FlashMinor":2,"FlashMinor2":6,"FromTag":"","GoodEvent":1,"HID":1497903076,"HTTPError":0,"HasGCLID":0,"HistoryLength":18,"HitColor":"F","IPNetworkID":39091,"Income":4,"Interests":831,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1383551618298,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":8,"NetMinor":0,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":8,"RefererHash":8953033390543080202,"RefererRegionID":254,"RegionID":168,"RemoteIP":349213808,"ResolutionDepth":24,"ResolutionHeight":1341,"ResolutionWidth":2267,"ResponseEndTiming":352,"ResponseStartTiming":247,"Robotness":0,"SearchEngineID":22,"SearchPhrase":"","SendTiming":226,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":0,"URL":"https://example.com/page1","URLCategoryID":17,"URLHash":3772920185893657988,"URLRegionID":230,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":40,"UserAgentMinor":"60","UserID":6054010646003259871,"WatchID":2707997008820556496,"WindowClientHeight":491,"WindowClientWidth":971,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":24,"Age":39,"BrowserCountry":"RU","BrowserLanguage":"de","CLID":120,"ClientEventTime":1388479268162,"ClientIP":879182714,"ClientTimeZone":8,"CodeVersion":118,"ConnectTiming":130,"CookieEnable":1,"CounterClass":4,"CounterID":20457,"DNSTiming":116,"DontCountHits":0,"EventDate":1398636678586,"EventTime":1377512320315,"FUniqID":4105333365780863476,"FetchTiming":134,"FlashMajor":0,"FlashMinor":1,"FlashMinor2":2,"FromTag":"","GoodEvent":1,"HID":1075924021,"HTTPError":0,"HasGCLID":0,"HistoryLength":11,"HitColor":"D","IPNetworkID":89192,"Income":3,"Interests":67,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1376709141045,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":7,"NetMinor":5,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":16,"RefererHash":2436866122978591865,"RefererRegionID":86,"RegionID":191,"RemoteIP":134520288,"ResolutionDepth":24,"ResolutionHeight":1279,"ResolutionWidth":1419,"ResponseEndTiming":894,"ResponseStartTiming":479,"Robotness":0,"SearchEngineID":17,"SearchPhrase":"","SendTiming":362,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":1,"URL":"https://test.org/home","URLCategoryID":0,"URLHash":3407268209500603110,"URLRegionID":253,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":9,"UserAgentMinor":"85","UserID":1843598226871335914,"WatchID":560501627336861806,"WindowClientHeight":602,"WindowClientWidth":892,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":18,"Age":39,"BrowserCountry":"DE","BrowserLanguage":"de","CLID":289,"ClientEventTime":1379776952993,"ClientIP":1027725832,"ClientTimeZone":7,"CodeVersion":911,"ConnectTiming":303,"CookieEnable":1,"CounterClass":4,"CounterID":71511,"DNSTiming":186,"DontCountHits":0,"EventDate":1381534177979,"EventTime":1397892466322,"FUniqID":8257177815506515056,"FetchTiming":57,"FlashMajor":17,"FlashMinor":2,"FlashMinor2":5,"FromTag":"","GoodEvent":1,"HID":816585482,"HTTPError":0,"HasGCLID":1,"HistoryLength":14,"HitColor":"S","IPNetworkID":98423,"Income":2,"Interests":661,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1397884199874,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":6,"NetMinor":6,"OS":5,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":3,"RefererHash":780835966186954572,"RefererRegionID":186,"RegionID":127,"RemoteIP":2020094432,"ResolutionDepth":24,"ResolutionHeight":1265,"ResolutionWidth":1534,"ResponseEndTiming":496,"ResponseStartTiming":427,"Robotness":0,"SearchEngineID":17,"SearchPhrase":"","SendTiming":436,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":-1,"URL":"https://example.com/page1","URLCategoryID":4,"URLHash":8837578503112226862,"URLRegionID":167,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":47,"UserAgentMinor":"78","UserID":3110401386920732576,"WatchID":1179055880340754198,"WindowClientHeight":922,"WindowClientWidth":1913,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":20,"Age":67,"BrowserCountry":"BR","BrowserLanguage":"pt","CLID":271,"ClientEventTime":1391886801379,"ClientIP":1358157566,"ClientTimeZone":-12,"CodeVersion":841,"ConnectTiming":226,"CookieEnable":0,"CounterClass":2,"CounterID":11562,"DNSTiming":146,"DontCountHits":0,"EventDate":1391800734360,"EventTime":1401421378745,"FUniqID":7114494935800995020,"FetchTiming":213,"FlashMajor":19,"FlashMinor":0,"FlashMinor2":3,"FromTag":"","GoodEvent":1,"HID":1044618145,"HTTPError":0,"HasGCLID":1,"HistoryLength":11,"HitColor":"T","IPNetworkID":76766,"Income":0,"Interests":984,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1389831614358,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":6,"NetMinor":7,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":10,"RefererHash":1849012677247247675,"RefererRegionID":237,"RegionID":1,"RemoteIP":1969250189,"ResolutionDepth":24,"ResolutionHeight":906,"ResolutionWidth":1337,"ResponseEndTiming":531,"ResponseStartTiming":128,"Robotness":0,"SearchEngineID":21,"SearchPhrase":"","SendTiming":21,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":8,"URL":"","URLCategoryID":12,"URLHash":668133440172155894,"URLRegionID":191,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":59,"UserAgentMinor":"86","UserID":7210143902042282079,"WatchID":4854212809280521528,"WindowClientHeight":831,"WindowClientWidth":909,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":21,"Age":73,"BrowserCountry":"BR","BrowserLanguage":"ko","CLID":642,"ClientEventTime":1394787990055,"ClientIP":1405033021,"ClientTimeZone":-9,"CodeVersion":258,"ConnectTiming":266,"CookieEnable":0,"CounterClass":0,"CounterID":89754,"DNSTiming":129,"DontCountHits":1,"EventDate":1390222953202,"EventTime":1405000919056,"FUniqID":1907706885883398765,"FetchTiming":852,"FlashMajor":7,"FlashMinor":4,"FlashMinor2":1,"FromTag":"","GoodEvent":1,"HID":1900455713,"HTTPError":0,"HasGCLID":0,"HistoryLength":13,"HitColor":"S","IPNetworkID":34079,"Income":2,"Interests":97,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1404109598830,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":5,"NetMinor":6,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":0,"RefererHash":8021316753329122000,"RefererRegionID":177,"RegionID":141,"RemoteIP":959596500,"ResolutionDepth":24,"ResolutionHeight":1014,"ResolutionWidth":884,"ResponseEndTiming":203,"ResponseStartTiming":64,"Robotness":0,"SearchEngineID":28,"SearchPhrase":"","SendTiming":312,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":4,"URL":"https://shop.io/product","URLCategoryID":13,"URLHash":5748999244175687991,"URLRegionID":1,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":89,"UserAgentMinor":"70","UserID":2655509612952465516,"WatchID":2171358706064044520,"WindowClientHeight":516,"WindowClientWidth":1818,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":13,"Age":57,"BrowserCountry":"GB","BrowserLanguage":"ja","CLID":905,"ClientEventTime":1403137577840,"ClientIP":838578928,"ClientTimeZone":-10,"CodeVersion":954,"ConnectTiming":68,"CookieEnable":1,"CounterClass":0,"CounterID":53861,"DNSTiming":171,"DontCountHits":0,"EventDate":1400525094336,"EventTime":1377919039632,"FUniqID":6032044365008456156,"FetchTiming":421,"FlashMajor":15,"FlashMinor":6,"FlashMinor2":5,"FromTag":"","GoodEvent":1,"HID":1566455975,"HTTPError":0,"HasGCLID":1,"HistoryLength":17,"HitColor":"F","IPNetworkID":67504,"Income":3,"Interests":156,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1395212503181,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":5,"NetMinor":2,"OS":5,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":11,"RefererHash":729837680370990854,"RefererRegionID":136,"RegionID":29,"RemoteIP":1195158425,"ResolutionDepth":24,"ResolutionHeight":1203,"ResolutionWidth":1406,"ResponseEndTiming":1069,"ResponseStartTiming":150,"Robotness":0,"SearchEngineID":6,"SearchPhrase":"","SendTiming":201,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":5,"URL":"https://test.org/home","URLCategoryID":4,"URLHash":993120896258627922,"URLRegionID":248,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":5,"UserAgentMinor":"97","UserID":9118573761976147156,"WatchID":985872150750401952,"WindowClientHeight":632,"WindowClientWidth":1400,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":28,"Age":27,"BrowserCountry":"IN","BrowserLanguage":"it","CLID":560,"ClientEventTime":1401522232835,"ClientIP":1714451437,"ClientTimeZone":1,"CodeVersion":568,"ConnectTiming":364,"CookieEnable":0,"CounterClass":4,"CounterID":15143,"DNSTiming":47,"DontCountHits":1,"EventDate":1386225859896,"EventTime":1386816993859,"FUniqID":5259650533482851979,"FetchTiming":421,"FlashMajor":3,"FlashMinor":4,"FlashMinor2":5,"FromTag":"","GoodEvent":1,"HID":166168647,"HTTPError":0,"HasGCLID":1,"HistoryLength":12,"HitColor":"T","IPNetworkID":50174,"Income":0,"Interests":505,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1405026994560,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":8,"NetMinor":8,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":4,"RefererHash":3899895570905178120,"RefererRegionID":113,"RegionID":293,"RemoteIP":189569090,"ResolutionDepth":24,"ResolutionHeight":1077,"ResolutionWidth":1676,"ResponseEndTiming":646,"ResponseStartTiming":289,"Robotness":0,"SearchEngineID":13,"SearchPhrase":"","SendTiming":290,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":9,"URL":"https://example.com/page1","URLCategoryID":15,"URLHash":7543860406179362628,"URLRegionID":253,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":13,"UserAgentMinor":"67","UserID":3381067093520520546,"WatchID":7307997627646078889,"WindowClientHeight":968,"WindowClientWidth":601,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":23,"Age":50,"BrowserCountry":"DE","BrowserLanguage":"pt","CLID":592,"ClientEventTime":1381154427785,"ClientIP":1104267163,"ClientTimeZone":2,"CodeVersion":445,"ConnectTiming":249,"CookieEnable":1,"CounterClass":3,"CounterID":22840,"DNSTiming":52,"DontCountHits":1,"EventDate":1401831951526,"EventTime":1396885740899,"FUniqID":5519381571819702690,"FetchTiming":80,"FlashMajor":5,"FlashMinor":8,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":1215252983,"HTTPError":0,"HasGCLID":1,"HistoryLength":2,"HitColor":"S","IPNetworkID":53093,"Income":0,"Interests":822,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1401977895999,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":0,"NetMinor":1,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":5,"RefererHash":2687381410508073438,"RefererRegionID":144,"RegionID":194,"RemoteIP":232251722,"ResolutionDepth":24,"ResolutionHeight":1243,"ResolutionWidth":1139,"ResponseEndTiming":148,"ResponseStartTiming":316,"Robotness":0,"SearchEngineID":20,"SearchPhrase":"","SendTiming":233,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":6,"URL":"","URLCategoryID":8,"URLHash":4105516630998936354,"URLRegionID":275,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":24,"UserAgentMinor":"84","UserID":507695518094295399,"WatchID":239852618408735473,"WindowClientHeight":554,"WindowClientWidth":1469,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":25,"Age":25,"BrowserCountry":"IN","BrowserLanguage":"pt","CLID":124,"ClientEventTime":1403839550596,"ClientIP":335273486,"ClientTimeZone":-12,"CodeVersion":972,"ConnectTiming":499,"CookieEnable":1,"CounterClass":2,"CounterID":41591,"DNSTiming":4,"DontCountHits":0,"EventDate":1388852071632,"EventTime":1389975428234,"FUniqID":7134931861440058950,"FetchTiming":410,"FlashMajor":13,"FlashMinor":0,"FlashMinor2":5,"FromTag":"","GoodEvent":1,"HID":292803280,"HTTPError":0,"HasGCLID":0,"HistoryLength":3,"HitColor":"T","IPNetworkID":45085,"Income":0,"Interests":627,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1374708592853,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":4,"NetMinor":1,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":3,"RefererHash":5007286112899270947,"RefererRegionID":249,"RegionID":69,"RemoteIP":594170336,"ResolutionDepth":24,"ResolutionHeight":669,"ResolutionWidth":1700,"ResponseEndTiming":454,"ResponseStartTiming":389,"Robotness":0,"SearchEngineID":18,"SearchPhrase":"","SendTiming":130,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":1,"URL":"https://news.net/article","URLCategoryID":13,"URLHash":5857652274812030464,"URLRegionID":118,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":14,"UserAgentMinor":"40","UserID":2385889176388156899,"WatchID":2946335356101520142,"WindowClientHeight":845,"WindowClientWidth":1153,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":9,"Age":23,"BrowserCountry":"FR","BrowserLanguage":"ja","CLID":747,"ClientEventTime":1380517225293,"ClientIP":525144915,"ClientTimeZone":-10,"CodeVersion":538,"ConnectTiming":31,"CookieEnable":1,"CounterClass":3,"CounterID":97877,"DNSTiming":178,"DontCountHits":1,"EventDate":1376268217408,"EventTime":1395137030038,"FUniqID":2552620702904300251,"FetchTiming":732,"FlashMajor":2,"FlashMinor":2,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":520777410,"HTTPError":0,"HasGCLID":0,"HistoryLength":15,"HitColor":"D","IPNetworkID":355,"Income":1,"Interests":595,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1395286699032,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":6,"NetMinor":9,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":12,"RefererHash":30234009970559518,"RefererRegionID":268,"RegionID":154,"RemoteIP":1385531886,"ResolutionDepth":24,"ResolutionHeight":1244,"ResolutionWidth":2431,"ResponseEndTiming":1621,"ResponseStartTiming":458,"Robotness":0,"SearchEngineID":12,"SearchPhrase":"","SendTiming":223,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":5,"URL":"https://example.com/page2","URLCategoryID":17,"URLHash":1565029146417613649,"URLRegionID":232,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":3,"UserAgentMinor":"3","UserID":5635329612123598351,"WatchID":2610606025549664099,"WindowClientHeight":475,"WindowClientWidth":1509,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":29,"Age":28,"BrowserCountry":"DE","BrowserLanguage":"ja","CLID":954,"ClientEventTime":1401451015125,"ClientIP":2018905671,"ClientTimeZone":7,"CodeVersion":586,"ConnectTiming":469,"CookieEnable":1,"CounterClass":3,"CounterID":44447,"DNSTiming":57,"DontCountHits":1,"EventDate":1392666958898,"EventTime":1379486028366,"FUniqID":4048070251068431430,"FetchTiming":848,"FlashMajor":9,"FlashMinor":7,"FlashMinor2":6,"FromTag":"","GoodEvent":1,"HID":964363253,"HTTPError":0,"HasGCLID":1,"HistoryLength":11,"HitColor":"D","IPNetworkID":34268,"Income":0,"Interests":466,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1401327037830,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":1,"NetMinor":1,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":11,"RefererHash":291199763284995529,"RefererRegionID":280,"RegionID":239,"RemoteIP":1643129650,"ResolutionDepth":24,"ResolutionHeight":675,"ResolutionWidth":1273,"ResponseEndTiming":773,"ResponseStartTiming":18,"Robotness":0,"SearchEngineID":4,"SearchPhrase":"","SendTiming":285,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Product List","TraficSourceID":4,"URL":"https://test.org/home","URLCategoryID":2,"URLHash":409139370919958454,"URLRegionID":209,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":5,"UserAgentMinor":"8","UserID":7704372898345597022,"WatchID":3387316553187093046,"WindowClientHeight":981,"WindowClientWidth":1076,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":7,"Age":70,"BrowserCountry":"RU","BrowserLanguage":"ru","CLID":223,"ClientEventTime":1380933480076,"ClientIP":558696719,"ClientTimeZone":-10,"CodeVersion":633,"ConnectTiming":442,"CookieEnable":0,"CounterClass":1,"CounterID":2552,"DNSTiming":91,"DontCountHits":0,"EventDate":1395265347693,"EventTime":1376158140310,"FUniqID":8175186074506093674,"FetchTiming":588,"FlashMajor":11,"FlashMinor":7,"FlashMinor2":3,"FromTag":"","GoodEvent":1,"HID":151960465,"HTTPError":0,"HasGCLID":0,"HistoryLength":5,"HitColor":"D","IPNetworkID":8207,"Income":3,"Interests":609,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1394801872693,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":4,"NetMinor":2,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":16,"RefererHash":7809412592660940736,"RefererRegionID":267,"RegionID":215,"RemoteIP":823533480,"ResolutionDepth":24,"ResolutionHeight":1419,"ResolutionWidth":1391,"ResponseEndTiming":588,"ResponseStartTiming":423,"Robotness":0,"SearchEngineID":19,"SearchPhrase":"","SendTiming":473,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":4,"URL":"https://test.org/home","URLCategoryID":18,"URLHash":8406530215512405308,"URLRegionID":41,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":76,"UserAgentMinor":"87","UserID":8941516145002875777,"WatchID":3117032085611389125,"WindowClientHeight":743,"WindowClientWidth":1414,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":2,"Age":60,"BrowserCountry":"US","BrowserLanguage":"it","CLID":370,"ClientEventTime":1398775660181,"ClientIP":287624456,"ClientTimeZone":-11,"CodeVersion":632,"ConnectTiming":78,"CookieEnable":1,"CounterClass":4,"CounterID":2325,"DNSTiming":164,"DontCountHits":0,"EventDate":1375142038627,"EventTime":1383473471818,"FUniqID":1474576707636241655,"FetchTiming":913,"FlashMajor":6,"FlashMinor":1,"FlashMinor2":2,"FromTag":"","GoodEvent":1,"HID":683846509,"HTTPError":0,"HasGCLID":1,"HistoryLength":13,"HitColor":"T","IPNetworkID":32210,"Income":2,"Interests":298,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1391253718081,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":6,"NetMinor":0,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":16,"RefererHash":5101682219692531203,"RefererRegionID":46,"RegionID":299,"RemoteIP":821306688,"ResolutionDepth":24,"ResolutionHeight":866,"ResolutionWidth":1291,"ResponseEndTiming":1590,"ResponseStartTiming":246,"Robotness":0,"SearchEngineID":7,"SearchPhrase":"","SendTiming":4,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":5,"URL":"https://test.org/home","URLCategoryID":13,"URLHash":643545950182100014,"URLRegionID":82,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":80,"UserAgentMinor":"12","UserID":3525496988060480084,"WatchID":4603084999235319211,"WindowClientHeight":951,"WindowClientWidth":956,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":23,"Age":17,"BrowserCountry":"IN","BrowserLanguage":"fr","CLID":278,"ClientEventTime":1395854321769,"ClientIP":979574515,"ClientTimeZone":-11,"CodeVersion":589,"ConnectTiming":359,"CookieEnable":1,"CounterClass":1,"CounterID":90988,"DNSTiming":121,"DontCountHits":0,"EventDate":1390004106190,"EventTime":1387369574633,"FUniqID":1280490688044051897,"FetchTiming":725,"FlashMajor":5,"FlashMinor":6,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":1715133319,"HTTPError":0,"HasGCLID":0,"HistoryLength":19,"HitColor":"D","IPNetworkID":36909,"Income":2,"Interests":564,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1404171496651,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":7,"NetMinor":7,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":2,"RefererHash":5461850876994985910,"RefererRegionID":14,"RegionID":47,"RemoteIP":1251772130,"ResolutionDepth":24,"ResolutionHeight":1124,"ResolutionWidth":1144,"ResponseEndTiming":1007,"ResponseStartTiming":18,"Robotness":0,"SearchEngineID":27,"SearchPhrase":"","SendTiming":292,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":1,"URL":"","URLCategoryID":11,"URLHash":72095073287738643,"URLRegionID":58,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":97,"UserAgentMinor":"29","UserID":515072362543036221,"WatchID":2478165122407481248,"WindowClientHeight":746,"WindowClientWidth":1239,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":5,"Age":70,"BrowserCountry":"DE","BrowserLanguage":"it","CLID":842,"ClientEventTime":1399915014611,"ClientIP":1982744085,"ClientTimeZone":9,"CodeVersion":374,"ConnectTiming":192,"CookieEnable":1,"CounterClass":0,"CounterID":13360,"DNSTiming":41,"DontCountHits":1,"EventDate":1401070116737,"EventTime":1385711252058,"FUniqID":3483695770520982556,"FetchTiming":70,"FlashMajor":8,"FlashMinor":0,"FlashMinor2":7,"FromTag":"","GoodEvent":1,"HID":257893864,"HTTPError":0,"HasGCLID":1,"HistoryLength":11,"HitColor":"F","IPNetworkID":79758,"Income":3,"Interests":73,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1404634863274,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":3,"NetMinor":3,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":8,"RefererHash":8644932471822975739,"RefererRegionID":47,"RegionID":25,"RemoteIP":1920718301,"ResolutionDepth":24,"ResolutionHeight":623,"ResolutionWidth":1095,"ResponseEndTiming":555,"ResponseStartTiming":433,"Robotness":0,"SearchEngineID":16,"SearchPhrase":"","SendTiming":170,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":2,"URL":"https://shop.io/product","URLCategoryID":10,"URLHash":4621874275545697774,"URLRegionID":237,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":73,"UserAgentMinor":"60","UserID":84419044144853970,"WatchID":5435518433424437692,"WindowClientHeight":908,"WindowClientWidth":1342,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":14,"Age":20,"BrowserCountry":"GB","BrowserLanguage":"en","CLID":862,"ClientEventTime":1381971703414,"ClientIP":434142692,"ClientTimeZone":4,"CodeVersion":473,"ConnectTiming":177,"CookieEnable":0,"CounterClass":0,"CounterID":98694,"DNSTiming":181,"DontCountHits":1,"EventDate":1394615762875,"EventTime":1400908636417,"FUniqID":2767562786187497829,"FetchTiming":428,"FlashMajor":5,"FlashMinor":2,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":1124032977,"HTTPError":0,"HasGCLID":0,"HistoryLength":6,"HitColor":"F","IPNetworkID":52611,"Income":1,"Interests":862,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1380959095196,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":1,"NetMinor":4,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":18,"RefererHash":4776328949160371919,"RefererRegionID":60,"RegionID":200,"RemoteIP":315671474,"ResolutionDepth":24,"ResolutionHeight":1031,"ResolutionWidth":2505,"ResponseEndTiming":1687,"ResponseStartTiming":489,"Robotness":0,"SearchEngineID":16,"SearchPhrase":"","SendTiming":350,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":4,"URL":"","URLCategoryID":3,"URLHash":8993447203454778807,"URLRegionID":274,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":50,"UserAgentMinor":"4","UserID":4566644014864009357,"WatchID":2180391954052570327,"WindowClientHeight":1118,"WindowClientWidth":1484,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":17,"Age":64,"BrowserCountry":"FR","BrowserLanguage":"pt","CLID":602,"ClientEventTime":1382170335221,"ClientIP":1000433410,"ClientTimeZone":-11,"CodeVersion":847,"ConnectTiming":298,"CookieEnable":0,"CounterClass":1,"CounterID":78487,"DNSTiming":131,"DontCountHits":1,"EventDate":1377155125074,"EventTime":1382799768329,"FUniqID":7617828908121173486,"FetchTiming":224,"FlashMajor":11,"FlashMinor":4,"FlashMinor2":3,"FromTag":"","GoodEvent":1,"HID":1446982521,"HTTPError":0,"HasGCLID":0,"HistoryLength":10,"HitColor":"S","IPNetworkID":35720,"Income":2,"Interests":561,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1401675051829,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":9,"NetMinor":9,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":3,"RefererHash":7407292833308935089,"RefererRegionID":64,"RegionID":231,"RemoteIP":1517093723,"ResolutionDepth":24,"ResolutionHeight":1297,"ResolutionWidth":800,"ResponseEndTiming":1248,"ResponseStartTiming":107,"Robotness":0,"SearchEngineID":18,"SearchPhrase":"","SendTiming":225,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":6,"URL":"","URLCategoryID":19,"URLHash":3604546194184577351,"URLRegionID":297,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":28,"UserAgentMinor":"68","UserID":7377053271096213452,"WatchID":9135265121151690079,"WindowClientHeight":462,"WindowClientWidth":904,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":16,"Age":16,"BrowserCountry":"JP","BrowserLanguage":"pt","CLID":541,"ClientEventTime":1385688842611,"ClientIP":892675583,"ClientTimeZone":6,"CodeVersion":342,"ConnectTiming":288,"CookieEnable":1,"CounterClass":3,"CounterID":20726,"DNSTiming":46,"DontCountHits":1,"EventDate":1395414874791,"EventTime":1397634016815,"FUniqID":6045590152555535561,"FetchTiming":758,"FlashMajor":17,"FlashMinor":8,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":1784142180,"HTTPError":0,"HasGCLID":0,"HistoryLength":4,"HitColor":"D","IPNetworkID":31298,"Income":2,"Interests":586,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1396077516103,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":3,"NetMinor":1,"OS":5,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":10,"RefererHash":152986467072692989,"RefererRegionID":173,"RegionID":161,"RemoteIP":745316946,"ResolutionDepth":24,"ResolutionHeight":935,"ResolutionWidth":1412,"ResponseEndTiming":1636,"ResponseStartTiming":70,"Robotness":0,"SearchEngineID":27,"SearchPhrase":"","SendTiming":348,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":7,"URL":"https://example.com/page2","URLCategoryID":2,"URLHash":8715014976308727170,"URLRegionID":202,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":74,"UserAgentMinor":"8","UserID":5605451771760445059,"WatchID":4787735280812579200,"WindowClientHeight":815,"WindowClientWidth":1010,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":0,"Age":58,"BrowserCountry":"BR","BrowserLanguage":"ko","CLID":340,"ClientEventTime":1380148957451,"ClientIP":1518193716,"ClientTimeZone":-2,"CodeVersion":759,"ConnectTiming":73,"CookieEnable":1,"CounterClass":4,"CounterID":81587,"DNSTiming":94,"DontCountHits":0,"EventDate":1375136748396,"EventTime":1375628958957,"FUniqID":5294463367548745977,"FetchTiming":526,"FlashMajor":18,"FlashMinor":0,"FlashMinor2":2,"FromTag":"","GoodEvent":1,"HID":1113545046,"HTTPError":0,"HasGCLID":0,"HistoryLength":11,"HitColor":"F","IPNetworkID":33207,"Income":1,"Interests":660,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1396772537914,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":0,"NetMinor":8,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":19,"RefererHash":6584580984525462919,"RefererRegionID":277,"RegionID":239,"RemoteIP":1915003780,"ResolutionDepth":24,"ResolutionHeight":1413,"ResolutionWidth":2539,"ResponseEndTiming":1043,"ResponseStartTiming":121,"Robotness":0,"SearchEngineID":18,"SearchPhrase":"","SendTiming":45,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":0,"URL":"https://shop.io/product","URLCategoryID":14,"URLHash":150795115859813073,"URLRegionID":171,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":12,"UserAgentMinor":"83","UserID":2155704536270615999,"WatchID":366644257593956536,"WindowClientHeight":700,"WindowClientWidth":1298,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":19,"Age":25,"BrowserCountry":"BR","BrowserLanguage":"ru","CLID":532,"ClientEventTime":1374404107797,"ClientIP":2014028644,"ClientTimeZone":4,"CodeVersion":811,"ConnectTiming":27,"CookieEnable":0,"CounterClass":0,"CounterID":91812,"DNSTiming":129,"DontCountHits":1,"EventDate":1385630311293,"EventTime":1405177126768,"FUniqID":8900741636065599302,"FetchTiming":726,"FlashMajor":14,"FlashMinor":3,"FlashMinor2":3,"FromTag":"","GoodEvent":1,"HID":1113518345,"HTTPError":0,"HasGCLID":1,"HistoryLength":8,"HitColor":"S","IPNetworkID":10951,"Income":1,"Interests":684,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1388105880136,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":6,"NetMinor":5,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":8,"RefererHash":2851661262273571842,"RefererRegionID":54,"RegionID":192,"RemoteIP":35767085,"ResolutionDepth":24,"ResolutionHeight":816,"ResolutionWidth":2268,"ResponseEndTiming":1165,"ResponseStartTiming":121,"Robotness":0,"SearchEngineID":28,"SearchPhrase":"","SendTiming":11,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":-1,"URL":"https://example.com/page2","URLCategoryID":10,"URLHash":7644705219195338979,"URLRegionID":138,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":17,"UserAgentMinor":"89","UserID":8695406620814049832,"WatchID":8872157912030258235,"WindowClientHeight":783,"WindowClientWidth":1664,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":15,"Age":74,"BrowserCountry":"BR","BrowserLanguage":"fr","CLID":712,"ClientEventTime":1401394347609,"ClientIP":693513119,"ClientTimeZone":3,"CodeVersion":275,"ConnectTiming":133,"CookieEnable":1,"CounterClass":2,"CounterID":88813,"DNSTiming":42,"DontCountHits":0,"EventDate":1387603693695,"EventTime":1385236973207,"FUniqID":8811335213119400115,"FetchTiming":718,"FlashMajor":2,"FlashMinor":7,"FlashMinor2":0,"FromTag":"","GoodEvent":1,"HID":2023315586,"HTTPError":0,"HasGCLID":1,"HistoryLength":13,"HitColor":"D","IPNetworkID":96693,"Income":2,"Interests":445,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1395726070867,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":0,"NetMinor":2,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":19,"RefererHash":3250845171530774223,"RefererRegionID":156,"RegionID":53,"RemoteIP":155399777,"ResolutionDepth":24,"ResolutionHeight":1422,"ResolutionWidth":1090,"ResponseEndTiming":284,"ResponseStartTiming":187,"Robotness":0,"SearchEngineID":21,"SearchPhrase":"","SendTiming":183,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":6,"URL":"https://example.com/page1","URLCategoryID":17,"URLHash":7108891153884699820,"URLRegionID":131,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":2,"UserAgentMinor":"34","UserID":8297878130355066024,"WatchID":8389705661434125597,"WindowClientHeight":469,"WindowClientWidth":1709,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":23,"Age":19,"BrowserCountry":"CN","BrowserLanguage":"zh","CLID":394,"ClientEventTime":1400971229826,"ClientIP":219244700,"ClientTimeZone":10,"CodeVersion":404,"ConnectTiming":303,"CookieEnable":0,"CounterClass":3,"CounterID":79373,"DNSTiming":108,"DontCountHits":1,"EventDate":1380108349555,"EventTime":1401664850355,"FUniqID":1242415196983778783,"FetchTiming":471,"FlashMajor":2,"FlashMinor":0,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":455127649,"HTTPError":0,"HasGCLID":0,"HistoryLength":5,"HitColor":"F","IPNetworkID":67516,"Income":4,"Interests":607,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1381169820579,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":2,"NetMinor":5,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":12,"RefererHash":2132307900008171696,"RefererRegionID":179,"RegionID":108,"RemoteIP":1150891693,"ResolutionDepth":24,"ResolutionHeight":647,"ResolutionWidth":1584,"ResponseEndTiming":1394,"ResponseStartTiming":413,"Robotness":0,"SearchEngineID":18,"SearchPhrase":"","SendTiming":78,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":0,"URL":"https://news.net/article","URLCategoryID":6,"URLHash":6513754252556930522,"URLRegionID":252,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":50,"UserAgentMinor":"9","UserID":7825436896833517548,"WatchID":5756572173312422224,"WindowClientHeight":847,"WindowClientWidth":1011,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":8,"Age":40,"BrowserCountry":"KR","BrowserLanguage":"ru","CLID":716,"ClientEventTime":1377782455992,"ClientIP":2119597521,"ClientTimeZone":-10,"CodeVersion":606,"ConnectTiming":308,"CookieEnable":1,"CounterClass":4,"CounterID":84487,"DNSTiming":105,"DontCountHits":0,"EventDate":1382812762118,"EventTime":1376374312256,"FUniqID":151908747411091313,"FetchTiming":691,"FlashMajor":12,"FlashMinor":5,"FlashMinor2":5,"FromTag":"","GoodEvent":1,"HID":1139910795,"HTTPError":0,"HasGCLID":1,"HistoryLength":16,"HitColor":"F","IPNetworkID":12440,"Income":2,"Interests":880,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1383673686898,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":6,"NetMinor":4,"OS":5,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":17,"RefererHash":254218380935409097,"RefererRegionID":12,"RegionID":29,"RemoteIP":1803257614,"ResolutionDepth":24,"ResolutionHeight":1408,"ResolutionWidth":1557,"ResponseEndTiming":163,"ResponseStartTiming":144,"Robotness":0,"SearchEngineID":20,"SearchPhrase":"","SendTiming":98,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":-1,"URL":"https://shop.io/product","URLCategoryID":0,"URLHash":3536651065197137202,"URLRegionID":33,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":57,"UserAgentMinor":"8","UserID":8766723724478243867,"WatchID":1576066709200626785,"WindowClientHeight":923,"WindowClientWidth":1587,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":1,"Age":49,"BrowserCountry":"DE","BrowserLanguage":"zh","CLID":295,"ClientEventTime":1399657755266,"ClientIP":2124611462,"ClientTimeZone":10,"CodeVersion":958,"ConnectTiming":278,"CookieEnable":0,"CounterClass":2,"CounterID":94078,"DNSTiming":38,"DontCountHits":1,"EventDate":1380819685784,"EventTime":1386318373449,"FUniqID":3828678186405456672,"FetchTiming":665,"FlashMajor":1,"FlashMinor":1,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":1408521481,"HTTPError":0,"HasGCLID":1,"HistoryLength":10,"HitColor":"T","IPNetworkID":59244,"Income":2,"Interests":371,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1402455101068,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":0,"NetMinor":6,"OS":5,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":1,"RefererHash":4286395310538829157,"RefererRegionID":262,"RegionID":53,"RemoteIP":1362534941,"ResolutionDepth":24,"ResolutionHeight":1150,"ResolutionWidth":2003,"ResponseEndTiming":104,"ResponseStartTiming":436,"Robotness":0,"SearchEngineID":17,"SearchPhrase":"","SendTiming":439,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":-1,"URL":"","URLCategoryID":2,"URLHash":4712419268381472725,"URLRegionID":222,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":39,"UserAgentMinor":"19","UserID":2671067031355210528,"WatchID":8039365387163857580,"WindowClientHeight":472,"WindowClientWidth":1399,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":7,"Age":54,"BrowserCountry":"US","BrowserLanguage":"ja","CLID":80,"ClientEventTime":1383675363724,"ClientIP":1044339405,"ClientTimeZone":1,"CodeVersion":994,"ConnectTiming":213,"CookieEnable":0,"CounterClass":1,"CounterID":74823,"DNSTiming":3,"DontCountHits":0,"EventDate":1398771129834,"EventTime":1382453120339,"FUniqID":4165111800657671642,"FetchTiming":410,"FlashMajor":15,"FlashMinor":3,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":1676609742,"HTTPError":0,"HasGCLID":0,"HistoryLength":0,"HitColor":"F","IPNetworkID":36058,"Income":4,"Interests":771,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1391868359016,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":8,"NetMinor":0,"OS":7,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":0,"RefererHash":3865172325985381003,"RefererRegionID":91,"RegionID":56,"RemoteIP":351857523,"ResolutionDepth":24,"ResolutionHeight":1020,"ResolutionWidth":1156,"ResponseEndTiming":1569,"ResponseStartTiming":348,"Robotness":0,"SearchEngineID":28,"SearchPhrase":"","SendTiming":415,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":7,"URL":"https://test.org/home","URLCategoryID":12,"URLHash":8556726175548668573,"URLRegionID":123,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":76,"UserAgentMinor":"44","UserID":4335154551772881702,"WatchID":3638714276446460917,"WindowClientHeight":894,"WindowClientWidth":1194,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":8,"Age":77,"BrowserCountry":"CN","BrowserLanguage":"ko","CLID":899,"ClientEventTime":1396496684243,"ClientIP":1783552822,"ClientTimeZone":-1,"CodeVersion":615,"ConnectTiming":60,"CookieEnable":1,"CounterClass":3,"CounterID":22958,"DNSTiming":187,"DontCountHits":1,"EventDate":1377856054598,"EventTime":1404446270733,"FUniqID":1371916393052346320,"FetchTiming":783,"FlashMajor":9,"FlashMinor":0,"FlashMinor2":0,"FromTag":"","GoodEvent":1,"HID":616150620,"HTTPError":0,"HasGCLID":1,"HistoryLength":18,"HitColor":"D","IPNetworkID":93452,"Income":4,"Interests":421,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1391841221971,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":7,"NetMinor":2,"OS":7,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":0,"RefererHash":2886091291440399965,"RefererRegionID":57,"RegionID":99,"RemoteIP":1200463604,"ResolutionDepth":24,"ResolutionHeight":954,"ResolutionWidth":822,"ResponseEndTiming":151,"ResponseStartTiming":214,"Robotness":0,"SearchEngineID":26,"SearchPhrase":"","SendTiming":310,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":4,"URL":"","URLCategoryID":0,"URLHash":7408587586240159722,"URLRegionID":198,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":57,"UserAgentMinor":"74","UserID":7828482389629788615,"WatchID":4128090904942453401,"WindowClientHeight":1158,"WindowClientWidth":1479,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":26,"Age":49,"BrowserCountry":"IN","BrowserLanguage":"de","CLID":247,"ClientEventTime":1402183986046,"ClientIP":444986234,"ClientTimeZone":-7,"CodeVersion":822,"ConnectTiming":109,"CookieEnable":0,"CounterClass":1,"CounterID":15618,"DNSTiming":190,"DontCountHits":1,"EventDate":1388397350356,"EventTime":1384374786336,"FUniqID":3101031035046577551,"FetchTiming":775,"FlashMajor":16,"FlashMinor":0,"FlashMinor2":7,"FromTag":"","GoodEvent":1,"HID":1783170553,"HTTPError":0,"HasGCLID":0,"HistoryLength":1,"HitColor":"T","IPNetworkID":89662,"Income":1,"Interests":480,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1374616501886,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":0,"NetMinor":3,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":17,"RefererHash":7849896779287019012,"RefererRegionID":284,"RegionID":105,"RemoteIP":1169310987,"ResolutionDepth":24,"ResolutionHeight":1236,"ResolutionWidth":1363,"ResponseEndTiming":1411,"ResponseStartTiming":295,"Robotness":0,"SearchEngineID":17,"SearchPhrase":"","SendTiming":27,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":4,"URL":"","URLCategoryID":2,"URLHash":4249556676461955321,"URLRegionID":83,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":32,"UserAgentMinor":"70","UserID":2447391282273585435,"WatchID":619486363908775275,"WindowClientHeight":661,"WindowClientWidth":1236,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":4,"Age":33,"BrowserCountry":"KR","BrowserLanguage":"zh","CLID":367,"ClientEventTime":1376519489202,"ClientIP":1881766144,"ClientTimeZone":-8,"CodeVersion":275,"ConnectTiming":299,"CookieEnable":1,"CounterClass":2,"CounterID":97044,"DNSTiming":76,"DontCountHits":1,"EventDate":1383004244598,"EventTime":1393783931822,"FUniqID":4822636219178489237,"FetchTiming":522,"FlashMajor":2,"FlashMinor":6,"FlashMinor2":5,"FromTag":"","GoodEvent":1,"HID":1570428393,"HTTPError":0,"HasGCLID":1,"HistoryLength":7,"HitColor":"F","IPNetworkID":44464,"Income":4,"Interests":172,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1388659347286,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":6,"NetMinor":2,"OS":7,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":14,"RefererHash":2159491522725988866,"RefererRegionID":16,"RegionID":176,"RemoteIP":1174873180,"ResolutionDepth":24,"ResolutionHeight":786,"ResolutionWidth":1810,"ResponseEndTiming":191,"ResponseStartTiming":197,"Robotness":0,"SearchEngineID":9,"SearchPhrase":"","SendTiming":351,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":5,"URL":"https://example.com/page2","URLCategoryID":3,"URLHash":1893555993448202015,"URLRegionID":41,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":82,"UserAgentMinor":"17","UserID":9006556188188630914,"WatchID":9055183144181912368,"WindowClientHeight":1150,"WindowClientWidth":1503,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":24,"Age":60,"BrowserCountry":"IN","BrowserLanguage":"de","CLID":577,"ClientEventTime":1395275456002,"ClientIP":1093954114,"ClientTimeZone":9,"CodeVersion":128,"ConnectTiming":308,"CookieEnable":0,"CounterClass":3,"CounterID":60164,"DNSTiming":64,"DontCountHits":0,"EventDate":1376719909537,"EventTime":1377718736800,"FUniqID":6788347658744970651,"FetchTiming":567,"FlashMajor":19,"FlashMinor":4,"FlashMinor2":2,"FromTag":"","GoodEvent":1,"HID":464473565,"HTTPError":0,"HasGCLID":1,"HistoryLength":15,"HitColor":"F","IPNetworkID":90875,"Income":3,"Interests":577,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1392227863600,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":0,"NetMinor":2,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":15,"RefererHash":3759481032280737039,"RefererRegionID":297,"RegionID":274,"RemoteIP":2074036576,"ResolutionDepth":24,"ResolutionHeight":848,"ResolutionWidth":1831,"ResponseEndTiming":1087,"ResponseStartTiming":216,"Robotness":0,"SearchEngineID":4,"SearchPhrase":"","SendTiming":321,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":7,"URL":"https://example.com/page1","URLCategoryID":12,"URLHash":5640301059128278769,"URLRegionID":203,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":84,"UserAgentMinor":"19","UserID":39635906188057940,"WatchID":8311841025432725447,"WindowClientHeight":747,"WindowClientWidth":1650,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":28,"Age":67,"BrowserCountry":"JP","BrowserLanguage":"pt","CLID":216,"ClientEventTime":1403124950820,"ClientIP":2116291040,"ClientTimeZone":-8,"CodeVersion":452,"ConnectTiming":28,"CookieEnable":1,"CounterClass":4,"CounterID":34598,"DNSTiming":152,"DontCountHits":0,"EventDate":1395013305199,"EventTime":1392008351900,"FUniqID":1031832765030733741,"FetchTiming":524,"FlashMajor":9,"FlashMinor":9,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":730155876,"HTTPError":0,"HasGCLID":0,"HistoryLength":19,"HitColor":"D","IPNetworkID":55037,"Income":2,"Interests":667,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1404933784097,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":8,"NetMinor":4,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":10,"RefererHash":3598591232157689711,"RefererRegionID":156,"RegionID":126,"RemoteIP":1426267432,"ResolutionDepth":24,"ResolutionHeight":1039,"ResolutionWidth":944,"ResponseEndTiming":1816,"ResponseStartTiming":159,"Robotness":0,"SearchEngineID":6,"SearchPhrase":"","SendTiming":64,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":3,"URL":"https://shop.io/product","URLCategoryID":18,"URLHash":908731833741233882,"URLRegionID":198,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":47,"UserAgentMinor":"16","UserID":6369283361561103539,"WatchID":6109283286145165893,"WindowClientHeight":1097,"WindowClientWidth":1125,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":27,"Age":76,"BrowserCountry":"GB","BrowserLanguage":"ja","CLID":897,"ClientEventTime":1403367694166,"ClientIP":1512752857,"ClientTimeZone":1,"CodeVersion":447,"ConnectTiming":13,"CookieEnable":1,"CounterClass":0,"CounterID":3909,"DNSTiming":6,"DontCountHits":1,"EventDate":1396410522283,"EventTime":1394185540682,"FUniqID":6374984961547616473,"FetchTiming":284,"FlashMajor":2,"FlashMinor":7,"FlashMinor2":2,"FromTag":"","GoodEvent":1,"HID":697462760,"HTTPError":0,"HasGCLID":0,"HistoryLength":15,"HitColor":"D","IPNetworkID":52609,"Income":0,"Interests":829,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1389659656780,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":5,"NetMinor":9,"OS":5,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":18,"RefererHash":452432292317715478,"RefererRegionID":97,"RegionID":185,"RemoteIP":1821741298,"ResolutionDepth":24,"ResolutionHeight":803,"ResolutionWidth":1217,"ResponseEndTiming":591,"ResponseStartTiming":4,"Robotness":0,"SearchEngineID":4,"SearchPhrase":"","SendTiming":354,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":0,"URL":"https://example.com/page1","URLCategoryID":15,"URLHash":6340201345015594720,"URLRegionID":163,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":39,"UserAgentMinor":"88","UserID":4992529323330296992,"WatchID":6265961365991554054,"WindowClientHeight":671,"WindowClientWidth":1250,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":13,"Age":36,"BrowserCountry":"GB","BrowserLanguage":"ko","CLID":220,"ClientEventTime":1375352571579,"ClientIP":398670504,"ClientTimeZone":9,"CodeVersion":780,"ConnectTiming":13,"CookieEnable":1,"CounterClass":2,"CounterID":62975,"DNSTiming":107,"DontCountHits":0,"EventDate":1388713807350,"EventTime":1400084958628,"FUniqID":7277821627977784403,"FetchTiming":588,"FlashMajor":18,"FlashMinor":9,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":86984609,"HTTPError":0,"HasGCLID":1,"HistoryLength":19,"HitColor":"T","IPNetworkID":21138,"Income":0,"Interests":961,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1376353586864,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":0,"NetMinor":8,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":10,"RefererHash":5720368189276282026,"RefererRegionID":241,"RegionID":125,"RemoteIP":591332244,"ResolutionDepth":24,"ResolutionHeight":1030,"ResolutionWidth":1047,"ResponseEndTiming":1536,"ResponseStartTiming":46,"Robotness":0,"SearchEngineID":22,"SearchPhrase":"","SendTiming":151,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":8,"URL":"https://test.org/home","URLCategoryID":2,"URLHash":2629183416757029477,"URLRegionID":152,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":81,"UserAgentMinor":"77","UserID":8068775278167099773,"WatchID":7769436015749707468,"WindowClientHeight":429,"WindowClientWidth":926,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":19,"Age":21,"BrowserCountry":"FR","BrowserLanguage":"pt","CLID":857,"ClientEventTime":1374246473258,"ClientIP":1804487796,"ClientTimeZone":3,"CodeVersion":162,"ConnectTiming":190,"CookieEnable":1,"CounterClass":1,"CounterID":57059,"DNSTiming":42,"DontCountHits":0,"EventDate":1389656642291,"EventTime":1401450130548,"FUniqID":7906282107404418040,"FetchTiming":62,"FlashMajor":3,"FlashMinor":3,"FlashMinor2":5,"FromTag":"","GoodEvent":1,"HID":542429577,"HTTPError":0,"HasGCLID":0,"HistoryLength":6,"HitColor":"D","IPNetworkID":13296,"Income":4,"Interests":626,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1389161860460,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":1,"NetMinor":2,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":16,"RefererHash":8854530577497200571,"RefererRegionID":152,"RegionID":169,"RemoteIP":2144871499,"ResolutionDepth":24,"ResolutionHeight":855,"ResolutionWidth":1937,"ResponseEndTiming":1762,"ResponseStartTiming":396,"Robotness":0,"SearchEngineID":12,"SearchPhrase":"","SendTiming":209,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Product List","TraficSourceID":6,"URL":"https://shop.io/product","URLCategoryID":11,"URLHash":3453040912644462959,"URLRegionID":38,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":43,"UserAgentMinor":"80","UserID":6965958794128754705,"WatchID":2416261488745939125,"WindowClientHeight":742,"WindowClientWidth":735,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":3,"Age":49,"BrowserCountry":"IN","BrowserLanguage":"ja","CLID":322,"ClientEventTime":1398581127923,"ClientIP":84011081,"ClientTimeZone":1,"CodeVersion":815,"ConnectTiming":365,"CookieEnable":1,"CounterClass":0,"CounterID":30466,"DNSTiming":17,"DontCountHits":1,"EventDate":1381622080018,"EventTime":1380943530405,"FUniqID":66457298750105708,"FetchTiming":923,"FlashMajor":15,"FlashMinor":0,"FlashMinor2":1,"FromTag":"","GoodEvent":1,"HID":102582293,"HTTPError":0,"HasGCLID":0,"HistoryLength":8,"HitColor":"T","IPNetworkID":60900,"Income":1,"Interests":452,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1386178148155,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":1,"NetMinor":5,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":1,"RefererHash":1023596709778845519,"RefererRegionID":4,"RegionID":110,"RemoteIP":101729975,"ResolutionDepth":24,"ResolutionHeight":1393,"ResolutionWidth":2295,"ResponseEndTiming":476,"ResponseStartTiming":409,"Robotness":0,"SearchEngineID":0,"SearchPhrase":"","SendTiming":34,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":0,"URL":"https://example.com/page2","URLCategoryID":3,"URLHash":1034898233384765694,"URLRegionID":53,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":18,"UserAgentMinor":"92","UserID":6699015150670201086,"WatchID":5621206703673261489,"WindowClientHeight":732,"WindowClientWidth":1610,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":20,"Age":72,"BrowserCountry":"RU","BrowserLanguage":"it","CLID":867,"ClientEventTime":1396635847957,"ClientIP":2006614946,"ClientTimeZone":-2,"CodeVersion":496,"ConnectTiming":45,"CookieEnable":0,"CounterClass":4,"CounterID":49295,"DNSTiming":143,"DontCountHits":1,"EventDate":1400852956347,"EventTime":1377098007277,"FUniqID":875914715922076151,"FetchTiming":312,"FlashMajor":17,"FlashMinor":1,"FlashMinor2":7,"FromTag":"","GoodEvent":1,"HID":1070773741,"HTTPError":0,"HasGCLID":1,"HistoryLength":15,"HitColor":"T","IPNetworkID":27066,"Income":3,"Interests":286,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1388850627931,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":2,"NetMinor":6,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":9,"RefererHash":3321121517346232213,"RefererRegionID":94,"RegionID":172,"RemoteIP":578585613,"ResolutionDepth":24,"ResolutionHeight":1017,"ResolutionWidth":1576,"ResponseEndTiming":410,"ResponseStartTiming":138,"Robotness":0,"SearchEngineID":2,"SearchPhrase":"","SendTiming":419,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Product List","TraficSourceID":3,"URL":"https://shop.io/product","URLCategoryID":5,"URLHash":632389695228180220,"URLRegionID":46,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":57,"UserAgentMinor":"89","UserID":7770734788845559432,"WatchID":7109734832775541715,"WindowClientHeight":487,"WindowClientWidth":1092,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":3,"Age":9,"BrowserCountry":"FR","BrowserLanguage":"ja","CLID":861,"ClientEventTime":1379657882441,"ClientIP":345749177,"ClientTimeZone":4,"CodeVersion":414,"ConnectTiming":327,"CookieEnable":1,"CounterClass":0,"CounterID":94511,"DNSTiming":48,"DontCountHits":1,"EventDate":1395715702827,"EventTime":1381981165094,"FUniqID":7349647927087491111,"FetchTiming":22,"FlashMajor":14,"FlashMinor":3,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":1415196665,"HTTPError":0,"HasGCLID":0,"HistoryLength":12,"HitColor":"T","IPNetworkID":49985,"Income":2,"Interests":153,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1391119199135,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":8,"NetMinor":5,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":7,"RefererHash":1969122200289504055,"RefererRegionID":55,"RegionID":156,"RemoteIP":412067889,"ResolutionDepth":24,"ResolutionHeight":1259,"ResolutionWidth":1599,"ResponseEndTiming":889,"ResponseStartTiming":237,"Robotness":0,"SearchEngineID":28,"SearchPhrase":"","SendTiming":312,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":2,"URL":"https://news.net/article","URLCategoryID":17,"URLHash":3561651019506509819,"URLRegionID":288,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":27,"UserAgentMinor":"28","UserID":9036023558650575450,"WatchID":3120247645488250025,"WindowClientHeight":480,"WindowClientWidth":950,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":21,"Age":60,"BrowserCountry":"CN","BrowserLanguage":"en","CLID":682,"ClientEventTime":1382070462185,"ClientIP":586966830,"ClientTimeZone":1,"CodeVersion":148,"ConnectTiming":44,"CookieEnable":0,"CounterClass":4,"CounterID":54934,"DNSTiming":21,"DontCountHits":0,"EventDate":1378623925243,"EventTime":1376756340236,"FUniqID":4958271693553435894,"FetchTiming":467,"FlashMajor":6,"FlashMinor":0,"FlashMinor2":1,"FromTag":"","GoodEvent":1,"HID":285486914,"HTTPError":0,"HasGCLID":0,"HistoryLength":13,"HitColor":"T","IPNetworkID":86900,"Income":2,"Interests":974,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1381464451690,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":0,"NetMinor":3,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":10,"RefererHash":1702797906713648920,"RefererRegionID":86,"RegionID":40,"RemoteIP":1947027402,"ResolutionDepth":24,"ResolutionHeight":873,"ResolutionWidth":2484,"ResponseEndTiming":1342,"ResponseStartTiming":52,"Robotness":0,"SearchEngineID":29,"SearchPhrase":"","SendTiming":277,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":6,"URL":"https://test.org/home","URLCategoryID":16,"URLHash":3211926687519583431,"URLRegionID":94,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":56,"UserAgentMinor":"7","UserID":2376451293918900087,"WatchID":9173115419201285991,"WindowClientHeight":734,"WindowClientWidth":1076,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":17,"Age":42,"BrowserCountry":"JP","BrowserLanguage":"es","CLID":499,"ClientEventTime":1399707018429,"ClientIP":604085844,"ClientTimeZone":2,"CodeVersion":485,"ConnectTiming":374,"CookieEnable":1,"CounterClass":2,"CounterID":42137,"DNSTiming":56,"DontCountHits":0,"EventDate":1395063585332,"EventTime":1382932171026,"FUniqID":728969905038202808,"FetchTiming":498,"FlashMajor":17,"FlashMinor":6,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":1543826535,"HTTPError":0,"HasGCLID":0,"HistoryLength":6,"HitColor":"S","IPNetworkID":37905,"Income":4,"Interests":638,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1397154896199,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":3,"NetMinor":8,"OS":7,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":16,"RefererHash":3990423378041114419,"RefererRegionID":172,"RegionID":147,"RemoteIP":2086126835,"ResolutionDepth":24,"ResolutionHeight":698,"ResolutionWidth":2209,"ResponseEndTiming":631,"ResponseStartTiming":347,"Robotness":0,"SearchEngineID":3,"SearchPhrase":"","SendTiming":214,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Product List","TraficSourceID":3,"URL":"https://example.com/page2","URLCategoryID":9,"URLHash":4070209684557095113,"URLRegionID":247,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":34,"UserAgentMinor":"45","UserID":878865366235137718,"WatchID":6077829840383314656,"WindowClientHeight":405,"WindowClientWidth":1740,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":5,"Age":58,"BrowserCountry":"US","BrowserLanguage":"es","CLID":160,"ClientEventTime":1387876493344,"ClientIP":752366835,"ClientTimeZone":-2,"CodeVersion":909,"ConnectTiming":393,"CookieEnable":0,"CounterClass":3,"CounterID":4367,"DNSTiming":49,"DontCountHits":1,"EventDate":1391186779245,"EventTime":1373871829798,"FUniqID":3380408177687958936,"FetchTiming":290,"FlashMajor":11,"FlashMinor":2,"FlashMinor2":0,"FromTag":"","GoodEvent":1,"HID":996971297,"HTTPError":0,"HasGCLID":1,"HistoryLength":9,"HitColor":"D","IPNetworkID":91307,"Income":2,"Interests":316,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1382953206597,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":5,"NetMinor":8,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":4,"RefererHash":3662436438183721660,"RefererRegionID":3,"RegionID":9,"RemoteIP":792812350,"ResolutionDepth":24,"ResolutionHeight":951,"ResolutionWidth":1355,"ResponseEndTiming":1258,"ResponseStartTiming":234,"Robotness":0,"SearchEngineID":12,"SearchPhrase":"","SendTiming":230,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":7,"URL":"https://example.com/page1","URLCategoryID":17,"URLHash":1104146592741498712,"URLRegionID":293,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":1,"UserAgentMinor":"32","UserID":3720368389456454321,"WatchID":6159071744432894143,"WindowClientHeight":479,"WindowClientWidth":1499,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":27,"Age":61,"BrowserCountry":"US","BrowserLanguage":"ru","CLID":999,"ClientEventTime":1388480947412,"ClientIP":1447585154,"ClientTimeZone":3,"CodeVersion":176,"ConnectTiming":299,"CookieEnable":0,"CounterClass":1,"CounterID":56742,"DNSTiming":149,"DontCountHits":1,"EventDate":1383257640865,"EventTime":1392163044681,"FUniqID":2081860616514930291,"FetchTiming":222,"FlashMajor":13,"FlashMinor":6,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":289061977,"HTTPError":0,"HasGCLID":1,"HistoryLength":10,"HitColor":"T","IPNetworkID":8985,"Income":3,"Interests":737,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1398570164517,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":8,"NetMinor":8,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":14,"RefererHash":3281461184765114840,"RefererRegionID":219,"RegionID":68,"RemoteIP":216588621,"ResolutionDepth":24,"ResolutionHeight":737,"ResolutionWidth":2051,"ResponseEndTiming":1627,"ResponseStartTiming":42,"Robotness":0,"SearchEngineID":23,"SearchPhrase":"","SendTiming":366,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":-1,"URL":"https://example.com/page2","URLCategoryID":11,"URLHash":8540714429694800542,"URLRegionID":105,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":72,"UserAgentMinor":"88","UserID":3366499944137909159,"WatchID":900576155621692479,"WindowClientHeight":460,"WindowClientWidth":869,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":0,"Age":48,"BrowserCountry":"US","BrowserLanguage":"es","CLID":472,"ClientEventTime":1374596694113,"ClientIP":787641239,"ClientTimeZone":-7,"CodeVersion":257,"ConnectTiming":359,"CookieEnable":1,"CounterClass":1,"CounterID":4096,"DNSTiming":29,"DontCountHits":1,"EventDate":1387115890171,"EventTime":1392289298531,"FUniqID":8563992468494392831,"FetchTiming":637,"FlashMajor":16,"FlashMinor":3,"FlashMinor2":6,"FromTag":"","GoodEvent":1,"HID":2050590202,"HTTPError":0,"HasGCLID":0,"HistoryLength":13,"HitColor":"D","IPNetworkID":61519,"Income":4,"Interests":427,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1376910558171,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":8,"NetMinor":9,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":12,"RefererHash":3920621726644658256,"RefererRegionID":92,"RegionID":73,"RemoteIP":1463513942,"ResolutionDepth":24,"ResolutionHeight":1172,"ResolutionWidth":1041,"ResponseEndTiming":528,"ResponseStartTiming":26,"Robotness":0,"SearchEngineID":21,"SearchPhrase":"","SendTiming":282,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":2,"URL":"https://news.net/article","URLCategoryID":11,"URLHash":9134968635645062995,"URLRegionID":156,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":58,"UserAgentMinor":"15","UserID":7517413871605291079,"WatchID":2590473910216139062,"WindowClientHeight":456,"WindowClientWidth":1014,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":13,"Age":17,"BrowserCountry":"JP","BrowserLanguage":"ja","CLID":185,"ClientEventTime":1381135550185,"ClientIP":1411346672,"ClientTimeZone":5,"CodeVersion":869,"ConnectTiming":127,"CookieEnable":1,"CounterClass":0,"CounterID":12276,"DNSTiming":171,"DontCountHits":0,"EventDate":1402927146700,"EventTime":1375413766548,"FUniqID":2216645692207663792,"FetchTiming":905,"FlashMajor":15,"FlashMinor":8,"FlashMinor2":1,"FromTag":"","GoodEvent":1,"HID":1142741834,"HTTPError":0,"HasGCLID":1,"HistoryLength":7,"HitColor":"D","IPNetworkID":7976,"Income":2,"Interests":934,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1379875891495,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":4,"NetMinor":4,"OS":2,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":10,"RefererHash":1852303398807420027,"RefererRegionID":238,"RegionID":292,"RemoteIP":666616509,"ResolutionDepth":24,"ResolutionHeight":1210,"ResolutionWidth":2536,"ResponseEndTiming":1162,"ResponseStartTiming":378,"Robotness":0,"SearchEngineID":11,"SearchPhrase":"","SendTiming":463,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":4,"URL":"https://example.com/page2","URLCategoryID":6,"URLHash":6227976609281473430,"URLRegionID":136,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":71,"UserAgentMinor":"90","UserID":7441598016976482420,"WatchID":1469036396535855431,"WindowClientHeight":1091,"WindowClientWidth":1245,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":1,"Age":8,"BrowserCountry":"CN","BrowserLanguage":"pt","CLID":39,"ClientEventTime":1383104615045,"ClientIP":384647176,"ClientTimeZone":2,"CodeVersion":833,"ConnectTiming":211,"CookieEnable":1,"CounterClass":2,"CounterID":72864,"DNSTiming":128,"DontCountHits":1,"EventDate":1398666351882,"EventTime":1383130621867,"FUniqID":1655298776729722315,"FetchTiming":887,"FlashMajor":7,"FlashMinor":7,"FlashMinor2":1,"FromTag":"","GoodEvent":1,"HID":1994468907,"HTTPError":0,"HasGCLID":0,"HistoryLength":9,"HitColor":"S","IPNetworkID":18650,"Income":3,"Interests":972,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1394361146555,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":5,"NetMinor":8,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":11,"RefererHash":8834677180290589300,"RefererRegionID":183,"RegionID":189,"RemoteIP":1653229458,"ResolutionDepth":24,"ResolutionHeight":1217,"ResolutionWidth":1824,"ResponseEndTiming":696,"ResponseStartTiming":236,"Robotness":0,"SearchEngineID":24,"SearchPhrase":"","SendTiming":328,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":8,"URL":"https://example.com/page2","URLCategoryID":17,"URLHash":438798166330847540,"URLRegionID":171,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":75,"UserAgentMinor":"23","UserID":3304110819954722505,"WatchID":9031667833444079474,"WindowClientHeight":589,"WindowClientWidth":1409,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":19,"Age":0,"BrowserCountry":"BR","BrowserLanguage":"en","CLID":983,"ClientEventTime":1397317276001,"ClientIP":425734313,"ClientTimeZone":5,"CodeVersion":818,"ConnectTiming":317,"CookieEnable":0,"CounterClass":1,"CounterID":46990,"DNSTiming":152,"DontCountHits":1,"EventDate":1389038537685,"EventTime":1377801984639,"FUniqID":6535544140713845235,"FetchTiming":93,"FlashMajor":19,"FlashMinor":7,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":1619129000,"HTTPError":0,"HasGCLID":0,"HistoryLength":13,"HitColor":"F","IPNetworkID":57862,"Income":1,"Interests":156,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1389702949586,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":4,"NetMinor":9,"OS":3,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://shop.io/product","RefererCategoryID":0,"RefererHash":4711163522428034901,"RefererRegionID":217,"RegionID":16,"RemoteIP":815883764,"ResolutionDepth":24,"ResolutionHeight":1417,"ResolutionWidth":1726,"ResponseEndTiming":65,"ResponseStartTiming":97,"Robotness":0,"SearchEngineID":15,"SearchPhrase":"","SendTiming":158,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":8,"URL":"https://news.net/article","URLCategoryID":1,"URLHash":2802654358056641398,"URLRegionID":193,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":99,"UserAgentMinor":"19","UserID":7809672435527599004,"WatchID":893035166108891065,"WindowClientHeight":533,"WindowClientWidth":711,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":25,"Age":58,"BrowserCountry":"DE","BrowserLanguage":"es","CLID":68,"ClientEventTime":1391794432585,"ClientIP":1739809017,"ClientTimeZone":-10,"CodeVersion":737,"ConnectTiming":191,"CookieEnable":1,"CounterClass":1,"CounterID":56987,"DNSTiming":15,"DontCountHits":0,"EventDate":1382881822043,"EventTime":1383110540172,"FUniqID":7884103459178027273,"FetchTiming":467,"FlashMajor":10,"FlashMinor":4,"FlashMinor2":5,"FromTag":"","GoodEvent":1,"HID":1998686303,"HTTPError":0,"HasGCLID":0,"HistoryLength":11,"HitColor":"T","IPNetworkID":29259,"Income":3,"Interests":545,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1375865844166,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":2,"NetMinor":6,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":7,"RefererHash":1487725707319795855,"RefererRegionID":69,"RegionID":174,"RemoteIP":726888329,"ResolutionDepth":24,"ResolutionHeight":1346,"ResolutionWidth":1541,"ResponseEndTiming":674,"ResponseStartTiming":260,"Robotness":0,"SearchEngineID":4,"SearchPhrase":"","SendTiming":140,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":3,"URL":"https://shop.io/product","URLCategoryID":17,"URLHash":2092411146789817076,"URLRegionID":118,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":98,"UserAgentMinor":"1","UserID":6543495594753448298,"WatchID":4195459315188648175,"WindowClientHeight":738,"WindowClientWidth":685,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":15,"Age":17,"BrowserCountry":"KR","BrowserLanguage":"de","CLID":928,"ClientEventTime":1374678140385,"ClientIP":557165738,"ClientTimeZone":6,"CodeVersion":478,"ConnectTiming":104,"CookieEnable":0,"CounterClass":4,"CounterID":69063,"DNSTiming":90,"DontCountHits":1,"EventDate":1390568271102,"EventTime":1393157622016,"FUniqID":2168349832691976965,"FetchTiming":139,"FlashMajor":19,"FlashMinor":7,"FlashMinor2":0,"FromTag":"","GoodEvent":1,"HID":1662368873,"HTTPError":0,"HasGCLID":0,"HistoryLength":17,"HitColor":"T","IPNetworkID":16229,"Income":4,"Interests":923,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1385808094560,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":2,"NetMinor":9,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":19,"RefererHash":1322876867747879830,"RefererRegionID":176,"RegionID":44,"RemoteIP":34520581,"ResolutionDepth":24,"ResolutionHeight":841,"ResolutionWidth":2375,"ResponseEndTiming":710,"ResponseStartTiming":15,"Robotness":0,"SearchEngineID":9,"SearchPhrase":"","SendTiming":332,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Search Results","TraficSourceID":5,"URL":"https://example.com/page1","URLCategoryID":3,"URLHash":2910888235173236690,"URLRegionID":40,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":2,"UserAgentMajor":3,"UserAgentMinor":"99","UserID":597961108076515548,"WatchID":3449653523569203353,"WindowClientHeight":1098,"WindowClientWidth":1092,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":1,"Age":29,"BrowserCountry":"RU","BrowserLanguage":"pt","CLID":192,"ClientEventTime":1394981363031,"ClientIP":893446693,"ClientTimeZone":8,"CodeVersion":487,"ConnectTiming":361,"CookieEnable":0,"CounterClass":1,"CounterID":49049,"DNSTiming":107,"DontCountHits":0,"EventDate":1373896018448,"EventTime":1384331186522,"FUniqID":4636264759398153547,"FetchTiming":929,"FlashMajor":10,"FlashMinor":4,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":221385156,"HTTPError":0,"HasGCLID":1,"HistoryLength":3,"HitColor":"D","IPNetworkID":56374,"Income":4,"Interests":446,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1389306703472,"MobilePhone":2,"MobilePhoneModel":"","NetMajor":5,"NetMinor":0,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":0,"RefererHash":3420737835207868747,"RefererRegionID":55,"RegionID":53,"RemoteIP":1114381938,"ResolutionDepth":24,"ResolutionHeight":1422,"ResolutionWidth":1356,"ResponseEndTiming":1914,"ResponseStartTiming":253,"Robotness":0,"SearchEngineID":12,"SearchPhrase":"","SendTiming":150,"Sex":0,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Product List","TraficSourceID":2,"URL":"https://test.org/home","URLCategoryID":0,"URLHash":8875933885672131141,"URLRegionID":120,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":13,"UserAgentMinor":"99","UserID":4004724258405478737,"WatchID":149193047789584152,"WindowClientHeight":561,"WindowClientWidth":1726,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":21,"Age":17,"BrowserCountry":"RU","BrowserLanguage":"fr","CLID":341,"ClientEventTime":1382866826417,"ClientIP":2140825122,"ClientTimeZone":-1,"CodeVersion":218,"ConnectTiming":36,"CookieEnable":1,"CounterClass":4,"CounterID":32305,"DNSTiming":101,"DontCountHits":1,"EventDate":1374934814112,"EventTime":1383573374867,"FUniqID":3095407171706348011,"FetchTiming":415,"FlashMajor":17,"FlashMinor":9,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":381333764,"HTTPError":0,"HasGCLID":0,"HistoryLength":12,"HitColor":"T","IPNetworkID":96634,"Income":0,"Interests":735,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1395900039739,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":7,"NetMinor":1,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":9,"RefererHash":8368914817093435358,"RefererRegionID":195,"RegionID":129,"RemoteIP":1038457078,"ResolutionDepth":24,"ResolutionHeight":1317,"ResolutionWidth":1606,"ResponseEndTiming":1507,"ResponseStartTiming":318,"Robotness":0,"SearchEngineID":17,"SearchPhrase":"","SendTiming":180,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Home Page","TraficSourceID":5,"URL":"","URLCategoryID":0,"URLHash":1000129287573463846,"URLRegionID":122,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":3,"UserAgentMajor":60,"UserAgentMinor":"43","UserID":7241485010250898962,"WatchID":3203106730338516936,"WindowClientHeight":1060,"WindowClientWidth":683,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":12,"Age":38,"BrowserCountry":"RU","BrowserLanguage":"pt","CLID":753,"ClientEventTime":1374399778522,"ClientIP":1128417409,"ClientTimeZone":-2,"CodeVersion":159,"ConnectTiming":57,"CookieEnable":0,"CounterClass":4,"CounterID":36455,"DNSTiming":30,"DontCountHits":1,"EventDate":1382463734706,"EventTime":1384278318495,"FUniqID":1429045844476930791,"FetchTiming":415,"FlashMajor":15,"FlashMinor":2,"FlashMinor2":8,"FromTag":"","GoodEvent":1,"HID":1728105817,"HTTPError":0,"HasGCLID":1,"HistoryLength":3,"HitColor":"F","IPNetworkID":46248,"Income":2,"Interests":751,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1404757796720,"MobilePhone":4,"MobilePhoneModel":"","NetMajor":8,"NetMinor":4,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://test.org/home","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":1,"RefererHash":2545247300236309483,"RefererRegionID":59,"RegionID":72,"RemoteIP":1355379908,"ResolutionDepth":24,"ResolutionHeight":854,"ResolutionWidth":2289,"ResponseEndTiming":733,"ResponseStartTiming":235,"Robotness":0,"SearchEngineID":3,"SearchPhrase":"","SendTiming":305,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":5,"URL":"https://test.org/home","URLCategoryID":15,"URLHash":8749366036763747629,"URLRegionID":97,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":70,"UserAgentMinor":"78","UserID":519326510453124106,"WatchID":3100392821893187123,"WindowClientHeight":780,"WindowClientWidth":1679,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":15,"Age":4,"BrowserCountry":"US","BrowserLanguage":"pt","CLID":883,"ClientEventTime":1390447452681,"ClientIP":1634738382,"ClientTimeZone":-6,"CodeVersion":416,"ConnectTiming":247,"CookieEnable":0,"CounterClass":2,"CounterID":27938,"DNSTiming":36,"DontCountHits":0,"EventDate":1383315709154,"EventTime":1398223507056,"FUniqID":6825593186878263924,"FetchTiming":771,"FlashMajor":5,"FlashMinor":4,"FlashMinor2":3,"FromTag":"","GoodEvent":1,"HID":1106644002,"HTTPError":0,"HasGCLID":1,"HistoryLength":13,"HitColor":"S","IPNetworkID":91453,"Income":0,"Interests":865,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1403846182793,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":3,"NetMinor":7,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":7,"RefererHash":5547219724671670361,"RefererRegionID":141,"RegionID":174,"RemoteIP":1760602331,"ResolutionDepth":24,"ResolutionHeight":656,"ResolutionWidth":801,"ResponseEndTiming":671,"ResponseStartTiming":12,"Robotness":0,"SearchEngineID":2,"SearchPhrase":"","SendTiming":89,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":2,"URL":"https://test.org/home","URLCategoryID":11,"URLHash":1104988500646745371,"URLRegionID":256,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":46,"UserAgentMinor":"43","UserID":3142512549742353804,"WatchID":4631166107204665304,"WindowClientHeight":629,"WindowClientWidth":1389,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":24,"Age":1,"BrowserCountry":"KR","BrowserLanguage":"pt","CLID":551,"ClientEventTime":1405272070909,"ClientIP":1169413658,"ClientTimeZone":-7,"CodeVersion":685,"ConnectTiming":191,"CookieEnable":0,"CounterClass":1,"CounterID":6576,"DNSTiming":121,"DontCountHits":1,"EventDate":1381952840335,"EventTime":1390196387581,"FUniqID":3551874433054541889,"FetchTiming":0,"FlashMajor":8,"FlashMinor":7,"FlashMinor2":7,"FromTag":"","GoodEvent":1,"HID":552979419,"HTTPError":0,"HasGCLID":0,"HistoryLength":17,"HitColor":"F","IPNetworkID":58255,"Income":3,"Interests":266,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1388618907609,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":6,"NetMinor":0,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"","RefererCategoryID":19,"RefererHash":7579840501496310740,"RefererRegionID":103,"RegionID":98,"RemoteIP":507155087,"ResolutionDepth":24,"ResolutionHeight":1285,"ResolutionWidth":2437,"ResponseEndTiming":1289,"ResponseStartTiming":65,"Robotness":0,"SearchEngineID":0,"SearchPhrase":"","SendTiming":211,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"News Article","TraficSourceID":2,"URL":"https://shop.io/product","URLCategoryID":11,"URLHash":8014821690947458829,"URLRegionID":140,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":59,"UserAgentMinor":"9","UserID":3869457415774630084,"WatchID":8416670094910755431,"WindowClientHeight":901,"WindowClientWidth":1722,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":11,"Age":39,"BrowserCountry":"RU","BrowserLanguage":"ja","CLID":789,"ClientEventTime":1397193000336,"ClientIP":10748715,"ClientTimeZone":2,"CodeVersion":544,"ConnectTiming":426,"CookieEnable":0,"CounterClass":2,"CounterID":1483,"DNSTiming":47,"DontCountHits":1,"EventDate":1378525772892,"EventTime":1386179341996,"FUniqID":3487414081152219734,"FetchTiming":553,"FlashMajor":1,"FlashMinor":8,"FlashMinor2":9,"FromTag":"","GoodEvent":1,"HID":272359030,"HTTPError":0,"HasGCLID":0,"HistoryLength":12,"HitColor":"T","IPNetworkID":19271,"Income":1,"Interests":9,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1394632976133,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":6,"NetMinor":0,"OS":1,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page2","RefererCategoryID":19,"RefererHash":7137188586597072148,"RefererRegionID":296,"RegionID":291,"RemoteIP":905708345,"ResolutionDepth":24,"ResolutionHeight":788,"ResolutionWidth":2109,"ResponseEndTiming":1429,"ResponseStartTiming":215,"Robotness":0,"SearchEngineID":27,"SearchPhrase":"","SendTiming":379,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":9,"URL":"https://example.com/page2","URLCategoryID":14,"URLHash":1543258994896335333,"URLRegionID":287,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":61,"UserAgentMinor":"78","UserID":2139701716929867786,"WatchID":279770902475580286,"WindowClientHeight":1119,"WindowClientWidth":1028,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":21,"Age":47,"BrowserCountry":"GB","BrowserLanguage":"en","CLID":341,"ClientEventTime":1393467217309,"ClientIP":1105445561,"ClientTimeZone":-5,"CodeVersion":835,"ConnectTiming":343,"CookieEnable":0,"CounterClass":1,"CounterID":7749,"DNSTiming":76,"DontCountHits":1,"EventDate":1387900949275,"EventTime":1392624470013,"FUniqID":6201897106323823197,"FetchTiming":905,"FlashMajor":3,"FlashMinor":1,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":1067913061,"HTTPError":0,"HasGCLID":1,"HistoryLength":3,"HitColor":"S","IPNetworkID":24065,"Income":0,"Interests":451,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":0,"JavaEnable":0,"JavascriptEnable":1,"LocalEventTime":1391393823044,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":9,"NetMinor":1,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://shop.io/product","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://test.org/home","RefererCategoryID":7,"RefererHash":8333152034228738204,"RefererRegionID":96,"RegionID":71,"RemoteIP":464164285,"ResolutionDepth":24,"ResolutionHeight":854,"ResolutionWidth":1426,"ResponseEndTiming":644,"ResponseStartTiming":336,"Robotness":0,"SearchEngineID":3,"SearchPhrase":"","SendTiming":224,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Contact Us","TraficSourceID":8,"URL":"https://example.com/page2","URLCategoryID":2,"URLHash":77090838923751129,"URLRegionID":128,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":1,"UserAgentMajor":58,"UserAgentMinor":"17","UserID":5163613576052558355,"WatchID":5767230264735807018,"WindowClientHeight":654,"WindowClientWidth":828,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":8,"Age":8,"BrowserCountry":"KR","BrowserLanguage":"en","CLID":789,"ClientEventTime":1405311269256,"ClientIP":1365017328,"ClientTimeZone":-7,"CodeVersion":114,"ConnectTiming":78,"CookieEnable":0,"CounterClass":3,"CounterID":65536,"DNSTiming":36,"DontCountHits":0,"EventDate":1400670045281,"EventTime":1402792568258,"FUniqID":5270590311071690306,"FetchTiming":258,"FlashMajor":1,"FlashMinor":9,"FlashMinor2":7,"FromTag":"","GoodEvent":1,"HID":118521670,"HTTPError":0,"HasGCLID":0,"HistoryLength":12,"HitColor":"F","IPNetworkID":10817,"Income":4,"Interests":172,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":0,"IsOldCounter":1,"IsParameter":0,"IsRefresh":0,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1384549642407,"MobilePhone":1,"MobilePhoneModel":"","NetMajor":1,"NetMinor":3,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page1","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":8,"RefererHash":7404871475852502867,"RefererRegionID":139,"RegionID":12,"RemoteIP":1017320629,"ResolutionDepth":24,"ResolutionHeight":1119,"ResolutionWidth":836,"ResponseEndTiming":840,"ResponseStartTiming":290,"Robotness":0,"SearchEngineID":28,"SearchPhrase":"","SendTiming":100,"Sex":1,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Product List","TraficSourceID":7,"URL":"https://news.net/article","URLCategoryID":5,"URLHash":2177021921089102980,"URLRegionID":173,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":48,"UserAgentMinor":"48","UserID":5461192681923852354,"WatchID":6394159393863643196,"WindowClientHeight":777,"WindowClientWidth":1204,"WindowName":0,"WithHash":0} +{"index":{}} +{"AdvEngineID":6,"Age":69,"BrowserCountry":"CN","BrowserLanguage":"es","CLID":925,"ClientEventTime":1379277583220,"ClientIP":19373410,"ClientTimeZone":6,"CodeVersion":141,"ConnectTiming":154,"CookieEnable":0,"CounterClass":0,"CounterID":88573,"DNSTiming":174,"DontCountHits":1,"EventDate":1380845035591,"EventTime":1378911530787,"FUniqID":2185763569318460155,"FetchTiming":771,"FlashMajor":2,"FlashMinor":0,"FlashMinor2":3,"FromTag":"","GoodEvent":1,"HID":1975306808,"HTTPError":0,"HasGCLID":1,"HistoryLength":12,"HitColor":"S","IPNetworkID":8541,"Income":1,"Interests":234,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":1,"IsNotBounce":0,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1394605390276,"MobilePhone":3,"MobilePhoneModel":"","NetMajor":9,"NetMinor":8,"OS":4,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://example.com/page2","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://example.com/page1","RefererCategoryID":1,"RefererHash":3680988261092529943,"RefererRegionID":151,"RegionID":8,"RemoteIP":1875709408,"ResolutionDepth":24,"ResolutionHeight":888,"ResolutionWidth":1739,"ResponseEndTiming":834,"ResponseStartTiming":197,"Robotness":0,"SearchEngineID":5,"SearchPhrase":"","SendTiming":105,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"","TraficSourceID":3,"URL":"https://example.com/page1","URLCategoryID":13,"URLHash":345003794695167069,"URLRegionID":228,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":5,"UserAgentMajor":23,"UserAgentMinor":"29","UserID":5691528058931868129,"WatchID":2659686430421164094,"WindowClientHeight":970,"WindowClientWidth":849,"WindowName":0,"WithHash":1} +{"index":{}} +{"AdvEngineID":15,"Age":63,"BrowserCountry":"KR","BrowserLanguage":"en","CLID":752,"ClientEventTime":1384265139323,"ClientIP":1007792620,"ClientTimeZone":2,"CodeVersion":806,"ConnectTiming":319,"CookieEnable":0,"CounterClass":4,"CounterID":21953,"DNSTiming":5,"DontCountHits":0,"EventDate":1395789892066,"EventTime":1392593133962,"FUniqID":3732859785350930068,"FetchTiming":955,"FlashMajor":3,"FlashMinor":9,"FlashMinor2":4,"FromTag":"","GoodEvent":1,"HID":757275524,"HTTPError":0,"HasGCLID":0,"HistoryLength":9,"HitColor":"S","IPNetworkID":88143,"Income":2,"Interests":50,"IsArtifical":0,"IsDownload":0,"IsEvent":0,"IsLink":0,"IsMobile":0,"IsNotBounce":1,"IsOldCounter":0,"IsParameter":0,"IsRefresh":1,"JavaEnable":1,"JavascriptEnable":1,"LocalEventTime":1382597969779,"MobilePhone":0,"MobilePhoneModel":"","NetMajor":4,"NetMinor":8,"OS":6,"OpenerName":0,"OpenstatAdID":"","OpenstatCampaignID":"","OpenstatServiceName":"","OpenstatSourceID":"","OriginalURL":"https://news.net/article","PageCharset":"UTF-8","ParamCurrency":"","ParamCurrencyID":0,"ParamOrderID":"","ParamPrice":0,"Params":"","Referer":"https://news.net/article","RefererCategoryID":3,"RefererHash":8412775606973326503,"RefererRegionID":50,"RegionID":131,"RemoteIP":1609908305,"ResolutionDepth":24,"ResolutionHeight":1084,"ResolutionWidth":2263,"ResponseEndTiming":62,"ResponseStartTiming":140,"Robotness":0,"SearchEngineID":14,"SearchPhrase":"","SendTiming":453,"Sex":2,"SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","Title":"Product List","TraficSourceID":7,"URL":"https://shop.io/product","URLCategoryID":11,"URLHash":1468111145634639481,"URLRegionID":168,"UTMCampaign":"","UTMContent":"","UTMMedium":"","UTMSource":"","UTMTerm":"","UserAgent":4,"UserAgentMajor":92,"UserAgentMinor":"89","UserID":4102193423840591337,"WatchID":2027411064773594575,"WindowClientHeight":1192,"WindowClientWidth":623,"WindowName":0,"WithHash":1} + diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q1.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q1.json new file mode 100644 index 0000000000000..e359a5785ebaa --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q1.json @@ -0,0 +1,10 @@ +{ + "size": 0, + "aggs": { + "count": { + "sum": { + "field": "GoodEvent" + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q10.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q10.json new file mode 100644 index 0000000000000..b2324e7dc71f1 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q10.json @@ -0,0 +1,46 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "RegionID": { + "terms": { + "field": "RegionID", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "c": "desc" + }, + { + "_key": "asc" + } + ] + }, + "aggregations": { + "sum(AdvEngineID)": { + "sum": { + "field": "AdvEngineID" + } + }, + "avg(ResolutionWidth)": { + "avg": { + "field": "ResolutionWidth" + } + }, + "dc(UserID)": { + "cardinality": { + "field": "UserID" + } + }, + "c": { + "value_count": { + "field": "_index" + } + } + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q11.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q11.json new file mode 100644 index 0000000000000..6dc9ae89cbfe5 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q11.json @@ -0,0 +1,55 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "MobilePhoneModel", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "MobilePhoneModel": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "aggregations": { + "MobilePhoneModel": { + "terms": { + "field": "MobilePhoneModel", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "u": "desc" + }, + { + "_key": "asc" + } + ] + }, + "aggregations": { + "u": { + "cardinality": { + "field": "UserID" + } + } + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q12.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q12.json new file mode 100644 index 0000000000000..dc019d6a3c9ff --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q12.json @@ -0,0 +1,84 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "bool": { + "must": [ + { + "exists": { + "field": "MobilePhoneModel", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "MobilePhoneModel": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + { + "exists": { + "field": "MobilePhone", + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "MobilePhoneModel", + "MobilePhone", + "UserID" + ], + "excludes": [] + }, + "aggregations": { + "MobilePhone|MobilePhoneModel": { + "multi_terms": { + "terms": [ + { + "field": "MobilePhone" + }, + { + "field": "MobilePhoneModel" + } + ], + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "u": "desc" + }, + { + "_key": "asc" + } + ] + }, + "aggregations": { + "u": { + "cardinality": { + "field": "UserID" + } + } + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q13.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q13.json new file mode 100644 index 0000000000000..77744d5e9ef68 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q13.json @@ -0,0 +1,48 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "SearchPhrase", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "SearchPhrase": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "aggregations": { + "SearchPhrase": { + "terms": { + "field": "SearchPhrase", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q14.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q14.json new file mode 100644 index 0000000000000..979f8aaade2bd --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q14.json @@ -0,0 +1,55 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "SearchPhrase", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "SearchPhrase": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "aggregations": { + "SearchPhrase": { + "terms": { + "field": "SearchPhrase", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "u": "desc" + }, + { + "_key": "asc" + } + ] + }, + "aggregations": { + "u": { + "cardinality": { + "field": "UserID" + } + } + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q15.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q15.json new file mode 100644 index 0000000000000..d1132e89f90cc --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q15.json @@ -0,0 +1,76 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "bool": { + "must": [ + { + "exists": { + "field": "SearchPhrase", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "SearchPhrase": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + { + "exists": { + "field": "SearchEngineID", + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "SearchPhrase", + "SearchEngineID" + ], + "excludes": [] + }, + "aggregations": { + "SearchEngineID|SearchPhrase": { + "multi_terms": { + "terms": [ + { + "field": "SearchEngineID" + }, + { + "field": "SearchPhrase" + } + ], + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q16.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q16.json new file mode 100644 index 0000000000000..7cef435b15293 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q16.json @@ -0,0 +1,24 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "UserID": { + "terms": { + "field": "UserID", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q17.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q17.json new file mode 100644 index 0000000000000..fa7592608c8ba --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q17.json @@ -0,0 +1,31 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "UserID|SearchPhrase": { + "multi_terms": { + "terms": [ + { + "field": "UserID" + }, + { + "field": "SearchPhrase" + } + ], + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q18.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q18.json new file mode 100644 index 0000000000000..5caf570c68706 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q18.json @@ -0,0 +1,32 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "composite_buckets": { + "composite": { + "size": 10, + "sources": [ + { + "UserID": { + "terms": { + "field": "UserID", + "missing_bucket": false, + "order": "asc" + } + } + }, + { + "SearchPhrase": { + "terms": { + "field": "SearchPhrase", + "missing_bucket": false, + "order": "asc" + } + } + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q19.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q19.json new file mode 100644 index 0000000000000..30dd2b86c5275 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q19.json @@ -0,0 +1,49 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "UserID|m|SearchPhrase": { + "multi_terms": { + "terms": [ + { + "field": "UserID" + }, + { + "script": { + "source": "{\"langType\":\"calcite\",\"script\":\"rO0ABXQCZnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJFWFRSQUNUIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ1ZHQiOiAiRVhQUl9USU1FU1RBTVAiLAogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9\"}", + "lang": "opensearch_compounded_script", + "params": { + "utcTimestamp": 1765261854238124000, + "SOURCES": [ + 2, + 0 + ], + "DIGESTS": [ + "minute", + "EventTime" + ] + } + }, + "value_type": "long" + }, + { + "field": "SearchPhrase" + } + ], + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q2.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q2.json new file mode 100644 index 0000000000000..04f351b3fff95 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q2.json @@ -0,0 +1,36 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "AdvEngineID", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "AdvEngineID": { + "value": 0, + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "AdvEngineID" + ], + "excludes": [] + }, + "track_total_hits": 2147483647 +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q20.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q20.json new file mode 100644 index 0000000000000..0268da24b570c --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q20.json @@ -0,0 +1,19 @@ +{ + "from": 0, + "size": 10000, + "timeout": "1m", + "query": { + "term": { + "UserID": { + "value": 435090932899640449, + "boost": 1.0 + } + } + }, + "_source": { + "includes": [ + "UserID" + ], + "excludes": [] + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q21.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q21.json new file mode 100644 index 0000000000000..eb0da407c85ab --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q21.json @@ -0,0 +1,20 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "wildcard": { + "URL": { + "wildcard": "*google*", + "boost": 1.0 + } + } + }, + "_source": { + "includes": [ + "URL" + ], + "excludes": [] + }, + "track_total_hits": 2147483647 +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q22.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q22.json new file mode 100644 index 0000000000000..ce7e166b7ab02 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q22.json @@ -0,0 +1,71 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "wildcard": { + "URL": { + "wildcard": "*google*", + "boost": 1.0 + } + } + }, + { + "bool": { + "must": [ + { + "exists": { + "field": "SearchPhrase", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "SearchPhrase": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "URL", + "SearchPhrase" + ], + "excludes": [] + }, + "aggregations": { + "SearchPhrase": { + "terms": { + "field": "SearchPhrase", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q23.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q23.json new file mode 100644 index 0000000000000..c6f24869acc8c --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q23.json @@ -0,0 +1,101 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "wildcard": { + "Title": { + "wildcard": "*Google*", + "boost": 1.0 + } + } + }, + { + "bool": { + "must": [ + { + "exists": { + "field": "SearchPhrase", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "SearchPhrase": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + { + "bool": { + "must_not": [ + { + "wildcard": { + "URL": { + "wildcard": "*.google.*", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "URL", + "SearchPhrase", + "UserID", + "Title" + ], + "excludes": [] + }, + "aggregations": { + "SearchPhrase": { + "terms": { + "field": "SearchPhrase", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "c": "desc" + }, + { + "_key": "asc" + } + ] + }, + "aggregations": { + "dc(UserID)": { + "cardinality": { + "field": "UserID" + } + }, + "c": { + "value_count": { + "field": "_index" + } + } + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q24.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q24.json new file mode 100644 index 0000000000000..5ec81f085af0c --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q24.json @@ -0,0 +1,131 @@ +{ + "from": 0, + "size": 10, + "timeout": "1m", + "query": { + "wildcard": { + "URL": { + "wildcard": "*google*", + "boost": 1.0 + } + } + }, + "_source": { + "includes": [ + "EventDate", + "URLRegionID", + "HasGCLID", + "Income", + "Interests", + "Robotness", + "BrowserLanguage", + "CounterClass", + "BrowserCountry", + "OriginalURL", + "ClientTimeZone", + "RefererHash", + "TraficSourceID", + "HitColor", + "RefererRegionID", + "URLCategoryID", + "LocalEventTime", + "EventTime", + "UTMTerm", + "AdvEngineID", + "UserAgentMinor", + "UserAgentMajor", + "RemoteIP", + "Sex", + "JavaEnable", + "URLHash", + "URL", + "ParamOrderID", + "OpenstatSourceID", + "HTTPError", + "SilverlightVersion3", + "MobilePhoneModel", + "SilverlightVersion4", + "SilverlightVersion1", + "SilverlightVersion2", + "IsDownload", + "IsParameter", + "CLID", + "FlashMajor", + "FlashMinor", + "UTMMedium", + "WatchID", + "DontCountHits", + "CookieEnable", + "HID", + "SocialAction", + "WindowName", + "ConnectTiming", + "PageCharset", + "IsLink", + "IsArtifical", + "JavascriptEnable", + "ClientEventTime", + "DNSTiming", + "CodeVersion", + "ResponseEndTiming", + "FUniqID", + "WindowClientHeight", + "OpenstatServiceName", + "UTMContent", + "HistoryLength", + "IsOldCounter", + "MobilePhone", + "SearchPhrase", + "FlashMinor2", + "SearchEngineID", + "IsEvent", + "UTMSource", + "RegionID", + "OpenstatAdID", + "UTMCampaign", + "GoodEvent", + "IsRefresh", + "ParamCurrency", + "Params", + "ResolutionHeight", + "ClientIP", + "FromTag", + "ParamCurrencyID", + "ResponseStartTiming", + "ResolutionWidth", + "SendTiming", + "RefererCategoryID", + "OpenstatCampaignID", + "UserID", + "WithHash", + "UserAgent", + "ParamPrice", + "ResolutionDepth", + "IsMobile", + "Age", + "SocialSourceNetworkID", + "OpenerName", + "OS", + "IsNotBounce", + "Referer", + "NetMinor", + "Title", + "NetMajor", + "IPNetworkID", + "FetchTiming", + "SocialNetwork", + "SocialSourcePage", + "CounterID", + "WindowClientWidth" + ], + "excludes": [] + }, + "sort": [ + { + "EventTime": { + "order": "asc", + "missing": "_first" + } + } + ] +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q25.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q25.json new file mode 100644 index 0000000000000..e848629fd9ac8 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q25.json @@ -0,0 +1,43 @@ +{ + "from": 0, + "size": 10, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "SearchPhrase", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "SearchPhrase": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "SearchPhrase" + ], + "excludes": [] + }, + "sort": [ + { + "EventTime": { + "order": "asc", + "missing": "_first" + } + } + ] +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q26.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q26.json new file mode 100644 index 0000000000000..cac94c9a611d6 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q26.json @@ -0,0 +1,43 @@ +{ + "from": 0, + "size": 10, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "SearchPhrase", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "SearchPhrase": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "SearchPhrase" + ], + "excludes": [] + }, + "sort": [ + { + "SearchPhrase": { + "order": "asc", + "missing": "_first" + } + } + ] +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q27.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q27.json new file mode 100644 index 0000000000000..ecf0a5bdb49f0 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q27.json @@ -0,0 +1,49 @@ +{ + "from": 0, + "size": 10, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "SearchPhrase", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "SearchPhrase": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "SearchPhrase" + ], + "excludes": [] + }, + "sort": [ + { + "EventTime": { + "order": "asc", + "missing": "_first" + } + }, + { + "SearchPhrase": { + "order": "asc", + "missing": "_first" + } + } + ] +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q28.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q28.json new file mode 100644 index 0000000000000..9631eb91fe4ba --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q28.json @@ -0,0 +1,87 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "bool": { + "must": [ + { + "exists": { + "field": "URL", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "URL": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + { + "exists": { + "field": "CounterID", + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "URL", + "CounterID" + ], + "excludes": [] + }, + "aggregations": { + "composite_buckets": { + "composite": { + "size": 10000, + "sources": [ + { + "CounterID": { + "terms": { + "field": "CounterID", + "missing_bucket": false, + "order": "asc" + } + } + } + ] + }, + "aggregations": { + "l": { + "avg": { + "script": { + "source": "{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}", + "lang": "opensearch_compounded_script", + "params": { + "utcTimestamp": 1765261854565507000, + "SOURCES": [ + 0 + ], + "DIGESTS": [ + "URL" + ] + } + } + } + } + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q29.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q29.json new file mode 100644 index 0000000000000..59dd9f6df46da --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q29.json @@ -0,0 +1,109 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "Referer", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "Referer": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "Referer" + ], + "excludes": [] + }, + "aggregations": { + "composite_buckets": { + "composite": { + "size": 10000, + "sources": [ + { + "k": { + "terms": { + "script": { + "source": "{\"langType\":\"calcite\",\"script\":\"rO0ABXQCGXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IDIKICAgICAgfQogICAgfQogIF0KfQ==\"}", + "lang": "opensearch_compounded_script", + "params": { + "utcTimestamp": 1765261854612898000, + "SOURCES": [ + 0, + 2, + 2 + ], + "DIGESTS": [ + "Referer", + "^https?://(?:www\\.)?([^/]+)/.*$", + "$1" + ] + } + }, + "missing_bucket": false, + "order": "asc" + } + } + } + ] + }, + "aggregations": { + "l": { + "avg": { + "script": { + "source": "{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}", + "lang": "opensearch_compounded_script", + "params": { + "utcTimestamp": 1765261854612898000, + "SOURCES": [ + 0 + ], + "DIGESTS": [ + "Referer" + ] + } + } + } + }, + "min(Referer)": { + "top_hits": { + "from": 0, + "size": 1, + "version": false, + "seq_no_primary_term": false, + "explain": false, + "fields": [ + { + "field": "Referer" + } + ], + "sort": [ + { + "Referer": { + "order": "asc" + } + } + ] + } + } + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q3.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q3.json new file mode 100644 index 0000000000000..48f70aeabba26 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q3.json @@ -0,0 +1,22 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "sum(AdvEngineID)": { + "sum": { + "field": "AdvEngineID" + } + }, + "count()": { + "value_count": { + "field": "_index" + } + }, + "avg(ResolutionWidth)": { + "avg": { + "field": "ResolutionWidth" + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q30.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q30.json new file mode 100644 index 0000000000000..58f78a7266001 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q30.json @@ -0,0 +1,17 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "sum(ResolutionWidth)": { + "sum": { + "field": "ResolutionWidth" + } + }, + "sum(ResolutionWidth+1)_COUNT": { + "value_count": { + "field": "ResolutionWidth" + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q31.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q31.json new file mode 100644 index 0000000000000..fd3aa65fdbcf4 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q31.json @@ -0,0 +1,102 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "bool": { + "must": [ + { + "exists": { + "field": "SearchPhrase", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "SearchPhrase": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + { + "exists": { + "field": "SearchEngineID", + "boost": 1.0 + } + }, + { + "exists": { + "field": "ClientIP", + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "SearchPhrase", + "SearchEngineID", + "IsRefresh", + "ClientIP", + "ResolutionWidth" + ], + "excludes": [] + }, + "aggregations": { + "SearchEngineID|ClientIP": { + "multi_terms": { + "terms": [ + { + "field": "SearchEngineID" + }, + { + "field": "ClientIP" + } + ], + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "c": "desc" + }, + { + "_key": "asc" + } + ] + }, + "aggregations": { + "sum(IsRefresh)": { + "sum": { + "field": "IsRefresh" + } + }, + "avg(ResolutionWidth)": { + "avg": { + "field": "ResolutionWidth" + } + }, + "c": { + "value_count": { + "field": "_index" + } + } + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q32.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q32.json new file mode 100644 index 0000000000000..7ce892abb1106 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q32.json @@ -0,0 +1,102 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "bool": { + "must": [ + { + "exists": { + "field": "SearchPhrase", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "SearchPhrase": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + { + "exists": { + "field": "WatchID", + "boost": 1.0 + } + }, + { + "exists": { + "field": "ClientIP", + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "WatchID", + "SearchPhrase", + "IsRefresh", + "ClientIP", + "ResolutionWidth" + ], + "excludes": [] + }, + "aggregations": { + "WatchID|ClientIP": { + "multi_terms": { + "terms": [ + { + "field": "WatchID" + }, + { + "field": "ClientIP" + } + ], + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "c": "desc" + }, + { + "_key": "asc" + } + ] + }, + "aggregations": { + "sum(IsRefresh)": { + "sum": { + "field": "IsRefresh" + } + }, + "avg(ResolutionWidth)": { + "avg": { + "field": "ResolutionWidth" + } + }, + "c": { + "value_count": { + "field": "_index" + } + } + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q33.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q33.json new file mode 100644 index 0000000000000..e4e0d5d350443 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q33.json @@ -0,0 +1,48 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "WatchID|ClientIP": { + "multi_terms": { + "terms": [ + { + "field": "WatchID" + }, + { + "field": "ClientIP" + } + ], + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "c": "desc" + }, + { + "_key": "asc" + } + ] + }, + "aggregations": { + "sum(IsRefresh)": { + "sum": { + "field": "IsRefresh" + } + }, + "avg(ResolutionWidth)": { + "avg": { + "field": "ResolutionWidth" + } + }, + "c": { + "value_count": { + "field": "_index" + } + } + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q34.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q34.json new file mode 100644 index 0000000000000..81df56b9adff8 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q34.json @@ -0,0 +1,24 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "URL": { + "terms": { + "field": "URL", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q35.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q35.json new file mode 100644 index 0000000000000..81df56b9adff8 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q35.json @@ -0,0 +1,24 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "URL": { + "terms": { + "field": "URL", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q36.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q36.json new file mode 100644 index 0000000000000..e1cb4a6063c5e --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q36.json @@ -0,0 +1,30 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "exists": { + "field": "ClientIP", + "boost": 1.0 + } + }, + "aggregations": { + "ClientIP": { + "terms": { + "field": "ClientIP", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q37.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q37.json new file mode 100644 index 0000000000000..2b1baee159a30 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q37.json @@ -0,0 +1,102 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "term": { + "CounterID": { + "value": 62, + "boost": 1.0 + } + } + }, + { + "range": { + "EventDate": { + "from": "2013-07-01T00:00:00.000Z", + "to": "2013-07-31T00:00:00.000Z", + "include_lower": true, + "include_upper": true, + "format": "date_time", + "boost": 1.0 + } + } + }, + { + "term": { + "DontCountHits": { + "value": 0, + "boost": 1.0 + } + } + }, + { + "term": { + "IsRefresh": { + "value": 0, + "boost": 1.0 + } + } + }, + { + "bool": { + "must": [ + { + "exists": { + "field": "URL", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "URL": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "EventDate", + "URL", + "DontCountHits", + "IsRefresh", + "CounterID" + ], + "excludes": [] + }, + "aggregations": { + "URL": { + "terms": { + "field": "URL", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q38.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q38.json new file mode 100644 index 0000000000000..8fb2f3f585ca5 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q38.json @@ -0,0 +1,102 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "term": { + "CounterID": { + "value": 62, + "boost": 1.0 + } + } + }, + { + "range": { + "EventDate": { + "from": "2013-07-01T00:00:00.000Z", + "to": "2013-07-31T00:00:00.000Z", + "include_lower": true, + "include_upper": true, + "format": "date_time", + "boost": 1.0 + } + } + }, + { + "term": { + "DontCountHits": { + "value": 0, + "boost": 1.0 + } + } + }, + { + "term": { + "IsRefresh": { + "value": 0, + "boost": 1.0 + } + } + }, + { + "bool": { + "must": [ + { + "exists": { + "field": "Title", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "Title": { + "value": "", + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "EventDate", + "DontCountHits", + "IsRefresh", + "Title", + "CounterID" + ], + "excludes": [] + }, + "aggregations": { + "Title": { + "terms": { + "field": "Title", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q39.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q39.json new file mode 100644 index 0000000000000..440839f125bb5 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q39.json @@ -0,0 +1,109 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "term": { + "CounterID": { + "value": 62, + "boost": 1.0 + } + } + }, + { + "range": { + "EventDate": { + "from": "2013-07-01T00:00:00.000Z", + "to": "2013-07-31T00:00:00.000Z", + "include_lower": true, + "include_upper": true, + "format": "date_time", + "boost": 1.0 + } + } + }, + { + "term": { + "IsRefresh": { + "value": 0, + "boost": 1.0 + } + } + }, + { + "bool": { + "must": [ + { + "exists": { + "field": "IsLink", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "IsLink": { + "value": 0, + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + { + "term": { + "IsDownload": { + "value": 0, + "boost": 1.0 + } + } + }, + { + "exists": { + "field": "URL", + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "EventDate", + "URL", + "IsDownload", + "IsLink", + "IsRefresh", + "CounterID" + ], + "excludes": [] + }, + "aggregations": { + "URL": { + "terms": { + "field": "URL", + "size": 1010, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q4.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q4.json new file mode 100644 index 0000000000000..cbc3569702fc8 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q4.json @@ -0,0 +1,12 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "avg(UserID)": { + "avg": { + "field": "UserID" + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q40.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q40.json new file mode 100644 index 0000000000000..c28f0158209a1 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q40.json @@ -0,0 +1,111 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "term": { + "CounterID": { + "value": 62, + "boost": 1.0 + } + } + }, + { + "range": { + "EventDate": { + "from": "2013-07-01T00:00:00.000Z", + "to": "2013-07-31T00:00:00.000Z", + "include_lower": true, + "include_upper": true, + "format": "date_time", + "boost": 1.0 + } + } + }, + { + "term": { + "IsRefresh": { + "value": 0, + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "EventDate", + "TraficSourceID", + "AdvEngineID", + "URL", + "SearchEngineID", + "IsRefresh", + "Referer", + "CounterID" + ], + "excludes": [] + }, + "aggregations": { + "TraficSourceID|SearchEngineID|AdvEngineID|Src|Dst": { + "multi_terms": { + "terms": [ + { + "field": "TraficSourceID" + }, + { + "field": "SearchEngineID" + }, + { + "field": "AdvEngineID" + }, + { + "script": { + "source": "{\"langType\":\"calcite\",\"script\":\"rO0ABXQGCnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQU5EIiwKICAgICAgICAia2luZCI6ICJBTkQiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAgICAgImtpbmQiOiAiRVFVQUxTIiwKICAgICAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAib3AiOiB7CiAgICAgICAgICAgICJuYW1lIjogIj0iLAogICAgICAgICAgICAia2luZCI6ICJFUVVBTFMiLAogICAgICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlNNQUxMSU5UIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiA0LAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogNSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn0=\"}", + "lang": "opensearch_compounded_script", + "params": { + "utcTimestamp": 1765261855083734000, + "SOURCES": [ + 0, + 2, + 0, + 2, + 0, + 2 + ], + "DIGESTS": [ + "SearchEngineID", + 0, + "AdvEngineID", + 0, + "Referer", + "" + ] + } + } + }, + { + "field": "URL" + } + ], + "size": 1010, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q41.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q41.json new file mode 100644 index 0000000000000..5f9af04f31b7f --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q41.json @@ -0,0 +1,102 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "term": { + "CounterID": { + "value": 62, + "boost": 1.0 + } + } + }, + { + "range": { + "EventDate": { + "from": "2013-07-01T00:00:00.000Z", + "to": "2013-07-31T00:00:00.000Z", + "include_lower": true, + "include_upper": true, + "format": "date_time", + "boost": 1.0 + } + } + }, + { + "term": { + "IsRefresh": { + "value": 0, + "boost": 1.0 + } + } + }, + { + "terms": { + "TraficSourceID": [ + -1.0, + 6.0 + ], + "boost": 1.0 + } + }, + { + "term": { + "RefererHash": { + "value": 3594120000172545465, + "boost": 1.0 + } + } + }, + { + "exists": { + "field": "URLHash", + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "EventDate", + "RefererHash", + "TraficSourceID", + "URLHash", + "IsRefresh", + "CounterID" + ], + "excludes": [] + }, + "aggregations": { + "URLHash|EventDate": { + "multi_terms": { + "terms": [ + { + "field": "URLHash" + }, + { + "field": "EventDate", + "value_type": "long" + } + ], + "size": 110, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q42.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q42.json new file mode 100644 index 0000000000000..5a90a6c648263 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q42.json @@ -0,0 +1,107 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "term": { + "CounterID": { + "value": 62, + "boost": 1.0 + } + } + }, + { + "range": { + "EventDate": { + "from": "2013-07-01T00:00:00.000Z", + "to": "2013-07-31T00:00:00.000Z", + "include_lower": true, + "include_upper": true, + "format": "date_time", + "boost": 1.0 + } + } + }, + { + "term": { + "IsRefresh": { + "value": 0, + "boost": 1.0 + } + } + }, + { + "term": { + "DontCountHits": { + "value": 0, + "boost": 1.0 + } + } + }, + { + "term": { + "URLHash": { + "value": 2868770270353813622, + "boost": 1.0 + } + } + }, + { + "exists": { + "field": "WindowClientWidth", + "boost": 1.0 + } + }, + { + "exists": { + "field": "WindowClientHeight", + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "EventDate", + "URLHash", + "DontCountHits", + "WindowClientHeight", + "IsRefresh", + "CounterID", + "WindowClientWidth" + ], + "excludes": [] + }, + "aggregations": { + "WindowClientWidth|WindowClientHeight": { + "multi_terms": { + "terms": [ + { + "field": "WindowClientWidth" + }, + { + "field": "WindowClientHeight" + } + ], + "size": 10000, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q43.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q43.json new file mode 100644 index 0000000000000..fe3721abb7c77 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q43.json @@ -0,0 +1,84 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "term": { + "CounterID": { + "value": 62, + "boost": 1.0 + } + } + }, + { + "range": { + "EventDate": { + "from": "2013-07-01T00:00:00.000Z", + "to": "2013-07-15T00:00:00.000Z", + "include_lower": true, + "include_upper": true, + "format": "date_time", + "boost": 1.0 + } + } + }, + { + "term": { + "IsRefresh": { + "value": 0, + "boost": 1.0 + } + } + }, + { + "term": { + "DontCountHits": { + "value": 0, + "boost": 1.0 + } + } + }, + { + "exists": { + "field": "EventTime", + "boost": 1.0 + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "_source": { + "includes": [ + "EventDate", + "EventTime", + "DontCountHits", + "IsRefresh", + "CounterID" + ], + "excludes": [] + }, + "aggregations": { + "composite_buckets": { + "composite": { + "size": 1010, + "sources": [ + { + "M": { + "date_histogram": { + "field": "EventTime", + "missing_bucket": false, + "order": "asc", + "fixed_interval": "1m" + } + } + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q5.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q5.json new file mode 100644 index 0000000000000..80317df39575e --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q5.json @@ -0,0 +1,18 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "exists": { + "field": "UserID", + "boost": 1.0 + } + }, + "aggregations": { + "dc(UserID)": { + "cardinality": { + "field": "UserID" + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q6.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q6.json new file mode 100644 index 0000000000000..09943aa083777 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q6.json @@ -0,0 +1,18 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "exists": { + "field": "SearchPhrase", + "boost": 1.0 + } + }, + "aggregations": { + "dc(SearchPhrase)": { + "cardinality": { + "field": "SearchPhrase" + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q7.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q7.json new file mode 100644 index 0000000000000..4ef1e8ddf33af --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q7.json @@ -0,0 +1,17 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "min(EventDate)": { + "min": { + "field": "EventDate" + } + }, + "max(EventDate)": { + "max": { + "field": "EventDate" + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q8.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q8.json new file mode 100644 index 0000000000000..66e882b0fe313 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q8.json @@ -0,0 +1,48 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "query": { + "bool": { + "must": [ + { + "exists": { + "field": "AdvEngineID", + "boost": 1.0 + } + } + ], + "must_not": [ + { + "term": { + "AdvEngineID": { + "value": 0, + "boost": 1.0 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1.0 + } + }, + "aggregations": { + "AdvEngineID": { + "terms": { + "field": "AdvEngineID", + "size": 10000, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "_count": "desc" + }, + { + "_key": "asc" + } + ] + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q9.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q9.json new file mode 100644 index 0000000000000..d72ba5bf44fc3 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/dsl/q9.json @@ -0,0 +1,31 @@ +{ + "from": 0, + "size": 0, + "timeout": "1m", + "aggregations": { + "RegionID": { + "terms": { + "field": "RegionID", + "size": 10, + "min_doc_count": 1, + "shard_min_doc_count": 0, + "show_term_doc_count_error": false, + "order": [ + { + "u": "desc" + }, + { + "_key": "asc" + } + ] + }, + "aggregations": { + "u": { + "cardinality": { + "field": "UserID" + } + } + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/mapping.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/mapping.json new file mode 100644 index 0000000000000..3424a925af5c1 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/mapping.json @@ -0,0 +1,323 @@ +{ + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "AdvEngineID": { + "type": "short" + }, + "Age": { + "type": "short" + }, + "BrowserCountry": { + "type": "keyword" + }, + "BrowserLanguage": { + "type": "keyword" + }, + "CLID": { + "type": "integer" + }, + "ClientEventTime": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time||epoch_millis" + }, + "ClientIP": { + "type": "integer" + }, + "ClientTimeZone": { + "type": "short" + }, + "CodeVersion": { + "type": "integer" + }, + "ConnectTiming": { + "type": "integer" + }, + "CookieEnable": { + "type": "short" + }, + "CounterClass": { + "type": "short" + }, + "CounterID": { + "type": "integer" + }, + "DNSTiming": { + "type": "integer" + }, + "DontCountHits": { + "type": "short" + }, + "EventDate": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time||epoch_millis" + }, + "EventTime": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time||epoch_millis" + }, + "FUniqID": { + "type": "long" + }, + "FetchTiming": { + "type": "integer" + }, + "FlashMajor": { + "type": "short" + }, + "FlashMinor": { + "type": "short" + }, + "FlashMinor2": { + "type": "short" + }, + "FromTag": { + "type": "keyword" + }, + "GoodEvent": { + "type": "short" + }, + "HID": { + "type": "integer" + }, + "HTTPError": { + "type": "short" + }, + "HasGCLID": { + "type": "short" + }, + "HistoryLength": { + "type": "short" + }, + "HitColor": { + "type": "keyword" + }, + "IPNetworkID": { + "type": "integer" + }, + "Income": { + "type": "short" + }, + "Interests": { + "type": "short" + }, + "IsArtifical": { + "type": "short" + }, + "IsDownload": { + "type": "short" + }, + "IsEvent": { + "type": "short" + }, + "IsLink": { + "type": "short" + }, + "IsMobile": { + "type": "short" + }, + "IsNotBounce": { + "type": "short" + }, + "IsOldCounter": { + "type": "short" + }, + "IsParameter": { + "type": "short" + }, + "IsRefresh": { + "type": "short" + }, + "JavaEnable": { + "type": "short" + }, + "JavascriptEnable": { + "type": "short" + }, + "LocalEventTime": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time||epoch_millis" + }, + "MobilePhone": { + "type": "short" + }, + "MobilePhoneModel": { + "type": "keyword" + }, + "NetMajor": { + "type": "short" + }, + "NetMinor": { + "type": "short" + }, + "OS": { + "type": "short" + }, + "OpenerName": { + "type": "integer" + }, + "OpenstatAdID": { + "type": "keyword" + }, + "OpenstatCampaignID": { + "type": "keyword" + }, + "OpenstatServiceName": { + "type": "keyword" + }, + "OpenstatSourceID": { + "type": "keyword" + }, + "OriginalURL": { + "type": "keyword" + }, + "PageCharset": { + "type": "keyword" + }, + "ParamCurrency": { + "type": "keyword" + }, + "ParamCurrencyID": { + "type": "short" + }, + "ParamOrderID": { + "type": "keyword" + }, + "ParamPrice": { + "type": "long" + }, + "Params": { + "type": "keyword" + }, + "Referer": { + "type": "keyword" + }, + "RefererCategoryID": { + "type": "short" + }, + "RefererHash": { + "type": "long" + }, + "RefererRegionID": { + "type": "integer" + }, + "RegionID": { + "type": "integer" + }, + "RemoteIP": { + "type": "integer" + }, + "ResolutionDepth": { + "type": "short" + }, + "ResolutionHeight": { + "type": "short" + }, + "ResolutionWidth": { + "type": "short" + }, + "ResponseEndTiming": { + "type": "integer" + }, + "ResponseStartTiming": { + "type": "integer" + }, + "Robotness": { + "type": "short" + }, + "SearchEngineID": { + "type": "short" + }, + "SearchPhrase": { + "type": "keyword" + }, + "SendTiming": { + "type": "integer" + }, + "Sex": { + "type": "short" + }, + "SilverlightVersion1": { + "type": "short" + }, + "SilverlightVersion2": { + "type": "short" + }, + "SilverlightVersion3": { + "type": "integer" + }, + "SilverlightVersion4": { + "type": "short" + }, + "SocialSourceNetworkID": { + "type": "short" + }, + "SocialSourcePage": { + "type": "keyword" + }, + "Title": { + "type": "keyword" + }, + "TraficSourceID": { + "type": "short" + }, + "URL": { + "type": "keyword" + }, + "URLCategoryID": { + "type": "short" + }, + "URLHash": { + "type": "long" + }, + "URLRegionID": { + "type": "integer" + }, + "UTMCampaign": { + "type": "keyword" + }, + "UTMContent": { + "type": "keyword" + }, + "UTMMedium": { + "type": "keyword" + }, + "UTMSource": { + "type": "keyword" + }, + "UTMTerm": { + "type": "keyword" + }, + "UserAgent": { + "type": "short" + }, + "UserAgentMajor": { + "type": "short" + }, + "UserAgentMinor": { + "type": "keyword" + }, + "UserID": { + "type": "long" + }, + "WatchID": { + "type": "long" + }, + "WindowClientHeight": { + "type": "short" + }, + "WindowClientWidth": { + "type": "short" + }, + "WindowName": { + "type": "integer" + }, + "WithHash": { + "type": "short" + } + } + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q1.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q1.ppl new file mode 100644 index 0000000000000..bad036a4f270a --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q1.ppl @@ -0,0 +1 @@ +source = clickbench | stats count() diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q10.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q10.ppl new file mode 100644 index 0000000000000..a7d0c198dbca7 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q10.ppl @@ -0,0 +1 @@ +source = clickbench | stats sum(AdvEngineID), count() as c, avg(ResolutionWidth), dc(UserID) by RegionID | sort - c | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q11.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q11.ppl new file mode 100644 index 0000000000000..8d483893ee151 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q11.ppl @@ -0,0 +1 @@ +source = clickbench | where MobilePhoneModel != '' | stats dc(UserID) as u by MobilePhoneModel | sort - u | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q12.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q12.ppl new file mode 100644 index 0000000000000..b33534923fe2f --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q12.ppl @@ -0,0 +1 @@ +source = clickbench | where MobilePhoneModel != '' | stats dc(UserID) as u by MobilePhone, MobilePhoneModel | sort - u | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q13.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q13.ppl new file mode 100644 index 0000000000000..2401de5095b22 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q13.ppl @@ -0,0 +1 @@ +source = clickbench | where SearchPhrase != '' | stats count() as c by SearchPhrase | sort - c | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q14.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q14.ppl new file mode 100644 index 0000000000000..98f22fe24941a --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q14.ppl @@ -0,0 +1 @@ +source = clickbench | where SearchPhrase != '' | stats dc(UserID) as u by SearchPhrase | sort - u | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q15.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q15.ppl new file mode 100644 index 0000000000000..ff6c5c5f9eb07 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q15.ppl @@ -0,0 +1 @@ +source = clickbench | where SearchPhrase != '' | stats count() as c by SearchEngineID, SearchPhrase | sort - c | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q16.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q16.ppl new file mode 100644 index 0000000000000..157e75680e1b1 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q16.ppl @@ -0,0 +1 @@ +source = clickbench | stats count() by UserID | sort - `count()` | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q17.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q17.ppl new file mode 100644 index 0000000000000..0ad47efdd3693 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q17.ppl @@ -0,0 +1 @@ +source = clickbench | stats count() by UserID, SearchPhrase | sort - `count()` | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q18.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q18.ppl new file mode 100644 index 0000000000000..03f06e60e3259 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q18.ppl @@ -0,0 +1 @@ +source = clickbench | stats count() by UserID, SearchPhrase | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q19.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q19.ppl new file mode 100644 index 0000000000000..ac7c3cc785ac6 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q19.ppl @@ -0,0 +1 @@ +source = clickbench | eval m = extract(minute from EventTime) | stats count() by UserID, m, SearchPhrase | sort - `count()` | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q2.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q2.ppl new file mode 100644 index 0000000000000..b52e0e25ca873 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q2.ppl @@ -0,0 +1 @@ +source = clickbench | where AdvEngineID!=0 | stats count() diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q20.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q20.ppl new file mode 100644 index 0000000000000..ce8b135c8274f --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q20.ppl @@ -0,0 +1 @@ +source = clickbench | where UserID = 435090932899640449 | fields UserID diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q21.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q21.ppl new file mode 100644 index 0000000000000..f0916f1ea0f04 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q21.ppl @@ -0,0 +1 @@ +source = clickbench | where like(URL, '%google%') | stats count() diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q22.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q22.ppl new file mode 100644 index 0000000000000..70081a8b5ffce --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q22.ppl @@ -0,0 +1 @@ +source = clickbench | where like(URL, '%google%') and SearchPhrase != '' | stats count() as c by SearchPhrase | sort - c | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q23.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q23.ppl new file mode 100644 index 0000000000000..a7458812255ab --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q23.ppl @@ -0,0 +1 @@ +source = clickbench | where like(Title, '%Google%') and not like(URL, '%.google.%') and SearchPhrase != '' | stats count() as c, dc(UserID) by SearchPhrase | sort - c | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q24.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q24.ppl new file mode 100644 index 0000000000000..e5eee2b10e54e --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q24.ppl @@ -0,0 +1 @@ +source = clickbench | where like(URL, '%google%') | sort EventTime | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q25.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q25.ppl new file mode 100644 index 0000000000000..24ee7f027943a --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q25.ppl @@ -0,0 +1 @@ +source = clickbench | where SearchPhrase != '' | sort EventTime | fields SearchPhrase | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q26.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q26.ppl new file mode 100644 index 0000000000000..b15493f836702 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q26.ppl @@ -0,0 +1 @@ +source = clickbench | where SearchPhrase != '' | fields SearchPhrase | sort SearchPhrase | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q27.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q27.ppl new file mode 100644 index 0000000000000..4437852d8948d --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q27.ppl @@ -0,0 +1 @@ +source = clickbench | where SearchPhrase != '' | sort EventTime, SearchPhrase | fields SearchPhrase | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q28.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q28.ppl new file mode 100644 index 0000000000000..c93dd211ab90f --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q28.ppl @@ -0,0 +1 @@ +source = clickbench | where URL != '' | stats avg(length(URL)) as l, count() as c by CounterID | where c > 100000 | sort - l | head 25 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q29.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q29.ppl new file mode 100644 index 0000000000000..19ed59cb3cf7c --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q29.ppl @@ -0,0 +1 @@ +source = clickbench | Referer != '' | eval k = regexp_replace(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') | stats avg(length(Referer)) as l, count() as c, min(Referer) by k | where c > 100000 | sort - l | head 25 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q3.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q3.ppl new file mode 100644 index 0000000000000..8f2bce48fc064 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q3.ppl @@ -0,0 +1 @@ +source = clickbench | stats sum(AdvEngineID), count(), avg(ResolutionWidth) diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q30.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q30.ppl new file mode 100644 index 0000000000000..34685b26c5ebf --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q30.ppl @@ -0,0 +1 @@ +source = clickbench | stats sum(ResolutionWidth), sum(ResolutionWidth+1), sum(ResolutionWidth+2), sum(ResolutionWidth+3), sum(ResolutionWidth+4), sum(ResolutionWidth+5), sum(ResolutionWidth+6), sum(ResolutionWidth+7), sum(ResolutionWidth+8), sum(ResolutionWidth+9) diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q31.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q31.ppl new file mode 100644 index 0000000000000..537ec1565bba3 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q31.ppl @@ -0,0 +1 @@ +source = clickbench | where SearchPhrase != '' | stats count() as c, sum(IsRefresh), avg(ResolutionWidth) by SearchEngineID, ClientIP | sort - c | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q32.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q32.ppl new file mode 100644 index 0000000000000..7d18c7953e4df --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q32.ppl @@ -0,0 +1 @@ +source = clickbench | where SearchPhrase != '' | stats count() as c, sum(IsRefresh), avg(ResolutionWidth) by WatchID, ClientIP | sort - c | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q33.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q33.ppl new file mode 100644 index 0000000000000..5cadfab7ef0b7 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q33.ppl @@ -0,0 +1 @@ +source = clickbench | stats count() as c, sum(IsRefresh), avg(ResolutionWidth) by WatchID, ClientIP | sort - c | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q34.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q34.ppl new file mode 100644 index 0000000000000..f7f147accb219 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q34.ppl @@ -0,0 +1 @@ +source = clickbench | stats count() as c by URL | sort - c | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q35.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q35.ppl new file mode 100644 index 0000000000000..e9faf66bfdd99 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q35.ppl @@ -0,0 +1 @@ +source = clickbench | eval const = 1 | stats count() as c by const, URL | sort - c | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q36.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q36.ppl new file mode 100644 index 0000000000000..f9d633e4f5117 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q36.ppl @@ -0,0 +1 @@ +source = clickbench | eval `ClientIP - 1` = ClientIP - 1, `ClientIP - 2` = ClientIP - 2, `ClientIP - 3` = ClientIP - 3 | stats count() as c by ClientIP, `ClientIP - 1`, `ClientIP - 2`, `ClientIP - 3` | sort - c | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q37.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q37.ppl new file mode 100644 index 0000000000000..0e7e8563285a1 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q37.ppl @@ -0,0 +1 @@ +source = clickbench | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and DontCountHits = 0 and IsRefresh = 0 and URL != '' | stats count() as PageViews by URL | sort - PageViews | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q38.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q38.ppl new file mode 100644 index 0000000000000..ea48c98e2bd35 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q38.ppl @@ -0,0 +1 @@ +source = clickbench | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and DontCountHits = 0 and IsRefresh = 0 and Title != '' | stats count() as PageViews by Title | sort - PageViews | head 10 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q39.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q39.ppl new file mode 100644 index 0000000000000..32b2d3cc3f7b3 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q39.ppl @@ -0,0 +1 @@ +source = clickbench | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and IsLink != 0 and IsDownload = 0 | stats count() as PageViews by URL | sort - PageViews | head 10 from 1000 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q4.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q4.ppl new file mode 100644 index 0000000000000..7a567f19b6942 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q4.ppl @@ -0,0 +1 @@ +source = clickbench | stats avg(UserID) diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q40.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q40.ppl new file mode 100644 index 0000000000000..1327762ad3359 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q40.ppl @@ -0,0 +1 @@ +source = clickbench | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 | eval Src=case(SearchEngineID = 0 and AdvEngineID = 0, Referer else ''), Dst=URL | stats count() as PageViews by TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst | sort - PageViews | head 10 from 1000 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q41.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q41.ppl new file mode 100644 index 0000000000000..17a373f376111 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q41.ppl @@ -0,0 +1 @@ +source = clickbench | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and TraficSourceID in (-1, 6) and RefererHash = 3594120000172545465 | stats count() as PageViews by URLHash, EventDate | sort - PageViews | head 10 from 100 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q42.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q42.ppl new file mode 100644 index 0000000000000..cff7ee534ad94 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q42.ppl @@ -0,0 +1 @@ +source = clickbench | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and DontCountHits = 0 and URLHash = 2868770270353813622 | stats count() as PageViews by WindowClientWidth, WindowClientHeight | sort - PageViews | head 10 from 10000 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q43.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q43.ppl new file mode 100644 index 0000000000000..990e3450fa713 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q43.ppl @@ -0,0 +1 @@ +source = clickbench | where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-15 00:00:00' and IsRefresh = 0 and DontCountHits = 0 | eval M = date_format(EventTime, '%Y-%m-%d %H:00:00') | stats count() as PageViews by M | sort M | head 10 from 1000 diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q5.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q5.ppl new file mode 100644 index 0000000000000..0c6974ee95514 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q5.ppl @@ -0,0 +1 @@ +source = clickbench | stats dc(UserID) diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q6.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q6.ppl new file mode 100644 index 0000000000000..a1896c31f1d69 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q6.ppl @@ -0,0 +1 @@ +source = clickbench | stats dc(SearchPhrase) diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q7.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q7.ppl new file mode 100644 index 0000000000000..5f92ddc6fb6a7 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q7.ppl @@ -0,0 +1 @@ +source = clickbench | stats min(EventDate), max(EventDate) diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q8.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q8.ppl new file mode 100644 index 0000000000000..28c29067cd425 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q8.ppl @@ -0,0 +1 @@ +source = clickbench | where AdvEngineID!=0 | stats count() by AdvEngineID | sort - `count()` diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q9.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q9.ppl new file mode 100644 index 0000000000000..ac5a40dc2ca06 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q9.ppl @@ -0,0 +1 @@ +source = clickbench | stats dc(UserID) as u by RegionID | sort -u | head 10 diff --git a/sandbox/qa/build.gradle b/sandbox/qa/build.gradle new file mode 100644 index 0000000000000..b5b39ea4ed3cd --- /dev/null +++ b/sandbox/qa/build.gradle @@ -0,0 +1,7 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ From fc3273495b3ec014d9586c80c9a4116f4bdc9168 Mon Sep 17 00:00:00 2001 From: Rishab Nahata Date: Sat, 2 May 2026 11:57:45 +0530 Subject: [PATCH 029/115] Add partition strategy setting and partition assignment for flexible shard-to-partition mapping (#21165) * Add partition_strategy setting and PartitionAssignment for flexible shard-to-partition mapping --------- Signed-off-by: Rishab Nahata --- .../cluster/metadata/IndexMetadata.java | 18 +++ .../cluster/metadata/IngestionSource.java | 62 +++++++- .../metadata/MetadataCreateIndexService.java | 23 +++ .../SourcePartitionAssignment.java | 87 +++++++++++ .../metadata/IngestionSourceTests.java | 67 +++++++- .../MetadataCreateIndexServiceTests.java | 80 ++++++++++ .../SourcePartitionAssignmentTests.java | 146 ++++++++++++++++++ 7 files changed, 478 insertions(+), 5 deletions(-) create mode 100644 server/src/main/java/org/opensearch/indices/pollingingest/SourcePartitionAssignment.java create mode 100644 server/src/test/java/org/opensearch/indices/pollingingest/SourcePartitionAssignmentTests.java diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 6d4f4360c22bb..3a30d0688f734 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -960,6 +960,20 @@ public Iterator> settings() { Property.Final ); + /** + * Defines the strategy for mapping source stream partitions to OpenSearch shards. + * "simple" (default): 1:1 mapping where shard N consumes partition N. + * "modulo": each shard consumes all partitions where partition % numShards == shardId. + */ + public static final String SETTING_INGESTION_SOURCE_PARTITION_STRATEGY = "index.ingestion_source.source_partition_strategy"; + public static final Setting INGESTION_SOURCE_PARTITION_STRATEGY_SETTING = new Setting<>( + SETTING_INGESTION_SOURCE_PARTITION_STRATEGY, + IngestionSource.SourcePartitionStrategy.SIMPLE.getName(), + IngestionSource.SourcePartitionStrategy::fromString, + Property.IndexScope, + Property.Final + ); + /** * Defines if all-active pull-based ingestion is enabled. In this mode, replicas will directly consume from the * streaming source and process the updates. In the default document replication mode, this setting must be enabled. @@ -1327,6 +1341,9 @@ public IngestionSource getIngestionSource() { final TimeValue pointerBasedLagUpdateInterval = INGESTION_SOURCE_POINTER_BASED_LAG_UPDATE_INTERVAL_SETTING.get(settings); final IngestionMessageMapper.MapperType mapperType = INGESTION_SOURCE_MAPPER_TYPE_SETTING.get(settings); final Map mapperSettings = INGESTION_SOURCE_MAPPER_SETTINGS.getAsMap(settings); + final IngestionSource.SourcePartitionStrategy sourcePartitionStrategy = INGESTION_SOURCE_PARTITION_STRATEGY_SETTING.get( + settings + ); // Warmup settings final IngestionSource.WarmupConfig warmupConfig = new IngestionSource.WarmupConfig( @@ -1345,6 +1362,7 @@ public IngestionSource getIngestionSource() { .setPointerBasedLagUpdateInterval(pointerBasedLagUpdateInterval) .setMapperType(mapperType) .setMapperSettings(mapperSettings) + .setSourcePartitionStrategy(sourcePartitionStrategy) .setWarmupConfig(warmupConfig) .build(); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java index 6b1400c305e08..af07f11ec696c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java @@ -25,6 +25,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_MAPPER_TYPE_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_MAX_POLL_SIZE; import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_NUM_PROCESSOR_THREADS_SETTING; +import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_PARTITION_STRATEGY_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_POINTER_BASED_LAG_UPDATE_INTERVAL_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_POLL_TIMEOUT; import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING; @@ -48,6 +49,7 @@ public class IngestionSource { private final IngestionMessageMapper.MapperType mapperType; private final Map mapperSettings; private final WarmupConfig warmupConfig; + private final SourcePartitionStrategy sourcePartitionStrategy; private IngestionSource( String type, @@ -62,7 +64,8 @@ private IngestionSource( TimeValue pointerBasedLagUpdateInterval, IngestionMessageMapper.MapperType mapperType, Map mapperSettings, - WarmupConfig warmupConfig + WarmupConfig warmupConfig, + SourcePartitionStrategy sourcePartitionStrategy ) { this.type = type; this.pointerInitReset = pointerInitReset; @@ -77,6 +80,7 @@ private IngestionSource( this.mapperType = mapperType; this.mapperSettings = mapperSettings != null ? Collections.unmodifiableMap(mapperSettings) : Collections.emptyMap(); this.warmupConfig = warmupConfig; + this.sourcePartitionStrategy = sourcePartitionStrategy; } public String getType() { @@ -131,6 +135,10 @@ public WarmupConfig getWarmupConfig() { return warmupConfig; } + public SourcePartitionStrategy getSourcePartitionStrategy() { + return sourcePartitionStrategy; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -148,7 +156,8 @@ public boolean equals(Object o) { && Objects.equals(pointerBasedLagUpdateInterval, ingestionSource.pointerBasedLagUpdateInterval) && Objects.equals(mapperType, ingestionSource.mapperType) && Objects.equals(mapperSettings, ingestionSource.mapperSettings) - && Objects.equals(warmupConfig, ingestionSource.warmupConfig); + && Objects.equals(warmupConfig, ingestionSource.warmupConfig) + && Objects.equals(sourcePartitionStrategy, ingestionSource.sourcePartitionStrategy); } @Override @@ -166,7 +175,8 @@ public int hashCode() { pointerBasedLagUpdateInterval, mapperType, mapperSettings, - warmupConfig + warmupConfig, + sourcePartitionStrategy ); } @@ -203,9 +213,45 @@ public String toString() { + mapperSettings + ", warmupConfig=" + warmupConfig + + ", sourcePartitionStrategy='" + + sourcePartitionStrategy + + '\'' + '}'; } + /** + * Strategy for mapping source stream partitions to OpenSearch shards. + */ + @PublicApi(since = "3.7.0") + public enum SourcePartitionStrategy { + SIMPLE("simple"), + MODULO("modulo"); + + private final String name; + + SourcePartitionStrategy(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public static SourcePartitionStrategy fromString(String name) { + for (SourcePartitionStrategy strategy : values()) { + if (strategy.getName().equalsIgnoreCase(name)) { + return strategy; + } + } + throw new IllegalArgumentException("Unknown partition strategy: [" + name + "]. Valid values are [simple, modulo]"); + } + + @Override + public String toString() { + return name; + } + } + /** * Class encapsulating the configuration of a pointer initialization. */ @@ -281,6 +327,7 @@ public static class Builder { ); private IngestionMessageMapper.MapperType mapperType = INGESTION_SOURCE_MAPPER_TYPE_SETTING.getDefault(Settings.EMPTY); private Map mapperSettings = new HashMap<>(); + private SourcePartitionStrategy sourcePartitionStrategy = INGESTION_SOURCE_PARTITION_STRATEGY_SETTING.getDefault(Settings.EMPTY); // Warmup configuration private TimeValue warmupTimeout = INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING.getDefault(Settings.EMPTY); private long warmupLagThreshold = INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING.getDefault(Settings.EMPTY); @@ -300,6 +347,7 @@ public Builder(IngestionSource ingestionSource) { this.pointerBasedLagUpdateInterval = ingestionSource.pointerBasedLagUpdateInterval; this.mapperType = ingestionSource.mapperType; this.mapperSettings = new HashMap<>(ingestionSource.mapperSettings); + this.sourcePartitionStrategy = ingestionSource.sourcePartitionStrategy; // Copy warmup config WarmupConfig wc = ingestionSource.warmupConfig; this.warmupTimeout = wc.timeout(); @@ -366,6 +414,11 @@ public Builder setMapperSettings(Map mapperSettings) { return this; } + public Builder setSourcePartitionStrategy(SourcePartitionStrategy sourcePartitionStrategy) { + this.sourcePartitionStrategy = sourcePartitionStrategy; + return this; + } + public Builder setWarmupTimeout(TimeValue warmupTimeout) { this.warmupTimeout = warmupTimeout; return this; @@ -397,7 +450,8 @@ public IngestionSource build() { pointerBasedLagUpdateInterval, mapperType, mapperSettings, - warmupConfig + warmupConfig, + sourcePartitionStrategy ); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 1a3c581fa1d13..8a8214b530b50 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1277,6 +1277,29 @@ private static void validateSearchOnlyReplicasSettings(Settings indexSettings) { * Also validates that mapper_settings keys are recognized for the configured mapper_type. */ static void validateIngestionSourceSettings(Settings settings, ClusterState state) { + // Partition strategy validation. The setting key itself was introduced in V_3_7_0; reject any explicit + // value (including [simple], the default) on mixed clusters where some nodes don't recognize the key. + // And in that case the index metadata replicated to older nodes would carry unknown settings. + // Also, older nodes would silently fall back to the default mapping while the user configured + // a different strategy (e.g., modulo), which might cause correctness issues. + if (IndexMetadata.INGESTION_SOURCE_PARTITION_STRATEGY_SETTING.exists(settings)) { + Version minNodeVersion = state.nodes().getMinNodeVersion(); + if (minNodeVersion.before(Version.V_3_7_0)) { + throw new IllegalArgumentException( + "index.ingestion_source.source_partition_strategy requires all nodes in the cluster to be on version [" + + Version.V_3_7_0 + + "] or later, but the minimum node version is [" + + minNodeVersion + + "]" + ); + } + // TODO: For source_partition_strategy=simple, surface a warning when numSourcePartitions > numShards + // (excess source partitions are silently never consumed) and an error when + // numSourcePartitions < numShards (shards beyond numSourcePartitions-1 fail to initialize). + // Requires consumerFactory.getSourcePartitionCount() which is added in a follow-up PR + // (multi-partition consumer factory). The check will be wired here once available. + } + if (IndexMetadata.INGESTION_SOURCE_MAPPER_TYPE_SETTING.exists(settings) == false) { return; } diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/SourcePartitionAssignment.java b/server/src/main/java/org/opensearch/indices/pollingingest/SourcePartitionAssignment.java new file mode 100644 index 0000000000000..1d9aeb42a6584 --- /dev/null +++ b/server/src/main/java/org/opensearch/indices/pollingingest/SourcePartitionAssignment.java @@ -0,0 +1,87 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices.pollingingest; + +import org.opensearch.cluster.metadata.IngestionSource.SourcePartitionStrategy; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Computes which source stream partitions a given OpenSearch shard should consume, + * based on the configured {@link SourcePartitionStrategy}. + */ +public class SourcePartitionAssignment { + + private SourcePartitionAssignment() { + // utility class + } + + /** + * Computes the list of source partition IDs that a shard should consume. + * + * @param shardId the OpenSearch shard ID + * @param numShards total number of shards in the index + * @param numSourcePartitions total number of partitions in the source stream + * @param strategy the partition assignment strategy + * @return unmodifiable list of partition IDs assigned to this shard + * @throws IllegalArgumentException if numSourcePartitions is less than numShards for SIMPLE strategy, + * or if no partitions are assigned to the shard + */ + public static List assignSourcePartitions( + int shardId, + int numShards, + int numSourcePartitions, + SourcePartitionStrategy strategy + ) { + if (numSourcePartitions <= 0) { + throw new IllegalArgumentException("Number of source partitions must be positive, got: " + numSourcePartitions); + } + assert shardId >= 0 && shardId < numShards : "Shard ID [" + shardId + "] must be >= 0 and < numShards [" + numShards + "]"; + + // TODO - support "RANGE" below when we implement https://github.com/opensearch-project/OpenSearch/issues/21267 + switch (strategy) { + case SIMPLE: + if (shardId >= numSourcePartitions) { + throw new IllegalArgumentException( + "Shard [" + + shardId + + "] cannot be assigned a partition: source has only [" + + numSourcePartitions + + "] partitions but shard ID requires partition [" + + shardId + + "]. Use source_partition_strategy=modulo to map multiple partitions per shard." + ); + } + return List.of(shardId); + + case MODULO: + if (numSourcePartitions < numShards) { + throw new IllegalArgumentException( + "Number of source partitions [" + + numSourcePartitions + + "] must be >= number of shards [" + + numShards + + "] for modulo partition strategy" + ); + } + List result = new ArrayList<>(); + for (int p = 0; p < numSourcePartitions; p++) { + if (p % numShards == shardId) { + result.add(p); + } + } + return Collections.unmodifiableList(result); + + default: + throw new IllegalArgumentException("Unsupported partition strategy: " + strategy); + } + } +} diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java index f9a1ec9bb3f7d..c2a7448fd1bac 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java @@ -108,7 +108,7 @@ public void testToString() { .setErrorStrategy(DROP) .build(); String expected = - "IngestionSource{type='type',pointer_init_reset='PointerInitReset{type='RESET_BY_OFFSET', value=1000}',error_strategy='DROP', params={key=value}, maxPollSize=1000, pollTimeout=1000, numProcessorThreads=1, blockingQueueSize=100, allActiveIngestion=false, pointerBasedLagUpdateInterval=10s, mapperType='DEFAULT', mapperSettings={}, warmupConfig=WarmupConfig[timeout=-1, lagThreshold=100]}"; + "IngestionSource{type='type',pointer_init_reset='PointerInitReset{type='RESET_BY_OFFSET', value=1000}',error_strategy='DROP', params={key=value}, maxPollSize=1000, pollTimeout=1000, numProcessorThreads=1, blockingQueueSize=100, allActiveIngestion=false, pointerBasedLagUpdateInterval=10s, mapperType='DEFAULT', mapperSettings={}, warmupConfig=WarmupConfig[timeout=-1, lagThreshold=100], sourcePartitionStrategy='simple'}"; assertEquals(expected, source.toString()); } @@ -274,4 +274,69 @@ public void testSetWarmupConfig() { assertEquals(TimeValue.timeValueMinutes(15), source.getWarmupConfig().timeout()); assertEquals(200, source.getWarmupConfig().lagThreshold()); } + + // ---- SourcePartitionStrategy enum tests ---- + + public void testSourcePartitionStrategyGetName() { + assertEquals("simple", IngestionSource.SourcePartitionStrategy.SIMPLE.getName()); + assertEquals("modulo", IngestionSource.SourcePartitionStrategy.MODULO.getName()); + } + + public void testSourcePartitionStrategyToString() { + // toString() should match getName() + assertEquals("simple", IngestionSource.SourcePartitionStrategy.SIMPLE.toString()); + assertEquals("modulo", IngestionSource.SourcePartitionStrategy.MODULO.toString()); + } + + public void testSourcePartitionStrategyFromString() { + assertEquals(IngestionSource.SourcePartitionStrategy.SIMPLE, IngestionSource.SourcePartitionStrategy.fromString("simple")); + assertEquals(IngestionSource.SourcePartitionStrategy.MODULO, IngestionSource.SourcePartitionStrategy.fromString("modulo")); + } + + public void testSourcePartitionStrategyFromStringIsCaseInsensitive() { + assertEquals(IngestionSource.SourcePartitionStrategy.SIMPLE, IngestionSource.SourcePartitionStrategy.fromString("SIMPLE")); + assertEquals(IngestionSource.SourcePartitionStrategy.SIMPLE, IngestionSource.SourcePartitionStrategy.fromString("Simple")); + assertEquals(IngestionSource.SourcePartitionStrategy.MODULO, IngestionSource.SourcePartitionStrategy.fromString("MODULO")); + assertEquals(IngestionSource.SourcePartitionStrategy.MODULO, IngestionSource.SourcePartitionStrategy.fromString("Modulo")); + } + + public void testSourcePartitionStrategyFromStringInvalid() { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> IngestionSource.SourcePartitionStrategy.fromString("unknown_strategy") + ); + assertTrue(e.getMessage().contains("Unknown partition strategy")); + } + + // ---- IngestionSource sourcePartitionStrategy field tests ---- + + public void testSourcePartitionStrategyDefault() { + // Default builder should produce SIMPLE strategy + IngestionSource source = new IngestionSource.Builder("type").build(); + assertEquals(IngestionSource.SourcePartitionStrategy.SIMPLE, source.getSourcePartitionStrategy()); + } + + public void testSourcePartitionStrategySetAndGet() { + IngestionSource source = new IngestionSource.Builder("type").setSourcePartitionStrategy( + IngestionSource.SourcePartitionStrategy.MODULO + ).build(); + assertEquals(IngestionSource.SourcePartitionStrategy.MODULO, source.getSourcePartitionStrategy()); + } + + public void testSourcePartitionStrategyAffectsEquals() { + IngestionSource simpleSource = new IngestionSource.Builder("type").setSourcePartitionStrategy( + IngestionSource.SourcePartitionStrategy.SIMPLE + ).build(); + IngestionSource moduloSource = new IngestionSource.Builder("type").setSourcePartitionStrategy( + IngestionSource.SourcePartitionStrategy.MODULO + ).build(); + assertNotEquals(simpleSource, moduloSource); + assertNotEquals(simpleSource.hashCode(), moduloSource.hashCode()); + + IngestionSource moduloSource2 = new IngestionSource.Builder("type").setSourcePartitionStrategy( + IngestionSource.SourcePartitionStrategy.MODULO + ).build(); + assertEquals(moduloSource, moduloSource2); + assertEquals(moduloSource.hashCode(), moduloSource2.hashCode()); + } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 4a59a77654d05..8a84e59db1ab1 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -3912,4 +3912,84 @@ public void testValidateIngestionSourceSettingsDeleteAndCreateValueSame() { assertTrue(e.getMessage().contains("cannot be the same")); } + // ---- source_partition_strategy validation tests ---- + + public void testValidateIngestionSourceSettingsPartitionStrategyOnCurrentVersion() { + // source_partition_strategy explicitly set on a current-version cluster — should pass + DiscoveryNodes nodes = DiscoveryNodes.builder().add(newNode("node1")).build(); + ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes).build(); + + Settings settings = Settings.builder().put(IndexMetadata.SETTING_INGESTION_SOURCE_PARTITION_STRATEGY, "modulo").build(); + + // Should not throw + MetadataCreateIndexService.validateIngestionSourceSettings(settings, state); + } + + public void testValidateIngestionSourceSettingsPartitionStrategySimpleOnCurrentVersion() { + // Even setting the default value (simple) explicitly should pass on current-version cluster + DiscoveryNodes nodes = DiscoveryNodes.builder().add(newNode("node1")).build(); + ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes).build(); + + Settings settings = Settings.builder().put(IndexMetadata.SETTING_INGESTION_SOURCE_PARTITION_STRATEGY, "simple").build(); + + MetadataCreateIndexService.validateIngestionSourceSettings(settings, state); + } + + public void testValidateIngestionSourceSettingsPartitionStrategyOnMixedClusterRejected() { + // source_partition_strategy setting key was introduced in V_3_7_0. Any explicit value (including + // the default 'simple') should be rejected if the cluster has nodes < V_3_7_0 — otherwise + // those nodes would receive replicated index metadata containing an unknown setting key. + final Set roles = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE)) + ); + DiscoveryNode oldNode = new DiscoveryNode("old_node", buildNewFakeTransportAddress(), emptyMap(), roles, Version.V_3_5_0); + DiscoveryNodes nodes = DiscoveryNodes.builder().add(newNode("node1")).add(oldNode).build(); + ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes).build(); + + Settings settings = Settings.builder().put(IndexMetadata.SETTING_INGESTION_SOURCE_PARTITION_STRATEGY, "modulo").build(); + + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> MetadataCreateIndexService.validateIngestionSourceSettings(settings, state) + ); + assertTrue(e.getMessage().contains("index.ingestion_source.source_partition_strategy requires all nodes")); + assertTrue(e.getMessage().contains(Version.V_3_7_0.toString())); + assertTrue(e.getMessage().contains(Version.V_3_5_0.toString())); + } + + public void testValidateIngestionSourceSettingsPartitionStrategySimpleAlsoRejectedOnMixedCluster() { + // Even the default value 'simple' set explicitly is rejected on a mixed cluster — the version + // check guards the setting KEY itself, regardless of value. Once any non-default strategy can + // be set, older nodes that don't recognize the key would fall back to the default 1:1 mapping + // and read from the wrong source partitions until upgraded. + final Set roles = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE)) + ); + DiscoveryNode oldNode = new DiscoveryNode("old_node", buildNewFakeTransportAddress(), emptyMap(), roles, Version.V_3_5_0); + DiscoveryNodes nodes = DiscoveryNodes.builder().add(newNode("node1")).add(oldNode).build(); + ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes).build(); + + Settings settings = Settings.builder().put(IndexMetadata.SETTING_INGESTION_SOURCE_PARTITION_STRATEGY, "simple").build(); + + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> MetadataCreateIndexService.validateIngestionSourceSettings(settings, state) + ); + assertTrue(e.getMessage().contains("index.ingestion_source.source_partition_strategy requires all nodes")); + } + + public void testValidateIngestionSourceSettingsPartitionStrategyAbsentOnMixedClusterPasses() { + // Without the explicit source_partition_strategy setting, no metadata is replicated — old nodes are unaffected. + final Set roles = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE)) + ); + DiscoveryNode oldNode = new DiscoveryNode("old_node", buildNewFakeTransportAddress(), emptyMap(), roles, Version.V_3_5_0); + DiscoveryNodes nodes = DiscoveryNodes.builder().add(newNode("node1")).add(oldNode).build(); + ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes).build(); + + // No source_partition_strategy in settings — validation should pass even on mixed cluster + Settings settings = Settings.builder().build(); + + MetadataCreateIndexService.validateIngestionSourceSettings(settings, state); + } } diff --git a/server/src/test/java/org/opensearch/indices/pollingingest/SourcePartitionAssignmentTests.java b/server/src/test/java/org/opensearch/indices/pollingingest/SourcePartitionAssignmentTests.java new file mode 100644 index 0000000000000..1ed8efc5f1151 --- /dev/null +++ b/server/src/test/java/org/opensearch/indices/pollingingest/SourcePartitionAssignmentTests.java @@ -0,0 +1,146 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices.pollingingest; + +import org.opensearch.cluster.metadata.IngestionSource.SourcePartitionStrategy; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +public class SourcePartitionAssignmentTests extends OpenSearchTestCase { + + // --- SIMPLE strategy tests --- + + public void testSimpleStrategy_OneToOneMapping() { + List partitions = SourcePartitionAssignment.assignSourcePartitions(0, 4, 4, SourcePartitionStrategy.SIMPLE); + assertEquals(List.of(0), partitions); + + partitions = SourcePartitionAssignment.assignSourcePartitions(3, 4, 4, SourcePartitionStrategy.SIMPLE); + assertEquals(List.of(3), partitions); + } + + public void testSimpleStrategy_MorePartitionsThanShards() { + // shard 0 still gets partition 0, even if there are more partitions + List partitions = SourcePartitionAssignment.assignSourcePartitions(0, 4, 64, SourcePartitionStrategy.SIMPLE); + assertEquals(List.of(0), partitions); + } + + public void testSimpleStrategy_ShardIdExceedsPartitionCount() { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> SourcePartitionAssignment.assignSourcePartitions(4, 8, 4, SourcePartitionStrategy.SIMPLE) + ); + assertTrue(e.getMessage().contains("cannot be assigned a partition")); + assertTrue(e.getMessage().contains("Use source_partition_strategy=modulo")); + } + + // --- MODULO strategy tests --- + + public void testModuloStrategy_EqualPartitionsAndShards() { + // 4 partitions, 4 shards → each shard gets exactly 1 partition (same as simple) + for (int s = 0; s < 4; s++) { + List partitions = SourcePartitionAssignment.assignSourcePartitions(s, 4, 4, SourcePartitionStrategy.MODULO); + assertEquals(List.of(s), partitions); + } + } + + public void testModuloStrategy_DoublePartitions() { + // 8 partitions, 4 shards → each shard gets 2 partitions + assertEquals(List.of(0, 4), SourcePartitionAssignment.assignSourcePartitions(0, 4, 8, SourcePartitionStrategy.MODULO)); + assertEquals(List.of(1, 5), SourcePartitionAssignment.assignSourcePartitions(1, 4, 8, SourcePartitionStrategy.MODULO)); + assertEquals(List.of(2, 6), SourcePartitionAssignment.assignSourcePartitions(2, 4, 8, SourcePartitionStrategy.MODULO)); + assertEquals(List.of(3, 7), SourcePartitionAssignment.assignSourcePartitions(3, 4, 8, SourcePartitionStrategy.MODULO)); + } + + public void testModuloStrategy_ManyPartitions() { + // 64 partitions, 4 shards → each shard gets 16 partitions + List shard0 = SourcePartitionAssignment.assignSourcePartitions(0, 4, 64, SourcePartitionStrategy.MODULO); + assertEquals(16, shard0.size()); + assertEquals(0, (int) shard0.get(0)); + assertEquals(4, (int) shard0.get(1)); + assertEquals(60, (int) shard0.get(15)); + + List shard3 = SourcePartitionAssignment.assignSourcePartitions(3, 4, 64, SourcePartitionStrategy.MODULO); + assertEquals(16, shard3.size()); + assertEquals(3, (int) shard3.get(0)); + assertEquals(63, (int) shard3.get(15)); + } + + public void testModuloStrategy_SingleShard() { + // 1 shard → consumes ALL partitions + List partitions = SourcePartitionAssignment.assignSourcePartitions(0, 1, 64, SourcePartitionStrategy.MODULO); + assertEquals(64, partitions.size()); + for (int i = 0; i < 64; i++) { + assertEquals(i, (int) partitions.get(i)); + } + } + + public void testModuloStrategy_UnevenDistribution() { + // 5 partitions, 3 shards → uneven (shard 0 gets [0,3], shard 1 gets [1,4], shard 2 gets [2]) + assertEquals(List.of(0, 3), SourcePartitionAssignment.assignSourcePartitions(0, 3, 5, SourcePartitionStrategy.MODULO)); + assertEquals(List.of(1, 4), SourcePartitionAssignment.assignSourcePartitions(1, 3, 5, SourcePartitionStrategy.MODULO)); + assertEquals(List.of(2), SourcePartitionAssignment.assignSourcePartitions(2, 3, 5, SourcePartitionStrategy.MODULO)); + } + + public void testModuloStrategy_FewerPartitionsThanShards() { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> SourcePartitionAssignment.assignSourcePartitions(0, 8, 4, SourcePartitionStrategy.MODULO) + ); + assertTrue(e.getMessage().contains("must be >= number of shards")); + } + + // --- Error cases --- + + public void testInvalidShardId() { + AssertionError e = expectThrows( + AssertionError.class, + () -> SourcePartitionAssignment.assignSourcePartitions(-1, 4, 8, SourcePartitionStrategy.MODULO) + ); + assertTrue(e.getMessage().contains("Shard ID")); + } + + public void testZeroSourcePartitions() { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> SourcePartitionAssignment.assignSourcePartitions(0, 4, 0, SourcePartitionStrategy.MODULO) + ); + assertTrue(e.getMessage().contains("must be positive")); + } + + public void testResultIsUnmodifiable() { + List partitions = SourcePartitionAssignment.assignSourcePartitions(0, 4, 64, SourcePartitionStrategy.MODULO); + expectThrows(UnsupportedOperationException.class, () -> partitions.add(99)); + } + + // --- All partitions are covered (completeness check) --- + + public void testAllPartitionsCovered() { + int numShards = 4; + int numPartitions = 64; + boolean[] covered = new boolean[numPartitions]; + + for (int s = 0; s < numShards; s++) { + List assigned = SourcePartitionAssignment.assignSourcePartitions( + s, + numShards, + numPartitions, + SourcePartitionStrategy.MODULO + ); + for (int p : assigned) { + assertFalse("Partition " + p + " assigned to multiple shards", covered[p]); + covered[p] = true; + } + } + + for (int p = 0; p < numPartitions; p++) { + assertTrue("Partition " + p + " not assigned to any shard", covered[p]); + } + } +} From 79a799456ec862b55cc9dc4ba7e4846a45000614 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Sat, 2 May 2026 17:03:17 -0500 Subject: [PATCH 030/115] Fix flaky MergeStatsIT warmer-counter assertions (#21461) The three tests in MergeStatsIT kicked off forceMerge() without awaiting the returned ActionFuture, then read cross-node MergedSegmentWarmerStats counters in a single shot. Example failure [here][1]. Await forceMerge, and wrap the stats collection and assertions in assertBusy so the counter comparison retries until the warmer push has landed. For testShardStats the result map is now re-initialised inside the assertBusy body so a partial earlier attempt cannot corrupt the next comparison. [1]: https://build.ci.opensearch.org/job/gradle-check/75593/testReport/junit/org.opensearch.merge/MergeStatsIT/testShardStats/ Signed-off-by: Andrew Ross --- .../org/opensearch/merge/MergeStatsIT.java | 215 ++++++++++-------- 1 file changed, 117 insertions(+), 98 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/merge/MergeStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/merge/MergeStatsIT.java index 0cd6a1fb4e149..c929b82c8ed75 100644 --- a/server/src/internalClusterTest/java/org/opensearch/merge/MergeStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/merge/MergeStatsIT.java @@ -75,38 +75,44 @@ public void testNodesStats() throws Exception { ClusterState state = getClusterState(); List nodes = state.nodes().getNodes().values().stream().map(DiscoveryNode::getName).toList(); - // ensure merge is executed + // Wait for the force merge itself to finish. The warmer push to the replica is triggered + // during the merge but its receive-side accounting on the replica completes asynchronously, + // so we still need to poll the cross-node counters below. for (String index : indices) { - client().admin().indices().forceMerge(new ForceMergeRequest(index).maxNumSegments(2)); + client().admin().indices().forceMerge(new ForceMergeRequest(index).maxNumSegments(2)).get(); } final NodesStatsRequest nodesStatsRequest = new NodesStatsRequest("data:true"); nodesStatsRequest.indices(CommonStatsFlags.ALL); for (String node : nodes) { - NodesStatsResponse response = client(node).admin().cluster().nodesStats(nodesStatsRequest).get(); - - // Shard stats - List allNodesStats = response.getNodes(); - assertEquals(2, allNodesStats.size()); - for (NodeStats nodeStats : allNodesStats) { - assertNotNull(nodeStats.getIndices()); - MergeStats mergeStats = nodeStats.getIndices().getMerge(); - assertNotNull(mergeStats); - assertMergeStats(mergeStats, StatsScope.AGGREGATED); - MergedSegmentWarmerStats mergedSegmentWarmerStats = mergeStats.getWarmerStats(); - assertNotNull(mergedSegmentWarmerStats); - assertMergedSegmentWarmerStats(mergedSegmentWarmerStats, StatsScope.AGGREGATED); - } + assertBusy(() -> { + NodesStatsResponse response = client(node).admin().cluster().nodesStats(nodesStatsRequest).get(); + + List allNodesStats = response.getNodes(); + assertEquals(2, allNodesStats.size()); + for (NodeStats nodeStats : allNodesStats) { + assertNotNull(nodeStats.getIndices()); + MergeStats mergeStats = nodeStats.getIndices().getMerge(); + assertNotNull(mergeStats); + assertMergeStats(mergeStats, StatsScope.AGGREGATED); + MergedSegmentWarmerStats mergedSegmentWarmerStats = mergeStats.getWarmerStats(); + assertNotNull(mergedSegmentWarmerStats); + assertMergedSegmentWarmerStats(mergedSegmentWarmerStats, StatsScope.AGGREGATED); + } - assertEquals( - "Expected sent size by node 2 to be equal to recieved size by node 1.", - allNodesStats.get(0).getIndices().getMerge().getWarmerStats().getTotalReceivedSize(), - allNodesStats.get(1).getIndices().getMerge().getWarmerStats().getTotalSentSize() - ); - assertEquals( - "Expected sent size by node 1 to be equal to recieved size by node 2.", - allNodesStats.get(0).getIndices().getMerge().getWarmerStats().getTotalSentSize(), - allNodesStats.get(1).getIndices().getMerge().getWarmerStats().getTotalReceivedSize() - ); + // Primary-sent and replica-received byte counters are maintained on different + // nodes and updated by different callbacks in the warmer flow, so they only + // reconcile once the async warmer push has fully completed on both sides. + assertEquals( + "Expected sent size by node 2 to be equal to recieved size by node 1.", + allNodesStats.get(0).getIndices().getMerge().getWarmerStats().getTotalReceivedSize(), + allNodesStats.get(1).getIndices().getMerge().getWarmerStats().getTotalSentSize() + ); + assertEquals( + "Expected sent size by node 1 to be equal to recieved size by node 2.", + allNodesStats.get(0).getIndices().getMerge().getWarmerStats().getTotalSentSize(), + allNodesStats.get(1).getIndices().getMerge().getWarmerStats().getTotalReceivedSize() + ); + }, 30, TimeUnit.SECONDS); } } @@ -118,52 +124,61 @@ public void testShardStats() throws Exception { ClusterState state = getClusterState(); List nodes = state.nodes().getNodes().values().stream().map(DiscoveryNode::getName).toList(); - // ensure merge is executed + // Wait for the force merge itself to finish. The warmer push to the replica is triggered + // during the merge but its receive-side accounting on the replica completes asynchronously, + // so we still need to poll the cross-shard counters below. for (String index : indices) { - client().admin().indices().forceMerge(new ForceMergeRequest(index).maxNumSegments(2)); + client().admin().indices().forceMerge(new ForceMergeRequest(index).maxNumSegments(2)).get(); } - Map> shardsSentAndReceivedSize = new HashMap<>(); - for (String node : nodes) { - IndicesStatsResponse response = client(node).admin().indices().stats(new IndicesStatsRequest()).get(); - - // Shard stats - ShardStats[] allShardStats = response.getShards(); - assertEquals(4, allShardStats.length); - - for (ShardStats shardStats : allShardStats) { - StatsScope type = shardStats.getShardRouting().primary() ? StatsScope.PRIMARY_SHARD : StatsScope.REPLICA_SHARD; - CommonStats commonStats = shardStats.getStats(); - assertNotNull(commonStats); - MergeStats mergeStats = commonStats.getMerge(); - assertNotNull(mergeStats); - assertMergeStats(mergeStats, type); - MergedSegmentWarmerStats mergedSegmentWarmerStats = mergeStats.getWarmerStats(); - assertNotNull(mergedSegmentWarmerStats); - assertMergedSegmentWarmerStats(mergedSegmentWarmerStats, type); - - String primaryOrReplica = type.equals(StatsScope.PRIMARY_SHARD) ? "[P]" : "[R]"; - shardsSentAndReceivedSize.put(shardStats.getShardRouting().shardId() + primaryOrReplica, new HashMap<>() { - { - put("RECEIVED", mergedSegmentWarmerStats.getTotalReceivedSize()); - put("SENT", mergedSegmentWarmerStats.getTotalSentSize()); - } - }); + assertBusy(() -> { + // Re-collect stats on every attempt; the primary-sent and replica-received byte + // counters are maintained on different nodes and updated by different callbacks + // in the warmer flow, so they only reconcile once the async warmer push has + // fully completed on both sides. + Map> shardsSentAndReceivedSize = new HashMap<>(); + + for (String node : nodes) { + IndicesStatsResponse response = client(node).admin().indices().stats(new IndicesStatsRequest()).get(); + + // Shard stats + ShardStats[] allShardStats = response.getShards(); + assertEquals(4, allShardStats.length); + + for (ShardStats shardStats : allShardStats) { + StatsScope type = shardStats.getShardRouting().primary() ? StatsScope.PRIMARY_SHARD : StatsScope.REPLICA_SHARD; + CommonStats commonStats = shardStats.getStats(); + assertNotNull(commonStats); + MergeStats mergeStats = commonStats.getMerge(); + assertNotNull(mergeStats); + assertMergeStats(mergeStats, type); + MergedSegmentWarmerStats mergedSegmentWarmerStats = mergeStats.getWarmerStats(); + assertNotNull(mergedSegmentWarmerStats); + assertMergedSegmentWarmerStats(mergedSegmentWarmerStats, type); + + String primaryOrReplica = type.equals(StatsScope.PRIMARY_SHARD) ? "[P]" : "[R]"; + shardsSentAndReceivedSize.put(shardStats.getShardRouting().shardId() + primaryOrReplica, new HashMap<>() { + { + put("RECEIVED", mergedSegmentWarmerStats.getTotalReceivedSize()); + put("SENT", mergedSegmentWarmerStats.getTotalSentSize()); + } + }); + } } - } - for (int shard = 0; shard <= 1; shard++) { - assertEquals( - "Expected sent size by primary shard to be equal to recieved size by replica shard.", - shardsSentAndReceivedSize.get("[" + indices[0] + "][" + shard + "][R]").get("RECEIVED"), - shardsSentAndReceivedSize.get("[" + indices[0] + "][" + shard + "][P]").get("SENT") - ); - assertEquals( - "Expected sent size by replica shard to be equal to recieved size by primary shard.", - shardsSentAndReceivedSize.get("[" + indices[0] + "][" + shard + "][R]").get("SENT"), - shardsSentAndReceivedSize.get("[" + indices[0] + "][" + shard + "][P]").get("RECEIVED") - ); - } + for (int shard = 0; shard <= 1; shard++) { + assertEquals( + "Expected sent size by primary shard to be equal to recieved size by replica shard.", + shardsSentAndReceivedSize.get("[" + indices[0] + "][" + shard + "][R]").get("RECEIVED"), + shardsSentAndReceivedSize.get("[" + indices[0] + "][" + shard + "][P]").get("SENT") + ); + assertEquals( + "Expected sent size by replica shard to be equal to recieved size by primary shard.", + shardsSentAndReceivedSize.get("[" + indices[0] + "][" + shard + "][R]").get("SENT"), + shardsSentAndReceivedSize.get("[" + indices[0] + "][" + shard + "][P]").get("RECEIVED") + ); + } + }, 30, TimeUnit.SECONDS); } public void testIndicesStats() throws Exception { @@ -173,41 +188,45 @@ public void testIndicesStats() throws Exception { ClusterState state = getClusterState(); List nodes = state.nodes().getNodes().values().stream().map(DiscoveryNode::getName).toList(); - // ensure merge is executed + // Wait for the force merge itself to finish. The warmer push to the replica is triggered + // during the merge but its receive-side accounting on the replica completes asynchronously, + // so we still need to poll the aggregated warmer counters below. for (String index : indices) { - client().admin().indices().forceMerge(new ForceMergeRequest(index).maxNumSegments(2)); + client().admin().indices().forceMerge(new ForceMergeRequest(index).maxNumSegments(2)).get(); } for (String node : nodes) { - IndicesStatsResponse response = client(node).admin().indices().stats(new IndicesStatsRequest()).get(); - - // Shard stats - Map allIndicesStats = response.getIndices(); - assertEquals(1, allIndicesStats.size()); - for (String index : indices) { - IndexStats indexStats = allIndicesStats.get(index); - CommonStats totalStats = indexStats.getTotal(); - CommonStats priStats = indexStats.getPrimaries(); - assertNotNull(totalStats); - assertNotNull(priStats); - - MergeStats totalMergeStats = totalStats.getMerge(); - assertNotNull(totalMergeStats); - MergeStats priMergeStats = priStats.getMerge(); - assertNotNull(priMergeStats); - - assertMergeStats(priMergeStats, StatsScope.PRIMARY_SHARD); - assertMergeStats(totalMergeStats, StatsScope.AGGREGATED); - - MergedSegmentWarmerStats totalMergedSegmentWarmerStats = totalMergeStats.getWarmerStats(); - MergedSegmentWarmerStats priMergedSegmentWarmerStats = priMergeStats.getWarmerStats(); - - assertNotNull(totalMergedSegmentWarmerStats); - assertNotNull(priMergedSegmentWarmerStats); - - assertMergedSegmentWarmerStats(priMergedSegmentWarmerStats, StatsScope.PRIMARY_SHARD); - assertMergedSegmentWarmerStats(totalMergedSegmentWarmerStats, StatsScope.AGGREGATED); - } + assertBusy(() -> { + IndicesStatsResponse response = client(node).admin().indices().stats(new IndicesStatsRequest()).get(); + + // Shard stats + Map allIndicesStats = response.getIndices(); + assertEquals(1, allIndicesStats.size()); + for (String index : indices) { + IndexStats indexStats = allIndicesStats.get(index); + CommonStats totalStats = indexStats.getTotal(); + CommonStats priStats = indexStats.getPrimaries(); + assertNotNull(totalStats); + assertNotNull(priStats); + + MergeStats totalMergeStats = totalStats.getMerge(); + assertNotNull(totalMergeStats); + MergeStats priMergeStats = priStats.getMerge(); + assertNotNull(priMergeStats); + + assertMergeStats(priMergeStats, StatsScope.PRIMARY_SHARD); + assertMergeStats(totalMergeStats, StatsScope.AGGREGATED); + + MergedSegmentWarmerStats totalMergedSegmentWarmerStats = totalMergeStats.getWarmerStats(); + MergedSegmentWarmerStats priMergedSegmentWarmerStats = priMergeStats.getWarmerStats(); + + assertNotNull(totalMergedSegmentWarmerStats); + assertNotNull(priMergedSegmentWarmerStats); + + assertMergedSegmentWarmerStats(priMergedSegmentWarmerStats, StatsScope.PRIMARY_SHARD); + assertMergedSegmentWarmerStats(totalMergedSegmentWarmerStats, StatsScope.AGGREGATED); + } + }, 30, TimeUnit.SECONDS); } } From c23afc2f4d9f9b1d3553b062549d668342d52b18 Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Sun, 3 May 2026 00:10:14 -0700 Subject: [PATCH 031/115] Add cancellation checkpoints in field data loading and aggregation paths (#21318) Signed-off-by: Kaushal Kumar Signed-off-by: Kaushal Kumar Co-authored-by: Ankit Jain --- .../ordinals/GlobalOrdinalsBuilder.java | 107 +++++++++- .../aggregations/AggregatorFactories.java | 4 + .../internal/ExitableDirectoryReader.java | 80 ++++++++ .../ordinals/GlobalOrdinalsBuilderTests.java | 192 ++++++++++++++++++ .../search/SearchCancellationTests.java | 68 +++++++ .../AggregatorFactoriesCancellationTests.java | 81 ++++++++ 6 files changed, 531 insertions(+), 1 deletion(-) create mode 100644 server/src/test/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilderTests.java create mode 100644 server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesCancellationTests.java diff --git a/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java index 06219b69266b5..9b9fe1b9f5789 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java +++ b/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java @@ -34,10 +34,13 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.OrdinalMap; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.packed.PackedInts; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.breaker.CircuitBreaker; @@ -70,17 +73,39 @@ public static IndexOrdinalsFieldData build( CircuitBreakerService breakerService, Logger logger, Function> scriptFunction + ) throws IOException { + return build(indexReader, indexFieldData, breakerService, logger, scriptFunction, () -> {}); + } + + /** + * Build global ordinals for the provided {@link IndexReader}, with periodic cancellation checks + * between segment iterations. + */ + public static IndexOrdinalsFieldData build( + final IndexReader indexReader, + IndexOrdinalsFieldData indexFieldData, + CircuitBreakerService breakerService, + Logger logger, + Function> scriptFunction, + Runnable cancellationCheck ) throws IOException { assert indexReader.leaves().size() > 1; long startTimeNS = System.nanoTime(); final LeafOrdinalsFieldData[] atomicFD = new LeafOrdinalsFieldData[indexReader.leaves().size()]; final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()]; + // cancellableSubs wraps each segment's SortedSetDocValues with a cancellation-aware termsEnum() + // for OrdinalMap.build(), which only calls termsEnum() and getValueCount(). + // atomicFD retains the original unwrapped values to preserve SingletonSortedSetDocValues + // type for DocValues.unwrapSingleton(). + final SortedSetDocValues[] cancellableSubs = new SortedSetDocValues[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { + cancellationCheck.run(); atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i)); subs[i] = atomicFD[i].getOrdinalsValues(); + cancellableSubs[i] = new CancellableTermsSortedSetDocValues(subs[i], cancellationCheck); } - final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); + final OrdinalMap ordinalMap = OrdinalMap.build(null, cancellableSubs, PackedInts.DEFAULT); final long memorySizeInBytes = ordinalMap.ramBytesUsed(); breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(memorySizeInBytes); @@ -140,4 +165,84 @@ public void close() {} ); } + /** + * Thin wrapper around {@link SortedSetDocValues} that adds cancellation checks + * to {@link #termsEnum()} iteration. Used only for the {@code subs} array passed + * to {@link OrdinalMap#build}, which only calls {@link #termsEnum()} and + * {@link #getValueCount()}. This avoids wrapping the stored field data values + * which must preserve their concrete type for {@code DocValues.unwrapSingleton()}. + */ + private static class CancellableTermsSortedSetDocValues extends SortedSetDocValues { + private final SortedSetDocValues in; + private final Runnable cancellationCheck; + + CancellableTermsSortedSetDocValues(SortedSetDocValues in, Runnable cancellationCheck) { + this.in = in; + this.cancellationCheck = cancellationCheck; + } + + @Override + public TermsEnum termsEnum() throws IOException { + TermsEnum te = in.termsEnum(); + return new FilterLeafReader.FilterTermsEnum(te) { + private static final int CHECK_INTERVAL = (1 << 10) - 1; // 1023 + private int calls; + + @Override + public BytesRef next() throws IOException { + if ((calls++ & CHECK_INTERVAL) == 0) { + cancellationCheck.run(); + } + return in.next(); + } + }; + } + + @Override + public long getValueCount() { + return in.getValueCount(); + } + + // Methods below are required by SortedSetDocValues but not called by OrdinalMap.build() + @Override + public int nextDoc() throws IOException { + return in.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + return in.advance(target); + } + + @Override + public boolean advanceExact(int target) throws IOException { + return in.advanceExact(target); + } + + @Override + public long nextOrd() throws IOException { + return in.nextOrd(); + } + + @Override + public int docValueCount() { + return in.docValueCount(); + } + + @Override + public BytesRef lookupOrd(long ord) throws IOException { + return in.lookupOrd(ord); + } + + @Override + public int docID() { + return in.docID(); + } + + @Override + public long cost() { + return in.cost(); + } + } + } diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java b/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java index 65c9eafbbe328..2f8c2299ebd78 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java @@ -41,6 +41,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.tasks.TaskCancelledException; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedObjectNotFoundException; import org.opensearch.core.xcontent.ToXContentObject; @@ -340,6 +341,9 @@ private List createTopLevelAggregators(SearchContext searchContext, // These aggregators are going to be used with a single bucket ordinal, no need to wrap the PER_BUCKET ones List aggregators = new ArrayList<>(); for (int i = 0; i < factories.length; i++) { + if (searchContext.isCancelled()) { + throw new TaskCancelledException("cancelled while creating aggregators"); + } /* * Top level aggs only collect from owningBucketOrd 0 which is * *exactly* what CardinalityUpperBound.ONE *means*. diff --git a/server/src/main/java/org/opensearch/search/internal/ExitableDirectoryReader.java b/server/src/main/java/org/opensearch/search/internal/ExitableDirectoryReader.java index 1c737bc9b5891..2434416285183 100644 --- a/server/src/main/java/org/opensearch/search/internal/ExitableDirectoryReader.java +++ b/server/src/main/java/org/opensearch/search/internal/ExitableDirectoryReader.java @@ -39,6 +39,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.suggest.document.CompletionTerms; @@ -210,6 +211,85 @@ public BytesRef next() throws IOException { checkAndThrowWithSampling(); return in.next(); } + + @Override + public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + // Don't reuse when wrapping, since the wrapper type differs from the delegate type + final PostingsEnum postings = in.postings(null, flags); + return new ExitablePostingsEnum(postings, queryCancellation); + } + } + + /** + * Wrapper class for {@link PostingsEnum} that checks for query cancellation or timeout + * during document iteration. This closes the gap where field data loading iterates + * postings (e.g., {@code OrdinalsBuilder.addDoc()}) without cancellation checks. + */ + private static class ExitablePostingsEnum extends PostingsEnum { + + private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 13) - 1; // 8191 + + private final PostingsEnum in; + private final QueryCancellation queryCancellation; + private int calls; + + private ExitablePostingsEnum(PostingsEnum in, QueryCancellation queryCancellation) { + this.in = in; + this.queryCancellation = queryCancellation; + } + + private void checkAndThrowWithSampling() { + if ((calls++ & MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) { + queryCancellation.checkCancelled(); + } + } + + @Override + public int nextDoc() throws IOException { + checkAndThrowWithSampling(); + return in.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + queryCancellation.checkCancelled(); + return in.advance(target); + } + + @Override + public int docID() { + return in.docID(); + } + + @Override + public long cost() { + return in.cost(); + } + + @Override + public int freq() throws IOException { + return in.freq(); + } + + @Override + public int nextPosition() throws IOException { + return in.nextPosition(); + } + + @Override + public int startOffset() throws IOException { + return in.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return in.endOffset(); + } + + @Override + public BytesRef getPayload() throws IOException { + return in.getPayload(); + } } // delegates to PointValues but adds query cancellation checks diff --git a/server/src/test/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilderTests.java b/server/src/test/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilderTests.java new file mode 100644 index 0000000000000..111b36e0a76d1 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilderTests.java @@ -0,0 +1,192 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.fielddata.ordinals; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.tasks.TaskCancelledException; +import org.opensearch.index.fielddata.IndexOrdinalsFieldData; +import org.opensearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class GlobalOrdinalsBuilderTests extends OpenSearchTestCase { + + public void testBuildWithCancellationBetweenSegments() throws IOException { + try (Directory dir = newDirectory()) { + RandomIndexWriter w = new RandomIndexWriter(random(), dir); + w.w.getConfig().setMergePolicy(NoMergePolicy.INSTANCE); + + // Create 3 segments with distinct terms + for (int seg = 0; seg < 3; seg++) { + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(new StringField("field", "seg" + seg + "_term" + i, Field.Store.NO)); + w.addDocument(doc); + } + w.flush(); + } + + try (IndexReader reader = w.getReader()) { + w.close(); + assertTrue("Need multiple segments for global ordinals", reader.leaves().size() > 1); + + IndexOrdinalsFieldData fieldData = mockFieldData("field", reader); + + // Build without cancellation — should succeed + assertNotNull( + GlobalOrdinalsBuilder.build( + reader, + fieldData, + new NoneCircuitBreakerService(), + logger, + AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION, + () -> {} + ) + ); + + // Build with immediate cancellation — should throw between segments + expectThrows( + TaskCancelledException.class, + () -> GlobalOrdinalsBuilder.build( + reader, + fieldData, + new NoneCircuitBreakerService(), + logger, + AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION, + () -> { + throw new TaskCancelledException("cancelled"); + } + ) + ); + } + } + } + + public void testBuildWithDelayedCancellation() throws IOException { + try (Directory dir = newDirectory()) { + RandomIndexWriter w = new RandomIndexWriter(random(), dir); + w.w.getConfig().setMergePolicy(NoMergePolicy.INSTANCE); + + for (int seg = 0; seg < 3; seg++) { + Document doc = new Document(); + doc.add(new StringField("field", "term" + seg, Field.Store.NO)); + w.addDocument(doc); + w.flush(); + } + + try (IndexReader reader = w.getReader()) { + w.close(); + assertTrue(reader.leaves().size() > 1); + + IndexOrdinalsFieldData fieldData = mockFieldData("field", reader); + + // Cancel after first segment — should still throw + AtomicBoolean cancelled = new AtomicBoolean(false); + expectThrows( + TaskCancelledException.class, + () -> GlobalOrdinalsBuilder.build( + reader, + fieldData, + new NoneCircuitBreakerService(), + logger, + AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION, + () -> { + if (cancelled.get()) { + throw new TaskCancelledException("cancelled after first segment"); + } + cancelled.set(true); // arm cancellation after first check passes + } + ) + ); + } + } + } + + public void testOriginalBuildMethodStillWorks() throws IOException { + try (Directory dir = newDirectory()) { + RandomIndexWriter w = new RandomIndexWriter(random(), dir); + w.w.getConfig().setMergePolicy(NoMergePolicy.INSTANCE); + + for (int seg = 0; seg < 2; seg++) { + Document doc = new Document(); + doc.add(new StringField("field", "term" + seg, Field.Store.NO)); + w.addDocument(doc); + w.flush(); + } + + try (IndexReader reader = w.getReader()) { + w.close(); + assertTrue(reader.leaves().size() > 1); + + IndexOrdinalsFieldData fieldData = mockFieldData("field", reader); + + // Original method (no Runnable param) should still work + assertNotNull( + GlobalOrdinalsBuilder.build( + reader, + fieldData, + new NoneCircuitBreakerService(), + logger, + AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION + ) + ); + } + } + } + + private static IndexOrdinalsFieldData mockFieldData(String fieldName, IndexReader reader) { + IndexOrdinalsFieldData fieldData = mock(IndexOrdinalsFieldData.class); + when(fieldData.getFieldName()).thenReturn(fieldName); + when(fieldData.load(any(LeafReaderContext.class))).thenAnswer(invocation -> { + LeafReaderContext ctx = invocation.getArgument(0); + return new AbstractLeafOrdinalsFieldData(AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION) { + @Override + public SortedSetDocValues getOrdinalsValues() { + try { + SortedSetDocValues dv = ctx.reader().getSortedSetDocValues(fieldName); + return dv != null ? dv : DocValues.emptySortedSet(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public java.util.Collection getChildResources() { + return Collections.emptyList(); + } + + @Override + public void close() {} + }; + }); + return fieldData; + } +} diff --git a/server/src/test/java/org/opensearch/search/SearchCancellationTests.java b/server/src/test/java/org/opensearch/search/SearchCancellationTests.java index a7a2a9ed11b19..d022486e80bc1 100644 --- a/server/src/test/java/org/opensearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/opensearch/search/SearchCancellationTests.java @@ -34,18 +34,22 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.TestUtil; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.util.io.IOUtils; @@ -70,6 +74,7 @@ public class SearchCancellationTests extends OpenSearchTestCase { private static final String STRING_FIELD_NAME = "foo"; private static final String POINT_FIELD_NAME = "point"; + private static final String SORTED_SET_FIELD_NAME = "sorted_set"; private static Directory dir; private static IndexReader reader; @@ -98,6 +103,7 @@ private static void indexRandomDocuments(RandomIndexWriter w, int numDocs) throw } doc.add(new StringField(STRING_FIELD_NAME, sb.toString(), Field.Store.NO)); doc.add(new IntPoint(POINT_FIELD_NAME, i, i + 1)); + doc.add(new SortedSetDocValuesField(SORTED_SET_FIELD_NAME, new BytesRef(sb.toString()))); w.addDocument(doc); } } @@ -229,6 +235,68 @@ public void testExitableDirectoryReader() throws IOException { pointValues2.intersect(new PointValuesIntersectVisitor()); } + public void testExitablePostingsEnum() throws IOException { + AtomicBoolean cancelled = new AtomicBoolean(false); + Runnable cancellation = () -> { + if (cancelled.get()) { + throw new TaskCancelledException("cancelled"); + } + }; + ContextIndexSearcher searcher = new ContextIndexSearcher( + reader, + IndexSearcher.getDefaultSimilarity(), + IndexSearcher.getDefaultQueryCache(), + IndexSearcher.getDefaultQueryCachingPolicy(), + true, + null, + searchContext + ); + searcher.addQueryCancellation(cancellation); + + // Get terms through the ExitableDirectoryReader wrapping chain (cancellation disabled initially) + Terms terms = searcher.getIndexReader().leaves().get(0).reader().terms(STRING_FIELD_NAME); + TermsEnum termsEnum = terms.iterator(); + termsEnum.next(); // advance to first term + + // Get a PostingsEnum — should be wrapped in ExitablePostingsEnum + PostingsEnum postingsEnum = termsEnum.postings(null, PostingsEnum.NONE); + + // Iterate without cancellation — should work fine + assertNotEquals(DocIdSetIterator.NO_MORE_DOCS, postingsEnum.nextDoc()); + + // Cancel and get a fresh PostingsEnum — first nextDoc() should throw + // because ExitablePostingsEnum checks on calls == 0 (first call) + cancelled.set(true); + PostingsEnum postingsEnum2 = termsEnum.postings(null, PostingsEnum.NONE); + expectThrows(TaskCancelledException.class, postingsEnum2::nextDoc); + + // Also verify advance throws when cancelled + PostingsEnum postingsEnum3 = termsEnum.postings(null, PostingsEnum.NONE); + expectThrows(TaskCancelledException.class, () -> postingsEnum3.advance(0)); + } + + public void testExitablePostingsEnumNoOpWhenCancellationDisabled() throws IOException { + // Without cancellation, PostingsEnum should work normally (backward compat) + ContextIndexSearcher searcher = new ContextIndexSearcher( + reader, + IndexSearcher.getDefaultSimilarity(), + IndexSearcher.getDefaultQueryCache(), + IndexSearcher.getDefaultQueryCachingPolicy(), + true, + null, + searchContext + ); + // No cancellation added — isEnabled() returns false, so terms() returns raw Terms + Terms terms = searcher.getIndexReader().leaves().get(0).reader().terms(STRING_FIELD_NAME); + assertNotNull(terms); + TermsEnum termsEnum = terms.iterator(); + assertNotNull(termsEnum.next()); + PostingsEnum postingsEnum = termsEnum.postings(null, PostingsEnum.NONE); + assertNotNull(postingsEnum); + // Should iterate without issues — no wrapping, no overhead + assertNotEquals(DocIdSetIterator.NO_MORE_DOCS, postingsEnum.nextDoc()); + } + private static class PointValuesIntersectVisitor implements PointValues.IntersectVisitor { @Override public void visit(int docID) {} diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesCancellationTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesCancellationTests.java new file mode 100644 index 0000000000000..b8db958740f3b --- /dev/null +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesCancellationTests.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.tasks.TaskCancelledException; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; + +import static org.opensearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS; +import static org.mockito.Mockito.when; + +/** + * Tests that {@link AggregatorFactories#createTopLevelAggregators} checks for task cancellation + * between aggregator factory creates. + */ +public class AggregatorFactoriesCancellationTests extends AggregatorTestCase { + + public void testCreateTopLevelAggregatorsThrowsWhenCancelled() throws IOException { + try (Directory directory = newDirectory()) { + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(new SortedSetDocValuesField("field", new BytesRef("value" + i))); + indexWriter.addDocument(doc); + } + indexWriter.close(); + + try (IndexReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newIndexSearcher(reader); + + MultiBucketConsumerService.MultiBucketConsumer bucketConsumer = new MultiBucketConsumerService.MultiBucketConsumer( + DEFAULT_MAX_BUCKETS, + new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST) + ); + SearchContext searchContext = createSearchContext( + searcher, + createIndexSettings(), + new MatchAllDocsQuery(), + bucketConsumer, + keywordField("field") + ); + + // Build AggregatorFactories from a builder with an actual aggregation + TermsAggregationBuilder aggBuilder = new TermsAggregationBuilder("terms").field("field").size(10); + QueryShardContext qsc = searchContext.getQueryShardContext(); + AggregatorFactories.Builder factoriesBuilder = new AggregatorFactories.Builder().addAggregator(aggBuilder); + AggregatorFactories factories = factoriesBuilder.build(qsc, null); + + // Verify it works when not cancelled + when(searchContext.isCancelled()).thenReturn(false); + List aggregators = factories.createTopLevelAggregators(searchContext); + assertFalse(aggregators.isEmpty()); + + // Now mark as cancelled — should throw TaskCancelledException + when(searchContext.isCancelled()).thenReturn(true); + expectThrows(TaskCancelledException.class, () -> factories.createTopLevelAggregators(searchContext)); + } + } + } +} From fead3a928236b61f9c759c61e738b541a148ab9f Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Sun, 3 May 2026 10:03:34 -0400 Subject: [PATCH 032/115] Update Transport.assertDefaultThreadContext to allow all Task.REQUEST_HEADERS (#21462) * Update Transport.assertDefaultThreadContext to allow all Task.REQUEST_HEADERS Signed-off-by: Craig Perkins --- .../org/opensearch/transport/Transports.java | 9 ++++-- .../opensearch/transport/TransportsTests.java | 32 +++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 server/src/test/java/org/opensearch/transport/TransportsTests.java diff --git a/server/src/main/java/org/opensearch/transport/Transports.java b/server/src/main/java/org/opensearch/transport/Transports.java index e6e9b7e5edb7e..014544fc2135f 100644 --- a/server/src/main/java/org/opensearch/transport/Transports.java +++ b/server/src/main/java/org/opensearch/transport/Transports.java @@ -37,6 +37,7 @@ import org.opensearch.tasks.Task; import java.util.Arrays; +import java.util.Map; /** * Utility class for transport @@ -81,9 +82,11 @@ public static boolean assertNotTransportThread(String reason) { } public static boolean assertDefaultThreadContext(ThreadContext threadContext) { - assert threadContext.getRequestHeadersOnly().isEmpty() - || threadContext.getRequestHeadersOnly().size() == 1 && threadContext.getRequestHeadersOnly().containsKey(Task.X_OPAQUE_ID) - : "expected empty context but was " + threadContext.getRequestHeadersOnly() + " on " + Thread.currentThread().getName(); + final Map requestHeaders = threadContext.getRequestHeadersOnly(); + assert requestHeaders.isEmpty() || Task.REQUEST_HEADERS.containsAll(requestHeaders.keySet()) : "expected empty context but was " + + requestHeaders + + " on " + + Thread.currentThread().getName(); return true; } } diff --git a/server/src/test/java/org/opensearch/transport/TransportsTests.java b/server/src/test/java/org/opensearch/transport/TransportsTests.java new file mode 100644 index 0000000000000..e7fdca0fa01bb --- /dev/null +++ b/server/src/test/java/org/opensearch/transport/TransportsTests.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.transport; + +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.tasks.Task; +import org.opensearch.test.OpenSearchTestCase; + +public class TransportsTests extends OpenSearchTestCase { + + public void testAssertDefaultThreadContextAllowsTaskRequestHeaders() { + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + threadContext.putHeader(Task.X_OPAQUE_ID, "opaque-id"); + threadContext.putHeader(Task.X_REQUEST_ID, "1234567890abcdef1234567890abcdef"); + + assertTrue(Transports.assertDefaultThreadContext(threadContext)); + } + + public void testAssertDefaultThreadContextRejectsNonTaskRequestHeaders() { + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + threadContext.putHeader("custom-header", "value"); + + expectThrows(AssertionError.class, () -> Transports.assertDefaultThreadContext(threadContext)); + } +} From c6cda3d6f505afc904037b97a95972791c7d76bd Mon Sep 17 00:00:00 2001 From: Abhishek Som <32438991+abhishek00159@users.noreply.github.com> Date: Mon, 4 May 2026 09:09:09 +0530 Subject: [PATCH 033/115] Added support for DSL Exists query to convert in RelNode format (#21243) * Added support for DSL Exists query to convert in RelNode format Signed-off-by: Abhishek Som * Updating UTs and adding validation related to field in converion context Signed-off-by: Abhishek Som * Fixing sandbox build issue & adding bug url. Signed-off-by: Abhishek Som --------- Signed-off-by: Abhishek Som Co-authored-by: Abhishek Som --- sandbox/plugins/dsl-query-executor/README.md | 8 +++ .../java/org/opensearch/dsl/DslQueryIT.java | 23 ++++++++ .../dsl/converter/ConversionContext.java | 32 +++++++++++ .../dsl/query/ExistsQueryTranslator.java | 42 ++++++++++++++ .../dsl/query/QueryRegistryFactory.java | 1 + .../dsl/query/TermQueryTranslator.java | 10 +--- .../dsl/query/ExistsQueryTranslatorTests.java | 57 +++++++++++++++++++ 7 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/ExistsQueryTranslator.java create mode 100644 sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/query/ExistsQueryTranslatorTests.java diff --git a/sandbox/plugins/dsl-query-executor/README.md b/sandbox/plugins/dsl-query-executor/README.md index 81228148044ee..04f9d697be3a9 100644 --- a/sandbox/plugins/dsl-query-executor/README.md +++ b/sandbox/plugins/dsl-query-executor/README.md @@ -18,6 +18,14 @@ _search request - `analytics-engine` — provides `QueryPlanExecutor` and `EngineContext` via Guice (declared as `extendedPlugins`) - `analytics-framework` — provides Calcite and shared SPI interfaces +## Supported Queries + +| DSL Query | Calcite Representation | +|-----------|------------------------| +| `term` | `=($field, value)` — equality filter | +| `match_all` | Skipped (boolean literal `TRUE`) | +| `exists` | `IS NOT NULL($field)` — field existence check (boost not supported) | + ## Running locally ```bash diff --git a/sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslQueryIT.java b/sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslQueryIT.java index 65996caf74c76..5e321a034919b 100644 --- a/sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslQueryIT.java +++ b/sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslQueryIT.java @@ -58,4 +58,27 @@ public void testFailsForMultipleIndices() { () -> client().search(new SearchRequest(INDEX, "test-index-2").source(new SearchSourceBuilder())).actionGet() ); } + + public void testExistsQuery() { + createTestIndex(); + assertOk(search(new SearchSourceBuilder().query(QueryBuilders.existsQuery("name")))); + } + + public void testExistsQueryWithBoostFails() { + createTestIndex(); + expectThrows(Exception.class, () -> search(new SearchSourceBuilder().query(QueryBuilders.existsQuery("name").boost(2.0f)))); + } + + // TODO: Enable once BooleanQueryTranslatorExists is supported + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/21442") + public void testExistsQueryWithBool() { + createTestIndex(); + assertOk( + search( + new SearchSourceBuilder().query( + QueryBuilders.boolQuery().must(QueryBuilders.existsQuery("name")).filter(QueryBuilders.termQuery("brand", "brandX")) + ) + ) + ); + } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/ConversionContext.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/ConversionContext.java index 99cdcbba9b2da..76d08db13fa1d 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/ConversionContext.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/ConversionContext.java @@ -11,7 +11,9 @@ import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; import org.opensearch.dsl.aggregation.AggregationMetadata; import org.opensearch.search.builder.SearchSourceBuilder; @@ -89,4 +91,34 @@ public AggregationMetadata getAggregationMetadata() { public ConversionContext withAggregationMetadata(AggregationMetadata metadata) { return new ConversionContext(searchSource, cluster, table, metadata); } + + /** + * Looks up a field by name and returns a RexNode input reference. + * + * @param fieldName the field name to look up + * @return a RexNode representing the field reference + * @throws ConversionException if the field is not found in the schema + */ + public RexNode makeFieldRef(String fieldName) throws ConversionException { + RelDataTypeField field = getRowType().getField(fieldName, false, false); + if (field == null) { + throw new ConversionException("Field '" + fieldName + "' not found in schema"); + } + return getRexBuilder().makeInputRef(field.getType(), field.getIndex()); + } + + /** + * Looks up a field by name and returns the field descriptor. + * + * @param fieldName the field name to look up + * @return the RelDataTypeField descriptor + * @throws ConversionException if the field is not found in the schema + */ + public RelDataTypeField getField(String fieldName) throws ConversionException { + RelDataTypeField field = getRowType().getField(fieldName, false, false); + if (field == null) { + throw new ConversionException("Field '" + fieldName + "' not found in schema"); + } + return field; + } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/ExistsQueryTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/ExistsQueryTranslator.java new file mode 100644 index 0000000000000..63dd27e9fd204 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/ExistsQueryTranslator.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.query; + +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.opensearch.dsl.converter.ConversionContext; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.index.query.AbstractQueryBuilder; +import org.opensearch.index.query.ExistsQueryBuilder; +import org.opensearch.index.query.QueryBuilder; + +/** + * Converts an {@link ExistsQueryBuilder} to a Calcite IS NOT NULL RexNode. + */ +public class ExistsQueryTranslator implements QueryTranslator { + + @Override + public Class getQueryType() { + return ExistsQueryBuilder.class; + } + + @Override + public RexNode convert(QueryBuilder query, ConversionContext ctx) throws ConversionException { + ExistsQueryBuilder existsQuery = (ExistsQueryBuilder) query; + String fieldName = existsQuery.fieldName(); + float boost = existsQuery.boost(); + + if (boost != AbstractQueryBuilder.DEFAULT_BOOST) { + throw new ConversionException("boost is unsupported for Exists query type"); + } + + RexNode fieldRef = ctx.makeFieldRef(fieldName); + return ctx.getRexBuilder().makeCall(SqlStdOperatorTable.IS_NOT_NULL, fieldRef); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/QueryRegistryFactory.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/QueryRegistryFactory.java index 5313c1d40253b..e61ee8aedaa4e 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/QueryRegistryFactory.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/QueryRegistryFactory.java @@ -20,6 +20,7 @@ public static QueryRegistry create() { QueryRegistry registry = new QueryRegistry(); registry.register(new TermQueryTranslator()); registry.register(new MatchAllQueryTranslator()); + registry.register(new ExistsQueryTranslator()); // TODO: add other query translators return registry; } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/TermQueryTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/TermQueryTranslator.java index 9f43be3cf63da..1c34c86b0eed8 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/TermQueryTranslator.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/TermQueryTranslator.java @@ -8,7 +8,6 @@ package org.opensearch.dsl.query; -import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.opensearch.dsl.converter.ConversionContext; @@ -36,13 +35,8 @@ public RexNode convert(QueryBuilder query, ConversionContext ctx) throws Convers String fieldName = termQuery.fieldName(); Object value = termQuery.value(); - RelDataTypeField field = ctx.getRowType().getField(fieldName, false, false); - if (field == null) { - throw new ConversionException("Field '" + fieldName + "' not found in schema"); - } - - RexNode fieldRef = ctx.getRexBuilder().makeInputRef(field.getType(), field.getIndex()); - RexNode literal = ctx.getRexBuilder().makeLiteral(value, field.getType(), true); + RexNode fieldRef = ctx.makeFieldRef(fieldName); + RexNode literal = ctx.getRexBuilder().makeLiteral(value, ctx.getField(fieldName).getType(), true); return ctx.getRexBuilder().makeCall(SqlStdOperatorTable.EQUALS, fieldRef, literal); } diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/query/ExistsQueryTranslatorTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/query/ExistsQueryTranslatorTests.java new file mode 100644 index 0000000000000..ff252742d858c --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/query/ExistsQueryTranslatorTests.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.query; + +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.opensearch.dsl.TestUtils; +import org.opensearch.dsl.converter.ConversionContext; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.index.query.ExistsQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.test.OpenSearchTestCase; + +public class ExistsQueryTranslatorTests extends OpenSearchTestCase { + + private final ExistsQueryTranslator translator = new ExistsQueryTranslator(); + private final ConversionContext ctx = TestUtils.createContext(); + + public void testConvertsExistsQueryToIsNotNull() throws ConversionException { + RexNode result = translator.convert(QueryBuilders.existsQuery("name"), ctx); + + assertTrue(result instanceof RexCall); + RexCall call = (RexCall) result; + assertEquals(SqlKind.IS_NOT_NULL, call.getKind()); + assertEquals(1, call.getOperands().size()); + assertTrue(call.getOperands().get(0) instanceof RexInputRef); + } + + public void testResolvesCorrectFieldIndex() throws ConversionException { + RexNode result = translator.convert(QueryBuilders.existsQuery("brand"), ctx); + + RexCall call = (RexCall) result; + RexInputRef fieldRef = (RexInputRef) call.getOperands().get(0); + // brand is the 3rd field (index 2) in TestUtils schema: name, price, brand, rating + assertEquals(2, fieldRef.getIndex()); + } + + public void testThrowsForUnknownField() { + expectThrows(ConversionException.class, () -> translator.convert(QueryBuilders.existsQuery("nonexistent"), ctx)); + } + + public void testThrowsForBoost() { + expectThrows(ConversionException.class, () -> translator.convert(QueryBuilders.existsQuery("name").boost(2.0f), ctx)); + } + + public void testReportsCorrectQueryType() { + assertEquals(ExistsQueryBuilder.class, translator.getQueryType()); + } +} From f886331afba2560ff02c71f137c86e0900dca3a1 Mon Sep 17 00:00:00 2001 From: Abhishek Som <32438991+abhishek00159@users.noreply.github.com> Date: Mon, 4 May 2026 17:03:39 +0530 Subject: [PATCH 034/115] Added support for Terms query to convert in RelNode format (#21239) * Added support for Terms query to convert in RelNode format - Added TermsQueryTranslator to convert terms queries to Calcite Inbuilt MakeIn function which uses SEARCH and EQUALS operator - Registered TermsQueryTranslator in QueryRegistryFactory - Added integration tests for terms query including validation tests for boost, _name, and value_type - Updated README to document Terms query support Signed-off-by: Abhishek Som * Added few more UTs and throwing ConversionException for terms lookup Signed-off-by: Abhishek Som * Fixed the failing TermsQueryTranslatorTests by updating the test expectations to match the actual behavior of RexBuilder.makeIn(): Signed-off-by: Abhishek Som * Added UTs for different types and Boolean, Long, String is working because of equivalent present in SQL types Signed-off-by: Abhishek Som * Updated UTs for geo type data type which are not supported currently Signed-off-by: Abhishek Som * sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslQueryIT.java Signed-off-by: Abhishek Som * Fixed Failing test with updated fields. Signed-off-by: Abhishek Som * Updating failed sandbox test Signed-off-by: Abhishek Som --------- Signed-off-by: Abhishek Som Co-authored-by: Abhishek Som --- sandbox/plugins/dsl-query-executor/README.md | 6 + .../java/org/opensearch/dsl/DslQueryIT.java | 34 ++++ .../dsl/query/QueryRegistryFactory.java | 1 + .../dsl/query/TermsQueryTranslator.java | 69 ++++++++ .../java/org/opensearch/dsl/TestUtils.java | 10 +- .../dsl/converter/ProjectConverterTests.java | 17 +- .../executor/DslQueryPlanExecutorTests.java | 5 +- .../dsl/query/TermsQueryTranslatorTests.java | 163 ++++++++++++++++++ .../dsl/result/ExecutionResultTests.java | 5 +- 9 files changed, 301 insertions(+), 9 deletions(-) create mode 100644 sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/TermsQueryTranslator.java create mode 100644 sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/query/TermsQueryTranslatorTests.java diff --git a/sandbox/plugins/dsl-query-executor/README.md b/sandbox/plugins/dsl-query-executor/README.md index 04f9d697be3a9..3b2bc297d0787 100644 --- a/sandbox/plugins/dsl-query-executor/README.md +++ b/sandbox/plugins/dsl-query-executor/README.md @@ -2,6 +2,12 @@ A front-end sandbox plugin to the analytics engine that intercepts `_search` requests, converts DSL queries into Calcite RelNode logical plans, and executes them through the analytics engine's query pipeline. +## Supported Query Types + +- **Term** — equality filter +- **Terms** — multi-value equality filter (uses query Filter with SEARCH and EQUALS) +- **Match All** — matches all documents + ## Architecture ``` diff --git a/sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslQueryIT.java b/sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslQueryIT.java index 5e321a034919b..33e6f22833967 100644 --- a/sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslQueryIT.java +++ b/sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslQueryIT.java @@ -10,6 +10,7 @@ import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix; import org.opensearch.action.search.SearchRequest; +import org.opensearch.dsl.converter.ConversionException; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.builder.SearchSourceBuilder; @@ -35,6 +36,39 @@ public void testTermQuery() { assertOk(search(new SearchSourceBuilder().query(QueryBuilders.termQuery("name", "laptop")))); } + public void testTermsQuery() { + createTestIndex(); + assertOk(search(new SearchSourceBuilder().query(QueryBuilders.termsQuery("name", "laptop", "phone")))); + } + + public void testTermsQueryWithBoostThrowsException() { + createTestIndex(); + expectThrows( + ConversionException.class, + () -> search(new SearchSourceBuilder().query(QueryBuilders.termsQuery("name", "laptop").boost(2.0f))) + ); + } + + public void testTermsQueryWithNameThrowsException() { + createTestIndex(); + expectThrows( + ConversionException.class, + () -> search(new SearchSourceBuilder().query(QueryBuilders.termsQuery("name", "laptop").queryName("my_query"))) + ); + } + + public void testTermsQueryWithValueTypeThrowsException() { + createTestIndex(); + expectThrows( + ConversionException.class, + () -> search( + new SearchSourceBuilder().query( + QueryBuilders.termsQuery("name", "laptop").valueType(org.opensearch.index.query.TermsQueryBuilder.ValueType.BITMAP) + ) + ) + ); + } + public void testWildcardQueryWithUnresolvedNode() { createTestIndex(); // Wildcard query is not converted to standard Rex — wraps in UnresolvedQueryCall. diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/QueryRegistryFactory.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/QueryRegistryFactory.java index e61ee8aedaa4e..f0bc550d59782 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/QueryRegistryFactory.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/QueryRegistryFactory.java @@ -19,6 +19,7 @@ private QueryRegistryFactory() {} public static QueryRegistry create() { QueryRegistry registry = new QueryRegistry(); registry.register(new TermQueryTranslator()); + registry.register(new TermsQueryTranslator()); registry.register(new MatchAllQueryTranslator()); registry.register(new ExistsQueryTranslator()); // TODO: add other query translators diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/TermsQueryTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/TermsQueryTranslator.java new file mode 100644 index 0000000000000..eaca1ae473758 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/query/TermsQueryTranslator.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.query; + +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexNode; +import org.opensearch.dsl.converter.ConversionContext; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.index.query.AbstractQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.TermsQueryBuilder; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Converts a {@link TermsQueryBuilder} to a Calcite IN RexNode. + */ +public class TermsQueryTranslator implements QueryTranslator { + + @Override + public Class getQueryType() { + return TermsQueryBuilder.class; + } + + @Override + public RexNode convert(QueryBuilder query, ConversionContext ctx) throws ConversionException { + + TermsQueryBuilder termsQuery = (TermsQueryBuilder) query; + + if (termsQuery.termsLookup() != null) { + throw new ConversionException("Terms query does not support terms lookup"); + } + if (termsQuery.boost() != AbstractQueryBuilder.DEFAULT_BOOST) { + throw new ConversionException("Terms query does not support non-default boost"); + } + if (termsQuery.queryName() != null) { + throw new ConversionException("Terms query does not support _name"); + } + if (termsQuery.valueType() != TermsQueryBuilder.ValueType.DEFAULT) { + throw new ConversionException("Terms query does not support non-default value_type"); + } + + String fieldName = termsQuery.fieldName(); + List values = termsQuery.values(); + + if (values == null || values.isEmpty()) { + throw new ConversionException("Terms query must have values"); + } + + RelDataTypeField field = ctx.getRowType().getField(fieldName, false, false); + if (field == null) { + throw new ConversionException("Field '" + fieldName + "' not found in schema"); + } + + RexNode fieldRef = ctx.getRexBuilder().makeInputRef(field.getType(), field.getIndex()); + List literals = values.stream() + .map(value -> ctx.getRexBuilder().makeLiteral(value, field.getType(), true)) + .collect(Collectors.toList()); + + return ctx.getRexBuilder().makeIn(fieldRef, literals); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/TestUtils.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/TestUtils.java index df5515bc3b916..fe3cb407e5626 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/TestUtils.java +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/TestUtils.java @@ -37,7 +37,9 @@ * Mockito can't mock Calcite classes due to classloader conflicts with OpenSearch's * RandomizedRunner, so tests use real objects built here. * - * Standard test schema: name (VARCHAR), price (INTEGER), brand (VARCHAR), rating (DOUBLE). + * Standard test schema: name (VARCHAR), price (INTEGER), brand (VARCHAR), rating (DOUBLE), + * created_date (DATE), is_active (BOOLEAN), timestamp (BIGINT), location (GEOMETRY), + * status (VARCHAR), binary_data (VARBINARY). */ public class TestUtils { @@ -75,6 +77,12 @@ public RelDataType getRowType(RelDataTypeFactory tf) { .add("price", tf.createTypeWithNullability(tf.createSqlType(SqlTypeName.INTEGER), true)) .add("brand", tf.createTypeWithNullability(tf.createSqlType(SqlTypeName.VARCHAR), true)) .add("rating", tf.createTypeWithNullability(tf.createSqlType(SqlTypeName.DOUBLE), true)) + .add("created_date", tf.createTypeWithNullability(tf.createSqlType(SqlTypeName.DATE), true)) + .add("is_active", tf.createTypeWithNullability(tf.createSqlType(SqlTypeName.BOOLEAN), true)) + .add("timestamp", tf.createTypeWithNullability(tf.createSqlType(SqlTypeName.BIGINT), true)) + .add("location", tf.createTypeWithNullability(tf.createSqlType(SqlTypeName.GEOMETRY), true)) + .add("status", tf.createTypeWithNullability(tf.createSqlType(SqlTypeName.VARCHAR), true)) + .add("binary_data", tf.createTypeWithNullability(tf.createSqlType(SqlTypeName.VARBINARY), true)) .build(); } }); diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/converter/ProjectConverterTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/converter/ProjectConverterTests.java index 9dba2004ad067..03959d18df492 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/converter/ProjectConverterTests.java +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/converter/ProjectConverterTests.java @@ -16,6 +16,8 @@ import org.opensearch.search.fetch.subphase.FetchSourceContext; import org.opensearch.test.OpenSearchTestCase; +import java.util.List; + public class ProjectConverterTests extends OpenSearchTestCase { private final ProjectConverter converter = new ProjectConverter(); @@ -85,9 +87,12 @@ public void testExcludesFields() throws ConversionException { RelNode result = converter.convert(scan, ctx); assertTrue(result instanceof LogicalProject); - assertEquals(2, result.getRowType().getFieldCount()); - assertEquals("name", result.getRowType().getFieldNames().get(0)); - assertEquals("brand", result.getRowType().getFieldNames().get(1)); + assertEquals(8, result.getRowType().getFieldCount()); + List fieldNames = result.getRowType().getFieldNames(); + assertTrue(fieldNames.contains("name")); + assertTrue(fieldNames.contains("brand")); + assertFalse(fieldNames.contains("price")); + assertFalse(fieldNames.contains("rating")); } public void testExcludesWithWildcard() throws ConversionException { @@ -96,7 +101,7 @@ public void testExcludesWithWildcard() throws ConversionException { RelNode result = converter.convert(scan, ctx); assertTrue(result instanceof LogicalProject); - assertEquals(3, result.getRowType().getFieldCount()); + assertEquals(9, result.getRowType().getFieldCount()); assertFalse(result.getRowType().getFieldNames().contains("rating")); } @@ -111,7 +116,7 @@ public void testWildcardNoMatchReturnsEmptyProjection() throws ConversionExcepti } public void testWildcardIncludesWithExcludes() throws ConversionException { - // Include all fields matching "* ", exclude "rating" + // Include all fields matching "*", exclude "rating" SearchSourceBuilder source = new SearchSourceBuilder().fetchSource( new FetchSourceContext(true, new String[] { "*" }, new String[] { "rating" }) ); @@ -119,7 +124,7 @@ public void testWildcardIncludesWithExcludes() throws ConversionException { RelNode result = converter.convert(scan, ctx); assertTrue(result instanceof LogicalProject); - assertEquals(3, result.getRowType().getFieldCount()); + assertEquals(9, result.getRowType().getFieldCount()); assertFalse(result.getRowType().getFieldNames().contains("rating")); } diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/executor/DslQueryPlanExecutorTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/executor/DslQueryPlanExecutorTests.java index 3dbd2acfd1ef8..fff14d61d1cb0 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/executor/DslQueryPlanExecutorTests.java +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/executor/DslQueryPlanExecutorTests.java @@ -42,7 +42,10 @@ public void testExecuteDelegatesEachPlanToExecutor() { assertEquals(QueryPlans.Type.HITS, result.getType()); assertNotNull(result.getPlan()); assertSame(scan, result.getPlan().relNode()); - assertEquals(List.of("name", "price", "brand", "rating"), result.getFieldNames()); + assertEquals( + List.of("name", "price", "brand", "rating", "created_date", "is_active", "timestamp", "location", "status", "binary_data"), + result.getFieldNames() + ); } // TODO: add test with multiple plans (HITS + AGGREGATION) to verify iteration order diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/query/TermsQueryTranslatorTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/query/TermsQueryTranslatorTests.java new file mode 100644 index 0000000000000..3ea95b6e01372 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/query/TermsQueryTranslatorTests.java @@ -0,0 +1,163 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.query; + +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.opensearch.dsl.TestUtils; +import org.opensearch.dsl.converter.ConversionContext; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.indices.TermsLookup; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Date; + +public class TermsQueryTranslatorTests extends OpenSearchTestCase { + + private final TermsQueryTranslator translator = new TermsQueryTranslator(); + private final ConversionContext ctx = TestUtils.createContext(); + + public void testSingleValueUsesEquals() throws ConversionException { + RexNode result = translator.convert(QueryBuilders.termsQuery("name", "laptop"), ctx); + + assertTrue(result instanceof RexCall); + RexCall call = (RexCall) result; + assertEquals(SqlKind.EQUALS, call.getKind()); + assertEquals(2, call.getOperands().size()); + } + + public void testMultipleStringValuesUsesSearch() throws ConversionException { + RexNode result = translator.convert(QueryBuilders.termsQuery("name", "laptop", "phone"), ctx); + + assertTrue(result instanceof RexCall); + RexCall call = (RexCall) result; + assertEquals(SqlKind.OR, call.getKind()); + } + + public void testResolvesCorrectFieldIndex() throws ConversionException { + RexNode result = translator.convert(QueryBuilders.termsQuery("brand", "brandX", "brandY"), ctx); + + RexCall call = (RexCall) result; + assertEquals(SqlKind.OR, call.getKind()); + // OR expression has nested structure, get field from first operand + RexCall firstEquals = (RexCall) call.getOperands().get(0); + RexInputRef fieldRef = (RexInputRef) firstEquals.getOperands().get(0); + assertEquals(2, fieldRef.getIndex()); + } + + public void testIntegerValues() throws ConversionException { + RexNode result = translator.convert(QueryBuilders.termsQuery("price", new Object[] { 1200, 1500 }), ctx); + + RexCall call = (RexCall) result; + assertEquals(SqlKind.OR, call.getKind()); + // OR expression has nested structure, get field from first operand + RexCall firstEquals = (RexCall) call.getOperands().get(0); + RexInputRef fieldRef = (RexInputRef) firstEquals.getOperands().get(0); + assertEquals(1, fieldRef.getIndex()); + } + + public void testDoubleValuesUsesSearch() throws ConversionException { + RexNode result = translator.convert(QueryBuilders.termsQuery("rating", new Object[] { 4.5, 4.8, 5.0 }), ctx); + + RexCall call = (RexCall) result; + assertEquals(SqlKind.OR, call.getKind()); + } + + public void testThrowsForUnknownField() { + expectThrows(ConversionException.class, () -> translator.convert(QueryBuilders.termsQuery("nonexistent", "value"), ctx)); + } + + public void testThrowsForEmptyValues() { + expectThrows(IllegalArgumentException.class, () -> translator.convert(QueryBuilders.termsQuery("name", (Object[]) null), ctx)); + } + + public void testThrowsForBoost() { + expectThrows(ConversionException.class, () -> translator.convert(QueryBuilders.termsQuery("name", "laptop").boost(2.0f), ctx)); + } + + public void testThrowsForQueryName() { + expectThrows( + ConversionException.class, + () -> translator.convert(QueryBuilders.termsQuery("name", "laptop").queryName("my_query"), ctx) + ); + } + + public void testThrowsForTermsLookup() { + TermsLookup termsLookup = new TermsLookup("lookup_index", "1", "terms"); + expectThrows(ConversionException.class, () -> translator.convert(QueryBuilders.termsLookupQuery("name", termsLookup), ctx)); + } + + public void testThrowsForValueType() { + expectThrows( + ConversionException.class, + () -> translator.convert(QueryBuilders.termsQuery("name", "laptop").valueType(TermsQueryBuilder.ValueType.BITMAP), ctx) + ); + } + + public void testReportsCorrectQueryType() { + assertEquals(TermsQueryBuilder.class, translator.getQueryType()); + } + + // Supported types: VARCHAR, INTEGER, DOUBLE, BOOLEAN, BIGINT + // Date type still throws ClassCastException from Calcite's RexBuilder.makeLiteral() + + // TODO: Enable when date type support is added + public void testDateType() { + expectThrows( + ClassCastException.class, + () -> translator.convert( + QueryBuilders.termsQuery("created_date", new Object[] { new Date(1704067200000L), new Date(1706745600000L) }), + ctx + ) + ); + } + + public void testBooleanType() throws ConversionException { + RexNode result = translator.convert(QueryBuilders.termsQuery("is_active", new Object[] { true, false }), ctx); + + RexCall call = (RexCall) result; + assertEquals(SqlKind.OR, call.getKind()); + } + + public void testLongType() throws ConversionException { + RexNode result = translator.convert(QueryBuilders.termsQuery("timestamp", new Object[] { 1234567890L, 9876543210L }), ctx); + + RexCall call = (RexCall) result; + assertEquals(SqlKind.OR, call.getKind()); + } + + public void testGeoPointType() { + expectThrows( + IllegalArgumentException.class, + () -> translator.convert(QueryBuilders.termsQuery("location", new Object[] { "40.7128,-74.0060", "34.0522,-118.2437" }), ctx) + ); + } + + public void testKeywordType() throws ConversionException { + RexNode result = translator.convert(QueryBuilders.termsQuery("status", new Object[] { "active", "pending" }), ctx); + + RexCall call = (RexCall) result; + assertEquals(SqlKind.OR, call.getKind()); + } + + // TODO: Enable when binary type support is added + public void testBinaryType() { + expectThrows( + ClassCastException.class, + () -> translator.convert( + QueryBuilders.termsQuery("binary_data", new Object[] { "U29tZSBiaW5hcnkgYmxvYg==", "QW5vdGhlciBibG9i" }), + ctx + ) + ); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/ExecutionResultTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/ExecutionResultTests.java index e0cb002e22ced..25d60e6a90981 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/ExecutionResultTests.java +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/ExecutionResultTests.java @@ -24,7 +24,10 @@ public void testExecutionResultCarriesPlanAndRows() { assertSame(plan, result.getPlan()); assertSame(rows, result.getRows()); assertEquals(QueryPlans.Type.HITS, result.getType()); - assertEquals(List.of("name", "price", "brand", "rating"), result.getFieldNames()); + assertEquals( + List.of("name", "price", "brand", "rating", "created_date", "is_active", "timestamp", "location", "status", "binary_data"), + result.getFieldNames() + ); } public void testRejectsNullArguments() { From fca9ef3ceec31b3564b3a4cb6c1070a19beb696b Mon Sep 17 00:00:00 2001 From: expani1729 <110471048+expani@users.noreply.github.com> Date: Mon, 4 May 2026 08:51:18 -0700 Subject: [PATCH 035/115] Planner changes to convert Predicates to Lucene QueryBuilder and insert Placeholder in Substrait (#21450) * Add intra-operator delegation tagging and predicate for Lucene conversion Signed-off-by: expani * Integrated Substrait Placeholder addition Signed-off-by: expani * Refactor Signed-off-by: expani * Added tests, PR comments Signed-off-by: expani --------- Signed-off-by: expani --- .../spi/BackendCapabilityProvider.java | 9 + .../spi/DelegatedPredicateFunction.java | 57 +++ .../spi/DelegatedPredicateSerializer.java | 42 ++ .../analytics/spi/FilterCapability.java | 3 + .../analytics-backend-datafusion/build.gradle | 5 + .../DataFusionFragmentConvertor.java | 9 +- .../be/datafusion/DataFusionPlugin.java | 3 +- .../main/resources/delegation_functions.yaml | 11 + .../DataFusionFragmentConvertorTests.java | 109 ++++- ...DelegationForIndexFullConversionTests.java | 441 ++++++++++++++++++ .../analytics-backend-lucene/build.gradle | 24 + .../lucene/LuceneAnalyticsBackendPlugin.java | 194 ++++++++ .../LuceneAnalyticsBackendPluginTests.java | 286 ++++++++++++ .../exec/action/FragmentExecutionRequest.java | 2 +- .../analytics/planner/CapabilityRegistry.java | 15 + .../analytics/planner/PlannerImpl.java | 2 +- .../planner/dag/FragmentConversionDriver.java | 131 +++++- .../analytics/planner/dag/StagePlan.java | 19 +- .../planner/rel/AnnotatedPredicate.java | 7 + .../planner/rel/OpenSearchAggregate.java | 9 + .../planner/rel/OpenSearchFilter.java | 18 +- .../planner/rel/OpenSearchProject.java | 8 +- .../planner/rel/OpenSearchRelNode.java | 18 + .../planner/rel/OperatorAnnotation.java | 10 + .../analytics/planner/MockBackend.java | 10 + .../dag/FragmentConversionDriverTests.java | 307 ++++++++++++ 26 files changed, 1709 insertions(+), 40 deletions(-) create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedPredicateFunction.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedPredicateSerializer.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/resources/delegation_functions.yaml create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java create mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendCapabilityProvider.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendCapabilityProvider.java index bba6d511d99d5..03b5b7284a683 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendCapabilityProvider.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendCapabilityProvider.java @@ -72,4 +72,13 @@ default Set acceptedDelegations() { default Map scalarFunctionAdapters() { return Map.of(); } + + /** + * Per-function serializers for delegated predicates this backend can accept. + * Keyed by {@link ScalarFunction} — the framework dispatches to the matching + * serializer during fragment conversion when a predicate is delegated to this backend. + */ + default Map delegatedPredicateSerializers() { + return Map.of(); + } } diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedPredicateFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedPredicateFunction.java new file mode 100644 index 0000000000000..5158a08176978 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedPredicateFunction.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; + +/** + * Placeholder function for delegated predicates in the Calcite plan. + * + *

      When a predicate is delegated to another backend (e.g., a MATCH_PHRASE predicate + * delegated from DataFusion to Lucene), the original expression is serialized and sent + * as opaque bytes. In the Calcite plan that goes to Isthmus/Substrait, the original + * expression is replaced with {@code delegated_predicate(annotationId)} — a function + * that always evaluates to TRUE and carries the annotation ID so the driving backend + * can look up the delegated query at execution time. + * + * @opensearch.internal + */ +public final class DelegatedPredicateFunction { + + /** The function name used in Calcite plans and Substrait serialization. */ + public static final String NAME = "delegated_predicate"; + + /** Singleton Calcite SqlFunction: {@code delegated_predicate(INT) → BOOLEAN}. */ + public static final SqlFunction FUNCTION = new SqlFunction( + NAME, + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.NUMERIC, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private DelegatedPredicateFunction() {} + + /** Builds a {@code delegated_predicate(annotationId)} RexCall. */ + public static RexNode makeCall(RexBuilder rexBuilder, int annotationId) { + RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); + RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER); + return rexBuilder.makeCall(FUNCTION, rexBuilder.makeLiteral(annotationId, intType, false)); + } +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedPredicateSerializer.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedPredicateSerializer.java new file mode 100644 index 0000000000000..7935b3702dd44 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedPredicateSerializer.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.apache.calcite.rex.RexCall; + +import java.util.List; + +/** + * Per-function serializer for delegated predicates. Registered by backends that accept + * delegation, keyed by {@link ScalarFunction} in + * {@link BackendCapabilityProvider#delegatedPredicateSerializers()}. + * + *

      Each implementation knows how to extract field names and query parameters from a + * {@link RexCall} and serialize them into backend-specific bytes that can be deserialized + * at the data node to create the appropriate query. + * + *

      TODO(same-backend-combining): When tree normalization combines adjacent same-backend + * predicates under AND/OR into a single BooleanQuery, serializers will need to handle + * composite predicate shapes — not just single-function leaves. + * + * @opensearch.internal + */ +@FunctionalInterface +public interface DelegatedPredicateSerializer { + + /** + * Serializes a delegated predicate into backend-specific bytes. + * + * @param call the original RexCall expression (e.g., MATCH($1, 'hello world')) + * @param fieldStorage per-column storage metadata; {@link org.apache.calcite.rex.RexInputRef} + * indices in {@code call} index into this list + * @return backend-specific serialized bytes + */ + byte[] serialize(RexCall call, List fieldStorage); +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterCapability.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterCapability.java index 3065b001a4013..3f31d5cba773d 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterCapability.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterCapability.java @@ -23,6 +23,9 @@ */ public sealed interface FilterCapability { + /** The scalar function this capability covers. */ + ScalarFunction function(); + /** Standard comparison filter (EQUALS, GT, IN, LIKE, etc.) on field types in given formats. */ record Standard(ScalarFunction function, Set fieldTypes, Set formats) implements FilterCapability { } diff --git a/sandbox/plugins/analytics-backend-datafusion/build.gradle b/sandbox/plugins/analytics-backend-datafusion/build.gradle index f638dd7a412af..77761cc69bf4e 100644 --- a/sandbox/plugins/analytics-backend-datafusion/build.gradle +++ b/sandbox/plugins/analytics-backend-datafusion/build.gradle @@ -82,6 +82,11 @@ dependencies { calciteCompile "com.google.guava:guava:${versions.guava}" calciteTestCompile "com.google.guava:guava:${versions.guava}" + // Planner + Lucene backend for end-to-end delegation unit tests + testImplementation project(':sandbox:plugins:analytics-engine') + testImplementation project(':sandbox:plugins:analytics-backend-lucene') + testCompileOnly 'org.immutables:value-annotations:2.8.8' + // ── internalClusterTest: end-to-end coordinator-reduce IT ─────────────────── // Pulls in every sibling plugin needed to construct a parquet-backed composite // index, dispatch a multi-shard PPL aggregate, and exercise DatafusionReduceSink. diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index 1af21101c1fe4..b8ec352673174 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -29,6 +29,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.analytics.planner.rel.OpenSearchStageInputScan; +import org.opensearch.analytics.spi.DelegatedPredicateFunction; import org.opensearch.analytics.spi.FragmentConvertor; import java.util.ArrayList; @@ -42,6 +43,7 @@ import io.substrait.isthmus.SubstraitRelVisitor; import io.substrait.isthmus.TypeConverter; import io.substrait.isthmus.expression.AggregateFunctionConverter; +import io.substrait.isthmus.expression.FunctionMappings; import io.substrait.isthmus.expression.ScalarFunctionConverter; import io.substrait.isthmus.expression.WindowFunctionConverter; import io.substrait.plan.Plan; @@ -77,6 +79,11 @@ public class DataFusionFragmentConvertor implements FragmentConvertor { private static final Logger LOGGER = LogManager.getLogger(DataFusionFragmentConvertor.class); + /** Maps DelegatedPredicateFunction to its Substrait extension name for Isthmus conversion. */ + private static final List ADDITIONAL_SCALAR_SIGS = List.of( + FunctionMappings.s(DelegatedPredicateFunction.FUNCTION, DelegatedPredicateFunction.NAME) + ); + private final SimpleExtension.ExtensionCollection extensions; public DataFusionFragmentConvertor(SimpleExtension.ExtensionCollection extensions) { @@ -245,7 +252,7 @@ private SubstraitRelVisitor createVisitor(RelNode relNode) { AggregateFunctionConverter aggConverter = new AggregateFunctionConverter(extensions.aggregateFunctions(), typeFactory); ScalarFunctionConverter scalarConverter = new ScalarFunctionConverter( extensions.scalarFunctions(), - List.of(), + ADDITIONAL_SCALAR_SIGS, typeFactory, typeConverter ); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java index d70aa344a7df6..8395f4a5b2f49 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java @@ -146,7 +146,8 @@ private static SimpleExtension.ExtensionCollection loadSubstraitExtensions() { ClassLoader previous = t.getContextClassLoader(); try { t.setContextClassLoader(DataFusionPlugin.class.getClassLoader()); - return DefaultExtensionCatalog.DEFAULT_COLLECTION; + SimpleExtension.ExtensionCollection delegationExtensions = SimpleExtension.load(List.of("/delegation_functions.yaml")); + return DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions); } finally { t.setContextClassLoader(previous); } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/delegation_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/delegation_functions.yaml new file mode 100644 index 0000000000000..1d9df8a93e6e0 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/delegation_functions.yaml @@ -0,0 +1,11 @@ +%YAML 1.2 +--- +urn: extension:org.opensearch:delegation_functions +scalar_functions: + - name: delegated_predicate + description: Placeholder for a predicate delegated to another backend. Returns TRUE at plan level; at execution time the driving backend calls into the delegation API using the annotationId. + impls: + - args: + - name: annotationId + value: i32 + return: boolean diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java index 3d156927e45f8..1e3e5811fde15 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java @@ -26,6 +26,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ImmutableBitSet; import org.opensearch.analytics.planner.rel.OpenSearchStageInputScan; +import org.opensearch.analytics.spi.DelegatedPredicateFunction; import org.opensearch.test.OpenSearchTestCase; import java.util.List; @@ -35,6 +36,7 @@ import io.substrait.proto.AggregateFunction; import io.substrait.proto.AggregateRel; import io.substrait.proto.AggregationPhase; +import io.substrait.proto.Expression; import io.substrait.proto.FilterRel; import io.substrait.proto.Plan; import io.substrait.proto.PlanRel; @@ -69,7 +71,8 @@ public void setUp() throws Exception { ClassLoader prev = t.getContextClassLoader(); try { t.setContextClassLoader(DataFusionFragmentConvertorTests.class.getClassLoader()); - extensions = DefaultExtensionCatalog.DEFAULT_COLLECTION; + SimpleExtension.ExtensionCollection delegationExtensions = SimpleExtension.load(List.of("/delegation_functions.yaml")); + extensions = DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions); } finally { t.setContextClassLoader(prev); } @@ -264,4 +267,108 @@ public void testAttachFragmentOnTop_Sort() throws Exception { assertEquals(List.of(DatafusionReduceSink.INPUT_ID), aggInput.getRead().getNamedTable().getNamesList()); } + /** + * A filter containing {@code delegated_predicate(42)} converts to Substrait + * with the placeholder preserved as a scalar function call in the FilterRel condition. + */ + public void testConvertShardScanFragment_DelegatedPredicatePlaceholder() throws Exception { + RelNode scan = buildTableScan("test_index", "A", "B"); + RexNode placeholder = DelegatedPredicateFunction.makeCall(rexBuilder, 42); + RelNode filter = LogicalFilter.create(scan, placeholder); + + byte[] bytes = newConvertor().convertShardScanFragment("test_index", filter); + + Plan plan = decodeSubstrait(bytes); + Rel root = rootRel(plan); + assertTrue("root must be a FilterRel", root.hasFilter()); + FilterRel filterRel = root.getFilter(); + assertTrue("FilterRel must carry a condition", filterRel.hasCondition()); + assertTrue("condition must be a scalar function", filterRel.getCondition().hasScalarFunction()); + logger.info("Substrait condition (single delegated):\n{}", filterRel.getCondition()); + Expression.ScalarFunction scalarFunc = filterRel.getCondition().getScalarFunction(); + assertFalse("scalar function must have arguments", scalarFunc.getArgumentsList().isEmpty()); + // Verify the argument is literal i32 = 42 + assertEquals(42, scalarFunc.getArguments(0).getValue().getLiteral().getI32()); + } + + /** + * AND(A > 10, delegated_predicate(7)) — mixed native + delegated. + * Substrait AND has two children: GT scalar function and delegated_predicate scalar function. + */ + public void testConvertShardScanFragment_MixedNativeAndDelegated() throws Exception { + RelNode scan = buildTableScan("test_index", "A", "B"); + RexNode nativePred = rexBuilder.makeCall( + SqlStdOperatorTable.GREATER_THAN, + rexBuilder.makeInputRef(scan, 0), + rexBuilder.makeLiteral(10, typeFactory.createSqlType(SqlTypeName.INTEGER), true) + ); + RexNode delegated = DelegatedPredicateFunction.makeCall(rexBuilder, 7); + RexNode andCondition = rexBuilder.makeCall(SqlStdOperatorTable.AND, nativePred, delegated); + RelNode filter = LogicalFilter.create(scan, andCondition); + + byte[] bytes = newConvertor().convertShardScanFragment("test_index", filter); + Plan plan = decodeSubstrait(bytes); + FilterRel filterRel = rootRel(plan).getFilter(); + // Root condition is AND (scalar function with 2 args) + assertTrue("condition must be a scalar function", filterRel.getCondition().hasScalarFunction()); + Expression.ScalarFunction andFunc = filterRel.getCondition().getScalarFunction(); + assertEquals("AND must have 2 arguments", 2, andFunc.getArgumentsCount()); + // Second arg should contain delegated_predicate with literal 7 + Expression delegatedArg = andFunc.getArguments(1).getValue(); + assertTrue("second AND arg must be a scalar function", delegatedArg.hasScalarFunction()); + assertEquals(7, delegatedArg.getScalarFunction().getArguments(0).getValue().getLiteral().getI32()); + } + + /** + * AND(A > 10, OR(delegated_predicate(1), NOT(delegated_predicate(2)))) — complex boolean tree. + * Verifies nested AND/OR/NOT with delegation placeholders and their annotation IDs survive + * Substrait conversion. + */ + public void testConvertShardScanFragment_ComplexBooleanTreeWithDelegation() throws Exception { + RelNode scan = buildTableScan("test_index", "A", "B"); + RexNode nativePred = rexBuilder.makeCall( + SqlStdOperatorTable.GREATER_THAN, + rexBuilder.makeInputRef(scan, 0), + rexBuilder.makeLiteral(10, typeFactory.createSqlType(SqlTypeName.INTEGER), true) + ); + RexNode delegated1 = DelegatedPredicateFunction.makeCall(rexBuilder, 1); + RexNode delegated2 = DelegatedPredicateFunction.makeCall(rexBuilder, 2); + RexNode notDelegated2 = rexBuilder.makeCall(SqlStdOperatorTable.NOT, delegated2); + RexNode orClause = rexBuilder.makeCall(SqlStdOperatorTable.OR, delegated1, notDelegated2); + RexNode andCondition = rexBuilder.makeCall(SqlStdOperatorTable.AND, nativePred, orClause); + RelNode filter = LogicalFilter.create(scan, andCondition); + + byte[] bytes = newConvertor().convertShardScanFragment("test_index", filter); + Plan plan = decodeSubstrait(bytes); + logger.info("Substrait plan (complex boolean tree):\n{}", plan); + FilterRel filterRel = rootRel(plan).getFilter(); + + // Root: AND with 2 args + Expression.ScalarFunction andFunc = filterRel.getCondition().getScalarFunction(); + assertEquals("AND must have 2 arguments", 2, andFunc.getArgumentsCount()); + + // arg[0]: GT (native predicate) — has field ref and literal 10 + Expression gtArg = andFunc.getArguments(0).getValue(); + assertTrue("first AND arg must be a scalar function (GT)", gtArg.hasScalarFunction()); + assertEquals(10, gtArg.getScalarFunction().getArguments(1).getValue().getLiteral().getI32()); + + // arg[1]: OR with 2 args + Expression orArg = andFunc.getArguments(1).getValue(); + assertTrue("second AND arg must be a scalar function (OR)", orArg.hasScalarFunction()); + Expression.ScalarFunction orFunc = orArg.getScalarFunction(); + assertEquals("OR must have 2 arguments", 2, orFunc.getArgumentsCount()); + + // OR arg[0]: delegated_predicate(1) + Expression dp1 = orFunc.getArguments(0).getValue(); + assertTrue("OR first arg must be scalar function", dp1.hasScalarFunction()); + assertEquals(1, dp1.getScalarFunction().getArguments(0).getValue().getLiteral().getI32()); + + // OR arg[1]: NOT(delegated_predicate(2)) + Expression notExpr = orFunc.getArguments(1).getValue(); + assertTrue("OR second arg must be scalar function (NOT)", notExpr.hasScalarFunction()); + Expression dp2 = notExpr.getScalarFunction().getArguments(0).getValue(); + assertTrue("NOT arg must be scalar function", dp2.hasScalarFunction()); + assertEquals(2, dp2.getScalarFunction().getArguments(0).getValue().getLiteral().getI32()); + } + } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java new file mode 100644 index 0000000000000..953f50afa1d09 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java @@ -0,0 +1,441 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.planner.CapabilityRegistry; +import org.opensearch.analytics.planner.FieldStorageResolver; +import org.opensearch.analytics.planner.PlannerContext; +import org.opensearch.analytics.planner.PlannerImpl; +import org.opensearch.analytics.planner.dag.DAGBuilder; +import org.opensearch.analytics.planner.dag.FragmentConversionDriver; +import org.opensearch.analytics.planner.dag.PlanForker; +import org.opensearch.analytics.planner.dag.QueryDAG; +import org.opensearch.analytics.planner.dag.Stage; +import org.opensearch.analytics.planner.dag.StagePlan; +import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; +import org.opensearch.analytics.spi.BackendCapabilityProvider; +import org.opensearch.analytics.spi.DelegatedPredicateFunction; +import org.opensearch.analytics.spi.DelegationType; +import org.opensearch.analytics.spi.EngineCapability; +import org.opensearch.analytics.spi.ExchangeSinkProvider; +import org.opensearch.analytics.spi.FieldType; +import org.opensearch.analytics.spi.FilterCapability; +import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.analytics.spi.ScanCapability; +import org.opensearch.be.lucene.LuceneAnalyticsBackendPlugin; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.GroupShardsIterator; +import org.opensearch.cluster.routing.OperationRouting; +import org.opensearch.cluster.routing.ShardIterator; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import io.substrait.extension.DefaultExtensionCatalog; +import io.substrait.extension.SimpleExtension; +import io.substrait.proto.Expression; +import io.substrait.proto.FilterRel; +import io.substrait.proto.Plan; +import io.substrait.proto.Rel; +import io.substrait.proto.SimpleExtensionDeclaration; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * End-to-end delegation test: MATCH predicates flow through the full pipeline + * (marking → forking → FragmentConversionDriver → Substrait) using the real + * {@link LuceneAnalyticsBackendPlugin} for query serialization and the real + * {@link DataFusionFragmentConvertor} for Substrait conversion. + * + *

      Verifies both the delegated query bytes (MatchQueryBuilder round-trip) and + * the Substrait plan structure (delegated_predicate placeholders with correct annotation IDs + * and preserved AND/OR/NOT boolean structure). + */ +public class FilterDelegationForIndexFullConversionTests extends OpenSearchTestCase { + + private static final SqlFunction MATCH_FUNCTION = new SqlFunction( + "MATCH", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private static final NamedWriteableRegistry WRITEABLE_REGISTRY = new NamedWriteableRegistry( + List.of(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchQueryBuilder.NAME, MatchQueryBuilder::new)) + ); + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private RelOptCluster cluster; + private AnalyticsSearchBackendPlugin dfBackend; + private AnalyticsSearchBackendPlugin luceneBackend; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + cluster = RelOptCluster.create(new HepPlanner(new HepProgramBuilder().build()), rexBuilder); + + // Load Substrait extensions with delegation_functions.yaml merged in, + // same as DataFusionPlugin.loadSubstraitExtensions() does at startup. + Thread thread = Thread.currentThread(); + ClassLoader previous = thread.getContextClassLoader(); + SimpleExtension.ExtensionCollection extensions; + try { + SimpleExtension.ExtensionCollection delegationExtensions = SimpleExtension.load(List.of("/delegation_functions.yaml")); + extensions = DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions); + } finally { + thread.setContextClassLoader(previous); + } + + // Lightweight DF backend wrapping the real DataFusionFragmentConvertor. + // Avoids instantiating DataFusionPlugin which requires native libraries. + // Only capabilities and fragment conversion are needed — no execution. + DataFusionFragmentConvertor convertor = new DataFusionFragmentConvertor(extensions); + dfBackend = new StubDfBackend(convertor); + luceneBackend = new LuceneAnalyticsBackendPlugin(null); + } + + /** + * AND(status = 200, MATCH(message, 'hello world')) — mixed native + delegated. + * Planner assigns id=0 to equals (native), id=1 to MATCH (delegated). + */ + public void testMixedNativeAndDelegated() throws Exception { + RexNode condition = rexBuilder.makeCall( + SqlStdOperatorTable.AND, + makeEquals(0, SqlTypeName.INTEGER, 200), + makeMatch(1, "hello world") + ); + StagePlan plan = runPipeline(condition); + + assertEquals("should have 1 delegated query", 1, plan.delegatedQueries().size()); + assertMatchQueryBuilder(plan.delegatedQueries(), "message", "hello world"); + + SubstraitResult substrait = substraitResult(plan.convertedBytes()); + logger.info("Substrait plan (mixed E2E):\n{}", substrait.plan()); + // Root: AND + Expression.ScalarFunction andFunc = substrait.filterRel().getCondition().getScalarFunction(); + assertEquals("and", resolveFunctionName(substrait.plan(), andFunc.getFunctionReference())); + assertEquals("AND must have 2 arguments", 2, andFunc.getArgumentsCount()); + // arg[1]: delegated_predicate(1) — annotation id=1 maps to MATCH 'hello world' + assertDelegatedPredicate(substrait.plan(), andFunc.getArguments(1).getValue(), 1); + assertMatchQueryForAnnotation(plan.delegatedQueries(), 1, "message", "hello world"); + } + + /** + * AND(status = 200, OR(MATCH(message, 'hello'), NOT(MATCH(message, 'goodbye')))) — complex tree. + * Planner assigns id=0 to equals (native), id=1 to first MATCH, id=2 to second MATCH. + */ + public void testComplexBooleanTree() throws Exception { + RexNode condition = rexBuilder.makeCall( + SqlStdOperatorTable.AND, + makeEquals(0, SqlTypeName.INTEGER, 200), + rexBuilder.makeCall( + SqlStdOperatorTable.OR, + makeMatch(1, "hello"), + rexBuilder.makeCall(SqlStdOperatorTable.NOT, makeMatch(1, "goodbye")) + ) + ); + StagePlan plan = runPipeline(condition); + + assertEquals("should have 2 delegated queries", 2, plan.delegatedQueries().size()); + + SubstraitResult substrait = substraitResult(plan.convertedBytes()); + logger.info("Substrait plan (complex E2E):\n{}", substrait.plan()); + + // Root: AND + Expression.ScalarFunction andFunc = substrait.filterRel().getCondition().getScalarFunction(); + assertEquals("and", resolveFunctionName(substrait.plan(), andFunc.getFunctionReference())); + assertEquals("AND must have 2 arguments", 2, andFunc.getArgumentsCount()); + + // arg[1]: OR + Expression orExpr = andFunc.getArguments(1).getValue(); + assertTrue("second AND arg must be scalar function", orExpr.hasScalarFunction()); + assertEquals("or", resolveFunctionName(substrait.plan(), orExpr.getScalarFunction().getFunctionReference())); + Expression.ScalarFunction orFunc = orExpr.getScalarFunction(); + assertEquals("OR must have 2 arguments", 2, orFunc.getArgumentsCount()); + + // OR arg[0]: delegated_predicate(1) → MATCH 'hello' + assertDelegatedPredicate(substrait.plan(), orFunc.getArguments(0).getValue(), 1); + assertMatchQueryForAnnotation(plan.delegatedQueries(), 1, "message", "hello"); + + // OR arg[1]: NOT(delegated_predicate(2)) → MATCH 'goodbye' + Expression notExpr = orFunc.getArguments(1).getValue(); + assertTrue("OR second arg must be scalar function", notExpr.hasScalarFunction()); + assertEquals("not", resolveFunctionName(substrait.plan(), notExpr.getScalarFunction().getFunctionReference())); + assertDelegatedPredicate(substrait.plan(), notExpr.getScalarFunction().getArguments(0).getValue(), 2); + assertMatchQueryForAnnotation(plan.delegatedQueries(), 2, "message", "goodbye"); + } + + // ---- Pipeline ---- + + private StagePlan runPipeline(RexNode condition) { + Map> fields = Map.of( + "status", + Map.of("type", "integer", "index", true), + "message", + Map.of("type", "keyword", "index", true) + ); + PlannerContext context = buildContext("parquet", fields, List.of(dfBackend, luceneBackend)); + RelOptTable table = mockTable( + "test_index", + new String[] { "status", "message" }, + new SqlTypeName[] { SqlTypeName.INTEGER, SqlTypeName.VARCHAR } + ); + LogicalFilter filter = LogicalFilter.create(new TableScan(cluster, cluster.traitSet(), List.of(), table) { + }, condition); + + RelNode marked = PlannerImpl.markAndOptimize(filter, context); + QueryDAG dag = DAGBuilder.build(marked, context.getCapabilityRegistry(), mockClusterService()); + PlanForker.forkAll(dag, context.getCapabilityRegistry()); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); + + Stage leaf = dag.rootStage(); + while (!leaf.getChildStages().isEmpty()) { + leaf = leaf.getChildStages().getFirst(); + } + return leaf.getPlanAlternatives().getFirst(); + } + + // ---- Helpers ---- + + private RexNode makeEquals(int fieldIndex, SqlTypeName fieldType, Object value) { + return rexBuilder.makeCall( + SqlStdOperatorTable.EQUALS, + rexBuilder.makeInputRef(typeFactory.createSqlType(fieldType), fieldIndex), + rexBuilder.makeLiteral(value, typeFactory.createSqlType(fieldType), true) + ); + } + + private RexNode makeMatch(int fieldIndex, String query) { + return rexBuilder.makeCall( + MATCH_FUNCTION, + rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), fieldIndex), + rexBuilder.makeLiteral(query) + ); + } + + private void assertMatchQueryBuilder(Map delegatedQueries, String expectedField, String expectedQuery) + throws IOException { + for (byte[] queryBytes : delegatedQueries.values()) { + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(queryBytes), WRITEABLE_REGISTRY)) { + MatchQueryBuilder matchQuery = (MatchQueryBuilder) input.readNamedWriteable(QueryBuilder.class); + if (matchQuery.fieldName().equals(expectedField) && matchQuery.value().equals(expectedQuery)) { + return; + } + } + } + fail("No MatchQueryBuilder found with field=[" + expectedField + "], query=[" + expectedQuery + "]"); + } + + private record SubstraitResult(Plan plan, FilterRel filterRel) { + } + + private SubstraitResult substraitResult(byte[] convertedBytes) throws Exception { + Plan plan = Plan.parseFrom(convertedBytes); + Rel root = plan.getRelations(0).getRoot().getInput(); + assertTrue("root must be a FilterRel", root.hasFilter()); + return new SubstraitResult(plan, root.getFilter()); + } + + /** Resolves a function_reference to its function name from the plan's extension declarations. */ + private String resolveFunctionName(Plan plan, int functionReference) { + for (SimpleExtensionDeclaration decl : plan.getExtensionsList()) { + if (decl.hasExtensionFunction() && decl.getExtensionFunction().getFunctionAnchor() == functionReference) { + String fullName = decl.getExtensionFunction().getName(); + int colonIndex = fullName.indexOf(':'); + return colonIndex >= 0 ? fullName.substring(0, colonIndex) : fullName; + } + } + fail("No extension function found for reference " + functionReference); + return null; + } + + /** Asserts a scalar function expression is delegated_predicate with the expected annotation ID. */ + private void assertDelegatedPredicate(Plan plan, Expression expr, int expectedAnnotationId) { + assertTrue("expression must be a scalar function", expr.hasScalarFunction()); + Expression.ScalarFunction func = expr.getScalarFunction(); + assertEquals( + "function must be delegated_predicate", + DelegatedPredicateFunction.NAME, + resolveFunctionName(plan, func.getFunctionReference()) + ); + assertEquals("annotation ID must match", expectedAnnotationId, func.getArguments(0).getValue().getLiteral().getI32()); + } + + /** Asserts the delegated query bytes for a specific annotation ID deserialize to the expected MatchQueryBuilder. */ + private void assertMatchQueryForAnnotation( + Map delegatedQueries, + int annotationId, + String expectedField, + String expectedQuery + ) throws IOException { + assertTrue("annotation ID " + annotationId + " must be in delegatedQueries", delegatedQueries.containsKey(annotationId)); + try ( + StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(delegatedQueries.get(annotationId)), WRITEABLE_REGISTRY) + ) { + MatchQueryBuilder matchQuery = (MatchQueryBuilder) input.readNamedWriteable(QueryBuilder.class); + assertEquals("field name for annotation " + annotationId, expectedField, matchQuery.fieldName()); + assertEquals("query text for annotation " + annotationId, expectedQuery, matchQuery.value()); + } + } + + @SuppressWarnings("unchecked") + private PlannerContext buildContext( + String primaryFormat, + Map> fieldMappings, + List backends + ) { + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.sourceAsMap()).thenReturn(Map.of("properties", fieldMappings)); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.getIndex()).thenReturn(new Index("test_index", "uuid")); + when(indexMetadata.getSettings()).thenReturn(Settings.builder().put("index.composite.primary_data_format", primaryFormat).build()); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + when(indexMetadata.getNumberOfShards()).thenReturn(2); + Metadata metadata = mock(Metadata.class); + when(metadata.index("test_index")).thenReturn(indexMetadata); + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.metadata()).thenReturn(metadata); + Function fieldStorageFactory = FieldStorageResolver::new; + return new PlannerContext(new CapabilityRegistry(backends, fieldStorageFactory), clusterState); + } + + private RelOptTable mockTable(String tableName, String[] fieldNames, SqlTypeName[] fieldTypes) { + RelDataTypeFactory.Builder builder = typeFactory.builder(); + for (int index = 0; index < fieldNames.length; index++) { + builder.add(fieldNames[index], typeFactory.createSqlType(fieldTypes[index])); + } + RelOptTable table = mock(RelOptTable.class); + when(table.getQualifiedName()).thenReturn(List.of(tableName)); + when(table.getRowType()).thenReturn(builder.build()); + return table; + } + + private ClusterService mockClusterService() { + ClusterService clusterService = mock(ClusterService.class); + ClusterState clusterState = mock(ClusterState.class); + OperationRouting routing = mock(OperationRouting.class); + when(clusterService.state()).thenReturn(clusterState); + when(clusterService.operationRouting()).thenReturn(routing); + when(routing.searchShards(any(), any(), any(), any())).thenReturn(new GroupShardsIterator(List.of())); + return clusterService; + } + + /** + * Lightweight DF backend wrapping the real {@link DataFusionFragmentConvertor} + * without instantiating {@link DataFusionPlugin} (which requires native libraries). + * Declares the same capabilities as the real DF backend — only fragment conversion + * and capability declarations are exercised, not execution. + */ + private static class StubDfBackend implements AnalyticsSearchBackendPlugin { + private static final Set TYPES = new HashSet<>(); + static { + TYPES.addAll(FieldType.numeric()); + TYPES.addAll(FieldType.keyword()); + TYPES.addAll(FieldType.date()); + TYPES.add(FieldType.BOOLEAN); + } + + private final DataFusionFragmentConvertor convertor; + + StubDfBackend(DataFusionFragmentConvertor convertor) { + this.convertor = convertor; + } + + @Override + public String name() { + return "mock-parquet"; + } + + @Override + public BackendCapabilityProvider getCapabilityProvider() { + return new BackendCapabilityProvider() { + @Override + public Set supportedEngineCapabilities() { + return Set.of(EngineCapability.SORT); + } + + @Override + public Set scanCapabilities() { + return Set.of(new ScanCapability.DocValues(Set.of("parquet"), TYPES)); + } + + @Override + public Set filterCapabilities() { + Set caps = new HashSet<>(); + for (ScalarFunction op : Set.of( + ScalarFunction.EQUALS, + ScalarFunction.NOT_EQUALS, + ScalarFunction.GREATER_THAN, + ScalarFunction.LESS_THAN + )) { + caps.add(new FilterCapability.Standard(op, TYPES, Set.of("parquet"))); + } + return caps; + } + + @Override + public Set supportedDelegations() { + return Set.of(DelegationType.FILTER); + } + }; + } + + @Override + public ExchangeSinkProvider getExchangeSinkProvider() { + return context -> null; + } + + @Override + public FragmentConvertor getFragmentConvertor() { + return convertor; + } + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/build.gradle b/sandbox/plugins/analytics-backend-lucene/build.gradle index 2263ad1064ab9..8bd898068e6b8 100644 --- a/sandbox/plugins/analytics-backend-lucene/build.gradle +++ b/sandbox/plugins/analytics-backend-lucene/build.gradle @@ -13,6 +13,14 @@ opensearchplugin { classname = 'org.opensearch.be.lucene.LucenePlugin' } +// Calcite (via analytics-engine) requires Guava which OpenSearch forbids on compile classpath. +// Use custom config to bypass, same as analytics-engine. +configurations { + calciteTestCompile + testCompileClasspath { exclude group: 'com.google.guava' } +} +sourceSets.test.compileClasspath += configurations.calciteTestCompile + dependencies { // Shared types and SPI interfaces (EngineBridge, AnalyticsBackEndPlugin, etc.) // Also provides calcite-core transitively via api. @@ -20,11 +28,27 @@ dependencies { implementation "org.apache.logging.log4j:log4j-api:${versions.log4j}" implementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" + + // Planner infrastructure for end-to-end delegation tests + testImplementation project(':sandbox:plugins:analytics-engine') + + // Guava for test compilation — Calcite API exposes guava types + calciteTestCompile "com.google.guava:guava:${versions.guava}" + testRuntimeOnly "com.google.guava:guava:${versions.guava}" + testRuntimeOnly 'com.google.guava:failureaccess:1.0.2' + + // Calcite annotation compatibility + testCompileOnly 'org.immutables:value-annotations:2.8.8' } test { systemProperty 'tests.security.manager', 'false' } +tasks.withType(JavaCompile).configureEach { + // Calcite annotation warnings with JDK 25 — harmless + options.compilerArgs -= '-Werror' +} + // TODO: Remove once back-end is built out with test suite testingConventions.enabled = false diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java new file mode 100644 index 0000000000000..a99bc90e35f15 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java @@ -0,0 +1,194 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; +import org.opensearch.analytics.spi.BackendCapabilityProvider; +import org.opensearch.analytics.spi.DelegatedPredicateSerializer; +import org.opensearch.analytics.spi.DelegationType; +import org.opensearch.analytics.spi.EngineCapability; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.FieldType; +import org.opensearch.analytics.spi.FilterCapability; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilder; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Analytics SPI extension for the Lucene backend. Declares filter capabilities + * for full-text and standard predicates, and provides {@link DelegatedPredicateSerializer} + * implementations for serializing delegated queries into {@link QueryBuilder} bytes. + * + *

      At the data node, the serialized bytes are deserialized back into a {@link QueryBuilder}, + * which uses the field name encoded within it to look up the appropriate + * {@link org.opensearch.index.mapper.MappedFieldType} and create the Lucene query. + * + * @opensearch.internal + */ +public class LuceneAnalyticsBackendPlugin implements AnalyticsSearchBackendPlugin { + + private static final String LUCENE_FORMAT = LuceneDataFormat.LUCENE_FORMAT_NAME; + private static final Set LUCENE_FORMATS = Set.of(LUCENE_FORMAT); + + private static final Set STANDARD_OPS = Set.of( + ScalarFunction.EQUALS, + ScalarFunction.NOT_EQUALS, + ScalarFunction.GREATER_THAN, + ScalarFunction.GREATER_THAN_OR_EQUAL, + ScalarFunction.LESS_THAN, + ScalarFunction.LESS_THAN_OR_EQUAL, + ScalarFunction.IS_NULL, + ScalarFunction.IS_NOT_NULL, + ScalarFunction.IN, + ScalarFunction.LIKE + ); + + private static final Set FULL_TEXT_OPS = Set.of( + ScalarFunction.MATCH, + ScalarFunction.MATCH_PHRASE, + ScalarFunction.FUZZY, + ScalarFunction.WILDCARD, + ScalarFunction.REGEXP + ); + + private static final Set STANDARD_TYPES = new HashSet<>(); + static { + STANDARD_TYPES.addAll(FieldType.numeric()); + STANDARD_TYPES.addAll(FieldType.keyword()); + STANDARD_TYPES.addAll(FieldType.text()); + STANDARD_TYPES.addAll(FieldType.date()); + STANDARD_TYPES.add(FieldType.BOOLEAN); + } + + private static final Set FULL_TEXT_TYPES = new HashSet<>(); + static { + FULL_TEXT_TYPES.addAll(FieldType.keyword()); + FULL_TEXT_TYPES.addAll(FieldType.text()); + } + + private static final Set FILTER_CAPS; + static { + Set caps = new HashSet<>(); + for (ScalarFunction op : STANDARD_OPS) { + caps.add(new FilterCapability.Standard(op, STANDARD_TYPES, LUCENE_FORMATS)); + } + for (ScalarFunction op : FULL_TEXT_OPS) { + for (FieldType type : FULL_TEXT_TYPES) { + caps.add(new FilterCapability.FullText(op, type, LUCENE_FORMATS, Set.of())); + } + } + FILTER_CAPS = caps; + } + + // TODO: add serializers for MATCH_PHRASE, FUZZY, WILDCARD, REGEXP, and standard ops (for performance delegation) + private static final Map SERIALIZERS = Map.of( + ScalarFunction.MATCH, + LuceneAnalyticsBackendPlugin::serializeMatch + ); + + private final LucenePlugin plugin; + + public LuceneAnalyticsBackendPlugin(LucenePlugin plugin) { + this.plugin = plugin; + } + + @Override + public String name() { + return LuceneDataFormat.LUCENE_FORMAT_NAME; + } + + @Override + public BackendCapabilityProvider getCapabilityProvider() { + return new BackendCapabilityProvider() { + @Override + public Set supportedEngineCapabilities() { + return Set.of(); + } + + @Override + public Set filterCapabilities() { + return FILTER_CAPS; + } + + @Override + public Set acceptedDelegations() { + return Set.of(DelegationType.FILTER); + } + + @Override + public Map delegatedPredicateSerializers() { + return SERIALIZERS; + } + }; + } + + private static final Logger LOGGER = LogManager.getLogger(LuceneAnalyticsBackendPlugin.class); + + // ---- Serializers ---- + + private static byte[] serializeMatch(RexCall call, List fieldStorage) { + String fieldName = extractFieldName(call, 0, fieldStorage); + String queryText = extractStringLiteral(call, 1); + MatchQueryBuilder queryBuilder = new MatchQueryBuilder(fieldName, queryText); + byte[] bytes = serializeQueryBuilder(queryBuilder); + LOGGER.debug( + "Serialized MATCH delegation: field=[{}], query=[{}], QueryBuilder=[{}], bytes={}", + fieldName, + queryText, + queryBuilder, + bytes.length + ); + return bytes; + } + + // ---- Helpers ---- + + private static String extractFieldName(RexCall call, int operandIndex, List fieldStorage) { + RexNode operand = call.getOperands().get(operandIndex); + if (!(operand instanceof RexInputRef inputRef)) { + throw new IllegalArgumentException( + "Expected RexInputRef at operand " + operandIndex + ", got: " + operand.getClass().getSimpleName() + ); + } + return FieldStorageInfo.resolve(fieldStorage, inputRef.getIndex()).getFieldName(); + } + + private static String extractStringLiteral(RexCall call, int operandIndex) { + RexNode operand = call.getOperands().get(operandIndex); + if (!(operand instanceof RexLiteral literal)) { + throw new IllegalArgumentException( + "Expected RexLiteral at operand " + operandIndex + ", got: " + operand.getClass().getSimpleName() + ); + } + return literal.getValueAs(String.class); + } + + private static byte[] serializeQueryBuilder(QueryBuilder queryBuilder) { + try (BytesStreamOutput output = new BytesStreamOutput()) { + output.writeNamedWriteable(queryBuilder); + return BytesReference.toBytes(output.bytes()); + } catch (IOException exception) { + throw new IllegalStateException("Failed to serialize delegated query: " + queryBuilder, exception); + } + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java new file mode 100644 index 0000000000000..9f441489365a6 --- /dev/null +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java @@ -0,0 +1,286 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.lucene; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.planner.CapabilityRegistry; +import org.opensearch.analytics.planner.FieldStorageResolver; +import org.opensearch.analytics.planner.PlannerContext; +import org.opensearch.analytics.planner.PlannerImpl; +import org.opensearch.analytics.planner.dag.DAGBuilder; +import org.opensearch.analytics.planner.dag.FragmentConversionDriver; +import org.opensearch.analytics.planner.dag.PlanForker; +import org.opensearch.analytics.planner.dag.QueryDAG; +import org.opensearch.analytics.planner.dag.Stage; +import org.opensearch.analytics.planner.dag.StagePlan; +import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; +import org.opensearch.analytics.spi.BackendCapabilityProvider; +import org.opensearch.analytics.spi.DelegationType; +import org.opensearch.analytics.spi.EngineCapability; +import org.opensearch.analytics.spi.ExchangeSinkProvider; +import org.opensearch.analytics.spi.FieldType; +import org.opensearch.analytics.spi.FilterCapability; +import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.analytics.spi.ScanCapability; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.GroupShardsIterator; +import org.opensearch.cluster.routing.OperationRouting; +import org.opensearch.cluster.routing.ShardIterator; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * End-to-end test: MATCH predicate flows through FragmentConversionDriver with the real + * {@link LuceneAnalyticsBackendPlugin} serializer, producing valid MatchQueryBuilder bytes. + */ +public class LuceneAnalyticsBackendPluginTests extends OpenSearchTestCase { + + private static final SqlFunction MATCH_FUNCTION = new SqlFunction( + "MATCH", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private static final NamedWriteableRegistry WRITEABLE_REGISTRY = new NamedWriteableRegistry( + List.of(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchQueryBuilder.NAME, MatchQueryBuilder::new)) + ); + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private RelOptCluster cluster; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + cluster = RelOptCluster.create(new HepPlanner(new HepProgramBuilder().build()), rexBuilder); + } + + /** + * MATCH(message, 'hello world') through full pipeline → delegatedQueries contains + * valid MatchQueryBuilder bytes with correct field name and query text. + */ + public void testMatchPredicateDelegationEndToEnd() throws IOException { + // DF backend: drives the plan, supports delegation, has a stub convertor + AnalyticsSearchBackendPlugin dfBackend = new StubDfBackend(); + // Real Lucene backend: accepts delegation, provides MATCH serializer + AnalyticsSearchBackendPlugin luceneBackend = new LuceneAnalyticsBackendPlugin(null); + + Map> fields = Map.of("message", Map.of("type", "keyword", "index", true)); + PlannerContext context = buildContext("parquet", fields, List.of(dfBackend, luceneBackend)); + + RexNode condition = rexBuilder.makeCall( + MATCH_FUNCTION, + rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 0), + rexBuilder.makeLiteral("hello world") + ); + RelOptTable table = mockTable("test_index", new String[] { "message" }, new SqlTypeName[] { SqlTypeName.VARCHAR }); + LogicalFilter filter = LogicalFilter.create(new TableScan(cluster, cluster.traitSet(), List.of(), table) { + }, condition); + + RelNode marked = PlannerImpl.markAndOptimize(filter, context); + QueryDAG dag = DAGBuilder.build(marked, context.getCapabilityRegistry(), mockClusterService()); + PlanForker.forkAll(dag, context.getCapabilityRegistry()); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); + + // Find the leaf stage (shard scan with filter) + Stage leaf = dag.rootStage(); + while (!leaf.getChildStages().isEmpty()) { + leaf = leaf.getChildStages().getFirst(); + } + StagePlan plan = leaf.getPlanAlternatives().getFirst(); + + // Verify delegation happened + assertFalse("delegatedQueries should not be empty", plan.delegatedQueries().isEmpty()); + assertEquals("should have exactly one delegated query", 1, plan.delegatedQueries().size()); + + // Deserialize and verify the MatchQueryBuilder + byte[] queryBytes = plan.delegatedQueries().values().iterator().next(); + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(queryBytes), WRITEABLE_REGISTRY)) { + QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); + assertTrue("Should be MatchQueryBuilder", deserialized instanceof MatchQueryBuilder); + MatchQueryBuilder matchQuery = (MatchQueryBuilder) deserialized; + assertEquals("message", matchQuery.fieldName()); + assertEquals("hello world", matchQuery.value()); + } + } + + // ---- Minimal infrastructure ---- + + @SuppressWarnings("unchecked") + private PlannerContext buildContext( + String primaryFormat, + Map> fieldMappings, + List backends + ) { + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.sourceAsMap()).thenReturn(Map.of("properties", fieldMappings)); + + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.getIndex()).thenReturn(new Index("test_index", "uuid")); + when(indexMetadata.getSettings()).thenReturn(Settings.builder().put("index.composite.primary_data_format", primaryFormat).build()); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + when(indexMetadata.getNumberOfShards()).thenReturn(2); + + Metadata metadata = mock(Metadata.class); + when(metadata.index("test_index")).thenReturn(indexMetadata); + + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.metadata()).thenReturn(metadata); + + Function fieldStorageFactory = FieldStorageResolver::new; + return new PlannerContext(new CapabilityRegistry(backends, fieldStorageFactory), clusterState); + } + + private RelOptTable mockTable(String tableName, String[] fieldNames, SqlTypeName[] fieldTypes) { + RelDataTypeFactory.Builder builder = typeFactory.builder(); + for (int index = 0; index < fieldNames.length; index++) { + builder.add(fieldNames[index], typeFactory.createSqlType(fieldTypes[index])); + } + RelOptTable table = mock(RelOptTable.class); + when(table.getQualifiedName()).thenReturn(List.of(tableName)); + when(table.getRowType()).thenReturn(builder.build()); + return table; + } + + private ClusterService mockClusterService() { + ClusterService clusterService = mock(ClusterService.class); + ClusterState clusterState = mock(ClusterState.class); + OperationRouting routing = mock(OperationRouting.class); + when(clusterService.state()).thenReturn(clusterState); + when(clusterService.operationRouting()).thenReturn(routing); + when(routing.searchShards(any(), any(), any(), any())).thenReturn(new GroupShardsIterator(List.of())); + return clusterService; + } + + /** Minimal DF backend that drives the plan with delegation support. */ + private static class StubDfBackend implements AnalyticsSearchBackendPlugin { + private static final Set TYPES = new HashSet<>(); + static { + TYPES.addAll(FieldType.numeric()); + TYPES.addAll(FieldType.keyword()); + TYPES.addAll(FieldType.date()); + TYPES.add(FieldType.BOOLEAN); + } + + @Override + public String name() { + return "mock-parquet"; + } + + @Override + public BackendCapabilityProvider getCapabilityProvider() { + return new BackendCapabilityProvider() { + @Override + public Set supportedEngineCapabilities() { + return Set.of(EngineCapability.SORT); + } + + @Override + public Set scanCapabilities() { + return Set.of(new ScanCapability.DocValues(Set.of("parquet"), TYPES)); + } + + @Override + public Set filterCapabilities() { + Set caps = new HashSet<>(); + for (ScalarFunction op : Set.of( + ScalarFunction.EQUALS, + ScalarFunction.NOT_EQUALS, + ScalarFunction.GREATER_THAN, + ScalarFunction.GREATER_THAN_OR_EQUAL, + ScalarFunction.LESS_THAN, + ScalarFunction.LESS_THAN_OR_EQUAL + )) { + caps.add(new FilterCapability.Standard(op, TYPES, Set.of("parquet"))); + } + return caps; + } + + @Override + public Set supportedDelegations() { + return Set.of(DelegationType.FILTER); + } + }; + } + + @Override + public ExchangeSinkProvider getExchangeSinkProvider() { + return context -> null; + } + + @Override + public FragmentConvertor getFragmentConvertor() { + return new FragmentConvertor() { + @Override + public byte[] convertShardScanFragment(String tableName, RelNode fragment) { + return ("shard:" + tableName).getBytes(StandardCharsets.UTF_8); + } + + @Override + public byte[] convertFinalAggFragment(RelNode fragment) { + return "reduce".getBytes(StandardCharsets.UTF_8); + } + + @Override + public byte[] attachFragmentOnTop(RelNode fragment, byte[] innerBytes) { + return innerBytes; + } + + @Override + public byte[] attachPartialAggOnTop(RelNode partialAggFragment, byte[] innerBytes) { + return innerBytes; + } + }; + } + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java index 3c5e4aaf1c86d..ffc20d47fdb16 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java @@ -98,7 +98,7 @@ public ActionRequestValidationException validate() { /** * A single plan alternative: a backend ID paired with its serialized fragment bytes. - * Produced by {@code FragmentConversionDriver.convert()} using the backend's + * Produced by {@code FragmentConversionDriver.convertAll()} using the backend's * {@code FragmentConvertor}. */ public static class PlanAlternative { diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java index 67b1ee6c8dfd0..f41e1c76bd456 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java @@ -37,6 +37,17 @@ *

      Single-format lookups return the stored list directly — no allocation at query time. * Multi-format aggregations build a new list by collecting across entries. * + *

      TODO(refactor): This class has 10+ HashMaps with near-identical shapes, 4 redundant + * key record types, and per-call list allocations in {@code *ForField} methods: + *

        + *
      • Unify key types (ScanKey, AggregateKey, ScalarKey) into a single record
        • + *
      • Derive {@code *CapableBackends} sets directly from backend capabilities, not as + * side effects of index population
        • + *
      • Pre-flatten format maps to eliminate per-call allocation in {@code allBackends} + * and {@code *ForField} methods
        • + *
      • Extract repeated constructor indexing pattern into a shared helper
        • + *
      + * * @opensearch.internal */ public class CapabilityRegistry { @@ -90,6 +101,10 @@ public CapabilityRegistry( for (DelegationType type : caps.acceptedDelegations()) { delegationAcceptors.computeIfAbsent(type, k -> new ArrayList<>()).add(name); } + // Runtime validation in FragmentConversionDriver ensures a DelegatedPredicateSerializer + // exists for each function actually delegated to this backend. Startup validation is + // intentionally omitted — a backend may accept delegation for a subset of its filter + // capabilities, and which functions are delegated depends on the query. for (ScanCapability cap : caps.scanCapabilities()) { for (FieldType fieldType : cap.supportedFieldTypes()) { addToFormatMap(scanIndex, new ScanKey(cap.getClass(), fieldType), cap.formats(), name); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java index 07ce76a8a3e51..e5c42dba04494 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java @@ -68,7 +68,7 @@ public static RelNode createPlan(RelNode rawRelNode, PlannerContext context) { * Phase 1 (RBO marking) + Phase 2 (CBO exchange insertion). * Package-private so planner rule tests can inspect the marked+optimized tree. */ - static RelNode markAndOptimize(RelNode rawRelNode, PlannerContext context) { + public static RelNode markAndOptimize(RelNode rawRelNode, PlannerContext context) { LOGGER.info("Input RelNode:\n{}", RelOptUtil.toString(rawRelNode)); // Phase 1: RBO — pre-marking logical optimizations then marking rules, single HepPlanner diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java index a0f806678b2ff..5e0983f1218af 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java @@ -9,6 +9,12 @@ package org.opensearch.analytics.planner.dag; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.analytics.planner.CapabilityRegistry; import org.opensearch.analytics.planner.rel.AggregateMode; import org.opensearch.analytics.planner.rel.OpenSearchAggregate; @@ -16,11 +22,18 @@ import org.opensearch.analytics.planner.rel.OpenSearchRelNode; import org.opensearch.analytics.planner.rel.OpenSearchStageInputScan; import org.opensearch.analytics.planner.rel.OpenSearchTableScan; +import org.opensearch.analytics.planner.rel.OperatorAnnotation; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; +import org.opensearch.analytics.spi.DelegatedPredicateSerializer; +import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.ScalarFunction; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.function.Function; /** * Drives fragment conversion for all {@link StagePlan} alternatives in a {@link QueryDAG}. @@ -47,6 +60,8 @@ */ public class FragmentConversionDriver { + private static final Logger LOGGER = LogManager.getLogger(FragmentConversionDriver.class); + private FragmentConversionDriver() {} /** @@ -65,16 +80,83 @@ private static void convertStage(Stage stage, CapabilityRegistry registry) { for (StagePlan plan : stage.getPlanAlternatives()) { AnalyticsSearchBackendPlugin backend = registry.getBackend(plan.backendId()); FragmentConvertor convertor = backend.getFragmentConvertor(); - byte[] bytes = convert(plan.resolvedFragment(), convertor); - converted.add(plan.withConvertedBytes(bytes)); + IntraOperatorDelegationBytes delegationBytes = new IntraOperatorDelegationBytes(registry); + byte[] bytes = convert(plan.resolvedFragment(), convertor, delegationBytes); + converted.add(plan.withConvertedBytes(bytes, delegationBytes.getResult())); } stage.setPlanAlternatives(converted); } + /** + * Lazily accumulates serialized delegated query bytes during fragment conversion. + * Only allocates the map when the first delegated annotation is encountered. + */ + static final class IntraOperatorDelegationBytes { + private final CapabilityRegistry registry; + private Map result; + + IntraOperatorDelegationBytes(CapabilityRegistry registry) { + this.registry = registry; + } + + /** + * Creates an annotation resolver scoped to a specific operator. Compares each + * annotation's viable backend against the operator's backend: native annotations + * are unwrapped, delegated ones are serialized and replaced with a placeholder. + */ + Function resolverFor(OpenSearchRelNode operator, RexBuilder rexBuilder) { + String operatorBackend = operator.getViableBackends().getFirst(); + List fieldStorage = operator.getOutputFieldStorage(); + return annotation -> { + String annotationBackend = annotation.getViableBackends().getFirst(); + if (annotationBackend.equals(operatorBackend)) { + LOGGER.debug("Native annotation [id={}]: backend [{}] matches operator", annotation.getAnnotationId(), operatorBackend); + return annotation.unwrap(); + } + RexNode original = annotation.unwrap(); + if (!(original instanceof RexCall originalCall) || !(originalCall.getOperator() instanceof SqlFunction sqlFunction)) { + throw new IllegalStateException("Delegated expression must be a SqlFunction call: " + original); + } + ScalarFunction function = ScalarFunction.fromSqlFunction(sqlFunction); + DelegatedPredicateSerializer serializer = registry.getBackend(annotationBackend) + .getCapabilityProvider() + .delegatedPredicateSerializers() + .get(function); + if (serializer == null) { + throw new IllegalStateException( + "No DelegatedPredicateSerializer for [" + + function + + "] on backend [" + + annotationBackend + + "]. CapabilityRegistry should have rejected this at startup." + ); + } + byte[] serialized = serializer.serialize(originalCall, fieldStorage); + LOGGER.debug( + "Delegated annotation [id={}]: {} from operator [{}] to [{}], serialized {} bytes", + annotation.getAnnotationId(), + function, + operatorBackend, + annotationBackend, + serialized.length + ); + if (result == null) { + result = new HashMap<>(); + } + result.put(annotation.getAnnotationId(), serialized); + return annotation.makePlaceholder(rexBuilder); + }; + } + + Map getResult() { + return result != null ? result : Map.of(); + } + } + /** * Dispatches conversion based on the fragment's leaf and top node types. */ - static byte[] convert(RelNode resolvedFragment, FragmentConvertor convertor) { + static byte[] convert(RelNode resolvedFragment, FragmentConvertor convertor, IntraOperatorDelegationBytes delegationBytes) { RelNode leaf = findLeaf(resolvedFragment); if (leaf instanceof OpenSearchTableScan scan) { @@ -83,17 +165,19 @@ static byte[] convert(RelNode resolvedFragment, FragmentConvertor convertor) { // Partial agg at top: convert everything below it, then attach partial agg on top. // strippedInputs passed to stripAnnotations for schema validity (LogicalAggregate needs its inputs). if (resolvedFragment instanceof OpenSearchAggregate agg && agg.getMode() == AggregateMode.PARTIAL) { - List strippedInputs = agg.getInputs().stream().map(FragmentConversionDriver::strip).toList(); + List strippedInputs = agg.getInputs().stream().map(input -> strip(input, delegationBytes)).toList(); byte[] innerBytes = convertor.convertShardScanFragment(tableName, strippedInputs.getFirst()); - RelNode strippedAgg = agg.stripAnnotations(strippedInputs); + Function resolver = delegationBytes.resolverFor(agg, agg.getCluster().getRexBuilder()); + RelNode strippedAgg = agg.stripAnnotations(strippedInputs, resolver); return convertor.attachPartialAggOnTop(strippedAgg, innerBytes); } - return convertor.convertShardScanFragment(tableName, strip(resolvedFragment)); + RelNode stripped = strip(resolvedFragment, delegationBytes); + return convertor.convertShardScanFragment(tableName, stripped); } if (leaf instanceof OpenSearchStageInputScan) { - return convertReduceFragment(resolvedFragment, convertor); + return convertReduceFragment(resolvedFragment, convertor, delegationBytes); } throw new IllegalStateException( @@ -116,20 +200,26 @@ static byte[] convert(RelNode resolvedFragment, FragmentConvertor convertor) { * when shuffle joins are implemented (check if all inputs are StageInputScan * and dispatch to a dedicated convertJoinFragment method). */ - private static byte[] convertReduceFragment(RelNode node, FragmentConvertor convertor) { + private static byte[] convertReduceFragment(RelNode node, FragmentConvertor convertor, IntraOperatorDelegationBytes delegationBytes) { // Find the ExchangeReducer and collect operators above it - return convertReduceNode(node, convertor, false); + return convertReduceNode(node, convertor, false, delegationBytes); } - private static byte[] convertReduceNode(RelNode node, FragmentConvertor convertor, boolean finalAggConverted) { + private static byte[] convertReduceNode( + RelNode node, + FragmentConvertor convertor, + boolean finalAggConverted, + IntraOperatorDelegationBytes delegationBytes + ) { if (node instanceof OpenSearchExchangeReducer) { // Strip ExchangeReducer — StageInputScan below it is the schema source // This should never be reached directly; handled by the parent (final agg) - return convertor.convertFinalAggFragment(strip(node.getInputs().getFirst())); + return convertor.convertFinalAggFragment(strip(node.getInputs().getFirst(), delegationBytes)); } if (node instanceof OpenSearchRelNode openSearchNode) { - List strippedInputs = node.getInputs().stream().map(FragmentConversionDriver::strip).toList(); - RelNode strippedNode = openSearchNode.stripAnnotations(strippedInputs); + List strippedInputs = node.getInputs().stream().map(input -> strip(input, delegationBytes)).toList(); + Function resolver = delegationBytes.resolverFor(openSearchNode, node.getCluster().getRexBuilder()); + RelNode strippedNode = openSearchNode.stripAnnotations(strippedInputs, resolver); if (!finalAggConverted) { // First OpenSearchRelNode above ExchangeReducer = final agg @@ -138,34 +228,35 @@ private static byte[] convertReduceNode(RelNode node, FragmentConvertor converto && node.getInputs().getFirst() instanceof OpenSearchExchangeReducer; if (childIsExchangeReducer) { // Strip ExchangeReducer, keep StageInputScan as leaf for schema - RelNode stageInputScan = strip(node.getInputs().getFirst().getInputs().getFirst()); + RelNode stageInputScan = strip(node.getInputs().getFirst().getInputs().getFirst(), delegationBytes); List finalAggInputs = List.of(stageInputScan); - RelNode finalAggFragment = openSearchNode.stripAnnotations(finalAggInputs); + RelNode finalAggFragment = openSearchNode.stripAnnotations(finalAggInputs, resolver); return convertor.convertFinalAggFragment(finalAggFragment); } } // Operator above final agg — convert child first, then attach - byte[] innerBytes = convertReduceNode(node.getInputs().getFirst(), convertor, false); + byte[] innerBytes = convertReduceNode(node.getInputs().getFirst(), convertor, false, delegationBytes); return convertor.attachFragmentOnTop(strippedNode, innerBytes); } throw new IllegalStateException("Unexpected reduce stage node: " + node.getClass().getSimpleName()); } /** Recursively strips annotations bottom-up. Keeps OpenSearchStageInputScan as-is. */ - private static RelNode strip(RelNode node) { + private static RelNode strip(RelNode node, IntraOperatorDelegationBytes delegationBytes) { if (node instanceof OpenSearchStageInputScan) { return node; // kept for schema inference at reduce stage } if (node instanceof OpenSearchExchangeReducer) { - return strip(node.getInputs().getFirst()); + return strip(node.getInputs().getFirst(), delegationBytes); } List strippedChildren = new ArrayList<>(node.getInputs().size()); for (RelNode input : node.getInputs()) { - strippedChildren.add(strip(input)); + strippedChildren.add(strip(input, delegationBytes)); } if (node instanceof OpenSearchRelNode openSearchNode) { - return openSearchNode.stripAnnotations(strippedChildren); + Function resolver = delegationBytes.resolverFor(openSearchNode, node.getCluster().getRexBuilder()); + return openSearchNode.stripAnnotations(strippedChildren, resolver); } return node; } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/StagePlan.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/StagePlan.java index 69abb4a89f87f..515a1d0537231 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/StagePlan.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/StagePlan.java @@ -11,26 +11,29 @@ import org.apache.calcite.rel.RelNode; import org.opensearch.analytics.spi.FragmentConvertor; +import java.util.Map; + /** * A single plan alternative for a {@link Stage}. Contains a resolved fragment * where every operator's viableBackends and every annotation's viableBackends * are narrowed to exactly one backend, plus the converted bytes produced by * the backend's {@link FragmentConvertor}. * - * @param resolvedFragment fragment with all viableBackends narrowed to single choices - * @param backendId the primary backend for this plan - * @param convertedBytes backend-specific serialized plan bytes (null before conversion) + * @param resolvedFragment fragment with all viableBackends narrowed to single choices + * @param backendId the primary backend for this plan + * @param convertedBytes backend-specific serialized plan bytes (null before conversion) + * @param delegatedQueries serialized delegated queries keyed by annotationId (empty if no delegation) * @opensearch.internal */ -public record StagePlan(RelNode resolvedFragment, String backendId, byte[] convertedBytes) { +public record StagePlan(RelNode resolvedFragment, String backendId, byte[] convertedBytes, Map delegatedQueries) { /** Creates a StagePlan before conversion (bytes not yet available). */ public StagePlan(RelNode resolvedFragment, String backendId) { - this(resolvedFragment, backendId, null); + this(resolvedFragment, backendId, null, Map.of()); } - /** Returns a copy with converted bytes populated. */ - public StagePlan withConvertedBytes(byte[] bytes) { - return new StagePlan(resolvedFragment, backendId, bytes); + /** Returns a copy with converted bytes and delegated queries populated. */ + public StagePlan withConvertedBytes(byte[] bytes, Map delegatedQueries) { + return new StagePlan(resolvedFragment, backendId, bytes, delegatedQueries); } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java index 2ef04d8cb9f7e..372c5cf693aa3 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/AnnotatedPredicate.java @@ -9,12 +9,14 @@ package org.opensearch.analytics.planner.rel; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSyntax; import org.apache.calcite.sql.type.ReturnTypes; +import org.opensearch.analytics.spi.DelegatedPredicateFunction; import java.util.List; @@ -82,6 +84,11 @@ public RexNode withAdaptedOriginal(RexNode adaptedOriginal) { return new AnnotatedPredicate(type, adaptedOriginal, viableBackends, annotationId); } + @Override + public RexNode makePlaceholder(RexBuilder rexBuilder) { + return DelegatedPredicateFunction.makeCall(rexBuilder, annotationId); + } + @Override protected String computeDigest(boolean withType) { return "ANNOTATED_PREDICATE(id=" + annotationId + ", backends=" + viableBackends + ", " + original + ")"; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchAggregate.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchAggregate.java index 03e146d53ad93..5d86fcb0372c0 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchAggregate.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchAggregate.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Function; /** * OpenSearch custom Aggregate carrying viable backend list and per-call annotations. @@ -179,8 +180,16 @@ public RelNode copyResolved(String backend, List children, List strippedChildren) { + return stripAnnotations(strippedChildren, OperatorAnnotation::unwrap); + } + + @Override + public RelNode stripAnnotations(List strippedChildren, Function annotationResolver) { List strippedCalls = new ArrayList<>(); for (AggregateCall aggCall : getAggCallList()) { + // TODO: when aggregate delegation is implemented, use annotationResolver + // to replace delegated AggregateCallAnnotations with placeholders instead + // of just filtering them out. List cleanRexList = aggCall.rexList.stream().filter(rex -> !(rex instanceof AggregateCallAnnotation)).toList(); strippedCalls.add( AggregateCall.create( diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchFilter.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchFilter.java index 8503b7e286ccb..fc93cf1f78133 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchFilter.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchFilter.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.List; import java.util.ListIterator; +import java.util.function.Function; /** * OpenSearch custom Filter carrying viable backend list and per-predicate annotations. @@ -96,7 +97,12 @@ public RelNode copyResolved(String backend, List children, List strippedChildren) { - return LogicalFilter.create(strippedChildren.getFirst(), stripCondition(getCondition())); + return stripAnnotations(strippedChildren, OperatorAnnotation::unwrap); + } + + @Override + public RelNode stripAnnotations(List strippedChildren, Function annotationResolver) { + return LogicalFilter.create(strippedChildren.getFirst(), resolveCondition(getCondition(), annotationResolver)); } private RexNode replaceAnnotations(RexNode node, ListIterator annotationIterator) { @@ -115,15 +121,15 @@ private RexNode replaceAnnotations(RexNode node, ListIterator annotationResolver) { + if (node instanceof AnnotatedPredicate predicate) return annotationResolver.apply(predicate); if (node instanceof RexCall call) { List newOperands = new ArrayList<>(); boolean changed = false; for (RexNode operand : call.getOperands()) { - RexNode stripped = stripCondition(operand); - newOperands.add(stripped); - if (stripped != operand) changed = true; + RexNode resolved = resolveCondition(operand, annotationResolver); + newOperands.add(resolved); + if (resolved != operand) changed = true; } return changed ? call.clone(call.getType(), newOperands) : call; } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java index 89a94d157fe89..7b7002233a2ac 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Function; /** * OpenSearch custom Project carrying viable backend list and per-expression annotations. @@ -116,10 +117,15 @@ public RelNode copyResolved(String backend, List children, List strippedChildren) { + return stripAnnotations(strippedChildren, OperatorAnnotation::unwrap); + } + + @Override + public RelNode stripAnnotations(List strippedChildren, Function annotationResolver) { List strippedExprs = new ArrayList<>(); for (RexNode expr : getProjects()) { if (expr instanceof AnnotatedProjectExpression annotated) { - strippedExprs.add(annotated.unwrap()); + strippedExprs.add(annotationResolver.apply(annotated)); } else { // Plain expressions have no annotation to strip — pass through. strippedExprs.add(expr); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchRelNode.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchRelNode.java index 130b9ea33a96c..0322300f1eb31 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchRelNode.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchRelNode.java @@ -9,10 +9,12 @@ package org.opensearch.analytics.planner.rel; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexNode; import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.analytics.spi.FragmentConvertor; import java.util.List; +import java.util.function.Function; /** * Marker interface for all OpenSearch custom RelNodes that carry backend assignment @@ -67,4 +69,20 @@ default List getAnnotations() { * @param strippedChildren children already stripped */ RelNode stripAnnotations(List strippedChildren); + + /** + * Returns a clean standard Calcite RelNode with annotations resolved via the given function. + * The resolver decides per-annotation what to return: the unwrapped original for native + * annotations, or a placeholder (e.g., {@code delegated_predicate(annotationId)}) for + * delegated ones. + * + *

      Default delegates to {@link #stripAnnotations(List)} — correct for operators + * with no annotations (Sort, Scan, ExchangeReducer, StageInputScan). + * + * @param strippedChildren children already stripped + * @param annotationResolver maps each annotation to its replacement RexNode + */ + default RelNode stripAnnotations(List strippedChildren, Function annotationResolver) { + return stripAnnotations(strippedChildren); + } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OperatorAnnotation.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OperatorAnnotation.java index 832c25afb0253..6b0ffe00826fd 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OperatorAnnotation.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OperatorAnnotation.java @@ -8,6 +8,7 @@ package org.opensearch.analytics.planner.rel; +import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import java.util.List; @@ -33,4 +34,13 @@ public interface OperatorAnnotation { /** Returns a copy of this annotation wrapping a different (adapted) inner expression. */ RexNode withAdaptedOriginal(RexNode adaptedOriginal); + + /** + * Returns a placeholder RexNode for this annotation when delegated. + * Each annotation type produces the appropriate placeholder shape: + * predicates return BOOLEAN, project expressions return their original type, etc. + */ + default RexNode makePlaceholder(RexBuilder rexBuilder) { + throw new UnsupportedOperationException("makePlaceholder not implemented for " + getClass().getSimpleName()); + } } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java index 804c96cdf1547..33b70bd35ca5f 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java @@ -11,6 +11,7 @@ import org.opensearch.analytics.spi.AggregateCapability; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; import org.opensearch.analytics.spi.BackendCapabilityProvider; +import org.opensearch.analytics.spi.DelegatedPredicateSerializer; import org.opensearch.analytics.spi.DelegationType; import org.opensearch.analytics.spi.EngineCapability; import org.opensearch.analytics.spi.FilterCapability; @@ -74,6 +75,11 @@ public Set acceptedDelegations() { public Map scalarFunctionAdapters() { return self.scalarFunctionAdapters(); } + + @Override + public Map delegatedPredicateSerializers() { + return self.delegatedPredicateSerializers(); + } }; } @@ -109,4 +115,8 @@ protected Set acceptedDelegations() { protected Map scalarFunctionAdapters() { return Map.of(); } + + protected Map delegatedPredicateSerializers() { + return Map.of(); + } } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java index a51fc8c100d73..1d2c4f81f16ba 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java @@ -11,11 +11,20 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.analytics.planner.BasePlannerRulesTests; import org.opensearch.analytics.planner.MockDataFusionBackend; +import org.opensearch.analytics.planner.MockLuceneBackend; +import org.opensearch.analytics.planner.PlannerContext; import org.opensearch.analytics.planner.rel.AggregateCallAnnotation; import org.opensearch.analytics.planner.rel.AnnotatedPredicate; import org.opensearch.analytics.planner.rel.AnnotatedProjectExpression; @@ -24,10 +33,19 @@ import org.opensearch.analytics.planner.rel.OpenSearchProject; import org.opensearch.analytics.planner.rel.OpenSearchSort; import org.opensearch.analytics.planner.rel.OpenSearchTableScan; +import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; +import org.opensearch.analytics.spi.DelegatedPredicateFunction; +import org.opensearch.analytics.spi.DelegatedPredicateSerializer; +import org.opensearch.analytics.spi.DelegationType; +import org.opensearch.analytics.spi.FieldStorageInfo; import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.ScalarFunction; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -207,6 +225,295 @@ public void testTwoStageSortOnAggregateOnFilteredScan() { assertShardScanConverted(convertor, dag.rootStage().getChildStages().getFirst()); } + // ---- Delegation tagging tests ---- + + private static final SqlFunction MATCH_PHRASE_FUNCTION = new SqlFunction( + "MATCH_PHRASE", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + private static final SqlFunction FUZZY_FUNCTION = new SqlFunction( + "FUZZY", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BOOLEAN, + null, + OperandTypes.ANY, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + + /** Records serialization calls for delegation tests. */ + private static class RecordingSerializer implements DelegatedPredicateSerializer { + int callCount; + final List serializedFunctions = new ArrayList<>(); + + @Override + public byte[] serialize(RexCall call, List fieldStorage) { + callCount++; + serializedFunctions.add(call.getOperator().getName()); + return ("delegated:" + call.getOperator().getName()).getBytes(StandardCharsets.UTF_8); + } + } + + private List delegationBackends(RecordingConvertor dfConvertor, RecordingSerializer serializer) { + MockDataFusionBackend df = new MockDataFusionBackend() { + @Override + protected Set supportedDelegations() { + return Set.of(DelegationType.FILTER); + } + + @Override + public FragmentConvertor getFragmentConvertor() { + return dfConvertor; + } + }; + MockLuceneBackend lucene = new MockLuceneBackend() { + @Override + protected Set acceptedDelegations() { + return Set.of(DelegationType.FILTER); + } + + @Override + protected Map delegatedPredicateSerializers() { + Map map = new HashMap<>(super.delegatedPredicateSerializers()); + map.put(ScalarFunction.MATCH_PHRASE, serializer); + map.put(ScalarFunction.FUZZY, serializer); + map.put(ScalarFunction.MATCH, serializer); + map.put(ScalarFunction.WILDCARD, serializer); + map.put(ScalarFunction.REGEXP, serializer); + return map; + } + }; + return List.of(df, lucene); + } + + private QueryDAG buildDelegationDag( + RexNode condition, + RecordingConvertor dfConvertor, + RecordingSerializer serializer, + String[] fieldNames, + SqlTypeName[] fieldTypes, + Map> fields + ) { + var backends = delegationBackends(dfConvertor, serializer); + var context = buildContext("parquet", fields, backends); + LogicalFilter filter = LogicalFilter.create(stubScan(mockTable("test_index", fieldNames, fieldTypes)), condition); + RelNode cboOutput = runPlanner(filter, context); + LOGGER.info("Marked+CBO:\n{}", RelOptUtil.toString(cboOutput)); + QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService()); + PlanForker.forkAll(dag, context.getCapabilityRegistry()); + FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()); + return dag; + } + + /** Single-field delegation helper. */ + private QueryDAG buildSingleFieldDelegationDag(RexNode condition, RecordingConvertor dfConvertor, RecordingSerializer serializer) { + return buildDelegationDag( + condition, + dfConvertor, + serializer, + new String[] { "message" }, + new SqlTypeName[] { SqlTypeName.VARCHAR }, + Map.of("message", Map.of("type", "keyword", "index", true)) + ); + } + + /** Two-field delegation helper (integer status + keyword message). */ + private QueryDAG buildTwoFieldDelegationDag(RexNode condition, RecordingConvertor dfConvertor, RecordingSerializer serializer) { + return buildDelegationDag( + condition, + dfConvertor, + serializer, + new String[] { "status", "message" }, + new SqlTypeName[] { SqlTypeName.INTEGER, SqlTypeName.VARCHAR }, + Map.of("status", Map.of("type", "integer", "index", true), "message", Map.of("type", "keyword", "index", true)) + ); + } + + // ---- Shared delegation assertions ---- + + private static Stage leafStage(QueryDAG dag) { + Stage stage = dag.rootStage(); + while (!stage.getChildStages().isEmpty()) { + stage = stage.getChildStages().getFirst(); + } + return stage; + } + + private void assertDelegationResult( + StagePlan plan, + RecordingConvertor dfConvertor, + RecordingSerializer serializer, + int expectedDelegatedCount, + boolean expectPlaceholder, + boolean expectNativeEquals, + List expectedFunctions + ) { + assertEquals("delegatedQueries count", expectedDelegatedCount, plan.delegatedQueries().size()); + assertEquals("serializer call count", expectedDelegatedCount, serializer.callCount); + assertEquals("serialized functions", expectedFunctions, serializer.serializedFunctions); + + String strippedPlan = RelOptUtil.toString(dfConvertor.shardScanFragment); + LOGGER.info("Stripped plan:\n{}", strippedPlan); + + if (expectPlaceholder) { + assertTrue( + "Stripped plan should contain " + DelegatedPredicateFunction.NAME, + strippedPlan.contains(DelegatedPredicateFunction.NAME) + ); + assertFalse("Stripped plan should not contain MATCH_PHRASE", strippedPlan.contains("MATCH_PHRASE")); + assertFalse("Stripped plan should not contain FUZZY", strippedPlan.contains("FUZZY")); + } else { + assertFalse( + "Stripped plan should not contain " + DelegatedPredicateFunction.NAME, + strippedPlan.contains(DelegatedPredicateFunction.NAME) + ); + } + + if (expectNativeEquals) { + assertTrue("Stripped plan should contain native equals", strippedPlan.contains("=")); + } + + // No annotation markers should survive stripping + assertDoesntContainOperators(dfConvertor.shardScanFragment, ANNOTATION_MARKERS); + } + + // ---- Single predicate ---- + + /** Single delegated MATCH_PHRASE — replaced with placeholder, one entry in delegatedQueries. */ + public void testSingleDelegatedPredicate() { + RecordingConvertor dfConvertor = new RecordingConvertor(); + RecordingSerializer serializer = new RecordingSerializer(); + QueryDAG dag = buildSingleFieldDelegationDag(makeFullTextCall(MATCH_PHRASE_FUNCTION, 0, "hello world"), dfConvertor, serializer); + StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); + assertDelegationResult(plan, dfConvertor, serializer, 1, true, false, List.of("MATCH_PHRASE")); + } + + /** Single native equals — no delegation, empty delegatedQueries. */ + public void testSingleNativePredicate() { + RecordingConvertor dfConvertor = new RecordingConvertor(); + RecordingSerializer serializer = new RecordingSerializer(); + QueryDAG dag = buildTwoFieldDelegationDag(makeEquals(0, SqlTypeName.INTEGER, 200), dfConvertor, serializer); + StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); + assertDelegationResult(plan, dfConvertor, serializer, 0, false, true, List.of()); + } + + // ---- AND conditions ---- + + /** AND(native, delegated) — equals unwrapped, MATCH_PHRASE replaced. */ + public void testAndNativeAndDelegated() { + RecordingConvertor dfConvertor = new RecordingConvertor(); + RecordingSerializer serializer = new RecordingSerializer(); + QueryDAG dag = buildTwoFieldDelegationDag( + makeAnd(makeEquals(0, SqlTypeName.INTEGER, 200), makeFullTextCall(MATCH_PHRASE_FUNCTION, 1, "timeout error")), + dfConvertor, + serializer + ); + StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); + assertDelegationResult(plan, dfConvertor, serializer, 1, true, true, List.of("MATCH_PHRASE")); + } + + /** AND(delegated, delegated) — both replaced, two entries in delegatedQueries. */ + public void testAndTwoDelegated() { + RecordingConvertor dfConvertor = new RecordingConvertor(); + RecordingSerializer serializer = new RecordingSerializer(); + QueryDAG dag = buildSingleFieldDelegationDag( + makeAnd(makeFullTextCall(MATCH_PHRASE_FUNCTION, 0, "hello"), makeFullTextCall(FUZZY_FUNCTION, 0, "wrld")), + dfConvertor, + serializer + ); + StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); + assertDelegationResult(plan, dfConvertor, serializer, 2, true, false, List.of("MATCH_PHRASE", "FUZZY")); + } + + // ---- OR conditions ---- + + /** OR(native, delegated) — structure preserved, delegated replaced. */ + public void testOrNativeAndDelegated() { + RecordingConvertor dfConvertor = new RecordingConvertor(); + RecordingSerializer serializer = new RecordingSerializer(); + QueryDAG dag = buildTwoFieldDelegationDag( + rexBuilder.makeCall( + org.apache.calcite.sql.fun.SqlStdOperatorTable.OR, + makeEquals(0, SqlTypeName.INTEGER, 200), + makeFullTextCall(MATCH_PHRASE_FUNCTION, 1, "timeout error") + ), + dfConvertor, + serializer + ); + StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); + assertDelegationResult(plan, dfConvertor, serializer, 1, true, true, List.of("MATCH_PHRASE")); + assertTrue("OR structure should be preserved", RelOptUtil.toString(dfConvertor.shardScanFragment).contains("OR")); + } + + // ---- Interleaved AND/OR/NOT ---- + + /** AND(native, OR(delegated, NOT(delegated))) — nested boolean structure with delegation. */ + public void testInterleavedAndOrNot() { + RecordingConvertor dfConvertor = new RecordingConvertor(); + RecordingSerializer serializer = new RecordingSerializer(); + RexNode notFuzzy = rexBuilder.makeCall( + org.apache.calcite.sql.fun.SqlStdOperatorTable.NOT, + makeFullTextCall(FUZZY_FUNCTION, 1, "wrld") + ); + RexNode orClause = rexBuilder.makeCall( + org.apache.calcite.sql.fun.SqlStdOperatorTable.OR, + makeFullTextCall(MATCH_PHRASE_FUNCTION, 1, "timeout error"), + notFuzzy + ); + RexNode condition = makeAnd(makeEquals(0, SqlTypeName.INTEGER, 200), orClause); + QueryDAG dag = buildTwoFieldDelegationDag(condition, dfConvertor, serializer); + StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); + assertDelegationResult(plan, dfConvertor, serializer, 2, true, true, List.of("MATCH_PHRASE", "FUZZY")); + String strippedPlan = RelOptUtil.toString(dfConvertor.shardScanFragment); + assertTrue("AND structure should be preserved", strippedPlan.contains("AND")); + assertTrue("OR structure should be preserved", strippedPlan.contains("OR")); + assertTrue("NOT structure should be preserved", strippedPlan.contains("NOT")); + } + + // ---- Error paths ---- + + /** Delegated annotation with no serializer registered → IllegalStateException. */ + public void testMissingSerializerThrows() { + RecordingConvertor dfConvertor = new RecordingConvertor(); + // Lucene mock accepts delegation but has NO serializers at all + MockLuceneBackend lucene = new MockLuceneBackend() { + @Override + protected Set acceptedDelegations() { + return Set.of(DelegationType.FILTER); + } + }; + MockDataFusionBackend df = new MockDataFusionBackend() { + @Override + protected Set supportedDelegations() { + return Set.of(DelegationType.FILTER); + } + + @Override + public FragmentConvertor getFragmentConvertor() { + return dfConvertor; + } + }; + Map> fields = Map.of("message", Map.of("type", "keyword", "index", true)); + PlannerContext context = buildContext("parquet", fields, List.of(df, lucene)); + LogicalFilter filter = LogicalFilter.create( + stubScan(mockTable("test_index", new String[] { "message" }, new SqlTypeName[] { SqlTypeName.VARCHAR })), + makeFullTextCall(MATCH_PHRASE_FUNCTION, 0, "hello world") + ); + RelNode marked = runPlanner(filter, context); + QueryDAG dag = DAGBuilder.build(marked, context.getCapabilityRegistry(), mockClusterService()); + PlanForker.forkAll(dag, context.getCapabilityRegistry()); + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> FragmentConversionDriver.convertAll(dag, context.getCapabilityRegistry()) + ); + assertTrue(exception.getMessage().contains("No DelegatedPredicateSerializer")); + assertTrue(exception.getMessage().contains("MATCH_PHRASE")); + } + // ---- RecordingConvertor ---- /** Records which convertor method was called and what was passed. */ From dc59652bea9e9cf3ac7d945a52de53dacac43d72 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Mon, 4 May 2026 12:23:38 -0500 Subject: [PATCH 036/115] Extend OpenSearchTestCase in BlockTransferManagerTests and BlockIndexInputTests (#21355) Switch both to OpenSearchTestCase to inherit the annotation and align with the sibling TransferManagerTestCase precedent, which already uses @ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class) on top of OpenSearchTestCase. Signed-off-by: Andre Kurait --- .../opensearch/storage/indexinput/BlockIndexInputTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/opensearch/storage/indexinput/BlockIndexInputTests.java b/server/src/test/java/org/opensearch/storage/indexinput/BlockIndexInputTests.java index 48bf49379aa04..4a92ec489d1da 100644 --- a/server/src/test/java/org/opensearch/storage/indexinput/BlockIndexInputTests.java +++ b/server/src/test/java/org/opensearch/storage/indexinput/BlockIndexInputTests.java @@ -13,8 +13,8 @@ import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter; +import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; @@ -28,7 +28,7 @@ * Unit tests for BlockIndexInput. */ @ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class) -public class BlockIndexInputTests extends LuceneTestCase { +public class BlockIndexInputTests extends OpenSearchTestCase { private static final String FILE_NAME = "_1.cfe"; private static final String BLOCK_FILE_0 = "_1.cfe_block_0"; From 60633f0c8a03cb8c50e7dfd3928a9339418b6c36 Mon Sep 17 00:00:00 2001 From: Ziwen Wan <38177263+ZiwenWan@users.noreply.github.com> Date: Mon, 4 May 2026 10:43:30 -0700 Subject: [PATCH 037/115] Skip zone awareness when auto-expand replicas set to all (#21217) * Skip zone awareness when auto-expand replicas set to all When index.auto_expand_replicas is set to "0-all", zone awareness checks in AwarenessAllocationDecider were blocking shard placement on clusters where all nodes reside in the same zone, causing under-replication. This change makes the decider return YES immediately when the index is configured to auto-expand to all nodes, bypassing awareness constraints in that case. Relates to opensearch-project/OpenSearch#14603 Original implementation: opensearch-project/OpenSearch#16031 Signed-off-by: ZiwenWan Co-authored-by: Shuyi Z * Add test coverage for AutoExpandReplicas.autoExpandToAll() Extends testParseSettings to assert autoExpandToAll() returns true for "0-all" and "1-all" configurations, false for a fixed upper bound, and false when auto-expand is disabled. Signed-off-by: ZiwenWan --------- Signed-off-by: ZiwenWan Co-authored-by: Shuyi Z --- .../allocation/AwarenessAllocationIT.java | 64 +++++++++++++++++++ .../cluster/metadata/AutoExpandReplicas.java | 4 ++ .../decider/AwarenessAllocationDecider.java | 4 ++ .../metadata/AutoExpandReplicasTests.java | 6 ++ .../allocation/AwarenessAllocationTests.java | 35 ++++++++++ 5 files changed, 113 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java index c0b16b288e1ae..50022a12556c6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java @@ -533,4 +533,68 @@ public void testAwarenessBalanceWithForcedAwarenessCreateAndUpdateIndex() { assertAcked(client().admin().indices().prepareUpdateSettings("test-idx").setSettings(newsettings)); }); } + + public void testAwarenessZonesWithAutoExpand() { + Settings commonSettings = Settings.builder() + .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true) + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a") + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone") + .build(); + + logger.info("--> starting 2 nodes on same zone"); + List nodes = internalCluster().startNodes( + Settings.builder().put(commonSettings).put("node.attr.zone", "a").build(), + Settings.builder().put(commonSettings).put("node.attr.zone", "a").build() + ); + String A = nodes.get(0); + String B = nodes.get(1); + + logger.info("--> waiting for nodes to form a cluster"); + ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet(); + assertThat(health.isTimedOut(), equalTo(false)); + + createIndex( + "test", + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all") + .build() + ); + + if (randomBoolean()) { + assertAcked(client().admin().indices().prepareClose("test")); + } + + logger.info("--> waiting for shards to be allocated"); + health = client().admin() + .cluster() + .prepareHealth() + .setIndices("test") + .setWaitForEvents(Priority.LANGUID) + .setWaitForGreenStatus() + .setWaitForNoRelocatingShards(true) + .execute() + .actionGet(); + assertThat(health.isTimedOut(), equalTo(false)); + + ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); + assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4)); + + final Map counts = new HashMap<>(); + int replicaCount = 0; + + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { + for (ShardRouting shardRouting : indexShardRoutingTable) { + if (!shardRouting.primary()) { + replicaCount++; + } + counts.merge(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1, Integer::sum); + } + } + } + assertThat(counts.get(A), anyOf(equalTo(1), equalTo(2))); + assertThat(counts.get(B), anyOf(equalTo(1), equalTo(2))); + assertThat(replicaCount, equalTo(2)); + } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AutoExpandReplicas.java b/server/src/main/java/org/opensearch/cluster/metadata/AutoExpandReplicas.java index bfc474bc75a53..7ad089755d788 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/AutoExpandReplicas.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/AutoExpandReplicas.java @@ -139,6 +139,10 @@ public boolean isEnabled() { return enabled; } + public boolean autoExpandToAll() { + return enabled && maxReplicas == Integer.MAX_VALUE; + } + private OptionalInt getDesiredNumberOfReplicas(IndexMetadata indexMetadata, RoutingAllocation allocation) { if (enabled) { int numMatchingDataNodes = (int) allocation.nodes() diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java index 17b8aa1d3cbb5..4ff24eb63e5e3 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java @@ -51,6 +51,7 @@ import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING; /** * This {@link AllocationDecider} controls shard allocation based on @@ -161,6 +162,9 @@ private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, Rout } IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index()); + if (INDEX_AUTO_EXPAND_REPLICAS_SETTING.get(indexMetadata.getSettings()).autoExpandToAll()) { + return allocation.decision(Decision.YES, NAME, "allocation awareness is ignored, this index is set to auto-expand to all"); + } int shardCount = shardRouting.isSearchOnly() ? indexMetadata.getNumberOfSearchOnlyReplicas() : indexMetadata.getNumberOfReplicas() + 1; // 1 for primary diff --git a/server/src/test/java/org/opensearch/cluster/metadata/AutoExpandReplicasTests.java b/server/src/test/java/org/opensearch/cluster/metadata/AutoExpandReplicasTests.java index ca59f1af13279..46ac74894a31e 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/AutoExpandReplicasTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/AutoExpandReplicasTests.java @@ -79,17 +79,23 @@ public void testParseSettings() { assertEquals(0, autoExpandReplicas.getMinReplicas()); assertEquals(5, autoExpandReplicas.getMaxReplicas(8)); assertEquals(2, autoExpandReplicas.getMaxReplicas(3)); + assertFalse(autoExpandReplicas.autoExpandToAll()); autoExpandReplicas = AutoExpandReplicas.SETTING.get(Settings.builder().put("index.auto_expand_replicas", "0-all").build()); assertEquals(0, autoExpandReplicas.getMinReplicas()); assertEquals(5, autoExpandReplicas.getMaxReplicas(6)); assertEquals(2, autoExpandReplicas.getMaxReplicas(3)); + assertTrue(autoExpandReplicas.autoExpandToAll()); autoExpandReplicas = AutoExpandReplicas.SETTING.get(Settings.builder().put("index.auto_expand_replicas", "1-all").build()); assertEquals(1, autoExpandReplicas.getMinReplicas()); assertEquals(5, autoExpandReplicas.getMaxReplicas(6)); assertEquals(2, autoExpandReplicas.getMaxReplicas(3)); + assertTrue(autoExpandReplicas.autoExpandToAll()); + autoExpandReplicas = AutoExpandReplicas.SETTING.get(Settings.builder().put("index.auto_expand_replicas", "false").build()); + assertFalse(autoExpandReplicas.isEnabled()); + assertFalse(autoExpandReplicas.autoExpandToAll()); } public void testInvalidValues() { diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessAllocationTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessAllocationTests.java index d954e4675aa9a..d673fefbec405 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessAllocationTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessAllocationTests.java @@ -1119,4 +1119,39 @@ public void testAllocationAwarenessWhenNotEnabled() { decisions.get(0).getExplanation() ); } + + public void testIgnoredByAutoExpandReplicasToAll() { + final Settings settings = Settings.builder() + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone") + .build(); + + final AllocationService strategy = createAllocationService(settings); + + final IndexMetadata.Builder metadataBuilder = IndexMetadata.builder("test") + .settings( + settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 100) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all") + ); + + final Metadata metadata = Metadata.builder().put(metadataBuilder).build(); + + final DiscoveryNodes nodes = DiscoveryNodes.builder() + .add(newNode("A-0", singletonMap("zone", "a"))) + .add(newNode("A-1", singletonMap("zone", "a"))) + .add(newNode("A-2", singletonMap("zone", "a"))) + .add(newNode("B-0", singletonMap("zone", "b"))) + .build(); + + final ClusterState clusterState = applyStartedShardsUntilNoChange( + ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(Settings.EMPTY)) + .metadata(metadata) + .routingTable(RoutingTable.builder().addAsNew(metadata.index("test")).build()) + .nodes(nodes) + .build(), + strategy + ); + + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(0)); + } } From 2b482e7974ff66078cef6b6db0256a16b6053773 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Mon, 4 May 2026 14:45:05 -0500 Subject: [PATCH 038/115] Fail write op when primary closes mid-replication (#21456) Writes from a primary that was demoted during a network partition could be acknowledged to the client even when its replicas would reject replication. The stale primary's in-flight replica requests were cancelled with PrimaryShardClosedException during IndexShard close; WriteActionReplicasProxy.failShardIfNeeded then skipped the cluster manager call (short-circuit introduced in #4133 to fix #803) and, after #21305 added the missing listener.onResponse(null) to that branch, resolved the replication listener successfully, so the ReplicationOperation finished with successful=1, failures=[] and the write was ack'd despite never being applied on any live copy. See [this example failure of ConcurrentSeqNoVersioningIT][1] for how this can produce incorrect results. Intercept PrimaryShardClosedException in ReplicationOperation's replica listener and call finishAsFailed with RetryOnPrimaryException directly. The coordinator then retries against the current primary instead of the client seeing a silent ack. Add a deterministic unit test that reproduces the silent-ack behaviour and would fail without this interceptor. Moving the short-circuit logic up to ReplicationOperation simplifies things and ensures the final fallback is to fail the replica, which is the behavior that existed prior to #4133. [1]: https://build.ci.opensearch.org/job/gradle-check/75218/testReport/org.opensearch.versioning/ConcurrentSeqNoVersioningIT/testSeqNoCASLinearizability/ Signed-off-by: Andrew Ross --- .../replication/ReplicationOperation.java | 13 +++ .../replication/TransportWriteAction.java | 27 ++--- .../ReplicationOperationTests.java | 98 +++++++++++++++++++ .../TransportWriteActionTests.java | 46 --------- 4 files changed, 120 insertions(+), 64 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java index 12d3502184ac4..0fbe4b532596c 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java +++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java @@ -54,6 +54,7 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.rest.RestStatus; import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.index.shard.PrimaryShardClosedException; import org.opensearch.index.shard.ReplicationGroup; import org.opensearch.node.NodeClosedException; import org.opensearch.threadpool.ThreadPool; @@ -273,6 +274,18 @@ public void onFailure(Exception replicaException) { ), replicaException ); + // When the primary shard is closed mid-replication, we can't know whether the replica observed this + // op. Fail the op instead so the coordinator retries against the new primary. + if (ExceptionsHelper.unwrapCause(replicaException) instanceof PrimaryShardClosedException) { + finishAsFailed( + new RetryOnPrimaryException( + primary.routingEntry().shardId(), + "primary shard was closed while replicating to " + shard, + replicaException + ) + ); + return; + } // Only report "critical" exceptions // TODO: Reach out to the cluster-manager node to get the latest shard state then report. if (TransportActions.isShardNotAvailableException(replicaException) == false) { diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java index bea834528dbf2..4249b4fa910a8 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportWriteAction.java @@ -54,7 +54,6 @@ import org.opensearch.index.engine.Engine; import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.PrimaryShardClosedException; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.Translog.Location; import org.opensearch.indices.IndicesService; @@ -573,23 +572,15 @@ public void failShardIfNeeded( if (TransportActions.isShardNotAvailableException(exception) == false) { logger.warn(new ParameterizedMessage("[{}] {}", replica.shardId(), message), exception); } - // If a write action fails due to the closure of the primary shard - // then the replicas should not be marked as failed since they are - // still up-to-date with the (now closed) primary shard - if (exception instanceof PrimaryShardClosedException == false) { - shardStateAction.remoteShardFailed( - replica.shardId(), - replica.allocationId().getId(), - primaryTerm, - true, - message, - exception, - listener - ); - } else { - // always call listener - listener.onResponse(null); - } + shardStateAction.remoteShardFailed( + replica.shardId(), + replica.allocationId().getId(), + primaryTerm, + true, + message, + exception, + listener + ); } @Override diff --git a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java index 6eb697d493bf4..31c775a35b708 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java @@ -67,6 +67,7 @@ import org.opensearch.index.shard.IndexShardNotStartedException; import org.opensearch.index.shard.IndexShardState; import org.opensearch.index.shard.IndexShardTestUtils; +import org.opensearch.index.shard.PrimaryShardClosedException; import org.opensearch.index.shard.ReplicationGroup; import org.opensearch.node.NodeClosedException; import org.opensearch.test.OpenSearchTestCase; @@ -88,6 +89,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -99,6 +101,7 @@ import static org.opensearch.cluster.routing.TestShardRouting.newShardRouting; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -764,6 +767,101 @@ public void failShard(String message, Exception exception) { assertListenerThrows("should throw exception to trigger retry", listener, RetryOnPrimaryException.class); } + public void testPrimaryClosedDuringFullReplicationTriggersRetry() throws Exception { + runPrimaryClosedDuringReplicationTest((replicasProxy, indexShardRoutingTable) -> new FanoutReplicationProxy<>(replicasProxy)); + } + + public void testPrimaryClosedDuringPrimaryTermValidationTriggersRetry() throws Exception { + // Remote-store write path: primary writes to remote, replicas receive only primary-term validation requests via + // ReplicationModeAwareProxy. The PrimaryShardClosedException intercept lives in ReplicationOperation's replica + // listener, so it must trip the retry path regardless of which proxy delivered the failure. + runPrimaryClosedDuringReplicationTest( + (replicasProxy, indexShardRoutingTable) -> new ReplicationModeAwareProxy<>( + ReplicationMode.PRIMARY_TERM_VALIDATION, + buildRemoteStoreEnabledDiscoveryNodes(indexShardRoutingTable), + replicasProxy, + replicasProxy, + true + ) + ); + } + + private void runPrimaryClosedDuringReplicationTest( + BiFunction> proxyFactory + ) throws Exception { + final String index = "test"; + final ShardId shardId = new ShardId(index, "_na_", 0); + + // Deterministic setup: one primary and two started replicas, all tracked. Two replicas so that the non-closed + // replica exercises the successful path alongside the closed one. + final ClusterState initialState = state( + index, + true, + ShardRoutingState.STARTED, + ShardRoutingState.STARTED, + ShardRoutingState.STARTED + ); + IndexMetadata indexMetadata = initialState.getMetadata().index(index); + final long primaryTerm = indexMetadata.primaryTerm(0); + final IndexShardRoutingTable indexShardRoutingTable = initialState.getRoutingTable().shardRoutingTable(shardId); + final ShardRouting primaryShard = indexShardRoutingTable.primaryShard(); + + final Set inSyncAllocationIds = indexMetadata.inSyncAllocationIds(0); + final Set trackedShards = new HashSet<>(); + for (ShardRouting shr : indexShardRoutingTable.shards()) { + trackedShards.add(shr.allocationId().getId()); + } + final ReplicationGroup replicationGroup = new ReplicationGroup(indexShardRoutingTable, inSyncAllocationIds, trackedShards, 0); + final Set expectedReplicas = getExpectedReplicas(shardId, initialState, trackedShards); + assertThat("test requires two replicas", expectedReplicas, hasSize(2)); + final ShardRouting closedReplica = expectedReplicas.iterator().next(); + + // Simulate a PrimaryShardClosedException on the chosen replica's performOn. This mirrors what + // PendingReplicationActions.close() does to in-flight replica requests when IndexShard closes. + final Map simulatedFailures = new HashMap<>(); + simulatedFailures.put(closedReplica, new PrimaryShardClosedException(shardId)); + + final AtomicBoolean failShardCalled = new AtomicBoolean(false); + final TestReplicaProxy replicasProxy = new TestReplicaProxy(simulatedFailures) { + @Override + public void failShardIfNeeded( + ShardRouting replica, + long term, + String message, + Exception exception, + ActionListener shardActionListener + ) { + failShardCalled.set(true); + shardActionListener.onResponse(null); + } + }; + + Request request = new Request(shardId); + PlainActionFuture listener = new PlainActionFuture<>(); + final TestPrimary primary = new TestPrimary(primaryShard, () -> replicationGroup, threadPool); + final TestReplicationOperation op = new TestReplicationOperation( + request, + primary, + listener, + replicasProxy, + primaryTerm, + proxyFactory.apply(replicasProxy, indexShardRoutingTable) + ); + op.execute(); + + assertTrue("request was not processed on primary", request.processedOnPrimary.get()); + assertTrue("listener is not marked as done", listener.isDone()); + assertFalse( + "failShardIfNeeded must not be invoked for PrimaryShardClosedException; the op should fail earlier", + failShardCalled.get() + ); + assertListenerThrows( + "primary shard closed during replication must surface as a retry-able failure, not a silent ack", + listener, + RetryOnPrimaryException.class + ); + } + public void testAddedReplicaAfterPrimaryOperation() throws Exception { final String index = "test"; final ShardId shardId = new ShardId(index, "_na_", 0); diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java index 5b04fcff5df03..cc5eca2233dac 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportWriteActionTests.java @@ -58,7 +58,6 @@ import org.opensearch.index.IndexService; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.shard.PrimaryShardClosedException; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesService; @@ -74,7 +73,6 @@ import org.opensearch.transport.TransportException; import org.opensearch.transport.TransportService; import org.opensearch.transport.client.transport.NoNodeAvailableException; -import org.hamcrest.MatcherAssert; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -97,7 +95,6 @@ import static java.util.Collections.emptyMap; import static org.opensearch.test.ClusterServiceUtils.createClusterService; import static org.hamcrest.Matchers.arrayWithSize; -import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.any; @@ -401,49 +398,6 @@ public void testReplicaProxy() throws InterruptedException, ExecutionException { } } - public void testPrimaryClosedDoesNotFailShard() { - final CapturingTransport transport = new CapturingTransport(); - final TransportService transportService = transport.createTransportService( - clusterService.getSettings(), - threadPool, - TransportService.NOOP_TRANSPORT_INTERCEPTOR, - x -> clusterService.localNode(), - null, - Collections.emptySet(), - NoopTracer.INSTANCE - ); - transportService.start(); - transportService.acceptIncomingRequests(); - final ShardStateAction shardStateAction = new ShardStateAction(clusterService, transportService, null, null, threadPool); - final TestAction action = new TestAction( - Settings.EMPTY, - "internal:testAction", - transportService, - clusterService, - shardStateAction, - threadPool - ); - final String index = "test"; - final ShardId shardId = new ShardId(index, "_na_", 0); - final ClusterState state = ClusterStateCreationUtils.stateWithActivePrimary(index, true, 1, 0); - ClusterServiceUtils.setState(clusterService, state); - final long primaryTerm = state.metadata().index(index).primaryTerm(0); - final ShardRouting shardRouting = state.routingTable().shardRoutingTable(shardId).replicaShards().get(0); - - // Assert that failShardIfNeeded is a no-op for the PrimaryShardClosedException failure - final AtomicInteger callbackCount = new AtomicInteger(0); - action.newReplicasProxy() - .failShardIfNeeded( - shardRouting, - primaryTerm, - "test", - new PrimaryShardClosedException(shardId), - ActionListener.wrap(callbackCount::incrementAndGet) - ); - MatcherAssert.assertThat(transport.getCapturedRequestsAndClear(), emptyArray()); - MatcherAssert.assertThat(callbackCount.get(), equalTo(1)); - } - private class TestAction extends TransportWriteAction { private final boolean withDocumentFailureOnPrimary; From 00a0ea78dc1d64678757a9c5cf2645b9ffa3b716 Mon Sep 17 00:00:00 2001 From: bowenlan Date: Mon, 4 May 2026 17:05:51 -0700 Subject: [PATCH 039/115] Skip root transfer for byte-serialized via handler marker (#21454) * Skip root transfer for byte-serialized responses via handler marker FlightTransportResponse.nextResponse() used to transfer the shared root into a response-owned root unconditionally. Byte-serialized responses copy bytes into Java fields inside handler.read() and don't retain the vectors, so the transfer is only needed on the native Arrow path. --------- Signed-off-by: bowenlan-amzn Signed-off-by: Bowen Lan --- .../arrow/flight/NativeArrowTransportIT.java | 71 ++++- .../transport/ArrowAllocatorProvider.java | 49 ++++ .../flight/transport/ArrowBatchResponse.java | 105 +++---- .../transport/ArrowBatchResponseHandler.java | 30 ++ .../transport/FlightOutboundHandler.java | 33 +-- .../flight/transport/FlightTransport.java | 17 +- .../transport/FlightTransportResponse.java | 26 +- .../MetricsTrackingResponseHandler.java | 5 + .../flight/transport/VectorStreamInput.java | 237 +++++++++------ .../transport/ArrowBatchResponseTests.java | 47 ++- .../ArrowStreamSerializationTests.java | 2 +- .../transport/FlightOutboundHandlerTests.java | 18 +- .../FlightTransportResponseTests.java | 117 ++++++++ .../transport/VectorStreamInputTests.java | 270 ++++++++++++++++-- .../NativeArrowStreamTransportExampleIT.java | 24 +- .../example/stream/ExampleAllocator.java | 24 ++ .../stream/NativeArrowStreamDataResponse.java | 11 +- .../stream/StreamTransportExamplePlugin.java | 39 +++ .../TransportNativeArrowStreamDataAction.java | 27 +- .../TraceableTransportResponseHandler.java | 5 + .../transport/TransportResponseHandler.java | 10 + .../transport/TransportService.java | 5 + 22 files changed, 915 insertions(+), 257 deletions(-) create mode 100644 plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowAllocatorProvider.java create mode 100644 plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseHandler.java create mode 100644 plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/FlightTransportResponseTests.java create mode 100644 plugins/examples/stream-transport-example/src/main/java/org/opensearch/example/stream/ExampleAllocator.java diff --git a/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/NativeArrowTransportIT.java b/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/NativeArrowTransportIT.java index 287a57f244d92..d3363a10d5a2c 100644 --- a/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/NativeArrowTransportIT.java +++ b/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/NativeArrowTransportIT.java @@ -23,8 +23,9 @@ import org.opensearch.action.ActionType; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.TransportAction; +import org.opensearch.arrow.flight.transport.ArrowAllocatorProvider; import org.opensearch.arrow.flight.transport.ArrowBatchResponse; -import org.opensearch.arrow.flight.transport.ArrowFlightChannel; +import org.opensearch.arrow.flight.transport.ArrowBatchResponseHandler; import org.opensearch.arrow.flight.transport.FlightStreamPlugin; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.inject.Inject; @@ -37,7 +38,6 @@ import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.threadpool.ThreadPool; -import org.opensearch.transport.StreamTransportResponseHandler; import org.opensearch.transport.StreamTransportService; import org.opensearch.transport.TransportChannel; import org.opensearch.transport.TransportException; @@ -144,7 +144,7 @@ public void testBatchesSurviveStreamAdvanceAndClose() throws Exception { TestArrowAction.NAME, new TestArrowRequest(batchCount, rowsPerBatch, 1), TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STREAM).build(), - new StreamTransportResponseHandler() { + new ArrowBatchResponseHandler() { @Override public void handleStreamResponse(StreamTransportResponse streamResponse) { try { @@ -264,7 +264,7 @@ private void assertBatchIntegrity(ReceivedBatch batch) { } } - /** Deep-copies data from a VectorSchemaRoot. */ + /** Deep-copies data out of the Arrow batch so the root can be closed immediately. */ static class ReceivedBatch { final int rowCount; final int batchId; @@ -272,11 +272,11 @@ static class ReceivedBatch { final List names; final List values; - ReceivedBatch(VectorSchemaRoot root) { - this.rowCount = root.getRowCount(); - IntVector batchIdVector = (IntVector) root.getVector("batch_id"); - VarCharVector nameVector = (VarCharVector) root.getVector("name"); - IntVector valueVector = (IntVector) root.getVector("value"); + ReceivedBatch(VectorSchemaRoot batch) { + this.rowCount = batch.getRowCount(); + IntVector batchIdVector = (IntVector) batch.getVector("batch_id"); + VarCharVector nameVector = (VarCharVector) batch.getVector("name"); + IntVector valueVector = (IntVector) batch.getVector("value"); this.batchIds = new ArrayList<>(); this.names = new ArrayList<>(); this.values = new ArrayList<>(); @@ -349,11 +349,29 @@ private TestArrowAction() { * batches via sendResponseBatch(). The framework does zero-copy transfer * on the executor thread. */ + public static class TestAllocatorHolder { + private final BufferAllocator allocator; + + TestAllocatorHolder(BufferAllocator allocator) { + this.allocator = allocator; + } + + BufferAllocator get() { + return allocator; + } + } + public static class TransportTestArrowAction extends TransportAction { + private final BufferAllocator allocator; @Inject - public TransportTestArrowAction(StreamTransportService streamTransportService, ActionFilters actionFilters) { + public TransportTestArrowAction( + StreamTransportService streamTransportService, + ActionFilters actionFilters, + TestAllocatorHolder allocatorHolder + ) { super(TestArrowAction.NAME, actionFilters, streamTransportService.getTaskManager()); + this.allocator = allocatorHolder.get(); streamTransportService.registerRequestHandler( TestArrowAction.NAME, ThreadPool.Names.GENERIC, @@ -368,7 +386,6 @@ protected void doExecute(Task task, TestArrowRequest request, ActionListener { + static class TestArrowResponseHandler extends ArrowBatchResponseHandler { private final List batches; private final CountDownLatch latch; private final AtomicReference failure; @@ -453,13 +470,15 @@ public void handleStreamResponse(StreamTransportResponse stre try { TestArrowResponse response; while ((response = streamResponse.nextResponse()) != null) { - batches.add(new ReceivedBatch(response.getRoot())); + try (VectorSchemaRoot batch = response.getRoot()) { + batches.add(new ReceivedBatch(batch)); + } } streamResponse.close(); - latch.countDown(); } catch (Exception e) { failure.set(e); streamResponse.cancel("Test error", e); + } finally { latch.countDown(); } } @@ -482,11 +501,35 @@ public TestArrowResponse read(StreamInput in) throws IOException { } public static class NativeArrowTestPlugin extends Plugin implements ActionPlugin { + private final BufferAllocator allocator = ArrowAllocatorProvider.newChildAllocator("native-arrow-test", Long.MAX_VALUE); + public NativeArrowTestPlugin() {} + @Override + public Collection createComponents( + org.opensearch.transport.client.Client client, + org.opensearch.cluster.service.ClusterService clusterService, + ThreadPool threadPool, + org.opensearch.watcher.ResourceWatcherService resourceWatcherService, + org.opensearch.script.ScriptService scriptService, + org.opensearch.core.xcontent.NamedXContentRegistry xContentRegistry, + org.opensearch.env.Environment environment, + org.opensearch.env.NodeEnvironment nodeEnvironment, + org.opensearch.core.common.io.stream.NamedWriteableRegistry namedWriteableRegistry, + org.opensearch.cluster.metadata.IndexNameExpressionResolver indexNameExpressionResolver, + java.util.function.Supplier repositoriesServiceSupplier + ) { + return List.of(new TestAllocatorHolder(allocator)); + } + @Override public List> getActions() { return List.of(new ActionHandler<>(TestArrowAction.INSTANCE, TransportTestArrowAction.class)); } + + @Override + public void close() { + allocator.close(); + } } } diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowAllocatorProvider.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowAllocatorProvider.java new file mode 100644 index 0000000000000..30a23928954e5 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowAllocatorProvider.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.transport; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.security.AccessController; +import java.security.PrivilegedAction; + +/** + * Node-level Arrow allocator shared across plugins. + * + *

      Every caller of {@link #newChildAllocator(String, long)} gets a child of one + * {@link RootAllocator}. Cross-plugin buffer handoffs (e.g., producer → Flight stream, + * Flight stream → consumer) pass Arrow's {@link org.apache.arrow.memory.AllocationManager} + * associate check, which requires {@code source.getRoot() == target.getRoot()}. + * + * @opensearch.experimental + */ +@ExperimentalApi +@SuppressWarnings("removal") +public final class ArrowAllocatorProvider { + + private static final RootAllocator ROOT = AccessController.doPrivileged( + (PrivilegedAction) () -> new RootAllocator(Long.MAX_VALUE) + ); + + private ArrowAllocatorProvider() {} + + /** + * Creates a named child of the shared root with an independent memory limit. + * Callers own the returned allocator and must close it. + * + * @param name descriptive name for debugging (e.g., "flight", "analytics-search") + * @param limit maximum bytes this child can allocate + * @return a new child allocator + */ + public static BufferAllocator newChildAllocator(String name, long limit) { + return ROOT.newChildAllocator(name, 0, limit); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponse.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponse.java index c5091007b6f91..e66f4fc47d2d3 100644 --- a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponse.java +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponse.java @@ -17,87 +17,90 @@ import java.io.IOException; /** - * Base class for transport responses carrying native Arrow data. + * Base class for transport responses carrying native Arrow data. Subclasses must provide + * two constructors — one for sending (wraps a populated root) and one for receiving + * (takes ownership of vectors from the Flight stream via {@link StreamInput}): * - *

      The producer creates vectors using the channel's allocator and populates them freely - * on any thread. When the executor processes this batch, it does a zero-copy transfer - * of the producer's buffers into the channel's shared root — no memcpy, no serialization. - * After transfer, the framework closes the producer's root, releasing its buffers back - * to the allocator. + * 

      {@code
+ * public class MyResponse extends ArrowBatchResponse {
+ *     public MyResponse(VectorSchemaRoot root) { super(root); }       // send side
+ *     public MyResponse(StreamInput in) throws IOException { super(in); } // receive side
+ * }
+ * }
      * - *

      Allocator guidelines: The allocator used for producer roots must outlive the - * gRPC stream — do not create and close a child allocator per request. gRPC's zero-copy - * write path retains buffer references beyond stream completion, and closing the allocator - * while gRPC still holds these references causes memory accounting errors. Use either the - * channel allocator (via {@code ArrowFlightChannel.from(channel).getAllocator()}) or a - * long-lived application allocator. The framework creates the shared root from the - * producer's allocator to ensure same-allocator transfer, which avoids an Arrow bug with - * cross-allocator transfer of foreign-backed buffers from C data import. + *

      Send side: The producer populates a {@link VectorSchemaRoot} and wraps it. + * The framework zero-copy transfers the vectors into the Flight stream — no memcpy, + * no serialization. * - *

      Usage (send side): * 

      {@code
- * BufferAllocator allocator = ArrowFlightChannel.from(channel).getAllocator();
  * VectorSchemaRoot producerRoot = VectorSchemaRoot.create(schema, allocator);
- * // populate producerRoot on any thread...
+ * // populate producerRoot...
  * channel.sendResponseBatch(new MyResponse(producerRoot));
  * // producerRoot is now owned by the framework — don't reuse or close it
  * }
      * - *

      Usage (receive side): - * 

      {@code
- * public class MyResponse extends ArrowBatchResponse {
- *     public MyResponse(VectorSchemaRoot root) { super(root); }
- *     public MyResponse(StreamInput in) throws IOException { super(in); }
- * }
- * }
      + *

      Receive side: The framework calls {@code handler.read(in)} where {@code in} is + * a {@link VectorStreamInput.NativeArrow} holding vectors transferred from the Flight stream. + * The {@link #ArrowBatchResponse(StreamInput)} constructor claims ownership of those vectors. + * + *

      Allocator rules: + *

        + *
      • Send side: Use a child of {@link ArrowAllocatorProvider}. All allocators + * must share the same root so zero-copy transfers pass Arrow's + * {@code AllocationManager} associate check. The framework creates the Flight + * stream root from the producer's allocator to ensure same-allocator transfer — + * this avoids an Arrow bug with cross-allocator transfer of foreign-backed + * buffers from C data import.
        • + *
      • Send side: Allocators must outlive the gRPC stream — gRPC's zero-copy write + * path retains buffer references beyond stream completion. Do not create and close a + * child allocator per request.
        • + *
      • Receive side: The framework transfers vectors from the Flight stream's + * allocator into the response. The consumer can then transfer them into its own + * allocator — which must also be a child of {@link ArrowAllocatorProvider}.
        • + *
      * * @opensearch.experimental */ @ExperimentalApi public abstract class ArrowBatchResponse extends ActionResponse { - private final VectorSchemaRoot producerRoot; + private final VectorSchemaRoot batchRoot; /** - * Creates a response with the given producer root (send side). - * @param producerRoot the root populated by the producer + * Send-side constructor: wraps a root populated by the producer. + * @param batchRoot the root to send; ownership transfers to the transport */ - protected ArrowBatchResponse(VectorSchemaRoot producerRoot) { - this.producerRoot = producerRoot; + protected ArrowBatchResponse(VectorSchemaRoot batchRoot) { + this.batchRoot = batchRoot; } /** - * Deserializes a response from a StreamInput (receive side). - * @param in the stream input containing the Arrow root - * @throws IOException if deserialization fails + * Receive-side constructor: claims ownership of the consumer root from the input. + * @param in must be a {@link VectorStreamInput.NativeArrow}; throws otherwise + * @throws IOException if reading fails */ protected ArrowBatchResponse(StreamInput in) throws IOException { super(in); - this.producerRoot = ((VectorStreamInput) in).getRoot(); + if (in instanceof VectorStreamInput.NativeArrow nativeIn) { + this.batchRoot = nativeIn.getRoot(); + nativeIn.claimOwnership(); + } else { + throw new IllegalStateException( + "ArrowBatchResponse decoded from a non-native-Arrow StreamInput (" + + (in == null ? "null" : in.getClass().getName()) + + "). Wrapping handlers around ArrowBatchResponseHandler must forward " + + "TransportResponseHandler#skipsDeserialization()." + ); + } } - /** - * Returns the producer's root. On the send side, this is the root populated - * by the producer. On the receive side, this is the root from the Flight stream. - */ + /** Returns the Arrow root holding the response vectors. */ public VectorSchemaRoot getRoot() { - return producerRoot; - } - - /** - * Zero-copy transfers the producer's vectors into the target root. - * Called by the framework on the executor thread before {@code putNext()}. - * After transfer, the producer's buffers are moved to the target — the producer - * root becomes empty. - * - * @param target the channel's shared root (bound to the Flight stream via start()) - */ - void transferTo(VectorSchemaRoot target) { - FlightUtils.transferRoot(producerRoot, target); + return batchRoot; } @Override public final void writeTo(StreamOutput out) throws IOException { - // no-op: the framework handles transfer via transferTo() + // no-op: the framework transfers vectors directly via FlightUtils.transferRoot() } } diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseHandler.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseHandler.java new file mode 100644 index 0000000000000..6083c9bde388f --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseHandler.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.transport; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.transport.StreamTransportResponseHandler; + +/** + * Receive-side base for handlers that consume {@link ArrowBatchResponse}. Pins + * {@link #skipsDeserialization()} to {@code true} so the Flight transport routes to the native + * Arrow path. + * + * @opensearch.experimental + */ +@ExperimentalApi +public abstract class ArrowBatchResponseHandler implements StreamTransportResponseHandler { + /** Constructor. */ + protected ArrowBatchResponseHandler() {} + + @Override + public final boolean skipsDeserialization() { + return true; + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightOutboundHandler.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightOutboundHandler.java index eb0f90b83c675..76d7840ec3712 100644 --- a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightOutboundHandler.java +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightOutboundHandler.java @@ -6,17 +6,10 @@ * compatible open source license. */ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - package org.opensearch.arrow.flight.transport; import org.apache.arrow.flight.FlightRuntimeException; +import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; @@ -36,6 +29,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.List; import java.util.Set; /** @@ -154,21 +148,22 @@ private void processBatchTask(BatchTask task) { try { VectorStreamOutput out; if (task.response() instanceof ArrowBatchResponse arrowResponse) { - // Native Arrow path: zero-copy transfer producer's vectors into shared root - VectorSchemaRoot sharedRoot = flightChannel.getRoot(); - if (sharedRoot == null) { - // Create shared root using the producer's allocator for same-allocator transfer. + // Native Arrow path: zero-copy transfer producer's vectors into stream root + VectorSchemaRoot streamRoot = flightChannel.getRoot(); + if (streamRoot == null) { + // Create stream root using the producer's allocator for same-allocator transfer. // This avoids an Arrow bug where cross-allocator transferOwnership of foreign-backed // buffers (from C data import) doesn't properly free the ArrowArray C struct. // The producer's allocator must be long-lived (not closed per-request). - sharedRoot = VectorSchemaRoot.create( - arrowResponse.getRoot().getSchema(), - arrowResponse.getRoot().getFieldVectors().get(0).getAllocator() - ); + List fieldVectors = arrowResponse.getRoot().getFieldVectors(); + if (fieldVectors.isEmpty()) { + throw new IllegalStateException("Native Arrow batch has no field vectors"); + } + streamRoot = VectorSchemaRoot.create(arrowResponse.getRoot().getSchema(), fieldVectors.getFirst().getAllocator()); } - arrowResponse.transferTo(sharedRoot); - arrowResponse.getRoot().close(); // release producer's buffers — safe, they've been moved - out = VectorStreamOutput.forNativeArrow(sharedRoot); + FlightUtils.transferRoot(arrowResponse.getRoot(), streamRoot); + arrowResponse.getRoot().close(); + out = VectorStreamOutput.forNativeArrow(streamRoot); } else { out = VectorStreamOutput.create(flightChannel.getAllocator(), flightChannel.getRoot()); task.response().writeTo(out); diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransport.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransport.java index 15800c5245254..cb1ac42587b8d 100644 --- a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransport.java +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransport.java @@ -16,7 +16,6 @@ import org.apache.arrow.flight.OSFlightClient; import org.apache.arrow.flight.OSFlightServer; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.Version; @@ -54,8 +53,6 @@ import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -97,7 +94,7 @@ class FlightTransport extends TcpTransport { private final AtomicInteger nextExecutorIndex = new AtomicInteger(0); private final ThreadPool threadPool; - private RootAllocator rootAllocator; + private BufferAllocator flightAllocator; private BufferAllocator serverAllocator; private BufferAllocator clientAllocator; @@ -146,14 +143,14 @@ public FlightTransport( protected void doStart() { boolean success = false; try { - rootAllocator = AccessController.doPrivileged((PrivilegedAction) () -> new RootAllocator(Integer.MAX_VALUE)); - serverAllocator = rootAllocator.newChildAllocator("server", 0, rootAllocator.getLimit()); - clientAllocator = rootAllocator.newChildAllocator("client", 0, rootAllocator.getLimit()); + flightAllocator = ArrowAllocatorProvider.newChildAllocator("flight", Integer.MAX_VALUE); + serverAllocator = flightAllocator.newChildAllocator("server", 0, flightAllocator.getLimit()); + clientAllocator = flightAllocator.newChildAllocator("client", 0, flightAllocator.getLimit()); if (statsCollector != null) { - statsCollector.setBufferAllocator(rootAllocator); + statsCollector.setBufferAllocator(flightAllocator); statsCollector.setThreadPool(threadPool); } - flightProducer = new ArrowFlightProducer(this, rootAllocator, SERVER_HEADER_KEY, statsCollector); + flightProducer = new ArrowFlightProducer(this, flightAllocator, SERVER_HEADER_KEY, statsCollector); bindServer(); success = true; if (statsCollector != null) { @@ -268,7 +265,7 @@ protected void stopInternal() { } serverAllocator.close(); clientAllocator.close(); - rootAllocator.close(); + flightAllocator.close(); gracefullyShutdownELG(bossEventLoopGroup, "os-grpc-boss-ELG"); gracefullyShutdownELG(workerEventLoopGroup, "os-grpc-worker-ELG"); diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransportResponse.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransportResponse.java index 9a1588a882326..bcbcdbfd9ee73 100644 --- a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransportResponse.java +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/FlightTransportResponse.java @@ -44,6 +44,7 @@ class FlightTransportResponse implements StreamTran private final NamedWriteableRegistry namedWriteableRegistry; private final HeaderContext headerContext; private final TransportResponseHandler handler; + private final boolean isNativeHandler; private final FlightTransportConfig config; private final long correlationId; @@ -64,6 +65,7 @@ class FlightTransportResponse implements StreamTran FlightTransportConfig config ) { this.handler = Objects.requireNonNull(handler); + this.isNativeHandler = handler.skipsDeserialization(); this.correlationId = correlationId; this.flightClient = Objects.requireNonNull(flightClient); this.headerContext = Objects.requireNonNull(headerContext); @@ -121,10 +123,9 @@ public T nextResponse() { boolean hasNext = firstBatchConsumed ? flightStream.next() : (firstBatchConsumed = true); if (!hasNext) return null; - VectorSchemaRoot sharedRoot = flightStream.getRoot(); - currentBatchSize = FlightUtils.calculateVectorSchemaRootSize(sharedRoot); - VectorSchemaRoot ownedRoot = transferToOwnedRoot(sharedRoot); - try (VectorStreamInput input = new VectorStreamInput(ownedRoot, namedWriteableRegistry)) { + VectorSchemaRoot streamRoot = flightStream.getRoot(); + currentBatchSize = FlightUtils.calculateVectorSchemaRootSize(streamRoot); + try (VectorStreamInput input = newStreamInput(streamRoot)) { input.setVersion(initialHeader.getVersion()); return handler.read(input); } @@ -145,19 +146,10 @@ long getCurrentBatchSize() { return currentBatchSize; } - /** - * Transfers the shared root's vectors into a response-owned root so the returned response - * is independent of FlightStream's lifecycle. The next call to {@code flightStream.next()} - * clears the shared root, and {@code stream.close()} releases its vectors — both would wipe - * the data out from under an async listener if we handed back a reference to the shared root. - */ - private static VectorSchemaRoot transferToOwnedRoot(VectorSchemaRoot sharedRoot) { - VectorSchemaRoot ownedRoot = VectorSchemaRoot.create( - sharedRoot.getSchema(), - sharedRoot.getFieldVectors().getFirst().getAllocator() - ); - FlightUtils.transferRoot(sharedRoot, ownedRoot); - return ownedRoot; + private VectorStreamInput newStreamInput(VectorSchemaRoot streamRoot) { + return isNativeHandler + ? VectorStreamInput.forNativeArrow(streamRoot, namedWriteableRegistry) + : VectorStreamInput.forByteSerialized(streamRoot, namedWriteableRegistry); } @Override diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/MetricsTrackingResponseHandler.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/MetricsTrackingResponseHandler.java index 04d22e5746141..2768ce106ddbe 100644 --- a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/MetricsTrackingResponseHandler.java +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/MetricsTrackingResponseHandler.java @@ -94,6 +94,11 @@ public String executor() { return delegate.executor(); } + @Override + public boolean skipsDeserialization() { + return delegate.skipsDeserialization(); + } + /** * A stream response wrapper that tracks metrics for batches. */ diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/VectorStreamInput.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/VectorStreamInput.java index 6951805560572..7393679e890ce 100644 --- a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/VectorStreamInput.java +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/transport/VectorStreamInput.java @@ -22,93 +22,68 @@ /** * A {@link StreamInput} backed by a {@link VectorSchemaRoot} from the Flight transport. * + *

      Two factories, mirroring {@link VectorStreamOutput}: + *

        + *
      • {@link #forByteSerialized} — reads bytes directly from the stream root. Used when the + * response is not an {@link ArrowBatchResponse}: {@code handler.read()} copies bytes into + * the response's Java fields, so no ownership transfer is needed.
        • + *
      • {@link #forNativeArrow} — zero-copy transfers the stream root's vectors into a + * consumer root before reading, so the returned {@link ArrowBatchResponse} is + * independent of the FlightStream lifecycle.
        • + *
      + * + *

      The caller ({@link FlightTransportResponse#nextResponse}) picks the factory based on whether + * the registered handler is an {@link ArrowBatchResponseHandler}. + * * @opensearch.internal */ -class VectorStreamInput extends StreamInput { +abstract class VectorStreamInput extends StreamInput { - private final VarBinaryVector vector; - private final VectorSchemaRoot root; - private final NamedWriteableRegistry registry; - private int row = 0; - private ByteBuffer buffer = null; + protected final VectorSchemaRoot root; + protected final NamedWriteableRegistry registry; - /** - * Creates a new VectorStreamInput. - * @param root the Arrow root containing the data - * @param registry the named writeable registry - */ - public VectorStreamInput(VectorSchemaRoot root, NamedWriteableRegistry registry) { + protected VectorStreamInput(VectorSchemaRoot root, NamedWriteableRegistry registry) { this.root = root; - vector = (VarBinaryVector) root.getVector("0"); this.registry = registry; } /** - * Returns the underlying {@link VectorSchemaRoot}. + * Byte-serialized path: the stream root carries a single {@code VarBinary} column of chunked + * bytes written by {@link VectorStreamOutput.ByteSerialized}. Reads are over the stream root; + * FlightStream retains ownership. */ - public VectorSchemaRoot getRoot() { - return root; + static VectorStreamInput forByteSerialized(VectorSchemaRoot streamRoot, NamedWriteableRegistry registry) { + return new ByteSerialized(streamRoot, registry); } - @Override - public byte readByte() throws IOException { - // Check if buffer has remaining bytes - if (buffer != null && buffer.hasRemaining()) { - return buffer.get(); - } - // No buffer or buffer exhausted, read from vector - if (row >= vector.getValueCount()) { - throw new EOFException("No more rows available in vector"); + /** + * Transfers the stream root's vectors into a consumer root so the returned response + * outlives the next FlightStream batch. The consumer root is released by {@link NativeArrow#close()} + * unless the response takes ownership via {@link NativeArrow#claimOwnership()}. + */ + static VectorStreamInput forNativeArrow(VectorSchemaRoot streamRoot, NamedWriteableRegistry registry) { + if (streamRoot.getFieldVectors().isEmpty()) { + throw new IllegalStateException("Native Arrow batch has no field vectors"); } - byte[] v = vector.get(row++); - if (v.length == 0) { - throw new IOException("Empty byte array in vector at row " + (row - 1)); + VectorSchemaRoot consumerRoot = VectorSchemaRoot.create( + streamRoot.getSchema(), + streamRoot.getFieldVectors().getFirst().getAllocator() + ); + try { + FlightUtils.transferRoot(streamRoot, consumerRoot); + } catch (Throwable t) { + consumerRoot.close(); + throw t; } - // Wrap the byte array in buffer for future reads - buffer = ByteBuffer.wrap(v); - return buffer.get(); // Read the first byte + return new NativeArrow(consumerRoot, registry); } - @Override - public void readBytes(byte[] b, int offset, int len) throws IOException { - if (offset < 0 || len < 0 || offset + len > b.length) { - throw new IllegalArgumentException("Invalid offset or length"); - } - int remaining = len; - - // First, exhaust any remaining bytes in the buffer - if (buffer != null && buffer.hasRemaining()) { - int bufferBytes = Math.min(buffer.remaining(), remaining); - buffer.get(b, offset, bufferBytes); - offset += bufferBytes; - remaining -= bufferBytes; - if (!buffer.hasRemaining()) { - buffer = null; // Clear buffer if exhausted - } - } - - // Read from vector if more bytes are needed - while (remaining > 0) { - if (row >= vector.getValueCount()) { - throw new EOFException("No more rows available in vector"); - } - byte[] v = vector.get(row++); - if (v.length == 0) { - throw new IOException("Empty byte array in vector at row " + (row - 1)); - } - if (v.length <= remaining) { - // The entire vector row can be consumed - System.arraycopy(v, 0, b, offset, v.length); - offset += v.length; - remaining -= v.length; - } else { - // Partial read from vector row - System.arraycopy(v, 0, b, offset, remaining); - // Store remaining bytes in buffer without copying - buffer = ByteBuffer.wrap(v, remaining, v.length - remaining); - remaining = 0; - } - } + /** + * Returns the underlying {@link VectorSchemaRoot}. For {@link NativeArrow} this is the + * consumer root; {@link ArrowBatchResponse} grabs it via the receive-side constructor. + */ + public VectorSchemaRoot getRoot() { + return root; } @Override @@ -129,25 +104,127 @@ public NamedWriteableRegistry namedWriteableRegistry() { return registry; } - @Override - public void close() throws IOException { - if (vector != null) { - vector.close(); - } - } - @Override public int read() throws IOException { throw new UnsupportedOperationException(); } @Override - public int available() throws IOException { + public int available() { throw new UnsupportedOperationException(); } + /** + * No-op: bounds checks happen at read time, not as a pre-check. + * {@link ByteSerialized#readByte} and {@link ByteSerialized#readBytes} throw + * {@link EOFException} when the column is exhausted. + */ @Override - protected void ensureCanReadBytes(int length) throws EOFException { + protected void ensureCanReadBytes(int length) {} + // ── Byte serialization ── + + static final class ByteSerialized extends VectorStreamInput { + private final VarBinaryVector vector; + private int row = 0; + private ByteBuffer buffer = null; + + ByteSerialized(VectorSchemaRoot root, NamedWriteableRegistry registry) { + super(root, registry); + this.vector = (VarBinaryVector) root.getVector("0"); + } + + @Override + public byte readByte() throws IOException { + if (buffer != null && buffer.hasRemaining()) { + return buffer.get(); + } + if (row >= vector.getValueCount()) { + throw new EOFException("No more rows available in vector"); + } + byte[] v = vector.get(row++); + if (v.length == 0) { + throw new IOException("Empty byte array in vector at row " + (row - 1)); + } + buffer = ByteBuffer.wrap(v); + return buffer.get(); + } + + @Override + public void readBytes(byte[] b, int offset, int len) throws IOException { + if (offset < 0 || len < 0 || offset + len > b.length) { + throw new IllegalArgumentException("Invalid offset or length"); + } + int remaining = len; + + if (buffer != null && buffer.hasRemaining()) { + int bufferBytes = Math.min(buffer.remaining(), remaining); + buffer.get(b, offset, bufferBytes); + offset += bufferBytes; + remaining -= bufferBytes; + if (!buffer.hasRemaining()) { + buffer = null; + } + } + + while (remaining > 0) { + if (row >= vector.getValueCount()) { + throw new EOFException("No more rows available in vector"); + } + byte[] v = vector.get(row++); + if (v.length == 0) { + throw new IOException("Empty byte array in vector at row " + (row - 1)); + } + if (v.length <= remaining) { + System.arraycopy(v, 0, b, offset, v.length); + offset += v.length; + remaining -= v.length; + } else { + System.arraycopy(v, 0, b, offset, remaining); + buffer = ByteBuffer.wrap(v, remaining, v.length - remaining); + remaining = 0; + } + } + } + + /** + * No-op: the stream root belongs to {@link org.apache.arrow.flight.FlightStream}, which + * clears the vectors on the next {@code next()} and closes them on stream close. + */ + @Override + public void close() {} + } + + // ── Native Arrow ── + + static final class NativeArrow extends VectorStreamInput { + private boolean transferred = false; + + NativeArrow(VectorSchemaRoot root, NamedWriteableRegistry registry) { + super(root, registry); + } + + @Override + public byte readByte() { + throw new UnsupportedOperationException("Native Arrow responses read vectors directly from getRoot()"); + } + + @Override + public void readBytes(byte[] b, int offset, int len) { + throw new UnsupportedOperationException("Native Arrow responses read vectors directly from getRoot()"); + } + + /** Response claims the consumer root; {@link #close()} becomes a no-op. */ + void claimOwnership() { + transferred = true; + } + + /** Releases the consumer root unless {@link #claimOwnership()} was called. */ + @Override + public void close() { + if (!transferred && root != null) { + root.close(); + } + } } } diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseTests.java index 03582398f600e..ffb3c36946484 100644 --- a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseTests.java +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowBatchResponseTests.java @@ -84,8 +84,7 @@ public void testTransferToMovesBuffers() { src.setRowCount(2); VectorSchemaRoot dst = VectorSchemaRoot.create(schema, allocator); - TestResponse response = new TestResponse(src); - response.transferTo(dst); + FlightUtils.transferRoot(src, dst); assertEquals(2, dst.getRowCount()); IntVector dstVec = (IntVector) dst.getVector("val"); @@ -115,9 +114,9 @@ public void testDestinationSurvivesSourceClose() { src.setRowCount(2); VectorSchemaRoot dst = VectorSchemaRoot.create(schema, allocator); - new TestResponse(src).transferTo(dst); + FlightUtils.transferRoot(src, dst); - // Close the source — simulates FlightStream clearing/closing its shared root. + // Close the source — simulates FlightStream clearing/closing its stream root. src.close(); assertEquals(2, dst.getRowCount()); @@ -129,6 +128,44 @@ public void testDestinationSurvivesSourceClose() { dst.close(); } + public void testStreamInputConstructorCapturesRootAndMarksTransferred() throws IOException { + VectorSchemaRoot shared = VectorSchemaRoot.create(schema, allocator); + ((IntVector) shared.getVector("val")).allocateNew(); + ((IntVector) shared.getVector("val")).setSafe(0, 42); + ((IntVector) shared.getVector("val")).setValueCount(1); + shared.setRowCount(1); + + org.opensearch.core.common.io.stream.NamedWriteableRegistry registry = + new org.opensearch.core.common.io.stream.NamedWriteableRegistry(java.util.Collections.emptyList()); + VectorStreamInput.NativeArrow in = (VectorStreamInput.NativeArrow) VectorStreamInput.forNativeArrow(shared, registry); + VectorSchemaRoot consumerRoot = in.getRoot(); + + TestResponse response = new TestResponse(in); + assertSame(consumerRoot, response.getRoot()); + + // claimOwnership must have fired — close() is a no-op, consumer root survives. + in.close(); + assertEquals(42, ((IntVector) response.getRoot().getVector("val")).get(0)); + + response.getRoot().close(); + shared.close(); + } + + public void testStreamInputConstructorRejectsByteSerializedInput() throws IOException { + VectorSchemaRoot shared = VectorSchemaRoot.create( + new Schema(List.of(new Field("0", FieldType.nullable(new ArrowType.Binary()), null))), + allocator + ); + org.opensearch.core.common.io.stream.NamedWriteableRegistry registry = + new org.opensearch.core.common.io.stream.NamedWriteableRegistry(java.util.Collections.emptyList()); + try (VectorStreamInput in = VectorStreamInput.forByteSerialized(shared, registry)) { + IllegalStateException e = expectThrows(IllegalStateException.class, () -> new TestResponse(in)); + assertTrue("message should point at skipsDeserialization()", e.getMessage().contains("skipsDeserialization")); + } finally { + shared.close(); + } + } + public void testTransferToWithMultipleVectors() { Schema multiSchema = new Schema( List.of( @@ -147,7 +184,7 @@ public void testTransferToWithMultipleVectors() { src.setRowCount(1); VectorSchemaRoot dst = VectorSchemaRoot.create(multiSchema, allocator); - new TestResponse(src).transferTo(dst); + FlightUtils.transferRoot(src, dst); assertEquals(1, dst.getRowCount()); assertEquals(1, ((IntVector) dst.getVector("a")).get(0)); diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowStreamSerializationTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowStreamSerializationTests.java index e85225bae0c42..d4b2603120513 100644 --- a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowStreamSerializationTests.java +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/ArrowStreamSerializationTests.java @@ -54,7 +54,7 @@ public void testInternalAggregationSerializationDeserialization() throws IOExcep output.writeNamedWriteable(original); VectorSchemaRoot unifiedRoot = output.getRoot(); - try (VectorStreamInput input = new VectorStreamInput(unifiedRoot, registry)) { + try (VectorStreamInput input = VectorStreamInput.forByteSerialized(unifiedRoot, registry)) { StringTerms deserialized = input.readNamedWriteable(StringTerms.class); assertEquals(String.valueOf(original), String.valueOf(deserialized)); } diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/FlightOutboundHandlerTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/FlightOutboundHandlerTests.java index f4eed5d2a36f2..93e3551c9d9c8 100644 --- a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/FlightOutboundHandlerTests.java +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/FlightOutboundHandlerTests.java @@ -224,7 +224,7 @@ public void testProcessBatchTaskNativeArrowFirstBatch() throws Exception { vec.setValueCount(1); producerRoot.setRowCount(1); - // First batch: sharedRoot is null, so it should be created + // First batch: streamRoot is null, so it should be created when(mockFlightChannel.getRoot()).thenReturn(null); CountDownLatch latch = new CountDownLatch(1); @@ -242,7 +242,7 @@ public void testProcessBatchTaskNativeArrowFirstBatch() throws Exception { assertNotNull(sentRoot); assertEquals(1, sentRoot.getRowCount()); assertEquals(42, ((IntVector) sentRoot.getVector("val")).get(0)); - // Clean up the shared root created by the handler + // Clean up the stream root created by the handler sentRoot.close(); return null; }).when(mockFlightChannel).sendBatch(any(), any(VectorStreamOutput.class)); @@ -265,13 +265,13 @@ public void testProcessBatchTaskNativeArrowFirstBatch() throws Exception { } } - public void testProcessBatchTaskNativeArrowWithExistingSharedRoot() throws Exception { + public void testProcessBatchTaskNativeArrowWithExistingStreamRoot() throws Exception { try (RootAllocator allocator = new RootAllocator()) { Schema schema = new Schema(List.of(new Field("val", FieldType.nullable(new ArrowType.Int(32, true)), null))); - // Simulate existing shared root (second batch scenario) - VectorSchemaRoot sharedRoot = VectorSchemaRoot.create(schema, allocator); - when(mockFlightChannel.getRoot()).thenReturn(sharedRoot); + // Simulate existing stream root (second batch scenario) + VectorSchemaRoot streamRoot = VectorSchemaRoot.create(schema, allocator); + when(mockFlightChannel.getRoot()).thenReturn(streamRoot); VectorSchemaRoot producerRoot = VectorSchemaRoot.create(schema, allocator); IntVector vec = (IntVector) producerRoot.getVector("val"); @@ -285,8 +285,8 @@ public void testProcessBatchTaskNativeArrowWithExistingSharedRoot() throws Excep doAnswer(invocation -> { VectorStreamOutput out = invocation.getArgument(1); VectorSchemaRoot sentRoot = out.getRoot(); - // Should reuse the existing shared root - assertSame(sharedRoot, sentRoot); + // Should reuse the existing stream root + assertSame(streamRoot, sentRoot); assertEquals(1, sentRoot.getRowCount()); assertEquals(99, ((IntVector) sentRoot.getVector("val")).get(0)); return null; @@ -311,7 +311,7 @@ public void testProcessBatchTaskNativeArrowWithExistingSharedRoot() throws Excep ); assertTrue("Task should complete", latch.await(5, TimeUnit.SECONDS)); - sharedRoot.close(); + streamRoot.close(); } } diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/FlightTransportResponseTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/FlightTransportResponseTests.java new file mode 100644 index 0000000000000..65592d6a38245 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/FlightTransportResponseTests.java @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.transport; + +import org.opensearch.core.transport.TransportResponse; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.transport.StreamTransportResponseHandler; +import org.opensearch.transport.TransportException; +import org.opensearch.transport.TransportResponseHandler; + +import java.io.IOException; + +public class FlightTransportResponseTests extends OpenSearchTestCase { + + public void testArrowHandlerSkipsDeserialization() { + assertTrue(new TestArrowHandler().skipsDeserialization()); + } + + public void testNonArrowHandlerDoesNotSkip() { + assertFalse(new TestByteHandler().skipsDeserialization()); + } + + public void testWrapperForwardsTrueFromArrowHandler() { + assertTrue(new ForwardingWrapper<>(new TestArrowHandler()).skipsDeserialization()); + } + + public void testWrapperForwardsFalseFromNonArrowHandler() { + assertFalse(new ForwardingWrapper<>(new TestByteHandler()).skipsDeserialization()); + } + + public void testRealMetricsTrackingWrapperForwards() { + // MetricsTrackingResponseHandler in production path; null tracker is fine for this check. + MetricsTrackingResponseHandler wrapped = new MetricsTrackingResponseHandler<>(new TestArrowHandler(), null); + assertTrue(wrapped.skipsDeserialization()); + } + + private static final class TestArrowHandler extends ArrowBatchResponseHandler { + @Override + public TestArrowResponse read(org.opensearch.core.common.io.stream.StreamInput in) { + throw new UnsupportedOperationException(); + } + + @Override + public void handleResponse(TestArrowResponse response) {} + + @Override + public void handleException(TransportException exp) {} + + @Override + public String executor() { + return "same"; + } + } + + private static final class TestByteHandler implements StreamTransportResponseHandler { + @Override + public TransportResponse read(org.opensearch.core.common.io.stream.StreamInput in) { + throw new UnsupportedOperationException(); + } + + @Override + public void handleResponse(TransportResponse response) {} + + @Override + public void handleException(TransportException exp) {} + + @Override + public String executor() { + return "same"; + } + } + + private static final class ForwardingWrapper implements TransportResponseHandler { + private final TransportResponseHandler delegate; + + ForwardingWrapper(TransportResponseHandler delegate) { + this.delegate = delegate; + } + + @Override + public T read(org.opensearch.core.common.io.stream.StreamInput in) throws IOException { + return delegate.read(in); + } + + @Override + public void handleResponse(T response) { + delegate.handleResponse(response); + } + + @Override + public void handleException(TransportException exp) { + delegate.handleException(exp); + } + + @Override + public String executor() { + return delegate.executor(); + } + + @Override + public boolean skipsDeserialization() { + return delegate.skipsDeserialization(); + } + } + + private static final class TestArrowResponse extends ArrowBatchResponse { + TestArrowResponse() { + super((org.apache.arrow.vector.VectorSchemaRoot) null); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/VectorStreamInputTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/VectorStreamInputTests.java index 37b470dc29bdc..1e4734b88a404 100644 --- a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/VectorStreamInputTests.java +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/transport/VectorStreamInputTests.java @@ -10,6 +10,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -21,6 +22,7 @@ import org.junit.After; import org.junit.Before; +import java.io.IOException; import java.util.Collections; import java.util.List; @@ -44,27 +46,263 @@ public void tearDown() throws Exception { super.tearDown(); } - public void testGetRootReturnsRoot() { - Schema schema = new Schema(List.of(new Field("0", FieldType.nullable(new ArrowType.Binary()), null))); - VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); - VarBinaryVector vec = (VarBinaryVector) root.getVector("0"); + public void testByteSerializedReadsFromSharedRoot() throws IOException { + VectorSchemaRoot shared = newByteSerializedRoot(); + try (VectorStreamInput input = VectorStreamInput.forByteSerialized(shared, registry)) { + assertTrue(input instanceof VectorStreamInput.ByteSerialized); + assertSame("ByteSerialized holds the stream root — no transfer", shared, input.getRoot()); + } + shared.close(); + } + + public void testNativeArrowTransfersIntoOwnedRoot() throws IOException { + VectorSchemaRoot shared = newNativeArrowRoot(); + IntVector srcVec = (IntVector) shared.getVector("val"); + srcVec.allocateNew(); + srcVec.setSafe(0, 42); + srcVec.setValueCount(1); + shared.setRowCount(1); + + VectorStreamInput.NativeArrow input = (VectorStreamInput.NativeArrow) VectorStreamInput.forNativeArrow(shared, registry); + try { + assertNotSame("NativeArrow transfers into a fresh consumer root", shared, input.getRoot()); + IntVector dstVec = (IntVector) input.getRoot().getVector("val"); + assertEquals(1, input.getRoot().getRowCount()); + assertEquals(42, dstVec.get(0)); + assertEquals("source must be drained", 0, shared.getRowCount()); + + // Close the stream root immediately — consumer root must survive. + shared.close(); + assertEquals("consumer root survives stream root close", 42, dstVec.get(0)); + + // Simulate ArrowBatchResponse taking ownership, then close the consumer root on the response side. + input.claimOwnership(); + } finally { + input.close(); // no-op after claimOwnership + input.getRoot().close(); + } + } + + public void testByteSerializedCloseIsNoOpOnSharedRoot() throws IOException { + VectorSchemaRoot shared = newByteSerializedRoot(); + VarBinaryVector vec = (VarBinaryVector) shared.getVector("0"); vec.allocateNew(); - vec.setValueCount(0); - root.setRowCount(0); + vec.setSafe(0, new byte[] { 1, 2, 3 }); + vec.setValueCount(1); + shared.setRowCount(1); + + VectorStreamInput input = VectorStreamInput.forByteSerialized(shared, registry); + input.close(); + + // Shared root must remain fully usable — FlightStream owns its lifecycle. + assertEquals(1, shared.getRowCount()); + assertEquals(3, ((VarBinaryVector) shared.getVector("0")).get(0).length); + shared.close(); + } + + public void testNativeArrowCloseReleasesRootIfNotTransferred() throws IOException { + // read() throws or never runs: the consumer root must be released by close(), not leaked. + VectorSchemaRoot shared = newNativeArrowRoot(); + IntVector srcVec = (IntVector) shared.getVector("val"); + srcVec.allocateNew(); + srcVec.setSafe(0, 7); + srcVec.setValueCount(1); + shared.setRowCount(1); - VectorStreamInput input = new VectorStreamInput(root, registry); - assertSame(root, input.getRoot()); - root.close(); + long beforeClose; + try (VectorStreamInput.NativeArrow input = (VectorStreamInput.NativeArrow) VectorStreamInput.forNativeArrow(shared, registry)) { + beforeClose = allocator.getAllocatedMemory(); + assertTrue("consumer root should hold memory before close", beforeClose > 0); + } + // After try-with-resources: close() ran, transferred==false, root should be released. + assertTrue( + "consumer root must be released when not transferred (was " + beforeClose + ", now " + allocator.getAllocatedMemory() + ")", + allocator.getAllocatedMemory() < beforeClose + ); + shared.close(); } - public void testCloseWithNullVector() throws Exception { - // Create a root with no vector named "0" so vector field is null - Schema schema = new Schema(List.of(new Field("other", FieldType.nullable(new ArrowType.Utf8()), null))); - VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + public void testNativeArrowCloseIsNoOpAfterMarkTransferred() throws IOException { + // ArrowBatchResponse(StreamInput) calls claimOwnership to take ownership. + // After that, close() must leave the root alone so the response can use it. + VectorSchemaRoot shared = newNativeArrowRoot(); + IntVector srcVec = (IntVector) shared.getVector("val"); + srcVec.allocateNew(); + srcVec.setSafe(0, 7); + srcVec.setValueCount(1); + shared.setRowCount(1); - VectorStreamInput input = new VectorStreamInput(root, registry); - // close() should not throw even though vector is null + VectorStreamInput.NativeArrow input = (VectorStreamInput.NativeArrow) VectorStreamInput.forNativeArrow(shared, registry); + VectorSchemaRoot consumerRoot = input.getRoot(); + input.claimOwnership(); input.close(); - root.close(); + + // Consumer root must remain usable — ArrowBatchResponse owns it after handoff. + assertEquals(1, consumerRoot.getRowCount()); + assertEquals(7, ((IntVector) consumerRoot.getVector("val")).get(0)); + consumerRoot.close(); + shared.close(); + } + + public void testForNativeArrowRejectsEmptySchema() { + Schema emptySchema = new Schema(List.of()); + VectorSchemaRoot shared = VectorSchemaRoot.create(emptySchema, allocator); + try { + IllegalStateException e = expectThrows(IllegalStateException.class, () -> VectorStreamInput.forNativeArrow(shared, registry)); + assertTrue(e.getMessage().contains("no field vectors")); + } finally { + shared.close(); + } + } + + public void testByteSerializedReadsBytesFromSharedVector() throws IOException { + VectorSchemaRoot shared = newByteSerializedRoot(); + VarBinaryVector vec = (VarBinaryVector) shared.getVector("0"); + vec.allocateNew(); + vec.setSafe(0, new byte[] { 10, 20, 30 }); + vec.setValueCount(1); + shared.setRowCount(1); + + try (VectorStreamInput input = VectorStreamInput.forByteSerialized(shared, registry)) { + assertEquals((byte) 10, input.readByte()); + assertEquals((byte) 20, input.readByte()); + assertEquals((byte) 30, input.readByte()); + } + shared.close(); + } + + public void testNativeArrowRejectsByteReads() throws IOException { + VectorSchemaRoot shared = newNativeArrowRoot(); + try (VectorStreamInput input = VectorStreamInput.forNativeArrow(shared, registry)) { + expectThrows(UnsupportedOperationException.class, input::readByte); + expectThrows(UnsupportedOperationException.class, () -> input.readBytes(new byte[1], 0, 1)); + // Do not call input.getRoot().close() — the try-with-resources close() releases the + // consumer root (transferred==false). + } + shared.close(); + } + + public void testReadByteEofWhenRowsExhausted() throws IOException { + VectorSchemaRoot shared = newByteSerializedRoot(); + ((VarBinaryVector) shared.getVector("0")).allocateNew(); + shared.setRowCount(0); + try (VectorStreamInput input = VectorStreamInput.forByteSerialized(shared, registry)) { + expectThrows(java.io.EOFException.class, input::readByte); + } + shared.close(); + } + + public void testReadByteRejectsEmptyRow() throws IOException { + VectorSchemaRoot shared = newByteSerializedRoot(); + VarBinaryVector vec = (VarBinaryVector) shared.getVector("0"); + vec.allocateNew(); + vec.setSafe(0, new byte[0]); + vec.setValueCount(1); + shared.setRowCount(1); + try (VectorStreamInput input = VectorStreamInput.forByteSerialized(shared, registry)) { + expectThrows(IOException.class, input::readByte); + } + shared.close(); + } + + public void testReadBytesInvalidOffsetThrows() throws IOException { + VectorSchemaRoot shared = newByteSerializedRoot(); + try (VectorStreamInput input = VectorStreamInput.forByteSerialized(shared, registry)) { + byte[] target = new byte[4]; + expectThrows(IllegalArgumentException.class, () -> input.readBytes(target, -1, 2)); + expectThrows(IllegalArgumentException.class, () -> input.readBytes(target, 0, -1)); + expectThrows(IllegalArgumentException.class, () -> input.readBytes(target, 3, 5)); + } + shared.close(); + } + + public void testReadBytesSpansMultipleRowsWithLeftover() throws IOException { + // Row 0: 3 bytes, row 1: 4 bytes. Read 5 bytes — spans both rows, leaves 2 in buffer + // for a follow-up readByte. + VectorSchemaRoot shared = newByteSerializedRoot(); + VarBinaryVector vec = (VarBinaryVector) shared.getVector("0"); + vec.allocateNew(); + vec.setSafe(0, new byte[] { 1, 2, 3 }); + vec.setSafe(1, new byte[] { 4, 5, 6, 7 }); + vec.setValueCount(2); + shared.setRowCount(2); + + try (VectorStreamInput input = VectorStreamInput.forByteSerialized(shared, registry)) { + byte[] out = new byte[5]; + input.readBytes(out, 0, 5); + assertArrayEquals(new byte[] { 1, 2, 3, 4, 5 }, out); + // Remaining buffered bytes from row 1 feed readByte. + assertEquals((byte) 6, input.readByte()); + assertEquals((byte) 7, input.readByte()); + } + shared.close(); + } + + public void testReadBytesEofWhenRowsExhausted() throws IOException { + VectorSchemaRoot shared = newByteSerializedRoot(); + VarBinaryVector vec = (VarBinaryVector) shared.getVector("0"); + vec.allocateNew(); + vec.setSafe(0, new byte[] { 1, 2 }); + vec.setValueCount(1); + shared.setRowCount(1); + + try (VectorStreamInput input = VectorStreamInput.forByteSerialized(shared, registry)) { + byte[] out = new byte[4]; + expectThrows(java.io.EOFException.class, () -> input.readBytes(out, 0, 4)); + } + shared.close(); + } + + public void testReadBytesRejectsEmptyRow() throws IOException { + VectorSchemaRoot shared = newByteSerializedRoot(); + VarBinaryVector vec = (VarBinaryVector) shared.getVector("0"); + vec.allocateNew(); + vec.setSafe(0, new byte[0]); + vec.setValueCount(1); + shared.setRowCount(1); + + try (VectorStreamInput input = VectorStreamInput.forByteSerialized(shared, registry)) { + byte[] out = new byte[2]; + expectThrows(IOException.class, () -> input.readBytes(out, 0, 2)); + } + shared.close(); + } + + public void testReadBytesZeroLengthIsNoOp() throws IOException { + VectorSchemaRoot shared = newByteSerializedRoot(); + try (VectorStreamInput input = VectorStreamInput.forByteSerialized(shared, registry)) { + input.readBytes(new byte[4], 0, 0); // must not throw, must not advance + } + shared.close(); + } + + public void testReadBytesDrainsBufferThenAdvancesRow() throws IOException { + // readByte advances row 0 into the internal buffer; readBytes must then drain the + // buffer (1 byte left) before pulling row 1. + VectorSchemaRoot shared = newByteSerializedRoot(); + VarBinaryVector vec = (VarBinaryVector) shared.getVector("0"); + vec.allocateNew(); + vec.setSafe(0, new byte[] { 10, 20 }); + vec.setSafe(1, new byte[] { 30, 40 }); + vec.setValueCount(2); + shared.setRowCount(2); + + try (VectorStreamInput input = VectorStreamInput.forByteSerialized(shared, registry)) { + assertEquals((byte) 10, input.readByte()); + byte[] out = new byte[3]; + input.readBytes(out, 0, 3); + assertArrayEquals(new byte[] { 20, 30, 40 }, out); + } + shared.close(); + } + + private VectorSchemaRoot newByteSerializedRoot() { + Schema schema = new Schema(List.of(new Field("0", FieldType.nullable(new ArrowType.Binary()), null))); + return VectorSchemaRoot.create(schema, allocator); + } + + private VectorSchemaRoot newNativeArrowRoot() { + Schema schema = new Schema(List.of(new Field("val", FieldType.nullable(new ArrowType.Int(32, true)), null))); + return VectorSchemaRoot.create(schema, allocator); } } diff --git a/plugins/examples/stream-transport-example/src/internalClusterTest/java/org/opensearch/example/stream/NativeArrowStreamTransportExampleIT.java b/plugins/examples/stream-transport-example/src/internalClusterTest/java/org/opensearch/example/stream/NativeArrowStreamTransportExampleIT.java index 4cb4e68fc5889..bcd44d8dae736 100644 --- a/plugins/examples/stream-transport-example/src/internalClusterTest/java/org/opensearch/example/stream/NativeArrowStreamTransportExampleIT.java +++ b/plugins/examples/stream-transport-example/src/internalClusterTest/java/org/opensearch/example/stream/NativeArrowStreamTransportExampleIT.java @@ -11,13 +11,14 @@ import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Field; +import org.opensearch.arrow.flight.transport.ArrowBatchResponseHandler; import org.opensearch.arrow.flight.transport.FlightStreamPlugin; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.threadpool.ThreadPool; -import org.opensearch.transport.StreamTransportResponseHandler; import org.opensearch.transport.StreamTransportService; import org.opensearch.transport.TransportException; import org.opensearch.transport.TransportRequestOptions; @@ -112,18 +113,18 @@ public void testNativeArrowMultipleBatches() throws Exception { } } - /** Deep-copies data from the root since FlightStream reuses it between next() calls. */ + /** Deep-copies data out of the Arrow batch so the root can be closed immediately. */ static class ReceivedBatch { final int rowCount; final List fieldNames; final List names; final List ages; - ReceivedBatch(VectorSchemaRoot root) { - this.rowCount = root.getRowCount(); - this.fieldNames = root.getSchema().getFields().stream().map(f -> f.getName()).toList(); - VarCharVector nameVector = (VarCharVector) root.getVector("name"); - IntVector ageVector = (IntVector) root.getVector("age"); + ReceivedBatch(VectorSchemaRoot batch) { + this.rowCount = batch.getRowCount(); + this.fieldNames = batch.getSchema().getFields().stream().map(Field::getName).toList(); + VarCharVector nameVector = (VarCharVector) batch.getVector("name"); + IntVector ageVector = (IntVector) batch.getVector("age"); this.names = new ArrayList<>(); this.ages = new ArrayList<>(); for (int i = 0; i < rowCount; i++) { @@ -133,8 +134,7 @@ static class ReceivedBatch { } } - /** Standard handler — read() uses the normal StreamInput contract. */ - static class NativeArrowResponseHandler implements StreamTransportResponseHandler { + static class NativeArrowResponseHandler extends ArrowBatchResponseHandler { private final List batches; private final CountDownLatch latch; private final AtomicReference failure; @@ -150,13 +150,15 @@ public void handleStreamResponse(StreamTransportResponseThe framework handles everything: - *

        - *
      • Send side: zero-copy transfers the root's buffers into the Flight stream
        • - *
      • Receive side: provides the root via {@link #getRoot()} — no deserialization
        • - *
      - * - *

      No writeTo/read override needed. The base class handles both. + * Example native Arrow response. Extend {@link ArrowBatchResponse} and provide two constructors: + * one wrapping a {@link VectorSchemaRoot} (send side) and one taking {@link StreamInput} (receive side). */ class NativeArrowStreamDataResponse extends ArrowBatchResponse { diff --git a/plugins/examples/stream-transport-example/src/main/java/org/opensearch/example/stream/StreamTransportExamplePlugin.java b/plugins/examples/stream-transport-example/src/main/java/org/opensearch/example/stream/StreamTransportExamplePlugin.java index bbc5952ca3f17..b862e7f046601 100644 --- a/plugins/examples/stream-transport-example/src/main/java/org/opensearch/example/stream/StreamTransportExamplePlugin.java +++ b/plugins/examples/stream-transport-example/src/main/java/org/opensearch/example/stream/StreamTransportExamplePlugin.java @@ -8,17 +8,51 @@ package org.opensearch.example.stream; +import org.apache.arrow.memory.BufferAllocator; import org.opensearch.action.ActionRequest; +import org.opensearch.arrow.flight.transport.ArrowAllocatorProvider; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.script.ScriptService; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.client.Client; +import org.opensearch.watcher.ResourceWatcherService; +import java.util.Collection; import java.util.List; +import java.util.function.Supplier; public class StreamTransportExamplePlugin extends Plugin implements ActionPlugin { + private final BufferAllocator allocator = ArrowAllocatorProvider.newChildAllocator("stream-transport-example", Long.MAX_VALUE); + public StreamTransportExamplePlugin() {} + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + return List.of(new ExampleAllocator(allocator)); + } + @Override public List> getActions() { return List.of( @@ -26,4 +60,9 @@ public StreamTransportExamplePlugin() {} new ActionHandler<>(NativeArrowStreamDataAction.INSTANCE, TransportNativeArrowStreamDataAction.class) ); } + + @Override + public void close() { + allocator.close(); + } } diff --git a/plugins/examples/stream-transport-example/src/main/java/org/opensearch/example/stream/TransportNativeArrowStreamDataAction.java b/plugins/examples/stream-transport-example/src/main/java/org/opensearch/example/stream/TransportNativeArrowStreamDataAction.java index 99fee3d870eb1..2d71bad7ca9de 100644 --- a/plugins/examples/stream-transport-example/src/main/java/org/opensearch/example/stream/TransportNativeArrowStreamDataAction.java +++ b/plugins/examples/stream-transport-example/src/main/java/org/opensearch/example/stream/TransportNativeArrowStreamDataAction.java @@ -18,7 +18,6 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.TransportAction; -import org.opensearch.arrow.flight.transport.ArrowFlightChannel; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; import org.opensearch.tasks.Task; @@ -37,24 +36,26 @@ * *

      Demonstrates the pipelined producer pattern: *

        - *
      1. Get the channel's allocator via {@link ArrowFlightChannel#from(TransportChannel)}
        2. - *
      2. For each batch, create a producer root using the channel allocator
        3. - *
      3. Populate the root with typed vectors (VarChar, Int, etc.)
        4. - *
      4. Send via {@code sendResponseBatch()} — the framework does zero-copy transfer - * of the producer's buffers into the channel's shared root on the executor thread
        5. - *
      5. The producer root is closed by the framework after transfer — don't reuse it
        6. + *
      6. Receive an allocator owned by the plugin (closed in {@link StreamTransportExamplePlugin#close()})
        7. + *
      7. For each batch, create a {@link VectorSchemaRoot}, populate it, and wrap it in a response
        8. + *
      8. Send via {@code sendResponseBatch()} — the framework zero-copy transfers + * the vectors into the Flight stream on the executor thread
        9. + *
      9. Call {@code completeStream()} when done
        10. *
      - * - *

      The channel allocator must be used directly (not a per-request child allocator) - * because gRPC's zero-copy write path retains buffer references beyond stream completion. */ public class TransportNativeArrowStreamDataAction extends TransportAction { private static final String[] NAMES = { "Alice", "Bob", "Carol", "Dave", "Eve" }; + private final BufferAllocator allocator; @Inject - public TransportNativeArrowStreamDataAction(StreamTransportService streamTransportService, ActionFilters actionFilters) { + public TransportNativeArrowStreamDataAction( + StreamTransportService streamTransportService, + ActionFilters actionFilters, + ExampleAllocator exampleAllocator + ) { super(NativeArrowStreamDataAction.NAME, actionFilters, streamTransportService.getTaskManager()); + this.allocator = exampleAllocator.get(); streamTransportService.registerRequestHandler( NativeArrowStreamDataAction.NAME, ThreadPool.Names.GENERIC, @@ -69,10 +70,6 @@ protected void doExecute(Task task, NativeArrowStreamDataRequest request, Action } private void handleStreamRequest(NativeArrowStreamDataRequest request, TransportChannel channel, Task task) throws IOException { - // Get the channel's allocator. Use this directly for producer roots to ensure - // same-allocator transfer (avoids Arrow's cross-allocator foreign buffer bug). - BufferAllocator allocator = ArrowFlightChannel.from(channel).getAllocator(); - Schema schema = new Schema( List.of( new Field("name", FieldType.nullable(new ArrowType.Utf8()), null), diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java index 5d3bd6c4daf73..9cb3b90f31084 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/handler/TraceableTransportResponseHandler.java @@ -100,6 +100,11 @@ public String executor() { return delegate.executor(); } + @Override + public boolean skipsDeserialization() { + return delegate.skipsDeserialization(); + } + @Override public String toString() { return delegate.toString(); diff --git a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java index d7c14eaf53303..541debca344c0 100644 --- a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java +++ b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java @@ -102,6 +102,16 @@ default void handleStreamResponse(StreamTransportResponse response) { */ default void handleRejection(Exception exp) {} + /** + * True if this handler consumes the response payload directly (e.g. Flight's native Arrow + * path) instead of going through byte-level deserialization. Wrappers must forward their + * delegate's value. + */ + @ExperimentalApi + default boolean skipsDeserialization() { + return false; + } + default TransportResponseHandler wrap(Function converter, Writeable.Reader reader) { final TransportResponseHandler self = this; return new TransportResponseHandler() { diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index b5ec44b96480f..37dbb8e3a69da 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -1613,6 +1613,11 @@ public String executor() { return delegate.executor(); } + @Override + public boolean skipsDeserialization() { + return delegate.skipsDeserialization(); + } + @Override public String toString() { return getClass().getName() + "/" + delegate.toString(); From 8b609bd36112963169ba337fe91538b43bfb126d Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Mon, 4 May 2026 18:28:15 -0700 Subject: [PATCH 040/115] Enable PPL fillnull on the analytics-engine route via DataFusion COALESCE (#21472) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Analytics Backend / DataFusion] Wire CEIL into project capabilities DataFusion natively supports CEIL; declaring it as a project capability lets the analytics-engine planner route Project nodes containing CEIL through DataFusion. Same pattern as the COALESCE addition: one entry in STANDARD_PROJECT_OPS, no convertor changes needed (Substrait default extensions handle the conversion). Exercised by CalciteFillNullCommandIT.testFillNullWithFunctionOnOtherField, which calls 'fillnull with ceil(num1) in num0' — the COALESCE replacement is a CEIL of another field. Without CEIL declared project-capable, the planner would fail with 'No backend supports scalar function [CEIL] among [datafusion]' even after COALESCE is wired. CEIL is also one of the ~100 Bucket-1 functions in the PPL→DataFusion audit (direct DataFusion mapping, S0). This commit is the templated shape for any other Bucket-1 scalar — add the constant to STANDARD_PROJECT_OPS, no other changes. Signed-off-by: Kai Huang * [Analytics Engine] Strip nested AnnotatedProjectExpression wrappers recursively OpenSearchProjectRule.annotateExpr recurses into a project call's operands, wrapping every sub-call in an AnnotatedProjectExpression so the planner can track viable backends per sub-expression. The previous OpenSearchProject.stripAnnotations only unwrapped the top-level wrapper — for a project like PLUS(CEIL(value), value), the strip left an inner Annotated(CEIL(...)) sitting inside the outer call. Substrait isthmus has no converter for ANNOTATED_PROJECT_EXPR, so conversion failed with 'Unable to convert call ANNOTATED_PROJECT_EXPR'. Walk each project expression with a RexShuttle that recursively unwraps every AnnotatedProjectExpression it encounters before handing the plan off to the backend FragmentConvertor. The shuttle's super.visitCall keeps the operand-traversal default; the override intercepts wrappers and re-feeds the unwrapped result through the shuttle so deeply-nested wrappers also get stripped. Adds ProjectRuleTests#testStripAnnotationsRecursivelyUnwrapsNestedExpressions covering the nested-call shape. The bug is independent of which scalar functions any backend declares project-capable; it would fire for any project with a nested scalar call once both the outer and inner operators are project-capable on the same backend. Signed-off-by: Kai Huang * [Analytics Backend / DataFusion] Wire COALESCE into project capabilities for PPL fillnull PPL fillnull lowers via CalciteRelNodeVisitor.visitFillNull into a LogicalProject(COALESCE(field, replacement)) tree. The analytics-engine planner's OpenSearchProjectRule rejects any project-list scalar call unless at least one viable backend declares it project-capable, with 'No backend supports scalar function [COALESCE] among [datafusion]'. DataFusion natively supports COALESCE, but the backend plugin had not declared any projectCapabilities, so every fillnull query failed the planner check before reaching Substrait conversion. Add a STANDARD_PROJECT_OPS set mirroring the existing STANDARD_FILTER_OPS pattern, seeded with COALESCE, and override projectCapabilities() to fan it out across the same field-types and formats the backend already advertises for filters. No Rust-side or convertor changes are needed — the default Substrait extension catalog loaded by DataFusionPlugin.loadSubstraitExtensions handles COALESCE natively, and DataFusion's runtime executes it directly. Lifts CalciteFillNullCommandIT on the force-routed analytics-engine path from 2/13 passing (only the SQL-plugin preflight type-check tests) to 12/13 passing. The remaining failure is testFillNullWithFunctionOnOtherField, which uses a nested CEIL inside the COALESCE replacement — handled in follow-up commits. Signed-off-by: Kai Huang * [QA] Add FillNullCommandIT for the analytics-engine REST path Self-contained integration test for PPL fillnull on the analytics-engine route, mirroring CalciteFillNullCommandIT from opensearch-project/sql so the analytics-engine path can be verified inside core without cross-plugin dependencies on the SQL plugin. Each test sends a PPL query through POST /_analytics/ppl (exposed by the test-ppl-frontend plugin) which runs the same UnifiedQueryPlanner → CalciteRelNodeVisitor → Substrait → DataFusion pipeline as the SQL plugin's force-routed analytics path. Covers all 13 fillnull surface forms: - with X in fields — single value, named fields - with X in f1,f2 — single value, multiple fields - using f1=X, f2=Y — per-field replacement - using f0=f1 — replacement is a field reference - with ceil(num1) in num0 — replacement contains a nested call (exercises recursive AnnotatedProjectExpression strip) - chained fillnulls — multiple commands - value=X (all fields) — Calcite-specific syntax - value=X f1 — Calcite-specific, named fields - value='N/A' str2 — Calcite-specific, string value - using int0=8589934592 — BIGINT into INTEGER (numeric coercion) - mixed-type errors — preflight type-incompatibility validation Provisions the 'calcs' dataset (parquet-backed) once per class via DatasetProvisioner (which uses refresh=true, sidestepping the LuceneCommitter.getSafeCommitInfo TODO that hangs refresh=wait_for in the SQL plugin's IT path). Result: 13/13 pass against the QA-module's testcluster — no SQL plugin, no force_routing setting, no parquet-IT system properties needed. Signed-off-by: Kai Huang --------- Signed-off-by: Kai Huang --- .../DataFusionAnalyticsBackendPlugin.java | 18 + .../planner/rel/OpenSearchProject.java | 22 +- .../analytics/planner/ProjectRuleTests.java | 44 ++ .../analytics/qa/FillNullCommandIT.java | 445 ++++++++++++++++++ .../test/resources/datasets/calcs/bulk.json | 35 ++ .../resources/datasets/calcs/mapping.json | 98 ++++ 6 files changed, 661 insertions(+), 1 deletion(-) create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FillNullCommandIT.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/calcs/bulk.json create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/calcs/mapping.json diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index b77bd4aef0954..09993221052c9 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -17,6 +17,7 @@ import org.opensearch.analytics.spi.FieldType; import org.opensearch.analytics.spi.FilterCapability; import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.ProjectCapability; import org.opensearch.analytics.spi.ScalarFunction; import org.opensearch.analytics.spi.ScanCapability; import org.opensearch.analytics.spi.SearchExecEngineProvider; @@ -61,6 +62,13 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP ScalarFunction.LIKE ); + // Project-side scalar functions DataFusion can evaluate natively. Each entry corresponds to a + // PPL command/function we want the analytics-engine planner to route through DataFusion. Add + // here only after verifying the function deserializes through Substrait isthmus into a plan + // DataFusion's native runtime can execute (see DataFusionFragmentConvertor for the conversion + // path). COALESCE is the lowering target of PPL `fillnull`. + private static final Set STANDARD_PROJECT_OPS = Set.of(ScalarFunction.COALESCE, ScalarFunction.CEIL); + private static final Set AGG_FUNCTIONS = Set.of( AggregateFunction.SUM, AggregateFunction.SUM0, @@ -107,6 +115,16 @@ public Set filterCapabilities() { return Set.copyOf(caps); } + @Override + public Set projectCapabilities() { + Set formats = Set.copyOf(plugin.getSupportedFormats()); + Set caps = new HashSet<>(); + for (ScalarFunction op : STANDARD_PROJECT_OPS) { + caps.add(new ProjectCapability.Scalar(op, Set.copyOf(SUPPORTED_FIELD_TYPES), formats, true)); + } + return Set.copyOf(caps); + } + @Override public Set aggregateCapabilities() { Set formats = Set.copyOf(plugin.getSupportedFormats()); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java index 7b7002233a2ac..e21ef6e00e145 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java @@ -18,8 +18,10 @@ import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; import org.opensearch.analytics.planner.RelNodeUtils; import org.opensearch.analytics.spi.FieldStorageInfo; @@ -122,10 +124,28 @@ public RelNode stripAnnotations(List strippedChildren) { @Override public RelNode stripAnnotations(List strippedChildren, Function annotationResolver) { + // OpenSearchProjectRule.annotateExpr recurses into operands when validating viable + // backends, so a top-level call like COALESCE(num0, CEIL(num1)) ends up with the inner + // CEIL also wrapped. The supplied annotationResolver controls how each top-level + // wrapper is unwrapped (defaults to OperatorAnnotation::unwrap, returning the original + // RexNode); a RexShuttle then sweeps the resolver's result to strip any remaining + // nested wrappers. Substrait conversion only recognizes the underlying RexCall shape, + // so every wrapper at every depth must be removed before the plan is handed to a + // backend's FragmentConvertor. + RexShuttle nestedAnnotationStripper = new RexShuttle() { + @Override + public RexNode visitCall(RexCall call) { + if (call instanceof AnnotatedProjectExpression nested) { + return nested.getOriginal().accept(this); + } + return super.visitCall(call); + } + }; List strippedExprs = new ArrayList<>(); for (RexNode expr : getProjects()) { if (expr instanceof AnnotatedProjectExpression annotated) { - strippedExprs.add(annotationResolver.apply(annotated)); + RexNode resolved = annotationResolver.apply(annotated); + strippedExprs.add(resolved.accept(nestedAnnotationStripper)); } else { // Plain expressions have no annotation to strip — pass through. strippedExprs.add(expr); diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java index 155b42b1f416d..fcc2a69250a65 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java @@ -236,6 +236,50 @@ public void testNestedScalarFunctions() { assertAnnotation(result.getProjects().get(0), MockDataFusionBackend.NAME); } + public void testStripAnnotationsRecursivelyUnwrapsNestedExpressions() { + // PLUS(CEIL(value), value) — a scalar call with another scalar call as an operand. + // The project rule recurses into operands (annotateExpr lines 127-139), so both PLUS + // and the inner CEIL get wrapped in AnnotatedProjectExpression. stripAnnotations must + // remove every wrapper at every depth before the plan reaches the backend + // FragmentConvertor — Substrait isthmus has no converter for ANNOTATED_PROJECT_EXPR and + // would throw "Unable to convert call". (COALESCE would be the natural shape here since + // PPL fillnull lowers to it, but Calcite's makeCall simplifies COALESCE on non-nullable + // operands away into the first arg, defeating the test. PLUS+CEIL preserves the + // nested-call structure we want to exercise.) + RexNode value = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1); + RexNode ceilCall = rexBuilder.makeCall(SqlStdOperatorTable.CEIL, value); + RexNode plusCall = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, ceilCall, value); + OpenSearchProject annotated = runProject(plusCall); + + // Sanity: confirm the rule produced the nested-wrapper shape this test exercises. + RexNode topLevel = annotated.getProjects().get(0); + assertTrue("Outer PLUS must be annotated", topLevel instanceof AnnotatedProjectExpression); + RexCall outerOriginal = (RexCall) ((AnnotatedProjectExpression) topLevel).getOriginal(); + assertTrue( + "Inner CEIL must also be annotated (recursive annotateExpr behavior)", + outerOriginal.getOperands().get(0) instanceof AnnotatedProjectExpression + ); + + // Strip and assert no AnnotatedProjectExpression survives anywhere in the RexNode tree. + RelNode stripped = annotated.stripAnnotations(annotated.getInputs()); + assertTrue("Stripped plan should be a plain LogicalProject", stripped instanceof LogicalProject); + for (RexNode expr : ((LogicalProject) stripped).getProjects()) { + assertNoAnnotationInTree(expr); + } + } + + private static void assertNoAnnotationInTree(RexNode node) { + assertFalse( + "Expression tree must not contain AnnotatedProjectExpression after strip: " + node, + node instanceof AnnotatedProjectExpression + ); + if (node instanceof RexCall call) { + for (RexNode operand : call.getOperands()) { + assertNoAnnotationInTree(operand); + } + } + } + // ---- Mixed backends in one projection ---- public void testMixedBackendsInProjection() { diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FillNullCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FillNullCommandIT.java new file mode 100644 index 0000000000000..0ee6a52cf29f1 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FillNullCommandIT.java @@ -0,0 +1,445 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for PPL {@code fillnull} on the analytics-engine route. + * + *

      Mirrors {@code CalciteFillNullCommandIT} from the {@code opensearch-project/sql} + * repository so that the analytics-engine path can be verified inside core without + * cross-plugin dependencies on the SQL plugin. Each test sends a PPL query through + * {@code POST /_analytics/ppl} (exposed by the {@code test-ppl-frontend} plugin), + * which runs the same {@code UnifiedQueryPlanner} → {@code CalciteRelNodeVisitor} → + * Substrait → DataFusion pipeline as the SQL plugin's force-routed analytics path. + * + *

      Covers all 13 fillnull surface forms: + *

        + *
      • {@code with X in fields} — single value, named fields
        • + *
      • {@code using f=X, ...} — per-field replacement, including non-literal expressions
        • + *
      • {@code with ceil(...) in ...} — replacement contains a nested scalar call
        • + *
      • {@code value=X} — Calcite-specific syntax, all fields and named fields
        • + *
      • type-incompatibility errors raised in {@code CalciteRelNodeVisitor} preflight
        • + *
      + * + *

      Provisions the {@code calcs} dataset (parquet-backed) once per class via + * {@link DatasetProvisioner}; {@link AnalyticsRestTestCase#preserveIndicesUponCompletion()} + * keeps it across test methods. + */ +public class FillNullCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + /** + * Lazily provision the calcs dataset on first invocation. Must be called inside a test + * method (not {@code setUp()}) — {@link org.opensearch.test.rest.OpenSearchRestTestCase}'s + * static {@code client()} is not initialized until after {@code @BeforeClass}, but is + * reliably available inside test bodies. Mirrors the pattern in {@code PplClickBenchIT}. + */ + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + // ── with-clause: single value into named fields ───────────────────────────── + + public void testFillNullSameValueOneField() throws IOException { + assertRows( + "source=" + DATASET.indexName + " | fields str2, num0 | fillnull with -1 in num0", + row("one", 12.3), + row("two", -12.3), + row("three", 15.7), + row(null, -15.7), + row("five", 3.5), + row("six", -3.5), + row(null, 0), + row("eight", -1), + row("nine", 10), + row("ten", -1), + row("eleven", -1), + row("twelve", -1), + row(null, -1), + row("fourteen", -1), + row("fifteen", -1), + row("sixteen", -1), + row(null, -1) + ); + } + + public void testFillNullSameValueTwoFields() throws IOException { + assertRows( + "source=" + DATASET.indexName + " | fields num0, num2 | fillnull with -1 in num0,num2", + row(12.3, 17.86), + row(-12.3, 16.73), + row(15.7, -1), + row(-15.7, 8.51), + row(3.5, 6.46), + row(-3.5, 8.98), + row(0, 11.69), + row(-1, 17.25), + row(10, -1), + row(-1, 11.5), + row(-1, 6.8), + row(-1, 3.79), + row(-1, -1), + row(-1, 13.04), + row(-1, -1), + row(-1, 10.98), + row(-1, 7.87) + ); + } + + // ── using-clause: per-field replacement ───────────────────────────────────── + + public void testFillNullVariousValuesOneField() throws IOException { + assertRows( + "source=" + DATASET.indexName + " | fields str2, num0 | fillnull using num0 = -1", + row("one", 12.3), + row("two", -12.3), + row("three", 15.7), + row(null, -15.7), + row("five", 3.5), + row("six", -3.5), + row(null, 0), + row("eight", -1), + row("nine", 10), + row("ten", -1), + row("eleven", -1), + row("twelve", -1), + row(null, -1), + row("fourteen", -1), + row("fifteen", -1), + row("sixteen", -1), + row(null, -1) + ); + } + + public void testFillNullVariousValuesTwoFields() throws IOException { + assertRows( + "source=" + DATASET.indexName + " | fields num0, num2 | fillnull using num0 = -1, num2 = -2", + row(12.3, 17.86), + row(-12.3, 16.73), + row(15.7, -2), + row(-15.7, 8.51), + row(3.5, 6.46), + row(-3.5, 8.98), + row(0, 11.69), + row(-1, 17.25), + row(10, -2), + row(-1, 11.5), + row(-1, 6.8), + row(-1, 3.79), + row(-1, -2), + row(-1, 13.04), + row(-1, -2), + row(-1, 10.98), + row(-1, 7.87) + ); + } + + public void testFillNullWithOtherField() throws IOException { + // Replacement is a reference to another field, not a literal. + assertRows( + "source=" + DATASET.indexName + " | fillnull using num0 = num1 | fields str2, num0", + row("one", 12.3), + row("two", -12.3), + row("three", 15.7), + row(null, -15.7), + row("five", 3.5), + row("six", -3.5), + row(null, 0), + row("eight", 11.38), + row("nine", 10), + row("ten", 12.4), + row("eleven", 10.32), + row("twelve", 2.47), + row(null, 12.05), + row("fourteen", 10.37), + row("fifteen", 7.1), + row("sixteen", 16.81), + row(null, 7.12) + ); + } + + // ── nested-call replacement: exercises the recursive AnnotatedProjectExpression strip ── + + public void testFillNullWithFunctionOnOtherField() throws IOException { + assertRows( + "source=" + DATASET.indexName + " | fillnull with ceil(num1) in num0 | fields str2, num0", + row("one", 12.3), + row("two", -12.3), + row("three", 15.7), + row(null, -15.7), + row("five", 3.5), + row("six", -3.5), + row(null, 0), + row("eight", 12), + row("nine", 10), + row("ten", 13), + row("eleven", 11), + row("twelve", 3), + row(null, 13), + row("fourteen", 11), + row("fifteen", 8), + row("sixteen", 17), + row(null, 8) + ); + } + + public void testFillNullWithFunctionMultipleCommands() throws IOException { + // Two chained fillnulls — first numeric (num0 from num1), then string (str2 → 'unknown'). + assertRows( + "source=" + DATASET.indexName + " | fillnull with num1 in num0 | fields str2, num0 | fillnull with 'unknown' in str2", + row("one", 12.3), + row("two", -12.3), + row("three", 15.7), + row("unknown", -15.7), + row("five", 3.5), + row("six", -3.5), + row("unknown", 0), + row("eight", 11.38), + row("nine", 10), + row("ten", 12.4), + row("eleven", 10.32), + row("twelve", 2.47), + row("unknown", 12.05), + row("fourteen", 10.37), + row("fifteen", 7.1), + row("sixteen", 16.81), + row("unknown", 7.12) + ); + } + + // ── value= syntax (Calcite-specific) ──────────────────────────────────────── + + public void testFillNullValueSyntaxAllFields() throws IOException { + // No field list → applies to every field in the projection. + assertRows( + "source=" + DATASET.indexName + " | fields num0, num2 | fillnull value=0", + row(12.3, 17.86), + row(-12.3, 16.73), + row(15.7, 0), + row(-15.7, 8.51), + row(3.5, 6.46), + row(-3.5, 8.98), + row(0, 11.69), + row(0, 17.25), + row(10, 0), + row(0, 11.5), + row(0, 6.8), + row(0, 3.79), + row(0, 0), + row(0, 13.04), + row(0, 0), + row(0, 10.98), + row(0, 7.87) + ); + } + + public void testFillNullValueSyntaxWithFields() throws IOException { + assertRows( + "source=" + DATASET.indexName + " | fields str2, num0 | fillnull value=-1 num0", + row("one", 12.3), + row("two", -12.3), + row("three", 15.7), + row(null, -15.7), + row("five", 3.5), + row("six", -3.5), + row(null, 0), + row("eight", -1), + row("nine", 10), + row("ten", -1), + row("eleven", -1), + row("twelve", -1), + row(null, -1), + row("fourteen", -1), + row("fifteen", -1), + row("sixteen", -1), + row(null, -1) + ); + } + + public void testFillNullValueSyntaxWithStringValue() throws IOException { + assertRows( + "source=" + DATASET.indexName + " | fields str2, int0 | fillnull value='N/A' str2", + row("one", 1), + row("two", null), + row("three", null), + row("N/A", null), + row("five", 7), + row("six", 3), + row("N/A", 8), + row("eight", null), + row("nine", null), + row("ten", 8), + row("eleven", 4), + row("twelve", 10), + row("N/A", null), + row("fourteen", 4), + row("fifteen", 11), + row("sixteen", 4), + row("N/A", 8) + ); + } + + // ── type-restriction errors (raised in CalciteRelNodeVisitor preflight) ──── + + public void testFillNullWithMixedTypeFieldsError() { + // value=0 (INTEGER) on a projection containing a VARCHAR field must fail with the + // type-incompatibility message from validateFillNullTypeCompatibility. + assertErrorContains( + "source=" + DATASET.indexName + " | fields str2, int0 | fillnull value=0", + "replacement value type INTEGER is not compatible with field 'str2'" + ); + } + + public void testFillNullWithStringOnNumericAndStringMixedFields() { + assertErrorContains( + "source=" + DATASET.indexName + " | fields num0, str2 | fillnull value='test' num0 str2", + "replacement value type VARCHAR is not compatible with field 'num0'" + ); + } + + // ── numeric type-family coercion (BIGINT into INTEGER field) ─────────────── + + public void testFillNullWithLargeIntegerOnIntField() throws IOException { + // 8_589_934_592 = 2^33, larger than Integer.MAX_VALUE. NUMERIC type family should + // accept BIGINT into an INTEGER field without failing the compatibility check. + assertRows( + "source=" + DATASET.indexName + " | fields int0 | fillnull using int0=8589934592", + row(1), + row(8589934592L), + row(8589934592L), + row(8589934592L), + row(7), + row(3), + row(8), + row(8589934592L), + row(8589934592L), + row(8), + row(4), + row(10), + row(8589934592L), + row(4), + row(11), + row(4), + row(8) + ); + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + /** + * Construct an expected row from positional values. Element order must match the PPL + * output column order (set by the {@code fields} clause / projection inferred from the query). + */ + private static List row(Object... values) { + return Arrays.asList(values); + } + + /** + * Send a PPL query to {@code POST /_analytics/ppl} and assert the response's {@code rows} + * match the expected list element-by-element using a numeric-tolerant comparator + * (Java JSON parsing returns Integer/Long/Double interchangeably, but PPL doesn't + * preserve that distinction at the API surface). + */ + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRows(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' field for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size()); + for (int i = 0; i < expected.length; i++) { + List want = expected[i]; + List got = actualRows.get(i); + assertEquals( + "Column count mismatch at row " + i + " for query: " + ppl, + want.size(), + got.size() + ); + for (int j = 0; j < want.size(); j++) { + assertCellEquals( + "Cell mismatch at row " + i + ", col " + j + " for query: " + ppl, + want.get(j), + got.get(j) + ); + } + } + } + + /** + * Send a PPL query expecting the planner to reject it; assert the resulting HTTP error + * body contains {@code expectedSubstring} (typically the validation message text). + */ + private void assertErrorContains(String ppl, String expectedSubstring) { + try { + Map response = executePpl(ppl); + fail("Expected query to fail with [" + expectedSubstring + "] but got response: " + response); + } catch (ResponseException e) { + String body; + try { + body = org.opensearch.test.rest.OpenSearchRestTestCase.entityAsMap(e.getResponse()).toString(); + } catch (IOException ioe) { + body = e.getMessage(); + } + assertTrue( + "Expected response body to contain [" + expectedSubstring + "] but was: " + body, + body.contains(expectedSubstring) + ); + } catch (IOException e) { + fail("Unexpected IOException: " + e); + } + } + + /** Send {@code POST /_analytics/ppl} and return the parsed JSON body. */ + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } + + /** + * Compare two cells with numeric tolerance. JSON parsing produces Integer/Long/Double + * values that may not match {@code .equals()} across types even when numerically equal + * (e.g. expected {@code 0} (Integer) vs actual {@code 0.0} (Double) for a null-replaced + * DOUBLE column). Treat any two {@link Number} instances as equal if their {@code double} + * values compare equal; otherwise fall back to {@link java.util.Objects#equals}. + */ + private static void assertCellEquals(String message, Object expected, Object actual) { + if (expected == null || actual == null) { + assertEquals(message, expected, actual); + return; + } + if (expected instanceof Number && actual instanceof Number) { + double e = ((Number) expected).doubleValue(); + double a = ((Number) actual).doubleValue(); + if (Double.compare(e, a) != 0) { + fail(message + ": expected <" + expected + "> but was <" + actual + ">"); + } + return; + } + assertEquals(message, expected, actual); + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/calcs/bulk.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/calcs/bulk.json new file mode 100644 index 0000000000000..d0b4a1fd8c4d6 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/calcs/bulk.json @@ -0,0 +1,35 @@ +{"index": {}} +{"key": "key00", "num0": 12.3, "num1": 8.42, "num2": 17.86, "num3": -11.52, "num4": null, "str0": "FURNITURE", "str1": "CLAMP ON LAMPS", "str2": "one", "str3": "e", "int0": 1, "int1": -3, "int2": 5, "int3": 8, "bool0": true, "bool1": true, "bool2": false, "bool3": true, "date0": "2004-04-15", "date1": "2004-04-01", "date2": "1977-04-20", "date3": "1986-03-20", "time0": "1899-12-30T21:07:32Z", "time1": "19:36:22", "datetime0": "2004-07-09T10:17:35Z", "datetime1": null, "zzz": "a"} +{"index": {}} +{"key": "key01", "num0": -12.3, "num1": 6.71, "num2": 16.73, "num3": -9.31, "num4": 10.85, "str0": "FURNITURE", "str1": "CLOCKS", "str2": "two", "str3": "e", "int0": null, "int1": -6, "int2": -4, "int3": 13, "bool0": false, "bool1": true, "bool2": false, "bool3": null, "date0": "1972-07-04", "date1": "2004-04-02", "date2": "1995-09-03", "date3": null, "time0": "1900-01-01T13:48:48Z", "time1": "02:05:25", "datetime0": "2004-07-26T12:30:34Z", "datetime1": null, "zzz": "b"} +{"index": {}} +{"key": "key02", "num0": 15.7, "num1": 9.78, "num2": null, "num3": -12.17, "num4": -13.47, "str0": "OFFICE SUPPLIES", "str1": "AIR PURIFIERS", "str2": "three", "str3": "e", "int0": null, "int1": null, "int2": 5, "int3": 2, "bool0": null, "bool1": true, "bool2": false, "bool3": null, "date0": "1975-11-12", "date1": "2004-04-03", "date2": "1997-09-19", "date3": "1997-02-02", "time0": "1900-01-01T18:21:08Z", "time1": "09:33:31", "datetime0": "2004-08-02T07:59:23Z", "datetime1": null, "zzz": "c"} +{"index": {}} +{"key": "key03", "num0": -15.7, "num1": 7.43, "num2": 8.51, "num3": -7.25, "num4": -6.05, "str0": "OFFICE SUPPLIES", "str1": "BINDER ACCESSORIES", "str2": null, "str3": "e", "int0": null, "int1": -4, "int2": -5, "int3": 5, "bool0": true, "bool1": false, "bool2": false, "bool3": null, "date0": "2004-06-04", "date1": "2004-04-04", "date2": "1980-07-26", "date3": null, "time0": "1900-01-01T18:51:48Z", "time1": "22:50:16", "datetime0": "2004-07-05T13:14:20Z", "datetime1": null, "zzz": "d"} +{"index": {}} +{"key": "key04", "num0": 3.5, "num1": 9.05, "num2": 6.46, "num3": 12.93, "num4": 8.32, "str0": "OFFICE SUPPLIES", "str1": "BINDER CLIPS", "str2": "five", "str3": null, "int0": 7, "int1": null, "int2": 3, "int3": 9, "bool0": false, "bool1": false, "bool2": true, "bool3": true, "date0": "2004-06-19", "date1": "2004-04-05", "date2": "1997-05-30", "date3": "1996-03-07", "time0": "1900-01-01T15:01:19Z", "time1": null, "datetime0": "2004-07-28T23:30:22Z", "datetime1": null, "zzz": "e"} +{"index": {}} +{"key": "key05", "num0": -3.5, "num1": 9.38, "num2": 8.98, "num3": -19.96, "num4": 10.71, "str0": "OFFICE SUPPLIES", "str1": "BINDING MACHINES", "str2": "six", "str3": null, "int0": 3, "int1": null, "int2": 2, "int3": 7, "bool0": null, "bool1": false, "bool2": true, "bool3": false, "date0": null, "date1": "2004-04-06", "date2": "1980-11-07", "date3": "1979-04-01", "time0": "1900-01-01T08:59:39Z", "time1": "19:57:33", "datetime0": "2004-07-22T00:30:23Z", "datetime1": null, "zzz": "f"} +{"index": {}} +{"key": "key06", "num0": 0, "num1": 16.42, "num2": 11.69, "num3": 10.93, "num4": null, "str0": "OFFICE SUPPLIES", "str1": "BINDING SUPPLIES", "str2": null, "str3": "e", "int0": 8, "int1": null, "int2": 9, "int3": 18, "bool0": true, "bool1": null, "bool2": false, "bool3": null, "date0": null, "date1": "2004-04-07", "date2": "1977-02-08", "date3": null, "time0": "1900-01-01T07:37:48Z", "time1": null, "datetime0": "2004-07-28T06:54:50Z", "datetime1": null, "zzz": "g"} +{"index": {}} +{"key": "key07", "num0": null, "num1": 11.38, "num2": 17.25, "num3": 3.64, "num4": -10.24, "str0": "OFFICE SUPPLIES", "str1": "BUSINESS ENVELOPES", "str2": "eight", "str3": "e", "int0": null, "int1": 2, "int2": 0, "int3": 3, "bool0": false, "bool1": null, "bool2": true, "bool3": false, "date0": null, "date1": "2004-04-08", "date2": "1974-05-03", "date3": null, "time0": "1900-01-01T19:45:54Z", "time1": "19:48:23", "datetime0": "2004-07-12T17:30:16Z", "datetime1": null, "zzz": "h"} +{"index": {}} +{"key": "key08", "num0": 10, "num1": 9.47, "num2": null, "num3": -13.38, "num4": 4.77, "str0": "TECHNOLOGY", "str1": "ANSWERING MACHINES", "str2": "nine", "str3": null, "int0": null, "int1": 3, "int2": -6, "int3": 17, "bool0": null, "bool1": null, "bool2": false, "bool3": false, "date0": null, "date1": "2004-04-09", "date2": "1976-09-09", "date3": "1983-05-22", "time0": "1900-01-01T09:00:59Z", "time1": "22:20:14", "datetime0": "2004-07-04T22:49:28Z", "datetime1": null, "zzz": "i"} +{"index": {}} +{"key": "key09", "num0": null, "num1": 12.4, "num2": 11.5, "num3": -10.56, "num4": null, "str0": "TECHNOLOGY", "str1": "BUSINESS COPIERS", "str2": "ten", "str3": "e", "int0": 8, "int1": 3, "int2": -9, "int3": 2, "bool0": null, "bool1": true, "bool2": false, "bool3": null, "date0": null, "date1": "2004-04-10", "date2": "1998-08-12", "date3": null, "time0": "1900-01-01T20:36:00Z", "time1": null, "datetime0": "2004-07-23T21:13:37Z", "datetime1": null, "zzz": "j"} +{"index": {}} +{"key": "key10", "num0": null, "num1": 10.32, "num2": 6.8, "num3": -4.79, "num4": 19.39, "str0": "TECHNOLOGY", "str1": "CD-R MEDIA", "str2": "eleven", "str3": "e", "int0": 4, "int1": null, "int2": -3, "int3": 11, "bool0": true, "bool1": true, "bool2": false, "bool3": null, "date0": null, "date1": "2004-04-11", "date2": "1974-03-17", "date3": "1999-08-20", "time0": "1900-01-01T01:31:32Z", "time1": "00:05:57", "datetime0": "2004-07-14T08:16:44Z", "datetime1": null, "zzz": "k"} +{"index": {}} +{"key": "key11", "num0": null, "num1": 2.47, "num2": 3.79, "num3": -10.81, "num4": 3.82, "str0": "TECHNOLOGY", "str1": "CONFERENCE PHONES", "str2": "twelve", "str3": null, "int0": 10, "int1": -8, "int2": -4, "int3": 2, "bool0": false, "bool1": true, "bool2": true, "bool3": null, "date0": null, "date1": "2004-04-12", "date2": "1994-04-20", "date3": null, "time0": "1899-12-30T22:15:40Z", "time1": "04:40:49", "datetime0": "2004-07-25T15:22:26Z", "datetime1": null, "zzz": "l"} +{"index": {}} +{"key": "key12", "num0": null, "num1": 12.05, "num2": null, "num3": -6.62, "num4": 3.38, "str0": "TECHNOLOGY", "str1": "CORDED KEYBOARDS", "str2": null, "str3": null, "int0": null, "int1": null, "int2": 0, "int3": 11, "bool0": null, "bool1": false, "bool2": true, "bool3": true, "date0": null, "date1": "2004-04-13", "date2": "2001-02-04", "date3": null, "time0": "1900-01-01T13:53:46Z", "time1": "04:48:07", "datetime0": "2004-07-17T14:01:56Z", "datetime1": null, "zzz": "m"} +{"index": {}} +{"key": "key13", "num0": null, "num1": 10.37, "num2": 13.04, "num3": -18.43, "num4": null, "str0": "TECHNOLOGY", "str1": "CORDLESS KEYBOARDS", "str2": "fourteen", "str3": null, "int0": 4, "int1": null, "int2": 4, "int3": 18, "bool0": null, "bool1": false, "bool2": true, "bool3": true, "date0": null, "date1": "2004-04-14", "date2": "1988-01-05", "date3": "1996-05-13", "time0": "1900-01-01T04:57:51Z", "time1": null, "datetime0": "2004-07-19T22:21:31Z", "datetime1": null, "zzz": "n"} +{"index": {}} +{"key": "key14", "num0": null, "num1": 7.1, "num2": null, "num3": 6.84, "num4": -14.21, "str0": "TECHNOLOGY", "str1": "DOT MATRIX PRINTERS", "str2": "fifteen", "str3": "e", "int0": 11, "int1": null, "int2": -8, "int3": 18, "bool0": true, "bool1": false, "bool2": true, "bool3": null, "date0": null, "date1": "2004-04-15", "date2": "1972-07-12", "date3": "1986-11-08", "time0": "1899-12-30T22:42:43Z", "time1": "18:58:41", "datetime0": "2004-07-31T11:57:52Z", "datetime1": null, "zzz": "o"} +{"index": {}} +{"key": "key15", "num0": null, "num1": 16.81, "num2": 10.98, "num3": -10.98, "num4": 6.75, "str0": "TECHNOLOGY", "str1": "DVD", "str2": "sixteen", "str3": "e", "int0": 4, "int1": null, "int2": -9, "int3": 11, "bool0": false, "bool1": null, "bool2": false, "bool3": true, "date0": null, "date1": "2004-04-16", "date2": "1995-06-04", "date3": null, "time0": "1899-12-30T22:24:08Z", "time1": null, "datetime0": "2004-07-14T07:43:00Z", "datetime1": null, "zzz": "p"} +{"index": {}} +{"key": "key16", "num0": null, "num1": 7.12, "num2": 7.87, "num3": -2.6, "num4": null, "str0": "TECHNOLOGY", "str1": "ERICSSON", "str2": null, "str3": null, "int0": 8, "int1": -9, "int2": 6, "int3": 0, "bool0": null, "bool1": null, "bool2": false, "bool3": null, "date0": null, "date1": "2004-04-17", "date2": "2002-04-27", "date3": "1992-01-18", "time0": "1900-01-01T11:58:29Z", "time1": "12:33:57", "datetime0": "2004-07-28T12:34:28Z", "datetime1": null, "zzz": "q"} + diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/calcs/mapping.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/calcs/mapping.json new file mode 100644 index 0000000000000..2e0c14e79054f --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/calcs/mapping.json @@ -0,0 +1,98 @@ +{ + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings" : { + "properties" : { + "key" : { + "type" : "keyword" + }, + "num0" : { + "type" : "double" + }, + "num1" : { + "type" : "double" + }, + "num2" : { + "type" : "double" + }, + "num3" : { + "type" : "double" + }, + "num4" : { + "type" : "double" + }, + "str0" : { + "type" : "keyword" + }, + "str1" : { + "type" : "keyword" + }, + "str2" : { + "type" : "keyword" + }, + "str3" : { + "type" : "keyword" + }, + "int0" : { + "type" : "integer" + }, + "int1" : { + "type" : "integer" + }, + "int2" : { + "type" : "integer" + }, + "int3" : { + "type" : "integer" + }, + "bool0" : { + "type" : "boolean" + }, + "bool1" : { + "type" : "boolean" + }, + "bool2" : { + "type" : "boolean" + }, + "bool3" : { + "type" : "boolean" + }, + "date0" : { + "type" : "date", + "format": "year_month_day" + }, + "date1" : { + "type" : "date", + "format": "year_month_day" + }, + "date2" : { + "type" : "date", + "format": "year_month_day" + }, + "date3" : { + "type" : "date", + "format": "year_month_day" + }, + "time0" : { + "type" : "date", + "format": "date_time_no_millis" + }, + "time1" : { + "type" : "date", + "format": "hour_minute_second" + }, + "datetime0" : { + "type" : "date", + "format": "date_time_no_millis" + }, + "datetime1" : { + "type" : "date" + }, + "zzz" : { + "type" : "keyword" + } + } + } +} From d7573c045ed219afcc413262368e02b9a3e439ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20L=C3=BCgger?= <7452348+timlueg@users.noreply.github.com> Date: Tue, 5 May 2026 04:48:32 +0200 Subject: [PATCH 041/115] Speed up Script Engine initialization (#21463) * compile whitespace pattern only once. Use isBlank to avoid regex replacement for empty or whitespace-only annotation check. Signed-off-by: tim <7452348+timlueg@users.noreply.github.com> --- .../painless/spi/AllowlistLoader.java | 21 ++++++++++++------- .../painless/PainlessScriptEngine.java | 4 +++- .../lookup/PainlessLookupBuilder.java | 13 ++++++++---- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistLoader.java b/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistLoader.java index c2ba64d3fc169..daaf0909716bf 100644 --- a/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistLoader.java +++ b/modules/lang-painless/spi/src/main/java/org/opensearch/painless/spi/AllowlistLoader.java @@ -47,10 +47,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; /** Loads and creates a {@link Allowlist} from one to many text files. */ public final class AllowlistLoader { + private static final Pattern WHITESPACE = Pattern.compile("\\s+"); + /** * Loads and creates a {@link Allowlist} from one to many text files using only the base annotation parsers. * See {@link #loadFromResourceFiles(Class, Map, String...)} for information on how to structure an allowlist @@ -312,9 +315,9 @@ public static Allowlist loadFromResourceFiles(Class resource, Map resource, Map resource, Map parseAllowlistAnnotations(Map annotations; - if ("".equals(line.replaceAll("\\s+", ""))) { + if (line.isBlank()) { annotations = Collections.emptyList(); } else { line = line.trim(); diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessScriptEngine.java index 5067df7063437..2ab1caf52c754 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessScriptEngine.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessScriptEngine.java @@ -90,10 +90,12 @@ public PainlessScriptEngine(Settings settings, Map, List, Compiler> contextsToCompilers = new HashMap<>(); Map, PainlessLookup> contextsToLookups = new HashMap<>(); + Map, PainlessLookup> allowlistsToLookups = new HashMap<>(); for (Map.Entry, List> entry : contexts.entrySet()) { ScriptContext context = entry.getKey(); - PainlessLookup lookup = PainlessLookupBuilder.buildFromAllowlists(entry.getValue()); + List allowlists = List.copyOf(entry.getValue()); + PainlessLookup lookup = allowlistsToLookups.computeIfAbsent(allowlists, PainlessLookupBuilder::buildFromAllowlists); contextsToCompilers.put( context, new Compiler(context.instanceClazz, context.factoryClazz, context.statefulFactoryClazz, lookup) diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupBuilder.java index 4c6910d16f8e6..e4a118528b999 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupBuilder.java @@ -65,9 +65,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.regex.Pattern; import static org.opensearch.painless.WriterConstants.DEF_TO_B_BYTE_IMPLICIT; @@ -1925,20 +1927,23 @@ public PainlessLookup build() { classesToPainlessClasses.put(painlessClassBuilderEntry.getKey(), painlessClassBuilderEntry.getValue().build()); } - if (javaClassNamesToClasses.values().containsAll(canonicalClassNamesToClasses.values()) == false) { + Set> javaClasses = new HashSet<>(javaClassNamesToClasses.values()); + Set> canonicalClasses = new HashSet<>(canonicalClassNamesToClasses.values()); + Set> painlessClasses = classesToPainlessClasses.keySet(); + + if (javaClasses.containsAll(canonicalClasses) == false) { throw new IllegalArgumentException( "the values of java class names to classes " + "must be a superset of the values of canonical class names to classes" ); } - if (javaClassNamesToClasses.values().containsAll(classesToPainlessClasses.keySet()) == false) { + if (javaClasses.containsAll(painlessClasses) == false) { throw new IllegalArgumentException( "the values of java class names to classes " + "must be a superset of the keys of classes to painless classes" ); } - if (canonicalClassNamesToClasses.values().containsAll(classesToPainlessClasses.keySet()) == false - || classesToPainlessClasses.keySet().containsAll(canonicalClassNamesToClasses.values()) == false) { + if (canonicalClasses.equals(painlessClasses) == false) { throw new IllegalArgumentException( "the values of canonical class names to classes " + "must have the same classes as the keys of classes to painless classes" ); From 20c8fe836753ea10c9a8947c7b2ba9f449199a31 Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Tue, 5 May 2026 08:35:11 +0530 Subject: [PATCH 042/115] Add timestamp filter support for queries in analytics core (#21436) * Add timestamp filter support for queries in analytics core Signed-off-by: Vinay Krishna Pudyodu * Introduces RexNodeTransformer SPI so backends can declare RexNode transformations Signed-off-by: Vinay Krishna Pudyodu * pr comments Signed-off-by: Vinay Krishna Pudyodu * Changed RexNode to RexCall to be specific Signed-off-by: Vinay Krishna Pudyodu * removed caching of converter Signed-off-by: Vinay Krishna Pudyodu * Refactored after adding BackendPlanAdapter Signed-off-by: Vinay Krishna Pudyodu * Replaced SqlKind.OTHER to OTHER_FUNCTION Signed-off-by: Vinay Krishna Pudyodu * spotless fix Signed-off-by: Vinay Krishna Pudyodu * Added todo to support date_nanos Signed-off-by: Vinay Krishna Pudyodu * fix failing test Signed-off-by: Vinay Krishna Pudyodu * spotless fix Signed-off-by: Vinay Krishna Pudyodu --------- Signed-off-by: Vinay Krishna Pudyodu --- .../spi/AnalyticsSearchBackendPlugin.java | 1 + .../analytics/spi/ScalarFunction.java | 36 ++-- .../analytics-backend-datafusion/build.gradle | 9 +- .../licenses/core-0.67.0.jar.sha1 | 1 - .../licenses/core-0.89.1.jar.sha1 | 1 + .../licenses/isthmus-0.67.0.jar.sha1 | 1 - .../licenses/isthmus-0.89.1.jar.sha1 | 1 + .../DataFusionAnalyticsBackendPlugin.java | 7 + .../DataFusionFragmentConvertor.java | 51 +----- .../be/datafusion/SubstraitPlanRewriter.java | 114 ++++++++++++ .../datafusion/TimestampFunctionAdapter.java | 115 ++++++++++++ .../SubstraitPlanRewriterTests.java | 163 ++++++++++++++++++ .../TimestampFunctionAdapterTests.java | 62 +++++++ .../planner/rules/OpenSearchFilterRule.java | 2 +- .../rules/OpenSearchTableScanRule.java | 20 ++- 15 files changed, 515 insertions(+), 69 deletions(-) delete mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/core-0.67.0.jar.sha1 create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/core-0.89.1.jar.sha1 delete mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.67.0.jar.sha1 create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.89.1.jar.sha1 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SubstraitPlanRewriter.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/TimestampFunctionAdapter.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/SubstraitPlanRewriterTests.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/TimestampFunctionAdapterTests.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java index 4f138b762eca8..e580b9824e36d 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java @@ -69,4 +69,5 @@ default FragmentConvertor getFragmentConvertor() { default ExchangeSinkProvider getExchangeSinkProvider() { return null; } + } diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java index 2a1d4f8129187..7185a3e8cb630 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java @@ -36,22 +36,22 @@ public enum ScalarFunction { IS_NOT_NULL(Category.COMPARISON, SqlKind.IS_NOT_NULL), IN(Category.COMPARISON, SqlKind.IN), LIKE(Category.COMPARISON, SqlKind.LIKE), - PREFIX(Category.COMPARISON, SqlKind.OTHER), + PREFIX(Category.COMPARISON, SqlKind.OTHER_FUNCTION), // ── Full-text search ───────────────────────────────────────────── - MATCH(Category.FULL_TEXT, SqlKind.OTHER), - MATCH_PHRASE(Category.FULL_TEXT, SqlKind.OTHER), - FUZZY(Category.FULL_TEXT, SqlKind.OTHER), - WILDCARD(Category.FULL_TEXT, SqlKind.OTHER), - REGEXP(Category.FULL_TEXT, SqlKind.OTHER), + MATCH(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + MATCH_PHRASE(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + FUZZY(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + WILDCARD(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + REGEXP(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), // ── String ─────────────────────────────────────────────────────── - UPPER(Category.STRING, SqlKind.OTHER), - LOWER(Category.STRING, SqlKind.OTHER), + UPPER(Category.STRING, SqlKind.OTHER_FUNCTION), + LOWER(Category.STRING, SqlKind.OTHER_FUNCTION), TRIM(Category.STRING, SqlKind.TRIM), - SUBSTRING(Category.STRING, SqlKind.OTHER), - CONCAT(Category.STRING, SqlKind.OTHER), - CHAR_LENGTH(Category.STRING, SqlKind.OTHER), + SUBSTRING(Category.STRING, SqlKind.OTHER_FUNCTION), + CONCAT(Category.STRING, SqlKind.OTHER_FUNCTION), + CHAR_LENGTH(Category.STRING, SqlKind.OTHER_FUNCTION), // ── Math ───────────────────────────────────────────────────────── PLUS(Category.MATH, SqlKind.PLUS), @@ -59,8 +59,8 @@ public enum ScalarFunction { TIMES(Category.MATH, SqlKind.TIMES), DIVIDE(Category.MATH, SqlKind.DIVIDE), MOD(Category.MATH, SqlKind.MOD), - ABS(Category.MATH, SqlKind.OTHER), - SIN(Category.MATH, SqlKind.OTHER), + ABS(Category.MATH, SqlKind.OTHER_FUNCTION), + SIN(Category.MATH, SqlKind.OTHER_FUNCTION), CEIL(Category.MATH, SqlKind.CEIL), FLOOR(Category.MATH, SqlKind.FLOOR), @@ -72,7 +72,10 @@ public enum ScalarFunction { COALESCE(Category.SCALAR, SqlKind.COALESCE), NULLIF(Category.SCALAR, SqlKind.NULLIF), - EXTRACT(Category.SCALAR, SqlKind.EXTRACT); + EXTRACT(Category.SCALAR, SqlKind.EXTRACT), + + // ── Datetime ──────────────────────────────────────────────────── + TIMESTAMP(Category.SCALAR, SqlKind.OTHER_FUNCTION); /** * Category of scalar function. @@ -104,11 +107,12 @@ public SqlKind getSqlKind() { /** * Maps a Calcite SqlKind to a ScalarFunction, or null if not recognized. - * Skips OTHER to avoid ambiguity (multiple functions share OTHER). + * Skips OTHER_FUNCTION — multiple functions share this kind, + * so they must be resolved by name via {@link #fromSqlFunction(SqlFunction)}. */ public static ScalarFunction fromSqlKind(SqlKind kind) { for (ScalarFunction func : values()) { - if (func.sqlKind == kind && func.sqlKind != SqlKind.OTHER) { + if (func.sqlKind == kind && func.sqlKind != SqlKind.OTHER_FUNCTION) { return func; } } diff --git a/sandbox/plugins/analytics-backend-datafusion/build.gradle b/sandbox/plugins/analytics-backend-datafusion/build.gradle index 77761cc69bf4e..5f55090e74137 100644 --- a/sandbox/plugins/analytics-backend-datafusion/build.gradle +++ b/sandbox/plugins/analytics-backend-datafusion/build.gradle @@ -73,10 +73,8 @@ dependencies { compileOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" // Substrait — Calcite RelNode to Substrait plan conversion for DataFusion native runtime - implementation "io.substrait:isthmus:0.67.0" - implementation "io.substrait:core:0.67.0" - // Jackson datatype module required by Substrait's DefaultExtensionCatalog (not transitively - // bundled by the analytics-engine parent plugin). + implementation "io.substrait:isthmus:0.89.1" + implementation "io.substrait:core:0.89.1" implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}" calciteCompile "com.google.guava:guava:${versions.guava}" @@ -193,6 +191,7 @@ tasks.named('thirdPartyAudit').configure { ignoreMissingClasses( // SqlDdlParserImpl is generated by Calcite at build time and not bundled in the // calcite-core jar; substrait-isthmus references it through reflection in optional code paths. - 'org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl' + 'org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl', + 'org.apache.calcite.server.ServerDdlExecutor' ) } diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/core-0.67.0.jar.sha1 b/sandbox/plugins/analytics-backend-datafusion/licenses/core-0.67.0.jar.sha1 deleted file mode 100644 index f0389973b3195..0000000000000 --- a/sandbox/plugins/analytics-backend-datafusion/licenses/core-0.67.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e843a3f25fadaf71c5743ccfcf3d2f28587354de \ No newline at end of file diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/core-0.89.1.jar.sha1 b/sandbox/plugins/analytics-backend-datafusion/licenses/core-0.89.1.jar.sha1 new file mode 100644 index 0000000000000..ea8e7e75240dc --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/core-0.89.1.jar.sha1 @@ -0,0 +1 @@ +9ffa7d00ebb71c64d0f2fac3cee6950132f82579 \ No newline at end of file diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.67.0.jar.sha1 b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.67.0.jar.sha1 deleted file mode 100644 index fd5b6b13576b7..0000000000000 --- a/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.67.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3deab1e77fa2549a7ce2d35f8961e4c7bbe30644 \ No newline at end of file diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.89.1.jar.sha1 b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.89.1.jar.sha1 new file mode 100644 index 0000000000000..d969f5a3f0930 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/isthmus-0.89.1.jar.sha1 @@ -0,0 +1 @@ +5ec1c27f852ce87754d3030ea3ebce63bfce0333 \ No newline at end of file diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 09993221052c9..e226d6b1074b1 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -19,11 +19,13 @@ import org.opensearch.analytics.spi.FragmentConvertor; import org.opensearch.analytics.spi.ProjectCapability; import org.opensearch.analytics.spi.ScalarFunction; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; import org.opensearch.analytics.spi.ScanCapability; import org.opensearch.analytics.spi.SearchExecEngineProvider; import org.opensearch.index.engine.dataformat.DataFormatRegistry; import java.util.HashSet; +import java.util.Map; import java.util.Set; /** @@ -136,6 +138,11 @@ public Set aggregateCapabilities() { } return Set.copyOf(caps); } + + @Override + public Map scalarFunctionAdapters() { + return Map.of(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter()); + } }; } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index b8ec352673174..a555f8f53e0b0 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -34,12 +34,11 @@ import java.util.ArrayList; import java.util.List; -import java.util.Optional; import io.substrait.expression.AggregateFunctionInvocation; import io.substrait.expression.Expression; import io.substrait.extension.SimpleExtension; -import io.substrait.isthmus.ImmutableFeatureBoard; +import io.substrait.isthmus.ConverterProvider; import io.substrait.isthmus.SubstraitRelVisitor; import io.substrait.isthmus.TypeConverter; import io.substrait.isthmus.expression.AggregateFunctionConverter; @@ -54,9 +53,7 @@ import io.substrait.relation.NamedScan; import io.substrait.relation.Project; import io.substrait.relation.Rel; -import io.substrait.relation.RelCopyOnWriteVisitor; import io.substrait.relation.Sort; -import io.substrait.util.EmptyVisitationContext; /** * Converts Calcite RelNode fragments to Substrait protobuf bytes @@ -135,7 +132,7 @@ private byte[] convertToSubstrait(RelNode fragment) { Plan.Root substraitRoot = Plan.Root.builder().input(substraitRel).names(fieldNames).build(); Plan plan = Plan.builder().addRoots(substraitRoot).build(); - plan = new TableNameModifier().modifyTableNames(plan); + plan = SubstraitPlanRewriter.rewrite(plan); io.substrait.proto.Plan protoPlan = new PlanProtoConverter().toProto(plan); byte[] bytes = protoPlan.toByteArray(); @@ -248,24 +245,23 @@ private static RelNode rewriteStageInputScans(RelNode node) { private SubstraitRelVisitor createVisitor(RelNode relNode) { RelDataTypeFactory typeFactory = relNode.getCluster().getTypeFactory(); TypeConverter typeConverter = TypeConverter.DEFAULT; - - AggregateFunctionConverter aggConverter = new AggregateFunctionConverter(extensions.aggregateFunctions(), typeFactory); ScalarFunctionConverter scalarConverter = new ScalarFunctionConverter( extensions.scalarFunctions(), ADDITIONAL_SCALAR_SIGS, typeFactory, typeConverter ); + AggregateFunctionConverter aggConverter = new AggregateFunctionConverter(extensions.aggregateFunctions(), typeFactory); WindowFunctionConverter windowConverter = new WindowFunctionConverter(extensions.windowFunctions(), typeFactory); - - return new SubstraitRelVisitor( + ConverterProvider converterProvider = new ConverterProvider( typeFactory, + extensions, scalarConverter, aggConverter, windowConverter, - typeConverter, - ImmutableFeatureBoard.builder().build() + typeConverter ); + return new SubstraitRelVisitor(converterProvider); } // ── Plan serde helpers ────────────────────────────────────────────────────── @@ -285,39 +281,6 @@ private static byte[] serializePlan(Plan plan) { return new PlanProtoConverter().toProto(plan).toByteArray(); } - // ── NamedScan prefix stripper ─────────────────────────────────────────────── - - /** - * Strips Calcite catalog prefixes (e.g. "opensearch") from {@link NamedScan} table - * names — DataFusion expects the bare index / stage-input id. - */ - private static class TableNameModifier { - Plan modifyTableNames(Plan plan) { - TableNameVisitor visitor = new TableNameVisitor(); - List modifiedRoots = new ArrayList<>(); - for (Plan.Root root : plan.getRoots()) { - Optional modifiedRel = root.getInput().accept(visitor, null); - if (modifiedRel.isPresent()) { - modifiedRoots.add(Plan.Root.builder().from(root).input(modifiedRel.get()).build()); - } else { - modifiedRoots.add(root); - } - } - return Plan.builder().from(plan).roots(modifiedRoots).build(); - } - - private static class TableNameVisitor extends RelCopyOnWriteVisitor { - @Override - public Optional visit(NamedScan namedScan, EmptyVisitationContext context) { - List names = namedScan.getNames(); - if (names.size() > 1) { - return Optional.of(NamedScan.builder().from(namedScan).names(List.of(names.get(names.size() - 1))).build()); - } - return super.visit(namedScan, context); - } - } - } - // ── Calcite TableScan wrappers for OpenSearchStageInputScan rewrite ───────── /** diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SubstraitPlanRewriter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SubstraitPlanRewriter.java new file mode 100644 index 0000000000000..d9840dc4c41ab --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SubstraitPlanRewriter.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import io.substrait.expression.Expression; +import io.substrait.expression.ImmutableExpression; +import io.substrait.plan.Plan; +import io.substrait.relation.ExpressionCopyOnWriteVisitor; +import io.substrait.relation.Rel; +import io.substrait.relation.RelCopyOnWriteVisitor; +import io.substrait.util.EmptyVisitationContext; + +/** + * Single-pass post-processor for Substrait plans before serialization to protobuf. + * + *

      Applies two kinds of rewrites: + *

        + *
      • Rel-level — structural changes like table name stripping, handled by + * {@link RelCopyOnWriteVisitor} overrides.
        • + *
      • Expression-level — literal/type fixes handled by + * {@link ExpressionCopyOnWriteVisitor} overrides. Adding a new expression rewrite + * only requires overriding the corresponding {@code visit} method.
        • + *
      + * + * @opensearch.internal + */ +class SubstraitPlanRewriter { + + private SubstraitPlanRewriter() {} + + static Plan rewrite(Plan plan) { + PlanRelVisitor visitor = new PlanRelVisitor(); + + List roots = new ArrayList<>(); + for (Plan.Root root : plan.getRoots()) { + Optional modified = root.getInput().accept(visitor, null); + roots.add(modified.isPresent() ? Plan.Root.builder().from(root).input(modified.get()).build() : root); + } + return Plan.builder().from(plan).roots(roots).build(); + } + + /** + * Rel-level visitor. Handles structural rewrites (table name stripping) and delegates + * expression rewrites to {@link PlanExpressionVisitor}. + */ + private static class PlanRelVisitor extends RelCopyOnWriteVisitor { + + private final PlanExpressionVisitor expressionVisitor = new PlanExpressionVisitor(this); + + // Rewrite expressions inside filter conditions + @Override + public Optional visit(io.substrait.relation.Filter filter, EmptyVisitationContext ctx) { + Optional newInput = filter.getInput().accept(this, ctx); + Optional rewritten = filter.getCondition().accept(expressionVisitor, ctx); + if (newInput.isEmpty() && rewritten.isEmpty()) return Optional.empty(); + return Optional.of( + io.substrait.relation.Filter.builder() + .from(filter) + .input(newInput.orElse(filter.getInput())) + .condition(rewritten.orElse(filter.getCondition())) + .build() + ); + } + } + + /** + * Expression-level visitor. Override a {@code visit} method to add a new rewrite. + * The base class handles recursion into function arguments, casts, if-then, etc. + */ + private static class PlanExpressionVisitor extends ExpressionCopyOnWriteVisitor { + + PlanExpressionVisitor(PlanRelVisitor relVisitor) { + super(relVisitor); + } + + // Isthmus hardcodes timestamp literals to precision 6 (microseconds). + // Parquet stores Timestamp(MILLISECOND), so convert to precision 3. + @Override + public Optional visit(Expression.PrecisionTimestampLiteral pts, EmptyVisitationContext ctx) { + if (pts.precision() != 3) { + return Optional.of( + ImmutableExpression.PrecisionTimestampLiteral.builder() + .value(toMillis(pts.value(), pts.precision())) + .precision(3) + .nullable(pts.nullable()) + .build() + ); + } + return Optional.empty(); + } + } + + private static long toMillis(long value, int precision) { + return switch (precision) { + case 0 -> value * 1000L; + case 6 -> TimeUnit.MICROSECONDS.toMillis(value); + case 9 -> TimeUnit.NANOSECONDS.toMillis(value); + default -> throw new IllegalArgumentException( + "Unsupported timestamp precision: " + precision + ". Expected 0 (seconds), 6 (micros), or 9 (nanos)." + ); + }; + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/TimestampFunctionAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/TimestampFunctionAdapter.java new file mode 100644 index 0000000000000..88d6d80afc5b3 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/TimestampFunctionAdapter.java @@ -0,0 +1,115 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.TimestampString; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; +import java.util.List; + +/** + * Converts {@code TIMESTAMP(varchar_literal)} into a {@code TIMESTAMP} literal with + * precision derived from the field's mapping type (date→3, date_nanos→9). + * + *

      Registered as a {@link ScalarFunctionAdapter} for {@code ScalarFunction.TIMESTAMP}. + * {@link org.opensearch.analytics.planner.dag.BackendPlanAdapter} calls this after plan + * forking, passing the {@code TIMESTAMP(varchar)} RexCall directly. + * + * @opensearch.internal + */ +class TimestampFunctionAdapter implements ScalarFunctionAdapter { + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + if (original.getOperands().size() != 1 + || !(original.getOperands().get(0) instanceof RexLiteral literal) + || literal.getType().getSqlTypeName() != SqlTypeName.VARCHAR) { + return original; + } + int precision = resolveTimestampPrecision(original, fieldStorage); + if (precision < 0) { + return original; + } + String value = literal.getValueAs(String.class); + if (value == null) { + return original; + } + RexBuilder rexBuilder = cluster.getRexBuilder(); + return rexBuilder.makeTimestampLiteral(parseTimestamp(value), precision); + } + + /** + * Resolves timestamp precision from field storage. Scans all fields for date/date_nanos + * since the TIMESTAMP(varchar) call itself has no field reference — the field ref is + * in the parent comparison (e.g., $0 in >($0, TIMESTAMP('...'))). + */ + private int resolveTimestampPrecision(RexCall call, List fieldStorage) { + for (FieldStorageInfo field : fieldStorage) { + String mappingType = field.getMappingType(); + // TODO: date_nanos is not yet mapped by OpenSearchSchemaBuilder (falls through to VARCHAR), + // so this branch is currently unreachable — kept for when date_nanos schema support lands. + if ("date_nanos".equals(mappingType)) return 9; + if ("date".equals(mappingType)) return 3; + } + return -1; + } + + TimestampString parseTimestamp(String input) { + try { + LocalDate date = LocalDate.parse(input); + return toTimestampString(date.atStartOfDay()); + } catch (DateTimeParseException ignored) {} + + try { + OffsetDateTime odt = OffsetDateTime.parse(input); + return toTimestampString(LocalDateTime.ofInstant(odt.toInstant(), ZoneOffset.UTC)); + } catch (DateTimeParseException ignored) {} + + try { + Instant instant = Instant.parse(input); + return toTimestampString(LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); + } catch (DateTimeParseException ignored) {} + + try { + LocalDateTime ldt = LocalDateTime.parse(input); + return toTimestampString(ldt); + } catch (DateTimeParseException ignored) {} + + return new TimestampString(input); + } + + private TimestampString toTimestampString(LocalDateTime ldt) { + TimestampString ts = new TimestampString( + ldt.getYear(), + ldt.getMonthValue(), + ldt.getDayOfMonth(), + ldt.getHour(), + ldt.getMinute(), + ldt.getSecond() + ); + int nanos = ldt.getNano(); + if (nanos > 0) { + ts = ts.withNanos(nanos); + } + return ts; + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/SubstraitPlanRewriterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/SubstraitPlanRewriterTests.java new file mode 100644 index 0000000000000..4a1cb807916c3 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/SubstraitPlanRewriterTests.java @@ -0,0 +1,163 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +import io.substrait.expression.Expression; +import io.substrait.expression.FieldReference; +import io.substrait.expression.ImmutableExpression; +import io.substrait.extension.DefaultExtensionCatalog; +import io.substrait.extension.SimpleExtension; +import io.substrait.plan.Plan; +import io.substrait.relation.Filter; +import io.substrait.relation.NamedScan; +import io.substrait.type.NamedStruct; +import io.substrait.type.TypeCreator; + +public class SubstraitPlanRewriterTests extends OpenSearchTestCase { + + private static final TypeCreator R = TypeCreator.of(false); + + public void testTimestampPrecision6ConvertedTo3() { + long epochMicros = 1704067200000000L; // 2024-01-01T00:00:00Z in micros + long expectedMillis = 1704067200000L; + + Expression literal = ImmutableExpression.PrecisionTimestampLiteral.builder() + .value(epochMicros) + .precision(6) + .nullable(false) + .build(); + + Plan plan = buildFilterPlan(literal); + Plan rewritten = SubstraitPlanRewriter.rewrite(plan); + + Expression condition = getFilterCondition(rewritten); + assertTrue(condition instanceof Expression.PrecisionTimestampLiteral); + Expression.PrecisionTimestampLiteral pts = (Expression.PrecisionTimestampLiteral) condition; + assertEquals(3, pts.precision()); + assertEquals(expectedMillis, pts.value()); + } + + public void testTimestampPrecision9ConvertedTo3() { + long epochNanos = 1704067200000000000L; // 2024-01-01T00:00:00Z in nanos + long expectedMillis = 1704067200000L; + + Expression literal = ImmutableExpression.PrecisionTimestampLiteral.builder().value(epochNanos).precision(9).nullable(false).build(); + + Plan plan = buildFilterPlan(literal); + Plan rewritten = SubstraitPlanRewriter.rewrite(plan); + + Expression condition = getFilterCondition(rewritten); + assertTrue(condition instanceof Expression.PrecisionTimestampLiteral); + Expression.PrecisionTimestampLiteral pts = (Expression.PrecisionTimestampLiteral) condition; + assertEquals(3, pts.precision()); + assertEquals(expectedMillis, pts.value()); + } + + public void testTimestampPrecision3Unchanged() { + long epochMillis = 1704067200000L; + + Expression literal = ImmutableExpression.PrecisionTimestampLiteral.builder() + .value(epochMillis) + .precision(3) + .nullable(false) + .build(); + + Plan plan = buildFilterPlan(literal); + Plan rewritten = SubstraitPlanRewriter.rewrite(plan); + + Expression condition = getFilterCondition(rewritten); + assertTrue(condition instanceof Expression.PrecisionTimestampLiteral); + Expression.PrecisionTimestampLiteral pts = (Expression.PrecisionTimestampLiteral) condition; + assertEquals(3, pts.precision()); + assertEquals(epochMillis, pts.value()); + } + + public void testTimestampInsideScalarFunction() { + long epochMicros = 1704067200000000L; + long expectedMillis = 1704067200000L; + + Expression tsLiteral = ImmutableExpression.PrecisionTimestampLiteral.builder() + .value(epochMicros) + .precision(6) + .nullable(false) + .build(); + + FieldReference fieldRef = FieldReference.newRootStructReference(0, R.precisionTimestamp(3)); + + SimpleExtension.ExtensionCollection extensions = DefaultExtensionCatalog.DEFAULT_COLLECTION; + SimpleExtension.ScalarFunctionVariant gtFunc = extensions.getScalarFunction( + SimpleExtension.FunctionAnchor.of(DefaultExtensionCatalog.FUNCTIONS_COMPARISON, "gt:any_any") + ); + + Expression gtCall = Expression.ScalarFunctionInvocation.builder() + .declaration(gtFunc) + .addArguments(fieldRef, tsLiteral) + .outputType(R.BOOLEAN) + .build(); + + Plan plan = buildFilterPlan(gtCall); + Plan rewritten = SubstraitPlanRewriter.rewrite(plan); + + Expression condition = getFilterCondition(rewritten); + assertTrue(condition instanceof Expression.ScalarFunctionInvocation); + Expression.ScalarFunctionInvocation rewrittenGt = (Expression.ScalarFunctionInvocation) condition; + Expression arg1 = (Expression) rewrittenGt.arguments().get(1); + assertTrue(arg1 instanceof Expression.PrecisionTimestampLiteral); + Expression.PrecisionTimestampLiteral pts = (Expression.PrecisionTimestampLiteral) arg1; + assertEquals(3, pts.precision()); + assertEquals(expectedMillis, pts.value()); + } + + public void testBareNameUnchanged() { + NamedScan scan = NamedScan.builder() + .names(List.of("parquet_dates")) + .initialSchema(NamedStruct.of(List.of("col0"), R.struct(R.I64))) + .build(); + + Plan plan = buildPlan(scan); + Plan rewritten = SubstraitPlanRewriter.rewrite(plan); + + NamedScan rewrittenScan = (NamedScan) rewritten.getRoots().get(0).getInput(); + assertEquals(List.of("parquet_dates"), rewrittenScan.getNames()); + } + + public void testUnsupportedPrecisionThrows() { + Expression literal = ImmutableExpression.PrecisionTimestampLiteral.builder().value(12345L).precision(4).nullable(false).build(); + + Plan plan = buildFilterPlan(literal); + expectThrows(IllegalArgumentException.class, () -> SubstraitPlanRewriter.rewrite(plan)); + } + + // --- helpers --- + + private static Plan buildFilterPlan(Expression condition) { + NamedScan scan = NamedScan.builder() + .names(List.of("test_table")) + .initialSchema(NamedStruct.of(List.of("col0"), R.struct(R.precisionTimestamp(3)))) + .build(); + + Filter filter = Filter.builder().input(scan).condition(condition).build(); + + return buildPlan(filter); + } + + private static Plan buildPlan(io.substrait.relation.Rel rel) { + Plan.Root root = Plan.Root.builder().input(rel).addNames("col0").build(); + return Plan.builder().addRoots(root).build(); + } + + private static Expression getFilterCondition(Plan plan) { + Filter filter = (Filter) plan.getRoots().get(0).getInput(); + return filter.getCondition(); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/TimestampFunctionAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/TimestampFunctionAdapterTests.java new file mode 100644 index 0000000000000..4dfe4c7670af6 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/TimestampFunctionAdapterTests.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.util.TimestampString; +import org.opensearch.test.OpenSearchTestCase; + +public class TimestampFunctionAdapterTests extends OpenSearchTestCase { + + private final TimestampFunctionAdapter transformer = new TimestampFunctionAdapter(); + + public void testIsoWithTAndZ() { + TimestampString ts = transformer.parseTimestamp("2024-01-01T00:00:00Z"); + assertEquals("2024-01-01 00:00:00", ts.toString()); + } + + public void testIsoWithTNoZ() { + TimestampString ts = transformer.parseTimestamp("2024-01-15T10:30:00"); + assertEquals("2024-01-15 10:30:00", ts.toString()); + } + + public void testDateOnly() { + TimestampString ts = transformer.parseTimestamp("2024-01-01"); + assertEquals("2024-01-01 00:00:00", ts.toString()); + } + + public void testTimezoneOffsetPositive() { + TimestampString ts = transformer.parseTimestamp("2024-01-01T10:00:00+05:30"); + assertEquals("2024-01-01 04:30:00", ts.toString()); + } + + public void testTimezoneOffsetNegative() { + TimestampString ts = transformer.parseTimestamp("2024-01-01T10:00:00-05:00"); + assertEquals("2024-01-01 15:00:00", ts.toString()); + } + + public void testWithMilliseconds() { + TimestampString ts = transformer.parseTimestamp("2024-01-01T10:30:00.123Z"); + assertEquals("2024-01-01 10:30:00.123", ts.toString()); + } + + public void testWithNanoseconds() { + TimestampString ts = transformer.parseTimestamp("2024-01-01T10:30:00.123456789Z"); + assertEquals("2024-01-01 10:30:00.123456789", ts.toString()); + } + + public void testWithMillisAndTimezone() { + TimestampString ts = transformer.parseTimestamp("2024-01-01T10:30:00.500+05:30"); + assertEquals("2024-01-01 05:00:00.5", ts.toString()); + } + + public void testSpaceSeparatorPassthrough() { + TimestampString ts = transformer.parseTimestamp("2024-01-01 10:30:00"); + assertEquals("2024-01-01 10:30:00", ts.toString()); + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java index af3b5e48fd1d0..dc23601f523dd 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java @@ -80,7 +80,7 @@ public void onMatch(RelOptRuleCall call) { List childViableBackends = openSearchInput.getViableBackends(); List childFieldStorage = openSearchInput.getOutputFieldStorage(); - // Annotate every leaf predicate with viable backends + // Annotate every leaf predicate with viable backends. RexNode annotatedCondition = annotateCondition(filter.getCondition(), childFieldStorage, childViableBackends); // Compute operator-level viable backends: must be viable for child AND handle predicates diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTableScanRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTableScanRule.java index 014ac7f074b74..caf3da092a30d 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTableScanRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchTableScanRule.java @@ -8,8 +8,10 @@ package org.opensearch.analytics.planner.rules; +import org.apache.calcite.plan.RelOptAbstractTable; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.type.RelDataTypeField; import org.opensearch.analytics.planner.CapabilityRegistry; @@ -90,10 +92,12 @@ public void onMatch(RelOptRuleCall call) { ); } + RelOptTable indexNameTable = new IndexNameTable(scan.getTable(), tableName); + call.transformTo( OpenSearchTableScan.create( scan.getCluster(), - scan.getTable(), + indexNameTable, viableBackends, fieldStorage, indexMetadata.getNumberOfShards(), @@ -101,4 +105,18 @@ public void onMatch(RelOptRuleCall call) { ) ); } + + /** + * Wraps a {@link RelOptTable} with just the bare index name as the qualified name. + * Isthmus reads {@code getQualifiedName()} when creating {@code NamedScan} — this ensures + * the Substrait plan contains only the index name, not the Calcite catalog prefix. + * + *

      TODO: Move table name stripping to the SQL/PPL plugin before dispatching the RelNode + * to the analytics engine, so the scan rule always receives bare index names. + */ + private static class IndexNameTable extends RelOptAbstractTable { + IndexNameTable(RelOptTable delegate, String indexName) { + super(delegate.getRelOptSchema(), indexName, delegate.getRowType()); + } + } } From b7ab90866a03cb8737f1b142aff1a870f7cdd982 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Tue, 5 May 2026 14:49:12 +0530 Subject: [PATCH 043/115] Indexed parquet query native changes (#21365) * Implementation for Lucene + Parquet via Datafusion Indexed stream Signed-off-by: bharath-techie * different row selection strategies Signed-off-by: bharath-techie * making datafusion query config Signed-off-by: bharath-techie * correct page pruning logic and add more operators support Signed-off-by: bharath-techie * adding metrics Signed-off-by: bharath-techie * fix errors Signed-off-by: bharath-techie * FFM changes for IndexFilterProvider (#60) Signed-off-by: Bukhtawar Khan * fuzz tests Signed-off-by: bharath-techie * fixing pushdown cases Signed-off-by: bharath-techie * moving collector/provider to analytics core Signed-off-by: bharath-techie * fixes for e2e , bitset perf improvements, more metrics, more configs, more fuzz tests, formatting fixes Signed-off-by: bharath-techie * fix for sandbox build and some more fuzz scenarios Signed-off-by: bharath-techie * spotless Signed-off-by: bharath-techie * and single collector fixes and javadoc fix and forbidden api fix Signed-off-by: bharath-techie * sandbox fix Signed-off-by: bharath-techie * fixing page pruning correctness issues for misaligned pages Signed-off-by: bharath-techie * ci fixes Signed-off-by: bharath-techie * fixes for context Signed-off-by: bharath-techie --------- Signed-off-by: bharath-techie Signed-off-by: Bukhtawar Khan Co-authored-by: Bukhtawar Khan --- .../src/main/resources/minimumGradleVersion | 2 +- gradle/missing-javadoc.gradle | 4 +- sandbox/libs/analytics-framework/build.gradle | 2 + .../analytics/backend/ExecutionContext.java | 2 - .../analytics/spi/IndexFilterProvider.java | 84 + .../spi/IndexFilterProviderFactory.java | 36 + .../exec/CollectorQueryLifecycleManager.java | 10 +- .../index/engine/exec/IndexFilterContext.java | 0 .../engine/exec/IndexFilterProvider.java | 3 +- .../index/engine/exec/SegmentCollector.java | 53 + .../index/engine/exec/package-info.java | 12 + .../analytics-backend-datafusion/build.gradle | 56 + .../rust/Cargo.toml | 3 + .../rust/benches/query_bench.rs | 82 +- .../rust/src/api.rs | 249 +-- .../rust/src/datafusion_query_config.rs | 233 +++ .../rust/src/executor.rs | 21 +- .../rust/src/ffm.rs | 38 +- .../rust/src/indexed_executor.rs | 641 +++++++ .../rust/src/indexed_table/bool_tree.rs | 636 +++++++ .../src/indexed_table/eval/bitmap_tree.rs | 1669 +++++++++++++++++ .../rust/src/indexed_table/eval/mod.rs | 767 ++++++++ .../indexed_table/eval/single_collector.rs | 549 ++++++ .../rust/src/indexed_table/ffm_callbacks.rs | 222 +++ .../rust/src/indexed_table/index.rs | 75 + .../rust/src/indexed_table/metrics.rs | 253 +++ .../rust/src/indexed_table/mod.rs | 72 + .../rust/src/indexed_table/page_pruner.rs | 1206 ++++++++++++ .../rust/src/indexed_table/parquet_bridge.rs | 264 +++ .../rust/src/indexed_table/partitioning.rs | 149 ++ .../rust/src/indexed_table/row_selection.rs | 655 +++++++ .../rust/src/indexed_table/segment_info.rs | 66 + .../rust/src/indexed_table/stream.rs | 965 ++++++++++ .../src/indexed_table/substrait_to_tree.rs | 607 ++++++ .../rust/src/indexed_table/table_provider.rs | 485 +++++ .../tests_e2e/boolean_algebra.rs | 1310 +++++++++++++ .../indexed_table/tests_e2e/fuzz/config.rs | 664 +++++++ .../indexed_table/tests_e2e/fuzz/corpus.rs | 508 +++++ .../indexed_table/tests_e2e/fuzz/harness.rs | 1259 +++++++++++++ .../src/indexed_table/tests_e2e/fuzz/mod.rs | 49 + .../indexed_table/tests_e2e/fuzz/oracle.rs | 416 ++++ .../src/indexed_table/tests_e2e/fuzz/seed.rs | 156 ++ .../src/indexed_table/tests_e2e/fuzz/tests.rs | 198 ++ .../indexed_table/tests_e2e/fuzz/tree_gen.rs | 537 ++++++ .../src/indexed_table/tests_e2e/metrics.rs | 213 +++ .../rust/src/indexed_table/tests_e2e/mod.rs | 389 ++++ .../indexed_table/tests_e2e/multi_segment.rs | 1073 +++++++++++ .../indexed_table/tests_e2e/null_columns.rs | 507 +++++ .../indexed_table/tests_e2e/page_pruning.rs | 1161 ++++++++++++ .../indexed_table/tests_e2e/schema_drift.rs | 361 ++++ .../tests_e2e/streaming_at_scale.rs | 1032 ++++++++++ .../rust/src/lib.rs | 3 + .../rust/src/local_executor.rs | 26 +- .../rust/src/partition_stream.rs | 28 +- .../rust/src/query_executor.rs | 7 +- .../rust/src/query_memory_pool_tracker.rs | 3 +- .../rust/src/runtime_manager.rs | 14 +- .../rust/tests/local_exec_test.rs | 36 +- .../be/datafusion/DatafusionContext.java | 31 +- .../be/datafusion/DatafusionSearcher.java | 17 +- .../indexfilter/CollectorRegistry.java | 53 + .../indexfilter/FilterProviderRegistry.java | 118 ++ .../indexfilter/FilterTreeCallbacks.java | 160 ++ .../datafusion/indexfilter/package-info.java | 12 + .../be/datafusion/nativelib/NativeBridge.java | 113 +- .../DataFusionQueryExecutionTests.java | 1 + .../DatafusionResultStreamTests.java | 53 +- .../indexfilter/IndexFilterCallbackTests.java | 179 ++ .../analytics-backend-lucene/build.gradle | 2 + .../be/lucene/LuceneIndexFilterProvider.java | 108 +- sandbox/plugins/analytics-engine/build.gradle | 2 + .../plugins/dsl-query-executor/build.gradle | 2 + .../opensearch/parquet/bridge/RustBridge.java | 3 + .../plugins/test-ppl-frontend/build.gradle | 2 + .../index/engine/exec/SegmentCollector.java | 37 - 75 files changed, 20633 insertions(+), 381 deletions(-) create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/IndexFilterProvider.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/IndexFilterProviderFactory.java rename {server => sandbox/libs/analytics-framework}/src/main/java/org/opensearch/index/engine/exec/CollectorQueryLifecycleManager.java (87%) rename {server => sandbox/libs/analytics-framework}/src/main/java/org/opensearch/index/engine/exec/IndexFilterContext.java (100%) rename {server => sandbox/libs/analytics-framework}/src/main/java/org/opensearch/index/engine/exec/IndexFilterProvider.java (87%) create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/SegmentCollector.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/package-info.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/bitmap_tree.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/mod.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/single_collector.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/index.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/metrics.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/mod.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/page_pruner.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/parquet_bridge.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/partitioning.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/row_selection.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/segment_info.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/substrait_to_tree.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/table_provider.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/boolean_algebra.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/config.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/corpus.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/mod.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/oracle.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/seed.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/tests.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/tree_gen.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/metrics.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/mod.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/null_columns.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/schema_drift.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/streaming_at_scale.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/CollectorRegistry.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterProviderRegistry.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/package-info.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/indexfilter/IndexFilterCallbackTests.java delete mode 100644 server/src/main/java/org/opensearch/index/engine/exec/SegmentCollector.java diff --git a/buildSrc/src/main/resources/minimumGradleVersion b/buildSrc/src/main/resources/minimumGradleVersion index 815da58b7a9ed..ccfb75e5120ed 100644 --- a/buildSrc/src/main/resources/minimumGradleVersion +++ b/buildSrc/src/main/resources/minimumGradleVersion @@ -1 +1 @@ -7.4.1 +9.4.1 diff --git a/gradle/missing-javadoc.gradle b/gradle/missing-javadoc.gradle index 1261d7464c103..56ef7d4f94092 100644 --- a/gradle/missing-javadoc.gradle +++ b/gradle/missing-javadoc.gradle @@ -308,7 +308,9 @@ class MissingJavadocTask extends DefaultTask { opts << [ '--missing-method', String.join(',', javadocMissingMethod) ] } opts << [ '-quiet' ] - opts << [ '--release', 21 ] + + // To support modules with JDK 25 and above as well + opts << [ '--release', Math.max(project.java.sourceCompatibility.majorVersion.toInteger(), 21) ] opts << '-Xdoclint:all,-missing' // Temporary file that holds all javadoc options for the current task. diff --git a/sandbox/libs/analytics-framework/build.gradle b/sandbox/libs/analytics-framework/build.gradle index 4463c14064947..3e9c18d648ecf 100644 --- a/sandbox/libs/analytics-framework/build.gradle +++ b/sandbox/libs/analytics-framework/build.gradle @@ -14,6 +14,8 @@ def calciteVersion = '1.41.0' +java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } + configurations { calciteCompile compileClasspath { exclude group: 'com.google.guava' } diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java index 9a09f7d8faa67..78946812c02d0 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java @@ -8,7 +8,6 @@ package org.opensearch.analytics.backend; -import org.apache.arrow.memory.BufferAllocator; import org.opensearch.action.search.SearchShardTask; import org.opensearch.index.engine.exec.IndexReaderProvider.Reader; @@ -24,7 +23,6 @@ public class ExecutionContext { private final Reader reader; private final SearchShardTask task; private byte[] fragmentBytes; - private BufferAllocator allocator; /** * Constructs an execution context. diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/IndexFilterProvider.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/IndexFilterProvider.java new file mode 100644 index 0000000000000..3354aa5a98094 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/IndexFilterProvider.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.Closeable; +import java.lang.foreign.MemorySegment; + +/** + * Produces doc-id bitsets for one index-backed filter leaf. + * + *

      Conceptually a compiled query bound to a shard: built once per query + * per shard from an opaque serialized query payload, then used to create + * cheap per-segment matchers that materialize doc-id bitsets on demand. + * The SPI is backend-agnostic — any index implementation (inverted, sparse + * vector, columnar, etc.) can satisfy it. + * + *

      Lifecycle is driven by the native engine: + *

        + *
      1. Native upcalls {@code createProvider(queryBytes)} on the registered + * {@link IndexFilterProviderFactory}; this produces a provider and + * registers it in the backend's internal provider registry, returning + * a {@code providerKey}.
        2. + *
      2. Native upcalls {@code createCollector(providerKey, seg, min, max)} + * per (segment, row-group range). Internally this routes to + * {@link #createCollector(int, int, int)} on this provider.
        3. + *
      3. Native upcalls {@code collectDocs(collectorKey, min, max, out)} + * per row group while iterating.
        4. + *
      4. Native upcalls {@code releaseCollector(collectorKey)} when done with + * a segment, {@code releaseProvider(providerKey)} when the query ends.
        5. + *
      + * + *

      The SPI uses a {@link MemorySegment} destination buffer so that + * implementations can write doc-id bitsets directly into caller-owned + * (possibly native) memory without an intermediate {@code long[]} + * allocation. Implementations write words using + * {@code out.setAtIndex(ValueLayout.JAVA_LONG, i, word)}. + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface IndexFilterProvider extends Closeable { + + /** + * Create a collector for one (segment, [minDoc, maxDoc)) range. + * + * @return a provider-internal collector key {@code >= 0}, or {@code -1} + * if the collector cannot be created (e.g. empty range). + */ + int createCollector(int segmentOrd, int minDoc, int maxDoc); + + /** + * Fill {@code out} with the matching doc-id bitset for the given + * collector over doc range {@code [minDoc, maxDoc)}. + * + *

      Bit layout: the word at index {@code i} contains matches for docs + * {@code [minDoc + i*64, minDoc + (i+1)*64)}, LSB-first within each word. + * Implementations write words using + * {@code out.setAtIndex(ValueLayout.JAVA_LONG, i, word)}. + * + * @param collectorKey provider-internal collector key returned by + * {@link #createCollector(int, int, int)}. + * @param minDoc inclusive lower bound of the doc range. + * @param maxDoc exclusive upper bound of the doc range. + * @param out destination {@link MemorySegment} buffer; + * implementation may write up to + * {@code out.byteSize() / 8} words. + * @return number of words actually written, or {@code -1} on error. + */ + int collectDocs(int collectorKey, int minDoc, int maxDoc, MemorySegment out); + + /** + * Release resources for a collector when the native engine is done + * iterating its segment. + */ + void releaseCollector(int collectorKey); +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/IndexFilterProviderFactory.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/IndexFilterProviderFactory.java new file mode 100644 index 0000000000000..ae0b1c6a8bfb2 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/IndexFilterProviderFactory.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Builds an {@link IndexFilterProvider} from the serialized query bytes that + * appear in the substrait plan's {@code index_filter(bytes)} call. + * + *

      Exactly one factory is registered per JVM, typically by the analytics + * plugin that owns the backend (e.g. inverted index, sparse vector, etc.). The native engine calls + * it once per Collector leaf per query; the returned provider stays alive + * for the query's duration. + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface IndexFilterProviderFactory { + + /** + * Build a provider from opaque query bytes. Implementations typically + * deserialize the bytes into a backend-native query, compile it against + * the current catalog snapshot, and wrap the compiled form as an + * {@link IndexFilterProvider}. + * + * @throws Exception on any failure (wrapped and routed to Rust as {@code -1}). + */ + IndexFilterProvider create(byte[] queryBytes) throws Exception; +} diff --git a/server/src/main/java/org/opensearch/index/engine/exec/CollectorQueryLifecycleManager.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/CollectorQueryLifecycleManager.java similarity index 87% rename from server/src/main/java/org/opensearch/index/engine/exec/CollectorQueryLifecycleManager.java rename to sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/CollectorQueryLifecycleManager.java index da24f5d7757e5..577a683b508dc 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/CollectorQueryLifecycleManager.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/CollectorQueryLifecycleManager.java @@ -11,6 +11,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import java.io.Closeable; +import java.lang.foreign.MemorySegment; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -53,14 +54,15 @@ public int registerCollector(SegmentCollector collector) { * @param key the collector key returned by {@link #registerCollector} * @param minDoc inclusive lower bound * @param maxDoc exclusive upper bound - * @return packed {@code long[]} bitset of matching doc IDs, or empty array if key is invalid + * @param out destination {@link MemorySegment} to write the packed bitset into + * @return the number of 64-bit words written into {@code out}, or {@code 0} if key is invalid */ - public long[] collectDocs(int key, int minDoc, int maxDoc) { + public int collectDocs(int key, int minDoc, int maxDoc, MemorySegment out) { SegmentCollector collector = collectors.get(key); if (collector == null) { - return new long[0]; + return 0; } - return collector.collectDocs(minDoc, maxDoc); + return collector.collectDocs(minDoc, maxDoc, out); } /** diff --git a/server/src/main/java/org/opensearch/index/engine/exec/IndexFilterContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/IndexFilterContext.java similarity index 100% rename from server/src/main/java/org/opensearch/index/engine/exec/IndexFilterContext.java rename to sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/IndexFilterContext.java diff --git a/server/src/main/java/org/opensearch/index/engine/exec/IndexFilterProvider.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/IndexFilterProvider.java similarity index 87% rename from server/src/main/java/org/opensearch/index/engine/exec/IndexFilterProvider.java rename to sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/IndexFilterProvider.java index 2d5224c48d162..0aab8aa5b03a9 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/IndexFilterProvider.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/IndexFilterProvider.java @@ -12,6 +12,7 @@ import java.io.Closeable; import java.io.IOException; +import java.lang.foreign.MemorySegment; /** * Provides index-level filtering (partition pruning, segment filtering) for a given data format. @@ -28,7 +29,7 @@ public interface IndexFilterProvider e int createCollector(C context, int segmentOrd, int minDoc, int maxDoc); - long[] collectDocs(C context, int collectorKey, int minDoc, int maxDoc); + int collectDocs(C context, int collectorKey, int minDoc, int maxDoc, MemorySegment out); void releaseCollector(C context, int collectorKey); } diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/SegmentCollector.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/SegmentCollector.java new file mode 100644 index 0000000000000..32871256ec856 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/SegmentCollector.java @@ -0,0 +1,53 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.exec; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.Closeable; +import java.lang.foreign.MemorySegment; + +/** + * A per-segment document collector returned by + * {@link IndexFilterProvider#createCollector}. + *

      + * Callers should use try-with-resources to ensure cleanup. + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface SegmentCollector extends Closeable { + + /** + * Collect matching document IDs in the given range into the provided + * {@link MemorySegment}. + * + *

      Bit layout: the {@code out} segment receives a packed bitset where + * word {@code j} bit {@code i} (LSB-first) represents the doc at + * relative position {@code j*64 + i} within {@code [minDoc, maxDoc)}. + * That is, bit {@code k} represents absolute doc id {@code minDoc + k}. + * The caller must provide a segment of at least + * {@code ceilDiv(maxDoc - minDoc, 64) * 8} bytes. Implementations + * MUST NOT skip trailing zero words. + * + *

      Forward-only: successive calls MUST use non-decreasing, + * non-overlapping {@code [minDoc, maxDoc)} ranges. Backing iterators + * are one-shot cursors and cannot seek backwards; violating the + * invariant silently yields wrong results for ranges already passed. + * + * @param minDoc inclusive lower bound + * @param maxDoc exclusive upper bound + * @param out destination {@link MemorySegment} to write the packed bitset into + * @return the number of 64-bit words written into {@code out} + */ + int collectDocs(int minDoc, int maxDoc, MemorySegment out); + + @Override + default void close() {} +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/package-info.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/package-info.java new file mode 100644 index 0000000000000..acafbbc2bb06f --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/index/engine/exec/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Index-level filter execution: segment collectors and FFM-bridged doc collection. + */ +package org.opensearch.index.engine.exec; diff --git a/sandbox/plugins/analytics-backend-datafusion/build.gradle b/sandbox/plugins/analytics-backend-datafusion/build.gradle index 5f55090e74137..e2504d6f49c55 100644 --- a/sandbox/plugins/analytics-backend-datafusion/build.gradle +++ b/sandbox/plugins/analytics-backend-datafusion/build.gradle @@ -113,6 +113,62 @@ test { dependsOn ':sandbox:libs:dataformat-native:buildRustLibrary' } +// ═══════════════════════════════════════════════════════════════════ +// Rust unit + fuzz tests for the opensearch-datafusion crate. +// Run as part of `check` so `./gradlew check` (and CI) exercises the +// randomized E2E suite alongside the Java tests. +// ═══════════════════════════════════════════════════════════════════ +def rustWorkspaceDir = file("${project(':sandbox:libs:dataformat-native').projectDir}/rust") + +task cargoTest(type: Exec) { + description = 'Run Rust unit + fuzz tests for opensearch-datafusion crate' + group = 'verification' + workingDir rustWorkspaceDir + + def cargoExecutable = 'cargo' + def possibleCargoPaths = [ + System.getenv('HOME') + '/.cargo/bin/cargo', + '/usr/local/bin/cargo', + 'cargo' + ] + for (String path : possibleCargoPaths) { + if (new File(path).exists()) { cargoExecutable = path; break } + } + + commandLine cargoExecutable, 'test', '-p', 'opensearch-datafusion', '--lib' + + // Seed forwarding for the randomized fuzz suite: + // + // 1. `-PindexedE2eSeed=` overrides everything (explicit). + // 2. Otherwise, tie the Rust fuzz seed to OpenSearch's build-wide + // `tests.seed` (the same seed Lucene-style tests use; already + // printed by `GlobalBuildInfoPlugin` as "Random Testing Seed"). + // When a CI run's Java tests fail with `-Dtests.seed=ABC123`, the + // Rust fuzz seed is the same — one reproducer fits all. + // 3. If neither is set (rare; usually only on fresh local runs where + // `tests.seed` isn't configured yet), let `master_seed()` on the + // Rust side generate a fresh system-time seed and print the + // reproducer. + def explicitSeed = project.hasProperty('indexedE2eSeed') + ? project.property('indexedE2eSeed').toString() + : System.getProperty('tests.seed') + if (explicitSeed != null && !explicitSeed.isEmpty()) { + environment 'INDEXED_E2E_SEED', explicitSeed + } + + // Rebuild trigger: any Rust source in this crate. + inputs.files fileTree("${projectDir}/rust/src") + inputs.file "${projectDir}/rust/Cargo.toml" + // Marker file so gradle treats this task as cached when inputs don't change. + outputs.file "${projectDir}/rust/target/gradle-cargoTest.stamp" + doLast { + file("${projectDir}/rust/target").mkdirs() + file("${projectDir}/rust/target/gradle-cargoTest.stamp").text = new Date().toString() + } +} + +check.dependsOn cargoTest + internalClusterTest { jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED' diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml index 2d975b319e18f..71c04d9f9001a 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml +++ b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml @@ -38,10 +38,13 @@ log = { workspace = true } num_cpus = { workspace = true } mimalloc = { workspace = true } native-bridge-common = { workspace = true } +async-trait = { workspace = true } +roaring = "0.10" [dev-dependencies] criterion = { workspace = true } tempfile = { workspace = true } +rand = "0.8" [[bench]] name = "query_bench" diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/benches/query_bench.rs b/sandbox/plugins/analytics-backend-datafusion/rust/benches/query_bench.rs index 8f361dcf3aa46..25b1a993c289f 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/benches/query_bench.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/benches/query_bench.rs @@ -6,10 +6,10 @@ use datafusion::execution::runtime_env::RuntimeEnvBuilder; use futures::TryStreamExt; use object_store::local::LocalFileSystem; use object_store::ObjectStore; -use opensearch_datafusion_jni::query_executor; -use opensearch_datafusion_jni::runtime_manager::RuntimeManager; +use opensearch_datafusion::api::DataFusionRuntime; +use opensearch_datafusion::query_executor; +use opensearch_datafusion::runtime_manager::RuntimeManager; use std::sync::Arc; -use opensearch_datafusion_jni::api::DataFusionRuntime; fn create_test_parquet(dir: &std::path::Path, rows: usize) { use arrow::datatypes::{DataType, Field, Schema}; @@ -25,8 +25,12 @@ fn create_test_parquet(dir: &std::path::Path, rows: usize) { let vals: Vec = ids.iter().map(|i| i * 10).collect(); let batch = RecordBatch::try_new( schema.clone(), - vec![Arc::new(Int64Array::from(ids)), Arc::new(Int64Array::from(vals))], - ).unwrap(); + vec![ + Arc::new(Int64Array::from(ids)), + Arc::new(Int64Array::from(vals)), + ], + ) + .unwrap(); let path = dir.join("bench.parquet"); let file = File::create(&path).unwrap(); @@ -40,7 +44,8 @@ fn setup() -> (RuntimeManager, DataFusionRuntime, tempfile::TempDir) { let runtime_env = RuntimeEnvBuilder::new() .with_memory_pool(Arc::new(GreedyMemoryPool::new(256 * 1024 * 1024))) .with_disk_manager_builder(DiskManagerBuilder::default()) - .build().unwrap(); + .build() + .unwrap(); let df_runtime = DataFusionRuntime { runtime_env }; let tmp = tempfile::tempdir().unwrap(); (mgr, df_runtime, tmp) @@ -56,10 +61,14 @@ fn get_substrait(mgr: &RuntimeManager, df: &DataFusionRuntime, dir: &str, sql: & let ctx = datafusion::prelude::SessionContext::new(); let url = ListingTableUrl::parse(dir).unwrap(); let opts = ListingOptions::new(Arc::new(ParquetFormat::new())) - .with_file_extension(".parquet").with_collect_stat(true); + .with_file_extension(".parquet") + .with_collect_stat(true); let schema = opts.infer_schema(&ctx.state(), &url).await.unwrap(); - let cfg = ListingTableConfig::new(url).with_listing_options(opts).with_schema(schema); - ctx.register_table("t", Arc::new(ListingTable::try_new(cfg).unwrap())).unwrap(); + let cfg = ListingTableConfig::new(url) + .with_listing_options(opts) + .with_schema(schema); + ctx.register_table("t", Arc::new(ListingTable::try_new(cfg).unwrap())) + .unwrap(); let plan = ctx.sql(sql).await.unwrap().logical_plan().clone(); let sub = to_substrait_plan(&plan, &ctx.state()).unwrap(); let mut buf = Vec::new(); @@ -95,12 +104,12 @@ fn bench_execute_query(c: &mut Criterion) { let exec = mgr.cpu_executor(); async { let ptr = query_executor::execute_query( - url, metas, "t".into(), plan, &df_runtime, exec, + url, metas, "t".into(), plan, &df_runtime, exec, None, &opensearch_datafusion::datafusion_query_config::DatafusionQueryConfig::default(), ).await.unwrap(); // Consume and free the stream let mut stream = unsafe { Box::from_raw(ptr as *mut datafusion::physical_plan::stream::RecordBatchStreamAdapter< - opensearch_datafusion_jni::cross_rt_stream::CrossRtStream, + opensearch_datafusion::cross_rt_stream::CrossRtStream, >) }; let mut count = 0u64; @@ -133,12 +142,24 @@ fn bench_stream_next(c: &mut Criterion) { let exec = mgr.cpu_executor(); async { let ptr = query_executor::execute_query( - url, metas, "t".into(), plan, &df_runtime, exec, - ).await.unwrap(); + url, + metas, + "t".into(), + plan, + &df_runtime, + exec, + None, + &opensearch_datafusion::datafusion_query_config::DatafusionQueryConfig::default( + ), + ) + .await + .unwrap(); let mut stream = unsafe { - Box::from_raw(ptr as *mut datafusion::physical_plan::stream::RecordBatchStreamAdapter< - opensearch_datafusion_jni::cross_rt_stream::CrossRtStream, - >) + Box::from_raw( + ptr as *mut datafusion::physical_plan::stream::RecordBatchStreamAdapter< + opensearch_datafusion::cross_rt_stream::CrossRtStream, + >, + ) }; let mut batches = 0u64; while let Some(_) = stream.try_next().await.unwrap() { @@ -169,12 +190,24 @@ fn bench_aggregation(c: &mut Criterion) { let exec = mgr.cpu_executor(); async { let ptr = query_executor::execute_query( - url, metas, "t".into(), plan, &df_runtime, exec, - ).await.unwrap(); + url, + metas, + "t".into(), + plan, + &df_runtime, + exec, + None, + &opensearch_datafusion::datafusion_query_config::DatafusionQueryConfig::default( + ), + ) + .await + .unwrap(); let mut stream = unsafe { - Box::from_raw(ptr as *mut datafusion::physical_plan::stream::RecordBatchStreamAdapter< - opensearch_datafusion_jni::cross_rt_stream::CrossRtStream, - >) + Box::from_raw( + ptr as *mut datafusion::physical_plan::stream::RecordBatchStreamAdapter< + opensearch_datafusion::cross_rt_stream::CrossRtStream, + >, + ) }; while let Some(_) = stream.try_next().await.unwrap() {} } @@ -185,5 +218,10 @@ fn bench_aggregation(c: &mut Criterion) { std::mem::forget(mgr); } -criterion_group!(benches, bench_execute_query, bench_stream_next, bench_aggregation); +criterion_group!( + benches, + bench_execute_query, + bench_stream_next, + bench_aggregation +); criterion_main!(benches); diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs index 973b50f11fac9..a47d5f95fa2f6 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs @@ -8,9 +8,8 @@ //! Bridge-agnostic API layer. //! -//! All functions in this module use plain Rust types — no JNI, no FFI-specific -//! types. Both the current JNI bridge (`lib.rs`) and a future FFM/C bridge can -//! call these functions directly. +//! All functions in this module use plain Rust types — no FFI-specific types. +//! The FFM bridge (`ffm.rs`) calls into these functions directly. //! //! # Pointer contract //! @@ -31,120 +30,6 @@ //! - `stream_next`: async. The bridge layer wraps with `block_on` or `spawn`. //! - `stream_get_schema`, `stream_close` must NOT be called //! concurrently on the same stream pointer. -//! -//! # FFM bridge example -//! -//! When migrating from JNI to JDK FFM (Foreign Function & Memory API), create an -//! `ffi_bridge.rs` that exports `extern "C"` functions calling this API. The JNI -//! bridge (`lib.rs`) and FFM bridge are interchangeable — only the type conversion -//! layer differs. -//! -//! ```rust,ignore -//! // ffi_bridge.rs — extern "C" bridge for JDK FFM (replaces lib.rs JNI bridge) -//! // -//! // Java side uses java.lang.foreign.Linker to call these functions directly. -//! // Strings are passed as (pointer, length) pairs. Byte arrays likewise. -//! // No JNIEnv, no JString, no GlobalRef — pure C ABI. -//! -//! use crate::api; -//! use crate::runtime_manager::RuntimeManager; -//! use std::sync::{Arc, OnceLock}; -//! -//! static RUNTIME_MANAGER: OnceLock> = OnceLock::new(); -//! -//! /// Initialize the Tokio runtime manager. -//! /// Java: MethodHandle = linker.downcallHandle(lib.find("df_init"), FunctionDescriptor.ofVoid(JAVA_INT)); -//! #[no_mangle] -//! pub extern "C" fn df_init(cpu_threads: i32) { -//! RUNTIME_MANAGER.get_or_init(|| Arc::new(RuntimeManager::new(cpu_threads as usize))); -//! } -//! -//! /// Create a global DataFusion runtime. Returns pointer as i64, or 0 on error. -//! /// Java: MethodHandle = linker.downcallHandle(lib.find("df_create_runtime"), -//! /// FunctionDescriptor.of(JAVA_LONG, JAVA_LONG, ADDRESS, JAVA_LONG, JAVA_LONG)); -//! #[no_mangle] -//! pub extern "C" fn df_create_runtime( -//! memory_limit: i64, -//! spill_dir_ptr: *const u8, -//! spill_dir_len: i64, -//! spill_limit: i64, -//! ) -> i64 { -//! let spill_dir = unsafe { -//! std::str::from_utf8_unchecked( -//! std::slice::from_raw_parts(spill_dir_ptr, spill_dir_len as usize) -//! ) -//! }; -//! api::create_global_runtime(memory_limit, spill_dir, spill_limit).unwrap_or(0) -//! } -//! -//! /// Execute a query. Returns stream pointer as i64, or 0 on error. -//! /// Error message written to (err_buf_ptr, err_buf_len), actual length returned via err_len_out. -//! /// Java: MethodHandle = linker.downcallHandle(lib.find("df_execute_query"), -//! /// FunctionDescriptor.of(JAVA_LONG, JAVA_LONG, ADDRESS, JAVA_LONG, ADDRESS, JAVA_LONG, JAVA_LONG)); -//! #[no_mangle] -//! pub extern "C" fn df_execute_query( -//! shard_view_ptr: i64, -//! table_name_ptr: *const u8, -//! table_name_len: i64, -//! plan_ptr: *const u8, -//! plan_len: i64, -//! runtime_ptr: i64, -//! ) -> i64 { -//! let manager = RUNTIME_MANAGER.get().expect("not initialized"); -//! let table_name = unsafe { -//! std::str::from_utf8_unchecked( -//! std::slice::from_raw_parts(table_name_ptr, table_name_len as usize) -//! ) -//! }; -//! let plan_bytes = unsafe { -//! std::slice::from_raw_parts(plan_ptr, plan_len as usize) -//! }; -//! manager.io_runtime.block_on(unsafe { -//! api::execute_query(shard_view_ptr, table_name, plan_bytes, runtime_ptr, manager) -//! }).unwrap_or(0) -//! } -//! -//! /// Get next batch. Returns FFI_ArrowArray pointer, 0 for end-of-stream, -1 on error. -//! #[no_mangle] -//! pub extern "C" fn df_stream_next(stream_ptr: i64) -> i64 { -//! let manager = RUNTIME_MANAGER.get().expect("not initialized"); -//! manager.io_runtime.block_on(unsafe { api::stream_next(stream_ptr) }).unwrap_or(-1) -//! } -//! -//! /// Close a stream. Safe with 0. -//! #[no_mangle] -//! pub extern "C" fn df_stream_close(stream_ptr: i64) { -//! unsafe { api::stream_close(stream_ptr) }; -//! } -//! -//! // Java side (JDK 22+): -//! // -//! // try (Arena arena = Arena.ofConfined()) { -//! // SymbolLookup lib = SymbolLookup.libraryLookup("libopensearch_datafusion.so", arena); -//! // Linker linker = Linker.nativeLinker(); -//! // -//! // var init = linker.downcallHandle( -//! // lib.find("df_init").get(), -//! // FunctionDescriptor.ofVoid(ValueLayout.JAVA_INT) -//! // ); -//! // init.invoke(Runtime.getRuntime().availableProcessors()); -//! // -//! // var createRuntime = linker.downcallHandle( -//! // lib.find("df_create_runtime").get(), -//! // FunctionDescriptor.of(JAVA_LONG, JAVA_LONG, ADDRESS, JAVA_LONG, JAVA_LONG) -//! // ); -//! // MemorySegment spillDir = arena.allocateFrom("/tmp/spill"); -//! // long runtimePtr = (long) createRuntime.invoke(512_000_000L, spillDir, spillDir.byteSize(), 256_000_000L); -//! // -//! // var executeQuery = linker.downcallHandle( -//! // lib.find("df_execute_query").get(), -//! // FunctionDescriptor.of(JAVA_LONG, JAVA_LONG, ADDRESS, JAVA_LONG, ADDRESS, JAVA_LONG, JAVA_LONG) -//! // ); -//! // MemorySegment tableName = arena.allocateFrom("my_table"); -//! // MemorySegment plan = arena.allocateFrom(MemoryLayout.sequenceLayout(planBytes.length, JAVA_BYTE), planBytes); -//! // long streamPtr = (long) executeQuery.invoke(shardViewPtr, tableName, tableName.byteSize(), plan, plan.byteSize(), runtimePtr); -//! // } -//! ``` use std::io::Cursor; use std::num::NonZeroUsize; @@ -152,18 +37,18 @@ use std::path::PathBuf; use std::sync::Arc; use arrow::ipc::reader::StreamReader; -use arrow_array::{Array, StructArray}; use arrow_array::ffi::FFI_ArrowArray; use arrow_array::RecordBatch; +use arrow_array::{Array, StructArray}; use arrow_schema::ffi::FFI_ArrowSchema; use datafusion::common::DataFusionError; use datafusion::datasource::listing::ListingTableUrl; use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; use datafusion::execution::memory_pool::{GreedyMemoryPool, TrackConsumersPool}; use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use datafusion::execution::RecordBatchStream; use datafusion::execution::SessionStateBuilder; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::execution::RecordBatchStream; use datafusion::prelude::SessionConfig; use futures::TryStreamExt; @@ -183,8 +68,14 @@ pub struct QueryStreamHandle { } impl QueryStreamHandle { - pub fn new(stream: RecordBatchStreamAdapter, query_context: QueryTrackingContext) -> Self { - Self { stream, _query_tracking_context: query_context } + pub fn new( + stream: RecordBatchStreamAdapter, + query_context: QueryTrackingContext, + ) -> Self { + Self { + stream, + _query_tracking_context: query_context, + } } } @@ -203,7 +94,10 @@ pub async fn create_object_metas( }; let path = object_store::path::Path::from(full_path.as_str()); let meta = store.head(&path).await.map_err(|e| { - DataFusionError::Execution(format!("Failed to get object meta for {}: {}", full_path, e)) + DataFusionError::Execution(format!( + "Failed to get object meta for {}: {}", + full_path, e + )) })?; metas.push(meta); } @@ -217,7 +111,7 @@ pub struct DataFusionRuntime { } /// Opaque shard view handle returned to the caller. -pub(crate) struct ShardView { +pub struct ShardView { pub table_path: ListingTableUrl, pub object_metas: Arc>, } @@ -277,9 +171,11 @@ pub fn create_reader( let default_rt = RuntimeEnvBuilder::new().build()?; let store = default_rt.object_store(&table_url)?; - let object_metas = tokio_rt_manager.io_runtime.block_on( - create_object_metas(store.as_ref(), table_path, filenames), - )?; + let object_metas = tokio_rt_manager.io_runtime.block_on(create_object_metas( + store.as_ref(), + table_path, + filenames, + ))?; let shard_view = ShardView { table_path: table_url, @@ -302,7 +198,7 @@ pub unsafe fn close_reader(ptr: i64) { /// Caller must call `stream_close` exactly once to free it. /// /// This is an async function — the bridge layer decides how to run it -/// (`block_on` for synchronous JNI, `spawn` for async delivery). +/// (`block_on` for synchronous delivery, `spawn` for async delivery). /// /// # Safety /// `shard_view_ptr` and `runtime_ptr` must be valid, non-zero pointers. @@ -313,37 +209,76 @@ pub async unsafe fn execute_query( runtime_ptr: i64, manager: &RuntimeManager, context_id: i64, + query_config: crate::datafusion_query_config::DatafusionQueryConfig, ) -> Result { let shard_view = &*(shard_view_ptr as *const ShardView); let runtime = &*(runtime_ptr as *const DataFusionRuntime); - - let table_path = shard_view.table_path.clone(); - let object_metas = shard_view.object_metas.clone(); let cpu_executor = manager.cpu_executor(); // Create per-query context — auto-registers in the global registry let global_pool = runtime.runtime_env.memory_pool.clone(); let query_context = QueryTrackingContext::new(context_id, global_pool); - let query_memory_pool = query_context.memory_pool() + let query_memory_pool = query_context + .memory_pool() .map(|p| p as Arc); - let stream_ptr = crate::query_executor::execute_query( - table_path, - object_metas, - table_name.to_string(), - plan_bytes.to_vec(), - runtime, - cpu_executor, - query_memory_pool, - ) - .await?; - - // Reconstruct the stream from the raw pointer returned by query_executor + // Peek at the substrait extensions list to see if this is an indexed query. + // The `index_filter` UDF name appears there if Calcite planted any + // index_filter(bytes) calls. Cheap — just bytes inspection. + let is_indexed = plan_bytes_mentions_index_filter(plan_bytes); + + let stream_ptr = if is_indexed { + let qc = Arc::new(query_config); + crate::indexed_executor::execute_indexed_query( + plan_bytes.to_vec(), + table_name.to_string(), + shard_view, + qc.target_partitions.max(1), + runtime, + cpu_executor, + query_memory_pool, + qc, + ) + .await? + } else { + crate::query_executor::execute_query( + shard_view.table_path.clone(), + shard_view.object_metas.clone(), + table_name.to_string(), + plan_bytes.to_vec(), + runtime, + cpu_executor, + query_memory_pool, + &query_config, + ) + .await? + }; + + // Reconstruct the stream from the raw pointer returned by the executor. let stream = *Box::from_raw(stream_ptr as *mut RecordBatchStreamAdapter); let handle = QueryStreamHandle::new(stream, query_context); Ok(Box::into_raw(Box::new(handle)) as i64) } +/// Cheap check: scan the substrait plan bytes for the `index_filter` function +/// name. If the planner emitted any `index_filter(bytes)` UDF call, the name +/// will be present in the plan's extension declarations. +/// +/// False positives take the indexed path and then fail in +/// `execute_indexed_query` when `classify_filter` returns `None` +/// ("execute_indexed_query called with no index_filter(...) in plan"). There +/// is no automatic retry on the vanilla path — a false positive is a hard +/// query error. In practice this is unreachable because the needle is not a +/// valid DataFusion identifier anywhere else a plan would naturally contain +/// it; the failure mode is documented here to keep the dispatch contract +/// explicit. +fn plan_bytes_mentions_index_filter(plan_bytes: &[u8]) -> bool { + // The substrait plan carries extension-function names as UTF-8 strings. + // Substring match is sufficient for dispatch. + const NEEDLE: &[u8] = b"index_filter"; + plan_bytes.windows(NEEDLE.len()).any(|w| w == NEEDLE) +} + /// Returns the Arrow schema for the given stream as a heap-allocated FFI_ArrowSchema pointer. /// /// # Safety @@ -365,9 +300,7 @@ pub unsafe fn stream_get_schema(stream_ptr: i64) -> Result /// # Safety /// `stream_ptr` must be a valid, non-zero pointer. Must not be called concurrently /// on the same stream. -pub async unsafe fn stream_next( - stream_ptr: i64, -) -> Result { +pub async unsafe fn stream_next(stream_ptr: i64) -> Result { let handle = &mut *(stream_ptr as *mut QueryStreamHandle); let result = handle.stream.try_next().await?; @@ -406,10 +339,10 @@ pub unsafe fn sql_to_substrait( runtime_ptr: i64, manager: &RuntimeManager, ) -> Result, DataFusionError> { - use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig}; use datafusion::datasource::file_format::parquet::ParquetFormat; - use datafusion::execution::cache::{CacheAccessor, DefaultListFilesCache}; + use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig}; use datafusion::execution::cache::cache_manager::CacheManagerConfig; + use datafusion::execution::cache::{CacheAccessor, DefaultListFilesCache}; use datafusion_substrait::logical_plan::producer::to_substrait_plan; use prost::Message; @@ -451,7 +384,9 @@ pub unsafe fn sql_to_substrait( let listing_options = ListingOptions::new(Arc::new(ParquetFormat::new())) .with_file_extension(".parquet") .with_collect_stat(true); - let schema = listing_options.infer_schema(&ctx.state(), &table_path).await?; + let schema = listing_options + .infer_schema(&ctx.state(), &table_path) + .await?; let config = ListingTableConfig::new(table_path) .with_listing_options(listing_options) .with_schema(schema); @@ -460,7 +395,8 @@ pub unsafe fn sql_to_substrait( let plan = ctx.sql(sql).await?.logical_plan().clone(); let substrait = to_substrait_plan(&plan, &ctx.state())?; let mut buf = Vec::new(); - substrait.encode(&mut buf) + substrait + .encode(&mut buf) .map_err(|e| DataFusionError::Execution(format!("Substrait encode failed: {}", e)))?; Ok(buf) }) @@ -691,12 +627,13 @@ pub unsafe fn register_memtable( })?; let struct_array = StructArray::from(array_data); let raw = RecordBatch::from(struct_array); - let aligned = RecordBatch::try_new(Arc::clone(&table_schema), raw.columns().to_vec()).map_err(|e| { - DataFusionError::Execution(format!( - "Failed to align imported batch to registered schema for '{}': {}", - input_id, e - )) - })?; + let aligned = RecordBatch::try_new(Arc::clone(&table_schema), raw.columns().to_vec()) + .map_err(|e| { + DataFusionError::Execution(format!( + "Failed to align imported batch to registered schema for '{}': {}", + input_id, e + )) + })?; batches.push(aligned); } diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs new file mode 100644 index 0000000000000..3725761e3fa6f --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs @@ -0,0 +1,233 @@ +//! Per-query tuning knobs shared by the vanilla and indexed query paths. +//! +//! Populated from Java (cluster / index / request settings) and passed to +//! Rust once at query start via a `#[repr(C)]` wire struct. Read out at +//! setup time and copied into hot-path fields — never dereferenced on a +//! per-batch or per-row hot path. + +use crate::indexed_table::stream::FilterStrategy; +use crate::indexed_table::eval::single_collector::CollectorCallStrategy; + +/// Query-scoped configuration. Owned by value after FFM decode. +#[derive(Debug, Clone)] +pub struct DatafusionQueryConfig { + // Common + pub batch_size: usize, + // Single query concurrency + pub target_partitions: usize, + /// DataFusion's own decode-time predicate pushdown on the vanilla path. + pub parquet_pushdown_filters: bool, + + // Indexed-only + pub min_skip_run_default: usize, + pub min_skip_run_selectivity_threshold: f64, + /// Whether IndexedStream asks parquet to apply the residual predicate + /// during decode (via `RowFilter` pushdown). Narrow row-granular + /// selections benefit; block-granular ones don't. + pub indexed_pushdown_filters: bool, + pub force_strategy: Option, + pub force_pushdown: Option, + pub cost_predicate: u32, + pub cost_collector: u32, + /// Maximum number of Collector-leaf FFM calls issued in parallel per + /// RG prefetch. 1 = today's fully-sequential behaviour (lowest CPU, + /// fastest short-circuit). `target_partitions × max_collector_parallelism` + /// bounds total concurrent Lucene threads; default is 1 + /// + /// At higher values, short-circuit savings in AND/OR groups are + /// sacrificed (see `BitmapTreeEvaluator::prefetch`): collectors + /// beyond the first may run even if their result is not needed. + pub max_collector_parallelism: usize, + /// How the SingleCollectorEvaluator narrows collector doc ranges + /// relative to page-pruning results. `PageRangeSplit` is the default + /// — only one collector, so multiple FFM calls per RG is acceptable. + pub single_collector_strategy: CollectorCallStrategy, + /// How the bitmap tree evaluator narrows collector doc ranges. + /// `TightenOuterBounds` is the default — multiple collectors in the + /// tree means `PageRangeSplit` would multiply FFM calls. + pub tree_collector_strategy: CollectorCallStrategy, +} + +impl Default for DatafusionQueryConfig { + fn default() -> Self { + Self { + batch_size: 8192, + // TODO: change this default value ? + target_partitions: 4, + parquet_pushdown_filters: false, + min_skip_run_default: 1024, // Todo: tune based on benchmarks + min_skip_run_selectivity_threshold: 0.03, // Todo : tune based on benchmarks + indexed_pushdown_filters: true, + force_strategy: None, + force_pushdown: None, + cost_predicate: 1, + cost_collector: 10, // TODO : should this be collector leaf specific + max_collector_parallelism: 1, + single_collector_strategy: CollectorCallStrategy::PageRangeSplit, + tree_collector_strategy: CollectorCallStrategy::TightenOuterBounds, + } + } +} + +/// FFM wire format. Must stay in lockstep with the Java `MemoryLayout`. +/// +/// All fields have fixed sizes and natural alignment so Java and Rust +/// produce the same byte layout on all target platforms. Enum-ish +/// `Option<_>` fields are encoded with a `-1` sentinel for `None`. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct WireDatafusionQueryConfig { + pub batch_size: i64, + pub target_partitions: i64, + pub min_skip_run_default: i64, + pub min_skip_run_selectivity_threshold: f64, + /// 0 = false, 1 = true + pub parquet_pushdown_filters: i32, + /// 0 = false, 1 = true + pub indexed_pushdown_filters: i32, + /// -1 = None, 0 = RowSelection, 1 = BooleanMask + pub force_strategy: i32, + /// -1 = None, 0 = false, 1 = true + pub force_pushdown: i32, + pub cost_predicate: i32, + pub cost_collector: i32, + pub max_collector_parallelism: i32, + /// 0 = FullRange, 1 = TightenOuterBounds, 2 = PageRangeSplit + pub single_collector_strategy: i32, + /// 0 = FullRange, 1 = TightenOuterBounds, 2 = PageRangeSplit + pub tree_collector_strategy: i32, +} + +impl DatafusionQueryConfig { + /// Decode from a raw FFM pointer. Null (`0`) returns defaults. + /// + /// # Safety + /// `ptr` must be 0, or a valid pointer to a `WireDatafusionQueryConfig` + /// whose memory is live for the duration of this call. + pub unsafe fn from_ffm_ptr(ptr: i64) -> Self { + if ptr == 0 { + return Self::default(); + } + let wire = &*(ptr as *const WireDatafusionQueryConfig); + Self::from_wire(wire) + } + + fn from_wire(w: &WireDatafusionQueryConfig) -> Self { + let force_strategy = match w.force_strategy { + 0 => Some(FilterStrategy::RowSelection), + 1 => Some(FilterStrategy::BooleanMask), + _ => None, + }; + let force_pushdown = match w.force_pushdown { + 0 => Some(false), + 1 => Some(true), + _ => None, + }; + Self { + batch_size: w.batch_size as usize, + target_partitions: w.target_partitions as usize, + parquet_pushdown_filters: w.parquet_pushdown_filters != 0, + min_skip_run_default: w.min_skip_run_default as usize, + min_skip_run_selectivity_threshold: w.min_skip_run_selectivity_threshold, + indexed_pushdown_filters: w.indexed_pushdown_filters != 0, + force_strategy, + force_pushdown, + cost_predicate: w.cost_predicate as u32, + cost_collector: w.cost_collector as u32, + max_collector_parallelism: (w.max_collector_parallelism as usize).max(1), + single_collector_strategy: match w.single_collector_strategy { + 0 => CollectorCallStrategy::FullRange, + 1 => CollectorCallStrategy::TightenOuterBounds, + _ => CollectorCallStrategy::PageRangeSplit, + }, + tree_collector_strategy: match w.tree_collector_strategy { + 0 => CollectorCallStrategy::FullRange, + 2 => CollectorCallStrategy::PageRangeSplit, + _ => CollectorCallStrategy::TightenOuterBounds, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_match_legacy_constants() { + let c = DatafusionQueryConfig::default(); + assert_eq!(c.batch_size, 8192); + assert_eq!(c.target_partitions, 4); + assert!(!c.parquet_pushdown_filters); + assert_eq!(c.min_skip_run_default, 1024); + assert!((c.min_skip_run_selectivity_threshold - 0.03).abs() < 1e-9); + assert!(c.indexed_pushdown_filters); + assert_eq!(c.force_strategy, None); + assert_eq!(c.force_pushdown, None); + assert_eq!(c.cost_predicate, 1); + assert_eq!(c.cost_collector, 10); + } + + #[test] + fn wire_decode_null_pointer_gives_defaults() { + let c = unsafe { DatafusionQueryConfig::from_ffm_ptr(0) }; + let d = DatafusionQueryConfig::default(); + assert_eq!(c.batch_size, d.batch_size); + assert_eq!(c.min_skip_run_default, d.min_skip_run_default); + assert_eq!(c.cost_collector, d.cost_collector); + } + + #[test] + fn wire_decode_round_trips_all_fields() { + let wire = WireDatafusionQueryConfig { + batch_size: 16384, + target_partitions: 8, + min_skip_run_default: 512, + min_skip_run_selectivity_threshold: 0.07, + parquet_pushdown_filters: 1, + indexed_pushdown_filters: 0, + force_strategy: 1, + force_pushdown: 0, + cost_predicate: 3, + cost_collector: 17, + max_collector_parallelism: 4, + single_collector_strategy: 2, + tree_collector_strategy: 1, + }; + let ptr = &wire as *const _ as i64; + let c = unsafe { DatafusionQueryConfig::from_ffm_ptr(ptr) }; + assert_eq!(c.batch_size, 16384); + assert_eq!(c.target_partitions, 8); + assert_eq!(c.min_skip_run_default, 512); + assert!((c.min_skip_run_selectivity_threshold - 0.07).abs() < 1e-9); + assert!(c.parquet_pushdown_filters); + assert!(!c.indexed_pushdown_filters); + assert_eq!(c.force_strategy, Some(FilterStrategy::BooleanMask)); + assert_eq!(c.force_pushdown, Some(false)); + assert_eq!(c.cost_predicate, 3); + assert_eq!(c.cost_collector, 17); + } + + #[test] + fn wire_decode_force_fields_none_sentinels() { + let wire = WireDatafusionQueryConfig { + batch_size: 8192, + target_partitions: 4, + min_skip_run_default: 1024, + min_skip_run_selectivity_threshold: 0.03, + parquet_pushdown_filters: 0, + indexed_pushdown_filters: 1, + force_strategy: -1, + force_pushdown: -1, + cost_predicate: 1, + cost_collector: 10, + max_collector_parallelism: 2, + single_collector_strategy: 2, + tree_collector_strategy: 1, + }; + let ptr = &wire as *const _ as i64; + let c = unsafe { DatafusionQueryConfig::from_ffm_ptr(ptr) }; + assert_eq!(c.force_strategy, None); + assert_eq!(c.force_pushdown, None); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs index 2b6b53426e179..e5d3a16c46493 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs @@ -34,7 +34,6 @@ pub enum JobError { Panic { msg: String }, } - struct State { handle: Option, start_shutdown: Arc, @@ -100,10 +99,7 @@ impl DedicatedExecutor { let state = State { handle: Some(handle), start_shutdown: notify_shutdown, - completed_shutdown: rx_shutdown - .map_err(Arc::new) - .boxed() - .shared(), + completed_shutdown: rx_shutdown.map_err(Arc::new).boxed().shared(), thread: Some(thread), }; Self { @@ -189,7 +185,10 @@ mod tests { async fn test_spawn_runs_on_different_thread() { let exec = test_exec(1); let caller_id = std::thread::current().id(); - let spawned_id = exec.spawn(async { std::thread::current().id() }).await.unwrap(); + let spawned_id = exec + .spawn(async { std::thread::current().id() }) + .await + .unwrap(); assert_ne!(caller_id, spawned_id); exec.join_blocking(); } @@ -200,11 +199,17 @@ mod tests { let exec = test_exec(2); let t1 = exec.spawn({ let b = barrier.clone(); - async move { b.wait(); 11 } + async move { + b.wait(); + 11 + } }); let t2 = exec.spawn({ let b = barrier.clone(); - async move { b.wait(); 22 } + async move { + b.wait(); + 22 + } }); barrier.wait(); assert_eq!(t1.await.unwrap(), 11); diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index 6a531a7839bfa..6ea0b510e3eee 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -60,9 +60,9 @@ pub unsafe extern "C" fn df_create_global_runtime( spill_dir_len: i64, spill_limit: i64, ) -> i64 { - let spill_dir = str_from_raw(spill_dir_ptr, spill_dir_len).map_err(|e| format!("df_create_global_runtime: {}", e))?; - api::create_global_runtime(memory_pool_limit, spill_dir, spill_limit) - .map_err(|e| e.to_string()) + let spill_dir = str_from_raw(spill_dir_ptr, spill_dir_len) + .map_err(|e| format!("df_create_global_runtime: {}", e))?; + api::create_global_runtime(memory_pool_limit, spill_dir, spill_limit).map_err(|e| e.to_string()) } #[no_mangle] @@ -79,12 +79,17 @@ pub unsafe extern "C" fn df_create_reader( files_len_ptr: *const i64, files_count: i64, ) -> i64 { - let table_path = str_from_raw(table_path_ptr, table_path_len).map_err(|e| format!("df_create_reader: {}", e))?; + let table_path = str_from_raw(table_path_ptr, table_path_len) + .map_err(|e| format!("df_create_reader: {}", e))?; let mut filenames = Vec::with_capacity(files_count as usize); for i in 0..files_count as usize { let ptr = *files_ptr.add(i); let len = *files_len_ptr.add(i); - filenames.push(str_from_raw(ptr, len).map_err(|e| format!("df_create_reader: {}", e))?.to_string()); + filenames.push( + str_from_raw(ptr, len) + .map_err(|e| format!("df_create_reader: {}", e))? + .to_string(), + ); } let mgr = get_rt_manager()?; api::create_reader(table_path, filenames, &mgr).map_err(|e| e.to_string()) @@ -105,12 +110,25 @@ pub unsafe extern "C" fn df_execute_query( plan_len: i64, runtime_ptr: i64, context_id: i64, + // Pointer to a `WireDatafusionQueryConfig`. `0` = use defaults. + query_config_ptr: i64, ) -> i64 { let mgr = get_rt_manager()?; - let table_name = str_from_raw(table_name_ptr, table_name_len).map_err(|e| format!("df_execute_query: {}", e))?; + let table_name = str_from_raw(table_name_ptr, table_name_len) + .map_err(|e| format!("df_execute_query: {}", e))?; let plan_bytes = slice::from_raw_parts(plan_ptr, plan_len as usize); + let query_config = + crate::datafusion_query_config::DatafusionQueryConfig::from_ffm_ptr(query_config_ptr); mgr.io_runtime - .block_on(api::execute_query(shard_view_ptr, table_name, plan_bytes, runtime_ptr, &mgr, context_id)) + .block_on(api::execute_query( + shard_view_ptr, + table_name, + plan_bytes, + runtime_ptr, + &mgr, + context_id, + query_config, + )) .map_err(|e| e.to_string()) } @@ -148,8 +166,10 @@ pub unsafe extern "C" fn df_sql_to_substrait( out_len: *mut i64, ) -> i64 { let mgr = get_rt_manager()?; - let table_name = str_from_raw(table_name_ptr, table_name_len).map_err(|e| format!("df_sql_to_substrait: table_name: {}", e))?; - let sql = str_from_raw(sql_ptr, sql_len).map_err(|e| format!("df_sql_to_substrait: sql: {}", e))?; + let table_name = str_from_raw(table_name_ptr, table_name_len) + .map_err(|e| format!("df_sql_to_substrait: table_name: {}", e))?; + let sql = + str_from_raw(sql_ptr, sql_len).map_err(|e| format!("df_sql_to_substrait: sql: {}", e))?; let bytes = api::sql_to_substrait(shard_view_ptr, table_name, sql, runtime_ptr, &mgr) .map_err(|e| e.to_string())?; if bytes.len() > out_cap as usize { diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs new file mode 100644 index 0000000000000..12ce9342f5989 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs @@ -0,0 +1,641 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Indexed query executor — decodes substrait, classifies the filter tree, +//! builds providers per leaf, runs the query. +//! +//! Per-leaf lifecycle at query time (one compiled-query + per-segment matcher +//! per Collector leaf): +//! 1. `createProvider(query_bytes)` FFM upcall → `provider_key` (once per +//! Collector leaf, once per query). +//! 2. `createCollector(provider_key, seg, min, max)` FFM upcall → collector +//! (once per SegmentChunk × Collector leaf). +//! 3. `collectDocs(collector, min, max, out)` FFM upcall (once per row group). +//! 4. `releaseCollector(collector)` when RG scan completes. +//! 5. `releaseProvider(provider_key)` when the tree is dropped. + +use std::sync::Arc; + +use datafusion::{ + physical_plan::execute_stream, + execution::SessionStateBuilder, + execution::runtime_env::RuntimeEnvBuilder, + execution::context::SessionContext, + common::DataFusionError, + prelude::*, + arrow::datatypes::SchemaRef, + catalog::Session, + common::tree_node::{TreeNode, TreeNodeRecursion}, + datasource::{TableProvider, TableType}, + execution::cache::cache_manager::CacheManagerConfig, + execution::cache::{CacheAccessor, DefaultListFilesCache, TableScopedPath}, + execution::memory_pool::MemoryPool, + execution::object_store::ObjectStoreUrl, + logical_expr::Expr, + physical_expr::expressions::Column, + physical_expr::PhysicalExpr, + physical_optimizer::pruning::PruningPredicate, + physical_plan::stream::RecordBatchStreamAdapter, + physical_plan::ExecutionPlan +}; +use datafusion_substrait::logical_plan::consumer::from_substrait_plan; +use prost::Message; +use substrait::proto::Plan; + +use crate::api::DataFusionRuntime; +use crate::cross_rt_stream::CrossRtStream; +use crate::executor::DedicatedExecutor; +use crate::indexed_table::bool_tree::BoolNode; +use crate::indexed_table::eval::bitmap_tree::{BitmapTreeEvaluator, CollectorLeafBitmaps}; +use crate::indexed_table::eval::single_collector::SingleCollectorEvaluator; +use crate::indexed_table::eval::{CollectorCallStrategy, RowGroupBitsetSource, TreeBitsetSource}; +use crate::indexed_table::ffm_callbacks::{create_provider, FfmSegmentCollector, ProviderHandle}; +use crate::indexed_table::index::RowGroupDocsCollector; +use crate::indexed_table::page_pruner::PagePruner; +use crate::indexed_table::segment_info::build_segments; +use crate::indexed_table::substrait_to_tree::{ + classify_filter, create_index_filter_udf, expr_to_bool_tree, extract_filter_expr, + ExtractionResult, FilterClass, +}; +use crate::indexed_table::table_provider::{ + EvaluatorFactory, IndexedTableConfig, IndexedTableProvider, SegmentFileInfo, +}; + +use std::collections::{BTreeSet, HashMap}; +use std::fmt; + +use crate::api::ShardView; +use crate::datafusion_query_config::DatafusionQueryConfig; +use crate::indexed_table::bool_tree::residual_bool_to_physical_expr; +use crate::indexed_table::metrics::StreamMetrics; +use crate::indexed_table::page_pruner::{build_pruning_predicate, PagePruneMetrics}; + +/// Execute an indexed query. +/// +/// `shard_view` carries the segment's parquet paths (populated when the reader +/// was built from a catalog snapshot). `num_partitions` comes from the caller's +/// session config. `query_memory_pool` is the per-query tracker (same as +/// vanilla path) — `None` disables tracking and uses the global pool. +pub async fn execute_indexed_query( + substrait_bytes: Vec, + table_name: String, + shard_view: &ShardView, + num_partitions: usize, + runtime: &DataFusionRuntime, + cpu_executor: DedicatedExecutor, + query_memory_pool: Option>, + query_config: Arc, +) -> Result { + // Share caches with the global runtime (same as vanilla path): list-files + // pre-populated with the reader's object_metas, file-metadata and + // file-statistics inherited from the global runtime for cross-query reuse. + let list_file_cache = Arc::new(DefaultListFilesCache::default()); + let table_scoped_path = TableScopedPath { + table: None, + path: shard_view.table_path.prefix().clone(), + }; + list_file_cache.put(&table_scoped_path, shard_view.object_metas.clone()); + + let mut runtime_env_builder = RuntimeEnvBuilder::from_runtime_env(&runtime.runtime_env) + .with_cache_manager( + CacheManagerConfig::default() + .with_list_files_cache(Some(list_file_cache)) + .with_file_metadata_cache(Some( + runtime.runtime_env.cache_manager.get_file_metadata_cache(), + )) + .with_files_statistics_cache( + runtime.runtime_env.cache_manager.get_file_statistic_cache(), + ), + ); + if let Some(pool) = query_memory_pool { + runtime_env_builder = runtime_env_builder.with_memory_pool(pool); + } + let runtime_env = runtime_env_builder + .build() + .map_err(|e| DataFusionError::Execution(format!("runtime env: {}", e)))?; + + let mut config = SessionConfig::new(); + config.options_mut().execution.parquet.pushdown_filters = query_config.parquet_pushdown_filters; + // Indexed path fans out via IndexedExec partitions (derived from + // num_partitions), not DataFusion's. But DF wants a sane value here + // for any post-scan operators it may add. + config.options_mut().execution.target_partitions = num_partitions.max(1); + config.options_mut().execution.batch_size = query_config.batch_size; + let state = SessionStateBuilder::new() + .with_config(config) + .with_runtime_env(Arc::from(runtime_env)) + .with_default_features() + .build(); + let ctx = SessionContext::new_with_state(state); + ctx.register_udf(create_index_filter_udf()); + + // Resolve the object store for this shard's table URL (file://, s3://, + // gs://, ... whatever the global runtime has registered). We pass this + // store down to the parquet bridge so per-RG reads go through it instead + // of hitting the local filesystem directly. + let store = ctx + .state() + .runtime_env() + .object_store(&shard_view.table_path)?; + + let (segments, schema) = build_segments(Arc::clone(&store), shard_view.object_metas.as_ref()) + .await + .map_err(DataFusionError::Execution)?; + + let placeholder: Arc = Arc::new(PlaceholderProvider { + schema: schema.clone(), + }); + ctx.register_table(&table_name, placeholder)?; + + let plan = Plan::decode(substrait_bytes.as_slice()) + .map_err(|e| DataFusionError::Execution(format!("decode substrait: {}", e)))?; + let logical_plan = from_substrait_plan(&ctx.state(), &plan).await?; + + let filter_expr = extract_filter_expr(&logical_plan); + let extraction = match filter_expr { + None => None, + Some(ref expr) => Some( + expr_to_bool_tree(expr, &schema) + .map_err(|e| DataFusionError::Execution(format!("expr_to_bool_tree: {}", e)))?, + ), + }; + let classification = match &extraction { + None => FilterClass::None, + Some(e) => classify_filter(&e.tree), + }; + + // Derive the parquet pushdown predicate from the BoolNode tree. + // `scan()` ignores DataFusion's filters argument (which contains + // the `index_filter` UDF marker whose body panics) and uses this + // field instead. + // + // SingleCollector: residual (non-Collector top-AND children) → + // PhysicalExpr for `ParquetSource::with_predicate`. In + // row-granular mode parquet narrows Collector-matching rows via + // RowSelection and drops residual-failing rows via pushdown. + // In block-granular mode the evaluator's `on_batch_mask` applies + // both mask and residual post-decode, and pushdown is suppressed + // by the stream's `will_build_mask` guard (to avoid misalignment). + // Tree: None — BitmapTreeEvaluator walks the whole BoolNode in + // `on_batch_mask` using arrow kernels; no pushdown needed. + let pushdown_predicate: Option> = match &classification { + FilterClass::SingleCollector => extraction.as_ref().and_then(|e| { + let residual_bool = extract_single_collector_residual(&e.tree); + residual_bool + .as_ref() + .and_then(residual_bool_to_physical_expr) + }), + FilterClass::Tree | FilterClass::None => None, + }; + + let predicate_columns = collect_predicate_column_indices(extraction.as_ref()); + + let factory: EvaluatorFactory = match classification { + FilterClass::None => { + return Err(DataFusionError::Execution( + "execute_indexed_query called with no index_filter(...) in plan".into(), + )); + } + FilterClass::SingleCollector => { + let extraction = extraction.as_ref().ok_or_else(|| { + DataFusionError::Internal( + "classify_filter returned SingleCollector but extraction is None".into(), + ) + })?; + let bytes = single_collector_bytes(&extraction.tree).ok_or_else(|| { + DataFusionError::Internal( + "SingleCollector classified but leaf extraction failed".into(), + ) + })?; + let provider = + Arc::new(create_provider(&bytes).map_err(|e| DataFusionError::External(e.into()))?); + let schema_for_pruner = schema.clone(); + + // Extract the residual (non-Collector children of top-level + // AND) as a BoolNode and convert to PhysicalExpr. Used for: + // - Page-stats pruning in candidate stage (via PruningPredicate). + // - Parquet `with_predicate` pushdown in row-granular mode. + // - `on_batch_mask` refinement in block-granular mode. + // + // SingleCollector is always AND(Collector, residual...) so + // the residual has zero Collectors — no Literal(true) + // substitution needed (unlike bool_tree_to_pruning_expr + // which handles arbitrary trees). + let residual_bool = extract_single_collector_residual(&extraction.tree); + let residual_expr = residual_bool + .as_ref() + .and_then(residual_bool_to_physical_expr); + let residual_pruning_predicate: Option> = residual_expr + .as_ref() + .and_then(|expr| build_pruning_predicate(expr, Arc::clone(&schema_for_pruner))); + + let call_strategy = query_config.single_collector_strategy; + Arc::new( + move |segment: &SegmentFileInfo, chunk, stream_metrics: &StreamMetrics| { + let collector = FfmSegmentCollector::create( + provider.key(), + segment.segment_ord, + chunk.doc_min, + chunk.doc_max, + ) + .map_err(|e| { + format!( + "FfmSegmentCollector::create(provider={}, seg={}, doc_range=[{},{})): {}", + provider.key(), + segment.segment_ord, + chunk.doc_min, + chunk.doc_max, + e + ) + })?; + let pruner = Arc::new(PagePruner::new( + &schema_for_pruner, + Arc::clone(&segment.metadata), + )); + let eval: Arc = + Arc::new(SingleCollectorEvaluator::new( + Arc::new(collector) as Arc, + pruner, + residual_pruning_predicate.clone(), + residual_expr.clone(), + Some(PagePruneMetrics::from_stream_metrics(stream_metrics)), + stream_metrics.ffm_collector_calls.clone(), + call_strategy, + )); + Ok(eval) + }, + ) + } + FilterClass::Tree => { + let extraction = extraction.ok_or_else(|| { + DataFusionError::Internal( + "classify_filter returned Tree but extraction is None".into(), + ) + })?; + // Normalize: push NOTs to leaves (De Morgan) then flatten nested + // same-kind connectives. Flatten after push_not_down so the + // connective changes from De Morgan (e.g. NOT(AND(...)) -> OR(NOT...)) + // get absorbed into the surrounding Or if applicable. + let tree = extraction.tree.push_not_down().flatten(); + // One provider per Collector leaf (DFS order). + let leaf_bytes = tree.collector_leaves(); + let mut providers: Vec> = Vec::with_capacity(leaf_bytes.len()); + for bytes in &leaf_bytes { + providers.push(Arc::new( + create_provider(bytes).map_err(|e| DataFusionError::External(e.into()))?, + )); + } + let tree = Arc::new(tree); + let schema_for_pruner = schema.clone(); + let cost_predicate = query_config.cost_predicate; + let cost_collector = query_config.cost_collector; + let max_collector_parallelism = query_config.max_collector_parallelism; + let collector_strategy = query_config.tree_collector_strategy; + + // Build one `PruningPredicate` per unique `Predicate` leaf + // in the tree. Key = `Arc::as_ptr(expr) as usize` — the + // same `Arc` reaches the tree walker at + // candidate stage. Predicates that fail to translate or + // resolve to always-true are omitted; the walker's + // fallback treats missing entries as "no pruning for this + // leaf" (safe: universe bitmap). + let mut leaf_exprs: Vec> = Vec::new(); + collect_predicate_exprs(&tree, &mut leaf_exprs); + let pruning_predicates: Arc>> = Arc::new( + leaf_exprs + .iter() + .filter_map(|expr| { + let result = build_pruning_predicate(expr, Arc::clone(&schema_for_pruner)); + result.map(|pp| (Arc::as_ptr(expr) as *const () as usize, pp)) + }) + .collect(), + ); + + Arc::new( + move |segment: &SegmentFileInfo, chunk, stream_metrics: &StreamMetrics| { + // Build one collector per Collector leaf for this chunk. + let mut per_leaf: Vec<(i32, Arc)> = + Vec::with_capacity(providers.len()); + for (idx, provider) in providers.iter().enumerate() { + let collector = FfmSegmentCollector::create( + provider.key(), + segment.segment_ord, + chunk.doc_min, + chunk.doc_max, + ) + .map_err(|e| format!("leaf {} collector: {}", idx, e))?; + per_leaf.push(( + provider.key(), + Arc::new(collector) as Arc, + )); + } + + let resolved = tree.resolve(&per_leaf).map_err(|e| { + format!("tree.resolve for segment {}: {}", segment.segment_ord, e) + })?; + let resolved = Arc::new(resolved); + + let pruner = Arc::new(PagePruner::new( + &schema_for_pruner, + Arc::clone(&segment.metadata), + )); + + let eval: Arc = Arc::new(TreeBitsetSource { + tree: resolved, + evaluator: Arc::new(BitmapTreeEvaluator), + leaves: Arc::new(CollectorLeafBitmaps { + ffm_collector_calls: stream_metrics.ffm_collector_calls.clone(), + }), + page_pruner: pruner, + cost_predicate, + cost_collector, + max_collector_parallelism, + pruning_predicates: Arc::clone(&pruning_predicates), + page_prune_metrics: Some(PagePruneMetrics::from_stream_metrics( + stream_metrics, + )), + collector_strategy, + }); + Ok(eval) + }, + ) + } + }; + + ctx.deregister_table(&table_name)?; + // Extract the scheme+authority portion of the table URL for + // DataFusion's FileScanConfig. The full URL includes the path + // (e.g. "file:///Users/.../parquet/"); ObjectStoreUrl wants only + // the scheme+authority ("file:///"). + let url_str = shard_view.table_path.as_str(); + let parsed = url::Url::parse(url_str) + .map_err(|e| DataFusionError::Execution(format!("parse table_path URL: {}", e)))?; + let store_url = ObjectStoreUrl::parse(format!("{}://{}", parsed.scheme(), parsed.authority()))?; + let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig { + schema: schema.clone(), + segments, + store: Arc::clone(&store), + store_url, + evaluator_factory: factory, + target_partitions: num_partitions.max(1), + force_strategy: query_config.force_strategy, + force_pushdown: query_config.force_pushdown, + pushdown_predicate, + query_config: Arc::clone(&query_config), + predicate_columns, + })); + ctx.register_table(&table_name, provider)?; + + let logical_plan = from_substrait_plan(&ctx.state(), &plan).await?; + let dataframe = ctx.execute_logical_plan(logical_plan).await?; + let physical_plan = dataframe.create_physical_plan().await?; + let df_stream = execute_stream(physical_plan, ctx.task_ctx()) + .map_err(|e| DataFusionError::Execution(format!("execute_stream: {}", e)))?; + + let cross_rt_stream = CrossRtStream::new_with_df_error_stream(df_stream, cpu_executor); + let schema = cross_rt_stream.schema(); + let wrapped = RecordBatchStreamAdapter::new(schema, cross_rt_stream); + Ok(Box::into_raw(Box::new(wrapped)) as i64) +} + +// ── Helpers ─────────────────────────────────────────────────────────── + +/// Collect all `Predicate(expr)` leaves in DFS order. Used by the +/// dispatcher to build a per-leaf `PruningPredicate` cache keyed by +/// `Arc::as_ptr` identity. +fn collect_predicate_exprs(tree: &BoolNode, out: &mut Vec>) { + match tree { + BoolNode::And(c) | BoolNode::Or(c) => { + c.iter().for_each(|ch| collect_predicate_exprs(ch, out)) + } + BoolNode::Not(inner) => collect_predicate_exprs(inner, out), + BoolNode::Collector { .. } => {} + BoolNode::Predicate(expr) => out.push(Arc::clone(expr)), + } +} + +fn collect_predicate_column_indices(extraction: Option<&ExtractionResult>) -> Vec { + let Some(e) = extraction else { return vec![] }; + let mut exprs = Vec::new(); + collect_predicate_exprs(&e.tree, &mut exprs); + let mut indices = BTreeSet::new(); + for expr in &exprs { + let _ = expr.apply(|node| { + if let Some(col) = node.as_any().downcast_ref::() { + indices.insert(col.index()); + } + Ok(TreeNodeRecursion::Continue) + }); + } + indices.into_iter().collect() +} +/// For a tree classified as `SingleCollector`, walk it to find the single +/// Collector leaf and return its query bytes. +fn single_collector_bytes(tree: &BoolNode) -> Option> { + match tree { + BoolNode::Collector { query_bytes } => Some(Arc::clone(query_bytes)), + BoolNode::And(children) => { + let mut all: Vec> = Vec::new(); + for child in children { + if let Some(b) = single_collector_bytes(child) { + all.push(b); + } + } + match all.len() { + 0 => None, + 1 => Some(all.remove(0)), + _ => { + let mut merged = Vec::new(); + for (i, b) in all.iter().enumerate() { + if i > 0 { + merged.push(b'\n'); + } + merged.extend_from_slice(b); + } + Some(Arc::from(merged.as_slice())) + } + } + } + _ => None, + } +} + +/// For a tree classified as `SingleCollector`, return the residual +/// (all non-Collector parts of the AND tree, re-assembled into a +/// single BoolNode). Recursively strips Collector leaves from nested +/// ANDs. Returns `None` if the tree is a bare Collector or the entire +/// tree is collectors-only (no residual predicates). +fn extract_single_collector_residual(tree: &BoolNode) -> Option { + fn strip_collectors(node: &BoolNode) -> Option { + match node { + BoolNode::Collector { .. } => None, + BoolNode::Predicate(_) => Some(node.clone()), + BoolNode::And(children) => { + let residuals: Vec = + children.iter().filter_map(strip_collectors).collect(); + match residuals.len() { + 0 => None, + 1 => Some(residuals.into_iter().next().unwrap()), + _ => Some(BoolNode::And(residuals)), + } + } + // OR/NOT with no collectors pass through unchanged (they're + // pure-predicate subtrees in a SingleCollector-classified tree). + other => Some(other.clone()), + } + } + strip_collectors(tree) +} + +// ── Placeholder provider used only for substrait consume pass ───────── + +struct PlaceholderProvider { + schema: SchemaRef, +} + +impl fmt::Debug for PlaceholderProvider { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PlaceholderProvider").finish() + } +} + +#[async_trait::async_trait] +impl TableProvider for PlaceholderProvider { + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + fn table_type(&self) -> TableType { + TableType::Base + } + async fn scan( + &self, + _state: &dyn Session, + _projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> Result, DataFusionError> { + Err(DataFusionError::Internal( + "PlaceholderProvider should not be scanned".into(), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::indexed_table::bool_tree::BoolNode; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::common::ScalarValue; + use datafusion::logical_expr::Operator; + use datafusion::physical_expr::expressions::{BinaryExpr, Column as PhysColumn, Literal}; + use datafusion::physical_expr::PhysicalExpr; + use std::sync::Arc; + + fn collector(tag: &[u8]) -> BoolNode { + BoolNode::Collector { + query_bytes: Arc::from(tag), + } + } + + fn pred() -> BoolNode { + let left: Arc = Arc::new(PhysColumn::new("price", 0)); + let right: Arc = Arc::new(Literal::new(ScalarValue::Int32(Some(0)))); + BoolNode::Predicate(Arc::new(BinaryExpr::new(left, Operator::Eq, right))) + } + + fn is_predicate(node: &BoolNode) -> bool { + matches!(node, BoolNode::Predicate(_)) + } + + // ── extract_single_collector_residual ───────────────────────────── + + #[test] + fn residual_bare_collector_is_none() { + assert!(extract_single_collector_residual(&collector(b"x")).is_none()); + } + + #[test] + fn residual_and_collector_plus_predicate() { + let tree = BoolNode::And(vec![collector(b"x"), pred()]); + let r = extract_single_collector_residual(&tree).unwrap(); + assert!(is_predicate(&r)); + } + + #[test] + fn residual_and_only_collectors_is_none() { + let tree = BoolNode::And(vec![collector(b"x"), collector(b"y")]); + assert!(extract_single_collector_residual(&tree).is_none()); + } + + #[test] + fn residual_nested_and_strips_collectors() { + // AND(C₁, AND(C₂, P)) → residual is P + let tree = BoolNode::And(vec![ + collector(b"x"), + BoolNode::And(vec![collector(b"y"), pred()]), + ]); + let r = extract_single_collector_residual(&tree).unwrap(); + assert!(is_predicate(&r)); + } + + #[test] + fn residual_deeply_nested_and() { + // AND(P₁, AND(C₁, AND(C₂, P₂))) → AND(P₁, P₂) + let p1 = pred(); + let p2 = pred(); + let tree = BoolNode::And(vec![ + p1, + BoolNode::And(vec![ + collector(b"a"), + BoolNode::And(vec![collector(b"b"), p2]), + ]), + ]); + let r = extract_single_collector_residual(&tree).unwrap(); + match r { + BoolNode::And(children) => { + assert_eq!(children.len(), 2); + assert!(children.iter().all(is_predicate)); + } + _ => panic!("expected AND, got {:?}", r), + } + } + + #[test] + fn residual_nested_and_with_or_predicate() { + // AND(C, AND(P, OR(P, P))) → AND(P, OR(P, P)) + let tree = BoolNode::And(vec![ + collector(b"x"), + BoolNode::And(vec![ + pred(), + BoolNode::Or(vec![pred(), pred()]), + ]), + ]); + let r = extract_single_collector_residual(&tree).unwrap(); + match r { + BoolNode::And(children) => { + assert_eq!(children.len(), 2); + assert!(is_predicate(&children[0])); + assert!(matches!(children[1], BoolNode::Or(_))); + } + _ => panic!("expected AND, got {:?}", r), + } + } + + #[test] + fn residual_nested_and_all_collectors_is_none() { + // AND(AND(C₁, C₂), AND(C₃, C₄)) → no residual + let tree = BoolNode::And(vec![ + BoolNode::And(vec![collector(b"a"), collector(b"b")]), + BoolNode::And(vec![collector(b"c"), collector(b"d")]), + ]); + assert!(extract_single_collector_residual(&tree).is_none()); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs new file mode 100644 index 0000000000000..63854c4088105 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs @@ -0,0 +1,636 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Boolean query tree representation. +//! +//! **No wire format.** The tree is built from the Substrait plan's filter +//! expression (see [`crate::indexed_table::substrait_to_tree`]), never +//! serialized, never crosses the FFM boundary. +//! +//! Two flavors: +//! +//! - [`BoolNode`] — unresolved. Produced by `expr_to_bool_tree`. +//! `Collector` leaves carry the serialized query bytes +//! (as extracted from the `index_filter(bytes)` UDF call); +//! `Predicate` leaves carry an arbitrary DataFusion +//! [`PhysicalExpr`](datafusion::physical_expr::PhysicalExpr) — +//! comparisons, IN, IS NULL, arithmetic, whatever produces a boolean. +//! - [`ResolvedNode`] — resolved. Produced by +//! `BoolNode::resolve(collectors)` — `Collector` leaves get turned +//! into `(provider_key, Arc)` pairs by the +//! caller; Predicate leaves pass through unchanged. This is what the +//! evaluator walks. + +use std::sync::Arc; + +use datafusion::physical_expr::PhysicalExpr; + +use super::index::RowGroupDocsCollector; + +/// A node in the boolean query tree (unresolved). +#[derive(Debug, Clone)] +pub enum BoolNode { + And(Vec), + Or(Vec), + Not(Box), + /// index-backend query payload. The caller (typically the indexed_executor) + /// upcalls into Java with these bytes at query-resolve time to get a + /// `provider_key`, then creates per-segment collectors. Bytes are opaque + /// to Rust; only the Java factory knows how to interpret them. + Collector { + query_bytes: Arc<[u8]>, + }, + /// Arbitrary boolean-valued DataFusion expression. At refinement + /// time, `expr.evaluate(batch)` produces the per-row mask; at page- + /// prune time, the expression is handed to DataFusion's + /// `PruningPredicate` directly. + Predicate(Arc), +} + +/// Resolved tree. `Collector` leaves carry the provider-key returned by the +/// Java factory plus the concrete collector reference; `Predicate` leaves +/// carry the same `Arc` as [`BoolNode::Predicate`]. +#[derive(Debug)] +pub enum ResolvedNode { + And(Vec), + Or(Vec), + Not(Box), + Collector { + provider_key: i32, + collector: Arc, + }, + Predicate(Arc), +} + +impl BoolNode { + /// Count `Collector` leaf occurrences in the tree (DFS). + pub fn collector_leaf_count(&self) -> usize { + match self { + BoolNode::And(children) | BoolNode::Or(children) => { + children.iter().map(|c| c.collector_leaf_count()).sum() + } + BoolNode::Not(child) => child.collector_leaf_count(), + BoolNode::Collector { .. } => 1, + BoolNode::Predicate(_) => 0, + } + } + + /// Return the serialized query bytes for each `Collector` leaf in DFS order. + /// Caller uses this to issue one `createProvider(bytes)` upcall per leaf. + /// + /// # Ordering invariant + /// + /// This method MUST walk children in the same order as + /// [`Self::resolve`] consumes them. Both visit And/Or children left-to- + /// right, recurse into Not, then yield leaves. The positional pairing in + /// `resolve` (via the `*next` index) relies on this invariant; if you + /// change one traversal you MUST change the other in lockstep, or + /// collector-to-leaf matching will silently become wrong. + pub fn collector_leaves(&self) -> Vec> { + let mut out = Vec::new(); + self.collect_leaves(&mut out); + out + } + + fn collect_leaves(&self, out: &mut Vec>) { + match self { + BoolNode::And(children) | BoolNode::Or(children) => { + for c in children { + c.collect_leaves(out); + } + } + BoolNode::Not(child) => child.collect_leaves(out), + BoolNode::Collector { query_bytes } => { + out.push(Arc::clone(query_bytes)); + } + BoolNode::Predicate(_) => {} + } + } + + /// De Morgan's NOT push-down normalization. + /// After this, `Not` only appears directly above `Collector` or `Predicate` leaves. + pub fn push_not_down(self) -> BoolNode { + match self { + BoolNode::And(children) => { + BoolNode::And(children.into_iter().map(|c| c.push_not_down()).collect()) + } + BoolNode::Or(children) => { + BoolNode::Or(children.into_iter().map(|c| c.push_not_down()).collect()) + } + BoolNode::Not(child) => push_not_into(*child), + leaf => leaf, + } + } + + /// Collapse nested same-kind connectives: + /// `And(And(x, y), z)` → `And(x, y, z)`, similarly for `Or`. + /// + /// Substrait decodes N-ary AND/OR as left-deep binary trees. Flattening + /// cuts evaluator recursion depth and lets Path C allocate one Phase 1 + /// bitmap per conceptual child instead of one per binary split. + /// Idempotent and semantic-preserving. + pub fn flatten(self) -> BoolNode { + match self { + BoolNode::And(children) => { + let mut out = Vec::with_capacity(children.len()); + for c in children { + match c.flatten() { + BoolNode::And(inner) => out.extend(inner), + other => out.push(other), + } + } + BoolNode::And(out) + } + BoolNode::Or(children) => { + let mut out = Vec::with_capacity(children.len()); + for c in children { + match c.flatten() { + BoolNode::Or(inner) => out.extend(inner), + other => out.push(other), + } + } + BoolNode::Or(out) + } + BoolNode::Not(child) => BoolNode::Not(Box::new(child.flatten())), + leaf => leaf, + } + } + + /// Resolve the tree: walk in DFS order, consuming pre-built `(provider_key, + /// collector)` pairs (one per `Collector` leaf, same DFS order as + /// [`Self::collector_leaves`]) and expanding `Predicate` IDs into + /// `(column, op, value)`. + /// + /// Caller is responsible for creating the collectors — typically by + /// upcalling Java `createProvider(query_bytes)` per leaf to get a + /// `provider_key`, then `createCollector(provider_key, seg, min, max)` + /// per chunk. + /// + /// # Ordering invariant + /// + /// The `collectors` slice is consumed positionally; its order must match + /// the DFS order produced by [`Self::collector_leaves`]. See that method + /// for the traversal contract. A mismatch causes collector-to-leaf + /// misalignment with no runtime error — wrong data, silent. + pub fn resolve( + &self, + collectors: &[(i32, Arc)], + ) -> Result { + let mut next = 0usize; + self.resolve_rec(collectors, &mut next) + } + + fn resolve_rec( + &self, + collectors: &[(i32, Arc)], + next: &mut usize, + ) -> Result { + match self { + BoolNode::And(children) => { + let resolved: Result, _> = children + .iter() + .map(|c| c.resolve_rec(collectors, next)) + .collect(); + Ok(ResolvedNode::And(resolved?)) + } + BoolNode::Or(children) => { + let resolved: Result, _> = children + .iter() + .map(|c| c.resolve_rec(collectors, next)) + .collect(); + Ok(ResolvedNode::Or(resolved?)) + } + BoolNode::Not(child) => { + let resolved_child = child.resolve_rec(collectors, next)?; + // Fast-path: NOT over a `Predicate(col op literal)` folds + // into `Predicate(col flipped_op literal)`. Saves one + // kleene-`not()` kernel per batch in the refinement stage + // and one universe subtraction per RG in the candidate + // stage. Falls back to wrapping `Not` when the child + // isn't a recognizable comparison. + match resolved_child { + ResolvedNode::Predicate(ref expr) => match try_negate_cmp_expr(expr) { + Some(flipped) => Ok(ResolvedNode::Predicate(flipped)), + None => Ok(ResolvedNode::Not(Box::new(ResolvedNode::Predicate( + Arc::clone(expr), + )))), + }, + other => Ok(ResolvedNode::Not(Box::new(other))), + } + } + BoolNode::Collector { .. } => { + let (provider_key, collector) = collectors + .get(*next) + .ok_or_else(|| format!("collector index {} out of range", *next))?; + *next += 1; + Ok(ResolvedNode::Collector { + provider_key: *provider_key, + collector: Arc::clone(collector), + }) + } + BoolNode::Predicate(expr) => Ok(ResolvedNode::Predicate(Arc::clone(expr))), + } + } +} + +/// If `expr` is a `BinaryExpr(col, cmp, literal)` with an invertible +/// comparison operator, return the same expression with the operator +/// negated. Otherwise `None`. +/// +/// Used by `BoolNode::resolve_rec` to fold `Not(Predicate(cmp))` into a +/// single flipped `Predicate` so the refinement stage doesn't have to +/// call `not_kleene()` per batch. +fn try_negate_cmp_expr( + expr: &Arc, +) -> Option> { + use datafusion::logical_expr::Operator; + use datafusion::physical_expr::expressions::BinaryExpr; + + let bin = expr.as_any().downcast_ref::()?; + let flipped = match *bin.op() { + Operator::Eq => Operator::NotEq, + Operator::NotEq => Operator::Eq, + Operator::Lt => Operator::GtEq, + Operator::LtEq => Operator::Gt, + Operator::Gt => Operator::LtEq, + Operator::GtEq => Operator::Lt, + _ => return None, + }; + Some(Arc::new(BinaryExpr::new( + Arc::clone(bin.left()), + flipped, + Arc::clone(bin.right()), + ))) +} + +fn push_not_into(child: BoolNode) -> BoolNode { + match child { + // De Morgan's: NOT(AND(a, b, ...)) → OR(NOT(a), NOT(b), ...) + BoolNode::And(children) => { + BoolNode::Or(children.into_iter().map(push_not_into).collect()).push_not_down() + } + // De Morgan's: NOT(OR(a, b, ...)) → AND(NOT(a), NOT(b), ...) + BoolNode::Or(children) => { + BoolNode::And(children.into_iter().map(push_not_into).collect()).push_not_down() + } + // Double negation + BoolNode::Not(inner) => inner.push_not_down(), + // NOT(Collector) / NOT(Predicate) — stay wrapped; evaluator handles the negation + leaf => BoolNode::Not(Box::new(leaf)), + } +} + +/// Convert a Collector-free `BoolNode` (the residual of a +/// `SingleCollector`-classified tree, or any subtree guaranteed to +/// have no `Collector` leaves) into a single +/// `Arc` suitable for parquet's `with_predicate` +/// pushdown or DataFusion's `Expr::evaluate(batch)`. +/// +/// Contrast with `page_pruner::bool_tree_to_pruning_expr`: +/// - That helper replaces `Collector` leaves with `Literal(true)` so +/// the result can feed DataFusion's `PruningPredicate` rewriter +/// (which evaluates only against per-page stats, not cell values). +/// - This helper assumes no Collectors are present (appropriate for +/// a SingleCollector residual). Returns `None` if a Collector is +/// encountered (shouldn't happen for a well-formed residual). +/// +/// NOT handling: emits `NotExpr`. Callers that need De Morgan +/// normalization should `push_not_down` first. +pub fn residual_bool_to_physical_expr( + node: &BoolNode, +) -> Option> { + use datafusion::logical_expr::Operator; + use datafusion::physical_expr::expressions::{BinaryExpr, NotExpr}; + + match node { + BoolNode::Predicate(expr) => Some(Arc::clone(expr)), + BoolNode::And(children) => { + if children.is_empty() { + return None; + } + let mut iter = children.iter(); + let mut acc = residual_bool_to_physical_expr(iter.next().unwrap())?; + for c in iter { + let child = residual_bool_to_physical_expr(c)?; + acc = Arc::new(BinaryExpr::new(acc, Operator::And, child)); + } + Some(acc) + } + BoolNode::Or(children) => { + if children.is_empty() { + return None; + } + let mut iter = children.iter(); + let mut acc = residual_bool_to_physical_expr(iter.next().unwrap())?; + for c in iter { + let child = residual_bool_to_physical_expr(c)?; + acc = Arc::new(BinaryExpr::new(acc, Operator::Or, child)); + } + Some(acc) + } + BoolNode::Not(child) => { + let inner = residual_bool_to_physical_expr(child)?; + Some(Arc::new(NotExpr::new(inner))) + } + BoolNode::Collector { .. } => None, + } +} + +// ════════════════════════════════════════════════════════════════════════════ +// Tests +// ════════════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod tests { + use super::*; + use crate::indexed_table::index::RowGroupDocsCollector; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::common::ScalarValue; + use datafusion::logical_expr::Operator; + use datafusion::physical_expr::expressions::{BinaryExpr, Column as PhysColumn, Literal}; + use datafusion::physical_expr::PhysicalExpr; + + #[derive(Debug)] + struct StubCollector(u8); + impl RowGroupDocsCollector for StubCollector { + fn collect_packed_u64_bitset(&self, _: i32, _: i32) -> Result, String> { + Ok(vec![self.0 as u64]) + } + } + + fn collector(bytes: &[u8]) -> BoolNode { + BoolNode::Collector { + query_bytes: Arc::from(bytes), + } + } + + fn predicate(col: &str, op: Operator, v: i32) -> BoolNode { + let schema = Schema::new(vec![Field::new(col, DataType::Int32, false)]); + let col_idx = schema.index_of(col).unwrap(); + let left: Arc = Arc::new(PhysColumn::new(col, col_idx)); + let right: Arc = Arc::new(Literal::new(ScalarValue::Int32(Some(v)))); + BoolNode::Predicate(Arc::new(BinaryExpr::new(left, op, right))) + } + + // ── collector_leaf_count / collector_leaves ─────────────────────── + + #[test] + fn leaf_count_counts_only_collectors() { + let tree = BoolNode::And(vec![ + collector(b"a"), + BoolNode::Or(vec![collector(b"b"), predicate("x", Operator::Eq, 1)]), + predicate("y", Operator::Eq, 2), + ]); + assert_eq!(tree.collector_leaf_count(), 2); + } + + #[test] + fn leaves_dfs_order() { + let tree = BoolNode::And(vec![ + collector(b"x"), + BoolNode::Or(vec![collector(b"y"), collector(b"z")]), + ]); + let leaves = tree.collector_leaves(); + assert_eq!(leaves.len(), 3); + assert_eq!(&*leaves[0], b"x"); + assert_eq!(&*leaves[1], b"y"); + assert_eq!(&*leaves[2], b"z"); + } + + // ── push_not_down (De Morgan) ───────────────────────────────────── + + #[test] + fn not_collector_stays_wrapped() { + let tree = BoolNode::Not(Box::new(collector(b"x"))); + let n = tree.push_not_down(); + assert!(matches!(n, BoolNode::Not(b) if matches!(*b, BoolNode::Collector { .. }))); + } + + #[test] + fn de_morgan_not_and_to_or() { + let tree = BoolNode::Not(Box::new(BoolNode::And(vec![ + collector(b"a"), + collector(b"b"), + ]))); + match tree.push_not_down() { + BoolNode::Or(children) => { + assert_eq!(children.len(), 2); + for c in &children { + assert!(matches!(c, BoolNode::Not(_))); + } + } + other => panic!("expected Or, got {:?}", other), + } + } + + #[test] + fn de_morgan_not_or_to_and() { + let tree = BoolNode::Not(Box::new(BoolNode::Or(vec![ + predicate("a", Operator::Eq, 1), + predicate("b", Operator::Eq, 2), + ]))); + match tree.push_not_down() { + BoolNode::And(children) => { + assert_eq!(children.len(), 2); + for c in &children { + assert!(matches!(c, BoolNode::Not(_))); + } + } + other => panic!("expected And, got {:?}", other), + } + } + + #[test] + fn double_negation_cancels() { + let tree = BoolNode::Not(Box::new(BoolNode::Not(Box::new(collector(b"x"))))); + let n = tree.push_not_down(); + assert!(matches!(n, BoolNode::Collector { .. })); + } + + #[test] + fn nested_not_recurses_through_and_or() { + let tree = BoolNode::Not(Box::new(BoolNode::And(vec![ + BoolNode::Or(vec![collector(b"a"), collector(b"b")]), + collector(b"c"), + ]))); + match tree.push_not_down() { + BoolNode::Or(outer) => { + assert_eq!(outer.len(), 2); + assert!(matches!(outer[0], BoolNode::And(_))); + assert!(matches!(outer[1], BoolNode::Not(_))); + } + other => panic!("expected Or, got {:?}", other), + } + } + + // ── flatten ─────────────────────────────────────────────────────── + + #[test] + fn flatten_collapses_nested_and() { + let tree = BoolNode::And(vec![ + BoolNode::And(vec![collector(b"a"), collector(b"b")]), + collector(b"c"), + ]); + match tree.flatten() { + BoolNode::And(children) => { + assert_eq!(children.len(), 3); + for c in &children { + assert!(matches!(c, BoolNode::Collector { .. })); + } + } + other => panic!("expected flat And with 3 children, got {:?}", other), + } + } + + #[test] + fn flatten_collapses_nested_or() { + let tree = BoolNode::Or(vec![ + collector(b"a"), + BoolNode::Or(vec![ + collector(b"b"), + BoolNode::Or(vec![collector(b"c"), collector(b"d")]), + ]), + ]); + match tree.flatten() { + BoolNode::Or(children) => assert_eq!(children.len(), 4), + other => panic!("expected flat Or with 4 children, got {:?}", other), + } + } + + #[test] + fn flatten_preserves_mixed_connectives() { + let tree = BoolNode::And(vec![ + collector(b"a"), + BoolNode::Or(vec![collector(b"b"), collector(b"c")]), + BoolNode::And(vec![collector(b"d"), collector(b"e")]), + ]); + match tree.flatten() { + BoolNode::And(children) => { + assert_eq!(children.len(), 4); + assert!(matches!(children[1], BoolNode::Or(_))); + } + other => panic!("expected And with 4 children, got {:?}", other), + } + } + + #[test] + fn flatten_descends_into_not() { + let tree = BoolNode::Not(Box::new(BoolNode::And(vec![ + BoolNode::And(vec![collector(b"a"), collector(b"b")]), + collector(b"c"), + ]))); + match tree.flatten() { + BoolNode::Not(inner) => match *inner { + BoolNode::And(children) => assert_eq!(children.len(), 3), + other => panic!("expected And under Not, got {:?}", other), + }, + other => panic!("expected Not, got {:?}", other), + } + } + + // ── resolve ──────────────────────────────────────────────────────── + + #[test] + fn resolve_replaces_collector_bytes_with_refs() { + let tree = BoolNode::And(vec![collector(b"a"), collector(b"b")]); + let a: Arc = Arc::new(StubCollector(1)); + let b: Arc = Arc::new(StubCollector(2)); + let resolved = tree.resolve(&[(10, a), (20, b)]).unwrap(); + match resolved { + ResolvedNode::And(children) => { + assert_eq!(children.len(), 2); + match (&children[0], &children[1]) { + ( + ResolvedNode::Collector { + provider_key: p1, .. + }, + ResolvedNode::Collector { + provider_key: p2, .. + }, + ) => { + assert_eq!(*p1, 10); + assert_eq!(*p2, 20); + } + _ => panic!("expected Collector pair"), + } + } + other => panic!("expected And, got {:?}", other), + } + } + + #[test] + fn resolve_passes_predicate_expr_through() { + let tree = predicate("status", Operator::Eq, 1); + let resolved = tree.resolve(&[]).unwrap(); + assert!(matches!(resolved, ResolvedNode::Predicate(_))); + } + + #[test] + fn resolve_out_of_range_errors() { + let tree = collector(b"x"); + let err = tree.resolve(&[]).unwrap_err(); + assert!(err.contains("out of range"), "got: {}", err); + } + + #[test] + fn resolve_not_collector_still_wraps() { + let tree = BoolNode::Not(Box::new(collector(b"x"))); + let c: Arc = Arc::new(StubCollector(0)); + let resolved = tree.resolve(&[(1, c)]).unwrap(); + match resolved { + ResolvedNode::Not(inner) => { + assert!(matches!(*inner, ResolvedNode::Collector { .. })); + } + other => panic!("expected Not(Collector), got {:?}", other), + } + } + + // ── Not(Predicate) op-flip during resolve ───────────────────────── + + /// Extract `(op)` from a `ResolvedNode::Predicate` whose child is a + /// `BinaryExpr(col, op, literal)`. Panics otherwise. + fn predicate_op(node: &ResolvedNode) -> Operator { + use datafusion::physical_expr::expressions::BinaryExpr; + match node { + ResolvedNode::Predicate(expr) => { + let bin = expr + .as_any() + .downcast_ref::() + .expect("expected BinaryExpr leaf"); + *bin.op() + } + other => panic!("expected Predicate, got {:?}", other), + } + } + + #[test] + fn resolve_not_predicate_flips_op() { + // Not(price > 10) should resolve to price <= 10, not + // Not(Predicate(price > 10)). + let tree = BoolNode::Not(Box::new(predicate("price", Operator::Gt, 10))); + let resolved = tree.resolve(&[]).unwrap(); + assert_eq!(predicate_op(&resolved), Operator::LtEq); + } + + #[test] + fn resolve_not_predicate_flip_table() { + let cases = [ + (Operator::Lt, Operator::GtEq), + (Operator::LtEq, Operator::Gt), + (Operator::Gt, Operator::LtEq), + (Operator::GtEq, Operator::Lt), + (Operator::Eq, Operator::NotEq), + (Operator::NotEq, Operator::Eq), + ]; + for (orig, expected) in cases { + let tree = BoolNode::Not(Box::new(predicate("x", orig, 0))); + let resolved = tree.resolve(&[]).unwrap(); + assert_eq!(predicate_op(&resolved), expected, "flipping {:?}", orig); + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/bitmap_tree.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/bitmap_tree.rs new file mode 100644 index 0000000000000..ce78f0535738c --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/bitmap_tree.rs @@ -0,0 +1,1669 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! `BitmapTreeEvaluator` — the default [`TreeEvaluator`] implementation. +//! +//! # Two-stage evaluation +//! +//! The tree is evaluated in two stages per row group: +//! +//! 1. **Candidate stage** (`prefetch`) — builds a *superset* candidate set of +//! doc IDs for the RG. Works entirely in the RoaringBitmap domain: +//! compact, cheap intersections, O(set-bits) operations. The stage walks +//! the tree once, producing: +//! - a top-level `RoaringBitmap` of candidate doc IDs (superset of the +//! exact match set — this is what decides which parquet rows to read); +//! - a side-table of per-leaf bitmaps, keyed by Collector leaf identity. +//! +//! Collector leaves ask an external [`LeafBitmapSource`] for their bitmap +//! (today that means an FFM upcall to the Java-side index). Predicate +//! leaves use parquet page statistics via the caller's [`PagePruner`]. +//! The reason this is a superset, not the exact answer: predicate bitmaps +//! come from page-level stats and are inherently coarse (pages are +//! supersets of the rows that actually match the predicate). +//! +//! 2. **Refinement stage** (`on_batch`) — runs per record batch, after +//! parquet delivered the decoded rows. Walks the same tree using Arrow +//! `BooleanArray` kernels (`and_kleene`, `or_kleene`, `not`, cmp ops) to +//! produce the *exact* per-row answer. Collector leaves look up their +//! Phase 1 bitmap from the side-table and slice it to batch coordinates; +//! Predicate leaves re-evaluate the comparison on actual column data. +//! +//! Why two stages and not one: Phase 1's bitmap-domain work decides *which +//! parquet rows to read at all* — for a selective query over a large RG, +//! we read only the few pages that could possibly match. Phase 2 then +//! filters those rows down to the exact answer. One-stage evaluation would +//! either read the whole RG (wasteful) or trust the coarse superset +//! (wrong, since predicate stats are supersets). +//! +//! # Child ordering +//! +//! The candidate stage sorts AND/OR children by [`subtree_cost`] before +//! walking (cheap-first), which lets a narrow Predicate leaf — or a +//! Predicate-dominated nested subtree — short-circuit a whole AND group +//! before any expensive Collector leaf work. The refinement stage walks +//! children in their *original* tree order, which is fine because Arrow +//! kernels don't short-circuit internally and leaf identity is by +//! `Arc::as_ptr`, not DFS position. See [`subtree_cost`] and the +//! `collect_collector_leaves` doc for the identity mechanism that lets +//! these two orderings coexist safely. +//! +//! Plus [`CollectorLeafBitmaps`] — the default [`LeafBitmapSource`] impl that +//! expands index-backed `RowGroupDocsCollector` output into RoaringBitmaps. +//! A different `LeafBitmapSource` could back Collector leaves by parquet +//! stats, external bitmap stores, or anything else implementing the trait. + +use std::collections::HashMap; +use std::sync::Arc; + +use datafusion::arrow::array::{Array, AsArray, BooleanArray}; +use datafusion::arrow::compute::kernels::cmp::{eq, gt, gt_eq, lt, lt_eq, neq}; +use datafusion::arrow::compute::{and_kleene as and, not, or_kleene as or}; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::logical_expr::{ColumnarValue, Operator}; +use datafusion::physical_expr::expressions::{BinaryExpr, Column as PhysColumn, Literal}; +use roaring::RoaringBitmap; + +use super::{LeafBitmapSource, RgEvalContext, TreeEvaluator, TreePrefetch}; +use crate::indexed_table::bool_tree::ResolvedNode; +use crate::indexed_table::page_pruner::{PagePruneMetrics, PagePruner}; +use crate::indexed_table::row_selection::{packed_bits_to_boolean_array, PositionMap}; +use datafusion::physical_optimizer::pruning::PruningPredicate; + +/// In-process Rust `TreeEvaluator`. Stateless — all per-RG state lives in the +/// `TreePrefetch` value threaded through `RowGroupBitsetSource`. +pub struct BitmapTreeEvaluator; + +impl TreeEvaluator for BitmapTreeEvaluator { + fn prefetch( + &self, + tree: &ResolvedNode, + ctx: &RgEvalContext, + leaves: &dyn LeafBitmapSource, + page_pruner: &PagePruner, + pruning_predicates: &HashMap>, + page_prune_metrics: Option<&PagePruneMetrics>, + ) -> Result { + let mut per_leaf = Vec::new(); + let mut dfs_counter = 0usize; + // Root call passes `under_all_and_path = true` — root's (empty) + // ancestor chain is trivially all-AND, so if the root short-circuits + // to empty, the candidate set is empty and refinement won't run. + let candidates = prefetch_node( + tree, + ctx, + leaves, + page_pruner, + pruning_predicates, + page_prune_metrics, + &mut dfs_counter, + &mut per_leaf, + /* under_all_and_path */ true, + )?; + Ok(TreePrefetch { + candidates, + per_leaf, + min_doc: ctx.min_doc, + }) + } + + fn on_batch( + &self, + tree: &ResolvedNode, + state: &TreePrefetch, + batch: &RecordBatch, + rg_first_row: i64, + position_map: &PositionMap, + batch_offset: usize, + batch_len: usize, + ) -> Result { + on_batch_node( + tree, + state, + batch, + rg_first_row, + position_map, + batch_offset, + batch_len, + ) + } +} + +// Candidate stage: Filters the parquet data with candidate superset [ page pruning + lucene bitset ] +// [ either via filter exec or filter pushdown ] tree walker +// +// Walks the resolved tree to produce the top-level superset RoaringBitmap +// plus the per-leaf bitmap side-table. +// +// The `dfs` counter tracks the caller's position in a depth-first traversal. +// It's used only to assign a stable `leaf_dfs_index` to each leaf so a +// `LeafBitmapSource` implementation can identify which leaf it's being asked +// about. We advance `dfs` on every leaf whether we actually evaluate it or +// not (see the short-circuit branches in AND/OR) so downstream walkers that +// reproduce the DFS order (`collect_collector_leaves`, `skip_dfs`) stay in +// sync with this one. +// +// Note: the stored per-leaf bitmap entries use `Arc::as_ptr(collector)` as +// the key, not `leaf_dfs_index`. DFS position changes between +// `prefetch_node` (which sorts children by cost) and `on_batch_node` (which +// walks in original order), but `Arc` identity is stable across both. See +// the refinement-stage walker for the lookup. +// +// The `under_all_and_path` flag tracks whether every ancestor (up to root) +// is an AND node. When true, an empty candidate result here propagates all +// the way up — `TreeBitsetSource::prefetch_rg` returns `None`, the RG is +// skipped entirely, and the refinement stage never runs. In that case we +// can drop Collector bitmap materialisation in short-circuited branches +// (no one will look them up). When false, some ancestor is OR or NOT, +// which can recover from an empty subtree — refinement may still run and +// will need the bitmaps in `out`, so we materialise them defensively. +// +// Propagation rule: +// - Root call: `under_all_and_path = true` (no ancestors). +// - Recurse into an AND child: pass the flag unchanged. +// - Recurse into an OR or NOT child: pass `false`. +// The universe-saturation short-circuit in OR is NOT affected — saturation +// produces a non-empty candidate set, so the RG is always read and +// refinement always runs. Bitmaps must be materialised regardless. + +fn prefetch_node( + node: &ResolvedNode, + ctx: &RgEvalContext, + leaves: &dyn LeafBitmapSource, + page_pruner: &PagePruner, + pruning_predicates: &HashMap>, + page_prune_metrics: Option<&PagePruneMetrics>, + dfs: &mut usize, + out: &mut Vec<(usize, RoaringBitmap)>, + under_all_and_path: bool, +) -> Result { + match node { + ResolvedNode::And(children) => { + let mut indices: Vec = (0..children.len()).collect(); + indices.sort_by_key(|&i| subtree_cost(&children[i], ctx, page_pruner, pruning_predicates)); + + let mut result_bitmap: Option = None; + let mut ranges: Option> = ctx.collector_call_ranges.clone(); + for &i in &indices { + let child_ctx = if ranges != ctx.collector_call_ranges { + RgEvalContext { + collector_call_ranges: ranges.clone(), + ..ctx.clone() + } + } else { + ctx.clone() + }; + let child_bitmap = prefetch_node( + &children[i], + &child_ctx, + leaves, + page_pruner, + pruning_predicates, + page_prune_metrics, + dfs, + out, + under_all_and_path, // AND preserves the all-AND path + )?; + result_bitmap = Some(match result_bitmap { + None => child_bitmap, + Some(mut a) => { + a &= &child_bitmap; + a + } + }); + + // Tighten collector call ranges from the accumulator bitmap, + // intersected with inherited ranges so nested ANDs never + // widen beyond what the parent already narrowed to. + if let Some(ref bm) = result_bitmap { + if !bm.is_empty() { + let new = ranges_from_bitmap(bm, ctx); + ranges = Some(match ranges { + Some(inherited) => intersect_range_lists(&inherited, &new), + None => new, + }); + } + } + + // Short circuit case + // 1. Skip if subtree only consists of AND [ since all bits are not set here, no need to evaluate ] + // 2. Collect if subtree is mixed with OR/NOT, which can produce set bits and recover + if result_bitmap.as_ref().unwrap().is_empty() { + // Remaining children still need to advance `dfs` so leaf + // IDs remain stable. + for &j in indices.iter().skip_while(|&&x| x != i).skip(1) { + if under_all_and_path { + // Empty propagates to root → RG skipped → bitmaps + // unused. Just advance the counter. + skip_dfs(&children[j], dfs); + } else { + // OR/NOT ancestor can recover + collect_collector_leaves(&children[j], ctx, leaves, dfs, out)?; + } + } + break; + } + } + Ok(result_bitmap.unwrap_or_default()) + } + ResolvedNode::Or(children) => { + let mut indices: Vec = (0..children.len()).collect(); + + // sort the children by cost to prune children better + indices.sort_by_key(|&i| subtree_cost(&children[i], ctx, page_pruner, pruning_predicates)); + let total_docs = (ctx.max_doc - ctx.min_doc) as u64; + + let mut result_bitmap = RoaringBitmap::new(); + for (arr_index, &val) in indices.iter().enumerate() { + let filtered_bitmap = prefetch_node( + &children[val], + ctx, + leaves, + page_pruner, + pruning_predicates, + page_prune_metrics, + dfs, + out, + // OR breaks all-AND propagation for its subtree. + false, + )?; + result_bitmap |= &filtered_bitmap; + + // Short circuit case + if result_bitmap.len() >= total_docs { + // If all values match, then result bitmap length will be + // same as total docs. In that case, we don't have to evaluate predicates + // since we know all bits are matching. + // We simply call collectors so that the bitsets are appended to 'out' + for &j in indices.iter().skip(arr_index + 1) { + collect_collector_leaves(&children[j], ctx, leaves, dfs, out)?; + } + break; + } + } + Ok(result_bitmap) + } + // Mainly needed for collectors, predicate expressions are inversed where possible + // and wouldn't usually hit this + ResolvedNode::Not(child) => { + // NOT breaks all-AND propagation — inverting empty gives universe, + // which is non-empty, so the RG will be read and refinement will + // run. Materialise bitmaps below. + let child_bm = prefetch_node( + child, + ctx, + leaves, + page_pruner, + pruning_predicates, + page_prune_metrics, + dfs, + out, + /* under_all_and_path */ false, + )?; + // Candidate-stage is a superset. Inverting a superset does + // NOT yield a superset of the true NOT — it yields a subset + // (wrong for candidate stage). + // Two cases : + // 1. Predicate : If the child's bitmap was computed + // from anything non-exact (Predicate leaves use coarse page + // stats), fall back to the full universe and let refinement pick + // the exact set. + // 2. Collector : If the child contained only Collector leaves + // (exact bitmaps), inversion is safe. + if subtree_has_predicate(child) { + let mut universe = RoaringBitmap::new(); + let span = (ctx.max_doc - ctx.min_doc) as u32; + universe.insert_range(0..span); + Ok(universe) + } else { + let mut universe = RoaringBitmap::new(); + let span = (ctx.max_doc - ctx.min_doc) as u32; + universe.insert_range(0..span); + universe -= &child_bm; + Ok(universe) + } + } + ResolvedNode::Collector { collector, .. } => { + let leaf_idx = *dfs; + *dfs += 1; + let key = Arc::as_ptr(collector) as *const () as usize; + let bm = leaves.leaf_bitmap(node, leaf_idx, ctx)?; + out.push((key, bm.clone())); + Ok(bm) + } + ResolvedNode::Predicate(expr) => { + let leaf_idx = *dfs; + *dfs += 1; + let _ = leaf_idx; // predicate leaves don't need per-leaf storage + Ok(predicate_page_bitmap( + expr, + ctx, + page_pruner, + pruning_predicates, + page_prune_metrics, + )) + } + } +} + +/// Walk a subtree without combining into the parent accumulator, but still +/// populate the per-leaf bitmap side-table that the refinement stage will +/// read from later. +/// +/// Called when the parent's candidate-stage accumulator has short-circuited +/// (AND reached empty, OR reached the universe) and so this subtree's +/// contribution is no longer needed for the candidate superset. We can't +/// just skip the subtree entirely though — the refinement stage walks the +/// whole tree and will look up every Collector leaf's bitmap in the +/// side-table. Missing entries there would panic at refinement time. So we +/// still materialise the bitmaps (but skip the expensive AND/OR combine and +/// skip the page-pruner work for Predicate leaves, since those never enter +/// the side-table). +/// +/// Also advances the `dfs` counter in lockstep with the main walker so +/// downstream leaf_dfs_index assignments stay consistent. +fn collect_collector_leaves( + node: &ResolvedNode, + ctx: &RgEvalContext, + leaves: &dyn LeafBitmapSource, + dfs: &mut usize, + out: &mut Vec<(usize, RoaringBitmap)>, +) -> Result<(), String> { + match node { + ResolvedNode::And(children) | ResolvedNode::Or(children) => { + for child in children { + collect_collector_leaves(child, ctx, leaves, dfs, out)?; + } + } + ResolvedNode::Not(child) => collect_collector_leaves(child, ctx, leaves, dfs, out)?, + ResolvedNode::Collector { collector, .. } => { + let leaf_idx = *dfs; + *dfs += 1; + let key = Arc::as_ptr(collector) as *const () as usize; + let bm = leaves.leaf_bitmap(node, leaf_idx, ctx)?; + out.push((key, bm)); + } + ResolvedNode::Predicate(_) => { + *dfs += 1; + } + } + Ok(()) +} + +/// Advance the `dfs` counter over a subtree without doing any bitmap work. +/// Used at an AND short-circuit point when we know the whole candidate +/// result will be empty and the RG will be skipped — there's no refinement +/// stage to prepare bitmaps for, so we only need to keep leaf-ID assignment +/// stable. See the `under_all_and_path` handling in `prefetch_node`. +fn skip_dfs(node: &ResolvedNode, dfs: &mut usize) { + match node { + ResolvedNode::And(children) | ResolvedNode::Or(children) => { + for c in children { + skip_dfs(c, dfs); + } + } + ResolvedNode::Not(child) => skip_dfs(child, dfs), + ResolvedNode::Collector { .. } | ResolvedNode::Predicate(_) => *dfs += 1, + } +} + +fn predicate_page_bitmap( + expr: &Arc, + ctx: &RgEvalContext, + page_pruner: &PagePruner, + pruning_predicates: &HashMap>, + page_prune_metrics: Option<&PagePruneMetrics>, +) -> RoaringBitmap { + // Identity key: same Arc used at build time is the same Arc we see here. + let key = Arc::as_ptr(expr) as *const () as usize; + let pruning_predicate = match pruning_predicates.get(&key) { + Some(pp) => pp, + // No pruning predicate available (schema mismatch at build time, or + // `always_true`): conservative fallback is "every row in scope is a + // candidate" — return a full-range bitmap so AND/OR with other + // leaves combines correctly. + None => { + let mut bm = RoaringBitmap::new(); + bm.insert_range(0u32..((ctx.max_doc - ctx.min_doc) as u32)); + return bm; + } + }; + // Evaluate page pruning for this single conjunct. + let selection = page_pruner.prune_rg(pruning_predicate, ctx.rg_idx, page_prune_metrics); + let mut bm = RoaringBitmap::new(); + match selection { + Some(sel) => { + // The selection is RG-relative. Translate to min_doc-relative + // space (the bitmap the tree evaluator walks over). Each + // kept selector covers a contiguous row range; insert it as + // a range in one call. `RoaringBitmap::insert_range` handles + // a full page of rows in O(log n) per container (or O(1) for + // full-container runs), vs. the naive one-bit-at-a-time loop + // which is O(rows_kept) with per-insert overhead. + let rg_offset = (ctx.rg_first_row as i32 - ctx.min_doc) as i64; + let span = (ctx.max_doc - ctx.min_doc) as i64; + let mut rg_pos: i64 = 0; + for s in sel.iter() { + if !s.skip { + // Selector covers [rg_pos, rg_pos + s.row_count) in + // RG-relative space; shift into scope-relative space + // and clamp to [0, span) since the scope bitmap only + // covers rows inside [min_doc, max_doc). + let start_rel = rg_pos + rg_offset; + let end_rel = start_rel + s.row_count as i64; + let lo = start_rel.max(0); + let hi = end_rel.min(span); + if lo < hi { + bm.insert_range(lo as u32..hi as u32); + } + } + rg_pos += s.row_count as i64; + } + } + None => { + // No pruning applicable (no page index or column missing) — + // conservative: every row in scope is a candidate. + bm.insert_range(0u32..((ctx.max_doc - ctx.min_doc) as u32)); + } + } + bm +} + +/// Derive collector call ranges from a bitmap based on the strategy in `ctx`. +/// +/// - `FullRange`: returns `[(min_doc, max_doc)]` (no narrowing). +/// - `TightenOuterBounds`: returns `[(first_set + min_doc, last_set + min_doc + 1)]`. +/// - `PageRangeSplit`: returns contiguous runs of set bits as absolute ranges. +fn ranges_from_bitmap(bm: &RoaringBitmap, ctx: &RgEvalContext) -> Vec<(i32, i32)> { + use super::CollectorCallStrategy; + match ctx.collector_strategy { + CollectorCallStrategy::FullRange => vec![(ctx.min_doc, ctx.max_doc)], + CollectorCallStrategy::TightenOuterBounds => { + match (bm.min(), bm.max()) { + (Some(lo), Some(hi)) => { + vec![(ctx.min_doc + lo as i32, ctx.min_doc + hi as i32 + 1)] + } + _ => vec![(ctx.min_doc, ctx.max_doc)], + } + } + CollectorCallStrategy::PageRangeSplit => { + // Extract contiguous runs of set bits as absolute doc ranges. + let mut ranges = Vec::new(); + let mut iter = bm.iter(); + let Some(first) = iter.next() else { + return vec![]; + }; + let mut run_start = first; + let mut run_end = first; // inclusive + for bit in iter { + if bit == run_end + 1 { + run_end = bit; + } else { + ranges.push(( + ctx.min_doc + run_start as i32, + ctx.min_doc + run_end as i32 + 1, + )); + run_start = bit; + run_end = bit; + } + } + ranges.push(( + ctx.min_doc + run_start as i32, + ctx.min_doc + run_end as i32 + 1, + )); + ranges + } + } +} + +/// Intersect two sorted, non-overlapping range lists. Both inputs are +/// `(start, end)` half-open intervals in absolute doc-id space. The +/// result contains only the portions where both lists overlap. +fn intersect_range_lists(a: &[(i32, i32)], b: &[(i32, i32)]) -> Vec<(i32, i32)> { + let mut out = Vec::new(); + let (mut i, mut j) = (0, 0); + while i < a.len() && j < b.len() { + let lo = a[i].0.max(b[j].0); + let hi = a[i].1.min(b[j].1); + if lo < hi { + out.push((lo, hi)); + } + if a[i].1 < b[j].1 { + i += 1; + } else { + j += 1; + } + } + out +} + +/// Cost weights used by `subtree_cost` to order AND/OR children in the +/// candidate stage. Tuning knobs, not a hard contract. +/// +/// - Predicate = 1: page-stats-only, no I/O, a handful of array lookups. +/// - Collector = 10: requires materialising an actual doc-id bitset over +/// FFM — posting-list iteration on the Java side, bitset transport + +/// RoaringBitmap expansion on the Rust side. Relative cost is +/// workload-dependent (Lucene posting iteration is fast for narrow +/// queries, slower for wide ones) so "10" is a conservative default. +/// Tune (or make config-driven) if profiling shows it matters. + +/// Internal scale factor for cost computation. All costs are multiplied +/// by this so integer division preserves meaningful selectivity differences. +/// A predicate keeping 1/8 pages costs `1000 * 1/8 = 125` vs one keeping +/// 5/8 pages at `1000 * 5/8 = 625`. Collector cost `10 * 1000 = 10_000`. +pub(crate) const COST_SCALE: u32 = 1000; + +/// Recursively compute the accumulated cost of a subtree for +/// candidate-stage ordering. +/// +/// For `Predicate` leaves with a matching `PruningPredicate`, the cost +/// is weighted by page-level selectivity: `cost_predicate * COST_SCALE * (surviving_pages / total_pages)`. +/// More selective predicates (fewer surviving pages) get lower cost and +/// are evaluated first in AND nodes, producing tighter ranges for +/// subsequent Collector siblings. +/// +/// Falls back to the static `cost_predicate * COST_SCALE` when page stats are +/// unavailable (no page index, expression not translatable, etc.). +/// +/// `Not` passes through to its child; `And`/`Or` sum their children. +pub(crate) fn subtree_cost( + node: &ResolvedNode, + ctx: &RgEvalContext, + page_pruner: &PagePruner, + pruning_predicates: &HashMap>, +) -> u32 { + match node { + ResolvedNode::Predicate(expr) => { + let base = ctx.cost_predicate * COST_SCALE; + let key = Arc::as_ptr(expr) as *const () as usize; + if let Some(pp) = pruning_predicates.get(&key) { + if let Some(page_counts) = page_pruner.page_row_counts(ctx.rg_idx) { + let total = page_counts.len() as u32; + if total > 0 { + if let Some(sel) = page_pruner.prune_rg(pp, ctx.rg_idx, None) { + // Count pages with at least one selected row. + // RowSelection merges adjacent same-decision + // selectors, so we walk the selection and map + // row offsets back to page boundaries. + let mut kept_pages = 0u32; + let mut row_offset = 0usize; + let mut page_idx = 0usize; + let mut page_start = 0usize; + let mut page_end = page_counts[0]; + for s in sel.iter() { + let seg_end = row_offset + s.row_count; + while page_idx < total as usize { + if !s.skip && row_offset < page_end && seg_end > page_start { + kept_pages += 1; + // Advance to next page to avoid double-counting. + page_idx += 1; + if page_idx < total as usize { + page_start = page_end; + page_end += page_counts[page_idx]; + } + } else if page_end <= seg_end { + page_idx += 1; + if page_idx < total as usize { + page_start = page_end; + page_end += page_counts[page_idx]; + } + } else { + break; + } + } + row_offset = seg_end; + } + return (base * kept_pages + total - 1) / total; + } + } + } + } + base + } + ResolvedNode::Collector { .. } => ctx.cost_collector * COST_SCALE, + ResolvedNode::Not(child) => subtree_cost(child, ctx, page_pruner, pruning_predicates), + ResolvedNode::And(children) | ResolvedNode::Or(children) => children + .iter() + .map(|c| subtree_cost(c, ctx, page_pruner, pruning_predicates)) + .sum(), + } +} + +/// True if `node` contains any `Predicate` leaf (transitively). +/// Used to decide if a `Not(child)` Phase 1 result is safe to invert via +/// universe subtraction. See the `Not` arm in `prefetch_node` for why. +fn subtree_has_predicate(node: &ResolvedNode) -> bool { + match node { + ResolvedNode::Predicate(_) => true, + ResolvedNode::Collector { .. } => false, + ResolvedNode::And(cs) | ResolvedNode::Or(cs) => cs.iter().any(subtree_has_predicate), + ResolvedNode::Not(c) => subtree_has_predicate(c), + } +} + +// Refinement stage [ Post Decode, where we need the actual decoded values to evaluate ] : tree walker +// +// Runs after parquet has delivered a decoded record batch. Walks the same +// tree again — in original order this time, not cost-sorted — and combines +// per-row BooleanArrays using Arrow's 3VL-safe `and_kleene`/`or_kleene`/`not` +// kernels. Collector leaves read their cached bitmap from the side-table +// (keyed by `Arc::as_ptr(collector)`, which is stable across the cost-sort +// used in the candidate stage). Predicate leaves evaluate the actual +// comparison against the batch's column data. Short-circuits on +// definitively-all-false for AND and definitively-all-true for OR +// (Kleene-safe: both check `null_count == 0` first). + +fn on_batch_node( + node: &ResolvedNode, + state: &TreePrefetch, + batch: &RecordBatch, + rg_first_row: i64, + position_map: &PositionMap, + batch_offset: usize, + batch_len: usize, +) -> Result { + match node { + ResolvedNode::And(children) => { + let mut optional_result_bitmap: Option = None; + for child in children { + let child_bitmap = on_batch_node( + child, + state, + batch, + rg_first_row, + position_map, + batch_offset, + batch_len, + )?; + optional_result_bitmap = Some(match optional_result_bitmap { + None => child_bitmap, + Some(result_bitmap) => { + and(&result_bitmap, &child_bitmap).map_err(|e| e.to_string())? + } + }); + // Short-circuit: if every row is definitively false + // (no nulls, zero trues), any further `FALSE AND x` is + // still FALSE in SQL 3VL. Safe to stop. + if let Some(ref result_bitmap) = optional_result_bitmap { + if result_bitmap.null_count() == 0 && result_bitmap.true_count() == 0 { + return Ok(result_bitmap.clone()); + } + } + } + Ok(optional_result_bitmap.unwrap_or_else(|| all_true(batch_len))) + } + ResolvedNode::Or(children) => { + let mut optional_result_bitmap: Option = None; + for child in children { + let child_bitmap = on_batch_node( + child, + state, + batch, + rg_first_row, + position_map, + batch_offset, + batch_len, + )?; + optional_result_bitmap = Some(match optional_result_bitmap { + None => child_bitmap, + Some(result_bitmap) => { + or(&result_bitmap, &child_bitmap).map_err(|e| e.to_string())? + } + }); + // Short-circuit: if every row is definitively true + // (no nulls, zero falses), any further `TRUE OR x` is + // still TRUE in SQL 3VL. Safe to stop. + if let Some(ref result_bitmap) = optional_result_bitmap { + if result_bitmap.null_count() == 0 && result_bitmap.false_count() == 0 { + return Ok(result_bitmap.clone()); + } + } + } + Ok(optional_result_bitmap.unwrap_or_else(|| all_false(batch_len))) + } + ResolvedNode::Not(child) => { + let child_bitmap = on_batch_node( + child, + state, + batch, + rg_first_row, + position_map, + batch_offset, + batch_len, + )?; + not(&child_bitmap).map_err(|e| e.to_string()) + } + ResolvedNode::Collector { collector, .. } => { + let key = Arc::as_ptr(collector) as *const () as usize; + let bitmap = state + .per_leaf + .iter() + .find_map(|(i, bm)| if *i == key { Some(bm) } else { None }) + .ok_or_else(|| format!("Phase 2: leaf bitmap missing for key {:#x}", key))?; + Ok(bitmap_to_batch_mask( + bitmap, + state.min_doc, + rg_first_row, + position_map, + batch_offset, + batch_len, + )) + } + ResolvedNode::Predicate(expr) => predicate_to_batch_mask(batch, expr), + } +} + +/// Translate a Collector leaf's bitmap (in min-doc-relative coordinates) to +/// a per-batch `BooleanArray`. +/// +/// With block-granular RowSelection the delivered rows are a compacted +/// subset of the RG, not a contiguous span. `position_map` lets us recover +/// which RG-relative position each delivered row came from; from there we +/// compute the absolute doc id and look it up in `bm`. +/// +/// `batch_offset` is the delivered-row index of the first row in this +/// batch; delivered row `batch_offset + i` maps to RG position +/// `position_map.rg_position(batch_offset + i)`. +fn bitmap_to_batch_mask( + bm: &RoaringBitmap, + min_doc: i32, + rg_first_row: i64, + position_map: &PositionMap, + batch_offset: usize, + batch_len: usize, +) -> BooleanArray { + // Convert batch-row index -> min-doc-relative bitmap index. + // delivered row i -> rg_position(batch_offset + i) -> abs_doc -> bit. + // + // For Identity position map, rg_position(k) == k, so the mapping is + // linear: delivered row i -> bit (rg_first_row + batch_offset + i) - min_doc. + // We iterate the set bits of `bm` within the batch's coverage and + // translate back, instead of per-row `bm.contains()`. + let words = batch_len.div_ceil(64); + let mut out = vec![0u64; words]; + + let anchor = rg_first_row - min_doc as i64; // rg_pos -> bit: rg_pos + anchor + match position_map { + PositionMap::Identity { .. } => { + // delivered row i -> rg_pos = batch_offset + i -> bit = batch_offset + i + anchor. + // Enumerate set bits in `bm` within [anchor + batch_offset, anchor + batch_offset + batch_len). + let lo = (batch_offset as i64 + anchor).max(0); + let hi = (batch_offset as i64 + anchor + batch_len as i64).max(0); + if hi > 0 && lo <= u32::MAX as i64 { + let lo_u32 = lo as u32; + let hi_u32 = hi.min(u32::MAX as i64) as u32; + for b in bm.range(lo_u32..hi_u32) { + // delivered index = bit - anchor - batch_offset + let delivered = (b as i64 - anchor - batch_offset as i64) as usize; + if delivered < batch_len { + out[delivered >> 6] |= 1u64 << (delivered & 63); + } + } + } + } + PositionMap::Bitmap { .. } | PositionMap::Runs { .. } => { + // General case — fall back to per-row lookup but use packed-bit + // assembly so we avoid the Vec + BooleanArray::from copy. + for i in 0..batch_len { + let rg_pos = match position_map.rg_position(batch_offset + i) { + Some(p) => p, + None => continue, + }; + let abs_doc = rg_first_row + rg_pos as i64; + let bit = abs_doc - min_doc as i64; + if bit >= 0 && bit <= u32::MAX as i64 && bm.contains(bit as u32) { + out[i >> 6] |= 1u64 << (i & 63); + } + } + } + } + packed_bits_to_boolean_array(out, batch_len) +} + +// Evaluate an arbitrary boolean `PhysicalExpr` against a batch; return +// the resulting per-row mask. Uses DataFusion's expression evaluator — +// handles all operators, IN, IS NULL, LIKE, arithmetic, CAST, UDFs etc. +// +// Fast-path for `col OP literal` comparisons: skip the expression walk +// and dispatch directly to the arrow kernel. This is the dominant shape +// in production (Predicate leaves are almost always simple comparisons) +// and the kernel call is 3–5x cheaper than going through +// `BinaryExpr::evaluate` + column/literal dispatch. +fn predicate_to_batch_mask( + batch: &RecordBatch, + expr: &Arc, +) -> Result { + // Fast-path: detect `col OP literal` and call the kernel directly. + if let Some(bin) = expr.as_any().downcast_ref::() { + if let (Some(col), Some(lit)) = ( + bin.left().as_any().downcast_ref::(), + bin.right().as_any().downcast_ref::(), + ) { + match batch.column_by_name(col.name()) { + None => { + // Column absent from batch schema: SQL UNKNOWN. + let nulls: Vec> = (0..batch.num_rows()).map(|_| None).collect(); + return Ok(BooleanArray::from(nulls)); + } + Some(col_arr) => { + let scalar = lit.value().to_scalar().map_err(|e| e.to_string())?; + let kernel_result = match *bin.op() { + Operator::Eq => eq(col_arr, &scalar), + Operator::NotEq => neq(col_arr, &scalar), + Operator::Lt => lt(col_arr, &scalar), + Operator::LtEq => lt_eq(col_arr, &scalar), + Operator::Gt => gt(col_arr, &scalar), + Operator::GtEq => gt_eq(col_arr, &scalar), + _ => { + // Non-comparison op (And/Or/Plus/...) — fall + // through to the general evaluator path. + return evaluate_via_df(batch, expr); + } + }; + return kernel_result.map_err(|e| e.to_string()); + } + } + } + } + evaluate_via_df(batch, expr) +} + +/// General-case evaluator — `expr.evaluate(batch)` with schema-drift +/// safety check. Used for non-`col OP literal` shapes (IN, IS NULL, +/// arithmetic, NOT-wrapped, …). +fn evaluate_via_df( + batch: &RecordBatch, + expr: &Arc, +) -> Result { + // Schema drift: if the expression references any column not present + // in this batch's schema, SQL semantics demand UNKNOWN for every + // row. Return an all-NULL BooleanArray so kleene AND/OR combine + // correctly and `filter_record_batch` drops the UNKNOWN rows. + let batch_schema = batch.schema(); + let referenced = datafusion::physical_expr::utils::collect_columns(expr); + for col in &referenced { + if batch_schema.index_of(col.name()).is_err() { + let nulls: Vec> = (0..batch.num_rows()).map(|_| None).collect(); + return Ok(BooleanArray::from(nulls)); + } + } + + let result = expr + .evaluate(batch) + .map_err(|e| format!("expr.evaluate: {}", e))?; + match result { + ColumnarValue::Array(arr) => { + if arr.data_type() == &datafusion::arrow::datatypes::DataType::Boolean { + Ok(arr.as_boolean().clone()) + } else { + Err(format!( + "predicate evaluation produced non-boolean array: {:?}", + arr.data_type() + )) + } + } + ColumnarValue::Scalar(sv) => match sv { + datafusion::common::ScalarValue::Boolean(Some(b)) => { + Ok(BooleanArray::from(vec![b; batch.num_rows()])) + } + datafusion::common::ScalarValue::Boolean(None) => { + let nulls: Vec> = (0..batch.num_rows()).map(|_| None).collect(); + Ok(BooleanArray::from(nulls)) + } + other => Err(format!( + "predicate evaluation produced non-boolean scalar: {:?}", + other + )), + }, + } +} + +fn all_true(n: usize) -> BooleanArray { + BooleanArray::from(vec![true; n]) +} +fn all_false(n: usize) -> BooleanArray { + BooleanArray::from(vec![false; n]) +} + +/// CollectorLeafBitmaps — default LeafBitmapSource for today's flow +/// +/// Expands index-backed `RowGroupDocsCollector` output into RoaringBitmaps. +/// Pulls the collector directly off the `ResolvedNode::Collector` passed to +/// it — no separate indexing required, so this impl is fully stateless. +pub struct CollectorLeafBitmaps { + /// Incremented once per call to [`Self::leaf_bitmap`] — one FFM + /// round-trip to Java per Collector leaf per RG. `None` for tests + /// that don't care about metrics. + pub ffm_collector_calls: Option, +} + +impl CollectorLeafBitmaps { + /// Construct a `CollectorLeafBitmaps` with no metrics. + pub fn without_metrics() -> Self { + Self { + ffm_collector_calls: None, + } + } +} + +impl LeafBitmapSource for CollectorLeafBitmaps { + fn leaf_bitmap( + &self, + collector_node: &ResolvedNode, + _leaf_dfs_index: usize, // This is not used in this implementation + ctx: &RgEvalContext, + ) -> Result { + let collector = match collector_node { + ResolvedNode::Collector { collector, .. } => collector, + _ => { + return Err("CollectorLeafBitmaps: non-Collector node passed to leaf_bitmap".into()) + } + }; + // Use the narrowed call ranges if available (set by AND evaluator + // after earlier children shrink the candidate set). Each range + // produces one FFM call; results are merged into one bitmap in + // min_doc-relative coordinates. + // Use narrowed call ranges if available (set by AND evaluator). + let call_ranges = ctx + .collector_call_ranges + .clone() + .unwrap_or_else(|| vec![(ctx.min_doc, ctx.max_doc)]); + + let mut result_bitmap = RoaringBitmap::new(); + for (call_min, call_max) in &call_ranges { + let bitset = collector.collect_packed_u64_bitset(*call_min, *call_max)?; + if let Some(ref c) = self.ffm_collector_calls { + c.add(1); + } + let offset = (*call_min - ctx.min_doc) as u32; + let num_docs = (*call_max - *call_min) as u32; + let bytes: &[u8] = unsafe { + std::slice::from_raw_parts(bitset.as_ptr() as *const u8, bitset.len() * 8) + }; + let mut chunk = RoaringBitmap::from_lsb0_bytes(offset, bytes); + let upper = offset + num_docs; + if upper < u32::MAX { + chunk.remove_range(upper..); + } + result_bitmap |= chunk; + } + Ok(result_bitmap) + } +} + +// ══════════════════════════════════════════════════════════════════════ +// Tests +// ══════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod tests { + use super::*; + use crate::indexed_table::bool_tree::ResolvedNode; + use crate::indexed_table::index::RowGroupDocsCollector; + use datafusion::arrow::array::Int32Array; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::arrow::record_batch::RecordBatch; + use datafusion::common::ScalarValue; + use datafusion::parquet::arrow::arrow_reader::{ + ArrowReaderMetadata, ArrowReaderOptions, RowSelection, RowSelector, + }; + use datafusion::parquet::arrow::ArrowWriter; + use datafusion::physical_expr::expressions::{BinaryExpr, Column as PhysColumn, Literal}; + use std::collections::{HashMap, HashSet}; + + /// Deterministic bitmap source for tests. + struct FixedLeafBitmaps { + bitmaps: Vec, + } + impl LeafBitmapSource for FixedLeafBitmaps { + fn leaf_bitmap( + &self, + _tree: &ResolvedNode, + idx: usize, + _ctx: &RgEvalContext, + ) -> Result { + Ok(self.bitmaps[idx].clone()) + } + } + + fn test_ctx() -> RgEvalContext { + RgEvalContext { + rg_idx: 0, + rg_first_row: 0, + rg_num_rows: 16, + min_doc: 0, + max_doc: 16, + cost_predicate: 1, + cost_collector: 10, + collector_call_ranges: None, + collector_strategy: super::super::CollectorCallStrategy::TightenOuterBounds, + } + } + + fn empty_pruner() -> PagePruner { + // Build a minimal PagePruner with no filters — candidate_row_ids_for_filter + // won't be called since we use no Predicate nodes in these tests. + // We need a schema + metadata. Simplest: write a tiny parquet and load it. + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![0i32; 16]))], + ) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + let mut writer = ArrowWriter::try_new(tmp.reopen().unwrap(), schema.clone(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + let meta = ArrowReaderMetadata::load( + &tmp.reopen().unwrap(), + ArrowReaderOptions::new().with_page_index(true), + ) + .unwrap(); + PagePruner::new(meta.schema(), meta.metadata().clone()) + } + + fn collector_leaf(idx: usize) -> ResolvedNode { + // Use a no-op collector — LeafBitmapSource supplies bitmaps, not the collector + #[derive(Debug)] + struct Dummy; + impl RowGroupDocsCollector for Dummy { + fn collect_packed_u64_bitset(&self, _: i32, _: i32) -> Result, String> { + Ok(vec![]) + } + } + let _ = idx; + ResolvedNode::Collector { + provider_key: 0, + collector: Arc::new(Dummy), + } + } + + fn bm(docs: &[u32]) -> RoaringBitmap { + let mut r = RoaringBitmap::new(); + for &d in docs { + r.insert(d); + } + r + } + + #[test] + fn and_of_two_collectors_intersects_phase1() { + let tree = ResolvedNode::And(vec![collector_leaf(0), collector_leaf(1)]); + let leaves = FixedLeafBitmaps { + bitmaps: vec![bm(&[1, 2, 3, 4]), bm(&[3, 4, 5])], + }; + let pruner = empty_pruner(); + let result = BitmapTreeEvaluator + .prefetch(&tree, &test_ctx(), &leaves, &pruner, &HashMap::new(), None) + .unwrap(); + assert_eq!(result.candidates, bm(&[3, 4])); + assert_eq!(result.per_leaf.len(), 2); + } + + #[test] + fn or_of_two_collectors_unions_phase1() { + let tree = ResolvedNode::Or(vec![collector_leaf(0), collector_leaf(1)]); + let leaves = FixedLeafBitmaps { + bitmaps: vec![bm(&[1, 2]), bm(&[2, 3])], + }; + let pruner = empty_pruner(); + let result = BitmapTreeEvaluator + .prefetch(&tree, &test_ctx(), &leaves, &pruner, &HashMap::new(), None) + .unwrap(); + assert_eq!(result.candidates, bm(&[1, 2, 3])); + } + + #[test] + fn not_collector_complements_against_universe() { + let tree = ResolvedNode::Not(Box::new(collector_leaf(0))); + let leaves = FixedLeafBitmaps { + bitmaps: vec![bm(&[0, 1, 2])], + }; + let pruner = empty_pruner(); + let result = BitmapTreeEvaluator + .prefetch(&tree, &test_ctx(), &leaves, &pruner, &HashMap::new(), None) + .unwrap(); + // Universe is [0, 16). Minus {0,1,2} = {3..15} + let expected: RoaringBitmap = (3u32..16).collect(); + assert_eq!(result.candidates, expected); + } + + #[test] + fn phase2_collector_uses_cached_bitmap() { + let tree = collector_leaf(0); + let leaves = FixedLeafBitmaps { + bitmaps: vec![bm(&[1, 3, 5])], + }; + let pruner = empty_pruner(); + let state = BitmapTreeEvaluator + .prefetch(&tree, &test_ctx(), &leaves, &pruner, &HashMap::new(), None) + .unwrap(); + + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![0i32; 8]))]).unwrap(); + // Batch covers docs [0, 8). Match bitmap {1,3,5}. + // Full-scan position map: delivered index == RG position. + let pm = PositionMap::from_selection(&RowSelection::from(vec![RowSelector::select(8)])); + let mask = BitmapTreeEvaluator + .on_batch(&tree, &state, &batch, 0, &pm, 0, 8) + .unwrap(); + let expected = + BooleanArray::from(vec![false, true, false, true, false, true, false, false]); + assert_eq!(mask, expected); + } + + /// Identity position map over `rg_num_rows`. Delivered index == RG + /// position — matches the pre-block-granular full-scan behaviour and + /// keeps the per-test expected values unchanged. + fn identity_pm(rg_num_rows: usize) -> PositionMap { + PositionMap::from_selection(&RowSelection::from(vec![RowSelector::select(rg_num_rows)])) + } + + #[test] + fn bitmap_to_batch_mask_anchors_correctly() { + // min_doc = 100, bitmap has {1, 5} (min_doc-relative). + // rg_first_row = 100, batch starts at offset 0, length 8. + // For each row i: rg_pos = i, abs_doc = 100 + i, + // rel_doc = abs_doc - min_doc = i. bits set at i=1 and i=5. + let bm = { + let mut b = RoaringBitmap::new(); + b.insert(1); + b.insert(5); + b + }; + let pm = identity_pm(8); + let mask = bitmap_to_batch_mask( + &bm, /*min_doc*/ 100, /*rg_first_row*/ 100, &pm, 0, 8, + ); + let got: Vec = (0..8).map(|i| mask.value(i)).collect(); + assert_eq!( + got, + vec![false, true, false, false, false, true, false, false] + ); + } + + #[test] + fn bitmap_to_batch_mask_handles_batch_offset_within_rg() { + // min_doc = 0, rg_first_row = 0, batch starts at rg offset 4, len 4. + // Identity position map over rg_num_rows=16. + // For row i: rg_pos = 4 + i, abs_doc = 4 + i, rel = 4 + i. + // Bitmap bits {0, 5, 9} → rows where (4+i) in {0,5,9} → i=1, i=5 (out of range), so only i=1. + let bm = { + let mut b = RoaringBitmap::new(); + b.insert(0); + b.insert(5); + b.insert(9); + b + }; + let pm = identity_pm(16); + let mask = bitmap_to_batch_mask(&bm, 0, 0, &pm, 4, 4); + let got: Vec = (0..4).map(|i| mask.value(i)).collect(); + assert_eq!(got, vec![false, true, false, false]); + } + + #[test] + fn bitmap_to_batch_mask_empty_bitmap_produces_all_false() { + let bm = RoaringBitmap::new(); + let pm = identity_pm(5); + let mask = bitmap_to_batch_mask(&bm, 0, 0, &pm, 0, 5); + assert_eq!(mask.true_count(), 0); + assert_eq!(mask.len(), 5); + } + + #[test] + fn bitmap_to_batch_mask_zero_length_batch() { + let bm = { + let mut b = RoaringBitmap::new(); + b.insert(0); + b + }; + let pm = identity_pm(1); + let mask = bitmap_to_batch_mask(&bm, 0, 0, &pm, 0, 0); + assert_eq!(mask.len(), 0); + } + + #[test] + fn bitmap_to_batch_mask_respects_position_map() { + // RG has 10 rows; RowSelection selects rows [0..3] and [7..10], + // skipping [3..7]. Delivered rows = 6 (3 + 3). + // delivered idx 0 → rg_pos 0 + // delivered idx 1 → rg_pos 1 + // delivered idx 2 → rg_pos 2 + // delivered idx 3 → rg_pos 7 + // delivered idx 4 → rg_pos 8 + // delivered idx 5 → rg_pos 9 + // Bitmap (min_doc-relative, min_doc = 0, rg_first_row = 0) {2, 8}. + // Expected mask per delivered index: [F,F,T,F,T,F] + let sel = RowSelection::from(vec![ + RowSelector::select(3), + RowSelector::skip(4), + RowSelector::select(3), + ]); + let pm = PositionMap::from_selection(&sel); + let bm = { + let mut b = RoaringBitmap::new(); + b.insert(2); + b.insert(8); + b + }; + let mask = bitmap_to_batch_mask(&bm, 0, 0, &pm, 0, 6); + let got: Vec = (0..6).map(|i| mask.value(i)).collect(); + assert_eq!(got, vec![false, false, true, false, true, false]); + } + + // ── Phase 2 short-circuit ───────────────────────────────────────── + + /// Evaluator that counts how many times its `leaf_bitmap` was called — + /// used to observe Phase 2 short-circuit by wrapping predicate leaves as + /// collectors whose bitmaps are the "predicate mask". + /// + /// We can't directly inspect Phase 2 calls since they go through + /// `on_batch_node`, but we can observe them by making Phase 2 evaluation + /// visible via side effect on a counting LeafBitmapSource. + /// + /// For Phase 2 specifically, `ResolvedNode::Collector` uses + /// `state.per_leaf` lookup (cached Phase 1 bitmaps), not the + /// LeafBitmapSource. So short-circuit observation has to be at the + /// `on_batch_node` level — we use a custom node tree and assert on the + /// resulting mask shape with deliberately-wrong siblings. + /// + /// The strategy: construct AND(all_false_child, poison_child) where + /// `poison_child` would `panic!` if evaluated. If the test passes, + /// short-circuit prevented evaluation of the poison child. + + /// Build a ResolvedNode::Collector whose cached Phase 1 bitmap is `bm`. + fn cached_collector(bm: RoaringBitmap) -> (ResolvedNode, (usize, RoaringBitmap)) { + #[derive(Debug)] + struct Poison; + impl RowGroupDocsCollector for Poison { + fn collect_packed_u64_bitset(&self, _: i32, _: i32) -> Result, String> { + unreachable!("Phase 2 must not call collect") + } + } + let collector: Arc = Arc::new(Poison); + let key = Arc::as_ptr(&collector) as *const () as usize; + let node = ResolvedNode::Collector { + provider_key: 0, + collector, + }; + (node, (key, bm)) + } + + #[test] + fn phase2_and_short_circuits_on_all_false() { + // AND(all_false_leaf, poison_leaf). The poison leaf's bitmap is + // absent from `state.per_leaf`, so evaluating it would error with + // "leaf bitmap missing". If short-circuit fires, poison is skipped + // and we get the zero mask without erroring. + let (false_leaf, false_entry) = cached_collector(RoaringBitmap::new()); + let (poison_leaf, _poison_entry) = cached_collector({ + let mut b = RoaringBitmap::new(); + b.insert(999); // doesn't matter — shouldn't be looked up + b + }); + + let tree = ResolvedNode::And(vec![false_leaf, poison_leaf]); + // Register ONLY the false leaf. If short-circuit misfires, Phase 2 + // will try to look up `poison_entry` and fail with "leaf bitmap missing". + let state = TreePrefetch { + candidates: RoaringBitmap::new(), + per_leaf: vec![false_entry], + min_doc: 0, + }; + + let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![0i32; 4]))]).unwrap(); + + let mask = on_batch_node(&tree, &state, &batch, 0, &identity_pm(4), 0, 4) + .expect("AND should short-circuit on all-false acc, skipping poison leaf"); + assert_eq!(mask.true_count(), 0); + } + + #[test] + fn phase2_or_short_circuits_on_all_true() { + // OR(all_true_leaf, poison_leaf). Same setup as AND case but inverted. + let (true_leaf, true_entry) = cached_collector({ + let mut b = RoaringBitmap::new(); + b.insert_range(0..4); + b + }); + let (poison_leaf, _) = cached_collector({ + let mut b = RoaringBitmap::new(); + b.insert(999); + b + }); + + let tree = ResolvedNode::Or(vec![true_leaf, poison_leaf]); + let state = TreePrefetch { + candidates: RoaringBitmap::new(), + per_leaf: vec![true_entry], + min_doc: 0, + }; + + let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![0i32; 4]))]).unwrap(); + + let mask = on_batch_node(&tree, &state, &batch, 0, &identity_pm(4), 0, 4) + .expect("OR should short-circuit on all-true acc, skipping poison leaf"); + assert_eq!(mask.true_count(), 4); + } + + // ── Candidate-stage skip of bitmap materialization ──────────────── + // + // The tests below prove that when an AND short-circuits at a point + // where every ancestor is AND (so the whole candidate set is doomed + // to be empty and the RG will be skipped), the walker does NOT ask + // the `LeafBitmapSource` for the remaining Collector leaves' bitmaps. + // The symmetric case (AND under OR/NOT) must still materialise. + + /// LeafBitmapSource returning bitmaps by DFS index, panicking on forbidden indices. + struct PoisonLeafBitmaps { + allowed: HashMap, + forbidden: HashSet, + } + impl LeafBitmapSource for PoisonLeafBitmaps { + fn leaf_bitmap( + &self, + _tree: &ResolvedNode, + idx: usize, + _ctx: &RgEvalContext, + ) -> Result { + if self.forbidden.contains(&idx) { + panic!("leaf_bitmap called for forbidden leaf {}", idx); + } + Ok(self.allowed.get(&idx).cloned().unwrap_or_default()) + } + } + + #[test] + fn candidate_root_and_short_circuit_skips_forbidden_collector() { + // Tree: AND(Collector, Collector). Root is AND. + // - Leaves are cost-equal, stable-sort preserves input order; so + // DFS index 0 = first Collector, DFS index 1 = second. + // - First returns empty → AND short-circuits. + // - Because we're under root-AND, the whole candidate set is + // doomed empty and the RG will be skipped. The walker must NOT + // call LeafBitmapSource for the second Collector. + let tree = ResolvedNode::And(vec![collector_leaf(0), collector_leaf(1)]); + let mut allowed = HashMap::new(); + allowed.insert(0, RoaringBitmap::new()); // empty → trigger short-circuit + let mut forbidden = HashSet::new(); + forbidden.insert(1); // any call for leaf 1 panics + let leaves = PoisonLeafBitmaps { allowed, forbidden }; + let pruner = empty_pruner(); + + let result = BitmapTreeEvaluator + .prefetch(&tree, &test_ctx(), &leaves, &pruner, &HashMap::new(), None) + .unwrap(); + assert!(result.candidates.is_empty()); + } + + #[test] + fn candidate_and_short_circuit_under_or_still_materialises() { + // Tree: OR(AND(empty_leaf, other_leaf), standalone_leaf). + // Cost sort at root OR: [standalone_leaf (10), AND (20)]. + // DFS order: + // idx 0 = standalone_leaf (evaluated first by cost sort), + // idx 1 = empty_leaf (AND's first child), + // idx 2 = other_leaf (AND's second child). + // + // The AND short-circuits on idx 1 (empty). Because the path to + // root contains an OR (not all-AND), the walker must still + // materialise idx 2's bitmap so refinement can look it up. + let tree = ResolvedNode::Or(vec![ + ResolvedNode::And(vec![collector_leaf(0), collector_leaf(1)]), + collector_leaf(2), + ]); + let mut allowed = HashMap::new(); + allowed.insert(0, { + let mut b = RoaringBitmap::new(); + b.insert(5); + b + }); + allowed.insert(1, RoaringBitmap::new()); // empty → short-circuit + allowed.insert(2, { + let mut b = RoaringBitmap::new(); + b.insert(7); + b + }); + let leaves = PoisonLeafBitmaps { + allowed, + forbidden: HashSet::new(), + }; + let pruner = empty_pruner(); + + let result = BitmapTreeEvaluator + .prefetch(&tree, &test_ctx(), &leaves, &pruner, &HashMap::new(), None) + .unwrap(); + // OR contributes {5} from standalone_leaf → non-empty candidates. + assert!(!result.candidates.is_empty()); + // All 3 collector leaves must have per_leaf entries — AND + // short-circuit under OR does NOT skip materialisation. + assert_eq!( + result.per_leaf.len(), + 3, + "expected 3 per_leaf entries; got {}", + result.per_leaf.len() + ); + } + + #[test] + fn candidate_and_short_circuit_under_not_still_materialises() { + // Tree: NOT(AND(empty_leaf, other_leaf)). + // Inner AND short-circuits on empty_leaf. NOT inverts empty to + // universe → candidates non-empty → RG read → refinement will + // look up other_leaf's bitmap. + let tree = ResolvedNode::Not(Box::new(ResolvedNode::And(vec![ + collector_leaf(0), + collector_leaf(1), + ]))); + let mut allowed = HashMap::new(); + allowed.insert(0, RoaringBitmap::new()); // triggers short-circuit + allowed.insert(1, { + let mut b = RoaringBitmap::new(); + b.insert(9); + b + }); + let leaves = PoisonLeafBitmaps { + allowed, + forbidden: HashSet::new(), + }; + let pruner = empty_pruner(); + + let result = BitmapTreeEvaluator + .prefetch(&tree, &test_ctx(), &leaves, &pruner, &HashMap::new(), None) + .unwrap(); + // NOT inverts empty AND → universe. + assert_eq!(result.candidates.len(), 16); + // Both collector leaves materialised. + assert_eq!(result.per_leaf.len(), 2); + } + + // ── subtree_cost ───────────────────────────────────────────────── + + fn test_predicate_node() -> ResolvedNode { + let left: std::sync::Arc = + std::sync::Arc::new(PhysColumn::new("x", 0)); + let right: std::sync::Arc = + std::sync::Arc::new(Literal::new(ScalarValue::Int32(Some(0)))); + ResolvedNode::Predicate(std::sync::Arc::new(BinaryExpr::new( + left, + Operator::Eq, + right, + ))) + } + + #[test] + fn subtree_cost_leaf_nodes() { + let ctx = test_ctx(); + let pruner = empty_pruner(); + let pp = HashMap::new(); + assert_eq!( + subtree_cost(&test_predicate_node(), &ctx, &pruner, &pp), + ctx.cost_predicate * COST_SCALE + ); + assert_eq!(subtree_cost(&collector_leaf(0), &ctx, &pruner, &pp), ctx.cost_collector * COST_SCALE); + } + + #[test] + fn subtree_cost_not_passes_through() { + let ctx = test_ctx(); + let pruner = empty_pruner(); + let pp = HashMap::new(); + let wrapped = ResolvedNode::Not(Box::new(test_predicate_node())); + assert_eq!(subtree_cost(&wrapped, &ctx, &pruner, &pp), ctx.cost_predicate * COST_SCALE); + } + + #[test] + fn subtree_cost_sums_children() { + let ctx = test_ctx(); + let pruner = empty_pruner(); + let pp = HashMap::new(); + let tree = ResolvedNode::And(vec![ + test_predicate_node(), + test_predicate_node(), + collector_leaf(0), + ]); + assert_eq!( + subtree_cost(&tree, &ctx, &pruner, &pp), + (2 * ctx.cost_predicate + ctx.cost_collector) * COST_SCALE + ); + } + + #[test] + fn subtree_cost_predicate_heavy_nested_beats_single_collector() { + let nested = ResolvedNode::And(vec![ + test_predicate_node(), + test_predicate_node(), + test_predicate_node(), + ]); + let single_collector = collector_leaf(0); + let ctx = test_ctx(); + let pruner = empty_pruner(); + let pp = HashMap::new(); + assert!( + subtree_cost(&nested, &ctx, &pruner, &pp) < subtree_cost(&single_collector, &ctx, &pruner, &pp), + ); + } + + #[test] + fn subtree_cost_collector_heavy_nested_exceeds_single_collector() { + let nested = ResolvedNode::And(vec![collector_leaf(0), collector_leaf(1)]); + let single_collector = collector_leaf(0); + let ctx = test_ctx(); + let pruner = empty_pruner(); + let pp = HashMap::new(); + assert!(subtree_cost(&nested, &ctx, &pruner, &pp) > subtree_cost(&single_collector, &ctx, &pruner, &pp)); + } + + // ── intersect_range_lists unit tests ──────────────────────────── + + #[test] + fn intersect_empty_with_anything() { + assert_eq!(intersect_range_lists(&[], &[(0, 10)]), vec![]); + assert_eq!(intersect_range_lists(&[(0, 10)], &[]), vec![]); + assert_eq!(intersect_range_lists(&[], &[]), vec![]); + } + + #[test] + fn intersect_non_overlapping() { + // [0,5) and [10,15) → empty + assert_eq!(intersect_range_lists(&[(0, 5)], &[(10, 15)]), vec![]); + } + + #[test] + fn intersect_partial_overlap() { + // [0,10) ∩ [5,15) → [5,10) + assert_eq!(intersect_range_lists(&[(0, 10)], &[(5, 15)]), vec![(5, 10)]); + } + + #[test] + fn intersect_one_contains_other() { + // [0,20) ∩ [5,10) → [5,10) + assert_eq!(intersect_range_lists(&[(0, 20)], &[(5, 10)]), vec![(5, 10)]); + } + + #[test] + fn intersect_multiple_ranges() { + // a: [0,5), [10,20), [30,40) + // b: [3,12), [15,35) + // intersections: [3,5), [10,12), [15,20), [30,35) + let a = vec![(0, 5), (10, 20), (30, 40)]; + let b = vec![(3, 12), (15, 35)]; + assert_eq!( + intersect_range_lists(&a, &b), + vec![(3, 5), (10, 12), (15, 20), (30, 35)] + ); + } + + #[test] + fn intersect_identical() { + let a = vec![(10, 20), (30, 40)]; + assert_eq!(intersect_range_lists(&a, &a), vec![(10, 20), (30, 40)]); + } + + // ── ranges_from_bitmap unit tests ─────────────────────────────── + + #[test] + fn ranges_full_range_strategy() { + let mut ctx = test_ctx(); + ctx.collector_strategy = super::super::CollectorCallStrategy::FullRange; + let mut bm = RoaringBitmap::new(); + bm.insert_range(4..8); + // FullRange ignores the bitmap, returns [min_doc, max_doc) + assert_eq!(ranges_from_bitmap(&bm, &ctx), vec![(0, 16)]); + } + + #[test] + fn ranges_tighten_outer_bounds_strategy() { + let mut ctx = test_ctx(); + ctx.collector_strategy = super::super::CollectorCallStrategy::TightenOuterBounds; + let mut bm = RoaringBitmap::new(); + bm.insert_range(4..8); + bm.insert(12); + // TightenOuterBounds: [min_doc + bm.min(), min_doc + bm.max() + 1) + assert_eq!(ranges_from_bitmap(&bm, &ctx), vec![(4, 13)]); + } + + #[test] + fn ranges_page_range_split_contiguous() { + let mut ctx = test_ctx(); + ctx.collector_strategy = super::super::CollectorCallStrategy::PageRangeSplit; + let mut bm = RoaringBitmap::new(); + bm.insert_range(4..8); + // Single contiguous run → one range + assert_eq!(ranges_from_bitmap(&bm, &ctx), vec![(4, 8)]); + } + + #[test] + fn ranges_page_range_split_with_gap() { + let mut ctx = test_ctx(); + ctx.collector_strategy = super::super::CollectorCallStrategy::PageRangeSplit; + let mut bm = RoaringBitmap::new(); + bm.insert_range(2..5); // bits 2,3,4 + bm.insert_range(8..11); // bits 8,9,10 + bm.insert(14); // bit 14 + // Three contiguous runs → three ranges + assert_eq!( + ranges_from_bitmap(&bm, &ctx), + vec![(2, 5), (8, 11), (14, 15)] + ); + } + + #[test] + fn ranges_page_range_split_empty_bitmap() { + let mut ctx = test_ctx(); + ctx.collector_strategy = super::super::CollectorCallStrategy::PageRangeSplit; + let bm = RoaringBitmap::new(); + assert_eq!(ranges_from_bitmap(&bm, &ctx), vec![]); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/mod.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/mod.rs new file mode 100644 index 0000000000000..f59a3968f95a9 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/mod.rs @@ -0,0 +1,767 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Row-group-level bitset sources — the pluggability seam for where +//! boolean tree evaluation happens. +//! +//! [`IndexedStream`](crate::indexed_table::stream::IndexedStream) only depends +//! on [`RowGroupBitsetSource`]. The source of the bitset is abstracted. +//! +//! # Invariant — row-group-at-a-time +//! +//! The trait methods operate on one RG. There is no `prefetch_shard` or +//! `evaluate_full_filter` method. Even when tree evaluation eventually moves +//! elsewhere: +//! +//! - Bitsets stay small (~512 bytes per RG). +//! - Prefetch overlaps the next RG's bitset with the current RG's parquet read. +//! - Memory stays bounded regardless of shard size. +//! +//! # Pluggable tree evaluation (multi-filter tree path) +//! +//! For tree queries, evaluation has two orthogonal concerns: +//! +//! 1. **Tree evaluation strategy** ([`TreeEvaluator`]) — the algorithm that +//! walks the tree, combines bitmaps, produces superset candidates + +//! exact per-batch mask. Today: [`bitmap_tree::BitmapTreeEvaluator`]. +//! This is extensible to different implementations. +//! 2. **Leaf bitmap source** ([`LeafBitmapSource`]) — given a `Collector` +//! leaf, produce its RoaringBitmap for this RG. Today: backend-backed +//! (FFM upcall + bitset expansion). +//! +//! [`TreeBitsetSource`] composes any `TreeEvaluator` with any +//! `LeafBitmapSource` and exposes the composite as a `RowGroupBitsetSource`. +//! Swapping impls requires only passing different `Arc`s at construction. + +pub mod bitmap_tree; +pub mod single_collector; + +use std::any::Any; +use std::sync::Arc; + +use datafusion::arrow::array::BooleanArray; +use datafusion::arrow::record_batch::RecordBatch; +use roaring::RoaringBitmap; + +use super::bool_tree::ResolvedNode; +use super::page_pruner::PagePruneMetrics; +use super::page_pruner::PagePruner; +use super::row_selection::PositionMap; +use super::stream::RowGroupInfo; +use datafusion::arrow::buffer::Buffer; +use datafusion::physical_optimizer::pruning::PruningPredicate; +use std::collections::{HashMap, HashSet}; +use std::time::Instant; + +/// How a collector's doc-range is narrowed relative to page-pruning or +/// accumulator results. Shared by both the single-collector and +/// bitmap-tree evaluator paths. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CollectorCallStrategy { + /// Call collector once for the full `[min_doc, max_doc)` range. + /// One FFM call, simple. + FullRange, + /// Tighten to `[first_surviving, last_surviving)` before calling. + /// Skips leading/trailing dead ranges. One FFM call, never regresses. + TightenOuterBounds, + /// Call collector once per contiguous surviving range. Fewer docs + /// scanned per call but more FFM calls. Best when the collector is + /// expensive and pruning is heavy. + PageRangeSplit, +} + +/// Per-row-group bitset producer. Plugs into `IndexedStream`. +pub trait RowGroupBitsetSource: Send + Sync { + /// Build candidate[pre-scan] bitset for this RG. `None` = skip RG entirely. + fn prefetch_rg( + &self, + rg: &RowGroupInfo, + min_doc: i32, + max_doc: i32, + ) -> Result, String>; + + /// Produce exact per-batch `BooleanArray` mask for refinement-stage [post-scan] + /// filtering. + /// + /// - `rg_state` is the `context` returned by the last `prefetch_rg` for + /// this RG — evaluators downcast it to their own per-RG state type. + /// - `position_map` translates delivered batch-row indices to RG-relative + /// positions (identity under full-scan; non-trivial under + /// block-granular RowSelection). + /// - `None` = no refinement mask needed (e.g. `SingleCollectorEvaluator` + /// relies on DataFusion's own predicate pushdown, so the candidate + /// stage's RowSelection is authoritative). + fn on_batch_mask( + &self, + rg_state: &dyn Any, + rg_first_row: i64, + position_map: &PositionMap, + batch_offset: usize, + batch_len: usize, + batch: &RecordBatch, + ) -> Result, String>; + + /// Whether `IndexedStream` should build a post-decode `current_mask` from + /// candidate offsets on the full-scan strategy. `true` for evaluators + /// whose `on_batch_mask` returns `None` (e.g. `SingleCollectorEvaluator` — + /// candidates are the only per-row filter available post-decode). + /// `false` for evaluators whose `on_batch_mask` returns an exact refinement + /// mask (e.g. `TreeBitsetSource` — refinement is authoritative and would + /// ignore `current_mask` anyway). Default `true` keeps the current + /// behaviour for any future evaluator that forgets to override. + fn needs_row_mask(&self) -> bool { + true + } + + /// Whether this evaluator requires parquet's `with_predicate` pushdown + /// to be OFF. `true` when the evaluator applies its own refinement in + /// `on_batch_mask` over the full delivered batch (using `PositionMap` + /// for Collector lookups) — pushdown would drop rows mid-decode and + /// misalign indices. + /// + /// Default `false`: pushdown decided by the stream's base policy. + /// Overridden to `true` by evaluators that must see the complete + /// RowSelection-delivered rowset (e.g. + /// `SingleCollectorEvaluator` when it owns the residual filter in + /// `on_batch_mask`, or `TreeBitsetSource` which always refines). + fn forbid_parquet_pushdown(&self) -> bool { + false + } +} + +/// Output of `prefetch_rg`. +pub struct PrefetchedRg { + /// Candidate doc-id bitmap, RG-relative (bit 0 = first row of the RG + /// doc range). `IndexedStream` converts this to a `RowSelection` using + /// `min_skip_run` and keeps the matching `PositionMap` alongside for + /// post-decode alignment. + pub candidates: RoaringBitmap, + /// Time spent producing the bitset (nanoseconds). For metrics. + pub eval_nanos: u64, + /// Opaque per-RG state threaded to `on_batch_mask` via `rg_state: &dyn Any`. + /// Evaluators downcast to their own concrete type. + pub context: Box, + /// Optional: pre-built Arrow `Buffer` holding `candidates` in + /// Arrow's native LSB-first bit layout, length = rg_num_rows. When + /// `Some`, `IndexedStream::build_mask` wraps a `BooleanBuffer` view + /// over this buffer (zero-copy) instead of rematerialising from the + /// `RoaringBitmap`. Set by evaluators that already produced the + /// packed bits internally (e.g. `SingleCollectorEvaluator`). + pub mask_buffer: Option, +} + +impl PrefetchedRg { + /// Helper for evaluators with no per-RG state (e.g. the single-collector + /// path, which doesn't do refinement [post-scan]). + pub fn without_context(candidates: RoaringBitmap, eval_nanos: u64) -> Self { + Self { + candidates, + eval_nanos, + context: Box::new(()), + mask_buffer: None, + } + } +} + +/// Multi-filter tree path: pluggable tree evaluator + leaf bitmap source +/// +/// Context for evaluating a tree against one row group. +#[derive(Debug, Clone)] +pub struct RgEvalContext { + pub rg_idx: usize, + pub rg_first_row: i64, + pub rg_num_rows: i64, + pub min_doc: i32, + pub max_doc: i32, + /// Candidate-stage leaf-reorder cost for `ResolvedNode::Predicate`. + /// Plumbed from `DatafusionQueryConfig`; read on the hot path. + pub cost_predicate: u32, + /// Candidate-stage leaf-reorder cost for `ResolvedNode::Collector`. + pub cost_collector: u32, + /// Narrowed doc-id ranges for Collector FFM calls. Computed by the + /// AND evaluator from the accumulator bitmap after earlier children + /// shrink the candidate set. + /// `None` = no narrowing (use full `[min_doc, max_doc)`). + /// `Some(ranges)` = call collector once per range. + pub collector_call_ranges: Option>, + /// Controls how the AND evaluator narrows collector ranges from the + /// accumulator bitmap. + pub collector_strategy: CollectorCallStrategy, +} + +/// Candidate-stage output of a `TreeEvaluator`. `candidates` is a superset +/// bitmap of doc IDs relative to `ctx.min_doc`; `per_leaf` maps leaf +/// identity (implementation-defined — pointer or index) to that leaf's +/// bitmap in the same domain, which the refinement stage looks up per +/// batch. +pub struct TreePrefetch { + pub candidates: RoaringBitmap, + pub per_leaf: Vec<(usize, RoaringBitmap)>, + /// Anchor doc ID (same as `ctx.min_doc` at prefetch time) so the + /// refinement stage can convert batch offsets to doc IDs. + pub min_doc: i32, +} + +/// Produces per-leaf bitmaps for one row group. +/// +/// Identified by DFS index in `tree`. Bitmap domain is `[ctx.min_doc, ctx.max_doc)`. +pub trait LeafBitmapSource: Send + Sync { + fn leaf_bitmap( + &self, + tree: &ResolvedNode, + leaf_dfs_index: usize, + ctx: &RgEvalContext, + ) -> Result; +} + +/// Pluggable tree-evaluation strategy. The algorithm that walks the tree, +/// combines per-leaf bitmaps, produces candidates + per-batch masks. +pub trait TreeEvaluator: Send + Sync { + /// Candidate stage: walk the tree for one row group and produce a + /// superset RoaringBitmap of candidate doc IDs plus the per-leaf + /// bitmap side-table that the refinement stage will read. + /// + /// `pruning_predicates` maps each `Predicate(expr)` leaf (keyed by + /// its + /// `Arc::as_ptr` identity) to a pre-built `PruningPredicate`. Empty + /// map = no page-level predicate pruning; each Predicate leaf falls + /// back to "every row is a candidate" (safe, identity for the + /// candidate stage). + fn prefetch( + &self, + tree: &ResolvedNode, + ctx: &RgEvalContext, + leaves: &dyn LeafBitmapSource, + page_pruner: &PagePruner, + pruning_predicates: &HashMap>, + page_prune_metrics: Option<&PagePruneMetrics>, + ) -> Result; + + /// Refinement stage: produce the exact per-row `BooleanArray` for one + /// record batch, consuming the candidate-stage `state` for the RG this + /// batch belongs to. + /// + /// `position_map` translates delivered batch-row index to RG-relative + /// position (identity under full-scan; non-trivial under block-granular + /// RowSelection). `batch_offset` is the delivered-row index of the + /// first row in this batch. + fn on_batch( + &self, + tree: &ResolvedNode, + state: &TreePrefetch, + batch: &RecordBatch, + rg_first_row: i64, + position_map: &PositionMap, + batch_offset: usize, + batch_len: usize, + ) -> Result; +} + +/// Composes a `TreeEvaluator` + `LeafBitmapSource` + `PagePruner` + resolved +/// tree into a `RowGroupBitsetSource`. +/// +/// Usage: +/// ```ignore +/// let source = TreeBitsetSource { +/// tree: Arc::new(resolved), +/// evaluator: Arc::new(BitmapTreeEvaluator), // or JavaTreeEvaluator +/// leaves: Arc::new(CollectorLeafBitmaps::without_metrics()), // or ParquetStatsLeaves +/// page_pruner: Arc::new(pruner), +/// }; +/// ``` +/// +/// # Batch projection requirement +/// +/// The refinement stage evaluates `Predicate` leaves via Arrow cmp kernels +/// on the current `RecordBatch`. Every column referenced by a +/// `ResolvedNode::Predicate` in the tree **must be present in the batch** +/// at eval time, i.e. the physical plan's projection must include +/// predicate columns, not just the final +/// SELECT list. In production, substrait plans emitted by the planner project +/// predicate columns as part of the filter node, so this is naturally +/// satisfied. Test harnesses that bypass substrait and select only output +/// columns must explicitly expand the SELECT to include predicate columns. +pub struct TreeBitsetSource { + pub tree: Arc, + pub evaluator: Arc, + pub leaves: Arc, + pub page_pruner: Arc, + /// Pre-extracted from `DatafusionQueryConfig` at source-construction + /// time so `prefetch_rg` doesn't need an `Arc` deref on the hot path. + pub cost_predicate: u32, + pub cost_collector: u32, + /// Max number of Collector leaves whose bitmaps are produced in + /// parallel per RG prefetch. 1 = sequential (preserves short-circuit + /// savings). Higher values trade short-circuit savings for latency + /// reduction on multi-collector trees; bounded by caller's config. + pub max_collector_parallelism: usize, + /// Per-predicate `PruningPredicate` cache, keyed by + /// `Arc::as_ptr(resolved_predicate) as usize`. Built once per query at + /// dispatch time by the caller. Empty = page-level predicate pruning + /// disabled (the tree path still works, each Predicate leaf falls + /// back to "every row is a candidate"). + pub pruning_predicates: Arc>>, + /// Counters recorded by `page_pruner.prune_rg` at each Predicate + /// leaf in the tree walk. Populated from the stream's + /// `PartitionMetrics` at dispatch time. + pub page_prune_metrics: Option, + /// Controls how the AND evaluator narrows collector doc ranges. + /// `TightenOuterBounds` (default) uses a single `[min, max)` range. + /// `FullRange` disables narrowing. `PageRangeSplit` is not + /// recommended here — multiple FFM calls per collector per RG can + /// be expensive in multi-collector trees. + pub collector_strategy: CollectorCallStrategy, +} + +impl RowGroupBitsetSource for TreeBitsetSource { + fn prefetch_rg( + &self, + rg: &RowGroupInfo, + min_doc: i32, + max_doc: i32, + ) -> Result, String> { + let t = Instant::now(); + let ctx = RgEvalContext { + rg_idx: rg.index, + rg_first_row: rg.first_row, + rg_num_rows: rg.num_rows, + min_doc, + max_doc, + cost_predicate: self.cost_predicate, + cost_collector: self.cost_collector, + collector_call_ranges: None, + collector_strategy: self.collector_strategy, + }; + + // Optional: materialise all Collector leaves in parallel before + // running the tree walk. Preserves correctness; sacrifices AND/OR + // short-circuit savings (all collectors run even if an earlier + // AND child already emptied the accumulator). Governed by + // `max_collector_parallelism`: 1 = sequential (today). + let precomputed = if self.max_collector_parallelism > 1 { + Some(precompute_collector_leaves( + &self.tree, + &ctx, + &*self.leaves, + self.max_collector_parallelism, + )?) + } else { + None + }; + + // Use the precomputed cache as the LeafBitmapSource if present; + // otherwise delegate directly to the original source (sequential). + let leaves_ref: &dyn LeafBitmapSource = match &precomputed { + Some(c) => c, + None => &*self.leaves, + }; + + let prefetch = self + .evaluator + .prefetch( + &self.tree, + &ctx, + leaves_ref, + &self.page_pruner, + &self.pruning_predicates, + // Don't pass metrics here — per-leaf prune_rg calls would + // inflate counts. We compute final page-level metrics below + // after the bitmap tree is fully resolved. + None, + ) + .map_err(|e| format!("TreeBitsetSource::prefetch_rg(rg={}): {}", rg.index, e))?; + if prefetch.candidates.is_empty() { + // All candidates pruned — record that every page was pruned. + if let Some(ref m) = self.page_prune_metrics { + if let Some(page_row_counts) = self.page_pruner.page_row_counts(rg.index) { + let num_pages = page_row_counts.len(); + if let Some(ref c) = m.pages_total { + c.add(num_pages); + } + if let Some(ref c) = m.pages_pruned { + c.add(num_pages); + } + } + } + return Ok(None); + } + // `prefetch.candidates` is in min_doc-relative space [0, max_doc - min_doc). + // `PrefetchedRg.candidates` is in RG-relative space [0, rg.num_rows). + // anchor = (min_doc - rg.first_row) shifts each relative bit. + // + // Fast path: if `anchor == 0`, clone directly — no shift + // needed. Otherwise walk the source in sorted order and + // coalesce consecutive bits into `insert_range` calls so we + // get one O(log n) call per run instead of O(1) per bit. + let anchor = (min_doc as i64) - rg.first_row; + let rg_candidates = if anchor == 0 { + prefetch.candidates.clone() + } else { + let mut rg_candidates = RoaringBitmap::new(); + let mut run_start: Option = None; + let mut run_end: u32 = 0; // inclusive + let mut flush = |bm: &mut RoaringBitmap, start: u32, end_inclusive: u32| { + // Range API is half-open; end_inclusive+1 handles the + // edge case at u32::MAX via saturating add (roaring + // clamps at u32::MAX internally). + let end = end_inclusive.saturating_add(1); + bm.insert_range(start..end); + }; + for rel in prefetch.candidates.iter() { + let shifted = rel as i64 + anchor; + if shifted < 0 || shifted > u32::MAX as i64 { + continue; + } + let v = shifted as u32; + match run_start { + None => { + run_start = Some(v); + run_end = v; + } + Some(_) if v == run_end + 1 => { + run_end = v; + } + Some(s) => { + flush(&mut rg_candidates, s, run_end); + run_start = Some(v); + run_end = v; + } + } + } + if let Some(s) = run_start { + flush(&mut rg_candidates, s, run_end); + } + rg_candidates + }; + + // Compute final page-level pruning metrics from the resolved + // bitmap. A page is "pruned" if zero candidate bits fall within + // its row range; "kept" otherwise. This reflects the actual + // page-level decision after AND/OR/NOT combination, not the + // per-leaf intermediate results. + if let Some(ref m) = self.page_prune_metrics { + if let Some(page_row_counts) = self.page_pruner.page_row_counts(rg.index) { + let num_pages = page_row_counts.len(); + let mut pruned = 0usize; + let mut row_offset = 0u32; + for &count in &page_row_counts { + let page_end = row_offset + count as u32; + if rg_candidates.range(row_offset..page_end).next().is_none() { + pruned += 1; + } + row_offset = page_end; + } + if let Some(ref c) = m.pages_total { + c.add(num_pages); + } + if let Some(ref c) = m.pages_pruned { + c.add(pruned); + } + } + } + + Ok(Some(PrefetchedRg { + candidates: rg_candidates, + eval_nanos: t.elapsed().as_nanos() as u64, + context: Box::new(prefetch), + mask_buffer: None, + })) + } + + fn on_batch_mask( + &self, + rg_state: &dyn Any, + rg_first_row: i64, + position_map: &PositionMap, + batch_offset: usize, + batch_len: usize, + batch: &RecordBatch, + ) -> Result, String> { + let state = rg_state.downcast_ref::().ok_or_else(|| { + "TreeBitsetSource::on_batch_mask: rg_state is not TreePrefetch".to_string() + })?; + let mask = self.evaluator.on_batch( + &self.tree, + state, + batch, + rg_first_row, + position_map, + batch_offset, + batch_len, + )?; + Ok(Some(mask)) + } + + /// `TreeBitsetSource` always returns `Some(mask)` from `on_batch_mask` — + /// the refinement mask is the exact per-row answer. `finalize_batch` + /// ignores `current_mask` in that branch, so building it from candidates + /// is wasted work. + fn needs_row_mask(&self) -> bool { + false + } + + /// BitmapTree walks the BoolNode in `on_batch_mask` using + /// `PositionMap` for Collector lookups. If parquet's pushdown + /// dropped rows mid-decode, our delivered batch would have a + /// different size than the PositionMap expects, causing + /// misaligned Collector lookups. Plus, the pushdown predicate + /// (if any reached us via `scan(filters)`) could contain the + /// `index_filter(...)` UDF marker whose body panics. + /// + /// So: always forbid parquet pushdown for BitmapTree. Phase 2 + /// will do the actual filter and produce filtered values. + fn forbid_parquet_pushdown(&self) -> bool { + true + } +} + +/// LeafBitmapSource that serves from a pre-populated map keyed by +/// `Arc::as_ptr(collector)`. Falls back to the inner source for leaves +/// not in the map (shouldn't happen in practice — we populate the map +/// with every Collector leaf in the tree before invoking the evaluator). +struct PrecomputedLeafCache<'a> { + map: HashMap, + fallback: &'a dyn LeafBitmapSource, +} + +impl<'a> LeafBitmapSource for PrecomputedLeafCache<'a> { + fn leaf_bitmap( + &self, + tree: &ResolvedNode, + leaf_dfs_index: usize, + ctx: &RgEvalContext, + ) -> Result { + if let ResolvedNode::Collector { collector, .. } = tree { + let key = Arc::as_ptr(collector) as *const () as usize; + if let Some(bm) = self.map.get(&key) { + return Ok(bm.clone()); + } + } + self.fallback.leaf_bitmap(tree, leaf_dfs_index, ctx) + } +} + +/// Walk the resolved tree and collect (key, collector-node-reference) +/// pairs for every Collector leaf, in DFS order (matching the +/// evaluator's walk order — we don't care about order beyond determinism). +/// Duplicates (same Arc pointing at the same collector instance) are +/// deduplicated by `Arc::as_ptr` so we don't call Lucene twice for the +/// same leaf. +fn collect_unique_collector_nodes<'a>( + node: &'a ResolvedNode, + out: &mut Vec<(usize, &'a ResolvedNode)>, + seen: &mut HashSet, +) { + match node { + ResolvedNode::And(children) | ResolvedNode::Or(children) => { + for c in children { + collect_unique_collector_nodes(c, out, seen); + } + } + ResolvedNode::Not(c) => collect_unique_collector_nodes(c, out, seen), + ResolvedNode::Collector { collector, .. } => { + let key = Arc::as_ptr(collector) as *const () as usize; + if seen.insert(key) { + out.push((key, node)); + } + } + ResolvedNode::Predicate(_) => {} + } +} + +/// Materialise all Collector leaves of `tree` by running their +/// `LeafBitmapSource::leaf_bitmap` calls in parallel via `std::thread::scope`, +/// bounded by `max_parallel`. Returns a cache keyed by `Arc::as_ptr(collector)`. +/// +/// Uses an `Arc`-driven round-robin over pre-spawned worker +/// threads so we never exceed `max_parallel` concurrent Lucene calls. +/// On error, returns the first error encountered. +fn precompute_collector_leaves<'a>( + tree: &'a ResolvedNode, + ctx: &RgEvalContext, + leaves: &'a dyn LeafBitmapSource, + max_parallel: usize, +) -> Result, String> { + let mut collectors: Vec<(usize, &ResolvedNode)> = Vec::new(); + let mut seen = HashSet::new(); + collect_unique_collector_nodes(tree, &mut collectors, &mut seen); + + // Zero or one collector → no benefit from parallelism, fall back to + // an empty cache (evaluator will use the fallback synchronously). + if collectors.len() <= 1 { + return Ok(PrecomputedLeafCache { + map: HashMap::new(), + fallback: leaves, + }); + } + + let n = collectors.len(); + let parallel = max_parallel.min(n).max(1); + + // Bounded parallelism via std::thread::scope + a work queue Mutex. + // Each worker pulls the next collector to evaluate, calls + // leaf_bitmap, writes result into a shared Vec>> + // at the collector's index. + let mut results: Vec>> = (0..n).map(|_| None).collect(); + let next_idx = std::sync::atomic::AtomicUsize::new(0); + let results_mutex = std::sync::Mutex::new(&mut results); + + std::thread::scope(|scope| { + let mut handles = Vec::with_capacity(parallel); + for _worker in 0..parallel { + let collectors_ref = &collectors; + let leaves_ref = leaves; + let ctx_ref = ctx; + let next_idx_ref = &next_idx; + let results_mutex_ref = &results_mutex; + handles.push(scope.spawn(move || { + loop { + let i = next_idx_ref.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if i >= collectors_ref.len() { + break; + } + let (_key, node) = collectors_ref[i]; + // Use i as the leaf_dfs_index — the cache doesn't + // use it for lookup (keys by Arc::as_ptr), so any + // stable value works. + let result = leaves_ref.leaf_bitmap(node, i, ctx_ref); + let mut guard = results_mutex_ref.lock().unwrap(); + guard[i] = Some(result); + } + })); + } + // Scope ensures all threads complete before returning. + for h in handles { + let _ = h.join(); + } + }); + + // Assemble results. Fail fast on the first error. + let mut map = HashMap::with_capacity(n); + for (i, slot) in results.into_iter().enumerate() { + let bm = + slot.ok_or_else(|| format!("precompute: worker did not populate slot {}", i))??; + map.insert(collectors[i].0, bm); + } + + Ok(PrecomputedLeafCache { + map, + fallback: leaves, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::indexed_table::bool_tree::ResolvedNode; + use crate::indexed_table::index::RowGroupDocsCollector; + use crate::indexed_table::page_pruner::PagePruner; + use datafusion::arrow::array::Int32Array; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::arrow::record_batch::RecordBatch; + use datafusion::parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; + use datafusion::parquet::arrow::ArrowWriter; + + fn empty_pruner() -> Arc { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![0i32; 4]))], + ) + .unwrap(); + let tmp = tempfile::NamedTempFile::new().unwrap(); + let mut writer = ArrowWriter::try_new(tmp.reopen().unwrap(), schema.clone(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + let meta = ArrowReaderMetadata::load( + &tmp.reopen().unwrap(), + ArrowReaderOptions::new().with_page_index(true), + ) + .unwrap(); + Arc::new(PagePruner::new(meta.schema(), meta.metadata().clone())) + } + + /// Leaf source that returns empty bitmaps — enough to compose a + /// TreeBitsetSource purely for testing its `needs_row_mask` override. + struct NoopLeaves; + impl LeafBitmapSource for NoopLeaves { + fn leaf_bitmap( + &self, + _tree: &ResolvedNode, + _idx: usize, + _ctx: &RgEvalContext, + ) -> Result { + Ok(roaring::RoaringBitmap::new()) + } + } + + /// Evaluator that mirrors the shape of BitmapTreeEvaluator for the trait + /// needs_row_mask test (we don't import BitmapTreeEvaluator here to avoid + /// a circular dependency with the bitmap_tree module's own tests). + struct NoopTreeEvaluator; + impl TreeEvaluator for NoopTreeEvaluator { + fn prefetch( + &self, + _tree: &ResolvedNode, + _ctx: &RgEvalContext, + _leaves: &dyn LeafBitmapSource, + _page_pruner: &PagePruner, + _pruning_predicates: &HashMap>, + _page_prune_metrics: Option<&PagePruneMetrics>, + ) -> Result { + Ok(TreePrefetch { + candidates: roaring::RoaringBitmap::new(), + per_leaf: Vec::new(), + min_doc: 0, + }) + } + fn on_batch( + &self, + _tree: &ResolvedNode, + _state: &TreePrefetch, + _batch: &RecordBatch, + _rg_first_row: i64, + _position_map: &PositionMap, + _batch_offset: usize, + batch_len: usize, + ) -> Result { + Ok(BooleanArray::from(vec![false; batch_len])) + } + } + + #[test] + fn tree_bitset_source_does_not_need_row_mask() { + // `TreeBitsetSource::on_batch_mask` returns `Some(refinement_mask)`. + // `finalize_batch` ignores `current_mask` in that branch, so + // `IndexedStream` should skip building it. + + #[derive(Debug)] + struct Dummy; + impl RowGroupDocsCollector for Dummy { + fn collect_packed_u64_bitset(&self, _: i32, _: i32) -> Result, String> { + Ok(vec![]) + } + } + let source = TreeBitsetSource { + tree: Arc::new(ResolvedNode::Collector { + provider_key: 0, + collector: Arc::new(Dummy), + }), + evaluator: Arc::new(NoopTreeEvaluator), + leaves: Arc::new(NoopLeaves), + page_pruner: empty_pruner(), + cost_predicate: 1, + cost_collector: 10, + max_collector_parallelism: 1, + pruning_predicates: std::sync::Arc::new(HashMap::new()), + page_prune_metrics: None, + collector_strategy: CollectorCallStrategy::TightenOuterBounds, + }; + assert!(!source.needs_row_mask()); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/single_collector.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/single_collector.rs new file mode 100644 index 0000000000000..92eefa73739f9 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/eval/single_collector.rs @@ -0,0 +1,549 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Single-collector evaluator — one backend collector plus DataFusion for +//! residual predicates. +//! +//! When the filter has exactly one `index_filter(...)` call AND'd with +//! (possibly zero, one, or many) parquet-native predicates, this evaluator +//! runs. Per RG: +//! +//! 1. Call the single collector → bitset. +//! 2. Apply page pruning (AND/OR mode depending on how the query combined them). +//! 3. Hand the bitset offsets to `IndexedStream` as a RowSelection. +//! 4. `on_batch_mask` returns `None` — DataFusion's +//! `with_predicate(residual).with_pushdown_filters(true)` applies the +//! residual predicates during decode, so indices stay aligned and no +//! post-filtering is needed. + +use std::sync::Arc; + +use datafusion::arrow::array::BooleanArray; +use datafusion::arrow::record_batch::RecordBatch; +use roaring::RoaringBitmap; + +use super::{PrefetchedRg, RowGroupBitsetSource}; +use crate::indexed_table::index::RowGroupDocsCollector; +use crate::indexed_table::page_pruner::{PagePruneMetrics, PagePruner}; +use crate::indexed_table::row_selection::{ + bitmap_to_packed_bits, packed_bits_to_boolean_array, row_selection_to_bitmap, PositionMap, +}; +use datafusion::physical_optimizer::pruning::PruningPredicate; +use std::time::Instant; + +/// Re-exported from parent module for backward compatibility. +pub use super::CollectorCallStrategy; +use crate::indexed_table::stream::RowGroupInfo; + +/// Per-RG state the evaluator keeps for refinement. In row-granular +/// mode parquet narrowed fully via `with_predicate` + `RowSelection` +/// and nothing is needed here. In block-granular mode we need the +/// Collector candidate bitmap to build a post-decode mask. +/// +/// `mask_buffer` is the candidate bitmap in Arrow's native LSB-first bit +/// layout, wrapped as a refcounted `Buffer`. Sharing an `Arc` lets +/// `on_batch_mask` and `build_mask` build zero-copy `BooleanBuffer` +/// views via `BooleanBuffer::new(buf.clone(), bit_offset, bit_len)`. +/// Length of the underlying buffer covers `mask_len` bits (= rg_num_rows). +struct SingleCollectorState { + candidates: RoaringBitmap, + mask_buffer: datafusion::arrow::buffer::Buffer, + mask_len: usize, +} + +/// Evaluator holding one collector and applying per-RG page pruning. +/// +/// Always AND-intersects the collector bitmap with page pruning. The +/// `BitsetMode::Or` branch that previously existed was never emitted by +/// the classifier (reserved for a future `OR(Collector, predicates)` +/// extension) and has been removed; an OR-between-Collector-and-predicates +/// shape routes to the multi-filter tree path today. +pub struct SingleCollectorEvaluator { + collector: Arc, + page_pruner: Arc, + /// Residual pruning predicate: the non-Collector portion of the + /// top-level AND, translated to a `PruningPredicate`. `None` means + /// no residual predicate applies (nothing to prune with). + pruning_predicate: Option>, + /// Raw residual expression (non-Collector children of the top-level + /// AND, converted to a single `PhysicalExpr`). + /// + /// Used in two modes: + /// + /// - **Row-granular** (`min_skip_run = 1`): the same expression is + /// stashed on `IndexedTableConfig.pushdown_predicate` and handed + /// to parquet's `with_predicate` for decode-time filtering. + /// Combined with the Collector-bitmap `RowSelection`, parquet + /// delivers exact `Collector ∧ residual` rows. `on_batch_mask` + /// returns `None` (nothing left to do). + /// + /// - **Block-granular** (`min_skip_run > 1`): pushdown is OFF + /// (alignment risk with coalesced selection). `on_batch_mask` + /// evaluates this expression against the decoded batch and + /// AND-combines with the Collector bitmap mask to produce the + /// exact result. + residual_expr: Option>, + /// Counters recorded by `page_pruner.prune_rg`. Built from the + /// stream's `PartitionMetrics` at evaluator construction. + page_prune_metrics: Option, + /// Incremented once per `prefetch_rg` call (once per RG) — the + /// Collector path always performs one FFM round-trip to Java. + ffm_collector_calls: Option, + call_strategy: CollectorCallStrategy, +} + +impl SingleCollectorEvaluator { + pub fn new( + collector: Arc, + page_pruner: Arc, + pruning_predicate: Option>, + residual_expr: Option>, + page_prune_metrics: Option, + ffm_collector_calls: Option, + call_strategy: CollectorCallStrategy, + ) -> Self { + Self { + collector, + page_pruner, + pruning_predicate, + residual_expr, + page_prune_metrics, + ffm_collector_calls, + call_strategy, + } + } +} + +impl RowGroupBitsetSource for SingleCollectorEvaluator { + fn prefetch_rg( + &self, + rg: &RowGroupInfo, + min_doc: i32, + max_doc: i32, + ) -> Result, String> { + let t = Instant::now(); + + // Page-prune to discover which row ranges survive. + let page_ranges: Option> = self.pruning_predicate.as_ref().and_then(|pp| { + self.page_pruner + .prune_rg(pp, rg.index, self.page_prune_metrics.as_ref()) + .map(|sel| { + let mut ranges = Vec::new(); + let mut rg_pos: i64 = 0; + for s in sel.iter() { + if s.skip { + rg_pos += s.row_count as i64; + } else { + let abs_min = min_doc + rg_pos as i32; + let abs_max = min_doc + rg_pos as i32 + s.row_count as i32; + ranges.push((abs_min, abs_max)); + rg_pos += s.row_count as i64; + } + } + ranges + }) + }); + + // Dispatch collector call strategy. + let call_ranges: Vec<(i32, i32)> = match self.call_strategy { + CollectorCallStrategy::FullRange => vec![(min_doc, max_doc)], + CollectorCallStrategy::TightenOuterBounds => match &page_ranges { + Some(r) if r.is_empty() => return Ok(None), + Some(r) => vec![(r.first().unwrap().0, r.last().unwrap().1)], + None => vec![(min_doc, max_doc)], + }, + CollectorCallStrategy::PageRangeSplit => match &page_ranges { + Some(r) if r.is_empty() => return Ok(None), + Some(r) => r.clone(), + None => vec![(min_doc, max_doc)], + }, + }; + + // Call collector for each range, merge into one RG-relative bitmap. + let mut candidates = RoaringBitmap::new(); + for (r_min, r_max) in &call_ranges { + let bitset = self + .collector + .collect_packed_u64_bitset(*r_min, *r_max) + .map_err(|e| { + format!( + "collector.collect_packed_u64_bitset(rg={}, [{}, {})): {}", + rg.index, r_min, r_max, e + ) + })?; + if let Some(ref c) = self.ffm_collector_calls { + c.add(1); + } + let offset = (*r_min as i64 - rg.first_row) as u32; + let num_docs = (*r_max - *r_min) as u32; + let bytes: &[u8] = unsafe { + std::slice::from_raw_parts(bitset.as_ptr() as *const u8, bitset.len() * 8) + }; + let mut chunk = RoaringBitmap::from_lsb0_bytes(offset, bytes); + let upper = offset.saturating_add(num_docs); + if upper < u32::MAX { + chunk.remove_range(upper..); + } + candidates |= chunk; + } + + // For FullRange and TightenOuterBounds, AND with page bitmap + // to remove rows in dead pages that the collector scanned. + if self.call_strategy != CollectorCallStrategy::PageRangeSplit { + if let Some(ref ranges) = page_ranges { + let mut allowed = RoaringBitmap::new(); + for (r_min, r_max) in ranges { + let lo = (*r_min as i64 - rg.first_row) as u32; + let hi = (*r_max as i64 - rg.first_row) as u32; + allowed.insert_range(lo..hi); + } + candidates &= allowed; + } + } + + if candidates.is_empty() { + return Ok(None); + } + + // Materialise the final RG-relative bitmap as an Arrow `Buffer` + // in Arrow's native LSB-first layout. This is the ONLY + // representation the hot paths (`on_batch_mask`, `build_mask`) + // need; they construct zero-copy `BooleanBuffer` views via + // `BooleanBuffer::new(buf.clone(), bit_offset, bit_len)`. + let mask_len = rg.num_rows as usize; + let packed_bits = bitmap_to_packed_bits(&candidates, mask_len as u32); + let mask_buffer = datafusion::arrow::buffer::Buffer::from_vec(packed_bits); + Ok(Some(PrefetchedRg { + candidates: candidates.clone(), + eval_nanos: t.elapsed().as_nanos() as u64, + context: Box::new(SingleCollectorState { + candidates, + mask_buffer: mask_buffer.clone(), + mask_len, + }), + mask_buffer: Some(mask_buffer), + })) + } + + fn on_batch_mask( + &self, + rg_state: &dyn std::any::Any, + _rg_first_row: i64, + position_map: &PositionMap, + batch_offset: usize, + batch_len: usize, + batch: &RecordBatch, + ) -> Result, String> { + // No residual → no post-decode work. Stream's current_mask + // (if built) handles Collector narrowing. + let Some(ref residual) = self.residual_expr else { + return Ok(None); + }; + // Apply Collector bitmap AND residual predicate over the + // delivered batch. In row-granular mode (pushdown ON) this + // re-applies what parquet already did — redundant but correct. + // In block-granular mode (pushdown OFF) this is the only + // place the residual gets applied. + let state = rg_state + .downcast_ref::() + .ok_or_else(|| { + "SingleCollectorEvaluator: rg_state is not SingleCollectorState".to_string() + })?; + + // Build Collector mask over delivered rows via PositionMap. + // All paths produce a `BooleanArray` whose underlying + // `Buffer` is a refcounted view into `state.mask_buffer` — + // zero allocation for Identity, at most one small packed + // Vec for Runs. + let collector_mask: BooleanArray = match position_map { + // Identity: delivered row i == rg_position (batch_offset + i). + // BooleanBuffer::new adjusts bit_offset without copying the + // underlying Buffer. The returned BooleanArray points into + // state.mask_buffer; lifecycle is Arc-managed. + PositionMap::Identity { .. } => { + let bb = datafusion::arrow::buffer::BooleanBuffer::new( + state.mask_buffer.clone(), + batch_offset, + batch_len, + ); + BooleanArray::new(bb, None) + } + // Every delivered row is by construction a candidate — mask is all-true. + PositionMap::Bitmap { .. } => BooleanArray::new( + datafusion::arrow::buffer::BooleanBuffer::new_set(batch_len), + None, + ), + // Runs: gather per-row bit from the shared mask_buffer into + // a new packed Vec (small — bounded by batch_len/64). + PositionMap::Runs { .. } => { + let words = batch_len.div_ceil(64); + let mut out = vec![0u64; words]; + let src_bytes = state.mask_buffer.as_slice(); + for i in 0..batch_len { + let delivered_idx = batch_offset + i; + let rg_pos = position_map.rg_position(delivered_idx).ok_or_else(|| { + format!( + "SingleCollectorEvaluator: delivered_idx {} out of range", + delivered_idx + ) + })?; + // Read bit rg_pos from the packed buffer (LSB-first). + let hit = rg_pos < state.mask_len + && (src_bytes[rg_pos >> 3] >> (rg_pos & 7)) & 1 == 1; + if hit { + out[i >> 6] |= 1u64 << (i & 63); + } + } + packed_bits_to_boolean_array(out, batch_len) + } + }; + + // Evaluate residual against the batch. The residual may use + // full-schema column indices; remap to batch positions by name. + let remapped_residual = remap_expr_to_batch(residual, batch) + .map_err(|e| format!("SingleCollectorEvaluator: remap residual: {}", e))?; + let residual_value = remapped_residual + .evaluate(batch) + .map_err(|e| format!("SingleCollectorEvaluator: residual.evaluate: {}", e))?; + let residual_array = residual_value + .into_array(batch_len) + .map_err(|e| format!("SingleCollectorEvaluator: residual into_array: {}", e))?; + let residual_mask = residual_array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + "SingleCollectorEvaluator: residual did not produce BooleanArray".to_string() + })?; + + // AND with kleene semantics (NULL → exclude). + let combined = datafusion::arrow::compute::kernels::boolean::and_kleene( + &collector_mask, + residual_mask, + ) + .map_err(|e| format!("SingleCollectorEvaluator: and_kleene: {}", e))?; + Ok(Some(combined)) + } + + /// When we have a residual to apply in `on_batch_mask`, pushdown + /// must be OFF in **block-granular mode** because we use + /// `PositionMap` to look up RG positions over the full delivered + /// rowset — pushdown would drop rows and misalign. In + /// **row-granular mode** (`min_skip_run == 1`), pushdown is safe + /// and desirable: parquet applies the residual in lockstep with + /// decoding, `on_batch_mask` returns `None`, and output is + /// exact. But the evaluator doesn't know min_skip_run — the + /// stream does. The stream guards this via its + /// `alignment_risk = min_skip_run != 1 && needs_row_mask()` + /// check plus `forbid_parquet_pushdown`. We return `false` here + /// and rely on `needs_row_mask = true` (default when residual is + /// present) to trigger the stream's alignment guard in block + /// mode; in row-granular mode that guard is inactive and + /// pushdown proceeds. + fn forbid_parquet_pushdown(&self) -> bool { + false + } + + /// Stream's `current_mask` construction consults this. When + /// residual is set, we return `true` so the stream knows our + /// `on_batch_mask` uses PositionMap (alignment risk) — this flag + /// flips the stream's `alignment_risk` computation which + /// suppresses pushdown in block-granular mode. In row-granular + /// mode (min_skip_run == 1) the stream ignores this flag's + /// pushdown impact and pushes anyway (which is what we want: + /// parquet applies residual during decode of already-narrowed + /// rowset, on_batch_mask returns None below). + /// + /// Without residual, we return `true` too — stream builds + /// `current_mask` from Collector bitmap to narrow post-decode + /// (legacy path for SingleCollector without a residual wasn't + /// used in production but kept for defensive correctness). + fn needs_row_mask(&self) -> bool { + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::parquet::arrow::arrow_reader::ArrowReaderMetadata; + use datafusion::parquet::arrow::arrow_reader::ArrowReaderOptions; + use datafusion::parquet::arrow::ArrowWriter; + use std::fmt; + use std::sync::Arc; + use tempfile::NamedTempFile; + + /// Stub collector: returns a pre-defined set of doc IDs, encoded into + /// the bitset the trait contract requires. + #[derive(Debug)] + struct StubCollector { + docs: Vec, + } + + impl RowGroupDocsCollector for StubCollector { + fn collect_packed_u64_bitset( + &self, + min_doc: i32, + max_doc: i32, + ) -> Result, String> { + let span = (max_doc - min_doc) as usize; + let mut bitset = vec![0u64; (span + 63) / 64]; + for &doc in &self.docs { + if doc >= min_doc && doc < max_doc { + let idx = (doc - min_doc) as usize; + bitset[idx / 64] |= 1u64 << (idx % 64); + } + } + Ok(bitset) + } + } + + fn minimal_page_pruner() -> Arc { + // Build a 1-row-group parquet with no filters — page pruner becomes a no-op + // (filter_row_ids returns input, candidate_row_ids returns [first_row, first_row+num_rows)). + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let batch = datafusion::arrow::record_batch::RecordBatch::try_new( + schema.clone(), + vec![Arc::new(datafusion::arrow::array::Int32Array::from( + vec![0i32; 8], + ))], + ) + .unwrap(); + let tmp = NamedTempFile::new().unwrap(); + { + let mut writer = + ArrowWriter::try_new(tmp.reopen().unwrap(), schema.clone(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + } + let file = tmp.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index(true); + let meta = ArrowReaderMetadata::load(&file, options).unwrap(); + let pruner = PagePruner::new(meta.schema(), meta.metadata().clone()); + Arc::new(pruner) + } + + #[test] + fn path_b_and_mode_collects_docs_and_returns_offsets() { + let collector = Arc::new(StubCollector { + docs: vec![0, 3, 7], + }) as Arc; + let pruner = minimal_page_pruner(); + let eval = SingleCollectorEvaluator::new(collector, pruner, None, None, None, None, CollectorCallStrategy::FullRange); + + let rg = RowGroupInfo { + index: 0, + first_row: 0, + num_rows: 8, + }; + let prefetched = eval.prefetch_rg(&rg, 0, 8).unwrap().expect("has matches"); + let got: Vec = prefetched.candidates.iter().collect(); + assert_eq!(got, vec![0u32, 3, 7]); + } + + #[test] + fn on_batch_mask_returns_none_for_path_b() { + let collector = Arc::new(StubCollector { docs: vec![0] }) as Arc; + let pruner = minimal_page_pruner(); + let eval = SingleCollectorEvaluator::new(collector, pruner, None, None, None, None, CollectorCallStrategy::FullRange); + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let batch = datafusion::arrow::record_batch::RecordBatch::try_new( + schema, + vec![Arc::new(datafusion::arrow::array::Int32Array::from(vec![ + 1, 2, 3, + ]))], + ) + .unwrap(); + // Empty position map is fine; SingleCollectorEvaluator ignores it. + let pm = PositionMap::from_selection( + &datafusion::parquet::arrow::arrow_reader::RowSelection::from(Vec::< + datafusion::parquet::arrow::arrow_reader::RowSelector, + >::new()), + ); + assert!(eval + .on_batch_mask(&(), 0, &pm, 0, 3, &batch) + .unwrap() + .is_none()); + } + + #[test] + fn single_collector_needs_row_mask() { + // SingleCollectorEvaluator returns None from on_batch_mask, so + // IndexedStream must build current_mask from candidate offsets + // (it's the only post-decode filter we have on this path). + let collector = Arc::new(StubCollector { docs: vec![0] }) as Arc; + let pruner = minimal_page_pruner(); + let eval = SingleCollectorEvaluator::new(collector, pruner, None, None, None, None, CollectorCallStrategy::FullRange); + assert!(eval.needs_row_mask()); + } + + #[test] + fn empty_match_returns_none() { + let collector = Arc::new(StubCollector { docs: vec![] }) as Arc; + let pruner = minimal_page_pruner(); + let eval = SingleCollectorEvaluator::new(collector, pruner, None, None, None, None, CollectorCallStrategy::FullRange); + let rg = RowGroupInfo { + index: 0, + first_row: 0, + num_rows: 8, + }; + assert!(eval.prefetch_rg(&rg, 0, 8).unwrap().is_none()); + } + + #[test] + fn empty_pruning_predicates_leave_collector_unchanged() { + // With no pruning predicates, the evaluator is a pass-through for + // the collector bitmap: every doc the collector returns remains a + // candidate. (Contrast with the old BitsetMode::Or path, which + // would have unioned with page-pruner-derived "anything-allowed" + // row IDs — semantics that were never wired up in production.) + let collector = Arc::new(StubCollector { + docs: vec![0, 3, 7], + }) as Arc; + let pruner = minimal_page_pruner(); + let eval = SingleCollectorEvaluator::new(collector, pruner, None, None, None, None, CollectorCallStrategy::FullRange); + + let rg = RowGroupInfo { + index: 0, + first_row: 0, + num_rows: 8, + }; + let prefetched = eval.prefetch_rg(&rg, 0, 8).unwrap().expect("has matches"); + let got: Vec = prefetched.candidates.iter().collect(); + assert_eq!(got, vec![0u32, 3, 7]); + } + + // Keep the `fmt` import used + #[allow(dead_code)] + fn _use(_: &dyn fmt::Debug) {} +} + +/// Remap Column indices in a PhysicalExpr to match the batch schema by name. +fn remap_expr_to_batch( + expr: &Arc, + batch: &RecordBatch, +) -> Result, String> { + use datafusion::common::tree_node::TreeNode; + use datafusion::physical_expr::expressions::Column; + + expr.clone() + .transform(|e| { + if let Some(col) = e.as_any().downcast_ref::() { + if let Ok(new_idx) = batch.schema().index_of(col.name()) { + if new_idx != col.index() { + let remapped = Arc::new(Column::new(col.name(), new_idx)) + as Arc; + return Ok(datafusion::common::tree_node::Transformed::yes(remapped)); + } + } + } + Ok(datafusion::common::tree_node::Transformed::no(e)) + }) + .map(|t| t.data) + .map_err(|e| format!("remap_expr_to_batch: {}", e)) +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs new file mode 100644 index 0000000000000..3b418762f4be8 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs @@ -0,0 +1,222 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! FFM upcall surface for index-filter providers and collectors. +//! +//! Four callback slots, populated once at startup by +//! `df_register_filter_tree_callbacks` (see `ffm.rs`): +//! +//! - `createProvider(queryBytes, queryBytesLen) -> providerKey|-1` +//! - `createCollector(providerKey, segmentOrd, minDoc, maxDoc) -> collectorKey|-1` +//! - `collectDocs(collectorKey, minDoc, maxDoc, outBuf, outWordCap) -> wordsWritten|-1` +//! - `releaseCollector(collectorKey)` +//! - `releaseProvider(providerKey)` +//! +//! `ProviderHandle` and `FfmSegmentCollector` are the lifetime wrappers — +//! they call the release callbacks on drop. + +use std::sync::atomic::{AtomicPtr, Ordering}; + +use super::index::RowGroupDocsCollector; + +// ── Callback signatures ─────────────────────────────────────────────── + +type CreateProviderFn = unsafe extern "C" fn(*const u8, i64) -> i32; +type ReleaseProviderFn = unsafe extern "C" fn(i32); +type CreateCollectorFn = unsafe extern "C" fn(i32, i32, i32, i32) -> i32; +type CollectDocsFn = unsafe extern "C" fn(i32, i32, i32, *mut u64, i64) -> i64; +type ReleaseCollectorFn = unsafe extern "C" fn(i32); + +static CREATE_PROVIDER: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); +static RELEASE_PROVIDER: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); +static CREATE_COLLECTOR: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); +static COLLECT_DOCS: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); +static RELEASE_COLLECTOR: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); + +/// Registered by Java at startup. Stores function pointers into atomic +/// slots. Each call to this entry replaces the slots wholesale. +/// +/// Not annotated `#[ffm_safe]` because that macro is specific to the +/// `-> i64` error-pointer convention. We use a manual `catch_unwind` +/// instead, though the body (atomic stores) can't realistically panic. +#[no_mangle] +pub unsafe extern "C" fn df_register_filter_tree_callbacks( + create_provider: CreateProviderFn, + release_provider: ReleaseProviderFn, + create_collector: CreateCollectorFn, + collect_docs: CollectDocsFn, + release_collector: ReleaseCollectorFn, +) { + // catch_unwind is defense-in-depth: atomic stores shouldn't panic, + // but if they ever did (e.g. allocator OOM if we grew the atomics), + // unwinding across the FFM boundary is UB. Swallow the panic + // silently — there's no way to report it back to Java for a + // `-> ()` function. + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + CREATE_PROVIDER.store(create_provider as *mut (), Ordering::Release); + RELEASE_PROVIDER.store(release_provider as *mut (), Ordering::Release); + CREATE_COLLECTOR.store(create_collector as *mut (), Ordering::Release); + COLLECT_DOCS.store(collect_docs as *mut (), Ordering::Release); + RELEASE_COLLECTOR.store(release_collector as *mut (), Ordering::Release); + })); +} + +fn load_create_provider() -> Result { + let p = CREATE_PROVIDER.load(Ordering::Acquire); + if p.is_null() { + return Err("FilterTree callbacks not registered".into()); + } + Ok(unsafe { std::mem::transmute::<*mut (), CreateProviderFn>(p) }) +} +fn load_release_provider() -> Option { + let p = RELEASE_PROVIDER.load(Ordering::Acquire); + if p.is_null() { + None + } else { + Some(unsafe { std::mem::transmute::<*mut (), ReleaseProviderFn>(p) }) + } +} +fn load_create_collector() -> Result { + let p = CREATE_COLLECTOR.load(Ordering::Acquire); + if p.is_null() { + return Err("FilterTree callbacks not registered".into()); + } + Ok(unsafe { std::mem::transmute::<*mut (), CreateCollectorFn>(p) }) +} +fn load_collect_docs() -> Result { + let p = COLLECT_DOCS.load(Ordering::Acquire); + if p.is_null() { + return Err("FilterTree callbacks not registered".into()); + } + Ok(unsafe { std::mem::transmute::<*mut (), CollectDocsFn>(p) }) +} +fn load_release_collector() -> Option { + let p = RELEASE_COLLECTOR.load(Ordering::Acquire); + if p.is_null() { + None + } else { + Some(unsafe { std::mem::transmute::<*mut (), ReleaseCollectorFn>(p) }) + } +} + +// ── ProviderHandle — owns `releaseProvider` on drop ─────────────────── + +/// Returned from `create_provider`. Drop releases the provider. +pub struct ProviderHandle { + key: i32, +} + +impl ProviderHandle { + pub fn key(&self) -> i32 { + self.key + } +} + +impl std::fmt::Debug for ProviderHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProviderHandle") + .field("key", &self.key) + .finish() + } +} + +impl Drop for ProviderHandle { + fn drop(&mut self) { + if let Some(release) = load_release_provider() { + unsafe { release(self.key) }; + } + } +} + +/// Create a provider from serialized backend query bytes by upcalling Java. +pub fn create_provider(query_bytes: &[u8]) -> Result { + let create = load_create_provider()?; + let key = unsafe { create(query_bytes.as_ptr(), query_bytes.len() as i64) }; + if key < 0 { + return Err(format!( + "createProvider failed: len={} -> {}", + query_bytes.len(), + key + )); + } + Ok(ProviderHandle { key }) +} + +// ── FfmSegmentCollector — owns `releaseCollector` on drop ───────────── + +#[derive(Debug)] +pub struct FfmSegmentCollector { + key: i32, +} + +impl FfmSegmentCollector { + /// Ask Java for a collector keyed by `provider_key` for the given segment/doc range. + pub fn create( + provider_key: i32, + segment_ord: i32, + doc_min: i32, + doc_max: i32, + ) -> Result { + let create = load_create_collector()?; + let key = unsafe { create(provider_key, segment_ord, doc_min, doc_max) }; + if key < 0 { + return Err(format!( + "createCollector(provider={}, seg={}) failed: {}", + provider_key, segment_ord, key + )); + } + Ok(FfmSegmentCollector { key }) + } +} + +impl RowGroupDocsCollector for FfmSegmentCollector { + fn collect_packed_u64_bitset(&self, min_doc: i32, max_doc: i32) -> Result, String> { + if max_doc <= min_doc { + return Ok(Vec::new()); + } + let span = (max_doc - min_doc) as usize; + let word_count = span.div_ceil(64); + let mut buf = vec![0u64; word_count]; + let collect_fn = load_collect_docs()?; + let n = unsafe { + collect_fn( + self.key, + min_doc, + max_doc, + buf.as_mut_ptr(), + word_count as i64, + ) + }; + if n < 0 { + return Err(format!("collectDocs(key={}) failed: {}", self.key, n)); + } + // Defensive: the Java callback is contracted to return + // `wordsWritten <= outWordCap`. If it lied, the buffer already + // overflowed, but truncating won't recover the clobbered heap. + // Detect the violation and fail loudly so the Java callback bug + // is surfaced before downstream code consumes the tainted bitset. + let n = n as usize; + if n > word_count { + return Err(format!( + "collectDocs(key={}) reported wordsWritten={} > capacity={}; \ + callback contract violated (possible heap overflow)", + self.key, n, word_count, + )); + } + buf.truncate(n); + Ok(buf) + } +} + +impl Drop for FfmSegmentCollector { + fn drop(&mut self) { + if let Some(release) = load_release_collector() { + unsafe { release(self.key) }; + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/index.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/index.rs new file mode 100644 index 0000000000000..544acca702047 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/index.rs @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Core index traits — the contract between the index and the query engine. +//! +//! These traits are all `IndexedExec`/`IndexedStream` need. How the searcher +//! was created (FFM upcall to Java, in-process native index, test stub) is +//! irrelevant here. +//! +//! ```text +//! ShardSearcher (shard-scoped compiled query — once per shard) +//! └── RowGroupDocsCollector (per-segment matcher — once per segment) +//! └── collect_packed_u64_bitset(range) → Vec +//! ``` + +use std::fmt::Debug; +use std::sync::Arc; + +/// A collector that retrieves matching doc IDs as a packed bitset for a row +/// group's doc-id range within a segment. +/// +/// May be called multiple times with increasing ranges (forward-only iteration). +/// +/// # Bit layout contract +/// +/// [`collect_packed_u64_bitset`](Self::collect_packed_u64_bitset) returns a +/// word-packed bitset matching Lucene's `FixedBitSet.getBits()` exactly: +/// +/// - Word `j` covers the 64 doc-id-relative positions `j*64 .. (j+1)*64`. +/// - Bit `i` of word `j` (i.e. `word & (1u64 << i) != 0`) represents the +/// doc at relative position `j*64 + i`, i.e. absolute doc ID +/// `min_doc + j*64 + i`. +/// - Length is `ceil((max_doc - min_doc) / 64)` words. The last word may +/// have unused high bits set past `max_doc - min_doc`; consumers MUST +/// clamp by relative position before using a bit. +/// +/// # Empty-range contract +/// +/// If `max_doc <= min_doc`, implementations MUST return `Ok(Vec::new())` +/// (zero-length bitset). This is a no-op case and must not error. Callers +/// rely on this — e.g. `IndexedStream` skips filter-bitset fetch on empty +/// row groups by calling with `max_doc == min_doc`. +pub trait RowGroupDocsCollector: Send + Sync + Debug { + fn collect_packed_u64_bitset(&self, min_doc: i32, max_doc: i32) -> Result, String>; +} + +/// A searcher scoped to a single shard (index), created once per query. +/// +/// Represents a shard-scoped compiled form of the query — typically expensive +/// to build (parses query, compiles automata / prepares iterators, etc.) but +/// cheap to bind to individual segments via [`collector`]. +pub trait ShardSearcher: Send + Sync + Debug { + /// Number of segments in this shard. + fn segment_count(&self) -> usize; + + /// Max doc ID for a specific segment. + fn segment_max_doc(&self, segment_ord: usize) -> Result; + + /// Create a collector for a specific segment and doc ID range. + /// + /// The collector only returns docs in `[doc_min, doc_max)`. One collector + /// per segment per query, cheap to construct from the shard-scoped + /// compiled query this searcher represents. + fn collector( + &self, + segment_ord: usize, + doc_min: i32, + doc_max: i32, + ) -> Result, String>; +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/metrics.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/metrics.rs new file mode 100644 index 0000000000000..773927ea07156 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/metrics.rs @@ -0,0 +1,253 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Metrics for indexed search execution plans. +//! +//! - [`PartitionMetrics`] — registered against the parent `ExecutionPlanMetricsSet`, +//! visible in `EXPLAIN ANALYZE`. +//! - [`StreamMetrics`] — lightweight handles passed to each RG stream for recording. + +use std::sync::Arc; + +use datafusion::physical_plan::metrics::{ + Count, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet, Time, +}; + +/// Lightweight metric handles passed from `IndexedExec` to the streaming loop. +/// +/// All fields are `Option` because standalone uses of `IndexedExec` (i.e. not +/// under a multi-segment parent) have no shared parent metrics to update. +#[derive(Clone)] +pub struct StreamMetrics { + pub output_rows: Option, + pub elapsed_compute: Option

      - * Carries the DataFusion query plan, engine searcher, optional {@link IndexFilterTree}, - * and the native result stream handle after execution. + * Carries the DataFusion query plan, engine searcher, and the native result + * stream handle after execution. * * @opensearch.experimental */ @@ -30,7 +29,6 @@ public class DatafusionContext implements SearchExecutionContext + * A single entry point: {@link NativeBridge#executeQueryAsync} handles both vanilla + * parquet and indexed (index_filter-bearing) plans. The native side classifies the + * substrait plan and dispatches internally; Java is oblivious to which path runs. + *

      * After {@link #search}, the result stream handle is available on the context * via {@link DatafusionContext#getStreamHandle()}. * @@ -41,18 +45,6 @@ public DatafusionSearcher(ReaderHandle readerHandle) { @Override public void search(DatafusionContext context) throws IOException { - if (context.getFilterTree() == null) { - searchVanilla(context); - } else { - searchWithFilterTree(context); - } - } - - private void searchWithFilterTree(DatafusionContext context) { - throw new UnsupportedOperationException("Indexed query path not yet wired"); - } - - private void searchVanilla(DatafusionContext context) throws IOException { DatafusionQuery query = context.getDatafusionQuery(); if (query == null) { throw new IllegalStateException("DatafusionQuery must be set before search"); @@ -65,6 +57,7 @@ private void searchVanilla(DatafusionContext context) throws IOException { query.getSubstraitBytes(), runtimeHandle.get(), query.getContextId(), + 0L, new ActionListener<>() { @Override public void onResponse(Long streamPtr) { diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/CollectorRegistry.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/CollectorRegistry.java new file mode 100644 index 0000000000000..235dbd11fa73f --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/CollectorRegistry.java @@ -0,0 +1,53 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion.indexfilter; + +import org.opensearch.analytics.spi.IndexFilterProvider; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Per-segment collector bookkeeping, keyed by small positive ints for + * cheap FFM marshaling. + * + *

      This is the hot-path registry: {@code collectDocs} and + * {@code releaseCollector} upcalls only touch this map. The provider + * reference is captured at collector-creation time in + * {@link CollectorHandle}, so no second map lookup is needed. + */ +public final class CollectorRegistry { + + private final ConcurrentHashMap collectors = new ConcurrentHashMap<>(); + private final AtomicInteger nextKey = new AtomicInteger(1); + + /** Creates an empty collector registry. */ + public CollectorRegistry() {} + + int registerCollector(IndexFilterProvider provider, int innerCollectorKey) { + int key = nextKey.getAndIncrement(); + collectors.put(key, new CollectorHandle(provider, innerCollectorKey)); + return key; + } + + CollectorHandle collector(int key) { + return collectors.get(key); + } + + void unregisterCollector(int key) { + collectors.remove(key); + } + + /** + * Maps an outer collector key to the provider instance + the + * provider's own inner collector key. + */ + record CollectorHandle(IndexFilterProvider provider, int innerCollectorKey) { + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterProviderRegistry.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterProviderRegistry.java new file mode 100644 index 0000000000000..4f91936b1ce05 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterProviderRegistry.java @@ -0,0 +1,118 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion.indexfilter; + +import org.opensearch.analytics.spi.IndexFilterProvider; +import org.opensearch.analytics.spi.IndexFilterProviderFactory; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Query-level provider lifecycle: deserialize query bytes into an + * {@link IndexFilterProvider}, track it by int key, close it when done. + * + *

      Called once per query (cold path). Separated from the per-segment + * collector registry so the hot path ({@code collectDocs}) never touches + * the provider map. + */ +public final class FilterProviderRegistry { + + private final AtomicReference factory = new AtomicReference<>(); + private final ConcurrentHashMap providers = new ConcurrentHashMap<>(); + private final AtomicInteger nextKey = new AtomicInteger(1); + private final CollectorRegistry collectors; + + /** + * Creates a provider registry wired to the given collector registry. + * {@code createCollector} delegates to the provider then registers + * the result in {@code collectors}. + */ + public FilterProviderRegistry(CollectorRegistry collectors) { + this.collectors = collectors; + } + + /** + * Set the factory that deserializes query bytes into providers. + * Safe to call once; throws on double-set. + */ + public void setFactory(IndexFilterProviderFactory f) { + if (f == null) { + throw new IllegalArgumentException("factory must not be null"); + } + if (factory.compareAndSet(null, f) == false) { + throw new IllegalStateException("IndexFilterProviderFactory already set"); + } + } + + IndexFilterProviderFactory factory() { + return factory.get(); + } + + /** + * Create a provider from the factory and register it. + * + * @return provider key {@code >= 1}, or {@code -1} on failure + */ + int createProvider(byte[] queryBytes) { + IndexFilterProviderFactory f = factory.get(); + if (f == null) { + return -1; + } + try { + IndexFilterProvider provider = f.create(queryBytes); + if (provider == null) { + return -1; + } + int key = nextKey.getAndIncrement(); + providers.put(key, provider); + return key; + } catch (Exception e) { + return -1; + } + } + + /** + * Look up a registered provider by key. + */ + IndexFilterProvider provider(int key) { + return providers.get(key); + } + + /** + * Unregister and close a provider. Returns silently if key is unknown. + */ + void releaseProvider(int key) throws IOException { + IndexFilterProvider provider = providers.remove(key); + if (provider != null) { + provider.close(); + } + } + + /** + * Look up the provider for {@code providerKey}, ask it to create a + * collector for the given segment range, and register the result in + * the {@link CollectorRegistry}. + * + * @return outer collector key {@code >= 1}, or {@code -1} on failure + */ + int createCollector(int providerKey, int segmentOrd, int minDoc, int maxDoc) { + IndexFilterProvider provider = providers.get(providerKey); + if (provider == null) { + return -1; + } + int inner = provider.createCollector(segmentOrd, minDoc, maxDoc); + if (inner < 0) { + return -1; + } + return collectors.registerCollector(provider, inner); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java new file mode 100644 index 0000000000000..87aa07aad0f30 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java @@ -0,0 +1,160 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion.indexfilter; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.be.datafusion.indexfilter.CollectorRegistry.CollectorHandle; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Static callback targets invoked by the native engine via FFM upcalls. + * + *

      Delegates to two instance-based registries installed at plugin startup: + *

        + *
      • {@link FilterProviderRegistry} — query-level provider create/release (cold path)
        • + *
      • {@link CollectorRegistry} — per-segment collector create/collect/release (hot path)
        • + *
      + * + *

      Error-handling contract

      + *

      Every method catches all {@link Throwable}s and returns {@code -1} + * (or silently returns for void methods). A Java exception escaping through + * an FFM upcall stub crashes the JVM. + */ +public final class FilterTreeCallbacks { + + private static final Logger LOGGER = LogManager.getLogger(FilterTreeCallbacks.class); + + /** Both registries in one snapshot — one AtomicReference read per upcall. */ + record Registries(FilterProviderRegistry providers, CollectorRegistry collectors) { + } + + private static final AtomicReference REGISTRIES = new AtomicReference<>(); + + private FilterTreeCallbacks() {} + + /** + * Install the registries. Called once at plugin startup. + * Tests may call with {@code null} to reset. + */ + public static void setRegistries(FilterProviderRegistry providers, CollectorRegistry collectors) { + REGISTRIES.set(providers == null ? null : new Registries(providers, collectors)); + } + + // ── Provider lifecycle (cold path, once per query) ──────────────── + + /** + * {@code createProvider(queryBytes, queryBytesLen) -> providerKey|-1}. + */ + public static int createProvider(MemorySegment queryBytesPtr, long queryBytesLen) { + try { + Registries reg = REGISTRIES.get(); + if (reg == null) { + return -1; + } + MemorySegment view = queryBytesPtr.reinterpret(queryBytesLen); + byte[] bytes = view.toArray(ValueLayout.JAVA_BYTE); + return reg.providers().createProvider(bytes); + } catch (Throwable t) { + LOGGER.error("createProvider failed", t); + return -1; + } + } + + /** + * {@code releaseProvider(providerKey)}. Never throws. + */ + public static void releaseProvider(int providerKey) { + try { + Registries reg = REGISTRIES.get(); + if (reg != null) { + reg.providers().releaseProvider(providerKey); + } + } catch (Throwable t) { + LOGGER.error(new ParameterizedMessage("releaseProvider({}) failed", providerKey), t); + } + } + + // ── Collector lifecycle (hot path, per segment per query) ───────── + + /** + * {@code createCollector(providerKey, segmentOrd, minDoc, maxDoc) -> collectorKey|-1}. + */ + public static int createCollector(int providerKey, int segmentOrd, int minDoc, int maxDoc) { + try { + Registries reg = REGISTRIES.get(); + if (reg == null) { + return -1; + } + return reg.providers().createCollector(providerKey, segmentOrd, minDoc, maxDoc); + } catch (Throwable t) { + LOGGER.error( + new ParameterizedMessage( + "createCollector(providerKey={}, seg={}, [{}, {})) failed", + providerKey, + segmentOrd, + minDoc, + maxDoc + ), + t + ); + return -1; + } + } + + /** + * {@code collectDocs(collectorKey, minDoc, maxDoc, outPtr, outWordCap) -> wordsWritten|-1}. + * + *

      Single map lookup into {@link CollectorRegistry}. The provider + * reference is already captured in the {@link CollectorHandle}. + */ + public static long collectDocs(int collectorKey, int minDoc, int maxDoc, MemorySegment outPtr, long outWordCap) { + try { + Registries reg = REGISTRIES.get(); + if (reg == null) { + return -1L; + } + CollectorHandle handle = reg.collectors().collector(collectorKey); + if (handle == null) { + return -1L; + } + int maxWords = (int) Math.min(outWordCap, (long) Integer.MAX_VALUE); + MemorySegment view = outPtr.reinterpret((long) maxWords * Long.BYTES); + int n = handle.provider().collectDocs(handle.innerCollectorKey(), minDoc, maxDoc, view); + return (n < 0) ? -1L : n; + } catch (Throwable t) { + LOGGER.error(new ParameterizedMessage("collectDocs(collectorKey={}, [{}, {})) failed", collectorKey, minDoc, maxDoc), t); + return -1L; + } + } + + /** + * {@code releaseCollector(collectorKey)}. Never throws. + */ + public static void releaseCollector(int collectorKey) { + try { + Registries reg = REGISTRIES.get(); + if (reg == null) { + return; + } + CollectorHandle handle = reg.collectors().collector(collectorKey); + if (handle == null) { + return; + } + handle.provider().releaseCollector(handle.innerCollectorKey()); + reg.collectors().unregisterCollector(collectorKey); + } catch (Throwable t) { + LOGGER.error(new ParameterizedMessage("releaseCollector({}) failed", collectorKey), t); + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/package-info.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/package-info.java new file mode 100644 index 0000000000000..0fc9ca455cbd4 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Index filter bridge between Java and the DataFusion Rust backend via FFM. + */ +package org.opensearch.be.datafusion.indexfilter; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index 059e34b6747f8..afed97c2325b4 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -51,6 +51,7 @@ public final class NativeBridge { private static final MethodHandle STREAM_NEXT; private static final MethodHandle STREAM_CLOSE; private static final MethodHandle SQL_TO_SUBSTRAIT; + private static final MethodHandle REGISTER_FILTER_TREE_CALLBACKS; private static final MethodHandle CREATE_LOCAL_SESSION; private static final MethodHandle CLOSE_LOCAL_SESSION; private static final MethodHandle REGISTER_PARTITION_STREAM; @@ -113,6 +114,7 @@ public final class NativeBridge { ValueLayout.ADDRESS, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG ) ); @@ -203,10 +205,117 @@ public final class NativeBridge { ValueLayout.JAVA_LONG ) ); + + // void df_register_filter_tree_callbacks(createCollector, collectDocs, releaseCollector) + REGISTER_FILTER_TREE_CALLBACKS = linker.downcallHandle( + lib.find("df_register_filter_tree_callbacks").orElseThrow(), + FunctionDescriptor.ofVoid( + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS + ) + ); + + // Hand the five filter-tree upcall stubs to Rust now. No explicit + // caller step required — as soon as this class is loaded, callbacks + // are installed and `df_execute_indexed_query` can dispatch into Java. + installFilterTreeCallbacks(linker); } private NativeBridge() {} + private static void installFilterTreeCallbacks(Linker linker) { + try { + java.lang.foreign.Arena arena = java.lang.foreign.Arena.global(); + Class cb = org.opensearch.be.datafusion.indexfilter.FilterTreeCallbacks.class; + var lookup = java.lang.invoke.MethodHandles.lookup(); + + MethodHandle createProvider = lookup.findStatic( + cb, + "createProvider", + java.lang.invoke.MethodType.methodType(int.class, java.lang.foreign.MemorySegment.class, long.class) + ); + MethodHandle releaseProvider = lookup.findStatic( + cb, + "releaseProvider", + java.lang.invoke.MethodType.methodType(void.class, int.class) + ); + MethodHandle createCollector = lookup.findStatic( + cb, + "createCollector", + java.lang.invoke.MethodType.methodType(int.class, int.class, int.class, int.class, int.class) + ); + MethodHandle collectDocs = lookup.findStatic( + cb, + "collectDocs", + java.lang.invoke.MethodType.methodType( + long.class, + int.class, + int.class, + int.class, + java.lang.foreign.MemorySegment.class, + long.class + ) + ); + MethodHandle releaseCollector = lookup.findStatic( + cb, + "releaseCollector", + java.lang.invoke.MethodType.methodType(void.class, int.class) + ); + + java.lang.foreign.MemorySegment createProviderStub = linker.upcallStub( + createProvider, + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG), + arena + ); + java.lang.foreign.MemorySegment releaseProviderStub = linker.upcallStub( + releaseProvider, + FunctionDescriptor.ofVoid(ValueLayout.JAVA_INT), + arena + ); + java.lang.foreign.MemorySegment createCollectorStub = linker.upcallStub( + createCollector, + FunctionDescriptor.of( + ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT + ), + arena + ); + java.lang.foreign.MemorySegment collectDocsStub = linker.upcallStub( + collectDocs, + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG + ), + arena + ); + java.lang.foreign.MemorySegment releaseCollectorStub = linker.upcallStub( + releaseCollector, + FunctionDescriptor.ofVoid(ValueLayout.JAVA_INT), + arena + ); + NativeCall.invokeVoid( + REGISTER_FILTER_TREE_CALLBACKS, + createProviderStub, + releaseProviderStub, + createCollectorStub, + collectDocsStub, + releaseCollectorStub + ); + } catch (Throwable t) { + throw new ExceptionInInitializerError(t); + } + } + // ---- Tokio runtime management (no Arena needed — no string/buffer args) ---- public static void initTokioRuntimeManager(int cpuThreads) { @@ -262,6 +371,7 @@ public static void executeQueryAsync( byte[] substraitPlan, long runtimePtr, long contextId, + long queryConfigPtr, ActionListener listener ) { try { @@ -281,7 +391,8 @@ public static void executeQueryAsync( call.bytes(substraitPlan), (long) substraitPlan.length, runtimePtr, - contextId + contextId, + queryConfigPtr ); listener.onResponse(result); } catch (Throwable t) { diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java index 533b200fc786e..a9f1b13eeadd9 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java @@ -103,6 +103,7 @@ private List executeQuery(String sql) { substraitBytes, runtimeHandle.get(), 0L, + 0L, listener ) ); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java index d1dda1b1997a1..3907fd8a69309 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java @@ -144,6 +144,7 @@ public void testNativeQueryFailureDoesNotLeak() { new byte[] { 0, 1, 2 }, runtimeHandle.get(), 0L, + 0L, new ActionListener<>() { @Override public void onResponse(Long ptr) { @@ -180,17 +181,25 @@ public void testCloseAfterNativeStreamNextFailure() throws Exception { runtimeHandle.get() ); CompletableFuture future = new CompletableFuture<>(); - NativeBridge.executeQueryAsync(readerHandle.getPointer(), "test_table", substrait, tempRuntime.get(), 0L, new ActionListener<>() { - @Override - public void onResponse(Long p) { - future.complete(p); - } + NativeBridge.executeQueryAsync( + readerHandle.getPointer(), + "test_table", + substrait, + tempRuntime.get(), + 0L, + 0L, + new ActionListener<>() { + @Override + public void onResponse(Long p) { + future.complete(p); + } - @Override - public void onFailure(Exception e) { - future.completeExceptionally(e); + @Override + public void onFailure(Exception e) { + future.completeExceptionally(e); + } } - }); + ); long streamPtr = future.join(); DatafusionResultStream stream = new DatafusionResultStream( @@ -223,17 +232,25 @@ public void testDoubleCloseIsHarmless() throws Exception { private DatafusionResultStream createStream(String sql) { byte[] substrait = NativeBridge.sqlToSubstrait(readerHandle.getPointer(), "test_table", sql, runtimeHandle.get()); CompletableFuture future = new CompletableFuture<>(); - NativeBridge.executeQueryAsync(readerHandle.getPointer(), "test_table", substrait, runtimeHandle.get(), 0L, new ActionListener<>() { - @Override - public void onResponse(Long ptr) { - future.complete(ptr); - } + NativeBridge.executeQueryAsync( + readerHandle.getPointer(), + "test_table", + substrait, + runtimeHandle.get(), + 0L, + 0L, + new ActionListener<>() { + @Override + public void onResponse(Long ptr) { + future.complete(ptr); + } - @Override - public void onFailure(Exception e) { - future.completeExceptionally(e); + @Override + public void onFailure(Exception e) { + future.completeExceptionally(e); + } } - }); + ); long streamPtr = future.join(); BufferAllocator childAllocator = testRootAllocator.newChildAllocator("test-stream", 0, Long.MAX_VALUE); return new DatafusionResultStream( diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/indexfilter/IndexFilterCallbackTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/indexfilter/IndexFilterCallbackTests.java new file mode 100644 index 0000000000000..5ea0da63d9a8f --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/indexfilter/IndexFilterCallbackTests.java @@ -0,0 +1,179 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion.indexfilter; + +import org.opensearch.analytics.spi.IndexFilterProvider; +import org.opensearch.analytics.spi.IndexFilterProviderFactory; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +/** + * Tests the Java-side provider/collector registries + FFM callback dispatch + * glue without going through the full substrait → native pipeline. + */ +public class IndexFilterCallbackTests extends OpenSearchTestCase { + + private FilterProviderRegistry providers; + private CollectorRegistry collectors; + + @Override + public void setUp() throws Exception { + super.setUp(); + collectors = new CollectorRegistry(); + providers = new FilterProviderRegistry(collectors); + FilterTreeCallbacks.setRegistries(providers, collectors); + } + + @Override + public void tearDown() throws Exception { + FilterTreeCallbacks.setRegistries(null, null); + super.tearDown(); + } + + public void testCreateCollectReleaseRoundTrip() { + MockProvider provider = new MockProvider(new long[] { 0x5L, 0x0L }); + // Register provider directly via lifecycle, bypassing factory upcall. + int providerKey = providers.createProvider(new byte[0]); + // That returns -1 because no factory is set. Register manually instead. + providerKey = registerProviderDirectly(provider); + + try (Arena arena = Arena.ofConfined()) { + int collectorKey = FilterTreeCallbacks.createCollector(providerKey, 0, 0, 64); + assertTrue("collectorKey >= 0", collectorKey >= 0); + + MemorySegment buf = arena.allocate(Long.BYTES); + int n = provider.collectDocs(collectors.collector(collectorKey).innerCollectorKey(), 0, 64, buf); + assertEquals(1, n); + assertEquals(0x5L, buf.getAtIndex(ValueLayout.JAVA_LONG, 0)); + + FilterTreeCallbacks.releaseCollector(collectorKey); + assertNull("collector removed from registry", collectors.collector(collectorKey)); + } finally { + FilterTreeCallbacks.releaseProvider(providerKey); + assertNull("provider removed from registry", providers.provider(providerKey)); + } + } + + public void testCreateWithUnknownProviderReturnsError() { + assertEquals(-1, FilterTreeCallbacks.createCollector(Integer.MAX_VALUE, 0, 0, 16)); + } + + public void testReleaseWithUnknownCollectorIsSafe() { + FilterTreeCallbacks.releaseCollector(Integer.MAX_VALUE); + } + + public void testReleaseWithUnknownProviderIsSafe() { + FilterTreeCallbacks.releaseProvider(Integer.MAX_VALUE); + } + + public void testCreateProviderDispatchesToRegisteredFactory() throws IOException { + byte[] expected = new byte[] { 1, 2, 3, 4 }; + StubFactory factory = new StubFactory(expected); + providers.setFactory(factory); + + try (Arena arena = Arena.ofConfined()) { + MemorySegment seg = arena.allocate(expected.length); + MemorySegment.copy(expected, 0, seg, ValueLayout.JAVA_BYTE, 0, expected.length); + + int key = FilterTreeCallbacks.createProvider(seg, expected.length); + assertTrue("providerKey >= 0", key >= 0); + assertEquals("factory invoked exactly once", 1, factory.callCount); + + IndexFilterProvider registered = providers.provider(key); + assertNotNull("provider registered under returned key", registered); + assertSame("registered provider is the one factory produced", factory.lastProvider, registered); + + FilterTreeCallbacks.releaseProvider(key); + assertNull(providers.provider(key)); + } + } + + public void testCreateProviderWithNoFactoryReturnsError() { + // Fresh lifecycle with no factory set. + CollectorRegistry emptyColl = new CollectorRegistry(); + FilterProviderRegistry empty = new FilterProviderRegistry(emptyColl); + FilterTreeCallbacks.setRegistries(empty, emptyColl); + try (Arena arena = Arena.ofConfined()) { + MemorySegment seg = arena.allocate(1); + int key = FilterTreeCallbacks.createProvider(seg, 1); + assertEquals("no factory → -1", -1, key); + } + } + + /** + * Helper: register a provider directly into the lifecycle's internal map + * for tests that bypass the factory. Uses reflection-free approach by + * setting a factory that returns the given provider, calling createProvider, + * then the factory is consumed. + */ + private int registerProviderDirectly(IndexFilterProvider provider) { + // Use a one-shot factory that returns the given provider. + FilterProviderRegistry directLifecycle = new FilterProviderRegistry(collectors); + directLifecycle.setFactory(bytes -> provider); + int key = directLifecycle.createProvider(new byte[0]); + // Swap the lifecycle so FilterTreeCallbacks sees this provider. + // We need to keep the collectors registry. + this.providers = directLifecycle; + FilterTreeCallbacks.setRegistries(directLifecycle, collectors); + return key; + } + + /** Stub factory that records its input and emits a MockProvider. */ + private static final class StubFactory implements IndexFilterProviderFactory { + private final byte[] expectedBytes; + int callCount = 0; + IndexFilterProvider lastProvider; + + StubFactory(byte[] expectedBytes) { + this.expectedBytes = expectedBytes; + } + + @Override + public IndexFilterProvider create(byte[] queryBytes) { + callCount++; + assertArrayEquals("factory receives the exact bytes from upcall", expectedBytes, queryBytes); + lastProvider = new MockProvider(new long[] { 0xAL }); + return lastProvider; + } + } + + /** In-memory provider that returns canned bitset words. */ + private static final class MockProvider implements IndexFilterProvider { + private final long[] cannedWords; + private int nextCollector = 1; + + MockProvider(long[] cannedWords) { + this.cannedWords = cannedWords; + } + + @Override + public int createCollector(int segmentOrd, int minDoc, int maxDoc) { + return nextCollector++; + } + + @Override + public int collectDocs(int collectorKey, int minDoc, int maxDoc, MemorySegment out) { + int n = Math.min(cannedWords.length, (int) (out.byteSize() / Long.BYTES)); + for (int i = 0; i < n; i++) { + out.setAtIndex(ValueLayout.JAVA_LONG, i, cannedWords[i]); + } + return n; + } + + @Override + public void releaseCollector(int collectorKey) {} + + @Override + public void close() throws IOException {} + } +} diff --git a/sandbox/plugins/analytics-backend-lucene/build.gradle b/sandbox/plugins/analytics-backend-lucene/build.gradle index 8bd898068e6b8..3275ec49f65e5 100644 --- a/sandbox/plugins/analytics-backend-lucene/build.gradle +++ b/sandbox/plugins/analytics-backend-lucene/build.gradle @@ -13,6 +13,8 @@ opensearchplugin { classname = 'org.opensearch.be.lucene.LucenePlugin' } +java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } + // Calcite (via analytics-engine) requires Guava which OpenSearch forbids on compile classpath. // Use custom config to bypass, same as analytics-engine. configurations { diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneIndexFilterProvider.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneIndexFilterProvider.java index 9851e07d33bbc..71e29fc3aac01 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneIndexFilterProvider.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneIndexFilterProvider.java @@ -8,17 +8,21 @@ package org.opensearch.be.lucene; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; +import org.apache.lucene.util.FixedBitSet; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.engine.exec.CollectorQueryLifecycleManager; import org.opensearch.index.engine.exec.IndexFilterProvider; import org.opensearch.index.engine.exec.SegmentCollector; import java.io.IOException; -import java.util.BitSet; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; /** * Lucene-backed {@link IndexFilterProvider}. @@ -59,9 +63,12 @@ public int createCollector(LuceneIndexFilterContext context, int segmentOrd, int * @param key the collector key * @param minDoc the minimum document ID * @param maxDoc the maximum document ID + * @param out destination {@link MemorySegment} to write the packed bitset into + * @return the number of 64-bit words written into {@code out} */ - public long[] collectDocs(LuceneIndexFilterContext context, int key, int minDoc, int maxDoc) { - return context.getCollectorManager().collectDocs(key, minDoc, maxDoc); + @Override + public int collectDocs(LuceneIndexFilterContext context, int key, int minDoc, int maxDoc, MemorySegment out) { + return context.getCollectorManager().collectDocs(key, minDoc, maxDoc, out); } /** @@ -89,46 +96,83 @@ private SegmentCollector createCollectorInternal(LuceneIndexFilterContext contex } } - private static final SegmentCollector EMPTY_COLLECTOR = (min, max) -> new long[0]; + private static final SegmentCollector EMPTY_COLLECTOR = (min, max, out) -> { + if (max <= min) { + return 0; + } + int wordCount = (max - min + 63) >>> 6; + for (int i = 0; i < wordCount; i++) { + out.setAtIndex(ValueLayout.JAVA_LONG, i, 0L); + } + return wordCount; + }; - private static class LuceneSegmentCollector implements SegmentCollector { + /** + * Per-segment cursor over matching docs. + * + *

      Forward-only: successive {@link #collectDocs(int, int, MemorySegment)} calls MUST use + * non-decreasing, non-overlapping {@code [minDoc, maxDoc)} ranges. The + * Lucene {@link DocIdSetIterator} is a one-shot cursor and cannot seek + * backwards. + * + *

      Bit layout: the {@code out} {@link MemorySegment} receives a packed bitset where + * word {@code j} bit {@code i} (LSB-first) represents the doc at relative + * position {@code j*64 + i} within the caller's {@code [minDoc, maxDoc)} + * range. That is, bit {@code k} represents absolute doc id + * {@code minDoc + k}. Word count is always {@code ceilDiv(maxDoc - minDoc, 64)} + * regardless of how many bits are set. + */ + private static final class LuceneSegmentCollector implements SegmentCollector { + private static final Logger logger = LogManager.getLogger(LuceneSegmentCollector.class); private final DocIdSetIterator iterator; - private final int collectorMinDoc; - private final int collectorMaxDoc; + /** Partition bounds — the iterator only produces matches in this range. */ + private final int partitionMinDoc; + private final int partitionMaxDoc; + /** Cursor: resumes from here on the next collectDocs call. */ private int currentDoc = -1; - LuceneSegmentCollector(DocIdSetIterator iterator, int minDoc, int maxDoc) { + LuceneSegmentCollector(DocIdSetIterator iterator, int partitionMinDoc, int partitionMaxDoc) { this.iterator = iterator; - this.collectorMinDoc = minDoc; - this.collectorMaxDoc = maxDoc; + this.partitionMinDoc = partitionMinDoc; + this.partitionMaxDoc = partitionMaxDoc; } @Override - public long[] collectDocs(int minDoc, int maxDoc) { - int effectiveMin = Math.max(minDoc, collectorMinDoc); - int effectiveMax = Math.min(maxDoc, collectorMaxDoc); - if (effectiveMin >= effectiveMax) { - return new long[0]; + public int collectDocs(int minDoc, int maxDoc, MemorySegment out) { + if (maxDoc <= minDoc) { + return 0; } - - BitSet bitset = new BitSet(effectiveMax - effectiveMin); - try { - int docId = currentDoc; - if (docId == DocIdSetIterator.NO_MORE_DOCS || docId >= collectorMaxDoc) { - return new long[0]; - } - if (docId < effectiveMin) { - docId = iterator.advance(effectiveMin); + // Use FixedBitSet for cache-friendly heap-array bit manipulation, + // then bulk-copy into the native MemorySegment at the boundary. + int span = maxDoc - minDoc; + FixedBitSet bits = new FixedBitSet(span); + + int scanFrom = Math.max(minDoc, partitionMinDoc); + int scanTo = Math.min(maxDoc, partitionMaxDoc); + + if (scanFrom < scanTo) { + try { + int docId = currentDoc; + if (docId != DocIdSetIterator.NO_MORE_DOCS) { + if (docId < scanFrom) { + docId = iterator.advance(scanFrom); + } + while (docId != DocIdSetIterator.NO_MORE_DOCS && docId < scanTo) { + bits.set(docId - minDoc); + docId = iterator.nextDoc(); + } + currentDoc = docId; + } + } catch (IOException e) { + logger.warn("IOException during collectDocs, returning partial bitset", e); } - while (docId != DocIdSetIterator.NO_MORE_DOCS && docId < effectiveMax) { - bitset.set(docId - effectiveMin); - docId = iterator.nextDoc(); - } - currentDoc = docId; - } catch (IOException e) { - return new long[0]; } - return bitset.toLongArray(); + + // Single bulk copy: heap long[] → native MemorySegment. + long[] words = bits.getBits(); + int wordCount = (span + 63) >>> 6; + MemorySegment.copy(words, 0, out, ValueLayout.JAVA_LONG, 0, wordCount); + return wordCount; } } } diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle index c0bec17f95bc1..038550b9cf12a 100644 --- a/sandbox/plugins/analytics-engine/build.gradle +++ b/sandbox/plugins/analytics-engine/build.gradle @@ -22,6 +22,8 @@ opensearchplugin { classname = 'org.opensearch.analytics.AnalyticsPlugin' } +java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } + repositories { maven { name = 'OpenSearch Snapshots' diff --git a/sandbox/plugins/dsl-query-executor/build.gradle b/sandbox/plugins/dsl-query-executor/build.gradle index 31ccdbc395a10..350eeda1ee4fc 100644 --- a/sandbox/plugins/dsl-query-executor/build.gradle +++ b/sandbox/plugins/dsl-query-executor/build.gradle @@ -14,6 +14,8 @@ opensearchplugin { extendedPlugins = ['analytics-engine'] } +java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } + // Guava comes transitively from calcite-core — forbidden on compile classpaths // by OpenSearch policy. Calcite API exposes ImmutableList in type annotations, // so the compiler needs Guava. Bypass via custom config (same pattern as analytics-engine). diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java index ca4ffa0c9c7d9..4c5d8d60272c6 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java @@ -22,6 +22,9 @@ import java.nio.file.Path; import java.util.List; +/** + * FFM bridge to the native Rust parquet writer library. + */ public class RustBridge { private static final MethodHandle CREATE_WRITER; diff --git a/sandbox/plugins/test-ppl-frontend/build.gradle b/sandbox/plugins/test-ppl-frontend/build.gradle index acb7a94a6c000..8a578d2056be7 100644 --- a/sandbox/plugins/test-ppl-frontend/build.gradle +++ b/sandbox/plugins/test-ppl-frontend/build.gradle @@ -15,6 +15,8 @@ apply plugin: 'opensearch.opensearchplugin' +java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } + // SQL Unified Query API version (aligned with OpenSearch build version) def sqlUnifiedQueryVersion = '3.6.0.0-SNAPSHOT' diff --git a/server/src/main/java/org/opensearch/index/engine/exec/SegmentCollector.java b/server/src/main/java/org/opensearch/index/engine/exec/SegmentCollector.java deleted file mode 100644 index 772244d88436f..0000000000000 --- a/server/src/main/java/org/opensearch/index/engine/exec/SegmentCollector.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.engine.exec; - -import org.opensearch.common.annotation.ExperimentalApi; - -import java.io.Closeable; - -/** - * A per-segment document collector returned by - * {@link IndexFilterProvider#createCollector}. - *

      - * Callers should use try-with-resources to ensure cleanup. - * - * @opensearch.experimental - */ -@ExperimentalApi -public interface SegmentCollector extends Closeable { - - /** - * Collect matching document IDs in the given range. - * - * @param minDoc inclusive lower bound - * @param maxDoc exclusive upper bound - * @return packed {@code long[]} bitset of matching doc IDs - */ - long[] collectDocs(int minDoc, int maxDoc); - - @Override - default void close() {} -} From 7b6de2405c08dae7ff1c4a1cec21721825aafc4a Mon Sep 17 00:00:00 2001 From: Vishwas garg Date: Tue, 5 May 2026 17:48:14 +0530 Subject: [PATCH 044/115] Sandbox/foyer page cache (#21202) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add foyer cache to sandbox module Signed-off-by: Vishwas Garg * Add foyer cache to sandbox module Signed-off-by: Vishwas Garg * Make format cache block_size and io_engine configurable; fix compile errors Settings (Java): - Add PageCacheSettings.java with format_cache.block_size (default 64MB) and format_cache.io_engine (auto|io_uring|psync, default auto) in org.opensearch.index.store.remote.filecache — same package as FileCacheSettings. - Register both in ClusterSettings.BUILT_IN_CLUSTER_SETTINGS. - Imports use org.opensearch.core.common.unit.ByteSizeUnit/Value. Page cache interface: - Update PageCache.java Javadoc: SSD-tier analog of OS page cache, agnostic of entry granularity and backing implementation. Java FFM layer: - FoyerPageCache: constructor takes blockSizeBytes + ioEngine params. - FoyerBridge.createCache: add blockSizeBytes + ioEngine params; update FFM FunctionDescriptor with two new arguments. Rust: - foyer_cache.rs: build_io_engine_config(choice) accepts auto/io_uring/psync; FoyerCache::new takes block_size_bytes + io_engine params; clone io_engine before async move closure to avoid borrow-after-move. - ffm.rs: foyer_create_cache adds block_size_bytes (u64) and io_engine_ptr/io_engine_len params. - tests.rs: update all FoyerCache::new() (4-param) and foyer_create_cache() (6-param) call sites; add BLOCK_SIZE/IO_ENGINE constants; add 3 new validation tests for blockSizeBytes <= 0 and null ioEngine. - dataformat-native/rust/Cargo.toml: remove stale foyer-page-cache workspace member (sandbox/modules/foyer-page-cache/rust no longer exists). Signed-off-by: Vishwas Garg * Update the cache interface from page-cache to block-cache Signed-off-by: Vishwas Garg * Fix spotless formatting violations in BlockCacheSettings Signed-off-by: Vishwas Garg * Add replaceSignatureFiles jdk-signatures to the forbiddenApisMain, forbiddenApisTest Signed-off-by: Vishwas Garg * Move block cache to a plugin and wire it with native s3 repo Signed-off-by: Bukhtawar Khan * Move block cache to a plugin and wire it with Node.java Signed-off-by: Bukhtawar Khan * Fix up Signed-off-by: Bukhtawar Khan * Removed block cache ITs, will be added when integrating with UnifiedCacheService Signed-off-by: Vishwas Garg --------- Signed-off-by: Vishwas Garg Signed-off-by: Bukhtawar Khan Co-authored-by: Vishwas Garg Co-authored-by: Bukhtawar Khan --- .../libs/dataformat-native/rust/Cargo.toml | 1 + .../dataformat-native/rust/lib/Cargo.toml | 3 +- .../plugins/block-cache-foyer/build.gradle | 46 ++ .../foyer/BlockCacheFoyerPlugin.java | 129 ++++++ .../blockcache/foyer/FoyerBlockCache.java | 111 +++++ .../blockcache/foyer/FoyerBridge.java | 123 ++++++ .../src/main/rust/Cargo.toml | 22 + .../src/main/rust/src/foyer/ffm.rs | 71 +++ .../src/main/rust/src/foyer/foyer_cache.rs | 271 ++++++++++++ .../src/main/rust/src/foyer/mod.rs | 13 + .../src/main/rust/src/foyer/tests.rs | 409 ++++++++++++++++++ .../src/main/rust/src/lib.rs | 11 + .../src/main/rust/src/range_cache.rs | 111 +++++ .../src/main/rust/src/traits.rs | 44 ++ .../foyer/BlockCacheFoyerPluginTests.java | 42 ++ .../common/settings/ClusterSettings.java | 5 + .../remote/filecache/BlockCacheSettings.java | 84 ++++ .../main/java/org/opensearch/node/Node.java | 1 + .../org/opensearch/plugins/BlockCache.java | 55 +++ .../plugins/BlockCacheProvider.java | 47 ++ .../opensearch/plugins/BlockCacheStats.java | 41 ++ 21 files changed, 1639 insertions(+), 1 deletion(-) create mode 100644 sandbox/plugins/block-cache-foyer/build.gradle create mode 100644 sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/BlockCacheFoyerPlugin.java create mode 100644 sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/FoyerBlockCache.java create mode 100644 sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/FoyerBridge.java create mode 100644 sandbox/plugins/block-cache-foyer/src/main/rust/Cargo.toml create mode 100644 sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/ffm.rs create mode 100644 sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/foyer_cache.rs create mode 100644 sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/mod.rs create mode 100644 sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/tests.rs create mode 100644 sandbox/plugins/block-cache-foyer/src/main/rust/src/lib.rs create mode 100644 sandbox/plugins/block-cache-foyer/src/main/rust/src/range_cache.rs create mode 100644 sandbox/plugins/block-cache-foyer/src/main/rust/src/traits.rs create mode 100644 sandbox/plugins/block-cache-foyer/src/test/java/org/opensearch/blockcache/foyer/BlockCacheFoyerPluginTests.java create mode 100644 server/src/main/java/org/opensearch/index/store/remote/filecache/BlockCacheSettings.java create mode 100644 server/src/main/java/org/opensearch/plugins/BlockCache.java create mode 100644 server/src/main/java/org/opensearch/plugins/BlockCacheProvider.java create mode 100644 server/src/main/java/org/opensearch/plugins/BlockCacheStats.java diff --git a/sandbox/libs/dataformat-native/rust/Cargo.toml b/sandbox/libs/dataformat-native/rust/Cargo.toml index 25e2ee18e52e2..6adb44c9a62b4 100644 --- a/sandbox/libs/dataformat-native/rust/Cargo.toml +++ b/sandbox/libs/dataformat-native/rust/Cargo.toml @@ -11,6 +11,7 @@ members = [ "../../../plugins/native-repository-azure/src/main/rust", "../../../plugins/native-repository-fs/src/main/rust", "../../../libs/tiered-storage/src/main/rust", + "../../../plugins/block-cache-foyer/src/main/rust", ] [workspace.dependencies] diff --git a/sandbox/libs/dataformat-native/rust/lib/Cargo.toml b/sandbox/libs/dataformat-native/rust/lib/Cargo.toml index 8a1ec6dd176ac..354c8f5019a43 100644 --- a/sandbox/libs/dataformat-native/rust/lib/Cargo.toml +++ b/sandbox/libs/dataformat-native/rust/lib/Cargo.toml @@ -10,12 +10,13 @@ name = "opensearch_native" crate-type = ["cdylib"] [dependencies] -opensearch-datafusion = { path = "../../../../plugins/analytics-backend-datafusion/rust" } +opensearch-datafusion = { path = "../../../../plugins/analytics-backend-datafusion/rust" } opensearch-parquet-format = { path = "../../../../plugins/parquet-data-format/src/main/rust" } opensearch-repository-s3 = { workspace = true } opensearch-repository-gcs = { workspace = true } opensearch-repository-azure = { workspace = true } opensearch-repository-fs = { workspace = true } +opensearch-block-cache = { path = "../../../../plugins/block-cache-foyer/src/main/rust" } native-bridge-common = { workspace = true } opensearch-tiered-storage = { path = "../../../../libs/tiered-storage/src/main/rust" } mimalloc = { workspace = true } diff --git a/sandbox/plugins/block-cache-foyer/build.gradle b/sandbox/plugins/block-cache-foyer/build.gradle new file mode 100644 index 0000000000000..a3929b3db79de --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/build.gradle @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +opensearchplugin { + description = 'Foyer-backed node-level block cache for native repositories.' + classname = 'org.opensearch.blockcache.foyer.BlockCacheFoyerPlugin' + extendedPlugins = [] +} + +apply plugin: 'opensearch.internal-cluster-test' + +java { + sourceCompatibility = JavaVersion.toVersion(25) + targetCompatibility = JavaVersion.toVersion(25) +} + +dependencies { + implementation project(':sandbox:libs:dataformat-native') + compileOnly "org.apache.logging.log4j:log4j-api:${versions.log4j}" + testImplementation project(':test:framework') + internalClusterTestImplementation project(':test:framework') +} + +testingConventions.enabled = false + +tasks.named('compileInternalClusterTestJava').configure { + sourceCompatibility = JavaVersion.toVersion(25) + targetCompatibility = JavaVersion.toVersion(25) +} + +tasks.matching { it.name in ['test', 'internalClusterTest'] }.configureEach { + jvmArgs += ['--enable-native-access=ALL-UNNAMED'] + systemProperty 'native.lib.path', + project(':sandbox:libs:dataformat-native').ext.nativeLibPath.absolutePath + dependsOn ':sandbox:libs:dataformat-native:buildRustLibrary' +} + +// missingJavadoc hardcodes --release 21 which hides FFM types (stable since JDK 22). +tasks.matching { it.name == 'missingJavadoc' }.configureEach { + enabled = false +} diff --git a/sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/BlockCacheFoyerPlugin.java b/sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/BlockCacheFoyerPlugin.java new file mode 100644 index 0000000000000..fa78a61af17e3 --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/BlockCacheFoyerPlugin.java @@ -0,0 +1,129 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.blockcache.foyer; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.plugins.BlockCache; +import org.opensearch.plugins.BlockCacheProvider; +import org.opensearch.plugins.Plugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.script.ScriptService; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.client.Client; +import org.opensearch.watcher.ResourceWatcherService; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; + +/** + * Plugin entry point for the Foyer-backed node-level block cache. + * + *

      Implements {@link BlockCacheProvider}: core publishes this SPI as an + * extension point for consumers to discover via + * {@code pluginsService.filterPlugins(BlockCacheProvider.class)} when they + * need a node-level block cache. Consumers are responsible for resolving + * the cache themselves. + * + *

      {@code extendedPlugins = []} — this plugin does not extend any other + * plugin, and no other plugin extends it. + * + * @opensearch.experimental + */ +public class BlockCacheFoyerPlugin extends Plugin implements BlockCacheProvider { + + private static final Logger logger = LogManager.getLogger(BlockCacheFoyerPlugin.class); + + // Foyer cache defaults. Pinned here for deterministic bootstrap; can be promoted + // to node settings in a follow-up without changing the SPI surface. + private static final long DEFAULT_DISK_BYTES = 1L << 30; // 1 GiB + private static final String DEFAULT_DISK_DIR_NAME = "foyer-block-cache"; + private static final long DEFAULT_BLOCK_SIZE_BYTES = 64L * 1024L * 1024L; // 64 MiB + private static final String DEFAULT_IO_ENGINE = "auto"; + + private final AtomicBoolean componentsCreated = new AtomicBoolean(false); + private volatile FoyerBlockCache cache; + + /** No-arg constructor required by the plugin framework. */ + public BlockCacheFoyerPlugin() {} + + /** + * Settings constructor (alternate signature used by PluginsService). + * + * @param settings node settings; currently unused — Foyer defaults are pinned + */ + public BlockCacheFoyerPlugin(final Settings settings) {} + + @Override + public Optional getBlockCache() { + return Optional.ofNullable(cache); + } + + @Override + public Collection createComponents( + final Client client, + final ClusterService clusterService, + final ThreadPool threadPool, + final ResourceWatcherService resourceWatcherService, + final ScriptService scriptService, + final NamedXContentRegistry xContentRegistry, + final Environment environment, + final NodeEnvironment nodeEnvironment, + final NamedWriteableRegistry namedWriteableRegistry, + final IndexNameExpressionResolver indexNameExpressionResolver, + final Supplier repositoriesServiceSupplier + ) { + if (componentsCreated.compareAndSet(false, true) == false) { + throw new IllegalStateException("BlockCacheFoyerPlugin.createComponents called more than once"); + } + + final String diskDir; + if (environment.dataFiles().length == 0) { + diskDir = System.getProperty("java.io.tmpdir") + "/" + DEFAULT_DISK_DIR_NAME; + } else { + diskDir = environment.dataFiles()[0].resolve(DEFAULT_DISK_DIR_NAME).toString(); + } + + try { + cache = new FoyerBlockCache(DEFAULT_DISK_BYTES, diskDir, DEFAULT_BLOCK_SIZE_BYTES, DEFAULT_IO_ENGINE); + } catch (final Throwable t) { + throw new IllegalStateException("Failed to initialise Foyer block cache (diskDir=" + diskDir + ")", t); + } + logger.info("BlockCacheFoyerPlugin created FoyerBlockCache (diskDir={})", diskDir); + return List.of(cache); + } + + /** + * Close the cache. Idempotent; safe to call multiple times. {@link + * FoyerBlockCache#close()} is itself idempotent via an {@code AtomicBoolean}. + */ + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + final FoyerBlockCache c = cache; + if (c != null) { + c.close(); + logger.info("BlockCacheFoyerPlugin closed"); + } + } + } +} diff --git a/sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/FoyerBlockCache.java b/sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/FoyerBlockCache.java new file mode 100644 index 0000000000000..bf9ea4ace62ac --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/FoyerBlockCache.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.blockcache.foyer; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.plugins.BlockCache; +import org.opensearch.plugins.BlockCacheStats; + +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Foyer-backed implementation of {@link BlockCache}. + * + *

      Holds the native cache handle privately. Callers interact with this + * class through the {@link BlockCache} interface. Native-aware callers that + * need the underlying handle must cast to {@code FoyerBlockCache} and call + * {@link #nativeCachePtr()}. Core code never performs that cast. + * + * @opensearch.experimental + */ +public final class FoyerBlockCache implements BlockCache { + + private static final Logger logger = LogManager.getLogger(FoyerBlockCache.class); + + /** Opaque native handle returned by {@code foyer_create_cache}. Always positive. */ + private final long cachePtr; + + /** Guards against double-close per the {@link AutoCloseable} contract. */ + private final AtomicBoolean closed = new AtomicBoolean(false); + + /** + * Create the native Foyer cache and acquire its handle. + * + * @param diskBytes maximum disk capacity in bytes; must be {@code > 0} + * @param diskDir directory where Foyer stores cache data; must not be null or blank + * @param blockSizeBytes Foyer disk block size in bytes; must be {@code > 0}. + * Typically read from {@code format_cache.block_size} (default 64 MB). + * @param ioEngine I/O engine selection: {@code "auto"}, {@code "io_uring"}, or + * {@code "psync"}. Typically read from {@code format_cache.io_engine}. + * @throws IllegalArgumentException if {@code diskBytes <= 0}, {@code blockSizeBytes <= 0}, + * or {@code diskDir} is blank + * @throws NullPointerException if {@code diskDir} or {@code ioEngine} is null + * @throws IllegalStateException if the native call fails to return a valid handle + */ + public FoyerBlockCache(long diskBytes, String diskDir, long blockSizeBytes, String ioEngine) { + if (diskBytes <= 0) { + throw new IllegalArgumentException("diskBytes must be > 0, got: " + diskBytes); + } + Objects.requireNonNull(diskDir, "diskDir must not be null"); + if (diskDir.isBlank()) { + throw new IllegalArgumentException("diskDir must not be blank"); + } + if (blockSizeBytes <= 0) { + throw new IllegalArgumentException("blockSizeBytes must be > 0, got: " + blockSizeBytes); + } + Objects.requireNonNull(ioEngine, "ioEngine must not be null"); + this.cachePtr = FoyerBridge.createCache(diskBytes, diskDir, blockSizeBytes, ioEngine); + } + + /** + * Returns the opaque native cache pointer. + * + *

      Native-aware callers only. This method lives outside + * the {@link BlockCache} interface to prevent leakage of the native handle + * into general-purpose code. Callers must first verify the runtime type + * with {@code instanceof FoyerBlockCache} before calling this method. + * + * @return the positive {@code long} handle to the native cache instance + */ + public long nativeCachePtr() { + return cachePtr; + } + + /** + * Returns a point-in-time snapshot of cache counters. + * + *

      Foyer exposes its counters through the native library; bridging them + * into this record is a follow-up. Until then, this method returns a + * zero-valued snapshot so that callers that poll stats for logging or + * node-stats reporting continue to function without special-casing. + * + * @return zero-valued snapshot; never {@code null} + */ + @Override + public BlockCacheStats stats() { + // TODO: bridge real Foyer counters through FFM once the Rust-side accessor exists. + return new BlockCacheStats(0L, 0L, 0L, 0L, 0L); + } + + /** + * Destroys the native cache. Idempotent — safe to call multiple times. + * + *

      Only the first invocation actually destroys the cache; subsequent + * calls are no-ops. This satisfies the {@link BlockCache#close()} contract. + */ + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + FoyerBridge.destroyCache(cachePtr); + logger.info("FoyerBlockCache closed"); + } + } +} diff --git a/sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/FoyerBridge.java b/sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/FoyerBridge.java new file mode 100644 index 0000000000000..9d5791a9ae14e --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/main/java/org/opensearch/blockcache/foyer/FoyerBridge.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.blockcache.foyer; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.nativebridge.spi.NativeCall; +import org.opensearch.nativebridge.spi.NativeLibraryLoader; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; + +/** + * FFM bridge for the Foyer block cache lifecycle. + * + *

      Exposes two operations: {@link #createCache} and {@link #destroyCache}. + * These map to the {@code foyer_create_cache} and {@code foyer_destroy_cache} + * symbols exported by the native library. + * + *

      Cache access operations ({@code get}, {@code put}, {@code evict}) are not + * exposed here — they are called directly from the native layer without + * crossing the Java boundary. + * + *

      {@link #createCache} returns an opaque {@code long} handle that represents + * the native cache instance. The handle must be passed to {@link #destroyCache} + * exactly once when the cache is no longer needed. + * + * @opensearch.experimental + */ +public final class FoyerBridge { + + private static final Logger logger = LogManager.getLogger(FoyerBridge.class); + + private static final MethodHandle FOYER_CREATE_CACHE; + private static final MethodHandle FOYER_DESTROY_CACHE; + + static { + SymbolLookup lib = NativeLibraryLoader.symbolLookup(); + Linker linker = Linker.nativeLinker(); + + // i64 foyer_create_cache(u64 disk_bytes, *const u8 dir_ptr, u64 dir_len, + // u64 block_size_bytes, + // *const u8 io_engine_ptr, u64 io_engine_len) + FOYER_CREATE_CACHE = linker.downcallHandle( + lib.find("foyer_create_cache").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, // return: opaque i64 handle + ValueLayout.JAVA_LONG, // disk_bytes: u64 + ValueLayout.ADDRESS, // dir_ptr: *const u8 + ValueLayout.JAVA_LONG, // dir_len: u64 + ValueLayout.JAVA_LONG, // block_size_bytes: u64 + ValueLayout.ADDRESS, // io_engine_ptr: *const u8 + ValueLayout.JAVA_LONG // io_engine_len: u64 + ) + ); + + // i64 foyer_destroy_cache(i64 ptr) — 0=success, <0=error pointer + FOYER_DESTROY_CACHE = linker.downcallHandle( + lib.find("foyer_destroy_cache").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, // return: 0=ok, <0=error + ValueLayout.JAVA_LONG // ptr + ) + ); + logger.info("FFM downcall handles resolved: foyer_create_cache, foyer_destroy_cache"); + } + + /** + * Create a Foyer block cache. + * + * @param diskBytes maximum disk space the cache may use, in bytes + * @param diskDir path to the directory where Foyer stores cache data + * @param blockSizeBytes Foyer disk block size in bytes (see {@code format_cache.block_size}) + * @param ioEngine I/O engine: {@code "auto"}, {@code "io_uring"}, or {@code "psync"} + * (see {@code format_cache.io_engine}) + * @return an opaque handle representing the cache instance; always positive on success + * @throws RuntimeException if the native call fails or the directory is invalid + */ + public static long createCache(long diskBytes, String diskDir, long blockSizeBytes, String ioEngine) { + try (var call = new NativeCall()) { + var dir = call.str(diskDir); + var engine = call.str(ioEngine); + long ptr = call.invoke(FOYER_CREATE_CACHE, diskBytes, dir.segment(), dir.len(), blockSizeBytes, engine.segment(), engine.len()); + if (ptr <= 0) { + throw new IllegalStateException("foyer_create_cache returned invalid pointer: " + ptr); + } + logger.info( + "Foyer block cache created: diskBytes={}, blockSizeBytes={}, ioEngine={}, dir={}", + diskBytes, + blockSizeBytes, + ioEngine, + diskDir + ); + return ptr; + } + } + + /** + * Destroy a cache previously created by {@link #createCache}. + * + *

      After this call the handle is invalid and must not be used again. + * + * @param ptr the handle returned by {@link #createCache} + * @throws RuntimeException if the native call returns an error (invalid ptr) + */ + public static void destroyCache(long ptr) { + try (var call = new NativeCall()) { + call.invoke(FOYER_DESTROY_CACHE, ptr); + } + logger.info("Foyer block cache destroyed"); + } + + private FoyerBridge() {} +} diff --git a/sandbox/plugins/block-cache-foyer/src/main/rust/Cargo.toml b/sandbox/plugins/block-cache-foyer/src/main/rust/Cargo.toml new file mode 100644 index 0000000000000..6b7d398c910db --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/main/rust/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "opensearch-block-cache" +version = "0.1.0" +edition = "2021" +description = "Disk block cache backed by Foyer — rlib member of the dataformat-native workspace" +license = "Apache-2.0" +workspace = "../../../../../libs/dataformat-native/rust" + +[lib] +name = "opensearch_block_cache" +crate-type = ["rlib"] + +[dependencies] +foyer = "0.22" +bytes = { workspace = true } +dashmap = { workspace = true } +tokio = { workspace = true } +log = { workspace = true } +native-bridge-common = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/ffm.rs b/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/ffm.rs new file mode 100644 index 0000000000000..c7a7a54412974 --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/ffm.rs @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! FFM lifecycle entry points exported to Java. + +use std::sync::Arc; +use native_bridge_common::ffm_safe; +use crate::foyer::foyer_cache::FoyerCache; + +/// Create a [`FoyerCache`] and return an opaque `Arc` handle as `i64`. +/// +/// # Parameters +/// - `disk_bytes` — total disk capacity in bytes. +/// - `dir_ptr` / `dir_len` — UTF-8 path to the cache directory. +/// - `block_size_bytes` — Foyer disk block size in bytes. Must be ≥ the largest +/// entry ever put into the cache. Set via `format_cache.block_size` (default 64 MB). +/// - `io_engine_ptr` / `io_engine_len` — I/O engine selection: `"auto"`, +/// `"io_uring"`, or `"psync"`. Set via `format_cache.io_engine` (default `"auto"`). +/// +/// # Safety +/// `dir_ptr` must point to `dir_len` consecutive valid UTF-8 bytes. +/// `io_engine_ptr` must point to `io_engine_len` consecutive valid UTF-8 bytes. +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn foyer_create_cache( + disk_bytes: u64, + dir_ptr: *const u8, + dir_len: u64, + block_size_bytes: u64, + io_engine_ptr: *const u8, + io_engine_len: u64, +) -> i64 { + if dir_ptr.is_null() { + return Err("dir_ptr is null".to_string()); + } + let dir = std::str::from_utf8(std::slice::from_raw_parts(dir_ptr, dir_len as usize)) + .map_err(|e| format!("invalid UTF-8 in dir path: {}", e))?; + let io_engine = if io_engine_ptr.is_null() { + "auto" + } else { + std::str::from_utf8(std::slice::from_raw_parts(io_engine_ptr, io_engine_len as usize)) + .unwrap_or("auto") + }; + Ok(Arc::into_raw(Arc::new(FoyerCache::new( + disk_bytes as usize, + dir, + block_size_bytes as usize, + io_engine, + ))) as i64) +} + +/// Destroy a [`FoyerCache`] previously created by [`foyer_create_cache`]. +/// +/// Returns `0` on success, `< 0` (error pointer) if `ptr` is invalid. +/// +/// # Safety +/// `ptr` must be a value returned by [`foyer_create_cache`] not yet destroyed. +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn foyer_destroy_cache(ptr: i64) -> i64 { + if ptr <= 0 { + return Err(format!("foyer_destroy_cache: invalid ptr {}", ptr)); + } + drop(Arc::from_raw(ptr as *const FoyerCache)); + Ok(0) +} diff --git a/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/foyer_cache.rs b/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/foyer_cache.rs new file mode 100644 index 0000000000000..3b32d92af61fd --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/foyer_cache.rs @@ -0,0 +1,271 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! [`FoyerCache`] — a [`PageCache`] implementation backed by Foyer. + +use std::path::PathBuf; +use std::sync::Arc; +use bytes::Bytes; +use dashmap::DashMap; +use foyer::{BlockEngineConfig, DeviceBuilder, Event, EventListener, FsDeviceBuilder, + HybridCache, HybridCacheBuilder, IoEngineConfig, PsyncIoEngineConfig}; +#[cfg(target_os = "linux")] +use foyer::UringIoEngineConfig; + +use crate::range_cache::{CacheKey, SEPARATOR}; +use crate::traits::PageCache; + +// ── I/O engine selection ────────────────────────────────────────────────────── + +/// Return `true` if the running Linux kernel is >= `(major, minor)`. +/// +/// Reads `/proc/sys/kernel/osrelease` (e.g. `"5.15.0-91-generic"`) and +/// compares the major/minor version numbers. Returns `false` on any parse +/// error so the caller can fall back safely. +#[cfg(target_os = "linux")] +fn kernel_version_at_least(required_major: u32, required_minor: u32) -> bool { + let release = match std::fs::read_to_string("/proc/sys/kernel/osrelease") { + Ok(s) => s, + Err(_) => return false, + }; + let mut parts = release.trim().split('.'); + let major: u32 = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0); + let minor: u32 = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0); + major > required_major || (major == required_major && minor >= required_minor) +} + +/// Select the I/O engine based on the operator-configured `choice`. +/// +/// | `choice` | Behaviour | +/// |------------|-----------| +/// | `"auto"` | Detect at runtime: io_uring on Linux ≥ 5.1, psync otherwise (default). | +/// | `"io_uring"` | Force io_uring. Fails at node startup if io_uring is unavailable (e.g. blocked by seccomp/AppArmor in locked-down container environments). | +/// | `"psync"` | Force synchronous pread/pwrite. Use when io_uring is restricted or when predictable syscall-level profiling is needed. | +/// +/// Invalid values are treated as `"auto"` with a warning. +fn build_io_engine_config(choice: &str) -> Box { + match choice { + "io_uring" => { + log::info!("[block-cache] io_engine=io_uring forced by config"); + #[cfg(target_os = "linux")] + return UringIoEngineConfig::new().boxed(); + #[cfg(not(target_os = "linux"))] + panic!("[block-cache] io_engine=io_uring requested but io_uring is not supported on non-Linux platforms"); + } + "psync" => { + log::info!("[block-cache] io_engine=psync forced by config"); + return PsyncIoEngineConfig::new().boxed(); + } + other => { + if other != "auto" { + log::warn!("[block-cache] unknown io_engine='{}'; falling back to auto-detect", other); + } + // "auto" — detect by kernel version (existing logic) + #[cfg(target_os = "linux")] + { + let release = std::fs::read_to_string("/proc/sys/kernel/osrelease") + .unwrap_or_else(|_| "unknown".to_string()); + let release = release.trim(); + if kernel_version_at_least(5, 1) { + log::info!( + "[block-cache] kernel {} — io_uring available, using UringIoEngineConfig", + release + ); + return UringIoEngineConfig::new().boxed(); + } else { + log::warn!( + "[block-cache] kernel {} — io_uring unavailable (requires >= 5.1), \ + falling back to PsyncIoEngineConfig", + release + ); + } + } + PsyncIoEngineConfig::new().boxed() + } + } +} + +// ── Key index eviction listener ─────────────────────────────────────────────── + +/// Foyer event listener that removes evicted keys from the key index. +/// +/// Shared between [`FoyerCache`] and Foyer via `Arc`. When Foyer evicts, +/// replaces, or removes an entry, `on_leave` is called, which removes the key +/// from the prefix-to-keys index. This prevents `key_index` from growing +/// unbounded as Foyer's LRU evicts entries from disk. +/// +/// # Key index prefix extraction +/// +/// The index key is derived by splitting each cache key on [`SEPARATOR`]. +/// Keys that contain `SEPARATOR` (range entries) use everything before it as +/// the index key. +struct KeyIndexListener { + key_index: Arc>>, +} + +impl EventListener for KeyIndexListener { + type Key = String; + type Value = Vec; + + fn on_leave(&self, reason: Event, key: &String, _value: &Vec) { + match reason { + Event::Evict | Event::Replace | Event::Remove => { + let index_key = if let Some(sep_pos) = key.find(SEPARATOR) { + &key[..sep_pos] + } else { + key.as_str() + }; + if let Some(mut keys) = self.key_index.get_mut(index_key) { + keys.retain(|k| k != key); + if keys.is_empty() { + drop(keys); + self.key_index.remove(index_key); + } + } + } + Event::Clear => {} + } + } +} + +// ── FoyerCache ──────────────────────────────────────────────────────────────── + +/// Disk block cache with prefix-based eviction support backed by Foyer. +/// +/// Wraps a Foyer [`HybridCache`] configured as a disk-only store, together +/// with a concurrent key index that maps each index prefix to its cached entry +/// keys. The key index allows removing all cached entries sharing a common +/// prefix in O(n) without requiring Foyer to support prefix-scan semantics. +/// +/// Keys are opaque strings supplied by the caller. The index key is derived as +/// everything before the first [`SEPARATOR`]. See [`PageCache`] for key format +/// conventions. +/// +/// The key index is kept in sync with Foyer's internal state via an +/// [`EventListener`] — stale keys are removed automatically when Foyer evicts +/// entries via LRU. +/// +/// Thread-safe: both [`HybridCache`] and [`DashMap`] are `Send + Sync`. +pub struct FoyerCache { + inner: HybridCache>, + /// Maps each index prefix to the list of Foyer keys stored under that prefix. + /// Shared with [`KeyIndexListener`] for automatic stale-key removal. + pub(crate) key_index: Arc>>, + /// Keeps the Tokio runtime alive for the lifetime of the cache. + _runtime: Arc, +} + +impl FoyerCache { + /// Initialise the cache synchronously. + /// + /// # Parameters + /// - `disk_bytes` — total disk capacity for this cache. + /// - `disk_dir` — directory on the local SSD where Foyer stores its data files. + /// - `block_size_bytes` — Foyer disk block size. Must be ≥ the largest entry ever + /// put into the cache. Configurable via `format_cache.block_size`. + /// - `io_engine` — I/O engine selection: `"auto"`, `"io_uring"`, or `"psync"`. + /// Configurable via `format_cache.io_engine`. + /// + /// # Panics + /// Panics if the Tokio runtime cannot be created or if Foyer fails to + /// build the cache (e.g. insufficient disk space or invalid path). + pub fn new( + disk_bytes: usize, + disk_dir: impl Into, + block_size_bytes: usize, + io_engine: &str, + ) -> Self { + let disk_dir = disk_dir.into(); + let key_index: Arc>> = Arc::new(DashMap::new()); + let listener = Arc::new(KeyIndexListener { key_index: Arc::clone(&key_index) }); + + let rt = tokio::runtime::Runtime::new() + .expect("[block-cache] failed to create Tokio runtime"); + let dir_clone = disk_dir.clone(); + let io_engine = io_engine.to_string(); + let io_engine_for_log = io_engine.clone(); // clone for use in log after the closure + let inner = rt.block_on(async move { + HybridCacheBuilder::>::new() + .with_name("block-cache") + .with_event_listener(listener) + .memory(1) + // Disable the in-memory tier — this cache is disk-only. + // Foyer is a hybrid (DRAM + disk) cache; setting the memory capacity + // to 1 byte opts out of DRAM caching. All entries go directly to the + // disk tier (FsDevice) below. + .storage() + .with_io_engine_config(build_io_engine_config(&io_engine)) + .with_engine_config( + // block_size must be >= the largest entry ever put into the cache. + // DataFusion reads Parquet row groups of up to 64 MB; Lucene blocks are + // also 64 MB. A block_size smaller than the entry causes a silent drop + // (put succeeds but entry is not stored, resulting in a cache miss). + // Configurable via format_cache.block_size (default: 64 MB). + BlockEngineConfig::new( + FsDeviceBuilder::new(dir_clone) + .with_capacity(disk_bytes) + .build() + .expect("[block-cache] FsDevice build failed") + ) + .with_block_size(block_size_bytes) + ) + .build() + .await + .expect("[block-cache] HybridCache build failed") + }); + log::info!( + "[block-cache] ready: disk={}B, block_size={}B, io_engine={}, dir={}", + disk_bytes, block_size_bytes, io_engine_for_log, disk_dir.display() + ); + Self { inner, key_index, _runtime: Arc::new(rt) } + } + + /// Derive the index key from a cache key: everything before the first [`SEPARATOR`]. + /// For keys without [`SEPARATOR`] (e.g. Lucene block paths), the full key is its + /// own index entry. + fn index_key(key: &str) -> &str { + if let Some(pos) = key.find(SEPARATOR) { &key[..pos] } else { key } + } +} + +impl PageCache for FoyerCache { + async fn get(&self, key: &CacheKey) -> Option { + match self.inner.get(&key.as_str().to_string()).await { + Ok(Some(e)) => Some(Bytes::copy_from_slice(e.value())), + _ => None, + } + } + + fn put(&self, key: &CacheKey, data: Bytes) { + let raw = key.as_str(); + let k = raw.to_string(); + self.inner.insert(k.clone(), data.to_vec()); + let idx = Self::index_key(raw).to_string(); + self.key_index.entry(idx).or_default().push(k); + } + + fn evict_prefix(&self, prefix: &str) { + // Collect all index entries whose key starts with `prefix` + let matching: Vec = self.key_index + .iter() + .filter(|e| e.key().starts_with(prefix)) + .map(|e| e.key().clone()) + .collect(); + + for idx_key in matching { + if let Some((_, keys)) = self.key_index.remove(&idx_key) { + for k in keys { self.inner.remove(&k); } + } + } + } + + async fn clear(&self) { + self.key_index.clear(); + let _ = self.inner.clear().await; + } +} diff --git a/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/mod.rs b/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/mod.rs new file mode 100644 index 0000000000000..28be08976f535 --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/mod.rs @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pub mod foyer_cache; +pub mod ffm; + +#[cfg(test)] +mod tests; diff --git a/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/tests.rs b/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/tests.rs new file mode 100644 index 0000000000000..675b46b1768d8 --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/main/rust/src/foyer/tests.rs @@ -0,0 +1,409 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Unit tests for [`FoyerCache`] and the FFM lifecycle API. + +use std::sync::Arc; +use bytes::Bytes; +use tempfile::TempDir; + +use crate::foyer::foyer_cache::FoyerCache; +use crate::foyer::ffm::{foyer_create_cache, foyer_destroy_cache}; +use crate::range_cache::range_cache_key; +use crate::traits::PageCache; + +// ── Test helpers ────────────────────────────────────────────────────────────── + +const BLOCK_SIZE: usize = 64 * 1024 * 1024; // 64 MB default for tests +const IO_ENGINE: &str = "auto"; + +fn test_cache() -> (FoyerCache, TempDir) { + let dir = TempDir::new().expect("failed to create temp dir"); + let cache = FoyerCache::new(64 * 1024 * 1024, dir.path(), BLOCK_SIZE, IO_ENGINE); + (cache, dir) +} + +fn put_range(cache: &FoyerCache, path: &str, start: u64, end: u64, data: &[u8]) { + cache.put(&range_cache_key(path, start, end), Bytes::copy_from_slice(data)); +} + +fn block_on(f: F) -> F::Output { + tokio::runtime::Runtime::new().expect("test runtime").block_on(f) +} + +// ── put + get round-trip ────────────────────────────────────────────────────── + +#[test] +fn get_returns_exact_bytes_that_were_put() { + let (cache, _dir) = test_cache(); + let data = b"hello foyer cache"; + let key = range_cache_key("/data/file.parquet", 0, 100); + cache.put(&key, Bytes::from_static(data)); + let result = block_on(cache.get(&key)); + assert_eq!(result.as_deref(), Some(data.as_slice())); +} + +#[test] +fn multiple_ranges_for_same_file_are_independent() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/a.parquet", 0, 4096, b"range0"); + put_range(&cache, "/data/a.parquet", 4096, 8192, b"range1"); + put_range(&cache, "/data/a.parquet", 8192, 12288, b"range2"); + assert_eq!(block_on(cache.get(&range_cache_key("/data/a.parquet", 0, 4096))).as_deref(), Some(b"range0".as_slice())); + assert_eq!(block_on(cache.get(&range_cache_key("/data/a.parquet", 4096, 8192))).as_deref(), Some(b"range1".as_slice())); + assert_eq!(block_on(cache.get(&range_cache_key("/data/a.parquet", 8192, 12288))).as_deref(), Some(b"range2".as_slice())); +} + +#[test] +fn multiple_files_are_independent() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/a.parquet", 0, 100, b"file_a"); + put_range(&cache, "/data/b.parquet", 0, 100, b"file_b"); + put_range(&cache, "/data/c.parquet", 0, 100, b"file_c"); + assert_eq!(block_on(cache.get(&range_cache_key("/data/a.parquet", 0, 100))).as_deref(), Some(b"file_a".as_slice())); + assert_eq!(block_on(cache.get(&range_cache_key("/data/b.parquet", 0, 100))).as_deref(), Some(b"file_b".as_slice())); + assert_eq!(block_on(cache.get(&range_cache_key("/data/c.parquet", 0, 100))).as_deref(), Some(b"file_c".as_slice())); +} + +#[test] +fn large_value_round_trips_correctly() { + let (cache, _dir) = test_cache(); + let data: Vec = (0u32..1_000_000).map(|i| (i % 251) as u8).collect(); + let key = range_cache_key("/data/large.parquet", 0, data.len() as u64); + cache.put(&key, Bytes::copy_from_slice(&data)); + let result = block_on(cache.get(&key)).expect("large value should be retrievable"); + assert_eq!(result.as_ref(), data.as_slice()); +} + +#[test] +fn put_same_key_twice_replaces_value() { + let (cache, _dir) = test_cache(); + let key = range_cache_key("/data/file.parquet", 0, 100); + cache.put(&key, Bytes::from_static(b"original")); + cache.put(&key, Bytes::from_static(b"updated")); + let result = block_on(cache.get(&key)); + assert_eq!(result.as_deref(), Some(b"updated".as_slice())); +} + +// ── get miss cases ──────────────────────────────────────────────────────────── + +#[test] +fn get_returns_none_for_unknown_key() { + let (cache, _dir) = test_cache(); + let result = block_on(cache.get(&range_cache_key("/never/inserted.parquet", 0, 100))); + assert!(result.is_none()); +} + +#[test] +fn get_returns_none_for_wrong_range_on_known_path() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/file.parquet", 0, 100, b"data"); + assert!(block_on(cache.get(&range_cache_key("/data/file.parquet", 1, 100))).is_none()); + assert!(block_on(cache.get(&range_cache_key("/data/file.parquet", 0, 99))).is_none()); + assert!(block_on(cache.get(&range_cache_key("/data/file.parquet", 200, 300))).is_none()); +} + +// ── evict_prefix ────────────────────────────────────────────────────────────── + +#[test] +fn evict_prefix_removes_all_ranges_for_file() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/target.parquet", 0, 4096, b"range0"); + put_range(&cache, "/data/target.parquet", 4096, 8192, b"range1"); + put_range(&cache, "/data/target.parquet", 8192, 12288, b"range2"); + cache.evict_prefix("/data/target.parquet"); + assert!(!cache.key_index.contains_key("/data/target.parquet")); + put_range(&cache, "/data/target.parquet", 0, 4096, b"new"); + assert_eq!( + block_on(cache.get(&range_cache_key("/data/target.parquet", 0, 4096))), + Some(Bytes::from_static(b"new")) + ); +} + +#[test] +fn evict_prefix_does_not_affect_other_files() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/target.parquet", 0, 100, b"target"); + put_range(&cache, "/data/other.parquet", 0, 100, b"other"); + cache.evict_prefix("/data/target.parquet"); + assert!(block_on(cache.get(&range_cache_key("/data/other.parquet", 0, 100))).is_some()); + assert!(block_on(cache.get(&range_cache_key("/data/target.parquet", 0, 100))).is_none()); +} + +#[test] +fn evict_prefix_on_nonexistent_prefix_is_noop() { + let (cache, _dir) = test_cache(); + cache.evict_prefix("/never/inserted.parquet"); + cache.evict_prefix(""); +} + +#[test] +fn evict_prefix_twice_is_safe() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/file.parquet", 0, 100, b"data"); + cache.evict_prefix("/data/file.parquet"); + cache.evict_prefix("/data/file.parquet"); +} + +#[test] +fn after_evict_prefix_new_put_is_retrievable() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/file.parquet", 0, 100, b"first"); + cache.evict_prefix("/data/file.parquet"); + put_range(&cache, "/data/file.parquet", 0, 100, b"second"); + let result = block_on(cache.get(&range_cache_key("/data/file.parquet", 0, 100))); + assert_eq!(result.as_deref(), Some(b"second".as_slice())); +} + +// ── clear ───────────────────────────────────────────────────────────────────── + +#[test] +fn clear_removes_all_entries() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/a.parquet", 0, 100, b"a"); + put_range(&cache, "/data/b.parquet", 0, 100, b"b"); + block_on(cache.clear()); + assert!(block_on(cache.get(&range_cache_key("/data/a.parquet", 0, 100))).is_none()); + assert!(block_on(cache.get(&range_cache_key("/data/b.parquet", 0, 100))).is_none()); +} + +#[test] +fn clear_on_empty_cache_is_safe() { + let (cache, _dir) = test_cache(); + block_on(cache.clear()); +} + +#[test] +fn cache_is_usable_after_clear() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/file.parquet", 0, 100, b"before"); + block_on(cache.clear()); + put_range(&cache, "/data/file.parquet", 0, 100, b"after"); + let result = block_on(cache.get(&range_cache_key("/data/file.parquet", 0, 100))); + assert_eq!(result.as_deref(), Some(b"after".as_slice())); +} + +// ── key_index integrity ─────────────────────────────────────────────────────── + +#[test] +fn key_index_is_empty_after_clear() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/a.parquet", 0, 100, b"a"); + put_range(&cache, "/data/b.parquet", 0, 100, b"b"); + block_on(cache.clear()); + assert!(cache.key_index.is_empty()); +} + +#[test] +fn key_index_has_no_entry_for_evicted_file() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/target.parquet", 0, 100, b"data"); + put_range(&cache, "/data/other.parquet", 0, 100, b"other"); + cache.evict_prefix("/data/target.parquet"); + assert!(!cache.key_index.contains_key("/data/target.parquet")); + assert!(cache.key_index.contains_key("/data/other.parquet")); +} + +// ── concurrent access ───────────────────────────────────────────────────────── + +#[test] +fn concurrent_puts_to_different_files_do_not_corrupt() { + let (cache, _dir) = test_cache(); + let cache = Arc::new(cache); + let handles: Vec<_> = (0..16).map(|i| { + let cache = Arc::clone(&cache); + std::thread::spawn(move || { + let key = range_cache_key(&format!("/data/file_{}.parquet", i), 0, 1024); + cache.put(&key, Bytes::copy_from_slice(&vec![i as u8; 1024])); + }) + }).collect(); + for h in handles { h.join().expect("thread panicked"); } + for i in 0u8..16 { + let key = range_cache_key(&format!("/data/file_{}.parquet", i), 0, 1024); + let result = block_on(cache.get(&key)).expect("entry should be retrievable"); + assert!(result.iter().all(|&b| b == i)); + } +} + +#[test] +fn concurrent_put_and_get_same_file_does_not_panic() { + let (cache, _dir) = test_cache(); + let cache = Arc::new(cache); + let writer_cache = Arc::clone(&cache); + let writer = std::thread::spawn(move || { + for i in 0u64..100 { + let key = range_cache_key("/data/shared.parquet", i * 100, (i + 1) * 100); + writer_cache.put(&key, Bytes::from_static(b"data")); + } + }); + let reader_cache = Arc::clone(&cache); + let reader = std::thread::spawn(move || { + for i in 0u64..100 { + let key = range_cache_key("/data/shared.parquet", i * 100, (i + 1) * 100); + let _ = block_on(reader_cache.get(&key)); + } + }); + writer.join().expect("writer panicked"); + reader.join().expect("reader panicked"); +} + +#[test] +fn concurrent_evict_and_put_does_not_panic() { + let (cache, _dir) = test_cache(); + let cache = Arc::new(cache); + let writer_cache = Arc::clone(&cache); + let writer = std::thread::spawn(move || { + for i in 0u64..50 { + let key = range_cache_key("/data/file.parquet", i * 100, (i + 1) * 100); + writer_cache.put(&key, Bytes::from_static(b"data")); + } + }); + let evictor_cache = Arc::clone(&cache); + let evictor = std::thread::spawn(move || { + for _ in 0..50 { evictor_cache.evict_prefix("/data/file.parquet"); } + }); + writer.join().expect("writer panicked"); + evictor.join().expect("evictor panicked"); +} + +// ── disk / capacity cases ───────────────────────────────────────────────────── + +#[test] +fn put_and_get_work_after_cache_nears_capacity() { + let dir = TempDir::new().unwrap(); + let cache = FoyerCache::new(1 * 1024 * 1024, dir.path(), BLOCK_SIZE, IO_ENGINE); + let chunk = vec![0u8; 512 * 1024]; + for i in 0u64..4 { + let key = range_cache_key("/data/file.parquet", i * 524288, (i + 1) * 524288); + cache.put(&key, Bytes::copy_from_slice(&chunk)); + } + let fresh_key = range_cache_key("/data/file.parquet", 0, 100); + cache.put(&fresh_key, Bytes::from_static(b"fresh")); + let result = block_on(cache.get(&fresh_key)); + assert_eq!(result.as_deref(), Some(b"fresh".as_slice())); +} + +// ── KeyIndexListener behaviour ──────────────────────────────────────────────── + +#[test] +fn lru_eviction_removes_stale_keys_from_key_index() { + let dir = TempDir::new().unwrap(); + let cache = FoyerCache::new(1 * 1024 * 1024, dir.path(), BLOCK_SIZE, IO_ENGINE); + const CHUNK_SIZE: usize = 256 * 1024; + const TOTAL_WRITES: usize = 8; + let chunk = vec![0xABu8; CHUNK_SIZE]; + for i in 0u64..TOTAL_WRITES as u64 { + let key = range_cache_key("/data/big.parquet", i * CHUNK_SIZE as u64, (i + 1) * CHUNK_SIZE as u64); + cache.put(&key, Bytes::copy_from_slice(&chunk)); + } + std::thread::sleep(std::time::Duration::from_millis(500)); + let key_count = cache.key_index.get("/data/big.parquet").map(|v| v.len()).unwrap_or(0); + assert!(key_count < TOTAL_WRITES, "expected < {} entries after LRU eviction; got {}", TOTAL_WRITES, key_count); +} + +#[test] +fn replace_event_does_not_duplicate_key_in_key_index() { + let (cache, _dir) = test_cache(); + let key = range_cache_key("/data/file.parquet", 0, 100); + cache.put(&key, Bytes::from_static(b"version_1")); + cache.put(&key, Bytes::from_static(b"version_2")); + std::thread::sleep(std::time::Duration::from_millis(100)); + let count = cache.key_index.get("/data/file.parquet").map(|v| v.len()).unwrap_or(0); + assert_eq!(count, 1, "same key put twice should result in 1 key_index entry; got {}", count); + let result = block_on(cache.get(&key)); + assert_eq!(result.as_deref(), Some(b"version_2".as_slice())); +} + +#[test] +fn event_remove_after_evict_prefix_does_not_panic_or_corrupt_key_index() { + let (cache, _dir) = test_cache(); + put_range(&cache, "/data/file.parquet", 0, 100, b"data"); + put_range(&cache, "/data/file.parquet", 100, 200, b"more"); + cache.evict_prefix("/data/file.parquet"); + std::thread::sleep(std::time::Duration::from_millis(100)); + assert!(!cache.key_index.contains_key("/data/file.parquet")); + put_range(&cache, "/data/file.parquet", 0, 100, b"fresh"); + assert_eq!(block_on(cache.get(&range_cache_key("/data/file.parquet", 0, 100))).as_deref(), Some(b"fresh".as_slice())); +} + +// ── FFM lifecycle ───────────────────────────────────────────────────────────── + +#[test] +fn ffm_create_returns_positive_pointer() { + let dir = TempDir::new().unwrap(); + let dir_str = dir.path().to_str().unwrap(); + let engine = IO_ENGINE.as_bytes(); + let ptr = unsafe { foyer_create_cache( + 64 * 1024 * 1024, + dir_str.as_ptr(), dir_str.len() as u64, + BLOCK_SIZE as u64, + engine.as_ptr(), engine.len() as u64, + )}; + assert!(ptr > 0); + let result = unsafe { foyer_destroy_cache(ptr) }; + assert_eq!(result, 0); +} + +#[test] +fn ffm_create_with_null_ptr_returns_error() { + let engine = IO_ENGINE.as_bytes(); + let ptr = unsafe { foyer_create_cache( + 64 * 1024 * 1024, + std::ptr::null(), 10, + BLOCK_SIZE as u64, + engine.as_ptr(), engine.len() as u64, + )}; + assert!(ptr < 0); + if ptr < 0 { unsafe { native_bridge_common::error::native_error_free(-ptr); } } +} + +#[test] +fn ffm_create_with_invalid_utf8_returns_error() { + let invalid_utf8 = [0xFF, 0xFE, 0xFD]; + let engine = IO_ENGINE.as_bytes(); + let ptr = unsafe { foyer_create_cache( + 64 * 1024 * 1024, + invalid_utf8.as_ptr(), invalid_utf8.len() as u64, + BLOCK_SIZE as u64, + engine.as_ptr(), engine.len() as u64, + )}; + assert!(ptr < 0); + if ptr < 0 { unsafe { native_bridge_common::error::native_error_free(-ptr); } } +} + +#[test] +fn ffm_destroy_with_zero_ptr_returns_error() { + let result = unsafe { foyer_destroy_cache(0) }; + assert!(result < 0); + if result < 0 { unsafe { native_bridge_common::error::native_error_free(-result); } } +} + +#[test] +fn ffm_destroy_with_negative_ptr_returns_error() { + let result = unsafe { foyer_destroy_cache(-1) }; + assert!(result < 0); + if result < 0 { unsafe { native_bridge_common::error::native_error_free(-result); } } +} + +#[test] +fn ffm_create_destroy_lifecycle_no_leak() { + let engine = IO_ENGINE.as_bytes(); + for _ in 0..3 { + let dir = TempDir::new().unwrap(); + let dir_str = dir.path().to_str().unwrap(); + let ptr = unsafe { foyer_create_cache( + 16 * 1024 * 1024, + dir_str.as_ptr(), dir_str.len() as u64, + BLOCK_SIZE as u64, + engine.as_ptr(), engine.len() as u64, + )}; + assert!(ptr > 0); + let result = unsafe { foyer_destroy_cache(ptr) }; + assert_eq!(result, 0); + } +} diff --git a/sandbox/plugins/block-cache-foyer/src/main/rust/src/lib.rs b/sandbox/plugins/block-cache-foyer/src/main/rust/src/lib.rs new file mode 100644 index 0000000000000..0d66471395d19 --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/main/rust/src/lib.rs @@ -0,0 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pub mod range_cache; +pub mod traits; +pub mod foyer; diff --git a/sandbox/plugins/block-cache-foyer/src/main/rust/src/range_cache.rs b/sandbox/plugins/block-cache-foyer/src/main/rust/src/range_cache.rs new file mode 100644 index 0000000000000..10854b74957e9 --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/main/rust/src/range_cache.rs @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Cache key helpers for [`PageCache`] consumers. +//! +//! ## Enforced key construction +//! +//! [`PageCache::get`] and [`PageCache::put`] accept [`CacheKey`], not `&str`. +//! [`CacheKey`] has no public constructor and no `From<&str>` impl — raw strings +//! are rejected at compile time. Callers must use the helpers in this module. +//! +//! ## Key conventions +//! +//! - **Range entries** (byte-range reads): key = `"path\x1Fstart-end"`. +//! Use [`range_cache_key`] to build the key; pass `path` directly to +//! [`PageCache::evict_prefix`] to evict all ranges for a file. +//! +//! - **Block entries** (fixed-size block reads, e.g. Lucene): key = full block +//! path (already unique, no separator needed). Pass the block path directly to +//! `put`/`get`, and the segment base path to [`PageCache::evict_prefix`] to +//! evict all blocks for a segment. A `block_cache_key()` helper will be added +//! when the Lucene cache consumer is integrated. +//! +//! Add new key-format helpers here as additional cache consumers are integrated. +//! +//! [`PageCache`]: crate::traits::PageCache +//! [`PageCache::get`]: crate::traits::PageCache::get +//! [`PageCache::put`]: crate::traits::PageCache::put +//! [`PageCache::evict_prefix`]: crate::traits::PageCache::evict_prefix + +/// The separator between a file path and its byte-range suffix in range keys. +/// +/// `\x1F` (ASCII Unit Separator, decimal 31) cannot appear in any filesystem +/// path or object-store URL — S3/GCS/Azure percent-encode it as `%1F`. +/// +/// Used by [`range_cache_key`] when building keys, and by [`FoyerCache`] +/// internally when parsing keys to derive the index prefix. +/// +/// [`FoyerCache`]: crate::foyer::foyer_cache::FoyerCache +pub(crate) const SEPARATOR: char = '\x1f'; + +// ── CacheKey newtype ────────────────────────────────────────────────────────── + +/// Opaque cache key. +/// +/// Cannot be constructed from a raw string — use the helpers in this module +/// (e.g. [`range_cache_key`]). This enforces the [`SEPARATOR`] convention at +/// compile time: any caller that tries to pass a `&str` directly to +/// [`PageCache::get`] or [`PageCache::put`] will get a compile error. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CacheKey(String); + +impl CacheKey { + /// Return the inner string representation of this key. + /// + /// Use this in doc-tests and when passing the key to Foyer internals. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +// ── Range entry helpers ─────────────────────────────────────────────────────── + +/// Build a cache key for a byte-range read. +/// +/// Key format: `"path\x1Fstart-end"`. +/// +/// # Example +/// ``` +/// use opensearch_page_cache::range_cache::range_cache_key; +/// let key = range_cache_key("data/nodes/0/_0.parquet", 0, 4096); +/// assert_eq!(key.as_str(), "data/nodes/0/_0.parquet\x1f0-4096"); +/// ``` +pub fn range_cache_key(path: &str, start: u64, end: u64) -> CacheKey { + CacheKey(format!("{}{}{}-{}", path, SEPARATOR, start, end)) +} + +// ── Future key-format helpers ───────────────────────────────────────────────── +// Add new helpers here when additional cache consumers are integrated. +// For example, block_cache_key() for Lucene IndexInput block caching. + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn range_key_contains_separator() { + let key = range_cache_key("/data/file.parquet", 0, 4096); + assert_eq!(key.as_str(), "/data/file.parquet\x1f0-4096"); + } + + #[test] + fn range_keys_for_same_path_share_index_prefix() { + let k0 = range_cache_key("/data/file.parquet", 0, 4096); + let k1 = range_cache_key("/data/file.parquet", 4096, 8192); + assert!(k0.as_str().starts_with("/data/file.parquet")); + assert!(k1.as_str().starts_with("/data/file.parquet")); + } + + #[test] + fn range_keys_for_different_paths_do_not_share_prefix() { + let k = range_cache_key("/data/other.parquet", 0, 4096); + assert!(!k.as_str().starts_with("/data/file.parquet")); + } + +} diff --git a/sandbox/plugins/block-cache-foyer/src/main/rust/src/traits.rs b/sandbox/plugins/block-cache-foyer/src/main/rust/src/traits.rs new file mode 100644 index 0000000000000..4ba82fd42826a --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/main/rust/src/traits.rs @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! [`PageCache`] trait — the abstraction for disk caching with typed keys. + +use bytes::Bytes; +use crate::range_cache::CacheKey; + +/// A disk block cache. +/// +/// Keys are [`CacheKey`] values — opaque newtypes that can only be constructed +/// via the helpers in [`crate::range_cache`]. This enforces the `\x1F` separator +/// convention at compile time and prevents accidental use of raw strings. +/// +/// ## Eviction +/// +/// `evict_prefix` still accepts `&str` because the eviction prefix is the bare +/// file path (no separator) — there is nothing to encode, and any valid path +/// string is a correct eviction prefix. +/// +/// Implementations must be `Send + Sync` so they can be shared across async +/// tasks and threads. +pub trait PageCache: Send + Sync { + /// Look up a cached entry. Returns `Some(Bytes)` on hit, `None` on miss. + fn get(&self, key: &CacheKey) + -> impl std::future::Future> + Send; + + /// Insert bytes under the given key. + fn put(&self, key: &CacheKey, data: Bytes); + + /// Evict all entries whose key starts with `prefix`. A no-op if nothing matches. + /// + /// For range entries: pass the file path — evicts all byte-range keys for that file. + /// For block entries: pass the segment base path — evicts all block keys for that segment. + fn evict_prefix(&self, prefix: &str); + + /// Remove all entries from the cache. + fn clear(&self) -> impl std::future::Future + Send; +} diff --git a/sandbox/plugins/block-cache-foyer/src/test/java/org/opensearch/blockcache/foyer/BlockCacheFoyerPluginTests.java b/sandbox/plugins/block-cache-foyer/src/test/java/org/opensearch/blockcache/foyer/BlockCacheFoyerPluginTests.java new file mode 100644 index 0000000000000..11c5a65765d86 --- /dev/null +++ b/sandbox/plugins/block-cache-foyer/src/test/java/org/opensearch/blockcache/foyer/BlockCacheFoyerPluginTests.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.blockcache.foyer; + +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchTestCase; + +/** + * Unit tests for {@link BlockCacheFoyerPlugin}. + * + *

      Focuses on the pure-Java wiring of the plugin that does not require the + * native library: + *

        + *
      • Both constructor variants (no-arg and {@code Settings}-arg).
        • + *
      • {@link BlockCacheFoyerPlugin#getBlockCache()} returns + * {@code Optional.empty()} before {@code createComponents} has run.
        • + *
      + * + *

      Tests that exercise {@code createComponents} are out of scope here because + * it constructs a real {@link FoyerBlockCache} which requires the native + * library. Those paths are covered by integration tests. + */ +public class BlockCacheFoyerPluginTests extends OpenSearchTestCase { + + public void testNoArgConstructor() { + final BlockCacheFoyerPlugin plugin = new BlockCacheFoyerPlugin(); + assertNotNull(plugin); + assertTrue("handle is empty before createComponents", plugin.getBlockCache().isEmpty()); + } + + public void testSettingsConstructor() { + final BlockCacheFoyerPlugin plugin = new BlockCacheFoyerPlugin(Settings.EMPTY); + assertNotNull(plugin); + assertTrue(plugin.getBlockCache().isEmpty()); + } +} diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 0cdce74e2adc3..7323a3f1454b6 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -123,6 +123,7 @@ import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.remote.RemoteStorePressureSettings; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; +import org.opensearch.index.store.remote.filecache.BlockCacheSettings; import org.opensearch.index.store.remote.filecache.FileCacheSettings; import org.opensearch.indices.ClusterMergeSchedulerConfig; import org.opensearch.indices.IndexingMemoryController; @@ -766,6 +767,10 @@ public void apply(Settings value, Settings current, Settings previous) { TieringUtils.JVM_USAGE_TIERING_THRESHOLD_PERCENT, TieringUtils.FILECACHE_ACTIVE_USAGE_TIERING_THRESHOLD_PERCENT, + // Settings related to block cache + BlockCacheSettings.BLOCK_SIZE_SETTING, + BlockCacheSettings.IO_ENGINE_SETTING, + // Settings related to Remote Refresh Segment Pressure RemoteStorePressureSettings.REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED, RemoteStorePressureSettings.BYTES_LAG_VARIANCE_FACTOR, diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/BlockCacheSettings.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/BlockCacheSettings.java new file mode 100644 index 0000000000000..809ecb6e26673 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/BlockCacheSettings.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.remote.filecache; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; + +import java.util.Set; + +/** + * Settings for the node-level block cache backed by Foyer. + * + *

      All settings are {@link Setting.Property#NodeScope}: they are applied once at + * node startup when the cache is constructed, and require a node restart to take + * effect. The cache cannot be reconfigured on a live node. + * + *

      "Block" here is used in the storage sense — a contiguous, variable-size byte + * range read as an indivisible I/O unit — not a fixed-size disk sector. + * Entry granularity is determined by the calling layer (Parquet column chunks, + * Lucene segment files) and may range from kilobytes to tens of megabytes. + * + * @opensearch.experimental + */ +@ExperimentalApi +public final class BlockCacheSettings { + + /** + * Block size for the format cache disk tier. + * + *

      Must be ≥ the largest entry ever put into the cache. DataFusion reads + * Parquet row groups of up to 64 MB; Lucene blocks are also up to 64 MB. + * A block size smaller than an entry causes a silent drop — the put succeeds but + * the entry is not stored, resulting in a cache miss on the next read. + * + *

      Default: 64 MB. Range: [1 MB, 256 MB]. + * + *

      Configure in {@code opensearch.yml}: + * 

      {@code
+     * format_cache.block_size: 64mb
+     * }
      + */ + public static final Setting BLOCK_SIZE_SETTING = Setting.byteSizeSetting( + "format_cache.block_size", + new ByteSizeValue(64, ByteSizeUnit.MB), + new ByteSizeValue(1, ByteSizeUnit.MB), + new ByteSizeValue(256, ByteSizeUnit.MB), + Setting.Property.NodeScope + ); + + /** + * I/O engine for the format cache disk tier. + * + *
        + *
      • {@code auto} (default) — selects io_uring on Linux ≥ 5.1, + * falls back to psync otherwise.
        • + *
      • {@code io_uring} — force io_uring regardless of kernel detection. + * Fails at startup if io_uring is unavailable (e.g. blocked by seccomp + * or AppArmor in locked-down container environments).
        • + *
      • {@code psync} — force synchronous pread/pwrite. Use when io_uring is + * restricted or when predictable syscall-level profiling is needed.
        • + *
      + * + *

      Configure in {@code opensearch.yml}: + * 

      {@code
+     * format_cache.io_engine: auto
+     * }
      + */ + public static final Setting IO_ENGINE_SETTING = new Setting<>("format_cache.io_engine", "auto", value -> { + if (!Set.of("auto", "io_uring", "psync").contains(value)) { + throw new IllegalArgumentException("[format_cache.io_engine] must be one of: auto, io_uring, psync; got: " + value); + } + return value; + }, Setting.Property.NodeScope); + + private BlockCacheSettings() {} +} diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index eaa62fb9f9526..bdab090610b6f 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -992,6 +992,7 @@ protected Node(final Environment initialEnvironment, Collection clas remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings); CacheModule cacheModule = new CacheModule(pluginsService.filterPlugins(CachePlugin.class), settings); CacheService cacheService = cacheModule.getCacheService(); + final SegmentReplicator segmentReplicator = new SegmentReplicator(threadPool); final IndicesService indicesService = new IndicesService( settings, diff --git a/server/src/main/java/org/opensearch/plugins/BlockCache.java b/server/src/main/java/org/opensearch/plugins/BlockCache.java new file mode 100644 index 0000000000000..f7a9040e1651c --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/BlockCache.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.Closeable; + +/** + * Node-scoped block cache contract — backend-neutral. + * + *

      This interface deliberately carries only lifecycle and observability + * methods. Backend-specific surface (e.g. Caffeine's pin/unpin reference + * counting, or Foyer's native cache pointer) lives on concrete subtypes and + * is consumed by code that explicitly knows which backend it is talking to. + * Core only ever uses the two methods declared here. + * + *

      A block cache stores variable-size contiguous byte ranges (file ranges, + * Parquet column chunks, remote-object ranges, etc.). The exact key and + * value shape is an implementation detail and is not part of this interface + * — different backends may use path-and-offset keys, repository-and-range + * keys, native pointers, or anything else. + * + *

      Implementations must be thread-safe and idempotent on {@link #close()}. + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface BlockCache extends Closeable { + + /** + * Release all resources held by this cache. Idempotent: calling more than + * once must be a no-op. + */ + @Override + void close(); + + /** + * Returns a point-in-time snapshot of cache counters. + * + *

      Implementations that do not track a particular metric should return + * zero for that field rather than throwing. The snapshot is not + * guaranteed to be internally consistent across concurrent cache + * activity. + * + * @return counter snapshot; never {@code null} + */ + BlockCacheStats stats(); +} diff --git a/server/src/main/java/org/opensearch/plugins/BlockCacheProvider.java b/server/src/main/java/org/opensearch/plugins/BlockCacheProvider.java new file mode 100644 index 0000000000000..a5f27a00af284 --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/BlockCacheProvider.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.Optional; + +/** + * SPI implemented by a {@link Plugin} that publishes a node-scoped + * {@link BlockCache}. + * + *

      Core resolves the cache at node boot by filtering plugins that implement + * this interface. Consumers that want to use the cache discover it through + * their own plugin hooks — this SPI only concerns publication, not fan-out. + * + *

      Expected to be implemented by at most one plugin per node. If multiple + * plugins publish a cache, core picks the first one discovered and logs a + * warning. + * + *

      Returning {@link Optional#empty()} is the same as not implementing the + * interface at all — consumers see no cache and fall back to no-cache + * behaviour. This lets implementing plugins no-op at runtime based on node + * settings without changing their SPI participation. + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface BlockCacheProvider { + + /** + * Returns the node-scoped {@link BlockCache} published by this plugin, or + * {@link Optional#empty()} if the plugin is present but has decided not to + * publish a cache (e.g. cache disabled by settings). + * + *

      Called at node boot, after {@code createComponents} completes. + * + * @return the cache, or {@link Optional#empty()}; never {@code null} + */ + Optional getBlockCache(); +} diff --git a/server/src/main/java/org/opensearch/plugins/BlockCacheStats.java b/server/src/main/java/org/opensearch/plugins/BlockCacheStats.java new file mode 100644 index 0000000000000..bf60b54ec6f36 --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/BlockCacheStats.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Point-in-time snapshot of {@link BlockCache} counters. + * + *

      Emitted for node-stats reporting and logging. The exact metric set + * captured by any given implementation may be richer; this record carries + * only the universally available counters that every {@code BlockCache} + * implementation can be expected to surface. + * + *

        + *
      • {@code hits} — cumulative number of lookups served from the cache.
        • + *
      • {@code misses} — cumulative number of lookups that did not find an + * entry in the cache.
        • + *
      • {@code evictions} — cumulative number of entries removed from the + * cache to make room for new entries.
        • + *
      • {@code memoryBytesUsed} — current number of bytes occupied by entries + * in the in-memory tier.
        • + *
      • {@code diskBytesUsed} — current number of bytes occupied by entries in + * the on-disk tier (zero for implementations without a disk tier).
        • + *
      + * + *

      Values are a snapshot at the moment the record is constructed; they are + * not guaranteed to be internally consistent with each other across + * concurrent cache activity. + * + * @opensearch.experimental + */ +@ExperimentalApi +public record BlockCacheStats(long hits, long misses, long evictions, long memoryBytesUsed, long diskBytesUsed) { +} From abae5d055fcaf280040b9d2d70aee98f5abeecef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9D=90=87=F0=9D=90=9A=F0=9D=90=AB=F0=9D=90=A2?= =?UTF-8?q?=F0=9D=90=AC=F0=9D=90=A1=20=F0=9D=90=8D=F0=9D=90=9A=F0=9D=90=AB?= =?UTF-8?q?=F0=9D=90=9A=F0=9D=90=AC=F0=9D=90=A2=F0=9D=90=A6=F0=9D=90=A1?= =?UTF-8?q?=F0=9D=90=9A=F0=9D=90=A7=20=F0=9D=90=8A?= <163456392+HarishNarasimhanK@users.noreply.github.com> Date: Tue, 5 May 2026 20:02:49 +0530 Subject: [PATCH 045/115] Port two-tier cache system (metadata + statistics) from feature/datafusion to main branch (#21225) Signed-off-by: Harish Narasimhan Co-authored-by: Harish Narasimhan --- .../rust/Cargo.toml | 2 + .../rust/src/api.rs | 16 +- .../rust/src/cache.rs | 149 ++++ .../rust/src/custom_cache_manager.rs | 528 +++++++++++++ .../rust/src/eviction_policy.rs | 379 ++++++++++ .../rust/src/ffm.rs | 216 +++++- .../rust/src/lib.rs | 4 + .../rust/src/statistics_cache.rs | 705 ++++++++++++++++++ .../be/datafusion/DataFusionPlugin.java | 14 +- .../be/datafusion/DataFusionService.java | 37 +- .../be/datafusion/cache/CacheManager.java | 106 +++ .../be/datafusion/cache/CacheSettings.java | 79 ++ .../be/datafusion/cache/CacheUtils.java | 121 +++ .../be/datafusion/nativelib/NativeBridge.java | 163 +++- .../be/datafusion/DataFusionServiceTests.java | 98 +++ .../DatafusionCacheManagerTests.java | 218 ++++++ .../src/test/resources/hits1.parquet | Bin 0 -> 1598981 bytes .../src/test/resources/hits2.parquet | Bin 0 -> 76975 bytes 18 files changed, 2827 insertions(+), 8 deletions(-) create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/cache.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/custom_cache_manager.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/eviction_policy.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/statistics_cache.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheManager.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheSettings.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheUtils.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionCacheManagerTests.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/resources/hits1.parquet create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/resources/hits2.parquet diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml index 71c04d9f9001a..1c5e72ea4e2ed 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml +++ b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml @@ -39,7 +39,9 @@ num_cpus = { workspace = true } mimalloc = { workspace = true } native-bridge-common = { workspace = true } async-trait = { workspace = true } +chrono = { workspace = true } roaring = "0.10" +thiserror = { workspace = true } [dev-dependencies] criterion = { workspace = true } diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs index a47d5f95fa2f6..d6bfab0c9fd37 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs @@ -46,13 +46,15 @@ use datafusion::datasource::listing::ListingTableUrl; use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; use datafusion::execution::memory_pool::{GreedyMemoryPool, TrackConsumersPool}; use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use datafusion::execution::cache::cache_manager::CacheManagerConfig; use datafusion::execution::RecordBatchStream; -use datafusion::execution::SessionStateBuilder; +use datafusion::execution::{SessionState, SessionStateBuilder}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::prelude::SessionConfig; use futures::TryStreamExt; use crate::cross_rt_stream::CrossRtStream; +use crate::custom_cache_manager::CustomCacheManager; use crate::local_executor::LocalSession; use crate::partition_stream::PartitionStreamSender; use crate::query_memory_pool_tracker::QueryTrackingContext; @@ -108,6 +110,7 @@ pub async fn create_object_metas( /// Contains the DataFusion RuntimeEnv (memory pool, disk spill, cache). pub struct DataFusionRuntime { pub runtime_env: datafusion::execution::runtime_env::RuntimeEnv, + pub custom_cache_manager: Option, } /// Opaque shard view handle returned to the caller. @@ -122,6 +125,7 @@ pub struct ShardView { /// Caller must call `close_global_runtime` exactly once to free it. pub fn create_global_runtime( memory_pool_limit: i64, + cache_manager_ptr: i64, spill_dir: &str, spill_limit: i64, ) -> Result { @@ -134,12 +138,20 @@ pub fn create_global_runtime( NonZeroUsize::new(5).unwrap(), )); + let (cache_manager_config, custom_cache_manager) = if cache_manager_ptr != 0 { + let mgr = unsafe { *Box::from_raw(cache_manager_ptr as *mut CustomCacheManager) }; + (mgr.build_cache_manager_config(), Some(mgr)) + } else { + (CacheManagerConfig::default(), None) + }; + let runtime_env = RuntimeEnvBuilder::new() .with_memory_pool(memory_pool) .with_disk_manager_builder(disk_manager) + .with_cache_manager(cache_manager_config) .build()?; - let runtime = DataFusionRuntime { runtime_env }; + let runtime = DataFusionRuntime { runtime_env, custom_cache_manager }; Ok(Box::into_raw(Box::new(runtime)) as i64) } diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/cache.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/cache.rs new file mode 100644 index 0000000000000..e84b372c35882 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/cache.rs @@ -0,0 +1,149 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::sync::{Arc, Mutex}; + +use datafusion::execution::cache::cache_manager::FileMetadataCache; +use datafusion::execution::cache::cache_unit::DefaultFilesMetadataCache; +use datafusion::execution::cache::CacheAccessor; +use log::error; +use object_store::ObjectMeta; + +// Cache type constants +pub const CACHE_TYPE_METADATA: &str = "METADATA"; +pub const CACHE_TYPE_STATS: &str = "STATISTICS"; + +// Helper function to log cache operations +fn log_cache_error(operation: &str, error: &str) { + error!("[CACHE ERROR] {} operation failed: {}", operation, error); +} + +// Wrapper to make Mutex implement FileMetadataCache +pub struct MutexFileMetadataCache { + pub inner: Mutex, +} + +impl MutexFileMetadataCache { + pub fn new(cache: DefaultFilesMetadataCache) -> Self { + Self { + inner: Mutex::new(cache), + } + } + + pub fn clear(&self) { + if let Ok(cache) = self.inner.lock() { + cache.clear(); + } + } + + pub fn update_cache_limit(&self, new_limit: usize) { + if let Ok(cache) = self.inner.lock() { + cache.update_cache_limit(new_limit); + } + } + + pub fn cache_limit(&self) -> usize { + if let Ok(cache) = self.inner.lock() { + cache.cache_limit() + } else { + 0 + } + } +} + + +// Implement CacheAccessor which is required by FileMetadataCache +impl CacheAccessor> for MutexFileMetadataCache { + type Extra = ObjectMeta; + + fn get(&self, k: &ObjectMeta) -> Option> { + match self.inner.lock() { + Ok(cache) => cache.get(k), + Err(e) => { log_cache_error("get", &e.to_string()); None } + } + } + + fn get_with_extra(&self, k: &ObjectMeta, extra: &Self::Extra) -> Option> { + match self.inner.lock() { + Ok(cache) => cache.get_with_extra(k, extra), + Err(e) => { log_cache_error("get_with_extra", &e.to_string()); None } + } + } + + fn put(&self, k: &ObjectMeta, v: Arc) -> Option> { + match self.inner.lock() { + Ok(cache) => cache.put(k, v), + Err(e) => { log_cache_error("put", &e.to_string()); None } + } + } + + fn put_with_extra(&self, k: &ObjectMeta, v: Arc, e: &Self::Extra) -> Option> { + match self.inner.lock() { + Ok(cache) => cache.put_with_extra(k, v, e), + Err(err) => { log_cache_error("put_with_extra", &err.to_string()); None } + } + } + + fn remove(&self, k: &ObjectMeta) -> Option> { + match self.inner.lock() { + Ok(cache) => cache.remove(k), + Err(e) => { log_cache_error("remove", &e.to_string()); None } + } + } + + fn contains_key(&self, k: &ObjectMeta) -> bool { + match self.inner.lock() { + Ok(cache) => cache.contains_key(k), + Err(e) => { log_cache_error("contains_key", &e.to_string()); false } + } + } + + fn len(&self) -> usize { + match self.inner.lock() { + Ok(cache) => cache.len(), + Err(e) => { log_cache_error("len", &e.to_string()); 0 } + } + } + + fn clear(&self) { + match self.inner.lock() { + Ok(cache) => cache.clear(), + Err(e) => log_cache_error("clear", &e.to_string()), + } + } + + fn name(&self) -> String { + match self.inner.lock() { + Ok(cache) => cache.name(), + Err(e) => { log_cache_error("name", &e.to_string()); "cache_error".to_string() } + } + } +} + +impl FileMetadataCache for MutexFileMetadataCache { + fn cache_limit(&self) -> usize { + match self.inner.lock() { + Ok(cache) => cache.cache_limit(), + Err(e) => { log_cache_error("cache_limit", &e.to_string()); 0 } + } + } + + fn update_cache_limit(&self, limit: usize) { + match self.inner.lock() { + Ok(cache) => cache.update_cache_limit(limit), + Err(e) => log_cache_error("update_cache_limit", &e.to_string()), + } + } + + fn list_entries(&self) -> std::collections::HashMap { + match self.inner.lock() { + Ok(cache) => cache.list_entries(), + Err(e) => { log_cache_error("list_entries", &e.to_string()); std::collections::HashMap::new() } + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/custom_cache_manager.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/custom_cache_manager.rs new file mode 100644 index 0000000000000..6d373f5320da6 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/custom_cache_manager.rs @@ -0,0 +1,528 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use std::sync::Arc; +use datafusion::execution::cache::cache_manager::{FileMetadataCache, FileStatisticsCache, CacheManagerConfig}; +use datafusion::execution::cache::cache_unit::DefaultFileStatisticsCache; +use datafusion::execution::cache::CacheAccessor; +use crate::statistics_cache::compute_parquet_statistics; +use tokio::runtime::Runtime; +use crate::cache::MutexFileMetadataCache; +use crate::statistics_cache::CustomStatisticsCache; +use object_store::path::Path; +use object_store::ObjectMeta; +use datafusion::datasource::physical_plan::parquet::metadata::DFParquetMetadata; +use log::{debug, error}; + +/// Create ObjectMeta from a local file path. +fn create_object_meta_from_file(file_path: &str) -> Result, datafusion::common::DataFusionError> { + use chrono::{DateTime, Utc}; + use datafusion::common::DataFusionError; + + let metadata = std::fs::metadata(file_path) + .map_err(|e| DataFusionError::Execution(format!("Failed to get file metadata for {}: {}", file_path, e)))?; + + let file_size = metadata.len(); + + let modified = metadata.modified() + .map(|t| DateTime::::from(t)) + .unwrap_or_else(|_| Utc::now()); + + let object_meta = ObjectMeta { + location: Path::from(file_path), + last_modified: modified, + size: file_size, + e_tag: None, + version: None, + }; + + Ok(vec![object_meta]) +} + +/// Custom CacheManager that holds cache references directly +pub struct CustomCacheManager { + /// Direct reference to the file metadata cache + file_metadata_cache: Option>, + /// Direct reference to the statistics cache + statistics_cache: Option> +} + +impl CustomCacheManager { + /// Create a new CustomCacheManager + pub fn new() -> Self { + Self { + file_metadata_cache: None, + statistics_cache: None + } + } + + /// Set the file metadata cache + pub fn set_file_metadata_cache(&mut self, cache: Arc) { + self.file_metadata_cache = Some(cache); + debug!("[CACHE INFO] File metadata cache set in CustomCacheManager"); + } + + /// Set the statistics cache + pub fn set_statistics_cache(&mut self, cache: Arc) { + self.statistics_cache = Some(cache); + debug!("[CACHE INFO] Statistics cache set in CustomCacheManager"); + } + + /// Get the statistics cache + pub fn get_statistics_cache(&self) -> Option> { + self.statistics_cache.clone() + } + + /// Get the file metadata cache as Arc for DataFusion + pub fn get_file_metadata_cache_for_datafusion(&self) -> Option> { + self.file_metadata_cache.as_ref().map(|cache| cache.clone() as Arc) + } + + /// Build a CacheManagerConfig from the caches stored in this CustomCacheManager + pub fn build_cache_manager_config(&self) -> CacheManagerConfig { + let mut config = CacheManagerConfig::default(); + + // Add file metadata cache if available + if let Some(cache) = self.get_file_metadata_cache_for_datafusion() { + config = config.with_file_metadata_cache(Some(cache.clone())) + .with_metadata_cache_limit(cache.cache_limit()); + } + + // Add statistics cache if available - use CustomStatisticsCache directly + if let Some(stats_cache) = &self.statistics_cache { + config = config.with_files_statistics_cache(Some(stats_cache.clone() as Arc)); + } else { + // Default statistics cache if none set + let default_stats = Arc::new(DefaultFileStatisticsCache::default()); + config = config.with_files_statistics_cache(Some(default_stats)); + } + + config + } + + /// Add multiple files to all applicable caches + pub fn add_files(&self, file_paths: &[String]) -> Result, String> { + let mut results = Vec::new(); + + for file_path in file_paths { + let mut any_success = false; + let mut errors = Vec::new(); + + // Add to metadata cache + match self.metadata_cache_put(file_path) { + Ok(true) => { + any_success = true; + } + Ok(false) => { + debug!("[CACHE INFO] File not added for metadata cache: {}", file_path); + } + Err(e) => { + errors.push(format!("Metadata cache: {}", e)); + } + } + + // Add to statistics cache + if let Some(_) = &self.statistics_cache { + match self.statistics_cache_compute_and_put(file_path) { + Ok(true) => { + any_success = true; + } + Ok(false) => { + debug!("[CACHE INFO] File not added for statistics cache: {}", file_path); + } + Err(e) => { + errors.push(format!("Statistics cache: {}", e)); + } + } + } + + let success = if !errors.is_empty() && !any_success { + false + } else { + any_success + }; + + results.push((file_path.clone(), success)); + } + + Ok(results) + } + + /// Remove multiple files from all caches + pub fn remove_files(&self, file_paths: &[String]) -> Result, String> { + let mut results = Vec::new(); + + for file_path in file_paths { + let mut any_removed = false; + let mut errors = Vec::new(); + + // Remove from metadata cache + match create_object_meta_from_file(file_path) { + Ok(object_metas) => { + if let Some(cache) = &self.file_metadata_cache { + match cache.inner.lock() { + Ok(cache_guard) => { + if let Some(object_meta) = object_metas.first() { + if cache_guard.remove(object_meta).is_some() { + any_removed = true; + } else { + debug!("[CACHE INFO] File not found in metadata cache: {}", file_path); + } + } + } + Err(e) => { + errors.push(format!("Metadata cache: Cache remove failed: {}", e)); + } + } + } else { + errors.push("No metadata cache configured".to_string()); + } + } + Err(e) => { + errors.push(format!("Failed to get object metadata: {}", e)); + } + } + + // Remove from statistics cache + if let Some(cache) = &self.statistics_cache { + let path = Path::from(file_path.clone()); + // Use the CacheAccessor remove method to properly update memory tracking + if cache.remove(&path).is_some() { + any_removed = true; + } + } + + let removed = if !errors.is_empty() && !any_removed { + false + } else { + any_removed + }; + + results.push((file_path.clone(), removed)); + } + + Ok(results) + } + + /// Check if a file exists in any cache + pub fn contains_file(&self, file_path: &str) -> bool { + let mut found = false; + + // Check metadata cache + match create_object_meta_from_file(file_path) { + Ok(object_metas) => { + if let Some(cache) = &self.file_metadata_cache { + if let Some(object_meta) = object_metas.first() { + if cache.get(object_meta).is_some() { + found = true; + } + } + } + } + Err(e) => { + error!("Failed to get object metadata for {}: {}", file_path, e); + } + } + + // Check statistics cache + if let Some(cache) = &self.statistics_cache { + let path = Path::from(file_path); + if cache.contains_key(&path) { + found = true; + } + } + + found + } + + /// Check if a file exists in a specific cache type + pub fn contains_file_by_type(&self, file_path: &str, cache_type: &str) -> bool { + match cache_type { + crate::cache::CACHE_TYPE_METADATA => { + create_object_meta_from_file(file_path) + .ok() + .and_then(|metas| metas.first().cloned()) + .and_then(|meta| self.file_metadata_cache.as_ref()?.get(&meta)) + .is_some() + } + crate::cache::CACHE_TYPE_STATS => { + self.statistics_cache + .as_ref() + .map_or(false, |cache| cache.contains_key(&Path::from(file_path))) + } + _ => false + } + } + + /// Update the file metadata cache size limit + pub fn update_metadata_cache_limit(&self, new_limit: usize) { + if let Some(cache) = &self.file_metadata_cache { + cache.update_cache_limit(new_limit); + } + } + + /// Update the statistics cache size limit + pub fn update_statistics_cache_limit(&self, new_limit: usize) -> Result<(), String> { + if let Some(cache) = &self.statistics_cache { + cache.update_size_limit(new_limit) + .map_err(|e| format!("Failed to update statistics cache limit: {:?}", e)) + } else { + Err("No statistics cache configured".to_string()) + } + } + + /// Get total memory consumed by all caches + pub fn get_total_memory_consumed(&self) -> usize { + let mut total = 0; + + // Add metadata cache memory + if let Some(cache) = &self.file_metadata_cache { + if let Ok(cache_guard) = cache.inner.lock() { + total += cache_guard.memory_used(); + } + } + + // Add statistics cache memory + if let Some(cache) = &self.statistics_cache { + total += cache.memory_consumed(); + } + + total + } + + /// Clear all caches + pub fn clear_all(&self) { + if let Some(cache) = &self.file_metadata_cache { + cache.clear(); + } + if let Some(cache) = &self.statistics_cache { + cache.clear(); + } + } + + /// Clear specific cache type + pub fn clear_cache_type(&self, cache_type: &str) -> Result<(), String> { + match cache_type { + crate::cache::CACHE_TYPE_METADATA => { + if let Some(cache) = &self.file_metadata_cache { + cache.clear(); + Ok(()) + } else { + Err("No metadata cache configured".to_string()) + } + } + crate::cache::CACHE_TYPE_STATS => { + if let Some(cache) = &self.statistics_cache { + cache.clear(); + Ok(()) + } else { + Err("No statistics cache configured".to_string()) + } + } + _ => Err(format!("Unknown cache type: {}", cache_type)) + } + } + + /// Get memory consumed by specific cache type + pub fn get_memory_consumed_by_type(&self, cache_type: &str) -> Result { + match cache_type { + crate::cache::CACHE_TYPE_METADATA => { + if let Some(cache) = &self.file_metadata_cache { + if let Ok(cache_guard) = cache.inner.lock() { + Ok(cache_guard.memory_used()) + } else { + Err("Failed to lock metadata cache".to_string()) + } + } else { + Err("No metadata cache configured".to_string()) + } + } + crate::cache::CACHE_TYPE_STATS => { + if let Some(cache) = &self.statistics_cache { + Ok(cache.memory_consumed()) + } else { + Err("No statistics cache configured".to_string()) + } + } + _ => Err(format!("Unknown cache type: {}", cache_type)) + } + } + + /// Internal method to put metadata into cache + fn metadata_cache_put(&self, file_path: &str) -> Result { + if !file_path.to_lowercase().ends_with(".parquet") { + return Ok(false); // Skip unsupported formats + } + + let object_metas = create_object_meta_from_file(file_path) + .map_err(|e| format!("Failed to get object metadata: {}", e))?; + + let object_meta = object_metas.first() + .ok_or_else(|| "No object metadata returned".to_string())?; + + let store = Arc::new(object_store::local::LocalFileSystem::new()); + + // Get cache reference for DataFusion metadata loading + let cache_ref = self.file_metadata_cache.as_ref() + .ok_or_else(|| "No file metadata cache configured".to_string())?; + + let metadata_cache = cache_ref.clone() as Arc; + + // Use DataFusion's metadata loading by passing reference to file_metadata_cache to get complete metadata + // IMPORTANT: When a cache is provided to DFParquetMetadata, fetch_metadata() will: + // 1. Enable page index loading (with_page_indexes(true)) + // 2. Load the complete metadata including column and offset indexes + // 3. Automatically put the metadata into the cache (lines 155-160 in datafusion's metadata.rs) + // This ensures we cache exactly what DataFusion would cache during query execution + let _parquet_metadata = Runtime::new() + .map_err(|e| format!("Failed to create Tokio Runtime: {}", e))? + .block_on(async { + let df_metadata = DFParquetMetadata::new(store.as_ref(), object_meta) + .with_file_metadata_cache(Some(metadata_cache)); + + // fetch_metadata() performs the cache put operation internally + df_metadata.fetch_metadata().await + .map_err(|e| format!("Failed to fetch metadata: {}", e)) + })?; + + // Verify the metadata was cached properly + match cache_ref.inner.lock() { + Ok(cache_guard) => { + if cache_guard.contains_key(object_meta) { + Ok(true) + } else { + debug!("[CACHE ERROR] Failed to cache metadata for: {}", file_path); + Ok(false) + } + } + Err(e) => Err(format!("Failed to verify cache: {}", e)) + } + } + + /// Compute and put statistics into cache + pub fn statistics_cache_compute_and_put(&self, file_path: &str) -> Result { + let cache = self.statistics_cache.as_ref() + .ok_or_else(|| "No statistics cache configured".to_string())?; + + let path = Path::from(file_path.to_string()); + + // Check if already cached + if cache.contains_key(&path) { + return Ok(true); + } + + // Compute statistics + match compute_parquet_statistics(file_path) { + Ok(stats) => { + let meta = ObjectMeta { + location: path.clone(), + last_modified: chrono::Utc::now(), + size: std::fs::metadata(file_path) + .map(|m| m.len()) + .unwrap_or(0), + e_tag: None, + version: None, + }; + + cache.put_with_extra(&path, Arc::new(stats), &meta); + Ok(true) + } + Err(e) => { + Err(format!("Failed to compute statistics for {}: {}", file_path, e)) + } + } + } + + /// Batch compute and cache statistics for multiple files + pub fn statistics_cache_batch_compute_and_put(&self, file_paths: &[String]) -> Result { + let cache = self.statistics_cache.as_ref() + .ok_or_else(|| "No statistics cache configured".to_string())?; + + let mut success_count = 0; + let mut failed_files = Vec::new(); + + for file_path in file_paths { + let path = Path::from(file_path.clone()); + + if cache.contains_key(&path) { + success_count += 1; + continue; + } + + match compute_parquet_statistics(file_path) { + Ok(stats) => { + let meta = ObjectMeta { + location: path.clone(), + last_modified: chrono::Utc::now(), + size: std::fs::metadata(file_path) + .map(|m| m.len()) + .unwrap_or(0), + e_tag: None, + version: None, + }; + + cache.put_with_extra(&path, Arc::new(stats), &meta); + success_count += 1; + } + Err(e) => { + debug!("[STATS CACHE ERROR] Failed to compute statistics for {}: {}", file_path, e); + failed_files.push(file_path.clone()); + } + } + } + + if !failed_files.is_empty() { + debug!("[STATS CACHE WARNING] Failed to compute statistics for {} files: {:?}", + failed_files.len(), failed_files); + } + + Ok(success_count) + } + + /// Get or compute statistics + pub fn statistics_cache_get_or_compute(&self, file_path: &str) -> Result { + let cache = self.statistics_cache.as_ref() + .ok_or_else(|| "No statistics cache configured".to_string())?; + + let path = Path::from(file_path.to_string()); + + if cache.get(&path).is_some() { + return Ok(true); + } + + self.statistics_cache_compute_and_put(file_path) + } + + /// Get statistics cache hit count + pub fn statistics_cache_hit_count(&self) -> usize { + self.statistics_cache.as_ref() + .map(|cache| cache.hit_count()) + .unwrap_or(0) + } + + /// Get statistics cache miss count + pub fn statistics_cache_miss_count(&self) -> usize { + self.statistics_cache.as_ref() + .map(|cache| cache.miss_count()) + .unwrap_or(0) + } + + /// Get statistics cache hit rate + pub fn statistics_cache_hit_rate(&self) -> f64 { + self.statistics_cache.as_ref() + .map(|cache| cache.hit_rate()) + .unwrap_or(0.0) + } + + /// Reset statistics cache stats + pub fn statistics_cache_reset_stats(&self) { + if let Some(cache) = &self.statistics_cache { + cache.reset_stats(); + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/eviction_policy.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/eviction_policy.rs new file mode 100644 index 0000000000000..6fe2a7402b3b8 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/eviction_policy.rs @@ -0,0 +1,379 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! # Cache Policy Module +//! +//! Simple pluggable cache eviction policies for statistics cache. + +use datafusion::common::instant; +use instant::Instant; +use thiserror::Error; + +/// Error types for cache operations +#[derive(Debug, Error)] +pub enum CacheError { + #[error("Policy lock error: {reason}")] + PolicyLockError { reason: String }, +} + +/// Result type for cache operations +pub type CacheResult = Result; + +/// Core trait for cache eviction policies +pub trait CachePolicy: Send + Sync { + /// Called when a cache entry is accessed + fn on_access(&mut self, key: &str, size: usize); + + /// Called when a cache entry is inserted + fn on_insert(&mut self, key: &str, size: usize); + /// Called when a cache entry is removed + fn on_remove(&mut self, key: &str); + + /// Select entries for eviction to reach target size + /// Returns keys to evict, ordered by eviction priority + fn select_for_eviction(&self, target_size: usize) -> Vec; + + /// Reset policy state + fn clear(&mut self); + + /// Get the name of this policy + fn policy_name(&self) -> &'static str; +} + +/// Policy types +#[derive(Debug, Clone)] +pub enum PolicyType { + Lru, + Lfu, +} + +/// Simple cache entry metadata +#[derive(Debug, Clone)] +pub struct CacheEntryMetadata { + pub size: usize, + pub last_accessed: Instant, + pub access_count: usize, +} + +impl CacheEntryMetadata { + pub fn new(_key: String, size: usize) -> Self { + Self { + size, + last_accessed: Instant::now(), + access_count: 1, + } + } + + pub fn on_access(&mut self) { + self.last_accessed = Instant::now(); + self.access_count += 1; + } +} + +/// LRU (Least Recently Used) policy +pub struct LruPolicy { + entries: dashmap::DashMap, + total_size: std::sync::atomic::AtomicUsize, +} + +impl LruPolicy { + pub fn new() -> Self { + Self { + entries: dashmap::DashMap::new(), + total_size: std::sync::atomic::AtomicUsize::new(0), + } + } +} + +impl Default for LruPolicy { + fn default() -> Self { + Self::new() + } +} + +impl CachePolicy for LruPolicy { + fn on_access(&mut self, key: &str, size: usize) { + match self.entries.get_mut(key) { + Some(mut entry) => { + entry.on_access(); + } + None => { + let metadata = CacheEntryMetadata::new(key.to_string(), size); + self.entries.insert(key.to_string(), metadata); + self.total_size + .fetch_add(size, std::sync::atomic::Ordering::Relaxed); + } + } + } + + fn on_insert(&mut self, key: &str, size: usize) { + let metadata = CacheEntryMetadata::new(key.to_string(), size); + + if let Some(old_entry) = self.entries.insert(key.to_string(), metadata) { + let old_size = old_entry.size; + self.total_size + .fetch_sub(old_size, std::sync::atomic::Ordering::Relaxed); + } + + self.total_size + .fetch_add(size, std::sync::atomic::Ordering::Relaxed); + } + + fn on_remove(&mut self, key: &str) { + if let Some((_, entry)) = self.entries.remove(key) { + self.total_size + .fetch_sub(entry.size, std::sync::atomic::Ordering::Relaxed); + } + } + + fn select_for_eviction(&self, target_size: usize) -> Vec { + if target_size == 0 { + return Vec::new(); + } + + // Collect entries with access times + let mut entries: Vec<_> = self + .entries + .iter() + .map(|entry| { + let key = entry.key().clone(); + let last_accessed = entry.value().last_accessed; + (key, last_accessed) + }) + .collect(); + + // Sort by access time (oldest first) + entries.sort_by_key(|(_, last_accessed)| *last_accessed); + + // Select entries for eviction until target size is reached + let mut candidates = Vec::new(); + let mut freed_size = 0; + + for (key, _) in entries { + if freed_size >= target_size { + break; + } + if let Some(entry) = self.entries.get(&key) { + freed_size += entry.size; + candidates.push(key); + } + } + + candidates + } + + fn clear(&mut self) { + self.entries.clear(); + self.total_size + .store(0, std::sync::atomic::Ordering::Relaxed); + } + + fn policy_name(&self) -> &'static str { + "lru" + } +} + +/// LFU (Least Frequently Used) policy +pub struct LfuPolicy { + entries: dashmap::DashMap, + total_size: std::sync::atomic::AtomicUsize, +} + +impl LfuPolicy { + pub fn new() -> Self { + Self { + entries: dashmap::DashMap::new(), + total_size: std::sync::atomic::AtomicUsize::new(0), + } + } +} + +impl Default for LfuPolicy { + fn default() -> Self { + Self::new() + } +} + +impl CachePolicy for LfuPolicy { + fn on_access(&mut self, key: &str, size: usize) { + match self.entries.get_mut(key) { + Some(mut entry) => { + entry.on_access(); + } + None => { + let metadata = CacheEntryMetadata::new(key.to_string(), size); + self.entries.insert(key.to_string(), metadata); + self.total_size + .fetch_add(size, std::sync::atomic::Ordering::Relaxed); + } + } + } + + fn on_insert(&mut self, key: &str, size: usize) { + let metadata = CacheEntryMetadata::new(key.to_string(), size); + + if let Some(old_entry) = self.entries.insert(key.to_string(), metadata) { + let old_size = old_entry.size; + self.total_size + .fetch_sub(old_size, std::sync::atomic::Ordering::Relaxed); + } + + self.total_size + .fetch_add(size, std::sync::atomic::Ordering::Relaxed); + } + + fn on_remove(&mut self, key: &str) { + if let Some((_, entry)) = self.entries.remove(key) { + self.total_size + .fetch_sub(entry.size, std::sync::atomic::Ordering::Relaxed); + } + } + + fn select_for_eviction(&self, target_size: usize) -> Vec { + if target_size == 0 { + return Vec::new(); + } + + // Collect entries with access counts + let mut entries: Vec<_> = self + .entries + .iter() + .map(|entry| { + let key = entry.key().clone(); + let access_count = entry.value().access_count; + let last_accessed = entry.value().last_accessed; + (key, access_count, last_accessed) + }) + .collect(); + + // Sort by access count (least frequent first), then by time for tie-breaking + entries.sort_by(|(_, count_a, time_a), (_, count_b, time_b)| { + count_a.cmp(count_b).then(time_a.cmp(time_b)) + }); + + // Select entries for eviction until target size is reached + let mut candidates = Vec::new(); + let mut freed_size = 0; + + for (key, _, _) in entries { + if freed_size >= target_size { + break; + } + if let Some(entry) = self.entries.get(&key) { + freed_size += entry.size; + candidates.push(key); + } + } + + candidates + } + + fn clear(&mut self) { + self.entries.clear(); + self.total_size + .store(0, std::sync::atomic::Ordering::Relaxed); + } + + fn policy_name(&self) -> &'static str { + "lfu" + } +} + +/// Create a cache policy instance +pub fn create_policy(policy_type: PolicyType) -> Box { + match policy_type { + PolicyType::Lru => Box::new(LruPolicy::new()), + PolicyType::Lfu => Box::new(LfuPolicy::new()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + use std::time::Duration; + + #[test] + fn test_cache_entry_metadata() { + let mut metadata = CacheEntryMetadata::new("test_key".to_string(), 1024); + assert_eq!(metadata.size, 1024); + assert_eq!(metadata.access_count, 1); + + let initial_access_time = metadata.last_accessed; + thread::sleep(Duration::from_millis(1)); + + metadata.on_access(); + assert_eq!(metadata.access_count, 2); + assert!(metadata.last_accessed > initial_access_time); + } + + #[test] + fn test_create_policy() { + let lru_policy = create_policy(PolicyType::Lru); + assert_eq!(lru_policy.policy_name(), "lru"); + + let lfu_policy = create_policy(PolicyType::Lfu); + assert_eq!(lfu_policy.policy_name(), "lfu"); + } + + #[test] + fn test_lru_policy_basic_operations() { + let mut policy = LruPolicy::new(); + assert_eq!(policy.policy_name(), "lru"); + + policy.on_insert("key1", 100); + policy.on_insert("key2", 200); + policy.on_access("key1", 100); + policy.on_remove("key1"); + policy.clear(); + } + + #[test] + fn test_lru_policy_victim_selection() { + let mut policy = LruPolicy::new(); + + policy.on_insert("oldest", 100); + thread::sleep(Duration::from_millis(1)); + + policy.on_insert("middle", 100); + thread::sleep(Duration::from_millis(1)); + + policy.on_insert("newest", 100); + thread::sleep(Duration::from_millis(1)); + + // Access middle entry to make it more recent + policy.on_access("middle", 100); + + let candidates = policy.select_for_eviction(150); + assert_eq!(candidates.len(), 2); + assert!(candidates.contains(&"oldest".to_string())); + assert!(!candidates.contains(&"middle".to_string())); + } + + #[test] + fn test_lfu_policy_victim_selection() { + let mut policy = LfuPolicy::new(); + + policy.on_insert("rarely_used", 100); + policy.on_insert("sometimes_used", 100); + policy.on_insert("frequently_used", 100); + + // Create frequency patterns + policy.on_access("sometimes_used", 100); + + for _ in 0..3 { + policy.on_access("frequently_used", 100); + } + + let candidates = policy.select_for_eviction(150); + assert_eq!(candidates.len(), 2); + assert!(candidates.contains(&"rarely_used".to_string())); + assert!(candidates.contains(&"sometimes_used".to_string())); + assert!(!candidates.contains(&"frequently_used".to_string())); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index 6ea0b510e3eee..7c9a67827f326 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -16,7 +16,14 @@ use native_bridge_common::ffm_safe; use parking_lot::RwLock; use crate::api; +use crate::api::DataFusionRuntime; +use crate::cache; +use crate::custom_cache_manager::CustomCacheManager; +use crate::eviction_policy::PolicyType; use crate::runtime_manager::RuntimeManager; +use crate::statistics_cache::CustomStatisticsCache; + +use datafusion::execution::cache::cache_unit::DefaultFilesMetadataCache; static TOKIO_RUNTIME_MANAGER: RwLock>> = RwLock::new(None); @@ -56,13 +63,15 @@ pub extern "C" fn df_shutdown_runtime_manager() { #[no_mangle] pub unsafe extern "C" fn df_create_global_runtime( memory_pool_limit: i64, + cache_manager_ptr: i64, spill_dir_ptr: *const u8, spill_dir_len: i64, spill_limit: i64, ) -> i64 { let spill_dir = str_from_raw(spill_dir_ptr, spill_dir_len) .map_err(|e| format!("df_create_global_runtime: {}", e))?; - api::create_global_runtime(memory_pool_limit, spill_dir, spill_limit).map_err(|e| e.to_string()) + api::create_global_runtime(memory_pool_limit, cache_manager_ptr, spill_dir, spill_limit) + .map_err(|e| e.to_string()) } #[no_mangle] @@ -209,6 +218,19 @@ pub unsafe extern "C" fn df_close_local_session(ptr: i64) { api::close_local_session(ptr); } +#[no_mangle] +pub extern "C" fn df_create_custom_cache_manager() -> i64 { + let manager = CustomCacheManager::new(); + Box::into_raw(Box::new(manager)) as i64 +} + +#[no_mangle] +pub unsafe extern "C" fn df_destroy_custom_cache_manager(ptr: i64) { + if ptr != 0 { + let _ = Box::from_raw(ptr as *mut CustomCacheManager); + } +} + #[ffm_safe] #[no_mangle] pub unsafe extern "C" fn df_register_partition_stream( @@ -312,3 +334,195 @@ pub unsafe extern "C" fn df_register_memtable( .map(|_| 0) .map_err(|e| e.to_string()) } + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_create_cache( + cache_manager_ptr: i64, + cache_type_ptr: *const u8, + cache_type_len: i64, + size_limit: i64, + eviction_type_ptr: *const u8, + eviction_type_len: i64, +) -> i64 { + if cache_manager_ptr == 0 { + return Err("df_create_cache: null cache manager pointer".to_string()); + } + let cache_type = str_from_raw(cache_type_ptr, cache_type_len) + .map_err(|e| format!("df_create_cache: cache_type: {}", e))?; + let eviction_type = str_from_raw(eviction_type_ptr, eviction_type_len) + .map_err(|e| format!("df_create_cache: eviction_type: {}", e))?; + + let policy_type = match eviction_type.to_uppercase().as_str() { + "LRU" => PolicyType::Lru, + "LFU" => PolicyType::Lfu, + _ => return Err(format!("df_create_cache: unsupported eviction type: {}", eviction_type)), + }; + + // Safety: cache_manager_ptr must be a valid pointer from df_create_custom_cache_manager + let manager = &mut *(cache_manager_ptr as *mut CustomCacheManager); + + match cache_type { + cache::CACHE_TYPE_METADATA => { + let inner_cache = DefaultFilesMetadataCache::new(size_limit as usize); + let metadata_cache = Arc::new(cache::MutexFileMetadataCache::new(inner_cache)); + manager.set_file_metadata_cache(metadata_cache); + } + cache::CACHE_TYPE_STATS => { + let stats_cache = Arc::new(CustomStatisticsCache::new( + policy_type, + size_limit as usize, + 0.8, + )); + manager.set_statistics_cache(stats_cache); + } + _ => { + return Err(format!("df_create_cache: invalid cache type: {}", cache_type)); + } + } + Ok(0) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_cache_manager_add_files( + runtime_ptr: i64, + files_ptr: *const *const u8, + files_len_ptr: *const i64, + files_count: i64, +) -> i64 { + if runtime_ptr == 0 { + return Err("df_cache_manager_add_files: null runtime pointer".to_string()); + } + // Safety: runtime_ptr must be a valid pointer from df_create_global_runtime + let runtime = &*(runtime_ptr as *const DataFusionRuntime); + let manager = runtime.custom_cache_manager.as_ref() + .ok_or_else(|| "df_cache_manager_add_files: no cache manager configured".to_string())?; + + let mut file_paths = Vec::with_capacity(files_count as usize); + for i in 0..files_count as usize { + let ptr = *files_ptr.add(i); + let len = *files_len_ptr.add(i); + file_paths.push(str_from_raw(ptr, len) + .map_err(|e| format!("df_cache_manager_add_files: {}", e))?.to_string()); + } + + manager.add_files(&file_paths) + .map_err(|e| format!("df_cache_manager_add_files: {}", e))?; + Ok(0) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_cache_manager_remove_files( + runtime_ptr: i64, + files_ptr: *const *const u8, + files_len_ptr: *const i64, + files_count: i64, +) -> i64 { + if runtime_ptr == 0 { + return Err("df_cache_manager_remove_files: null runtime pointer".to_string()); + } + let runtime = &*(runtime_ptr as *const DataFusionRuntime); + let manager = runtime.custom_cache_manager.as_ref() + .ok_or_else(|| "df_cache_manager_remove_files: no cache manager configured".to_string())?; + + let mut file_paths = Vec::with_capacity(files_count as usize); + for i in 0..files_count as usize { + let ptr = *files_ptr.add(i); + let len = *files_len_ptr.add(i); + file_paths.push(str_from_raw(ptr, len) + .map_err(|e| format!("df_cache_manager_remove_files: {}", e))?.to_string()); + } + + manager.remove_files(&file_paths) + .map_err(|e| format!("df_cache_manager_remove_files: {}", e))?; + Ok(0) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_cache_manager_clear(runtime_ptr: i64) -> i64 { + if runtime_ptr == 0 { + return Err("df_cache_manager_clear: null runtime pointer".to_string()); + } + let runtime = &*(runtime_ptr as *const DataFusionRuntime); + let manager = runtime.custom_cache_manager.as_ref() + .ok_or_else(|| "df_cache_manager_clear: no cache manager configured".to_string())?; + manager.clear_all(); + Ok(0) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_cache_manager_clear_by_type( + runtime_ptr: i64, + cache_type_ptr: *const u8, + cache_type_len: i64, +) -> i64 { + if runtime_ptr == 0 { + return Err("df_cache_manager_clear_by_type: null runtime pointer".to_string()); + } + let cache_type = str_from_raw(cache_type_ptr, cache_type_len) + .map_err(|e| format!("df_cache_manager_clear_by_type: {}", e))?; + let runtime = &*(runtime_ptr as *const DataFusionRuntime); + let manager = runtime.custom_cache_manager.as_ref() + .ok_or_else(|| "df_cache_manager_clear_by_type: no cache manager configured".to_string())?; + manager.clear_cache_type(cache_type) + .map_err(|e| format!("df_cache_manager_clear_by_type: {}", e))?; + Ok(0) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_cache_manager_get_memory_by_type( + runtime_ptr: i64, + cache_type_ptr: *const u8, + cache_type_len: i64, +) -> i64 { + if runtime_ptr == 0 { + return Err("df_cache_manager_get_memory_by_type: null runtime pointer".to_string()); + } + let cache_type = str_from_raw(cache_type_ptr, cache_type_len) + .map_err(|e| format!("df_cache_manager_get_memory_by_type: {}", e))?; + let runtime = &*(runtime_ptr as *const DataFusionRuntime); + let manager = runtime.custom_cache_manager.as_ref() + .ok_or_else(|| "df_cache_manager_get_memory_by_type: no cache manager configured".to_string())?; + let size = manager.get_memory_consumed_by_type(cache_type) + .map_err(|e| format!("df_cache_manager_get_memory_by_type: {}", e))?; + Ok(size as i64) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_cache_manager_get_total_memory(runtime_ptr: i64) -> i64 { + if runtime_ptr == 0 { + return Err("df_cache_manager_get_total_memory: null runtime pointer".to_string()); + } + let runtime = &*(runtime_ptr as *const DataFusionRuntime); + let manager = runtime.custom_cache_manager.as_ref() + .ok_or_else(|| "df_cache_manager_get_total_memory: no cache manager configured".to_string())?; + Ok(manager.get_total_memory_consumed() as i64) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_cache_manager_contains_by_type( + runtime_ptr: i64, + cache_type_ptr: *const u8, + cache_type_len: i64, + file_path_ptr: *const u8, + file_path_len: i64, +) -> i64 { + if runtime_ptr == 0 { + return Err("df_cache_manager_contains_by_type: null runtime pointer".to_string()); + } + let cache_type = str_from_raw(cache_type_ptr, cache_type_len) + .map_err(|e| format!("df_cache_manager_contains_by_type: cache_type: {}", e))?; + let file_path = str_from_raw(file_path_ptr, file_path_len) + .map_err(|e| format!("df_cache_manager_contains_by_type: file_path: {}", e))?; + let runtime = &*(runtime_ptr as *const DataFusionRuntime); + let manager = runtime.custom_cache_manager.as_ref() + .ok_or_else(|| "df_cache_manager_contains_by_type: no cache manager configured".to_string())?; + Ok(if manager.contains_file_by_type(file_path, cache_type) { 1 } else { 0 }) +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs index c159480542fb2..d12fa24fa3acc 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs @@ -12,8 +12,11 @@ //! `extern "C"` functions for JDK FFM. pub mod api; +pub mod cache; pub mod cross_rt_stream; +pub mod custom_cache_manager; pub mod datafusion_query_config; +pub mod eviction_policy; pub mod executor; pub mod ffm; pub mod indexed_executor; @@ -24,3 +27,4 @@ pub mod partition_stream; pub mod query_executor; pub mod query_memory_pool_tracker; pub mod runtime_manager; +pub mod statistics_cache; diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/statistics_cache.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/statistics_cache.rs new file mode 100644 index 0000000000000..2971648bed1db --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/statistics_cache.rs @@ -0,0 +1,705 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +use crate::eviction_policy::{ + create_policy, CacheError, CachePolicy, CacheResult, PolicyType, +}; +use arrow_array::Array; +use datafusion::common::stats::{ColumnStatistics, Precision}; +use datafusion::common::ScalarValue; +use dashmap::DashMap; +use datafusion::execution::cache::CacheAccessor; +use datafusion::physical_plan::Statistics; +use object_store::{path::Path, ObjectMeta}; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use std::fs::File; + +/// Trait to calculate heap memory size for statistics objects +trait HeapSize { + fn heap_size(&self) -> usize; +} + +impl HeapSize for Statistics { + fn heap_size(&self) -> usize { + std::mem::size_of::() + + self.num_rows.heap_size() + + self.total_byte_size.heap_size() + + self.column_statistics.heap_size() + } +} + +impl HeapSize +for Precision +{ + fn heap_size(&self) -> usize { + match self { + Precision::Exact(val) => std::mem::size_of::() + val.heap_size(), + Precision::Inexact(val) => std::mem::size_of::() + val.heap_size(), + Precision::Absent => std::mem::size_of::(), + } + } +} + +impl HeapSize for usize { + fn heap_size(&self) -> usize { + 0 // Primitive types don't have heap allocation + } +} + +impl HeapSize for Vec { + fn heap_size(&self) -> usize { + std::mem::size_of::() + + (self.capacity() * std::mem::size_of::()) + + self.iter().map(|item| item.heap_size()).sum::() + } +} + +impl HeapSize for ColumnStatistics { + fn heap_size(&self) -> usize { + std::mem::size_of::() + + self.null_count.heap_size() + + self.max_value.heap_size() + + self.min_value.heap_size() + + self.distinct_count.heap_size() + } +} + +impl HeapSize for ScalarValue { + fn heap_size(&self) -> usize { + match self { + ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => { + std::mem::size_of::() + s.capacity() + } + ScalarValue::Binary(Some(b)) | ScalarValue::LargeBinary(Some(b)) => { + std::mem::size_of::() + b.capacity() + } + ScalarValue::List(arr) => { + // Estimate list array memory size + std::mem::size_of::() + std::mem::size_of_val(arr.as_ref()) + (arr.len() * 8) + } + ScalarValue::Struct(arr) => { + // Estimate struct array memory size + std::mem::size_of::() + std::mem::size_of_val(arr.as_ref()) + (arr.len() * 16) + } + _ => std::mem::size_of::(), // Primitive types and nulls + } + } +} + +/// Extension trait to add memory_size method to Statistics +trait StatisticsMemorySize { + fn memory_size(&self) -> usize; +} + +impl StatisticsMemorySize for Statistics { + fn memory_size(&self) -> usize { + std::mem::size_of::() + + self.num_rows.heap_size() + + self.total_byte_size.heap_size() + + self.column_statistics.heap_size() + } +} + +/// Combined memory tracking and policy-based eviction cache +/// +/// This cache leverages DashMap's built-in concurrency from DefaultFileStatisticsCache +/// and adds memory tracking + policy-based eviction on top. +pub struct CustomStatisticsCache { + /// The underlying DataFusion statistics cache (DashMap-based, already thread-safe) + inner_cache: DashMap)>, + /// The eviction policy (thread-safe) + policy: Arc>>, + /// Size limit for the cache in bytes + size_limit: AtomicUsize, + /// Eviction threshold (0.0 to 1.0) + eviction_threshold: f64, + /// Memory usage tracker - maps cache keys to their memory consumption (thread-safe) + memory_tracker: Arc>>, + /// Total memory consumed by all entries (thread-safe) + total_memory: Arc>, + /// Cache hit count (thread-safe) + hit_count: Arc>, + /// Cache miss count (thread-safe) + miss_count: Arc>, +} + +impl CustomStatisticsCache { + /// Create a new custom statistics cache + pub fn new(policy_type: PolicyType, size_limit: usize, eviction_threshold: f64) -> Self { + Self { + inner_cache: DashMap::new(), + policy: Arc::new(Mutex::new(create_policy(policy_type))), + size_limit: AtomicUsize::new(size_limit), + eviction_threshold, + memory_tracker: Arc::new(Mutex::new(HashMap::new())), + total_memory: Arc::new(Mutex::new(0)), + hit_count: Arc::new(Mutex::new(0)), + miss_count: Arc::new(Mutex::new(0)), + } + } + + /// Create with default configuration + pub fn with_default_config() -> Self { + Self::new(PolicyType::Lru, 100 * 1024 * 1024, 0.8) // 100MB default + } + + /// Get the underlying cache for compatibility + pub fn inner(&self) -> &DashMap)> { + &self.inner_cache + } + + /// Get total memory consumed by all cached statistics + pub fn memory_consumed(&self) -> usize { + self.total_memory.lock().map(|guard| *guard).unwrap_or(0) + } + + /// Get cache hit count + pub fn hit_count(&self) -> usize { + self.hit_count.lock().map(|guard| *guard).unwrap_or(0) + } + + /// Get cache miss count + pub fn miss_count(&self) -> usize { + self.miss_count.lock().map(|guard| *guard).unwrap_or(0) + } + + /// Get cache hit rate (returns value between 0.0 and 1.0) + pub fn hit_rate(&self) -> f64 { + let hits = self.hit_count(); + let misses = self.miss_count(); + let total = hits + misses; + if total == 0 { 0.0 } else { hits as f64 / total as f64 } + } + + /// Reset hit and miss counters + pub fn reset_stats(&self) { + if let Ok(mut hits) = self.hit_count.lock() { *hits = 0; } + if let Ok(mut misses) = self.miss_count.lock() { *misses = 0; } + } + + /// Update the cache size limit + pub fn update_size_limit(&self, new_limit: usize) -> CacheResult<()> { + self.size_limit.store(new_limit, Ordering::Relaxed); + let current_size = self.current_size()?; + if current_size > new_limit { + let target_eviction = current_size - (new_limit as f64 * self.eviction_threshold) as usize; + let candidates = { + if let Ok(policy_guard) = self.policy.lock() { + policy_guard.select_for_eviction(target_eviction) + } else { vec![] } + }; + for candidate_key in candidates { + if let Ok(path) = self.parse_key_to_path(&candidate_key) { + self.remove_internal(&path); + } + } + } + Ok(()) + } + + /// Switch to a different eviction policy + pub fn set_policy(&self, policy_type: PolicyType) -> CacheResult<()> { + let mut policy_guard = self.policy.lock().map_err(|e| CacheError::PolicyLockError { + reason: format!("Failed to acquire policy lock: {}", e), + })?; + let mut new_policy = create_policy(policy_type); + if let Ok(tracker) = self.memory_tracker.lock() { + for (key, size) in tracker.iter() { + new_policy.on_insert(key, *size); + } + } + *policy_guard = new_policy; + Ok(()) + } + + /// Get current policy name + pub fn policy_name(&self) -> CacheResult { + let policy_guard = self.policy.lock().map_err(|e| CacheError::PolicyLockError { + reason: format!("Failed to acquire policy lock: {}", e), + })?; + Ok(policy_guard.policy_name().to_string()) + } + + /// Get current cache size according to policy (uses actual memory consumption) + pub fn current_size(&self) -> CacheResult { + Ok(self.memory_consumed()) + } + + /// Manually trigger eviction (requires &mut self) + pub fn evict(&mut self, target_size: usize) -> CacheResult { + if target_size == 0 { return Ok(0); } + + let candidates = { + let policy_guard = self.policy.lock().map_err(|e| CacheError::PolicyLockError { + reason: format!("Failed to acquire policy lock: {}", e), + })?; + policy_guard.select_for_eviction(target_size) + }; + + let mut freed_size = 0; + for key in candidates { + let entry_size = if let Ok(tracker) = self.memory_tracker.lock() { + tracker.get(&key).copied().unwrap_or(0) + } else { 0 }; + + if entry_size > 0 { + if let Ok(path) = self.parse_key_to_path(&key) { + if self.inner_cache.remove(&path).is_some() { + if let Ok(mut tracker) = self.memory_tracker.lock() { + if let Ok(mut total) = self.total_memory.lock() { + tracker.remove(&key); + *total = total.saturating_sub(entry_size); + } + } + if let Ok(mut policy_guard) = self.policy.lock() { + policy_guard.on_remove(&key); + } + freed_size += entry_size; + } + if freed_size >= target_size { break; } + } + } + } + Ok(freed_size) + } + + /// Parse cache key back to Path + fn parse_key_to_path(&self, key: &str) -> CacheResult { + Ok(Path::from(key)) + } + + /// Remove entry internally (works with &self since inner_cache is thread-safe) + fn remove_internal(&self, k: &Path) -> Option> { + let key = k.to_string(); + let result = self.inner_cache.remove(k); + if result.is_some() { + if let Ok(mut tracker) = self.memory_tracker.lock() { + if let Ok(mut total) = self.total_memory.lock() { + if let Some(old_size) = tracker.remove(&key) { + *total = total.saturating_sub(old_size); + } + } + } + if let Ok(mut policy_guard) = self.policy.lock() { + policy_guard.on_remove(&key); + } + } + result.map(|x| x.1 .1) + } +} + +// Implement CacheAccessor - DashMap handles concurrency, we just need to handle the &mut self requirement +impl CacheAccessor> for CustomStatisticsCache { + type Extra = ObjectMeta; + + fn get(&self, k: &Path) -> Option> { + let result = self.inner_cache.get(k); + + if result.is_some() { + if let Ok(mut hits) = self.hit_count.lock() { *hits += 1; } + let key = k.to_string(); + let memory_size = if let Ok(tracker) = self.memory_tracker.lock() { + tracker.get(&key).copied().unwrap_or(0) + } else { 0 }; + if let Ok(mut policy_guard) = self.policy.lock() { + policy_guard.on_access(&key, memory_size); + } + } else { + if let Ok(mut misses) = self.miss_count.lock() { *misses += 1; } + } + + result.map(|s| Some(Arc::clone(&s.value().1))).unwrap_or(None) + } + + fn get_with_extra(&self, k: &Path, _extra: &Self::Extra) -> Option> { + self.get(k) + } + + fn put(&self, k: &Path, v: Arc) -> Option> { + let meta = ObjectMeta { + location: k.clone(), + last_modified: chrono::Utc::now(), + size: 0, + e_tag: None, + version: None, + }; + self.put_with_extra(k, v, &meta) + } + + fn put_with_extra( + &self, + k: &Path, + v: Arc, + e: &Self::Extra, + ) -> Option> { + let key = k.to_string(); + let memory_size = v.memory_size(); + + let eviction_candidates = if let Ok(tracker) = self.memory_tracker.lock() { + if let Ok(total) = self.total_memory.lock() { + let current_size = *total; + let size_limit = self.size_limit.load(Ordering::Relaxed); + let threshold = (size_limit as f64 * self.eviction_threshold) as usize; + if current_size + memory_size > threshold { + let target_eviction = (current_size + memory_size) - (size_limit as f64 * 0.6) as usize; + if let Ok(policy_guard) = self.policy.lock() { + policy_guard.select_for_eviction(target_eviction) + } else { vec![] } + } else { vec![] } + } else { vec![] } + } else { vec![] }; + + for candidate_key in eviction_candidates { + if let Ok(path) = self.parse_key_to_path(&candidate_key) { + self.remove_internal(&path); + } + } + + let result = self.inner_cache.insert(k.clone(), (e.clone(), v)).map(|x| x.1); + + if let Ok(mut tracker) = self.memory_tracker.lock() { + if let Ok(mut total) = self.total_memory.lock() { + if let Some(old_size) = tracker.get(&key) { + *total = total.saturating_sub(*old_size); + } + tracker.insert(key.clone(), memory_size); + *total += memory_size; + } + } + + if let Ok(mut policy_guard) = self.policy.lock() { + policy_guard.on_insert(&key, memory_size); + } + + result + } + + fn remove(&self, k: &Path) -> Option> { + let key = k.to_string(); + let result = self.inner_cache.remove(k); + if result.is_some() { + if let Ok(mut tracker) = self.memory_tracker.lock() { + if let Ok(mut total) = self.total_memory.lock() { + if let Some(old_size) = tracker.remove(&key) { + *total = total.saturating_sub(old_size); + } + } + } + if let Ok(mut policy_guard) = self.policy.lock() { + policy_guard.on_remove(&key); + } + } + result.map(|x| x.1 .1) + } + + fn contains_key(&self, k: &Path) -> bool { + self.inner_cache.get(k).is_some() + } + + fn len(&self) -> usize { + self.memory_tracker.lock().map(|t| t.len()).unwrap_or(0) + } + + fn clear(&self) { + self.inner_cache.clear(); + if let Ok(mut tracker) = self.memory_tracker.lock() { tracker.clear(); } + if let Ok(mut total) = self.total_memory.lock() { *total = 0; } + if let Ok(mut policy_guard) = self.policy.lock() { policy_guard.clear(); } + self.reset_stats(); + } + + fn name(&self) -> String { + format!( + "CustomStatisticsCache({})", + self.policy_name().unwrap_or_else(|_| "unknown".to_string()) + ) + } +} + +impl datafusion::execution::cache::cache_manager::FileStatisticsCache for CustomStatisticsCache { + fn list_entries(&self) -> std::collections::HashMap { + std::collections::HashMap::new() + } +} + +impl Default for CustomStatisticsCache { + fn default() -> Self { + Self::with_default_config() + } +} + +/// Compute statistics from a parquet file using DataFusion's built-in functionality +pub fn compute_parquet_statistics(file_path: &str) -> Result> { + use datafusion::parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + use datafusion::datasource::physical_plan::parquet::metadata::DFParquetMetadata; + use object_store::local::LocalFileSystem; + use object_store::path::Path; + + let file = File::open(file_path)?; + let builder = ParquetRecordBatchReaderBuilder::try_new(file)?; + let metadata = builder.metadata(); + let schema = builder.schema().clone(); + + // Create ObjectStore and ObjectMeta for the file + let _store: Arc = Arc::new(LocalFileSystem::new()); + let path = Path::from(file_path); + let file_metadata = std::fs::metadata(file_path)?; + let _object_meta = ObjectMeta { + location: path, + last_modified: chrono::DateTime::from(file_metadata.modified()?), + size: file_metadata.len(), + e_tag: None, + version: None, + }; + + // Use DataFusion's method to extract statistics from parquet metadata + // statistics_from_parquet_metadata is an associated function that takes metadata and schema + let statistics = DFParquetMetadata::statistics_from_parquet_metadata(metadata, &schema)?; + Ok(statistics) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use datafusion::common::stats::Precision; + + fn create_test_statistics() -> Statistics { + Statistics { + num_rows: Precision::Exact(1000), + total_byte_size: Precision::Exact(50000), + column_statistics: vec![], + } + } + + fn create_test_path(name: &str) -> Path { + Path::from(format!("/test/{}.parquet", name)) + } + + fn create_test_meta(path: &Path) -> ObjectMeta { + ObjectMeta { + location: path.clone(), + last_modified: Utc::now(), + size: 1000, + e_tag: None, + version: None, + } + } + + #[test] + fn test_custom_stats_cache_creation() { + let cache = CustomStatisticsCache::new(PolicyType::Lru, 1024 * 1024, 0.8); + assert_eq!(cache.policy_name().unwrap(), "lru"); + assert_eq!(cache.memory_consumed(), 0); + assert_eq!(cache.len(), 0); + } + + #[test] + fn test_memory_tracking_with_policy() { + let cache = CustomStatisticsCache::with_default_config(); + assert_eq!(cache.memory_consumed(), 0); + assert_eq!(cache.len(), 0); + + let path = create_test_path("file1"); + let meta = create_test_meta(&path); + let stats = Arc::new(create_test_statistics()); + cache.put_with_extra(&path, stats, &meta); + + assert!(cache.memory_consumed() > 0); + assert_eq!(cache.len(), 1); + assert!(cache.get(&path).is_some()); + } + + #[test] + fn test_policy_based_eviction_with_memory() { + let cache = CustomStatisticsCache::new(PolicyType::Lru, 1000, 0.8); + for i in 0..10 { + let path = create_test_path(&format!("file{}", i)); + let meta = create_test_meta(&path); + let stats = Arc::new(create_test_statistics()); + cache.put_with_extra(&path, stats, &meta); + } + assert!(cache.memory_consumed() <= 1000); + assert!(cache.len() > 0); + } + + #[test] + fn test_manual_eviction_with_memory_tracking() { + let mut cache = CustomStatisticsCache::with_default_config(); + for i in 0..5 { + let path = create_test_path(&format!("file{}", i)); + let meta = create_test_meta(&path); + let stats = Arc::new(create_test_statistics()); + cache.put_with_extra(&path, stats, &meta); + } + let memory_before = cache.memory_consumed(); + assert!(memory_before > 0); + let freed = cache.evict(memory_before / 2).unwrap(); + assert!(freed > 0); + assert!(cache.memory_consumed() < memory_before); + } + + #[test] + fn test_policy_switching_with_memory() { + let cache = CustomStatisticsCache::with_default_config(); + for i in 0..3 { + let path = create_test_path(&format!("file{}", i)); + let meta = create_test_meta(&path); + let stats = Arc::new(create_test_statistics()); + cache.put_with_extra(&path, stats, &meta); + } + let memory_before = cache.memory_consumed(); + assert_eq!(cache.policy_name().unwrap(), "lru"); + cache.set_policy(PolicyType::Lfu).unwrap(); + assert_eq!(cache.policy_name().unwrap(), "lfu"); + assert_eq!(cache.memory_consumed(), memory_before); + } + + #[test] + fn test_remove_with_memory_tracking() { + let cache = CustomStatisticsCache::with_default_config(); + let path1 = create_test_path("file1"); + let path2 = create_test_path("file2"); + let meta1 = create_test_meta(&path1); + let meta2 = create_test_meta(&path2); + let stats = Arc::new(create_test_statistics()); + + cache.put_with_extra(&path1, stats.clone(), &meta1); + cache.put_with_extra(&path2, stats, &meta2); + let memory_with_two = cache.memory_consumed(); + assert_eq!(cache.len(), 2); + + cache.remove(&path1); + assert_eq!(cache.len(), 1); + assert!(cache.memory_consumed() < memory_with_two); + + cache.remove(&path2); + assert_eq!(cache.memory_consumed(), 0); + assert_eq!(cache.len(), 0); + } + + #[test] + fn test_clear_with_memory_tracking() { + let cache = CustomStatisticsCache::with_default_config(); + for i in 0..3 { + let path = create_test_path(&format!("file{}", i)); + let meta = create_test_meta(&path); + let stats = Arc::new(create_test_statistics()); + cache.put_with_extra(&path, stats, &meta); + } + assert!(cache.memory_consumed() > 0); + cache.clear(); + assert_eq!(cache.memory_consumed(), 0); + assert_eq!(cache.len(), 0); + } + + #[test] + fn test_hit_count_tracking() { + let cache = CustomStatisticsCache::with_default_config(); + let path = create_test_path("file1"); + let meta = create_test_meta(&path); + let stats = Arc::new(create_test_statistics()); + cache.put_with_extra(&path, stats, &meta); + + assert!(cache.get(&path).is_some()); + assert_eq!(cache.hit_count(), 1); + assert!(cache.get(&path).is_some()); + assert_eq!(cache.hit_count(), 2); + assert_eq!(cache.miss_count(), 0); + } + + #[test] + fn test_miss_count_tracking() { + let cache = CustomStatisticsCache::with_default_config(); + let path = create_test_path("nonexistent"); + assert!(cache.get(&path).is_none()); + assert_eq!(cache.miss_count(), 1); + assert!(cache.get(&path).is_none()); + assert_eq!(cache.miss_count(), 2); + assert_eq!(cache.hit_count(), 0); + } + + #[test] + fn test_hit_rate_calculation() { + let cache = CustomStatisticsCache::with_default_config(); + assert_eq!(cache.hit_rate(), 0.0); + + let path1 = create_test_path("file1"); + let meta1 = create_test_meta(&path1); + let stats = Arc::new(create_test_statistics()); + cache.put_with_extra(&path1, stats, &meta1); + + cache.get(&path1); // hit + cache.get(&path1); // hit + assert_eq!(cache.hit_rate(), 1.0); + + let path2 = create_test_path("missing"); + cache.get(&path2); // miss + assert!((cache.hit_rate() - 0.6666666666666666).abs() < 0.0001); + } + + #[test] + fn test_reset_stats() { + let cache = CustomStatisticsCache::with_default_config(); + let path = create_test_path("file1"); + let meta = create_test_meta(&path); + let stats = Arc::new(create_test_statistics()); + cache.put_with_extra(&path, stats, &meta); + + cache.get(&path); + let path2 = create_test_path("missing"); + cache.get(&path2); + assert_eq!(cache.hit_count(), 1); + assert_eq!(cache.miss_count(), 1); + + cache.reset_stats(); + assert_eq!(cache.hit_count(), 0); + assert_eq!(cache.miss_count(), 0); + assert_eq!(cache.len(), 1); // entries still exist + } + + #[test] + fn test_clear_resets_stats() { + let cache = CustomStatisticsCache::with_default_config(); + let path = create_test_path("file1"); + let meta = create_test_meta(&path); + let stats = Arc::new(create_test_statistics()); + cache.put_with_extra(&path, stats, &meta); + cache.get(&path); + + cache.clear(); + assert_eq!(cache.hit_count(), 0); + assert_eq!(cache.miss_count(), 0); + assert_eq!(cache.len(), 0); + } + + #[test] + fn test_concurrent_operations() { + use std::thread; + let cache = Arc::new(CustomStatisticsCache::with_default_config()); + let mut handles = vec![]; + + for i in 0..10 { + let cache_clone = Arc::clone(&cache); + let handle = thread::spawn(move || { + let path = create_test_path(&format!("concurrent{}", i)); + let meta = create_test_meta(&path); + let stats = Arc::new(create_test_statistics()); + cache_clone.put_with_extra(&path, stats, &meta); + assert!(cache_clone.get(&path).is_some()); + }); + handles.push(handle); + } + + for handle in handles { handle.join().unwrap(); } + assert!(cache.len() > 0); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java index 8395f4a5b2f49..a9e9ecfe27d0e 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.be.datafusion.cache.CacheSettings; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Setting; @@ -125,6 +126,7 @@ public Collection createComponents( .memoryPoolLimit(memoryPoolLimit) .spillMemoryLimit(spillMemoryLimit) .spillDirectory(spillDir) + .clusterSettings(clusterService.getClusterSettings()) .build(); dataFusionService.start(); logger.debug("DataFusion plugin initialized — memory pool {}B, spill limit {}B", memoryPoolLimit, spillMemoryLimit); @@ -171,7 +173,17 @@ ClusterService getClusterService() { @Override public List> getSettings() { - return List.of(DATAFUSION_MEMORY_POOL_LIMIT, DATAFUSION_SPILL_MEMORY_LIMIT, DATAFUSION_REDUCE_INPUT_MODE); + return List.of( + DATAFUSION_MEMORY_POOL_LIMIT, + DATAFUSION_SPILL_MEMORY_LIMIT, + DATAFUSION_REDUCE_INPUT_MODE, + CacheSettings.METADATA_CACHE_SIZE_LIMIT, + CacheSettings.STATISTICS_CACHE_SIZE_LIMIT, + CacheSettings.METADATA_CACHE_EVICTION_TYPE, + CacheSettings.STATISTICS_CACHE_EVICTION_TYPE, + CacheSettings.METADATA_CACHE_ENABLED, + CacheSettings.STATISTICS_CACHE_ENABLED + ); } @Override diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java index 48d87a6ecfc18..b626ff15592c3 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java @@ -12,8 +12,11 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.be.datafusion.cache.CacheManager; +import org.opensearch.be.datafusion.cache.CacheUtils; import org.opensearch.be.datafusion.nativelib.NativeBridge; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; +import org.opensearch.common.settings.ClusterSettings; import java.io.IOException; import java.util.Collection; @@ -36,6 +39,7 @@ public class DataFusionService extends AbstractLifecycleComponent { private final long spillMemoryLimit; private final String spillDirectory; private final int cpuThreads; + private final ClusterSettings clusterSettings; /** Handle to the native DataFusion global runtime (memory pool + cache). */ private volatile NativeRuntimeHandle runtimeHandle; @@ -43,6 +47,9 @@ public class DataFusionService extends AbstractLifecycleComponent { /** Shared Arrow allocator for all DataFusion result streams on this node. */ private volatile RootAllocator rootAllocator; + /** Cache manager for pre-warming and managing native caches. */ + private volatile CacheManager cacheManager; + /** Counter for generating unique child allocator names. */ private final AtomicLong allocatorCounter = new AtomicLong(); @@ -51,6 +58,7 @@ private DataFusionService(Builder builder) { this.spillMemoryLimit = builder.spillMemoryLimit; this.spillDirectory = builder.spillDirectory; this.cpuThreads = builder.cpuThreads; + this.clusterSettings = builder.clusterSettings; } /** Creates a new builder. */ @@ -64,9 +72,19 @@ protected void doStart() { NativeBridge.initTokioRuntimeManager(cpuThreads); logger.debug("Tokio runtime manager initialized with {} CPU threads", cpuThreads); - long ptr = NativeBridge.createGlobalRuntime(memoryPoolLimit, 0L, spillDirectory, spillMemoryLimit); + long cacheManagerPtr = 0L; + if (clusterSettings != null) { + cacheManagerPtr = CacheUtils.createCacheConfig(clusterSettings); + } + + long ptr = NativeBridge.createGlobalRuntime(memoryPoolLimit, cacheManagerPtr, spillDirectory, spillMemoryLimit); this.runtimeHandle = new NativeRuntimeHandle(ptr); this.rootAllocator = new RootAllocator(memoryPoolLimit); + + if (clusterSettings != null) { + this.cacheManager = new CacheManager(runtimeHandle); + } + logger.debug("DataFusion service started — memory pool {}B, spill limit {}B", memoryPoolLimit, spillMemoryLimit); } @@ -123,6 +141,13 @@ public BufferAllocator newChildAllocator() { return alloc.newChildAllocator("datafusion-stream-" + allocatorCounter.getAndIncrement(), 0, alloc.getLimit()); } + /** + * Returns the cache manager, or null if caching is not configured. + */ + public CacheManager getCacheManager() { + return cacheManager; + } + /** * Notifies the native cache that new files are available for caching. * @param filePaths absolute paths of the new files @@ -166,6 +191,7 @@ public static class Builder { private long spillMemoryLimit = Runtime.getRuntime().maxMemory() / 8; private String spillDirectory = System.getProperty("java.io.tmpdir"); private int cpuThreads = Runtime.getRuntime().availableProcessors(); + private ClusterSettings clusterSettings; private Builder() {} @@ -205,6 +231,15 @@ public Builder cpuThreads(int threads) { return this; } + /** + * Sets the cluster settings for cache configuration. + * @param clusterSettings the cluster settings + */ + public Builder clusterSettings(ClusterSettings clusterSettings) { + this.clusterSettings = clusterSettings; + return this; + } + /** Builds the {@link DataFusionService}. */ public DataFusionService build() { return new DataFusionService(this); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheManager.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheManager.java new file mode 100644 index 0000000000000..64af4ec7af147 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheManager.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion.cache; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.be.datafusion.NativeRuntimeHandle; +import org.opensearch.be.datafusion.nativelib.NativeBridge; + +import java.util.List; + +/** + * Manages cache lifecycle for DataFusion caches. + * Holds the cache manager pointer for runtime cache operations. + */ +public class CacheManager { + private static final Logger logger = LogManager.getLogger(CacheManager.class); + + NativeRuntimeHandle runtimeHandle; + + public CacheManager(NativeRuntimeHandle runtimeHandle) { + this.runtimeHandle = runtimeHandle; + } + + public void addFilesToCacheManager(List files) { + try { + if (files == null || files.isEmpty()) { + return; + } + String[] filesArray = files.toArray(new String[0]); + NativeBridge.cacheManagerAddFiles(runtimeHandle.get(), filesArray); + } catch (Exception e) { + logger.error("Error adding files to cache manager", e); + } + } + + public void removeFilesFromCacheManager(List files) { + try { + if (files == null || files.isEmpty()) { + return; + } + String[] filesArray = files.toArray(new String[0]); + NativeBridge.cacheManagerRemoveFiles(runtimeHandle.get(), filesArray); + } catch (Exception e) { + logger.error("Error removing files from cache manager", e); + } + } + + public void clearAllCache() { + try { + NativeBridge.cacheManagerClear(runtimeHandle.get()); + } catch (Exception e) { + logger.error("Error clearing cache manager", e); + } + } + + public void clearCacheForCacheType(CacheUtils.CacheType cacheType) { + try { + NativeBridge.cacheManagerClearByCacheType(runtimeHandle.get(), cacheType.getCacheTypeName()); + } catch (Exception e) { + logger.error("Error clearing cache for cache type", e); + } + } + + public long getMemoryConsumed(CacheUtils.CacheType cacheType) { + try { + return NativeBridge.cacheManagerGetMemoryConsumedForCacheType(runtimeHandle.get(), cacheType.getCacheTypeName()); + } catch (Exception e) { + logger.error("Error getting memory consumed for cache type", e); + return 0; + } + } + + public long getTotalMemoryConsumed() { + try { + return NativeBridge.cacheManagerGetTotalMemoryConsumed(runtimeHandle.get()); + } catch (Exception e) { + logger.error("Error getting total memory consumed", e); + return 0; + } + } + + public void updateSizeLimit(CacheUtils.CacheType cacheType, long sizeLimit) { + try { + // TODO: Add updateSizeLimitForCacheType FFM function when needed + logger.warn("updateSizeLimit not yet implemented for FFM bridge"); + } catch (Exception e) { + logger.error("Error updating size limit", e); + } + } + + public boolean getEntryFromCacheType(CacheUtils.CacheType cacheType, String filePath) { + try { + return NativeBridge.cacheManagerGetItemByCacheType(runtimeHandle.get(), cacheType.getCacheTypeName(), filePath); + } catch (Exception e) { + logger.error("Error getting entry from cache", e); + return false; + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheSettings.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheSettings.java new file mode 100644 index 0000000000000..0b02a7d6bf4dc --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheSettings.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion.cache; + +import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; + +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + +public class CacheSettings { + + public static final String METADATA_CACHE_SIZE_LIMIT_KEY = "datafusion.metadata.cache.size.limit"; + public static final String STATISTICS_CACHE_SIZE_LIMIT_KEY = "datafusion.statistics.cache.size.limit"; + public static final Setting METADATA_CACHE_SIZE_LIMIT = new Setting<>( + METADATA_CACHE_SIZE_LIMIT_KEY, + "250mb", + (s) -> ByteSizeValue.parseBytesSizeValue(s, new ByteSizeValue(1000, ByteSizeUnit.KB), METADATA_CACHE_SIZE_LIMIT_KEY), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public static final Setting STATISTICS_CACHE_SIZE_LIMIT = new Setting<>( + STATISTICS_CACHE_SIZE_LIMIT_KEY, + "100mb", + (s) -> ByteSizeValue.parseBytesSizeValue(s, new ByteSizeValue(0, ByteSizeUnit.KB), STATISTICS_CACHE_SIZE_LIMIT_KEY), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public static final Setting METADATA_CACHE_EVICTION_TYPE = new Setting( + "datafusion.metadata.cache.eviction.type", + "LRU", + Function.identity(), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public static final Setting STATISTICS_CACHE_EVICTION_TYPE = new Setting( + "datafusion.statistics.cache.eviction.type", + "LRU", + Function.identity(), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public static final String METADATA_CACHE_ENABLED_KEY = "datafusion.metadata.cache.enabled"; + public static final Setting METADATA_CACHE_ENABLED = Setting.boolSetting( + METADATA_CACHE_ENABLED_KEY, + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public static final String STATISTICS_CACHE_ENABLED_KEY = "datafusion.statistics.cache.enabled"; + public static final Setting STATISTICS_CACHE_ENABLED = Setting.boolSetting( + STATISTICS_CACHE_ENABLED_KEY, + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public static final List> CACHE_SETTINGS = Arrays.asList( + METADATA_CACHE_SIZE_LIMIT, + METADATA_CACHE_EVICTION_TYPE, + STATISTICS_CACHE_SIZE_LIMIT, + STATISTICS_CACHE_EVICTION_TYPE + ); + + public static final List> CACHE_ENABLED = Arrays.asList(METADATA_CACHE_ENABLED, STATISTICS_CACHE_ENABLED); +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheUtils.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheUtils.java new file mode 100644 index 0000000000000..16f55daf8a983 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/cache/CacheUtils.java @@ -0,0 +1,121 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion.cache; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.be.datafusion.nativelib.NativeBridge; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.unit.ByteSizeValue; + +import static org.opensearch.be.datafusion.cache.CacheSettings.METADATA_CACHE_ENABLED; +import static org.opensearch.be.datafusion.cache.CacheSettings.METADATA_CACHE_EVICTION_TYPE; +import static org.opensearch.be.datafusion.cache.CacheSettings.METADATA_CACHE_SIZE_LIMIT; +import static org.opensearch.be.datafusion.cache.CacheSettings.STATISTICS_CACHE_ENABLED; +import static org.opensearch.be.datafusion.cache.CacheSettings.STATISTICS_CACHE_EVICTION_TYPE; +import static org.opensearch.be.datafusion.cache.CacheSettings.STATISTICS_CACHE_SIZE_LIMIT; + +/** + * Utility class for cache initialization and configuration. + * Contains the CacheType enum and methods for creating cache configurations. + */ +public final class CacheUtils { + private static final Logger logger = LogManager.getLogger(CacheUtils.class); + + // Private constructor to prevent instantiation + private CacheUtils() {} + + /** + * Cache type enumeration with associated settings. + */ + public enum CacheType { + METADATA("METADATA", METADATA_CACHE_ENABLED, METADATA_CACHE_SIZE_LIMIT, METADATA_CACHE_EVICTION_TYPE), + + STATISTICS("STATISTICS", STATISTICS_CACHE_ENABLED, STATISTICS_CACHE_SIZE_LIMIT, STATISTICS_CACHE_EVICTION_TYPE); + + private final String cacheTypeName; + private final Setting enabledSetting; + private final Setting sizeLimitSetting; + private final Setting evictionTypeSetting; + + CacheType( + String cacheTypeName, + Setting enabledSetting, + Setting sizeLimitSetting, + Setting evictionTypeSetting + ) { + this.cacheTypeName = cacheTypeName; + this.enabledSetting = enabledSetting; + this.sizeLimitSetting = sizeLimitSetting; + this.evictionTypeSetting = evictionTypeSetting; + } + + public boolean isEnabled(ClusterSettings clusterSettings) { + return clusterSettings.get(enabledSetting); + } + + public Setting getEnabledSetting() { + return enabledSetting; + } + + public Setting getSizeLimitSetting() { + return sizeLimitSetting; + } + + public Setting getEvictionTypeSetting() { + return evictionTypeSetting; + } + + public ByteSizeValue getSizeLimit(ClusterSettings clusterSettings) { + return clusterSettings.get(sizeLimitSetting); + } + + public String getEvictionType(ClusterSettings clusterSettings) { + return clusterSettings.get(evictionTypeSetting); + } + + public String getCacheTypeName() { + return cacheTypeName; + } + } + + /** + * Creates and configures a CacheManagerConfig pointer with all enabled caches. + * + * @param clusterSettings OpenSearch cluster settings containing cache configuration + */ + public static long createCacheConfig(ClusterSettings clusterSettings) { + logger.info("Initializing cache configuration"); + + long cacheManagerPtr = NativeBridge.createCustomCacheManager(); + // Configure each enabled cache type + for (CacheType type : CacheType.values()) { + if (type.isEnabled(clusterSettings)) { + logger.info( + "Configuring {} cache: size={} bytes, eviction={}", + type.getCacheTypeName(), + type.getSizeLimit(clusterSettings).getBytes(), + type.getEvictionType(clusterSettings) + ); + + NativeBridge.createCache( + cacheManagerPtr, + type.cacheTypeName, + type.getSizeLimit(clusterSettings).getBytes(), + type.getEvictionType(clusterSettings) + ); + } else { + logger.debug("Cache type {} is disabled", type.getCacheTypeName()); + } + } + logger.info("Cache configuration completed"); + return cacheManagerPtr; + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index afed97c2325b4..387d18376cead 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -59,6 +59,16 @@ public final class NativeBridge { private static final MethodHandle SENDER_SEND; private static final MethodHandle SENDER_CLOSE; private static final MethodHandle REGISTER_MEMTABLE; + private static final MethodHandle CREATE_CUSTOM_CACHE_MANAGER; + private static final MethodHandle DESTROY_CUSTOM_CACHE_MANAGER; + private static final MethodHandle CREATE_CACHE; + private static final MethodHandle CACHE_MANAGER_ADD_FILES; + private static final MethodHandle CACHE_MANAGER_REMOVE_FILES; + private static final MethodHandle CACHE_MANAGER_CLEAR; + private static final MethodHandle CACHE_MANAGER_CLEAR_BY_TYPE; + private static final MethodHandle CACHE_MANAGER_GET_MEMORY_BY_TYPE; + private static final MethodHandle CACHE_MANAGER_GET_TOTAL_MEMORY; + private static final MethodHandle CACHE_MANAGER_CONTAINS_BY_TYPE; static { SymbolLookup lib = NativeLibraryLoader.symbolLookup(); @@ -77,6 +87,7 @@ public final class NativeBridge { CREATE_GLOBAL_RUNTIME = linker.downcallHandle( lib.find("df_create_global_runtime").orElseThrow(), FunctionDescriptor.of( + ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, @@ -218,6 +229,87 @@ public final class NativeBridge { ) ); + CREATE_CUSTOM_CACHE_MANAGER = linker.downcallHandle( + lib.find("df_create_custom_cache_manager").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG) + ); + + DESTROY_CUSTOM_CACHE_MANAGER = linker.downcallHandle( + lib.find("df_destroy_custom_cache_manager").orElseThrow(), + FunctionDescriptor.ofVoid(ValueLayout.JAVA_LONG) + ); + + // i64 df_create_cache(mgr_ptr, type_ptr, type_len, size_limit, eviction_ptr, eviction_len) + CREATE_CACHE = linker.downcallHandle( + lib.find("df_create_cache").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG + ) + ); + + // i64 df_cache_manager_add_files(runtime_ptr, files_ptr, files_len_ptr, files_count) + CACHE_MANAGER_ADD_FILES = linker.downcallHandle( + lib.find("df_cache_manager_add_files").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG + ) + ); + + CACHE_MANAGER_REMOVE_FILES = linker.downcallHandle( + lib.find("df_cache_manager_remove_files").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG + ) + ); + + CACHE_MANAGER_CLEAR = linker.downcallHandle( + lib.find("df_cache_manager_clear").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + + // i64 df_cache_manager_clear_by_type(runtime_ptr, type_ptr, type_len) + CACHE_MANAGER_CLEAR_BY_TYPE = linker.downcallHandle( + lib.find("df_cache_manager_clear_by_type").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) + ); + + CACHE_MANAGER_GET_MEMORY_BY_TYPE = linker.downcallHandle( + lib.find("df_cache_manager_get_memory_by_type").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) + ); + + CACHE_MANAGER_GET_TOTAL_MEMORY = linker.downcallHandle( + lib.find("df_cache_manager_get_total_memory").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + + // i64 df_cache_manager_contains_by_type(runtime_ptr, type_ptr, type_len, file_ptr, file_len) + CACHE_MANAGER_CONTAINS_BY_TYPE = linker.downcallHandle( + lib.find("df_cache_manager_contains_by_type").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG + ) + ); + // Hand the five filter-tree upcall stubs to Rust now. No explicit // caller step required — as soon as this class is loaded, callbacks // are installed and `df_execute_indexed_query` can dispatch into Java. @@ -336,7 +428,7 @@ public static void shutdownTokioRuntimeManager() { public static long createGlobalRuntime(long memoryLimit, long cacheManagerPtr, String spillDir, long spillLimit) { try (var call = new NativeCall()) { var dir = call.str(spillDir); - return call.invoke(CREATE_GLOBAL_RUNTIME, memoryLimit, dir.segment(), dir.len(), spillLimit); + return call.invoke(CREATE_GLOBAL_RUNTIME, memoryLimit, cacheManagerPtr, dir.segment(), dir.len(), spillLimit); } } @@ -551,9 +643,74 @@ public static long registerMemtable(long sessionPtr, String inputId, byte[] sche } } - public static void cacheManagerAddFiles(long runtimePtr, String[] filePaths) {} + public static long createCustomCacheManager() { + try { + return NativeLibraryLoader.checkResult((long) CREATE_CUSTOM_CACHE_MANAGER.invokeExact()); + } catch (Throwable t) { + throw t instanceof RuntimeException ? (RuntimeException) t : new RuntimeException(t); + } + } + + public static void destroyCustomCacheManager(long ptr) { + NativeCall.invokeVoid(DESTROY_CUSTOM_CACHE_MANAGER, ptr); + } + + public static void createCache(long cacheManagerPtr, String cacheType, long sizeLimit, String evictionType) { + try (var call = new NativeCall()) { + var type = call.str(cacheType); + var eviction = call.str(evictionType); + call.invoke(CREATE_CACHE, cacheManagerPtr, type.segment(), type.len(), sizeLimit, eviction.segment(), eviction.len()); + } + } + + public static void cacheManagerAddFiles(long runtimePtr, String[] filePaths) { + try (var call = new NativeCall()) { + var f = call.strArray(filePaths); + call.invoke(CACHE_MANAGER_ADD_FILES, runtimePtr, f.ptrs(), f.lens(), f.count()); + } + } - public static void cacheManagerRemoveFiles(long runtimePtr, String[] filePaths) {} + public static void cacheManagerRemoveFiles(long runtimePtr, String[] filePaths) { + try (var call = new NativeCall()) { + var f = call.strArray(filePaths); + call.invoke(CACHE_MANAGER_REMOVE_FILES, runtimePtr, f.ptrs(), f.lens(), f.count()); + } + } + + public static void cacheManagerClear(long runtimePtr) { + try (var call = new NativeCall()) { + call.invoke(CACHE_MANAGER_CLEAR, runtimePtr); + } + } + + public static void cacheManagerClearByCacheType(long runtimePtr, String cacheType) { + try (var call = new NativeCall()) { + var type = call.str(cacheType); + call.invoke(CACHE_MANAGER_CLEAR_BY_TYPE, runtimePtr, type.segment(), type.len()); + } + } + + public static long cacheManagerGetMemoryConsumedForCacheType(long runtimePtr, String cacheType) { + try (var call = new NativeCall()) { + var type = call.str(cacheType); + return call.invoke(CACHE_MANAGER_GET_MEMORY_BY_TYPE, runtimePtr, type.segment(), type.len()); + } + } + + public static long cacheManagerGetTotalMemoryConsumed(long runtimePtr) { + try (var call = new NativeCall()) { + return call.invoke(CACHE_MANAGER_GET_TOTAL_MEMORY, runtimePtr); + } + } + + public static boolean cacheManagerGetItemByCacheType(long runtimePtr, String cacheType, String filePath) { + try (var call = new NativeCall()) { + var type = call.str(cacheType); + var file = call.str(filePath); + long result = call.invoke(CACHE_MANAGER_CONTAINS_BY_TYPE, runtimePtr, type.segment(), type.len(), file.segment(), file.len()); + return result != 0; + } + } public static void initLogger() {} } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionServiceTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionServiceTests.java index f4b4185fe75c5..f6ec7d14a0661 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionServiceTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionServiceTests.java @@ -8,10 +8,19 @@ package org.opensearch.be.datafusion; +import org.opensearch.be.datafusion.cache.CacheSettings; import org.opensearch.be.datafusion.nativelib.NativeBridge; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; import org.opensearch.test.OpenSearchTestCase; import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.opensearch.common.settings.ClusterSettings.BUILT_IN_CLUSTER_SETTINGS; /** * Tests for DataFusionService lifecycle and NativeRuntimeHandle. @@ -93,4 +102,93 @@ public void testCacheFileOperationsDoNotThrow() { service.stop(); } + + public void testServiceWithCacheEnabled() { + ensureTokioInit(); + ClusterSettings clusterSettings = createCacheClusterSettings(Settings.EMPTY); + Path spillDir = createTempDir("spill"); + + DataFusionService service = DataFusionService.builder() + .memoryPoolLimit(64 * 1024 * 1024) + .spillMemoryLimit(32 * 1024 * 1024) + .spillDirectory(spillDir.toString()) + .cpuThreads(2) + .clusterSettings(clusterSettings) + .build(); + service.start(); + + assertNotNull(service.getCacheManager()); + assertNotNull(service.getNativeRuntime()); + assertTrue(service.getNativeRuntime().isOpen()); + + service.stop(); + } + + public void testServiceWithoutCacheReturnsNullCacheManager() { + ensureTokioInit(); + Path spillDir = createTempDir("spill"); + + DataFusionService service = DataFusionService.builder() + .memoryPoolLimit(64 * 1024 * 1024) + .spillMemoryLimit(32 * 1024 * 1024) + .spillDirectory(spillDir.toString()) + .cpuThreads(2) + .build(); + service.start(); + + assertNull(service.getCacheManager()); + + service.stop(); + } + + public void testPluginRegistersAllCacheSettings() { + List> settings = new DataFusionPlugin().getSettings(); + assertTrue(settings.contains(CacheSettings.METADATA_CACHE_SIZE_LIMIT)); + assertTrue(settings.contains(CacheSettings.STATISTICS_CACHE_SIZE_LIMIT)); + assertTrue(settings.contains(CacheSettings.METADATA_CACHE_EVICTION_TYPE)); + assertTrue(settings.contains(CacheSettings.STATISTICS_CACHE_EVICTION_TYPE)); + assertTrue(settings.contains(CacheSettings.METADATA_CACHE_ENABLED)); + assertTrue(settings.contains(CacheSettings.STATISTICS_CACHE_ENABLED)); + } + + public void testNativeBridgeCacheManagerLifecycle() { + ensureTokioInit(); + long ptr = NativeBridge.createCustomCacheManager(); + assertTrue(ptr != 0); + NativeBridge.destroyCustomCacheManager(ptr); + } + + public void testNativeBridgeCreateCacheOnManager() { + ensureTokioInit(); + long ptr = NativeBridge.createCustomCacheManager(); + NativeBridge.createCache(ptr, "METADATA", 250 * 1024 * 1024, "LRU"); + NativeBridge.createCache(ptr, "STATISTICS", 100 * 1024 * 1024, "LRU"); + NativeBridge.destroyCustomCacheManager(ptr); + } + + public void testRuntimeWithCacheManagerPointer() { + ensureTokioInit(); + long cachePtr = NativeBridge.createCustomCacheManager(); + NativeBridge.createCache(cachePtr, "METADATA", 250 * 1024 * 1024, "LRU"); + NativeBridge.createCache(cachePtr, "STATISTICS", 100 * 1024 * 1024, "LRU"); + + Path spillDir = createTempDir("spill"); + long runtimePtr = NativeBridge.createGlobalRuntime(64 * 1024 * 1024, cachePtr, spillDir.toString(), 32 * 1024 * 1024); + assertTrue(runtimePtr != 0); + + NativeBridge.closeGlobalRuntime(runtimePtr); + } + + private ClusterSettings createCacheClusterSettings(Settings settings) { + Set> all = new HashSet<>(BUILT_IN_CLUSTER_SETTINGS); + all.add(CacheSettings.METADATA_CACHE_ENABLED); + all.add(CacheSettings.METADATA_CACHE_SIZE_LIMIT); + all.add(CacheSettings.METADATA_CACHE_EVICTION_TYPE); + all.add(CacheSettings.STATISTICS_CACHE_ENABLED); + all.add(CacheSettings.STATISTICS_CACHE_SIZE_LIMIT); + all.add(CacheSettings.STATISTICS_CACHE_EVICTION_TYPE); + all.add(DataFusionPlugin.DATAFUSION_MEMORY_POOL_LIMIT); + all.add(DataFusionPlugin.DATAFUSION_SPILL_MEMORY_LIMIT); + return new ClusterSettings(settings, all); + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionCacheManagerTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionCacheManagerTests.java new file mode 100644 index 0000000000000..f09497c72564c --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionCacheManagerTests.java @@ -0,0 +1,218 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.be.datafusion.cache.CacheManager; +import org.opensearch.be.datafusion.cache.CacheSettings; +import org.opensearch.be.datafusion.cache.CacheUtils; +import org.opensearch.be.datafusion.nativelib.NativeBridge; +import org.opensearch.common.io.PathUtils; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchTestCase; + +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.opensearch.common.settings.ClusterSettings.BUILT_IN_CLUSTER_SETTINGS; + +public class DatafusionCacheManagerTests extends OpenSearchTestCase { + private DataFusionService service; + private CacheManager cacheManager; + + private void setup() { + NativeBridge.initTokioRuntimeManager(2); + + Set> clusterSettingsToAdd = new HashSet<>(BUILT_IN_CLUSTER_SETTINGS); + clusterSettingsToAdd.add(CacheSettings.METADATA_CACHE_ENABLED); + clusterSettingsToAdd.add(CacheSettings.METADATA_CACHE_SIZE_LIMIT); + clusterSettingsToAdd.add(CacheSettings.METADATA_CACHE_EVICTION_TYPE); + clusterSettingsToAdd.add(CacheSettings.STATISTICS_CACHE_ENABLED); + clusterSettingsToAdd.add(CacheSettings.STATISTICS_CACHE_SIZE_LIMIT); + clusterSettingsToAdd.add(CacheSettings.STATISTICS_CACHE_EVICTION_TYPE); + clusterSettingsToAdd.add(DataFusionPlugin.DATAFUSION_MEMORY_POOL_LIMIT); + clusterSettingsToAdd.add(DataFusionPlugin.DATAFUSION_SPILL_MEMORY_LIMIT); + + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, clusterSettingsToAdd); + Path spillDir = createTempDir("spill"); + + service = DataFusionService.builder() + .memoryPoolLimit(64 * 1024 * 1024) + .spillMemoryLimit(32 * 1024 * 1024) + .spillDirectory(spillDir.toString()) + .cpuThreads(2) + .clusterSettings(clusterSettings) + .build(); + service.start(); + cacheManager = service.getCacheManager(); + assertNotNull(cacheManager); + } + + private void cleanup() { + if (service != null) { + service.stop(); + } + } + + public void testAddFileToCache() { + setup(); + try { + String fileName = getResourceFile("hits1.parquet"); + cacheManager.addFilesToCacheManager(List.of(fileName)); + assertTrue(cacheManager.getEntryFromCacheType(CacheUtils.CacheType.METADATA, fileName)); + assertTrue(cacheManager.getMemoryConsumed(CacheUtils.CacheType.METADATA) > 0); + } finally { + cleanup(); + } + } + + public void testRemoveFileFromCache() { + setup(); + try { + String fileName = getResourceFile("hits1.parquet"); + cacheManager.addFilesToCacheManager(List.of(fileName)); + assertTrue(cacheManager.getEntryFromCacheType(CacheUtils.CacheType.METADATA, fileName)); + + cacheManager.removeFilesFromCacheManager(List.of(fileName)); + assertFalse(cacheManager.getEntryFromCacheType(CacheUtils.CacheType.METADATA, fileName)); + } finally { + cleanup(); + } + } + + public void testCacheClear() { + setup(); + try { + String fileName = getResourceFile("hits1.parquet"); + cacheManager.addFilesToCacheManager(List.of(fileName)); + assertTrue(cacheManager.getEntryFromCacheType(CacheUtils.CacheType.METADATA, fileName)); + + cacheManager.clearCacheForCacheType(CacheUtils.CacheType.METADATA); + assertFalse(cacheManager.getEntryFromCacheType(CacheUtils.CacheType.METADATA, fileName)); + } finally { + cleanup(); + } + } + + public void testAddMultipleFilesToCache() { + setup(); + try { + List fileNames = List.of(getResourceFile("hits1.parquet"), getResourceFile("hits2.parquet")); + cacheManager.addFilesToCacheManager(fileNames); + assertTrue(cacheManager.getEntryFromCacheType(CacheUtils.CacheType.METADATA, fileNames.getFirst())); + assertTrue(cacheManager.getEntryFromCacheType(CacheUtils.CacheType.METADATA, fileNames.getLast())); + } finally { + cleanup(); + } + } + + public void testGetNonExistentFile() { + setup(); + try { + assertFalse(cacheManager.getEntryFromCacheType(CacheUtils.CacheType.METADATA, "/path/nonexistent.parquet")); + } finally { + cleanup(); + } + } + + public void testCacheManagerTotalMemoryTracking() { + setup(); + try { + String fileName = getResourceFile("hits1.parquet"); + long initialMemory = cacheManager.getTotalMemoryConsumed(); + cacheManager.addFilesToCacheManager(List.of(fileName)); + long afterAddMemory = cacheManager.getTotalMemoryConsumed(); + assertTrue(afterAddMemory > initialMemory); + + cacheManager.removeFilesFromCacheManager(List.of(fileName)); + long afterRemoveMemory = cacheManager.getTotalMemoryConsumed(); + assertEquals(initialMemory, afterRemoveMemory); + } finally { + cleanup(); + } + } + + public void testAddFilesWithNullList() { + setup(); + try { + cacheManager.addFilesToCacheManager(null); + } finally { + cleanup(); + } + } + + public void testAddFilesWithEmptyList() { + setup(); + try { + cacheManager.addFilesToCacheManager(Collections.emptyList()); + } finally { + cleanup(); + } + } + + public void testRemoveFilesWithNullList() { + setup(); + try { + cacheManager.removeFilesFromCacheManager(null); + } finally { + cleanup(); + } + } + + public void testRemoveFilesWithEmptyList() { + setup(); + try { + cacheManager.removeFilesFromCacheManager(Collections.emptyList()); + } finally { + cleanup(); + } + } + + public void testExceptionHandlingWithInvalidFile() { + setup(); + try { + cacheManager.addFilesToCacheManager(List.of("/invalid/path/to/file.parquet")); + } finally { + cleanup(); + } + } + + public void testGetTotalMemoryConsumedReturnsZeroOnError() { + setup(); + try { + cacheManager.clearAllCache(); + long totalMemory = cacheManager.getTotalMemoryConsumed(); + assertTrue(totalMemory >= 0); + } finally { + cleanup(); + } + } + + public void testGetEntryFromCacheTypeReturnsFalseOnError() { + setup(); + try { + assertFalse(cacheManager.getEntryFromCacheType(CacheUtils.CacheType.METADATA, "/invalid/file.parquet")); + } finally { + cleanup(); + } + } + + private String getResourceFile(String fileName) { + try { + return PathUtils.get(getClass().getClassLoader().getResource(fileName).toURI()).toString(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Resource not found: " + fileName, e); + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/resources/hits1.parquet b/sandbox/plugins/analytics-backend-datafusion/src/test/resources/hits1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..647d8fb5235c2a3c1526f3aad45f52ab603e7582 GIT binary patch literal 1598981 zcmYg#Ra6{J+wA~@OK>0DWq`poxVyUqcS3-m!QI^n?hxEbaCi6M5F8RT@}B1VZ_Z7x zs;)=&eyn=Us7q*a5r7FUED4rqlnG9e2|&0Y&;u7IDhLGfvT*XWHsj)ikzk_maPtCB zZXN;P3FGG^!NlO=08?c}tgapJ3!31;hLOy`Fpavi?7Y~FH%*_cs27!2?wjeMMH;ECL zM-ZY4=HY^nf?>dQo1Jt=men2JRz%X9XRuGJ@ z6%NEh!VT?*1M~3z&($wL6ygr-pnV{yC-9mNasuWOfbxJqJir5H3xceJx%nVdAYRd5 z01*;6F!w7ikih@J;)VlpLrDR4uo)2W2F4F*LE#qQ7bFD)xKX&dcu7cL8-P>MUtky? z^aEfO!VQ>;L*eEYBvFR`_o|-t>I($;hM9N(E^Od{m~xV+K<@xDoRC0}fT;B=*+3K? zL1fP|$T+|kLI)V326J(WjsdQt;6a?w3^-U1JV+4w2?XMV*n(kPqOQ;bIDj6M0R-g+ z0w(}T0IX-igScUefC7LPh<+=G2WA9s5U1!J>^(d{P;lC$2S~&pcql3ug@>QZKorCWxdoB{ z5r54YCP20k9>^;X0EY|02tXG_0L+tuU=v^{9TmX?QtJ-@wKqD4l4EqNU;xW+!@tP3B2oazbNUzlw23=~2QV0XXL4gEu=l=t9!Tv|G5gD+}2P6k3S_nwl zpaBN~iLW4ms)Tp~_9Q^iHUJ3>0Kf+@hKWW1a`~^G_TZopMA&CIm=_X|e?SBQ&@rip zs=wlF0o*`O0d_DycqlV)0Ns8?1^;^X84R2q0KtTYAp(yOB%rn-xiDThSO^MK>otGD zuXMEFfyOgQ1K0ujFlZ#uw_JZ00YXr76cbjU?VzkkFkcj?F+h?xpB74u0OsK|(E!xF z0ZmR8ClnCB$?&V~YygHh2-b!Qm4XNKObP&*ZFImz9FQr#UQ}q!D>lAY&+6!4ZvKEj z0KiJ1ZJ~}Ruft{rO7gn#0a!z6z`&4!a6=1U>nao=0SkqP3ZMafzujdb4&q{|L;>J1 zgrJDhybdNyIFP_L3l!xw0)Ji~zzPQKM+F0!`UAyA1UB`rwt+5e!USN#BSHs|f&C-~ z;OqXi{xGosSR$`+YexpQMu0R-2nkw-0MyrZH`M=iMyR|-+u@Z_Fcz>fkdP%m4vKg$`u`*Z{o-5;WNkCOu#U1s4hIT2@K}v?mUFvfS^LJ1*Hu% zmmrHO2CNVPfI$MK0x-h};9v)EAWjq1*BJhV34&pVa3J1*4rmPu^d21;rI1*VKmZP) z-h~PPL_`HvQLj*6FCLfzJYN_xv>1MS3|frv7mD~vT+EzT!bs&R$Ip{2*DF3?#p?KJUoCYdX=5ZR(9??ADHP{W=e9H|iuR33*l0XGCWEQql8ubW#0d2CF{2<$!ep)njfg)}yc z1#k!!qgtVKGAvw?fK{)-YJTAzdM4Bq$sewS3ImIZJQvP+PiW^;6EzG{$20DKuOKBjcgL=>q^by}6yd-XES));~q-Du-pLjn{+ zLh43Cwj|zILMp;mr|qsKc)fa+E-!=q`?Ab0W_=FJC@V}M4HiSH!PO--S&e3+NdYuz z%AqjJ>@Ms6151Gs)*+(EKjX3#(Rc1dD`j{Cd0IP$IP*5!qh@K?s_&JEH!o zPCnxmu9V-j+&#F2PQv5DKH(HT{k-8I61OzV0^>8$QI`6)5M$z-hnq`qBUPMIf56@& ze1rO|eYRUh*)PI}ZV)#)@uAMLs1?%D%??a8cT9* z*l1JP3E@!njiCb*6)#Ql=RJV)@0S$}X@ z5lgK;=X3{)DBp?&y5H}yhn~Bp+I6otPlxE9gwBSouvXCy!>PlKmXoC5lu~CE**4n^ z)fzW8#C|P?FzLcG-9%EUb1ZjXvgkQ^f%Dyw^jMh-<1R{J2khWcIb1}~ON!1|q%Yf& zKKDC3Xf#KpL4MCLg~g&T-&<|}`?m=~YdrJI#DH&s-XF&ZVsSM6TsE?~;z?wro%%HmIV|Hm@D`6;#W6hSH z(@%sSO(YU;JJV6MvLb4Vj$$G?vg%;s?ClQg+ojX&gQqFS)8C&-f5;nA_!;4DJ!!Eo zA&PPU!=llJw|{^z1ED+ki|APys7t^=(_(NqGNXG!Cp2@>u;d* zwcx%{y!(rNv+__^jhc#NVqshmJBPM?-ujuwMjVrtc5}Ku92lKyH-eLZd+E@ILDy;B zJak5NHS=LusL@d^_U@g6WuuUj*(4!Rd6JN^871bxf@jO9`!_Cg?{1Ese&@F^sujEz zEEJ*obnP{TqRH-xojutbc*97AkjTP5V)*-x>Uht2#HL%%s};SSUkd?UUo?#qYOhok zcYjcCdD`k`fH#O$aE4*dE}feiiJ8HYZMvlr`>Pa{Jltthzxik)d+X}Qli_fs_{r?D z%W>qDg!MS#s(eYNKfbFdmu6J$cxbO5PI8U>x_xZ4^nGviY0zya*(Zgq4KKScLo)-x*vC#9;XTHiXIZmmM?(%s$pH<`flT6%aba;N* zDT^^fO`(}N`!X_kBwL9QPUHB&MhXFjhz=$j=gwW|_Qy<2Cn;bUUe71~@hQGY(S~9H z--XLXE@?ck#zxFu$$&l+loD6$X(Z1kWHhIj(S;a2FTjX-)eg@~@3LwP|5({T8l3*| zAEQZSazh%euezo2uu`IP`R|%$njyKl52nRINR0$N<&19GbOoh4AolBw(CaDw|?=zIBi@f8De({v2evn5%tWxVvc#v-ukUh$L!LewCl3CpA ze#GEoM-9f5qcGpu47dvz%O^b~j77>||B_R|xji1e({y7(SuN9H-=j0gy53X)VV746 z>)99g_*|ODB(55i&y1y6$4s0lE|aGhWcuMzHJ&gs#7MZMiUf}ll+&o&BEv-s$ zD~4DlpcT0L`MM*yVZp%V>KBwOnjMkZ49!_LE+M_f`A1!a(YDXTz&fP zA8)p;7KiWU^vsdpd>)j#kG1l^t6_-MT{L4P9 z5N$l3vK@Y&DABFu$7;b@-HS!HcS)k2dQ$<~wiVmw?2XffENDEo@;_~h`4DN*;8FTM zb-Jc{+vd+r--v|q-?Es*Kb*G=Be&j?@hn&uII}n2QJcP$n5Ir55xpCKVaH?!J-ta$ z@ZtUyBkyN#PLhWq_@V80%scLK8y-EAZaB>0HnGhck`by z%|~dIq9+cYVG4}PV2{+J!UB>}#e$+-%RWRHKKj-NwaFhQXoZ_v((6MMLe}tXq_A@`S5~x_ z@&f}##d@ADhB3j4q8@@Q?VNA&*+< zryG@mM;}>VrjHA5$!$T7Yk&6P{&~F-Z$aZm;F+#Ki1FEv>GMW zaO)|?3nwK_>~2}X3Gk}zO#aESJ^v>JqHLUT6?kJ4;G0C0(Z8|o{SFHJ=BPPltFp?(4_NkJIX96Pbg{%)Qc?*waOD3m{_iWXha zX!vR33({{td|x>?Icherkp19WOP#nR?Zkw&TR`k2T2g$l7p6Dks2Od^HLH1CBJ^0J z%nvyUU_Yak^%u|XVS@fTS{4$-d2XCn<(AL*3a$ogg69J|Dwoj*mz zZ{PIyylm^b2T^pTmaGga@->Twr1ufAHTk{EpqEkJJ9wnc%wOA7eUBm*)O6JMnXZ7% z(~DxM`5U9lXHG`sE*9%xQkX%9dwx>Y4a&=58mRL*YdbWvP^drw70r{ef2u>coo4@% z@PYr(K{-y~`_ohiqm*yHUj!dbo~hJAQ=cgINCdX{KuXy%2xd^?3q zakYbpzEtsTrM0Zn?7Ejcwm->Wgr&oj5~bsaH;)wU3G=tAOiHc2Ih3A#*&dSv#1de=Xp}BG-GRN(ZX$;X*>6v1)2nx_wLYN`7>1+8*pq zNzQF>)JgT;!8dzw% z7U4}+wP)k0vVeoye~g(iu#?TIu`bZM;)_$>=#qn2R=zFxdzn$#5%Ha}Xu;WnR+%C% zL0Jt1l$xL3ABb!NDK{psOEbB&pb{%jR%~F3zm@4F4}MTD--OVXqN6Izd=QkIT*L{H zBD;|lX=cz|4jZMDoWh;dc0<&(3`6W#IxOK7i6S*$U-H!UN0|9?TOV@IxzXF2+E1gY zsk^l4F@%n&XO4yOwukZucE_J}jN7-d3!1Sx=5pc(evR;^v-k{t%G!v{=*t7C|!iG&9Lz$qwJGYpTlgAkO9u1BCMAUDmt!q7E z7IH~;hh9dc_)Im&0<;`x1Ubs;hT&QcCtcK>AKW%wnjG=qEERrQm<-m`4-uUem8mV@ zU{ch>+YB+9!h0TKeo8QEW2Gl8wAy-{5)A#>-*KL)aTio1*?2R6W0y`Ex8X_pUgEu_qR2}^@^wyE*z#nyzTO*N<;_J9O5x!}N0m;Y;qaEo zR&V`Xd|qcM)%M|n&Bc3{dXY_Y?K8j6yPR634O}HRmlmsUl4kMz5mr=)>6Du>-~7`5 z79<<(=+gwl9b6O35Ogct*Kc-+M z^+qPz-58=b8qT=6vosGsa#xgwYe+P~23O1TQu3F?jr%pcg`2 ziV8eDyLzX_v;OR4+7^dB9mGh+WLiusM(A<)pmnh20&70|;QB;R_K1Jb^+|tME-hXb z9paE@E|R6v-AT4}D{_MAGZ|l3JrM3G;2X?=OoS)&&(@81$z-- zU1?UQcDJP8E3=dj;O>S!i0pL!if_oY9i?NLUnrff`VUrn#+eMVVu)+wU#ja82-#pU z|D$cMx_2VmxV|QV4!2vRCD16Q-h4%wP%|kun(4&z?<)T&JtcHQ79E+D`JpEuVMm(B zRr=PN>I=JCQFv!YGW%Dp4lGOD@AiXztCF4HK}HJCXA1$Yu?!)-l|CM8bBAY+6&1ne zOmFe`XBck(P1s>cJPp;q4W`TKb&CCrc$JgH%IPV6Vw2pFB1vg$o^Job?0g&BGCI{Y z>%SLtEu2xg*T_qEnq4$`vh&Z!7F|CpZXgTiw9+-KMLpP=7u|Q6t|IvluCN?M^U~+5 z@YZX0taaP^KwC>nRBD}29(4`_KA zm(52KDd1Pab(s0-&jdXz)$1=k&-ZJV?dJrANn|q?ZO~BF5SGVFrAZ}&N;GJTYV-MN z4I>GxFlpkF4h`9ci}=n3w|8ofsghXK<*PbVIHkU>|(& zgKlj_kYs8T;1CtsxO3|xtFtMB2*E5fDG6b#kT_kx*fEE8|KPjmt*Y`s(XA#9JC3n$ zYt#;U$@?$TW3(YZauSe_tFxcq8o|>kw@Lfyor7sw&Lv3vUDGc0yta^! ze1<5DH-=!LC?uJDFZNld@oB`5!knHCrj7+8*n=&2WtOswDLk(b_DI)Wx+HS@)2z3l zV)_?sw%Kf1_>dw!LF8-k11RwxIn#Msv|9ot%1`1H<~;fa$wmlx@|Y@_KL_6HpK`d1 zXQEzH-Dr8!<_RxuSI?zrl?I=bksch~zAAP3TSw1i!(HqmN~n)?@PrJLiNXH2uIOLu z;!<2m3K_xNq$X`vheB&jHSL9Q0(v1(%14hgh+ur`7j|`p; zlP;PGM3J&Wls8{n0Ahjkv&h?b={4|PT5M7DHSZ7m8^&U-t39O{-~mO1BypPt^9 zOXf^!1ZD!AiN>*z6AQ~HV^I*<`cEE->6f6RTW4lE_v3||LZm-4<#+%7evYI4(lr5D zIQV0%Melx@GJC)gCh)sD2zJG$y6NT$k1|GDVz&bptlRn{D&rC5oux0+;abm)&Fk1K zklSI((V#d$`esXxxpD4tErOUk{3nr`p_;bZ7a6}!wzRu(g#hF#LpjbW2%4K872lHW ztlx$K{dDYv(vr1{x%w_hJ?{5S^$U5jC;7Y^lv$gUnqWJXosu9$RrI>cOVOM6;@#~o zrrTMLpWtrrgKE=Pqy9+!t2R*sqtM+!`q*H5A$-iu*r)Nj*v>r~=(lUlO-2KifZX|a z&w@?)zsH8VL$^k+Wp@Sl$U>-9Nki#a4OZri73*pQHQL+XC3VD(oXVm%fgb1s)(gUN zOS5|2>-C=tIDgedTX;r7lCK6e3pB~UKPyB1pKbgQMe)T`dBZ3Ed@7yE@~IAQ*cZ{m zZJjU1VH!={So=@1!}XWae&2^gWM$p2O;=m;Jd6j@TiIhRQvLx(?CVc_v$SVC{Wdg$ z`g3R;jPUpf;WbloUne}Oy02pMvEmnXuKJ_V{Jsqu{7u*Wh;|vN0`f!g{QPW^_+c=~ zd%9ZgX@@RNc`5Vza+?$Pyorq^zCBAKwDgk4Wl>3e>nhoztxV-1P#Q`7>!#dC4xYK= z*%0POLNfeo=`X>XSf72p+mn&h9V&Ds>bWLO2`n)mg+F`~_tx)QnbPNIn-+IMU3TrP zIM-kqJojNVo5D(T+ka;4xncy%(+!SVr2m}!{s_l@~2RZ1&nq$`H38Tl=n>VgZrTkWR> zKebEVL$nXXZZ=-O^lEj?cNjl!bX@v~O4l{1;Em)ZZ)hx1%Lx@G3Z%FUc*L8v%5Fp3W_Mh^ zFWNe@HQ&lr&In&5J>p*W%Bd6_r;36vRNv8}i7&y;BrIaXW3G}Z%hyGK-lolV{k7lm z(bF;7^)?S0u13#K>5ZCg1b+<(Y7(Z{Ue$G?fa!-sVu+RCh0q$*SWs&xY_k!~;0nqi z$n3DRsm3NWB7P9T<+`3!w840*!H#p}w@FAPt_DhT!bs$$!!>|kWINrrv7GW4W55#% zSqKqu^#1+DN?u|OT`g7i0aIT-Kk4nE+_8| zVJ0UDbdlf>-r4S?BM~%?o9*q&&Y8yV_cK_esI;j6^NmO}FUAPeC0O7iLVG0l5lsFh z>kJWIG#z#Iito{BZBQ0UQgs%nPe^K(9P>pr%qd-X?NfHhK3qiiehFq6}rqtI~>27C%x|56?MiO!u<94fw)jI-j)Ud z!!)O+81iQ7TUuu_MJAs>>e%R*-5rC}#)bN36dvP{3d_$u4K~_?Pr}&$iY{jUZTE}$ zxlt|XMD%~^&sN8Sz~YOjR5xwjII{GTH0;Ahtk3+V;4jQe8@YEGDrzbiO)o+1lE~S9?yp^hz{OngcCkqgr86AkpgOluBFT z$pNInslvpseD^cuDttsssQu%eMP82~=_kC|m=(p}JVCM+*0IIW%v3f*F4EXN7T=Rj zTo$$;o!Zg0L&nv$R3t?xC!lT+mfKe$Ro4ejfD0zHdTw&pS zy!g}%wW^tXr2Io|#uO6^?(8S)eAdRR#!5BZT0OY1$x-ICO#-IqKls)AI`A)!gmh_w z54aO!wek^yl|hRY1WyDRCAgPyD=c`?6Rf)y%ZTa=BQ%icjKIE{6SCB{o0X|4&n$;! zSir>A&1Gl3aPqi}(KvtqpO5a?kGHWE#pE7VYm=LmS+E4(3vPAB2 zA1~Z$Ny|YM1ce(OvIvMVuFkzJY%;Z9X z_)vz};Gy@G;jK6ub&lzfx~!y}9%%HJUy6NKDvQ=Rrl<;DnA@N z=xHQ}=NQj2&W$98nY6FzPs3hSsn6E9W`FY^C|BNgsXaEMNH{+v<1jEwd)bFK!ug7w z-bWp^#IjL$_uv(C&{oMYG^7xFqv@dakmuo=zn)YxI^EWk!STQ;7(Nz`RXZKqN@?Dz>#Ze3R zIh{TWO8IKu`3P1x}l+L9ekf|&bXp0mV0=08w&5%1BG)LK; zRnKG17#$$Pao1YFBIo`(>YR3)cLQ9@O&D-$`Xtvh`;{=XL%56D&=)eDSi6M#rw3Fa z*hGHTy7&*2m)@&;a6_3O+Kz~{&psZ8+K2Y4Uz9sp-rn~!WrPZrWuDtv)zaTjEv+D+&R(ZG+^;rQ88EnaU~qin+wdEd4471YZjP1@C-s)M;b{(R`rLQBK@ zeS%8Dv0177j-?jf*gHN9&QuNRee{Ky0qYMPysL+a~X?;bDKN@_E;PrqZc4i2be5UBC+@y7s0MH}_Zx)|Be*nwM3r7)3DB zhaO&@TrMn98s4f;7-q5kY^B2K5lb_)k~l$I(^cd7NA-xd)SUS`5PmA`IsQSXLo71M-!ZUMeuhp2&_slBh^M+4-788b}+CWaYxO4M)L> zuh-e22p5%Bc^HCHI-66e+W%0sg&@ih@}XRXsT5b>aZ<)7MZeiRhxPzFw6|Mu(^3s# zjz^q0nbqDPL&HZ8f%hr#+4{eBw!YM4Sd z1v3E-H8VGiYUeV;=N-x9FO+I5xu!d=@9P@@i(~_2>Clx@D2ZiCw9pHWCIv-vl2D;! zV;vT$%2eaz2-6uY{0LDOmTYI@N*UvY)Hait%g6ID^L3`5HNT<icn8b-e-fA1A>4?pvF27xpfn@C8Y zk#997fxB&t04ZZoO7Kp=Wk2r=O3(RQOP)CGOVc!rhaZ8HH}lmBnbHmfW(9?66 z-+LKfxj1+)LoJM_i_WVGin98~MbiN{IPKP7+%R8Si@pBE>bK@MV%vKJui+)XW{L)G z_hc(YND4c)TSl~+`?r4%5@u0qZ3w?m@vorBJdQE~80*Cq&b#bs3IoMw8Nr($b*Eyc*Rvaj>fmd{p55YxIz@^{=F#)aSjH|EsKHr$vrLg7tkD zzh=!*lleMJUWA`$ks{3ooeG)3896fcb_PZ2 zvL$f1cuf*U@N$C!beyV(L;@i(C|VyIY9M@oObKEmx`$ZEesV#WL{1PexI?5?iA4GZ| zVs@Qfg7==2y=}RC!ed%C^_K=ibDa2iW_5xYPNO{Sb)ZuNd)A40zLBs|P?c1=ph#&` z!yD0q-f0Q2Q%ub^BEmmX>5c5|4-{u;Hc5ShtqG)s0`v>t`o7f{9Mo84TiyV(nQ)Si3naoK zYpSD~(uztr;)RN~2dkNUYr=Ico>yaPc5-9BG;d~AS<`X2!U#RJVF*98+9TsmVFmj3 zfgzkin*aRAG)|)%x9a>Q3#8xjUa}M3D;>Ej(ngEDx!_gpvHDTF$hu16(^NzMTw*u_S zns30;vwwYP<>-&R?8Nb_9~1Fvs*QP9B&x>(f6AzGZii=|3#)d`K&bw?zLa-Upm0V| z>^k}Hb%&?R*0>WuOB3xzw-$Cw^8#UShy-2J zPnaC^>AG_<$Sz%QJBT~S8JX+N znF&w4|$ujlJorlityr_j)% zz{-!wTkqpRR_VilVVY}%)4L23u6z~<*rOf%MzsA`MNag4lfm%F=JK9a!eNt*fVIcV zuJumY*jjAaD#eAzKB5*Y3)E((gPUb}oQP}IRv9q`x*}7j-yyhG4y!YtT`dGXZNxLx zdJ=+eAfb|9p6-K&(y|~+JQ5`oyv*8GT1slv&062HSMl$Xw2fVHNii~2s0q6#>-)qc z7d7vygW+mtOI8<8ABrTENcq*iIf-Ymi+DY2Z_SAXjch4k?3YU=%vnmb7Z1(4D!2;x z+4C?*zYuP)aR>;N4(9l9iiwQcAY?LsaQdU`6_~qgfxtKmDhq?(l&qlIDR-^*nvKD@ z=*cGU+F+RI=uzaRU1&zvmBylqksf3;LRi92%8`@0`*@yDp7?StQ7c zP?~IOK&D5_j=RjTONN^VPjssNDT`1Ij==StQsYk8aS)+#d(#^=UYA4D=V0oErDUb$ z9`jvKKkXwN0$zB+O`Y-&DRPZ3N)iry8;D9GMKe~_&^*)~+qOP+^bZsHlYG4WH5rc~`L zz{o`j=n8#jm)P{dMM>pNYI>yOp1#Y~#EOOIf8O{N?9XE(l*9{7%{GzL?f0yo1aD6i z{s|>X4=CMk4}$YEIZD_9kmb7~63N$^ly7;tqLA%co*2g*moXv_hv)jsEq4%?5z%;m ztJLc7*pnjt7crtF&$6cQYqvqxISx@c$)Ia2#j9-1c~xVJ;YXxZJPYpcRl9f;(H05S zF9O2UXQY+}4I_3APlAm)5zkyff8P@Lk(*3F=UF@RE!{Z^N_6c%TNE0C)SGdeEFKq_ z_-LbP`P1iR(Uc`Vb10EB=u%}2(nXzkDCVWaMVM5r55@^Z8MTZT)b@`mGKhaZE2H$@ zP{KoVOe(LAZ!q~KV~;U;DKhI&X&5pEYvS|?Y{0hTAw603{N+91#G4;Z^tfLq?gleO znG9Mv*eW=JPLsRJvC@8fg8aRbtVqeeL(G}yD?}oF+dEN@$!w~8U}AfH^VNF<_iP=%vnO?>-tp~_T0jZf9wY{NDM zVq5rEIlnK%UNN?!TVlcrLPUEp78|5kw#O-X!~E2m|2h% z{#~3GK~Ff7iQIgcD!eq~eln;zhQ}xXA2x=3bjXwq7C^@&mOg zPOzJ<8K)DGR?2mo<8Z7}5};K@*-;g2q?!J0g5LTYoA*uJ3>^CmgT{EpX*5Z>nu9+3 zBQyl3f}#KFL5kg3`${%HEN8?SyGot)PB~_o7QKJF)8%PMYi;Xh)|hjYWmd{&|i`MB>B0#QM^-!>maemAugJ-VqYeKebj;ajIi%sAb|pGIPK z!!Sz4RQZHzCbW)Y_ih`@clwrsKOiVLGxFcD0RB)1Rjqj3!&TTtn9N^L4?i2}Ovz1| zv%<8FY{AY+Hy_lburhZT-F1#C@E3(^%41+~TkY7ygY=m;9``-@+{d==yT&hvD#Tiw zcb$%KXa$(NFO=%{Ho!8HhPP+pJ#npw@9PfbiOZpe{jc4iKCc^Xvnz*JB1UTq;q?pm(-D+7`i&o-a@7XY; zqt_*}SILrzK1nb)tA-?7p7_FIt~+RW6DO2e(c|^+m=PHXEf0-%xZlhECJNJ2CXkGt zq4je@SNf6h>1){3OK=~EtkL5>O3`$pIIXXyDjsy+K6sI7G)Sgnr?MNZb;~^VxCtT-8hWh&@d*;IErp zK5_M}p-PkX|0#s3loObt1Z;~GhrRtb5udB;HOOU0TZPckcqw&|{LwA`?^jo>V`?vX zjf&VmLd`OQ<7ol2QO~!+RO%@kc>4H>(7>FfVLCki#|W)MEuE-B5R>yB#`I@~GWE`} z6jQHi3e+LAa*Q3p$L;8slqa#C2J$y8CYF<8Zq`gc(W2NUpCJ%t?=b>A$B)J&<6Zgxqkw8ZT`SPfNpaXaujKd3RIzxCI81TdP-Y zEYV1prTE^%oG1cnKd+mxcVeBtD~9pnM~xmB$`vZVStRJio6h*%c3MyW&%S-ISHAQa zpJZoiOi%ZU8nw&h3q12j{`5rA<>M%ffeu?RUbr$x$#%eFMpO7NO-^R-v(A>;X7`em3mD2v-rj#Yj{ec~LY)@vjM)jN6f?Jf zPDtPs!4A&Z3|Q&fE>8I#J^C&;=YlP&d;V3=lzl=lcqT0-wd^w)3PtE3meQm%b!DN! z(Y0ZySInIEEDaO)&bK%nmi0kuPVr%&Nk;tits~}Qs8ahFrZtE5ZSgDBH1X1PleZ2k zCpcT?=X>>!%O8+;Uc?JN+G%eO`H`V!acVN776$j%-hfDi$d^k+@54+Y(AUU@WNokW zD+Q~0tHN|9-IXVMb%Jj#O65lCG8>VzY7c{3OSY`umc}&4I1?qUgz;zBq~T9|8|xo- zQ=^+1qZ0lvWkSoOlakflqx6t}8QEhJRF_0Pztbh}9QJatI5xPn;GJM;w|7dy;v-~ zH-C)jUbt7NKwidqUaghIhZz?UoHiEBh*?x(Fl&;%|#82wd= z=SQKHT9C3Tl<%+EFUn6lpy%-9Gj&H|?X2UiO_|%sPT0;U?iD_3F_+`(&f}9fqw;;)fS6(e#$a*IZCkET&yMUOCfTg5-O z%|ks`OP{1|vM7x4mF1kmEd~7rh)ecKTh*IOyD9TO6Rlba+`Kou%#yn_^Y6mT3ZSOm4H%VeuY;%Y(cjVWF(7eNONa)cQnN7?mQCVG=MPWhDSQhg6hybciu^tA| ziOutOAOCmu(uw5L8j-Fb&AFepe(3#CtA2k@DNW($07k!D(RD3Vi&+85F^wOU`$}xM z++UmCNBp(ax1)6ZNy!U!@8N0w=G$>$eJby)ffX>`Gfw$?ujI?EG)Nslzs*a~QHY4$ zt=(xTRHfEw!6KiJi!kR)K)r7+Mt)9H`?fuFkyMz;6X2tQbi+gsqRfXjjeS{Ud$t}| z#m;u`l0eqvQ5ZV6G4;I>tS{J;e@WHu5#iGh2=Ms5wn1HQ{8cA5+1+!k;kbe+sx3p` z9kwRYti=D7?AB<;tZl)NM`7wp;$jPIG8YLGbYfSb)-~)WHM?5n>?^7X6KP$^KY1Up z)adr!bJC#Vm|B@m*hXL}xkPk7yV3WjM&05!17=+>ZCKw^Wkw^UVs}B1{*RY_+WFsw zA$}ty^uUi!d4@8zpOZg@x}+3-m*W4xOLBSkR)fUo^dj$UnFhU3D|z@`y&Go6)-L#I zBV``36r;(y=hHmylI#!!>Ml0>Qpxtks0NSxQTZ8!8+X24-cBE?UWE#=#bD&zl6pt* zEZm?wdFPXuqAh$lGb}eYi{|r7p~&1*u{w85w{F*jfoD$Lt=ge5HNBzi>Bugz*|C<# z%fBunv;!I1Ewy^ogOev!_b=?i8EGQ~W~iRu?C?b~kFXz)k91HZ`^^RGx!*QTrO1GM zjnC(OvhF|X1e_jk`HS!Pa*6wF2!!$8%^T_)6SNLp)-gQ2m!Ul5lRnq9`wojWz4X6> zg7=URBHNJfj>3z>3uM5VsLYB!6c8n9TCzC^HU8EnL8sQUmZw-MH3IT`_7EzA<;)oM zRg>_DDBwq@xKYPL;E!7*wi(_bufFM+T(`{6TN9_wzGMQy>fUWzU*5wfo~C|L~6F~MqaNPjp2g-8;usG2pUsd;uf z`H%yL-L{tmg=_Gb)$r&L^{LK-RN#EAqG_XA4SU3&HXCHVma7QX1YArZlpTZ`eO=FL z;%%e<2O~h-zso<${3x&AvB`=E3kDY&I+;LFpd?#|uHC7LH6|6e2P}7`O)v8qY!t}f zjc7zk6MCOC%u+?pJ@7}3u81cb5sWg&2oS8d#RfTutv(FR^jz$w0; zb1U6hH4iBxjM!#WU-JB~TF|In4|_+NS8ep!80b9om)Rm)l#fHPUQ*eQZVEo&y(%|y z(yrrtFiUDsdz7AJtEw+hw&q+nL5zToyHQ)pbI)n_xg#|u{C*wG;B*`J=T{7Z3g(5b zX3={}Z7P_yyyOKBzeP()uThJ;sl?JFmaJoOQ z--B>db=J}@&;By(dOemAK~02f{d0W1<<-TlMwzB@+ej90fk?GgwiS9lp+O}#>i&t-S>qVE zNr2WWbK#m?7zdP}AmRKJE|sZ2Cv4`o+7erD?;NlA5(8t6<*!!ESHOVfx6d~8`cP+l z2{#b1!GVSu+Nq^M_LO6aeXklbK81()puCkTFECApyL54;x4y9(T^E5#Dp`8f9w(OlP6ozw{0c~%P_kh%aA zY&f+o6H|j~phYOtKsqW%Zn|IZ#X|(_B)2R8VP)CvhkSEIm$p!&qq+whr z@M8`}~lf>EAwL3LBFy((dTgtYQKr9{iB0K^J3$>$}@Ou*j~2^1!z1 zF~y0v_ZA$-hus3>YFk32QdO|WI3@n59tRUsp%R*3>@cHCv8zaL=^4Nm)KQ5bovRG6 zx)ytrAhFKg(7yFK0>%eCk)iu6@t*-_QXeuDtZt;(C|pCutMx3Q7DSke>GcIv&f^he zKQlAuXNP=8UpW3}FU*8k*5Jk_z-U~X=tthJup>9G)6*GI+yVKrBm>_pDKmNtaBCZ>}UbR{20ojAqk`6zk4rzWz{g?n8reBiRjy5LZP{ zc04L6>C~TK5=xxoQFQM+CpVTk)PI#MlRx+ zp&N*G9$OJkg9d^ekZ`Ne2=f_+(~OfZxV^pXxrtyfAGE1F6pS!lRV-FkzXC?BdTgv^nN9pCJ_e zi3?RF{9cqm?2dI03{bHSM29CX+{dqDzED|6v%27Y?+|sA$uXzdAb^yifjf1io=@Xp z?vV`K=TPSEsvMuIK0dL4bB?_DF{&|YcxJrLgII06pm!EdtKLTEu0PqX6PXSA^~z8+ z{ZuG7`vl$>Bb{4o_Qm;sT>bsV%^}XeD&5u?fESB{6;PTf&XQThuw{XTIWnbfW6^G|}A|?1qHASA0d`#I}wP<=~Z7yy)ZK z-^m)#)oXd++>Xu6t*w)0ku?s2#0$)$A*Up*ixr?c)cHr(AIIY~s@YN8Xwt&Uyw@=E zby;YOM^c-NzXsREUO6+al=Tv69qcsuA(hWZcs1~Zq+1xIT`_GHD`y|hMf@5ULuxy3 zn86~UiCIJ6wmlIbTwti2yZ(^#_YzDaZ$Oi=G8CJks$@A{uQsV%=Ny2|>_j~AART3i zwGD=wz>Hf59}V4x4BizWt!yp~KkM4j*W(?5oJr`v{ zzFy@TrP!yxrOubhSGDn+xAC)HQpl_leisR?$BlV~0%}__v*7J99>v8x+8ejoE%fJh zRgNj@BJjs~CE6w9_hRAo9wyOfjoCXFf#0{I4td!>nv@e{y%HWBr^Kz-1=v)I zmanjsXaG6bn&J*i5+f~yVJa+#Lp(GXVEDmc38*-tirRuNzJO?G`*EClkakZ%3siv1 z`+B}Fd>(UtN?^WLSp+Bg$?h4wAHNCRb-S3%@Wm z9aDv3bQ<5OWF7;LE2f*h|C&fD#7?n?CZ*M*B|#+l?)BuHY?~FYL7_`F;(=tiWR&o- zcT_c3!~sctpn>OLNJ=o?veT)9w90buYB*tp%y$vo`+dNJA$Jj7+ga$dX2l|j#U)KN zw*Vbu$iqslG`oeLJ)uT}bH)cB6R3ieWVVVpNMOBq(MmFP0^!{p|)|4K(8kXI5O;BWwiQpjLeB&3Pym`8A4!9>K!DC{GHm*ofhs&7d&sj++_|z&xXjC$wjJ zd#O>;kmbT%@9CL|0AwPO4F59hNnNjLdbZ_EvdR*8&zL?WKaft(pKvu0#|VkMrIGTf z1T<)CQ^AnUD;{5ict@cdNj9pM03~SeOlR943y|UYVjQYnq!H`a@Wql+zvnb{ss1+H zi^P2%CmnJz-}X;DR6E?#HK~KaGp#rS?_2lFdV%N8c5K;*oDUoE#Jdz)n+kr<-TW$y z2t)~$8H}mPy6d}f3@Q&vp~?Prk%>@wRX^NS8bM5Y;7i=P zab5g+VG~=DcDx*OFVNbF&sfW}iU2*hCms>|kqw4!oXNU#;ug5U*~;32HR`M{chLhM z3KoCIG0d-jeWxx9;068~Ck?9iV%A!Ur^%u9jF2^*e^F|D8ncq$uhfk<(+z39?&`e;Milrsn5F_b9k=8s9rfAQL< z@9M;gwLhi%6#0;?gph>rS&8 z`qVp{ZbE;miQRpGrn_MkXyouCW#zvsDj%v5uHnaVk=rJe$$UMTZTTD@MYF&Hm0jr~ z4^@#b=+1PHU#5*-i9~_N)IJwkn-^zjpg`2u?7?>8TTQ94ccpfi6%O6ddQP!fF9#wA ztnakbORZ#3q!_kiyK)QngBm|Ek*!dCQVY_>RyHn2O~jA86MESu^Ji!<+%>wt`eh8Z z_25tz)We03Z!6ai&dYdd2Ok?EYb8dcLugM4Bc#A zM4zsg-MUpmsOMokIq@|;iAe@LdI>3UuoIL}@S7g3UDk+@c+Cqn_ z9FUf9PkHSO(7C^$BqyzG(IOhkf;KM}QVsi1IjqZIvd-dHFYI8lvw;(|EC;)zK|gU) z-@)L$a%)9MX}Npj_`FjmRRG5Rx?x z+G92z+XO#BqVIDOsZ|v@A*{L|e`D>yr9bdS{gkW&jJ%TJC+)S?Kdo z=g~GYSpb|lRgB4hm+f{F7Q~(g;lP2Wq9CWq;m0m>L8V?E1wffyx@epC$NCoYxafPJImsrXvcNLA@GVjm|@hfsrF-kGyUA+`OJ^kxZvA=Y^<=SwN zUJB{P3@E8<)|QPt9h98#ROo{IfLM=4g`|;2kaXjR^wH?Cv7tQlP(W#8@Fb^5M^QKcA!FT_;oIa=n#J zo^4{yKnbLT=+-b&yQ|K9qY&eqDznd#SEjc6yU2*?#G^w>Jxf=2Rp(t@bOa07Cd?aR zs|wYdU8IUI?#)hnNGc@p(7tRnge3(q!z3!#wSZbfbr`i`k!?h`2Cm#PB1BN(%@pS8 zkwt#72!GHG`K?zmyCoO2sWG=xG)LPppBrYl^M06-j)^&Kzx%pC?4%J5i>Z~bG64ne zZzkvcRk7L43uC3B4m_L%{nj=#k0+p(1Z+L)jhM129fN-B2`DS+Zs14(91uT$x>1_v zVkD^eP@i(m3?h-#;mvubN{zAyJq1Xeg!*9%^OWbb&qNl}SboOWrT+C}VWZt0bEZ=48){XcW@(rH|6UnE(nPW`ly(7zp6$w-TB}3e{ z)>O8_$V<^h3IxGMzebVvuN;{r0GJMCO+=gegV)nIaLTx zSMI_4fu&wuX_y?Xw2s2>7#`}=y0%z4`q&gd-K4@&2+O#<3MNZnzK?a&;6_Twlc{q? zzU}xpY*AvZwBH^}>)q!oT2rI(C`xtmaR3#c*vUY!X1-#>j~DUtIA^fU`3{>Cs-4up$aojV-$U<1CCDlYImcNUPfBj-kdt`kEG) z_JrQfawl{=VCNJJ0(A0Gci};pZtHF2VV|?Npfr5RM3!D%BwaP=TZ3%`s-^>&du;9H zM4qxU_FA7tS8aANOt^E4okSX+d;1T#@!`t!+Oo#L$8`Q3##Mob^L&*!RlNG5Ev7ts z1xLlvc~b1l5!rqL?L`Xj2@5CCG+uEqvVe2@9lVNTC5%g+!6MKQZUCD4pgd$hV5`veBlJi@Wop}bey(y&fn?bCB zeQOieNgO(*IuJpYILIBQtc?KSHb`2zR46V@5YdG1S^QaM--Qv5dGN-qa1tS#L^$KP z;#$mO+Pk}i03b0}Z)Yzr783b}R*p^#Seq+|2qh06j_s!&(RS_5w&8L>=Evw4USq<> zDiFC(XB#E^aHGBs_Tfb=MJo{QX-O-dE8eCDBAi#2^MO2C9<`hEpiI1=T>$X!Gg zsa&a^5()UQH~no~6;wzcCTdfwZrxmhG{LH5P_Ov==7`6^ow(Qti$64t8gi|f$>pNw z-V%~-StQl**KLa5#k=rlcncM#uS0;{EBM|zsd3q1#J8rVRwhyEh&^hakfC!Jik)AF z9<*ee=OJz?`rG36BB`C%u4ss>DW-u?e47HtZp!WIF3jJGY_hUTU)WgNj2=zU6ySO@ zGuvv)&+gVsdCQ^sFmE#%*H*w5h-bh3!gWZku2Or!WW=&t6Q$69Da*`7NlEa!4l822 z;yNzkZ!aIVtShw+I%kR|3e1nh0u;g1K&f`sNRL25S&K4&@>bR6ykh@6K3?{;F6eUG zDW-#rd8JmMlSV5Wk9Elk%X#V5^cgT~rrVqAt&=NTtgNt6VLR@`L8@g9ma)I3yqik7 z9lj)7SvbnX1Eg~ldz1<8g<%F)1ZIvQ3CQOl$Zl{z1!KaIw7KW?Vw^p32D1^~hq;&F z9T8TQf=G)3D#gNfa5WJTh@32FVW+9e1&iGT^)knwmdNYU7ml6iUdp#8d3xL5Yi#={ zGR9-RNL{>##XR4UrM}U6O~jY60lbCT)9=E(i+z6P*!B?GT+kwiLu1hhN7k*EbxS<=dA#nP&|Il3_&+mqzK)P`(wU8y;A}49O+8 zeh25gxwF2jk6M3-(&)_L!yjZ&SS=|pvs1Uo^@phurIsvW4svDIda_tPJ2{I}@zszg z^A%guCJDf@d>8_}(VEhatzLdRPT&`5k%rA0<)f~m3Ty>`4OP925?#z;rEtudfc!3& zaaNP=I!{l0Y?Q|D%z|hy8uT&fgg7T{fqBGBK4@MZhg%-ifRwhR$EW}YeIuXx}vJs-A201t6n*<*z zdX9s{pXxl1SDp*i3X^)~ii=>X!z42Yl;Z*2aZ4%FrKsC6CF!?g!Mu(~x4Q`t2Vb)q z`RQ&4Li1(j0v3i#^7}RUICXAa%9Y@&F~$4kOTo{;xKL)x?mtgSdMKQWq^9yQXn1oP z{@rhl$C4mCUw6|%u1|cu_427|#m4Bz9IB0gnB~a*Gw`Vz;tx+n@@si$r z_b4MaZj-=%zF;S9JxqhFIA4N6jcg;6 zY1Fr^GH=FZ<(XX*Mi(8J4;{=PA;4nJsAsYX@cxpdQLCK3ss){Bh)JJmHpzi5a0hxY z(pMh)F{W97yD1k^Vk!X;r@qhe@bQgc0?}{rtyxX_>1E=8;un>YVEPjB z1}RYOt$x%lOq)m5zJBa1_q$WMpimJgrv#J6Xh|)9%WVXkY?QlP#Y}usFYt__tSG<~ z$R|7Sw_PdXL$?PdATCJbGPOR3)+QHNlE@mdee_Mk?)Csym7f%)tAhs)neObVPSp(U zZC|t(iZVbHJ6?J0bg+c5tih?ma&IR8@5-<0^~v9;w0%I%m6&~LDbaLr8fVw3J1XO) zaRBQ0`Q!&05c0WV2+Tr866flGD!Uonq@ZLoWDNLNF!(UFz}M$nI?`5F45!YMUkcyG z*v=vEUT{-YzAQ(tRf0Pg)4Cp+V@#(55nk_Cjtej~ysAeKR(o-CX%$b|2JTm3q`RlW zTtWLwf`@&D&Tbt77R)eintD0q8l*9}&PBKqDjO2p&p&ThWpgMV6IdQRSdv8@9tTPx zg*=zuF_XL3tGKMr1PsJp>YAooSz>grEhRuLZ%=4&14u70bp3uN$)dhn^~@m;e?KFH z@vO^?vz;ThP>(bhJe7*Hh!OyJZ zGe_#F*Nh5_MP5AJY&8L{;Y7zO13H6&C9rn>MhT?LAi1bIP6xlIU9%b)akX7foKxea zQQ=4Md)4~lXy2w3N{|gU3CLjvQ>a;Be5Ra#leIb`&QvF><(toK2WEW=#~zedrm6-g zbe2o>Bp|6(Ay`|RJcPX;@X_aGB~QRC$pN2Xi_~Pqv5%9)!!xLN~E=82@p;N_%sV3&fT6Xz?E zb<^BCvsF{ZuNefhJEFQkGvrgZcnx^N<7~EEkm3r1g~A(`LAN0JCGv@M29s_yR{;Fh z(m1%K=&xitqa8>l8xSNz*-wfJ1OnMJ46&X`vI*+TC>@opg7vLF)V0t2%z!_&w*{0u zXh(@zicIzQ?tqC?JBZ1pqh_AO#2Eqb&+5)m#M+C?Kh)M+9T!0?XQ0o(g zY?EPVR3RQnrEHHjiiERNavoXfK&%s8_)5Q9vlABlVq+E+xa0yGq$eES7+Ea>vOhQ_ zFWoc}WiNW)1WKJdY7_bo;#%UnGA{x{7)WoG)&Wsp{qk{(;hE5ZYmDj zi>-Fv8Ap!hh$@lvzD!HC#n@&h<{{pPtTADifliyqMKl=dsCva7pRVw zb8O6^5)_sV?|r7?7g*{k8A6L33WXAXU|P4$?HK_FOLl>jmv&2-49<|{@vuJmUn~uX zZ^I%|vPq9uUJE;W7%L!%vDe2c_B-PkH?|g911Z3zIxE8o%f}$z2Mk=Bi6v+|Pwcdq z3<1B>lZU|{G(gR>dhpO1;#yaMz&yy%Qyl6cl)`aDn0Wv!Wigd9>S3^rj<0#CR&d%} ztzk#HMjddo$8a>WX2+;#gml^@K!O}t;auQ?y^4`*Anbc4YlLUDAYuuNr<|uk3%XE8 zD^8VM!NWHrbg@<1oM+UmLTp4ttBZ1b@?DVwueftrIb=8 zB@B%%XWM~b+_lN#s1o0QSwieH z)JtXU^*whHtmHPMxQ@lwo3GG3D9CCu^)<)57xwUp26vWsJ`=G;a?3d(>(mo1j8RK9 zMX@?WMDE>C1X35z7!JtTxwf?AOu%-w>yC}1796h2hM5z)i6E}+WM>omSK#zAX^}IQ3{G0Ehd?W z*DNi#4cag7<4rI={(++Vt9)je?BtdlJ4juG#>>!cI~z>)FjZ;qnqXJ0+F8EzSvmtZ zI$Jtu?kLw^vyCGT@hU9bgAw_Ep#N&f_NmpP^uAL&fJjr#b3pQrCggN>k5E^Fo|r1< zs97u-$19vHHE8=E?wE|1N8Uu3c5ZUPcTqk?DVMTK_ad=#ST?3e{0Iz57gkZAQrl*b zB(cY6YaDiu)FC5!06my19T6i>m%3(@NJI*Y#Flggf-+|H)qB}xb%acm%H3JhanyzcmlDmR-<|)73#UB_*ecy zE%UL;QH8e1H{BHL2v|}Wo!qSSh9C}F*f(P_HiDN_cVe!ktSddGHoTHsq9ew(Lsle` z2M_TT49f1|1%P_>nh8B?wv>IrG#Ou(a;r>|Ms&XpCE|L{Qo$^ca<9nB9m!VN>LNCE zO9IXA)?Ad746VbeFqNZ~r!61F0Q|^mu_?NGFh#H7nYVd*12=hXaIMWzA7v+kUzY*!=;+57^OveEpZX@<_H!K5ehR0X z3Rc^IBMqi&?5$*TJze8>t7Z+H??n!{yJuVJhgEuim0pc;6JSsw2y;3i@RXNyC`bJ|)3Z_2hlb5usy8uky$5Ut z=KXBgYt4@_Vx3C0hib8F7R?&T(psibOA6|EtZYZxdUxlVCrOe?Z6o?@wOq|ATfG>| z^5B&yufap|?(zT`R*H4nKx1S``**%5{adx~oYs(hT#6tzb< zFJ!S?Qi@BNbRCv62vW};Tu^O+DW597loXuT$OtoP(S>PZ#ODCEP9D}CEqp{^I1^O0T&GLGs57 zgwJZh{@Fm$Lu@WcFl$aZ(s(r$EEL)xN#vNLKI62D{M54|4(&SNtQ#NP2L|EnndNw6 zDm#2PvYO?Uid@|EyUGOd4|ETan+SSg&)iHXul-mPQf4U!S$EAo%pR>kPit-xCM&{c zM^WYm)Tm<*J7}(dEI8RosH7+He;iHY6wYyZrZ&GBC}SVDsq?aJ`>f@&#oRu9u6&z{ zU<7BUo=6wd;Cq}i5wbN7+yi@{w{i7PbU*ScwDl7>E3fWeuq!aX*j^uRj2`p5(Afa& zjm&k@0{6TspF!)6c~MUovH@@`434grXRvuPy12s%4!j_%%Av*kyG8X@nLt2C$djE& z*@q`p#Sny`1uQR=aF>4``$$kA?!E#iB(mV8TVX;^TNE0mVnIc*-lYS z6~37)syD!s*vlBqg`C~8Gj4JhOanEo+jVyI^Yj?`kmZ~X?Yvlcx21K*4EPZsQsv$# ztYP&mdV=m8jh40i=5(m}H=HG`i`(l|)lTfv z*jhTx#tOAv$xmbODOyH0P2+Lbc^kI5-7Z`Se7k zY-rC3^A`?(3fh~92jKP>c_5|*8%_E7g5+oMk(PiL3yM~ol&@hkDbe%#SG)Iim?FtAUn${+eu20cskq+c#ceoNZP=mR*XOh!HA)SMe5u;b zugY4>Wv-bBgt2+Jp0{nj#MJUXpEs3yK;>p`97saq?YDAUVJkgPMp(vqvNUN?i1GkDkT{z~5lTa$3pYQZEK3H~ieTlq#e^r`w#v>OJQeL>FQl zrNk7VX_u`F)dnbKjq)UVbm8Y`Zx==ahURHq?S0qc6R{q9J|nAb_a=@mT=n>Pp81nc zJC!jpB=6R|uYTRJeY;QvlM4(TWOH8-8)J-~-s_MI9ai;ks zg|zg4G+3mOMpKm{#xt;D2-*~BJnzFjR7uPv!bfC`H*T^WQm{(re%beAOOno?71QP~ z;8m3c_zxLUv_e+bmi9bY#ayfM*rB4xw*gp+%!F2$lzni`4ql}1TMKj2S?FaK zIY|p2pi6OG(Z8)?=*5gt(NvCr`?4 zE@i=*5ryM+^*AJ_pSfhIw;-(RaJOQxoGtb4qVHr+8%Qy-Fy3N0JO=f6sk(_MbB;>S zvXyvHk7GdQH~>`&a>2RwCSHa-U(jA0vJ>4I5ODxWaO)mW<)+D2FI%M%H45X6N*(GS zIWHopd|SvsQ7f z4$wG)f%cavOrGEmf#oV{1Xz6faacGb7K|GbJvlwz1FHOC3Z46wMnrm)3NOJMb(*?a zQg$IxD!MBrpdC_BWsBXiXPf0dQvr3P)TnH+BZGmuX4iKNbbDnVV62ypwuw^pybos0 z6xe3FS&KN_8UlxcleMixvBEPhj-+4XNG82=Bgwe~vmS{o&FT5rv2iW5BL+`lH#A?# z!}T-|dRc~vsC;9+n6S~N-oxdRf`R2NTAsBsBOw>`0ahXMk#=dP#IL2P{-3q8@rm zNWs;72R(cDn7Do~Bh(M{`1beyr794gxK??CsC=<5rbbC8hXZ1n0w)if+8%l_3!%ACA8g~5{7Uu}iku>PAuM8g6m;a@r;(=2ZGA%pMQ_gAut zjDQ+@8|ud5Y*Sdj(SyKo*$hDMYKO4-8LJS+=3tjk4n!f`Tk5QepDlyyM?C9CRb`$i z)7Kn_d{upnNhBfyIhj?;Fg&Okwk;h9xM1zot{$wAc44qmVM^#8^3}0r#0@`}#miE> zK=*fTPzns%xT@VjyH0Q4DpBTMn%pgQ1Mlh^*}+pHVsebM#KLxfza5vlU%o;4@mF`~VV;QtE6n_WN((b2x59xh*V!aVO zS2D+lF(*%E9jqPYRSjZMiQiD?kaA|obTQ!Y8pjFy3UCNd>&a#uWD!`r__A7Ec}xfH z#WHs%eO3=)$uwtcd}*+Qv_>vt!X;xs#=AzU-MZD{=Gg>+i;%wZ&BJGW%664^xnM>G)r_Q}8eUjrNDb`;xcdRlPPJAx110*UtE!n- zFroe^BPf9<^Lu!ZAK@=Rz6uP#&>(70DSO0H;20rggO;OVAIike$Wu`*2&94v_UAX> zD{f$p&%y9qbPM|;$o7K44U#^^GaLEYtMCRkov8VtV4Y6aD3Y0XtR3xa&-So?qcsqD z5+Y%v6CXj3+DX?z&Yl%WIAiQ#SS$Z@^b z7Bc%I50p-%Tz9Hmv$F{4{`JxDPPlWI6>Iv5PIxgV2?DCLtvAn=Pz&_->(}zf^e}EJ z)^V;^*P5qU$9M1pevjGj_{z3L|B}VFJ-nQEX#2cfmOr@EBtObhk>(PAL+0!Zbevq- zL2$0TcuUk~A|DPV z+H z%qqk0Ig&4i7_1=8bR~i`vs9Wzv$$GO=I$8qYuXetFi0g1w=cXc+F~o&dfI|kn8>zT z(&ujv(tcwcZ2B_{AI9(7$77YPcUmKI(6$Nd!GkkioQUzZa>S(@KS006N~7wglcIJ} zI|gB+``F}+JD$u#e4+v>@fo(9C(s3;sr=Z6&4M>f zT|)gDgj4))z9^-5R_yYwRG>)b>ETb=Kc=Jh7T|;edeE;{Mk|(FulS+P4f*mgl$Rjy z;u()i(Ih8p4X4?{!S~rU07bbH6paO~iiB-^d)di+WbeS%e}6e#K=Xs#HAt2Jl#;1? zLRGGR7e&lo;7@N*a|?%#Ibu;Lye3OJRs;w&!W-!|RWOQU+IN zb>w44Hwmx#*ax9jpSB=IJ%^IZPrl}r)zk^QYR4ABCH4VoeCO@f>`%lnedW9ktvh*( zH=+xiY{n33U$y|F&{-EbAIUv(oA5S2{=}+|y(2QpcbsXrx{XO$k}?K0=>tOr5o0JNI&jnfgVSm;Q5@) z?X;k#q|@1?t=J#a+;(0}k7;eNLpxQS&L;w~c`8DXB$f$SV2-&`cqT4@1;lhe z&9ULMWyQDhVWQ$KHU#Y{y{T}2Co`C~hw#Smy&`inYWVPsqCOxgxyBbkxeQxfQ{2^~ z5=(HDigh17=bOM1n=^M5MRePgfglkstHzvQA_=`VQ9)@%q|Rf5a_PKOfvr+H6Ye*) zfkg@{H}vB^)29*~`>ZAQ3~gYi*hTwhle`g6p^j>PvTQUcGI&N7C%`K4qev_Pet#aR zB!^2*_ac`VkU+Q5**@9XGxnD;ork#zg!rT4U+khnz4nd_-;AS9mK6?VELtiNp zjK!*6D7#JtQ%YwwZgW6DM8yoc!r8vq1wC4N$d=iIk)Z&RULTI|M~dI|Af)-SM8?PO zT>Z50P1^!l*Lvm_&z^PR-Yo1!NrF zg|5E&wG;6Q#|}zogKXM!KxXAju6*Ip(v>7MxvOT%liY}!DOax3HD(Spa~ja>gz$E_ zhTrcsCoy4DgM}xm&QwA>Mig^`o=;K(-JA9JcJ>e;!QmM=WoATJ9uuJLNgOc*@m8fXLwEFJ`4GMrcF47+R} z!1(O6x4e7t)IivBYK_inG?`)T6z=1LJ(>UHlN)M(uIgQbAo&FdPwFj7B!Wh0b(TY- zn1U?Nw6)23BPcfBX4#a+dY}$ouMLY5h`&PU_QdXM8Ss-FPf-PUS2Kt{kyLtY?M4QR z&l?RL2dAV~8Ez`3mY{G^_-f#d$)H_DQk*N#EkXT6>|yU1J(voyOK}G#HeLwX=IA3! zvFiyqY8a4B>u1)})!O#4R?z2vP-`W;rce`AEAR@xqVRJb%i3~GptuUx!ZD|NL)Y|b z67XLG2|8{xu8|c4KiB|G99mE75}C4Z+RHOYL8r$=MrWAf)lIXA@0Ow8_Tvb4Xf>u+ z4EtU9`CW_<#}wOLU&mNU1uN42cXIP3?TiVcHXH5sx;vA?sgDPVN>baQG@q5NlEFF*fwPlP5 zak9qLW1SJsFChL>Y4A18RV&Db7izlBV8<0?q31OZ_`5V4Ml~U<{L=mnx^r2hWGlah z3WD{T>j}7yf+J~e3~oe(d;JIKM#vy2WP12#rxGiUn*sBX)h5{|D5R?keE_a}{ruR- zv3miilqO!D-5#i>6VVA}A%8EzaKx+&w>NZ*tdx_?gmVBsBGiq#Y*P*L63{C?QSw(w zk{2J>b7>kQulmw|*tESw2-XEtvf5Y;qjq9EcL&`kkI-<% zO-EO@nVa7sY&{*+Aem!#{suF}m2JwK6**OeO&9`3wJh>N6Ah{Xfih>xFNq_)OU8StCx&P$_~I}UyaUsAh|4s z2Cq#c41wAygV^`7DwOZiJ*+nW%8g@r@I%IUM_7ezST1w*-4~Yprf99)&xJ|QQR!aQ}V}WUO*#y(OW|16ifFnR? z<#g!>Wr#DMV1^BRVVK z13W%WWUMPUHt`l8I6@Q0#+P4L8k`yaEjki9mw)WKe4tBNBz2664$4+b?<;p3-)XZZ zwq-AF9*e$uFH8$3BibIGU7~fJK!-p9!<$>;Lfv(alj6_@gqNw8&=-|V^zveAN`j@w zi^|AsG>C4T1s$Xkx|_asDbL2ek*bYCj&vgt_)zOG zhnu%OTr6kj3C|nolR3j%!FK6`bE$Vt*BYm57Y!EX)#l5K z2m5y!aufGu0T?29v>jJcj`ksz3lP{l0L}}WW7UR!xD)D zOu>S3K^7|4ssxI1Jx3Ykp*ffg+)?tB?Q=3(OO9P7|4{1SEA_1M0{YP4$={^`L*7aHo(l zQB7?ZN2=P3@I|mzK_AxxfdZ|6i5UaUfDph>hQlAv*zo||==CR}V*cr^D$`h`bF`kU zQvR1FMYiGV{J44zm-h^*Zs~aXx@c^mg1pXphbDhh85n{;(5}D$bWg3rYczR{qnJr^ z72Ir3A4i3Peuu1!A(~*0h+M?oyYAQX%JhzY{VlS~A#+}H`_CK^w6CXs?rW-Y z>GGi~ni(sL`}15I8_1rDHhVO<#ZPr_toFzh9)c;`cMCC!UqTgsljK z$u1`Wk>h{@yqC79=>a8C9Sg!;s!)w z(r#_Xc82&b^zv6m1wT0*$pmnK8OW@Dt6J2{-GrA_&+dxTzmvKZ=;$6)cY*?hzQJl3UjiiOt@}x~F ze95LF!Y;bKabea1<-5ivx~$H=-9f;lgqOU|N*9$pemy6r(!SynH(@S#LUF2kkkPPG zX(Hrf&PR4V^>=%DlT~Gtv%zd`4NAvOJz2i53qcvdo9F_EGc=MaJ;P%-E*!4Eb2kA& z0drnridT}<%IrDQ?VZ2G{4zyLE$C!@(a?Q}q&%cqL%M4rG^YzUMY!A$y=?BRVPwWNN0>Q|$_GPSm+L&}E16mw?ia&Lf@K#&H&C1I-BrI< zF>V1Do1}afD^aSe4d4c>TXIGlpDi!-j2l7Ag=vToG|G6?O5C# z*1zGKKGjQ>&_ML?a1y1i@ah7B1*52o*?}XF!?t!TQliSn9;vAyJHMz$_>JAD6H?i> z@plJLH^{}Kv^&NbZWzsQ*Pu;T(R%asSY2jJmN7#mvGn?QyfC})$3BV zqL_GgWZReKuUUfXKDlO?RByL#dn4I;L?O zrDuB^4*|y2>OFOW-3zYb*x1#o4q- zSqmur zttZGY7VzyfHrw!-?+~R|SIOXsv}I#C%XH5C&iJhA)x|&TeCTfb9Ts9`LmpLxE5evy zU2Y>IJR{Vyiqxs@k*^NIj-!tXu250$``&<7Q;3|TV4a16ZBP46`yR zurQhR<><1QwnOWqAc0vvot{yTtjbkR^&HtIHXU_*$v{kXjs+dIG_B7CO|WQ0%$h~Y zwaO*_^n#oDjjs6G2?>kNHKv0H!)v(Dy0*uUt`-h7N3cL)sEK%F^-t3k+d;T6Sd59S zWC~tdODkRyKnq%YDnW;66muS-do9zzm{UQhiQo_nN+Ct{WHdTN1lmoGHScXkz13OnbpiC3X;klq0z=X?a?O*4Mt=8cp~{(YRRXc#-cymb~4au-frMY}vcKvx>8884r#Jn%<}2C;tv5Xq+? zyid`s;+bY+ASanM4hx`Ty#nn51u=_<>x7ope)ckNSQf zrXU+y(IWFM92eJD3?B^(%pP(b$OHS~8_AoBGMa0I-WfR4fDYz7`v%;ZO7o zH=bQ<3T}yPqbe+2Zu6nXxbVzq0+P*_5DYI*a(cEkpzE{Hht~tQYng>X1gM1!X~YU2 zMCCs`9D{>`FX0cI?i+~lIx*SB>eLyRs=sjl5)V`_B_k$Lr(*+Y7Uz^#M#Wlc|DoK} zGBrjnc0!DyCsi#hNzT1{#xqqO_oz5}P*v_yxEo%-UbxO^@rUZ|6%+y_w){9L#Ud(1 zTvRv;*=C5w7}hlmMKY}}x(PaDYib=1^3>vJth^NfyR{jJ%x7z$mp3VV2pP)p4ir}R z{G1Wb(gEZj)w2qYm`?J1mXJ2iRVWVE7yORxo6y;Tr>vj6zpc8c^jNj+d+!D6`M7@4AMqspH3gz4hK&+Hcv}9ff_g zFSnt*P?_N2rAAa}f6m`i%ni^JS`Ta-THU=j-Xm_11s@GCyu4LSp^U0j`&io#t_SbX z9B(u-fLhqjAIbf1s|Q_0p;lOW2NKNm7?r7N4l<1;R+4P=d)x3yjN_@o9H`-5&+#MN z6B6k88{Gycl8{yPocCq}(jbTAuy87;A-uYCue(^p=qm3No?&Urvzl!#mP);D0GX|> z_-rVZj0U<6al=W4SfREsbnuZ@h-4+X8&Q%cSWo90gS){1+1jZ#CXY_!N|(X_y-6|? z?c|rr#Il_6IV2NfPwoUHRw(>P8184Q5_=EIU2S;rI{0CH405%etr00J#R`c} zVhMnU>8A!er{lg-#XPB^$X^eo!|i7PdoZXl+W-u!*_6}rf;rDMjr-`*hI|@5Kg$SX zqc|o42%Znuk@&O)v({k8L9j}M-hvK>^=(xmS#W-3f9-U>k;M8pW{Uhdxz=tuH#B4B_dw9@!Ga_N@5v^fy>HK zpXEXIoL=X0NA2%S{tsYD_L7a2>TayE$?POOy=r#zygr27tISTETD2i(H#oL17}3|rp0pG_ z7XV{0+|v;Cg#2FakI(r?0HB+$3G6vmNXpNHI5~=u2%RvDgBT8$X?T$p7mOVSQDvNz z^4a>bi%aU)&&O$V$Vr_*Qji3j_mVWW)J#3YUbGF32KSMEbpX`0ZWYp7lvc>G_M{I{ z%;2L>fUuATYD1~7wFx3dOAZ$@nD%NB85H51%9y(xJd#fZfGFbvn&v_1O;M-4$)ZhB z=2jcFv>a&va4hILh(gmNDVwfr5 ze{`E<#rdwUX~maW9*I)LmkNJAo)$mESq4fcru$FN>@;tZCoowq0Nx8!QiHS3f}Fk z{CF=nmBqrN=toxnV6R-wc*ygv2lo&)*bd}V6fg-l=BMTdPSu9QQd$F~N`0TNLCEZI z+s7A3RAm;JqLZ|tz}klTXT{{QBAaUdvh3Z*oV3a@MqN!Y_wxv;ST<9_$R62~ifGZo zNwLr&mDj#o4k>;;a!jpRmk4c^=iP63d=CqSU-m^@x-<)I#B=s}Rnb~p6<7yw6(oZm zETJ@SyEN2~W)JeAB+Z?TA00_{Tdo<0eg49pmJx;kTS+zNEYG{^m)$vRF3eRJdaTZ@ z!mCAhB3KO>eK0|3Q5Tb(NdJ;q%QFx=(C!W*1YWVaTs&{D#+_=n(A2Qs%jUDWmLyXV0b0NtBeuj)wI zpCUTBaBF<6lMLQ0G1ZR6z|dx9R|SI-&?_HF;qbexb~MlA^x8s7jx3eEHf+lFe`i;| zX)@zuh3#Du`HUOt_-%o1xIk}1k+mH7kc75Zlv^>l;|ucqhRRc~IVQAJ_;vRpTIa!2 zG`2u-CYu)_j(AC~B%Lny#;|3PgR&G^dm!QZwyga=q#y|hj+c!yp@$bzO;&uHy|xIg zOR+nB_7$h?*X2vRAE|&oY%!tOZh^YXJ5M<$=vy!(dX}76r*R20Kboj$Md2(O5~5qB zXJY1<8${3X=5}Xl`f{&x6qFnV*R3yVFbF^|-|Ez6 zXHUCG(EClk<*mMsm(x5LN``weCM08VSUghV@Dt_P(b7G5MN^_L(IqHA=+cP`F2HFz z>fSgO5*}JMOKKir%hiU9My3L0kALdWkXU{Ivb#Uy(`wYlLxVKn*7PF=6?{eNHo;!} zX_9{PKqd}C#3=zK77C67Ck3A-8BGguV9&2f&!3eY&58CVU);xczIwl?6{q8MJa4x{ z>a0XiHoG^EF>riNP}t~Z6c6OL)4x>UjMnQ>h%3Dn83qcxoNWi>%jZE+nG#~~rV(jh z=HR-ZdP~TWP^GYrJ+58U(M%CzFk#AO&1hXIgG9r;LSsdszHruvf{SV@$o@9W5jVE| zzOv=D1{PXP{02P%mTLk>?M^dfTZ9PkLBP1zOUtaPie%=jj~=Yla~CG1e`dLlm6tyK zv#Gr2jO-gVj)c&FM=sSNoRyMDGlV;+!Ybb8$6dA9OImC7#vMW%8j!hBQYV|Z_15k2 zW_?5)th9qVDNysPG`|JAB{nU?V+r+O|ATVX>bGR|8iY>ah3Hi>$8YAa+_V{>t2GD< zp!D4**BzGVh(e4?I=O0<%~V1Lkpvm)RUyEsnhGR%9*JeIZ9;XHC+&u&2b-zs9d0(R zn9=;nt2#Wgml?9pbdD_dLly2ciiC>bFuoaV_C7F=2_~v)QSoeSh->F6k!Xmu;`m7r z!Kc({QVPArHJQ@CHgk#CmBdS^5|L8|4uJ2w!eX&Aee4mb+7Jq=v~A)=9wk7*lZ1!Z@M zxfc4?xOHmvpt-nEHq@XLo)ng+Q--hPAO66lQo5rodaazhY=+-xx*4C#?e)UXt4-d; z?@m)Np}kI6L;=xsN4GnJYh20%apem^5gj@dnB!TfsT<8WS`Aq*G%}64>8iq03;|io zumpYLCw15CBWpJUC$CBFP)uBC$kv6FvXi_QK~}x60FL`|42Ful2~^Y|nYnIlwkt*x z*d+FNs$*kToZ<{$9)rqr0q;|1lox)2C{LZUNn`wt!xuiy`<1SU`^+%BoeXg@ zL4L+#5G@h6EGS)=>S!u|W#5=#N-$_NBc5kzoiA)_ z?&^S%7)s!6Bg;W@^;`k&B4Q=d&R)O-prGKdLEMhti3im(+QUs}*f}%*kp&f+Ia^@C1|F1o!m_Nutq3BkThX1FW++M+G!gZ>ahh)blnXrG%y5_i z?uvA;p48DJBpvaPc=|X|YEtrCgoCl#bOEC`vvCwN_0FwT>^vpOCl4SC4+1;0Rtt<@ z7h(8V%bjXAjakhNO8w=)MwPUkxb~+QP-=7;Bz${AiwTZyGzstK@hG3Y2h?1?CL<|l zkq=@BouLXP@enfZ3B0ozIv(@uNrt%Q6kW`OY2FaPQ7$sG5vosZ+p{3vYRp z#IXR>CkTIVqoXCSHv{b47@otMlvPG*!Z-=5iAQi^}SgE3j%n^cFw>e6=$V$!^-?0V=xN`b;%D2nPa8 zbv`E6n4T1ek!Eog;F2~OxEv+{p4TZArGuOqg`g+rE#=M&e^zk#zKHcf4j0e~LShV2 z7!qD|1a>Q*)>I4&tQX7GA)f=G6PqSt(fXtZ!+L!{#0yOWA>ZaQyVp(i=b`UvaY(L? z4d}v35jZsUoj*}cES<(Xf!9RByPulZlYYa2bG)lR@ozTH^dd^0Yl5WboA8t7CT|=- zMG-~2>P8un9*>^gkU!7o(MY;OEItDB7xVMI);HRvDb?k-+>jKVCcHL z(jk{{Xh?QOnM-RXZ4HOJNBu+kdKfciYvGB)N|xYbjiY-k$wan~R_$odXW<5}kU--_ z)=?WpUWWV}-|-F2jBx=0a^8fw-%z)rkH8G`zWpI&%Ga=*5qT9& zB{%NW_S>hLkCX*m(8XYN-If;6{4$56VX8{T=8$0af9u`;ktFrq;n^pQ5-ILfkO*hAwYAOWq@I9~}s4#TsfW#M;y5!17ThgUlf0vtN{FEWS?wcBj%F0x&W zKvsZbZ5kdRoVv#!Y~+DyJBQETobLilrztN~xelJRd^ajpUwVlm)(E9uBWZiU5S<8) zHQg|ZqH)V}0kxMrf!cQeB#JVSOk0p9>Rp^N8$pbgt~#sK7c+%YVU*>HH&E;z8$-@z zE05im@DJ-kdgSS39WO*lFiX?J(yde%o%V^hb{v2-p5v2$n1!UGrzLjFKIt;XRY?0QGMC<>@$KBJWe`6;yiR zAvB66_c+h6$s=N|CH$~!f7qniK8OTBuffZq% zum`uPM*h%x#Bf@@Or4bL!p(ZH)wJ?Mv#yzys;~jsu8wHluDFuHs2bPSK}i&VJ!>Mz zU^(L6`xGIJXzzj;EJ)s#Pibf%@kE&Mg4R=-#-+xc)}7T7Z!1pdYh0a`1o@l;jb|I% zxjpyP3f;x#0a9-5;OTQGES)vmhF3O#4^c~FHaY4cQ4GdZw`~y~SM^nd-P17$tDLth z4>zSSK9nzV?4GoxWaMjlORsL-V6Lgy$)4rQf6k9I^|)mcX{C6FBe0<_>A*m5*SjdO za!L-0gPt&OjDRp#`4dnfaUj#E_5_QaSG|l9;e1$4uOE?CyJHH%0Xor|jj`6;9l_uN ziiX>tkz*`y89AioL|Bs-q?a_%af-b%uZ7sSm=mlV=B zu%df%>v4rRiW+BnQx`c(_cq?2pA;#s>$cSbD7W>3aX*^WY#90lyL zA=zbS&*XYkhVaNsP}aBy-5Fa)!Xe!kxP^yNDetWSV1~#gyXOjlyV4O^H;-Aej)y3^ zp(oMRydQ*Lx$;`OnviIPp%lC%q#QkVe-J_qa#Cq+O~@qKrrQ?)It?L+I?W5;!nHBC zo5G881H?#*d9B70*wa-OUIq}u@!D`!GM4C)4f3d($k`5(uxv1#^82oB11C^D<{Fe6 zXfRoqzOuFZA_N908`y?r4=uQ};C936!m)Nx8n$YPC(*JaF4T-@YkTUc2_bE|eSi7^ zsXQ^OYCBpAjcn1UZ;Y0hdV=h%EHBceFY&_>uY-%BP|sV^i}yXpECrMZeRa{i@*G}D zDyIWLrGk*JN}|wpg#qSn*|Ja3P}Th4X*3y(;~=#j_%N8!Lkc*-haVKdXkcxWM=Fqp zR|00{bTxzqIwO9d6)hz9pVD45Kw)Qx0T#-3%!)p9A&euiy=hu*!Zboy#m_BNU5&4S zQyph9u@boxNqu<-)!JAMq7n^qM8Ja04XXTh^RflC>wY|e6#atd0R>Hav~y?PFAMK4 zhra91%8s_!lgt)G-548sk9(sWLstjf(mjFC3d_#UfdE9Shwi1qD#Z2 zw*X{Bt5A80vV2hUT9j=UTMOHK(2!j~RV(=DF9Xdg>>m{Vg@WFU|h411HYa%9&;VWA7U^Yttp70oB5wechw zwWYe{?&tC3*=IHW@RZG|Nqy}(znzKQ;nu&2D zmwJWbKa+Z#!9y*87ey|3{N>%ym*5Ah&rrs6v1%?nd$j0-45qslD-mpkYCkOKBjC^hBZ+-X)ZNuVH>33}#L6z2qRcK3H%qG#npoC%v9cSQo}FEqrnj zTl7@M-9`Erm}&*!azHbp0hw*G9jPDAu*igAFmD`WVTQ%|YBl00dPyp|msxc6cF2lI zl8=W@)=O3jNLvaF7G}CPrwL(dnw@4c4MgfuG)!04T+`}Da3G59o~o4+HG6PCWB; zh4y9A{fQ0r!exKiB@l^mkX5%&e3!{a&;dIXVn6^Um(fzc8<#{c{mT`gpTFI+-2?}) zrv|oy9Fl=AAAqfl*b3sp!|#{L33EXt?lRvzmrIEz7~qw2_HJoo;90cpjT95(Z2X)p zYgr>Bc-)(`4@~fz9-R*RVUqcqzvbF)nO(*dVnR&1vmG-3FoY*MFaY@I{?hg_{4n8e zE`GBN)Y+C==)&>Vo|?c+C4Po&4p9V2R#OOY*{eYFo8|UfcNbi~*7PiMuIvcHlyiwk zZdvTH^lWQhP`Hqd!pWFL_5V6oJBYM5AI z7nN&+q<=Ui1ycij*Vv|hE<+VMwN(7RC$HHd7%(IphA383>C|Ue@R%q=7V+U?S&)jw z%{s}SNnS-rM#zkoZO^{PVb=2tuLwapqv`=>evm_d6T4)_M<8w6RlKA%^Ohc7tRKx3 z4g0JpwxyIFHgjX%CX1$Go8!I+vpuOlfLSa4W-ST zG^YonXp>IcD4=te93Bgq&D3~FPN;8@SZj+xqdlc4{%dT^R2gcdYB1*=4R|8-w)kFW z%%b0nSkc+YduY`eto%rjCyY=h_)aEW>2PfxT3E z+Xu@KwBEOIv4tevnig)k_JyogQh@nQw>uT0TPN#*mSVx;5{Jl89u({MU5{&riy{^P zQU1OgByfPS#-gUZ8J_rMcNa>QtHaXeYM}yX+=+2`7Gsb=x&?RVlZW3h=C5mT)%#w* z5Co$E(6P)h_LFts)H(Uk>tTX4Sfqu}o7ngEg&T<`c965$+GonffY7p-dGGdCCpoFl^4-flnl`dx!gEBoLfdiF@dq9n6s|2cMjBi ze5&h}xNTiVW;N$LS2@5!c%3(0frK4Mempdw4Z`P!mCxS#cB8RhxXjAwtDVe&6 zevNZPF>k1SlIPfW+5tJJ$aS{_VVD<)Awv>_DD#}BwutpyK{-iX-(82TK2|NxJ?vR$ zE?2ZH<`<7gfv;17L;)*AV+scC31;`ULWzU4lF7)~L3m=hEPv+-y z6vlF}EjQS1MTsgytsHcxj7L&6`E3&`I?TCij;b2=?c$^70bS@t=OV2N^j476j)rqz zz?gUrM5$HZl@*UffCTX70`xle3aM~vuYD;sIgXe5!$G?$iJrnQ*HqeH? zFvc9HzB)Qmy!uM*Le zht+}Dmi@>a`S`6?MmLfm9)*2&GPPRzNjmD$J*x~Jem_{BL|Cf{MHA-J=w}R0k&3#o zi7EY-%$kIHP!6`+JcHEI~$1E?)iF;cqYS15UFP?r~etupd5;DW3yq`&6;m zIcfwCv#4sid=m1w`Ch1N8$eNe(0XhIatPDcv@OFh+IX6AxWPMoR)%&uR0d7(3QeK& zLareKz90?~7@Ju$KQbaLc~!SBGa?yGE{=(eltT0dK(zKuw!BYEj%SzLyVWr39oX;k zSwXB7e%o99^S4o7kT)uGFx4?N5fr@DxcXqnCCtxNUsiOHKXUH_+yK5~GOdwl716CX z-o0fi^c&S0Hz#^0H=!EQ*8VtJ{AJa4;A3mmvWk@nW+8k^UMPCl1;{9T=8yC#Y~8$W z<-2Td?tr+oY7?V_B~$Lpe4kbbE)7-d=QS8aXR9#g5#1BZs`AiCj;5UiS8937>p0tU)x?sz%djM0cp;MSnqJhFQ9BWVm^dDr}d0gx}N z8RSw>N;a$Yxc)1#vrQwkWzBz4Wx2iQWkO(&W2mLX<5(oi z!puzLPNCRib(?h&jh>(Y(XqFAkZ|?O=EA)I@w(iBYm3MqgC;JLX|t{IdPGMhCswPui5|y3mf>ahn848==DzUrxGn5a4=q)*a|2OVf)JRvod;n{gx1K#(MSB z-x&4<nU1rSs-XJr9LcmV2D#MgOB4;6fAp5*7SAJll%l%nbn8!yPftM zGd4~ z3HbYx>Ni`{*ICS!_q$KicPq)!Ju|me@lMqxR8dfZMR6-b*fXGN zaasjb@x@6&xLxAytKLn~Q;9bBbRSkX1FN`@6fm?IsAR_Cr<@O=@yCb3bWr4lX8cZ{~M@wxwyvxD-e%xj~aWXJI^ zd82vy5b{v&z}Oj}pdl$3=Vs2+Cv~aqNSJjJKE$Q*{~D@X3;`&L3*B#P;|CVyf$QB)x8}U)BKSd7+px6KH3y6BS$w zW##VETHDaKQiO?x@fk3=x2aP;&^HN;I$3w6`>TbeP7g7@BI*IK&pZ8c&d*fDJ-Sz) zoSFF|#%qO-qUl=RY$nZbCmig+Xfzi`8?uk(k}E96Ni1Ozm0ezOX<5Yv<#>f6q*bti z?>0Q8tzMF@eBq7v|Z3Dp*se@EL z6B0U?ObiX=Fb80Ul@Yi#%aPS{9^JQH#*0SvAo#vrt**k`7LJ$bmFP76o}tY$8f9vD zNnQx00@Rg#SV5FhibW`boLkK)KvZgR@c{$O_z@2`3&pv49qX2*^a?{>GfJIub`*MW z##dUg?2c$;|7(ome2OTeWTn-v=xAJ?GR)l+dzD^YA>-GKXTTPzmx zomwgmJ3IvIw#cLoRb7w70^cyS+{)U`{Mnj0TyhebydT-rtj?`#cB=Dte`WQGRHaIY1WN$)2)91ioK*Pl3O6lfs=D`Csg$vLw|x+zBgZpCWPC7U(t`?kb+8G`m2Ti`}rd&HAwV@yaxU zoyzdW&wRl)jVM z&5=S|OLRhpjlt%`61F!D@XQ{gPFHEzBfWT5vm?F8JRO}_XjxmJ#c}4QDAE-oVD@6z zb@RA%X(!U^iwzOMKRA#(4%J+R-XFp&#axE&Je|DiS?Hzd$$X1P%o>MEIcS}xI!9KE z?P(+mqYU$nI;`KkpoHrAm#Q1lZto#U@OEUe)zhN1uV2YHMP<0Q7}Wy5l&$XLjn0+D z))ARrOENm+uhMNROC%pRW!zcwcxFmBN%4s<(vm& zZpYdnACxkuiI8#qbfUa8^OV}rBK7LiV&T05umLxw$rGb>QRo)5l7o2LO>3`(20`UX zn(uf!gg0qNuM0acgObbMVc@K~kvCsCS8`a2be>L`k)NnN36btN} z`f&n+t{?Fn5R#j2OS5IoP`vHx5{O7|e~s1HWm|pK&4|8|L3&Fuy3`#69GgU7{HGzz z*Z5YbWa_=P*VCLGu3h47&tgrtWv!UoTkF6`G9^hXe=m$F-KIu>lu9keJcsI$rDD8| zvzhUcrlZ~W=n;)xTXf5(lr>j znt3-y(1*J$1%6t_W5o8A?4le4lD>2=ZCCVkDTdKKc0*fv8r>|~`zk|r)w&+qhPiwq zCEYbpOs^y#px;DXoW#mQC8NzEwP~jVVaBakj)X#C@ocO~ljYOG1poxlYfMc3@5+W{ z7B*kBG}~ueL39tsXTX$Fi>FovsSdh;8gAa>MhaY#A)hivZ-!%O*gYsexC+9I{)y@@ zJuEt~Ht6v{a%)-y!y^cNO0}63STUwo2Zze9&XnvOcn!KiCkk8u6x;I8@)CcwKXmg; zc`A(|WMEhVE&ynYII0ukBb08*fL;Wzjv-;_a&3EFPz1&kWX}{O4JC3qD%N$W8Cl)K zRGmcB2d243IOCZ6c9X!6WuLmo4zZ?3InIw7OOl;Dfk>@mw!{trP zh@;g-cxMu;+t&!85HDka?Ld+%A>W7bMS41J=a6=1kH8_@f5`zTr83YkN9h_xY1Pb% z3F3g< zx>G{Tn&>c4AI1w?vxCV76_k;PuM2vN)>rNPG zZv6OyRFfAczOsUjR!oPo^y45z^#b0S0Mp!2n}p%Sb#axm^?b_gqga0kh&*X3!_ znnA@obudeS#zTyh#l%^<@}x&tt~6Xg6Cyy=0RFlXaw}+oThG=F$vj`xJvK z-FpyLlW8G9fh#}0ZRcGIxL%y~LLD;3>T%dy+D``LpOI4aK6DwRTa zUPUQ|XU*bRX<8B&$l#n)T;5b3k}~x;OKI=4F51L_&Wwh0OUYP*Z(h0+DUWpHifWI| zWpBW;=I!1ek&8kI!p$>_)tS&(l;22p?q)KPduf4tf38R!Mz4GA*|jn+(!x1$8MS@h zEe$3uV^C^aSh-W(vkgq((Ril`3$`x7Y>5N4=1DRpyZbu^e0vMyZGzeHqge!7Xx_6t zJ7BjzFJoxTy*^InvQ+%`O?BH-9a$B7QMeMkIiy=s%A4850?_lkEz2x~8rU{;Hbc%# zyWL&D=OOP4GUwTlMY5$Js8k;5e?|Jd&Nu&^CO+6Gf;n+6aNgn{l%bQwclkhk0}yjq zjaE7;kh!t|NF`Z(BazcgY)FaLGYI&-9B`m>R$+y%mMuXj1?#=O)FBYxAgkt?$t8&5 zR*7})^h<)$;S<}P^ZRD9B<6$Y9wN?rgP=ejq*owSW{9Zs+}aUn2o&hsLetvDmkqz6 z0XaqI7fCn^$aXnmy7SR=L4O*si7j}G(;sb!wn!$Zx#jv`zanp0F@VC`1R(t-$rPB& z-F?;p=ioGFC`I!3#-~BZnM*G9 zE6IV|S)b0@>Xfbw?vSkx;W09=8?7O*;`jwy<-Zn;@8QDuntR{+~?GT9}XRs z09HNTiNWWBYIp-m?oUZ$3`mge`z9Y>wIOSMNxZn;>R+mabW6f&RqSSFrUBHKyMDRk0y?P#O@^_hB0?H`R013|7KKU`ngd>^ zz3>Q@8kbt|$5&af_`Y=f+o$v;BPVl+L*E^Ie~)>KWY{%h9xkF4aY?~e8hNW9*nT>! zjp|)L^EBQb1R?8KwW7=9W2+9FCf$_l_A#FE!{4zqEcXL{flP_eJv6{nQ=r;pbSe*Edy1|pq?jtlPvXbPF3=W z3Nl#$S*yk~+`dB+rXdx2L2Fgh;>2-2{J>^L!_&`F`N{--^sCxwbL-oCe$WAma~@hNI!1IDYP1X z(?2Bq798oG-rA2gtni1+=%YJ&FuVf`eXwIswH2XeX$a*gb$q20sagPcsQ;_rjD;O}zo6z959p-)b*A?t<6 zK<@HNP%2_4+-ML5T857z+9jt1MOrHCfBEX6G~l@Jr!0Sy1m~w5B zQJ0=*+eILsh>kRZyxdG3jH$X3w z0+B@}g1j-R-o+Sk;gM^Ek-?>oghge%1$Nka$z^NBx#c`{NF_tehu$Y@A!j{1YkLcrry0B1|bW_yo5 zZDzEoW-CqTi7$(zIIAJ{DSfBhjYV{)8?U(--8-~BC8w(+`l1LpQ+)O{m+Bbv6B<7q zi#VOyZkR4Y&7FHd37QYiwv(U^NYOpNC7zAl2+68F_8E@{@|QfSM8r3&^Jp+jWQ(2N zDG@t9YMn;dx)Kte?oWfNyj`&nV%+l^JL;NJroC-8i%baMxWetx&#N_Wn z#LpNA-ch*;T3T6ZnKZTDiwO`y=*)H zyKZR#fGiRPT#9gqdNKRsLg{(v(rk6r_4q+ZM9x3JhC$a3>=nxA5?gn=x9Ge=y#(1*?>P}!XPGOJ zp}H3es)oG99-GSPgV9t{^i4p*r=xxLVq=x}*b|`qI=LS(+t5gs{J7vXj#XXOC# zk~f0)kARE_WtlR^3R2dbw5A4#n^f#L{${;igkfFl-q$b)L7gajt$(PsVAA!D*<0py zbxt4SZ*y0#Zy*n@BaHhLL#5Bgh#(19oLleE9UAl8`h-eGoT!j55A#%;I)H3eYj32L z2(AEJP-ZsbZ@`qd$sWIhqAR-i(k;6hi7|Nt6ylp)?ytDXWb!Kc0?n05n$9>=ujg~c z`x5q4E|4f)MZ$x2vBUOz(V`OVaUz#yxt8VuG4lp>G^3jso3WG`RjL#ua8y~M#ZG;0 z7J4UL@?9H%2Rix#wqM)aJ<88R7nB%*lPpf83tI?a<8!f;ur?10wfth3(PL`a&j{$D zQ2V`+XH#Q5m`@S%RLetta3ip5iIQF3XQXn=BBY_bdHt@B_DqhW^|0x~BxZw)+Ltg_FQY%3r5c`O_)*)w@za{HBTUq4q zwCmMj@E)c}sJ#WWtUCeUaT||g=Aq2MAtMVf?9+nVO2RnueGlPi z;{veIGE#zLI~+K+4A8t{#>+gGT)WO7O2#WBs&EsD(N%F^9K26MQkZQVh}R)#gV~r~ z_0=?rJH=lJHIO3zxmx+8@$B=xT-zXgF)}dWs5vl^2;kl!v{@{bmh&>*m={JPYegob z0o<{K+8(YHT(GrIvp!oCSM~YVL>z<}{-^Qx2x{rTq+t)X3^_#=O|L9o0ECJ)Q80~5 zb7^PMzo<5AR$AT5!0H^KF8wlx(zdTy4bHHx_j!wkkRdCS^{@z^7aj1Rfd=Bwq9hSm z)g;NY&zSyI(Q&QJ^^smyW|l$bFE^F~eLoIZgF_BK zR@ohmo|h0^-Qt-Ob-O~!yInoG%c)}CR8g>V`)$ZA^I<6EC#EcaF=+SCDn zO%VM0=H8V2v$>FzmA>Q9LLq=bqjtzVskhh%t`|EFOnfWJK{rvZ5R7bLk64JL&p<^< z6wBYQ1DkVa;!~vNSK4PLr0Rc1@F-p$X;hXf@hW03a+870Hm3cDnNR>RG6wG2m!bo$8n{8&^Ss zWFek50H1O_FKk-p%{;A|IJ|pNq@8m17$anF+u_5);a88IS21q$RV4=lI<=RCCd`Re zRhiTeoS7P!tXFWDbx%qnvJmOC4$pdyS$j~jpQN$Wra}@wp-ZAUs1ib}ZOaSPmI1~v zugUV2MbsCejgSAdjfxam=6KnW8Fdykr69puqVx}0Bjf;4{9^!^=yU*wBx3civ%a28 zn9<7DVg{)cTFQ6_RUL8iYJ9?^F7}|G?tl_58zdPrT6W2M)T9(r7%|}$=Rw}Rz~lUy z((h?dK`+rB4t--ilFv96w$@9O3wX)11f_3tCO%zlQKm!DGgzAD1HRYNtKB`^f{9eD z&L&BO7>fVLk9cshAa}WRNZ|oYW7tLEkm59=H$u;~_c9W5JYaz%?h1;j4A5RmtdmGx zh&Lo-TL0iDXDwAcS;!m|=oN7NIM=N&DFL!3-rM|TR;cy`cj!%=1{nKlx~4j$4=Bqx{v_b5@BiVG7oDSI7K zEL3QtG{MxnZGXp*6l~M^I81U%QgZBS-q$G$d|WRdM-;+C&pjVS8NKo)mu2xwD7jM8 zZ-A*l0ioNvouQ*u#o^};OC6V@aOF%(JPj>yl(~3+8Ta|u+sLehnr|(L;}oDC7s|Vg zbWd*irHk(>O0B0$Flo@b;2;-hmo!zdkWEZ46^5JQ*?$Q?czR#heymV4E7j;oVr?iu}C{U(~F-I zxQSxOW*J$~gFy#Knp>7&p`jQKo1Vc+y<#kMt+N~P-Wy9v$W~D!TzagUaz)gHc-dx$ z7UEzol5Q2eo0@Fx+p$}vL9k2C&Tbz3RJyG3b2q5+Oj@J@Oa|PCk`*c8KQW12TL2;P9;&rcmu01)pQ{bq*%Y3GAcRmWtf~#TqIQtT zMf5OitfC{Ay+3x=wfJX|$bzoZhzzJK%|X?4TH)nH->J_7)-VV=FHCdxS@KnhqA$M! zhzmO|!2vw$GO_E91=aKKCSG_~WaToXc%-+Nivce(hs!%S4wJZPQn7{%jlr0y{M~|U z(;`@htwj#@8A!2Kc?7%IM*8Lr|)`GzWD3+sam&R@d`Ry<-DwXarN%+OO4wurDrU)wKE! zPwU}xTz`<_bJ+Ag;>?pjq8)@9=;oVlvc^`4uXW3$HlI?#JIch>I-mt{= zG$K{`nMz_LT}@z&=G!xTiEb?*Q9AuaH89DzkVbMq)coVlQxmVcLZX?qh;n#38SO?v zd7D1jGug2>Q&2ER;@;`8v9ujIHBUP0jBFO%Ay*t@q0J5!tAThi9aZ~lcsn-~&mfaK zot^oDC*LI?vB4Wy8`q~n*l~^^qq7J#5aCew9?vvAoFwR=xqBh3jLe+3KGrsAr4D!9 ziPPQbvWF4t`>mNIK0o#TfyqnzHzms-(phWCxEz|DofjIrO~F^=ZG#MSs0Dz4oqovF z=VXq2>jLUk9zLzrXe6>nrC#y73sA;j*9=h6&TBK1dgOmRn96{xPQMW_`=pF86Rvj0 zO5(5NhB?J8M=BH(K&CK+P{XV{-&h|vbM9Ma0jFnI1SRv)2RL!B6t~t7D@;qnH15pi zM6G8`0)z+L(#myuH7!&xB1cf=L!?JAV9{^1?3=<-K&&*~GhvDW=7V=a+cH%sAjdJp zdzGYUL5*!*wDJ%PsFf5Vt~F-oY1=fBTaIaYIT>}+Pwawiwu|z!h=Hju9DwDmpf@j4 zNEz(t%|;|2Ps!XSd&#Fp^Ugix@ZVRu;mFCqgjXJO*vKAXWb;Pv8M_kzIK1Ux z(|@{$lT8h3s;ImvwiRy-*NXZM!gg&05<7F5AmmUTu&YY}z83?j4^j4ipU5EZw-D9=F>?w!-})AJXc2Pm?l1cQ#hj@V!1E}Ca$x(YY-;8GvR0B^T7SP z;$CyHR*=nX zQvi$I#vaV8@sk4nfpp?_w#RiX=e1Ch2gJO zbdpy^&llX2_g{pvh50a42DpUQyR)Z6bQ%qLv?| zp-eKJ2Oc(}U$!Er91mssHVKF~aMD1u`Zx7)rIMi`dY2kK70ogsFHI)N*P4ru^k`1D z!41E|7sMe`J#8`2vR`OSx{~Y?lZMG1jd2*vCogv>DRr;YcqGJ0{&Ht$)-IQ;b}NKp@d6eEzg{B@#;0i?D>UUeeX4bEyQC(@Dm=oRyA3#4ER z5%9sC##g`7i0^YFs+Q$b`XZskfjl#sp~9fu8hjsYZSu(Ji<2eX$JhzPs8HkzuLC?% zhUe1KmFGvwA$BNB(G+JZLI4_$+yuGsM;MtrNk5z`)|d2ENLMo%AbJF^n>EI>=BL zKYrLkC$iCrWl_KmWed?;Dw|XiP=7v5>fzvC#qkn*^sCh*p zj1Kibw8bUz#aTvV{Nx`kQ{#^?nr=Z=f+W4Wf$*koWx2No?4-b2q|ps9*X*~64y@O@ zeBDb8ByO>G^12by@eJQ)O(r|x0y6=~movp#2%4{4T|BtV$*g?>#a$Hl>>c}6k4qWO z#cx?u3WzJ1bm4<5N*xU;SJFXNr@9=Y%LsLGxGS|zM%FkagGXe(H0!73u>s%lv7v9z ztwii|fc^y>D9b_#F2MICT%n>4tF5DTR6H1LJFj9vPEWZX5Ny8@EOAZpHMPAUg^L+; z_PD-LzH;zh(0Iulv3g)#a6WA#PKC$!)~qpkuM#vrB@7V?vnx(y>Jw8zFZj0d!fxg% z!!KoP|ESB3)I!4m1|~`vwyL1{wxmeTB;Y}1y%%^?j|INoh8I|Zh^nta{r<}ZIm{`q zuPA{AU-ovnOT~Wc=YTwAA=QzQE(SjxG3Y`Q7FJ%FMP##Al-z7-AS#PQ8vy7dC5|ZY z0u(GP+8t1)n}7ozp!qU%142V`pwLYw#uWrLi6x$*mv1Vt1K<;1Ty=m9Mm#dqAn~%0 zqR%&7(X;W(5WShIaUXPtROx+HloBaT&jEQeGiDK@Tba}y)xlXuF8xgDx+Ugt$%Py_ zh^}P4r)S{@JX{HaYKL1H7=?}B3fZ;fRv6?Qp3_rOy?AajB}}~j3~Wrbw8;YLRM{Q6 zwHtJu9txT+Qx9be%;Ijl_q zDpI3)*knn8(W8sE$%g~KS!!g+jV7mu1Z~$&XAZ)~jOPQll99;@j|yI7To>&9rTPuU zT!nnHi4ytVJOp5^N#Ao$5wV>g4sP!8yO=~Dr&bhKD-Nb7cHe`Fa*ji>?KsXgMu$m+ zZ_kFxd8dN)qG?i;xds!a`E{DtcrunmEdcGx_vERFfN1^jkmh;sv447dl|zDE)b7l! zX$C5mbpdx7g9Jy9U)t>k&Vay_+BkrcrcdN1a2qDS#sK``7X=lJ1LTU&S&nKIy2s04 z$$7-2VxjQrRkAuhuqfCnSGIrcFxoNLGq;D(z2g$?x)sf4l@SqeFfSC?_Pz#>lQ%~% z=Qr>*RZ5s;K6=UGj6q;>ts1i^lxo~Cab13glk_e$#XZLaw$Q!aEi!w!)9O5-kevYy zq+u4=& zAveSHQR$|U*S=w?BZ0Q5Qb9Cdz|KM^ZTxxcJOd|b5=%0}gOFuoO z&nmpftf(fT?9$gmHs^&;+cqb;XZOTNM`$#Z_-Q$Z?BUNQw1{}4JZHNW&sh;`qrK{ z->%C-z@(s^QJ7O;7K)44TM7h1(B?3x3J}g_fMK;G{njDCl^yNgvkQA+r?WHj1tZNl ze0ERJu}c8p`^psSN}f|M^sZ%M4a80zwnq*;FuY5tunEJh9>QIUVJL`f=nD_THTU{s+V%s$LfVN zfU$Ua$aK_m$2}urM#BI}dF1x!DF?)Ep~q*<69-Yq>rZ z+=vMoS_v%={-9tqH?81@`ix#okEU?XyOGi~kmACmSbtret}S9TJ;jzb&@Pag2XrJq z=v0N)P=^!CZ*(^Y8+_U`-|Z~MDX1`$%V^JGRVHExruJNR!WZQ{FH6<1KnP0XhBm64 zR&BnG8C;9xu#mfHQggyHx5}EHs`WfTmF%}}^9Gtw=$KAi8$Sx;c3B`YH9K|Fw`oY& zqO&fcdO>PKw#R4kCt96}Gx-N)6@AxDL-o4Y^d@{2beitss{^GYIDnB#kfcloyVgSQ zNhm?$ZMrI5JpC%Sx9dh3#4i1m_i&O!y2fMAr$YEHJr>f*ud$oa0b8(wa0E1abc-~x zm39*E$?fRs8}^C`g;EXDObRwln z>3U~Xgx5e=NbvuIM2Km!8CcY4x0eiwi(@Loxe%#;V_Qx}gKgug4e1FmBS$Iib;#wzVuJ@vP#R1Y+jC z@a(cw4%a}*@tFuFe3*GN#g9Sx4${TEsS;Je*#QMB;S^EbJqFMWrfzOr%EN2RI|~ZYltS$~-JK(fv7-#Eq?QKqVFEmI_oS1s=rhjV9qii? zPoJw{Gb;J+{0-~!9hlLut@{OL_uk||W}4pekbi>?gxg$|sU=!oV zlp+rjPpl~X@b19T(DV$*r}*i7?NRvg&&>*n%|{=b)bVv|cmg(5=Y^{kWFDqIwid5}D(ZQ@7`?6OQ%D$Lj3gbd6WIF@AJVgj|~r>o@9 z9rTyIgkIj+*|+8uvKoX;B8m1x@?IsJhzA~wwA?c0p+`VIDaI3t>dW-Z{2t|-kV4UW z_R-WBBbFpEWD~4fF{a89mVz4k=z1~G+JD{AhHIB|exJP#>M}`RAJw9x__tHG@5jTgXi0v#VL>`es>VQp$-b1;tKo2rVwX zVx^}u_e_AW#A45n*l1Fo({x01`UtahtOv-o54FpHu+3u3vy!BajKEqZ57=wWp^Y04 zq4GzW!C_ZWO_a zf_%veVqtCJ@W{oMQa}+hNvytp-vlA7wr*?()tLxw20cufs-Lv8i6Z^5^BLl$Y6Ibi zq!pf>Vf&&qx>|5ZVn*0X2O!~@lH0o}tE^|H9pNe|VzBE*@ahd$0Q8{|@Gfi#>MXr2 zGT4`6$A=?esX$Dul9GoIeuu3tsH&KaY{@OzpR%NLIB^RocW~Gsp*=ia(ZaL0=>e+t zkT=G9hu6hl66ONe`W_Uc`|?K!imU?3nmGR<139rGM?o%wgHmhAO)?03XCeK?}r`x0L!DsU~GlU@W9~^-Qxu~_G zFE5|n3WT&9Q9-|HS&H^SfI{FXy*pbf*xQz`Us6yv<@ZlkN(3*Eui-VsJq^3zvUlx# zSC!d_Vn`FRO3jg7Nq49YwfA|_*?&7ypNJ4JP_18EUeKGm@ul)8tKLgCyk%?b+`I!< zpofdh0ZjU+oZ+KHt{JS8GGU8D!sj;gev`m>S0}dge*k@h%?6&A*(0%u*?Wc{av)(; zM+xQ|A4Xu4O<`P6+S4tUa2d@Mk4>!Dd%-6OI0QP&|d`ykm|FJL@`CH&4;Aq7H2>560HsluMd z=<@6DuwW`@*Mv*K@)n=Kjy{Sq$Mi;PEpbbG@f}8x9_DTR-X0&M09sysHfJx=K=z!2 zndG^#O-de$KyWB*QJ9h3$f914i_e_SX$LiJO?qE1^S_QvociEO$Jo&nvioB=19}g! zvJ{EID2Sy!HepTUdVPE~jh@VuMa*ZO_ub4h25k26)0y2jiI;kk%Z(z*5I}beZjj*` z$#+mICDkt14XVpit|aU?8q%i?2#7bzZ^fAl4-mYca zPAWtE){riBDxRId`(nTfxfFNfGR7nCVx+-c)O!FrC@mYM>wXKR z!Rf7xrADdN1iyRCNw13q@`t&5DD*@GZ_>24X)4W`o~UfOi< z__iXlaP&P~^GGO*=gfcCd^L|E?0r1MEDa(o73uY@cG)9N?d-78IAkZ#gpf$YfYd5lYJ9g zxR?5jn-|vh(13ZYAkB7CJS!tYw8$yTSnrM<7C1ia6BmF~yB}a98%(v^&|7EjSjIER>nE$dV+wPPM5744SO9uLSG{$WoRxp^-Zd zraSpZBPKqsP32$_J#i}SxhuPP*7rkiK2kA+m2&CL2(AQ)1|9mP9SbYxhF-MqSp;Q_ z3>`L=Cbc2y^=g_s=R?sTaP3Y*$#a``d=vWSE#@x}1*uh>&N%8bB|(@FNt>#t+E=5N z;?W>KTsAeCho_W(z#YzD$?8nR`~FD-<4x>Oj*AbUI}32?T- z{UaB~#G+Zc5%L#ZDt{S}ElajCqnDdJ;DQ#n4Mf-^x*3{kSNMR4%?T>_&RRR9tfj*@ zg?l#S42o5uq|PTS;3rt-J6hw7oTo=Ch|<(2S_5ii7~;5@n)bfBbLi!pZ#SJHKLCwE ziKlA8P|Mw(OHzL;_2d@x;PvT$ljE5)$@+e0!p6WfmKL~a)uxXCF{4>@hKfpv6^3;* zz4h4x7e%1WZUqku7wu?C%TlcCQV^Svahvh`Cj2~#NGxEe@G_8QLCkz|gvXRMDbFev zFJ$XxK5fZGiuD9;HaS%6`x6^g`W2!5vWs;-q&sl~w{v>SiV>3c$ou{QMsV3!2F}EF z{4AhvfC1Z2Dil=U`Lv7NZY%)~hutK33|3`W!ri--MZHRr4zrpf^2;{tDxqloM1gpa zFeel*1+pDyNIw)Ke!I$cs(EB~l?{B`^}5&;pK4j_<5G@{e*{y>gdkge*A)siVFJ@o zjRj%P9Am3$^B>M2>XMQU8Vx@w97dVu#aU2@>6~q(ez%!OUw~?AVYLaOOTCo(Wo?0% zZF^Hq&-aU<&h1U_4M@T6y=KqO2YIOncNW=(kuOY>3U19n&7YwVnNEpY=ZT2Fa-)HAH2=rHTi? zAFDTdj2FR>D+iWooOdrgE~(cWJ0RHE$v|rOxdyv^mw%vOGKLa(@lnC{->YX|y?n<5Pvr1(VMwut}vS1S&(hrf0E>^z_>Ui8!-u zCn3uduNlDx^;7`#36M4fm3t1?3L)6`Ro$a2%^+p3Df0EFA)2*++$K)StDu;nSqzXC zRPZ`Igx@xHbSHsjq@04LtoSV45V1bd4EI)D@qD?ieyoX6w>o&41=1B&YPN9%`e;jb z^HfI8!SYF)Z^n5g%ho!Qb1al*+e7rk46ED-hHw;E-pWPtO2GJ{ELD~MA-(hBO<2Q_ z6sdk-ID2@7*3RomQc&l{Vv_>OKy^hKgbxF)Ct6nk==U5@yF35k(7}@~vWG%1JE<;rSMn{7LUY$1SSj7nr z$pLV|ebb27jW{k7TPuj9=O@a_l5T;sZ&7LNpH?RuVym40g>67l|Y6eD}E!f=YCjCunfZqDUrAz5G(?2 zgLCIxVt#l#ZYf#y!lUD5*Xt9(RP<0jSZ5*+Y}{WUr~vs-vVqlIm|POq#ydI7ADVhi z$vYShM&c5*GES{}lnfVNfetjlBa3t%Di4X+*c8ym%~ysivzBWw=9BN9s}+f{?{XryFe2i(K*eK* zy#`mu_Hd-BVNf7+kn3*oNZMWgMR`H?U`M%^Sf_cIv}Kj<$~N7*B)HG^ME^QQ1GBY} z9Vu)%#Wfts#KQ9XHoZ{>GeLYmE`s@1RCL|61LmluQh7#(9>~7|OhLn8sMI>8cY~TA7^Gay0 zX!0)`4|3&3tP9^92_y_7l#p2l;!sPmlu@`ix@U_7gkm=G zHh+7z;|&D^2%~0uWdz?v#g)N^6L=Eg?h;Q3yt$0!&i?*V<=)wLcZX6JmKH4m>t=p= zCDl(IZcX4uUCQ&uk8!Zfs0g_ZQ&Wlk$ZFRHMp%^V>Z}!7QeEU^Y*n!T&`79uJ9q@V zpVVra}?*s6kLExlWn2!(@uTc)U7eJVjodV#bWq(cu-jZT|9 zH?freQW0U5P;B%uMFz{J{-)T<0~+h(iy9G>?b1d7XMIj293rG{`phnm*KdbyRj7{|Oh=??`EH+lK zZa1Z~=i#GG!yw>#56F8PTV`mvJ+!KJXNYt}uB}VssO>}Q{~8C0Iv8yf%%(rsp6$;A z83Dh{xTJU7W!$w4aK_rzrpJ23HrK{(Wzcb{{p5ha<*?+j3ZQbd&Y zC9#Ws9%=(4qng%v2T|g31&z&lq^eyc|GMo=s3I`Vun7W9o{qB2LR(7P_Pqpc4Sy{$ zb=(QqS{oqai;LwGsLSelUz?KhrVibhr+om&Z|&Na-^vxPKSy7%dG0 zy|iwp_!oA$FVvIpDjE{ymED=~Y>lv17)0V!c$Jld$E~)7j1%}Ne8&e5myT_yaQmBU zwe>xT(cOAUW|=VSGMR-HerdWz#`wAsvKr2J-d_lhfQ@O+w8ypOcO!TAQW}Sl#y@3H zSGZtkDbL@{)Z%8;`0`hNpyNvLDl=U)D)5#ZgN%>3I1=)10DwRy&3{LkKP!{#r>ohG zuFh}F*2Z~nrOr;7*w=rImrcx2@bbA(r(348h7~s;bM_`U^>!PFf#)Vl5V$K2qrSNy zq+5*V%{t*xrIVCp{)9mm)M*>i42@n+?&ChB0iL7CZmWu9 zH6T{Ld;P+nZ6Z-lYZr&YL<_obowa@)eq1lmwII1za$2zshjr3Gnu)3@R?@jPpI4&? zgIii>Vu^4FQrYT5@RRYB|UheH{eUfjg)~#3`x7xmu=Bik@Eb$>Ll%kb>`hk z-I>KwGwZ%xKiQN~1RD*n&bEAwi9^6*rU;F}(%$R77%r3x5~8yGQ%AbE z7afxH3Kkx$fjszqM-DlfSXi;XH9X%;vLb|$7g6l6MQ)9|@|iC7s32&{uzxm(>Mb*Q z(ZiQiALX!aRw}VX@9$+3&*Mii^C?6jXRm%qG+(>NjLpnIUS(O!(xB5BLuToFdbB{A z1Ev*eb{A+*t9V9=Y|;dnnnxY!RKSnYGT|(X;c=XR=#6}R;E(_i7|dd>k!we{P^n%f znef_NK+n~=?-7t%BeU$$!)6ek914wPWg+Qa*P98B9k)n8LzfP-mzs|*mE`T?T{G#V z+>ji`8INr%mj}@RYGVw~ElZiPNZczC`)!jNp8OafIxvU|%BBT~K_O`YF=LXAk~vwR z-f!`jN~^T9*BOTRLbQI9$ArQe&KV*(aLPR4H(0KBDpto^q`YHDdZYN(^P`DkIg9mW zfCakq=rR9!Hp#ggSCjfhJX<$q#%p7yy%w|2DD##+ztfR566M}2DH|rGD@?{Gp}L$@ zwHo7jGl4VsU4e<@^WuHkopmk%bqhRxuyFLzN1?+kici}}NB*X%x34O7$*vx@}3Qh#vPoOax9#? zaN4iZ3M{VI2$fivB;h_C4RHI1*g*P9P!^D;px$BmXFI$8M0>xbobLZSr&q4fwuS;M6j%ZOtkm1RXw2d$U zUNVZ-J3d^6I8T6cs>Q0&k}rD>-b3DC5Li(5@C{tINhj3(*mXu#d6>+|X)2AaCl|E# zK^cNZ%XF%63@Dg+v)Lv@G-!Vj(ryzsM4cm>pU@MX+Ap0jt}S%*F(cko+^K7JtP^wX z$2H`7g4tD@B&y(|lbk!9#A|`Riq8^NJLVYSGQswm6c~qF#Zc7L$)lzGjNC598P6Js z`1{pov`jk+(yUslAXcQ_47xy*w`rJ~&yVEY`J=*q9DjWQaTiEbrc9>Z<9(OMr8$8r z6cSjyksF4MCQAINiFI4aK1CmH{+^=01%;4?Z`KLOD2jgIn1*2@^^->A+Fq>gP&AN^ zvdMzYOUniV2IDxSe0KFJ;-R8sV5LxnQt`(uC1$lX_HWWM#XNAE7q02zz{Zo#_N#aq zgkIBEHxyGqM1>iy6AQ`?1Cp*0L1nHs3S(UDMRibVRY0dzbboME>_8(3tr%usTeI*T z@2fGy#idTVH7TpV((PMWy6MLQTQ)#T~9p9d9)HGHJ zFTAb_?}^q54s&_Eba+(~hDE6n^(2`e!-4A4=4Dei5kj9dCRS)5>=4+COX+5pVU@I` z$EXAWxa#;=V8b0zv4qB+5j*ubYXlCFFb@wnm5ZbyRerY_9ikmhH{q7?o`GPnAc0$* zblu3K)C^`DIAKF=ny|B}Nu1*Xk1z>`9SJVL1G{x(S}i1cNnZG!cXOgpLQ0Ox4w}~{ z7RJtK)B%uc1CMZuvxg>j#|G|`GvA6Y6T6l8F^Ct-3n~Bm6Av+iy|B%ggRo$XyzHt; zkWWGv36Q1p>uF1XIRh!W=X{xIv&;KKopS({8BI2>&_;=uO_8Vg7iFzEE%F>vjG}38ZqyUxmAR2C4St>(;O6M zvz)*N#co#F!;|?8Kgi!cfg6A0L_84SUaV;>SuO!sHQ18VQ@B{M!732ehUYh z7K&Qhh~tSJs9ht&H$mJLuMp&%?$K8ENp4xLCDIb-9ZNLqC6_3>uoeWWMmojxuLLchD%*)_@J#@+jxbQj8R5ls-x#LEqr7~k%EQZBXcc^JZnBZ> z2y=najd2IR7Egxkl9)ciB`9(;ah$tY%?%fqdl0UGkm)I*Xt)bVYs9S8a2B+laM~cg zJ1)wwSYiX(Jl);RN!~~_!x8m{Ph{`gOWZ){3ihQia&HZEw#l^;pq+4^Gb}l$f<=Y% z0AU9Sw+I2W9;v=KHUvh zpe`^{34f}}F?>T!bX(w(4&vF-_BzkkIduhxm;npb3%G1W^|ety;?rP1lCHEn#@i|0 zY%WA>?gd&?D+dkZ)x~0#LfZE52sF3W?{YVtH&$K_XEQ~gVgFfJE-M{<7N57FsxUH0 zQq<^7;yhJ3klVSKWbWmm*iy&lk-8~uE~U8>eZB&BJ%C}e>hr58ZhzR>4GhZsoI6f z&(kH+2e;${XpkRVhhJ;Fo5&YRlem6LeS@0^<{*m>fD;E%Dk4WkBl{XmAM)pU@gZj> z&Fx2w-Xa;jO6DUi0HEn`OYY|!UQA&p?zLw=(mB@~1-5MwiUZb3>$1lKMQ(4^&gP+K^0`}izejYU%(taG zPhGtC4FgasCxVd2WjYqgN!s#YI>p!LT#^Y>hR~Uu-X?B(V@;aF0eWdX@K?{kYLV0$ z&$inLUl20Zz_urGddF79Im1OD0=Wzxksn7|n|{?Yv_tuDgbn8v$3)T%<*UCkgpY%C zs&V+wmr>Br!RtJn)^p3|Uaxxrj0aTUS~=MZyP#yl2bL2SEIz(sq6;FQbli-@?ANOM z?6~rs1I52uNqB*co2dw26-DQeLp@=id{N5#(07Zs?ByMyS%VQSyS0}`ypyjVPvKdp zmi;_ZYL$iQ1583r_r%V0G1DTx`}B5A!+*)r(`u`++18juC(RjI?>(vHl(FnQ#y693 zMKXl<>n?rt?9a5xyieAUQ}iZ(BtKZw4U*ijA}F{Qa7^Ck%* zPKqT1t)$Bu^Y;!LM6?(&!U=JE)(5Nu3sgpBSm0A!V)L$LI6}*3ti)ZGd`;1xgShXZ zNJ2t~Tgg=Bd&gQ%ld2gr9ptC{Q`vD$=6e+ARQ<~ccbQ2{JNiJl(XaMBdx#w@nc=Zu5gqMZbsN$gjO z=0@sCtK#7Droollm9TD}DN;YQPf&GI@$9vUSp@aUP#NM{NCtmpdb3bhA-&O-?J?FF zkxz&jv+>Ijk!?n1pYf<&Yn-2H?Zd__Mr-}is7^Fmpljr^a>+7ZN^661>kZ`|Qy(iy zL0*|Qr5&3wnZ53-=mB>6_v*URw^#3&H+JsMri16y*}aRJ~4P? z8di%)L(4&g4t<9?9TPc$u;8rVMt|xX*dw@ll7Myufu?=~Fw*3^rd2M^Dwj^&H08W~ zzS!>cAkQ{Udo{e`=WcysO15;jHi{$NyDlHmqx#`D7_V&u_aU>7sBhg@;zlc-ZHa=_9Y@(U?>kuW!2M+j1R#~ver6Dv3x5CN2Uqg z@<|(bnBGL(#Af8aL#1{dd^uTp7jemCS!D3OYZ)u&D!i54SIVn&=Z^cwSsqBc|tN;(1^^^6q z*&Er;rosahtZAiiXHgVW6{R;EBr$K+vgN>MTRn8jxb-*Z!CnZCT}zj6ks0VEJSJO< zX;zBKB=r>t>BA2_Q3@evm>KZIr*)jO5%_hYg2j@+57SP}I3w>u;@9G%Xc)+NTUbPY z8mMLG3h4)!1>^8uVM#F3fTJq+D?t&Y4j_{{XTQjN?tve?mMDZ4{dv;I8Y zjwSNRR@1jYP|_l{c#awB-q|)Q!5G)#E(s;HUr+-Jw5Agkt?$jbV!0C5L1t*Zu1 zDHDdDyH8Cqd(vprM~9~0tRRp&rrZ-tHrN>%&{7qQWzf>YWpdW~7`_b`HtR~y7wnZa zFxq#q78(Q;iKiDl^&5Q)cxe2s^~p^`r4vWAonod?nvJI&MrN@XD`0UWuo`7*DToDj z<8bViI`{0MYwgRo_4CW#oPSM4&EOATKs{O1G64rx=f)_>Q@_rw(GKwyMH%G;#b_FJ z$mh9F{9BK`dN-6XH|*=*)Pumb4{#N_C6{#bg={V5(&ev(R_lE4g+#|lWJ3-ph_hh+t)G}}ZMRv=A(51@`-)YE{ z*$B|Ig%c0@ynLXbWhih5vPUDMmK705ze&H13CXztfWg;HsWCr+-g^U$o3t1_6OV1_ zV$MsiH&#l8O*q0MY8BUYdP|zb_B8i^gO1}`fpgSE&=vdRh(vhR)P;~5q-n?_<3tK- zH$~33yvYisU}uBVO@%5?6KO$wK5gf<5j7%Nt8g}nn z*>&h|x}oc^E}G6IzrIFA0CrtqsM9*O&FR-HXPsFo_JR5|mi&PzZKp@?RMOn(2X!f` z4`62{vF*qp?&;W4zOq*$OraEJ*D#+u*4`e0C#5w^ap-qayf0!1?Mu?PsUuI-;kb#YC}bXMe-JSo z)l)B!nyI076Ny}KU7w-jT zh6|MrNLkF)BmVc5P#1nMs?+RwPeb%fPj`D3hP*Iuzg8d-XW1b@WcJkdBvfRpi3UwX zoe2IbdjT0%S!`+I8p@Q5&ptzV;VG_%Ijp(bau9!-Z6?cEP%_><6Mp4+*^~jDh%aeStAfN~Q2;i3ilDWwiWSrW>o5oy zUp!V*oM0Sq&#>IP71<#L8OzQYBRvB7Tn6iE>uI#;Uc*TEpqPH&($kB5OLxNRA;&fQ zpcL#$=I^gQKdgMC?Uw?#IW;og5~nO)6OplXhG5D8vO#Y&lWe-HL`^lR7Dc&{1jO zPX{|NFs5Z?F=)|{Z2!6&djdo)qN`#)k*4>0HVw{={A0P*cf0CkjhoCvuIf0bJ8Cvq z(>rHU#0FFl!V9a1-G)~t0T96%#vvG@hI+Im$ve8Bo^} z#@g!?ycfY0-Ai&CgV2qVBF+a``NmHyW^*uBAIPz8ep)zH4X`-Hsmu-oAP6PGIZFdB zahv*l7k>l>%66#_9!xCqhejwOfCq(U3gUtz5cp?Fpf$BAN6+pF`BgtSW`j%fLrycM z-&RB!bPg&z!ocTHMg3TLhdP&CJ`@_t`5oZMMckaW)=fkVJix{7Cnu5Xc~XT>m2dPr z6T&kZ-0f!x6g68x31};cZ1x+0F-DV&Vdwx@Ql46d=t2dg3Zy~5a1hGy;iS`qclI2D zj}J%Ff$3|yZ-&i@Vol8qA;4?kf<=eiXxRM_9uz>-heo9uuC*2%DXYi4G+Lc~KKd$? z+&~ponoi9v;MLo&)0I^(`1`$rCYTa6;=JVh)^PRr*%v*cWoI{DlHwojT`vU})oX{& zoraO2^9Y;E@J3yQyBUccV5uGnZ5;rz251+#)0kqJYI7R!h{c(uHG+%vtWp=?M2)rT zQt@r_of*io*%MW#jQqhBT@U!g)!xBz4x0M62Q3fUiAZ2n9=4xq zD20_0PL{EC&O0Da@y@f6FlV>~u@XyYm1If0ujZq%jo;{(WE_rRQ-*ayY_z1SY~-GiCl#G@bnc?+kzA-q08&q8Gq#3)3^& z*En9zN^CAMz+Oo@rdKl)HmAe4aZ1UFgHAb#Wr~%$XW{DjXxkQi!B|B^BfQ4-7w10S zPzI5gr<`j<7GYzP*1I`SREcIP=70j=20REN1zyHr?d}4Ux@#DErJ%(oI;csj&%`U; zZa8-aNwjl41s`IMk<^hyTW$vGEYkCnGmtxp6ef~jT=8dfk{V~*E5~}@HHYBk@<{+L z22J{yxByd)KV>aGa^@o`IKyjFCb&k{2xlm`GOp~l1Hb_e$v%hLwX&4qGDWViZ4=6H zVBeIWc(vutS(=91I;}CZa(Mxy%AUPq@Tl#99$=lp(?03wFN`737Me9m7ovLD4l{_#%+MVVKV4_g&%&}vKOEU$t z%LA=U&#+?-v}hQKO^4lDl#P;tf=iqz;^?;7dy z$~u|1I;BV=T3_M<_w<%VU|BpzvX^>edU^r#wnlGO!jt{D{a`XJUhH=HO}Y5uUIAG(5)tJT=o=IB1DPP_AQ{eIY$-}Pl&8Bk zq|LI$PJ;6|@4lxp1uWB;xduLd7Pf&O?@K1c(n00|p=*-G1IW7{D9uTRHwVB&Z_W*+|CiwFx8p5>`QXXxEr5#(ZJy#|}JQOAc2p ztc4e#jnS)phF1V>3MCb5rcoN1g#_?;I9k&P;6f*;i^V?5X93$4-Qm8lrW;wp71cHte zl6|{{4HbIU9)p^

      *>dMFFWtCNY(KfC|5y0m>#)63H|t%6wNY19-I+IA3G{Y5f>) zESbEguD`S=?J)l%KOLy1M$=%Hp*Mc zvqRHe1p=zZPR`zM6!Ssb!E7`mE;~b} zjYl@IE}#r5((Q#?$}nwIB|bP=SjRg7=JbR@7jZGA^ehG)7R*Jc(_9$! zPrzfJB$*BNiQHM?@cgxV85_#tV>y^S`Ck^IQ_9F<|hed3_QH)5J@3EwGoA5_xBv;nTpVMLMksg}C_kU2YY%(Ojloa}m&&IiC+{jCdW*eA#(v1JNr z>-zZ|b8u*hcV2!#&{=eopooy3O5NcZ8Sbg!t?;%;42^NhXv!3@#cWKZ;*CY^(TY)$ z83%OmV**0EQ$7~=#xIGZ=Dqhr(n!@?SNQzO((vwC~ z1)0TF{#<@g5EQYwo%3Bgaw7-53e(bU{%T9s`66hR{JjR9Ynworg``c!5tgO@p@7np z^02WRlQIg%l;6bM#{ix@Y(+)3JEFe8a_kfc-$@5`?50$QK`01^yyzJ*U*ft42hLBj zNSZT=xYo0uf_(DlxAqYxW%k%fP_@SU5Fk|d^Cl2m>A0P~g)}IkaY7ESpbU!oq+7C} zaOdSpNik13X4ppSFYjoh#X7m*H8VIPU2peQ0mDir9)VRRbMoVYiQjk24(nuIov_sw zSd1z>OQHmg6OUpGHpXh``1An_)hf-SB zwzNm@NfouxC0o*xDJmO~zlSezB|d6ilPBv@L^dU}ra+SYiG&KL&HG#!pxj&(%>Q2Vx?ACex z;bQ3ddE}=nl~Gi7XUCRns*Q~-D{_kOV5m+MY4N;LV0c9Wzm$GHSyj~b`jx0y3Dxc7 z-b`)L?dk!rzVPBxo32%WzO^mZ(f}CLc8!6ey@81bmCtRqXM?F5)lrV-Wlu6BS?r-m zeqxTc-}9BQ?1uoi{j(lJi2^xMei5;vz6JoO+8mhaV+dA2rnz+k@o0-s1X~GjZkt8J z9_eJ-bH{_#e%G;rE{M2!L4Z0)Egi0~;k(NgYHMv7an@_?BVagep8Zxs3F$npu0}=D zr(_0<8oKWlY5Q4Rksi{HP*mRpmEv74<-MMnmIgb9s!jpdKxBH9n}1OGR4~^JjGF_> zPe2%r)mSq4l5YHX{DL+h#{&vSydDb4S16`=F^0)Y)(|-+AZ>j0ovng;&e{O~{tE(M zb}jxr5yR_brPe;S6Jf-0>MzkS3)kZX0eqmzdvoeUmRc$lNQ(i@NEbDx_Z*>xOySW- ze&|28j4dMCT(7z2ROA4=r#Z^)9fTsrk#AbZ!R^nxysHU7OIfsbL*6}-K0GL!@+T*xaq=PC(K__UQ zYCO@vq}G!gnv99~6=u_o;xm7*`4;QKgr%WHv^gE}3oC|&&OqnN=?R71M45arvuqCp zseef3IlkFos@eX+x{yb2M2`Asl4sFi+9hn7>Xd_1oV9GNsgoSR z{poZ8Urwj^5=MyCFJPFO&YF$k=_hq{kdO>h#7bGjhSVg;5kq9|yOUL#;_y!hS8RZE zuVC)bygOR-`HZED$EJr60q!*6qFvG4G<2T~-23GYn6E^c`}kRq5j<7`XFZjDUn9#; zKY&1D{Et%{GlRo<*3tNZpRQLxA^qlcKp^7L)M(T)>JMVxIUL#Etq}?SHxBR#brPoRCk&uA(Yl z7gjVs2Cu)K2%qO=t3YQ##$w3}p60@cB+I&W*I@E%}8}ET)DqIb< zT#0f+ULZb*l@|!lUOcPt(ds_0@Wv-}Vaa$W6l|8}IpXtneo>uj+l>oEb}0a=5q%8t z4J-`L7F~0xJ(xFjX-^{8qY**Tyq>;ewU9rQw-iA+#i<1L&liBUDf@#?u_(Auyog zU~&Gf`K3peMX-wVCFmL|ZN&<==*Xj%go)=nm|K_K7iTw5oso1~DW_7scxgf(-7blQ zma!Y3YqLv;QJwa6(q2KV_}{r;8V!>+KzaTIEo0Jfd?)~*x6)oV_JX!@)-W~Sqh4O^ zv<;h6^8A0j5rPD{k+nQp z#!C55o2k%pAnP%ysHhHvGl%+4n_jKnD1WP7?%8p2MDnf8dmsE1GR7g;;6EZCef$Oj zbEkc$6x#C3S4iOQYWTT-t z6}u}+vk=SpXibCnc}jFTYc|<`w(E_EsdyqD0j02v)2q3%@YMzTwf!lbuNh`Bf6*1y2v%#+~_S+na7{NcMg;ae+ zE1dTQVPLrNxzrXP%bKS`0ivv9m}#%F%CKowBYSBArt0e1>FYY7Wmm@lo+z8=6FvoV z4vKPVA{;2Z00>n0)d{weW=ULMFfGSxQuQOyOjzn13 zwib3t3bLV*Qak~xFigz`_4en1=Ne=sm&J}03-Qfllma*Sv<-jN8eQo!a21%aXqLB& zn{&*=ZcN5NAq*K#^U$EuhJC!SS-jZ{l~g{jWd_D;@@yMU(XeN6pO##UT+JLtTu=v| zCR7GQNkZl~&WZ(T`8De7DV{H|)jd`wl{#cN6H|?`*z@?wmyFu*%Bb^&n<)?gK-PgB z8Plw|9KlFwZKR)EI8($z$5nT9g*(k|Qh&FH`22oezNu_&mONN}Wm}n{f}AAM8It-k zAjNDZf|<-yX|BMrD;j#)9`d4H_eB zw@_YZosR25Rq<7rO;M#q%j4`~Por14&RRosA=zKZ4ntF77r63gi-?mV3z#aCk$fL2 z=uR@ny^5zFo4pwtmLEOrJ>B}e`060%wJM)fXh-yt<{pSF5wi}GFj#pdgVd3(XRy%k z{I+b86KE;Q6)i1a>JE?dmF>}Zqms*rUN07!!b!Q(f{qZUlY-*IL`O2eElYpr(KZPt zk&{7PLfC5T!w_mRR_#FMHnjNP2%w)OG?>QOjPh_roL7AO<*t&v$o1ThBrm{Tvw55_ zTJ0Ip`lO^c4jWDAaxL0ULbIA<1oS37((8_+TWj{35`hoU6=x@Jm4si&xR^PyyNos< z7ri=D-j`vB<39dGM0=x0fimHqJi3?>2qlLoU@yuY4YjGM-hq0c?m@5}rT}WTG*QsA zLuTq#Vbz_D(GH-{F0IG}CCCpjziJx}oCYJ&%Vh|BGO>*Irj9Gb#tPppb}x!tXEUAC z8p@V-yJp&0^S;!|4LQN|*Vue6KJY#uaHkcfMUaP=29Y!`2AC$J+JDXMY-Rp75M-5HCx*!q54&^9l?JHL2_G<{Y1Bym z>&0j2jDcJsw<@T!`LrjpQ&^Kz?DJs&GFdO;HzuMn>BMy+sTc14sz~5UOgvAh071ZwTq%xH52gsu zH#t=HbppJoPAbTXzRVzL-^9L^1_t&OD!YLI=IO9ad1R{YXEa``cSuqq7Ex~QX1+C5 z!K1$2Rr%3qqbQk*R4*Kqf?)cB zPLsr=?4{Dcj)&m7rT39X^N3aYfn)CXu!e-(ByR?+B&N!2Q3;ynvgDKs73mCam6BDS zyefR6%n7&U7^45F-!o5# zC(^dk*c2Rv79Gf5Y++_ps{1ewXE^vgXGBTIii~@$3Vam)neR99@kcHemM}@_pQT&{ zr%W^yjvV(^t~KU(YQkX&fY4(qft@=tqF;nWq!PHMX#o|=nC=-@R^@}NMM>K*{aZfm zfhjlrQn5vgDn1qPMD2kYmwzXq?h=#?=#6DhimzJ8O+Z<5myK(^kYqt`?+?pr%LSSM zfs|m?u3~bpd!_SsXt*G)`AH~f0a(b&t=!v2u5Y@)I4Mn0UbSGuMf zf6N;k(ui`dXV6x@dCNT?wtzfD;D`u{a+*Etfr!*hN22oHxUxjh@4nzB14#|)AJ$)C zD|1laO^u+ggqva)y~}Sn>ZQ6rIFKf{K!%v=jrZ3#b^uUiRlsQYjxo4EcsOD_8WF*B zgL>JaGm&UjXBTpc4afpt$vvR}$NqNW*5@)R`>>#Vk|!CU5qGy@p{UhFyR5d>fDOuL zeiKU*FNGH;Y-L|)tKgB};t9vM5~#=-xJ~fK)G{}r-@}6uVAj;3zaAGnrMf3b2v-3) zdUY0z=W+yaqpMYZymdBq^_yhQYKGX1mchGN;W%e7K1;U<=8dw_ZDac$*I7*K?Co^Q zun>)!JOm7WVS|LI;Bdaky_QeLr14`gBX1had!ar7GGoXvzy{4v=>rluJ;A9R}L1{ujr+>Vhis@J$L0J~B>z0%rb9)pAl?L24;DC4QOOz4Enn?#Nwz%;W&_mZs*cx$!A!6VnWi8i^DD}`u< zE%owxrDOIDH6|h!yy(TDsYt%0Z!~;xt#DLYfxWG?8}TX(iU5}~*%Nh`1^I|IT;S9R z-;BkIfFeQ401*xzudp{@8%hxX{CjX$7Z=9yuV2F# zb2?mftP_EhmXQi?wc<2&Y&8>m!N+50&KggT zl5Z{mpD`TrN)Ims|8#38t^}-bj+ENfYCG}Bwy zr&wr#hoRF2lkUQggyb5W^qqd$ebSc3ual02MQtrb*<+0?Au&k zxRIsL$Hrh53zYkaMl9ubDN3@_ew+cnXto0$eBeVp#qc-(GV+D06xN66p(2bCvM|R! zy)8T!$()zH2!Fp&%Ep$Q_(W8V& zAy;!cVJ585M(iru3zD1amUAvuT%djINe~A)nPZzMrhSp34QF?(rl4+>Qvm4X z+6$%9!6!k>quAS?u!cJA; zR-{KprMn#f`HEny`gM(;B;e;#q?)wmGgQ#I0;_PoNcmm;G|Y>6pcT@<@JG2d8S~G}ejB=V) z$WM;}Jgm5 zj6DcS9DiOxIJ*3#6+3uR5zIzGyJuV^ob5$sz&WuR+&`h198Ze}3Ip1M7`a1Csa+nW zXmD6pOH*j3m4q@p`xpVcov<-T6}*7FPN5*cid^EjRBz~ zaS)sKEyL~JVR049J)pK$?nHGf`Eq#z}b3kctQ>y(o)qXl`OzuKWug~t{Rkf4>REG0dBeB*;c+@BXJ0V zy9(s2HUG@K*sc3?gHJ<`mUBSJPANHYM|u()$L^7suA65s*mXrFpq^2WMb`!4_1us$ zax33G>9gOqZ)CG-&IE-$8L8pxblw{l(<%V13C?0Uc4-}_MgbNL2)E)3nj-UOHl_tp z3Yiswt$Mq_yVrvv4K-b*_zG%AHGCTh`?QfV(Eb!$%Q=VCN0}Z#55gbFi|qaJqO<R5kUb2VgQ-zi9=H5s#UY6*QP)(#|@Zp8Oh~$W_W=k^;XF28} zy?w*ld^pxTU})^NeD`~3ZK+D-LhjPJUZY#u66x+QD@ z#eY^4tyKIy)$&2J#dRArs5$d zya!q*d)?kR!DnG3s;kozr(rV5b&%DbV}Cri-S{*H_2e4lVFySilQp9hM6=^@V?>Og z#1mOt^`Y;z=R0OhAzN4Kj){i#{wl$AUYwcsSPh~EU0g?FrO>2J7^tb$d(v@@ z!W<_KZdKT?E3vw>rHzuOQSSBrDa9)66BaJPqk%NpD`bd&jz$}@iiDqX!id27+sd$N zC%FaLRgm}Tka0q#JXEaIN+=Vsy7(>zcq@it{Ok~dd6m76pB=d{J%)^^Ot-%~kf{OD z^h}O9bftU+9H-mFht6Pd_9kY5Wyb`KGZd~BJ@Hm7zgdL(N~T6KF?bVs1on|hY6nEr z+%$$VHnwjD&1=U3n%CCDd|}Li^zq(k!X#Q->o^qX8JZ?8vuWpp&~E>^6#UNWJ5d_N zPLfV0K%pguxD&kvJxAU&ejc{+r$AOdh!UxH8cWKrFSHU=U1L|~8VKm!q-%OCA#9J1 z`Oc}i7q9ZEpfllUuxvahm=v`J4u@Qj+kCnv-q%tuY?Stt;g@gTp2W3vf|$C)DaS3W zLmITYb$ z2g@vkJ8`+ADSF|Gj(MTCo3|=&YlkP3xgM59nVS3G(N81Gi=TM@L{dynZAHp0!wdanlw0I zQ-^u*a*?3rps0gE1)M~In0Y0|2_UUl?s=9}B=#9X(yqVcJuMmNbZdEjLr^;umOk+1 z4XHA@z@^t)4lnSyY3^>*GECF_dX8GSS{1qf_pOIr#w3Lx-0e4lU9o{fxVxTP=ABQ! zS*Kl)CmuE}9P1WG_skSWbG>l25XMdC!mQG^ezmQvLg7E5trc3COSq~5lilwv4acZDC^$(t*Xz<2LM=+|| zzn2#36O(=XG!Nzf!Y-C!1-pb5yPAimL|Akrd8so`5i1mR5o)2&GC|Q(CV#VpU^Jtt zlZVKd8YZ5**@yLT!+YIWRyuXzZJ z$CIHBU1dySvWxuIw=0(=nKy7Fo&`#+j`2_>$dWVBrssilDU@{G*@6mn?=!+6k;JxJ0C{x)&CG#cUaP zDD}md9N7j9su*oDGe0vcSgwht|5@`F2k~4C@0hpfh|Hh!^VHJ91~p@XUEwQrKU!V* zLOEn%iF1gCS+I;qn;orPjXlQog9}#1s|9wO2UHO-ABG(VGcL9ll9w<@Me6SzlAepL z6NILuYY|z{C#U^hGvvzeqSJ0hbs}Gc^_kFV(ufft{_4$S^(n?ru+AC^QA2u5WKDQ! zbdM!0>~_ck`=T-xiC^JD&@MxV@+1?Z4LJBLStL`Vj&pxOrD_<+ArHcUJ?WE1YTdV` zkH3@Qewyx8%N4ro=UwK*v!zDN|L_2k>7Pa$lmZc`9niOvQcMml$C5n7mkw_FW65ob z(S=yU+3xJ>@xOG^avyt$$pbS6rMgS!7k%~jJW+!>mJb820?rwdPu0>vJ6q1vukSWV zUzj$#FoI=5qzu598c4Arb>YE`li6)czlmHO3KGyHKF% z>6R7SWYzPeRNyrZwAcJoDX~$7VO1Zn@K*}U%4j<~X-<8sMn^KUiemt#eCMUCCWJ+% z1NDyXXAw}t)++NSp_Yk!gN(gXzjT`ngFtr+3o&<@36Qr1hE21M;%2Vyf`cHvfVcr* z^9~-NIGqx7-OU!+mr$HuRY)|ON>*%|95{&isSmP#ntz}0l7tQ(uRhikRq6N*YitV7 zUuwu3=Odoe$f3i9uxi}JA9Tb~+pQ(7Hh|nf-HNQB>Ww*~@E#{3L=0jUuVZ&$66Ab3 zJC*H~Y55|_KgWbhmDKekqC9(zab(tI-jEvVl}l%er%qZlX8_(vnmdI_gQ)z z(xym3^HWs-eDUtTCnqCcTSu!`Y`c!{;JEz5_cJDkn#m15ZZ)Z%yv;3#;Mw?ErDzv& z7rm@98rjf#qZPiU?w?eYg(rhg(Uz`l);#;bLh$y3xg}JtBM>Pw2t+T`i?l#d?8fGr z=PCDxn2pa*)>@`7Qb~t9WcK8wW(6Qd3Z{!x%`<>PHSc?O+S-ePF935qKve8u;VzY%P*Zc7KA{joSKjz-%jxF2Lw^SOC@CDtrP5$#S;lO8 z&gj|@9}H?x2jgcBf@GjRPlF-w6N+x28*%huEovmy`gHNzGVaN9AuWo;zP7|=+~rI> z_Q7ny5olz-`wXG6_frI?Au)0A=2)y;ocp1$Yx)}1BarS&{S#F4&0n}M+6GjU+$x{Z zLEoGtiO7u$G6g>tuhs{$ohtf)c)QohS8Hiz57S#UU-b-K4I+`0c7B6*Ohjz=`#j5! z57>q1$n=zQQ!M0x7lgrg+-gjW=(pitC4t;|Esc(+9x1!cx|*Pv54)eY8{ za^FQo@Yt!5N7DQw3F;EISIsacPCZo*dxHlUW}q7?d#%anqeoZJz7ji5t==dRnY$oB z=30HrqLzkj{4Qq1WL$*huv$*x!xAaRD}mITCmf6IvlO;azPhyK%0g_Zz&lwVzz0rU zP;fkwirS5crDnwGFauTL%skmzlGx9oN}u9#w#n!>_4%gXg$BUqTQ>8-uYN@V-Pns1 zrA(QOU`Pg?(J4pmSrG@s(m1Pz+@xV`Xx-6Y7!UmtQO@#3GA!f z9OZV+-X}G}_1SxRZt-kLmI8yI_CU*|Y?UGLh|k)}n02>oI!;f^WH9^t&@B^lkx9hW z=|dOEj{ax*gm+3a4L)?W7jXIVu>DV=W}aNLG@!D%L9iPM`v$>t{wR z&k}&^UQ%?uATGfftDag``G#BpjFEnt`3i)Gl?S@*o0Z4U!yWciy=sk7WF#rhbB1)h zRcx7y{nb&T+2=O!C%&>D_8jeChL%${n;&}Vyb+QKdmrCOsrQ#21TpRU9^SiiR`eai zw~Mc7TbhyEsymEb%BB4%8zwlMImX}!r#B!WB^-Js8Ov6T6TzxF#QhIO=j@bW_!hxj z1uSr(R*BFAKF;U(YJT*g7g@acy#+`?X8S3V!GFsYe*^heFPuO65=CH!B-%33J+rvB zLhAC1$32ps+~Z3YKc1Edv8H~9q6J-}-2T)#ck>Y%q~nmsB7-hrDPo{al+^I?w8=F+ z+Dk8SAeP14ARSzCg)>;Moj=(rJaMfvBJwDm09^yN*YvH$(LK{Dh{tS?hy@e62w-|% z?RCLLi>P=+q69zKSkx1ud75y=0&tq@CAoqFc?5i#EZPPds}HDLWl0?&T({fL;>)@- zJQ-KqL?WakNtc6{hCcIP3Yd`!0T-1^>5prULv?}X7JqV2YdmC;)uIPr@ege}q`}^; zIgCBu1t#k#W^Djp?4t`yDu2hZ0Ra|-$O{^f^qa`Ms7XQR#knsacBw;S1}}vJbG`23kKJ8jGc^c^-q@pBC`-YhogEM=f0U!{7!1?MBZFwlrGL;%uw<(m3d^b z`+^;FZ~{@VZH(s$iaNDvoKk(RMj7@l3J~nw(w5P%6C8~_Iexq;#+Miuzx74mzZ|9q zaOfE>uF?oOp~@};T=*ERh+0o&1DOfk`B4|_<7D=f<2Zid8Xs>t;6y3z?JUPDvv`wO z!Tz!hknwbzX5MDOK>F{6CdM{$?5ZMi1W))s<>`)$&2qN(7w4mzIC&nHY2DgZ{QU*K zh|Om>S(!SgPhOy~bF9!Jl}ZwuQTj%hjQUT}WLeI!7m2y)l{|FCBTr`q4}bK+>z#X{!+B2%ELc)0~QtdA|HVaBe;)5eq1jlFI06} zx==tW5oQehjfpz=6=Hcj0TG1JBKH;{6J@Ub7p7Jl(_=$Rd?NI@^03OzK#Ms0$HPwR zM;r0D(&#;SxNR7zwxFE|>*b>8`Ux9Fnpr)1u^q1g8X|-}kfeJ067;3p!`UaqhLN^4 zSui572M)OeNHn}luMgO)lC4>NLSyCb?lD{`h00%3TQ`m+i<3iWwqAZnxXF?rP7{gD z<~-KxT=5Ib?U%CJ=Bk*P`b7{>1Ai4Mf-fVYOjQTO#w?UYLf?cess5N%ubr03>VeV&$jnDO~(@hOWUV ze%J(&FJ4520-Ftarke0pToGQSI(AKB%!Q#b*5>BhJ70_RvD~vBK*frXKyCMGXf#*E@y1&e@40F6j{XiMM9kVayIZ#4 zi)230wJtcxF}Cla%{tL12%+Z!vVjEHa)TXM&kYE!o8aYj(Q5&v7}$)G2m}2#TMy4o zb#0rYMk{L3^Hj>$sGV%uTE~q}HL`;n4#FkV2&t){JGzR*p|Ku><%3a(i`;i}UOF7? z1opV3%nVr%9|yDuK^)12v<4sp7bN+Ml>#Or&(PhT%L9C2U9p2t zLt_9bLDknZlcI^%#|1EAu~L&t*3Pj?aFd6cIg-SVc2lzWd8-G58CMX>R*LObL|<q8yi%qewom52plcK8pRtt(nqr)ZQox5?Y9^FqfOna92boYF#bpP8p}*;Y z#$wUNd@sKer+!Jy=mvaN5J<_UlvwyTJCO0FYQib85bcr?2>d{6RItRvDv^&cEPLc+ zQ0v%goaVLxs)4w-0ht61&bjiS(rLy)0vW%nB`TgnwlM6wSu;Q4jR4(+mYz~<&I}TI zIy#R8r3U@f+?6wz>@~r+8%eHInWeX}xzCj;TipZ}y6(BcD#6OKTJosWV~tJ>2~hLu z?Qsk4CdFDyLpJ6J<*Z{hhuiba#fFnn-<3g=7h0xvG|;RLm7TV7bp-<`0w-tmmos_H z>B+Ah^s`&|Ccp^A`wJC@(H%q$G2mLw4m07UX1$g&$-uTFj8NPSAOtkH-R7NE&8jt+P zQy<|_m{Xx0%O641UBY=vOcsN@_Nrl7AOuJatGU;%>e&iuqUEv5IRh_j<1+tuV5 z_|uSYI%qZb=pOXJg3x$uG=0W{>)S!TVxyi;Z0Zpkmux0_c$Bn8ZzYlFdN ztIV09uQ+CQK$=B!f+N}|@zRuV0lCc^SC0byEa;=oRCmS#I1hOpDk&H<_p~V)4i%BF z>zcXx8QzBEp_yI7QSCj61BLTG)1RO`XPvttn(sE~YG!2#Rffr4kBFH2WsET1>nfUx zpqKs~YQuO38lOBTAeaLr6HDCDJ{v5Hi&MS%hDGU6PYUKe9Y(=w-s_|*r;+~~F;Y7s z!1idEW^rC(@lEZ4-f3&4Vt3@v?j0#|XvuXGeLxH7D+8G9_%l%TZP}nz&(kvF^}6!b zjYtrHV=G8}fm&s$on4q81~S0#QBtohQ6^ts`B9N9qvT*9w}QkwBp7`ftCj#A%FV{% zEPoD%tKOB7l8fop4xSZ3BqPfsJX0C%6zdqG~0^uwx?7P<*ZP+9{tLj}?0`9K? z4-e$IsrM?b4S(sQ2KLygR`6imfq=jx@k(|^i*=9?k%unU1{T8Gyx5D`jlsy)tA}qMqf}PZY`!UwsLQaBuW-O(={{_j$Zg!=CwSz|DDau@Lj0{wJF#%hO{2G%EmlR5y8Yk-|era^K(D`g5N>#<${ zBG<4iu_jVevLGQEW%CsSqkTOinq(5Az(V299hBm<^TGcrtGL&Oj=8+Umbe<&C{X~s z?B1iXs}KOi-jpE{a=f>q`zDDaBB_)bNn<7TNWCcbxIX=Q*B3p@%vWdgdQ2|f(m4o7 z6l_`l@gR*Wy$yfL!NCPiLlC4<1`bv7b4Zj@{@Oz|xtkQr%U^nPSBJ_Z+pA zN@-O?b&YdFTPI<&l(0LobO_N@1XBst2(uos9A(7sL(42Oi_oz_c(+1D56gFYdDhlG z!!OmHAv>rA$#}8%jBavyH};G$rsu?lZ~|acsSG(@h$6T=fJVdXyJDLdAS*$TKu}Vq zw{`#;GJj$!8c6LK+|SZ{si-GujVui(6Zu+!LfiQbQ5J`Q4&(koy8v{=ziYBOpHYhI zGf+Bkr4=$nmt&#t0`61kDA&=|JwpyaFDD}J)vz@ibEK3<142jm7f_^OZ4?@O zq}bKAZDC3d-kX@Ktt}6TWgVk& zR0nxc*0wS#OsSx)#K}0S|9eg__N11%RjKm!z7^ivSG|RO+=jV_=?HQ^%PJ8qY4l<_ zrX#Tfz4b~kSFHMxR~$2BS)VP5s(~w$gT37yTl&>y)3KVMV2M+tSPtHW`kNlq-m0%^kU%#d4L z($9vNDl!tOTd|hNvwpaRkZ3_0mG3%6_0hcq`MVzf@9GJjpXv#0$T6viNhUI>N}5Oxzp9!$^f7@D>Tde(}np#IN< zdVh=nWPeTlR>2Ih2Op|T=bpd&3X6yDWy@p2!^_-740lVv+OoHW z1!vIYaxQ5Pa_2pBA2rKDP~EI|<4Zz>VWC;omOfvIfc~XCVuY@4h?`)~y!zkylv+5` zCK1OHF)=5S98s+J@WVY;DDiwM-Yr$#<}z?;y4LCwyuG9pbB7up^t+qWyeR~VnEkRq zBo>Hy%*^XST@j-Ug@5+`MW=CUJbtNTzM-V_1ul z?njzyhWxxMAo(e!<{QDX&$IxKa{xm?yuXd;ung97aChS;7Gxn36(rnnDZA84U9$A) zrO6kQb=2P>uD6ax40DhrdO*+qW-?n)bs&l3Vb_gx@_jr=2-V4?Cc1ZO_ub>8J|GU? zig16nFPL$D_v3ptRQm;|qr5Sc%Jl+DVAA!-LF29IB}@dZw~I3?o{XSl^Vtr(1lR~o zt)ieZImLu>7!7zV>=wCBslPNP0&R0WMJ1lDE2GMs*Ge~69m`TZqDfOD4uC#2GgR>d zlX^((ao6(fC>Kl%z7>asr{~&atEjld;_a3hZx!g?FP!swzt;Jnl~QJ(8JI=s;=6^ToMgy@odntln^e7MSWKa zezBl{#3y>?mItisjJbKFn<_g&H(;23SA^|-C2+NS?YX-g6`&gHxihabW|8DLNxI?= zV|yF|y_l)9ii84ywc1v8X1On<{R_iF_q(H+kG4?cK$S)GTA&!r*0awvc*uk*^ zN_r#bd$#{ru`3!O6Jussb;Mc=9*&PvfbXKA+FxYxpeh5VNP%c}&yrrc|K*w{s|kNN zn)$9^?c1=ZFF#*Nt&j^_Phnmh50oNEx?DD0$&pTjz#Fp_E*C662%ZivW%F^WT#m(U_h~+XUOlpepfY9)OdUR?J@^b6zT3<*KL-OjCgAX{{atYQO z-}8C745`+O`2m0b4!@r{@C}T;TJYRC^Pa8^c1vhlCfgqARCIkZ#1m`ev)A#|I{YZX zZP5V^XzSRHI1cq$Qp4CCSjosv$Hs2--bUP(-R<7aGJ=etso_lV1|fjf&9XH)pb2RX zR?a$b`*t;TTK+86@LQIF8gc`310IY$A^h9}zE@AmpGI35!^DZp8f{H?6Oru>L*BDE ze&+qx8APmqBS~rcU^M$*@DF%edAb0qPw=t9o_+UI_%ddoWr6SV5y28Z-SyIpeC;wiEe4^y?jT?A$j1S zz6KJqu(ZLc?YMUErwTBGu|t?FF&jU0W?BuB%9_W5CdgMH>1Z2nySm6&U|BQ4NHJn_ z9x8!Q#(X>~#m#}A2tIw`j6HM>eOH?MHKtsk*l&kqUlpmof^Jpc8@%V>>3BG)I))AD zCb(yxmT;zM{`R)^Q(Jy1>Fud2vzJYGcPq%St!|bdF6HK0)f2)z&#<h%dXVVecVAnIyYh$Mm3q0l$yfjkG((4Jjo%mHb?;HMzjt+sKc2vt^fSd%Jiboeat zcBRAKArb$YT za`8R6Mj-aCFq*1kM7Mv?dDr$AS^+1z>uUi%7ukn#!5Bp%nvEsNNw?{(gEw6(h<=xt zJ=qbP!tz!$2&_jL)N0}=O5}E6;|ID+sFPSgN;V`DPXb2DJu>@N>r#K2y`Yq4Bd)#} zVX{tTP|OtBA{6Xgh+L!wGMADY*8JyeAeK_D!T}2gAOQHFlE-pDk4qv?eh0X9JYb&q z&jOu3g%Ybx-Ul3)TobWf>Lc5&W&GNORhUvCVAH7$Ya^W1j^evS@C-(PYuC1df|UL1 zDY^?yQt(k1tm&#-5emG?Xdk3?_vTmYG5_e!>$uwzMR(E{=NZW^wlEp(&xE zJW(L`Bu2aw(0FnCoOW)p$hd*GvO!%Hm&!*7u?t*Fk$6BXd;)&KVp4X6RRt1`B+EI` zzC5%2?b6&F({1q~Du0sJu^kb)fA~e)9&6Zl&rdYHxjD>^sLmEFbC{=c`eZxpVL{#* zKHIVJTWth`#8m{GS7(5hc)6D-@k049qAz%DQ> zbWv18iM23;Y!`eUhW{OW-0f-RZDkYt%`F1`QLmBF&YYr2H7r9=r z@L}LRBGT1lQ!ul=n!O7;1mKlj9BS56Oikx>zc}H&A7H-&m-dCa`5Ltv-h%nBT zl+kJHvif_?XpTlI`0lU)!E#`X>1AlgoK=uBWmO%^@H-eQbZO6HqV>^l9QkO9kaT9SwlavR2&YNx<1m8*F%Pn)ZQCXWkXUq(YF>sKJN!v1y%gz?%qzJe%Q zj58m0tp$?n0NH3&@50{I0o&!X=@jFOMZe!85M1IVX9i)@0(Y89qG!$tpHbi7TimM3 z@iY5w@)FcEK#5Wmgc!QIRu@v=qS%rFEQmfJ2uZ<^ z=0F$V*^zvk*(1uJjT0qhW@-E`954K&$sP~rqNDiPtZPA`5!N@G711zkiA{tc89E&MR+I)~?aPZ117b8xkV{3OT) z=IV`G>RddLp#?~sD$SU5gU`~!?%RdwN{h``lR|{J^T(cOb|Dal))gkOiuiyT@}5A= zPv|oECSiQH!BzomIkURWyxYz+Or0vWJtR~3U=vg%MH{ak zX}LR)G<1|i$#M}^PM&LH3Jvk*Im;m%Qwi#eyMCaXjX6*Is1q%JZVcW9H1zbpGiR4g zdY+EC1thkS{!qJD%*G;hraPdFjT-J2VSKMsBtkj{POz{UM=yimoN|J!l6CkyWmC=E zsjm=RpYdtte@6)s}Lj}FmLHGh?z@J`+nEZ4AT!@5aK8|6;fj?G2-wvWpeJ{Xe zuYNE`nQBK`;T<5RgJ#=34X=nruxt@i;<;p=WgP5tUG}QpLFzjO+^?`rV>@j#Qi(wZ7cpQ*zywnU&qe z!Wqssm6D*oE5-)ss;W&>UWDdYZVF5^7BD%1kNKz%6Afk7X}{FV$y({PpypT`UMcLj zJngbusJ7tLWf-F1Osql1++oVndhvc^E{Y$CfMt{me%p`eAm?$(#mdfj@W+X4RrOr5 zhAwya1XEG6PUzi)w6|HXYPPmf$6T6kx(}rHEYj6rshH^hO%j9$;U>L)ontiONQ`>o%CaiQ=tN zH@Hu88!wF=5T;OQ6Q3)WTD>Nv!rDmGNJ;-bNc`1W@gJSYdMGL5v3Uyv=Y(i767pwN zY62;RztYb>kh!p5Z*HfTrI|!amsJ#@$Gkqn>Pr4&2kHd zsb1)l5a*eUP^0!q#(fdUvPKeIVH1t3G(MZAdSh!m5cP1WRrRNaX=j4P4jCMG$(@Ha{A?g&K4OJdoGyK?Htq>J z@J6#PY1c%PpvgIThVci77+ZQNGv4j5KdL!H#JU{mur1-qEXwjx;(eVQk{w*V9K=eyRs|bk)dE>HUo3T6CLng`URB4ch zbe&T(BTtg8yLpIUR%?e+3nu|ui7T}-f;Gd?QWn&yVT^;R!9wB&oHTrbWx6z6XUhOE zDdrJR$_0da8uk00A}_TUbx!PkO=74XQIS<|O@@G+;h`i=2?N~GmbIm~QqGiV(dRoD zU2g`t>#b1@Fbc@77qTlK49c}GPUoJbaR{#g6jNS46Zh>M4BvldGh)+(kja@^u)~g&o)c8pJG4=$(gDbv zZ5oHoeEz8Mj4eN>C@cv(M%~+$mT(kO($-H79V1dTlPKG=HzZ@~RME0y!&5r06u@eVR)u`02>N2k&(Z0q>XnI%I}SZ-}s&8*vg zSqFhPo(zHuH{x;S|u2+HQ1c}o$Kg=|BT#V`uq@te=fBJCYvPb^n=8ot|}4*=Hm zh6-GIH#~b!_~&jTSAf~2?R%W^SNC*Y`HkAAa$jU3u*=}%KYC1Xja`}OMM)ph1HTcL zQm%gTdRSL*7f-O|#Y{jUYNEC>y0_!S@AZ7!zj_D-T9hvPhSnY~9%7FWZ1P3Z#^nG; zQ)1rsrpP(B84_U=UlfHc#Q}aOcADkuwNN8_eF_DNj_pW8~W0a0t1v^Ls&i>Da#$c4Yu<`#BP)nGx7$|KtxGj z!h``3>IF0GBwtit5ZT(zz(hu|8gnNoQzLEe_x60}a)=P;g=H$^dFm+*#ysm+mzM_C z)9hsH{>-rC>p`!XAvE^HCp$b3lpTLIoP^tb5rVj|xkK(7s{DwMDrOWoMrQ$K8YRVl zJ3!fbq$ctNZ>UCRS2Dc|2>j+UWlpCc3i(tOE8`ru0`LkvZ?g*u_wHt6hu3}TYQEq! zI=*&E^kL#R7Le6eW_FL33*Ax<6Rxs=uK7l5 z^j%C*h!Oj2Wsk);6VE-`CSw<#larKB7e3 zEMi|t8cb22l+MKny8BcZ&F4RZSPBHvE8Nh*aiDbcOOSK!tJb||fosEV)^tQ0F6jbj z$D$U=op|dMN@JbvM*aCXh*CRLg&9l!JrL62U3oiu~`FWAqtABLBgoGNd&IR) z)7HIUSo@8rL|{r4176e&bvp*#cPwDLj1A-m1c7=cm%(DyeK>9nscpg=WU(aaZZTyD zITv=~Xx$rk)kG^&zf~RXIOe(tumjSD-h`&J?NxG7>I3`2y}Pk-G$F}E!{!Q5lJ1yS z9MJcp=U;!0CuT<5?y)CXbwb357guR8iUemL!chnHfF)R4LG?&$sd-g;QvnXvI+}rd z&AejHb*3ARXa{wr-1-1sffP*z2*XiemUHRzqDC8-rK<)`;O-~^<%sL?kiQP^O^=Aj zUn^ZAEg4MGrP(=1D-A0M#sd@6jQK{-IWW1-*C6hJc?e^!lDk(%?l6&a_=`yr4DZ1-W8*DANK7j*pLloyw6`^pt=SU>hbl|1-7S&v50-NPP-h#~RHpMt z)*(+VkCK{{f|B_xrS`ag2c}UcQ)}*?Ndx+k?D`0wUs`gBK1F4YY;sG3wib5uQw7KE z=HF78FBe$Sj)rKzYhJ=HlcvTUIM{ zF|9f>gkb8^4_DgcN{)!7(Rq^z5v=zuOpNQdMSqRD0*z>Sw(;6&bx5v7iJQpDna_w(97lmUfqWyGSz)UX`SapW+q109BtM^oLBb91Z z9#7*nh_OjKa>tNc!j>?@{Ghd5Y$~au1!ts!-v8K~dQk65;BK#AqC+y66Zz!9(Ii-xtDby3P>9?t^!Qg(x`(+til`9j{?3bYa|mrpe!Cz z@%pLa*vid;x>Xj%YsRGkP!-V$m5w=z!sw6`+Yt{FWteQ67s67~c*e)d~d*_A2<)p9i_rtk!IY@_cU+dJ$P+ z#L7cOKFNiD3AzH)TEts+(2Bk-sQV3kFK`!XT~SFSSclv&XGFrd^? zJsWQcx#^9RshqO$K)jlnf9v6{!mcnh0~`l{F+2LnkBgX~&NdzHExgYPII27~4f$fy z*i>H5aS8>2I2>hxiV6W=0Ij#n{>_fr!3)s5TH$sGaP~-AFa@hoS}FMCr1ZBX#_IB; z@5e6JYlQt972{FU!A@&A8U&*U8QaGWKQrVF#J#Ct2%|&ZS#Lo#7#wv0_`_`0r!JQw z!KflMelS;P6>8*riLo^*PI8V6)e;DcUHBb3>lP8T@d_?TBsvVJ@QQ%R9B{W+y3Ek0CQhY$4wU7iA(cTjIMhA|V)^Z<(w(ZXz|m-^DBA@g89kRaCS zEQ0Y-ijLnf&-T(AU`#nx@T%9j_hCIO<&HJ2pr@a5guSnKokN4()$jSWsV{E<}&wN?xyAvHP3q+;)0)9-hI01bHsd}_73LB~h7N3dqC5=xBBD^IOfvaaNX}?4OwAuEl+{sl=KaIHx>8o0avp$V^ z-JsCjRc_rpGd!{IE{F@Q)sm1j=dOSX+63fZxrdG28bk6`^B&|e>8XH+BTZZbZ@5K& z(+>LzaL%Io*8&o{45g|3DsP&TyieGl#pU+Y+o5BTCYpHnEi%Mzis*2hMve57+bSXE z4J$gU#4VI&Ofkk~#yb~N4qIdVDBAJ_`FO0Oq_T2^x9HGybg_@i_F`AC*4&AF2HK&8 z(TziOV@TY~kkHYR_ZSTpuahsE9xg?-j5!>avLeQFJmg@cD0q-SPRHmx-3YH-&7P%h znvP~(m7f&w4#0mYKH@Wos2yop4{v$5rt4N9za9G06DPKgMS(qxvIVr@rR_>vUOo50P>)Kz_6Et{s%U}lcPQu*p3}t_F*^{~#4AnS8M2aKh#-P{%0W-S8ur&S7 zNKiGXAkKn!<&PT|y^15HS9U<%0%>lVZ*QbwFl!uzMDwr)=|W*c0>52W@Q6!~$<;r4 z&eNda=2_?%&zj96vx1|hhgH`S`y5;t5b>KjOg{V-HvVANa6a)~Z)2S0ETGs%%!B%t zyPAK_IARy&&1I(Tw55Q_xfa*Nz515HEgat%2HvNnI-_RRBbN; z&+0c9rTrx0E0!?tOr@E0@6xapcXbK5!spIv` z&RLyN%j0$$>mHL~I)gy5w+$xcw+1<}R-rTz$l9#CDuM{=VPu9A#7A!ye;)r zsiI?JoDZEOUCWn3B_s(cP$G^kFmmvd>q?xBki_)KmZ_FMy^hkvW8M)ngx67aQoGto z0rv4pv$ceHVr8YQ{EQ_3T+~ZigP@&1hGi)fJ9&^*}t{pMAr zI$YECmHXw8zSn)dHKCuA>rh--fh`m1Mu41|^OobpVW6uXhjm{qKtzQ@uY6BLSDDW0 z?(1lH9xvc9Nrs%z+8QQc-qsB z8Ml%Fzjsy{JD75W?d8*(o}DC2>ItL@Tt7@Y_!p9UmE-3p(e$`9?6!~5)A5kVyUla@ z?jAP;sjWvCijgV@)yM~&gAw#KUnb7X2Y0+))DISLGWJYL1Y{;gu-dNG1Yhc(u-boI z%`fa~h-)WGO#-9Mx7Ke{m>y-PEtAK*k%I_)ypj(jb^Iw)$A_(v6$G?iX|k3?-KXSG zPH~c0hTt~&zY+;0k0rQdVp@nvTUqk8G(oyeM2JQmoM0STVYihfJPVN*(IH1t<@z{A zW=^r~bD!aiZ)%r{?4>bpF%qk8jypqeGP-zM*e@}nrVnuNFbrG3gDN_d`qaBhFhO0c zjf)7_pD9>G;7h5BPnoY#5qYG_0}QOUPm&~JNgrN=%z}$9zRF)8O}xuSjrxa{v&o7T zNk~~GRfb`KW)_ycGp-igVP#g8mP-5MUUN0F-|?xb-_0zgY3J3YPc=aV>=A5BLQJ5# z)lEAS{#D9W0;PKsKqggv`>4gyk|wO;Z5nygzADURG=>Yom)oai{a;oHrFwd7e;{?;1R&}R07+y&xmSDTk zs0K_+(|YqYSPw058M{2O(}IVokKUL>3tEZYeE*OJL;$LJETDZ8;Gkvm4|!;r zxGne1D^t?X2Pb_q#^A%fkJ6GuE^*BQ?`Q-BsyKw^Qk)V7u6k%)gxsW09&%-oW-#um zRTxic7vEciz0s5-4A|a=#*Ijpc15e6y5Z;Xp#)*}AN!6*3taZZXQVM9dfGikB7giO zPr#~n1Apx%K8v7X88IBRXq>;WW!beOuB@7*sS(Y!$Xh6z7t9oJ15j|@n<%?j z@7&z{jn1U--dW0i+M!58TK$<7S~&mF%GIH)J2cYW;!@C zDTGWGi-Mml+k2rt+|Z_lk(L~kzsQF5d)`j@u`9A?Q;`O|j5cb{I@D7451h*EUw1*; zI{MbOn@AhMl&#L<#P6>=NL!3+t((s^2>i^QC=_3=HOXx zMpTwKDnl_xrFL>N{|#+$mg{=yDQ0_uHm0&MhmkUD%lBM6v@i>K zlw;S-l#~2jxT0=mz03m=%YnUUFZQr_5t6S&^KKuWLt1lUX~Zijm!T^v`C6>Fp?Um* z2PgU;r07l8o2@-KbWsoo+z}lqlJ2D=)HE7RZv32oporKS)998(X%L2zWlI2Ce(`UL z0Xw5s$`COVUG628El6)2d|hg}RaK_m@Lo^>)LBALIzUkT)r0N}q(f6g@B~nbBRaYO zQ0;X4b`;a<_9{6w>tTqYr7vaSND;7a7RV|9W*GDPt)$ zrBg@}6l71th7x1hMKutIA$kH-K23s8=(WFGIn0%g5$Eo zDV?1_7aB|iAm*C3uZ7Yf8!D$57u)AF2H$v)#lvkhlu4|z5xQwGY(LCj zKSGtQMs8nPz>8YkhLT5{W>b$31kcg~-iFX|wX3l%k&z=`A9-`{+PJe4nXh|?&Xf7< zhTVTM9ajNR&i@wPVmf1zm0Z^NK~cF%RjXAyEi=%xPU~dUC%!4gfjwHCn7tMQYr4y; zWTEie!^x|~r!a0r)1Z&M=+)6x=Wi*QY^pCdwViP=$^?(!ts{RgIjo#yy&^?%(B8&O zQieF5r5HD#U$1#IGSSuu;bnnJxq3S2H4vYNok~UK9z6L^m z@iXZne2MpF&_r2Euc#7)$22_jlp34P-FVNy(uaA#4eASV7g<$s3-1eB{pwgQry$+P z4t>L73C&cnYU(fU90Gw?E(sC`;(`@`ZOq-@^8g@YBp|$_a(%DB+IA|q&}5_j5bWz4 zNTH90!P5qe+5l3@ z#6v!KUy?R0@c7iL$yvjwlwvz(;E&m?a~en81n!F6bbLIKTP0vTNv0BTOjxVlz^MtP zM$8%LYs;kW7)3y5j(~Cv=H?*Mg0Gakzs1_8zG~DE_%@rG3MVpxX|q9q2Ma)?v$W2B zCy#boRj_>_Z|B|RDe@-Qi3b9i?C(inPCWN2WjaZ3e?O$x1wb1Qc`D3tXTQB>@jCFGUs!Fhi#PiX)>p2n0s4JMN^1;UOq z7s3=h_8vI{SFR@6+0uYdECPCtT`m3Vm)?->Z6h-VVfW{3F_i?*$yP!Y4RAJ+=k34> zRap{(h`+P_Y0USBSCcAPxswMSfRDJkdQ(D6A{?$cZ^1=jd$?Og*$0ucWmLe zhq0F$<{xVhDeOiRg%9D;j*|$^g!l3}>1|I2p-JvTCE((4HQG+-WsaB`{^|o1qFd%2 zl1yNMn#_AVnI$bjJ04sSZ8K9KZ|G!A^2&zdRwKBtZC0G*#VR=}l|Ks`P_=;6eZ#l0 z(Kx(hWFeJKsg`nQWwf#U9{zTAnQD=9hfO0ZNplU+D6jvMFX-ZohQYxA~Dy5sYL8ICcg{Ph)J|(f2UlogjJKt6+W1FEk*=a0|TwC zuM%hszTNdweRX+yK$9m4DL+L10|l36!<7N2!l%8}dozA*jeWC>*P89nA<=93^vN5X z>ZA(=&I?5k@N#B9o*=|LiG~)hRSIfo>}1wE4yuuObn=tsW?sj_Iz+*W{D?*Jnejc# zDm#HPrMPVgE{WN=*1=gorc4nqbmw5psnhUwyB+9)(x{`fKV=QD3@A6F9~$^h@MJ< zt&PvX%_)nsIlsq>_JzrxifMQBLu0hj!v&6KaGBu5&tKeJfkz-(V;$vF{kkQq8Wn2N*PL>K%sIOZrt6ASyb%*yPKBu%vUH;Gs{Zbs6PR$~u`CL(mboHMDm^nOj8e!mLFVaA zRZu!u(x`>s>$fwR>1CN4@JV6G?zK*Xu1%|j7KA*L7a#SYPuDg!Ny205K;_@C6q)-r zJA>jd)43GY>31VOiVOF1Z$}Dfk>*Xsm1>K`^)Q>_k#}-h$lS^{foj#f!scBvzwMQ! z1%9DS!zJzn@^tqpprC;7%_Z7<4MJGfuC|9aJCHhZ_O_uh%K-Qp5)|acrlVRV&&b(7 zWej5j2XcYLA+59?pl6}&g3U*k{Dy={v(hKurJnnqn@Ru0Yuz?)MhTE)$)Scdfdv(J zgfSUva*&3ayMD7~nCL?f1XHoC*P0qKQ}BPgujCwV2|9$-+UL({UMj&XR$||_kqZ^q zh7>=ApFNCJtnQo0ysF@Y-b{*`sC{L*s`P!^RVZr^S$;HP6|6xv1h@xWx)d&xMHxF% z)Pi)wjA)^?x|p7}Q_UJF8;ig<=@%GltkYThmhWz=BDO&-*kiprYMZ0!)6|GV%5lRc zBa;?*(vh=wcottOW3aePEld7f&ibO{u^>dITen_qTcxu0u@JFC6qRZ12%Z}s3uh}? zH@f)hr*ZYq$B=tk(6W3TrU)QMX!H5}b_d;rgmG8ngb=+fOts_tq>S7dX zpExh<&YSBwlK*%+N#rmB~*g6DccR-xy!9H zo3`XkuamsmJtqoh-9}(;-xU;Q&a({xb!R1nD7>R$6x9M;t+n(#l9;CCz(@2hozC6L z0~!A$0xA*Vkf@>A^qw7d=GBnsS{gXXYp;ykR&PgAM?CwBNqm5#+8JQXRSIM}tb3Tj z_sD%>toHk>H;!D_RZv<9w48HxCJrw(+%aPU7A5TjC&Dye-j!0E{4N-a@*%l*w_lvB z6wtF(`~vl&_3%k3zH*-I4QY8LtB-#kEeG?Xld5OWdH4z?O5W{b=V_A-uf8k6w*s}h zkP5DurFU3QYf~t`(ULEEc938xojZ0WkS$IqVX&aEsWJ}*S`s2rE+@}NQt=b-90-iG z{x&4blZbkj~3N0;tJ>xgg9#1ggYJ4Bj!q&17Vn^<-wXlC6dMARG8Xcz%xQ-nA7N`G)%qTYg?i-!rNmaxrazb-D zPwC|KX%8sI&fi%O^iJ$+wZCu$mzD6K)Rxh$t`4n-BP`T->O26rzyN4k)4g;X{dWvf zrgSM}RLC{RE~cx9QclAy-C7+9)dw4gs5J#F;5iq`3~Qo&k|`I}XDL2~*WNI2^*hwj zaq&W!rF|Ym@V(2hdn>z=2PYh_YWJqK#g-~L+v!|gk${dqxq3AVN8tggi1BrdH5)92 zZb%^z*vm|N(hFZTRS8?e^7Ny+404W+6OnEtD4IQ)mC}o@Ip5mb-QQLBXoM}ou8JrW zdEAol#G9AZsp=UkdSDEUDb%Mk7G6LW+sk52K8H$ZBF6G*F+pM*MN-~Ac6uG%*C#!L>`l2<%moEr2jhs>SdzEwLkIv)S1&%1Nz+{V%;yE99K7xbEDzA}n_ z!9aI>AR^uA<|>q7qddZKq8Wc~n?W0uVsww9T%hcgz)cVYKaSN+lKN(Y#kX&Bjd)?| z%8i78&}+LnL}2SW>CgK?;-yc;SuXORZh-a7WMj$u7TkeHB86sKzr-u@!Uf~-QgOax zh_ksW8x*=XOD41^6+JD7NoDR!17bPL z0MvFXZR@V)eC&}H5Cl6#%?=QCJ|r3UgAcmXfO~RBPN%rkQ|s;^(GF<^P~`NAT4FDP z595xpC5JaY3MkHTyg90KHudp0!u0z={o zY;w{1LWv%fiOKNbaL%wzv(|Y=H7?!6e4Q9s;|{~&nk$u+1b47`Xy7?yy~8k&Ak3Hy zMIYoGPI0K3+)9>bspsmPJMjKtibH#_pD3=;%j_7w)^kBqjJ0p#cWR*U(P9tzZyzC7 z;#*>$Q`D7qK~J;Uh{l&ot|ho6-!5rpom)t>2FN=kj%Od$f*j*_*mIGI-+5-9_+U^) zWhKgJwubR2(RPFY zj3RSQh0WR@!6tsm$)zLn3h5b27;LZL^IS)bGw>$n0qv)t$(fGN1@ zw#p%1U57wnxJ@Ww5E6Poi3tzHwlnj^RDEfhbcY1cD92e<`oji3j!{;Heb8+(eAYz< z+80(v1)N;+sioI=;q`yk5PMRoaGPuPj$9pvE;>kqQ*t=1hL^b3G51CcVjY>8{d^kB zTSwSk^Xx9PS$>Q@!HRj_=J8(oP`=m;g%k_4j|tg;Y9ii8GuNXVgxssbm98Ny0jsii4VC+tdWLt@K}5btJmhZGaSMPMHbSgn1#Z|oRP z7j0r!dK1Fu&aKWWER&tkYGdvsU}U-5y2F_()rY^zeQ+kSJ;^YBQm{Y(o0<`yBl}mK zfL@JHV`v>t&jQs(b9ta-nZ1@$M6bx}MuD`3i*4g<=+NA~Du!i4yyEL2nOvJz*bpSv z{LJ7m5QclNCDwLq+b^?{m%5i5WDPc1zxdcyj_e&f#>Y+CM+a&(u{_9qDYJqz=#-TT z+0kmJL5}pm8YS8AIsvYUcUPS~SQ(Thr|M{URPo4SWIQQg5wz;uvVQ1-T}uKjJOn$V z=~3#{E*9Fth+YQXtfcHm6L$Fef+w|#79?rCM+VAoXx8GitCls>Tt2Knsyrs12k%EO zYcEV>ZKim%-$H*(2fHsOZ=Y+8w~=j*F#WOWOiu?#S#jr<;`r(uJ&Dh;i*%>pYrH z0LbV^@>HuASq|N3p9Hm=bnIqlNJhob>eq2V4S;mRH#j%33vYZwOSN%Xc{cm#p-3}H zDBv(0;y14cZ52KM$XDaJw3Ji~QUeCaH(E6o;#{jeb{_=P0_iY-T;Bk(&X{Hpik(st zcF{v&_v0j3EJmi)>JBdsV!5%p8tRFo&j$)_xikOfBXF{{to_ihyIS*|FM61wpke06 z7|>j{<9UEJpUA5otq0V?WitF*Vg0@xuqd|N4U2b;P|C77%nudyYQqdLtf3CLNI}_v zsFG{2-@D;&@RzNp^tq6Gye6v(B>1l=4&u-dwx%&~zkYhlp1De_%SBwS+^{4zrs!WZ zl<*Rx?eDzxpc9y>4m&Fht|Qq~QuUeT2@LQlnBubsadd~#44ZKDDe}k@GehgrebFYT z)f%HF#+Y1@`kCwr=8$%EDpjjd&CJ!Bt~Att z3g0PWYN3Bw&{4@&=gU+>dk^E{x<~;wG-se32nWQrroFCY?g>QoWJ@FTz!ATjWC*=( zJ%(-hrG+ohuHa}s&l;Ek-=Z;{|J-&lLBF-o)EA|@R{k}C(z%nVh0%CGm{4*~8yE$8 zl=8{o>z2FC;GCtf0Dx_rmMRwx0Fa7t2+LRcd3+LMji_9t9Mh&TH_8seW-gT>1jI;j z5SFX;xDgB)8Hxcj?oD0I$0f)^+ADT?rmPA3 z_#Ugpue!~~x$29#l;oXSkh9V5Q>2I5e~+rQW(htCaaaj;_1YuK!po%&#^B;aq!Zk| z3B0mt4z2DkffCb%V=z<9;VaMrUVcOty$(sXMyeH+(P)eb;u^&u@nS+uFk@P>;L6hE zuwM-%I=qB??Uq+e0XBmRtRM41)0ayEi zH6lM2#bbH6jy(uyV;EOnM3qQC%giKpialo-%ngLC?f|(2HTEkRcc;BH{F%eRPWU0r z99xLRG_i2gJ5aaAZ44mfxQ`?cBrk^^FUR+_?yYJTcWUg7h8Owt(i&afQiAVIRNo$7 zm2iJVyzreM_jVyo!8<5Om z_K>!pCynV+{x|S+43W_`qKwI5-GPvlXaFM=iSy*54h)Up^*H*$;{G1;GXt&_I)`&S z22$<&w~yX3aCCpMxKiv_IM#H2ZiZ3WffxDmS$%#1UqI%mrEUg_c=Fci3lHXWwA|6-J#uVw>005v^%s1L& zTjWMnT|T^wWIX|{zSGtwWjzx`B){fS9R+#9Vop(T^NUHg z&|FDE!lGb>{RS_4YZIZTfj{r)jt-$n;i@}nx&eaOGJa=Z=4`m=-P4;BC7`eqf{%Yj zcvc_!D0(z27D~0)n+6f@+NmkE0vyU10;m>Yo7Y^Z?+pM8N1RsQ*aGx3W<#8x08;To ze4Ftq#6g9G!A-LFR+2J(l5foKM?9_=C;~y5lS~PYkvi6S#ExUZ$}u=fIJPpCkKw-;IE^Zvv8}={E5KK1mB0d-yzH2rs(LKPR*R>8 zEFC%j{VjK&wcaI6*CDR&c69&*Dty^Hg*&*-*_SN9N5AGG>chBVkhjG>ObDKs#{*?! zv4o73y<^HNKT2v;dX10-W5_xQ(3vUItV?N{+%5rJzRD$;4;ZLZmn1_5O(=3dS9~1G zS`}h!@l{2B384#NH<6g>P)xzC_+r)?G}rDjjSb#MH+gc!9|J2wu{Y=ZMT1aT1J5FY z!{DY!TG*L7cNB6~IZF~wiPpsOood8T*u7Ig{cmQ}4bF_d=t?QJt$`k}q2AlIMo{j$ z0tzy)7+crF)VPFaL-# zt^_GTy~$nGl_s=a!H$SKOvG+oPBIT`O}kTM2t%};*Pc$tg$$Lguf6KVhk<|b$8buv zb@lY(BfHJX5TMGZ)n-+g8n4lZkcYJI^vVzjjU^A#F55iA&SghOLY#A@qK#*=@NEFD ztPTjyfYp(IFy;Hk?h^>Uo$OsWl#qg^zumf<^hS$RBv0C+51G__;T@3n1-(5INg=uv z_vuyj-N9#cfw#}r;7gcrzs3j;aNeSNuL^=*%$b79irAVgMiA)dbutxMp^P~A$W?|`^P!#UueWXOS`IG zE5y9q>oPD+c{e+MO(Z&CCw_F(5{F(nL~0y%?3_IfJ4NO1+S%Ui^?+uC@i;{=4MW1m z8NANa<774TIz`3p2A;zqj4(vX&;zG&;UJJ3)Is<^DI&H!F0>40x$!q9{;5y^qoN`S zgN^GTgG}0zrabD@)qv2;P$30>oe7JBKX}yVTRbvl02*D(kg62w*8m?>Dls(2Je)J| zeCY|13{xfe*sibrI(b{DgdN9_w#6z5`)_j19BoOoTb=4hnw4*l3i6>0T~}B`cf(eC z@ZzQe+}GxnW`A@aCo2Mrsv&lCnT z^n1}Dh4+MgufnQ*^SmiyP)O>h|M5U)5z0MjxuNw?i@3gPk`c&yG6GE}Dd z%2ZzfQq0@*te@nZj2V zoLEAhRa4O6IAv`RQ)GiL;f4bSxWMuE(QY7^l;c zr$+92#IGc;Nw`PZI{1P_HDm z!W{JrUlh$)a(n|QxrldZ46z?v&C#CFYh*~je-GgwlB0EyR~|JQn>9$}HT3l*ayKo7c47jz@+4Z>L5E-ljCF7Fk?ts`NLXMDS(qV`0nkRx!?$p`6)9bL1CS&S!7h$kn z&%g4%ee!^reJ!+91qR}OF1Xng( z=&sgCAGuruOgFhqi-K8@_#a`Oze+L&Q)PR8DTP;EHTU$bV@I9B<}{`z@mEbc>M^FR z3ag~BKZbZ>0a8+P-<9RZ&!Wd@bhZZ>PupOAcprc!Nu3?r;ROj4Ot!G(vl8NnxItxK z&{qK{-Hg~`vCvI&*^x5NDkrhcpBOQ{{RRJBF>H-Mc$vX!Y+?G6N$7b~2mK`pHP`7< z?^rtJ{Wi`P){Qoo)F=PkE?)$w3fpqllVTcs9qfY8hoVR&!z;rML{Fj_lIAe%sj8mzZ6A4 zddY>^SDKTrG0|D0<&zwkuv1#3-G<6Vz$wI3&^VmOOy#tTt|U{qs733bpO@!+`k}pu z@e#mrDm9bR3k<+w<$hk2dtED1}=}#N7Y=|o-w-Hef+}d>6Qy}4Y|w* zt912*4_ibZv3p{)Fapz4mO5heJ@xbz4c20iv@TUR1rjlNV^}U>D(w zaU`v%gLacjg4n6tu~bi}q<@9kHh3bDH7E9bM5H^OQm1Rb8~LQvWKxl=d`O8ZST5UDcHiUt5`7Uq%@e&UQoJpK&jP?+mUjlrFI2yyt?M?ShdR~rYAIZEZXwIW*6szUE}&sdB(?gM|g z{*`q-x^V7|C<P!$U7q2MaF=X5()%>g-GxCtd{&<=4Jr zX~#bl7wW&pc_of|_{aS!Te2}Lb>7`+$?T*b@R6M@LAfsg9(DJ)jAsNXkH?x%aacf`aj90VTEAcFh}I;HEF5WiYmT*x(<%(ksyFZ}s!EdILB+L*!!vnfUY;Sm`*F~Lx+PJCJ=m!Nj ztpDS24S7v(K)(E1gT~`NYQbTX17q)cnyos-P4%wwGg}Sg_K+;)#Q}puE&FE@!|$T{ z%GT=rhf5Wt=P2QhN^_oWY57vgQw~{n&8Iw3cnq@MIN!`(coFHh2-$Fnyw)lZjC(%M zJf!^nmn=;QkKa&aUq&|+hxpr3=lZXnEDc%RAXF;wfnRs=8~k-fZlq%hl_u40Ra&Yp z)fEciJ6hKl*^N}ZCZ}%ne&C050gERTWA?u*!X#)-d8{_W{;W;D3XRbJLX}B&hkzvc zo&sa}LHp=R0gcxEAn2;p$^s1ZXd;@v*p-ImoWLKk+ZLNjAIcN00olbFl$%0_@u{(N$D16@d$`Z46zHS-7?X{s}QhuIr|HV2z(fEbj>Tg|r+7v60yStoh zR&oPGfX*;S%U3fSWuQLAjDzhO{^2fPN{#npJf_89Vkka`KyBq`2JL}K`HfNAnHH;+ zod6<+UKRQ07uR6+J;O?_2UsO>xGGaXM^o=fkn_zvr#V>2)j@aZ%aON-A#`zV=DYNMVZ_fmxK#9Dch*N-;-_4IXtc6e>Ri2WMi3^C- zHd79tqo-Tb#&Yp4cTPe!R!*Ru@#!I~ofIr!^Ppj7cQo`vNUEpmNF zE%NS!=c|3IaSIE!N&)^%*8+TPH*_B?%}k0}f7$((l#BH37qv{a7i)knM!Z2Jo7+Gb z`Iv_UB9j_JPmM9Rl`O#OF+rep8s3*GfJ%$PZy2X|TeUE&>yN%w+bmLq`wr!JpcYMu zSvZ9>iWWW}-ocPdA}mY%AmAhEGuq=L z2kiv62!-sQFQQ@=D15E7r9MrMg*w6#Mi6*ZIO3XUPv+W@4$zZ$i>}QD1Us}<7;icK z&bzUqwBZuVnn@GE<>is+bAq7cU(no2X9wl%8+c3C+C7@T&O@B9A6EiNspCoXb3`QfZo$Ah5Cy--F$((1go5lphC*6PTh zGjBmXO4Rr@hF+R&0wraGRel-L(aL*_7dTkPKwfNY{au18ZZeozlq5O}PAYf`odl}EJOY@7pNGbJq@1N0#(i^XS-DjSaE__Ql1O)M-K|Q`#h* zm2aG8_M8a61|-jL2*{j7M{n8gH{#?kUAGcu8Yd-f@>X3x?zMS`JCuX=;r0 zDLnnqI?rfR@{IxP z?PLiij}4mMz|H3=5|yI};er-anYE@Mz24;1T==S(Sl{Y(2ltTXHQWW&pU1X3bGyx5 zXFcmwcV_WQDY8N-X&w@s`K%uys%;)k$mL@peT%9Pq@T{<&(thT53tBzy&DJ1_pKRS*5izgQD+i*@N2~z z0Mixldk!vF#MwB;l{ngvZ$q^YgEGzM_kg05SDU5ToTvo-B}Vf<%w$Q+D&6i4^Y_{- z`Vq3i6NGjun>GF5A4qT@O8vAdWt}D$`r4XrdvDWBBWq=3c!;z4E79MhjENryMEl02 zOXxtZ+@TlNsqJP(eEPlz@eVX5B)1TRG|w6=7KhDs9Kilf4d$e&CrCk&lP{Rr-SVGK z;ZX(Y2nC6lNr3`Qc=pF%i$HSMlCMKlfku+NE{a%CmmE~HwTICvn+t^hNuh~eer*h= zY&`x>{%xY3=YsUJE3j4S^u2imHeh)|ck!Zv6vrDf+QYJT(swS?z#K^#f=gYp@DZ3j zv@goW`0PanRs4IqB|K00=_c0{7;G{Yq2_?RTJaYE>JU&{586!Y%I6LdcXZPDNXj1` z|4V&xn;FLXkoum_krTy*-$YGhGf5APq-uCx4h;tvk2tkVlv%3P@)oDg(q2imj|cAz z%Ct0d%oB(7Zk=lyZOJje>QHlA)Jxlrs)&y z#qmxVi$(Ee>GrmESFDt3S~zv%;lLNU|sRAc7}{( zI30SD68K=nD3+XtJRYD77mRY-7P;E#Syhp%#9|X3#^Uwr?{wEoTWz>E$OYuL^X z^>98+NwL9)*_-(#x1SE|!uzcq6AWUCEl)pkt2?=so|q!Xk@{Z8awEKPCB!={L4{xP zeWR{Zj<%)UKku44i?-r988Kz?m8ok1%VfAD{0&Ti>r}h6-{!VwuMIHYK46J<@z~5r z3812!qfg5|9a10iqiBPYcV~XYDSQhgoHAh`ok*!*nt!bCegwSU&(IFqamz-q#qh5^ zE^5Zb+q@U`XTg2sYCbrM%$Sc3ho2&0r5ilSn2o`7&|4m^Q3e!3BXe>j$HMjcJl{9y zrfFfl%3c=y~Bi? zN-}gd-SIg~2wFA-jZ#4szae8UlNsgQ_rTk|i0k5O`-MaFkVh*muG_>j>@&fV5XyN9bw6vFopqEJtk z{cBtXx3yZo#*cfJ+jLFaM_lZ_t=WIXl%Qu*@L!Tgl6vt^r0X2^dYBGzFm6rXV7KXbVejdwfT#PXD3@_zT zccuNtd;Z6)J?AZDsmh(FwGFZ|kX`KAL?oW-aDfuBALgVs0=S@gFl1Oh(nj(t44Gikj<#*>?=?P!IU?LrJtNpdb)EN9 z_Mh&z^{tzT*gKCaSyAr!Gu!QzT&{hkWtwyt}v(D!e6B&d{8?+;&SO z0ZgsGvDz57l!80{G+-1@N<_Yq3B@q^s8oLP`yDN8oaAnViitj0M|PVbCi7Zym~&O( zeEW%&d`~XlL5m5$x%o3GVU-o;7vC$L!b}?Wbi&Hm&{~<(u2DW0=T9&5I5D!gg&1E{ zKmUrD_(w4gf=xd6OPDqGco8XoIB3WxT%RS^{8|CYWa>fvpVOVnTws^~FrY74yT{lV#F zf}wss+(l$ZbvZ7!tsbH4Nk4v(OyQ^Hb~}q@V9T@q8A8GUl_>CYNUx27YIBZpaP(jY z!Yd^triOuBNTP{%HiKSXOwo1n+Te%1`5q3x+dWRZEE z6G`JiLAZKRLa7L%t_t)I;HTD{DgUMVNd>lNr z(j?CF*8}!6EYk51kq|=29S2?pYD3=-@W&ug0pmxhhp>gI#10?zx+L=r}(9A1OiPtjRx{?uDVm@0}IT za0O|_MX{`dt&X!uO@`U3p@pI`(;jN~Ft)gfPm^QFWh+u@I*19iwPhxd6T={q!&Ug> z_EAJ#Pdhu9jv2?;ZlCIiE!RcM)ZGV9b%^@ab+hedw1W;dWGi#+`hAl*7?@C0>+VlZ zD!g92d|G_8R0`3Hvv88C-FCR(1>?Nd@3M(#`Qv0%8)4dYD;+BScneOY&Z<~E35{%0 z(mCp-){mcPQ$~p)Bu@V5D9Ij7ZUxswskGm-{&*Q1JV0!EVu-sm{!y~wUE<*^-LbEP znBqio6Q;47zh9>^TSZ$!sBr~iFFd4v%lnEO$sFhsDe)0e<9-Q2NL@N9j~Wd_W>hCu z4hXxsmVosnfYsBPKVlCoXye?ewM}5O>rZHO!sKT79)-j>gN&iK$;%bnpRO)b36bds zoAa91Xp3s~CCmlaE;nsG;~7&x{UNpntj^ zm9)00n#+?QlK!`f!h;}rd221!YZ}FWrDMv~BNV3S#_Ap#&kGGw#OnXSV;aD`pMGkf zNEm7_>Tsc*{z?F{bu2%3t7)T|OueE4)aE?&z`f|#j$_h<}ALh-mUYl3ue7G zY$fSX`2WGJvWl5_HW8kE;lM3V62=VH)MK$+iG&pvs6g0oPs<!l3i`YZZ3j=l2QmK}a&$U<(_qS-^pt?}}j^8K7}(vjf~a z<{B7L3TxXD=?)J+3YTj(OQ4}3hByAR*UHcwud!^&QRg^-KrRn6Ktp_v9ykA4BHP89 z)NHu~%qrIX-j1+CKN9hR%fj_N2x(GA7V?206k#6$@q3VYgLKnLF!T@By`iMPIDng) zAS)k2S|+wb%ML|c@ODl6AVZ6G$6vr-$%R`ILknu2U&%Ljxzjo%NB=Y!oLcRTG#kX? zHcI(=CReOw)4h7ME?fCF-tZVZtKUu#9aHhkbw=q)%jbB#km5T%)w}Xv>kA_OykBTbL?BJC;O7Yn+B@R*2ItIz)JDf9fI7P&gme0-MO8mLQ zDGtCi73a?2klz1nxP{BzTjcEAcnMr6I#x}Kx`X%*pR1-6Fd~=z_8DGMN)f(?BJy~W zqcr1)6E%sTqfSQ=yISkMPWr%jM(69eYUjos`rc?3iDLvCmyy=o#H>H{|^8{r;=a>=Hx40u{4-4>*je)M1 zlaR=LEg|jOQ%AbrFr=zJ7PBh8IPra|YQZp%CpH6W{kiApJg!lzAKm0JNH__Da> z&K*W5XY5&03l7QZZ^(+Fgs43RKhPww{H}BFe$%vB{DiE{!~d8)3_~*xQ0U5=k0hBo>+kpuybI1U8}4i zk}w<6N4&Q`Fgvj~wO{)6*Pn?|*N(hMXYoIxlCkPpXLy zg?#O=fD&l}yabJe19AjG;_v2&i_&JQo!X|WW_e=kmzRKHv)CZ@sM?k?o5w3ANx?$n zt)vaNnnU6>{nO4^U4JE;ZHTZ#{+*-!=~P`q>Ot1;zELXBCf78Ho=I?|{TZ-Cx_)Z) z+XQu6zqLDJid(e9&7X#GKe732oWJv+CF;A^IM-#9>7~wV*JTmij@Hvt7-OBUJ{!}d z6KacMjXMhm%Iwzr2?$6|JM)Yiuj8x2F;__R{l&|O5iL+B0MC|^D2S@*`06ExC=qh-bnwx&T?RfAP&-x-$ynty4^ zMWza7-13mm!l6HfSg^3zB9UV5e41ETu)fmvg;Q~?LDz)`XzVzDR1qe0=A}T}?@Z$} zgm)r4!eZn^vv~5NRb<<*ydZYJ;nGc#R5!ZmhOB7`b2r7koguFK38c6ih88>c2w7cb zrY2ta7L9vAS<~1?PLs>mV~vJdLrofO-6|<4^j|~6WUg=;rNMtMPtvBau~5jb!JZLx zi#-sTxyail665N-6LBPJw@7UZI&W+(k`jbaw9 zG8Oi=I10_*4F)>ySkNbN5p6VM`ITgc1PaOIWg1+MVv6B#f9SiuutOiOJiTs@p0DT* z7o2Rv$~L6MpZ)yeu=o*u8jy33X9Ni=4#1@(s!H07kvgbK~XUSM9tA()F}s?4}|@kFoq%f?oFkb*Q0jP3Fw@ z8IviyOep(!~9dym$KDiDTmX_ zA35wv06m!3B{5hI^OU$kC2qLEj@kIa!N44~`SZ+%)D1?NFJHbx3Cc}Z!Ol0ueC{!|NLjL;<%l9^2%7KSonV2Q;R|W2 zll-tk6$=4T{FEKSk6nMlA(fgIKO%}s9Ox9s)HyFTox4|Lck~NMm^r6s7ex~ZR1**s zKT@M4y}+c2Zvr_5B9I~~XL_oVupp-ConCPy*saEN>~^bZ4a>#sIMMWI{#6j;JQ?RT zBd|9QUyY;=Ybt_MKbw2WU$MoUT1n~^d44=MgeoJ3@S1PGYYfw^>ks7t(P=JWFJAuQ%st-DFY^vh^P04I9&^Z^VUV=Kl2;!|1n#K``Zdc$oO|!mPVdmy)d0 zR?w!^P+5xW=(s?Fuqs+mV+dKxjnpi)m6Hq~f2;CMM|7|}(Um-Bg( z=ua~@i4ziCk&nmTnOxesLPK@+965r0Q#V9onvev|e_pL8eA$4R1vUf9?BAo4ma)`z z)qTO4sIyA@m0vP*A~s0ziN7>*y&}vLRtiFV|LaF0${OrUTO%Vo`ub&|ep+(Z*O3zq zeb4QMGol>UC_6a$*6Wvw{D@ndv~Ht{%CN|_4?_ozse9TIBffNF1ccc8xMio~!dyHW zgyy_(vAVTkBCW5oSeY}M=VgwRz)6&_sJIQ)b8{HhD@qo#r+xy(IPy`va!a5eC99-h(y>BU!^ zx5MeXjKS?1gXdw<%Tk@VawH9lR5J-3WGp z3o)7lvr&@eIpvWLVq7x@7>?O=qnLMe)`D^SIJ0!2f|~V zr9$``2Oa+(gSJH^jU7T>PN?tAV8+oE<^a7xBEI}L946MG z80FG!lFJlWE7AK*bxz9V$H#{ola##0RE~bmXdMdOcUm&a1J6fY>HEAZ%j%6EV;Lnh zvwCqD!%iV(7fcbS-_@wL%ruE~Lj~9i;13|l-{6K955v;6bCInI4J8_VLV_P3l5OX| zVe}I3Lp}YFW<4RKEDXuwt{THy#(Hm9=mehvV438`lReodhPlYeXodH4F&sgmiExTm zNW#nw(2Gpl9f7N{(Vp)PI)0{MK#*L*!sQzAlptsud>W#B^r8>_hGFWBy^bIa8|o$c@Alo~*uQlX43QW#JH- zqV>WU;&y`gVey%cG#Y}{Ac9Z)HD^y#{z{LYTPs%x|4Kbo51tKKNBx01@x3xGsRvAwd0|8Q~U3t1hkyv4Gel zyD@$dg=4>_k>NI)gDuO*WMvF9G-xAP6wwdWEzj1uLN9KR3#rUT*3Ys(>?@p0gr>88 zGrh0Z6FOCPlf0}`{dsk+Lj12*M?4uP;-CpMv@zPfe!h4?hm6jq*0`TfvA>JgI#82B zJbWzcaYEc!%&4O@LyGmVOru=iVGpe$wQLBJ%`^pyKOBsaA&ghMTOW%t<0(R*c&Kh( zcm%rdOs3fL7LYiuly|j%YcHdH_10V8sP|P(ta#DWlDJSI`&EL%#4PKcSJ{WdIX7^; zM!-y847h(y)b5fz#pM=c2ti)(47|=3dlx_o`=})^gtEk|Nz`2T^Z=?J2&TEo48HssngCVCI`OLQasK|Lu}+eH8m$DE17zOfw-X7=4*T$y<=_yPwRne0iM(i=-HA5nu=b*^! zMn5>FO|Yq7h8Vvgm&!~iz)~&0Jkl`X_aBeT2~O}6|HtE>bw7xPvyN+yV@UH!uWOFqrh|ACNE~vI?zJ}5iU%HRADj@_F*-fno`G% z;_8yMDAc=Ror;Iu9Y6!bvdbF(_7|C(kIi@oyyYu%Qtk9uk7Be44U`Zpo=dmp5pA7D z7CrwTp20*_Me`HCG!JF_`B{4&vWsYaxx8y_deVDM&mZp#fM~D6a)*nmOJ4(iua(;S zP9o2JT(~ShpX(64-Pku$h5vHx@vUP2tn_1Q#mCykq?)uk_Nx6Xq#Pj;?Ox{{-{`bk z(VQId4!efns`6i{S>Tk)>mi>o2WTLW@JJw^R z*uys0sHANF{<{FNQB!Lss?wCscmb<0_zxs^LdJzGAlmtNp_XNC_C$&%hlN67YwdCt)|EQr|l8vrLr(= z{{=~#zQ-2`B9wQX{FvPlT=AVD#6}qDOD=ZjjrSLm+ zju&jn*lcL4O&i_4;UBsDMM9T4tw+z9 zL9GFokAe~282H)vyS!6P&RnHjW3Z6>QibAw0;@`ADQ+k_%B+OQ6Olzh-E=IHl=f0eSa{Z>3q zZJbg$FjUjYCS!C^(Xe2kTS?&dI(%+K07DM0Iv34}1}O zh0ynl?JIUY?LBksOeFld3h}yoDcUB>Z`929c#xDm`SCx0o}|$wtD@7_SMKg7()ga@ zB{R^Fk8~vbM{F!2$DjLWgC3bHKgbllsLbM?*>bdC{iL@ytSS*!7;Vy14J9HnhMr!=&{X4M zkuMN33F%hiH_khGm%OX~G*#Dr4M|Syj}-E&J`}&j4S~J*9bv*06 z)a*Rji?Xz`rtJXNY&dp~&;8E&lqm8b<3PFAGd$rZN!s>JD=)ACIv`y3ux=)~XP(Mo ztIvh?Qry+#2Q2L9<)YQa%5@i|1+AZAYa(nr8+RB4H4l{AN1Q40C}T-aasfWv&q>ap zlP}w3tIj+E#+nZZ$r~b^A7u6VI1y1O7rJS8&QXc7_ip;Clca;*ZmV~DD-Pd05;l69 z6&nfHC6)A9#GZ!m&MesE)VG-8^CcL*(@%zgc8!~|@E^q3=qD=5vy+~vvICZLnji{B z+Am8mefu*ifXW2Tgoi(tsyfX8oUjh=UcrJfh7XwzKa3n-lbGz+n3$RZj!LY1!KITn zwT9kJF9Dnf3Pp7bmpjx)b$Lc7*{?Yhcy!%{`u7@@N%H|Tt(;Yw`uGou8w4PcQ6&pT z+atl7Vm~L2K7x*8);JS9s6WeSgp0;!89(uZxI98Y&hg@(r%A{QmU@^tj)8_ygDG^J zkmV;NjEq*8zc=dey@sDPh@3lkYfglq-p8(pCeOEOV`iXgp35}0%8GBcYiC)+ycP$M z7F2U7mff9p54Mc_H2bq-2W>mI&b+NA!*Tuy)8i*%laQx>VHHkD^bI_rC4$)`8Ewq= z;Quflx%R20DdWgPzfu-DKk9j_kRx3Q{V%N?F~l$oG`Zt)gO2tcQdNFi)N^88%`W|Z zHstQ78)h7Hw*%&#_3Vh3-JK@+dTdrLO#A^OA6yMpKZ;#|mYMA2m8vIdrkeczYxQvV3;ctcSq`jb`FpUF{F*eT-7y+4dNwPR~WF8y?_;Z;@H zZm(g=c0(5y1XtL4J}v=i>RK?>r~mql;Z#i%G0!8x-bpY$y;vd<)(Y+M^S-LgIxfd~ zS+>p;StuXI=sePm({Oz&w1teQOLl%;ar9<#iIJ5w{`A97eLh#e^@S=P_eGA24%7 zs6VVs!&W(+$gdBTab`2(ZWwIgwA} zQX@1>RsoSe7R3iDb6GJ_Lkng)B)I211z_X%^Oe#h;`)roozdOBt_P1{Hx@hVOLFOk z`!^!#e1l5JrDI_xd0W5YdK%=0ZjE7%+RT)`&#k*Iu%^ic-bA9L^cURQj0w2^85rt| z45T^y`Zc_Bkge4+0i}|}==g-IFxSLjoI9%uyD0Qohc(gI|{wyGVBS6Lm7+?h83hyv@2s@Va-^VDP_K*tD$p*qe@Z?TDihDLXf+b z?%3sZ(~&U-^MqotMc5e2@n?13W4u1%-;P^Nc~%8Mx_8CbJ>n#hnXAjUggwjMkT$*L z6;zs&RC(Z?IV#jel%xELsLSvv?Bj9B=Ebl5OdU}#t8GfPxHW?vRIEeF}2k`M~g{z@mLbO zzz!-)(Q(pIW?U>UTfs(XzV~tr`fnh6 z({BVJBWgk)#MZu!L!PRYpxyOZ0aocnanbgBg50R(p(r?vA&@=iF z$DrV)7uWXT6935jVV{&&NJ}fcS&(S`xOqa) z_swxo8QY4LTaoFAmLjE=0%w}SS}v5W3}i4SnXQR&8@;18u82R(bzPKlCE~DVF{@2f zv2$RSRtwucx7mJ9av(D+ILV1>om%u|Oi4SL?2x$q9mH=I*aAx)GbFfBc!NTm2*fFvl>p%569&rFlt;vKYq)5$yJC5G%42QJ2u^G{eF6WE39pH z@SnF}QN(+O8r>VyfRZ9+ZDJu;7r!tNla3Pwv`+ry zt{8=wupaPDy0#QcF%6+BN81_JMuOIDJ;=`~u;u(O!IQy9Uw5sApPwP%PtWo-qGm0e zV9&#nbDUz1-IxNyGN(CqhA~swH$QI}oFmibC~y1xze3P3Q3Ul3BMD8%6Ld8eP&NAi znQ!ynw_(Blix9yRi);@C6-`1M$@gb6Oh~|Fn7-K^O!L{5=!r~a3ls=zVgV*j}nEHX7-n}WA*^(s?-IZIuC6b-;yv!AT5*6T1K?>G95hhMhuJd*8d-SnGi2r?DA7fWU- z=OPs23T~wXkHpxKle3$25mAvaO}=+n%#70nuNr8{QT27eq~I$C*RKs7A=Wv6JWffK z5T#a&>WdKH88fW*kP{~+^aV0g{~kfUmGY)r(DeQRlXFug!f58{9 z%MlJ9QeZmx4jebEKbKbaRoR(kLr#SdpEokmRTYkM;hP{gGP{%2}jX}eOe zPp?d2>l_Z3Hlao|5}k>b>WPIw3cLK3A! zbw80V_@|!&MmJje`NP zGZY&d``Q%yjY0wY91a_R0RX(Pu_6Ef01sn3ck^!`Rvvs5cs6!cumyqG@KK;SIEnvV zg7698+1WV$IYC@t%fZeKcCv#&v;Y7n*vAIq1^46SgvZW<=3vLihi2y?MuGm<2+hVr zYzY7XIRNk+Y^?Z-0AAv4cy=B(e0<;_0Kfs<`ez^@z}*BPpxN03IiWc?h!p``#K-_{ zU={$tP22(v;v|j(j|Sn#f!px^4L|_^cm(kPAYuh*Hg+%w2kSpL4ncFU&jkX&0aAbf zuoAa|9s%G;LGU0j^$L&<01SQ#&Bh8mfdFs`Zh)B*ivW0tX~4L|fzaGs#OB0!05+f# zBzUYAG&?Ww7>qAi3IKubA;6sBG&BJ`_+|ihVtHr|HZUMNH?a#e2RE@fG#i)TA7~I4 zG3~z;SpTU;3(W>Hhyv3GYr#wW07k}#=HMaLfd;XIJAg%r2aBQz_;(cq778c|01g2F z?BLA6$pFE!vJ>k;vvC+WgP9)x(2@}l7Atg*|>=nq1kwOgMlp&|9o-( zNV&qZbMo>I^ph2wuEMPw1sO-ek;8f)x!6y5k`rud` ztb&2?Y@F=)C%{`MXb_JD3Be@2!sLv-u-Jq0viwn1h4@bOwr#D?2d&L z>;tDExDNKJ!+=`_MgGbA1P;#*7W*GlZekj62rO6tSN<)~0T!B_L+}ASQ_QdcUO{%C zGz@?Z^b-a+1MUr+{Wnu_01(#-w1G32x!@C6nE*&|>~ctO4#btE`xX=&A(;~ zFcA>3+rN3p2BQsu1rwwJQbB|Kp@GhjV15JuFcUUtaMxlO@a-)efP*Ut!e9W*C>{Xh z_?KZ2Bv?g36mS<;FbFsC5cu)Zh3DV}dCv&CfQ9yF14cnZgFwXipgt&&2@EJ14#3u@ z3E&Vk1!+P8j{x9725ewvxBp;PVZcTd0M2rk6?}yV{cje)6Upk22Y!1v@&D$p0JyW6 zfkqHuFZglh09Jv+3G#!5wL}0`A%NpB!Gg^SG}t7d@dE{u$5&Bv`Y3ARjb5 z8y7eJp`Zph#>+oIZb+aZcmT){3e*J&GzY_T3a$dD!O8nyfd@~6A1-cU5NH<~CZ&Ol(YS+qNdQGqG(ulbQKWp7+Q1 zXRm#BS65YEbyan*b@uM2g9AD73i6W+fC&LuMF9N@g92SfKo=kwIDi`pgsMI-NZjgw zw4DW9eKmsu#s=Jh0fs?>aR_9AjKDyHRBV%iar~(SnGFYAec>PUH6V95P_rmJ6e|aS z037HG3C07Mf&h4Z1*G{4J%BjW7yp4A03K*S4J=5n3Oq3QD+kZPxVLvfmghht0pdUb z5)c3wU*old0|kco45a%jZ-CpQ0P-)8q`oe3NFdoKa4?Q-Lx3j&s5<02fWS?H24exD zLi)RZjnf?*Fboe8C5Hhu6AObB)_u8H2hz%(iwwq=I}JdE1ZmlW1@M3v1Ly&f6p;0m zrjl@9iCh7?y9M>-(XRxcfKmzg0S_QW1U)nbk&_G10_IQn6@jnlzC!$;lOQ|nP^_%} zZ6ImT@uO{!0IM$oJHvovcwqq$An;iA&p=Of!N0iH4~YNjXpIbNo&gP>a|a5b2MM4B z8D-Pg19=nwMeLw2SVSSgxcwnP^$_3z0m=cZFPJ&N0oGsr6aPnCb`YYpV1P@|6JRb3 z;O(o06!MFhVjv|EAi4oBzyM^Zpm>Xe185LHa2Y}X@<9Ox6oOy?=)i+o>OtWW=memF z_`~6^0vv$_*no;77yG-NKqbgYYA}%R$*7<%60l#GgT})GLKYO0F91isQbXb^;Mgz# zgD?BgP=FmskXm9$pbZ4z9|$VoEI6V1>U%6N~^P00Vr&g0zEul{5lS(PPoy2K_{a0763%PlIw92>_ZD{>N}2OKu2g zY#=DszceU(jVtJfC^0D8f10vOJz1Z$O!qDqs34_H#lGlB!B}X2jKfs+y9l^ zSzkziB-ntx-~h-kyfq+!12BMd5DS5U;9nk2erY-aAq2>V0p0a}0oVkJGG{I|=vn7i zT6Kat0&GEP#qRF{#sP?h0RDjjT*84`EWgMRfe7?L1aT20?)nuRIZz%0do{t>=`Y{` zz%QjNpz6s%OapiTQ63-&A_NPlRI}@s0khz~=)(?3Ljy?#e09$JB7-m-XqMsdHruWT zJ!yso6*YPYBrqWTHdd}OsCgAs$bcnafb_3KiAMsd13W?is6piUDu|%j%>uoyfhtx}0nW3YG( zB*35_4^4CwiGagyt3Oe6ES`YQ>*?-(2rPjN0uByeL`(>sQZgYys>lpJnMNT`D2l=y zA(dXE&S;{@JO&Y<-{S|4DKU))G@s59PKF7|;;`H3@P?#~&*5>qaN{vOMako*f4bUV zl8n#)FAxHrK&(QhkR+U*C@b)rGA>y(4y(=48G4CSQh{8N+QcHYTqe8y!LNxWTKoc$ zC<1YMH98`UjSOCA1a*3KalD$3{%Gh*s#?8X8*qlS8tpWz;|l7j6(&6uilaq!W88X! zKEF>0oY<%a^^_-q_AIRYU#@*C_-zpCQ)=1bH_ZC4wp$II*q zxeV>Pynn+0^s?p3hvR-zM&|^P9Z#?GMgX?a+`BAlO;-VdXr8xw)KRpKB)vW_Jqz8q z2>JAI?@!l_kc z$lGNGL?hA5dlR_tvBY%`7KLiOkTXYO@Yrm&#v6+z>DUC*!0}Z|Dig(eYWIbjRI%)a zWU~099u#C!8MGB_&*zS1Gg$8h5}^;p!x(}8w6QpoPg=7S z#o*O(YMO&A75!ej-ap~p43wNj7@V9~BXvg7MXfs@(dl579V5wXSe2toLpTDuz;LkN zr@plnOXcgAj51a(mdd@CmIw){{8v1fRpj>30{%;YqxeJgo4ZYfR63T9jCP<6frQ8g z_Kd1~rMhcNbWuyvIle3nS%Kb2G!TITf=a244{JC%yoQFsCU*d0P|{LR*#4NezIqkh zu=oN(H%?ir(Uh5`(F_x6y~$#(=sA>zV%}E*4MEj14nLH^I%<2da3x(4&Gx_kVE42i z;BEYsP6*WwVOsPlqr}-OEB1`Cj<|Mgoh6oN9m7x`1)(f}CJ2EjPTn?z@~uXJ(>g>6 zagy!yqYIKOv4tscXMBw4Cf{`QEL$4cpeW8LDhz9lzk6Yjs^=|rIOIpAmFkW_JL^ho zol>iMMq0o9Yu#^!%t|`n|0NkEvC{431VMWeAw{r^4eo#7D;x_g{!A}SA#n(QQwMj6 z;-RM@Gjf(6jg#e3V5Ar~9Z%F&O1g4*08Scq_9Rz~i*7OXBcK(lU=0y8bE$r&C}*k% zt7tE2_3?)h?1HOnJA^GjuxVwgu0$%QtTU4$vN7A{ZJDEy61myhTa;TvpO$`acw7IK zX-)!}&EL#=r#%AclHo$HIwTgq&T_pKMB{93W@S=TkE}LwymkiVtSn5)w5;E_2v(nY z!$&vSJn24`x8t5hgN9zgG3y*IqAjmp83Ms?1~+GjmK{-6X&!Q)JlB5EqHKSEWkq*& zP>0g};~5^URvoIA;g=bv)Ayfv%h7wK(5v+V8@3SLx^}2dxCdGK$!#Hq0ke$8Oh-h$c6hEuy^!dd6neAzFvv|f+7D{m7W<$nh4X5LSYp3$0wL})#ex36U`rn0yEe$%(AbJTSTh?4na5pD(#~zy z9(u&qnNH%JvSZ8s!T#%-NmQ#*n^0~BeIiUO`fZa6S0x%en-XsQ^Ky8cb~XQI{+JW~ zxxmHU;U!q64u}rJ=V7Hn;uR4*c}%Bk{!$%uK-6+}QWJPLFnQ*Rp?g7XcI(=QL{1;* zP+Vdp=v(^={9%CuSkF{gesfTGqnLDj>6iIkorvg|!Aaj1pBA@hXj~$V>yO1SN?{xR zd#3P29g^KwIhRt;Siw{7#5@1Db~=-V90}vS(YFB!YPU8phY4tPVMjMK>PZ<=qp`!$ z5)52LQmpn%nR5>o-|plWmYv&8Ds1+w2W?C*7-l6S56@PZ=DPd9 z(`>KskhG@Dc_m&>0(_XGtVMr&Bp^wVfj5qj{1g!Wy;J zib(=ex#}d&cvbk56h|?6a30>c>Uj=B5z3}Oyyyxoqw%;bfwfa8lLv?kasNXmm*_Hc z9PcaY$OSG0i)J4cG2{++5BALzK2XcDX-|q{r^d>R+c~WwL60$1>QPwVDuYh2iAuX+ z2O@4iutH6_#2fss0vJ{kdW$c*b3f;5J$Wllw{%TQr{hu{BYb~1zj=duZD%4K$AOx& zknNlL&J`}{e+hG4hsva?x^^Y`D}yrGeMFeu9@>)&O7_shEBv@}beH33F* zTHJgY3K%+Hic##MJR!ePAOIwyof**%X@R*OR9EG*#mtXrZOKiC=8dLh{!H)2;uf-c z-1r=YP6xa_1?zBCGWjkyjjMvgz$@9AtEyiqC2NsvCpqDa715E?i70*fj+{Ez+uru(zNK~`{StFLPq)b(Jt&nYsi8SzjCF@Mg1ct;hK<;n z%y}oF*uXsR)Y`1=UwrIUtJ|J0TAoG;j-U_cY>eYBm z0iz)+Du>JXK_@dU5)Q^3Dv|33G-Vv$*uwHFV~RpTThsXZ^1zQaDcon~)7?|a3-(m( z40Ca~Gw)xjZjTcjz3Z(SHy@md$C_zKRfiq7;sA#5zX#4pI8Mr3HH$9+3p{REoYlIN zxPPmp0$~G_UMKplMC2{H>Vd)nN)5hdNZ|4tK6HeR*#7pN0!6)~{;ErUQJ^yS3`k7}$j9*yNtp zP`MA}6ooGy$_qw~twc;SjUjePxYh0zm3RUf9*HY))W}E!Obo4VoVN@bWuYH`t{lZo z+NKLN0Asz+L*euiKhI{n_#jy1$faq)+EX65Q*U4K!2?{xisU~y?UWM^iXRPzwmc1^ z13jp2*>u!Vn_$~~HAp`#u%pxPo-d33ay#M1veYqKg#(9wzN$^9sQv~4z!w)nU;YDfM zpgf00brNN}b~wg}9&sQI^uAndNzy*Mmy1=4#!Z?{sE|2?==?Z+(2AGg5NvDaWroL9 ziu`S8ttWyU;h@LWk;y2<5+5bjTm?VasY-)?93>M5QyhgF(a+$8q(sT>4e}k|iqVL~ zyqVUu6j3pI1;3UP?JU2+&v55L;!EJ4+H~E&57Evdl|mm^-}WAjB9LK2$WYT8VvdHV zeH&Ur^`WaFz|Z#BcW^~4m9zC$gTdZQS+Iu7FjvP)_|7`@QEg+i@&HV8PmOE3-$91}Y-JCX3hHb=Wby}N zF3?Qc!ELzYz(a+vEEtvfI&3yLoNMF6g@qzhTc;Xw65LT0UA(?0bVI1wrxyLaPJzq^ zn2rcfFd=@EXI_bnK$YT(D4z|k2Hby|U@Gl(tY%6Dt)Zk3{0Dx826Y`lgFu4~X{l$^%y*ZFX5Uxe&?<^= zid&P{K2#nC^@(w6K6k0)bYwqHpMKvm^elpl#}r32Bz%0%aFhFr>e$8`n?q-{x$04! z8?N=(Oz2qUr@)AFj}*bJaCaMmTb{t2Jsv0UTwDsBloLwhGm5>&o#W6`uA1P)$|mgm zzBlk98OW6Yhe-O)tNZD%`s)$X`1{&k(1-C$5LEj_z`h!~Xm-s?eizy7`PE%$p3NGP zI^PoI%7&ZMkbOB|Z2`TW0mLv@xn#w5-D7h`b_I zt91v}RDbUwf&w9~1l{R?w7pdu4a9@HVYnh1t-Gf92)%qmKaaM?=)rTNNokw|v7||< z#WOYfMZ|mMF31vuqZ0`-#7Py{{T*}A_758|+QFu`Mq-P6SM4ka3_kzbloK*T9HT>i zf1)>PYvd%U4F(zgL~&qosd?D~8?Nmz-y`f|C*b9~=(Fd0RpKEQPdJCB8=( z(jfC+-fr@aNZih*z&`y+ym-qNVa7@6%ZW{{U8ap9r}!hJ*=0(G=1)vp<|BlXFnk%v z!k~Pr&P@&vQa%S^Lx>;6rnM(>4Jk?gL4KHYN*2FGO*`R~TC0ff{_{c_zj5h{Nw_@M zmsPt~oEECE)7&baJ-4t`dL!~hQrYbqhn`|cNAl@5@|T4#PwRgiCcwZ;_DE3mT|)3n z$9g=^xCt9R4rZY()9L4)%M=aA*Vr}}))T2;APWo+s^u4-PI&VDv)nKY(M)P)_7EFs zXEQ5+d-+zXRr-3eWSH?c%xSbe)`UKqhRFeXj2a;~s7OWJ>XbsWKwi8{?01U|jBTQ~ zql$xn2KxkS4MmkDY(nPs#lb##dY`1#-?Be==41?ly1+mk#d>*$l$z{cgH$+uf{`W= z#8QL_19c@Zs)1n$%a3Ju5lx#R63OEBWjz6LW0B%rdAJ&fhh~sqq!1IO8&A{e*bN3t zOk{R!N7bV1Zg^B_9uxZDThM;RBqQpG<}v{{k{5sb{P*uTI$LhyNIJynrqOaJ+zf;K z(BLYp%{r71Y4Bm|anfAHe18qn@XaZ9K4lN3PzAcL)S2PQHQm*~P@Nz^PKAu#QIeWW z=NQ2Ls!s!_l)?3Gh8^BYZAR>8#KPs7w~$F0cO}F`vzJ<3p)Gu=Jx*I+)cC0;W>I0hZAYxiUtuIa5Iy~2cy+6x1j8(f?N3pvBWHXrxTT($PuJCbo(W+I}7{0i2T; z7&LLvT#{&`_jTJ8rOTHu?=tNM%n+)y~RozH+|jOu)EJInW(wi z^7{NJ1-nyg4sovPOqKA516pW{F7+2frQWH|mLgJCEZ8o1k)jofEOO^Q%v9!4(hrUG zVUAhcq||bgO6SnJhJ)@sFRlceks*$L5T_fi(snyLy?Vu>Y+niH)Vnk%LGuC{>V8Dn zR%of84dWF=s%%|t#b2otj#et~hF%Q2Zpt_1?({`4VaipI+}zn?2s6K+uEMMeRO*^W zCw?Azl@$N==txqy9K#qp-aD_SPqaUQY{+?=>~galo-f0v9H|8J*?3ZJx1t}3Os9o> zk6Eh_viW&-B4=+k)Uj<{W5Y4H)kJv{kj3F9BM1>yT%+2sXq5hFs{@S86zZ+V9*l=%srLbTDi~PnGLs=;=qZ(nel^Y zcbwKZ;)*C z{R3UTpxco8%sVM|f+((ZbtKgw(v)mVGKr|%!Bz|%<5dogWIH4e!dy(|v@dy7AUS`< zKMdTR;$^7MWl)WRKgt;cjU6j#)VK3gLLf1x?tF%?cS{69j-kbx1)ccnCwOUnE(!%lMy6`ISp*DV|=M$)EG@^BB%3ilT|b1nl3$itxraE@?{hlXq&!%ulPG zMLAzC*CbcHGT|i*N%GVNRioa{1h##*s_gV+j>0U_?IcN3mki$q7)Fz0-cM?!ljUA# zA#cFbA&^_-e#mb8Mh#5AwUVAg7J;(OxsPYnz9nyQ&?8#V*ctA~#_q$^fmCHo3Wnc9KS<4%Q`c7SKvGlUg||0#+D$CAA#a@l>+&caOHs$1X(Ddv>^p zs=B9ZBKt3+)Nj?>0`c@tnJqbizwig2d;@5?ly)<1K34+A2H@~M1RZB7)5_Z_1 z)6B%-v7}RPMNABR(@Nq6dh;ZZrbhB1<9ByQVpZtesKoPkrJ`e-0S%^1akCnCv*5=yg=BTxLa`MR z_luf)^5J(2S~it>g4$-t-}4PQB?^K=t((*eH7>OcqoM{fKjp2y%!2W{5>g+7-I9}l zE7#KCtd4GE%N|A&wUXGmg!ViGUAAmUZJBKN;%ZumZ@Mo4et*vkY07ee-WV0%sWtRA z9CJu$66uEo=%xOfDvR0)GeXBEqEik94Nd>9%*%9jI%>t6<*u9-6)y0zLRhaROe69y zk`G)7od#zKn-MDwe^a0iIiGl$_=8DtAZ zIPHskF?)^7@z6_`gC0kxgPC90oCF^YOTK zC95mq#bRw-de5epw+{Ed2n?-Fe`2^V!J+BQDpl;?5$N=ZykpqeLY9QcY&gC4;T5-9m&t$9iybUz&aUf z85C`cn4piO!(VY|=S3?x=-6vmuVfumb>(JB7(T+G}*G?pNAp2G&EKv1zuu;bZ zpS*!zB2P`v0W#S9!Vga9n@4J7)Q6h1X8+S?0bIv-3WHPdF7>CU#XskXK4geshi`&p zK}D7U%11A^K+7MCi>t)7CD2PnBkIoJWyzE#^hJI|8JUFf2hz9X;!Fic@Jd0bkDXV4 zDCDV#MTS*i2eD$q)*hkuj0e(!+cSr{=9FR&hTT|wvDG}Hyp@(C$K0F(Deh0qZ7j%t z$<~PVXP3VbEjYQ?{}^ivutiICI7ixU!J?bWTd`)vPPT3Jh^c;ofnAmH57fMGQGU+E z?Fw2}qsdo)+biV1cRL<2){FSTzO3ZA<{~cDtA^34%#h6fb8N$-HY{(niHJ`}y8QP- zzQpPumZ`jQ;RKWagsZ9>u;1p#ekw&$#_IZqF4Uz(TScsLn_bLWR@5At=VmvL5nWJ0 z^%@Uct>1bXVI?XOJD;;MH?3kFjV4_tG*1}so)U{#^J==SNKj!<%8pEcwiA~|HnmM= zTzt|E&640^zgNd*zhFZZU|x?7=iQ}_lVgwCDXZo;u~Y}p7=ocy+6#@@^=i3QcZjME zQmIa`Gw@WvPS?ceU?D)ejC3U&P)iuT2vF;Wx`wV}%F83p`w0H1hzeq@TY9nc(Uq4j zcQQa99GlmbtU8rU_4X~4#7Tc^{cW@o+DUN2Cei~jfh|cDJv8A0>vZN|p(th^{Wlb$ zLYt{DkdvwY#3p-Yyc1(5>Yr=3!+#_0Y?ydh1L)eLi?-A=@1|txSr;q8Hco#Yji)|o zhdyI}7Y=viCZ<-R-Q z@hrC0HjAFsoNTlXpVZ~UZp~P}{3U2+U1thc7jMdiSfUTfU8>`M`;Y_&^gP=#Cqzi) zT9EHuJ7;GmLSZY@?hR1pG6`L|E9TELYNbDIja@QULlCp7F4F64q##{qh5eA(0o&5U_{?_OHSZMOTQIsJihPn4YlGp4 zJsP$6t#96PnWINhMiU^Koz#82@(1!rYg5GUwnX~W90Jh8EtAq)q1do(8DX7{(l)+M@wL6EALt;61!^bYWe-+aV|UimuHUU9ev;<&M?Aws z`Ia3HvF0@hfKUEaZ^xOrWp+>zGfGQeiAsoTCy(d?8f9_CU=j(&W6vLvE#Pe(WvRwiv zRu{V02o^?}9q0nU4eB*V0I;!@YgaV&>!R`HUXd=otVx4ReTY?6EXOU{^{>T9>_O6& z%$nJzaB4U=EV6YOGqlA*{Iw(thLBRC1fhpX5E4;E4c?tfzr}fA+Xp+bkA?5(=aM4w%0G8T8IUgry4L&1= zlRqq5$;h-mHa-&a4T0QZUa?2q8BO)|VTFCX#JiJqKPMbZrUY^n2i}6IWo~IkcW`7d ze?U$2CWoV*#UngdzpRQ(*2JL1|GAe$bJV!*+{rY>MCSh1Y={GGJE|b0M9)LrDz62I zA{pL7A*$LxT5qaj(A{L?Wvm!nB)qEW>5k^#CK6G43&G4ik{8_LaezP3?-6ECW3(yn zrS<_Q??*e}>|Qx5YLqrfy0xBoU0HUITs2r8wugM5G<*9N_&h5tK>mZl>z1@hhb2z} z);@RVXev^|H4>6v1v@gVa?b3VwlVTxFxFxfK)Z?#ayi+v-F?;FJP=%aA_`)5UwGCy zqmgj&7D^=GhOK@MpE^+JdSd{Bbk%_%+S;k+t^WX`T29Z%*N=vos`SdUt8nZXs}vl* zyk+tN{wl5e(uYYBXI&>ct3Oyj^OB$_z28yCyDx&BHiSjol{rOIA+&Znf`*u3!mMM6 zulUri2CU0jn#4tIo?ex6na?}~Y*%x-Yarg9+3hsaWxdSV`J_wndW203dfI{-6IA&l z5#tb$v121~OwF8EAnh5nt*`OQweHe7OXH#WJy?tdux3WE$`$q(uYqPhjz)#sD zd{~oA=ie9@E(zgb9GtK}h>l!4bgM2q1bgjTsYM31V#~tL1Apj`l2KV$JS|@#GVcxa z7m(PnQK`_Bp@t=)8=X9j=ps{B^TuPx9dmNZcXyOJ34N-hY5&GF$o4&Ph@#TmNb%Ty z*l}$R$3THG=7w`*m_x?&zKkXeDGBSP$?{H1{)Ph7o%1)r3XEa8IN zzMT9d{;1q@2sx-E3|+)MDDM(Y36Uuq z`g_XRFEHV`2_f)y4JV)mSp+g@*dP3$T}B84rcMAepHgRLI1Wx3Cjaf zUt6uLpBar>eS3w=+4_h065E0FMJXjXTG#yxyj%Z)N2bKEmE!-Rl?1EZg?;KToN9Ot zW?;Dr(Cbeiw&T4>q!E|AV!e7zO=d8|h@Wo%b!b#(`?0~tzM%xQ^x~K7oJ$l5O2jDSH zX4P4MR)qkwBhZ61U&UgZw6z0BWbKyyXi8@%pVkM=$-}i;O?a0VcUFGLUv0ne$9mNX z!o$atj0wXB)&Hb(@N!V-PHZDHqK2+wRTcv@DD7X6jma+!ojT5LM!7n*7n!U(@!Tpe5IGh7@T%I7C1AxPe_K>4oFG@nB`0!e7Lnk}hQxZp1yI-}GfwtO~%1XOUxaqe=uEX=T5W{7H5j zNg;IQH=+q@9;;d$Z}H>eTI!$F2VLo4E-y<2EtMm6(L0sdf?( z&0cdRhh+Y?r$?GXQ>+Yh+!weoUvl9}nl7v%s^P0%|BolEm{&qaD> zTM1qhdpRZ=xl#&SS|+!?j_eY@DWRD4`{(+xy5Gw6zGED6w@FEm5f~KfzAK3=*DVv+ zHOht0FHfX-w7ftZak{7K-es{)_x4Tw-jqS^6_lF8o2fV6VwrzRJZ8Z~6FU}YZ9;9o z4N$S_U0i6-GZPvW&qV5rzVLyYln#^q#Z)WH-6RmD`B`mPVMsHlqQ>SYYSAzs0RhD!N4 z)Q`h9ou)tGBMYGgD>uJ!`+)}MFCP!j_$MltSwx*c$JTs3CL~g>^dMU})TE8XdnK_) z<^IW7vt8xNVBEs9Mlo$&o=WcDGoPVqysNi>1>eaBiz0o6E`=>4xhKMbHGblCX>~aL zp;pK*Jti)dhHI?*!frwlQe?i@3e}+Ky(|h;NYxWcp*=n6$*d!48p!hn4{Tc#0G$_j zl?&VwoMTxkq&uUPUHYpIw;4kvoYXz94LqQP#Ix&PB-?Tm%eklfD7~tRcl{d@V+~|~ z&-O`;NqMruufi@C*&X|D-@z{!>-`)vvyAe+G63N^o2AA1ElQb)Cy&5Bdw$x>g6H4? zh1;jVXhV2&nrblT!{c)r5=HEY#MPl{a@%=KR{n!X(`aEHl78?~m?bI`8f81|s^zYC zE&($i+MWIjWrh;yEnim*&_4}42J1fKZ%>t8L^9Jvagqw2-T1lOyMjXw?V_EgoD3bS zwp%}aB(DeW{M;*hpM6we2D6^#UEG8zB1=pmx0Xn2`g{wIrYC%pW0BjjS%b8fq}YOKnf#j3*=vPRNlmbnCoX*X zPUZ=!Mgg)s{ivponX>vK@rHl6IP@ZO6xTgs97@#DegeQ>VJS?&P;HpiGi%H!eXKIwDINwS?9(s3M>82=)^ryB@VF_9w^+X zB#?4NLO>&K4k;2Wmffe^LVJ})45J)0nyny2SHb(3%gfYrBOzZV_VeYp5Ehby50jGd zwa5>)VUy4K%62^ZbR}CY)P+)f{79uJ?A`ttkyb70V#e2zSDL@6z0?^7FsT)+mD}_1 zW#drn5)WRq1WLPtw?HoRTG~h$;Sa{&WfRAHGFA5E;zp^o?^kqO*%|39>cr7VY+cPi zP`~#JovV7LozH8Pw6{EKrtEt8BJgJlqq@lO_T0CUK_Bt+rXsXV9}_?zJV6BztM$IN z9vi;>s$H`dlRjK7Z2MEvm^gOU;!9%NrN(?qR$0RIqXioCVGL~C@u0Vh5NTtel8+PV zVz3A1ot4hrWzQc7)qy&KtlLrY*FFd>Pl}mHK(u@AINq}z$1xRED3@qyWuOE@230ct zr`HcRyl|o0$W|0Qi|5mz*!+E>#&#k;%s}gtDnM}4)-~erl4z@2`Bu>w=qUmKT}*C*T3@3~(n2dArT;EOGMQyd7XL7J#3%Qll&}$o<;^^%`ccL+EFFAAZWCDIQ>CXq~OM4cwtM?NvAq5L^$VO z;OCHXp0opI4z;Zm7n$(opw!B`g#_!zPZv~4MQE!3X0PBKAqe#lXr2jr0z&inPC{&VA+BZ2UDQLH#(&KmOyws)&X4+bFC9EGt@|Rm z)NcABnt#(q9^fmVD~;P&szrx9_m$S%_eSGyglKR4N z517gmBT%4$z^{`@u(nnkAs^@oCc6BV+EXi-dzb^0#=G;fFXV@?Z@qnZt|1n(Bz!LS zU?!y=ZJfYPtkmgvYAp9FI+Hl+J~F%ND%?SZ+lP2Wm%5#iUPlF`x!00=wqs7 z)1?%TB87@Vi_j>2;13=|N_^ibe{|y>LgPBV)ry! zfYf&|FKUu=C-d`jLdhj`{3761U~3D$r$_5f908nkEx!`De_82*{AKUQTvyS#$QGTuvKfEuILqMfOVKCXA=x zThHfXqgupuZWc0LTPrK#jARnxpz@fDUR^~J9%#6@9dbjC`wJ++N1||i_Uv0N?T7h# zKzi(e3t5#CxFaU3a}iCr*j`a*>9kFW>OPuM=?Qm0(imoa?`S8aWVO6=-f&3Qp6a(g zzL+Q$UEJQLRwAd5~2W<$HqU z1al$}TIpEbBNrqh(8zj+^_h}aYYjz~)Fr!3v*Ks-&%vQ&iW*WKh16qBWiiqgVDj~yO($ls{iIS3T0(=G}@Qh$T zsEj$zKFYRiWI-&KZu&(e=CK(8g$YPG(R9BBML7f@W2)`i5iSH_4LZf-{KFfR|JZYV5 zT9b7XBT2)dq$bD&tN`Qw&;73kcq_%?V=&!GrJnp-WNlh8mg!@ z(B@QvRd|sVpYYk#Dy96&ZMh_^!@OAP)j^YReVPVV!_6U9{0z$di+$g}8%>%1gm^|_ zKWR=-kwrYweug?!?`_9=f8H2dJ;(@9(5+Ci5{~bFcy6GW{-`^n`$KOkidw zER|=Vt_wm-R(GddB*LFk)Vif+@ksU3{EsY?@zS$Y^$3e=-wAYSY9?TDa6SRdAtfJVJDKJm}9>Mj85aTJ% zowirLk|Rz*V2#P*!MiY*S7OMe4DgLPbva?<=yR|wWqQ3aB*38juu!&Dtd+~E$MYR0 zWrgY6ThY+p}pUERk__rOpek&xvx; z^1xHNPv-G)c?KS-t5%&cY?eWO-01ges%0+Ql?)3EvgL5B zhtMs^KZlKK2Nf~oidys&O;#y3VcIEiwkkdzPAM?3`NRb}F8KdhpFvR?{u;odlNdFR zcA=0Qtmp0--T-9n`h_%)FcFthyn21#S+m}`LKnbk82wKu!K1?lh|dCL5&cuCaraXRK3V zYh=D4`ObBf#v0a5W!x5j?kmR^9E5z($ zSMa0BorS&k8iJHkC}}DB{~0{sfygd?}ykoK;2NT za=?J$RKj}uA>3aT%2c|;7_m-^GJPQ2wYTEhi)m^Xm2eEK>FdIbbK2gC9eg?VRLYZ z5U0TmIHK_pdPK(A%X80mb zF{6-2SA*Pkn^HUHOjHbI3q8Ku_Pko{D{U{77NGa<{+6cBv$1WXn>r(Uk_zYOSAbS? zH{$<4)m?VZ@GDq{8zly+2}7om?B48Wj#s5BvJG58`xDF1u#+NpGj zKlLlat;dSWhw7RjgAk4lnBL>sr)tJ7sa2g{DjGfrSjk_@38w+hF;qJBJWIcbmfQB|cR+kY>rp08(%_rfKkjl&_B#TqBUcFZr%)XZ!pi}RG1lu&r z;(V|$bX!KD->6zP4#YEpx{%G$tK;J*vCa=`xkd?bsO?7xCNPC8q#g4hKU|ru5pDhn zxV9-XsPda^DIp7o8TP`N52s%8OTL!Reh!M? zrgxW~Tw2;pA4hJ2U6%eHgpJ7f2n8O!srl$cnEiP*#1v+3^yQs6sTS)WZ!*1uJr|!? zRkCp$ocXTkYgf6`G(J{vwDCuW(6NjGRILQD z@9Uzuwjt7)e`pw9BKRJ6&C%)FkAu$HAQu`RO?sJrUlU8Bk_G+zCNWFjF2+wBX*{Sn z*DSiPM3meM_g{~)0@iy-S~~TeLUh!*6k(~EaKL{(`5no&YOD(J8!Pbs8F4j4>!L8!<-Bm;Dhhj3=Ag)qgW zM&#qV{XVamwPaATxQCR~W3pj{Dd+b`nHnu^4yvxQ(X$P0WmsW+>uH}M5^l(1?hX_3 zvZ@%=pgAAj24ZRCx<rI&-5*=r)NXQ$l3PW)nqeY{!jbt0XW z^={4h)_>8@v!9xQK(%61Q!@Ol7U!cW*xppWo4#@~-gEI10hO|3=H9>&+w!vTyvSbX zkUy2{WV|E_I>c;S|9V8DLugr!V+RbEp^G>g<;Y!i{_H$&5B0MDmE0`Un7I29%Ks0z zBNw3ah}JZIm;TIYD*E6N61sbwL>GfQ?-Rh`i4bwch{`C%{yO2}UgDo+3iQ~@qS8{+ zAFDx_0UqFOmoh1pJZ&WZYaDLW&lXh7=%Pp#9kdv{J({k0+2fjMq_TRHbB(-w#htTk z-2)ev&O#@dOHK z6;ltaA&U=yrOn`$9f=3_&Q-;$r=YT^c#i2K4$ZETN&ThdQ03n34+I-Gz<2CL6-;G5kTm@`-TQIGkh6r#Fs*{99DqEYEVqA1S+r_Hc?fNhYQaPmbWG zsZPqIOL+9EY4ax0FB1S4Vo|vP>zb!P23*?7&D?b7)EHP{j|)=nJ9R%&Vi}h%eHr$1zNveuu-USd zLs{qJ5YIqdwPZ7XE8_8a-n5N~_Gl##CdEADIEgS1BUq5a!^D)|N3aP^x%&uQqZv2@ z-aU`RLdWE@8e{<((c%4>dGx*uiTi`5O+dm5ivhUn ziNDbJNYZXJ`XvlQnWoyMsOAfQ$rR@YM~Y>Ye^9sv41Fa<1XA+MYGLwVuj=EIL}>a& zV%ibaL<>Ml+rwBNtQIi33E8`4_VhT|S+ABuZHyDtmPm4f`8aGKSbMB?BZ>#4tuXZx zi&Sb=Dlu<|Q_uCPPt6NnmX-J2NMI2j+l=BRXj-R6n`UlCxtFD!K(oZkpfYS72@!+2 z;xwA;{E~_FY0518Iw22?ug3zkYY38RS>2Bobuc(^7khj$=?E85wq}Bi_Q46{bzcs;%cp`LrdjPjd`$SR<5mOnaMwz=oSHHlp!(J>2Rlt7@c=% z0o!JV%NltAd0qv$N*PwMgd_YOS72)51gQ{l2qlvSu5aF7VE&zJU>WHPCLa-zuNHl7p zkjv2Oc^T0>q7VE7L4A{#rjr8Dt*uA5Z{|i@94h}VdsYb=01MAQy0f8$06_!QLyn}r zT&7~;z_XT8{LqEoYS(R=Y)x$~ed7pE+{1h8pchQU_2Q@o98pnFv3-EA5TC}TOq#rf z48iL4)p(230E$^`D&Jduf!F$&ce&Kffoal|;!?6LgZB}a0nKETxOJ%R0DL3rm6Zz; z=DLs?Hv8EXtu%Tg#Nmn*5TNEt*T@<^%Ob7s|1Z1|mX zi0?{}M9r%Mlv0y8(mJJ;rp7q(MinYd#rTv*lqS!^TjPV-%=8s5fkH?eec{!09fp^9$ z6#X8$x^Gmbof8nc4hi3fp8|r8ZH(QrHFs422U$tfZ!7JNOwQ`;ILBoU?VOCob!#FS zS9U=u&ux=0RQwlQ%keH4^M)e~ksU#09@|-`xft6LW^^f0tMxiv8Hn}149}fWHDLV#{GdX&xe5!XUkME2myLm+2T8+PF5JlDA7o9j`o6Q zm0*fNY8-m@q*+@Y8BF5aUIcx)&%qD_+qydfB(^`il_gUe2|1XF5lxg+Z53&P7DZ97 zht$o;qZQw`BWW@@<+aqbiCtQ$t$5ic&&=%F{8$x>U~)=h#K(@z!+GC|pcCoXIaYB? zE3@hOdSorv^i*=ZTuuU7MgZ=5*cqm`iO<<*%_P5No zb@l)~K*QyZG+n3+L1#5eSsl2Iuu#yZQLy;VdYkcjK+Dz~@L41??hw!R%K{IlvYq5X z=kk=&lA5jH_R{&(2uU4q*cqT`*%#v69UnL9h=t+905r&4IRS`$7tEf<5qA8Pb%(Ns z`LHx_QFETNVd$Ca^w`5hB=C;f1(&9m-R|KLTE*RqC)nX(5lhsT_&snWIY0_VZfe%H z3r_8k_(nZiw(#Dk#_pIuT!6 ze-jyZ;qAzsdG$C^K0pl>(0=R1k-X)X3QFD6%gKq!jy8G2xeW=KCmbL}h+XrHLvJ2r zvPZp!0!*~pgb$|&h%bw@JX+`EqX++CP=|Vm%viNqXy&b4SE{}+Aj)e6SH;#OTD@Ym z$jPf$*!#ld$*n=!$xVJujt6P0Ppq;^`9g?%==%Zfqyo7KPTx4g+J~b41n)RfGt5!db?sAG1vK=v51-JqK#U~-i@yz-RVZ=kVsS>Bm&3=HnR zp}yQQ#OWTI$;%9?e*)`RMX#4`lS@OhX(qUVb<#r!sl}QR%-PLiEcysNkx}36N`T0Cajs8v~AK+J+yH5d(HH+#=sOnWT#WmEi#UYOv#`O6Fh)-CE(%>Eg zP*!|*HenbGDzVQvB%)DY%1$$iee&mch=vkf5aAzPbchcy&>L`>Qy#cgc~d%3H7Toa zaX679_txa~QU1W+B25)@J3x!*!jC+c>0~iYqUOlB*)XyhzvnYV7)~1%9e)HtoA&3< zIL;@eO~i^(0D#YW)z84I0x0~$D9$!*nH?fFZ%hRBddW*>l?@^BNG{zAZB6o-NR$%+ z+{bIE7BRppA&s{eMoIEyJ0MEWON55{jC5ZUgX?8&ocrCgjIRA_>JI==Yzdv{gq%3@ zMgb2hkO<2ygV(yWxW;ZR!F}N}op*ohyBxL@x8qvx=u_l#0ri{Skdm*rR}dd|V$L2# zXz1tSQ#Ual-lucnC2{_CGJ5RlV~Xn*+>lAZSj&S$WMrbJbah1Ze-&gW43zlN_EmsT zJwKz$22!~Vo&MVl4|}M`S}>#OLCG#aD`vR-G!trc3?^YSCJHm@^7ZS3GZN<$Em}Oy z29<}qzHT*se=H;AYZNb4E;>u#TDQJd0SZD)B3vxvf^wU7w_vqBDhGR(e~tpFa?Fhm z0Z5yta4e!@sV&dgLRt%~XuSu4>CJGR-O|QlbH^%pbk#?h65!VNb)de#XfNc|a*LJS%dS7$1WzI`sAX#2$ zWu!56J4wfK^?2K+Bcu=6>Pom)DUV<_8yn1(-Xt_O9^ZoSTgS2}XbdePBPtmdIa*8L z;*q;4{>ac`L_TGZ>>vmPL9J=sKz1GoQnpz2g=U%|fR&q26P`Q$3*rJ@?onM3&RM^0TSay*1JQlH7f)9s5mk9 zwk2Y3S*}urII^AihAHa{*_+LEC;xM0d0V`$AP2~AEM=Hta9M+5RThOMe0)F=r_A!$ zO=*vZL`(<22kpUg0J!REZ`DP2q2m^OtGfx${Mvr1=o->Z>xC6^2 zxoc943$&V|IyFh~{n)s*WVK*7=YNk;QF@l~&~G z^q@iOf2<_KGxGB9Lk$2Av-eklw`mPK)!gXV@qkE(s)@Ka@1m z#K;b)CZR?Z0Gyb+O0?z(v%m9$(E7HS)>7FJ<1st-Cjym!IFo(ikw9KjNZAFSHE49k zTRVWyv%Js?!|bIMJdb!0Q;^(~2>I+*iZW|{%Sc|D_3i?_dl4Qxr%z;tvJn_*3_ZE_ z{C?m{ZZRNGfcm><5S@5MjluKf&gxFDl@8j*$1p0H%1w8psLogM@gy(@!Vb*F)?fCN zs4rBz8mC?tw7%AdgH3X61JDYtbovS8u_(l>yt~=rUN?g=8S+M#px1y5YidME%?^!Bx6ZHg;z6N zmJ+Z@faZ^`gAs`iY>fiEz!P`E$m&_+ZKm`haXFgE5d@E1f+fyt(cdSObEY;#^{N@i zf2Tyb3K|eyk~PDMzhF9SJ}#wlyyT7YQK%;fl{_m2@2NzFt#^W7Ls;N#%>>a9h~qI8I&?+gM69&}h8;EKcIov$srGsprcyl=HIYpb;~ zl`B+!$}-v=P@m_)j8E=54*5U_6I#CK`O8KU?cTQ=qnpTY#EES zkQrQIq++BG!$t&}j0mQ*+BE0HZ8WB7TT?n@6^wKEG}tnLVpM3q ziLV3^aXeLa!rPD^N3}HQefIPMHF`_hJVY>&5^pK3f)8h{@ zkP?i?dt$*E=wN+;oppTM_mv7M?(Fa*xWh>m!%AIcprH>HPF+SYoYDJU$hZO2xEMR5gvsImoX>YlqHpM3qFO!hU+*x85Yw6zQ;&m!aG--O>Mjos$5m+C< z_vu+ehO@e;WYlfAX&44W6!e-gs^<#0OEuw@vO(Lbp$!NT=*XFMa5S3M{(aMo`)YD6 z%U_ZjSS8k7qdqVw4)Lz=^N;5>HINCbA{Q3-GPwmdP<+G-kZZFQC-nqHpD7e zH@zTg8sn(bdqp-JnGhhR&b3@YNJymZgz?%vAt*qZH7p(mo!0 zSibVLMFd}O2$~=$Gc9UK*$E1MbFf3x4PZSFYy zvtF&AhX67Vu|K>s+~zt`7;~-doQx2Rvf?!*Z#r44nUn|ddVXsf2mjTg_W+vepKz>RtA zkg!%g3iCtioKq*WQ58pm8pQ!7S!VBD>B8J}aI4d$e`BBE-B@)s!40%p%$_k)_Mc28 zzf%l+oqFI7JKvyYZtqV^>QCMv>x@#PnGC>r8rlKbbHS2w9T3BAn#R?jvtNXNKAOat zRO4hDCuqQL$X7mvfj@)oX^`K0F@R2#v68$82rI}3;0=ru3*Ac!5fi$2>hQ2I^_gf& z1Ta1vUM&-@?VesiqRn)Fny~)Fd^w}8Mn0EPIOWzC*si)2dN5+;Qk>;4b>pGkmwmFuMGOg#S-a zMfZ7&on+{5)-h==ZkA*;@c02GCx5p$1b%IN3Exs@T8z}ADWeodbk(RbcRh}U1xLNE z^11`wChsuwid1QZVDS3N8Ut`AQy$lwyc#g2m9t3O7Rdk>NFvJHM*q5|WN?QNj>z>uu%VNucjb@t} zQ%fdC4*jqL8u%1%D=dxBaD48Blv%Tt<7s)#g{gHLEoriY%;35#j4YARD8$Psy!>oedGFQCWgDw4sMwhpV{%d_~f7stoD*cJq# zX&`6v<#k*}cjGD=<*=pTgf3Mi2qLajouw`_@)fSnrVoQjS&NwLmS~w-$4nR%UDlp$ zbzNa%Q%XV%deD6qEa5zJNM`vF#%8FO$sCxI`Wnv|T5*G3-P(TKBO1bo_N~Fn<#)jo z+?giP9Pa5W(`5iKATIG(?ZT3f@iZ?Z@K@Hv@uo@nt_k8$Ib1i5D%Bc?;FG&&Qlg1f zOw3M@ty}mt^4`@ygkL4hP1pH1l(b+PHy&$B;ZX2Y-LEcE^yk76f<>@&aN@%FYSQqV zBa!UeVYQBooMdC&@s33}yqj%6&(Mtp%Ny6&@`rHwWx*~C0x5b<3`a_9lBu9xi372i zEnYtBJFA_6)<|19_Rc7Re8eXWQ|I?Pwk~$mfCtXVh^#G-UmAX{T8)#XmW5R~s|sIo zeKdL2o1sU{htTXWh*7kSpC!E$PvrvFDwBYs|5M45^{u8Pnb>wY$}1OCP>W$F%vD{l zdKNs0ynVP*zpldk1#(dpWtH+FV_7-6{ti+pX_le{V@ep-~hejvXJ-~a~+0dzoU6F^e_-bn44wUxs%e0M3 zt0QP*(B@Y)2x83Uh1M)&b4kRM^w;WJEQ=&_=D)VC@1Q@tjg%;o>ckj(vL|!+QnMs) zd9&rG^W=*v<9c+MMR|H65&=9Wze1ypWeIlMWW@zfLXxSc*<$O(`!pl(?9wsm9m1vN zkS$$FGa~^+PgES{4fy7u>OEgnThs#+h!xK-zu#~f^5bpIPEsTO8{6QMkFKbIZo5bU z1A(h#_iPN1lBXJ8uLQKRl4?^t?L8fihI#W7oHd^n(Kvx-jqs?d6k$f3zWRkGaN~DrBS11CLVe7=;+) z!UJqNO%{^0Sp-WVhb$qHjAZIS)9G!-bFPUY8B|}iB2GT)zdKha4c6Im)4X^%*UO`q z0bgTg8AsMb7IjqEW6s6T5>z%mUi=&aR`C@GvQ}U(FWey*8nn=#+#PBSpig9dP!=~m zB{k|b;&%g%{@|!{s;AUQ?fICBh`w#TennYKA-U)Cnu%?Ml_|3iBYj_q3?Hq@N4e;k zybR7-Ohnv9HR%Br;!vpIfr`t^lk?@Tu~JeL_S0`%t8R;G(Ra5M^TC>=VA<(l)srpI zW_xvmD4xtiYte>S5u8n`9E7mt^+e8wtG$mO#bSWyRx;&f;q49HHLaY`22>c}44Zle+E)E^>L|SaPZ|hKJ7J^QNKQ0%+CoJLb9M*#3zYeOiRT)_ zV}L>5UltB|{419+03T$_r%(igq|~~I+Q!=ehD!p zw3{V4a4drbP9sCtDhm*@c+*Jt3o5;=rY0)_$Nh}^*I`nbtAi=62 zXdTVHmzY!18hRUOiUMW6UkB9OI6WGIdiX;ce^B-^R&sl0BL4M-ED+I+)>RU%>Jmr# z(JQ}E8^(gT;eEt20p86Gj7C`#z^bTe!Ckpgf1nFEnwtYufi%VnikvLJX1(sA)GW6h+2KZ zslt_cckV_!)y&24xr^OA;Pe(iKEt<5b+QYsV%%P5gudh{gb`j+oFi6N*)}#3Sns|A z*cHa2oL^Jt?d7cw8=8x=!6TWFb{PFwQ&lstA6jF_ccua*wW>Ch$Bh79ncqapz#XEx zI@*?Pr?RKMJ#CNKX7rx<>IQyYL+@6G~`YpLhf}mW$O*m=((q z9Ojecsi)JDZZa5g2Lt5w@AcidOR$S0gPa$hx zc1}U{fE%s2k5TVlR+k( zTJ{eNAxXq1uCRW#-2wSo5J(~PP?^_zD;pm2kc|8bP}=iQzbi%2P2r zJeEUTO+zYER&>7cx<0@c(2=d>gp~AcqrwV^P4CvbYvA#2w}*KO!ydVCc*G~LDgKZ+ z@0ngFh7xlF_RO>`#B3vtZEHlEKz3Or9+M5v8IL($fmFNj;ttSH!EQzi1l33fRxy7q`+6(A{daDoe?ihNrNu=XJj61SYHC& zL>ir3w5`}IT;S{E=BtW7-sAuWVTV+Jr|Lq8iwV6FRw_ze;<{Tb<5+%i99qbsVCNIc zpgIZMMK%toiP)w(jc+?w5XB^Qx47fVylf)6q~{1d9r|A5fcov z@m_%#U;o|BijRKvl6=a6Ol^egl2@kv_3Ox>wYFbkAL zln&_1ti_1jjueW!%Bs$iCn&u|Vb7$xI^#ToC1$nnHtL(4(RJmlKr;xsMZWxc*m`fV z4IpAsRZxsWR(}PoLpO?R(Hz|hzBB_r6Jl`|$R^jYk}I7B>lpAw<~@10HPE*4fgrBl zBgK`l1T)EUYdNNtf!$6~+ohz6--qvKz%D$Q$ zsSM{P3|B8v+Yn2}#3Tf8`80H0e=z{84c3+NHqsqvHe=|@;#5}XQukg0dCNiPPvExK z;@hQc;>1?s#Hx$yyXu=Hb9crez(#EG=+z^<2cmiRIb%RY}|DLI5@5G z?X_#gxn6u}xR$ZmCvB-P^9@#9y}aL@Q1hkkYjka&$|S(0GSTcQYR-q1BpKeO3VdgghMxThhhn+e1z_Syys^51uQEF@>KfE;= zI6YZw%ZI$!@q9ev#u=^e2{*xdX`SsjXqA_ z#Fd^tL1{^?yg~pEK{=T$DYB&mG>n7Up|_UJB*M+!7tVCc<|@3^5=)X3 zSN6ixzSoQ-1hcfOXgiI-i~?#32VV#vV|)3z z=`X>?N3S9flkYW_I|+`-lT(^jc%YHA%G)vgy1hy#j7#_F-Ivr{Fts&W=+*n8(ZQEj z{@@M~TUrqL3|L|H0lbx*ncQH(Qlok+T#QewI_*V9YGPmcS&#LivOY0XT{`Z|!z##A zk!Bl2uQQ#~QZ{x#M;g6)@cAt#nCi2eJ=J7-9Y+8%sq+`#cLsLpY;+;^5MV%Zc5kZN z=#9Wb8C^?ik3L+C`;anbw`b)PIOjVvIgi6ULDdJ2p9{aTvAKHDPK1*W& zZ?b%c-1i@hoI$@4L&6^VM#HwJOqL0S>4&HKq2D;q*ML^&z-N+l=4JgQG(8yq^gygIoEhT(G`gdvkvOM|W<(*?Tt};> zF-k~B+RX>P!9Ey~nxZ@uUUER)K*$q{OoJqFRMZgx zk&ez7pRzi}ZC&-lf7S9BYk73JkH6S4t{8cglkBI|$NR=GS6=&Ehny7E`%kYGsjD@h zLkie$F)6k|U??~jz@lq;JD^Hj>oWost81IYR)B^9elNpjj&4p-)y6KOf8eb*)%A?&fGEQ9NKeX|D_%e*5K`y6+p zMu%VWXfL8l5;}Hk%No`gX{HLV5z)td@Hpiem&04pYpq@@a_)4FA}ThcW9FyGiRujt zF1z2U79)>aJDK!K$$4xE4ZsnT_?XK{*3Pvpxi?2!ISJ0-iv&LJRb9%91$l}m8jrA^4X6gUv9vC(^g6qYYW^`yQ5~#r9RFkP^nt* zytCdl1~Z4iG(%K;h2%Z`H@Fqh{(Io7ifX!EMn8i>wD#Br&c9)27OlOQ&jyKpxwP~z zhX|Q8i%Er2rv_3II0sHRE{Qho~KbnDr)hNEXN=~|9=lI1V!@?0B3HBpu!$XGa|Hw&zU$55rjKG!` zqG*Ke*^a2@ZAUGOhBD!Q&@CXF$NV00ooiEsU)aog@@H==A&|=giv+Smc5(B4Rp73U za}}R{pxlase@JqtU9++W!MquKFJXtt_TYkHO)hkCDV7g1Sfn!>_}%*P&Jpw5oF=0dMp`^g7b30^!Q}Q1`w&{M#~j(Z8wS8+ z`lepE4#$aQ1LfxVb^VT-28qFk+ylmg*{D4G=oEo9HZ|#Lp>bB>@g?ED_1yh-XCz63 zP>BTmuThY3?|Nd|RRj*$WTVV^rHp|Xu5VGMWQB(S8A?MoW;%?p!(nPKl9WzEfO)5p zc3r`i5o{B{kg$JZxT7C^~bUEx~Sboyi~^h^^cxuONIaJ?W{Q zaXNp(;c~{hOm$`!HSZ*_qPnwSWbjb3krSQ)=bM_%4LU+h_7ze9_(8?Oqj3nRB0PM` zsP{0*QSwwoJxZ`HXkJOsNU6ps<`=F=F_;O!ld|9mF~AW_^LScxt8t%g-Gt)9(MnlU zzWYhK--iuxH%){>H0UyLydZHfoZ8;1B$+X8!&CL(wMJBVoguc7Mn{^jGNeljp$2)F zo|Y)6#s@a+WX1Cg281bXzi*a!ROf)xZ`LQl!y^f3?&UPM#D-@NGdH8=H2_!*>4YXt z-=eaxQ#OX;KQKZk`loYv?_e~^O)FF!9tA3z)j3%YjyO>#mqPLFH2IP{j%oYs!gmTr zyH&#zkTF8;#Ia^+|_DxOph?!C67dB0Ti))$=H2L(0Imagfpr^=|A)G)+ zig}d4mI#7r#@4BT5m2YQFbZc-7g3L+oZ7kZ#klv5a_VI(zR2F4*O+e36Va1UJDSvK zhzm$b7;tB@90O7~7c3ZFQYBY&;*pzbmGO5{3M_femd|kr!4lmvIQ$v&q4z#>jeQJh z1Snlvr6@SY<>*HZnpZ!|lwIW+5(4I+suov)2K3s^AN%k@-WbOCBJlWy(qD5{&)^st z$xYJJXc5A$ReWDlz!>suq#E@`a+X2~AX5=Y53h`9`BrY5biEzV6+sF))ZwPJL^s%& zrwJghi=MEOV(!k+%0$VQ(>`JWvI6V9;+0*Awbe=1u~oHP7MBXhgfU;q1yqi1LzgdL zeI#9y(2GB0I~Z9CT0o+x>EJ1B=9b-LZuUAoLg=RoR2RW_5&YZqVaif{x)gFJ9=UHY zk%J5mPHFfRo_*hcP9I22?H#*>%%+Qa4=Q@3cOo#!?khSJL!`_L{+zh%z4)~j@=L&+ z=a^U8sa%<3M;>CuAb$Z-1`Po*pmJnu>VZ>DM#g|#7wM^I9#wQNcqBs)jfw#(ys5bi zdc6T1)jv8}V2&<#Y;nhA{;87X8jas*LrX3=3CSadSPEVi5=G*sIiFKgu^A~dgwt-$ zIzh={cjD}fM%Tq`Wb~at5f^}>xHyYG?k`}yB0M#XWN5+1~-)d($`^-NFm;;S?jwSy|=kSv2WWDNNF0Y zcSHI{$Hj~zBqi1AOm=GWhe8_tTqRih(JhhYG#@!KS*E`tyF^Ur9^r1ZH`Syy7DEia^;uON! zz!ebQG_%oSJOXr7`O8rsLZ_G$7!RLQ8`;(1`R20_(RS9O)yrl<_@|+n3yIwT| zws{_iUld=~JbhnS^(EwM-_+aHjvYFM%4{YZMj8k;O#JI!ds##Qj%^m<@lkzp2u{$9 z*kR7Nj4l$eC2B#DqoE9R`>T|&q-WDrHvkO>xT-Q(#Zzjg8llaqvyT~m)g8Hhe?~TW zil92}wqwKKxUm@FxW>XY43v&U-penIV z%2uXG13K=-WE>-QC0cPZZYC~~$}?NcZ{TC@#YTxuFLQk#ibqcGmkh!5&>XG<#O~A5 ziZ@Km8h2S}z_$r#XRys6_?`=a3vvC+l^W@n1?OW)WG)J>E`U=iMHW-DvjURGKIsh9 zHq6uEL7+TXrKzoc#lY9iB4|gqNtp2dQa3GtbCyabDjl((9bNqrWD`U~Ppu zSd*FfQ3x|FVS7<1DeeqTDRu(*YCGRfLpBjB42K9*15ufk5IoD(RB3n61GIp!%RbI$ zUf3+WvF+sq&&2)Wih(Ylz}$NQfpouwmg5p~*_u%TpzI*c2l4a8!YByN^*ZV}&=iiNL?3R~C#bpnH5f*h}og zB_FJFfeRKL9AEA%kP{7FL}2x^-k1&EQJ(<)@JdLEpz-W80*@8>+9HZgsDNzQPM8Oh z5}E2;Gq4(L$b<=Ihf0ChFYW_BcCYZ`e>MHcdHd<&B140EQdYepoLg4QtLJfMOM%M? zi@raWgcVOgxGDvy#1FHRoUw4L33#<~rLg2glNfH|%-lQt2IWq)Fkq)b&37@Z`sFPH zv$}Q(cr`ly?K)4*dVd)tU@|ZIu+wN7YE42XO^ehk!e+2BN*Jl8&gxW+`YP=ri_eau z=EB{S?kN#%sJvBhT168OWpq=ZGcV)`$X19LILtERyK5ScaV!V1zmi9aQHZnCBC3!+ zZ%-7P39c75U#c0e#rM!2pBKWyqvR3t7i(tQ!5nSQu?9r0XB+R?i{Y~`Vu z`SUK~uS9j#R2P+T5zi;Q=$RcbO)u)`Qd1tD@lh!g@VI;QgpsUcf#pf?%2Jyxx-(bd z#)J)7N})^p`wO#hXwE||T(kax6=q3lpupH$pNYxM?#Tr!WcXS}e|Q0XwaIi{1&M1S zzee0Q#U^X`Jf16NP0AP~U{?ckH_7}lgK16(Rm979I5N>UqdD(S?)dlUo*Td%Gw5t& zf7Og@;v%z6F2TL;el6u;-{}N;xl5^$ zlSn2`d~AR0mBe8iS(2m>UWhZ9oZbp;bP2!;Y~q|RVJf;Oj@FhYf@IBo?F)$v;lRhi zL8uzSgR?HfZq1$4j&MtPzq6=OOvB_{8tt|<7?$#q%q#n2&M|WGtD~37iL$0C+?;M1 zux)TM+&Oh~ydAW(ATmlPV@f3jj+xEl7Y1=`hO{+oJgKnD1Uq$m$kujdTHzwc7m<)& z)3Fa!LARqKxG^&m!I@_p1Y){ib)C4Nm)EF+#KCCW45MijQI0`0?I=~vyr;86)nVccE5lo1%ruHzpwI@q}> zt&AMID}-;H*ay^v5Htn#4K$jp!@x6C<31nxyN8VGS!J6kLJ&CM-YMJS%z}b|%E3}j z@&r+!0adPg!t_*ej;z?$+lI9%o9v|Q8j>ny2AG*vQYGO?t!FM5W#`U6DyKO^*>q`p znYRR&KgK9BoTobS8Ldz{->;N{pMhQ2sZ8F_wWb~&cpS~5f3?vse@7-k*XzwQ%rznH zR#|mpN&$SJSUkKf4@f(?oaTmK#VU|iaC*#-fCq%O6mKlgTYHu2K**^xGmWTiq-6$HztfJ z3|2mT;quoulGOQ})(Qvqccq=iIkh%qopqYfc@rG18w7KWYEKET_Rp9(WxU{%o^CJb zQW>aBIC<|h5IT}21?~9a4YwH-fQyzk>m!+}*y!|u|Dmb%suG0U%~b16=;_cnx18?) zqtDxx4tpU1otbh}L*TJ*8b0HnPlj-h`9-6_FOg1jMac<*%er_TL};sXVD z-Q8XBRKzs)3r|RKGW8c`;0dLt=a8v@l$>*K(Et;gAUu(E+pKPO`!1cw31cr?i9q_feIVRD8=VQG=uDP6rf8vop>+1WqR4>dR)i8?gAd~=G z`Br&$lXio`P=BaS_R8IMdEYuguWi^$n|;})&)n{SOZ0>~?cup8rM?LG7Ojxlpk_9@sTVWnm9s-`Wlwv@JD;J@S zw`Q2u(OZ8eO9zVBuv16_C=?r63;a+56ibOPicVK>ns{y8ZH(sL&h!9$992#xzvp`< z9)+w!t{-;`2&yb@78a9*+=B_-Ez5=xr!q6A7Afy{H+unO6DHg`&2XznWAt{Y2ve-K z%7CPEt(l5H-~K7*I@P3>Lde}T3%M{QwT5`9`!X`s?|lhfX1nfl7+HCUCoM&nTdzwT zZ@Pma8BpDjUv`Lg=yV!GoPJX-0Sv+rCwBGZ{BK8T3ryu1R+m~p+1VbU7^3O))sb)G=0tMOpsVR@-tAEPqvHdVaYvh0 zhtE9(mNHY$Q*-$?hVkY05(2h)ila9Qe*>S@Q4B}lw`EIWaSvwDao2oj>nl@3uS(x1v0yN@A zhygaw;!+>(ovWjOizPUt?n4jr!z#V$*TVs+!3!q>b})H7L4fgv*<{Cs)+ip*ZY z?B?j8a~L`LaJD|r(TqunX@PDQHzo3C|!8c?SU$;4WCTO z^}-OeGfxFvnub=Abh3pEICC5sCMeach5fn2^3sH@o{x;Ep@(W^SSal|+ms2#02=L@ zg!_QwXgRP!ds=y;<|U~Lz-a%{Ozg)TqeyFAs2}&G-7dm~k=}>D@=1+?SqL$W7u0AzrEWyB^1vCG2Blc(hyZ<~v-7g)(M5%TOHo z-iy{8AHGjN232zlwk32y8~`ZU9J;96f1bMv{GDk`tYYBKqYGJ%`{G<`Ks0vTq-tkC zWILn_*r1__3cUtA>@~yS`mp{ zU@|2@M>_m9PU~;RFsUS3tHi@x3~TN|0n}}!25WtWP$n;lUulz(VNh%XgMNf{vDC zZBe}@b=7en%b`~)nz=wnnH1+E_ncn#CM9A=&#jj;Uc z`88KUfI4#F1n*9@q^LljkOs!om8mliFGuasxOjjcn30#dB#Iw;`S9v2zk%@EnBZdj ze$XZ*5Ib8%Y}^CnStN0i-vgLkl<%PpKIuqBb>r#+Ap!U+vS0}tQlJJz^jvK32tbp-iKt;uux%qv6q96ttC(9HV{E4}_|fvAvT`<5tfjwz>Wecu2Wl6^VJ!KcVc;ZqBtZMVo!F8x$o*z%iZ zlqmyc;zvONY!rpS+6-($bVG8)c(a8^N^zEwZ>;&&$FS`@6o*rUjD6Z@Z0{lx%CzKh zw7M4;yWv?#4}nXi@EdaFF5S<8Y&mZis)1qY?~8QB3AEupu%fazn{4%LjAf?x4r=|a zkrkp^{cWbw#+&RLx{v@f<8ZPj9`W%Cx)GSVClU(1KX?xj*JxG3$zx0A*CmXg=NV!I zj5*SRfpcJC^+uA=Vzr(~w8t1#qDVzP7q7N)nft<3#DFTAD1*8}J_$&oRv8{F)$bvv z6^XXJo){8p?ZAgzB#PMi@Tq0G!je_wq|q%nE*Jp;sgs=AUF*9em4s%gJ;wdJS|s_S z_V^}r0M$5jCFg_A`rcaKQP`p$w$}nyI^ystHVHa9Rni7b`uHS8qBW(wo|$}f9^a|h zZ1AicwPy2P7gD_xb9fN&SP)pgT?kL9bE=L_T*V1sCh(p#z=%=!BDE`t=Ok&Rx%pT> z8>%X%Nz{RUT@A%kJxs`&A0IsPqdJjv`OSoAEFz0}7Mt@r@Ro8+N3x(|auwNy=nNk@ zyf{^}B@wpuMcERSJ4EnwTVF){P@P^9&?QV(H;Iq{NCpR%SmgV-dZ6Jk)3#P%^Gx-S z!>i1k3PGW{v1ULvLHk=r_kZW&MHw4WI#^HRj?knrTnu9u8-F_WDi9CM3UBc1~&l?L?3T`+jIz^Hzu-O(k<~fog?S^(ApC^Y?F5YWOmr)bl|GoI5 zdnu%hOrMT#&vM`nTS1`2&ohCOr%24Vci^;y=_>mx@$sd+ z{Crp6rWEIf>MP}$)-UH?(h8}yJg>Mq5v6`2SIuLBv|bE>zoM10dz~$OqAkc5bT7vz znr#DT|(M`#EK-F)4twS1ih& zl)B5$N?eFxmh>`VgTlf5)R-4_Yw+!aF1}0u2f@uG+iiEtv!cqI(K?t~v~-6N#8I8D zhbAk%*w7x&*0xv@zm=8BfMgKN1QL#^xyz7gjT$+gtv>0Q?B)Iv3VLVD>jpe6aAItX zWWs$v7~FHF3!5WwET_^J*XD;74FMy;G|g!|d+APcn?=Dqk!WBM^Z{oWmn(6vwq8dX zZ&hyWqBds%)t6JLDf<3+x`1T{#{@;O^fYaULhwfFUSm}&VR%Gf!B>e90?*CLS&-ep;z0ODogkh&(7Y)Sa^Gnw8Z>&i~17=#iKy#Yg~5mbzWlJnnbl_}GKA|-VpHENj*=o4V>r?#}@5c)i=WA4T02Ue;6 zJgHH%7!U8wVc$d?*5k`5+A;+Bl;SAGM+yi+V1+e95=EmVMzUKkLlP zY>~w{Um=KaI}_l^*-=k!LG3U#wW`xin#jD*9Km#*7d{)ykI&FEl%5;j6knIHf%T#^ zC`$p5UfK5A0wT7TP|WVh?-r6PIU!YKtRj5sUMR-vfNzVmRvZ-v+*Q#Ogrw2*#HJ7C zFV;$uS(0$(B$6jx?=~Xr7RXmKO?#qiJjREt#<}&nC?{1!ymcU^d^rR{WmVlSW)&Y> z=k(zKhBT4S!hcD>x|tHkZ@oH2;MDRhJ!k}8;&m3V$OFDVJ)1%A8h*l(jkWQ9b! zj?Kg6Cs=9^jpV;CoK$)p1tCh6*d9@I$C`>-u$Ol*xF&ecq;7fNgclOGZ>NPLQ>InC z_BOr*rWnuz0;nj?VHX^OxW$)1Gm$d5$fD18YbtmSI+)-&o3C<2A$LtJG^CZIMu-{1 z@~ZrPpMf!Np8$JB*abvVPY>?{fQY-pGHf$^bLwOErf-#~d z)TRq4dr>zReHe2x1 zdK4C{=zqfiKI8;c#-- z5DTx3_3UWw>U35BZGVnWRFOIln>BhjsX+wr2GC&LeJ39rHrhZ|zy(bwdo}XS&F?nb zg$6GaPPfeh2caz+qyRa4JnhS?@KL>PWZ_L7uU^GZ&x0eht>a9-kpkjfOqG^E_I9P` zXmY*}Ald7c@fB|}G()4~fh2+{gM8a+NK2u%97tx4)QiLhi}<3QKx`vh>04Qcd_;VG zZZgWN8D6G6W-V404VtZo3V&h+Y2Zq$zNcbKaq0a0&D|hTn3gFC;O#Q}@_GtZ7sfrp zlU^IQK-EJ{X_N>@IuAd}dOSKBRRoiKr;1NiDW? ztjg}Xtmb+gzNyDq^0mr+^`hx>IB_yz7a;I<-z(RKG7d*_(~)66yRYML;*m)d)CPiJ zDli;xh!RRX`n%oG??C^3$~pr|s-xjmFZAnD_0tOgFYm-djwN%Tq>>&ubvb>xk%|v` zdLxL9B^;5A6ckK5ab30Pr4PaxF{*bC(kJawWaY8a^$xL-3&?eXsm6r~jR>gan`AK7 z0+mU15Txcyz-o?khhk>4EpP(2s5U>DYY(yC25_@|^G?GvP5$-JIRK0Xhp6VpC8gh! zGk=^e!u<>t%}(5Fa9vJ$q@W5MZr7i@7dc$k%rPtjSS*$S3xMFOT3g;TYdA=Ko1&Yy zQtFuAp)fR9bBFWQ%;bQ2`&k9iY^qty{X;w{N}npWSYTxn?FhW-sU!bRGM4GN;kHjf zmU%cxVIrlSDkcdZ38d+~fsaown?RPIsnb<;<0oNRDzdEXtO{}Q+uy2H8k$89fVfVm zdP|7DdRU4syOl|VLR(=yZ}I{Bis(ymkc!sc_=WScG;k-ZiB3JS4pp`y$!O|G0_Vpe zt@2J1eP6mpyqsT<0!NR$1xaa~n(U){-YipADpB7nmwxN;uIlBPfi(_EPo2GpUMC&2 z*)!fGxCMfU6i>P)?4L`RkxSmLq9bj!zk!x}>WAJq>wY6r{NvdyoG-qxjuo4SlKZ{N z5zoQ!uY?>p4!;|TbBhX7c|&zK>SToTfU#R~nQhV}GJ3M~mBX=i@W;+%()|gAZ0ma9 z179=cXM|0jM?2>;Sx=g_L5Vw;s;1Z&w_)~PM(hntLG6&V;fjws4-GN_xh&go53hh5 z>KyOJn>sK;%*lf+;Z{w_hl2N)v2LeWwwuWzoZz2Joc$ z1+F&%J%An9yLWVC@&`^TYnoWEtg0%Rv~Xtb$xh@h?-HV8p%SRZ*b!2(Xg}=`0iOGo zmmUUVzo2z8Pf3-O9G@}d{V75j!eih0-Uvn?Z~!(GBjJN>hYHGh_4X8bRG=KoAjX7# zvo2sgU#8R)bXkOR1F6f^qSZ)2&~dd~Q{o{d=|0yosM(O=V{`5-S-UTBkcBP9apfkL zo_+?iN~*abTVyHKh1LV5Q7>YwT3vWr#0ocb*gh>oOD0BA;<32&7&c99Ng6_o<#kfK z)Uz8lyIm?P#oip7{*fj*kuGAiKYv-N_>&ac*tEiN?dC#Unb0J~7~=DL@LOh=d1Vkm zW-})2_}R8U0lMqS6GH(C+EO|#@-TpfRkB=NY3T@Y zwTUBmc0R4OF)Z&`f`6f&A4qe@mO9G}yMrzjU3xAv<33WT>GHX+r z`cwb?JlWH0J4<2`+yFB4b^+5>_|&L-m>ob3z9>K}!fT66-xwJ~nNPw<=UMdQ_}Y7a2>%O7Xbr|D*63wcEUJZ<9~j2-dZh1l>5znD+o_J6^OQ(1@a z7`Ftc0G2fNzR3X-HQz;PJQi@gF&SDpStVR-vAy^1_+otv%r!i?Gmqq zI&ZQH6(O3X4ebEv&p1!T|5%Y)+;V;x!D1rvoAvKld01Gp@Y#)c(UUdqG!RwCiv-CO z%p8wQHW0279*$L>mR)HN$ik;KZKculdf%XFy+eWoO~vK`JgD@!i^z@`YiK(WtklOm zfd@b-N@)H0Wb)C$WjRtTH?f)<-BENQIVJP`G+WIWTKCAwl5I`5Vu)4pPP1!5V)xuu z4wLJ+J21Nz`3tUi^{j`;nsLZ-o%&s9py#TPF*fcV8}uj_;{oR} zD$TG*e3Hymbesg>grk7f!Y6?{+$2_$DFA%?+=gGtaJ_H4EV@kK*eAc!#r`>?-F6Uj zUwF&fh)4-4qYvPWHlCv6lZ@_QO5iA~Vb`N`#FfQ`;E`U*A9g3X66Z@XA-056OlY$q zlY1Z9`_O5@?gdA|PC`LyqP0dxR9CYka6C`W=6y@k6%C29`d(_^Q$_e|RlrwxB+cPt?5zF1|S!NmFr zWAk{o(|1Un^g<8-9}>;-Nw%53)W*tpLYG>$(;4d<0sB#ePilwe1b?owrwlbBe!ujx zaMgv1I==DQLM)Lf_vqC74j(YjRjDeD-j|hB71=+0gaFc9_{<2?Cl+$Ld{Gdm_ zQQ-$xP91qdG^7$6A>R27^wfso|G^&Meu&GZDjK^F(k7oQ8Ck1X;aDp%7oKN&c96Z% z2Kg`V3@a`oW~1$zj4aa^w1Fhbqo31Nz2bGfy7H#F9~zERWREw@mZST#@B65`mZN3$6N|lw+ut3Y`OjK`-;#9QXGU@7=c+t}tC?CZtGyYkFn8Jkq@H9&RtB5qr<#()rI3eNB<7+qw!NEVk)K)G~BY)(5tv!uS2D&5Q)0omqd z1Fz)@cAnl{M7Mgwn|44Z6_vw#Eh~pQyuL7Yz2p_5%1*PR-ce*il~7_mn#ut}E6a>D zu*4K{WlidtPf+pVZA%S|MRzV~W}kZ@4hh;?MjZNy8Mepcg@5(zg1FA!>!I(8k~A8g zw}oKM*kZ_ohsA8!Y5ActT7XMv{S;*Sa)=}j@TE%@Ohhk$AmqEL znX+%3fbJV};86ooq8?)8b2NafS@Us_Y`4dQNK%7J9nfc9&Y_HE@f6_tQX-u*?h|Wl&|g#N{Vg?1iDFk5$xJfMrK{ z&laXe)TNY`wa`_g1-@Rb+c6RfSvR#Jf_PP%YNB>>k{`lnGPw55D(Gdufk{)jR|bm< z*aPMo5?01vpU`P-NJNA~Xn#^-sDSoF<#XWzcaD&CNdjdIji62O$d@jrxf29}pfopR zbV=_9J_=3pxZ|{PCySuP7=Tz@aF)Sm%{@^=nmtVgob+em@r{CBoF-UTET^ntbA|+y zV<1i9DSoS)Vfv)XNpHVH$1vD#o4h*aDu6m_T(GqHSdPO3LT^ zMqsnfnt%5c=_Y01Ikb2nM%J3)_gHx>54eMZ)#T!2p=o%H$kI~~_RP9j7rN535wO+0 zsrY79e{csqmCYVt=&f;#lGc(%>qG|NV_i|N)YIr{lC>X$TW~6pqIB;o`TP05OubSx zo}m?9$&c)q*G7o$Gdh2wkm zLt3oW9 zJiwVx44T^4aUAlyn_oRvB1v#+cTqCtVtJz%r}`y#Xpj*}+IX4<8)w8^?QuDs98}kg z8iwWv5`^8$j)A^u!5wP<%VqJ#FvQxgV28D6yw4w70&cBO506zs-B(IB{{z`jwzZtI zqpYSD3(P4m)b}aaaKS?GZPU7oX>^$>LG_tjPK1UwM+`RqHRKWASmEY=Z!TUVw^h&_ z^I>4J)B7tu5fq7VB;}?FOO)7AThGevw5$umKp9MKX!INLaY3*Pi_7uel)9pXjnZc#M5{2>D{ zh13pjo{v+#((Ompr?de9R=Y$_7Tuhtx6b3L#eYpYj2WY%V(5u6p|&wq?&yYIz2l>; z5}^v9%ou|rg83)yp6!MH+CE9K8VP8$+us0A_r@wWQxw;^>SUG1k;tG^QYfUyNzNA_ z3-FvlsVYtwH%Q>Lq_CrIb}bL=Gl)r~zkEKMi64lU@-E4%cSI4a)KhbjSGbHSr^62aw?}Dxn;XD;TRL7)?zSuW5P8tcU zJWs80bN9hGfRs=kzWJa*tp|ZJmmz62po)#aF4?$VXuy6M3XLvfZOHjBxm;0eRQgJ+ ziuFO697UwlNS{RO%DOtVg!(@q4vzAvfYy9= z`0)mB$Cw}P#PG^ZZ8QU!-Ex3N>HvDybr&n6RVQDtJaTju%vRjJ7z3F05L?{I_{Hmu#Vi(Bzw0XiQ#u zmz|p0xM18Qwm33`ZhBniElW8!0xF@{iZVvB#v?RyDn1bb(2Jy#U@}z0`laJibD=Gl z64!E)LY=7?Bv?r6{_>Hvwyf;)Mc(0(Dark2Z|kBLE=tIaZ0;O4md*r@3ROejjh;Fe znAYaWI$szR$$6tTF-i6?`A&djemn4SVz?)jQBb9}em%(^{laY&3R`0{pF%@f>M$s} z5b3m5#h+5~T{-j|fV|dSi6_l9L(r4`Bkja*B$Z~SV410Ebkb26UV5;~&5Iu~FA~&4 zw6uu1?3RH1B@3sd?{0^s(gJ>B;g1B_2ou(FO_r7K=kS)<@NkBJ9L&?3tvzDAOEv3R z3pOntM>L`vimPQz^D1#L;U{Kgbq}tI#?YbgnC|8)fUASMrh!6`r7=cfyY86;k#8i1 zg?(wnmE6rBbvGSJ8$`NQMo=ojPUwdk-D)q}Tx3Qu6e5JGSp#i_kIY6ic;SJG-=!Aw zw>cB6lGWfT;9JvS00Cu3J`XYdp2kqJJ;GV(0?tT-dJvp)Z6Q`0<lXX?4s72beLbyc}BI<*V zupD6RsI$prDhP+1B%ZG(i2yz(zt4LRzH*Ex_%@9EYJO_JfP2~}^N$NXaEUJa~9=N8+!%#eidAfHkI zvZsT3n#Oz7Qi%@K61S>NO{Xf6XTlgci+_ZUV-HG0EYC0MB_lWP zS9^>WwQl@Ea$e|4PJuq!ZP`+xaP%On8S$3qHRKl?uC4I^YAwb^ow_!6M5Bqwk5E%yo8JVv|&37L2*_pXmRGlb@m;?qeO?I zt}%*27d!2NbdYF(39efdHfU}|ss?EgaJEe4oW(CJCCyvB&09;zUo0^x$OV`!x4z>s7151xxah#9p{v$OL3R#G zrDI*BrKZ1%7uHzNVRPBRnp=gs2Ct08Rx|V$GX}h4ix8^JX;1 z#S>?KL^PT}-5X|WYp*JB!{BOn1-Qnv1!mi6R4#=A2d#Js5v7>u;#ZH%y6}2(4?AQ9 z?MSl5lyx4YiuW%QOj(35dcLmUu+`J6aXXh7*9tG z^Z-rg;`sDRLIuAZD)M3%_V_6z77J{>dvf`Ux*c)paSOTC$qoGs{Qy2=7O=NUa~&0! z2b%`SyA}MfgTQ1cDixgPUCZ_=P0DLaU|zm<9PYyhEg+VH2=8>e>VuC1*3)e-Aa=qC z6!ujEiO7aAPmUL0cYkW4$QH~C>KxD1n;p@$qV#c-Ll6IqF?7KVNE8RpmEePhg@A(T zUUm>V4$`kNqBz|nkkO~R$i!;D#_Le>LmRo$T~=Yd$6s7RplAopgZYq!YJl*2T#|F{ zQl^_8NJSyLYE4v9>;42oPWp*D> z7feq*L$vgPg)e)Z7R<+MR6j;K*gBcNRwKN zdN?Nkjf?T|#Ii?ux)AXldxSO$ayAc!L3p!9mA1WB7$6Im4h%>3$8h`F-vn7zi+HKv zz84Z-zKh9G$B4cPv)GvvqL^*3ElO}|ATGFMZb;f;h-A%E83OY5!wNbY^4}HHw-rPF zjJ$B{l$Ic#-YLz(DvZeC)4d}gP_+X?D0-Bb+Cg6C&zy^+hzJSN)2Sa!21xUTrB2-& zo7Z3VJs*UYVez+S!?m%Nh)# z_&jL(tBNX*%;#VuQjQC!$%XoaiFRwQ9N^;&LywIkO{&GnG$ehiJc%&QNvA@dWeY{f z7z!`FtTRs`{F+OB7rU>XMtRSWeFN}m11S#JeSaE^>1>c3Gu%q&Q zZynz7s&dyJVL8k7)ymq*wb9!RsG&pBSkxt+V&PZaH*M$+1}^AMOv%r<-R;i zn17SoYHZ11EQRccqkKcrg@xt$X7HCkQO%uPb85( zWEX=jmwr^RiZiT{)uboA0tnlv_Yv+00O!8YJKvwW-hCKMeBun)of)pa5-}tpC3vJU zct?DAJQXs}!QY|P(N6@MjWs629g;i5(DnEz?;`^VsA?$&Yll&6dPqx;iOv~8(KvHx zkD_|v!d2PCtifrYgo=#ke+MO=@hW7PF-)m%2BpcN<+KSqnY|sM&#icmQM@!=&iRvB zDy^7xo3+%_Mv%iEAYlcxO%J(9NCDs6TA$Hhow|Gyimiy0F#=Vr$gSYZHT?&sU70hImh1EBP$y$|Ef%Rz;eGV1YPfM*YTfpmt*1y8FEaNkE_0

      b=Bn{kIT2|5x!MMN~RJdtVhT0W~*%0jA; zrJx>BZIGl={a|eKmWZfM1ssoICgrM6Dnw$#-iY|zk7MDYLpll1bYT!c+TuN8t!n@)VQ zFyTZU*`6;G;g0tyI1?N>*Fy|28k8g~EETZlf1oW(jeVTt^1l}#-CJqlQwT4vS#HMW zj7oPBgfr+q*Q9(rzh|F}SX;H>5UVj!s zu@^;I!t}G*coK+e1WQv3P#uUr67-#@FEyM7_$#fiYb(Ul8CH?q$hU~^tl;#SZ}=a$ zqKj5&f9F`p6yRv*tanjHt$3r*XqiiU93;J zkNIqqpU?JICNwPxm_Wnk@S!cWm{ikpB`wOopSb07rM^iM^3-(aNzJs~R#h4<+$AOR z@bCaJ&i0+-W_;iq=V)qd2!h#j+6B`>EtV@$R=m5mVIOyPKBqk z+CcJ>`Yd(}PQAe|;O%|3>T5K5fzG0B*@J-Xlw?#*UA_aPyf#6O zvmwaY_+nISR){axfK~B1L!ry}9{o(7M4EhNGEf2@#g$ysxUa1g zaMno(i`rqXiWx46<~~Iu;F)%WcQ`E8mwHraJqB0G?C!pXR>`O<7w+m$mapT-nPdYWDE3ZBF>^Z~ z1#fwmm7|R&Bh=`$kYtCGV3DJ_|kl1+0sYr;@DN*tISAOu8Oqdha z5%(K2)z}F>OX55s>4A${L#7bdo1z{YlHT)1vQvj*43aluxT;k4ON%VsUQ^weJYhj3 z9jX`7U$5{Ise9cOVBOanecaYzRVH}h3EXgu{QhM6r|!W|^Bq`~yA?gO`E|izarjt@ zjhfSDh+;0^R9)uE?M-!jmm%-A7O&UVV$vsbXA1bw}_h-W9Gt+ zbv_E0ZFAp5U3kQ!W>*uOIK1RVHL^&9*vn!CyafAXBT8M{EJIk~VjN>eok@-m8EDq5 zwY3~f6>o-RhZfJVDW{lFg0kM{w`_HiWPs-=R$pq3Y({$-uD9i=S5hEgZLAcF^y$9I z6o*gt5hko)bRNbuw`=Ps8sxoG(zMzvg1cp2A2JhC4VSE=5TPg*7fco7p=} z!t44x*SVqaNFEMt5I#(kWZfbAb&iWA;49K<{`QHXN_IAhCm02N3%eLo3gASO^_&FU zTP~V-tW=ih@Wb!i13#&49a+QE?5>d5X6);nD;*^r_b$l?wH73Ldy2Q4VwFD<3p17p zP&aUvNx|e0#OQbQd1NI6Y(`85wIL3;5(7FC%3b;Q(GkM z%D84qkGh%0mt@xAPcrsS;YA8C9_sYgo)i#0D$XgIG zth!up(Rzast84?eGk)}t+Srf419BBG$fhJg=3XjNoQXz zCv!dO11ae|L?h#Za};?1G&EbmilgEALcd8-#xZb@#=oc>=<)Y-$E$;{*V)ky#JNX+ zJf@(lvfzu(=d$BSc^;|-a5q}uSsuWMR!dZ4c_DH>+NJGekLtLx*Xm22NekR^Q*;-RyLP!HA_iha(@(@GN%A=TnWFS#RXL z!ggo|l#2zP`M07p7;yMNq6<$vBkE|e+%sfMEl?rZYmA~d@9u$@jh4+YXKtzLlq+60 z9#KM8lkD<+jP2hJE!}(fdgUJMk`+hSEGi>f_OgHErPa}wWLzz-%4kAw1w<>Oq> zpFz{uR^398@3cGf7RU~AA-7M)J}XIl<|nUxVy^-+Rs?gNt0gFP;A%`FOvHAzjf zP-hKA@RhCyw5Q7|Qc#&^nr6D3CW_2r9*bJB33Q%)GA{d81=!WTi4W)jwnbX)ba?$S zKY~ObuGbheYvv}Yb+t)^CxYn>bA9Oac}ZjLlXmSQXYB^_pY+KoM=uH}AABM4On0sn z22;&^7s~0?g&Ig1-zQ*(Dxj2*`z8_?AiThJSBxb+9d}*ke)Q=8t<&rA~U# zD_-uzOUKcZ^vGbEHH8Y$sAmeA%COJ0Mp`khww@7W!gJqlxpeOrX3}l#=o!~ zo@6q^XYHGjimE~j-q-+Is=ha34of#b0VYx?r+^A3J0Zp_>bq-+&55f`a<}uoFqTmD z+`Z>~PHpDcorcZ`ua{`*lNfvEJfb&Z_2ZdrM0(E<{Ug|k-w0&k^R9}-0^c+3yC8#; z?7o6*V>(?3?QlIL)NtuTOckTP;sd?Lg_QT1DxpR2f;RG6an-driEQO|9qm z-V6lnQ9L!`V4HPi-=1zlSlv~61BDyb_@?9h<^8l83ntA15jbg*A<|cu zAuG(XxY9+KxUK4PQYI3&Mh(;spZD@}uVxq~3Xurq3e4*f7zMN&Z7IHC2}1eC*MxM# z*Nvdo5x&s-RTmq|^$FongLz-1rp<=1@GsyG&;(((^0#ZbMbe43 zMzjyC=0jxLD4uIx)hW|dPfR9&P1=H0GCs>|mPpFl%@4<)J@l`ee@^8m!!IFA#5Vu%`Nq` z1Y}PYq+V%lhkQ|k6h|EA2geI~X-Ng~`7z76hj90s(el1tSfVgoqNQd=3Jd@)IW>e! zyPT9b0^1fdjTG-dYQ+XDNbXJvuqnW;0d3+MPcTkB+Mt&zn7)%;O$S$&MZ4=7q4HQ3 z=bZx1NFK7|jEt&PFBfW%nQA;5-7iJcLID9U5(5ku1o_Ohgo(Xx6b<57h*;%$!A9)j zo9K9i9m8+9$MuU%IfZ9Y5wtNnYQr=C=6&e}#R%(P7#qGI*bYLe9{RvL!Z+|t5~f&t z2!d`mG_%t$pJ;;`Q=3~!jY_Gzy#g27qC189pc3Jmdgsp|PnVH}TLJh`981NUQw?ha znjkV)w@C+_x1B1Cze+?WfS%Q5cT(Z<{ek?&kEaPTW8( z7e^oEjGV(F)LMJ&+*F1L(Z~CGJw;dqM-O#0&IJ{i+$T_i_ zu|aGUplA8fw(Tm{jXPG~s zt$0TH&Pqm|veNAe++qCQ-thb-Z+sG!eVUtHOi1lgf)gdB5T|_2w}p;kO|QAQi9XJt z$p27z<2BY_N99?Wa}ry}ZIY-qR;}Y0U>hw&tMJfMZE_IagJ>Kv_}oa`x0rZd!D$uN z&-o-cHXgo{q*%MYIg@aH z6M{HnLeXQfEWn*V2MaBx6L2RMt+yo*=$VnBde8``F-?Psg%V8M@=RE=_AaKS*p09x@VOB^ypC|tmIH8rig7y#BqeA# zN}@R7zOQWkbt_~u6H^RT!`b2dj9gCxd#C!-!leYow*P*XzxWG8YsM?ha#`6Kac!dX zri%-Bo8LZGnWa$3c%|(1M|kKdzp&hF(CR0wELI7mX-U{-HFUj8;wyBunj&q=#D!Rd z$eC<8s2L)Mlci!;YBnVa#ww!%smf}k#HSd_kLVeK%chJwW<+xkt$Z!Ym<|@4y1A>R z;3VE(GPp_Z59Yv~Rnst4GfH$om4Dxz2#RuqCgjT3NW%a#vn)z>kO*tMDKTRpEdvNl z)24Xj^rtZt%Vvo|pKhvceB<#gD;R`-2Z+I>4pH8@*|b(NYC_~j+{YGxgFGinwkfyO z==N2_COw?3!!kW?p)vN-RT-p$)50+U+cL@Afc1?&kv$$A;cP;|J=8|E6HK7x;BDg` zIF2INgx~ZaYd7~K;JvXpEpXLrV8Z^X%}-tpAIe0m%-3?QB6Pt?ys`}`zBtV~#-;Lm zdVEL0RKr=ywq#k#Qp6papQwZtM%!;!?{i0uUuZryxeOt+K^LriKQ>aLu!;ao=dRaK z`pN2)G$uPZ`_s8==3`_nx}A8<= zy!F6IOiS7C6ZT7VN|$AB#z8gGqf1za@RW9tO>p&tG9$ZvH&uavRuLj7row!QPC1pV zhM(GW3{~hI3tgs`%{=Cj3wE^&!2#4A7rgACWfrdn0ols8a^s_R^Vj2aO9tA}aQ~ae zN7c>zDt6Y+3>@{ICO|_H#%7u&Fpr-^jDYdfdm*Uqn`;|bt*eT z8Z}{;+8i$~xFk*B9M(?*@c}atSYC#FvUvJN&x-2xd>r%-bmdpV{Mzwh)bXp4KeYuT9aMWvOIVHdVjr4C3$ogi z+~HBvhN2VKx+cTmdss&;M}Nu9Mil>U4D74RgL-X7;cdX{vEf?nbM%tBV-IM8^P`82 zDYtqbplReKzPB}-aDVm2}bZS~)s=usda-w>tNn-6vT(wLpT!Qk79Yr|Y(fG*p&V zW5u+%*g)z}8+?W?-rSPWU$HSb&fRQ&JHEc=FjA)>EDTwjHla5RGlmjpt0>A8fosS) zH1U`(6hkmAFAoL-JYG3R@EAk|GjrtTE?>-s1=nkIa+m~k(OJZmq{oq^nT0Rb^R8f_ zPj>c#qn$}&JFgtEt7LuQ7Qo?I2XsfCYt;9dxw+Vk%+e|v>#ix+!h;eL4Yev+STtBr zp8bKz=y-r#ML9HADBwRleJS0BV=Jewx_G8Rk->n(AXpKqWECUX*1+kW%kwNxgp01Q zXGQd`6j^lS0u*2SdM$~wI&=di%8rN*jOzz`&exv2y~z9g^i#;fEZr}*C#Xp_!3KBt zk7q@)msq|j^R%WzGE~8>gjL|}E!~;FqmMZh)eJ4BzebfsJv;g>bxlEjp$JL zIsn*KnPehJnXZMrZ!obsDIJ65qt0=)J%#z{_Nj!%IiqEp-6zn?hke#ujGgYHZ2s;3 z35f2`93VK8h%s*rW`9|^9fujcHpQ-)7R$@1v@DZBv}PX-Prc>4?3>+$91Fwj5v2Nr z+0m5RS%ricMTnNkF)Fq|n~e^6ZP(wv^+GKD(-5u#zt@>>+9ft#ktU6$UQzBB7R1Wz z8H$g;vZIJkilU4Rctp}9J9@ekxG^4AG96=&qCLi(*&(8}&k4j2$$GG`36^Qg%@|=^ z=QE-Rz=K;swH)kQVOfb55l-{NPC?AY(hG}GI9RTL);Cxyw_UOoW>hZ1X}S!ix#%*# zO;&SQ9;|6Q)581>tPw!1D&8T1taY7hh^Z+9MV71{rVmwSH0);72Sb?Zs*=DaH<#MC z@Bm9dw7*fZ(KKz2{Jco^E;ZkwhP(r6DrPW zrtRCHGStHhEF|anc0x0l=^_Wg*dV2xINEdG1LKb-T*S1LO@*)G24Zy}z2XI5Nz1XI zYXL(N_%e%=w9@c6QNfoU7v}|#O!=BBgpz)ju;YSrFVe1PAWsV-;v;9En{_uuT}wCU z;GpHx&dy1tZ!WT`(#BjICTHjNZf@aaC7u2*P;Bp(YLO4iAM>dhiv`7$kY2~|;<+aA z=z!$piUgxh86o4E)zRxL5C!JIG?mag3?FS9O&?Oy(&u?r(%Xl4{}-$(VC_YG;>^+P}hCqVluZ;4JEYxV|^?zZ4bJJ<;cqvNHT#} z0QHbm{%!focqF=f^^$DOxG60Vp!+WC9%6TbWhpT)Gp15#UJ14rUZG))ao9ijbMHXk z-qfg1lvy-|ka*h2R!k$U-{CDg+Ux1+a&ivk!dNY>w9NH$BFLRNYzYyo(!w*y?{l2# zWFg|MQRP#Ok_gUbLs+ixsC2QF+JP4eWe0b3aV6wAD-Xa8p$3)G#i&@aJNoIz?8SpG zruqYpT^k2VH<5T-_xZFgZyQHaP$q3kF0TfI^LFobn>HLKwTPM;Gb6TWmOY3Hc=;!p z*j_5vbv~$8Gz_va4*QB-JNR*=$?0 zyHezkF`*iASqR(QC|?@HJp~2g5Ubhrgpgbh$1-y9=SCsxWoj>ehVRlD=HPtc(Mkj! zzEF~ZlM93RZ;eY39ak?)Eq6+$oo6v|;`~`1`J?NVGjR}j8|yhu1xCRAN}}i6`YnrE ziX;~z!+J|!x|I8>(WzeAqR3Ieu`D3K23I8!7(pG4z0nmL>P;HJdVidWYgk9Lf1$b)O(> z2~{%CFu7t2G?xMd^-)daQbo3)ogDBQ+rmD(#?Y35m)HlySUtV>R}$0ViLmM&Ef;zP zz3_`>1|;O7puK-(>e=$h&ZIg*+>o8bvL}p)QlEJvXv69gawVfxbJW!(`5AnZt`ZUV zq~oqaQnRW=3kV^cNx6t0Da>^ZKIVg#0nZa)$Q3ZLG;nmG9>M$73k#^>716OS@x4w; zvi|Wh9r64UM{o$0I&9H(1E5cqs=aoF(MZlRDmn8vX0_9o;Iae?U!I<0mui;4@4AQW zD*lWz+ZI7UmV@&JjJN~D$2UADMvOF`p(v-wvAy(LyuHLqE2H0`Q@a@pim@7Q?IGGj zT>z+NJ9#=~k+k_Fo~@wfm^^tjA#7S^YHItXHEP;0iTFi23f^)MSS;JSlTm>6%Dd{k=@mejFC&aR|;uinQs* zgJM%gAF9O|=gZu{?`E9Py6s1k-DBl|s~L0`B`%IUkyW(iyM{7(1{;oDjOiuRa;M;Z z@$C8n*qk-b6>O3P&CS}O$Ud1mzamBoC@s{{I6<5CwY8?bd&8x5tGOPO;_Gqs496Ii%1|cWaoreGy&ZR{br;>Ww<%Xwn3<& zAWcWqh_oqBt~vp*ULGfIlpn7>1}H5_c*9vsHC@OjB6vVKkW3z16+j}8yL0YC<%Kmj z=Xxj{xjO)l#)NDQ3w_5tk^i$Ca5cdzwba4>@(+o`iqr!8FB)4=X@y@CE4Yojd=t9a zLxw3bN6CL)l@m`|SUI=1r+j`L6PhvIaIBYKB*8%rzDkrsGMytc0tY+K3oe=-UNUxE z{-pB`K{L`Ga_(!sUZkENarGqI3)mIUTKjYl9@7N!rA{_5EzvY4?454xW?p-FtT?RU zI!_f>wBLP7QR5*{MHPC-cP=Fgi6XxK`7Zd2_*$zGN) z_}6Z>`$#S`)#hH>g?1>suwf)1C2JiSyka6FD$xtVzf!O=!QL^#!@c=)bNY zBI0ypYtJEq!Enwxr45R;F^up5oWqmm|%9C2w?@TFA^#H4Q2`9WpN*;Oh8Q4-jM zwsR5>_(if)&uU1T@V%0f+$})u5UJjqFpBZ}B7WIL5ZbCx_b|{XiTgPo(dG=M2gy4&Cfn9iVmeGp5#`6+ zItYK49NSaM8PnR$+*=ygB9j0|g;OY>=NnuGc_?;6lem%tuOQr{U|HFqZr9k|Y<}R*!*9V2W$<>5E~#Kx z0oi8QjC(ppkJpBQQdTzgtBiy3*#hg39h1<>O`Nrr1Z%Su<8p)@amLgeU}OMjC+LWy z<8up8g%z?pSvxu{l~7ZRBzq%CKb=)?R-6E}S%&9#MmmNt+1!|-MYlRSEqY=BCQ4CwFam7zOge%44Xl1^av7GPv!eh_aHij=oo|5=ZrmQUA3oZ;_ojT{ORz9VGwO4#gtwBevBLpoI5+BZTjCgyQtT~1txVTlswe=^m< z(E{~cws$4Rg~2iF1ckBcnV!kqke3OhhiCxNtG~Z8a&jksb9YsSnwUn!h9uB@R=@PA z9uZaoIcQ>jlGVPGx3Wsfk#m?&Rp#Bjj2@bBG>AH!3JA9q-FyzQ=SZ?ywH6k}r}Tf1 zmidcZ1cX1Upo>#w*Cv2?UKjRgbCGN{MD{A#k{6+^d5X`B-$#pd_Z)|{QP6ML12OZ?VBP*UeqOdOj?yU_wptIZ4 zTVn`B-?e|}OG2X3oI#%1CczGRm%UsQ4I4IrvfEBW7S3Q6dxGnvjQHP^PrPQI-{Go0VKn`o$ilPi9jyK zF2V|?DI`}X+~;BFz(+bfI1I5w?e#Ei9(ieDH6%Qb>cGfpltzB_2z zFl>g+yR~^gKXxHdLxC`8gjr!b&mL&qLEUIX)u9MMT=2V63ot63%1Z4+87|p)%sve) zK`dzYjzT|V*WrWlH;VFu=XR_KeBmf}OH!(oBf@7lV|`ED{B*)89T^~U0Lkklz~9zu8=%45 zgaWe$d?gT0b6tf1eH~)&{5YQNv~^k8mHd;vAD_o43bhnbJYDhOO56=@M!$dHOsv=Z z(w^dXcmgeQ@EvMRh5YImaysqe(4jchNuOtvqx?*ld4=&I^_KiM=>*hDVZi6^ac;6> zk#+NYlb8~lrdL4s>NzWicqR`fY*X=+E77^o(SBJ`!e<0p0iN59Hm@Y$EF-vk?PMHD zXEpPnC!=wnh3P&rV=|CfUtzY|E#l<*jnM-jD-$`GQbK{y^4a9|W>`4yk&f{snq*^r;#nXc z;A?6i>cPNWDgBwbf16e%y%XS@FwZiVvFZ5cWAu9(?4qgB(c8Ss(-G}~k1Dcqsr)&L zc>K~u*)t(TDs(S1Z+M?JoFgGkJh`Kr>}81x;VA)5DahB13HZiW-^dh&QVV&W$WrKjom`2}$I`C=*{jh93` zJr+E0=3a_6+Z@-(TmNj<)4|igvh5%~J%;lWDT#<>?KF}kpghh`tE2il5}tcuo@#6L z3kxbpnetwzc)OA10S?6^9PHOT}~P~&AkmnEH#Gc}1N27G!%m3pr8-28gAC9X-i$=!dU>$GIYJdkj_ zGRVM_6&n6!HhvxW1?~-A!B23g?R)a$a~a_nP=PyAHYxm;UpzdAgVK7%i%DO53|ay3 zK{9KpMH)dAi;~Fv0u-As#p<9~cJ$zR$;j9mnvm7ZObiUJqqtwfwlEy@Kugf_BE>GN z`WaB<-&7fmuu5tyF6s2V`U;!OFDFG$X>21TvzVNs0>WeHv`9E^FHb&GGRfiXIJvxF z(paEH>sKMVrPdBUXAPP&ysiYo5wIg3lDQxw+pB{WggFr?3iG^qnb+CZuGCIyuKZ80 zJ@?JI=VGo(GF}zgehc>>D--IBl`}<`l_ALu4|&Y?G$ar*A~t$!y?n1chY(xJ$psyc z!_S%|__5dCn8pQ^iPZj!^xt_IudADA)ccyxke5p7xHKqoe~U>h9Rd$ZBwf9*{Mu`j zTbUXtmXeYWVmR39S3F!zi0xh#TPsV#4<}Ds69@Xtc?Z0t6+V@Z?UmSJP-3QHZ#fCTr6^@az_8%iKqL&t1*=3Em%yFV;lU5^~AyMc$`z^t#R2mR*H1lbSwOIwUDi(v1SKe&1&A0}b3ti_Ya2mYA6=>q4<2TNk?Wi0Jt%VFss9`Kh zMea@|zs;Zu6Q2dPAJ4a&*s{BO!oDB`msx3C)qVuLK?u}oW+Q(sb$hgD>#?0Yo9bl; zK>_tR;f>5;dWTT4&ak+Em+oPuluwyAPP#hnt9z}t;4nHv^9POAsfy5)DrHeqYt*T_ zf@bRO%a$@$wQaLLQ<=skcj{@zRbVa$K&{Fa8aF<722R6IL64Q%&x83V-k2%(iSYQ^ z6S)K6flpON@h~Y77BnRo=6mmbu^~#*O=e_7(qoNseEUTSiVJ1+2(nTVOnT!zg&L-> zsI}-rmgbyULG|$lTaLFA^oU}gr7!VjRpgIovm&BWl55g4UTrhd5jT~VVK7Tk8ILh` zrP8D&2OS-gK0RQMbkDWrMYeB-U7V3^A)I!Xq4ui32edGhJws(w|XyV;ivIh>V8TU%|K&{qV5@(W4 zvn-?-{UFOR5!cxSiaC0u%-*rzBfUVxs2w)T%zh4&)^vU%gV$<4p?aM0?~GQWHLLlJ zM`hTScm?q}0}G5Ob&<`z@rTd?v{9lmGB$Q?jC4%vCRgp!(gsjQp|XelMMYLo>g1iPP;f0_Yu)tJbVtRkMnD5|9X4OYj&Gk0u*r!2o ze|&H;f%i&kiaeu&a~THAFxU=Xu1U=E>fkWSRiCKmaA>r;wQOJB1jGE19m#Q&Rpg$_ z-odjkcf$+Hy*80Qe4YOGmbseyk>QAAFw8kW8fZ))B+~RM*2}UtFFrA&%!ffDK4uS@1*RY`&1>yc2a-lr zQfj#{LoOl^?*K*ZEj*&KGm`|5&o7^DLe9_4%*x>x>^Z?zi5_Bws90tsOTjO~|qJ0P=cJ1i%~ zfs-GG0GJ_9^pL{?Akhw0n`=Be+yfzvC9u-g8Q5%WuWoC85>KfL{V5Mg(+FcF$#V83 zqFXGeZ){%{u!}A8?0E!%*rZryZ=7ps9+@?5c~)9y&B3>FonSd*DDoVMHB`{*U0b@A zGlG{LKAKVLB}Hp}Eg#`redcU#JB}7(H)Y_;rHDuboEBQw4$+HchkQ7UsSfE@!?v(6 zvE1t9>WlWHE~IMOx{I6c-4K1F8fNC;$uO_$2{||Ql&Ls@Uv^Q^=Lb^ku78>fTfVl(43bldxHa2&)Y9uw7CS3T}VtU$ln>(#1!SQT_x=GE~?#0aq8j z{GR<~Lc#Jf4--_YXwNS`6r=H5n6m!3);Q`7 z#M`3?K=DAgG4?$pooW8hIgZWL%W#b2n!cs$b(B=ZjFL-L(G8U-4{Q}kWAj{wWSA?7 zS#dy{%}?4F_t*n>XR zkWnn010U787GWppO(v4}cgxVr29ew6o~n%$p@b_X681cYQVt!#O7KZfvvvV?##Qni zO5HGAERQ!SbX9%m%w`ByRMyLa#zLcl0x8ST#vZl6dQz$s%j^B{l74^n#NdlqdxMjQ zf?_FyH1O`ksLjz3^x}(yn*%rbuAV#F_umy4DWW7_^LW00#2_dAY%!%fo4nHlMx{op zT;?>dyv`ZDA)4BUWm5y-F&A<_%B^mLozcg_%b6gx1S@+0&*Rbsp19soYvh3B(dtQc zNQDubG{HFR2i-EPw^tA-?E6w3M36Cb$vZ1fpI zNX-}n8V3aczq57+8F)=007`NVpkwP`M%kAQ(t8dzRmVeWk4m0u7!M*95)EjA5Y|(XJjFS7!pGJ&-fXhtpuA51i;wVU_SV)d;y@pyic;MTuNi$R zfM#!ii{1trEu=%mIG^Yg%$MO!Q4~~8kU`a+f^!xhaG+~j%`^{UWBZa~p=F;8W?N>| zdN@o^i;q-30x-vN*h>~HHC6dAO}D8T>{+OpF#9L`QkP&QI&EUC=@4M*7O$bDEVWW9 zUXqyV^Y|?sMlX~w?p>=T^`a>mB!MjoU}TxOj{Yp!z4^VoHExjn?KMW^y`v?IIAhoC zr=<(9q#ZUU0`XNT+OA8wV`n@{eoYk(+C%Ft_1mdz5-IxiA<}Z`;+i`gZ{|1EFB&;{R zMncSqna1LNcdVlrvM2J^6O6D-?{_n5ZxvTakWTjj{|b+?G^FKrTOMetZVSk5#g6Z&Z0#1{h6jB2TeEktvX>ya zNo;G5TO8}m70~%$t6pzxh&%{yZDJJ+WQ%{^gn;rUl1GfirmC?j}zs0L^KOcP= z(wr39n$4Pq@K+&Q>Y^}S=e^bAwih)rQUWk0_f{x@%}Vw29=k_5zZV3R{i;alybn-T zmwGU$iI)W)jHHjIsqsimSV38A??p%kTja!yM&<{qbemX>wzJ9Nsoc zTq#Ujzgc~5@Za{M*HX`Uov0WwriMc{_&?kyx#uZz46RovI zS;`H3m+l-t^_rSrnA7DHbv-Dded@F29tFh}V5wkqot8H#scXu&;$m9EEB)E0bvhCk zk)62!r+~f0P6X3kv@VViINyIW%J{HWx9be&zPftwEJRgzpfMzhIfMCHcyT4fWK?|hkUraAYp$WM#YgXaJ67wT8Jz3cqh&h6a|pk9 zxuuuSwkm~Iizd8%y zj$tspN=mhHMlMRmb1R#4Uin1e0BloDoX!Y!7^0*PdvMwIslq_Z0~wJ@sABd~9H~P3 z6#y{^iGu)S=Ik_N&K0PaR0omZPX{p7cUO%eN>@I0U$t*#E-&)Qd5yz}L>mjOmfa}D zrJ+P!+Abx>e4ShyfuNL+XjE^i6$q!^L?3H=c!_OS_v44u;!w_$8TElZYAFnghx3jqW|pxA+YJGwXoKk$7)8ZQ z`>1o`HW1*(jGA?x^n?LjbA`%+OV651Sx}%-Cb}UK`H^??O*&llfF|v{fE3$venLWTo+NEgZ)dWo|eQ4G5z%#Nq$2weYQRVC5e8;%gx_kiX zz{IJWw(~BH?CIRbE#H~QG?DfiwIevC)8P5AzA-50?xm~Yv-7u`uL(eO91_NTUXY?SKnbNBQHt5NPM^_SMK%y)Dycq7fMFCuf}1>ajcxW99~4$c za?kzC;_kp+^S+F}jmNG^yyalx;R%`LS7+>z-aUF%-zOt+tBPGLc{#j$8cVdX#ng!) zu~Uj6+7&m(I&y}g{)*Ih{C^ zWqzUAfHUAGzv+K6Q8?{VFP?+8mVzxyKChaqnNw)3{I#-&Vu;8d7|yD^<&d zH@giN_;i6xh_7W#P>;!yWj%?lWo(=oK4HEl;>jr|xlkks1XICR;AFHHX1$eO3abFh zR6LMoxFjXkS;OEZtlbq3k2O+4kMyDez={xhO4zcGJ-&zfb{tscd;?pBPjvm6Lps=H8O%?}jD6Ba+qWfF+ozIy zQdED@wK|^n3Vg4DhDXKq3ykbdCx~3IhYR9QE<(u-$8r{6FB9-VGe~f^P+k&RXitcu zEHYRz51BOwyqR&tG~IBpOTo$;3s$nD;?1Jql2bM*`SEG;XDWV=bIfZLp3o+}kKGL2 zZPHL)0c-h3DHt=yQ(UQ4?pGYPRUN@LKr?2 z`CLZ0={IkOKOp%{&=@}i&=6bWLrGYzGnFw!D0e`(*X{HmsS?&18&=bxb*w}(J7x&C z18poO;n%kCok^%p#FsqMX^@9#k{(xSky#2ZqO+x^8+<+4FQcKXN~-~;e2K2I>yte} zY{7G8SVx5nh$3HVk*qFFGEGEsJAopv1$E|+b}sSOD-t8I0V8JA6|fQ+ub#(x^H^j4 zJI`~m-gx%G;)c!C$=cRrF-!deP*>`|gkMZRMdOYb?G+pg(U@gLqt1wPq*pBCd5__=gVTwmB`SGS9(~1XC&`y3wUa0rQ@>f=PxiwqWti+p|?sU zP_Q});qJsxEdX^oc%eO#mw;Ou|IE_MO~JKLffJnzkEaI04bts%LS0yuWMbpERutn~ zOr4m!cDSFh@7}KJU>b~5Z(4-tufG|6y!|lS|GWvzH+wPoMWVJJw!Ac z8K%5Obp-?^yt?6!JSbAG;aF2c8Jfa2y^&(cxC)`*%p05L#tz#ZYt<1(U_m^OKVizk zEsje(+UeosV-*A@bT)Q#2fD(?liN^Ajf)S09Ws!KPZxVK*M^m5$>EBXAKXh?*Cy@m zR`GeR9G}Wn_F(f6~v+! z-&pvq!Gme0)n|$Pp!djt0H&8*n2o3;xm9AMAZ35L!lS8BsW9n2zc@SI!Ey9nDmc>w z!DM1@`Wg<>;>4mg=aozKBN^?Ul5|v8;}oh-nQh9^YM9fua~R2jES&6|wWKv)ZFp0?Qd9fORTtTCbYAPD+~(?6ov1)T_po@X=j)o5My1(Tu{6l;O}e_Y9LZF5F#5*s zJ{zRt2H77KYCTKfy)lKrNdFh05ZN1K0Xeh~XhlmJe(#**qk*-K3FZGu^R|NuA3N;2 z9KVO0vA;u&d{`3}9!H3cxpND+wS1669BY$Rbvc1T+vzI>jhPzA)=q^Yq-r{}29c-9 z-OQVY4rivYu*an-R`v7TK7tw!NIRiNNzc=t`;PM(hBn-VztYbQ=h$q5ad&($_fk}g z?j*15ED9mg8e(}p$0t{YMX#yl;o;cJu+rZco2@FVa;Vu$I>jKno9z*s*GtwKKvHC| zqu|FBKq{*Ab%a~T@=-v|M=_biOfnA~E&EPyyWoo-3T?&!js&EL*q^I)2OH22VvRjx z2GUDqx`&Z8nh5>`d$A9B(N537D1B_!N@mUxx!dOLN=)y6Vks9RrF&0~e#wr0iJb8m z?^_K{z6A2GUUNFxBhLsf@3Pd-X8B6Qv@mS#12jPLaBU9|;Y&R`h?{)_JM@#x$Ca(m z@4`Psk8nmea~_p+LyTO9?jcK#(em^L-Fn9)Nt7eb*m;s$f}()9^R^23`6CcP3@gP1 z*+Sm$DVi=slj^BY0=NYyIeMj!pYU3+0KDVu-txM3!&-66S>D7wEK`Io!wz&wU`KyN zt{dgMRh_S%5D+}?o&H@IriWITqqlFaqm0KWh-WFV3dt676pS4>ac4$VpjV%_o)oZ` zR09(x*NA(70vxi=;^8tnKve-dlxpAD&dP?N^4ELYFW#-vYSwLDy43(MxzCUP@TFw>`v;0 z(4>L)qj zA%~c*g_g$xTMA&TB8OOW$Jfi;vuS+qs^x>WsC2a88o@#=$cEh%0;9Y9OC9S7)w@6^ zjxGuJ34+y$(fjx&^au&aqRkixi;9A0GYdzVjSk~ZjDX^lRT(aw_IMS@ED&E>(1MJ= z_HYLP2b;cv_rj>u)U+7fvxphsvFHl;!&$0?qO3r6N6{p&D>vamj>4hU>El` z8VRqLR|Rnd$(5w~ND1XsGch^Bw|pb9@%eFua}^)P=|)X?^IlpL{g(VF(N?{(HtNJ4 z*NJG#`6z9R;mI0KJqxN0+njZbR=)RVeA-gxTy4f_JS*d^&$mPu@XWB-d+2{YG$O4+ zzGb;|hU2ePBH123<2f6y+^cEX3DGj%d$1t|{hmYnT;Cams{ESecY_@%3bl1ApKQ zN|rJ7P7{;PRE8MT(UZwM@}ze05Ct7z&C!#@lpN)gumm@-%utq4M+pP|GppssF1+`I zo=HkNOH4$LAq#WavZu{54s0oIXp@%iL3^||A`e$QEe%y#6Cp*Ws34oQpgfcqYIb81 z3`M9JSYB5Irj61$X0NuF3)MXh^vi2?T8im9OS%P0bdEFPnNCOL6V|heKbT&XqaB>u zFj%zLp3aP#<1i$~$Os)S*md4<{p(bwdJvP#DTPhOBN%p#J>Z^}K({AL5deW!r-|=~ zOtgOrhuk-$_0ne=ueVOoEdAVa>^}p|Q}3gJgR^fZ-_#UHR1<~*nQy5jRaxr1&ZGzN z)+!f=(t!A)7FC9Pq!iC_cYUWcO71iFmdkSHm-S|eQ9jZ{ss%*ZT$|bTndP>;Wi{2a zro+L9xJV%Q^gOTO+lDoTOpifTnzk&BePH9-y*Dv2bfuMlUcV~7H7YqfFJBhxcRSMmK_gtqfdNlcm##W0!L$DU{r zFzoNS;yx5jY;04MToT(ipL?t2kEAkY_5O$Aj^A_})z+qz4wm4gey8+HFx9#lh1)Qn z5nLs;*~>CfGvQqGpTuz+P{2F>2VkYqJs=o>4ML)}wkX?u<#x_R1eLxfP}X#BS4~xK z$I;X9mHVP6#7f7P{+WRG7LR!8_OJy4vl{CYl+gw>S`(+V5K|~XY|#%2iG^wPi_i|Q z7}{3!kr++1jvA?C9W{9&^dW|nsUV8!v;b8So9E`+&SLJCa!=M93sYz~ie8etgLEO$ zmQkq6ij*~g=%xV$C?q^V^_5}qhGFvnc@w>>9c7UnSWt8tKH*(65x=7gg`VWx+R6EW z7m@PDk^rNDzkWJ}%S-5Gg+RftFfFZ^&LvyoL$)Sxf7LRfU(h$%w-L=8Fs3nyuHw-gdhy|iAlAWu( zq`)KnE?QUu$=7D9yinLMDtYtVs+0gymma5((IyxVs8Ykycprh7ls(2T37{aTeH_IcrXlJ=^aCUDjRgV<0pq?7$1B*x)prWj zja$TPXz!C0FUPT@Lcr}zzPu_qQy6b_iC^;)FFOQZgt&y(?>Xk>jm-nD@F*FD(CZ6w zoTaT@z6cST%g<${FG(UP@gSY}f!#~sB0ZDW1mrjaS4@Go;hJUBdQ*t7_~nDjt(To( zGUx5XaZ_5!D=P^be`o;3v4Ltu{2CUaBLYexZ}z5~Fo>v6QPfG>lgDZTlX)v(_=a-FCJ1Z9M*neu6k??gOOXYvW{J8D+Dq0Z@Cxu4i1QM za8m1GBYUtt2d2Fco6mv+fW{lK3@}F2Q|J;Fo!qErx>CJG!8E5tY`8fbBO3w=8RPvp&hB+0B5#h!A@B{4WBt`;@_g-7IvV->O?HyR0lreG!2_`s@0yP zhAjL$#?_Pay|hiVL2i~wK}6{9a3X>d9!*o;cq+z7GZvwbwLa0+g);K!P?wh`ZrUX_ z;QY1Pq~9K*8a)>v`DtVqJfwA?qU?o!IUAaRJ{(^HkRfXD5%e zBm)5FVrmGH3mSv7h!4_(TcZIg$a#|ZN&UHxQk8(>e|9FXd9~;5pbe2!HnirxUK*;K zmL9Pkp43Nt^*Ls%8m8K5Zu$*ILfJ)6_5I9Xglw|CFKx&M9x{`Wnd@=vg&zqj?9O*j zfX!gQj2D4&Mi!;WgMhrfaYo}LMwDWNxHlghN?o3tR<^itLR-z6I4m|H0E5Z7a7Z0u z#p3>1WXA91cSK<>iy&!3H^kCfa69vpRnV7&^UC1j3RRwdG-#>Lc08pE56ia0^$-*0 zv9D2>W6nc+VPOP~cpdQKYJ5O+hhRE5!cfLzzFvb@#{zkt@dUp)Ihw z(z}Lsbqk&>*(+JoZyASnw<6Y&j6MX>N9H(Abwha~J5!h+8p0SDO}YO%r+6!8LyZhi zRJr3zQ6T|KJ|b&EmB=cHrb)4FnQY}$5Vg6d5OxyUeZxtCjR8ka z=h02QsUR@)28}znc)X?n=RUg*@oCkIf47y8=W$v9-qL3Bq4a_R4|Bj+!LEijK!AFsRX?zB;Z@u|-H zKwOJbL0wDaZBD+$@9O#?YK}aE@!XV1|I$v-r5GVA`p(E>In;Pz-B(I~aU%iw%96+w zAQ+sO3NK`uNp29&I5HBfxQVK;@!6age$k}BwproI5dE0M?TpDS`9I;-NGDidNk`|M z0$1x`xEOyLj9TF`tNRdt=9EWNA^sZm2)&N|z1h+0TQT6CgMBxA z-9WZJ+tz=zxF{;$svY{$_`7RRfJ~~dimZus_C>iJ30lARVBjzr&>*vhefy6Ssx>#> z+Lvj3b)(u}$MHE5_k0dQw26i<=#rBsHG{U}ga&cj&{7^Vfv)bIG_Wx73fr10;k{@N z>a8h9Ikc0-I`VCHPW>W!M~BwH>Q3$^Q^bE?>^l3q)Fv#kx>BSkf@ji8KWwXi&Oq=0 zCInvaxX8?+G2}ez|Lsa#;rqynZosPha_;Wc}@#b1P`E0B>^Re z0K?j)j^L#(c+S|usfVq#A{L_D^oxB6OU{&4T?wE(11QhBjT@x2#pr^8^poY6M{LWT z6=9P0mq<^zbqLQ5d%hxJ3e*WY?9@>B#wvVa3|%(%#$AcQ!dh1uP}iJ_3*8&VJ7bX@ z<;0F-PutEiB>u(g9xeFwXU@4b7@;G}rbXzf_rgrWZE{&Q7P!z7gq?Mi7Lje1bVW4t z-UyE=9p_}$xzHZ!O`w-+=NbG^sCH{Z8)xS}Litz8!ETc;@%R(I@nd{4ek!rmy`0hG zKdc}>Aw`V4+Tmk)2hmcLZhrF`QG83l#z3+DeD)+Wc-qVlSmzv4HtpXpd-|?4Mkp-6 zP}nI^$_YrdX;4SLB%n$hqk=V9i6%zIgx2yDIt2(RyPX?Z(tKQ#^}yhc4+g7=Q-_3( zJAwEXP*T!jDPJtolK)E)w$8I5C@0$!BwqD8Yh3XWH8&sU zGo~kvxEse^5^&AifPt>_3>fjvoST0Ww{?Tcu!Ryy@dO%z={KO-Bi^7Zns8+`sE2}@ z6EezY@mHL7I$!NKujWwQ&Q`#`-T@Y`l%u@Ky4`DBh5#zj1T$`!>8YBXgU!4*np-A3 zNuqkhh;;|;oW>5ApR93mOO-?>YEK31Uo%`|Z0!ibqThOXk^bnjP0QHWwX$YTN81m? zK)BSJ-qNMK0-dJNg=|L>i(pA#a@{dyRtc%A@d!%TOHAn@wtsRN_ItxggW$ank`?7q z1>SIuim6$5K!N(*0tONLy;@dlg)ZY{Ad`)4TehVf1{?G9$C_@CdMvCpW%IR3Fr*fq z5P(ioJ}nPVK2;vur2{l0^kyXCh{M|tarud`7yiOV7&kEI5;qmtil$UBh{-ds$&E)) zj;U-0ms*14Sx*l|Y0NBJD3bB1IJHgM`uitzMq9Tz}c zQyy7H_iAhv_?g6g<^lA8zehz(_yx<_)V7?hfjOu2y?d>lct_~rN|X%H3Vr5*y9?n#=4D$no6S0H5HUWw z9dET<6xw2RK_cLSrC3D3Ix>0S#MctVPQgHqlCJ%w6!2D;)T1kzXRLjSiA-oIs%|n_$GqCVt6AUiq8W$yI`fhy-jhEkasL6Y!nxhs#PaCWdy|s{65cAy zU4-G52DKUQsoOK<_y^?LM!Drqw~Gpk$LN5vFu+uc=W51A;7LdeGwT70$C-!lC>&pX z#YRdNAxPux-D0aX!7iG_yWd;ueUCTAR`L~bsOUnVyQ$e!ND|)rB89@vI5H0w+Ej`6 z(aTFh%7V3z#XW`X{#K>pjhElV1@y2^L@S`MCUe(cn zOpr=Pu~lGx(d>7>Z)mY;Ie1N;g5-Qa8$#OfSYRdP?Y+4JEv($3a|CFsjVZ;vz9A&p z3xRRWd?1~6j=u^1ED3T99KOMqLb@X0y98rkI*0A6yRD=zIg#>P7Bo zh=5#8tCX2m3jcC|LsVQV;-(TogfFyKUj?MQ{cT)(0iPdPJ*DPwns^ZGoGosP=a=&2 zEn4ew?F%ZE;o+|<-g{_{#NbeL9(-5Zh2Wf3U`{k`{gCC(RdLo7>d{1sQUQjBI%8Q& zXSsbUY)=QNF}vWk#8=9n;ue7$3E5G78X{aUfFMI9#Di!=h_eMdbhgu&8iy_T0}{IO zvGG6!4+L+%-Lk}v(a3)@wF|$w5sSSPfqQ|2c9^1GmZc_=X$0`!C7l32o#uoaOoN=K z(Pt?e!0YT+?V=c7y}K`?FT&H$t^EgC3PaVQ+{jSvK&fSM_nGwaE->i)@_i~_Y6cQR+l&x~aJzrg zmpTgh#;5UWGd|f|t{6A5OI?bVxHBRdUGU7uBx4TwD^NZn)9Le93<*okQY&+oYF>*?FoP@Snw~9W!*93PVgpBkjRSKmtX*ZJ>pVVDUP`IsEKg8qc(DGUBTu_=wgKy+?Cq z!2?d-=Qf?#;2U5*H@YI;=OXBHSYZ-UUG6bxh01y#;NqOWd_Tc00T+_RX%($e>PNm3 zy>L;$VAaD6hkfQO&h&f2ZFRMXo`KwWkRpV%`AfpHF{1WLHY?v}U_*8>V3%)vu&@X@ z5AQ-?RJx&x7j7(@1v{q5kf)Jc?uqBs@A)vi`#cpxC)${BC*eQ(UHE98O|qNT^9XENcD(yJHyqD2``o#hs>vb(T>R3)8qfEJa=}1Hb>~Lb3{l0`z5MY7f`K#1%=H zYaJNX#NxKeZqLnVmk%Wwr=zOXawbk&MzJS`dVKP!RV*77e7uFhZFJ5)w>1lhhymmR5a?bp%zl-paRj3qRt1Xf-UUj?gP#@#@Ce``1N2*_4N2q z7caipMceFV7>9_0_Q&7~C=0P5L)=}PEQuI z$WAs_EvJ7qPe$>^jvy{H1)U%1A$>3DEHndtLPux*LOqcG+G>rGkltxKe2n@SMsTj; zo<6&5{9F1D8?tkwS319xl`KvXHq@gB)}Ih*c|c2tJ~%unu90^q+;T)?#{fh}L1in4 zl9u8*nyou!0*(Twz>;lAO zeh!3WUWcol*p8^g1kHt|ifV;=J*GT;QA@@_5ttgE*%(YBmB$ypC%GHiEa0IM0-hWW zkX*5`!LiDCS?Mxo;c%XrB|Tu7S0M7VRCPSpg6`BbVSbv}+9}#2oAOJXYE9eTEMW+c zZysqdNF56DvK$WzT>M_*Yg_65iJw6`n4}>-8@NF+6nCCcv#y0rG~@;IQLUDJ_<1+m z%a5i?`!O`dFMdID!*eeYm9MgV94;~+MC*lMR z*SgR(=JUn5+HSUXgA?h9(bHY<;sa?C z{>dX7Vkx-?WqoxNXZW05qpTXkJc@sEz0GcHPz4T;zH>q21%e8fQKZ0O8XExyYu$aZ z$0^NV32hb|SmZc)H{A+Lm1NmFP?)Vudsc#&Rzh!Z(UyYMfE4(cD2N(^oXvUa?+0wC zvcLCuUvNFdm||wiV5twvQ^PsqN(?EMkTV-47rGG1r3rz!x3IgTsl&BGj@<yhAQY;t{q{#Dc%?5uRQkMvI)7@%_IupTXY6^2fDLJn z+pM6`wCija-l9@G&#g=%FJ;?-^DJIzL^89x=HGfUb^3s~96L=DQeefq>yuJOc>l zHJbFn-I+Tx-O2`u-c5)ou^ys(%0PAMvCeIzA`iLVG5M0I9UaQUreT6#lYz4kuY8Y7<MnVZCvcMH zKKvbRMCfNtgup~tzks%3h$+(hrP!~@e@6DTG9j7}>YkE#G4bcCc0;lgmZ&JAbBb|6 z8?=#t`YlK?jQ}qPosK1$-4M}x5}qQ)9zFE{F&OtolE65x+gLaL1^d291|&zqPf#cv zum;aoCefI71{(m`@faqbIr(e6wvfRsEUazYEwvk8cY^A*g(Cb~VA&V-v`81f`mBmw z=582tdM<9XrIY6b6Xg1&+#3%VQgKJw6J+|TSA?{G7x|D>2X|mz*xLcMVhaqAFhf|f z0U#%-c(QjTFz6d+8`Sa$W@Q}R28@DY1y9ifQPCY@!ZC~X`z2wr%6|R1){Y+Jx{-J>*J(uwYc@nPvD_v5SQ;g>8yg)zg541#S$9`TGh*<<{n%!q#1E zsg7+c{Q!*40#)*D?b{dzd|viLbwiE>5Bhez3tODn0#X>cC3KVbdI@nmxD;F+@$}v^ z9bytx00ss+&_oWJ1$Do8cQdwGnk@{DXb!656o;PmMOKBDE%bkTIJ-gKR>i}CmG#0) z9d#(vUuw3V&*yFjg8{Ele&OjZh_BjmGD-!(X(o7;qkpL(s>7$1KQ z4|<&^KQP~sUB8J>zqPfQ!x33*L8c*JPB-!diqZ7CHbM2({Z4RPbn%F1NY;3kp;8Id z3&gyW$ivpcd{3!+Bzp5qm{^ooHu|kVQBLNy1Vs+f(|ebZ5iQOCAh6_V2p>Hk%397t zJg^Yt3Ux1*0=fDH1^}?1u%ao5_>!~}-MCkTpBwAN<}G5@SBj>D6U`2r-4DWb+)Sm| zfgBKLqK55K!i>bV{@y;nv^O_2lvgoQfmWoM3keh)RUdG z@zbo%d@dH+>Y*@cs)Vp}WL+5IE*62u(WY2s$BedS1qdnHpZj4Y@U6Kn@LUKs9Tt-B zvDwXTr`=O;E}du#mZ9?Hnie$+7gtyMN^a`e3zaYckxKeOore4|T6ZB(Hi9}Rs3HM1 z9d67SkNLBBW>X+-8x~s3Q@Hb6a!d#Mbu(b3gci_n6<4^aqbw<3yP9yO3zEEX_0UHL11PI2m}>T8*U#MTxn;V^pX#Qs zy=|~Knj;)xS5Mf^6Qo|`ai(S>EC08jW30v4FNtMLyV>eg>bSgm19W=vRi~??QkxXy z?5mSn3PuLw7V_4uho%O(NP3vZus(D=@Gqqwbcbkd4K@STQOhJlx)NI&fy9486XlLC z1pdpY3Sx<;RUJYdDLtr)*sOBC2!h||I<8o^NKwMA81xknxfbBe6?jMeMF z#rNCOkzo`ZbEDD zoO$gG9of-LQxaVslKC?=K-jq#x3*8HUcwEEae+Kx4DZ z81Di9{1`T0kU>OHVJ{=~8e6-fVkJvIV|Qs(DVrhTF2WkwxSFRZJahSKRK$B@)1NMZ z#hhC*e+U?h^=J_v022uVM;=$8(VJ?JVkQ~w?g}T~dU-C>A$Op%E7W4`A_;atvw5}_ z9{R0QhF#Pk5|xtIeuIO2gAYiF%u^^Rm-?X;3dDn${SNQ9C=)w%o|-4$Axj1st_ z3CH{sPU}V<!srs%ya7i0~-V;deWpOGLB9&i-Kj?{G$ zp{&4U{F)6xJZ#6Qz@ZT)*WHEWQ~vn1Mz!RPJ?57+Ho7FyT8KkJ%?43GZ=yB#qa63o zsO>Zwbb}UV)Z)~^&S~{otHG`g^&%tz9>`*2UPcZf9Fh|^t^ia4m{5Q<4j>E#rhWrB zNp(dRxsj*r#axd56o3woQV!Cv zx?5nvsu7kZo-vzBs_tdCFMjO7@gxb94yu$;*L%i#X&>lSo6!viS_1ei(4nDpe54t~ zM@2)~C7Xx7^-UlP1PtH|Qoe+79{*v|i4ZN0pM+SGU0m@f2uvD;kv3)TU|?33V9<^= z@> zCDuKES=TgpiBI>Dl}#Eg($-$)c}u#D5BU!6V^OfR;xpsKxi)`Hvh%I@3*;LVi1C(y zhg>#*)Gphh@dqg3uFtR$38k{QZm+PdNvK<1Q||l5Vn0sF93JPeZ*jZ_z3lzN6V>ku zieWJmdvA3T;ObDT)`vHU=(J%i8!aZZUl8nwr>Sg`~gZOeDFnKwDY zWK*r8B?j*g$K@LSfH&I8I~R#ujC`xrEJNiGdlSrik2M*6gXz{~WJy2;;N0^XYq z4gRW)5u#a`8|BBX18#TjRjspw8pwOggnG3ZI#9we<2zR6>V~Q#9zhjc-@gEisq&Su z`%#lJ_dSzGq80FL&uND7`;I_oO&YEPsx}*u8YAUTV7MXWGp|`FS0>)xje5;_W#Vmy zwyLOnV_HL96#TgBGo!i+&YmUC=6WRw?OLN}YXmHZMZ(a6(=hqHEmVg!Wrxcus`jWP zs48>g;o}`7E%bhca4PGvqa0IJ5YwaMy~@U;XJQT7!ufyFCFzOtJVBLg9jMBOiFOnS zlE@67$0C42#4_`dk*mOb_r5HmYA}$M7L{@NEM95V3|N5INY@9F3RyePL)lg)nejo< z4N7?WY>!EUeDTku12U`YU>OYSJJ86E;+<3r>GBc%IBHZ&iiVFk@;Gq}oIG9yJbxeQ zB(gfTJnb0%E?;f2o?d{i*Q@w5a^ARg zvF#LqP(}jV=K++%K(_^lr6d{M(;|R*=SGNAx5zxc<57_uwm7Q3@fzFBG!I+!96Gh3 zUq2iHloqE=19wrIbBh8O;*mt94A4O|(*RuJvon%*DTtP9%yhhfV1WbaJ`QYJ&J{Q8 z-qPCf1}wBGCiz-XI#;*$P5V{59z5GwYL(ujGexj>mi&?cVIDQaJRuhD@#YYKG66jS zd#!Jzb8BFv@+Osy>0kD2Ls^Gt5NwFNUjnE&!}9HSB%SSfEA28p5$4~lQTpa%bMi)Z z7FQOIIn)5xy0iFHS<-V@VO+bnZD>ACZrj_WO%#j_0&!4kcuvzxFBJC5(Q#(jubvG+ z>WnB9l<)ATI)R_^3$CV@KrxaSn_;SE;8NBk?<* zV!PKRjqR~KU=ZpF6!FMPq*m z1Y(>RduioOsN%qoB$$f=3p*&Vxx$P+ zS1rU1Z;j-9dR{6`R2iTCRG@%Z!i@cmD^kd|Dhke*ba)!Ag@_>WFl{jz+J=V83z}R@ z#@u(tq~B-@m@7_P*Ewh@c60231GhM#RL$9%*Lj&?6jaH+whIY+8Vj)(bE1oh+)pWq zw>pXjNnx;JwFLsP>a2@|wQEJ_-o*iMvZ9jC*R_DGk(&LMh-liadb?Xhmajd@Z$&mC zVRNb;L&$AURL>Z(A(GR3d+S~EdX`;Db-!bi5H`*+!6_%h5Y}F3!Z=^ew>~ z>R5;nQr4eL%~Zx1VPIw%;3;zty0oBE0_&tG{b-yDMXSLM$_yES2|}|WieP|t3Ntip zufOW5%yRDF^WFPBgcpq9gkq2VDCKjcMCgCvz9y! z00~VojH$Q4oA+PGM??5iZ%>MxYN;Oy z**j4!$yT!OP(B)eE{3Cucp8V5@mGz}Afb*vPr++@gBsw7;xxcO_{w5mxl_t<45S#d zcrXi`DjtPU1|PfKe6H0(pIj97X(BveH0oL#obrum3jT`WEgsueZI)TqF8kn26|P94 zkD?*b%^UI|Oxil!hC z+awtr0#eTJwvBg;!&3~b;?ybf2S0ygdX!{@e#V2w8^Va~GC5;wEr}2!$KR%c?IGv#U{dIzF4^kFEJZtigO;UinJ*Ol|Lmldh*X8Djom z=HDALe=j!bUEHA+Y}O1~n=h;61ZqtdA>e-$-QzG3uw~+NI7vrr{K_Qp z!-;@KUeK=iXv}RQ=ak)Hs#~8#Y8L;UO1>!j(K4n-jRp(vO8DA;@nh%ZA&qc(;Eqk(>EK#5V`Gz%@@2xyKPO5bX-Z;9#Nw=GVjK1{xqZztKh zf(_z)uQrm2t0AhSaJ6s*Ic~qP$wRRCJU-O`zp*>P6l%J8Be%^uP0|dM!)79G@A*3s zi9v_ufUq$SU#?(HWK=Po7g3i1A%Mh>KvaSAg$9M{x zu$PaZZJ8~o+X7Jg02}A|U7V)NO;j_@+1|vGl4Ju9(H*aF1lk^VLeC!^-%O$KG3YF- zd{K>42%xmkv!F*Q1D8i-SKP$m_(hvwTyQswuaf5WJkN1T9{PGs+=bqS8Vg@zqhtUEA~KZcCK3 z&FDcq)ogr4lE8*yu~dKmu(<6Ng(uTc2v>w3HtHzO_=GMHd=f_(Z>(Ug1hL>`wev=S z(RPxhc=$hQPuT-k8R@xDX=rkK_bIiw~4|?xXCRbveOCk1vWhv#W zi6kK7R+`B{2X2F3_ZTG^CVZ8pFU|?}wdD zNWZm$#UowiLJ-t6HI*%zv;36#QXwn`ogV`boqNpgN~$N>*^a zx{a8{jwiriBVS!Kc*$|S(OZDY?A3VZgot37(9KpmN)(UR=ovldn%+}zmhkvsZ$=eZ zVTVlrzK(g(?~8vdo&SQv`8gyn(GQv zRgnO)l{*GwJP0MH3L_Ahsd?R-mS`|));)n7wP8Y-OzH*FzhB&DOB_~@(YdTZQu-V} zq(v@!inLD_OJA?Mw5(RTbIAkx;#{2fmTT7UuNZ0pz?58#=s65`8L#67-y$V2z-!^t z@(kOL0oEe#c4B;9zbG#=HSiU|EzFV~_K6zKg_OM@s~zmVR-92Y5j9?JW_a|;FYB^+ zdERK2WkZZ6o8|$*KZ+)!XA5wA3*8Ijzv9Abg0{YQ(B#4N3(RD9N4`Qhkn?MqW>U6` zGZLiD1kj|R;9PD$0v;QIT)es_hKJ6aWft1|3FRwv@jP5vWa;GW|d`l4xsF~Lg_FqkNh~_-5t}fIR6c_ zO$|P*T9Gat1ZhiaG3)kxW*XiIvha2lXv;`g-$-nHJ~wglLLXabKG86D?6nI+2nr3|V@w+K-t~3sO%?J= zEmL*3i4K1Nuw0xhP?vJ^F%5Uj!D@>#eg?knYy^m!G+7&@bNWuE^|X$=bIgP%bBRh4 z`Ep*!O+rp-$J>voMpud}0T$(-C<}eZarfZw(-7g-&W&^%J|2R-jO(YsFubo_KQc>O z7sDmU-;}`B~bh4eor^sBnH9CR;-WS2wHjKGwqO^RU`$3)D5;p}_h{Kmw*UWVJ zc*eJF@r^z@-aDktf7%gWmixuip;Y#euC@m}v+8K=nepIcxXxjYtVr2_BYpb2vcq5k z?5>wqM4_Nwb%cR0B(eE+zl);lQ}H}PMX_elmfA3xn;7%>S2Cfc&XRQ5_(SL;P(_s)w~>QlM}Oc-iJL8;9HlSNYu2rh z=bW#+{7f?4jIy>Y=p37<8Z8jOfrxea``9Vk3vZNhKlbyYDz6L#++xp@r`>p37!I#WkEN(P!<*CMMH?&O*>@<~T0K!3_b6QHDb=*kw@klLVP^ZS ze}-w%_*R~j!9*8ntXD9I?~;Z~>zc3>;t2-1bD*9^74bO7Qf!kH!$ij9qbk_`mH(v& zKgfTm&*f=nh7!a<+-isfJ1o}ex;3yQ!~nvQQ@IO}uQTrGk#r1QKtDS8o}a^Bd|N*0 z)-A&-rV)ogOIr<+`8&*u?Dlb2+rr|2H@tY%K+!oz&%V(xE=e3Wopc+R;f+EE@~@W=3yT44Vm3-27X?roC9Cpm^M}>krD*YnI69V(C?K zC^V;`K40IF7fJ-<^@ZN1_`GC_bJk-7|IGbWDyY9ohEWd|l#~t~5n?&_22Z)eBG6l` zx#bN${<@RaSL{ha_IMtQI749l4r(}U{0&jW231EQTTl3f@^@Ys)20MJp68a)zyX!WfJR+@p#MUC=}z;Fo5yReQmPF^~#$ zgA^Oxj_z?jD7;$GSE7a^pP-0BVhUw%gsKvgG=P{EsEnC*>(Q3}JkoHV8TEdjLnsUS z#S`d(tP6wp#xb@O5scZ+Z2I&eWwUQZ)`7z0`*&Rj@7ts1Me=s)DSmmAiw-GPnqD;% zUCu_)6x%{?S1A|B2x$Wot^Jo|7B;u&tUXRM29IweckTdcbo^_hH%SSrkB^67s7 zhVAZgyN@3d>8*m2F2pVLOT}W2CPd~-fV-Q_O<%N$JoVQ<+%TPMBi)g6Q}B-J!UvT& zy>e|Ty$+~hE~(s!tlLu?H0;Jaga^}i8cyM--cLZV7j@?AdZpX;`>hb3+DsW$&4(sS zR?z1j$I&o`qq$~A%XsN%KaOjA3k}w_nHyTh4M{6Hu=dby-aDY0Kw!hSg=;=E!;^6% zs}jRF&G^EWrp&3*upgN{o_63U2PKVQaUdAi+LkO2Sf5*=7te6x+ZFa0$NA ziQH>huMsF0TLf~Tba>ZN`_PMJ(^_aU%Qo%~0HlvX*_uH&k!a>+w>5QmUAXJ{iA)zo zfj6AJdFw?qFt}J<5udSs4n`aym5QkZ=3T}5M zNxTV=)WRWzc)BT>&8{h9@VN%X%7RhlzMbb=OAs~FO(fr=%7_`W-2HCG9HLk^aoN%f z6lu}qfKAD%x52CK;8vA9d>~HYCP!#rMlDo=vOq^CW)kAU1`)%?799}R`tTxWQ}Hjd z?}<%mK6ue2isd|bRTQX1nrOTO0DT<5vZM+*jX4NAI}8^`0e|2CSDZ(e%yO`jiT1|t zXe5K8?18E6VwUmr?l{dtECCM5wdPU1zD0&zpayw2q}LfAHVuVBfW-^+FpGcWU&}fV zxerNyw7;=FI8d2-GkQDh)YjMT*J76&}``% z0s*$-=`%Nf&tp$lOn_)b7fZf1gxq{x-93|6%qh^~Dcqjb%XZ|6sT_d>MzAZ>quMIR z*`8a8Wc*__(io51T3LusS4B4k;-UAqp$%ghQUHS4m*nR+nq*Y?NdTYfm>^#ZL}2o` z<5DI7f&;DKBa2Y)JP11y_RUy8*I*3s+Q<(l3p;Lxw41L@N~w|j#&YihZ72bEYkZTy6Tc z;lA=iHcTJD8C5Bs-AG3wU3=CuzWNFilCSNYW{&34OxvJ(DjfHhN zve;GQUj1%^PNvN10Il9&Ak9zK_RPtm1|hdqOj>u9UgP!crth(qbWHanvKJ+C17o}A zm}-#JLO;VGF6;nAr0^I7^n^4y>7W3Jh@nd9HtIoYYc-2PuWk~-l#YMYEem>~g77=I z0##3aCAEu4)L{DNl0WpZKaHSr1K6?Z3m+G~<(Y8(zS!aAu@McT!w>Y~*n}{;!+oud zmbK-azI;*Uu#$`2THUHX?}T7=UVAKlh46UT7*~|~snYJVbPGv))b-II4v2$nmZpA$ z=hAb>G%QeQwYL^vQ>dn5{QSdFX`%oSEh#l?qc8vG8J^{xDdM6CL@Lk0-YX;{~t`(NGc+NIro(yKF zysttRGF&$&&S*Ij5$2wf=m0n}88aOdj6e14#oI^(l;P;l_yyPZxF~f~5N>2j6|T@& zmTL+=!4|bQsbGVCPkPo!f~z`q^pr1;@|Tpobc ziF!fGD!E0DS(PF?4V>Q?0BkJ~57@Xpqs9(_o?MnFxe*la!d%q`HGAjSK`$Kma9#Uq z*^cWi5;)0n5v=u*w$j==K1Z8S@6f$g0=Dh8oj)+FZwiVF7;u^Eu0aKVos|eF%0Hjz+4K7WwaTe%C34Wl&dq1i5$D2^14h-h(U>2R* z)hJ;@95G$w?8rQzXwyliXtt%A4TlOZxEa#XG^i51nvUqczg#mj00nQ;RucW9J~<3Q z8$N=T&4&4@d1ivZ9X2Aj!jRyRVzoO09SRv49&{2E`hnzhCt>dXn#&n9Yhz>}m%L72 z3u(ur=*hvb3!T$B!|`4Sep>)mbomU^zWAicrEf_itkw1GM)$_L<78}YE^ba^s_RMs z8nh@n`hvaf)kJUz51Ye; zsUBzt5Un1NaNP+)#2m#TL0Ejuq?2lorj~0&o56@0_r7bip;HzW?!<6(NVzsQ#Wbo` z-erwnQquLIAX;g9vQVIm8{*)g*xK6%2)t79lf)sGSNBH%HO|vZi1E9WW-qVbhrKDa zBO0m#SUr{uG8eT)LpH~}5&Cg9M5?oalzC9LWXNiaD1uQz9=a0w1w4O+A()HEgIvGo z`*C=*&y{d@{f;DF;v=z1K%NI_uhz|m%Xz9BQsn>vW72>AxVL2xC5|&}%AO!*VvmUz z_j)NB{b5|d@jJR73jhK%wa=YS4rCk;z}bS=p_|T}H%ZQQpL6l@BDT8Mk&X~x-NyAO z12}ACu*-P#VLNeiu2dJ3ZhBBg`g z7S)&j5sr{{Yu4&z!g_I629|!C_V8nwwvLpn3W7Qdv>cGo?Rm8ff|D<$Vz2H<>jmb0H zJ-obW1I|!edA09Zf5ptoidfM_9XAS31&;d*14)0Cg&RL!Y?Ems<}p_;9*`q)d3%C6 zP4Er31XtF9G2cUVNQArDMFHXL&RnQG1F}Z%gC^>NFY>Jh?*f!BcQ3-?lco#=19yC< zoT;HL6+GjoPpn1oPQr?&e45f}JvV^_8)N{zo54gr8fiPbMHYDAc$G*A4T68uLCft} z*3Ql7E_yl-4yTu31GcGA6v9d!9tUfMw-OfW*TOebdo(tU=SX@>Av!`gO(8y(MvUnL z;$$a8`~dR1G8h=MH#FT4=gP+FBydCFbTykeTQnUUddK~a?G73rl%eW|7pbrI%@Vu1 zmTH&;08mO%9$ltrP*ulhrhM{@Kyxkr+z3q!qiol3n(-@GaRuC%tbG?Mq!Rkso zORTO&?7&s_rk`-WC5hZsRnX68F6^c-H|;c}%Oevv(+-GKJRW&%d3QeQPIjDRpA0U2 zY&RY-4g!VRm_2h#L-53KNE;z4>2;S4aoi}Dgb+lb`z)Smg5;xjAE3t_91tW4p~@^dYwZM!77322Y1M0~>Yf`(LeW=RErBzP-Z zCv-78ZCylBk1Bvwnzex~NXS5r;0{=gK|p^wH3~^=%FWplE3v=8qRPej*duuCi(QUF zGaa)wAe5T!!_hG;tvR@ty&-V)IX>?UpZyA8fHsSgdp&Qra9wd(g@w`(n}ru5DF7!Y za{OhUF9lGv=P(JNhpcFx9|$MWN0dGGypa!Zb`Vq~<#_6(>E+UbH+J~^hPrnW9sUx7 zKEvzHl|uFswr^C!>RPPvgYnKIDVEP%nd$W_j)Y#fhq^!y2D&rf%L~BNJ{jy9>pwjr z(;6UiClWB_S$aRhD47)ed!k4cjalkhck)393^gL-&EhwW)Dr8iO83B?tBRw@W~Z{x zzZ{60u>&PK*Y1fvM)}AJ#Sv5Pz&=}uh6Ei|6(s-YjdOGrM7A+zV9M?;q8dXV?@7ps z$5{!bYmfmPIs+m&=KsAP;k`+ogW&pAXen=mlyh@>IS)3yv2z&87aNY^+-5z;OcS}vX9 zF{{5!Flqq6ZP`6zw-*fNO00{I`<;Ki%K^87uA&W)YX_3XT+Hb31%9)Ie5M&EXkMJ{ zJ_9AW6aW^sGW>RZjqA+O^0-c8Ew6fo%JEm@ghIDH&=5ruW_LSadUnb=r|7-VMi zoIEYLpg~E)zuw-jeOps#zA}67t+z4)K|{04;&ANhb`8|rm2K9xoZi}V=gO(6Yey4z z1i*FnJqgJ|lc9><@4xn zILHJ=Q&pXkeD6ETNpuw!0aYcnbZ*JPT61K5LqS5Y9Nzl32ZZ4=j)e^A0DIl`hX1+!21g= zGw&EPJ8MtHnhE@?y1C!LhHsB`fP>|P4*=@8&Rde2uWizww}@}MDW@$rkLbGgYBEv=74QLQrB{vOTUc5O{8jWX6BHf}AX_l950^ zoRmDtFHPuY4*x3n#?LbrzTV=@&D$b8GR4&Ul;NAszrydLB-ab+s$p!!*?|(W8?vx~&M5JUE1IJ7HnciGvg)hvoewleDGx`Q$ZPZZ%!ucG@Pk-9c& z`#~p!waiSP57vkcyL*rUwm5kXOi6T(f=x(qES|-%n(ty5s*P=P!N$$Yn$_$uE5E-g zO+I;&+9<_faar<}mKJ~5g&tyK8|EBpr$;>jGADx$y%(6?H_6o@iSvk75Jw}P?f)ob3y+XAXhpGdYd#*Kufd#t{U6ca#n6p=IH-mc(z_s@vm zpN46Fakr)or=xg>b<&|tA)cP^%{+U(4GrT^ITlDWtM$b|5n9wH;Bm$smIAuY*O9~~+ct{=W`mPqSw*lRW zKEOG*fZo6gKV|^vv$&$V@N-}V&K+5_nG9}JU_$MAxlj!RHp;`IC#t@ncN%^Y3ZY{L zvGN%39SC56YchrLYQ-<7V^6sY|4GZ`%aMBIK&-+UW(z00U{|>BMi#%fe|0}S@JtU1 zo`|0cHTw7qs#e1)8yNKb(7tTn1_2uP`$QIt5(&)ln_e>s8uTJfY~_QhTjxKr>Q!5|)y@`>nxbm4K(Vf|1&*$>d8%%}L1=-S+0@wb$J%T7LnCtuK>oF@VkcO|`D2|5Bq=IH`JD_VVd zhpS|}SZrfrb=`?k6S|G3XYM6a?lEx)d|9{4`!TXjHCw4+5A>X_P96y2{5|F+ywTzg znE}>dlzi7|OYu+N?HW~Q*K8OZ^N7$jb?Di??o>|Pw!P1z75?*Fsjrc!ao@^YBM4_u zo3Mxa#4;H$Tme6Hh5}?ObUOR;Yk?C1N?82)+M`dcoETTt*3`Kh&#)xZrDc3V5%Wwj^*w5G6fu@7K~m zdnFn+8+v~)C7b-iHW-$LXKjb=B`@@Fl6o9hjME7QBS#@_?xF%0@cxRjs8^day{-ElFkz`(J?^lC|`NOY^T=hSstgbBWnS+_AQ4}>@`6u&qT-33CAQeRUZsrNXI-sgYD z)nb0sYm>_|XB06YM~_tko)HZeFUYs5zW<8tP@V>`e_~AWd)wpG+PS4G1LN$>QpgNs z(tJmhn@3|A+6b;lT$;L-U2z9x#|Tr|dR>Fb7_|e&^{*jzt4|Wf^cQb8d(01KlMWDN z1=}@b=d6s;nCKnw&^UJY8K^J#s+Aq5x28cu5IHBqb->^W0AAZg#pGtWacyP#J}`}* z>fXzJf6Q^vsFabQjcdmh?X7t1<0`)bg*sGn&@t9cvA-)$9T+Tb(76T8F-9Lniq`iy zUy`IZ#FRhdY&ZH%IS!D^u?tF1GxR`|()!b8#~aLT&z(gE_xl}RfsEre0XaP*-;2&O z&l;kmnW5>r7EeYXE3b2C%o>9y3$-q0=qmx%GS2b=p9_cK_v~3B77Ex1?XXr2wER(t z0_EAzdmYf|vbJYqkzHcnGwmf_a8C4w3~Q|&bF3k57x}v%C84r_UBnog3w34LyOUYf zblrcy8X>1Xhbd{Tg<3#*VGwoj9NE)W9TL$Bg+$mei~QP+2LiEhXCASA0|qb$ju}{l zzw`2?r^yx44w2GtVzjB#G&Ud;7*hx;hqE)$4>sK~Fv)ZxNGNh`Fx3mtR!CqTnUq<1cJ=V(^L-VY+MP^lDS7Mg*Bj@EL3S$-U0$9iSn?ck{D&p7-F{-i>U!_;s7AM zssKemy1%ciZ24KQ_Hx5nR-jrwhl5+(4Z(ad5uopqiaEI${=G)Muy(}6-#^(}!KNgI zz}wM+XH#hz2)xIk(xQzz*NhccPnj5ufNkt{rlW@AXNG0dj^)>ro>Q}my_oIbiAZN1 zj>_`c7*c?|pZx6DZ5}dWNg%aQFdL(+?65+~I_U7h)u1a*IJg$$hSgqfzIA9R#)f9D z5QvXC0+57xZf0pJr?nTIR!4a@rn8U;zeoEvnx|x|XR3*x8f|73U!u@LW?q$ zS>%(7paQk!=%&~nvZ}F$Dzm?_>8kN+3id;&@*^fnQ8lFIaHfcCF}J(EJ_Ij~=F~)C zpq!~Xz^{ni3n|LCQ;h1x-6D1=nNW8cYE-ASdizhZGhSK9pLpL09oQi3InA0}Z zVMEtpN1d}kr|v37?swMjKjI~*e02|qQ+R&GseI5rfV@n)rhv*jsyo9-U06piKu8C$ zCLAe0Yt>m;16i)Htq~-foL7E+&QAI6a6Un@~ zx&{1bs|LsDUAxm1e=nluu<=qAQctk8xU&G2%4tfL6`R#a$ks*uS=s{Eyvh+*woUfH zB=EOmX1!@%u%6XYabUW3ywYA}!=snH-5A=X61MPDON*)8$@!N;4cR*AgeP71Z_7nW z{afT&*|(*|KaQ+TGG2=HMq+c)w91dST}KMXE)c^-V&AxhI3yObr4uCwJE$A(*gFfi zIc<{Y`w+fX-tA4LU0(KcbnTr0M59g!F?52^ycMp3_DAb7!s`5Cf|5_V>`5goRYjED z_iEwFH>`2hxrV~y8hba?o~DHK__Ku&XL4~pkTEoJK%P02z5?KjQmT(KYXJPFJfTq= zNVj2j0g55CV81aR9v<_hZ4!J)Xu?e$Q5i<(96aJhcr}7MxMz;5T zY*3{`4kG6KY*dM!A{d*7INJ@t)Ta6w70Xa#TM_}~OZkY@<#LSz;;c<*he0_1M_n-tGcu60#CW~Ff9Df$Hq1pl+ z1{)S_BgFu=&tX%E{fdB=YEc}TJ5(WI$58dQ8OYZMa|HDkE0QEZ3_b33<8&&UJUA6O zB6WbNGgvu8}<5OJd<}vjs9|c?84h=|c!N z3)8$fBp$o$1SHHeZ0>@E_C!{=8&I{{3p63ims+0phC#}0>1rb8sw*w)LX_NpV{mEC zsTv$1A?Doy!LhsoBQ4-LYjpX1gQmURR=WAf!0_uU*)XmJLkpL)&NmsKoofn6nPUH7 zo%DNUfhFzAr>7Xr-NP1eOJDJ5th_-N*$)7N>mW74)jl;mhXpvsVsB(CW3>B<7$;$A zQavELo_?gy`GG|=y$%Jg3NYxUqWN|*=#gmMUJ`PZ9PQ0Y9jY8<3~CiBm!WI;ws_Ow z+bB2 zi3}djEYL_f1=y0H;I~_0yb71|ixaOLv%>sW4yIyd_!Q6TvZtlEv0QtzeV`6|3#53R zi4tN}hReAfa-B>1x5yeL(Uk)_HH6Li&<(q3R|I5tY2FqS5c!g!wgr?$z%C5&sM{F$ z+;1xM1SR*+_S%EyCbYfSouNFlrlUfg3j?TxrB1A(s1azFNR3Eu>DW!38Qd>Qk4O!t zDF|nyqvcdiARP(f3B$DsWWw}PKN7yAZSTHDOpZ?&XuN|Fxs41q0-atnY}&4dU44dD9JcnutOem^*X%4hz5_Sx*K>~P|48OJfBw_$>GS5 z33kf&^OnYEP*6D^b|_oRz7%$?AmDi+o^|!}u(+hqP5?8DzWZ8WKJ)yL#%;BibJW_g zC|OrGM5GG}XdLbYVKVX1PD`4e>J{vnG6lL2XSQ5C%J0AZ9Eyngr8ZlC>aSTtTrLgI z*+i5LJO5uRCrn|IR7CH~<}|aWVSol~ynl8b=f|sc-PRo$&^vdvA|pQw!xF zoj5Cj6)hN)WFfI@qpgaO(Xy`#2q!#PMrm4YAv%T}sBx9$x1|+Mf}RNJ z;Xh2k*ep1|HYRpQY!F#a)9GNhueSusRGxXg1hKJ3BL{Ab7)~DmJ(0lLyV$5{hDj*O zu1v~Ukxi^B0d{H*Ueclc(tv~wuscpb-&XoW2V%X27H`-TWRI+th#J>cI!XcSjPPox zof`-olug5J5K{s^rW&L;eX}9eJ&V*c@*sJ#afB(0YIrWl#SR%eL7(#+E~e*V^9*A% zI&Hw+bAvf%38&cZUQ?U9v*z);r6Ild31n(o=+oXxWf6C4Z&>u z%lK#ed#)lTb=*4u-nh=70&v2Y;c^Z;r=#na5@rBzAuX5$G8HFQ3TnDs=8&`J zJ{tzQlsmYFhdnvRzb?c2uXNEB^8Vn{w@I8r4dV*;Y0QT3YOu3XKhbhp{dLdqe(X1r!A<>!o0_Xi|H%)z0gilVcY{+kMM5xlZK}JMy$~ zF~jh1rrh2(-I0D}|f4jKiJvBOBW>H6a1}-{C*YS}g;JVLV*;?|g9Z(F1xe<*fG$gIY zp_-6=91}#~D-K~b&`k#+Pl22L%Y%WCW{@*5WLB86By_3tV5WpR;$2Yk@HH@*Tn5Jy zwjv!kT*^yPtz{o%ljj|)f2GMl@(U>dU^Zh9Z4IX1DJ$xqar~sf^ zhmWoITsBa>!4uglF}@He;MW)22n4PNP11TT!;$b^sgGc5qM<|)Z8Rjzud5pOrU9;Ne4s93gZEv^X|mw#NP7N+Q1(o zvhq!Rizbf(lsLy&g7(#>x`H-V_BjMi6i%BtL|Wh1B(Q~cbRqIQPC&)xzY!1xJ!52s z)%w~T@ZM<_lnl?+5AV_Jw&lD?WIDHU{=I&6E5bB0sm zZ?6}7s?(YHkw!~IZW{fYjZU*?5<^D(?U+meNTIPTUJixi0D6^cX7b#&S+v*Z{!BhJ zMLhJpG|r*yc~Q|hbF^nxmr))Z`HL4Sar*7nI45F9b-HCc^0e?|d3_F<%gqJj1d31o zFn(Hkm_l`Jha3YO;hfR)d_n?EhQcYA;qn)E6cJ8$nkIO1)?W8I%9zu*qzwt`OaQO( z)|9+jijbW8H}2!9@K}20n4`Ijx^fr0H?Nk6z`QUMBH(5&XNipuRQIE#_GnLf`02#3 zZ1|Tvy5!4IQ14lpRo$_xU^0{V3kihAv3X?(pQ9yhUxA(4JoCGrK%w~!buCtdVtc*rk62RNQ913FQGCt;EK#PPy_!-) zheh$w{;cjDIhic%sF-0Wned%bJ7N^B-68=PQbt2W6BS{ptg)DVeR|-NxxGOGrd${~ zC8}uz%BRrLAyMjK)zR`gKa?x%>9SpSlEWTm)aayqu1>N(Ni6|2qhazTNvYwlOEY|>CA$u=CAwA_W zyg`KH1c~C5yRS|cY;#jyuyCBhvI+d^y-4Rocaa3?=ch;J8NzOK+wVTtIF3inS zx0(jzc(dh*Hp3%EO#yrSB|eUbhg%e16lzB$0?TzTRSJRS7XY|wn~Kp%lOlTyJB(Ar zqqN~aZi=#=K+nrXZgmMqPP!^B;7? z@T_eW5!=JTkXBT7y)uX`>1<3b7>W18aG&5-CM;>>(MS}QpNwd0aT=6d)x&;%t%AX+ zo*Mz$5~LxZWp#LyY6fL|V!qZU%Gwz)QIaf+0~qC&)vqA`HfDa((}LeX+Z;7Z1N3#9 zZ9@Fm2QDeYh$R&$uu;QTizyz~DzxM*`Q6)py$!)W(_L*)vcuZDIZp>lubdD2>4I64 zbc^C(M5;e5)w*fwVfqz6$hBRV$D*VH)3Kd;6-ViQ&JusKbMa(wKQd=d#k3|}CdZVz zQRwzzp>^&qMn#&zoXJ_;{2_@vZiN>{2gjBc{ZedJ4y{%js-?$+K8M(NDWTI4YdhPI z^=i4TyOCbw3iL9dnUv0FKdmgnj{W5ZpmV4doe_%(GlIx_=5RQX1^ zQHywoB}fBl2kux=GlXck$JhgzK*%*y)Ai>f{?RrZ0dCe|Qon${fr{yRxY;mO7>77z z`w=9{ND&-XCmL}CoiwZdFgWM6n%^RiCZQ{x0Ro=!V!stW$UU5&M~%fm!}&EFTx3}sLv7M? zccksDvrw&2iF{ULCk%tj9kZHk+Po2byr;8N)Bb{t^_@#r@>$QHsf^NR@i4W8P1X$4 zShK#Aln(bN=ki%-I<{#m$?nh{u+BW(K1T;!c&f0C-hnsoQo#4kw?5UV-nS}JJB%I= zdIhWbzx2k2g>BK4zc^>Bv1N~`*uF1+W-7_a2>}zFCXbi&Ku8LLz3>;i12n+%i^Amq z=m7LsKte_$CIKiYXcw|NiR+d0Xf1i9X^;SNe`6PR!GmOz_qY@^Kq`a4C)TbNF{b#9 z_F&5TI*_=n#}8(B7>73kC88+p1LeeWwQn-PUkI6);t`U?7<}r@61l^MnVAz4A?Vke z;HB7wdg{<_kY439J1%cr=pxKQQR$AxyI1Osff*)_BUw0#eQ$s$+E!v*(*d*~Xa6Rf zSfuw&OgbI38Hr}D<&q5CffdLz5t=wrXncB75%itpUq)jE#9(w9w>;)~^p0qrT0fgB z@KbKYW}@2!^C~Pd?cwmBb`3H7i756EE>)(-8UntYgwtJ_eT^?0zA{~W$$w&EcV)YN24O2H3-pXxG6ZVA~ z{b37Jsmhv>%8uuDIZHp?Q^cKm+W3bc5@<=A-e&?1AdMgB^8wXo^rAHjlj`maEvCsc z?-W2nElC8?qU^&i7l~1F*A6nb9)k-w6D-iF07&dxRVB@iN}EYrn}fXd3Avcmojqog zjfNvx*o>cvBtKO8gB&tnLV&O#%Rsd9O^$MbVJp;S2-qs%@Ot?N=N|VC%!Bal{5Osh z$V;^9Oq=3Bav;cRjQI=OT-wfmgA@!anH_~9Nsm+{w;bQ(+!b=9ZPlI$OQgFnBh*l$ zYFuwM!bDcXJfIQ7Q0)#boZSKIK|n5jmrak;rj9dgVxV~C!rpws&LZAc83U)nC?EHE zJq-boY(~W=gndBUN&surWP-MN!*odbH5Vt23p_TVmrmafsA-oQ(xBR-!O3fnnk&K% zmC_J_Ve79lpiYHu;LBFl@8;Ol4w&0+bF6xgcG-M(d~Dbj<^6FYz=!fTqeA>kp*{gJ zv@?pxNI|Asa?uT_w7lF9$lk%=Q2P)D1b0-l5`LBJEGbu-Wo%SFo%09lTCUcvhx;XR zsRqE|Imb)dI;jt`ExCi7T}}g^7{pL3tgzFbfWvJtA}pmr!;nC&ZIQo1&!z@9oajIU zMTYzKWqd%shx*4i5Yt>gv`jQ6|1S;KdfNN z(e$!D$J54qPQh~mEJ5nr293a&E@P}b38y=AYZchk?(NbR4S%aIiLKpV-_#QWMo5H8 zaxg+wb_bqg#7RD6T>#npf@o;|Z1dJ-MAsyIm`RRZbF^JHO0J|1x1PCNZu}NH15IUj zfi?yaLp2C$zN)A+QDMzR!wN)s+1hxgVK1vbxiH~>-?Q3e68F0Jte*Qw9{T8>5X!j1 zazN{~*QE!;=MIBINrsJGCcBZKKA72cvywrnn;5$rIL9NWz85&~xcEGgVgYtrLk7{; zJ$PpJciQWyu6fR@FVqKrz9$4~;Xyd~y>bBVh?MpGE$$@P_~ZP{EAs*;Y?dTJkSA&F zA1{nR058#@<2$j{w1t)ZfVWs4B5&)6MiP&xgJ+rgZk@2H(h4c0V^F*4)(+I@3`~ls z(KQeY09g=iRL=Qxfac1LG=4iFP1$vQlLqsX!DTk;O#z~pJHFg_{d@dFK?W07rxxF1 z$|Dlx{=)j~We$9TkRq{6>wav4A`#~L1ZP6;t0`BU)4(k()ssan1CrAbGCpo4S+{!UdTBy>6Y!0ZDiORPAimj+Qc_(n_DKYB zYw5C)eYP2}m0zP{Lt(Bokf_3DQhV*wGgC`RATnV*aY+&r2w{{Jrk~PcRIkPhYX}1rj=wee+N3P6XG#0zsvDA>zS-!Id<4} zfz+@Ll2s3X=^;_4Z>ml#-w{{au(LD#%r_FiLuHp)7pFCUIMRP&Gbd!01^k+u@NtgE z=%f@*O3HJQ`Snk<+-6OR!qgWlvVrw5M=m?l+n^(fV6H*8=Snrp34~1&(+(CykR|J# zCD-saeHldV$+Zn>D3L@c3S>!;<)~M$hNEugn`=h~`Va(FMEW|e14y#<$V>5MCQ4P@ zdh%2?x#MQ#<>}$4PfmBdl1(Bz$R+p-c#b`J-Ry19$gi>pGR}yDC9)qXFiaqYMsFrKLDunIWO!{|fwHbNb7#4Sl$M72xh`7rbtlMDFk`Cb2Rj01;+1Q3`kVm1# zf!G4RH|x^uNPYvA1);9-6dl_UW$zBWP!D`C(pnCJqlEDHg!OYi#+1VIE; zCS*1RbFAT)lqjG_I91lC+!l}e4KPVa!bOvBX$iG(Dv1tHX!QkXby8&^krMiCKE<38 zZ90oCmDHIrbzR*n7q7THNk5_gP@*gcdt32HPWd{k&W&A{%N}>Cb-p!3x4nuxA5t+q zfa)<%LM>@wmEE@>hqieu%n&*gILD|V{EMp)ntnk(t2{jPW~L7QGQj4IO#ErcY=AOt zC~9w;#xC7p9LSMK$;eHHZUYJPrt+Hc8--U_TLEbCzZnOrcIFkKU$FBen=~thx7iW9emvK|ifNPtOs(y3$J^mCz1zj{uoDAFjm}u9>qOB<*VR zPl_b!%l}R1(Qu+W)4!bbv?@WS4I;;bzGt%Jx>Cs|KSOaV;N4NyKTQBn+qhUc$nGM} zt3?#o4-2;20i#MqKrR8xfD7?s?$0kO)gm%%Mnjybv^+k^z2R}^@1t0H%zSp*Hu9LJ znwvOo-5PE-c~ExgFmyRD=976?LPSo%tmd6oU)sH%c6L*_O2up0Qt3t zLow@S8e1O6l@r!SCmsnt{hqbIrXDU!E5S^Uh!e=+fmPFk2wUe6!nLlrFUj+`G? zqAdQ>LpfffZwrkPIG+pz!F0~xRDBkG==H-^Gast=rMXuLDOc2jQ+Xuqa4&a{f5Z1z zARH&3aU_W;zbp#-ZMCabJ7j3>pnhOe+9ahrE7HlYr=|m;THM#mEsdgX6BAaEWN&1A z%nA~JbE?J8IPFZ*r}YTb{SbXR7Tccl9X7LTzcUVMO$xnG&P_Tj}sVX&!;gZ zKvr|*JkvebCYw4%)UTk?+rsjTV*q*VhPf3wjuewi#D0 z8fMVvlBP4AZAP`m%hZfu`+1mj*;z(*QZ{ODy72hdP;-A^d*M!JzBELER%*4Hv2x1XV+ELbk9Dfl_!eZCaqf zsWHaDAJUp_bXsDt)JJ49?#Z{F)l8tD)F~V?P(-i|*$(m)Wyzbgi&Uw&m zxV`Uew?$r=mFXyuy0)}4Ro0nAq15G?C-TE=dnsTaGxdBBs-)F4bkFM)!)W+mAvGb{ z(h*+IO`tPPG&=~fou|#X=9h3^)0lyEeGHz3&OGybvE=`#zW^E$+U?#)$(%9I)o(Xg zv%ehuRN(dpibhqnu)MRoaFArW^Nu+wyH?;#pQ9$UQ~PQzq;SPK3cDRx{r+6(iWDR< zj!Qg#C@c<{l6K{#TmHEz^ihYv4VSRF-K$n(B!ioS3Ith_XUlnVD$??^CSX97Y56sr zFXPckjuuVs)Z$yJ@jqeN@p}uzmC0AEj)U^n!pX*a+&Oi6OD%@BohyKbWMjB&FO3$d zcfmJkOX`n0UX@;p#w2$$Y*PIEg7;9pEIttL^kPHIV0!h@>Htu7@o<*KxCBuIyKl_V zww4KK7WL9!nDNYV;l|yLwM?!(@Kl4mAakcm)eHm)OCWErzE4g|=Xd#1Z^D84FGr^M z$(bZ+vh$X8zbHIFH?D=l6Cz!jo37gCPV9+g*MkNr0uiBBbIYeik#4qwl=f(1HYq-nCSJNjvB` zn{ynvYO6dalJmsP47eLS!N)nK@Qc?5H)j_L7W`y<^?n7$)7e!#xu`@UCd?q=c>@-= zjh({Rl%G(Y{d7t|f@DeFSZx;3xwrZREY<*t=<4k@H1e^hk@C&xboHsF70KT2ghUf> zVlJ2a{1;ttexm=5B`}AP1PpGXZlTw>Ky4EZUX>RDPU#eQ-$v_>O*JRp zJxGYhC>2mwVa@LXNz&V9fGjhe3Kg9KKOTgmvXz?fC{E}**xeZ zSwUg*K}KnJWgSpDG*O&Bl(@)`dAdDBTAYOJndGQLNLD5f2-jqV~W&We8s0*LgS^Y5r z>9!e9Zt(Ok&FPCa&MpEvYZ9gMo8qwk*EK30=vTRW-15czi%^}fGHKMKshGmD;IIs- z?1~n*k%rM*PACdfkHatnLuJW3t$P(x=zxfm?$aZ$)jG;Xh&E3)~X|)@zCZl;p=VnZ5D@+;jQO}$D zC=VWBTSJ0MIR2&n*HcS+tR^&)|sYNrk?hd1q2wL-zvq5{Cpit98+boe`O3+uv(^X9mL z`Lwka4F{MmQ3f{YGRY4pB*Zu=uTOS^A}1hsthgNVeDLhL71sLKl1T`?V#o*2q7IC^ zlWEvy__wy&Sf&sqIkx3HX?HNA0)2~%l>!dudDbSK4$9$_`=uhN4)%&t=@aY*iYQFP zpI-AapixfS6c?lDL6696)w;N2^2WOm+9G$K6RQkv1Z*ZbHM72((-thuL&Rqar~bpgJ%F)Bao%)z??PMSt%{ zkw>@M#HT6MsCFOuCNw?MRgGvSSvH5!O!ZJs4#!6Vxqn6-RR%Z<>!LjqB?1_2`l0Gc}KxZpw4a$^AM*%x&M;bK&A^3uK24xHwN zn05AieP8MuGKjC90s>pjeiQ>`Ig8!Ez>kC|?JamL>nS29d&ZmLG6dcLN%85lf5x|Ysc2`n^PAA(g1N~x*3 zgEt^N*<}i9#WUuePODsYoChzhTd2I=T<3tIee0_?xxeBw)=k&HMNc643v zcOsTl7)`JDveNd(%f?5^nghCYtTc_D#O0y8Xkp`XD)$_J!X_Iyo?J+4T3x6v{Uey* zMkbX&_Qc#BbduP~mLeHFj+#ZH6XS5CLs(MRmb;QbRZyJCrhmAsNw3s^He654D-AFX ztmFZm{~R>*6j7h z9r~$cqXQm$l+V^2XRz+z~hxqa4+{c`>Y%-Ggm zWmj6&~1$Ha>5!cW#&e7yPOC!>iGdj zpz5OpK2;a{T1aUgrv@&L)Xx*^f$vuqb8Gu{$Tls)v*m9w$?LDIIfD2o@s=_JzOl#rwB(t5pqiYaVh zNT}<1h2kryR^YHao;|NY>2|ttXB&QOsU({+zk6?|MW4iJI|Ev?v(4z5k>(E^q>Ruk zfdL8ZhT+04Y`J*eGs@zQ2?jN(2w05Ut#$lCp-36Y6hRs1-E+zr(t(z*+hKgeM>@BI zH9o%4*o!NR(mW-&2Zm)xB| z^yL*r#!tP@0)8h=tjss)z>nh7zyJgruNClKy^X;bGE*$e8IbKwxt~K2)k(c}Qdrn@ zuC0x>(d_L4DmxW=$>mDX*0?PuFjKf4KhBHkBi<_1TdN=I(;Tc`2ou%!VYf zS)OhRV zjZfT-7+67*n|G$0MY||1=pnL*%+04xjJuwCr-3{v%yu!iZ(&CaYm2c{;T{axtq6?4E9$7)z+`ZkW0+??;0rkRCA z&W?5c^++v@qp8H!s9w~O(d+O9@V9;4brNM7bj)boHa1r#-B$@{71kB%EV`XnD$LDcL-W*$pO&;1t2$j)VtrAoBc= zX2*VCe>=V}3{pRZ3D7$yYt78!+PuOzNGxy5Y^5lbvT6h+)7Nk}9b=(c8XKtC6Vd0& zqgv52Ck$#~^aON5jcpyav$;Z+JVY7KD11$lQaw3e#VvGnPjj{d0;{Cm&>Ns$CmE31 zhiZx0yuh%RPOR%D^w4&a7`ZhD>HLa=7oDug3V>|g$?6^EX39$KDgYbq zpsJ8?HDu2Bi{{*-N>0tct;SV}D$jh{ICl(uQhQSuW|tXw3aR$7%OI_3X0_{-F5u#c ze$~RcF%CG$`7(i^;fm638moAih@1czQv)wvtt{;Q$Am8@xfa2p%JO%bkChF>**49Y zzyhn9uL;QTvkA14974N~Mc82E_!slu)U5fS&?P*ys}BhlWV2Id~<-AHaI2za^mTpla>-Xzz9Zk^Rby zmun`-A#qZzAde?$QFv`U)srXu^kKczUzMz6hoc?yArG|e4~u=AZ5p#vK787-kt@uY zl34E$MIARWFiy;ZAT6k&lxf{>kU{;>1~^VLhpHB zL2&cM4gW>{U`8_Pwn+VPRjV^>V25CiGL{vW$x|}IlOw}(e{q)`{J8{i5ESN zk4Q_dA?J%Q!3hA&y}GLPtI@kQC)6W~D2xj^xD^yScA4q^`>Dp6apPkzsRLAz!_cZB zaGoqr}J>JrL*x+cawGixdi4JZVI*L|m0a+$g7OS=d9D6I1bmM|@S)2mnBeidzeDK5L}B zp9K?*-(~LU+OP(8?)g2x2h#Ypot_$UdJ_nBnJvaYl{w&KJzJz$$x4YuYXyOmmDxIo zh)ij~qa(-_Rf5uj5pHHL%5)>JFFSj*nJ+(}GKoeV*d$2y!wKI|rpqAxJSH?kp@UwGbd8ag!n;%gx1p5Z!5)9WTUlugqHe2QzL_X} zsi>+lE+x=uqn-d~4@cAD=uzrgW);Onx6&33xj}U1>xtQ0qMDFB>90L;AFi@ZbTOn@BxsmeV>C02J5Yi5N7l?U(DkcpvHHmrO$En3`>vuQ+~EH>U$tp z2Y~_;84r%<12=;AF8ldVQAVm^K03$Y-{nY8nB|Br+ zwhE!`tE!VX*WQt&+%B^P%LcMNH5xd@#w}$U+`}df$Po=u)X#zFek@0?(wL$bcp7!V zC)J_bU8TJ~{Gw%_3OP>T0Vi*81fTjm550&CwcFlRJbJYVc{;vuZJp&b?q}4_%qRz~ z_a(%$Q6oWEm6q;EJqqI~G|RCZ&%Fv@3C0n51@*)iw|1wRV5iQz@Sm)Au~fXwbK`1b5=qetLvc#%rMp+#SYVVV>fAx^$d014Mq^pVEpX)_M$sEp?gEH}Gel60HS|2S#N-3UB>JRG2Iim7}A4r|L?~|Yl zvvD9n%gCZOcaZ$0Mdd6&ant#-#WPhHHL}lwvPwx2!)1yG0x?M?I%adnJQ0}~$Bl(b z5*c^*r~qQ~CIf!>V;TKbAi{}0_pZ|}g+oLp{f z^bN)sR!U{Qp)%)RGZ|5uZrIf`)&j8g7frbwpm!K&ZpyIg5RRbT-T~!$aM1QWs#Ac_ z*@MoeZ-hA1oV@1)au?N7jRdW%)T7LXxN^)|5dnP)G-oId>G_>1tQzAPE1?z}iZ-fx zD0zIBKBHFGW9>8`-ga5c2>hJga3tWfS!t*`df-|Jkj(W!2!vu-4aLT_!sBhYHj#fW zYWJM4M@J9E0g7t_{dyxywo?wb3(^g=Z%as2v(OY^q(FHa53^KyYzr9mne&X$Uf15g zU~Cu5o1IL305d`t1O%^guCOb|rAp_W6YNWmA%0eAP6Ly@+D~lRFFkHYr`s9HQ&|9usg;t zdC05XwdO}29+iw+_X0@NQSPn+j~9);;W6u81}M-hZg z@ObhV7oZS4Ti>fgwVehhOIx+xh{{}W>^2UUN6XAE4syOY^Y)vSvY&=) zf;Jrn_mw75$2`2(LX`Z-qYWXUFPP{e=R*!MkQR!CFtYvZ=RPB@$7UMc{eh}|_^yB? z?vLhIObCmxWjM5wb9}9aERcl>0Oz^6gzeKM_*=%|`A_fs+L-z&n7}b>$ zbpF{B*f4_g%u!Xh*y564+Y|a;ti<_Wf+ZOaQ4toWipV)VzMHwaYcw-u_9ezGgIj>Q zmj`Zi28;l!%Z1JG?rse^H6z}SyIdAd?CCR%kl^{|+N}D4ptv}p`EBo%I{@2>|B#lc zD;9S=oAm%B)clpN6yHWzG5GTARiMP6=QF93TY2$9GUSI{nc&hdRSL#*jzIVy7nOSZMwTxoeg z_;u0hsPG5$mZ;ju5NBn5(V4nC5KKbA3vl?uQA53MnD~$>1MX0ICW3X{>&YAC1_o+> zgWrJqsBU&y{^G50OGjjHWSuu$^_Gv>M1UFMKog39Xg4_rd?9C z1vw_qM4_X?lHQxCCPl|ozsZ6ZBgM$0t$5Lrq$p;4nuUL$VW?z4Rl`Knr@21&YcINKt_r2r?P*{99eqWNI?0cx*nW207D9BP^|C`^^%KI2NWl@H-mt|0b z<*2rarg(zS(xF{Wv+ImeK`A$GAn{tWan)LSE8a{#c%PctFHdlAKWd)j8_0w*kL*e$ z_$6|qB$##1sHQE-VV`dQA-e5IQmeGp#c)uwG3h8e_*d-m+_{?LSY8_3W}MQ0OC+I* z;e=}e^9VI+J{C| zp>}{0MF=|3OO|j}3>xJWtLhie4rx?kX>F_s2GE$$oc5lzCT> zk_*}1^s(liFiK7bIaT(xoCm$nfE;`LCsq*@U1eBYJrjkpz%K3%#bNQ{Qrz8&ySo;L z;_mM5uEmQ>ad(H}6pCxh``zzPp4?>SoHH}YUC%3gvJT0XG zRyo4c&%Q?AnYF;&VpvPUvQKRKnn4;%+a6~@ku10gJKjlBRcj8q%51k>RG&%H0@((| z0<}G>?51LPaG^M4Dmq6>gjYMc6GVT^K(X{%>7eY2NHY6}sp3KJQ5HQKXz!d!m&^IS zbC~D-xH`5qe<3Sr^m>W@8QB-2@Upi~>KSW^<)E2f%KD^R;V~k(-~d8MLl(4MplQ_V zs53%7ha0WHDb{}dF2G1Zu|gX^5hv1f?maIAziC=>@y7{~q6gnGE|YrQ^+1=3hp)j& z|K7h=UeP-;(F`|9nG~Ici4$s>=Vu)@CCvZIz0wT@%uT|5#UinbUi+hW$AbxAZ~`|h zu@{yre%3YTdrjP*-Eu2Z(*ECq6$sHzh<)*E%f6oChxjiXmflC zR;nLv7SzZ&xVh~b7q*>6Z_1b(oj4(D^KI_F$GVgX7COCr1R>e;s&2o^?kz;JRgCic zn&z{sKMt?@;g!YpS0hwXNxYPs7hjsRvMWix~x^@3fpZQ8Y1Yr;+=7M)S$|QZn{(c3g)2Gjg=h4fs zYUJ|~+u;n)nxC@3&#|>jmTQVGiG&4Xw6JB4riPA;!ZbQ^C7sjpdWME}NNL(^&lM%I zpCLJIzFE0@P~x(>bRel*$=Zfrx;jtOYZDUw*cm?<%mCNO6}aOg$cP-QetH~xcN-xl zeiq4%dF(9kkp9C$)VU_wNL5dlsL&hv$8Y?OB6It>zMKK2*hW%f(}#)2JR*4T>O!Pc?b!L`z=Iy0fBQhv*T83 zw(eqv)=rpTtV6`VrvKH;%lKDL@q2|5-uka#k7$JUexha|^U6aun_=4>wbi8m+0;=v zY21v#iozq8Hg7JX>%QfP|0MD0_+PQQJI31FWXK%eB z7gR&UyCi#Jg+SuqmZ?Fg!>3p)kT6LLPwu}>KC3%-`Ygy7as41(YtkfI1*(9QucYNz zLalb^8H+VG{H5+9JuL6%7uFFr;X&L()OuuuqoVCbY>V}>#wS@-OHDJ04YNIRi&n|r zaIUO`$0?ib%^3Wx*$`(Nb~|IctQc($rrEdgqS;Hjm+$FSd?Alm!m{b6k?^{lxwEp`b$$t+AAHFngDlTs>A z7#v5=!#rWzO`X?j%2cKIjMKQMJU8rvc4U=5A-`oy?dX`@ z)r+JIA#;|Ak0eSzF?4vutj+5(F!=915g&&vBDq^p*>ofL0B2hd93GM{{kta04mmHE z9(_W%ZPNYDUvwc<(;3o*0;hFgG1dB+g(!W(1o37kS3v#y4?W!eANJKXY+0SPO=iB$ z9|vJqNn6_GIc|J{M=Gi&cviL`!lSoE;>Md_Gc@M+S}J*t;t2mK-=1AL`>XJ`muOP* zV^I867*(DwM;ft43GXa7sTz^V#fworwq8&MMkUUwN$x_xMc#iuh9nGQ%D3SaU8{;cvb;4h9^)Gg1a{7xy) zK*7srD{t7cNgSIos}k(IvF~l?FH2rb47&lKL~63onq%eMzNLJy;?ktA;KcGGV^qm1 zJ;ruc$!NniWGspT16AFp(ZR=(>hnrc&a;Qd-ulJ4dCtCCY}jZT)XNVMA)e(t^WO5% z05{Ij{8g=4X|TtO$>eZdP4KZ0f=K#t=r*v1ZE^&$tGsa8*1NzY(py={)do0q^wa zTa3HRig&WTmO(?NWNv}HYgexZ-$&s4DC4u-EMB!bqO{JJsip%X4UHc11TMqYmFs>b~DeuA&fjV;%6Hn?C;g$ht&)dqM z42z-v-67SF5A8ue5Bah`qmMUYA+7kTd=5q6Kh3{~2dF-$RLI7ho)RxalB;SNq>66P z7FZY9=vWco%iLvpBIEnErck8A&}@E%*St47KgxYD$cBLm6C?_}99aaTI*lP2+=Z3Fkqn#A zGi>$6VdbN>p@wf0DwLt3jZn2(&=nPS9QF*SIjZy71+0Q#1 zVpl#JEJK&JJmS9C?e3XO>Yme5y#t&L>86q2y|=kg92lbRnpc|Na`f3GHSDYY2O)Jw+azl>|8JXxHQ8uQizh2 zhNL57i{Tq!7fNq_+>jX3;%$U8d44LVmH1GQZI>)grq^|sIO=L^Tuf{;Tb)5EucAvi zn*+P2jY**7S4Zuc?jX8}IUmIR&}=k_lE-c*MeU#FLBhr2D_D@M%;!?&#rB`MT0IT; z{ibL1Waw>lQ&+}OMLCsS1mpt4cI{W6xTwbi&o0EJmYakB(<(=A<|*F3Z~gFD(>?RE z#Cf~h9 z$DaR8SB;@$b0u*z#c~V6m$#B|I_(CpL{=U}0V1s)e9(vh4HNHMPOvj5hhnokAZP~t zNN*mF&q6a;y2>Tw872FGQxc#grNK1!ZmLn}w~HXqEzrSS>2t5=m~w!+nm@ zJ{8A&kb0?bB;|c{Lg}O0=+1&1f11q3bBM_ynt3!Y@od{qZ-<|VpsCfCS>)P(5n<1} zdz^iYuFaw3o1~oiN-8qBP%#)2&+&u@mbGS-`+n~cs$)bx_~FP- z)8*yE81oS2Gv7{gxPPYyP{$^YA`j0-GtZoI9#yMNLS^!vcmyZ^h}_bv&bL~tM;)4= z@dBp3u#q-W%MwfKQMGEdk5myfzYvU_i(GeqX0GfjSmQ>DZE3u1+X##e3+;V$F4)4D z#aXiDbRhF#3FMR=BEd8mzz6ylldw;`{8I4^N zb!?-Q3D>ahWE*x$MoS1eTDT0SZGEz_$}Zt_d|;%$9n4TDqdgKG0l$*!{hl9ijy61Z`v>EMfaB#fCT9Xt zllStE+|>=GZLEX0Sdl-Ft#ciC^Y`Ixm#XRF2EY*bqhX#j!5pfnAr22ib zTd7drfmnK(lW0Y*?~EKX-P-(^bz^8G^HW-9J?@9jeEGqhW&gx4-Unxn?jrxkUe<3k zd$;YJCvMn+<(^Ev5?Yz-^gOU#+b%8_r}9<7r)HUXGD_)W!u)=usDGOM>UW~R&sxxX zGQ^K_YaY}E)#OxHH;xV7k$c^dND&YzCajrxi^_{W66oz>L2?Yss6Lyy+m3`WWo+$* z-hG^EcbrjqdLq&nryxE)yF|&1!WmF;&+UR6Aq z)6u5Y5E*9vr2`f%6DPOfBV_t4;;$hJmGaB2v~>Wtl8aoaR05T;O|T77kM6JCCz{`n z@QtlO56jEO9^Wv=_=%~0s}^c~w#u%V0X1+q+qK5ruHS;m*KhB(Kpi678~1mbis5!& z@JZ}7+pC$wixMKs1loByezA>kGQL0RZ>*y!_sn6e?JOcwk09HgFg89vlxNvEVG&s{ z`Q@i=jtoiIEY)Nt%3SYYN{%U*)p;CmGtoE0#r$nW&!I17%7s)d<3;uS9Y#E_MG0Ix zrIX|Hf=BnMVMJ`hRuRK)bVM~EI@)0OC&17cc(Bk9u5W!z9%tf5SAJB_nl&W%jI;W4 z+v1X!@GTp)GXfGM1123mu_kbCd@aWdD+=jE>NVI~!=-wg&Y)K!iwazu;=~*2?#63R zHi10-5uE?|w88s3?p-dpmvWhYtR%6ltU0ONIl~;SKK0|g^{z3FXSE-V)XJuJftUi- za5b|4Lx{z5yxn120Ch_m0fwo$qza|rKpTcKET)ww5l0I_^d&ka!|yR`hafHe$1a?; zYG;jdSQ`BZ!RJ%MUeLgk7t$kq$>)prxdj&$oDmUibaD1Gn3#p1uVTNB#U|KkHtOVj ze`_;zkEW2AM=RV{{*6<*uyt>wh^-BM3iqN_R~bN4Lv4!du7$0!#OP?!Tc><=?`P{Y zX!}>ZzT1mrAjzOuRZX{vJCSDsdo$CLJUzuzUbo@tDZFDHUG$yzt8KU0-Y;pN)L(1~ z^|Y<`N{CDYF;6sI&0)WOJk5b9uGvQ4A zxv9JOb4+KEMjmbl`7k#A$Km_VjHVLSNuAe6#ZLQ^KgTX5?9uFst(&HZ(lcXoJ(<&( z$t6ikJd{}?228}vd%la&V+tSUKzDP0 z(KK}xEyPx}HCL-xYfcw^K9;_&UZuX7vq`m}(AhoD)whGQMRe;H;8KhkkC4RbhT?LyEKgc%GiClIx}O%Tf+q(OZhk!Eo%)OPM#X>$-e+rUA(K?Sn6x@VTA2 zpTp)S^iHLd#2GKFswfL3*Af%h35#(!O$K5-i5ue~mV9L0K0Xws7-UX}J(b?{z$I=?!)gEJx{qk2b+P<8-CfG5c^%8X~VTmM_v=I zsM4Sb@nM%0$B12i4Pv}BHh{j5lN5#9abJ?V5yp^aY34fHt#rd%*~BR%3B{u@+Zo(H zr<5{k?SKDOJr{r!)ThUWHV(OhIG?yA^sAr;xc0b*Q7TvZEVfeRSTo0`%*dn%D3ikk zZunf`7xC4M=r*n%tS}`jlo}r zH*LQ-9JQ*VNv4oC)Tf-P_TYEaKW-1TdZdrp2qWD;X}QBvq@M?@J`oprAch(91po0t zvEWJPk;h(#&uTUPV24vAyvakG{vWxico)%m_PE}?uGvQ2uXRp54Si1Ax(k*3x0Hw} z>=8vBDb;C&tTgt;Mf?08GBZo!pB{L_Pr=f*H&aN-Sab?sH=@Sdku~b*@P&?z(QH_u zH~b3HTUB5d4kH!5<#^u2gn3qpwR3*(#J*3driZC-;9$m4{%YUpv2xN_#E+5~&?S7f zv!VWNa5|cVdeK5ou5i?Yeo?i}eUek*qMSb7@TV_vX&A&d%wmN1z62}{*bKZCjmQZ1 zYV-eUT1aU*6+IB$nNvURG0}dfzIn}>=CWj~?7cM||BAO5pQhiCHpMhZMrpyXP|$R_ z59)9~0S=W=&M0{z5bd#+M}|s~cWZ40PpgO$|#g}p7G0XOjtaxrn%MzX#MsVXC^*-uH|;%c17y^s7`hrz$m?}*#5zIe3I@W&2( zD#j#&6Lx5orgtY3)5wS=9u$V#!)O3sXQ&`kv5`E*L$aANt?F5J_Lu{C=p~ONGT0h}j6HNRqrFMvXpWA3RWcQGCj5|SVn0)B zq_W6XIp-=gAJo6ZLWVgTx8?((UIU`YA}j2uM(CO`ZJCC`N;bo zfeG%S{%5_?sx#Vg(vKu27$majn~N_8CB<711m(gTj0`S9!!WEiKLNtxIr+ZT%{(OK zO$1mwnMssWE^^kNgY%A{))V6U;-Mc!F@g#xwRNojnPd#iQwT%I)6wU6q(CFAH~jX2 z@scNCFG&Z(f$AyKr3jBx9esYB^gPiCYk8{Qu$T6^W&^Q5xM>U`5tr=CcKJv(s*J>P z4;Fip{IsVEz#rac$zkH7s;27jb{h7(JTe>i4QNs{caNj-quP<7R?yyiq^SW0-EsdL zsq*CAw_)d*@rN{sN>afihDhWQH2-%t@oa7A!ZKV5CG$%XDN9w?X$qBPzdA^*w%sx( zvhb4EOl4j0W7}BeZKq<3{!v+R^~Nh#V!)wkYU%T+mky(aV4zMir4?bymdcJE>sY7P zIYsU_fkwL(ltdIS7gThfka_FPRr(Ll!F(_b6+DOoD3l?Q_Hu*JROE5lT8#O!M1N$< zwPi0Kd33F2bXVRJ49J=4_{DyZDkA2oX%)d8^8un8d1Wa}ip_;ZZT7-fpAj}>ANzV$ z=dj~e$$%5sL?hpF8L}npBoo`Ldy}RBlgjF@Ulr5o5AiMmhgV`tgBZRQevSD9+H?-c z6gfg;c@vi^vDEuGEEI0=VbV6p(-Ui77K=Hina;Tetn1QEfBV^U+Y0pP<>F%LfeHUj zWaZTy6w2os!9}=lQB~+Lo2a{D4Sra^sM*VJp+^5%9BnYUWoWX$0hI`5c& z8?@&t*%E7e=nDyz$X+X4SmC(vJZ+72)u|axX74Ai)rd>ly*~|Kj<3?RK*FS&3vW?d z({6BGOEAQk=t|Y?#t%f~G!BTz*|6UF!qUy*>13aO1Et$mabDT&er9K~x8l zy=cO;wKzmRtyr?$50_Md+&dL8@y*AKY-<0o4qv_mjtLp1{+)wy6)`&g%xo=G(sx$H zgibo9rUTzC=Ps_Fv!^d0CN*&8O;Z@6Dljbio%@eu!tv1o+RJp-PqVAuG0Cj`5a=;` zJ6n+z$2aG}%%nx(DhlBrbq(F;{dMjvlC-wrie=4Yknmf$eZ?=&TX*x;d{@ z!(B=NXESAZg6_T!k_-*ii0zrPdO(wALv?Re?Z+BL`AWFju&QT`9y^By%AtH!XAxH9 z+Rs0qVwq7m)!?&^($@M#;By)3^r{P#2<8bCJ3W zCZ4Ex_}iSaktjl=;gF;jR$*k_%ua``Wn^W=bs?5N2KHq23L)to7{)c(3B2poH#bT= zr$}0oKoO-GrqkUfb(o$k@`J%n57&;fgDQE=Y^LsmG;_Ou*yqx?ZwweEqAO+BO6Y=7 zzr_+}5#`9pM~9G*ClJp!K1zx3HCogb-1A0KJ#fe_iM1MihwzWycT<$WUXp!O`YKP2 z++yZaXMikDH-p*jrY&;ofr2l^PydrOVaM!C@zn)0;eyAQit67Vt+2@!An>ZB(*re8l6*aDo0`Ym*W)SaRFR-|^)=*jtod$_^7MO7e4&RUwAbtWgdkY-lOwK@ zZZTy$YLE8l^_p(5%0l1isYhP0afa74AxVe&UmVH?{4*3-cmL$Y8}indUQrq1#TJ-D zBp)KTY9zf%(uq_p37(zP?*89WSC+YTjU}kmSS?mXt>wwnHmdbZdz-$!-rtu6Z8G`$ z=1Ga#Fi`v-slr&yb9@G7L?Vp??CXuSivlmLQB!s3e{=*jD=+x$Usd*WSEAcoV`C2lMoG8c{g%O`??kdyM<6`(8v5zx2Y%4 ze#^>u%y;y|9Z>V{x7PK;IQ^iVrjV;$MDey(v{yLf5YPzpAtzGJ!0O7 z9Ugd!xQWXQ-bFPKJ|-2L1n01UZN4kxYb>lwwZ;_cz5i<&_U{L$3L;L!9QMXAnMX|R^;zMp>a=>2>ocJ#o;db@K&^I9YP zDNj;F=t6Ig+2#60bRn@Qk-{FcYnC+AzJQY;&kT`0#&Qf70b{cN~q=s$t! zT;84SDQhpEbksp{$|UFiJ8ta3Ez*i~78OPC2}{=DU-*cM0bgzpaDqs^h-uuHe&-Zr zhnpi(1F!z?%m0-a$ohfxZ|kBu%*DTN=R)Mw0!#JUNke1tB=1F*I&)ZplscA+4??|v z>K@F1_-&{oXSvhbWt_K_IC^ivkR~fJA0F%hrKB%#7gus7xSxNXQP$_)1e%u$qkXCP z*kh1;lCEmc-dw}i-0s#eOia;5rnBnco(FT(Uvr+A3R{oR;dznDo3LN15U%LdS%E7a z)IDFm$rb&Vz}Hco-3@M*Pa zgmz%KaYZ98hFW`rzcAa*f~*5O&ah6QY*8Vc3d5NmT3%Pmy+v~@p1BmCzP!HkG#z(> zLUb4YKA`HQ3g4dKI9z-Fb=-P{=`Y>vN>HC;m|R~HILbn7`Ywb?$|L9Ieg-FLP8Xxm zL8M?ck!k3$lFL5pu$dv=71z-TKWFG@sNYH%;h@+Yjow@Rs)aVo!e8k^6hSO%#1Z~g zr<_2kHy5;BeX-I+@;TwSx8c3~7bX{{%a1?{$8o%dCqwlv{&y_&pyS$*-T{@3knW=b zglDEJu4C3OA3#FqutlJu0E6PRh>g$~&<*+1bIrhDYz+Rjg=s~a zt#XD8d?PSSJ}kQKv0B)R-}Kj};`k*^`KGF575zao_p(r-4c+vC*_pNS^ZnX|Lr%Hu z-QgNX!=u($scQ-sV`C`EkTTsrr3jq#L$U@bPHd4E6Y*ff9XC8Iub?)F$c>lA8bDr?zc z3LKrQ1E-?|SN3J=f2IxmDQ999TWnt8dsI~x6Puhv*P?~Kh08EZ65zwb)*13_NA~dB z${wy5!>mCBn! zJcyWNsiHm#=k8N-E7Td`)=As8Ex@ZDA8P4n7~+5-RHu0_FZ7hszZF6E#yqy+K~iK~ zd6TcY2_+n~)AdFvZi^4g^#A1{xxFa04{1~wmRM!ji>vZ`d7bJhV?BsU+B7d%)gZyu z@vQ!nLvyGtK_^$-*6bLNI8IYJq8Srw&4he!r!OX(|Eb6h3S!Zj}kLzKZEvXzo=iQHIN+2f@8l>nWD>dSz) z%YW6;a;&COpoCb*8ZrKemu0%t;k}|kQ3uKgC0Fu|BW0xIlV19g_KR^uoAwSE=P-y? zvYQZHjIk{{EjD4ekTxb~j=o9vz9kx6=5vM8pJ`rCV_2upJsp)?cs-i2 z27hqZkD;fXf*}z+y?z0%#A$vnHzs5sb^Heq7(`k9a8x*2>`cBfl3hb2Ik;7(=Siqb zgb$ZzrdU>G+`?E@IxxS582p>Tobzk>{cs;?F9i|NY| zr9+yuiR>#$9yNSsxN9QHNQ9W*)n=v4a+5AFoJ?Q>2KR@z%KKp^vySvBPIKP>2q1E} z;%_5y9h zAyz)sdT!P4{}eStzAcn(j`+=f(pJru5-ljKmVZH3%na||_s)y(_PwTO!pZe+Ar#i- zvX$JS5AjehOX=V8B7g*g*5A6 z`XTTx=*ln7@wQvKPj67k?s`EVm5x3w6=HD^gLA#I_3ZUiR#~ZFd=6Rn;nMP4j4FKV zH5kRb2Z;MFH8VCY0~kzP1-30Y`^WxEM>11=^ZoE=q@JemVt;v|Y88oCC`NBHqZlXS z>?rlc(~Bp>;=}D`4>Rewh`NP~+*N|~#RS!d>Thi(sIg*w^=+Qbb>7smOHXCQ_ig)` zC_ht~bS*eh9QFpt=v=>7#p^(I`B6XnHA@S1ExQ#62%P1~N)5Q>DKZtfT`X~_qt zJ0Ma`7>06u@fb=U*QY@3Gu#es2$o8vvqIOwZ0=7TQtav0>LTc7kJ^dzt7WF(bAo6> z!0H6JBTtVso5k0EzW?nozRSk#DXzQwC&b&>e)myKphi9OPA~fEigadyNm8F#k?S{d zhHgzU_(Ukjf6(9S4ZZ`x&%PM{-%RVcZ5JNVjl8a7d-k=YcS3iPYim?P`9O@+w)-Vt zt(mj8ap7b*f48>!<{-Gkw=6m{V@Nqw25tq>)t~s*7q=NcqpKJ}hh< zT!UK*9Zvjf<&>SaJxcDB8QIj6T~FX3+!F)%9Ca2HHkOa%<`{0y3$Gp$Y&c(h;Amzm z4qot57B4DOJC}`&%Zs~KeNM}hWcUxYNX_TrAyulNh_0;B$BkeKg^q-6oBM)~HNB1* zLW%J4VZ&=Be|rOYj%n7j%b1A0$fV zKmYWR5N+y&;`CXHr`SHB^c%*}>ykbT4SCx@&;_9fW&FgQeR_7fy{_A3#e?M)ipkcEQan-=DIpgnnHu$L#h(^!7xGbF) zpT~cAIGlzSLWkwH+q}*MGfSEL*f?cp!2~O-l-mm3OZL`^_ zkl9agCG=VAXFWgq6d0U$6B)lozFudO!4!gu$GDHhv))FL{YO!{y0coHs3~`}y4Yvm ztY-tGx6(u9o3K)=C~bI9YenyGAcTiE~h!^HDP;iK9=W5f(D*kH=f zxI3sVsWFkW|1nZ8q0)WawqN;7IaDQ^rw)rml#IRQSO4ez!=eS{4~L3Pw)UsYSSj9Y z#U)>@0rTr-#Z_shh< zzmREB$OLe^#QGiSGR~N2D>5%ksa#>r+Sa!XT{qfV#>~8RuT_(e;(V%5C+r)K^wH8& zvd~2-B5;m61*z9b83bmhH=W&ytB&?!#=qt>w$DKDp16=>iY@eJ2d_)J zA`O^9<>dzNPzt9ZYZm=QfCWRwZ5=CAyhfJ9hsRJX!m{4NTTRS;t^@vqNB?j8TuMgI zDxAMfvI3%!Fp=wr)YZU|)QR!FoWZ1JJgX0)tABFE+q}T39S1w$$3o*QHTqaM+ zsCZc6X|f4X((o#kbKVwnWMkteoNCak<-2K>=MMv*Q<$WTwiD#DmK2fL1swh8F2<$i ztTr4*$8_s)w;%!Nz05L+HYnw|+bYJXS=@PY8t3~swkR88^~4ADGvIBu6$zO$F>;3L zA`(MJT?W|zIwgu?f)@gE{luYOt+(-}F+chGdl<2KDqQf+sH3(_++*EiiKQ}+7A<@oUDGqouRrE@&nI3Nte#zZ~QeYeR|b~w=gHAjTkSH ztvi9C96FV#L{yJ{oR-->5oZETQZ+;5ID|TIdXHSY_rEw4C-1gbq$H%C{P(70de=;jaW&ucN zqgDPTD>aC(3~oc>k7w$wQMERQ#ZOe(-TbV<01Yr~RK@IYn~?{)e(NVS>Q6rGd9_7? zQaszvrWprFs5z`^l~7zhJ)mGe8e zzS!>B`(Z1}9MV+PzC0BKE(!*=W?8cZiWVbn zQL3_g1z=mq`F@NQTys%O!S+@S%9;5g7|(IGq2)|_srk*gNqT`KWE~^BQQxA$kbxXG z`oXx4mBAuFTU|5U(cHMXWXeDep&mr?_z&iKFkbSw)n=Y?6>Y8_OQf_f)Njxs)wRK(wI$!eXJc06 zZb=SzOeGsly#G)aW@ol4E945}8oxcc-kcU8+}TP|WhOX*ZvWm=`FCuLc%u@bG?$2- zX4ap*3%wEAZg!9CAWS%wQrCszh0f(X;Ns?Eth3W{HCk{uwwDjk=4&s++X9t zo3qo@NICXVem`HLQn&hq_>T*j_=cqBQ(g+cWRFgjhE%aK51!eu)}Dxr)2zrxY-`$4 zcbMtU*-FKew!iQ*(eW504r^}xPy3LSQmopsL z!OZ(Rl7KyEdp}^$JSvcHlATR-Be(Rm1&=+jc=GIo{ruc_<|xnF+XHFR#5J;vDQA|?q}|glS!|;$-Qb^ z7hnZIQ~zMrr6;M!X&PNYV^pw;K4x+ROw7dE)5W5x3EBL4+*`$IGo#_gQwLq_d!b1(spug4fq_a$zRHzgedxb}%B(vR5m!Zezc(C< z#T{@-`4@xfzQ3sUo8ok!0M-J&Pn*yJooZ-8T_mi8`cV5yG>2Oe<(SK6Jc?piv{jSM z+xR{=W`))0SO-W)>#=_Wj~r0EnPU4QJW-AkJg);pz2OkorWXgOBJ zv7Y|rpzh;b-qteN=~rKh2zJJ~xv_fU%rFf)j(acieG+wDw2+kf}> zQ&-4UZhfldshc-DnSY7TT;#f}znKdb*=c64{Kdo)<)JAJi|BL{%If6uH9>Co3yy7)9fiEcbWIWp;uaZn#5xM)tJ(!UeW;Q&(&JuS)j>jS%90Uoyox`@Xr; z&bl2}X%R0`7Jes*=pP!GyJO@s##TN~^Nz?PyKo>1im^-a%SfbBYzm!CHDQsFFPAoO z3E?p+X3C)yXd+h%^1I$vXsIuU+ouwA@x&Wu3KGZ)Br>irtr;P=7J6=$^E{gDt!cNA zy_|_HSM-%Vts>bcW3>!!Ohv)!Cp@`7rJ{&vDwNL)*Qgps^X*oYgkf72p6PlO?5b5O zCEO{1#F+h#BdI0od+OjBnd05s9#d79soG3ySYn@#xg%qTPebn0eKKLC5hnTVw9SUB z2C=)o_;_+bAHT4^E^_nvHXdAy1o9wL`1~5_UF(63Uef(vAiTS7~j8b*6Th-sUna^-+Sq4isUy#O-x{OzH``xnojtpr?i*7YD zf^H^*`whQGncE69(v0A5??tEgeQqmPW&LCl$r|ykbo8dl(6~Kpq}Baa1(3H0RK)_sv4T>>D;qoRFuDR1}9@?t@6N_Jy{{^@1;MNDTN-4rmGEx8og-p zWw7ZPCs%7W^ERun@)r|Z={Lz9!yFPUl$oiMO_N7+&f{&$cDZVQkY`-SWb|?seT#@S z^{oBuT}`0<_Fd$kl;m8?4}-QRe8n+W%L}_^K8p?pk^5$P%X6C~zrv{+D!9#myhaKa z+3srj6vqph^-}?v(&;dlykU9dJL?x#mMmsGdLUx5Yd%_s)#gUsz+y>Uuqa<=p_>)C zPL_;%3qjz`5oYSE#HzDB$Uc24MGmGmORX0dgu8dAkNdrjPXYJG3^FbZ77WaFCkx7| z6bJ?u7Mz-e3X6yWLuZEq4+g6)!vU>C3=+_GIKab$L6!UPz%1}m7#;$)JwO1)uiz1Z z@m@TL2&k4ELI6H*fQ2UjA-F*ZK$uw&dhi-EGI0G-2;v1234>sO9C{&aAP76e81zpF z0tWB0qQSF)z+S-MViMqAgcDK`E*LUNJA~g55%3T-I=mOiOac-JYMF#b2g7*50aF?O0TAWQ z5E)S3Bm@j9b%i*Cs81m(ph!KaF-Y?eFxi@bRD!;L0l+TfA=)7S{~*?&>S(}tm>OU) z=7(UwSfTEL-gqEuAk$nxg2F0P1+=0B*gDn$+SnWb%q&zuIYlX8Ql|z1PJ@w#@Panj zfef^E0%RUb0G!Jy;2YTmObBX1D#0$>AK)P%K6*$Y=*I@2PlX3^3VNOgY`m%g)Z=NW z7YLUNu#SrX@Fy9dY@j(Q=rt(T3NV3Q0H}mw@K#_LoeszxsPPQ42O{4FnDL&0{IKpo z$wBGVfDMr$5aA3JkWo}FV8WIV5b_}e7|U4!h;_XHeE$<*);1U?uD@0juVP83VKl z1KESIfdqhIdQXA)%H9Ay7uis2P(V0@0)`VJ42EfYft8xU;)4m0@&c5|F5lAfZp;O&;;=UTC>1Va*&BDpk_xMDCF`mAf){m3I=^= z0D3gD093UWP%5S(0N{ZTxV%(_IKx1p1z;E^QULD`I4v9mQwe0NqZ^V4qTEHbhM5D< zPFJlsQ0Ok_j+ArT{F09w{LAPFFzB0UwZ8Rt>mb>j8=7r~`wN z^nsLk&(g4hVJxMA;*k~tWd7(tideS*0Xn%5YZ%f?FleqEV7-V3w#u|8uwZhSRX|3~ z0K-!)KzPRs;0o*jIg}rU(t~DJ08B?zpz-|+TuOceyn6cqNV+xvoC6n767&s<0Wz%s z+;*;kv04s5@d!6S80G~`xVQqriv9-V2uc8#vsv{t$Yed zfL#QjAWR<852Nld23qG0?^T>-tgx6s5===vj;cy-5WCT)-D# z{euqe2a(P}D?!9vz=X;@P=Jvl2rrBcku?M~U<7zfn*&@vEi>699wA_pV%-5-I4cm62-8562?6oi0|D3_1B;g>5J3Kw4rIlu5YTuQ z2sC9P0G_WMkOsFjVB$vw0MJ1V6IkWrZ?KtUEUDh5nF5O*XI1~TXv0*o(m0`fxpARREg(}kdETqql8Z3)=- z-$;PmB>@j@z^4-cn$iF7OK$c+piH*VOqfel8PL@@P=&t*fEgYdfNhZm6#=1D0}e;# zKr@L7*!ubcP2wh?(4+$z2}=I}RRNhf0v=s4fQP3bVE5Sq*h{-@5N()~I%60IhenX~ z7$A!e7g&W^_=#5C*{0GJu{|B~XQ} zXy_P-?kB*?!v>V+{06W)@C5W@mjEb?htPf)l*?1lFf2fCAq*g?@&J;a|No#c=?w_} zQwF$_A_2Ri{{V2XQv?Fr!G`jJ;$a+v;U8JOYAeo&0S|2R7H z_?WKuj~_gkA7V;}nrLhNH7TO|mCB2^`pR<~5mn?>WzTmd~@C=iHzXcbjp0)H?3`eHLsD{hC$I zKLO4CuUH+bdQPtNThCa(^kuqb7g%HqCz5)$7T&Z8XW?h9MydQTiY3hdlm|Z7c~&{N z26nrIF}>Zm@YgQ>oU(=$jqAq~=c26%sz+I|gx}()`puZjSH~JFV_xThMaTFS{Wok~ z7|p$>uD}uBXjtI+7CLu6;#+7SzeO#Ao_VK$rz>$xaq2#{*6?+P8W0Q9bWg6n(wINX zCvzd_m|>Hn>4>DmXDn~P+l*3a6IWl|$TD(LSXJpM_%}P=EeD$bEQy5$V*iO{ADRXc zKRjf@uQGYOb{3NR_c%j(_VeV_@6nZSCVA2%@+4$;9>Jf3dcp5&W&F7!hsV|p!7OeT z@>{=NfpIDZ6G+h&fhsaEMV@}o`lp>{4O2bbwJjYwnx5d+c2n^Xy92Z3>K?Z)sg@m8s@CV;9l79_~v7tHB{%V1_BUf_Y!fVzv z*>{2!zV;LAa(b*T^oeg6Jz> zFlqZK$mZv34ENqMtf$F~tbfXnthLg5!zY6uaLdPT2yiup>f(dkY`xB^7tiCj$)_1* zhLdl7ma^<47H%H8f~%7v;C*p03|O%j3L@(AY-C^VdlLv$XVUZ3?LwOxrB$$ec!{}2 zjrTa!BApT}!S5sgLsy|QC<-QS-U`FodW=+cCpt?K7cxXtF)KN3MpqwIf#4C(kVf4B zaPzSChN_l2rG_-B%9t&m!ksOZjcmC)3o@@|!iBFrC_&u^DAwE)hFet&%#!d)+?%+F z2@l4>`Hgi9r>x6l>ht~Kr)v@Hc)2#u_umT~T%XA@FMkS1kHa>GJcOL*U*!2oKQd%^ zId^oc%X+`<%pm<{VnRpavm|XSqm8LftgHrRNk=65N1B%i&5%DKcaqE5H4~nq`?czf#qq zk`O__lB-K=iGJH!jI!cePaK5yZb8WDjl10atQ^qYivtYPaRKTFT@fO$&LJ+cCReCH zoobOhFASS)c)V(^F0#shnW)6R8i>!`mG#WtXDyQC-3<9o8mjjC4J*+=K3CgkRXuG{ zSF@ostD_O8f^|Y28TJ-yZexZXab^TnC6l{XxAMfQsIn|s^F1r=)X5;52sf3j4tGSf zW8r^SgRM^^A@K35A7=GS13{$@V&)-Y?fIZ6%HUE2a=+?&XtD3K|O}KHw?85-6d7)ilY@OH9m$_U)jT`H8tw=NEg+S&JBTz=y->mJqdG4UaH#Pq85%q zj1xc@salm4zOflqS{QDmN!A5^t8b=l`@5pppIzmVhjE5a+TTZ6_ecB1|05UQa~fW$ z+W`;`&oN@R=5yhL*-+4Tu*V`ZQXr%JD6VyuE~qK|9mxyyY=OAME?>DVy2R)w|gxolum1&q>qIRpG- zVrBSFG}NQrA*R7QOyq6lx<`!%T2sIHr>cFO^Gb*7}{O&F{B)=gqV(OWGy=eSj$yaU0O#vt>@hp0+A%_8rAjkitHnc=czKlgq20t7|8j5fB? zzHl`mxSgt@DHBreSrf0$or?4b!Fd;c~8B z{+f(pgyb@*inRLlq?#7hZ-7Bcw4A5z6(&lbs)%q#Dbu|&2qS5K6%sxi#r+FR7jIqQ zu}Gb9kWh6$if~I8+a&2M{NDL5Yp!-2rY#x`t0(X}BDF8asV)^}iK9C!=yivI4#%>B zofk2OD&KO~KU)~`gT`wu(nn!g*L8k`M64k-YNpeyvZo>Q|8$7QoXz<5T@E7H;e@&& zq!ub&$JQspNePiXCW%w~aZ6t}B3OP7N?t#MudJaHwTxNDZzKM~(b=L))Z02s%XWj< zvQX!Rs0G>)pt7Q~q|!>%D*SJ>{OC+ZyS~!nkOMP3Dbl1DBDnaL_C1p5ae+Uhe}tzU&cdD}bs(zJJN!1I10PH7p-~He=fan# z4WGR8JbW{BZitMJFybWVO>}6m&Wezf_W<)(t8-t!W(Z|tD(jmt4xbX&oyGUinGNN` zj!a&y1`|Dc6CS{&yR$#U)9GqlP2BDOr-W)#Ww z2S$p#xeCqRxS5am57ES)+QI;PS>^}2G)QJw#cpRsC(5oK(0}<)f{{Af*?_3d7AVbI z^VHHuhEwirhRYw+#ECD6F2F%J17uKjF1`^3IYkrsrcc9-`DOvE%2YzJ2#U_Ci}u=O zOg~SNH0i;LUB^L-*PaRN~R&7@%n| z`jgrlLRUUSSZT?nvhO`ECXWFR+i6$0+FR&Xd$lc?tp_7Po{&#~Q}zhS&131#JXGi4 zT%MeroTn=5ig5XL3NoG2#IVSax(pO}7aFGCg3MksxiHs@u`m}W}lH&51o z1GSd*Fsp^mYD9=?X(pGFd1`WHxXpbQIct_RzcaTMJ}SVC&oyhye{Ho|<;NBf>eMC8 zq(&?^uT!2BDZ-zskJ7W{{7p}sYOj5b)N9&ZN8D{emP@nnCSJdhsLto4Nry)$^ny~R zZmQ!VB&`psaMKLzinYro#UZTGI+=A98? zRLIPE2D@=xsxG#uYA&>}SvV3OI0?yqdX?v%ugS-+mr#Z0TNv5WsRPfQc@AAX1?Gqv zfTA8s&Xa~_TwVMKTWqiMv{+2X@%c3F4LgH`zMaAIkKV_QCdT{KR&BG8?4~u4g-5`~ zu;>IixE2OS>BMMRxu1tBE-~ya?F*E1Z!^%VX3q26mk8%yomopI2Rc-{z-+nnI&(I9 z13Vqk4Q1MX3-GALGevGwTbKuWsu`jJ$_m84>XtIwN&#QfOQCr{5lPEh#C)px$REa<>OtRIH~d? z?CR))7d85_(pQ%vp3lbfZQmG#zwsOFt%@#k%GH*Da)eF{2Qsr8N#huvX!9N>aVCjx zPb(WK(mK_sP#tten3QZo5G$V&JA88xWwMSp5@cpyRyJ%IDxgeI*rW4nt0_7&PzHSq z41Ib9@n&JW;;wC-$-^-a{(1tBT^|5JzaKViX5bzq(~bdf-9lOT2pwyP z<4eOX6CW{A_o3WZ`&DLp+hpb<3dHK6j%$U(@HYZd^TSm@V5;gDpCCCkLGaAouyT%e zG?VkMqAQ<#4@0^ig99I!6_USnz9Dbfj7+H=f>u`kibb9LjBopYVVXK$GDGn?BUWyE zSkzD*8;Pqg3Y9kffv0+(voW?)=QbBld&ACp66?!<1R^qK66=|o!z~wTvFg&~>r3D&l*_Xo zW~Wn)Aa*u0jG`d5IL9d=pnxhvpLc=r2!SP)j|lmS*?VvtU2C4ZUOR{qz9TiO{s z)yLd4p3#q0hLpvrtAPa5zoVzCPXk%20yE^x6`=HkGKjcJ7}i}Ex0arRc%*X#$lu!v z12zR46yWxw8t++=#I1Tv-aZ2r`W>9)pSJUDe0A7UI2}Y8c?Jdh;tJZhX*3G*iSdjC zdo@plmSVsSNbAWpCS7VYYZww)tvdBbsd)0H=QB&6bi3wNH6=Bk?zQ?97w6v)9Ut7

      %U)#pFME)(8^hrJdVZnc;Km-dzsAd&AHHp!S__{q<4a#z=UHxlUcK*b=1vPyaoX2dGGjf8LX)Mnv|oWVcQt`?11h0#7hOnUX*oDCR-9QhaZZQ~*+9jUWB$j;BvuV?u%`gRfS;P)Vi{_#V| zFRI|?Q7I5ToHDa&pF^GG2i(aiUEr16qpJ`XV-FFg< zVf)xrpK$575pLgI;fa%fleoP13QU+*hNx}@pvBfQ8*x=a9jv&dwfZo=goR~vB48}A ztLEC@Qoap9-3lDo@6oTA;^*yXZeAC|sov7|CIoxFHj=Ujchul8=H<{C&D5M|xA@~h zp4x@^vO5mGuFqgehsGeFUfT?>z8YsyC!RsZjY!yM`v#qQ%+3l6DTA+5(Hoc17{8LX0WQdNj{*C1cb zYb3QE!=$W0D66C^nqacIn$Xl6Al@HXRrqUg|Kte1t-fc(N$Ew#eHNHa{+HmC8r@;w zG3{zE+sM2nt%~85m-=|hWy5+STWSrbVi?E2A&?KLn~NTgxW36ndQ;7c)FqwQQBGC> zO`Wu>wnVkrol0`!vdEyvETYzBBaW>QGww_c7S~A(@n;i_9>0e6joN`8H2D=Xjnu`B z!F^jhdHWY;-0&FT#h7KoL!D9RIbnd(=`x!vi3JIww4X+mcMOXZA2qz{Gwtpw-8&=W zeO5k{xo!Co)c_$cASFMPlp<>#Ft+MC!>&rxvt{-*G;MPLfxAKsDY~Uojp=2KSVq5g$>9*)hb=fG83Cq#c zrNbC})HB9ymk;r${pi~zooz_m2nao|i|w+w1!{J{L;#0&gsG9~vC?!oG@KrZ;Hm_m z4RvBj1D?*poCcnxEQdiO|)NkoZ-ue^qj?|8hl$VOSORo+H zZv%QG`{RsQcBC_EsEdV4ep?1@_Z^-ha5lHGj0rMa7_vW-058)i`8L$KkHL#c*EPikXP3r3O8M(c`T>Di7j z)f(+0t17X`XHEJ2hED6E25L(S;=q#nQu`mCOiV*^VP}zRY%>-D4g7-iJL|vg&VCV$PfVcKn_vTLKy} z(e3m^{&yO7d_Rtnws*qMlttq(y42^k*9p(5o@6>?Mjs``p~S?5 zW@U``?O{(Eh0IjD8eCQ^x4z=GE|JCyDqdG~k?-3ZG}(zKf#@VF99@aEsEcN>@`=u? zBOlZG0z#xTHMe0(8AMqq=cxbR8gHN^-KT2j6iCz8(Xuynetn5M3c@|F3&P}o2MEA+ z=@dg+{tKh`<~ZcMxfQ$l9j5PCAIbJ{Qc!uFcZy5Hk=WU`hlqDt7;Am6&?f)p!AE;e znzZ{jDMJpnh8exh{ocBplGsWJV3E_cuMae*hR8pQv7J>BzOXXC{5uk@TId1$8&Bf) z72hG93bTPC_&L7C?*kRys0KPN)Nw9()D>DnfDl?*sN1X}3jI zn=PvCQDh)2t&T+h>2XMt{wy*tJq>$JjFg}VT=Pbq&`hp<&TtodqGb!T3X$g)V^|#u zlk%kF9fN4-1upbpzg4}X{mo_I9K$bX3ynP4>p@j5y230a6+2|%3`W`385*x97Z7z9 zv?JwFF=C$pLnrH$Fm*(ix@4lcDRL(iH*iiD^^sX|*y-LzW(7@K8cCO@3}UUpQr>-! z1uP{x!f~b4mJvhHO6M8FAptgS`NzU7lUDO>%P)X;#VYu`VF)(Wl&nbd+*}qZc*RAk05TC+kjgsGg1h4wnvEz>QT;*>nLN=QKesXZnTV;2vvE7=x^+F zqN4Hh5#oudR)^Go0xbv9%gwFDT$Qck>dF06K!zo`LY*{AJSPhci~K>4sJazhBF}%$ z5FObEP6<(&U?R!!7sUVeO|^phK1a zVrh2|z`=vs7cGzLn2w@9K(J0%xbX+>AQ`9)H^yDmRF!L2XQ{qnZ6@gkKwM4xngD#R z7*$()t#T}d@mg**vSg_n27P#w%3MKDxCC(O(p)%L)!?In$%jd=vyO|vUi46zN62NdO&e+F*<+_^^GAGwdWj2 z7NOI+NeesNeN8)BsV3UnM2`H3DEjBa(;kQ5*5jNKd9f>D-mOE{3^{)s2Fz*hDNtv% zy(>F7%1Ui8u_ZA@iHqM#>d4-@NPnSD>>~g5Bdna_MlJq~1H9Ia2N}CBXMn-?z_)B& z5{(k4g~@a5uYw|YvbrB}^O8s-7nCjzlpgazx}Thw$UGg_SAM;MA~q}r`aQ?_Hn%rQ z{M8a88+-7#0Hg>SBXw2-Rh>+|t`|4^hAuaYj)|?*d|lj0cGbpyN_9~UGdl=3D{F1! zdZF<@d7}n_@q2fPBVGtBmo7K4-5oluk$ia@r~jhP3zY1ajSO=8%$D+OFB}Zi`7Q9Q zdA;RI8(8zvR~{$((!SbKFqb9kQctXm6_&Cm68&uV1Y}sG;|zK7^V(uL-U!+wK4cWa z$ClD?534<((;{)kj#&B6Y_lDG74GkHqv}hjQ?kPx+W_~GQBV1`qm7)o#{hNTM9gEs z44NL&f@Gc9U;MDCktjdB%fi=!>+<~#RO-5!b6Lh3E=d@JVNKTdR`P1Jkt#n8V{I?! z?3!|M5A09F;-u+*O%Xa_4s7ptB_uO?TnGcd5jlu6YEGbTR*S8h&`|$U7)<$ z*;*?5V07nA07n&bGj@-*_mz3Hs}WlkHI@FWaLpf97=DSFgOFa^1x^qA14@_clx}!Q z)s2#;BSg-=1CD&Z0HHsxGkVEUC(7PwGX`~7J00?UJ0$Nr#+2`9R}{GB9wN*BL}6Z{ z&D(T07BP+kdTn&BRW4U}?8J8Nfl|H`ym&-op1U`$xfK&~_a)o01*y*iv(3F^vZytT-6hxX%Acd4Z-_=S z?r4K<0}Ycl>)23{-;H7_eMLj%zRgI)Kr3pKQ6_fBC#OcLeC?{E-qXJ3Tb?I7=Vf;-`M!!BwUI?+O>#NyctI?CQAKvq4SHAF4b z<(=f}SD5e%ClSsl?G2)Ok3X0dWvUkl8sP%h>mlUMJDJT91 zB5a5RdIGYz`j#$jEw7lnT(nHZ0jpUx!s%gs>#y@YDvx4>oIM0?*3z-$G>w7zJ!WV6dt&2VLGxMMR&UF)UQjy{)Lc zW(`6kv+K&Ne#oQRZ6=#xHtS>-!}f{V-B3Pgz@O`;!?7Q6nlirJLz4Xl8oGHQhVgYI zi`!!w%)7}-K6}DEOTPd{?wh^9Rn3z3A?>UoQ??j!62Km@JV`K|GVe0jxy#4Zljz7g z^B7jr6fo;(ZKHs28o}I-$Zep0FcMUY_!Mc?5i4lY z9TX%p>mZeGA-F^F6@)eUL2L&mbfnyr+d=(Vm?p<99=llpY=^(_jfTfrjpguW0_b1P z6V=o*n_hH%yv&>l1XPc8c>ppli9wH2>Pychu*M#T8aQ;8O&UG6+Q?vi&85dbXmbTU zEwbljG7%TM+%kU-?lV)nI;wlonX>id( zG)9JB#;W4X{Qf@mEAcqE<;;!~nX;(0VG~D;Cxy=LqA<_6B8&Ir8JP6^O3-F z(-2-oG|OH=(M7g|W0IS+kN@}_ER@I+JW}rnN`K`6f?oMQKyAN;kfW=%kCokK0`avj z?Ib-H;31amAVsGMpdnb1*p1wumdogUFe$?9D(_^*PC;)WnzglYUo%};!pQH4>>np# z01qld#bk3I`S*YLFczZVGt4L|PnSnYK`?>{)D^?jaGe+kUZ=H}i$CHWf@vwE3gHZ; z_~L3Lg|z<@%BD<^%-6B>7#-W3*r8a<%va!7yYtXDSeMq86~{3=q2W!=?qD&cI?thA z)tMd)E-zeey$M0HKSIPSvI(br`;APzd)7eNYGP9}ZK*FsF$721IS%Y9{jJ|9jdet$QX+Jh=)|=`XbexSsnlysgs23BJm+RtDSQ}N-S6#4WWe@1t z6ohG(%|>ne|An^VEhk*=uh$YCgD)DSFdthR;A*{!H&^%g83$yDuSI;)#3zx4mb4sS0i zSwK$19T*=q)=tu>o>4AOW?-a=WGv<$%NTqiC8_THG2vZ*0fKXMS`W28u!t~&%^TNp z?tO{$&dltrs*#QhBMW1Hl#XmE#;9e1xv}^i@&+1;hJy;_L)0J97U?wwao5wzPDgQG zgbV^hB`+PvRy7Yh;Xs^Q7PL0fhyycg5T_T9lhu^E;6!2#^^3MOCk%G=Cz{D_Dg*i$ zaT4(&e1G*aqx8tYW{Ql>N~KV z3AbSXCzn9RYcwH{zIwIl7oA33P{mWY$+BXo&fdUlL4c`3T`6az1j)vpjQ)>Sf#%7I z5~#*)6nlc6@`xRfWNbbz*Kx$+KqT$qXX;Xm>%L;D9y(@oZL6yN2R*2@Sn zChLN+@-3MkwTi@c;_|Sh4H{i&G#mlUdtqF(ivA|vB{P-py%c{vEi?1ug9kqjgzbASbVuz5*e-i4!)H)k!Pa}pZqd= z?f<0kgkeXUeUb8_spTU(0!1%X0*aoEATM}kR`17~C4~oNT)5ZT>d?)ai`gO;#BPzl zzqu#cNmkH7fqJN+aylILoYsz^cnx2Ds!%SA`WqaUyRSe+m1GwNQrZ{VOMMdF!eiz$ z_8OD`irr|mrHOQTz=RidaZP#sA1f7s>S)ZkDM>aCA%wdvW`v)KhFe0VX-ho8H|?!Z-BEt~)&3??-_qGMxZ?L#A05;u4~W&LqPj?|p$XEa`)~zctRV%eMrqxK>*m6Eaz^b_JQI z#Pj7*AB?z_wg(|TRfk%pKN_n`bINlx&Z!5<FeL6ru`8WZtm>bPO}a)9aX zV;K9I&S)g>`ca{;&%x6hW-0$u9h*Q2z*bkbtwrBgv%kf*SX`up)$pWAXS3((*MDH= z;*KEO1RZOY(B9-!6X_&nB8q^`<2(+)I6FYPbpqu-xL_norMArWh0Y2<@~#Mk5?hntVlPRYbS}DhjMM^v0z)*+3mXs#{o5ju77BOzu-vn>OIW8J*ah3Q%eTDQe9c{o3{%Fjd}OQqznq z=`|8cj@L1D{n69f-xyt`YiLLe4)E{a2+WV@TnCjLS4;3XE=YDfhT`CK z>f9|6<3U{z2~im_c+t42SXwqF&sbTc0Q+%}0F%(~kUbjgxV&?_R@5 z{5sc+)(L>SBx2~@bzU^8omyWwi5e#Z-Z9*`*Q_S;+e@Ab0`(%Z-jwIZDvE0Xz1dbG zDn~MhL8N&J&CMY4PY-i(Q(##WHyWqOd3OQ-W4=2w40B2?omhArE{Un?NzLDVM;i^)@tr z(AP#TkmT$pYd7Ez8|jR3QaT9!{Hx=d$jC=Y*(^D=wj8I(2Me7IRAN~g{wTkmS<~(z zxwA(n-cSt)hUwAlV;pGJ`1EiE{s#- zlJnHWWUqvbLe|?d_&7b4g*$YKUHxXpQ|Dbt7S3tG(WYM*8m0B|;?? zQ^2{q-H_S~^}FOSZt1*xA?z(jHK2BFsWo@gX3HdL}FQ@E$c zYEE4vPC{}FJ9UqeP}%h>lZBfF(e)!B{hoGdDGSF*U3@@1(0Yx>F0VadVJ#-0Ll>;T zK>2F&@WKK{sWpxj+)A#HF6|*_msWmK){NS6U?@~9BO?OjOU6p~UR+(G-LowR->4Zi zlpFKs6V5wgq{F`+FV40!lo-a=1f^+?euI9PlU#$V7`QB=+R z?PTp!T)=AW8im5E7_;}LmMp(*n5XdQaCpW$kY(<3nC<$uEPlC(sy}I4Lrlr#lDU5o z*-<=?k|w;3!;CT;n<++pz{BQ~p}!bTI?(clqtS3Qp%Wt6d6SVxuz4lHW05d*_zWz# zWDMwalm4B*qYm++8xI8bVq;M}n~DB*bYg$?V>2&%@exA!;%_5S3Z|pUf9bSPjN4g5 zHA+uWTM8A3p5~CAW%z~EpK!GE?(+DGo;daP=KkfX9EVJ3fm!ta5}*myHajH(m{bsf zDV2l)EC~Z~@IFpXMY7hCepp7H_KyJ_+ym7sWq!H!vBxGM-xxOe?j&e^u=?&i1Qut* zfQYN6(NTwQ%?QOo=FgQUcUfw-w!~6$%BJC^A*{=ZE`dnzB+6>8nqZ>4hJ3vVEn0CK z2mPkbs3{XZGSXO?zk{TIh*sX#nE_G}i9xL|FcSFFT~qugz}{J*9tS(B`6hEy#*oC} z)mE81zlHy?xIFD@rh=2*@@iMs`vu-jCSJiAAKK+{%C#4;p87hgpPYCZ%-MU_ng=`* zp5M}?fqZlNWZW0%{FK$`jgOZHfH|&yKnPoRSG0lHYZBD7$8#TFLXP}L6(d`wKhN)r zKSNK>RlI-GNt921Lc4C8&Y?u-c4J!x z&<|9I**%K;j_K51<_1Y^c^E|Y8Q;*z#1qDiktzuYY>&=Lp#9Gtu7Yxk<C^|~Vjo`!54#(y=nHsL^GnvTHi_?USY+|fhL6akLd&|HM2|fuu60v- zC!+joDB9f9Y^~W+731%{1uRVov|&HgKd#J&75|t=G^*gh)Sf`I4c}IWt@VPI^6Dg` z1kK|-)u1qRYjG-1cGLC%tTAOWJscIaWc`q}oFVUkIX5|H(9L01E_S?a^{S_V6>4aF zDu$mMD)Hui@8lUsv&IROe3PHiGS}e=&wSu1kr}(WkX0Gl{(d_t6Nd4B%YW7--KFZn zBo}niUzAYhB%!M?3u(^h$?h}iQb=|NNxd&ItrfapI5pe+1Tti@v?kg;p6(BJLDRG) zRsJ;Fiy!HJ}+$%gQVsf?%Uw;#IlE2+JSU1YpOh-WSDL3 zG$0%awR+5sOI}h~8`xM7hzn@57GtQ`usc=~YFHg6p`Gyc35#j>vrUn2uaQ|9=3)6E zyx)O%*JmhKdJ&kIt)`mqsoCv4;TG3<kCP?crLV)s|SXp8=s2}+g z$e1<+fBKX5HYA<4)g~$^v9Mj8(LoNHC-C;YwbqLhbcaceD@GhvL3Og}Yq-NE#2YfP zqme1jTy(TUa+*w92mT&S0yXEZAR;0S2?M`#J@Vl~*1vwd=@QL16U7H(Rm-(^3Nc?k zB~Ff~sv=^3`s!odpgY+}L2{oEeuv(WouG0d^vWk7;o&hUYXp7rCR54KY;Gm^%dsO36U#pKSMJ)n3 z^6c{BVnR|jP!5b(eN(g%C zYix%3aoBE91xh~V6C*`Noi%K+iHgVJj|0PX_pbm@XiF4FB3jzYnQoUZEr09 zDSoee*2sd^%>E!mL1#Ho!DOR!ev({y86KxMB+$tilaz?5I%`X}BEurz)nIi;2_e4N z9#mj8d8EY^N zM-&p|)ht^(GS9AAnA!_CbHY}8T~V)DGsXCvkgjK8hS)rwI4VMEEvX2VH&d@zhX}P1 zHjfYMET0-zW|Mwj8z}^*r50Io6?-b^VC`g|U z$MVtr8fIAS9Uz@O3EJ{!WBJ2%f|knW3wRzbC0=ZVo_KJ7PkC6w>Xz|NE(|GD z5_*Br&u^#he_SW{yS_U?;Mje*O7hvu3C@A3Q%4#TO_rR)SXNq?-6f`fy ze$7qO8*R*Y>+~27$Y<#6g}92+M&>S7M-(p3p&rCL3Et=w+hg^Vv$NtgKxRG$>xwBL zQ}Jjb@8YqXi1hW60XpPRZ_9Ol6ZvTlD%xc`3Ov04VK42AV79WQDpiw>QX&>kdXC$OdDUST z*erzl{Mzo@E6d|Nf#qwKe4qa!5^q#^>8sA+Cdwg170!(pY= zlPCFr{TlP2^GsdP9{kF9PWpxe_G@%nuzYED&HOfzp|(sk5{RSHG{Km^tLcV%fXAtc z$Rs0J7JbMQN!r#)_P0Wp7if!5Vp5o6Er-Zu>ukhcQ+r#;lZ(UvL67isJ=0j=Q2N5D zlsi&n>2%!1QC;d}`0Ri#8$mtQ-pPQTuf1$Pi~ySHl-$rWGsVTz>7Se zW0Sr0A@j}wprlx5Ehaa&&sNh5d26z{+5XG2wHC42fbgXcj0E}fU+ngR`Pz?XWq!$M z%SwvIc#82$HXDzZ)8epBZUx zl=i9}HA#@JtuE+cv#9FPcCdiR;J0)?+x{s7;5%2||I&g<;%60p&eh+E1UD!qEJ)MYw9#?15D%EOnipgNo#Cq8zJ z5;Xw}=TJsHNytEbB6ao{>2m}?8-Bwmkc*Ge&Of!|Rk_#S$dko%h07o7K>7U!tJr#g zMOB-HBCXMheN9gm#l}`j1N1qoHhX#5eTd*}jUb3KlS6M{@S}B{m)c%lH#u4X1=V%Y za`@vM&MieIWEJ;kbEtR?fh}MWf=)`v6~iuT+hb~tVqmq^I#&p#%V*$_vpY~3$Vf*q zscrLWg#aec8%(U0HbV;Oi zuOP;JXqLV@mGav`o!wTw5#jEw%=cn+{Sg^Fh%T4d&g2+2v)y{A`6|Xxvr)AdqJma? z11Y&p%5zy4Pmu~i$wTvL$pivlMusG zD*fxNEl1!fv!s6|jIxW)3gUcXRxtGC+1VRP>?3hYz?ug+ZrrN9u4oyJ-nqY zSgHmBOXue$&)5h#ZIzourre1lc=S9 zCGbV}GR;&q5Thl=EMfIA_bR85c{(y!KtR3m1Vt0J8aK+#Zq{-pic802D_)elf5V3NS7L%&)3cQ& z(8<)UQPTDyEU&Fom-4D~BVT6mqH<#FxQ4{+v1xMpDC=3KElZ^4ZWy>z7iG$iaoj?O zTRS=bBlVj0PmNrOeoB^L7N?{dFVP@)O-Nx=+T$hd&#OUR95+kUm*|E~pJ|2|N}iWf z-(W26>yYy|q|l3^i4t&CWhVd*niyQFf1sB_N^w^?Ohrltjt;aWJH&C$lWA@ZtYY0X z_`m;`wHVOM3nO?dZV4<|W)NqTG?r%_(4S`MJ{kWwsTk08FOhOy*o1%aOp;I67$(-` z+T^9##C?fnytQYr;V^4rnD)H-uojP& zqwN9Lxn-p!XKYdTq1Q+zjIz2dJ%6lHHHl~;m%cBviRyul?IkzEF`ajH0#$+s%zNS` zWY)BQ)gBi0grjLPqBn8P(X-Z6ed9%ovLD5>-O*{WYzMHTdgzjor<%65>-@SzmlZZ# zQ^s-vSv5k66B(x^6e8l%7`BxY!e#D!Y~zh`v}&!cScpg{bysEg3taNgldPL8EZ%G( znEyY+jEu^=VL27?X8#>s63dc&^8R(E@21nH^Rgrhezo`&*|Zm>81fQ>cy(;BnwjGz zYlpPmo#Ed}UAj=-*@Pl|sSAcm<{kJlQI{-IORS3t3VNc|W5)9an5vtwQkopBN)@Ka zptU%J;Gem7fX-{hhI2&^5^j4Pxw;!K+sizQ{dPPrJ+csHO&bF=>aqm$jAJPKCi!!u z(i=$@Fs5Rd#Pl}2V*kjKFYg!Pi_Wgb>gWt=ZnjZ1lo88_VxlJCLU-wm5Y(@t7wJ*f zVCiR`8cZo*N!yZbGO4>KMFzZ1n$e_pQUT1b2$#XUp$J9vww0crK+REIuvF^LMNr4~ z8s@nUyzKJ}jB+(4y;%Nz!lRzS3^{oMEV&zP$N8tl$&JmR+b=rpRXKhGLRNERO&&JD z5I^OV5Z=STnShi1=0;^nE&{azr*waXwGPl(n}Bu}+frwF#Fa%P)p0{orr9Dp17B4% zo!$7v_E@*87n1XClIOogn!jjo5_`KPs{!qVRKl8s#st(^UK85F)u4FZK$D7?X6o2l zGKBEXJfYM;zPtyr)Zox6Nh&>XS<{u*Ix@cBgqs>zk9e?n!aq|Pzz0V882vQRk%E&#i(QTd$y=`vQh|YD& zp=i{e{#`jsW#J|8Ol|wf8 z*iGlImZ_&e-h(;3F->Q@O-F>i3n5nOa`Lmx8pI8W18Mel&XMF_K{YmK>r-1L^@%#p z{O$T%3lOHS9x1u2jTm!*z`3+H%8$(es6TgME(^`G#JzZtv#OCC%XUHO7_8c7WhA#> zc>>d9`!>|=W1ZbqK1hP}%y{!;yrTuR4vJf^7g>FB(`GniQM_l1yjqVDf7P*3lw(qd zN%fxyOeX96&MGnI9uJhaHjBh{RO>R%<-GR)w8Yi|*Zn=gdjIPOXsN+N}inb#NpZPk#cM?eF%@< zCbVw00x)F5!Y1!!kjmBVM&#^Xfwwt521EI*K1%S&WZ_`#x5@CqIIER9Z<(zA+f#~I z+>5Xg`#5U6G=aepUL*BC9{aNtqyE`EB3q`5(n+J)4BcN?z#AMryesn+T*TWqJ&CB3 zuZNmcn5J%nX7B4vLmFoiOI#m-g5J`(3G8s?c0?;O&6jaxPUM!a>Vk&6>h2*9ETC|HnQv~fI@pShYmO2k?y|gN#(yU7>Tmx6w7|= zE*j5W`puQ#A)lFvn2ZVg);iX$`bjlXNUB{2y{>ZRPrum-ccQZOdr=iuRg~NkhzH zQs4h0>C6LLy#ELO;BAst302~u#GMBvu8BMD1SQUe4P}lNrS7=nT#8b69iuiQos~K^ z#n2g3vrdg|-GS~fwXIvRqqgDq`h0);Ba`NNKF9m>zCZ6nD*S+)&sFN|sX@?v#ec6_ zc+^#$OY)+LQfZSxxwoCr{q&g&`R8Iue z-b@KeJh{!xRZtmw#y>?`-DMhwUdUyT?pP%4RhCw|AFNug`CC)iGUfx!n0QMGp*)GQE-VWGMTVjlOSOChsyQaq> z|Aa&MpXh}|Wi$X;7ARe^qpJ5-q@s&<)Q6?~p>o!JdiF#WEV+eW<1L7R9Fu%m#wZ2H zIqN-EBS}2&@nN6seqaUFi9$= zMs7Vma&`94g-%e4ee0-HwfyQJG1qYZm&N}q?E z?J4E#h;0HBj|Xg|@(u#tzPe}xR1Eu8Pah^zzQPLnbT_hygy1^z&3Qyil?;SW1-Id* zTvWVFyTG>%N1(zNKD%lo$vvL!Bw2B#SsS6r5*zX3JWm?JQiFRtH?tVx$HVm06>K<2 z3FSms>P#<3`zrW})tqL*hmaMGMQd$=V$9_u(aBr6o5ShyukPz0Nxc&M=)_=@k(sVd zjmYML!d^}4t%A@pHv;4}FD8W0ZqtSU_%t({Nm?p<2Et+rQ&cWG;k+N|NdwrttESuy zv2dXR9XzSX4C`^a2)dl>n8UEOs~jw!+XDQtvA!#&b!^dPEBp&E`{4^z#3>^KCc)zE~x$>ZXLV>ohPGE^{n=Q)@u$6vFx)AlaQqH{z zTbW(ijQc_bHzl9pgvPp<{)DQa@|SKU&P;c8J;wngCl*lpNbFvST9(GCT4vJgDFrM*amxJm3WaB zSwp1Y%Y=MT%$gv7+s&kF=lR4_2-Qr~1O3rE+y>GyKgdU#+3_z?D0PvtvR*3%p)NrfHd(JW7@B9BVkRh03-dWgn3 zs(Xhsk9QN>4t7xFYYT_O2$%Pb>_R9489!iMD7Kk_AY~w0j1?U7rpj}~Y z0^oB-%hP-2d^nuLNpwX1Amy|l2_RCp$dFqYPy6+_Fmfkwc2TPX{*yu;V>z>3rmixA z2vA#PIDb(!?d1p)T4ZS@ur0T$Y+B8mlOwocUk!hIyB{`gOkN3CThLCfW5Y?UpOuC< z*iZ=tMX$?1RZt8nx5?OT6KoLp;yW|?>5fep!FpN|aVo=h*F?1V3bsv^ytPUzGZL8MCQ1bMFjtoiO+ zq}0b%kPUF&TGH@Cm}MK?(nx+%0dwEQ=4f*h9jKVramvMB2Y+2^4+o)*l(@!QY9?|ZEEZ>{(d1Z zh@Uy#Bu#%aoSbD`6@0FR4UxH}_23oxI`<~oruPrOP=2`#S5YPx%QM5WS6TU6M)|t9 z)ZYbKP0Ljen&B2j%~e>)K)KP>>_Uw#YJ}6DsLDIv0-E!OGnCWXVFp>_+lV8?8C+=u z0-n%wESLmE;V5X%DYAVXTza3nx^Jmxje|kit8*@K)Z7}66o0f0NxegN`W0!lM+^iT zNtt$LyY^+QA6oRzV*w^*D)9b)5nfIz@TMuImAU*et#4)#Jup-&xVry9e3f-mnBuQd zvMP(Ce_OY4E0$uBwb!^Wxi};t2$ZuX!SmwB0tl@$6l`6IHDRfR3+wa|#0}M4u%DXo z18$R_w!*)rs5qDJb-SCOz?^Y1_5eoUhDcVtPd9~g{H_Ml`PT$h8^E_22H~oVl*?+P ztk5RldtrofKC5iKph=;k14WZ;_iyi;<7MqX$#(ggCT(0qWW6bs=D>F=s!B)oX2OI- zkJDxL5;!nU<;HSEIZ2-DHXq_7D^Y8zYM2)4S_gU34Ro)O=TiYj)zBeQ!+0W<@?CY} zx@+D4NA#{8tjR{#wL2bF``Oe1$_s5%)^%f8mA{8v@0FlQaxHTao|e*S+OLUea9OUc z<}}A-rGjGwSfAa18;qR2GU_e!RU#W4lkxRi-j;R^~6f{H&rmNrKRWDl{2dqs#b1GY0)K_o_$?` z&XfA)G_4?3X+PFz4AP_{((SrI$qmYq@Mg(87!KhwI}@pMM>ZEX%xSIJe$p$vKu*eJ zy>P88qc9nqFKemNj5o?1RWlAZWhJ&q&P*fXzA_~U-&w5W%=fx$6v~rt zB)|F^iBDCT9-3E({uom35?d3xp#-tG+-;CLxk`~P9VX4*!1}qL1ITKnWJz=~z-q0h zE!1p@`LMbC4U+hILL4V&c~iRGM3$ephlW&3x3njEroaqs60YLDg_}Kv!tJmHs(63j z;%W&DDZ3fKN@^v)_h*4GsH^xHFQShX6Jjifw_xda(sTQZ_FYB^Gv-c~U*{WX@_h|R zp^+Zf89!1hVtDj<}7*M6{pR)yev>QS9$zR$pjac zzQPTWB^=i7sH6>xb)XNWzby-B$D;J1`dX!!0IbE=NA?_q{zeNr++#iDLnIbBG+RVI z+&Mi_`TXlch{O_@^;DN+{*&NF8_~kB^R)TEmw=LwbnP_1asl)*l~E8XzD}T>)x;%! z;Sd?O6rFmMp626g+*me}Z^*^BFfDQDWy`}IhVt(dszg{_b?uE8?=rb;OTc$}iCSNU(^Orn! zv?)h4FW^R8(-x@Tp}7W}DQ>HA0iIiu=)6<_6VYxNKTYke?U_v@|=l3+y_-a{o@M)&r?z6eIh zoy))ozmkD^7yqJ)C#Ws=1{ecXsH4&ryfCrvW^0R#L#rg2gx4jmbJ1~Rj)Z) zKDh{b%yvoRG>!{Jydr!#yPZ-O9a8f$4`0=tA#HYSP2^l5B8l6I2;Jrq^F!DmMdYG~4wjg1o`Cis zGaHze)K_=PF<9#HM~M6}nPov|=zdwIkpx4??}G{#T%vuRh?n3O!vTv)k7YtzTO_e_ zJaUy(urqq-I8x4G;Ul$~%=wo!4Wc87^gZvP&qYNvol|;AqEwg)_xnT7+K$5y8+pQ* z1UlZVgL9}U9@c5I;J6KEBsY*0?sz!`i_gfKe1oWTL(-(t% zzCLIRB!L{zg8|l(TLr^3ZWkQQQc0=qbL7GtY^_gz$jD;z&M}hvJrd*4a*)ZRTPDkI zr(td}dSMIgv&>RXJ9iSn7XEG^Nm4Zw3I~Q&QKa6!J}TIkK9hFFm7r`eSms#W;>J`g znoF}R2f9lBQpFZt(mspu%8Q#bN@d&k9tveD9VAIFtuN0q!H@xZd=Iu|v&q@N;R@d< z`Eo^fcjDqQmA0*4q5OWRmw362FEY!r*RlIser7o39TnbQUN_ZKd3Q}g%%X4^K~ZGc zKoMi4R7oAYckotS)k7dB)xdq0t}BgToXnX3^_@ob5OW0-lUzOU7Q~poTJnDO_%KVn ztzdXr3&ry>UomM00@!T3+=yg-w^b4|ZnML9153^DfWl#TrQI|MQ012Y>kTK#Rd#Yi z-7*7Rx@5n)-X0)NDxka1{ud2&TM!}k(i(Wu_r8pf__rECX`uRC-C4!LPwS9m46(sAf|}d9`${KikUT8p~jqA^4;5T0;>|?-6FiE^4HDf7qC*?0Nptsf_<99 zCzA$2927cU`Fny1y4EHEYkM860cQ_*c0josb0b-v?QPlr&eamxnFEIZOn|O>Hv~>6 z)t8ZT4UH14h8^{-@CYh={zi%Y4~+jfFHK#1w}bVd!hNB~`Lx&avOvT1O{%tEw3b(l zZ~Yxec9kAdx3{iZ>dv+UC~Mjy;U(<)S#7kfi+V|Sc1;A_Hg%#IhC`uOlxP6pK%R z^wT5IE&U5*>3o9<)r~#z3Xju)LeLKCP%xqTGt`o=l@FwhN?49RV-(4Yt+3fHDr065Y=jA$R7ETqhD!BH zJ-`eUmRa-$gOj*$4(W|9vs8B`z=cXWP)N@WpT==`SP!(~JnRdls4jP0bYa#!4OR}TA784|JqVul%(fe#6w_l!MmKT zC2|Tv&v_joF-N!8B2tGnF4hz3<0c^ZX8jYE(vJ*PWMEb`PCh?X;+h(Ed0w30m1xz8 z=t*ZFnyuz29?h}P*QA#=Db_1dHy}M+9*5=A;~p1WFy!X_kP|kLJx>N!#L6*>?P44U zJanZQiiJmYBZB!GW58O2P^ix&m(HSVR@DP-U<%&JcpX#acpR$hXJr%UW~7G}O69Ib z9Ks;Kmew|fV9{6`mUOTwXW~16{cfKtn|^wb#l2B}6M3n<$A-d`7RAn!=F0E2S;8;6 zw+eFCvP3SeW%XRiTja~eMw|@oWp?X-hY2W_v`HoE7gR4Y{RgfvOtNa3kq*0c)khg{4v}uE7{-E6VOX`U8&n0#zil61(21zUFMbqVv@gDj@>Grk~deXmIcAKj()y5eu8&a)Orq~KN~PC0nCmJ&W0!78SM>+H(t1I%TdsA-SujBNCCQbi2`U8RHuBIMz( z9dKbw0Fo=O?xv#aVH@QwdHqe9Eu9{~4bySa=BhwE;{Y_GMVJ|I9 z=mO|hsTZSE$zPki)Q!_bW9}x4s&n*cFYUQQ!r}Vr2keL1!Djj81Xy{022lSw(+=&& z4M!kb2jTO{&eY;l9H`hybCfna4GzXN_%0>z?x@uD(l;T4drO)5z8=?A)&+s)UfoH3 zD^>1^Ce~Kv7G8)OF0fH5u0R*Mp!HeQ*=h#R*2~C8 zEE11uOLC{ca7pSJbDTCM0x6E+rgLSA_JXUsBd%GM&RL@ENVH*mTV9dzZ8J(CDa!=u zeFV5(R^i!gQcT*|loFLoU5PtUd>rk5g&029^9RW0-Jr_Cny`{HT!SGQ;^~V%i(M9~ z!XKJmkdCAykMmdE;ztj3>vC3r#$>&uE$YL9+H!O;a9yVtt(PxUn$2uI^}Kf7Q&+V2 zAm6ebkSBA`B~$ow3Mjpr9X!>8CjcRc_nXR>=%IWcPrfy)D#~CEFn3BGdAlDpPY1mQR3k1F9r=njRR8 zDemen1@rJ55z9+qLE zpLCNKLU6mJ!n=SWO10IzHV~e$PtR^8F~dN}^)V$VI5~6B`fL<^&olfgAhiuR0y-!4 zH43q{2pBx3I_@pezWAsRGB zshs%}GOA~h;g^*qC=b-siWlEDw0D|n*m>8Esq_sn0N5H&wNmE2BxM1Mvjn74kQ#zN*SDX-nbqmRKp>juKX>3CgQqb+nDu11d5I zHL#IxsUqW}&AD3Llp@ruf~GQ{E^OHT1QB!#{siE?6_wEuCSPvG5b3%FW}q$R2K?=> zl0z^E>#J11g6b=%h|#bjU=f+lBh8M0Cmo}KfF-7YbI92Y{v{2D6`AHswOa7Ji+b*U zym77#Dx{{ryh~{{bXj*J#m2f}lqNNYIb)Umr{X=YDp}Q2FFeP3@D>r<6(oj->yiEH zK02_)948%bnIX(SK~$8arg)XSH(&+s?r_P14DDHLdZ zmOhC^IO7b5%yaVm{i5praITXGRX?Z4tpWCqM)Lfrk;Gx;tE@<04RK6`2?q3{qg*HD zjOG;5BV~F+4CeKEjt!FVH%7!$4|CW|JSX-OFq8wuHW13TPvR?FrFvbep>pU_rvqZT z?*0(PSJAzLQQL@{b}4uJa9RQ|2qAzQbms>UVNRmdUV`*^7uPOs5`TAS+&xffib-G! zHpDerO+m4x^qPx-kA3$}LY&x?1TgfXE=Z%)*790kBMap@hthc}TXBK%Xg=Bp)qH;n zoA2v|Z1RtYIC=6t)U@bh%WIEW?;2Ie z;j5|8$W~o*BIr{4qALXlXjjJOM@Zj&@MZ!>`%8^KQBWr6)@-i!EKTIo5K`!m0Ir^2^62V_<4wjZ3IDfV zFdJGS{WX7lf&)9gq=w9PVBp_r09t+Ug^?wa#!e_Y!{0lCDXORfb*0?1cT7i}Kiky1zD5SQ0OuFa~5DoGGQn9YhovG6O*fZ zCnBiu-*E-*liZc?rL6Cn_q$s#C+k_Zptri?nFj2@$kABQNgX**4pu@mZg+x+DUh_{2In!1qFCw>987cy0@Wx zG%`6y^0FX|_t&8i?i%RH2d+t{nMzqTAU4bS-%o_Lxf$ zU6eVfqVrEEKVxG&j-lnq{&*`oEcN7n%t7Zon`ngObw9$j&U8>~PaL%(w`(>tU%Hozy&Bb23Gf)aqtCNqCBL#0Ekuz0w-ykVkospddd%%o^Lbx zp+uFqW?L^su?g82vsxTsPGEUw+|{yXD@4o9mx^o6{SxLFHxqycqQFX%s^)_CR&-N4 zd101e0tM1LvOiZ{h&c42JQ)J#y84Gso3 zf4aPzj7aFKxqo7jO?s(RCiioj(aqmt;n8q}7u3jXgQiy%}%NKVy$< zOZF7W(n^L~hK@!Mtg$DkGI74GsAS%~xI4VnL7-48aHqTsBjeSv4c;_zFgCm>#--D~a^n{{7*f zXZ4(EaCkDR4*zGi5?GRTUn#T=Q1^V1%v)NUvo07zKgO&1@^Lx1!HNyZ8d#a%Kn|K#?uoB9)fS`7GjY76qI9pmzH`{PX*p%Mr9x9v9C7Zye;8Lmi0X&Sf zoH$wL#%OO@4w`c-s7~I-X5Unii6&Jci;UNNd3pnJwbsldTp`w!5OoQmX00u*zLId+ zaRY6zZbNg9*z&>R$ycCDA|jik=GrDA+fql8{$YuLfl}&*+{CWjSU9BGLytnID zwhToqS#P5&bKO>VKmMaVUqU0j^gyJ%IhKW{=~_35ni65tyx`$~ROpZs$r!XMy-c#Y~lndUc9D zV7yh*{!Veq;1$eun6?LEJ_(lprH7*B+CvDiL{Faz^a~F|+65J{4-3umUlRn}wO26* zj>VS1HLaB;Fdw~UoT~2|t~?9T-3?;Wb!SZ=o4XJkb`8W^;JuE2KGzQA`J!MP1;Z5g zsSmB?caXwGOxu{-G-$hExBS@K6F_q*35PeG_?vv_BU&G4?RjSgHfm|O?ClFA|@LZ^n;4i zy(;}wC|2-OBVQV~L5lRyOD4kg31XOk)T2TXgBDDD@<_7xL88-CJFJ8z4R(lcQ#Ds zGgX1`pl&;W{C2j7bb}q?)xM!H3TkLI(1T<%T>#badSv zdR!|Ql>ZnmOZ!Na8v0IT{%hpQuW=qqnQli|{H444OUItT!A#g4T*|UqjAP){Bt6gq zVW<)@fBXmbI=%V4n>fEr0ykg2lO{R`LS&I?kd74Od^|QxRo*3NY10# zbMt44lDXB&w-gA(RgoYt`wZ;eq{a#s2Qe&5+sd_Om`wH|p#68cD@vTB5O&eUg`kfk zQm!{N2m@9A%ionTO{irj5Xm(IGImAFoo{dey&1{-p9Db*l*2~Ufyp|c`BUPwU71+` zDs_-Nsl@e8w_ON>|Ma*4=w93rAJCmH@XM*QDVGUmn%_+ZZS_#4Lk}ivO%pBR><39b zAfQB!3`b=csmHD7!_))F_$sT^rLNFG;AMGq&0L@YSE7Mukq>+_)nb`B4d4=idJ_27 zaBya!vof^+cTv&nLz)~RFl3P z#a=Ah)Yqi!%Nc$-Pw1+I>N;N=V{hdsE5X?AOd=oCVK2(lz*z$uvj&{NNRx#$yXl+Rv3%q$Zv_Pwh)1Ry5b|yWOL`^dMa5BQKYt%7gXQ-Uq>-b+(AI}zp9jHbn6VxC})mr zdMQEm@JpI4e$~aurDEQOs-m8^^ulX!bzdvVn9Pwai8fTYVeS-}Uz3A+uIFsW?kP-w zPY^~%aoXil@DUnFo!<=~76x}@9rVx$RkzrUrtQ+%L18_`k9tV^;h_0EGC~nYse0&_Wrq^mOPXG~bJv zsomwNDjQ!-#h#qeOG8?xOq=ZLTAg?2s9^0qdiHxDL0mlE6-<(edg(zq*4R)f%GTES z18uXl<0+Kx($h95Bk(LIQ8ZpV8sn7?w>hsPX&541b(?`yd zttqZzcD#8x$9hpuJ4)O{;9J;3ejO|i9?2scoy(TgeIOTYoZ1CpD5LdI4f%-+0r~oS z^x+#{CHQ&6>xRSIKLG&RL6kUa6|H@y3$=v6Yc6ZT&=CRyr5QnyQoSK?`if%dAiYLG zk|!RM?5~Tf_WL?8?mty6-nUgL-HEEV#La=A?n`?7BD67YUv}rD)>kS3r1;Rc__Ycu z%cQpE046f`Y3trK`|Y&TH7z@JK-2?119foU!q9mh9hNg9fBDT#jf+*6?rpO4tigl$5Zwr-i z^0iJ$i9hS)#A=-wg4SlbbSg;57t0d#mw~Mz+@>^i?Hpddtfv0Exh3L%|F5{rjwMjW z)U9EKG8HOnOHwM8H_7!gW~-^i-(lq~aha~$=8i}VEtq% zZ!WnAHns)e!TTc8sc&^~NXs>OIQakD8P%KT=+5%rYLgWAy}q zTd1_d{MXm>uL&E>C9=8!BZPP5tM-`Z9{vofRX$@jnMxZ~E*~eeRB@hh+YBmKKEte> zuwA89oIYqyldq4ll2sd->(AxF%g*B* z&i;$6dU*+7E58c6tW(5UMP1-yxCUVBu46%$Ehvf0V^XEhN?u)Cjx8kLHe3==t$^!4 zg~}&B0{#abjPAL^t7pV8$@gnnz<>|oWDj)EA@(|x_3z9Z&A#P_zoS{%3e_cZaU*_c zlnyIQOa&MpH)4Qsiy1FXC5L@co~`v&@$lUq^23xE{`L#?31ruEHrZ-E;`E7)t(BwQ zv3%*_;ZA=rN7sowQqj+b{!?A+i)sO{y}vW))7$*eO;z1bp=E&#{mTf-+$Ze*$1dz> zxCc;O>%tE`cJQyeL-<%VpV!9jXKDQxaiZTmW8SY-y6K3`hFw0qz$3l9j5l!!%V^$! zLog))Qd57Xd}SHnbFSezZ&^SxL{*3FK7&W@Uf^Htd5}VIF08DEP(iX!(1TGyD{ah zU2OVtT^>owXZXth@=>FP;gAJ?aPY@xu=J?rV+t*O2~9fe*zd&)uL)>J2tI={TTz)`|lhejKbd zXR@{VD_P#)OH6yICWo)7yxOnU@;6^gb}$nrBR?HrfX1lxveYAZSC+IQJx zPXYM{)|vHSoFNK*QWNjA*}a8asy?9}S)#cuGCSj!8+ma@~B zzxeeG#UIG!$F26AsxWpFghNTcbtb|Eufe#_t_&#vIHwe|RzJ&+lXsvZ)@FY;{hKBjdKLeZR= znJ;yJOGuS#>w^v2Ga+D%x6@%ldB7O>DgR1Huw=JwP7Y-M^9#% zN7C8x%sN2kJKAYT#(ehK_&so7**Si#cmpLPgLXsO?Bbv_Z^A;NA{peYV&QrVKH8|F z(wF*iDrjkt+4r&fsf?OUm>% zXn3beY_*@jLNk>h{I@>fm^01@%DI*qh4RT+0MkMhNj{^R*?*-lr&}_!ty2j<>&8V` zG$jT8Qr#A*7#yx`FQ!wupN;MKfEBms;X&`Y$x-(H3)F8?ADF&FRbT1RkAs;t3!FRm zkZsTE#khB_8C4|F3bFWYhD942V`wku70A1HpdnK{Z`S+F z0KTLTt1kZwuSRTTZ!ge_RlZf3kNfj9`ANOdppcc!`@&Nytrg4f=QgaEVMGm2`5tdV zQG{QN^#IJ>?QH7;#?6ira=m`o{40$JFGI(tj03PGohjs@e(-j^i#Hl9(n>H*qqN zH5fxCm`l}_Y?urm4UpdR0Ku9xW{_h_V|b2Oia;NDsSaP-n3(;CZ1#J%KPTS#W0pGa z6|lyaXxA#`<;tWJ;NZy=pZwx5N@b(E!))=87rT`*-Y?4gyd=_~@afNN?;%K|EsZIV z%SkMyxECKkUO~W3rH-HJNg%v@J(>PK1pj`lCZV3Ze`_@0X?_xDEq{xN9q$_!nb_6f z=B1D&{|Q=3(I!y+E!8h_#CZ>KSi+#wdf|LX#T&SK4F~z|#~w{uzRDI3mtmxiCe~C^ z1$wD>v-~NOHtJ1Q7WEV3eU`(!WnSm!!;e@+WQ1RPC(kbLtFno4ZvaZ~7#_GkkPV+o zu}XXng17x~*1!BuBP8LKU==fGacqNsWQgsLL8;C3%aL~LS?2SH<~XfaUba-r1wzH; z*xcEVS@f3YW~w&FX36TsywtHBr#9s*r#`YFE4Uj$nnE~#uil+KPuLDhwOhzbNj4r^ zHi^afRrMOuyDEwi17+Rn!^RhoP9{e~%&sH=Htg0=Rgn zIUf_|va5kA_Ka!(;WAKV;g{}X+$){0axY(rv!8GTxPAR0l=hga77|7MU)eU2k1aLX z(AIPwiP-D$$*mN=Mm6PMgeB%ktP;`dV|i}oLiY7f57@)!=K<+gl$4cQN-5ZIiyuBt zV|R;H2xho3*`$>(WmR=k+0SS9Ck1B^=H=uW9J zvznf5mWKT}oVF5U<8TF`4lCOu?;%EQ^g71b;u>`g$GHl+OT zf>Va1>th#Ki?2OD4=oGptfa~oy>Q+rlx^yVeME|LvqoM)r40s(k((nmM?<5ls9hx9 z-$9`jiB8-Wfe2|@*ApjwhXbYbcYw&ESg5G_1vcI^4={(%WnOb#jw9W}BNcyV_m3L# zH)Lvw%;{!;)_*c^jYaHSU&plH=xLF1J`aG8CkIbz7@&b!K=myuhJV{b40Pfis!5xz zi~`=|a#>1=l$kL`oLnygBpy}nre#%r?X4tRaaaCzq=hFS>+iFT3M=^fWetvU=zjpw zHv^24VwZfVq-FgQmRD&CGrsYZA7`~@Lv#DGjegS@IK2=8J)~;F_nd0D@yGir$=2tP zbPvTM=ae&0{Jcz2e*udRxcHZU6w9lUmnQ#28#FEWow@evB^LaD`B7S@M0i7UkUy)B z;gHvM!oGX^fdcH(C0{f_YJ8}~lJ~jDk>39@@{$vrgLf~d{6!1Cw#sC&Yt(tg)iMHd z@~JsrA6tVEV`^i|V`TunQj?uf^P39sDqCnkv6EET)NpC^0ch}>UQ|Q=d5?n{{3TOI z7c%6%YG8bHUnD>kbqmquO}>s(N@?xWa0*9Ww!CwW`HrtNieyX3Z~+rjgtV~n>X3I> zX2pBJX8$8w5K0D8NU;W+bPB-?UEVig0h&Jst%a@?f}UrL<@6;K||Jg zO;HAyQnOZ~sT3-XzIptEZWypd75|Q7)r=(viPyDbD6bkbBL|bC`QxEk2&2dPu(_cJgu#18+97 zyBp~YoMkf7q_zsqiY$(oTCjV0Et}O%{=qV3@ z-_gv*Use^VB#O`_e=X$wal|Ugo)|N(%-A@tYLcp}zzVYr zZ33G0x3jGAWRl6#*$}GdR6>$OyaInAm%cI%UvCY(c-D$EZimReQbtwSi+tQ{0;eGF zCQ|mi%){hHSCU2+0CqVMr2eLp;gpd+gCaM7q0jDAW$%|Mr^3_43gs`u>Q?XOYbWI) z>DY!7#UW)Cn#ZAJy$3u;{Rr6pz6NfDbp$k9K4s)oI<3pNPYs*aIyM&%KKDH@z6x*p zyaPMfn1@EPS>aN{n4tCpOMBdk$Y0g{{tL?EwR9Q~yBooC6;&A9 z*!lmN;E z-#}^ix`XBO1ayT# z*b~+Nqovg$UN|13Zi(&Da&Zz6$x1LM$jd3LclSbon@>XlnQ#vnUf%}jeX|gGWn)U^ z(kjUJatTlwQy1|6rRP+Up2-mWyj?tY@f7IWNq50$pKyLZsi=wZ4n;V3IH83}~6TqIC zs=T03#AtqdoaZq~co|gVqc!;&6U%X3iOH6Z#+#q{|I@#xdv0LYRdFpGY9Z0N<) z(7*KsJ_aT5)jNwv-dzTIC7go=enTdre7N<02;r|GPn#ghzI(@T%FdNM_xm%ZnQ7r` z8)XCyJIJ6NeC8wq3n)Hvr7PS0@D#Ise95CpyWU~e0l2?I9tx^gQH1Qf z#McBmA*qrnHo5nl^ZE5VI7G`u9;-x$j3jwAV5HGRDPA5@Z&p4l1qG^5%SNUi1v*=( zUM?Qh4t=fetu1FavHtridT_d`fKg9nGZG|%DNunUeM`4Dl2j`8aoI~lqY=ulH17of zKzvCP;v{n-i}#td2bwY`_QNe}6vwGvX;$f5f!#0Y#peF{018-HoF;$N;(+E><)}Vr zWME2lxk?JBncFuCPe#bAUxK3>Gt_gJN(b5aD`I=89s+f$^YxcM0CSDc0M(L#03s~U zFMZzRFfvl&WkqA)^Wz&(+|o>|s+t)lQ6qT#-$v}CTsh{OyW3MDm6rhb^OfLXOU?j_ z)ZSorRoeB);|-j}K=_*WLabZ5Z{Ud+hcntj0^jB7B?g}y!^!-bM$X7P7{pwWh3vEx zgha0BzKY!7QPna~2z$4r8ZM$=@*7Eo5r&0m0IDMmBgkJ`T?VcXHgYO2QoD!?ZJSAI z>R97N~$LZX0ocSjS+8q^t5JD;T<^5 ziMFhONglhi;N(9eM6`)u^-2RivW+kd4>RU<)UkA#XZ|Sj*y4##{U; zC-{_Dd1;;lp1uAQj2krpwh~yu!|A($VctCsY2ijtY>jF)IwdBc5+U5OO<{a%Vm=5? zFO8NL-{<|A`HWddC1SP9gGBWf;3+l3nrkCs)1;Ha`nENJKx#^<+*S^+24nf?evM-& zqvzIQsc z+8kl0iU7~8T(#;hM|++bFLl+W#QD_#>$#FXjnxI`CcPw#U_Y&%lu^+{8_n%a+VRr} z!=b9tXFJt2?NlYUIC&kU|C^p3CS8_-1NU1gjPA}B2X8Y>#Im@;DPI^bXW!tYKUPtU zS7xIYj#Bwhk*fac{CmK;TRH-U`gU;rogDI~>7dUiM5#+wOY;XZypG2uHD~g}L!y|8 z=W|vfl+Y3>&!$5Rgws_Q(^6)X<`B~Bivf8#)!lDzb&eI^TQ;9*@ zod>!sDhD3DF@qrwbBj z8`BlMZuuNWQ7a`)emlt^+k^b6?GRr0;};`dX8y~+mYy?QvNXf6t6MI&oV~&hnhfMT z=EWuvp_kJ_8b2~Z(lZAsSDt7bMMK{AB40^KUGRGXYjKU1A9M&{~N7eNTb5ATdtv{LCN~x_C_T&jdsL3woIFEhmxp4ToT=2D1L# znG+tjh?l}Q?(PKh_2V4Dl9S-}BQr00er7SId%c+mDc&eqW$uyj`4_KLy*cemO+yZ;mm*AqCe zZZQs-ZuU5(QyU{7^$&6IZ99WFSMrfq3slAV=c?rMq66@}J~^D;dSxD7%X7-Cw;{m+ zI)hXAs+CxpLuuK<9fh+Q8YR(t^NvCgo4YgT8$q^U8PD58Pk}dSx1gDZG{n-jN5tWD zrEKEI#N|p%KOVcOB3ag$kg_9``P=0#U$-x3r=wIuiCZyo+SZhKxl44r98l|b;J#oDeT~`nzpX2kD>d(zN zvbu_)N`$!@%gqm1!=sIie(9>GK-Se@R~?f%VVCZVR@#so0h&`$MYZS166IJj)ZNS5 zm?bJ$r<&ZFV9qB_jda4XP9qz7ENCW2s-b0nlbA{r6T)%lVNV+8<8Lb))Q!hivw=(Q z0fy^*1tv?1OC{N|A3Sl8qAq#{qd)i$o{&@3$Pv?F*0X3L;5Z#oguo1hBcgqV3_S=@ z_*IDcX7hX@43QAhutc`Eq`bMN#B%?j6(n+T?_=C|#u^T7O$1Q~C&9N#?|M?TrL(SH}eoDY=lx zcvbN8Fr<$iL@kz4ovJ&`GO&%|BwEeZTB4OxH-~^dvH!&h{I`OyJ-8*7pW4B}sm5GM zJ0FuIE7!7gO(_=D@>%t3r(sVq+c^4lL_tcRxjD#zXf-6y;-T$tG<)A&mPK8O$c3HQ z+Q^}8o{&tqWKKf4$hDy<`Z~!7SP0@O=#Z)BJr0$Ymap0Kg4*?zBdvOdTM1t6R3_b_)n(_276#cipYW=k&PXzDfkBK~1GrYwHO^|L zp41F>KntK+1yD=T4{Y{+Q)FL)jWhjnJgaSVmQ$OI&p|#)kMNQIlqZ!`QE?66xM$UJizBDk3D+ZrJ3<5*AYDZJzj6 zHD-RdJqpbyib-`MsuurNsD*TY3)X!d$H zw2l{Uz^5850<+ht`$uA6o2$|hjfs;^Ys2-GuNLB{ppy)kZlp@&e?Z}h8SMDSl@XLx zOF@>^`JBOeRW$2C1bMC7IscPZp4fXAoL-?Wty*MVrEQ|oWqMNp@j6?~jqANrf1 ziuE^BHR1;*dhu!pnuz@)z*;{>llH@)nR&Xui`?pM1XX!p6A?I$iqh2#c3k}wNY*=P zxUuBz5d_}yT)9C8wfAZ>?oWA|O#Rw$$-omp@{|hl`F${3tl!xv6x$RC??{#hpK^a7 z8Kw-zdIQ1HH@9+pshGpK_N)$qYvaQuZUw*2BM_6hQ^Ta<0*^^tlvXx<9)$7W z7k>RsRbl_4A$0eqGTiAm$&J}gQ0JXr0mVSdo#DmFc1U<-#v7tSnF>FKJwAywiEjyS z%~XZH-%>ikCuLxLUp6*O*w5+B@Z6PjmL4j8;E(pnCh32Xw|>-z7^I04 z_|!wDAq42$v_Wwv;snyJ${C>4D)K$X`!*_LJBM$6Nl#}vv}Ydkih zd1;+UI*l%_UPh@r3L-GxS_GipQZBMdBjKW-F9b5l5hjUQ0oCiOTi0S$RlAHzyFIN6 zvy<1!{dNVQcy20;>~S8&n_o2S0EdJ-TD#cAc^Pv2Z6Z6g)io^AY%7elNL^-C8DV&F z^7vaw^htwTiYXqq)V&D(Y|t&eB~@*h$RJ+sNA~K7DUe$Kvf!P1mb$tlCE$54B*CPyh9BYy#Yo$( zP}{GS6MV3Ppz54d*jS!72IU7m^%Rj}m!vQwt1SW8&mJWXyn(4hz(lX z)R4Fjf%rFXgOC+f$LkLX&iemI-1R_~o9lF&{G;w&m#Rq7X>_B)8${Oc_j+!)OjdS5 zjaC5Jq|zN<2$*vrvVw}T;8k$;RlT5=M78n2u9WuIw;w<*r@~Nvjszhxa`{+343ao` z-xIGgAZp9xvaG7SO4@Sjwt6!BV}lW#ZS=SiX(jDYU1%r zoRbr;tb!(DS{NF28nNmQeaQsx>G@Tmv7Fjmji*LRgF68B%~yaE1$?@3=bheNk{3Yq zQIjG1TFnjB0xqYf)J0p8#P2;8si=c<`^S5zfd3AgyRN5okgq4g+_s%ZYVD;kzZ|Us zul@Ej2H4gXJno||0bW$q8V(UxE-R_oDzO_80Zq@c7k^5&colxSko2H%5(UJkLMbbB zYdx@xJNR3=yNe8Pfd-D+Mx6Y%0UX?g(vMN)?kb($V^pF- z5$W=ugVUMVF9Df)o%cWJ$$8#Z#quv-_7sS&)PMnc66&ZP*HqeIjmqh!V2^bvhyVAt z9QmjS)e6koOpdl<+kKQr_e4aA{7mT|61`9_Hz;X%LjkOo3`Z(I}vCK~XBjpJ2sRkx|rQQG|-C?jTm{!lI(@_xAnA z4?>$f&wZbBo$Ks!x$vo(A&+)LZ`h(UT+CGnW^0R?E1h=p3za?4lpiUR0B;mH30)a7YBxCeydFNy#G(4ApwUF0WpSGP`YJE5UAfbY_BpOP zIm2(I$R97>#gdA$DCDV5~5aK4{}kU2V62? zYzmDz^8P!3*Z0d1i4r{w4=}Hmv=33m{7=yC=Dm>1S(qT{_A#IScp*&_OOWtim8Tv2 z$TmPe?#TDgIc@r6z=x)Vzhh6{t0NHD&Fwk#pWZ?SXY}DemO5zRJ{`2``ywx|2ZPET zpYWe6Q-I&R9IW<&o@jb}lC>D`($S9~ocq7E0xs$C3R1MT0cY-HU?<2M?SNC7-Ab)i^Umz)Mu1wXA0trri7zKfk5}mvFY{*;4k-n-!3-3X;dixzBgmH)jke%$7?N z%xX#xNbl_TIw;LzliCesZw1qJq{-YLc$ij$pn7F8#ljYmip!OH+lV-E1 zX-7JYfs)#td%5E~Z(JV#h!;H43ZU+)x$668Fq|{0EOKBlcfCBH|8yJ#F3<;9SB7pt z;Tf0IS)TnCv1O1*LpjG(PMN49^#^wVMC*nl9dj>mBv+O|x_5ob^N&@^B=s20cd@%7XrSI!+NMkFfPALH?tXrRNPS}_CpHN^&H|2%5BQ6Yxp^v z4q(4^gdgs+8EjlqsBgTqxiPoOCBvUVls{+NrF?lSgf^W%ZaKb*XK186?MajskCn?1 zY}wPmw9CD^k?Q!DC~VLBIPt+zG~o5Bs8qeh!#z5|tJ6(IZLkIf!|7qQ3^O8y1;pJm zY-DeWIzYD>r}wd{tgi>=IRQ(fIn;yBU(}eF3{AkC{{;PDgJ#={~iWwFRC@&f%D_E%BPok?oFR~oeVQ@jFbVj;m(-@_{y`FxVJUfJ&-ZpT7I8|%$7XK z0sgi!s5j~)hRDPeu3?f+&_5FWkD`X#q8sF>%9Nvl^!zB#4V}U%8QO|>qZ_ji4BFll zW9lCxvySr$8*sHnWm?NIrc7aBCtox9qKWB|y>n5MrMim2%hESEh_fDHQBueUP9u!$ zG@*1%6=cJ%9POlcT|R)lgGs!ReVjPrW=v+aH{D#Fi-lYh`0|*lv@kZNTDQqXBzHC@ z;orx3abqiormlM{30+h=^em{BKzfXp^yC_u@~qb*wN`TI?e2jCl&G4nGUH}mbc!nJ zYP@Tuc6@pjlAH1~j`bTV>LgZ>(NChcdP$MKjuMR4(}SIj%>Rl#CmX)3B76gIvRhS% zSb)79GEkf;M`rS_<_nzcg;5C8Qr*8#)@y>y|BZ0?dw*f_R}8V1=*d_*OX}#tVTC%% z@bf(Xm@U|V^W(jCvZ=QI@)wdX1v&#v>$YB*-Y`lDP-a`e!1XYh$}SeHghTwU5&PO=pT$)2ad@AVuShnnJhyums&@>dio0 z=0JNnHQ&@BJ9`7EHyeA%ELFR_aoRi&efBANbqF-)$*Kx?n=cznk?0{$&OthvjyD*h z7HA-&&Lgp_dUJw+>0o*0D|EMoURRj47deRi#=BH1Uy>}-qu^NXGZEWTIf`l=_Qidx zO1;uo4@aO`wlUTAcy`4)aOk8E&e)g}Mi-fJ_~jn2Tk22cxYB+?upb!dH5e*q=^%eU zkEYp`XR?lSPQ#+c(G`8G<~I`eBKlcilx(B$^fScgni z(_fMYddXVO=l&+^!kdlSql~N2WQkqmY#4aq5GQFi1!K~ z^ItthdRzmgwtQ~dX*{N&E4N^I`;i z8qVafajeYAI0~fjxDC6^7LDQ31brvGYx79{3c&Aq8dFp%O=jym#-*Yp9VMT2n;IE{$P)uY_lYcq{7 z5mO#5b7ZStaXv>6`sOgCQRQ0X$!D=xdCZ%Vxz`VBrL)bJuMco-tC&G9 zyO__+6@}cepo->F!KVWcLAUsNaf(l{z6kyly8MuOoqvH35zrpTW3IstefezugKQ)b-Xa z@mi(7i)+3`6%Upp$$Vo@Z62Dqe46aJ1KEEjIl^Rs>Ll4n&k9#r(}cW;GlkcxZm37_ zQo`7TkNc{MA$YnYbBy0~0j!Z`9E&}PxCdjN-@hNe(Q+hja;ZDwyRA8g@MOZo)(7(7 z`rEnPwkj}K<{UL?8y(KGzp6@R8C%oKv1crEn*T4(TOubcC7_k@a5|a7@fe6a6={jT zl1rt)YOdSR%QTaGylMee0hBQ3_L7^=^AEOKx%=L5fAjwn2>d~*oWmXP*8=#-a#t+=`NP%u>b#G^aQN?RPhlluNwa%}aoqo!xli-FO$mR0}$SRBV)dR*GBAu9#8A zpS*0M%4>twWYk0HvxfuV0aKpn4Xn&43WoSGV}Y@!EQS?{4&eO#qd2~VlB_*QBNcG< zjo+#&86OkU4;=0*&HH$rL{B2kW&ZD6$?rJR{a=i=(orni9M11b0jS?Iy%Zesk7aZk+0$A>0a<^QUx@2PWU8)& zdcVvj-*m%%?;@U~$49l0SKA{OQ&plHB((cX1Eb^gl&8oOz+GYDTQ2WhgW?Y)@1Ilil$f&pEz@dw`P4)f1;p)nGjfP^lIhmi=-DP;d{C}%V~ONXs==g^usFFTt2Df z{XbTIda2*Ui;GsHB!5!Z%{ejr-H}y{Q&HlU*Bt0vEy_`-?{EwtJA%?C>9YFC>}Ave z(g~<4_|l{WR*gxRS>mV4T>k#T>y!i)u*Q`tm$aJ5P43xZmKwjOhs3Fw!TgK@IH;qNy^z%PLDx`+I`2#KEkd? zDaJ>|L11?8SGQP8)d3~={pGD>&<+!OkPhAq9?5@RVMny_T2@rLH^z{CS4H*~ssW7U zb|pY!=_2iWo4KSTZB|%VNquS4hcBp7iGK3oRxsB8Cz=sgu4HL;%fSpdalsg@A!R!G zbJ55jZ{@>T>7}RiRL>%r;cv~Tn`kqcs4Jhp4_k|6k6tX&NrijMvS!%iS~?$Z=U5u_-wq?N5XC@2isG zP_&W`GVcR$we%GLpuPRn!)bE{U!>23~gE0tAj6hMZju02^n|BI{Fh$g1(%9KB~4-@H5D4FpxR zz=6&fi?Z4R@(xWnM4J<)Y`lYG2r1UaxGw`8169Fd`CCuc*w@c=ahT!BQW7!)aFKYj zOw+B%Sznkwxy9?%TdX3ZNXNLQ$_rHcfy<6GV|kWMYJbgxy`}MI4)xoV9@(?VP6g)6 zv?jrFe#}UcD~qwKpW)j{ycRU#`lo2;Wo0)JAa}a1&MYQz5$di}OpT>^OFo@pORXSf zL>h8+26?Rm@1(n42D~HPkn>9cVfqLNt#N2xdm!A zWgHO0Ja@O)ubPy|9)Z=YprQ)b*p?&<=K`e%C}81tjZ|qlh!17}U~Rnd?Anqu8$xxH zN({w8V1gFyi%aFqN$#H3G+lf`#%aIub;ouw%=dj2c1VjxP}tXIV(9Kk=66B#F%o2U zCvk6Q3rcHa%^=aXdbozM4&1cS#`vHrfPKYJtDcybt@C?UV^NkNcQUFNrzzA{c09o& zR;r91@@+S(gEl3ipxbb~|E*#U+!>6{2^F8MNek0DhHQJ!VM zII~C|%Fol)B@VIALJnKJZ(1buO}gE96j>xfFoLj1c?&`U8R;@?b1)=%?EDcr;M^#r zbEIs%7r-xmgePDcSY1r6B^9>8te^30ev~c4V2odMe^x$^rba5?Sr5;@x^??yelf32#Qg;I3(_JJa65d4KONI;qnWyqkjW!)nY{?H z9GsrbzF-jk&@r>d$ji4Q?#1<_S2J&gypjYJv7~Fi$k(5Ux^I}YSPyvgOfQy;6zC!$ z{(&G1Tf$RSrC*XKc`4OR4cg@J@7OZtsJ7Ju9&eYt#~=eA>g5?jRrLt?q(5kMuhJ`v z3kHD}uuE4}DGqjNXC|59@Nd$4X{IYOFV%YD8AG$%8C7KtG%S`a_szo!f0~+CAX}fo z0Xn8K>;OQvkBjnmmj;D6WbgiT(2&i~n{EV-eN2VQ@1pOcK;K<6`Ps!WpZXP-qe$g0 zGv*am@GaS086Z&6QCdE2y5*~BUdlsh;ey*|cqRJ{(D(IL!EE_s24dP@6N|ou<;L`! zVqT>>1(aj^^JO2081IFSJdo_>9$3u!oeh7K?SB2Ct?`CBfG~(L9Khup7EH*(pG_N5 z&eEp8RWuW-V>CvE3w*M>7C+K{s?{ZZCt#(!>e(Fc9x@%UWh!AksodY!6Tr$VIoN4t zYP)1Vw?#O&Ivu=mX7Fv!kp!=*8_*Vp%{a7#cba+&68pp%9%!I)b(60LBVj|C4as&* zK7PhgF9Z1WoP!G8%aBw7cdVjb8{AFUWWs!_PSCO}nqu>GnPo>TVo+x6t7kRJl$a`4 zuW_iwOF2|{2Yb}ZO=|g*IFWL_{^QNjKyL3lk;B1lz!`v&Nw=-cI1o;3_{FaLYw>8W zU#`2M!s~P;kEl?xQK!(5DD`pp$M$g)zuOwXkDQC;<4H)snjiwWT{oA0a0!J9-G;e9?#9T1mRIY5_I-ic9VVr3hKiOMDmpY-kPre5c0>w}D0oVY?YO!`X4 z*JGprZbWtv<74Xx({(O{8WpsV?@f5f8|OikSNL z=*#odXH`BI%Y2~>OF8&n_)%-}qB7ulFO78}Zht%#Nd!A;>d3fbW?WBMa2Y#tY9w;LoUF2YNR09yne9QdzSh%$3Uq1eV=6q4=tD+xm{JMTw}Uw6TUAAh^lrsR zz2*mIC+j62OOu^)@;2_3{gBOY{d_&}B8$>_j$v#(sO2k9DRYeyT8yaC%+EHjGxkDRO%REY0@4Q>QPlHs~RC?HCy?=XR91i zu$NDTKNRtnRKlF2f80pTvB=#81gtVuc?0?MN3`ouSA6kLsU(EM1?tQ2;bx`rcVR$s z{>Qx!Qzd&=ux{OmSjxhu zy&(xMGNW>>5%z&O2!5iXw$X+=yf>BLWxq!j}J$%lEN7r*2;T7P4f$R8Q#Ba>SOiY&y2wZF~GkW=kArtjFjEhly4 zZi?O-m8`lhme`Mo-3LZqz$oGQfxO!Ap7>1pngI}U{W}Z?OH#9BZwIvVWj%cSy0Q&1 zZci&=)pr9q|1xKkrY8e7RF>}K$2>j6a*XS5uB#{)w?wYko6}U9XY;@{KNb`=T_!j3q+{aDIH2^DZZ71>!|5{QzsP}LrYV-ctQ>Fh= zI7y$cKwCV8J2Zw4lBOKCod2D-X>CP`W~xvF!%TM&35aDHN5SH@%6*qk3Q>Wqd)MzG z)PlD{0o^dasdu5eZ1QD4WOz&`q(0J^d+(xaE3PTGLw26yBUk7d;M|L;d8{_c*~@nDz^6VZ$pE8fL;OS?G4Fo`e0WhJqKBYoyGgK#SWh z7{%=c7RW25T<<9D`iQDXBfFJnlTqEgJ_+watF6zPxstw+Pv*Jx;!dNTL`kK~OGmwx z_*3q?Wmhk&Pj-&>>IyoF4*q7)1hEq~^ty5kO%0RenlAZUsEQ^@gLT}IfyJu5C( z30@d|%rh3FE(b58{^osXNDE_|V;LxOXcW>)=W4!=myGX(keBoZGwixb$<(==(EmUr zN`~vxwvU?ENiLEe|6+6Ade2)<4#Gngl-7G=XchizeXgR`8S4@92$tD&@P>LwlRtP6 z*jQV+?BoU-mAPc^ljtae*RthXA9ky~u2USW_Zhd=MaZk%fWSrN>!jgmr0k<9rVq#5?Ui#`W-*K? z+Lkh&XkQXqB^lZQ?G7Hfhe?;Od8n1D(kVBVV2H9*!790M8_2;zr>Qbl_jKQ@JQHR8 z4Nk^f=`BDk?9F7)LY}HdReE7ekxsyGc^mOSrKP+LV*CC}CgYjeV?!DSWuw7mn0z+U zjA3VrX2>T$VMM)!(16=j zZcK6QD+ksiy4#hrz0{q6qIJM=Mk*Ye;j56Ik}-;IJ255vIh4PgqE$eiUFeuw*J;LD`ewoRMAGIGlQ;O($&rlpJ8#Kd|u3# z+h^-ZN%k+fq4lqO?LeL{o!=z_S)mF#8VA#BFmLpE{5jJkI{Yp8bj`~r{i}-fz#3TG zxwp`Z@7p7i-BfmaqgTEmk6t%3umyHv!KVS49XIjepFf3;U+u?pw$cfV!^)M)>Y-u; zq<8~CE0+DW)+@62p9Qwp7jsdYRM8SyIvsj3G8G)lyT-%b^tws2I-P^~asHOlj{H4> z&DNBd?lYoLqCl62O=s13RD8^e0cfJ*Wa;mG>oQfEZ8*}aNet1A_VRk9g>}UfSZ!g0 z22$2^BFLjsD&`@3>~jHt?uG!M( z+|K5&`TECH=_=TcUB_P-T3)qF-pQN3=Mr@D8T0Yd4lC?o6BitI+B3(sHN zM^e8q%YkrwdKSLh0FjEZa`t^Bzq1NEh|>fn$cGw4SS;B<>>1u%A~B4II%>lC&ix9R zoB19G{n;;GkCf?A8Mlq%CAU&Qiue}yrf9Lej{IZeEWe)R)oYUJUu41ETG{?ZUb)7d zdaT4S*233~{*19MkabB2&yDebqSwp&EoKZA?FyX;$8f@Dtp=`N(i6_NyD?wAbt}rt%3YTNaIzN_@0tlXkgqw1-(Iw11Vq zjqykp0lN`IVm^G{8kg>)_|GkKxwa>71xKWU!`*WT0bENLET)7^r)LOGc8$t*$T8iX z9X|loZ@Ph zMrBL!)zRPkyBU$d%Y0iKom|SAM~rG^e=uOj2k7wKDmH*{GuA7#+qBEIz5K%$G-%@Z zL1acU@lE^qySrSV*uqU9+%_7b2(*-s#B#(O(z+P} zIr#=Za?Fk#<2IzrH^ca-iakNy`sB12p{g3DLE&<;-D#VA(*Xs!VZ9{i)xpV}hi9v7 zW=bmsxCQgX!vI8@bd9F?wZU?d96VCxo8Gc0T0~K#9 zw_mp2i!3-h$wxHR$va&TvQyeJ-(#wxGpbDM$^*^8^1PfV zFIwr9$3pOo33?Ys#XQjdkSZP|X<6L&k4(GL`LNkyDdYrJsnTu`kc#zEJr-ikbb&E) z%ZGf~*IN)rr$uaxPEHCBjy+J{eD&em-V zUv!Op_neu_$rd%0J_itf>U#&vyJLZby~@%NzF}X97!)@pdREhdJFD&^Lz9X6U;YCa zYpp6*^Q}~~uUQIq|Ko>tutH%7PiGk_Hy<{2#f)XKQQwLI_*R!p74@^~)GpugVNY)4 zx*j;Tn|Xf5vHWtmf3THx3G}JTnP-rapH$IiTz0=x)R0t+|C1`-U7qytFIjs2MfWGI zVTh9Lb{tO6N_-t^b6MGIV%f{bOXeVebT(ZYjJP#D(m=O+v+=9H6h05ov}C{+YELbN zbv9?SLq!|PihVrVuZmsB(^Vow0L48WQ%t?jzxizN^C4XK&kI%%q#%aCZ2Ts2rNXp{ zi!yLRCW#I*K$#SP>LMiawDW-TKucYUQdh=h^8f?RaincjcmjO1_%4Lfm@NG-A$Gl0 zaXVv6IwOPTV~C!5+-u{-NsdnX5q!Ni%5-uB&bw)|VD)%=<==%vlhC3cwq!vl$*^x7q0|L3LHADlr6ROEQoOzyPl%^=xwcuZiC&sasxXTe1%$rwS&}f|{!!9$g-K;?7oPjfB0he)s%Rr4bbw%>iuaT` zsiuvet84(him&GyOH*Z0F=E|U<&KuY5AzrxY*U%dT4;Ga6*gL1+3M2d7qm%FN}OS> z5j!Oy3?EdMda~%G>6XWiqV1IvV8Xi-h z)}qj6S^XfU%>D?xt@C`$(_UpA8X_Uu)%A5b5|ld9v$ShLELmL$7VHT7XE3MP^n&D$lEu zQSw0_pj43{m*-wY)_qS~?c7j8kA5i0WW0)50&6O*BgOxkVL}qLYrV>jOFgKyfXHuo zl01LI>xYH8?f}_+KI5GAev7exB!4$V`Lnc-*9z>U z1|9cHbR`z9wyK#ezfC|!=)v>LXlfe?C5MJ#bYdf<;V(#eZ{m(*)kr`*X2%*0Q#JGC zKqex669uPw1oK(cFyi5U3C12)iB)_@wT*vG)RvFd!53}nk^cr4EtF%O_&k>?o+NM6 zFDrLW=j@LyH*>|T1Uy&jwH!ZYS!A@Xi~Z0P;P0D0}Bm5S(pO zkv!rJ?d5`=z1Z$FtnuMIGg~sdqGFv!b8S;qX#+$k+E>hvu!d@h~CGNFBLrc%GAzDSJTr+>UmxWyLjYvb?T5USdXYO2aMSK(Ttp zZDkD7{0SYnXQmn8MxyuOv&AM6V2LClb%Bjk4A5S8_#v#ljWv*PlSE6yS`{MqX0yv5 zl~AyI^kVuMSNM<@UgMn(eatg8RW(UcF9Xq;tpo0miVm{rL3D%C=D~7zZGd|0S`Pna zH*9RPD(D6<;s%iScFTKuNJ7W|AwPp555~3pN_n#eK_YEf8#oC#qCvcHs#Whw1A zw!ijpFq@Qf0^=fJoJ=u^z~4gY#+a-U{@$Igz3|S)@#L_4wm98U=0t2M3~{Q`m6Fg! z6q(H+!0(UaYtQC!N&nMhCD_rDqQkx`P?tbg&cfM6{wjn3f?FeHeLcu`cF?TR1AnO3 z&$2*<%I_u-O^3^pQLu`*_NhO6+gl({hk>W(2k@V#utRdH%yh%?p(=%>pQZa~?tB?V zIx=z}I(%ngL^3P9QF)~ESsk`^GqHCwhmV)E0|<7mm8xkx=cI?d3BIBt3zsFg8GmxN zHx8snK_~jYl3M_Uhd$+n=ddZOoUW+3$DoJ*CXzU=T}F59cp z#wa#H;m-LXVn&|EAVzTuX?&Zh*QgaYmyS~b;B+>Hz$=W2l6e6v>Zgpmbsa;x*2doA z)pdm%$|}V6D$lS&)ijnDKJmIRVeZEA^;Hhx+7rl9D-~v3PE*s4K-xT-Y%f65Tph`= zGubUo1(wO^Tr}d9G8T2Dp|f2Z^IH%&-M+~%iIJ*$Bu8KVAcn7Gki<7)3I;Xim=67e z3M{z^zvY?tA>sQpP5FoNSI99n7-n)dq{Trq=F`aNTPnIoZXbj!Und$zGAfcv?A+2( zsyb&C8!y3Q&W(e;sB@++jH=m3JpeV)*Hv|f+8R5(pH(G0=x zm)4iiMsR4AD(GWmAQuaC=JPgH!;I8Bxs0L8Xd%t}2lb}u_#m*>l487?HDYBIzsmAApZosN| z4#B(7I{Z8m0j-^;PcEboGCZY?u zg$~r}ZTJQ_TYL4MFHcwO#J{RyG)%U5q8z@A){LkJZH-|m)X%MUs<$*V2R+j@0Cm?-XNx+P+%viprs(mb6t)g?{Uoau(;Vp(ZMq{&_s zVznxsA!-B{lv3!IbA>3+7mWaz?-xQJ&iY>sU7#TMwh` zv7oApt4)uwqACKNK_Q=`+(kaT3jDuX@Hh~+RYl-cyc zwc6BJxVy^WcHnd)y^t)btCYu1K$xGeN^rscr98MTXyoUVO6Un}nv<=Sa`KtH7*G%$ zZS=}<$b(L65n>-7B57}!HvW`*pH59RCiZ6gFFxOcKNSp;hTkErz28DdZ&O9hHDsmA zSO>^9E6kfeQ0OfcY000CJO`UzqDrSh1_=9{Q0_I-XBtN`dx5vasL0A71zhTpeXa6X zWmAU$;U`zP4}NwoBIKJWGw9ty7JQRs&0da=J-HS+rK?|N;6M&g{&ew$IJnPsE@_;e zC%E*TQdLVg1%Fhwf`mKP)HXkei~Sy;<-WOrrHFH%Euy$JDW(&E0B6XVhX1SNT-tL zryedNoyT*}k?n6lkuL?nh}>y=OY*#IB%bAeBDd zRgfeZ1pYyI(P#*KaeCbhc`oOV>jGPPbZ3mB1v-0 zYU({qrGr45QYWZwZ)7}~?%)_~ts$;>Ij`u^DjycYPfw^wM+jEYAi1;&IhxbN>X3ck zn+4Q=kyy+efsRc0&aA|l_s=8l<+AY=w4KhSK2prZ2-7T8(3=~r%#=<4a`W#f+e5~j zs<3f)4uuZ}R&K9ycaql*BDmAKn8fJb!*P6eU9dpj8;{J48Fa7&tf2c_2lU_)uA$_a z70frX58~PEX0r0#z64R)7=9 zZhSL)7wA@`5f@*a0^&KFf}57fhJL_f;-Xg$ zmKM)I1)hooop=f5PfM#mkX0ILHp#D7%>VkuAC#|WVue0aaZ1degi;oiS+(m~s+S5r zDK46_jr;Pe0WyDQG6=Den!rqgMNpsCWEJ{ykcIb%n`ed``W0ztk>leZ0}~LCiUq`5 zV#8(j37Evnxx~#@FkF&^a>lc~j;{dZFhDD^5NmP_xjYz_F{Zs&kAH}z%F)+?1zcJ| z4_z=6l-&EhKACuw=ezhPpRrq8xFc>9_p!f_Wm-(@$dRWA-0ZbB#>cqNOA}o)tUJ@> z(EuNUW$-I}_>=UQLwAezVWA7U0)HhVARI*bZ&RU% zWb@C~8iG@{#VC@o41Qc$NB+79KqaG^*~+n$@LTmp7_h&w@kCCDfkK?bsCfw7$_ofc zLshg42P&ZxE`+iFD)m+)-1MGwx@;Ds+K$fBwIW?KiiB@iy7!hy2|ar(oyo3v=tbBNrl$Rn1$HEPy_BUTY0+36KzbL$8tYH3$s}Q z2gw*sFJV$LSMwfUU=9W;UR2uuu4XiVq=w?Z)wrbvaaST4v7SUns{2Z3JJE<>E$nT%*Nn)8vv?@8$})7;bD%ChNR>%z zcz_pOu;6C@H?^JISF}$EmFp${0z>R1Khzxa8iA-B~o(>sezOV(q@#H zxg|5n8bjhs@+8#ItJAF2i}9WuJ>>BBoac6Z<)v=e;e^4+;y9HU2HJ(vWF^rkd1C~G zDy9l-5J2Z#9&AoadFC8q$e7fKK3T^ zu~}6%lIAlYj-Trdi|19vRB6W!1kAHkB)l4O~5K-NKWzGN}5Y-F^K z?86!2lF9xxQI!$$HgKWg8+qEt$}0G)noTmI8`8VAMbIJrU&td1_5x<=!kQ%40N^6X)_v z@c};XY&|oCx|a{*H!K^mS7%f4`M zdn(Hj3}V7Y8j>P`?M;D%0v)UQQspk!%)O3$vKX^Efw3W|bxnUcmQ(9?BE+%eBHe-e zLpqG-5#`^eL)@D*XKqIrHk;m*zaiN#t6Y{R9Q5i58O1Z@q3)?AwPl0XYuClI1%%v) z0-c-A4YpF2K}J@7xpB2B8_JM73KzycL24=B585-ID|62w(@?q`3@X|{+{6)-x4yE* zSy-)@;*>DJ3K~^at+wJH3v!R240dfg8*y$;H4+ z#=T`8!iL$K<>6UY7Fg*q+N?0NfW3|}P$}GzV=6wHgC;|IT4iL(#(z+*Bg!*XF1&>L zvg5uv93|X~OS3ENW~jYxq+D-2fH?l5tA8_+7<$BeXyMPwHx_5CV!o`V9$n7T8!v6`5dRHt^67;+kh37^p|IubQqhtT-Qzf%vHzO{CtfR*wwqe8q_am9d`zrM&l#s%WveA2LY@sYo9J z8G&(Nsl%qBtpVPHXP-;KEBC0tIxn^ZQXPtU^cS z=_nf?0V#Gj;MKeR%f+(}ej6y1Jq&A;umwIL1Y}w^T|7I#r zkzwI<|MHp@@Q0edSRbNUOLEPaF)Y6loOeysMJWL?N{Io8Yef>l7|S+5rl75)w(uH! z23;LD2p<7xH7$+2tf=vJPBs;<9zq=1ZAh!r<4%2L8e}`}058JTc!iapR0_!<)1^=2-j8WreVh+`N`0RdpGm5bOk!4y8&WcrEE)f-WSv{Dpw!)SqY=L_al`QrMKZ~vwI;C z{-+UU=SE3Nf#=itw-05or%L&E=)t<*!PQ~uOo&tN721$iQk>wk@vx40a{kqzU-H{n zOK|LHF8FXFf2vaz00y0GfL9rVBqx)=xmKuRXA34hPi8> z6F=d^zzo_1n4D6B59;rjK`anhn5gFt-vfz}dc+QhuObP5sp>jXXtySev1N28Q>O+< zSSxh>R(DctH9%m6W@EW-oe^n=fep=+{m(*xHz?my8S6Khp-|~fz>(v6j1^mt{i^aM zG}7G^=g(C~zIl*nY0`M!?8?iW=R9j;L0HLb@2`r;PeI-ehBp7uf=xscOC8i_l{CctqRoV` zSz~GXI^VcZ`QyZ0Y|Jg7Kuel5#%MlzkSd4YBKq=Fbx%n;2ry1lC9g}E-VRVxZ*T?K;FZfbsk z@i(7&QCWO|JRsF2r&6Z=f*570M3ylCN!8#$_W~rM@AZGoY7k5ZGhlznQZFvy2{0xAX*6n=gyRc zB;a+5c$o1+RupTS2qC$ZZ(&~IIRH$ci4n+&$S;?JaSD)%rTAN9jIhSj=yIrujJX#} z_$7%-To~5KAHC$Ix#mM4goZc=yf3uT$VrR5sX}dxOIf!6GKW(91_HNL&vrgbArmTJ zk^<*4+F-TP#qP6?v>3yNBkYXLorXl@ zHiGuJTFap}!E8LuQif)zP!b}LQCq%v%gU6ER^Ve3RWx3=Aw8of{A7t^6%?cW37p7hwHQ6$GndQT!=Mebv`#e9KBKcR15$6BCJp`o0VWn2Qs*J7jevpM z1O)-lE~+3^&bLAd|6UH=JTR2QdPOgIn5uI78h@r0)S=2+7KOQ?kMgc0K%Ju6JQ{(b zi7P-naG|BdYkOpvBQT`)HiJUG&^=B|k z0yU@HN7c7hqi%tEz*9U$PJRj>7Vb z>Y2N~@d(cpWpSW!rG2E`HV)^Evh*d*6B+~3JKn_l+Xu?jw}&6F7*F-!+aEk*x;S3@ zaNsXa*FY9l>u|cBx`Q&ZNWQ&{ehE#@ri@40>xr1d)>+wM@)_SI>&$NuIkx>*}B7?qO4w&!BlP#e{yfs|gks=3=fb%b=L*`cF$Mo*NLEfy~x$MfVfF&>o8(0W`3w&E}$fRo_Cm~;~^2ULpuuf{bj!L!3Cp|YDlw(xHP{fz{P$o0ixQ&WC`6R+( zCnyReHe1{Y9pzQVC|fsaor#Sr{}q{niJQ{16G+lP1!l{{iG0~}D|i&9{w&cjc&Bt& z7p#y+A29=RwimxKN%>FFN9~alllY$QD!PCJwKe7SvRlaBFL45XK4Atp>Bt`0dG=sA zD$k6BWG3s$_4~NnTwOWy`F4Kpy)IlqL+#o>s-h3dZabg%Bl9|q-_raIbff&)9EK5H z_1dLD4=zI$!py3WXpUiGg5CbhBuIXs>mmw|ds+NsV!$3TXg zU`;aU-Zl~Sht~#O7%;zG&WwT|RaDhTP!C)j@;@~3qAGm^eKzJxPdDGXUuD~20ikQu0eE{{Hw$C|yDX6U;`^~06;0%D2a|@Wr%XE6blqbTy9jZ)D~3x&z>LY^xalOG zR5S`1Ea)VQXQGO!S=`lzbia(V1MsyUhS084kwx7` z5AhH2R0P=8%6P6SW}M3klhdLJp;8yqdh)}PH3n6n6tFL2V)(?rUpC!@v#N6dEK1BT zAA?V9Rz^PTt#}akYYdPrx}&T!3NF1EY>kzuY=C#dCGTRhkJP_vwNz2rx?DxxlOEeS zpJ97ovkl6eJXy4l_QNLXK*7^okE0XyZypC)K)WYn-uv^V{y|Wq?4}qt{*ED4eTb)* z2B>hi%hD;F(H>HO;C$&~xt40$!46Lr??@tZe2^RX@Ap5X92URXlZIruhs%PfH$=uN z(w|=g8&j*a!@=O7xeWf*LGk^vVIz0x%z*H{SXd;*HZ0v844uZI_e1V<_RrS9)O@Kz zkDLuJkwc83MJKqB<$9`AYgOD>Ht)$>gP&TJ%c_eKBes*H$qpJff;YWeHR zkULSE4m#uALpdi(>bSfbS+WG(fi5kUm-nJ#%T-ZFUd%GzxG&j8w4i7W_#Uw0$3;y! z_pUc8H_G`maAh4`et%CDo_*_|ac1U($8+)wg)xeK6gjt<;0gg5{L$*c-tgagZP3jAHr zVtNt|uIEe`kvBnxwfm99QnAjm0^3dHoBk4!e51Tp4zGX?_KFPbAkhI8V>jv$ZsiB91?J z5l6qWkK%JOUX#oY_#@XYUvVnl1ut85w*ox#7E#!*c75 zvwCty!QB!+71Tp431dn+Lu!=&Ej<0AFv{T^FTX9qPJE>YLAGp(5b}53(Zfip-0>?Y zI%b!(gdk+aL>X}e7;XgY5zJI^7ome%OeFKzPrzcB{C2& zUaMS-2niWYcujW;u(_r+_ix*wm)IppO8L*DXznAj|6?;7SAubD1VG(EO6#4S z07&C9o(N-LMyS%kI*&F`vW62-NxvISyG^+|Np&Mrho$VpWcokI#aSJXumAW#hkSA& zSOC`eTESb0o1bzZ7a?Utq*De4^sWQ@lU=t#ucv%vwWwuVEXGG2If9x8 z(Z&(Vwx9p7knbTM-}nuU8WQ+9&w~jFBuRk<)a#{O%y83FA#=O)6}dSv&Dgi8$2$s= z=H5=!*VYL5a}VKZD#FrbCA!3R%OM&htVo(;FDBP0BlcL@OePsP()9_=$^1cPm`lse zhfY+lNAn3VX6s6@@1_yit<^A3uLKiSEOxd& zuQ*QEF|M5tft;r*H_PU)5tu_NzD18a%#-&Q1S8V_eE@Hk3O7Y#ea)r%XYSvp#i90{ z(5(roT6dd;w(=N(L4<*eYs;itczM>=+2vK87`t@~<%f6GaOU@^jBZl$A{Tz2s+=M9 zHgnt$s`5v8wAet7+g499zai%E!+29)5E;xTROoSFpk#==y^9BDURfjfT4;}K^zqDt z_13?BH%!0I`OGwGrB_I^OQxOhHwNLP@=qjF6mx@$#bczh5|%|(Yly>5oI~7WcgjP{ z`Le6DEQxsx>K(5}$p^7rlwZ3(LZrXf2K**6q<@temMb6djXx2#fi=6`XbuG_n_dEq z8Yx>t*}Vn4Ae^F;`^6I_#XcnD7pRP(f!1gQJYw{n%Nl4TqxKO=Iw|V&x=JU*$D9{+MXCF zrxILIfleI%*bjS)$J`OEE6S9M1AyP-FC#c#>M~K!da$gYas_08jgS9cj}2bl$4ua` zhw93vH(}T3-wH-JlIkHS77@X5Dio1>oF*>AeQi))OP`Reo4=v_DVC z)S$1g&;bX{Kg1YW7K16-(yM_P&SutMF zKMJ-|K8@gpZ6yY*CZ?e-MXCpJE$`?BPVJPj0V7dj<(i9|@OPdRu7GB!TUWxfzbmxi z+t8%4ydG-Lp6g+fpWwW{FXX=Ynpx7kXRS9F4&277_cr5ul{$wl(*q9!t#zXpoIS=@4KIg z9c`Bh#3;8uC=%K$@ehJU;@OSdwfKz}?y8I)#@4DD_L)~Az`nZ(GND8(hJzQsDa(DB zbnklkV;1MUKvkXsgi2e;#RXowoJ=!y^}2t*IILa+a_cFBFc$UEZ1gpaNt0ycVw`mU zLuQm1dTw9Ybcutx)dG;9nFr;zj}O{$3L@(zy@;Z$4!47?m-I}wDY+;~9i99=qIv&3 z6+a^{^f#-~1$RIj*%E2G4ncTIBYd)QJq?QpBoos1+mW?`7Dz~OU!LA|f&>7yEc;cs zJEt9;s`s)wCH-r&5_h&bjW+_AYNZ=8@faMJ5dRiLpY~+JG+&9EAz~mcbBuq;vdCXQ z@~s(%%@Uo@9b#-P)PuMzU^i8cbyVdn2!%5ZI%;`Yjtw+*L6&0_LSCLOGiE{nuAM-t zJ1FBx-husDC%T#ublDcsC41@2tgxdC_AFylVTCa|S#Q(~<%vJQaY4&sI$|+N^NNP@ z?ps!e3}I)mhNIcWA7yT?ptQea9U|T^zAndI+8tU-XAYI?SrCVRfy(m#vdg{DpI^s= z?94I^;DN+0W0j4q_~KB>Ln@pjZtYm7C>IO2+oJ(p(n`0g+$3X3RTu}kB$e zrHf{3#&b%xZaU6dA6phWjXnix%j3UsiByE$gMln*%s52i3k7eR4$iwIOLk^q0-I@v zyP3uXYO5KNzuhdB3q$Z2PV^#3@?i{C@s3^@u;EzHE^Suu5B1qY3J8U!aiEcA@~p>O zDGeI%n(OJ9K_e)uyL2avJhKl^?Ni|wp*e};oTxiX?=bbtD@Ty@ExOUd&|66QS%Max zFKWl(3``YogH&SC6Ot=1+c-E>$?kI126}B$UV-G( z#_>20TA2!fx~2hu+4JS|89^<9bOdEmjU*?Et#-VNKriE)LOsUJJy{;U1T6flJdf+T zwtX0P3Z_T=4`X+Sitjc)&1YBRQXcrrZw@B@KmM}j2It{YiFH(I@#uwXaG%@vh5`S& z#hR@nQRAgYxmlxK*Y3s!u<0fhS8{X}Pq;w&BP7lugJkX=p8A_Upwwr|@JM~XRd1kk z+>Q|o+yjGR-jVt|IikY*<(^$=!hAj(WZW`7I= zeNtVs^nAj2rkvD++G%-@ND8}guu~=NH&s+;I$^EkQHP&#(}>9ToMz_jW_aZR8@~=Bl5W?o*tS`#0MFp@?zC{)kZcqwKC-5bFOf@ zZsP@-v38ZR3z!GIbTcLj@7R;Dob{$_H6QS9qP7drFO#!iy|@L)PqSgiF;!$~=Xi4e zKS!8!H4CoD;?Ee=F%*W9d<&_*auI;cKy#BVXE>#`X2hEiRctLec-VHcvX~I%N_u4x z9t{N-#T$20;^du?5KDF?V0OhUxu(7@b={S$m3PmtNQd3Hn#$Sbpzdt5^ns)|V58rR z$7A*owCqQU|FQUd!+a)pAFP9WZDLyUXttaaU)xMp>Hc!k&BAbpL?z z?lPs@QY%?zJjPVI)E0EW!y72gYOrjozl0-}bLJQeQ&Qzj8!X>NAAyLY%`6{dUqJL> z3Ts0XG zkLpQG$UZqX94`6NbXbJLrMqO6>5P#&TkIlSSuszMKKaA+=F){KldX{)T-5#H6$UHG zctifZf^HOVM!d>t(Zl>ffgKB7qDSl_enU;T3DPgZN|kS>Sz+=P$wQL-7NEGaA3*On zvr>R37WN9$l?XCrbdxb^FYU-P(A(3WNLXJKC<>5%Ciy>rOHa(A0(f{@Tda|yW=5!& z10St+rxuipm@2SQ%9+H=lHc)*KYVP*z;n|~WM*zx`Rxk4_<>ov2IpbY5^UO_>XN&K zqgZ05RRKn>UGifiJ3?me;@zOnQ8<3u9-k%ss&Ocj%%qPM)qPMRbE|T{Y=mm9y)fCA z#4fx|e+5cmES~G2&D7Q7@Muof4&GBRU5o`zH?Fy$TE?N`67{&Pg{a9h+5aY-Ha}xU zDaxjQa~WO33PR~GX@K*TJL})gj0kZSz=!9Vg+rw19T1l?s1?!mhz#lVmZb&wxdArk zJ458x4s7k5>F-nzNN2A5(aSr&>ZFA6Lab zzzk21N|ihGyN2q>NLpg1euP8KbrixZZ3s8V@%XDc9y|O8z6>%Q$(lhv1L0YetS(Y= z;34q_blfnu5%S9R$fU=`pP z$ahKB20KXFT}GrGGSdgjIu|4{*DUP<$nyuNPws@IRmOCgOUV}*Q5B3VT*lT(Dyt{~ z)li@iX`cXX3LZS^8XqFAjSS_f3dv2i8My89)nxisP-u{rdJnmb$3Lrrd>RcE5>wV$ z=IXseMKkw;EJig)`O9`A(A_@r>PKjGxR$%<=~XexX1b-!;n(u z@XX_ZJo6yAtVg)?i*cq&=WCGHkIy~Gis|($ktS{H`gl~5s2^bd5hT1w&Jx~qHRZ|e zPjFB6?c;Pi4i-3g{m;oBz)^~*94i;oow<&dx5{}~a$zhi_P4*dx@&)k zZ+qY7=bHl==)5LAd%fj!I0}ArdPK;d25a>ifOm14OV`ordw!o9&bo56#y?Fk5@p9L zO#bXP3!2k~ryqaIHZ|{)8Pj{F>RjsoqkmF(A`5r#i74173Z}+6L*R_80@AIW(39R-C}rJCS%Pjn}aJ z)rWZFkT&VdY0GOfvslsT+5GLVD6dFE0}A$k0=Nrfn08tfpfP&~)4J}ko@RGEo;!@v zW}M5*_ZTm@C*a+En18m&WR%r!TFKJzV>o|OLWz7)hq)tCcz5t6hB|8TShZ-|Ev?>R zsLXEs?QUfbDobk@du1@3`xPr(hJ>(0!#`6eu>%*n8R**j$!mmL)`CA{dRm59MhJ<%#WbXjJtV zunx2CgmfPH5`N97@D}1qwO*GbWQac+fDk_z9DdN{rM2E=T zc;syL&}=y~f%W!S#$M~M;GhP6!P*8M$G3K*JxGg5;L&N#lC5s@ zoO2UD{M3a(+i5lb&UyJVBN#1jzyMa}*~E&LAExod60t($N?XA2M;(4h{+%UnHkrG{ zB0kQ%4LB{u^?wL~m(%*Ul0iH|Dqf;&b^y4AA*{zaiyvx)Gwq~rd9Lg*D^0vV1|dGb zYsE+*{c9w8h2<|#KDQwF!^}HQ_dGNmWGecQH){N)gVI$`21YWdt@IS7kHl+5)XA#l ztO-pM!^KwCB!tTZ<+)+naQTu~gba2vcHXDV7rPsvHaf+R2O9y(-;V%F&juE*l>R_s zD-}7!zXKpT>}TQ1Z@~oZTf++eaoQ2mcq@B))+SyrTQeN8q=bLAsm1_l2G4Dq#JgLU z+Ho>wBCBqf#Yds~7Pfo2jUAy`@^5n{e71ruG!8}YSg+L5M=9Xz!B&zSQ&CVZ2%K!{ z&kxH#valbteEpRE4AlKP_U48>Dz;J~OBzgO(kMch<>x+()K1xA_n~XSvl+<(5XTF+uSrFmBj;R*<+F#=|o% z^SAbYv$%StAoZNDtyKBlW?gYQ*BXk~Nx{27vIP|zq`W_K9iM2$NrT!<*#A$KSQF8M zrWxmyWvkIOqVmYn4g`?fKjf|Rx1rrGT9d-}I4_*oj>>h3PHu8q)0OrL-3arbf51fmn zIGI0rWoF5ZBE}fLnOR34R8((f1xXoINTzDh`qD&pyg7u0Rduj|@?AK&$G%wX)W5AD zNAj6^$U@$w%uk&Brx^PNJpsk=V%v$unwt?*Q7+EriQ`Wh=}?k0-8ca_k~VNW%M>l< z`!ZMI`^?&3IHPw}-wGT9qhFlK5AS7z0w3i8j!V;k-Q2I>COy4i3oCizlN+q&jqjLr z%FnE|SDsBI-7enp@?q114sg8fRWf!7<2ds4uJ7g(R#MO#z)pJ)aIQJen3MdNarq0| zUrudci8U(O=@R)TEA!6*9^YPP_>Z;ekWXEZX;4GP>-i%nRLL|P#@ga^qfDs3j11+e zs||U+pRVa37O{-#%bLoorF(uMqkTv?yli~LiASoQIk^saJZ}cD>i836dR482^m~?D zo&`eek7lupay4wa55LB{GdMrl8qJ4&MaiUqQ9c;X4}S>Q*6nMMqRt^EszHfBsalgw zy-4G<+GfDOn!jMh)mPgNdAk#@Rr`_WLWi)(vk|=ZtrGe&(pjWl6|WdA_tj1L^gI-Mz&jl8<99D?@ z(wFm{YKByltY=U~|AcH={FJ?XwSW)L!z{+3jVZEvTajGE5Su>h3WMzb6>%DBW>uC> z4ZR|armXK)RXbdc>kC8Zvm$ZfP_Unw<&x+w;O*)*P|v{Sz%1%IM*5AYLb)2o8qylG ztD?nx>{69?=29zBy2mqNr6ajo?wMGSlR0J*El$gMntGG0m_8 zpw0GDzJ-lohPN~_XGLhfgcMtDInbQf_ugizU1qeWI4kk$%$pqRqDO#bzn0@1py8fh zb=c*S0jg}R@5mbqcQHr}tqt>qdX7GSi8pphlvb-sttNxUT0|!7XAiHgv0O6mODkO5 z+t^`Wnk>kH=S<-@nvp)nAk}L>FX`87Kakb7D1+<&5C@L{UM+Ct4w0rq?>k3Xlg`PG_4u*t^Fm%| znw*YD)X{*$S0=Z!vSp^;V?=9c)c|b~F*E@-KdU%dUan>}!BN1Vcs@I{LfB15?L>Yb zP#c+fTJGCjNq%e%vYZ-XQR*qQRNCkBKm@_W@_eX;TJb%q9C!Q`P#}3wz07sGA%Dg-_@~!zt{8}cPk#mRf&!ftQtJPv( zfxq*um1QT%g~uRZd)41}_vdd*!ufW+EmVI%FBXcmrwoZpg?NdxD@X(FSXt&v&Sc&p zOw^K0e<``dzq(XlpB0|7AK!cqykmFHWngiRqJHbwOJ094kjg#jyPe zzcY%<9~cb$0R{In0DyN zo~S)2cW(et1NwvLx4HwjDU{m-T5eD2I)+od*pu6mbt)p;-D5wOHM^%>6R7X~`ha$$ z)~bopoPg;jnvRSC1wPQ)T^r%RdT~QkOc(xk&y1*|ReHmvZcD`e18q=PS~fnFiywm~ zwv=X-;4Yy(KmT`d40=Y3f+9#^7k3q!Dqs&(8?rj);+?E@sX?%I_*S0w(bXd@$~bMwI6eMjuja8WV$#Y zF+v}^XqMc|=75lbM771Jb{F^4XOMCiGZG?b2_SEWvzJ>>`1s?Il|h_js<(WuUY)W% z08pJE)WI{Ym}uMkAlH*r%Rw*|IhD?bmRowRW8U|)e`{N9blO-W#ZIar`15(L(avfj z)k?h`S`*JApQ+WknufyiV|NBU^s^n!t!bg3cpZg4_Gt+oifjjR^};C6mvBxar`j0Z zU6{O_w!JNXjlUgP%Ic#+Lu5u3l*$KsH8&8Lsk_G|vDMKQ)AJBK%d9ZDibsUB+9c)1 zNwTpu=h(C{Lp(ZvVxSmR?X0#41l*@kt z+8i@3NIvPuk=;~_qR$sBXJ3kTBaNvlEopBhbpVGk0Wme?4Su_8h7iP*nItjmIlwaC z+8NS>Y*zXDXJ&u+lZ{BXpppPHy9$?eu}SaB4bK(&& zoNw~6`~`=4jPmO8+eub)Dbp=qoP{>Arm?j@ffEFqS;0ii=Y*)YKSo@ygGM!<07z~? z>xknN1H11dh(fgmOM`iw$;qFf-65Dn?m21bJ5ZJ4U%T6x7PyWCcl^2|y%a zJdeZ;01wyovWn!0-p7^yjPZ_UbB>j9&`AEbtZLjPRLGUuMz64{oopmz#z@6kAa0K| zD58v+QB~q*6NWH}Vr4ST7j#{}jNOMuK@8i}Nf1iAcTEhXSDKk?$(!vNJthmr_s&%y zI{2A0gA2Ww`dY-Kj#NF)8-ra|l2n+-Yg}PEiA{<1k+dIS;(3*LxO`}&rsX(|j+vSV zmRMQ7{sj)TXB4<-pfb#wKc&m{X zsMwboB*Wige(DW0lE2n+v=L@pWA4uZV)RH(;onqceJLg4=*o?MPcK}nhwVhU`YYHU ztNXh&jKhBMf|<6c2E13jLa%-tuh{@ep3)glgA$X5_$2#CM_+#Rt8GO{{z4WlrYjhC zM?pn-Yb7hs`+;@!?QVq-)R@+gLLvSrP#HDF^BbOhhg1=1JcBb>(1$(zIShynBc4KD zf5`d=nHddaoLV$#gFuIywpAdd*Q{KragA;IWhTq7n_x}D`+*l1e&v92^4!LY;y6$| zy%vlY2hP0`AicQ{547q6rJqTI1si(DaJw3|-UBn&M{@B@Bk}w#nf)ajNoZ+d4&GtD z%G#7Fz5~mD!#6%nW_DqRzr4?Q=i74nHw!_jc{}(Bc?Xi~HH+b|eaOfc%EYJ2SoLyM zd#RtRS-;R-8;ExQ?j7qCl}- zfarkf2qSbN&0DU}r&}I3hNU!1hph)vT2W5#<#bE5YtgY!kZYb%NfcDgs3(nXLMHRg z*Z^s!jRgmpK~)IEB$mb31BmTAj0Q}^UoH7}w4E=fKZhu9=fdFKkqq}?3XOwaw~)?F z+1QUYQ5)As<;xy55_)R|nXj(G>AD{SsEg*phPRnHnp%n>a@BX z;w>||36A${m&~rjG+&mnqvhy6#;i#FT3IrblbmA)RRXi}X?mq5&a;_7>%~Gu)oCGoL{_;@(Lp1+|jlVtCMwO-1o-nBHw*l$G4=fj&e}pfgjfoYd+f%6FOPV7Z z$)V}caR~0^#}IDa7D(|jYA&sefGnQQgL2;Q3{VdK$>}{-YrAGC&poQcBKo1`%S1Al zW%erAPubl7p^|!~r@CMq6U<~^%@hojD&wsjvdH3U5paMQwSvGp;x&V%O$levo+@L; z;iy;{t1V#4#({a0uJXjYW=W8oT@CNwofj>?PGGL4++q+n667yahp@z7?z6hSHTmIW zLl%*#*JhW0w}Yg4Dnh-MR}Qv7TCg10wjvCD+g({+B(SbYW@1gkUs7uV4)P0oe1wdm z*k4n58f=9V8B$~d*rdvm8fTY67m*cY={HuId>I2_mM!Fb;%_o{?_Cxr^^|{B8p8q- z`ypqllB!5F4^gK_%j|aR%l zRa#ZrVJ&dGk?SoD_i{d;9S6v1g`Dq3GzKZ1W6@akjGZIf-(*Vfxgh41_CRWj8D2>a z-(vW}JBX6y-k@MhHxT)AEFafj@bQL#(~Y&Mcu^(?v|M$U&aXm!f5(C^i?s~c5Hk@K zsXVgnnOC~BJkN0ToAYtt71#w`y!>SMYDN!KX82+O(qaVVm*wa^nA<2b#8XDSWyN9t z#(EN+RuCd*)opm?6&n_E7sdJKC_^eAK^RbJq@|2S#S`BdY;TQ=)xF*u?p;fBj(*mV z+ks|bHL3d#+q?aM$$r$^o5DL_a3%Pj=gMe zL}$w-KXd|cb+qXh{S9guwe#{N`X|dFFEknKY;lroq5Yhk#~hIz^DK<@qDSGNWS4E`Mm+`(4Ig zq@6ex^8+zt*sZI=uwWOMIf0}U83F3B^HEL~Bw^`xx?~;$VOmBYH+P%<{!m-GMyG{T zMb8YXCg;8dd2987w#(UY@yk)9${ayO-B}Jkoo?o|!t0doNjN>OxAM1{d(I(nBR&h5 zZ`p3qOreF9F7Y?4Qc1&HmRE*z=&4bu^oT*3>;?CtT$m(InHC|G@iSy^?({{;Npx|0c%aYp#4_M>5!GsND8!+?AkKWL|c*(5!6O=IZ4@n zat_0MWbiG7*ZNNwg-odc@mmVd8IXcbd&exPA)P0}bEkQ7q%Eu1dnF^m3YRG_*}%8Y zopJKnNUsRvfE^)^pMzEx+gU*p>It<@R^P63A#8QwMz;N7J3!vULbOqln$VR)fOq6} z*xKBROqQihN;mp|A-%tVj8AGMG)E)0K`n?7xxAEL7tgelWQ3LpnADcN)LG5~CPy&d zc{99~TrOrQLpm_W1dU>`Y5|rT>Xg6J&q@XN9l?0D;$%|^I6P%K*xd6j&punn0+JJ; z9cltqlR}O9!J;UF6GpcXb14TBc#DUBSPb{bp)xWNo6$|A>IOSs_I}Ay7EJ+2v&{Tp z*%--d6^qd-&768NqaE{n)rOVqo5!Z!i$bNCej7x*M%&B(_uZ36*ZJr86ATiLh9z?@ z!bT$1YnbL~IpqEIy!Lu;fV26%pQlH>)>D`KnzS6R= zv&m?Yda2-F;d(1Zdf4!XM`nIY_N7?>}H+KVhINXXv zl81Rq;B%haRbd5At-36w(?--S!wd_QB@PxpcPR|v+YdRLA}wUP%nS-5Lf7G`X$-C8 zh6k(l90?y-l9??PKEu{}v4PbMXu?P*ngWKyYNuUDFy#JPraAXF`~3d@Ay*z&5dqHr zwPm#vj9BZz(cttg0ryL55iXdamqgKRl|7a`7^B`G;B3MqWOG6*M=K z+RICA0(^w}HWK_boBjSYfFUE>ODf`Jk@k4@iDIQ=Wp(XxJ)q!_<>mPyVQ||wa^(o^Lk<i*uMBxEH%n?0RvXt${?wo{k9dwMYZ!2Rx=XQ}J7)sVvm=Oq z^lvW#Z?c7+p$;kB1sfPQo)^|Eva<-PrcC6$);v4P467+uUOGz&e>DO%Tduv-Y0R!_ zp)BA@r#8moDOR%b0$(}n6K@cYnN(MDLeXc+F0km*nVB#9U$P7OAA55VT2vQvvZRH~ zeU+&xF55yr(15AZLJ&DyE40=yL%PZ!d;nyxks{Np6&lB~E69^#`ebXeKm?ZqxeY}& zGH66*qVYpY5XyOs7ii^QO@@T>6!GkprNcnzw`)Tn*6I{yU8@qC5U9({nrrzujC~$9 zTK_#e34+aPBU{VEvTmLM>o;dxanklxVC+qNy!<^GrtfRIT1%kCnl8a~jmx3=GBFll zoxKS2E6jpS7U{i28#O8WZ->2PrrTVwOh8kgsr7PiXSs#I)wC5_3ZT=R;-}gg4#b#tf`DtFfLoRfO z7{*QEZ%G`xB&)w;#K$P$PG_*ga4&$QmZ-K{T@|S_mjFP^Q2+%Ql1fx1D=yQ`4#RfN zXh10&3cWfWzz&X7gV@RL^^rj*Q1tc(v!A}PfWoS0#YmdQgsp(iB()M~u+)rfFHdh- z4tZ6t6yN>Cc|yymp{h=W*?g74$M`0eOKzQDrgs~P_FK7a!s zGm8S`4|Ozl)fz2iledv&fxyDQI8naQ2rFV-BjneEENR_C{_ai)lhPTieA-)V&gUvz z3&XX(^xMwgY93>oN7V73mc*`qxQAlk9nH#PHDV>dhtn5yJ(HvlGz;LVToo z707L|hE3(I1P`eXR8J1O;Bi~kmqAiheW|sM6WG}TK%bfg(b)w|lwt$p@W0roXCpwq zPbH7M4@z+Lj)LUFY&+=GDp%6^wQm9|745In%}nYpD>R_wzP{0NI|KYO2H^K?#>ZER zE9=Y*AGy4Np^lm9?POwaaBALHR+e~v%)ICPbY5ybqg!NXJ}XIdo0F{!`FIB7ebbz+ z{-dSb7L{SVUI9>1p@zh+uFnZkK{`O@BIKoi{awRlK?mk&t)*Xj{tE#OeZZlQZUkU) z*+h!ZF`m0gXKZ6^Q~^@M;YH$7MhDr%Jrznqk!Hpgf~?*U%Pup-PqV=~8`Yz-rR(cx zKHr%6)#Mw8MO+`zO`2Owox*aUUu$~c$*`v|o1b4>;l^w04lz8ua70t4U)ok==0~(X zmw(=do<17LNW+UujfX6uL|e=3{Fy(WJj!whR<|;cqe&6)e1y@k_A6p!`9Y4S%0RXy zP9H%V#dXl+2aaNxX8E@=Gc344a*#C30Q)}8bHVv2l)A%tT}YRSZmo$bo_tH{BD?#5 zy*Th&p~KRxaMSlzu6*(sOj-N6(-zNAD_V!eiD zgjk9*rOgT6>NtU;J9NW}mg*W2HGC-+8y##L%l9idwQ(;1_2IkV`}SbYZ|Dv?SynB! zD4I6@r^I(vr1t>l+_4Ky)+gP z_TIZzjQGC|?XQa}m5Z959O%t(HyVJM12ZjhKGpARG+`ds6Nr#PbCI%RKU*1}$H39! z*iU>fl$Ud6zL$hdhG_1zvLfZ-+aOd0RzV(GX?N-6#j$X~fFm^)B&FI#Tlq|#W}e!@ zWM(e>?5N(-y;BQNA1&qhh{dQUuk-;ysfOJHyKZD&ulU2V zGYpP0wUps0kFWfW%*$nFs)_2DUP0P?Yo%kM#QKS`5jAZ?KF9v6s>W|o`c#&EhOMx` zED6PR>1YkgkU(0qBLJGzs^b*)#@`{$q z?SbiK2NW@0yY-!$ zxkR!)bg>UMyto^Gn?MagSyCLICC{sJs5|0#IItxQnWT#tIS>a;A0>4g!@$)IDMv~B zU1nkvx&I+Q^!bkG_HF`cCq1_u#^F4$O6!Wpes3p9y*4ntxfFjk;^HScjA`4fRM<0B zWnRtT{5o^NKmv1qKQ4gDae32B50L;|B$%XB^?kH;MFe5FNj;2WaCS*m#QDu85Ke84 z?~9@sCEk!yKSin_%*5mX?3s|Bl9Bmi7Qge zaK+W*(h5h)jH=*3mr1Pu$T~=XoIkizdD}%I7IyLPTFw5Mh6PQ21kqK{A||0WyEvH; z#|24hTRFawEjBX4x|2y9r#I$V9i`}7D?(09gLN!d6|coQ2<()WT)k@=E#$Y4Ac%nV zcAVT^X_d&WFaUVwb9O;4H_o0hK!%FzF}Xwrsnv0EYt9|tj$h~2yiss5?`z5ff1)O_J)G7U6P4ZlAgBBLsuyJA{s zQSy^1iFm%l`;veN}wTC_5VPN-e!qY&b?$Yl`Y&kqpciSv=q*m zwhVk*lnp?pw*l$8){9@n*(CLlSK9!F9+VV8XiyRIXVYj8iR3ny{0Yq3NiCHzxoAL^ zS=dab6!LEJap;&@Wa?GLg+g9j zxl17YWU5D@Mo|VXx{xJOK)td=A41J{Y3ID46x{y3IhAkBqN*}}4?C{;8k~7^Grp}n zXk{=`b~_33wNjtMaOKf`)W2~~7L>`@1qb$7v{YLg;*_7Dm zE8)#-LSLMm$|tvB3d^Y0A?Gdt%6V~|woiDx14-y=CCRlJKD9Lrjb5z5^4~x3{0Doj zWa)ZhSB%t}!a`pw&XVJPE0ScIhWS`)tuS#1vRZ=Q!r)n@ebMd0JvknimgvlHWo$&n zpyry>c+Q*OzQTONRzp5@wY5!xncWni<0yE&;nqet3};<;aMqk7{7Cze|(S97xc9qjhW_Z&$Vh| zLn%4ON$xitO)yLfo6FFnfaQ}0GjBuvfs+QzG)C<*uo zDPk2JEJ@Q@n@r`vDm8;zXvw5S{^F|2Pd`Iq9kudZi}tXTsHY6~TN686_Dlj}7GxI5 zdm30nt+Yx;zl3a&-}rNpSuj9epBNu59va*I`hKSy`#zRdI%v3J{Kk3jh~!(xb#T}f zg8{)FGqwSk%$iI`~QIq+@i!! zLS-dnZP76K4m^X6b*59=mvMMrftDj3Bv-$%Ydzq9Hd>v{h}o8Q$$N z2COa{S|a%^*bmh1lF#;maIb4WxZBq&Tl!CcL)UE`A1C8ufJwk@ zD@c|Mgr=*^vEn4>tQ{nkHAeX4Jx<|j5NEWDwj1Knj(tDYxTi0^2QlfhSsD2q;wuS5 zAs()*XrFVO3kh0AH3j!^F)rS2yJV^j;RXhy`)&J?Z&431;udMpXSCKvz4aTcB?B+7 z_)lZD@md*urx_8#9z(ojdwl&ibMxHPGbtpm2S1!l zEC&|SHp&lJQo!iK1$gCG7XbraQKv>H)Rv5HHbK2ZAvr3T43KMku@)eeN=C;h1J7Mj zxXjY~t)6CFU3uF^f}wo}%Cc8bXei{-6zq%aiRPT8)%a4ovl8l%ROoI+bIIVY&Wcm3 ziesNmj!6W2?05jmYk$kiHfpA%inX49jkwH?KmHI1u1bRrv1RI_wm1Sv#4hrcJ4NVl z#U4z%lgho=O9d?zxhl(s43LnfN)?S-2_fnYZz2WRkVW2p=w%xB38a%J)j;3KRRb%r z62);qfmZY%lFl@9McJfN*5=dTR*0F@Th@EB;=Ifd`EM}C^G6w=d;TRzw%si0FTbic zGOdr-9SWot6t<5Bh@FB$z|5D_BSsRG0VFYB2FT5odF> z^6GH%c<}d?IMJHYy36t};0`IpIg(ln8W^+;*0HxPn2$HzfmssgEyp^ph?bu_W2L-f z#&na>YeC4LwZ3JsB4h&^$;xcKNA&Gzm1wTGQ{su~58j_>{PYvp3{Fs?2&cCfl zLW1bSz6P5eeHV{Wi#5>e)DvvJX(8u!&U8g_A4lc=_H5wyQ}yDEyRS>L`7Bac(}ud=o5{ zj@%y2wqIKf+bf|to4iN|p$G|WCs&@>MM$?uPmNowYuA_6HINwaEa}a zHCeT3T^u71hcLARe>%2rNoy?$&;&IozcJ?o8B*V`q1GOVhGg6o<=RE$$qGTAp;oGo zG>WitWkD=#s=gUB8vP!4ywu^ZeHLyk7Z+@Pb`aOX6?9yoVrq8%%qAK2PR zt+2B^90bdJV;BVXpfblp*p^m;4{xssF$!KmeS;BZcxz~d3lx&tVq{o6uy}X}JfOtJ zNPL?R@duk=4rjE6;1x9%qx25`tQiu6{+L)xzW4-Y(f*Oe9REQ@i%4(gzB9Lz9Ge3D zwovYSTNU~G>hfuu&m#>qTzs3`&XBJXVYdaES`e58UWK2vL5-ysLFwwrV5#T9n|E2-Ig8 zF^z3)r~ispMRaQ5izQn=0?EEq<>!T&S|7)DVSontkTyw>y)0?QB6m`N)~KFWirtaa zAM?~TT-JBC{P6{A>vc!0ydS_>{bDA0$;|32vgBqytd3~IAk3J8mQurpo*ZWMIH~ah zT>7LoT%U(S z2Fk(jcr8(VT8Alzr2n`Yn1SxDP0TX%T zkE@XBF3On58%+>{L(S6RT)AlJK2H?xH^%9~70SkpRJRX)kz56Nh+Nb{{-{ zB?csJTn{`bRw2`%4{KawbLiBW^OiwdIF>>)$tkWXwm~ebQAxtXAv{V@gvt2_&Q$Ze z>gt7~OO+3`b8*N7jWPUEO1Q?S=_ws_YH9te*0TK}(E9p!J4PCQ8n2O6`4N)%CtPB` zrv4Sa!6tswS08C*uf_?vTCB-ic8)`+M(LH*KN?E6ryuC@XDDfv5ROD0x5J>GY^`jZ zIEeeY2(<6;Q3y+S7pR~ph#eJH7jN!YQ4n*Q@|gc5MZVTr@nwh{s)9tI^j#e>{)N_y z%;cuVj;IU{n;vm(wa{a@`bhekA#LT&Kbd~lYjzAb0qL~?e|CR4o@T{}M-TL&Xfw7Q zYchP~QFF^*0&0Nxo9eRR*Pq$Z?1YNo+?-X!rJB=&W6$k~i(^a+>8x>jhZ@*%@F+AT zitKrcXtYW5s2tRjw2FEoT$KXdpr+qXU`J=K2)K0C06baYCgf4tRtv6G1_(i;uw5NQ z6Qx>(;+$v`KOAQj8S%wQ7#<~K#y+Vml{|;ZW!cob)Jpq=U^@7%LuB~-KM&mI6C`@CUfT*|k=9Exo$JFD_;ZL_eA?D>)o~-?51~nXaLLAuhHa3c;gwZyuC1iBtQ+U{Lf6ED(M#i_zp9%haB1jC?806|FX~xITsXy5lo9W8lAoyDqI!BS={*kN`Y;9H{2ZDpjgTE(RwSY`Q+oZp-e7bt%K{E-5q7E=Q0Ji+ z(x~=iD@3kFf=h3G0#E|Y#GbgZ{MGplcn81BCRdW)#SzC+Thj~=lJx5En6kyOXqBbo z)Qc1(e)CvfwprrM0wUVW4m-X`zBvWC(hh7gEF(+H?M76PJ=dK@M$61_l*+;mGEsBr z{#6V{(esu~>hT((7v;VaP@m^LUJ(x5N~+7(ml2Lp`5@c1`j!JDwNS>BO z8a#yr=j5g1#06KsF&%XYY)K@BSyM2l`~|Gm{slW-X=e1M`wQ-6>WSAz;NzY9(i<_` z=V`nWA8i{p>Oqg>egbb>*Z_$vwOlYbVN&yyc1y(#y;qK(`zLQa0c@Cllku9!>?^f(=P=M;5v2c7m zI8*XcC8!M|Ii7;Wa%6&)BvX6fsoZ~>qmp~O4Eeq`+~E%{C}H%DVEGL`CDT#sq-wsM zC1uY7ocOMQ{&r}h9NP-7nzDuo513hX@hiBy8Hrw6k12Yz@lKw4lOhA)c}ac}aTDXL zyV@!nG%kv}YU-D1hxUO4LBb%hU;3Jb36fa{9(ebm6+!VRaI=~NcC=dyH|}4=$C-Cg zYF<-s7PU89GGRIwlapcJ4eo%$YgNm?-;qIoG$RrTEcf@LbxjNxFlkdb$i%jCWGWnH zsTtEstP8eY)a2JP-pDMF*5j;f*{}~n8*GNOgA9v^(CUhfyVNWhh)_?C#cIj+F^+nr z%jLRQ##e1wCb52b81PZ^Ro!0H%(SQ%%9a zv-<(=y{Hg*R1ryY&5Rqtjaq4lu_z@%Vyjt(RPY3}TSE)f6|S5%u`?0yMNWCQogInc zms&&PcwI*Mgi^WDf*qi+&O{E!R-;B*w3nFc09>b!fyGX}W$bR|4`P$KJ(cISlpiaj zcK$ucDlwX-XkX^E7RKu0L~f&#yGrW$c!$jY3K)-?ZO4J$;h}Q+3n*ze7_;^Yherit zKbiLqLZyNmA|h*11Eo=DwrDor_kmV2YfG<)%PcDl9V5vHr6MO%o<=apBQv6t(Oj=f zr{aT=nulQ%l*Js&3bSw`)kpt*t{QeX?d|)gnN|m|g!Pg+twAj?BNTHxe>VQQqDaZr zTlvsNm|5cp50K#hoGavJlC6z>BB&g0X4jR|i*2p7oDnOxu31?yi5ydG3%aUdmB$gR zZsz~AUHK)`=njFRBJVn2$H`m$k#+a)!3>sV(u`ei0pCpk`~O~NkxO15&@8i%7KgRO z>&)wRG6@PrmAH#c139T-0baVBi+>MiE|`$qRwf>2p+ktLl|EbJ6B&;nmg2k^z@6Sf zk~Tq>xH$vGI_XTp98Byidr!iQs{X^rRKi@Oy{4}%oyOe7!8}4I#agmiQzH{;oh3h% z2Rqm3Jr7>^hVmKF;@B~!dS88@s8)xThC~ zAC!VRsb=hSab`KW>_6bNq|9YI5^hJgh__FG1BDcKmNh>!8=Xg5BFUqJvT`qIy}Bqsic|Y z4z;q8oLOF4-78t<8g?xDX;1<#v9u7(+$0})oPuy~Y?d_Vw$wdL0_s9xG)$T=W7Tne z#w-m+#DNE+*Kj_!(2Zr58qn?yyjE(u7brdSXS|4x^5qs-1+MNIa_*wVwOsko0@FQN z(oR|hm`8=)^5A<;XP{{emrtvLZoSR4O7LUHYci-QM@ig5jHLH~wb<+IFv)FgsZ5Ox zmgyClI!${9FV!Xi&9oitkYJu$Neq|~o|!09e}i|&d;sI2jPO8N6au1*S6%vi3hwfN z(yrPPf<{p$nudECR!C|f7q$5k{1!Mc)^tphUfFg5RD*&Yu1!&|sq_MKrpW1sFbLWY zbVVz1!-jHj_W5>%V&*+T`#SN=#=871%4N9WJMt@8btpTNXfPozvbz@0?5J)pD5DEZBA;^6 zi9r6Q8QE2?7|v*T6YUW;npqR2!Fm4qzE`-Ub>}oHt1A8we+e!c;_JWJ!i>-E*y*@5 zTrzear2C6mTFKFHq{HI|NlqEn#e5ynn6({?rPX#8@Yx^WTIkXE2(fQNN+sHakc>fp z*)0l-OjbLCq9Jm;pEDVgr=SMfj=KU~aMSzXIYge4SJlRCpde_t-8<)^!B<{LBHR~CmslHU^IDi81S>ldbb zl04`LoFR@V8Fj@fmFDv~nB8W9uk`;EkT}f9v2c=%IB7xSLX_S7wn*L5$uJ<|CCdad z3b};+xOFJ%pmlDrkG;~QT4!EuZid&C^LuQrTM4j20|^R{9c@spiW8AM8%+Q4l9>xM zeY_xso*x6bT4o&nn)t8~J$c&BRc^pRH#cfpGv#|T@Z5y;z$nQMkf_SA_0hCrm3#N0`|sYNYqKr^}kh6e5nQjZ+{AV_%J05gUo0s)t@qdc{8I4K+dTS7E+fZ*Gy^* zkmE+GxiQI@o*4t<#-on=Vu)OR!%8wPX6Bl|hm-l`wkfX=w<4LCi;RA#VENPK?5~#Q zfE7eZX6)iRJEHz5WYLXTYDjzed~AHO z{M#6IQ|_FVFLm?4`9IBMe`)X-LB8`e8|j`4qd;eE!Jw4dIRM=g2_Dy8&IGZ@2RGvV zjo7Fh1yh1_{T47G(Ywh{c`zDU{&tp+>cc#0$DmGl=oNvFne7d`D!?r}l^=-0am$zc zRcm-)=SqM^*6yt1_sB=$CR$Wf+{bJUO?m}i)iUM7%mOD7LNMh3Gg^bkvuH?@m&?6F z;tU+QKrZ4M2fG4wANd{A6?a$B9C_4~tu0s&93Gjey^KzwZew|?`2a!#Iyv&e_C7sPpO#am{3g&Bc+x#Aw2#rIVzQ*2)`9 z9V}QtVNdbXjrAnR8k0E3((C#0&v4v$e-&pr)sGKW96jijdx1aL4qJ z>z`txZ$8Yfzb3eaJD<`CfHu4xJ#Ayrf0DIna?Gv>hZTdcXr+W=0hdg3OwVrqK`LC&;jGD*HUH`f+VDvMTy^VHBiU zG+oB709)IbCBbU3FO|phVdhu5*(Io4n8AlM8cWRl0CvVrmvaC%=indU!fofnC*N=7 zjF2kD@sSLg-d0B60t_K)5)mm>TQ-$LIk;%%4%Mia7IJ*2O<|ya;N>l8>q`OP@aE#I zZ6~31;D#A#x@^!sn%)8xS0u?7Imm%$&i(=$jqt`poZ6ps9JehIA#mI5P)d0DK$a)| z249!ZZ-a(#@uhrzG9roelxtk)@bht?+LPvHVKrBY2i#m7?iFe ztR=OI`isZIEhBoWbQ&m4o>^`PFm)ztr6q2?b6D87Qh-ozGk>()y$96mXjPf?vhdsx zH6MO4{k^$cj2tgFw3yf4LOTr$IH#Aad>_sUAMONZxFWeSk90}v?|7tZcNFCMW>{Y! z7MF?xJ$n^KS%#;C-$ABa))?@teJt{Y-ekW17mz8_g%#K;Oj{;oGg8T5h$%S)A9YME z>Ew&bvy^H|tTD@zD%+9WiDAu8-_krhfz3Tn`qVe93m3GajJ;x0lSvrbQblX~K|M^QF&WxaxK@Y^L-fQ&QG{!-VS>tVrW-&Q*!$2z{CV z4usOa865r<4OlsDhBU#@pqS81Gp#R#k~&ha{1?euE3vt_=f>GRTC2wQlp)W-%2Q@RXCCp7ki~VaAi$d)By-xv7s?f_I+_{C z(KOiy+|MwXO!x>re~?DScOkm~xsp8yyE|hdi^8R{4&@GxpPh`D$?YpoN?=gaZ`c$! zH(B0rEu1jFGw1m4IjFpmY8zEFb=Qv?)W)fR?AFG1Tdn29mQuDkhnc93?r-%{9XC%i zO#AGXsAWZM8J+x=)6lz&bO7|L>fa;F++^{w@9_gsFg@?XQy0&UkCnE8R-(K!$(hRT zjc7W`V$S(9wl#eaS#p#M`ae6Js91D^t=BCSPsMy!8vP_;|<9+k?eO?!@ag@ z(;u2|v?Wr24CT*PBQ2koh45=?{r4q77&cFy;0(EE7W776G3u*UmIw)EXrw>}9+vwy z2|A%!@;Odc*_)5*J<2Rd;01hCKXCB?u&J8i`56cE{$wjdu2ti7HtD_37iMlfN%%KD z&am?Q)!!EjBg7iIP}%mEhhFF+2TLJxFEhd>3?Vs0GlN&*TuN&odxys7bF0l&@Xg{9 zNx6+@A-Vy`QpOCaE0>#Eafn@44IqzBmu^O;VU(wjk*|iuyTyG5qKW4@Kp&DUdrWXP z#q^)Yij74Wy*UB$G#mD}`2xq8K@%i=`&_yAHxt{T#uPQ+OoO5^aFd$gN5~D4Js+T@ zY&is*`X5Q>9T?;NKk$dXQ+JVti1P@}z6Xi3&p!JOA$IITaX|<*LR(@sDIq8gifgTC zg;a;Mlo%yODn?0bG_CgA7$H=-s`7h%`uiu`-Sd3L`}5wP_vqK%<^Z|wi z?LToI-Zy(gG?qBpKc?ZYs722$Tuz=LGA$o!SiZ?jep9I1CIvp^;K77gcHmf#^p3Tl zTX+JYp}bT)Ln!;7kc|pW+iS??=0>TEegrm*S9V^F74a_lvI!r3C3)J>7=Zd~YBp>F zxAWOQk)BK18a~EN43}+lfFc)8(X#7pIPe|B3;1ds2NVGGbj&g(t>!@C8G)crY)`4u z!|mW>Vmq+G*G+B}gItd~MZ|^3>_5;TsqL~zs${@i>s?FDmG1RnpDky(EuctdN9}Nw zOA^Ne&@c6zE^v{gAS9=+4q^?BI(RXU?b5T`!t_mxO{2yI2kZgLDJc+?)<nOtsW*u~f1?NKiEQQWav5CsvTAWzZom*FnluB1c)fkmf_O7qwAE9&0mf zQAO#soFO^-ceUVPm;CTHC|_wa^j=@ZBLwN5Anjy+l6C`LF!d96h8$WB%Uu5<;>}QA zh<{%LKfarZ-(pjmOXi30eW>n@#F3ia3Z{}v#5LA2=vL^IZ(hQN!e4f;DNLVYf$&|3 zb{`sTd_#zeFqJXdC&95&xhFJnRricSQ0I)72a^%r+s1=}G+6E~lRgC1dgq%|FN+P( zKt?6$aGOO8y1?D+hgEF1nQrZ(t|}Ku;5m?a*TGbzmMU7k>^Upmqb|(8 zNl1hZm3Bi`dV*EFOE;OSUVB&v>6q_9xH+@z@{FWl zG@f)T1WVL@v2p=bEb&RVr|u*)8#-WJQf51}K1oW-yInE~?=Y7{)`9Z)Jje&mL`#cm z@ZrXN80j5#TTSe2hSdKNmFqnf?Ejvg9fcc$sP?b^;*nly4(YJg9j7ggDa701jlx`u z!zUj$6c&jpJNyVTtxIsa^lD;|^uYZF@+7;V^5+DpK!|YMhn@=3Z-E=HQ*X0fO3hdo z$Mtld5_aKm8ecaQiNX*hRc^Qp8EeGJ>Eifg6cMiZP<~yw+j1k(PUCDSuyzxn2mXFq zQoc%TEl9*=Q`A*fsC;95yHn9Y5=UcfczWW6p@R^;;~Jxcd8u)89Z=m=jcQFQvt-{g z!=~UeOPRjCwf({ADnhuCwkIk@`vx)lT}c51o;O=|4~RG6jO7*v7f3XONd36=s?}T# zJ#M|cH4nTg$;x0YxlQ3ZmgdO(q(KNWTO^(gQ+;j*vul9li3{ZH26UgH+gRyby*bRi#g!DV*^X>q*&B7oswJvj(V6 z(G%S;P;OY~={Z4?ve>K=`TZ^M0H$|^1v<@A2Cp~xMf*_BgEna=-wkq~x9G^|_GxrL{V6neWEwWc&$F7{-lxV;0V z?L5Pw>SL$CZM9HPF)m2Hx&uchudbpz*oSIyQnhZ}ru#z>9ZBI@gREpM-NH)vA8>F@ z=6)O^E2t{JhsRU9Z4`+4fN)doLaZR)%TKd;?hWqKq{8b4j=G2jCX;p!kxX#2w+dD&w@M?|B79QnG!Yk1mJ8STq;Ad%KIszHMI>k{K+pT-J{*F!OMAN^c6RTT4WccPqG74PDVC^nQ= z405ji3Fd0}%_@1jc&+^W8y>9z+|ZNiLg@77RFszvXWc2%jC5}lpG=FC|AyUFzyQ|9 z)Ch~wode}*tWki-@ebt13f@NSc~d!^&RXl}cg9GwaS?eYZXCQOgqE8kJ6UMKpXxcm zN^fr~JO46lQoTN-KS0@)4EAxhGe%+8)+cdPv zo_bh3?2{nog_ohvzN!|@p?J4d&PB3=<0{TXe`U_b%JO}H_bGX|wjm)&o;GDuD`MQm zUAo0kLy3wL z^tNT!h0)ARkKu% z6Oj);L|LFS*97@=4BAeR>O_fp5P{#Nl<>CC4IBERsgFE<)3D&U+c@}tUIHai(MB)R zvQ)l}fMDmTOsy~VjK)%DM|_;}WYyK`q^V*U&gxp@;4ryb4EuSY`YN7LUh(Xz@U&I< zg=OY7cdk7C8{KKukW?Gh(Xw&5W!j|UX<*!41y{KB;@wDHS4}h?k4o^PkL^2S;;E{4 z(rpD+VWAA{>L5!DnEXXryMn0M_0a`WB5>sA_6IXbkHJPRblY`3FPx4`oL5gc~*g+=d{BgZ?|aXgd8vd?;QnDHp<kk;=5KyP_%PsWN@^BnVc~W_4uSP9s@n zCvgI6FLBUkDZPkKqHqaNjV%w%i^fXlh=>YiS+)QTyIn#UBGJ`Xf>oWK`(v5tmZ}DG zR57cdIWVSjF-7P%g)?Di#Oj#qAfhkmVP+X!+nj;bOK+Olx0q*_?&&9|MuF}%bVqNQ z*#a`Kh|V5oeQ^mzOZNnK_6cGUDAU!vHJ&|U8+24ImMhYTe-e67W@2$r9F z1cot2FBm0@nuFzqN=xde=T3wv_yZZmQ~@2)Iuur54VMp>!!O6`jx>N?)Ew3li=jdY z2@lgSwyvJ^p%mRnEy5Q@@zYS<(H~_Gd>muPOVdoxP6XMs#t?MDU|Bjp-lB$aL* zW!@2QhRFQdDt2+W@}9Rr-F1Ycx|w>*Y%9mK@E6V{ zTs3YayS5GN)@u>-Uyx{}0S4VE-UYF)Rt3NpbmwU3hW?WiRKACUs;JE7tqi}*tS!45 zxK-L`#s(SN#O;$`y8@-2s<6xp-BOQBBePXbKVv~nRiqP9t0AZ(Y7(X7?Z&t#;e z*4n15ToAT!7_2CHByQcrVM>(<$L&VDCz(V@}g-6nA8jnh={3NMu z<|JO|t|7dgz8tCSt)Ty`3-Bf@>-hrqa(keWE6Ib<8CvPF>oMsavt`I|G`+b>T1S^8 zikrF!C% zlPYujoa>OTh*u11ZLECNHa;6~U0x7~P|#l5?t$Ny>x7{aw4L$Hdg4~DRUECPZ5MYe z17w($zRwekldw+3|}xYm5v@#Y+$zr?1ecb>7F`p9ml6KJRm*;o|5ww z=zzF*c{nvsRX87qO<`@WSVt|}D zUueF}N3U7CnH49|G#P@I7VO6NX0M7M%iJi7UI(#o$!Ea3vCB_%G$*qtM^jV@(>jnp zuM&GnvT3hkN@Yb;#LGj<#e+F#8Uo(pq2 zXBe1Mv{@c+1Urea-HjQO=aTmek#}5K*tm9aY()gy=E=Pb4#}Z}enL!=)K?|nPuk(& zi`Jp8$1A41M};P=SN&hVN(fU*oQ{H(9teMl9j6r7@^WhzWdEk_h(kT5OyAlo@rAPU zd_0XB$Z6GzYZOn@9BvC%i>Z-B76bJ;dX5)QWup(ZplXz1hF=eo0j%-P35m*K6USHeGoV)I0yfGdPPJ)K`$gdSs-6< zP0kZqHH->xs63nrvAl|MLLrWE=vaBXb2+d7qw@wr|)P3uu&k@Gvv z>C)_dQ2NwH!zI1if#&yfz^hN!Vu>~lijQN$;t`;^#f=qQTm_&d*N_G)!0YI(tQ-b9 zPcB3=MH2##P^03i;+Y$(svKoZl<63U(z6)3e?TwnqP@YE3Y3#AQm6jxk(rnm-S4}V zC!{DCH8a)**UMDFM|loFl7#mrKTc~O?BfA8*e=V9k(+L4X*CkNMWsun*mxTZKY2b% zBDUpXJ!6!3YPd_~s|d(p%VyLa%6zv~i6P;fq(4lSS2t7>08>3I;f!IpkMe5Cn{%O# zJ9Yp11Ovuxajakqfo~s zV>lLBP(Dm%T-``5RihRG*! zubzC0e8&%6l{fKe1}Pt4gdVm9)gWbxIu46+Z-isx5urYNx!XZu0XY5XsB8=tPmGKi zAD=7dnt;P+RGeUVOc?l{*$evM8AHTa=NBjL01&{|kni(oY+OFc;pKia%SPG{L(2wYA$O6XqLsdA~QF zh_`pM>#};@711vq<+Tt<)8#V(r@j`s8r!1T;l}Ux*AF zjqRJJy2%_UnJ&AZLVrT3F-bgZB%$u)j?iKfVgXZ16i~KxlEXLQ zrbOdZkiRZOc+AqSHRfQI=6F~CP2=c(B-4wCov`LIiM120D;FZNb(0%V&z6~0&8fbWyKN#|<8t>6vM~c;&+&Sam&-jTl|w2RD#GWcX4<-vY&czP zh8B>P0}-c0qOQuF;+icqOQ(QI`5%vjm+$n6Dgmc=N&v61LIuBIqB zIHw??v1~c7w>B*%NBVC+qsl{ACovC+UfZgqhN(NzYM2|=kI}m9;bPm4O5J(~I6FjL z{i2DF(!R>iL<9> zh58btmp@T%PjV{-my5dc@0i0Rr4NU?L{Dl3(RnAJaQj=zx#4cRO7(Jxc{ZUBYu-fu zbZdizId~ghF*x5UTQ8dlHH(UqI_<#T{pxz&t=e_GB(VyG1Pgzjs=7e6(v{Lzk02Oe zlx|s#=A1rH`6~k{jNsup0=2(A1-{4XMXT{UhIIt*V(l`y9@>Mz1Gqdy)tE)Q3UwFP z3I#2F702VBs@S?y@-MhOurnx*VD#=VlA?O@!U8fMR{d~BpEU@Yr#L!9pKSiL^mygZM%HxExKfd^W9V3vsiw`O!1 zf_}*j+%&aP^$4=LYm^U)C`r0l})B;aFw8o#NQ@IR|S z(hL>xJRO%HHi5M47cRuqO`B&g5Op}PzLTttlwM=;a=!O8U1Y(iwFIjSe?u-=EV9swokqDA}&Yg&Rq^L5qLv^nmC^0}jr4#5- z+Es;$w2@l_v2U6!XMe5rj3lVS`iAr!YF0_iRBr95JE^OyCpT7b_>6a9Aw%@cH7p1l zv8tZ&1vWV0h%H*ePf7RBP;Qw>)$GPE6>j47X7odB~V~K&J({xi`S@b!=FT5NQe@RIZszZ@at5gyv9j}_b04li?jm$W-J6Ex} zrs+0yvE*{8pq?Of`s`@!xBNUw`x)M}SR2F7GU_H1y4h+ThjYW=Quci4#6K%hvvYzGCT$+g(q{2 zZt1T=K0C_cb#7I0H~Sl{C`yARdisGcrIk4Il%d>56m5|U>F6bYmf#A0ia1^L1fBXp z{uw9nAewSplX^vX`1`W7#PCX$rS3v8lN#&H>@~s6N+Xdg-XMPPd|pUlHW6!*#p0-x)}9VGshJ5KHmhQ)TQXQWF{ z54(J-TV~;Y;-X}S?wZYPIgL5d>|CiG&uY=DW^20=V&$zyZi}4m%vTFRTsU#Ya@kV_ zn{%6o8Q*-002->to`<1(?J#*uE!0w53&}qPrc#oZTCv09)6p}m6;&L=1T-XWbXt&R zEp4Z$Rm2pC_Y_|0(x_|+Ja49mcVl3>Lw9U|aJ<31L?zNbc*~{rS{7kay(OOM#T(#H zc^&atCsjhr^0Yzr5m83p$uOk@R+rodW(ws8C&>vK>t0tlL5H+3)DV*iojmlpV*eVUc6|RH2O=S-yyRu3dgaHhD3rZkp&+i%(+8l-<nNh*z|B=rZ1kn7VP@P_7H> z;Sxbz+nQjNoulv*w|oe)cS^(#yZ)2A2u;y%mFJnbV?GH)qtsOWuU^4c?P!!&v!xZm z3gh|%#cV_QcT6-#g8~2k8FIG`x8)!%;l9@o^Ues}-WmcaZYxv;b-;$QXLH~OVtol*Vz%qo!c1DAa)D|Lmv9!jdddu$ zHW1B}yTovyk#Y>x!x6>0y~o&^>s>UsybyGf;&xaAE(`pnpoK*A0WI3yF`RPgZKDW@ zVp%8uer>37FrIqgs6SF0j&agZw?#>-U51C4*Wi@;LGuB zs-77l|2{B$lKlwHh3ZiMr78r;O2eHeruAUiecjuY{111sTyJf5B8|Kqr7+I`A!e~N zuPR}Fo{VnXES3+TDr$C>*L^1ZH@PRZ)lqrUw+irl!3t&2pJv6jaz<$5U{U|%re?Av zBrg_O6xx|D9qyLb2oIgd%5{3ibd)t>zvov#zmC!~sz^1}gsN9Mzw5RGNY0!vZDhU&_0-c@I?hJQ?k21U7F%#i?FsL8NeZM{IzyW8i!YU% z&rmCfj_M*0lNo4^?sxK-C0IKZ<&um-*j#>sP1-F*4(`@-nu8^N59$TC|KF&cQjwHu z$?4dD@2PN;YN`xE$X4vGRjOF@#ccq+z3vP}!OUqWTMAP(*g#Gxa$eD^oQ3sCHrTdI&#nvZ7SxfGAD|N? zI~mHvC91SFah^;x85PAIfy>Kl1-;U1;4N~f1SzeSY(K{$FX`!RL6@X;7-2~cwr>fO zX$_IT#n0mlaa>W;XZFt!UG1oBtdFEdGGPhMr@JbbXuWEEm++g}F0+bL3p8tRHp0Y_ zit*sO$O#a1FkBUDyr-KCG`?)Jj2O%LP&_?Q8V!a5-_;A&%e^*+RUESnD{Rs?2iaur zB+pe7)E~I|6#Fdp{p~K)iDV1Xut?P=dtcWE%KS9LhoVSv(E%4>TOaG0?XapzNpABG zqN@XLJ;daH?Go$`>08%b2p;>~C|NlkDSg0{)+^9eoNnulE>C<^&|hx99G#9#9;qic zl;jxH$1^q@oMZLmRuBT{KTo-kGO1sq)h)5&A9CfXfCC2*4BM@JmW9&*ay^oRQd>XM zh?NvEEO`37<;7GQud**w8q0U1442$o$Yz`CuIX~SKiFTl7qjHm$X%@s29nO}SaiSc zL8iQj2kIdzvHREYMxiz=#)eOZ3b->XzK;1iHvzVG)W*ZW7o5 zVriypcH-C#^w91`m|R&8tLoi>O*PTA`oKkN2Fes3lOFB=qb$d4(7f<-oW7VUPluArkl>8&HCOzcI3M0^JJ7_ z4LY|egRC76E?c5JIE;)LtEUe`R|?=W-4ZUAIzYc_OLGdOVr>l4+EgB5 zX{0om44K_h9hWF(@))Q}yD(umijT9jRXh=;?cv@8PBE1{O)ptU4N+iLs9Adm85ky3|ILSBJ0{Q z-X=4qpeii7WMtqYvo@9G!R!E|u#xmUj;do%2$Msf@>L)Skn)4NY{^#^_FSs8Xiw5g z*$V+1n>)j*OnOlx^aC#akLhvm%Z7`b@ddr}U=ZBfSwcSJBN@UL*kST5b(vO2^GDg` z)B?_LvhJA5t8wi)wImn(&h({Jd)KflV{rlrz6kI3@lbAbebDm{-EPHyV-3cEkkcP5 z$O~n`wptu&;Zby(*g7C@(s)#Gf3)2E4PJ%I)1l3c@C?m^C<5sw>=Wdc_{oNLL62X>lHaA<5C1JVn2LCbB z3}@K@L)~(Xkl`xmHlGT;s6Ao*<^1+NCN9t8Rx8I+9dX;vSdh5Ro@i#4&T^+B&n?ki z;pilUllBiTM8m*qJ?j>D-1@#@!gX)0Bl8==_Alyj4Q1j3x5n<1ZQ!84g>3oStjeVM zcgUH$hN^W?>Hxzr>atbY@vX|GYS%6A$)OtzKuTYrw4yFPHiaWdPWOrrBk-eOoA{0} z=ruj@oUCkM5bdn;?%!F6BD`78u89f}r_0GMO!S-XNKyJxKl!6G{0@m(6)bjz0Gy;D z^02@hM;t~@0YepE#o?1t6`MGPu+h(ffADy~a_M_sBTQ}zYvBs;4AsXZ;RbfWiH+!H zM+q^Iku?p1y>FP6nv3P%a0aY1KPF5TFLLL}%k6A~khn@5duBrrD`f{eC|rqj$UFg; zL{=VD0a8eHySjDmjb5DG4t>R$1Jf*SBvZzj3xK?%h8Rbo*KxY710vQ@heIuN;2cUR zk3dZR7NW}-A(MY{Qz3h9d?Hr8Z!r#8uc|g;w=ptz{Y2lss~n=0q(%k=R$EVU%Jsp9 z1<-qkLmPy_Y*s0MYobh0S-}rbxWA&d0j;r>s+%;4oI=$gUik&J=32)R2Sz-%4>OfU zyHw9@E2+J~|Au;)6=ETEZs&dUx6m5U;)*OE-Y^0q=QoRwm0T4S-gqbzU(~ItT~TIr z9GMvMyHb%vkM+_&Kq>!WTwSFmj1bRY1ZYt-stv5lqrynqs(8wQlb21$=|MkA^nAx6 zSSmTQglu$U?XHfTlzj<^Woxc22u`7uxsf8d^N`Q8bjNut#jumg4Y5Hc4Me58rKc=1 z-T1;3u!wH0>Q0oPG#Z;#107=rNk=+;p_ zhINx>moO*11whQBrys=HD-D<5o&=NU1dC~^+d7M5FN-0VZ!Ma2-e~+_&YBXOYh)rJ zb2g&96&;nX>e}?$YPUn)ng)}3sMzXQRZ{C)T}xHcsJ=M7@ziPF0I`~aiDx>=`#9YO zse9!G(rOGgV&@L9oCm5h;~QMPOXdnr>#CmA1oOzILK*NfeIAt!rTHepOTAHOkOe!m zAvOHFgNq=k(%xE=k^=2?OghFM3C%z2iJ=l{HT+m2)*xK7++3JcN13J6?vp;P?zoVi zg&|5+9WEQ1?%LzwbS(36yrwjRPf!NH-^)mnw!gy! z?nOe)!K#n_j#21ucU1a0nYKNogU_Ir;&5CCX+fy+&90+a{G;SaZe!liX$Pl$78NJU z+8RkX^fHw^Berz49#&NWO+yLq%x)yTs)I#VJx6(ulg6SJ=Z%uv+mLgk zSE5oMdea<+_8L2g3WOG(FgeihbKHmvYb~YKR)W|HD$9ej@b)e=|K+wAS6@*=6830m zRqaX^xAehw{3u}!(Li0QZcN5PgqJH!8WbCeC~^gFaiLRO9!uJ^97WUl8TY3-LlN4h z3m7u=^LYD9!=-H9)#B(5wJlf<>i^L%zCh|)&0aZQKHdZYC(e+XKgGvtMocysl3Y#l zc0veK^&*cfTL+|O61ju(!C#w27nzh9&<^E4qpvhegEfBZ#7SYLb03Vos56L_*q0EA zurS%0!E2j{hY@1{1khUPFI2WP%SIm@dj0pHx349t%P*>K(3l*zMLts9A9oob+CRFZ z5m;U1mbTMi5Z|RTR2@BMwDea!70~rZs9=I5t!0)=Mc)U*R(%aorzjBe^TzmOlpV_y zNrD_@aZ6k;&*_OKnH2MqUMkVGvT|^2sL#>I>7^&7=@)F$;5GmroJ1b>heV`a++w6~ z30E8ek~>C#kbvd+ZiEl5u!n%cMQ@{RWVB)xwj*5nn^hlL*CKf%$xv2)+!)z>4`uY+ zU+yr_$QLVr4U4b^%BnKRT`meLK>JB8@bhN`!g@T>2#*XmCrh|!f~xMtWle}UbYxE1 zSMXT8hvgwEQ$=3Ou_MTc@$22qup*%1A`t_wdfJ(^OeIyKW$@d`{84vOv(YFh)U@qB z+?`PT1zZ!NjPBC&s-j8S_sZrjIO~ob2kWNlv1jCgVc6Jj`UI$k7|xkAGhivMRveql z#6mNhNucIVR~Juhu++3J?APB%+O)%2_e}Sy1WVf}CAW8jcm*cnQwlmu+v{)@Z0_b1 zO*8y*T{*s|=(*AIgCCTCZ6)V^x1F1$?*zpsOl|q&RkeA%Bb3v(x-36!I3;d^o5BVv zb8OEHV8N|YrdSTRi;xShAUW=W=ff|zsXr4Slg#F*DuvBqMD!)-H5Cv}+U-u1w{{@A zxwh$p=0Wb{I(Q{imfZ|smU_gheB9x`npK=#VoOQ=0L^NSD)Zb>vANB9Tqi)C@e}-p zi^y(zUIU0A~EiiL>CTB!kwi5305;H44R68S%VP`S$RP1!@x#;w0yQkpb(P zr!u;TDY0RDZPgR2B3E%IMW{w>G%ahZB3az>%UYmH+up8@qr@iz*TV4A zAU|~TREpVUm4Z0hMBG9J1}n0KOSeNjVd%;6vWaSRK+zv7&o>xs?h?nnL#2rR5>uc# z(*A2v62aw5Shm)_hKt&O2)3yO}C(qA}cW(#nq(~Ex%-GsX31$E7*c{NTh!i-8NSS zT#b)YhD9ZuRK7~`EEe*;xd30unirUyv-La8k(Lw*PT7aWvP;FoaQjwXGSZ;`!MhBq za9jWw#U}tze?5iHUi;9d^Z$Y8?I}@REbT)nh@i?55+QjR zf)Jjlo?fAPdT-=jN>_QK0m6YIHF2Pztwej2P>f{Ht15}BV2q?DgvjL(RKt>Fm<}1y zUF7vgOwXNERZJ3s1G1u2Sb1V5#N=?xjZ18OYpMcq0`yC~b$e|{S`yP2wn1UGSHQGP zGC``oD9Lfae9q{Om2#;!Lg631@J(bg`rxsIQmr`318x>vLVrvgthFe~kiYN3vuCR! zO7{{R?4lryg}R#I2_b?muqI-Uhy%q-V)wxveYF8#!7=H4!*FOpF@CV!-%l$+@fOg-K28=k1|fddbNc}l?;xq3g~I!_ZksR78r@OBKjewyQyPVr-IA;# zHLf!NqJOSxE0o)neJoQdz^9qYA5{eD<{pAB8}Bo$vf?NLDhBNb`|>Mk*d1nlD?wEk zO;IU_Nt0#a##BwpkA*|f3F{4XdQVm6vbq(4)soiL+^U9?_iMbr1oGyRE~E`eAZ4u@ z?w8W1@jkAq=sP-jQ?Ny=g1tnwh8}n>eNI_$5JYuUg93L^u^o)_55qX~sf5bRmfQj3 z$s6<<-0aDRfb{u>5?C=e(-o z>BH6LM0m0<6pj`)9QK!!s$DNJ$-v`AtXyntPF8WdVX$OhHAIPj7EpFX^M2H(kPHI^H5gWs@UeXy7LVpgi|VVS)9>F#_VDpg!43&A6*!7Xk#^$HP;Om<+W%( zBxr0VuNz{d6c%sm2J(7_!={rfz#hX=NN{o6Q4kYJMA2QHC`Dn!g5)OdGxAkLTj5AK z5pHnB@CIlX)+IF^e9Uczu9Gqqlu7wi={g;Wx+_?M{_1Ik0p>1>PT zT@yUkw#;qC?T8lDlz&sH+?cjW|AfJvUOHd{^I7L*Gh!&J;R(~#}K0b!=F)q(sru8Y}Z+I_tY zzg_+dtf|!#Qf;OdIQeWqO)A}H!q%^n{VNS910>7MaNA4Kd{^rkir-wF|81|#cMa49 z=9({f`QBq5K27|CRC$ZBr}YR4s3ms_%y!f7pOj^qt+_n;zcuDq(^r+s2Uz6sLm*U8 zJ0MhujZKl2?GQW@dN9}Fb)4bv&so$48U%{I*q|@B1^Tj`Gwh~@)q(>0E5C_NoGdFs`Mde`NGsg6 z3DbDvf7xs(dN)60By*@&qr&9p%SfY#llVSJrB467mmkkvbGzi5*Kt385zJ&?*JSR6 zs``DsXx6hz`Skk5AfX!8W8gXKd86@u7Cv(pJF44{k1kb}(>fl`*D|V9s+2JoXiI`K zMOzCC>R6k_<*L4M2Oi)GPHkbtNy2}n=)(6QN0c% zn#`NQHutv1<2mA2{7po`z&k?GfXcfAww(a#QT*H+I3VUb6@j1+nOV-BF(42H|O&ma4bEc@+etZaG^fZD44da0g{ecfXXyJfhP0UAd*4543{K#=EqULvj(l1J6l$lFjbqD;7QaG;I!4s zU{^lk9USf-=5%;zNM`Gk3=STLp2cahXAjO4H>uPpN#RX1pwvT;jBwG z8Xox~6xj6nnqA)4K&r`0SpAj18Q|s>=GQNPiARDsaeEtJwth^ zNnh?G{&<}+ay#+Tx`PmhH^Cyo=XtDAU*PdqMZ+N{mvanPwt)0eS!SE0T`-(-`9s4i zCr5J*-yQ~dkA7ppd86HAoX<0paUPW{z7c9}-pc>pp2?uwZve66qx`R*vY>NX@8*RG~MB3Mwk(V?9v6KJDaTHK- zQ3fb1cBxdyzU0eFP+7(^l~urxWV!bn@$R=PfP1eP#+&zo?{?%c%Dhk6+k!=i_q95O zTD!l5KTy2YrG;k2$&8^8m2r}T`uGO#cOJ!zz8n_SWEg+;(-l~%|I0{&t8(~ta4$=r zLZ*29G0%0stah*)u{vJW8v9+3HH(FU=5lH*Gp>zva{;abOqH0SZ!9Q;nAkX zc;!o#mQmqD9-DU+#2K2wmcQK;pQ3W#EOOG!6ouT=ikn~^Id#{df=yGFy+MU@zdXbv zAC$tmYUG=w^=P)ZrVsC@9R*um7kQyhxY>$DkC&itI;T0L4?8Qm#5~_8vZ=d;3{o_X zvuV?iHGOc5U-K(6`YJ1r*rU;2I;Z*M&A}i|{}SeUQH_Qe^!>!zoqWG{njzvlM)uSSk)A@90x|1vutu) z=Zw#)8cASAm_#)Mk>jef>=8abzNWg{-*mCGMU@zDpOSc|t?YA3O#m3B+Lz8(%xp$= z&i$~eDwp;-n6f0wAx($!n4|8VLfOmpiX_Lq%OSz-8E*SOEVk6k*#)j* ziwECk%T1HmMdJqiFK@ovF1Hpc)jK#@{^#OZCnZ#*)>IZ;T}5_;@8qqs-$S8guLAD7 z$Cm}X@7so!d9tlo6pjy zo2IG=R2s_qySkaiSu!?=Tn=^FjNUoZa+N zrkYyJ_qF=5kT2JOGm+>NToA_ws#?);GN85_hu1OIuvxLKHV$TJ6T!ODep9GnV>R=8 z^6q;x_}KftVUx^lY~tia-sqce_Tj&>o8)8|zj~)29A4#9*zpW=&-v1@%gHe;%s*1S zs`UC+^Z8$eP*(BmH1kYU8)*6wtN3sxzkcw@oT)s0AbY9Yoc~c#F@-x?3OeE+>+!PSyN^=@o#x%e+g zlDv?`_I`>8AqA1HGs0vIx3buu!`vy-Er3mB*5HxWzp>({DhugP+(y!yx8|p{IiAEW>S=0x{d?$?%m_#=!twhbchonfXE{Y9EL;cE#p{n zVm$JFOR)4n0)RSL2he0R45r$FJ5h2g8l^Ivij!hH1Bj36aTVmpbD+R`T^aXN+J4Iw zm8H|<2QYVQOp*LLj7|9NKp)|mEc3dGlz15f2DERBu&I{dl@ICtCrwYHA}!MGf%4vc zCS6>XKjw!3?0z?M9zIf%ss|Ucz?mPhugCH1)vf$J5oU~n#$U0U<4YO+*b)90u8hZ? zT&+rzbVJ<+TPkRYF+jJFc@Bm!$Rv#we%FX8hGc?SUw?;I`83KeW7P^r(#ug=p2H1M zkX9(q-#33GRn36V`93t#Z7c`5b_@8oQr9ZUC`#vxqqSke1apL7z$RD3w$y^PJ5iLR zxC-E^5-lS`t^nz~P1*FoN5HpUzEjFJHvHNfB^lD_u#qgeyMPwHz##2>0**_S0F-v0 z$N#R*5DWH#5Lw|M?rbvVB={E&tHUcZm9!I5B{QoY9(yep^_6@f56`BOaZ#n&NGO!6WBibceeLN zKf{ij%w8Fz5VwE)F?K+8rcfCa1veS}0Ej1dH%#*5Iu_UIE9U8&kSoWRxHU=Z8gIjU zLy>}NeE{CQ*__zxY3b7ZPfjRHj}4P%4%T{WIX{kH%haErVAlK1Io$i8Cd~pB?^@*OSVgz$4$=q$g#Z?x3w5|YPTi-QewR$BU+0uw}-rd1)$*Tgj zo#~wOt2u6)Y#--F-|EIOeoH2zQq4Vb?kWQ71EuMFpb`ddw&0CdU0GC%#<K5qJo|MlyO@a&;3skg6W z$~EsBKI!oq@WQ@J)S~mba|{Ofb+*8lqR11GtKIm1Z@yh#SL*y@`lrd#pWrAz4(9aF zm1pX%39LB$I^ec%X1BXT`Stk|<~sKYKkP9Bz3Ev1Tu`Rdp0Ngp*Oc9~ILtDq&STuD zi_B$IVAeG%w>Zp>$enkA6STh0icfuG*kqH;}dx5#wmz7lFS5YcAD3Z#WDYc_n5!( zZ+u)ep4Bybz{hn#EOv4SBUf7OWroiG^4x%OY>nOrE13)V zby#h%Z?BVO-_e-zK&Ii89}V*zdGAQDSNnsL12ZesHpYIU_*ui+_U@zJm;q62odgUINv8Tnhx4yfk2_ zXoJ#Hq}UBsMK@viP8Lv=ZjctS1oFRKX^B!Ukst3og6do9PMSex!cm^0XmXjA5-htq zfKQ*SfSo^4T?4+Pd#nsB<&D>mv)j+|3vu5UQ{L`5-+$c^%%aItC^r<@l_hc*s~fV+ zAONqsI?&3<`~@n6+CB>1?2zi%5gp5Sp$U0X@(t_v)C5lZ+re;r?c*KN@Bs@hI}&u6 zei>5xNwu{ot5kzr-Bv*>mt~WGj`P;GKM(@6SgRn1YOthAX|yg*0T5^Eae!;@1!rq6 zucAa?R?Fe&er@u(rri#1JIcD_ zByKw}zvTi*TVh;VyL`B{6CkXv$?>9+Ii>1u<}RZE*rqD9Y&aHA44mqebRs#3v`ncp zIiM3Ir<76ojj{$>{>~yli{uDqH-=|^AC-xsm{dvrih)Odtru36emyvco<9J(9FFh5jk!*lzx_@P4+oVGRfj8M7OfdM`k!k=~u zz!5Y!Q}?||)L-Z0t+AZ>;|t7f?+OHQT1VpQrWDbtXDr3i)$nr3N8m`oWRS*+Ukv>& z&nlM+08s7(Q2wIsLH2ZG8=m$C<_saaGRuc-@7tCs?`iDxVF@RhG?jT%&cKqKs?h)4 zW30K;7M9zhEi0%I#mDNGz_|MN!EM=9q~HLmIv&tZg39a zYw0wWnZ^*$ma&@t7EWsGa+XBu6cSS+kMFww9`>LOm-|1F#C%sgTm!N!7$jdZ!+gDA2g0r3*P6h!Z8XFJc7>fAVwK|5_VsDv5GpHp$*&k^b&N%DRs z|M?=bb-&{###Sw3_BhU_!~u5Zr_l}h3WE%M2VA?O6v3>cC`E3d6L^=W?*75SSC8eT zI5(WCy&~G zvk9X&NLz0MBY1*`iogBu+~K5}m#TI`64c@ZUEv@?`H( zG=uWjSkJ`9-9t@U_YMqA6S6$Q01`vQ^ct)?JPR-2BqK+X`|)nXY{P~}7Nh;6s_^f> z7T_B@qblz8x3c@A6WH$H57=orq9J6$pXSc=K(xyzO4UhI+5D6v^Gdn5K(H=V__&DR zYZ;Ia$YmY?nfr?Kbt~&)Bu0ezC}Si0r*f97es?=%SsKtU|DUa(^z3JAS;`|N6z2ZS z*>wMlZ47z>L=V$zPPTylD%ZA1LPzp(Uk0o%kWQCUS!tGKV!?=HD=VnmnI%u`3Pvo= z0p8C-`yDcHmQc5F2``MEaBQ@EogN94tO%hvb#xhc~+LL)#D# zn-1kQq)Z)PyD%nCYn0%Xb>+=ADN)+{(q$YEt%vG}r3pK!aEzzGfnYiKI-BiBe{3l` zg(Wp@W!PoyRXF1GUhHn-*9@>FfTg=kpvh6)6)azbrj<0J45HoZ9>n zn~Xy*Auu(gqI93Y`?+`clSM?F7`2%;(!qkeo#o8>e8yzYO&r(vgBT?FCJN})EE0U5 z@#(L8D8A1J9L3o)*cw|^eUVCtW{QfMD$AA0My&M7a=QrUOlSWc8GYnCoYu`csSepQ z5wMZ}MRt&?*;Mlb@NHCgejWBF3wZh^!|xhykj!w51^oI4-?2wxfHv^Ir9r&5`4@DEW;6M*TnmmY?=VnUrlMtH4}-8% zT7Vkl$5)WCNnF}|SR4{ymI3nu(DydgVCNTWvCWf}_}{$3J;f5x8VJ+Gp|bqj+@PWD zU6#6016KQYAX|pgh~pO2U!AA|9>!_rB33ZnkTe4>o=r_n zSyN(oB<&KY_|F&Yu*_aISLP0@KAmrbNylG7=DTRb#nPJ7Y8nBNgy^QKgyEM~mFjii zl;`!73iA3`79TT!k>9ut9wizvVZ~k1H z4IqDQ%m2zgL2M4yz2Rgwl7&W~l|{-@ehBo=tE*WOoKn{e4u(|)z`CANpM380+EVig zJMG#HzT@o6CXXig#qyPrC~HOo8k_3$JIU;J%K0lC^P=Tof?QzvRaLvn>OnvhW!fX{ zCh1qwt*x?!Lgzw2>y=wA=q74xQI=mcCnDmDj=*%oLC!z7hf!$qeGnK> zK^m#c!SIRz@PiniwkaPs)8qf}-dSHtSD;G_h)Pz0-B$Vh7nb=YrQ^81F0M@sl08(# zx|*c_L0%xQsfz6R0rRLWY^TOBmhe#7F#U{x-GXGKHqRS(`8>U4uu1n|+gt+>Nu7;! zneY>XTcTOUT_qtm=moV@S00(?2EeN)fxYi_XMZ7i@hMWRm65DXi_+vh4a$oRHaF2 zRBY3XY|g2mw#>&nW!yaw_v`%z7WuajpgG?rbLRq|wR*Z)dd}xZi}F>RI>YL^Q7A@w z4+11V|87`iIt-O|SM&}07#u(RKiE(?>Qcz%pLi?ur+6p#4<^DutdYtcR)Kh>g8I__ zSHq{G{>)NRgJa(loGJedGc0mOwSTTMk(r~-V`w)!*zn54nH*ljbb#~pZFpD7C}0&> zkz1|E6!C1%v$ zC7EDXQ8nrK5}Z1ru7k=s8T|LHMv`2bZN$pc7;~mHngfTbtP({@tPCQV%oQk?`anvH zj_~I@AV+AAW|l@ODhLQ2_I}-k(qBC1jyGX@zG`QXv%tsU^A5zq~2prZ(>WxffS%n8K7>_e#ciVMM}oD z-Vm=zqa$#rLY2@w!p+${EGe@Hv2bP`T4uKLddiFT+Nk zsC(@ZM>)oUgJADtpRwjVRWUp+%OQ;h!Z9j-$zJw$WVpwH5E3C;22tR}COPyBW80t` zH||OfHQECz6DC8Ao?|eTk5mOYT1VEEU+SSMy`}ou=0A69vi22wcBtOLx4VMqfhxpJ z_XC@5J}@P-9OQg3fJMb`=8#sgdoc~?f6MezT^7tTTr%Jh^mQvgNq$+6;G59JT}ZH6 zx{Y|`MJSDA&a~aS(*&xwLQS_jtpS4YZV7r7L_01 zdTYU%(F86|V;QG<0;-oPG;C0s>T=@@Bxt;#x@^=r-C8NE`zqyl31&r<`yz}ePh zSzj3}Y9{%6JinF=fg2s)$q7~5 z#*Tu1=YKt6z+#yWQm-GZ4u@zi(HpYvF6v6@c{%imY?_D2rJsS^lhv$hCAcIDM{wBA z-tKGy`~8Tu32b*pKQ}F*)CE}6pTMg1jd<9R&h9N!fh!pzf!tJ5)-%-yT-OmOsU=0( zB=8Ahqz|P=rPe*LEhi1k`g|Sap{wE-*ej}BnXifbsP19YofuM6voa;S2}d7JV{rLR zA%61`&wm;D8KXIiEzvCSt=dKoJgl%O6ML)3{@ILJkIX5_>JMeCK5RHt8^c5yUV&|P zQR?!EA0g#2Di(H?`JqW$@FhP)|81ma{cO@^d=BIWpxj6$yjfDQBGBA7otGj~%rmS~a2)v?80D~!#nz-|wKN{&R%vQ+jYto7+GN>Ap!s_ykJR17#-ljJ*pHySzPiIEN!1Xni)I@I(&OG9!k%vPa8^T(Y@@0lvuie}kdfdMb9p02 z+)g0nR^~vxUzzNB7oak#B2@AaFCk#aY$45ef}fjK@N3T}{P}z?n{?@MP2}ls?B1;s zB*SmOu;WxTaueNJU2bU{>p|VsK)y!VC(_l_j3~K8S{tk~N{Q``Gs%(;kQ06F?fjZl zOSL5_Rl&6(a_kg8JW@Szbv>u1yez}PhWr#?Ak70n|4!u%Zp_cHK0h|QL@#7PF_G9| z$$DvdYy?MZ`S>yPShycTNH_(?jZsa-dlUOiI%9eV+4&B~cH@d$)jcUT$&6*d#yf;T zn*^ZD9na5}#`AdLu3j81O>6RO+*^SEjvm&5m|%}eC#fwz&r&StOICb3I8)C2A4%sO z-&Fbi@e3R=p%lUn4^YC+1qvy%?67lzLKs5WVJASf$SNQbnJrq40#b2MR3w1K;_O#M z*7A)H5n{$tD6~uGub{)&c5wb=1Ipc_D*73JrI*O` zC=@vI1J_wF1$b!u9b9=2=zxP+}LZ~kr}$3du>w;J1^ybfs8O#>FSl%1(&g>tn-!)@3dlj>j3i9z@trM=R%wZb$`~N`tuQH8{ghvt zP>?X5Dk!ICu|`+1``VCs!EDaF6+iCzk9g*zD&9%{q{Q18k?oP!U**3rwma#X^FH!% z7f3d8j_H<3A9BU}l+jwsH(5SeHP-YP`wNJ5-($w`uR|`KZ1n1eOjA}LL7*Pf?((kz ze(NCR_)_N=sQtlKGZPPjL9!oac_=lH4MeK(pqqUK<6Y2=Q46PXu4@-L%@5gbNl%(S zOj4g)A_MvL&*?H+bVyaIM=dFjOz)P||xgPfT3@-?;-lWP`4CxKWjOrJEjfaHT2mc~0f@rheMl z7ZW2d{r?AwZopD9Tva6NrXpi(u+Nd0UIS9CY(tHCWrlpqloH8_nWfV2C_M1F-Zykk z3*fE+NbJj1KH3Az>rw}gY;(NVIeEW@%eBr05j9#a)pr>x7*ri7< zKTR}&cks71iqdr#YEDC3Ihp_q^!yn{g|#EuQj=kPP-f%BLm4!tyPJr^#&ynSmKQmV zPx+m4hm`rEF^-_DSoX`M)jq0Ew)ZBO!8_+x`9*$lR%`nQK&K z<_v&ztqi=~Qn2)7{+pPfWm(Q#vF1`FDI|L8@0)ZnRLI z$FMs-q>n;Qe*`A|cNRGQ($CP%)NFp{9#B5+gHvArUx|gDbvji)K3A_I|6u^nyL%A$dCxB}<6j;a?4qhpM<=qiW>8bE zWP26`C%X}*HIKn$(NlJcEAKL$^w0Ua%Jv>8>wlCpLmW*_kMUNP_ONs68G+Pl`g8J{ z$qYT^=e@UptHvrcLLL#*Cy&~Io5!mvXt}6ps;kQT$`qZqs9Miae|a@bdi@68V?R-R z9HNru{LNjT zZBhxP_lLF8!Nrf=aL21OA`=lPX(O9j!I>;HOOyH2kPhn&xOK~AbW=vLkJ@h!8mYpJ z7|~+dt)9f9dGZVXn5t9A*QWCx(v&F#y)>x)Q8)D`0b*O0pE_bzb`BaXfjgPIBk}1pZ{~L-*D>%ef-yx`8m*S$a}F z#VIqpV1@Wv%HF$7%vn{QPIG>k2BNk{9aOqg&cA#LT5Z1`-o-e&v2?f94CO8vnnARn zmvm^tU0znfHd1dlm;4Yd=M?TZS)&09=w1gGk^2al#DnOP#~cXF_-`Ob6Sm75?XqfQ z&Bi7j$7+0o-d?oR)c>x2Vjva7{%EkS2+@@RDVt>9>N~mP_b?hqHCJr?-P#TZ-{C zSw*|cv)@Ay_Ib55o=C40-)QK9*|G0OS3#1mYG&f^<>Rl+m!wihTH_yd#V; zrYtG<{05sG&=Ekw@=|juYbLe5R@tW8tbV#$SI@(A6>;U|nT4%cD zo$;XAp)(w4qONFsNU%!SITD{NrnOk9;m;1rA3=yH+UWJAcNvWY& z`wU|F18a4pr;c{MuBQPV8)N^Jq9_vAWhC}pcooRn5(ADZ1PCEBc8ftvOQApwe`pFH0KA#^HQl z#qs|c87xD*dpgX{n3-lcbMgM-=nf{g$~S9d3Ul)0+gT`rQ_L|k&KG+~&DzrBi^sz@ z`RODaTq`T4t?4v91d!NxC+PTAw&g-e`#r{$s#0dlU`uZ7%V~aG2$0tl@W3&@ybhyyUWOhKzEgSlxQE6mjvhy37>F8B3R#g3O zS6pNGKiVDR-P==coe$FiiBDi#x-&#gd0W}Q>thrmT#UUI&hZ`EU;iWO&)TXF3KU?Oe7@| zd5G&8-qXsH)g$?(*P0_Vm!@G(O!);dU#`pAuIoa-2@Y7ICzGqKUzHZWCp-TFh}&a*U`t2%+i;9fm-3F3drP^w` zyI9je`H;5})|M|t{BLrKR%^on9lc968fFFM{U>0kB)b{pay}~gw2Ik{CsT3i6nE8l zDYW*|*Cv@seF+&HlX9JDa?3=>m#RrWO2gz^@AK$YmrRDHl%oKj`HJ5irX};S6A1ZO z+Q!*Y6cG2*$X4&YeCOSIknw-$dd8wk_LTA#(CTqLZfut>Sv)+E&&>#+gyyNl7`a)m z6Fl%MRD4vI0>8)-9w{DamZBVj8M5_PB<#3Kq=mm3Af)5Ne1x2mYV{I`tTdQ#_anm>d-`ca!xWqvMcr5g^M^s*H(%F|i> z{TSTBu5PEU1^3G@8xRAAEZbpO8uuG0r%7U>T#p+wTFL3PW=IzPYevaQ_@$cP!aPZK zl$pBLjoGCwc;o8+@?I0@-gh59)O;N{&rY?;@^TZQ_?|b}-kKkNFbwnlmv{L6cU4h$ z8F>U^JJlhE-LAk*GtWycq#yw+X|}T){=&35%)mWuRw&o ze?piW!-&Bu-84Nxx9Z)dYz#y*z?s$C_^pqWzXf4N+60+VJz4Ji1)4CM$t6plh9>!| zVUjx^u;=QKRws-RhTE*CI92YgP!FFzRdcq3IrEHL@oUMoQ z4~3qDY)YqKWq9<&BhrK22lCwg1f&|!EqhxFjrqihU(Ce#o(+MOex=Q3R-t@p8 z{+N`s92h*_nKJ1+z;zVnhzBoDj;;pm&NI$V632O!hSi*oIy)%GREf5@`| zh}(H|n3VZhocLBu@Rn=3kl`ai+$NZS@i67=CoB8r+vH6Bd{S5ulaf&Fx-lKUrTqIa z?AcN|J2DKFX!^RgC3jHnf)FtbV!wVkO99Ta+`}wrSq#u`_{6A9BpXQTGa|Y__q*r`BIQ7Tr6 zA#!0TOj@iwt#SBcX_D2I>tB;qK@18}{caZrc|tk+>-n#G6G=@gjvAYci|Xwq4qYI# zdYb9Rw2EXHS;aZ>!9To0bLGsEk=sGBjqT8i{ZDbOAu4W{f2vR+VCPslpo7PeVcb^q z5V+9iV<}iG)Lr z-GI_|{bohvl!ZJrQT~QtW=%JaT-{3Ed>Ux`K`-3;gotW(kz>c0dz5JLv>~ldx-%oS(vXGt0_`(?(t+ehiWN zOS@ka(*Wu4)2OSJb?teO5lgfF3dELSOTFog2spX7++Ol{1}rsNFCExc<<_Qsecf;pDtTn6kniU3Sr0U2~0-fYR0oKOulhuEP4IsR*Gjy#d?`n~duG^(=;xn$CS!Qa8p4^CS_ z^g!ug@UyxB28pY&tbP!c-@P6PiOT3mh_+%r(Tlv<;%J7dxTZ(#*HNxcQmzf0#(HSl zuc}%P3t@hLo1a0GKUJoZJ-1lovQ=D9(+i|Y_l_26U_GyH_pM04D;-TAzveUH-5`Yw z3Z87LTO^ih65`0Tk`qrtpWb?`*m-3P(KD7t;`=(9OZh>B`CGk1ep+kOA@)naVL#=W zBMbFD<=M*CUTXE?<~rTU7j%6Yu=;{|Me^ZW@Bllun{W}q5z?gt$2gG3D=gIe%8tKi zr&X&AUB9!~uMORiXqm_WIo}?*T z`HE@jsl(-6rE)crGVE{Y)lVht05#tJB9-iqr*9=8OCl&+uONa~N;v2(dT8VFWmY*t zgQ@uzmDJME5zd7Dla`!+jL&6G0x%J^IWn*i0D9#vtl4fl_ZMErgNh>DICG$FlCL*b zJ+ls$U7|eo#r~`ru)RCDR;NY;sa@@jPt)S5dPh<-<)15K=ibmojCxryV?q9 z^@GUIf3nyO(9m*9_uK3={nbGQen$ra*b-?W$2OXUSb-IG{^{&Q7$d0d;LHwkF2!_E z*btj3PqzZL=PLga0w0w<@qb-S@KlPRv%@KW-eKm-#rI%h?@AuHcp*S`ZYDJTtQ>?W zW9~1nS=d7kZRI{v*1@99$&DH}XUFCB2FO&FF8z2GHaBLa=1QOSd{Gy2;qu;Gvz#A- z=Fj%TvW~05Dcp%;906LEj)$*dI%%?NP3#?mUihTo+!R(B`vW=V7-m=TtB2>4!|fY$r3nw8vEoB;EF5 zI3i!Y!!L9@jS6U{mp&9%!Fe~bGh14{#Sd0L4Ot(3iz{ntKP#X_VP|>s6MG}+i3Ocap3t(P>eR7MM9L0L@N zH3vt+9k}$VF<$YE?o_PvD>ph$CEF6r4~>)iiuv-rEZmX?KbaA&+&W8NWjdszZfokR z%I8b7(R_qzn0|!O-1TL}BM{n_=xNZ407`qM7sm}wN1<;wN%;lz6j+;aH1o$LWRPj)Wndn?l` zs5h$YuAQ=|%&mXfYy4ED-`v{+^FJ{Gqd(e2B03+RuCs9^S7G(J1_^VM z=+sy);pyQ*U2>2qm)_@D4=@OfDyjH*ncNa=AaWPN+4rW(Q(a&O)&caFb&pw@6ia7n z5V9?ReVGM2bOMj)sGQxHBntah7L>ySekTFsk$qvE33ro<{4xcq^uHggbd7Qj!IF13fB`YRH`axMNNtd~3uqe|2~`h= zh}1YH;MJ|~VAnyOnr^b7mX#+xb@C#IPB8qE(w6ckB?5#si2QB7%-45&#;oAJY)fQW zCaQ37OEB%^mpt+#l(a68a{@1=l*%3D|2F8L7ct!y2ome2eX6{*QLC&}$J80Z2&c-1 zFF3&HH(|b;RICYbUABnB}>JIU|!H7)LeF{&9_E?J*(ia(ojFN~~X9(UYKVDr^< zl56ikuAfyjU6<4}l|&T=$787;qfWMiLnPB=)lkbLzwH8l92?D$G|cAL-8THmTjoNs ze#$voW^aa9UY*AuofaZ)FDTDV_-W;>w*?`UC%OV%A#WG&gy%-yy7yI(@(DJ#4o$F z?7&P(6B#`pda}ZCj!a(7!x)^Yr~(hRkPladgVO%2J&B5jZGA6+mY&OICMXnd17MVf zWPCLcGfd@UsXBS_&dFn zzr^b6oclugu)XClN~%H_H^v;9vNGQzS@~8a5yHeQV`+A&tT_SUik_)NgMuY8D;+?Xu9wLlQNBgUZJu6*N3Ll4EHfqn7o4|~X;jjAwppYH36CMB zP}+mQPoj~$@(Z|B_6S#1`3pey-gFf5wx-BQnJQGeTXTYZ89*GGeaREYXx1M#2sUk& zT7oXeTS=0JH@c)!H>Ben-md6^YQ3`We%%nT(&=~u zq7*FHxLtWW87b+x((?_D{}W0=hUs>ZyPamKt{TndsS(ERJ;W)qRnaXnwk2|WQpHzb z1XD7y=LM@$Zqynm5XUUI;`nm*Sfudk}|n}7~SJ%Uk1 zgXE5;z~l5PfVU&vE#lLyf`1(17U!|-!+5LA2FKb6sn-fSCB6o}d+Gu|^yqoW3}~Hi zoKG*8Z4V;WWvZg9@mN-l?AXk89-3?BN$yz@2oTNDM825?_f{xxcb!P=h%IBJ z$Pb-OmrSi?)i70nYnyl$ugY3zzA7C5aD=~0!|cSpdO}WX*Ue@ zY8NO|y&-DW~MZ(WqEm8Vj%Bc*hvq(457#!^W@A8{O;Y<(aOFL><&yv)&n-^ zi;T1MyFOl3(F%|PWeR2cqTZZsWKFTXMgo^G8QmkQG=A@EIDYXoT{S=s?zggB=HY9ULqu9ziyoa6#I_hL9#uAan3 znxT7XmcPiUR;7ky-zH>sfy(P5d8<4?7x>_;1}c#SoR+thXWNGZ+(xi7MwMfN4x%(7gOS%;I(37WyX%EK`KfxT zx2=gZTfjFBF6NyM>;BFG^WnJ%%E0<@DwvA@%Mj#q8JC?h?=r{vP`T6r{ zQ<3c<%Cj|%_K!&GO9yeI116n%_P#8(@B25 z#q{!woRMf2N|u{vag3dH__ae<&_(;RD=}pY*HD)m9V5%w8BGp9cQSxe+EA*Wf%u=1 zCnqOf-cHBnEnO$x1mDfqO@>ZFq!V{Q(6j8D*0HZP#>(uFymQdZA$`F{K#UfroJz;@ zJ*wxny?YNYFoY5l`S(38{<6)CK>=!RclgXGP*f5m>SFq(q&D|9_A}sQjowGhGR=O( zyo@y2*d$EZPghvd&Opi1itpbEp7>&tu1Frj`^c}rwo zcVOfrSV^AAL>sNxhtc%p2Hec4I?Hlc)pV8a4*`AmWO;dg&nWp_ca}~cz&Wm70DSOi zo5E*~47HvH=gjm>xotc-gs?YsX(Y)vxEab zDcg`A81hBFAi0tw#(Yw_&T>i3GdyRh%Ik|72(o9nAd0L9Biv?bJ>=TESq|L?>9*0^ z0LGJ+kKTxhHs&<9dFR#qfV>qzJy=Bu{PEVjSU@=%Py_9B$);+nT1GLnP8?cuU%e8} zlfy|cnP5zNJPqO|zp2En^3o!Njopgt;Q#0ZOvKHQO{Go)znh#0Gn6U!*mEt8SyS zaf_rlgY)G4WUu5+y;F&82UF#txv=qvZlYc99x+|o1@0?D{Mahf&xCdQDWYdux|}Fh zHXO{@4b5(2w<5_QM@P1f<`^5(YcR$`78<&wv+c)M>b`kS4k%dgyvm$!)MV4m_&28E zXLW(w>?(UmSLckvQmq&)-m{h=hd)mB7+doVhOPlN4bl*sdf^*x5p=ux1@7*wR| zh+u2ua+Qap=H5!`u40Lt33I`JunYwDt63z2hgnR*x&*6jSH9t}b52uiBKYS$-8aYb z@kW4G%mH;!76c+T!+SM-bh@#LEb7C1_D~hmkcuGyrzdJP5gk9{;T3U5&xW<39T2dlDw68e8-e;335Y`LVSklA&zKtw7t^0;SE9g z@nwGegvw2a-o_lHF)$G-C3~gM7cQjIjfQ7#@J(L$uY<0>WU@&YoP>a7oJ%c|f749b zJICbfRCeibsrec-CvP|pt1`Gwgd8f5xACdKbm{0 z%1NWir6vz@mv7)@(VEj$Vj23naIA!r;Xk2K^(0w!FXGknD{h$fsGgG51nv0C&)DX- z>9MwJ^w{rfs(OGtpANj5dRpw?*WmvznFh>?X%s@5R(B77$1QQR4APk142?$Hq^{GR zOSN+8IJhX(027m%wgU3~;(i@H*0p5xSewyCm(gW?0fqioJ@ji&h(oPro;U|v26MO* z9DFhCQwM>GkG#*YBcYu}JbJ$$Gtwan}MU@pgl7 zxqLJOSQ}dqClZ+1Rod{d$;SRNw+@jx`}k4V|-si%mxbX0*MGMags#D=0ekyLn#ygeL~?&AGOio2{r zgYRJ6G9sprtDu*%Q+E@jncZIB?ZkC&{Tl@A-x-ph)6-3YDruLrH^G*lRBn#U{|yHG zT^DXVqs`m;lorT0=K+n@DyNUE9A|oAD&GP;9$c^o3t|$YLK0J$%1ufM+DrTdv_GYM z{iV&*D96`HT*#6fE5UchEIE4)a`m;4PZG>yW!Qec?#{9z`L#EMN>QHnXqh~}+;=zE z+d*$Yz4c~p>TMMtDFqHhv_g+!*`^{5WqV`JZWef`$xn8XL94NQI;IAYILfLweZf;Q zS8;dG=vkCk?&6iY>Wmf5Y4xRFOE4lZkRyGddesOrp1z?B23x4-9a>s+fhDN9e`J;&T0% zA<}Lp>D+yp=&Kn|a7~Q{Vq*^0a`Ldwxot>tL%&bu*d?lTxtygfTuP>5m2}k%#Y$n_ zoRZuED*5Et6<+80R-o1CVh;A3su?TIbZe*MJuduDM&=PKD(pkkbT~&I+5}7#KfzB6 z3P7JU_Tw0}%kI$8YQk3+O_o5_@Kgrh_msZcVxi?}AyI%*yBypELw`~R{-&j3rhS!y z$TozKA`W}xe-*H6a2iS`$irX2(0lMr z&T3%arE-+lA_d7m+m(J7wt~YkH*~^!IQOjNmMC7fM^iKfOS~@0oUkA8W z_2dUTStY<#`rW!AVT<%R45wJyFmN9USE4oolOdLEG7t0h#^jEsAdGR=23Old(sUz? zb4?l9`f3`;w++I1lKOyI0n3xorKQ(~!xL(Rpm_A!x{?gp;I?c!>pkE2J+)98erp!$ z#*K~G6S+52-C?UQ!<(Dv3+3%7TXTVmY8Bgshr#wEw@NgVZ;wIu*(!G$NA?bu)M>om z@J-m2N2c2;WJSu23t2YldJ2#(W|)sWbuXVN--fuqkrl<-FP&u^DNcyl64?733&*G~ z*0u7Pb}T8jv?pCwp$-%Fm3YwfBQi$1F)-b0Tux0=9#+{|m-)1oK_6ki{`@QEai^<< zKkt}NP$7Odntaz-!0wx;Ootq81!Mmjh2G5lGmThwTz28&qVrjDHxqexUrThz{A> zat8N6p67L8%GrE;jx1~K<=8M-&;>{@$m>+qH)AVdk+w;v39X^&ln;8Fx!7lg^Z3r( zVdRwj`}rncrVRfc3FxeAu}Ez9(v^H$`KgO}gLNTG?n&-y>TPxhJXYb9&wu0{)*~BUP8(Ay7tEIn$2m+=N2^f*de;;P0c z#R+n0k(QW^AkV#2OX(sYue@?QlDhmP6lSY-M+Zss@-`dlF)$JrI=CGJ)z%+{TG&Bu9}q5-DT6Whs&)6yA5*#Xi@F{^qr0gW zeFis(?+wr@mTGrOvKfS6KkHr$$-;UliTD`VEr@K-D$Oc2P35;y2w|@lP_E8S?qt!@G8XIW)y`HPak+{RUsc#z2QsE&O%^&b)fhsixpOi2D+Wp30@I7BM2Tq|Rnbr#F5q&&>E zkU8(JD?l3<6Oruav`svW9Ubzro7ZWe3I`MIVzc!qy{9UvB1#sXvvQ0JWj1-p!#h3j z0ertaJx?NOW)8Z`HJ-pU9FxAy?#7a@=qk^jgA)DILxxqBDbac+lk^Dp%Rl!c4XgTr z9OavF$f+;uK`g0oj5PZM5!)F7I()hzzjj$Ar}8V_zO;+3wDEynIr9K6cMHjw!(EP1 z!KQkz2t^&K9#E2|>poD5PZVZTWQCYMN}H_YEQYa4s8}Kf>vX_7nXRPmelV%aCXUMP z^%Q_KI9Uz_VIb>I{%^60#2_s8SN*Nx}kr_wfB@OPEN5* z74}7(d{c-`gl|Q@+mdhtUTJ)CDh0O&dHyYdW&$2^c zC4!1KI89b`F!PN1#W8S_MQtZlqKPpgwOS4`zz>KkZzS`xv0vutg6~VZEv?fC^yF*Q zWy1Z5uyo#sO8?)G@P3WqjsvQE9^V_wlx_d9yNrygm|UEOPDt0g?l)@5KB~%>WB(vH zH|a^j$5qZY6j609r;gst=iKypZ-f41c2-HvF#;2BQBUuu!;+k(yfX&nx>2Fdp` zO-fq=1#*ua1=3m7=!!S*UP$Vh4}J!2f+mI+D=sr2V+(oov2pgeT!^1@aWOgV$(xXcQkN7B0;jPTnChapM7-1%HF#Xdorq0Y#!CeYI9O&1C|;^_&VExS z4nVhK-AfO*ZoHW+hF19pr3EaMR}@-O)9%v7&R;SJ|@Y{gdVIT-l2QEh8+Rtwp>My2da1n z1g~&Ci7)>K=oUytkyd-8=qIkRGjl>n9veZ}`jPFvRw({=H z=(;=q#QgY-*rl`{1gpU_1%SVAD!Q^_j1kJxHeTBXXBE8y% zt6BM!gQ$2XLabO=tYdZ^$|pTws=Qh9wN^vFGt&+RrT0cDM=$UST{Kh~+j#3lOXP}7 ze~QX%hUf;?$^HkzHE5~mXz^XO^la(8wm6R^t;N)p`s0FFabKuFf$?U#u0N@4ChCrJ zbh^IGXKz(`X}Z90x}?6ttI#*ultVeg7@h8}FjD0`GAaW|X{3ys_}yq<@ea2FL@x3c z!={lY^5nl}iM;t(zE9%A!2^0d^5n&4C>)H?@esuQgyz*$sPF2Ic8YdHktN4S@gOT# zj}R%Rb3Q0rzChr2ps?k(1Mp0TT3CkHbR2k?*Q}8W4|hE44-n64#}8OHK0qRO@f(w; zgp+)6b#Eyr;lQ(v;nKDW#J=NEvsCu$3Ws^=2BdXPC7clKAjx&yUT+Ph%yL*qOy39b zdWE%5@7^mey?f)kvY43jaoSJ5iV?C~`TZa^D*a!=M{uQ&-5Vx6+e77ThAWNxxoWF3 zmuVZx+XV(lcfIE?Wf?DatE$uuO8ymu+N1MuMk8ynGJ{Ku-eo~(LZ!mg5uD^+WsH?q z-++q#1a8x?bd>MKqrl@kx_U06;u}$em5p>MKnHpG8{qM+&oN8y+-cR|vy%F`XE6-x zu1ZMt4iJBzDx426`ex%RCc7b_71^9F*_R6q-YNrDK(^l>N8;D0NPFNRbEVu;V-_Jj zzO^6|&zYn`VVU_Bpwvl6fDgXR!<>suErT?%+^UMUarp9ljEuxYX-A0|5iFWegWrdP z(y@q8#3`iiZCY<}*mGCNH;pYP+RyKj0Tn1z7WS}f=v#oleTh9P6Rw(GdGB%FyRi<{ z`c(C3XdWAB^v*Vrmue>=g39g~f3$f$K<%_V$KD)xo1t%xv#8p-db-ekw2pAPWOqO| zk%}U643u4;;!R(Ik8H~p+`#PNcR$54ExX1u9#)~Fz+{n!%y(`l}dtgm`@AP~~k2 zr7JdLc}87^y{bZzvxxC5{TsMnau(mXeNR|8O;x&N8)Xl&S_mv8RLLP6NRy z*){}=i>USWKkSu~GPqNJ!OWny{xe+*OJI_D55?8;XG3n~9_77*Jh5%9jG6{2Wl*Gt zLl@1)RHRQRk9l?6C>gYFy5c%j1x+H@6}i}FHH_nEmLHKt)2%qp04=mk+-?Pd%^azn z252+(J`EE&(oY-9S)f^54`*^81gd=vNc$9}Y5X6eL#ci6h`in9UR^fwsepbxH zz>E5zZG52w#zV6=Rm880t(uTQt)7KC$Q>;Y8~NqLM;)nHKNY4NX9W(Xist|pVy0KC zOXgzZ!(q^u%Z%Qp18FH~NIbN1vT-r1MuVJgph(OFEy%W0fY4uhbpc?rSvzg}Y;)~S ze{!!SJ>Vi=Q{JAYmF&sX%CrWNErSqx$piA`HOzfPDx~Ab$;Rj_*9pP}?kV7F z>2l-OGB?=gpDufKwfkKW%MIBb^#K9;aQhtKSPv7tmr46j!oyT#i99fft9@jvm1itZ zb!t5lgi0h<=IhxA^RvP_zv&+@-@FYxw|pMd!5*B&v)mrQH@Z;beGrR04*>LET_pH+ zsu!v>+8`x)SeA_Rm?9I~+x4EkOb@}M!ZbPcHdftz>>Y>lMfd8XAI5dancN*@c_PTQhXKGw|3y^$9>(nYn^JVRqq4ru zmr-Z5i+Fx;x=uy60LIIk!CELR=d+v&plKnWHsV3E7?3FkeuvH5DCaH45gZS?u3Jd! zhDS?vj%)#{H^-^Ew9`=~e>#-S3mM1LGclQosUIrv;nLA_|Bi}z<>R3irbYn+b!YLu zS5--x(KppEYN+|YsF_1PS_GwcMXhRE>y@=T*`#K+$-uK6`MM4%Zw96lGom7I+JiFU zU-ZX-IzVbJ9Epqt#!JQ{@Os^=|FhNd8q2S**#q*~%@E?g#qjIvD&Qa*Nz2lT4?$f< z6h^U_*k_pK#q46KLmmpGI!(sDV%cDGEW-;bc|dyxJAAx7;o^LUpiTNqWzbgA>Z zX~Wqo8US!aqY~a>ky_UCIY1i5_=_quNKP}s4T5Ce zYiud=@B&cg&Rh8C1r=?egEn;R%_B-tKRbajl{ubrsfupuo(#c;3++Wi6%8}$6<5gB zwz#TU&jAx>l&u?Tsc0~Q=FdkoJ6 zR|K*;J0W#c*nwS{j1WBVwwC3un58fiE=Duun23d5c`w;;c0tqGK7pF{SO)?U#spZ` z_b{+dT@h17ZkL0)N61u;o?v!PcVH&i6<0BrpS}p zWfK+bB2ypiZQ8ZZ`<6MFz%CBc&>gp4@FKfbmjQl z%DY?c)e9?*y=;}pnqz=uk&53c&pc*T^7Vxg+3@X>7sU9sEFzmmA+EJlbQpJBlFBh` z%fZ&1CZxb1zKsrWc;7)u$z{ushP};7z@TP{+(;#YoS2DZ%uuC6rNJ#&O-y8G44xj% z(Lg0z%A+r&CR9yOx}N3<=V<%EFDn_yNpi@MsmOHstk8@aAExSYxwiTc&$~)q+yyJ{ z9A(xRZ7c@3r82u6F>{TRNnhAo0?&lC_7|}ANJUprNl>DD=}76X&qY)0QOQQq{{`Ol zPCeHm=m+lX0>>8FJwhyHmGAI0^hvwAfM= z&4dSP!bHrX8whZC?_tC;CZT^HhR~=Tb~gNE10&@_xm!Ah>cdX)5!!X~>V#i&4jI}h zuYG6tNc%WE$*A|0gk1}DCJ^`TU!4;ini`S@OD!)jO=Nho4!ga2nd@P0P9yXrN4|6( z{N1iytS!oyZoHJ5hmd*q;UWaLK5P75n9NjJQ`XTT$`U4~OKwbG*uMZ513*u2kdoR3A!& z5{!#2aIXGQMXkaAn#P!zA?!!(ax?CsEqaYEuf;0nl6Q1_@AlCcSSM6oCm1iD4``IM zl#kbgHA63$F$}KKO|t4;%ZJ=iKm4pJJOuHA7S;>Fkbmi>mw%o>7B>5=HB36oVwnimGNV{W~^d1Ve>#5k2#1$)hO3%MhDrrMa_No1fN3vdL&T$qW zCIZJ4fq$7=kMd+mPhI6`N6-X&V$QR~x5Def2}IdwPS6K`qsb@u>x0?@WZ&0EShuQ3 z4CA9j7ma!KZr4Kg)>uaikxqJ5nP1X~)L5)7SXFdfKDJN|Gd$4kzV-3c(Hm+n3O`wr6sWnJy$(tkneReOk% zG=B*4J~jaFr?(3uc4c}I<#)LZ-kM|C(1X#t!N^2w@x2bW^lrr=7U?MxOH^DY%G{3(+PY%j1>0*t)f1!gP)!aiCNn45ZCoCu zp3kbvF8I8LT}qv(i1S*7c~!|hGUO%8EuX&2g{JDxtV=pDyTafYwX=%|Zn&R92b4PH zWL@0#OI;BCcMtK$P+jy{RTECg9RtzT@e3O}#hE!qwpjMr6)Zohp2u^kJeE znM-W`#ws`uQde%_ZW!>O^lIdB8bp8B!`|*znN6h5F&MTnQ5}ee!}1BU%#2|$(?cpY zOzPyLv9{|=r!~Km$S%oPG{SvY16pnla*%y z?M5ANIt~st9eAPMty01G2i`83Voa*?OX5YWSjs!48&rno8VDluA?p5`&fbroz~Wm||XQMi?#RyUBbGwgZtX#tn|h^n{? z1yPZKGlkYYJjDjVu>b!XVB=u!FbJ6>RR zJ<}zN1|e8)s64M+`q5+%!gc7hK$Slz{YRlqeAKM#%C^z`!Dg68*^D61{>TaPRb)6x z^5|;viY~d@5l&gFYSv@R77drI55UsVsYjjouGE&2V91h&uYh~^DO+E0(`Tf;pQ&=b zH^&>K67|rr6+<`$tE!(;5s!Rx827SnXTGKHE41(!*G1t_|gn4Y4zzmZib zLzf_YQ}5(Ucc{X!E>CS}++9|VBB-3t;uDty{zBJn{|aHvJ!$tF!LkZMnz4cM>^3y4 z|4F>qpIKPVo5c3R(HQ-Eb>Znel{1XxmSMl~X_e-5-xDYV#`Ir)0J1YBcralr_c000 zg)S8g0nqwR`+LcDL@vso#TKE%r&CrnN~-1|2@bLHu$u!$c3Bb1u5h)E?sb$c8zI*j zT`duw%hy_}u%n#pfVQlw@_OQW8^d|2(Bo2P72Ng6t#-HZX?ivPpO^t~d35H^SfvkB zpY(bzPpa+>hvIJk)eXrQsY6?5Ri$6%5fQ1h+2`3_(?uMPdtw#o8LJrH8;lRVv_ZB;v6Kf2I(*8j5Q{gtfNj9dTm9g88( z3-Wz(K)cD#$^E zuu`!}x&dybx(duYNUB$xZb%LMy{Hp}D-pi+sd1c~$jdt9`MS|Oy+T@aCm{5zio76$ zomhC&r=!LHI}>x^kREFnQO0q+JJeG^S1(^@C^i>fnxiYkytuTPt+@fpGp-YFMyP}P zq;<>xfnI2^+Muz{F6Xnnw1=9u}mA>PxfYkd2L-wahc3m#zAgLZlUZ<$Y5t;I} z6^HEzwuV&ROcZN!p1#ao^g*FjXuh66pf}py*jJUO$D%(CB3JH|E+w$&k&_w&-nZD| zP=~KOpbI~Ge{n|1lk z5^~sDskX&YPR@do(}t%{z;i;eG~j4(B5oQKCrV^SWo-HgkPpTl@`Dc^_3K?&)&+JNO*GX3(> zyTEMAqhRC|T7%^CGkjegy&3`wst;!j?B;JcU9alt2?SP08oR6X=-C{4;H+U7UtVm8 z#aH;c_f_RG+uE1v->`wZtzIpgsu1tx*G%mXAjNG}O-G7WqB}%A!wr3+qaPk!TKI97 zaE_6m9g+d(O~T;H-&^PAbPq?gv7EjLf{bYbRle8iCRML*XjG;58&tMN_Pq-sca2>t zuh{153cjwLBd8E8q$1Ei& z;?o(Fn8Wa7FJ7OH3svLWIc&D zGK-9lvn!2jWjRz(vz=>_sp|%6dPz!svjjUZhc!5-?775JV(Vo1NBnd?`7$}NkQZUQ zy;r7>GYvTr3NbKl`qPN2@}{+6`N$~?eOJxN|VSp#p25YkZF^V%)bh{}OdF zewLY(U%thXy5UK#xIkPua#zWhUqBaX8Db<>i(1R;JvlWiqXImZ zO)8!L06N;K>W$b7#$rf^E&6F$1asHtlQ;L9Ah~(Q2;N{s_dp%m zIjjP+h!Fb*%8qX#_-++_PNscrxj@X!p7Q7ju#@btB{z-1j=|c^z-)DPCR#;wWrK=M zNGN_%d+_xuV17TkTuzoJio|e(l%or8)q<4D65r>k<2H&vJ^NRIG<=*@QVT?hm*- zB?}*DhQ+g)VAaw%R-Dkr#t`}Z4nA88xoDOPzcs&3zOZBo*B5hJdRbTQV9X2>mXri{ zpkCKGPoCD}hxhL^9mf4xx?8wtvTSWEp6z0E=18>}@K1!P=7f}t zfqm~&nVV4dkp+B?EZfK|k2VMD8;gs*#tW$tnfskp4mgIEf~($qXcid;%jDW%P5)v- zxbZx;Ayvq$uj6}t5d?g8`e6CO{#ezo?aeoE8DcZ}{lFa9GZ~Qo{I01#(F_nNi$0E8 ztQ<+0!4txz4|ye?hNK1cQpD|#Vo0=Fi8c0n!t_!ASh*WcbM`dq73h9hWNEodY?9Pl zA?@^XJcx(b7i0U+leKSnPevK0OXh8}Gd6J?p3ue>ka~*^dG$u^K0Wcr3Hng>lBHax z)E>&2|7nSH;5`lqt#xrSXHZzA4#C=jvId5N{#R#f%}%@{tO4!#*x-PK-RWn|yb8 zG<`+s*sr`z^;)hYG!yU9aN)rvsX7{uGI>M!GDuV}d|#>fIgzT9Z)8ota_L#le|qk(x0in_=UND%f0>Uk5fvU9oh4%)L(>uOM*`s%SH#TdGbPWh$ucSb!8e z?Q)?EOkJXCHV}g>Z>VnU&IrLrIs9T((!!XOnnW9Vm&xRJ@=GWYE*grmrps{a2)^-g zm3SIUlpru28{<(JiJmf}9hT4E`G8mt^gON`D1It*Fb9>l90DT@#Bnp;(D49LuE!>>=l*_WR} zJ6u$y2c&s7xOMdBi0=fVf1JI#hwi^d<>t>Z!!yV6k@l-Bx9q$R_F|$)M|ts0 zJ5v``c{Z<-(2WMJneu%bvyg*Uo|1=WTa{Rc-oadU)X-jScE+y8>^}p>USZYNn z*f>S?xTD}AxwgzK*Y~y>l^0rstrzoaN$W;$as@C5TZ`~7zntP5(C^*PbcH;Vvo*(r4Vw7c_4&`c!aLC*~3t2EU?HM_*|vk z`S!@I(8O=kn!PD(n;Zk&keg|)yPSX84oz3H_<%r(KZC$70^wpftX9MOv_-fHdf-D* z71dZ;wF%egLE{nPkTxZ0OeU7vJ>l2s%5T9M%r3Al@y~rK)>G=mP1;CCl~1SrDqoF3(?uNiU{^{f4t3*O*ZilKfxJC1q$DmhK1l<1skD zM$!ErNoO7qWBxzzhkeHkO)4Qz%D6HQ%DBg!yFuj6wQ?6Cca9}@A!1oMH!QM5?y*>5 zmn;#xZ0@k%6*gLHSFGdr`s{E2m>tdYe2(|$eShA^luzvcupUlkzV0dq>@$~1q%X+# zK!vM*(+$ert8$<%x@#a3t+1|~JI-nr>REkcoj*e5`f;$W=Jx0aEXR0H+?(+gq*EA7 zdrk+a`u1@+cer|k1g(q8!RjUVqp{!3Ix1fw;mC5aLUgPg=zunNCMjP|jCCf<{f4Y; zv=el{sA45Z7w(EkD7-JrJ{rDKeo|r~9s-0bH$m0ZU10@v<#iS1F}WE*v$^|OUOpA< z_G6|cQuPYe(RZ}Os{|vfDwzMTp4CX2eSt5#%>UTmnfmbQ-y@un=tT4xF+V~IvJYEc)C zByAGXJhC-fS>#eJB;6B)h?plmE^#8q^^p1up0XaUZw-L&5s!DaEz^YPu5(hNt`mnA zPG}kZ3c+v<2_)n5II3r-*(iosO+;WycXfGH4==WBz6w(G#aoMs7)zQ9jtK5WOvH?6 z3oZ2cz8yreh6@lMPKwLDC~B&>bO3THG(Z{2T_Nw> z*@^-^#4-hjFjY?&#`3TmTdQz(oGR{kNU0ApT-9utrRR>)9v0+_E0K*Z|IwL>81z-< zrKgYhH(`4uK4X`u(vR+gVEWrtMq!H>yBu>jvY_n9R-jaFC$=5cl4FPnMEd4ZzNRY> z)Krxl?Bls|YKEbTD1?{hZ)BKF{WIEyG{^njhPDVW1X~qta+t4NFoSB;g=i@&FQCYE z(?d)!l){Q~#*6vqopGj!XCj*WRaFCkicQU=J!M*?-*IOi+sF$kRffB8r9l;VJuVdc!I2~nyx~sk^t^^#R8dEaBr87^oxgxB`mR$f zJV3@N7j3SdV&edDHeJzE7RrLo$WI^Y=1FB$q$^jCJq_?&)l^tlQ!Js-P?lFX4D~G4 zLxNvGwo8s8_6h%Kp==5})-pQYem$eK{9Y98D?KN2wKS=(kqM&uT9HIl9Qa3h`9J=V zXwp8*wrOR<3Z?&%#8^-XLdw7{mX@^)m7^U?#px^Tos#5xm^$B2I;!l@med`U;me(| zczljqK$2J=C2s;TkMgZ@&EH7FRqSekG)$?DGUZ7_m&@i9m__Imj5x6RVqQu3w4G=N48(Iq->TNltmm> zRG9qxBQI^$Jqhmve18wh*reLvs17OOTs~0>!WX?j`2sfk8$rEJJG{qi%z{F!G)Frt=%9C<{NRD72l#_&7sA+x${u_?uIal&;k< zxSU$(of)Sg1JB2j5y>b&rx$83gB&Q(u1su_ST9g2WS;({x`51?r65?EemB_;G)m~9 zoux&b6JJF-W%$~&-YI&jKNoko9Ud=E>4v;&TnOE4(6x<_Yi28S zh2$Rc_=A)7&#z#;2lP-ctX>j^u_ib4vc>TzKh zA~r8=V@#1G*Fd$}ui9Ts(bM`7mq+H!GCgYuA(q+}c@=2nNb&K};d15`Mqqd4vOc7T zPFD=dj-Mc*F^oODu1tN6fu2+;+MyKmN;3PI5y4W5bixREIqBH1e-m)a`IVbm~?f5IyO8cQQ-tkOtzCmml`HdHImWs?O+${2OO zN4NE{EIMnB_CruOm|&`ds0{Id+iJ_?>x(Ev_hK|z{m@(lDiqF;Mh!T8PraxL_7$}N z))t|b`PxwHKK$#EsUL~8@^U0cJ%U6K*cJ#UVRrWJUa(%B<;W8w~iI?xg1)AS)P@E&D>Wmv0y#zp46DlrZ5PaV0gv4;|~fsDJ_Pf z{(V{s<>%>fqrgJS$JNvGJvreN6&&a}LoO}{0XysQ1K?9uTHG8q3Neg{#<`;Bu0>}; z$uXuF>dH9RteIkB>P*^PcXg9a>x?+@cAC?Wli{6M zap*YO0IKBT|NMX9<7Ha^Xw@vBuoJxICF4* zMS5sAlhTkD5liI53Jl2>Ey1nGa14r(5zbKAFaX$<)YAxdUkVC+r4ra`>IoH4tm7)7 zPKQk8qByRlOlkor&#R8;uk?)Ga`PQJ&mmoF0$#?~mX_&G4Gxs28WnJ?yY)~K3g$!? zlN%#Y!Ezd+yPw?RRD}jpH^|LOc=`Hs6D-wRu zS~OR!B~T+~sO24$1zVYMrizQ{r{`Rf`nMSGkZ!GrvYfCTGd^kw?zzx1vV0VLj<_dp zS)BtMS96a8Sh=2DuNVF-YbY#+b2Fif^jTvhBS3vs-w}H@93dxCrUf7!O-B(%O?G1z zjvVZCD9TEdrcP!%sQQPIXcnUt4~ql4lN%u%ZFR+d4|cXB$e<6>cLsI)o5Qu{zeVi! zsf?Oo*F(e6$0Mpp-JP(c9j}?(%kbs)JyL#M z#Ae&6Vgj4=q7d9sg;hZDh!7MWFvO+%E(T<-dYofwM=lU@x}cv#4U=3yE+kKoJStnR zz`^NGV?h%tnu#V?sKbD>f|XgF$c>$KjZ`R!mHMcnK`Q1Rg9-ud*ug^GQ56H%G+lZ$ zH`1irCZrKz7}+>b;vT?=it@Pu#H~&PiXpP)IrOzl&uA%oR4oNkVLQsD-_7wN-N0LN zYFeQ8Wo?k0UXT`7cd9K|{s{7E0JUVz0??C^lbi7bINEV8rVBFf07y=XpiA=z zh_dST$#SU=Te(Gy5c`f*we&EC@9L2)m3MeADqm(a9a1x;$}V%3$usjaw^i72zU8#j zF)1j38R_Vm;R9hWp^afQKr2btN!;(VKT|{u_36!RF#E!PL;RaqDkNLfS~W{ z)=OZ0WE!F*BveM;jEXkKz*l(vD&XK>g#qpt~HTB=OQ%n8#fA_tsN@&H3nU-amiZO~T z$@D`v54P8_w3|tq<-76cdi{#InPoklLrndrmUJV1^ByRh5d#M+Zn#yT9elcl2U^oQ z#nkbbi3i*+nd3}HtGe^`j0yg8{41Vq{1qpEMXh;=#Z%^WX5#_9+?trQZ&S?)rjT2o zxp_(FwkRPzs`B8dT1*#mQQWNZ@Hy)_3(iU4MYMsm1O(#vAS$|;pv!5uV0Qt5~=i@>s~BrkkYpo#WPFVWmrXpj(p68@=dc&l_JMxjE>SWegxC> z>5d1i`CG<$KoXXGhL=$;P|HeYtB@kgY{<0h?{3{q+LqT08~6gv!Fw(;!dKOl->ABc zoqo%5x|U*?=j-`fh3iaOaLBOBi9-O<;}0I`eS_&6wPkVr4kBcZPvzlmNMDhLY&Y-+ z*wVOvc=!9eY~?}=_87K_^~bkiH>QUSzv2cip`H1SqrW@Xfa-ORb?2ty1=|w!J9~ z6;zEmdVa=hr00iAW@DafeSlS+c4JEuREG;{-C5<_LS8$lDqDLV<+Ya^fo#_{=rt=o z@Xxj?_tz(yijK3ht;g{ z?3YOCU&6vAU^J`yc_kQ7_oe}nc>wfTE7;@ylI+ZXCPQ}rn}Np7VmY6Rcy;7yejOOX zbDm>Z&hPK}SdFbF=Y#p7$8Rhry&nfr!^98Gqxs>Wg%0D+AWPtG(`L_5ryFJhV zk-4)KSavI$e?FZD2GqKtSW|kcl)TKV;~MiZr5wj{^cSGj4qIQ6&#+pbEJnIP(^e@^ z>hsxreh7$X@3qCSOD7kr-O!9jX1rEs*p*R2e&?UVH}Kr_eL(fZTZ5SVKbX7!0e%=G zEGi(Eoeo%R4o`XjGx6i4jXxUd&(1XItXgXf{=r!&jTM%-C-BmxR;a*(4>CgnVM$Vp zihHu(8BA-$CLEN#sw49iHslCu3>?jkwTiF^=ol%_OCu%py{NsSH z(=!yWs}}wlyvi_%$&Lsu=)m8C&aoQ(8b8cx&-v3sfZMjeK+;w<_}lIKhE2Y!5S^lZ zH185^M}bKW^kb*Zd-L(*2)0)H1H<)xz!Y2m$NT41FiL~=9Ae@VTG9u=!t85>GcGIh!$+9VzIj-N~jh%fM{=-uiw7mpC{o99kGahkvCsqP8 zuU?!^z&A|v{96#Jx$3}Lhf8PK70AcC8ipqQRIiiqLs-?R1eEE!mJoTU2(kSk-oM|6 zRdkHv82${3l}WRif4nldZZu}xoQEuI&LO6bAA`2@by4Ag8BI2vi4p8 z#y*Da>}t(sOOM4?qaLU`_K{64`?2dsS6KE>zc`~LTlKm5`#m6f+lwFGc(LLwczeZ^xvKLmZo-`@UYCUaO_!2EH7MOaKmq})^Y zb#?RMz;&K5{5oHLIQ%7Zc%KK~79V5YmY>-7A2(qO!IOaAT%7jOLkYop75H^#4Y*dT zY=FFkef~7V$dwcWcmz&l;KBEe2q_)Qvo96z?R}DON7u0wXEZ1Ha0hG59R-4p3uE-J z=*BBQU4^hlnE4esXq9G7IKmq^-;yc7q{S5^$n(;8vfsfahkwVD?=P@&KP)JzrEJ3~ z>UPH7ja9zr#Mpa+SjdZ#AiAq6K(G|&xeWvPp+q-U-lP%_Pb6Onaa$#@F@i zA{lgwY2p-bUZ@&J_EM1>nIljjD`#s`iGrvy(qbe?`7inL;z_1ib_gUm@jV-Vbdgv0 zIl#xPp}?^7dmdT(7*yDC+_0j17J0}yZyvec)ak4F1jUP62#@uh%45$rvrS4hM98>E zcKEx}h&&cJbH!v~CBbAJNgw(_;_d?yB;<#Sr^4dUYy7Rs621kc^Q~up_Vk46wdkpy ztMgP%GsswmL74L_z&*6H!ij}7h-qgni`Fn8z*ly>}Zg=}@%RFbR-26%yA8#ImC$-C~pMyR~|2h21^@k7NXyxT08g&lv%eztrE0*(sG6n&8q zBBu(`?;SB|+5$_KynMLkX?uwHCBURB&ELxDPVz2hNuf?46#RtDE089Y3* zyOE=X#gN{Uo<<9J&g$~_MyeDG;J3IVtiDlwHaIpQnvy;jSVkyXxfnMLyBw!Yuw@cNB z9_V72WI#<`U8YJF#i(AHN47B4rwCBkRsbkE{R4&`9AV_hP&-E8)v!<*{*+fUt{U+& z=`h3nS;V*SeE>D40f)M8C2&fnPObd!3{Ec!M!CzlNub}d(>#26fI(OrBBu$6;3K)|M) zy3DJQsx*oJZiA5RAcuq>WFfN#@Wi7mCjI*W8(!zeS^QVQeSa`hms3u=c6wYXIhfDG ze{TTtS=B+$$uzoap((t9&}cx1B#R+>kt$SpN&& z`r-nEz#tOjlG5+~Q^h`A;)foud2V(ggyJCHNrk3KD?O30vZiqQw29Qe=b*C1uR3bV8+U4zh;9D9$giJ{01QzBoX2DqwG<&?$ zDu-0q(sNbSY5a_d zoa$_w{0;Usf5ha2-GNeAGbd?!Z~5EJ2!0Lj%b;swneyvP{Btv$K^u}}mo(LFplOU< zPFa~I?F3NmqIk>nYzHa)h>sm%sS-?eLvd9GoNJ50(dxLfw#NB<+c}T%&voM4Ta*H< z>;$uRG0ao$u4TVtRCZn&rFRWo%d0K-0PE|DE-zkWTbD9z!aBqK> zZgTSRZ%I&@3S-!Fr&#O9-He>jh#wLhtS4KK=K}YWB2p8t9V^LmpVddaZ!QmvF6lW$ zn-XU!n?PV$DNuli#%Xy_Xi-kA09#{zF`^{36C+=!!iQ}pz#G_(Wn6y3SrqjJTEE~zlY0lE6%;~1O%?G0ho{DfKQ7Mc2IZ*2n6^7?^xBADDyNQ zB`Psqxd{dlm_(V%w0ErPpDFy{lgAJH?la2QTNowyBH#WYu|?Ya3);S8tJ_o_dubq_H@ zB{`STos*mya_|j5d_I!bd=nU@>^Z(o8_T=@I|&?`6$hcoNv=pFH9z>^bkHtl7s(tq zXt&od%=2f2Ikw=cNwZp5Qg&J9`(E|^9{fy!(hsy+wBJD2{)KO@Z#a;j+rbyqO@%nj zhDnC~$r}StG2gk#{6JNYa`LC@ebqV7$vtLYzWu&}NySp6<(P+DdOdPl<#<1H zzIHe$LB{{V6A3K=VWLtBKQCb*{(mr#3S$g{oH{dC^mq=SD|v+Sz{OU5NAcXR+&qX& zwZY#07L2cmH&FC}5EbR`0;FP>FN0xD#4PR-4MEXKyW{-T!yIXew znA=VUjTpfI{%)*s?l@R`sye1=wOHD>0c`$l1s+-Q5G-l06fs)RW>Qj#MYia~N9+z> z^TEuLy!kA*(sxF!{gK5othMJw4E;F%uoFVDUN3%G13Ew|1aQ8MKt7DIhYxP0f}?~&=g6fTMoZzX-` zbQ>V;l&XFI3Pk9)*a+90Mop78ttaQ!NtM}tpfrmrUvUhpwi_<7j^SHlQgx(9IddCa z`^h@DPq{wY+OT~2vk@=6BVATjPUc&}J0nfYOSen+=@^1{h;Ej^_RduK;s=i6x)~g% zVNfOQadxt_N;BeR#5rD?9mRYzbD3ar7Z#sd3&P2FGosXOj+d+mf{=Y_J0$@Ex7;%Y zmLjs-+{kPIM2JM%L7tF z#VcHX1~OLNfOLHw#Kl=H_}4IfIN82*o zvnRWYBsyXXZ(j*G#(j0fKBcE3sJE?mmQNH6d+eLHnqKiVV9>ao@=fGT;GQ| z6Np>@amVgtKh{LB*9a5Slu1o|L@pHkV%(6T;PI2kEW{&ezHlE47tlC3muJ~;M zyZ+kGcD98WapI-{g~i^ywXrXMJ5ZZ3uMvMDZ|3moZ+d7Y^j6YWBl%ZDJ+`{+7;eNO zePT;V+G`_AUL-Tzz$AaRe6QGAHU*j{pj^|&FYChd!}Jrs0`%h=a)8HxaQHP|cPSmBD$1cMUQj_&Eo`+hf%e+N zFilnk!?b85lPV9J0i_5%wz4cz=I7r(JFVP4kQ>KsMg=(~TpE+?Rt9v6swN|Ik;o_Y ztkN>IJY41b4rm8!DI=$SX+CDsa$^}U@gwVRypNBZp+=!xZNU*8n*gjIxdWUKtQDF3 zilgk6$j6}ChD|)?f;-o|%+a!O8B1Py%W0FVfL%_U0d}tJT&;6Zf_Nxuu6MzRmAWR@ zUZC#ox9B-FWW)y!{Iv29eur?;9%Sc8SPsIehMMK(_w4%R4J?^&EaBW0`IeQI8ms(q zDMTLpnytShMN2O22Vdt>OP}Z)P&M#5FLYEi)$0@dG&Y@{uzfgbYqw}r_qk3@>Ja-Y zRq|OR#&?5l&`@J)#pIwD+~QxIzMz>2CU3_o}a84!_i&Wvvj%f zD}I4a76&nec_!sMjhD7{Io#Hi)Rt>HiagK?3(NZAdd_%pAn8&3|M}&tLdw z*G~Kzc^Cp3P{a>S8-Yn%!hFRIY>+txu%7x5qIMuDRnArgE`z4LLY|0?j?@FV*lyTO3+o`E$T zT4O(TA}UA9T!1f8xWGr78K!cPtkt;Bj5Es}ipR>--R#hL6yWq3hA4ZogOyj-_`y$k zrW<&oJdM&*b@5U*QQtd2a`a~A8PwREB5VJF^kbrpaCsWfI{)!SO-)j@W#jZexh)T} zN=_UA|Ix-yMy0XO$$p^zw3rZi70Rr}36>dFhi{w0*?+z&#%cHE%842_{<2*)mafUO z_f7-!NvaB@_X#*f8p>b2WfdL~&U8N@K3_sv*U3J++qfy#>R6soIC{ zZpU=}PUZRCImdD$n>y`sXE^X&eu0x1pKXP$JG=pXWEq)ppS_-oWOe%|8J_Z! za?f`n=~W6pB@6kGHG0`-U{ma{GhJxY)Alep2qqU$hV0WhxsP^3Pf|yq;Q~`_&vD>9L(biT76i<*rae7qco3- z%JJ|%3N}tNoAOd>378fW6w1wgL^ZWUUs+cUlv%6Ws!4w(%Dt6OZEy<+cv2D?wNmE4 zMr}iCdd%O)ZEjB{DV}rPJC%Ef`9$p=uX!4h94-HSk zJTp{uZL*(C4}*2CQZ3eF10k;+>YlH~Km6?%OO71;hLb(_06VXu@`n8y#mAXe!1T9{ zKy6J;_}uA5pwb7V!Q}NI)}1B%I7IpJ|5OT5drzlBK9mA1nRWo?zd!zWt5fR z)&q{)B%Ov*_h3_Xt^%&Wo(4Fv1r$ltBfP~q1uD9^i0!{oUZ4N9=G80}SV)!zWsB8y zR=*SjH2ef%|D?N6AXMqc%NsehUexT6?t@s}UP>`Z$r+rQlsA-yqIpTLJaq!Y7)K<^XDnm@d!l8EKS_HW|rk~9PzhqzI%>v7{tYzI*D+(cI4*Vh?O5Rh*@$#EnWlk-E^Mp1c=@&b807>S;>X`~Dd z)w^kgs#cP*)RKrQrz(v4N`-;or==0g>oJK_d3q4Od>{-eAzQRI_v2JR8l&giWvdTB zi&I{;FDCO`%2^hgm%<#h?F|ww6iXzU!UH8a2qtu!xJ)@+#4-KPo8^AFn>V%w zaHR8RF`M%S$8jv)X_YEU)RYeg>X+^@a!+;r_U#7v0X1f-%KeWqwlpYBy9M&B3T42KiY;th&hpb$ z3c|ux9K}%OD7k$TvEZ%JaQ%`2@0-;SeU&!6FU3(jep6Xqo7Do>scV`MDk7R_*&5=V z2p*SJ7lQTQgU5wIM6Z4g)QT+JhW*a01Lq;HPL>4f1aQS^t;3S#`%d7v_fG!hNeu+~ z)feO)yATrhR1Hu+Jcr+{+R3iINQzdu6`mG(wv3bfzO}PR=C1+Ae05U;De=@uflFI! zi0@zEd|W}UOjJ0LypGLX`iU2&J%^8X)Ezh~CUM|z#>2P>jjbXXR@mpHAiL;Ofzp6a zNZ%8bj*_ZP+3!obw9CIQoC)X`nXpBLZ0)H>B_;pn%q-<#wFSDPI?}}DCsor~`*>fr zcsd>_ap^t>d~TqDkNOToa!ZvIaw)}Q>sHv^wm&(5FFpaQO_6xcwh(NEQ##*6h8FTS z!Yx#&cu0VBeF}&TZdgd#h-fHddpk3w_)Jb~`a{_4k&&!9`A;4x^Ak|5p~_;d9?5vE zlR}87OEXKgTOeUcRUtc2m9e_3?r+X2M6#-X_IU?M6ANy^0P^*~rpd8MnHfd`?ik89 z-i=2SxT<=3o;wOKM!K`LiNCRKZVpW%6M$$)SfSL8M$UL=+X)^DttEqWkk)N2`|Or& zNzu9hgpfpl_1$U13id|nc==L;rH68Cdqr_ZNdabu^smd<32{bL&~%gb4_A*wbz!Mg zTFpzzLB4XAq2xo`3-03 z+mlU(?E=H;14tT2H%{NJih+8qV&&6xQw?khRPnVtp)LAGR**yed2Plb4*B|LhNq1F z$fibBLC$qmZs{f@h)TIw)U{{Su95M}`I~o822sQ*m8AFxpm5R5tK-Z-Ecd(-F9oYu z-Uy-|HMbaPcv%f4yi9b0xLP}H zvLy?Yxud6*(aL6LNHVv&6iEmzC#9ztd8+C|1(~-H=+s`#Ccb#WCT`zF?yc$w)g5b# zF4^e5L0|#g6JAX!b*`%RnG_TukFK$#$HXv7l@$uq;UW0yOOeXb>Ke{MM=dhq#Pcga=BF;`Dpvw-=X6&i>HW1~m1i1DYq64t z>($_VdnwyDM~8klcjZ@GXJ(tM&^oIkEa;VRw30#YD{JU>E+E8FSp8)f-%?t!vcBC|Me}I@W9S4#mX|7tyc)Nlx+V=3-^QTik#f#7 z`KmmBOWh3-)>mC{0$TFMs5eF^ad7rJxUn$J?!5(?ci;hLa?@BtKD0*V%JYG+h>@HfMp8f2eomL0uWX!%8O-HxiNOMB{_ zNN07Gby5QaDSBufDL#%}ZLP^cq{bRiAeH7PJ|7HJGu2<}$TQ&mFaU^9AjV9{T4sH@ zo$4g#p}CPKPifpHk5qI}bT(THScy_V#qfqwYc!C19)prv6(G4^WIsi!TXR|l4z|;( zf>dfy1TJHC99|!jWLI{ibf0Fpe)bE9p&^Y7 z<#WQW<<<-%gn|jo`$r@Q-hj|pt*<3truJb&-3^dn$#0er%>&TfAI_*hj|R^kDCfpk z|5`kCQ!}DCL(NjUEX?@59@0n~kxit4YIgcTcezX3QoumF!OF|%Db1y=GC>O&g6pTO zYI1lF<5V5!%p^3B7MUI1z&2)7Ps~8g@ohn-N|w94T>`!i{?o{j+2;ZFdp)I`%a@DNvrHqBJ;J;RZS_}0Y1J-JFc% zlV?KFosi`c80x?EI6SH^Ww4VX1Fo817G4Ou{#O;4&S?$N-hLaM zB01d+UzzxXy^{;oR7`G(+(qe8-rCEUWU2Wl+o}Hr(D|`GGrrVQDoYQ4@MEn?$s4zg zZzD+S(bfgo<=sfAdP{+w7*=`uj%kDlJUR{>v)AUi1%I>m{dIYGvZ~qR?!$8SQ2$#7 z9(H2w_JkU~dx0>dq;r6LzK6Y+FhPKqRd=CL79(Fa%>a4cY=Ho3Y~ut5?sYmOc|539 z;RYka&LZW{E}(G8Zt$HN;T5H*DL^@-=d_W9hrs-$f1@$@%;Z4mi%L*<4(~pG!=B@y z+^toS9GpM}ZB072geo#@vGRhv$cE!?uD~hmrl6~Q-V#H-qMlKQaY59ZDw3s{C(EuA zW0(8w&_Pm*MH4ouZbqE{fHLjQD)w^Ca1mDX zC-n?}BBn9yzEI5S?GR{o<5xzAtm(!vzDQ(0nZeW$tP35HhDHaQA#n)%q z$$oI?WLzM>VNINd8}-7MA~l)-h2*nrXIT?gv1JP<#t*1%^NesMm1TshKjK zxavL^ZAG<3r#XTqBr8n_k*@REPkl18WKBc(^aMpht9oGk+8$AAs%1Fh@w%Kx7!A~x>R$C&-#TT;27bZfecxl0HSfS;;7}E1R(T%r>;;Ra`gASr zQ&70JyC6!N8sm_tbw;5C-{Ek>4nW@Bn*qE2^hc63yaU=6ONNA8<@hImMDnZ6%9f5G zD{#(&@5jmXEH>e*+bw7l8@}P5&H&&vN$aj>)sryq-C0t<1c+oU z34%nc7BgS?1Dk(TX7h3Kgdk;CHTmf%Y_Yv1PVU44`*V7J3t3VQN`A7;$dE$?h>KJe zpT1Y!h&@rY(;DfSHq_)Ci;Phoj>u*`3I*tt&GH zH|45;MzSH5lrMWG*NTLykm?tz`+Cd#0<@k(x}y#blp=Rz5WSSrvjFesO8v}BMSs!i zfbkCo8$^&^gt98=wocr&P`W>af_^frKWp*d%yMrihc7WXky2$4&pA~ms@o@`;otSp zTy`Y6oPa#U&7=E_?HipY={uW^erCzRmtw6Cgp!+T3(^a5!}8 zX*V&iVzxVpJ+pA&&!%!}6P7Vr(eDV3$9ig@>`ej^O;!PzE<@SGtMbvASdV#iq+Stn z;Fw<6NFGk$TW#IZUMl=xq-n006a-Wz?ca|WCSadtA(JN7jP4L=MZu*Z`$j#Vdk!W3 zo(whOv3`sEWMqv*PgU5Ul{^>+wwB)(T_kmS8hKK3I?VOkB>3XY;y^4g#*WJE>m^T? zJAJvVh^t3wm(T_(E+sKyK<%M^ zrL@RJn5C%9;VDYj`TJLA0`q1C%danu1Td~B5X0AHmcPcLp&V3@&A1B(KCf>5itTc$ zPJGm~G{UTGEH_*loj{`YQ5AItz<_1%G%#*l6KA+`i`JJ@O*yR7D(8Q$>KeM7+Ya2x zCRrt{bA2s9M~hFo7(HO{_cj2&n97`d*aFapG1b3BrpW81Ff!`=Rh3eI8NSNBS&G}u zkb2^|pV7e^RTMm@n^wo-kOnV#wxJ$Xo#f7BFC|btr0i8tyy9eMG8fAcW$+=S_5y)? zE!bDK4wBRel9}l;inK%)b|@bZ&AkG|9_g;eGA9+EiU_(V1*0+Bq`^QuPoz@@ugvZ}2aO|$Us#I z3WAhR6mfQ|RR{0}OCXAUH_-)Vjs*WI_JI%YOIOt}wc5Bh;lhN>6MX8~o!C)&xEA5s zkguxow|AD2@AA=j4?kAcoek(9 zYbY}_ysS(t6>Vc&>TNVH$dU7noZ+%~KL>KPAXZ}gMrVlI?~v=~0nTI@F&+JDn;zOl zESr&iL(2ic{VFkVg$|-jRSc^6P_&t+w3C*RF}&7e8elk`i)5oLStYrzoCN<%aw!jB z8CkFcg1D;}^%nQnhNt}1k;PrqZ3eQ>)fvv8;~_I1A`G^wBipZfgZ-wu>K!Y{Q~F;m zIB#4V^mbc0sXGR;2(8Fz&ry2fLlfk-Hy@fO;jM*K-UKeSQVD69VUc1e&%}P^8V{dp zsKDMFPkFnFQ=O!$%Qc~Hhm;~_2KE(hkG$@*kqp9k1Ie%dudmuLKo?N{1YTg~pd; z*)1C*MCRyPLz(#mJP0df$N#b^B1{RsFnPiuYpHvSK;6|>O#6^rlzgrzr6vNriUkfV zRDvEJ=|z1c;Txu1(ir{Xh#pF*v`P@8hwf-CBWgfp{ZF&ot%!H&s)GIo>hZlP`kD`y zC8FMokVSZSraNBns7_LpYuF*itUv&W^)ZK%uCn_Z!xN_d%#p!vQ z&QNk-gkh4WjrmAw4HSx}LkJ7T8Y!qNIn`vB3O21i02K}gwn-{hbed%DUU&eVWkcl9 zP_U2git$p5>^2-hxt(MLX$x9>5EqlbqBbm2w|wVYa8k!qh{Yg1%7>(45?Awemmy;( zz?{ZYy+>XZN0r#$DjHZ6cxt`U!`WtLPxxhgBR(*=DbZI`xiNdm%sWu|TY4aJ2WIMn zL6BdDn#Hy^4_-Jt=W`JBaI3q;@l z2Tt&{pA&s(F+Y@V1(w!*2}uoC=QdEc`>2YrwJ0D%J;2xsMzSnJhnE+7(7A>UkL6MJ29!}tl4()+RTj?oH zWz;v$6m3kl9o<#)$CYEU!Qr#QWNHJp+kXl+KxsY81bLd+Tu%ApluRCtZkWwIId^572V=dy|99CxwX2#nofF@Z5d%5eOA*cq) z4LETM4CCWyDi1$KYm)F{ccfhG0SjniAyCF0ZOdyY47%z&Smob+`PN9aX5Rh)D*EYV zVz|f{rlk^ROO=k>Oit|O4YOf*N>UzR3@^!Z)lZ|*)nCpL+;Fg;dAhBR^xgyR1gnVl zq7%H)n{))c$G-JZYhx?Pt@R+wAG;XYP_5^36Hvhj`i_-^yNI;3&l&lD$IMY^ja0eb z*VvgRmlsD9)+S)bAw8q2T&&F*ocW!9n)Q&fz`^BB@t3Sfqy;hUR0=Ip?wdFg*HN7> z1K?i|7&et{-bkVjfSv(FLz7aFEK6%g$0K{_6q+b9NzFlPMd6(v{m-8<~_Umv(^6@w;IV-Drp-Cwn6+uIYvT6b^TR1tERq zltPc{HmEqHKZ~8G+ze%cY)EMIp`)t*LxgI$Rx8g*Ky-Vy`N^Bp$oQRwtg6ozQytL$ zTRoyJ7#tcPzt`aCp6Vgp<#sL{HKZhq%YVf&-Bh0A<3F&H^{8-CZIO`%OUvpG9r)Ik zKeEt9Ct6V+Zs=)cNnbT0WM7UEE(6j)Bgb?QuM^2ZFoldRD2;K{RaFOzlfSBjsypWG zGvP0He{}kS(KxqWsT+aB2O#77H($ zKd>Y0(_A^7=8Tt9rJ{52zr_=j5@N+VOw*ABc+W373=?jPgmSV<85?aW!zh2<1$yn{ zSmUd*hDnBh&W!)+dBG(0sm?*JJ`$b|RGRB~0XTZo0$~hs{@Um1n!G#77!*P8!J%-| z@F^4TOJYox3>^`z7>{R=7{{Uf#F0iM79&Nr!fD$Mm8aB}J(VD+0d$C$v$>pcdKt3= zk8^Aj>7-ct)MF6J^k}5xKvnuAHXO|ONp+JwcAj0IO=Mb=ZtAAUsIx-14)Q7i=zOW? z2g#v23{^$XtV?aXypfnEwrZNq0vS^2QQql>Rc_zo+fxm){^uJ=DM&R4_(x^7w3^Qb zs6^I4e(mW@kb}LPSyC?rJ@UQkAZ*dKNNszN14Ex)Djk&tTxl|dx>f}i|KSH&&C;Xl zg6a8mXsZTo3~G%)q->xru}|UeCFb%^LbC$JOQlr3BKSu>%?9y1Ut!CK13}-B>8Ush z9D@~ZH$`uV8v;J0wFhs~sJCi?d8ib_ai5rVx}G*tY=3~g76K|{**-W@V+!cY?Hoq? zY7IY7@wW~X;VMlMflEdO?zT#qdtl*PJ*Nzztn>=pP#p$GHE64J90p9Ua+R%m&*#Zf zm6Vdy0yFG9C8?kxgve4N2&GQl#6q!p8&UG0I&hBCO`h6hOD^1x5^e3Cf+sj#oLWmu zKF~;Oi1RC@E`*}tNQ%R3?^6c}U2Q6`qN|q~ow3jZcW_!$ad~jCi$^+D^`Oiy+JzV- zQ4S{AuN4204UY}-cD{~wX-{H8Q0ghe#+7m{Igx{216~Y~}@J3cd9;-y$_t#oMb zmJjqueX*|ecu$k&@6eVv!UZKgjA!4KgJ_n=fni7W&{1$#O@Sn$y4J;*VA9IXgbPYksDQ(0t zeM5;1S^cZo7g3iPDo5TympLlMr`<4^Cyg+h$dm;94n6-gs(_RB=2A`7ss5JwF!DmR zhncBs&81N@BTJrbc3WrCdV@URgS{^2nO}pOPJ0y!E#@^q55oZ*S=-2wZ#r?P8| z!8>4mb5?Vp$EZiEs_~D3b6I^c*n>>k3d2Rv3U<=r(ZJ`@44BWOt}sWs%=K0jP(hli zP)C?!F#jafw2xj?Rc_g%J()J&{NF7y247Jb`F0?RdE2B2cwSl!*?tG*jnb!9c}{tH zseH|xkBX&@))pJPOxj;#qsbUL$U*Mb3nS&^VIb07kF6%hoJNAY#s4JL9BklQJ+_Mi znn@-N=8@$w5!@=oqOKkfkOMma&yD5iLR*wVyrZd+CzDmedA7=~|6weQXUkLe)aZga zOBK5+D<#iD8l{z;_1y$y+7Dd-%+X~$Qn?PO_E41tUyojfr#~(deVz25$iwU+L0xCP ztqvl_{}!ErMp{%2tBxzOs=8++tj{$DJv%F0+yc!mcu1NzR$)qaN?InMAh||slLJgO zl8R*e0?2XS24LGYJw!Syf8gcjiDWzLg~R212pZD3$%YjbIk&6af59@^>Go3GA!JpM z;cX0G=r_+@Y7A%7jT@BDGlRwvzR*l41tr#ksAbAR%6n^qa}|_TZ{Olf&<+)Z;2y@N z?N4MM49MXLluc&*HCe4Wb#(ql`b1Rt*G3eMFy96ebcJKmRNUW|?_sTvR789BFsBWr zHP6uABspY?Cxg%K4;eeF!o28%T?qxpF`R=WCZgOe(3s)^cKQ0hpooT`#Cbi<1TV;{ zjgDpwl}eYu`3rj55HL1$3=WDA)gCLYvrMzY(Ib4!3EJ+YWI470s;r*kOhs@MndR$b zl)r7B40k!k7X}vPFN@C`$!JeR5Wbkh-zh5ZCDYXjqrsNtSR(4-U@Ph<-wi}GEY>|M z<4|$c!mVhnCS9vS4tNh+z;kUQWkwesS*eG@(7%NdozkP)$^MV9l(C98pCnd+`&YtR z+*S9N8s9^FcIO1aVP&fXcA6^4->Np2H&v3PQ&(pe#O-L$l|f2RSyIF4DUMI@ug$cw zpuv@)?npIdp60fJf?}&7*rVEUO=a)EPLtc9iO1HH(rwM_abJY@kZLMU|CK7f@u`my z3r{wcm7OZ`qcx$QxWDs*W&Wz&LnU-aqC*OjF_8{FcShiqj+eE8hK;KEo{0I>3flKc zDbhKG;~6l*2%+v@LRG1z^0Hrl$KTw>GC^r|7i5_MA`Vxn_+WBdw6)M6>SN_rmT#*V zq~^N8m;Q2~3iMsaM__+=8fDMQ$j{TAsS=Z86iQwqB$K4)cGm8vn_zvmwsPo8^w|R{ z!*Q@GAxTG*0C+8Ir22D&;Hrnb@RA!onHT}HT&1VhmH0=T{@ylDrS@cbLi5Qe0uc?A zG8iizZo`mnWk<Wo}Q9j1pTD zrRsH@L^LfMA=kqAm3}^cpj&bUsedWjs!2hSv~OX~g$-Jpa2x7sA>}?8If#e29+L72 z9H&GVDc$si(Ne=Ac$2!sGj#d{e5EG@a^vnW0dQpHXYuR| zO8gEeFj`5I1?t&Vs(gGp@jYuKt8$BvS z631{1YqDeIPo@8j({t4gQ_3XpG|P}zqhP~_U1o>8*aC{(f91?Ya3u%R#s^J|qB^lC zah^4RkuP3*RrPLmkS14Z!51_4azv>rfUU3|Tcj|5J#|w#mT31#8JPiRxvCe|fm39X z8&9KAyx(u2dV8Vh=WpV~r)=hQ(qGj$ZPH2(1PfzAS;d42XjXy4kw|s*j1lsB z5$xlk3Ss^GI;{7*rc6G*g3|`;w1-NUA?A2wl4oZg&a;ZYss=Yo-QL(#0NY1689w;% z7BaUo(5|3o*Z|boZ}dlXsQ!?wuV9-rH1JE$YmBXF?-el-^?9 z0>nE^V(}yQ8d-R|HS|QBG!2!fl@N+D>^5#9<|~}rrKdKQYF8OI8P_8?Lq_*!mDuti zf<5ug#+H+VXOJIXnw>>*wiv*F;)A=uLD(9!>{*&%wS-Wqy&n{c)^jF`uPUn2`h@{S zSfXTyY6laxfRWtvv@mv!-p9?S8LvQQ5T=5uBIuSIpvA=GVnasFQ1=_UoT&mnebkdX z$$o0o= zp9G|h@5W3f2ZjA|tI5BCU`tPUg(g8M@X&lu$*qk=P`oKpu>ZGwEX_gql~S=(C8$C# z{mkX{Ty#d8@FMDtcTkNo;H6%lSWHvtyOxdfWFLQe{1SjC`Pq6)sm-Eh)Xv zAX%t4Xa4*%!-R#B6OLA!Ge$1-L_hjH(5Xr9J0OFudWIb3I#=ngmF1_;ITqx&2ij>) zRjE9b@yyxzNUhk?keGEG;w*lGEC|NF{-A4h;Bj%Sl?T;~6%o|{Cvha9+#$T>r6;RU z(%7!r4>3el$K-1pEfEsvZ}_qpn$zD13Xu)paP+3Ic)4wW>-7t4lA{{lEYq#gTIr-L zFe;-mmXvQPFqQIs{;IXp20dXWJBg?vL(+{*>^Hh59sbuz$hj(#@MMq?0>1jTllw1$ z%xe!o=A%dTLY`%{lIi`~_HVC&Te^)6&>SNo=#5GH{S>qB)f1dh1*K#Y5jSEw$m17$IG75<;jk#*B1ua_)jbv+r)O2f37207g@m?v?R9H+ zR1W+*@h_lFI^PVGtn%hU$@!Y`{_6>ophpgpo)yp{DEzE$`>2!snA9MxT0t(#TwDVc zG}>I48C^%$|z*%j_PvxmqcH=r*iL8ZleW6)iS8MQ_UPHp0kWNFj4cAHKc1{ z52jQE6KxTgZ{cM)l>8{^R1vV_X(}r(9-4^>t%yR^BLf63s~5J$mlHQi7FGo{ztXj# z+L!`Q`MI$(M-o!Zs$Z5X7?zY;Pn8IAiD^@GmW-w_v>YZ{BxNl~oX!)y#-2-fZ z*HgC*K{RHmTLP;eSXI~o8qe>I>F60CLDXu(x$QeeyytK<8aKz%i5h!w!SFil)H($r zAK6qM9R$^KvJ+tO@iRcu@KN&WXi${qV~{R&-<%}@gJ76uGVH-DyhlA$O>u66>dB}Q z$j?Kn=x$Rg>dOx=on(%s!V!=E0jAPQp}$I9jz)E->2jXBdA~aiQ1sNZhRa3G$dta{ zLO9$!G{@->qN4fmzYfq%li~iUBj5;mwPkZDr>C|rOp~`tWto2}ItoQFtriR?-h%7W zHC*c+kXKEs2{XO)8;kA!GaRrvMxR(t1=*Cx+fz@OCabHV#U-dPmD3)ISEzz>-+9BS ze;i=M!jIzX$sDrcwEYm}wSr`cY>W!AP%msRe>ZWOaN;L)7QbHD^|gM4zm?QeX2B+l zoa!FYU;bzev%acF*1@kGGDwwlc4&u^Lbc=qb-|rqRTfh40x3eK*J?c7(Dgh|vVWYE zPW%K*?`%LDwO1Ne?QXgQRU)gI1YSmSeXU1Lk|$O8yR9G+iKW$(cMEXKZC0J2j#FGy z9(eHj!S6x*5lN|X*X)eKbhDeKk@C{))8l$c9aSaciArDXqsNbg-$XQ*hWm_UIeP@g z)%5{jYh5wX4ud0L;?KT_!u_gIOO_te1AYP>)vrv<&-AUtF+vsSg;gy z0f3Wr+XQJ&{x{dfQSLG$pTqM~IIinCq4Gm6x^7ohQ(^rvHh&W1m$+SA-hxI6R7|Dy z%AHMG)8@Rt%i`d`;)hAys^~pw zg>c;73f{O?)zLkp7miWhWs_`Btxag4K3%l3|3}i92ex?s5B$N~B=m@@(gzhs>4Q>t z++mIcCGNP3p+TuTmy|k5Ra;+}rp3eQId!C7AQ7>Z&{h`CVt@w<1NN!R=B$%YQyGn% zq|Xapt^J~btJ=q@VuJU$o>AB-EoN0#jiEf%D zS?!_Q)6HDi_tn8tyKcuvx6+#c@N9LY;a}*XJ9eXeH78DABAa2pxws(41k+j}j6*46 zJ!HheoajRmrkBl^U_Rt@LeHe(Rt|-+K5=DW{JO*B=`84iEN}cL={9XzN>GwtLTda$ z^JyGSDC56%bosQA1xe@y+A47vPGL+$)FE77g_GuS#@=cfaM@*}LE>+YG0S7b6)|0{%GtgDb z0g}<@V-}>UOZFdB!=3;)ep2H_n7$`nMNy~Dl-LJsviU{)Sgt215l03YgriXJ6Azt# z90M2Dl;5was>@w*c;eO%G$x_SGeh2+%TIn*QS+5v2qU_eQNK!6{ktEJY z^aRIb+1?)SYU^hW3wUVhrBz7xaC}95Fks7S!-DL!aa#x62pp7%DHRZ`&+EPsvi^G* z`Gh&V{PhPyq~BVW=MDhnA6Em%Og*!YESL;w#?sFoL<_>)Hl@W$?b0fxt}3r} zNi)d{tNuqxC8Hp90y-CtLoUtK0}&`fKa~89 zVkC8^!)bHAhL>;la5z2?2!|XYpPlu%A@Wi+Z%77o2f)M>O$EwCz->PTKe(-1>PlZq zrm7I3X&fKrnJl_>j?`WQueqWZHI>SbK>P_~InCE5Ku9Tipuc=N3B)aYk&U0z3rBG~ zP~=ofetmhP0=m!9p}4Q+>Pd58h{YRV^S0{ZdkxIL{2W+$M#XEK>jOl-SFOm)!&RYm zx;7O4fXk%*$DA(s(@@HNJOLp!*S4qU%A-`qeV#-Jbsud({^JLw9zVq~zU43|&z~q= zO96LFl`1f;CD4yZFai=*${=o4l&QdOc3>){J}cRq@jUJs$``$I%48kcy=vRFF}ppT zIDXn7E>SmEBpCIEMr=d$xrrw2ufawF6j)dXZ78z^k|t!AJC(BwWkWaA(?yYRlb>iB zA!VzX|IqtJ5I~zYLB|QBWslWs7e5YUIW+)04G~Mmij(G3Av6es;mC=GYp%kr?YOve zdr8$08p4gyE)U$znJASB^2VZZ_+)NXBSCt1hF|a16Nbn~n^}8L-RIKQqZnPyh6lZ& zTm7<`vU%v#C6REmqEV>WglX2D?JbgbuDaqRf04_A(Ppb9xLc z)$HjY9s7cYmm5Ql^^URYA60Rsrsr5wWN9Y2=j#Z8<%ByLA z$9vG+Xg#!Pt&t2O1gpxL;izwW>o92--M>YnaY~eQ8VFMwe%>HGu`~cjw^Ebqa~^x{ zS6JqqFG1aa>K)lE8dA8aoDVO@A|aOQfpocE2kT%_5El2hGKlFpRS!Kq0a>=#jEdSG z6iZU-$1UA40%VJ;r5Odv#Id*H?t@16^C+JS{&^$h#&;;eTa|C{a)FmZ&8JWo7OQ>` z!}KyYP|vi`axtiNEeL8a#FK@nI@I?56g)bc9^l#$lXUmjfb2azZXozjK32x6u=L(J zph3U*d|BQFQmCkIheqgGLtvmJW!=`Z<}qbvgtVv%7aOY=4HQ?BE0gD}wPfa80O|=3 zvH>_N3AfR*-(}{as-;*9IN)`_=W34n(C zy3=tyd7@>L1rTzv)DiK9|MW}Bf1>~3sNSO|O;We2O^LD2)wz$rE1jqp&yq{OfJ^~B zKNj1FY`W{ZVI(B8J%CDgiPV|EKDf{3FeurA-xA92lLk7z@FQ zIax}{x#@VSN_HUP2wq>OnR0UF@ip#OW~;mJwx1gROQIYiOMZiS(@k*%hP7v+timNI zrw^rhnANE=@5?7iQ&n`bIf1J36XgQ9N2Uq->gQ0hx7-0sv+u*3uBf;~?N6^$EvFlu z-Vng+;1oA$>LmN$gC|4zy-5wT5+AK64?)p%FV~jG;|Sf)xYPY$=?7?Y7>@3Nm}0@k zs35k^N@S9)whw&$r+|0FCf1VCpLi*p@)mQY>iOG%y>FcSN*0!U ze$!QmRX_;Xy7@*i?q7EU+A9UA=zgI#dg~g3KJKgsdk!K- zOG#?j=2CaTxm+qN1V~(NPnQE>M!xKvX*Q7}8`m1Vx73r)*TLBJMkM%ClT*h3(oU-! z?v`H4l6a!~>!}p%W~?`BIiAGSPS^#H$?iu)94MaJ~TBmjAJ{T6oEocomUaEYA5_?o#L(NyQM-8gD9QZ^)g%PTA zVrJF4dnRQFF?lw!0~I?qR^%GZ695s`)M zIG?N;`a9Kx{7c_z>G(IB>N5r5(3n6zDJ%t-xPXtq#3q{IzV41@jSxhUo-h@&0d@FY zRVSgo)9YbO7Ef?ioV5B9*UstBy{7+f=vG^|G|@)nBp`b8M{;RaCbxb-rHcL}jWFBG zE{FW|*eH}nr;SwEx(H$&n(lyL3C=9k%ctNgD;s+DQt$3?1Bt*;j*N%0CuIMfkiA~#yvbpm^DlMjU`E}o0e z9}91eQgFzTnIKb|;{B&ee z;N<{iT2S27AR;@V8lo^bSQ2i+|DJS0)2Y;j{VgCA7oC$Y+Z|TIXlHjRj%4S@}mYHD2y?zARh5s?R8HV*?Jynby&A_2Cb9o zX*a@)INi_^2648gx|3bbtp{7_;Fb=PrU-EBO9nS$?eOam8E=9E+M>|Y;9Hb|X(4kW z;VMHXFkzJL+^zL3C{q^(Rf!|8_mUlFUFF)h@${D5U^wO06b$0^dfYY%+=2LZe~A!n zrRR3#2Bf4e24QM{95zJw9#T1<-Bt470zLC3*l=D8RG-|9{8n?Jx^TzSNl&V;u1lg& zaoi>jDBsL;G%Bn<5oJf4<1K>doi(Lw9tub`6>~tJq8@C@-xcm#v{Nz~a&&$5P$#9^ zc0sp&I16ouRLTCj))$^a3VE$hT$)IcVWBIGljX&^w9m|=N!!x%x~Ndk&R_@zV6ieD zgLTzl>aUKh+7>TXF+DOO`rpIiaRp z{{$BL6P68C!%@nP=YSKfbiYSl?F732zRLVQi`7=k+<&|jnJRUn6ky~YC_^^VrrV5w z$gc*aP?g*|1*O*Zwio;q2K0IK81dT6tJ zTn%)2YZac*T20_`WOMgd`>r7^RE>cSdVY1()~qzFrt(-x*#(*&-DHkat>yjCJX3Kz?ezhh}KAFc`ml|xIdZ=*1RtT=tiSlN7c(iI za!@{l4)LWfC+xYbDQqH3N9!^jk$ox`Wm8OUUu}V5S7zB5xn2yjo~4(~#R~G&mD2v; zUV1`WP<7I=Gj3Tmt!IHrtUs}1Q%q_T!1Hf84U(evf>ktfj)K0NvEpt6prZBM zSQ$TpV|-CJg)zT70h+dr(OM-EN>KnA)u4$9@F0XQ(+f}f(B*PKI6gDnJF6q2Dr2O4 z{Rd}eQOXP{p>@HV{P~!eX>Jwt(nZc+gZi!?hegqitS$pY4WIJ)^pvMn zTqb$i9=QC{(BRg9qcS=9(LwFsh9*8$n6-;QN6^C=Yi0ZJAyi;E}v0+@ZTCoChppzK*l5?hfZ z*$DaU7{>qA4`9ZX^guj3$yrbSILL8UFD>J)!V^K@C|bzA_(HH0-{Jdu?sL-Q1yE>M zK1*MI#q5Mtlty9))Is-(vzMO^IO0ueGzI% zf$m&`iEHtz+w$64L#)BKy7QQa>U`Sc^dfolBJXT^W>w5WcNU@wL*gIhE!n3UEuPVn zosx7Pzg%3jw@B91ge7ooJ)2%OLCL-hK5im(8eZw%E(;S74a@YRHL||}$Gbbn%n!~n zI)3XB()S1m%?(&vX``xQP8#kCsx)`{7{nk0tLx?cabdaJqJYq=_l6meeTCU}+|5W` z$y`D^CV#)z0iaX9)d+BBkhd3xpEQapWa=I_?l3+ccQr^L6e&GkF+6CCMHa~i5FD7T zy%I=sIGNBfb-_y>mhK!aO*7Jxwc-??oI7mzQQ(6ec+6HC6{~0yYt8SWf-c+3#g{q7 z(Aaz4MFi|6P-?u?#%>o92p59{4wYa-1{25{VC;dVpVd9$bwQf_$+7?W< z9EO{hEALV99LT@0s_Z_&?h^E}ZLmWh-F0=}W~uuu{G@$vu$^)@M-e%AZ1I66;;gmI zl9PAL87M);;VAl+o*;Ts3o?FTxmQkrMZeI%QmRBDBO^zet@6y*E{i-*vo0WLA0rVx zLAlpjaE!&eV>YN^86_9nu>v|u*OvF1rm0K=uEVdWN*ArEMTUZ!KSs8FZDSM(f1NrLjZF;*-MBm-0{dwJFT4(s^JUuRq zSh-vmni`(HX?n&qq_M5HG&ttU7wbYs*`xAq9;yPwQRm=!1?TW)Sdb*j;)u~2r3XRo z;$E;ft)3Q0&yrui247#)t*fMXF4A*GX{Ka^M{zenh;vtMQ#j2L{)7bn4YvYv`WrJ$ zg6ygUx91|ds&=o`FHKdX(cEoJG;LmXGLaFcp2!|zihpkdU+t_X)WHt$kJk1Tcoelq z%gqj`G7WTh9n>9?+)W)=0=G+()g7I(0G%yhwME$&PIN?QTO*dR@wZx;b0wmd%O`0i zKrNPNZ|)9~lHfcgE#wt9jHD#R4+-ZUl){H#8m_sxdn8_d?sw#wM^|)QQrIH6o<(@FTeVd{%bk68SEs)*0X5yFp zNfKYmB%M`*YzLlV;Az7VKM=I|L{E*zY&5NxcC>erOOsgXT+UnzCB~WR&W@-YjXDt* zjYbN5Pw8%#OdD!0#_v)x0Q7cL11Iy_YNg>ms4!y@!X(g3&D~GlORYS~q&2&an8_m2 zN}e5uO@8_bD0Pf8(e}oabjMeOI-{v(!O1ldGPu>33_#FQ^L&wf(!d*(ZAu2(5G`=4 zqJg*;Gq=cK3(G#EVs<|IH;sOJgqWx*G*M!C&PXDt1*g^@kublgp)PfkK(6lOKM_0^ zXdn)gED2-&!s>+C`e#c6<&?Ydo;er%EjxfwQ5FxrW`yBhanvWt#4kU6gre3nffv`$ zX&z*j(+#w2qj4mZF9y{MT`2aBx8=kkfI>vqUceA+2jHE9@q}mAVm+7#Zp5tfWWXDg4A314B?KPA%{sae=h z%v*q9vhx0*^|pflTROQ&zP=TUvx-z*aaIZp5^EbeRN*IiU&5CABM4XXR0yX9L zkvHyOEf8csPg<*HISp=kL-KH;*NsCvsVbo@cEuMRt)$Bw(9WWQz(3U8v*p%FFm5az zMzyu!McVH9w8r}HBtRjspJ!HZ&E+6QYwU8Kn^psh; zwo=ZH#KERDLWTW7&+u@)R@MeVTDSv2M6T#}A9)i%Rfu;CO0Vqvi_NuHmL7!x#(@V~ zN2&1$UH#QvtnNp>c&qF^2$Ne~0Sl^5Gb5yo&_*&=$@-DHb-5Ha1YZjDU=R5rnZMpr zmnmKKvgt_n%&9EWn#gm$rtw`wqHQa%X(LJlm<8?xse91nmZ25B zZY7*bRNTl|X)+QuVVe_`;P)PBN%(66>y-nd0bVFdnvR_g#5d~iH$V~QSajMipXSF_YGTF8xM;cO+5M#yL?`bw|w42NC^ zSM5)FQB66`B`vC`Z&0YRZf5V`GDI2oS!a6Na6{Iv%h%~HYe?Y6LqU5QSSC26S$ z9eX`giLyvJ5D8cg%|U!;sJyRk4x>P?kfce7K$+VVwc93dBaL0KpNwkreORSQR zlo5!m!e^1`xlynv>tH#mJhg=#Fp2H;3@3EuuFtJrUPo9%Qb$y@yf6+I{#Z>VdcY8y zL4-Iv|JE&o&~QrX%EznVBZDcqEwaUI(eh)+r)=cSg*wm)N}h3{Mc)C zwK8ci#xlD$C{fh{D~U#w7DwQsB%uB6%?72WH^P9bD&EW8=Zn~O#P7U95l5vESme|( zR}#psRgTsC9yPiDc{s8WhZ&D*QC2)b*nsh5Od#b|IGy4&wkY$H4Ta_8;Xo*Xu zxF*^faX}|YO~Wzi=`9_G8s%sZ_VrweX~P&dZmi7t$d##jIoZV;1{a;%%%y6cgu=lB zg0G%j@rIOb*_$KEuPaqL8mjD7#>>beI-_?bl|r69gpiu5hq92D_+a1E6NafPirP|l zFZ}IGm8brxayXsT-85->g0gkDvFmkcC)822s)o7%D{8_4*@nxre5pd6S$z`oR?$mLNaet5a6w{Zs*xTF{o{rIQnwz4%z`7OK^G%a z$(?BIzK|~F?P-1)`k^ZmpcXafGpB`}*8Nq% zKF>%_w#0zGoTCAG#&R-klBDkvGqpJ$0hd=*`Kp|8=;}j*xcqCKL_w_h3Fuu-Pi-W} zzu{zlspQH4sXS5a)mc?p3!U-ax(zm)Y{LNX)q+Uvb>-_g7oiJY6z3;Tjevw!qy-Sr zCS9)gLiPOb15hqOcQivFC687Ymu<9l@g5|4$vAMCAhiQ@%1{z@qqeEkj@^_|lXkwC zu~q80iU=c4j+3F;Mo`EDErTfY!h%i@IsZh6uH@iFWyxR=U!c4 zI6&?YwMKMp9ZEo1Yq3qiE{&UtA-zU-)I;)>H`0ELw`g7Nt`u&b{pu!R2#>}!)RI#& z2-m%Jfjz-s9fbjiN~y#ZZ+WO z>pMefKa4}c_*>8G53dcKQ5izriR>iiVA@RVuCq^A-wS$b3t~AYw+0W(!!Spa+kmCc z1}JdOjq+cK|P;wMyZxCC?o|?WpnvGK^B^HfazG(U2E1SvfzW zxXz@2;a51-n|p-`dNBjGc19k-rSE^oFIax9P`wl@7WdUU!x#TW^OX4sy2AJ zoRK98LeOfm_~;}AlRZ^s*x2O6Ew4j*P4p(Cypd}|sn-$crv!484{}+!V$K{7Pstm> z{g%B3mjZZ;sj68|>?z2cIeKwzmF}#vH%i>BQAVfcmUb($bYzD4_W&wiQdU-84Q(jLF!4m>hS~GZ~$$+nBs_^;DPN-j&Y6vY`Ge{|< z5|_E4Xu<_8A|aEiZRt)53!CJX2AteBPz<{)uP^q4={cb9iL z<&c)|fHbeDivo)N4%a>^NR6C}VgKj> z3np!OO;sY(E-z6D2qxjP%lR%aj(FtQr0`sP*7n14^?;EC-s2WWLy3~>+YJgd=zawz z>rng)n*6!2BwC%+-)R&fU2y##)Pt8$O+zu#d#8(-mzTg%5{BE# z=}f~eF^|1j=<{&qpY*^)nSIAFmG(4g_4eUmKCzeWUA)2eGE}A{YLAX^rxrv|mAv^@ zF>bpmDY%~MbJ|ad9yBJ>fJi|fs8o#B=v@ga+#`9Prl%7UpW|{v+n`;v?%Jn)n_diW zDEb1YLn_@)rWvWoZ_6_I>MZE^1=&$xC!xCIpTI+k!!bE?C_#4HtDLxSkUP2-l)vbm zAXXCw*N`7zW~gtm2;*eCy#EDbWUDgBI|hJ0tNsA&R6Oy>hBAB*g$Q6-ad^neF9nt!(TOM0%{YVI}3sgKd9j;OrQTkk>RYfAILkNo~9RQ?e%LKQ7N zJsv9gD=`jhEp@5ZK7~T14ZR*tKR-)0pLUtB+X$~6pa+|Pr@0-N$@083DEC?bUB)ET zXmo-Ey?8BD;aCfw%xo`VD#hW0VUShptu8+@)I*D+sz!oT{uJY^x9+4u^AFHWw$dzC zIJ`c&G}TOE!e1Z~v3Kb#MGJ%1Wms@lIII#X1{#Ny+hX}C{#vJ(RFkha8_6=WCF5NT zFOdyZy{XI-_aTl5&t`2MeE*A4D6|*!+?NQe$&HXJ4{=0&dmF#m0&Ec(5y5d)(#s~u zUxqnJ+Kx9D;f?i71{E|icv4ExCkx}{UCIf;c|8rJL5`t;yQV|vu^CIy9TVm%=R|$X zu(*XP`a4w(Tt}YvRL0yuS2;W^IRbZSYOsMyfZUP z+T;P+Q>jR}m-MWWxYFHiKws+$Y3oGYk(3Zj9z!Mt87S@I&G~TUa+)jEXKP=o783;y zl!w=JhXvW@@2hK^*%;vG{d=HK`g0bMCGuP#$@`S9!`jQ>EDl;3wNtWfI5&u;oy(S$8&6SAuoq zg^hUhrs*Y9U>~O5^3_J@nT+j;Tq@)bz@uHbT(UYd_Y3OkkUDbBWW+;|_X9nvFZ|fr zRsK5)f)P?nS44G#^o{PX3y-Fa)$J5;K40}rxvu6;_bBN)2)!e)O>*P8Mqgh89zP+) zf@bWliO?eWlmdE_<)x+G42aEc*OJpSwUnHoZ10Z})AK9<4r_`wcuTU~P=e`l&z}p^#s*VION)Gv z`*dP1jLIL4rz|uE-t3$zS{K6yHWp8le(g{k6EDMTs_gcvf^EL8>QZr@q$PnXrp~ae zXGX%04u!dtt0TE3dZ2ryT&xbCK6Q?LM_o1?sH7wf4o~M;uFx}$qbh@!PBjNm>#eZ} zhkz>lgf4x*CR|~X9`~kX{{UB5K+KmG6`zE-3(Ur`mK%dm6F2^ z4{v&14=u$#P~2A5r(Vp(E;0?m(a88$ysjdEi;8Qco6F(L?&Ek9lNL~FJUdRCAD%7$ z&V`t#=)M`yW?4TNMAk}pL!KGUoK(2u699jn8(4g-=gpPYWv)0WQ*|0XB_Rfiveo2^ z%+*>OY}5-X*_(KuL)wGH66y1He6sc|r;w$)l%5n=iz_m7w3zX`%GOKhIdhfE;g-rC z`%n3?R}x5%P2e9W*HFa8Hyu7aLG^`*$D1xIJHrt7Mx(3z5$Q6?E|s%Ta>nJ9o|9b( z%K6`p>lG8Nkq3AS<#rW@>Y}^j*o5y>Wv+~u{=b`TQf~nE=-28TR#V@Aj164$O@Uux zPA-Z&@m1|rrG%&oXp8Ll?UzaA^8(|EL$(?3Cq`dYIRG?$l`VDh%kR5if-yG zpo4L8V=NR&Rl%_`PdOo$=p|h^95lLj^tjh$|7`X(RW&EPd@ju?n;)Z4ezb&r9Y26M zu|+Q#1`@d6L66R#h0g5@mp|NwUvhduDl1bti={r7ifu_Xi~D}Il}B*W92o&Mb+>UU z<)eC*^gMpOS;_)2cx>>Bw8vU4&#PN=Sb9IYbw0)mTsCTqegP%PjvfAGfnOC8Mzh8yQ2pDT2=dKwz*OOf;cF)&IX-&=44RJqMb-*D%#<2LVx_i8A zI%cMgjt_$*XC=G;R4;Eu#gyF7;2mWR;P8Y_H@E{-ztoG4$S^Tm+-@7 zM)@nH45cDVm3+2FBXDsr5jmSGPF29`sb07l`GGU!Ilb&4x;hi%ThyCogs`QQAN*T zu-3-<4h=-WCeH!3j`lLNU3MPIMbS99@+Ly%{nk*x2t8rAtk`A*keo%+Fmr4h@WAB{ zMsoQ_$&Yuu`IsW*`;}W{vJwSKLC*~5#r75|Xo;QRU4Z?)Y9}#G_YBkiDJWrD(;+!L z%d6D8&~TZzo+JHT&oj^%3YErDl!PQC5y+vkO}p=fD?Q>QrQHUQWfYBtB)SnIa>Q*T zM6ABMp;SJRmXFsVK%}r6;1Ac!>LMu0hDrQ7ya6xiCF8l1#FNuJK3NXcfi+MS$cIq1 zG!SbGvsDUfBL^R-GEuL7WX@0usxpN9)m8MVoFHPBN{Mi!+wUQ9-hG^g&iX2kVY``G zBHlcN_)Bu+r?h~?>_-61S34$l@n^PTY6k@bI!L>U$mZ8Gpt*WMNOebjDBxyg_EV~7 z9tH(HL*&Q;*bJqcY#1u}JCswT9*jKL5b=T|;-X&knjEd~&A`m^^@K5%(5yYjs$B~Y zY4>7Es%wvoTp2wOVM|JtNqKkL$Qy$gte2iQ3Kx5sYHU#Up)7ykwJ7(HP3f7VX^_O# zNl6<}o+zPR_}<;sxW?3)pV61)7$3E5V)#X-a7dgKXtiOsxalYg9;OlbkSp;uy zJ+zvz0e4sFraIxwP!R|Nl&^%EOwqDe(0vwA1r#3w_0TqBQc0&0mz6zQzJM=SQt=TI zFHabH97GtJ%U||L%zXbSh&!$WnC;s~LjY7q=j}wh=0;i|Us^*+#V&s8^J+H>isYx! z<~XoEeeqcT3_y80P8uXy6pU{eknedU(k-$)t50{%zG`?cQoMwr>+NWYK4w zM0Y(dl|#+gq`b53Fm=k0sc?vETE%dR8ApO_=PBa|F+n8^S$T5S8$yoc?uY#c=5dak&OxQ#^-*f*}QL>eg-| zOe&yCF?kw6U4_G7IgS|Zubhz9AV)FQ;w^~Q!2fELR+YvNu*AbQvV(ZtvQ^exhfgh4 z^}gF*F!D4PKbWKL{0mk><(8SQ1fuSfeQQaw-j3fi*H5v$)M0*~EDlZXjILiy#m(@WymA@<_?;wsI zL}p_-<)|v;Wj-Jn2*cwU9D$-!)IeNY;a)^jJu6?eH++)14PM{nJT!S$w>A;iT9%$m zi4iCui=6M}Y?CTJO_5C6RAsl@AbB!;11f#PK0;hC*U8)9W zio)-T{%2`2i9-`+Skhb&DimqLQuRXsQ^jNRwso(|AdgQ3*XAn1a zBp@v6C=bIpt!Rs70JJYq3jkpWy*o+gQm-R$gLP)*;>z>2A3?F zfgUQy#z^H>x73_)jBo(ZE zS{aHzr*KE_@e(nt=O-Xt>_cUx9X=7K;>NsqU%-gsXSJySf5*#EUxJS>e@HB6f{f16 z>{O})s?Q*mCHbZ&RVWU86VDLP9$uUwRkb^BOx+%??NygJhL=0b#6ut1Xmy#Vua z6$pIcbAvqSHc>$nfx$j2*N(-b8L#kw%$w=5GhX@gIN=G(z19Lc>ZbeOk@T7lscn8 zj3av7b;JYV5u^5*i$FW_+K50&0;K*hKu=ifeU0<)s_Mn}*OOH7r1CzPuz4}OSZhn! zbZ-*4Lv--(szCfd>dJYox*Q7xjeHapElT=cHIyzIHyYEOh_or;xngc?_7IO))&mZZ zgk^oj$VB|+)*-DU0Z8}BckQq<6ANF6oOZ%ii)$XS6#vs3;l2bA}BEFM%gF61} z+RDTv$s&)FH6)B-g_c&>eDvS^iAy`Fv&0qAd0;zMr1-fIxtrxLw@zTu}HF9@OEN!uBy6!*xggaF%^ksElh7vP>{) zi>fS25ofEsH_vR7m)?NZE2&l^f9fT(q58bJTpVYii0h`y44>436V0y!h1h0-=EU#E zsGEx--913khanmd>;7(Xz5?s~EnQRL7~#@_Mt|7$<)fqz9lKbR$t*9_H#D3q#UZKx zH5$Ww75lTVhLIslqd0|c%!q?qPhI8c0>RhdwNzhlsA6bIdSNTXn0=A9yR=Y}K47t9 zp&|H6a2jH`C_>tN=Ss%qpX$emu&5dU_G^6Ms|)`x93lU$0HyIisGH2>77#~?3CRq$eXmnNVH(o??zl*OK+R;zq*_~WrQ2v zNeu2k^3A7^J^m{?(y@cGAy3<@i=@ZR{YN#UA zM7(z8FcS#PP0Z({%a+4@v%GRL4_Hu-(u^V!v|M+h6&`aS z6<%mol;fvp1tQ!rP%gciMw9DjAZ#+lRDC7#=C9Ib8Rwv1g<6+bal;i`5FvR@LDSGj z^8F}K1yO$v5_h;&P~u6BM5}|%Q9Khcl%=XL;cAlnY_^l1;TKxN)ryS%(B*^_K_?_< z6OOa!6WO}TrJ?eAC_ERAE_z6HM;@yOM!+yL1`q&-P+qmw2+A`LAuYsveJ+eG?2ncg zZ~kpKS#9BbAgYbhzAa6V@Aeuc0Kc%MY)N51$HG%7(^ghjT9m*X&_u&!z#+2*hR7jT z)7?{0mXq!xDhf}i=-|!W9xRQ})W31hRr=rmVNIJpqgALN{XVbo43*P{Y^Ye-Y}!<% zYGspYU$sbHpWLic*(52dwnXg*l_SHn-+%Gpvy+26-31uz`0L=>cONq1glJy6zi-YX zmWL5?pX1@XIsEnSfBZH39*FtX&-}K7DhmHm47%Ct4e*b8_pk@u(T$R_K(VXG;jxntA*2~EI zW_HFi$MM+<@$B;@)xvG{Qh>eD#!w$-^Ai09zokQoN{}>C}Lq zIc{<^s~k+@QJobRf5p_zUgB*vRb1m}7q3a0{J>vdY-W^dKGwO>!6^Ou0NZJ|7~Z*p zFRon#X0d-@;CzwGw-ySC${X~pmXQk>^fta{nf^5^eg6d`Nt*U!rNc4}Cc4XWDI?Ip zqSS7Wyb91)n_)Q1mxWXL4t$C0$3pv$0Y*nM`EuwzR`zR8K6_l1pN>Dz%Xc{}&H$kRIT_QVMuiQUEy3RGO^oi?2RoqyQNy$%M!4JudU`-QCZWDisuI!o9k z+`#~8synG|FmIPE<(PAoo4)?|sUnjvmw}WA$WoBEdofSJF2g40PZ}0kpnB0BP>pV0JH|Yo z-i-d?bA~3Fy}3XE4*4?p8%9Zkh0Ao+q~~!Mzj(9}yCXR#OG>_B#(|6B@5|!RyW)$a z`$-FGd|V;^ed%V89V}`O%Mj*3M>F<u&(<)_4}(mXb-@w9;gW8EVF< zcNMy;+0JlE!YbYlva+-k56E=sU4VJ}W#F~&6J|^5!XRV6WJjMZVx;Gr@B+7U%Zx#M zM(aYK%savgeqYKcFWzSEXJgsw4=sV!#8>&E>sa=iUu2l1TWgM_xrup4f6um7WO2Z~ zSMu=dIlMGh^7vLa9^R11Uu#yeg7;N!#T~z4mkwhYD(oa{*?5;ji3~G4wOG#Xi)9?l z0OAWJpQ@#@tGD5kDItctQ-R0o%wu^A>Vu)NW5B!#{rT%|FIINQ#7MP1XBCS^1MuHg z^Vw)HT5`6WlMSQS67_uO6S?I)xdbAsuxPown@r18H7z3hFh)(3qJ4+ z)b#TJPB3jTtM2*&Q{>fQI~7!$y;Y+a;L$#wv$jLWo{&N_j1P?j86JVDSC|rzwe$&> zB|q?^x+>kg?|sJABKYOiYHZ6pjh%<9%!crRi;%YuG zrUK@wUC0L3^aM}_X3FdF*|@iv&)b9nnk-`J@7_T* z{hztmMb)Mm_BFp)F$XQFS*b%i8lEZd`5^_@D~3%DSLOM$5iDj%9LE@O!yqWNDQm7g z4AOdFW(()y`D^w==AS!}w=)&yU0u#EMlA#`;}0?QoHq@NJW#lLB^%ZhQ^IFsOJPHe zO6~G@2Y#yeW|;?7)TMPjM~do6r>*_G9dO!}B}){2el>6IQkIJh@&IU{`v=@s+$;xsFN88p7QE?*Mt#jf}ot<+Mj4yK@PXRUaMzp+X}`%JIOH2nXG1?!()Yrtu~q7kr2qJJ(j2yN57Sn* zPGII;yCJfTSc>v_EYH8U(Xh+P`+R$OF*E;K3|f;+Z<9Op6_sat^2O46Ab9mEMyfV4 zrCffT!$&=mn7zqR9&2kD9;tbO9e(EI?b)U-vQT3A={_IJZ@z=kXDJP2HI$`B0Y|m|1QhvCO0oQ?7~yBCcJEg8`%0?71uhe6D0+b8oVN{=6g&r> z{yP-VwVux^s!-8K7AdW7*EY8ILS4`#A)56p+04mrp`<0|12t+oG<7A3gX}J8d6}g| zfIph4AYZZ-;~M%I!&Opu?%B%9Jvo5ipMHy7{NV$5D>Fc@53NkQuD>f;+O&dUy|1~f za-qKw693zr$<{TT*t#+vE?vrR{}{p)@0i)aS|x}M&WH4GrQjfJ!n+Sov#^q}<~$WU zM?bzuFk@7615}#9E^gj6vc%t#zncGISmlgTY7Q;sRJtjhv_l;@$>|HMrw3QT+!SHy zWw&7T%JK9QiemodOIh8~n()6f#1%-mQXM0^0I06#_-vm-x#@FRtoI)b&!1IgkzqMZ z)~%35T`p#MNpS#V!WP4?g{vYf2XsdS!7Q|uJ~hW2tyPD2SDM4#4_9K_gA#bQ{mXE{ z+jZE5`BfuH`##4ZFRW(B(;BDMbtX_ONoUoSmb0+54$N>@^`n~g2Q>Mby7T&(KE&8% zgnhieflumi+!uQ>qx(xncBq0Iy>(kPnRU$Y z$@8-~)%f^A+1mqC@@~3SHS{Q#E=_7L`; zl>d@?s&G@-2dsZ3UF_v<9y3q>5#&430Qhac?y}3`8+^87J%A0H&LjPw7&ghJtBH7( ziS^ZX*59WcXTrr4xhKuh8CK|7wItyXbF83+Fwt|QVOO1J6ydN4)s{PV*v>i%V#$`9 zu&nC+AgPZs8SkN=W&HjwgU(N5+j)%*yIdIy1l#-x3z%kSqV=ky;-&LE+@}^>>!_Y< z{x0K*RbNj$gD1}CmG4|N;KSLP)Y*c z0MPO)E7;fNDGNB$gX7xtBNX~~xD}@k4Nh_<0E-6i82QrsZ;(8HEsXF^Qy9zK(q!4G zYNw>CR)V`9F;%}@j?Jv}hWCa5?llU@7d>Q)UwjTO=9dDzHtpEcic0*<`Y*FC+{z({Z4ueSr$=P~N#xz9PDnt&Yt2O2w`oxEu2*a_MpJ<1}N2lB-c!Vct2ccA$TJr5-{ zi6zD*1F)5Pu#SAYnzyT~17^C>)|J;3@7Zvb!&wmSl&^z^L;7a%(sj6+q6Vv&ULUch zRF1~S+08B$1!J$y+duCxe~RLSD}HC`Gi&k#qp{%^e+Lls+HA(Xs9K~9FNWh3;!Kk- z&Vh|Tn)$6sDH)H48FY*OikCfSSksQ{xDII?UQM1hgRcJm80q_l3np<{X+E#SaAbG8 zfYaFn*_2iFz^-@$&{XWgc)oKy^1(AO&p%QKihq-LcORI8Tz4iA?>nB8ZIZ#E(vl+@ zCwfUOd6!D2a#3-biT40r(eJEhRWY)`smf{56(thwnUZTi;Ml?64Yvq?0*vaCj4Ds3 zF!y>ZOra_ElGZ#tNt%?vsq`2ia3T(bbm~^K@+}lX=Arr$*PAc4o7mJl)I^po@D6;3 zf%;tMlDtn}@+it@7$)(rHgcuR!6TirU8%BiKEH^VhkKG3P%~FJp_<~`3GwDxkhsGV zKnXCsp7bZBP`);21BR6VBwxiiJ%g?-od>eNja4|cs0*%u_ES8{mO{|;$?W#lHDEbL zfXZuXqZgYnSrs9$#iUG#|9z#x5z0((r z;Phq&Skv#{D%@J>qRy7;+TG|eBW<0@{PoL?fY=SAL~0B$60}Rp6!+^4^pgQN=8a)Bb*jU?BlS>Kv3?Je9+r%N+MS*!SwEsbCeGvM_f!+k zE5C8%6JLQ;Yjg+L`#)g&9lNmn2T%EI_cG3@zY4@1S=9jDloI>>MPT*5lfMQ{0bnZ! zupE!_0VECJi@3`yeUBou@yodNvn%vn5YFzdPiD%dDIRT0d^s0fwBGtlctimeFtGf?@O3jSm?a~gkTUo!IKmDh|oIlGIg z9~?Gpv<*v;TS=_HUps(s$pibU+0mRY4L9(EiK`jpzMTUzI9F8=I#;%Q!U_GHQiN`j zZzk!UPNBFUaPEDEb&kEul>()CYe`!r8pclo?Dxm>96s0> zId+ksJ)keSwA+caqM?0NS+8cC1rx&&PY&e1dXRaVGA;hx$asJL&O%!?XMk7rU~TDl zp7Z(SW0pKkxx>aZWZqRRc;sm|P?_=e_#zdGNVg*L%uCNtjS3EN4)mZ=<-g?BlHN2>i$#S|oVh$otV zV_4;+NUO_onB=AP0H*8f$p5qF%mLi4cJ0%YEP10o z(+%GTQBO_@sykDwY{z~iN-3ao z$Ct;#g}x399hd5mLQ-;#O)R=r z2pB%jDVJq40E0yZgdfZ)mU}grXYjY+@N#1LRf0~Eud-?V13*AoMzb8SATy{VWfSWv14B_Src!W%Q7(o8Rf~ZTi>9RGROK%H zK9iTZ=Z#$HaEYZn)O|4oAeS|vmmoxN>SvdeTjM}2F&=>qt@WfRc~<$|cF$#~^Di4& za$40)cr87H&_v>dX*^&vy-`}N@))UTkI&R*Lk_4snX>d7GsSHRNX#|Z#fH}Aav9rO z-4uMkou!-_V=f_b&0bB0-UXs9;(=|Dpi}9o+*%771I(u?1#)qFuytT1Z=AfD1QMMb z$+uIl1K-fUJTkKdpY`lv(A#|g@VfOGD;uoz)~ipL!?p^{`gSN(^_pH1C6%fhq+iwr z(Kh-xjCe)gBYs63?M_LM?H&1i-c!(f(IM=u%c=vZ@gMu{W8;h8#<7L7f53?=5>5TvYT~Uy!^~@=f9U7z`q1s+g(o^e8aW$5nD;(Z*-z&1uQs&8S0{Idy|3|9AvG=4M5lsE=I{>Vs`E8PthNnC{g z-~s~_^#YMURK)=ap^lK#e;BGIQm7$mR+%P=8I2xzTu){v)6w$6(`~f1Ft{e6y){>8 zScF{)Ijy3(K(fV1knY>xq1e!CAXA~Lf$>;nq`vqzbJy)@IOJm`?7X}1auelHQ!0PW z@(6_umv_RM=XN;aKb#Z$r4EZIQ&G8GL zFK1t77Zp^$&R#0~Ct9Jje+~N0x9T$L6=etA7|DsHsIq|sU1Ung2W;Q%;w{tWCi-N2 zHiz(SX&zA?2~FhjcOdGb)eK40RYQqYVzyxe131Min>8fg>;;CaRU8%fr&fvihDj!D zgLjOd%2r3W02;qX8h&Ml+GJ-RmtT$-LTN|zGK(y|$!90}83__<#a>g_v$lgINz3Su zU77N4kXct1WJ%vQcr18ZzOLc8EcykAusM(jC!KWoCAQtj&PC-cZr*uXupwf5QP^QizxP=U8^f zD!i<`0faBco8x3ExK89xd91wE3-;_*@zG!3<2e3Kw8;atySi&w(^bQjs>PP(s-SI) z)*&S*-=~A2O_gfa{ti;>uyGX=i6%#ch9y~9xg}m*6E|0}a*gEr1CV%Kb+9|65`HHU zbS}fHxI9v(jOpk;fFw`%sfb!rl+09to3ATK4(#yQ&zo`NUF=yyqflUIC&3)%nxP2cu?}g3SPI0^a{!^Zvkg=#8gim zPfPi6G3t5u@DlB2xRa%4{vSza9v}1d{r`74m=TE)TU^A%I&Xy72SM!fMi{llKGr5B zj6L?TPmqeG3|gW^qixzKtO`}zHo2g%I)eebzv zd7X35(olZOG*jfYxU_63Y{RKPMhU4r{5K4;X((hVRj#HIKZ0lcFaqLiBX?KYhpZy; zyX;}uG8YXx2GCPSxgw--zCA(80L;i=Wk;Dc8iKzxkmWS~m`8c_9INCG0GHM!bepUMQ%IjOLa+%!% zo|&QNHxAMDK^t^-QV;GSh*MB--+dV`I@E=iuPug--%jEk?uXdhALN|Nr?ohs>?3UE z6dAAb>P+?(4WzGLLD@f|Ckt)sf|`j* z=SnG(y)0c|>b^rA6NPgH&526=PWj6VIEnBSPFdb5&WB5m#A|_w=K!@H%a^7mq zSIbfLGTUfddbbQag6S)bbQv|$Q*;4wu1r3`8t(nc)6V|GMwV=b=NcuNR7vZM$$`lP z{q;16175cJoz{UJV%bB(&Ah6^2eAJUJ@K<=x)m}!6%?~3!W=zk@U)j6<>aGu%abR7 zqeeJA2#N}~mAA(6P43PKWDyFaw%?H4deCiDCl=G|Fux!8k{OiGZm^D~)7c*thY{p% z(Rs7ROZ;wDIUnisp=pz-08Db}Rrsh+I-BgIJ5|!3%0*1WCkw72pCg|^Y+CEKH*-~z zlAmgrC9*M%C3P)lg^ua$_W>=nHOdIU;r?c_!{P}1Z9suD?FVee@^W_l(2MAo<|@0j z9Qp=}xF@TKBt}%zuv%#4zvuwL$R)(?@G8zMyOh=KXoA4JtIDF~wLl{|c%S!N z`x^B}nOKxgo{EzG&G_(p0oJ%9FGpvcH3*(xA2a{gDk#OWyct6iXC!o5d{3JA16>pr+BoP$oKK+Jfx>4HI}%4Q7!E|K%@RD z7$F~8ENa>Vi0}KzY*etbGh|Oi~t} z`#R`Dl}LMO!jySwcigJfsRd3c{n&KL?v_L$uEZySp^EH=B3d?+>^F`ng5VES>7Bhek58bt*I3puN| zpJOM-w4;~bp5+|YZR@wimr4Gc?5n6b-`tEgmLs|cSo8BJit|5mPBW8(viW@&^IsT~ z19NtS8}i-S!Ea@R(hKC@zfC}U72e)ZUyJ@xhcDGs&JOyzHbx%S(q8i=PSLjmx;AZv z#mUlxO_TiC-)SnQhiyRZsA^XwStk0yN2 zr58$K3*JrDU{}eX$!qaovSnUx4s<0MAG!>;7v88_XD++@NPeuDFZZ;66PyE6o@)bs zAEDemsfcP{r0krjN9O;t#Oh zA4hZURdwr@c82M|$q9Dm8`S)7FSbgIu~}tCaeAqYUV$k5^#jjX6NgEdkOR<6RW`SL zv5!B*rm~zR^WgE1o`P>(6Hqr?77^-QkA1%xUq~{ipJcK6Z1K!}(;>Tbmze_=vi zQDmT8tz}dTGe;^fU_EDb<;TeaG}vPk`Eg5Q{HpUH`?-JYK1tTT#CnsBtXAc1 zV*R7QGgTSzB==_X*Y6JTo<%)5qZc($d?hPGet8ys64p(;$JDnIF-=5|4bcq+8#ZBu z+jZ}^UG`lzBdNWaN;#0Pm(+^o8<#Z_ydO6G8f2AYzkF?EDjk00%o=c0PX>F->NUY z%j--kre-K;7yo=L&$k1h@n)Se*XlfkqH3&(*uOV3WcFuJV1a@^-a2V|<-K#fd)SBU zr7)gh0aaMj!pKzPi!3^0#@nP(dseV@GhgWl=gCMWn#nZnXFaX&I|k1+^>7dQCq;qd%p}ddkliFk=NxU?j(SSag)0)cHCs_z6~f?J_HY7teph%I1DK6id}g zP9Ni5ae>T!01s?h%pd>u!<_HaAw|c2W2E01l+Q0uz$-KAfy{=Jy(|;n;=oqw`!Vw? zlgOWrPaOK*tdxSUQJE*)I4@n4v#FGL(YA|p!Ak8y9?;=Gez?fnF;s;^W+q?-^7Z5m zx{P;_yFDP<%lBEumI%&&K7*_zy)B;&sBom7N>{{|OWVl3uhD~X6?}|7ATsEAGbFK1 z5u%gNAwOG(Sw*C`vrl=0Q=1uPO@p6>jBPZ+sZ+!WWlw-RC_atAj=B?c%DY`Q&u7_I zGx)?((AhheXF&avhs+GT z2sB0Jc67#zdUCQ%<)Op%t^ZY>h}l?A@ZMJ8r^1#tT}+6nD^pS+ckoG`9;r%P(xw|v zU!!t50OT|5RKF7Xu$?w(KATr5XShY8cE`sB>NuWO|1Q{qx%GA-(}h;$kx9DrrvWwB zRK@(Rt!I74J4vHkaKJ@X9?M??9mMK_rkh=6(Q7@DV-^CP_-GQwo zMPpjZ>&W`0oI!R9LicS|7{O7IA;Sp1qQ5GRLUfNO~^nHq8p6!#zecdyEA z0=cQ9-7ybzP+@;f*l#PRRa4Ox>_4zdT{H{sbYCASP$cA`ClI%qg zoN9?t44D!CEhtx26(ty%#rg8mUf#{rsRjfqOL|J|e^4ZT6pT&3>iYblBwAjdWd>Cb zCw;qKPP5TXC@_Tk1iE!GFRz8{9cktgX${L`?W_{%H4cUM$;W&>sLJqykF)-pS%q@i z4jgF*K9(UREPsC)KN@Pawo5O+{vfMJR*|&@lyNnIt!<4pQm!Y9*TXy$85JpEou=@A zs&I%n2P6BdD$FEK-=nJu3OmDTuBL zGS6=&$SmR>a{O(GviS%sfV+REM4F^ivlJ+rFpw)R=TO=CKFhl8fx2`n7^<7tHIfleqieSJLaySN-z{;B zu9bUPfXY{tzdgZax^#OwAUsWR*5s+KfP#(BVwr#X0eT!*!{;g~zN3_v4u4ISAu_{N zes6AjN`^~aU1{?W09oGmu#;U;NWf}6JD0MHhBBc6mdR8>w~Y7|;$3Tt3Xf2ZM)HsY zcBgf7Us*iV(j&XVGyqS>!B!p{k*Biea&Q{wTmMTlSCTqG(w}CcLEq6Gbv`OCH9X0M zx{+p-{HC?k2|9mJdg6)T`lFx`$D2A3R?KcMTNaqiJaYkWcIV;x-O}T%|LFsH?xmXC z`;)Q?W)A96_c3tR3 zqZ2oJu#=s-5@YI@n9aKtInr|=Z@e^^k3FWxA1yo!i&UphSst>dIi!Dm6v8KTz?AS; zK*D~-Aje_?)ma5{?k%%aHg)As&otr3KU~krKaWGW zD4(Ipk|-nRnqi~V-o|FEoW%}X+(k0|%4jTa>53AiFOgo+(J!0s3`eJILOxy4E~Q*d>hjivgdNMC!MCz5o^EQeVsdo*&x*K8Br~r(6#4{LLqx)mIl)8}iqAwb5H} zJ>5zEfPAzpswM62vf{gu1;C*Mr=$+Y;MsdLNQt2Ei z9YOsJ;H0PcHYy0!f64{%>8lO20@?D{PgCR&bg3EgUaN=;dit!E2+p?-5e|?S6 zH$hbc7**CxwrKaO^FrSH@0WHTbzQ0P(sHaBBQ<_PG@gFPq+>`B(IjosCg*-erBH&@ z8Elc=RFW1W0YmgShw*ezmm4~df1wEpsx@V&8G4v|TjgvfUoJBPMh%q{ty5G;LSM@6 zzhju3v|P@C|9gzHDZheycKy%LevUdR@^iGyXqJM^>|HtrTd3PZ^d%P`5 z@OBNeOfm;>BHMzj?DR#1;~e9@fTI~&9h5eZZdr)0mvKKR)tC;l-UfZ<)kW>zrhrH0 zbu|;@^eC+Ax(w4r=~jV~i6oj(1x96UTjlO9CAauSi=e3kqAsueUp+I1@=%Y5c?A#2 ziWCIom6re*&-JoW<=U&P{S)PMYSmSaGz3~1S@F3@sIQUSq1BH(_A_f=_&9I)NhS1< z*|~NqbMNr#9eUW!*lBFw79%g^{hip2`X)a*#KSeWe!vWE%?rtcUjPzksq4o82Rh59 zdsaEx-q{q_j*hCdhwhF&jY*=cK9^#pjL|64jAY0biX+mU(?Y_-IDCq^M(EtRavc?^ zt@V1T9M6Fghikx0RdoNr!P>??6he@nip-%erK6Sn;kZ>GvAX^=D>+~8 zYR~+ue1319WvNG7vYXlj(d6f1E5|sVRgOa&t|_gUEH5<&LC5rDLjpV4c+%s%uB*!F zEpy{6hrB!l_Vz13*}5$<3vf~Woe@Ji;5=3p9-tz%oyKQfj5YCo`TR}Z!k{-jRhQNV z|7n%`*Ad8d{3cd&!eQ+1blsf$b-GPR{Z*Kv9a-6Oc_W(D-xGeBa>I&|jSs_l0~K$@ z^hXnO%c#RSFgwk9h6A$e@obsN1x87Hf?O*HB|Mf)8PQh06XgJAZqu6dJr(FFAL&FX zrZ2aV<-1|hBtUhal1!3~VoZ-bF~}@}rbXlAa5fYeud|Uy{sY?S0K+K;ig~q z_CQ=hDyF#{e_$D655V7Vzr%at`5pQ<)J%?aVyV0HkcG7f*{Avrr+qwhyxb9S>-#IV zYpYIjocDs2K&+#vlYIC(`^ZsA4cJ@iAZfLkop1jQfiU!qZjaf>;q`aS9J$XFBN=@j z2`{??F9e43_$MBQmuu-R`OJ%s;7-KXLJ!D@2GNCjqcY1H%6EfU4)U~({{*#lsQ z!rC%OJASKbb42T3HiL4%1HK(acuwp!vwt>O@YOpiqn8ZLfdpB%c+W5$zpb_!Vtmuw z9wQMy^RMz&D6hrj24FB9t+;w9??%kOvWZOj3w7t;gj${YgP9;}yFl*|&#~m4L~%9b z^~kwH@E47t5~Rr&_$kk$DbY#&B$;6?GLAb&>D(0ildYUBxgYW>$+Il7;2QAP=M@_& zuW+e_i9#19Z)1HMh{{juV3cNg_=j`2vA=?p0_|IV-w5ipJ_uX>qkBxBP@zP5q=+3X z_gf*rXp9@9ki%U8RVuoBg2|<22^B70bnvC*R9WyoQp~`nCL{=iqglIms7!7TvKgZM z4*7N|jPa_|C!&8{f3_ zgg_>@+}Uf-mk+jagxz$UimInFX_5({7N_gF-9m?Kd7yXl6VRo&Bg87y^Dl-}Ny{Zj zQ!)wZkC})kR@z%yHOFG+`aos#^yp#I8$H^$CS5^6_ZhHVuY9;xP9o8b=v_z z{o?aw)=p$3tx=j&I)tFvvePDg{MSObuRB9HI;`la*yXil^S_+)+LL_$&8vu3i+!x} z=C3~5hhx);ffk?6^vXj-d*M<#K18Vy$rL}Kt-sMXvI|s6Yq@1ZsDsrJr^hF-h3mQj z-J=dG8wv55#)EVZ4tjn~qjs@FKFmd{U7`Jk_})My=>-=1Ka?w*v?nuG1;a ziYhQcz>@A?v!MvxMB?WKoC%#p#t>+6)dl9qBoeW5^^j?wA(A`yvqZv>?d9SnxaECS zNaFrcm}0eJP1AG|WmaAi8xPfz!zYo*J`D4e=y#E!7xj2&k4jCH?%nxUaaJV{BG%Dd z>vVnPuHId(7YuUIH`Y@KvOLMf6WZ1Z1 z9P#_Y5i;vpGbAUTHS@{N^bXQ#FRkRO`RMSxiV#N;G=Z&*pZPlr6=OhEdC|=qk35G? zxqc2#82*u#cwMhJ`zKyISkIni6mVYzFQbM0xj)l~yG6Kf#8#`!_!HRwA}t6wK-vy{ zq-U6Q{D$*!>(=0h^G!;1m8&Ix$Zy2a1V%_9Ie^&Zj7WVyVguheyU5jE=Aqu$Hi|p`e?I$^uuZu*~g-HNzeNi_wvv2!zX0H6)nGYBLh6)+S z5MfzQc`}Xy{>^_Qabe%SiXyD>9b5bT8e&KD3p?(5I2IP7pM@w}T>{=ACyOj-De;e) zxwxyTwdGg>-?(#x!=IM!qhD2FeUzhbpgi7zr!%Xrg;>o2VPojSFAH?7!HzHW)<$xI zH;P@7*akOhHmP3ldCy=tNK6)U(10+eAr5n5q z2O|-N7@!hj(UEL|F)!n}FvuMw8+KT3xmc4(&c9BS(cCh#Oo|^jZL;Yc2T)hpfZE#N z#HrNI^2GmfGIQ&IpjQ7wfZ+K*u)}BcWR#^Ur6p2PFkZS?e2qr^V`O1N1ZNESpZHQl zFx(I5MET2MkCT1C=giJ61=rViGj7@KxXECN#S0UcK!mVOi?lJf3s1~~Bd<%-we zQv7ug3#ZDh&7})b&x7Y7Q|Ovm$a|eGf|yQAo}OVQ5!MS0Wd^DNS1{Rwh6W?#b{Bit zxSmxg<_WlOQostx=g%M)_v-02X-6%I97>{i1$N2OWw3p|gpQWIpUs&38;aoRTV?^` zOIH!v2h(7Z>k+8z=&6ORuZ(VqnYeHgarmlTnp<9L3{@QZ^7%ELLOWKEQzjZ0MfABS z(s($lQjAP^@rg@jkv!hheoT%&^*bue84JtHGE7BiJIL58-lyb>M?0NMEh zD()f?0K5cmGm`6aJFsp~7x5=sEnqZXT~fv&lb-=_dPAp^^Uy4y|EJCAE;U%Nw>=dh#rkdq-W%5ucHM7#nmRTii2=Ae_3R3?{oN(HFKSo;_7Urz`rbB~Rmbzi);d zBUE6Z2BdK^R#&keD#dVB)8-|Dv{>ZPIg0>){v`QWrvSX(ndL<^PfImsSETSRccgUP zfX`GjPqzZ`PC)JYhsoc$W-3}AM{*Hq#YBmXC@c=)v5B-SlpT6D^Tx#}+LR4up_Gp> z19IWD%nZcOu*-gO3FMp~zo#&=gr2u=3hF2vkD|9l%kDS$YPQaOp`wda9920)9yxCB zNp&1Imfxwui4;Frr$`%g#mQ$cSf#S!HIuj(ebAwzqnFG&g!eY%12bFxjDtLvl_SY` zFV7`wbmsSk4RHH7<(@?e25Yak0kM8o&py8VDWW#yDEwFjPB3ofl}d-5aPbcl5UtUw zWFB%$#^`TT5W$YOQGQ>(&XXAS(^~q!z(=m?{=)T`w#esh|=k`a@f0KN~{ z(CfLKCHiX?@q0_uz!Er_sLdGNJMR%0q;q(y{vVXDqg#_!Xpr;6QZ(kT%G)1CB%eQ9 z%v|~9Cd-^t0q=bJ3z{&hCdZk5l4I|tIODW%GF9MhY1b^m!8T?+@jr3-W z-zPMcqddXldgC$en$ijZD;Cohf3jeZ*csy~&qtVU8Fj<-;yw5V!F_ z+B4IQBa<|ZEDsM@H4woCK%gULj82ZDUgWFIhoUQ^wpjEcP|iW59y0*wM19zGxb7_U z;7tVTw!X)C>RS}yFx~cXp9c%Nww>jaDxJ&(n2b>hH!A${bQ7#sp&MbJpn`F7;2)Ng zppqhFQJF;tkiSqjeJe3OJ;QLtsx4sBQ~0yE#xa9syUAhxrBJEQ;Z(@!NLKdTw`Msv zLX``!A}Hp1D%=eeOYhoQD#nz0lS~&~nnHDC#{kexN9FD!AA7OCt=HnqMt5Z``8PP~ zS%-l)f2-6-jMCX0zqLhORk&Hj9e7X{&xACA z25e}lZj(MkPaTUx)`;yorpodd4wyDsv*epXB&U~7d00~&N?B>Fy9hUpC|Th^dF6Zd zGWcQ8&AKh9n0gOq+GN?Y5V%QlrA*vnmP)^sEcW{$cDL-W!5d?UTXPeXQ=j)_G)LFs z4c^n8RQsvQsTlk6?()fHz;CB3#7wdu221RUhQ&Yp)pQWrE{(%Bx<_F%gZ(5+x3mbq zixe&C21{^R+ezZ|)%9iNtfS4oi&k)zwE?b-L3>6ecxBuQ)b`9f81@4yshQN#QR4+~ z*>lxLi)_YsFX62{4Yd!coYr#x0ah`&AJ4s|{Vo!ky32M{86kM8z2#3}$HeBDCDLj) z+pA6>AHfJG$mFxE>B=!qNRit$;T(!}=F5{s=#G;* zMYCWyvQqyszU9;@B>Qlc|qy>uzTVFGeO$6$#kNBOEl>V^#~oh zWL>$*>F%1y4o?-^u=pOk^j4_EH7Ywwp6!p9eAYrG{-L$@7oVHN7?fi^VqKtXpWF`6 zC4tnl`bxQY1ZJ{W70pIt=O@dxkye7lR6{Y>)!pdMWa;W=8<~IOE&2<5yvb!}Wd8MX=zKA=X)W=z;KT260OI(O^Vp<9Q)OUl==kz%grT&eoWL6m40o#0 zlv;IwA3A%Xz^(Y8QJj39S2I)ZM7rH2Z+750FXM~Jq2$O^L)VmaQ-N4YmlD$Px5E?U zFye%?qT;{ORU}7V%gmKZy;mm9j+W2kiZdAlnkR4b7WXZIpiSnD>ERdHNqa{hru_;?reosIUzHRUCI9)2U z`pKTreB_OhVAd?%nR`fY7%1GHZ~WZQEEkKIj1CWWk^2X*_H@Z^FE8~lgIvkknu=p9 zTy&l?Uk;{Zs&TR+N0K+f`5S2NEA1bK*GKf?6R#IwCs*kXX}N%KIoApL?^7WqO;15! zN*E%z69|pdBL{89OB~{4T``kR4HPgc*qm@oVK0f^$Q~VOJn5`Tnk&cXKqFbYd&1#I zSaHoi`0FyAjc_8XP;Py}bLpL(0L4)bTlI9|-6hzC92L}KeRGn4{-Nb4jF^Wd|2+)W zCpw~b`(eAAR+8i!^vuhb-P*9G>Aby84N&?Qx?3?4wb&w!O(i@kn|eb}24(b;jMwGh zYIEkvyZiYn*H0`NZm3OT0l}wc_fCb%xg@{^2?PH&T45f zOq9~ko&zKxM!i!<@pq|U2l=Kk2H;m!sT;c*4k?&vh2&%mC~$hY>68VDSk9aDBR6u; z#@BTVVIBVdsC#3c zk(eGJev+Dq`vcdi&tJ+K(|GHMwen3Z$5;){kef3W%nTFiz-A9 zgIQ*YEI$b!MYLqUwaJ~7qyGXD>gq}s20v&n$!;J`@0$5qYHK0`C2{iewKSjHXafCS z*v9HQs$d)GV?(-jDIb^C3$ehuzTk-Rw&ZYJjinELJmstDun%$9h0wpU77CdDi%ULc zwRLm?;PVgc1@iN!Xu{3PmW*+sn1XKVT28(ZtsE{rm|-@iG-n|& zwq(4tm<`KR{er`rt#Vt*lRvVc*M3J+AB}-EtoVF=YwMEh-@;Vwbsg2Y#k}H^hDG^qValgVRu1@@(qi==I2UgIi zyQ%EDpbstHFL94^j>kS>J2#G_Ioo-mU}X{~|LJPLcrvwDh+$ECsrPMKKt@M5D0N^2 z3rVQ{gRTAdyqN;$me}QHI#lkVGmz%g;7CI%HyKy9s1dE%%X$)m3^XTO*9^${JS&LV z^mt`L6)?rOZNU`ZXeXJu2hD-#?l?@bryU)%J+hQ&DA`Y>pg#9L3b$In$7*{hx|yB5_qXq_(PXx~%)#49Tps7>3o1WS8oz z?765oB1OwS0c#JYkDpP9e9zVyKl6%nfqyA}A`tnF(Y&@qx$|aP4*oQfL_5*>GV%t; zz4Z#b9nd*_$2crG^B0QkodlkHl#)hi+k(|q%;R%)2UrFktZkDl$xNdt!G{^GM+Gy0 zICVU5vMgGX?pZ#0t~=0}_NXKILw+BUUlZCI?IIIwk6fuz%|af@=>0_%43 zYRv?{g8=YqRXGy+mXDBkb>iyzXA$cWidLE=C(e-;Yf!L%WSV9A>Yxj}AFhk43*)8H zL$Z@LSA@6w$h@`ou&k~?8+Cm>GgbDz3^D&wp}Mj^DGeL21RkiK8ku1%<9L2hIjJyN zWxUpF@tm(?x_rD2GVZe@N!L|sU)kCj+wexZ8!(XDO!9SmgD*eht5GUx15)R&Ck3@& z#^;9sO)$A zYQLx>?ConMGuQLcyrn}pG`nsp)qW%J)B1UiVs|p!eq^)dLFJ(-&*)jD zAH0tk^toZt%WyN69rU8)Y%T(Ik{e<9Xpofzz1+RHigK>yMJaVTp4?$XXdNa$G9o3f zIlES|W-0MU22e%;TG0-bu`+SCRfvI0eN?_IO!LS)^U-VznsP8ZRJez@bU%a)<%z*t z%bq0HnSmkaE`h24py7+{qZcEpy+tTCzZUQG_CcIVS|jlp-Na!->2+&kdeCF8T4?s{ zg(we@{A}{pP|?nMvgaGTC2q>6Nr#WIG))WP_*4~YA}4;sjCMZ3DL2c5OWtkB)<<7L zJQil*Lw?M`*ZV82$f%R<Kx+DyOlOXG{lxwJZ&HH>bV)G!4{RuIFI=&H#A1 zMDJsnpA9{a>Twf0B1<^^?3I{FM*6&^$~Oa;9eN%BJ>{cy@e&Et76!q#m6p_{k<>Vp z5)<*(mBBBWITG~<_chfPB477wBc;#AS`M&gZgbm@o2w7b#?LD6gi-eDQKMvye5Mj= z$zQv$);m>rtIX-dR+z*vS(2YNbpxJ+TI?*?g7}Ot5}@dBAz8FfBT^D-B=P6cXw9+- zd8wjj!ldbHG-kE06oq}pH7QMyv%04>)7-Wrxs`h8Re3Zgn-ncuCRY2FK%yQvexqNe zO`{W!j9hDy*S;6fV^u}sY4YorORG_}FSX{7Tj(svrg50D&r}Mz{LN6mja7a->D?gG zt%};{HY65d1A-s(nK~=p5AZF<=z7H7bF5T4RC2)2=~*w+b1)~~ z{{T;`0#t=9gLP%exwIs_%$(ZB$@C&lG!%gj4YZaebgk1t-G-9YABRFJFaQKM75Tc+;7iXVEMML?c7Vu+ALr|cV{qS|k z`us%Sig&Ah$!WZkoB@NSM#=5E2v1kt9eeRdh)n;rJp6>p*db4jg}7BuqOK`)Y9~EC z(Dk^=UxUAx(m)41C+pkmc2YIb3<81+hhu8U(Hf@0i>R&gca(KYEko8Wg{Zf)awPKu zi>6{frKOPNXQRe<2Jd0m=nV8>!9po$&9*;IE|dj-nl{2+1RgfDHjBul$ek2@FTlZgX)v(tM;!6bt~m z7EG2tlp_%1$9?!jhi&O4)0}W9R~XipXpW1@;UgE)UtfCW!qu}>!gATs7>)eiV#mO~7#&frpf9 zAifJ{8K-BW%u?Z0qLSIQWK~DBImmVmH@1bt#5V!P%~WQHAa5{@0ikJrfLH6V%_6ye8Drv9wiwCjmFbcH)}|F0 zcPhMWGiet}D0`di*NuEg{Y;RV?{OeKRAD=!vf%GZouasXA%|agjp;!ST?6FeVCeKU zX7)Aisl}<=Gu60kl0f$e7ANr{{5?UX4&{m-Q^%~(xk##Sr7Wg{8i!n_q$6WzWgo3G z&>@A*B=KX@i<0$Z0p$|5Nj1vE;n(s^l(c6s(bj;EXl-ssHdmNp^vNTm>phrcnGRig zBFiNv7DD1D57r5aL$Djp?y_wY3_VvlI?JX27RHAA*T&2g)!S;35^8?jj>Fj`8iJcp%4xdw(wy&uzJum#?l^6DGl#&Q*2 zEAIq>Z0%H86cOMYeOGO(FYyaF;`}2VV$co*CV}Tspi{wQu9*rSF@|TA$v<;BlUs^B z_6^)5gKH=lT&^2}b30Za+~Vj)STcVo+<`LQ-@%H3i^3fd%Y^Y<#1_<&F)K`+_8F=z z_RegDIqtfJxMwqGRK7|F8CSs*jLvz!7nWwcE;8VX>rts;!KueSvT|XxN;eGc8*g;7 zw%Lrlx9xV>Fak{Rthbnha)J6LiZH9>Ag~2#cdcwx~*h?XOGqeLrdncocxJDZUevn$@ zAlI&^U?e>5QfU>$S|mM<-n>vz3d${vH9AyOq8xJjNb4?{0qHyjaqILojQgc>v;|y~ zns7k*qA{@LraU>@&gJ!M2y`J)j;psCOhx!wTibyBBj&e!=!2d?u%riec(>l+iT-anfC zTq-Wq6+88eQ5E?{AU<2R)Yv$e~0aVqx%pHe-u9HsxKqI&|_Rb&8w6dv|lk!6enQo47Zdv z2Hn0;Z02w=7-~%}0PgJg;u2j~(NNz}>_(%P*wLH$t>od)Q799&>A4f1BXipE&BZIB z)NI}HKgg_mjFZ2+G|+co+(KkX|E>IMtIBC4XAW}IgH`z&Y1ES6ZBm79IJ_`Xy3p7O zb5!V-6YE%)A4e3fFHMtmufo4KRj4mJbSN~1qc0qnQ%A~LXF4&oB^wdff+$(s=akK8 z9GQVY#|&Yeu0~wUdV(mUlKLo#z$o^VKSFl8(F*gF!QAOeoWP)-cKGf;s5Ka_Cd^v8 zP#X5NLQ+a1C(==}QjX?A-KomxKzQ5V%D5kyFOF!-$H9=Xami(J**Ka{e5}Izp|EER zJQ!1xn^Jc@RAq@<4^(wP^_WJ;BUtKkS7@A>rSDU+_oCZ;4#@;8OS21bxHJc`f2Buf zeuWh#8sn&m`SRD5AC4iN@dwkw(yJCWpbJCw=0bmLb za02E1Aw_OKd6o)i={Y(rRY$XJn0JpZicjhQc*(ZQ`VIW`^#x{Yg1V&NzemWPPs-*%=;j@f-U>WA_knS6S>+8*iPF$Jz&)NfW z;ZfAxetL#uQn-0KR0%&`{1Tq8!RT9QT@yzAUSn^r+PvTfTOI9R=*jKgbZH)siXTXM z2O6a)maI;$dfU|Gq06eHABN|75Qtz?$?6T=TQQf6`f_;ZP2A2|LIuWeiB4v z-+a@B1t}XWtFM}52I!&GyQ|^5#1$aAvp&dlN;-MZcv7eWrAi`0%6ck%g{Uee1AwJs^6skGh%M|9H(YL`?|giN0dwO&>s zk9^9s15~eP1R>?%R9WJ z(TS$MJ@qHc+QSy|k_&GvNk$1!RQ{-Zty>hAzk`X=cW>!Ca^nY^gEdzotxKK4^)dlKL%&WkZHV-p?e*~_zummZU9fa zt}2tIet)=y5|^16li)~WX`WB|-sicZTO)SpVbWZ_z;sKeW1z-!_;FaQ>{$8lRrKB` zWP9o}n!{HyC}Sf_KnCH#a`h&#HsJ$I#5k2QjJS%e8P310J}U`vXPZ?bwckZPaE|Lq z`e8E$Tu!RblO9g@ysjl#aF4ZKVi>wuWC_ZE&G@{oMf`PvDqIVX`J$z75lehx3!8o4 zZ>7qsJ~J#!8=5+cvuvzvTnY=_|ASRrzk@EOePkWu{j5TK?IK;(?CU5$&ga2@q`}`e zRp~m}6MrfJtd_A&PU}l~r}D4HeJyV-tJ@))3r+(YC#(Dj`D8j^w@rE(8iv3C-du5tuwB zr7!WZYbwTzL9lI*4qHJezGkN~q&jUT5z?XrsY%BMt^q1r$k^TZ62GbZ+T=9a8tJQx zdPK(vdcCDRIs|8;%)Mn6>C}_qh>NSQu_V1nTHj2wK}uY<11i`>_6@|elquM_?vse> zb2r$^N|i8*FT~JVeiD+)DDvTCjvJh|KN*<~cbCcI2}zMA83`|zwlh8lkj?tc3`(`v zfN`I{4?hf#usj@~uRY2+9Am`hl}U8Sa-mGPOnqWEP=nb{4`ZowddZ7HvrsOl!^SNr zp(e^yHWrB`<9$;mK;$u%vlpdZ;+8ksf_}G5Cfd06bu|2^s&o~6ap(lAC9xFu<}<)*+X)K98L@}yZyAvP*#Ji?c>QbwmitDB6G zl&YT*6ZoJzLi%&EP6EqWE59AG0+@DpTl|dh1h7p;GzS~%$}Ooww_&b~$Td zvCQDup8OI2>R>7B{Ej3p854rAtGQv221}9Bauw9Gs2T4}u45Wsqqy?X14!TJ3Q2pg ziLd>nJS%b2v**f$ZFZeH5z~=7s?-+HEWBDWbT7BZYn%G|hHJZX&%=zp?!vTmkxF53 za~K^OvqWQ8ZMsmEN+s2mL^o)+$@^fHcjln9&Zv~0MpQ*10FWV&3`<;smh&m7?QcmE zLm0dgh}XU#{S%a{75^fJkg6Nnt<;iQNmvxylQA7rNS@ZaQRs|&s*`hQ!*s)Wa z@lH~MA}C^_xZwoEo{*J!M3yQhqjVR>tY1Hh(Z3_Jdhf+m0_ zu)PNvH9*FcWFpMWI_5tsBtY-1??q#2aQoxPnMmn9GkF=g-L_%XDW)VWH4`|Zq|tisQ8Iy^ zIkILEoJL2iV^VP;jVc3G)?a=di6ZNwQY`s=4;Jq&okB+wtFDAc!G2L6^Uw#!pwS=m zSZsAYfp8+Zk*wI$g-CHJAmuvnaF}kkc+QJXx`0mJADN53ifNI@Qqs08x%;oSBvubV0K^5pNx%1Hok;=B1E!tuM1xzUoChNc< z;BbCtD-})S!>7t3S7(M*PI*zxTA+Li<*WapVV}`$G!GmF?~yrGlcmN)g6(p5N|)=I z0hyf$B^VVuKnFMK6WS}706fodhDSC2bVraW+EZ>qZ+i89guuArUAT;a zt;DQLWKE+u=XkIOYI_z%u6FcG{5PL>5i#D z^-eRqt=WWJf zodD5FM(Lqaf6s*A8xguN6?}5NS==No`ly8dm3fmdstPJ#?e>VFJwPzP2z&=`Z&HVr^|M{zC0XD-q*VvPs zBh_0VoPG2Kd;^ue6uI&~CGY+L=N}RP)1sqretnfaPIg{09q6JEx2^y3;buD3X63(T zD%zoFgj8E(6%n>$@M}NCl-?b{JUMZKh~pbHjFata(rAA<+H%4O#$IXDmuP{ikF9B@ zTncqp)`*A`7aV(O(TI$AS4q1|mR~0dW&%=^>a*IwO0+;~f)wcCB){p4nC>d~Ibx=^ z2KtrtK$fUlkIL>Yo&Pkmp;C5Z8GZ%me@x}~$IxX*bCB5s;Q^*&--_cY>y#%Ezry9> z{9Rq8;36C3#w8BfC`?6s9nCR=?oIqRV-T>!oeTy=UZyF>XqwUxv~zAjke*Qa+vW22 z|6498OK<8C>lOXYFjk4#eiTDw$>1w!iPV#6UfI*u3QHs%OeCSYT_+plTT*8h>hQv0 zq@-tYIUCAoX8f3L7-M4nRDjfxPB$@C4#4uPGV-0v|f1mm0$(b#Ybx5*$5GEgd$qKo!|- zH9j~X^*m{19B<$_9+<6z+?a)rG_3{X`|LKr%|_xT{!5`^U(}3zu@?FJR~H1gCtFv1 zHk9BGH8U$^_B!P0fR6f}(`5?`h3&%e=G0?}A#J3CHu*4Mk}j3y0;ZBd^!g0b1^G!F z+n5Z6zE$CAKsaZdQ7^Cy1GzhsjY8ZDQSe*r>qERl9Ev&p3@VUFCQdmUGa zfo{?w6UhhgjAGAa&6&N&`CL`m_2pC|C)!kHG=Y6fc1pFU*edM@`suERF0Ob3n~?Gh zzmZl^04PqHD7$_)gB)RKhj@Zkf?V7H`zN0S2xK3I>9{F~lo(iRoH|$P-&JPWKu-*hc5U!rzZDFnA!X& zO?TCE_LSLGp>3T8{yM0s;6@GDi7jpiVO^jy(p z*d(ULOO$TtmsrLbI(2$|U*#VxnFA5^k1C4b9C{X7G;E`nNi?oYX%sNX7bB19eq9N_BNvIvk%q%YM66K8S!sc#^#S zjp^nO81A~t?b3Q+nWWdm)ZjXhCYlJB*VI?Zvz2YIEL_8Z7b;Jh%pn&RhAE6=p`MMT zhUMt6BEy1F>(SiVS=q)T74)n60-t-?hgJ|#E2v3)F?F71E|V~A&<+N07of*WT1vZS zndOK|Xby2<-X0==;>+@`A48-KZylU-^O0 z5}3XKca;6FW^#kv+K$Ptt^#X~9r0yS(Z}){+tWR$mDHJfH!X2lONYo2-HD^hHBNHS zHXwI##i<(Kpj# z&}0rJJLwlK+X*4GFyU9n6%c@62Dp^Bz|Z*=Z8bYOCl8-_>=K zqbUDN3dGCPROSIYiXt$DWg7bvCicbW%DQ?$(I<2rl1=$(_HYdu9k0rU zNh8__vVK=v&dzI>WvB3>j83IR&w#94IdW|ezD zmoGVil)hJF&VP6WXLPixxhm5aQQnvEJ~D=A*J=}L!r_Fdz>P6_p3JUSEZu=EC0=g! z!Ec?bau(pm*~Sr<{kW%9%Ak3i`gN;b?!AzfqEOz4anh zcv42{a?C|k9~ytBdt}?*OhUlgh3>8#W7%d|Umap@!W|mHj*dFw)JG*m%5HBax%H8z zPbyOZE;sjDex0W=NPgUbH6}T4fh>O=l(GI1gla(R6KIJTPm9ves8-mcVk_`Ec={vA`g3ov+-hf#2n4h%_SXOIxnv=r~?2t3L zEc1ReUtOv5gH{F2BDwjW-G+t%-#*a6@pM({gSUdw^5to^^Iv*UIz7#)H&cPS97V`& zXlTFMtVVk$mkJOx|t9oe)B7;$S+$D!b-bzmMC+T zlCe5vvC{J`D~u0O=ra7V?vpAA#ME zXuc|V7_wG&#u{>kbW$(By;l4b`0ge`=Y5QzLPB>PKkOlSk3$YOxA?N~GN5m#@^#`H zJ^M+WoKut!T!JO{se~uw=m#ji*!67VKZVX{q1-9#IKP%m*+H=GzypNm2}08{s|{-L zZIyExQ7D`vz1~B)&sHVbXmKA6-E}55e%my5Pg+7lUH!TH2G2XIVg^dzP+Ea3d5r~K zSA`wrMxV@7B%mZhHuL}C4%APT`H#3lkQxVR#v;f8X%wG`O+R?Y;DkVbzgO>n*6P=TrBujQl}`4t{H=}7a- zGkPkS|E`(BjZJ8jOzes!_*s>9loi`>B2J*vUp$R&`$kNsv`;tv`jOpa$djg%8o^LK zY&50f{mNK~sDh?a@hJZ~tJCD@Q8@@Ep&?wXwvoRm?>m@hXFURqoqY0a1JfGaYaQq>3MY&2FiI z*L;ouz%<|fz5v0$Xx;g`$mL40Uxhu0!vG1FR7&H6Vgm+qyc&e0^0@{>(l9yUmFZLX|k!)NXm}W zQL021wY6V0n&jn!S-exE<#wKvt9+L*!>;j~E5^u^lks?n6*oboT>UUx-Y6peT={&b zYy;)|0o-F^S!-ohn%yUV_h5^6;Q+8nephZ#939B{arXWH;i(vXw@lAJns}Wy^X2$J z7RyA)T;tjFa`bP>IO)F2 z%rO2cra_+{ZmUh{v4mU=a-!tz!)Yntd{T|_65+`0AYl}{ts!8UV-J6KVrs8_V!7nq z25{4Sz&6sKyOLF8r^uE___Tw{SOXa-9&@DdT`p_7%W{vs5_6NgO*$k%^y9kc%$I!; zZA*CoxUuCFtR9gbPU!hpUMjHF9VOe6VE-1nJ7$uKIWMvMff`rx=tbTgLtn?W&2n&s4oE9OXvaQn?*t?>+=sKm>KS^3$t3FG{U;P!SM`A*wr4+ z2se~B)}@uo&$q2yS=s<~RMh}c)lfz)@Vsrg^mdr~S}&)*1V`J80MN7)oLBdhQg zq~ytQK{$ju+4j1YJlFmN{T~m6~_7(-PMcI0z zbi8fpvVn;jr<`8qm{=hRUr6)GV>Ge?OnBN#d~@8DL2Y2UmAY?og>L8DTDMs1(H1pY zL%A-AZ5rzTf~w4u!&FpKdgV{Z^||sVN}HBorM^D`7xqqN8@e)AM>ghi zc5WL=p!{XxFX5&9A*msXAsj-Hhid7Yj~MAbH_edLG1%N6_j8>UdmcRXbp)sVW^zcf zb>b7pJ_cDU*(D!L<+X2~WVOqSfk{nN>I&KYkv#_JLp=(EAqL8xpDYgvaHZo=$OZL9 z9>l|d!{l-!zS(IQ28xW=0os^sFlxnzh&vcF2_X0<%H!t|m+dNHtL%9N3Hm&}j6gV0 zvCGc1Ah%MENSU$$gB+rDCNHK-LLZeflek8>0Xl}RpwFw47_l!#iXxS(yY$-6K7P4@ zXWCE&OE9;=UX*8LAR>g>JCO}${8sTRfYLu1N&$>EWMrli`}6LTTXaE{N2{Z>A5-)e zYeBqUhg4^ym$Q@Qsoti8<+$p~vre-TvR;}?tuiw8sR~E29A?N=?{~?^9|{wpH)wtz3eVCHv)3U7a^M830sMOT{hSdQ>v`Mjn)qXq`}WTZdRyd}75=Qvyyb z585S!>b%Bk8;YW4!+d{%GH+J95b-NY0ZWMz{yztU5b(4&m(B zu@f3%Rg4llp&>>Iiclj)h#j?JRg@6rOIk&0rX>}s6A8EJ&v<{{>+_y{ z6iMAeL28wpn=hRaAOFlL871RzQLsK2_aVpS8Q=yAT3d*vbpaqz?^BTZ8GOTEq zq<+*;PRhooU8;UpZY%Vw?8yjry-ZtIbwu*=~}n5#;X4P~wwi9n|^*Aj$Z_5gWs2mhm8~i^GDnfzj>s>7E#1uhuW2|j68p^*8NU`d!EW4 z9HfFXn(76O#qtB9{l3aLEJ{~#u$dzX63y>}Dw^M1LU-`}BRwKoyBU%%d%D1zYpNX7 z@4f*ym#I*x?o=X{Ui-nVLRwj2jEr1GY(hei>XP9lWj7)(F10p0Om8yG?WX}h%FR^O zGx{B;i2z@wvpcd=uXLeK!dqTK|Fx#c0?(6aKd zDi}m5+HOEG;cJW(LF1)MfsrQen!tUh>PB>)m@|~y{Cc!Pv%$#lAqDH&VCh(rV~f`_ zI^)X7_Tf|1S-3!EMbvNWE@fFCNNd^XAelf7MR;OZU1sb*q7UIM#Ruekdm|6MAWByyr)bYkzZTT#f+{MT%nO8$s=@M7FX4^|orfDF zwwlbO3ll>*1EuMOOKv&!jO*a=5R{6s@!W`73$XZ;H9D7tI3|H5RR31JKYu?)orbmv^Fs>+}cCBGyvTReO z%y%#2d%~Scp}kn+P`M*VD)T^1U>}<;mjAgTVHcsgjM)HceLKSCW)b#Na&Vff0N$9k zocoZi4A(ZzS49=ILJ&tLN|wn;mLT1Tz`>lAh}5Qe=Jl~ItFoh1!LCRGc{e@cOR4dX zk%|mT8z;4jxT5?}%}_z}6a^qea*jNXH;7eTif%JlPl=E^XV7Oiq6VYN<|RtYw-_xt z_2&ZXiCrGNjIYTNWY3fom6?57d98Qq@z=Ox$Mu0vyQcuN%t>Vz&5 zrhKJWC!rqT1W|1vAh#3CKmmB)9MufPDfMfyna|rB@v>7jFr&>|6@Zg6SOSkk=gOQf z%#mmi$yO=33e}w`wI)(r0s~5)C^%?0uWoVKDl@RK2)4@gl^jPx>14+3?olNLP1nB3WPPrp40h33yLH zBE3}iLO^l4y&Sh<2NmkRhc}?MpTTfMnI~IP0XMG{GGr|TDgSIfCZqmlop1Hnd$L^R zBZSq(IC1_eW6(UUv_#C~yu0d-CNiU`Im>jly9d2cmVM1pJ=U`WWlbs@ueJkT7v6zC z6+3c3(OuA_clT3m7t4~IsC{KgkUbP9#Y9LP%5;@F5JX)$?1Uy&tAwfy_f#k06WyS` zYr1p3jDL@gI4A*Du~-qp=X&xNsG;PSzRqHl-*sn@RQnRN*{Z5?lveRK^i>=vwdTTl z+H&;3DbgE5aF&zc0rxhTwzD*vzI{5~*_d~XDoO9V#W2ZCtDlp?!neSs8`}|tpQw=I zafNYMNP+@Es<>|Au^mWk(%o_B1R6 z%7GUUbaoAp^5kk)9Cv5CN{h>{D=Fs^BVdv?h%qM@tud{Flvb6MP(SALSeBa~owa0b z2O~&=cAM?W%-#g8N2@9SG>#5JsE3XgUn>BQ(A|%uhS{)TymI;w=8{*2RED(0+Nq>` zu*&+StbXJo4(uV)4~|qDqe)l~`F19XJRCCwSpYwGD`(A>eQ@t=e}oiU948f5xiaPT zCRZ|dIV3#E9NzrNj+er@aFyYO33xv09DH2wM{No8)ldft%Kebw`%3#U#=)3i9ENNf2RLU)k2hoT3~W0V0*4 zwM^YHTwO2;e)4n$akNFAsKA;DC66GltIA;3PtUrJ-c7RaTs`ifHUaZrmL4%t+-qIA zu%--UJ;>UR#*^l)1zWOE`6DW6b7M4Vat{I|89AA*ELq@yEmr=?g%`e8W!^Jx@Z((* zGNiVWQ+0K1k2KT3#kgcM9*e=?8P4=p*eFb`xl=kl!83T@2VL7N?fqDJnmT(wRqJwa zzi2y5JK4h4iTK$27AC)-Z*&B#%+Vh0IAOBHUuSb;^@N!)m(*b73wtU);03X9c|URCB-2pDKCQ!$1IyZLq!if zieeaj20WjjhYnQqutN4#0Anucx%0Ky7`tqZ#7Wk#zpH@w6+!@AdT6TnEQShNr6BH` z&*BCjyBo{Z(jjb{f=_jThpCUU{Dg3&4Qqn&H*yL{>Bz=qnBgyU~v4i=%tmmzpxE=)(An^J^)|MwuWce)-?58exJj9UxGylh#Of*eaxWbj^GbIPfmauKOPg4Ztl|`w*pfa@;v5G)4}8Kt zv8P1I;DelH7u}s8wdNqnW@kp5P?{)(KSiZfqG6TglAE6=cW*{p#a)}_j90y{ZWive zVm{)gLd)mwY?SVG_Hz*Z6qO7;%zRC-$wjEpI9p>WUjqg$J&$*0$l4Kz7Pk<21PrGrzhWl0~1Z6RA4 zrliTtpQy&8_AB3@9;9Pwj&9nsl~qsw5X`dSf)eg4)P~ zgvxL{`22M}ew0#pHb_<{AUR1jbLq7c7%bPr;`COYqWzVWE0a39++d_;mt>VtKC3lH zI91hSNt^;4F;02Qy*fyte!HNUwtC!Bq(+#be6h79>$N#diBAhl46`D}BFjQ`MV*i- zrYT^Z+bZoZxFVIQ^Lywnwi7pzg0El}?(HN9B+tU`Ol8#&J)tF!aQ7Ue$1RYJInk=} z2<3$7A>$3ZEviP?UDP00V|FFfI}#`Yl$U&r9#n?QFx6`sKjP;6UXS}8WvA#d#?e@n zpB^z3e&ARqm&+hs7S03Dt`TblN2E|$aVmQHF%|B&MP-P;qm6;K(PDxZ;@hId{c?$} z<(J^8h2%KdM%rFjFaZe|zNu!?+$aD^qsk-pg9=z))L{5%_7xUIxO$?Swq)~8L&NkJt~IH83UTD7hgfKeUhLOJJjRv#9H4pY<|oXBghwNFn#nWU~-d3AoI zs1ZJxiYWX}**Gif32V3k=i8yg&{Bwb0w{0BgLSGXVyutcCy!rEvqTUW}B<9L!)CDt@6sz(Dov_B>Sdv)kC?Z38JwScIj3uXt~ zaio6nIaoc8+Rl2`2sDAP+LVqx_wK;{v zK~*+F`>9>Ahj2R=Kqd@=ps~@x!%>~ zl%-2xD3K}zv{qR@O(2R`HcoWIge|j*@Vh99{99#c~WiYdY7+$fN2S z;e-K2#VdmiEwa!<+s2?R8J0+_nwf!|d*%iG4O(vh#@xZENYIYy!u*it6nzc)(a4A0 z*qUIJ%iRDMcT;;!cb}!RSF{GR&MS>xOo!3bcP#b0P#z{1s2q9+9>OzvXd4b7w39rT z0}y}sB{81iBdVfQm;t9lW8{^6CJvZ%Jay5b_Vyp8E;;3~Dm^L63WixQ zmV-YxgxBON%u>dVBrHhp5Z4|>~GH0k!0^}v_ z#91pvH|{jtnV{ISA2e?*CDy*2(MEWdiV{15Y8qAxLu6DP4Cj%<0bW`)o+H4o>k}gl zDW9Jp5#dH87w5Pml%$I`Nrz$Z8vH@SpBBgIFi z#~3Nn;d8S|_RyA+BZ;hz#-#^>rp*-{>0IV0(ZrY4iQ=(wxE zYf`GB{RH)u(cT>KY)T}^^lH53MN3PrHgONoc8fX?De6$60C~yE9M&8r5HVVgW+eWi zvWqNos~@NwuG@|%*IO#E&m1IY%7b#x^wbR!c8Z&u{TN#~UKjzU1l5(NYarNhJ~ z>+W1$MaOAE^#h1HWi5tAE{Jzj8;cpO!PbMiX{rofgoN6E+sKmjRnQ}c{sg%d>S@XF zhxlP&kGlf;NZ1}}l>p9M)AN?$9FOpZTWbEYaDH@ziazcppQ)CQ-+qg#JVV9IO!8v* ziqFgENlbKn=n#iIP+1BHTTPh@X&)&kd^#9tGE>XMq8; zpw-6no|lq&B&w%4R=Hx8f>2khUr1{6RCBVd-)uNAE8*l=rAL*)=ve#&RLGvkEkw)b zyNK#quZqPf+D^<7lmHkkZO@ofafKz1l)LSs@l9ebkh_PB4C&LtMd!=gMhdu{RYtZ= zf|7pF^M1qv5;g_tl>bV(z=#o6cB7R_dB?>ZEF~&%S*TM8%c-i0J8tFtxtb6byYL-4 z#J|(Q-Wz)4IOHSO&`vK=A_TfB3yK4WYXJ8Pgdf@Z@tb*j4 zev-Erc#&4NiaSMAUo5HY{3t_o|lK!w59N4zKB)TE~Z1Wrp$4k z9yS^8Lb6lJ)nOW1=;2{Ajk#Kqr^atX5;noYDxU=NQ|IcXJc4B3OAqF#anelm+B?LX znQ$c2S8XjGl}(+`3U9R}i%Hetj8(PJC@J)f>M<{VMQM&IJ_N zqC;%r8ijUcb4_!q#3`>v_rDov#$o=sWgRf4l3T1KWq>^oAMQYDN zs$bc9lZq_L<42y*h8Yz6Ok?!1OZauecMS4(4FJm>Qot@7iTd|4gAzfyXk z|8HP=nf$NfW#;h&#jY}h`856ZDDq<>>BlIV8)(|>^xBxyU9x$G3M=;O(2b(4_9SCXuhv|ME8J<=B{Yaf#=LStBs$w;iEjDfGIP(J00g{dve>VQhLOWo_PDSA|AQAZ z?J9dI5l; zs`qqSOSGHrF}d>93BFHz%XZ8GMyL!J!l#q;X%N4@80kLR#DvoavJ-sYPPw?sl`2E# zvAPqtSbwc_M#`$h-jAzCQZBO2wPir#E&hyhMJ1gFsN~d69YC!=^)Qnx8OlE=rh%L9 zRsyY>PYs*g8OH#;8!Le-p!aYO!zAz4Fq_sG+}IJsr#@lKmhuy@r1&!;Wgzb!o5S#T zvExbmBK9}t1ZsX9i=o8^9mV(IsDu0#fszLQYgQ zDJ?0arAR1OHhDgbQ|k1Z0dlBdBKFZhAZ9c(9(ZcRN%jz+l2HX%{JV@t>iod3^AGat znilM!eKX*g+nw2tZv-^c_XIC9X`S{PR;f@5JX`WJKQFbi-$f@ltnsgiv8ek6XYyq& zby$sIn7@r?&YnY9f}<%1Q~n;~+mG|>=&5|#@j0mV&ja4|+sfun6tEkwHavGn^@Kvc z7ORd?5#s2BQ6gKVn9ZC8=voirhkCwjxXVsnvs43JvK1FE-=B@#N4J)_W!Ts9Uzq&n zbu_AqIbm{JiP*{>fW)BuowT#C{ZkWpw$d1OKI^m*D}_t>RbP!z%2!(d+_q5E4_`7x zaV-|5jbIzk!x(d_%CBy65We}7iC>S{`0n)#9{DxMK&kwab89;nrnE#=IBKPeJq}hS zB*IjIrYkx5+U9ij{+HoY%@RW;J<*7l4Viq}OFdFaSHSy99T=F&gaku-GfZ%zxz1QCx5mEx%j+FYC?>6MK(cNw~(0?T> z89&>AvCQY!+mybOA9}$scI3F_AcfFneSL83t3UYhcmtMhYsIH`HWqdLQ~qgs3w|!r z?NsN&f-Z+*IE!*4IF%1gpf{&F{+R=T!lOMvX4p06zEG9#N{s|p`s_E-L?Of?kS(GW6&NX&q&|Fe6l%# zQjIAbb<2B3sCFtRTTbzUcDm3c2}9uGRq(~idSQ$^?|Jmg?yNIv6z6t-6JYFfkcC}W zQOlQ4L4Ezy!~bq|(IGd-uu0>cAVb1Ft{PLqv;V8wCDY#X)hboZvegu-}whJ$#|8ZrMJ3%f4=hPxY< zu2SONnst6>xG0>1Lyh!eU$x+(waYW6w@jXE@HaWEloPTKN zBrZPVg$DkK@iLs04T+l%;?bg>mN5AqneZabcxQX$-?jb?uT z@r>4g7vF!k0>b(6Cj0!BA{>(SCp%gHC*N&t&mi91?WO%weqE(R?O_8Xo7sauB~(jv z?;0SUmx>~qc>{Fup2EZTH@YaSI*a|ZUCOV=zUSdy6e<=y06g?p!3P(zK%s3%f#9xk zz_KjufaEGITxCNjQ`%L3rnkfh$(y!jH-;4aGAH*j-qzL}pI1Y}BtA|o(PkQZOPdw! zum1wR>wAr%-nTL0rNs$O?%WG*=|o2R%Af@xC+0Jg_A)14;#CAo$8cusw1CH|OyyOt z+dyRed%$Iy#$I+`Fz5|^3iw|Bfqz<}SXAQ${L>?W1zf=aFTN|8?7MC}x9k;rIo=Tf z1)6#8;lB*;2*>PjrjSO9{aM`*C47p8@_zha*bAi>%E&ZTn0=0?;g%59o9Z7G$~ihb zLbk>7{fq(3ux=FR*j=%w8o9{pkHe$Hdm~f*<22h$t#TyztyDS;xO;B}*D8X^^6U+V z*0ptXs_eSS!u;1kY#+tsN}ESK_RA(COj@soZH!j68V9d8m?D59jEaO3-8oiqakH1% z{!B4)i^m$17O@O?EIz{NO<%5n{~BI6urZ$&kAj}I>;e(@>zTgNWeOt)9mLFF!RB*M zWla3!33A_?1-@0?!3HkRH0<(RMGDU5_A6&*G3{K{nP-=mL3;EGsF0*dZ&`l@25L6MSyAkgFhTzvR#BLrudr&6X-UEGErhBye`DI>MB81KvvtbSz`ez@~1*i@#U5hVScFdpwhlZ@&N zA_jM3Jva9jQWz2Rk>QX%JNdQwZDisrPhMN- z0%@8YXDJ`Y1EowqzM~nDP5Uy0JKt2sY~;rW5n;ffEz>y&RGus;s)n@ur{ z@g2abFPoWDaNOI>_@O(Rv@&hYW^F-XmMql3HX?^SF^?se7x!eI9oC718s*D;C+wiB zaTb`)UtyS}{vE?6RoZ|(&g1<0w~bG)7jSsRiYp#-vDS+zJm*l??2(E$2j7RJ%6`eO z-vj{SVGBX2GQXkm=Onphj|l>s{v}WR@yN*5($mSlRB_qd`I8Aja_e$nI*-PvL~rVi z1It%Jh8sJ2q9ypvTEiykA@sof9i}1w5p9 zLU+$MLdCO@D?>ci8IkhCE1oU?fEnX9u)8bDI`aBFENep*v&T@G@dRu4W70dN-XwnZ zyTjhj<1Ur)RT=udDot@q<)>PGSV2@>AYWz~AgnOh18dZkM1IDN;wcqRf&<^FAdBZp z9JDPkvbfQ3TlZ$VX6lNTFc%52-@HjM3zEuA>nEG^egR%_ZL2H?pYT?flHhu}gY|}P z1I4SfWd+eA`QzUQHrZc!4PgekQmqZByRHPlZq?dw%jSHBzj%W;emU<7!g`7m__Qio zzoj#avW6-I2cM2C(MnmtvV+dW)lB#tl*wEQ%-E4CU3-1 zlwQxmX#-ML%4M?sT&rO+<-hds@KNcIelW_Y3bOPPBotMiCG^1hhzgVEi=m&ow-4s1 zwY{`NX;E<7II5z{_TzBDHJuu;8|50xUd&3@S2cp<@;kP*yPGRlcFtxgUmRn(bygV( za&4I6BpcYwj00@?`YG@tITztHbQ?=BtDbHf_Vek!x@Zq%Jnc$If*eQzb+dc$efti` zmX!(-n<~YyZ7kRJ#g-_^Uu-y}rw8NS#wa1EA!ZQQwzh@yAL7zkl@?kNT&5O+@**2NYyvSo)4U%Bi&Hm&uN?Thbdom|8 zHwFq=O(_$3tI)zRk<>* z{Bw--eh7nfQHjv;JJ|I!lq!7kWX6>v_grRu1)H7H#>gaw$`l~Aa=?PK`#8^8T>;k2 z?*EZZN)`FEDY)>rrHEiyf-?UGO?)b*mDQetw<9MS8S>Wz$Wc$S$>g~Jbjxanx!0Ia zsULA5WO@aOO(LvHmcC@;njfP zp#=n*0sEDPF8&;-3}yzd=2P#-tg^!e_7?eU{z9Yea6)PsDbcW2Id;&L9G_O)S>*-II!HwZxJ0x^D1RG|f*E)XhyfP4bH zxJWGFPJ%No)K5m0XVWpeJ!Y$bFM735nyVDy5?PgH_XprSZ>15W9Zn~lp)90XZ7b`T zut6Qdr)>IL%H~Lc17YaXl<#T`085kJ8`&~uJ-@zx%y&0pl?Yaiy#I_;bf!VavAPy0 zH!C21x9ZMHT0mjGSeF{P@?azA;Zxa&leG;%(wVB6&7f;WmVEJnt+h#D?vsD=$dHm8 z%$}3zQy)ls5U=(SOq?%(Y&QYq`w`(Nj%XteKTvFS{5?^D(nQ@C(^J^b?Of=-c9ZBJ zbnxP;M0phX$<6kBw`(TvhUq4=l>fm1qb~!#wS@4>z+|9V>mgV%{VYqqtcM!Xi<+8p zYNjh*IeRP0h;o2nK`86@WTnSVFS~1yGER(u2-o?tG(87u^ z7*CatG~i!;J!avxyMOb~e?K!!#Hdi(V#RyVYKr=`Ye}d}RC!XNVt{z6*ok5@5ART2 zS&m`i(WYCR(%=e6w#}b2J(a*=Z_CM&6CpfMdKA8|nW_=5p>6^73}{%Rr367IbdzWm zV^%3=$rg#vHtg~s5Kd#L3X{Yy#)A77McZ`8G(9MnPaf@ z3>Vw!TZXk>Fkq1VZt?3cDzYLi0ggzHLV&E!0tf2;&xn%hYmo&t^9(y7uDO0HiNG!+ ze`Zfc17^ID1R8Hz$KF>S;^6MblvFw#f zbT(D9(JJMYE#vrhuo>?x^;xgF@}%v0SCPz{4Cmk1mPf9>W+Ul07>+V4)o{2c_{e}| z!0rjPL&Y@@rPa{GD&m;UFk_W~;wC+0PkV2&3i4@e@EUOV^hN;CY^)KFGm|D;f8``A z*5D_%S}MSFF;cV{h57Q~Phc?NtPzV5HO*J9QNB*}TkN`aO*Xb>Ht;LA$w(k1Kdv^h zRm{Jba4C6_%_D~{@kl@y&bZVTJ{>Fzd+)g%y?|mOmBg%U6f>z>CQpJnr#-3`7ln+w;~SJuQ%+`*^cddB}OJa71UxDcHFj zI>R!fqd^wC5*L3xL0LMf2YJ#8B~<2l8bNY4-KELrKCq)sDj=t}J<%rXec0jC5@^J2 zj~QvQzba#H*9@CBH-vU)vp|Y*&zS$FA~c(d467Xg1+}q3&+Zp2hgQJVix~2UaI5Qp`cG!FnM>o}CQ#A~+4{TtZjJ-@1BYg7d&%TC+gh_*! z2C?+D+B~+dSR+;uYfE_RlWE=R^7-)gk%!Yk`^GH+8Ci&Z6X}_f@jw(T|Zn6H<8i7HK5tCK&ws0sD|Ud^PPK z{&b+vv8?Z_xMUq82wN0N*_-bf^Y$ZyKjT5IfxiQlHEnr~O6ZvcC4>dZaEwpH1UQ1_ z_b_yW^4EaI(tV6_F2-z?&Qlnm-Dk+K;(WkO&nYjN+Z9ZC9Fig*3e^cdWXy=;Al|Lg zEP(#FHl&T!U!pp|Fb)^miM}F?b~Pd4qquLWoa7LhJ`)<-?+v2NTx4kSvL`sw(u+?^ zsvD`29%wk_?ov26-I?pjvd=m8C$s>P3P}*jolzcTss+mQ{wvxkdT8uF7#x&4ajtLYaLskQ*X9}p_~UmYrAU5oIA9#wbNb08J{Vm*8-6 z>Oirt(m+g86{fzI#=`pMgc4hX!Q3oH2I_oJJ!z4{^lkLex)O7oPlN-P(aPtf$OJnx zJjpd2Dq_taeJIUS&L@F_HB|eQUV4NtOf9uBYf7#!wR*U0RBbJg4x>OCVwi2j8`?-p zqm3f&hqQXq;W+4j_6IiY(}q=3I5j}tR_CAF*SR#Uk)=R$T2jUHXCQSY$Q7!cw-m_x zBOv6fJq)p0&#om4CK{OGb7G0|qSw`(^U%o;P1sbDp6w@p_(Cy-W!dfPqeiCw;!~5> zYB4AH=2a=!}TpSvqy^xRS3*Wo|Ej+FwY6hqdX_@)_P>)0; z8ZH{~@=H2G@n%SY#BXFh-d}@)L-edFB8%WY%l-ib#fhNmqii;CzXp?e&_6{2_i*T= zpR$TJ>N2}C9geb6w|TPKpi=T=I6~}oPKL~n;IT9nJre$k_d8wW&<^(iVw3dT5P3Yv zOm6de2EiaoFikz6n^GGXSr-^4DS#spw(qWp=1E@H(+`YdTgWlBR<7s({ATich!H0Z zZm_+ZsuRjN<#_cy%_yan%j|DuDD+m>lP-^#eDgEDdzJ}xP)ehm4E&aLuTZA&<*Eqe zzl|95=w-lFvL<^zbQvPATaJGYP2;(@ZdaOIY{YZnv^gd=EK?_95u~1>Zm(ko^W5Zo zes#VxxX|EnCVn%|CTmrkg{3->U&QHZd(#~fR2Ke_lg^l?lYCD%EI+Aq3R#Bg%CCLwqcDIW?c1oK)_dH}65wiz-*|m#^4Pu*FJ(b*vZi6yjQZ z#K@4ZLtK>NRdv9qA8Ia{u)TnN=7*NF6TRP z%PsQZGs7mO`+#lT3uC#R7FESMoarZLdU1h`0moJTWDQAr!K?R8>|@P)SW;LGXf7r^ zg3DS#Gnw`gi+8zUi!oy4N#_Fi2Ss^Wz+n*lW6N+Feg1*DQem`?jQX1Al0WvtSvYPp z;Evo44PRAu`8G-LF4W4#VzmK%S~Gh~@orf5;@ix-wlt*Xsfz2=)C=s=`36EgQ_mm4yTjsoFLxDDWh{@i4iqbVHO5J}mc3g|~7EVEmy>r4(Zgks+Tk<>5*kRHrI@ zIvr_5$R`bW?yD)lW>KL{u_!Y!^je6d4^JxhC|%Xj67JU-GYVRZkKP~1Na8GLg)Yyi5tks2KTfE%0G+h z)r}pFSFQUhCK`~zCmihz9k4y^3r;;=#Wpuj;8VlSM!eiC+u+c=~Fp%sXu%s!H8h9=VBy0=XaMke|Ic>(YOi6O>L|Mb55;PbU74gUbq0q2#$R z>IOUuqi!yJUV-xc8o+-3yvnpa+Za)LU4P-OLLuyAOTKuHgK4h118Ufkg2b5wj#b!< ze%o7*r3+Mgl+;?pX3m!csT!XHKDYnJpKDP7?u#mbcBjg<`-cWLxYEIp$hs_}6~*Xe zbr+W=ew{h5+AWMIab19XmldK(5$Nj!P1-8TUw0Te@iUJ43VAa`DI-veCIETzHt9C^ z7I6CPGn?dt@?!ai82@FE$kZJmLX8f5f3rV~fO61w$^XQajCCs1TLy12u%#CP@RXm~ zd8-BRX1a$5l0iUK*uV7L2J-F_WdC$3cW*LCn#)aUQ_8^F@T{;91NTb^&fc4V&qq&b zE2d#Ur~76=y;)TlNYvAsNwqORa|u;$rINb3#$>a)rwY%B-y*+kH8M@BfJ(JOWtJq~ zX8|;3DkWnNo6}H4QUYXbIy2@Xk0q--dvp#1qNQK6?$4{i2=TR76Zak${`yvIvSBOO zNmwQN*=0n{kGQ!2k6&!_>j3Wi@>Pb zPwj%3zDQV?y$!}?dIw$8xLa&iRXARjfWBJHfziHDC8(V$;Dwr>fzoRsOr_Ks4yl6f zZYFo{vVvCz_`LW!ipgKQUCA;rj$Itov)ZbJoQl#}S!I9x7?vBY%A3;$!%22{0)5

      %=`Id{#>5UfycEn+^P^a(GaI$&FnHvuKx&<$Bjd|Ab%)C z1}@;A-wlC%q)=p0zU*v7A&^W#GJ@*fI8ijk9iqq?%Qo^AtF}wm=e+UBASmo{QmB0X ziq&4oAzvcfz$B-9SSNqv(5KBt_Br(YR&pT|*7yn+1!{(~48&|IB|YXtOZj?GD`kak zr`53pae*oD=FV#hk*;lccAdIU-+hf7l2iad3}`F{E|XpCU!j2A+zf(bDN&W(31RXn zr;l!$?JxKL=GVI8qiuxJSer?o|FOF5G!q3uGV4h4Sq_M{3+2UY7%bs;-A3QroCrDB z-pH57V}bEbdYglT*}DAws}ZUF7?La(!r*9clHA&lg+cH=nq>MarDy+D5Mb;IXlt7? zv^@_&VEZZcI8iU6593^eR9ii&wiKs{w$@;)y*4z@^( zs0cl+yo^(!s@9W`&;IY2zOrgRN#})NIj<6y-FjA2h$b}{Z(CSJc{0|($f0hcDUodR zxt>rC5Eq%q^su*;!sp1Qk;zSf+-C1ra!bL%nCebGq z#cK_C`&h;9jhOAR$fWJ>XZ%-(r5e+v)|b=-hFZ6t^@TkKK_XSEazRYCB-a2lNx(A8 z!vXAeyuXXiymYWU zOx|dN0!%JYmN)lj_(p)9?=O9?gLvfql$Q4s(M2}v_IjcRa%jzy^05l!l~<=%N=n*M z^L9Nnl+K&h0HK5{a-8mL$*=h>h>(H8N=)Ot-soWsWk+RKq^$V{X7*c7o}AqZw6(DS zW_{siGH+})$y2H5>r`D;v&z1p7-nnv7C$d3c$sFPQ_um6pFLPAy)ZCz+c^DcA6%Mz z)(1Ghy3L-xXboqLS9;TPvTvmIbMSYmDwBLsnVUQ<$(Zj+Fx|Wlw$k@G>zb^`H4se+ zD6zy!nuR5$sYv98U%RqpMQwm{xjTQdLtuwaUZ)UWGnx7VcJZ4sz?);3{^#F#q7PnI zl}qeUC1;AsVIc*#n(B1gO?Ow9U8-Yp^GYte3^~97UQw}S9_5(lq>2#e(~lK4Bcupy zBZ=&Uo7I(4wS#BGgmEMpK9bv%Q@i`loG<0hL>I}`2xRa_L)m|?2RwVIBR_0w1%ufO zcFisinIEWbIC1|w%S%jgG+yrlBTNbtdFvT~klCMFB6PbUOaEq)uk?b3xXE&H-xYV2 zNL3zT-&Q17ub5bTKFPt_!kA+D=|1Sy?j*djZw>g+YZaC==NlLZrEn|2x#B8GKNWrM zcmcC$63bTF=oyVs)eB6JDf5QB1u9cjRq*k9E%~ZdZVQ!i`laq{Bab$^a%ILW0JDPB zcG>)nfoi*e@#DVGj$^K?Kx!>U=9Sz4q9@*E+p7j7YI15f0DPIjbDbuEm^4u!D=oZ0 zc4O$LAaX)`8K)cZU;-sBg1>O7Na$j z0?F=%engIH1Khd=6=iTMP=fkd4pjQQE>d;9In5;H%pPa(Y!v%S>wO;HqF8)4DL;q( z=i)b0Rt<;z(HBl$K`*W)LccQEmBQF7XTie$)N|X(OMF@~zWgPsu)SdKi5GmG0X+eksa;PLCLr+$hQ%^G}!T~xgh9_zaWc0 zR4b9cR1W{KW*ky)`WL836{~#z7t~aC8fVeh0-tt+=GqrIc{1OC4lb%IRr^#hFhP%} zKgBF=jASOaRYe49$oAzdY5|&+cqH>kPt`zuU5XJQUawhR-0u^;<6-FE zV*NrKQM-=5Zob+NF;S>j_@}*6j5L{unNOvHFguD{w6}lI8Gl(8)7BOT`7n ztK|4_d?%4@*6bt*8WRk6sT#hhkl$}PO5fdxnC=@;BGylCKgOn4_+-GS=U+G3N!hy;w)YfRl0i@Pc^-GJzA9o+lyT}e0$Wy2UGy)o;_;nx| zxCNzP>JC-BA+|hZPg3ov@Ho;ei^|A4e^gm?2JKw=Lvlwunwwr$+ycpIpqbS7+V4BWgLh+RL?$dj2X;r)A6|MhEn zic6e9A$HY-sEaDuU12^8r+-5^ZEFrWg}*^b-2+^q;_<*RL3^H-BI1zeUgn(RPz)$lLpAblykGfBam)Ms5wl|k1(vDH1z z-4!9RrxOe0jLV>ce;qKnBy9)9@jXbFp}QN1hpHjfNlyrqZ&j_Z!Iz9!F{>Pr5&uIc zLf5S!;{gOREVG&_TvHdun`?z{WRw%X(p)R*Q1%^@UrK4DS$t2eR19f`-QZDqtzdx! z*AsED8x<>JC@7U%m@T(d=eibGU@XHaLM?knGgpxpFPwbB# z!PZ_}LFV1!yRs{QSnyiTTJ(Z8GN+Jp{Z=oiDqB7RfHPDP(kh>UApPFJ!7S53=SJzo z;8%8qK^Q?MxgrK(IDY8X-G2uZ1hzF(QTsOTWVuac!f>NN1&aAgy}F!#tDxvmZE~SQ z)r0hx&MF%Dk7=-!jX}tvxw_3yDe~U9e5s^SM^CLItICB?%^c>f%>&qhQ^BpPIhs@m zL=>HmXVO~}*)^ry`%6q8G@+3pc~asMc+*xz!8a)dFz4tIp7LWwK!RJwgqBF={2e_) zRb)%{Ld{8a$o8Qe@jKNqD0>GQ-pXj+-O&gv&GqH_Vb^V~klYEqWaV&CY8L((*xt!W9xB7paBS};$v%88Ujycbv{^8C4m*oMdqYU|3>UWQY$mH#)WDeztv z;*jt5LzFa8s3Lc+Fkh`5(ebF+c|)XGK93#W3I45Zi+=Dz&uTB*mFC)@-+u>0;_&t} zMjWPzqDVMGmM`jaqDb-sJ zwJcGdiDOFc%~K8Js5z%fuegI{(-zP{Q;OUhJ%O?S?&w(R*XGydHf`-H0!hnX^IY`I zQg!zV^^}3K@+tVRp%kOX(>73kHyIAfITSUJ1JN9Qi=&J;T+b+@9kvt`k59E9^b3`| zAMY(ozXG(a55QeaK1RIuu8_WLF&f=ij5Hh;3E`5he5TMCF z{K}O=be}6ll?^Sc@{?1M@)SMW5ME3%=z?IRmllMLA=NE?3Rsa-w}oqaExB@S9K7Y7 zn>D`*;0Tt%7Nu7XNc%87L7J#^wxk5qK59~0KriQ%YXqk@hFq z#{#M$(wYsdU8fqax%Hqya&S3H40fjO5?I5J+fjy|ZC8jeSkAo%f6l0Q=Ml=j*>je&qt0Tf4hqSG5}Lpd&4#d_Ls zZhBg8NeN?)T;0?QRVBqAvr9rfxEj^Pw-dr8ee#*DsVe75r$Zk}Ze!Si6>;2O_`_RI zMFW;SYmIC6}!o4%Opz%o<$a?hw<3PpMzRl$_cS<|f}RX^hZZ8^9#fP&Vt`69Z| z3t#A#FpLV6$0jPjA}Uj)x|A&TMUvXP8maPfBj~-^!6}^6?JjIW@pXyOaEw6PNUZ=L zp&>;8^$`$Z5si`Yaa~cPvRmlhbp=(RNGYX}M7pg+$aS}v(5?!c63~Kim(&D>o;Eci zK2qHFrp?((DE*Z&Rv_Rd?9|*l)z04$6)G&7g{&EiqKZI@A*16)+Gly?_dk zA))eqsga4=a;zNf0l>X2ec`foI>5a9WWrW)y0lcPa0AJ_!^w4N50f0&86nLrs+Ih; zG9(OBBpRp5gw>hq5HDMR@!+&sHmrgSkLwvq%}MDByuw1D3q+x%t!7ywn}IK5$$E~@aj3P>CHEEw+ph)>WL{NwzjzML4Y(+@4@va)Fr7i z5t1{;Mmw+yP^y*=P^Ce<VzED%sddEgALR-3ajpmIkuA6?VQ!?RY-{P+01STW6RjE?#`>dsLa z)pX^%m{j%A_13J1D#zUr$H;k*tF^4D1O6}5t)FT^NvX0@iza-fBt(xx)1y;n(KH56vS}|v$UZ!Www0-eHbqfP2t{%@ zoTyh3m1NCZ2Jf!|Ybx)DUr_tMqpTASf8Kr;LM2Fl+(E_74|!Z7aSft4U|}$lcq40L)jJzHn@UVggadP6O}4dnc+KGApttUpo{n*#M{zpMxL1s z6Qb z)Hija6J=JXK3iN{**hEfOri=M;L0`2`nG_B?p97!Oi=ftq1V=;0SyseC5WyhZz?}8h&3V%9n7Noa=y)-1B4>3< zHGB@^5)M_zJ0!U^TsMY%J6IU4%+E|iw{riFGw7t-$HOnEzy1@A$7IZq=pyOY8i@R? zBKB6jg}Z+rXry6}vvrnpr{UHWbZ2WIpWH~c#j(hbdvR)W!;4VTwa+ACIf#M4qW|kA zi(N}Khy+u0nCq-!O&_bKlI@lC|LY0NUlj*0N>OAP$x)S9PEb3A4Y;kijWRnbj{2Fz zUSb1#Rf|1QhdwbzRp=$%wx>So}&aE-S}Bz_0T_Z2wOUG}*6!Q*C9Mo2p0j zhb`uQ21=3))$nLYo<@?sRxUjka~ldtDU}aj1;?%&KaBFzS@F`BkBtOGeQICw>NvI^ zh>zfiq9R}f@ij3(Qw^2kou2ZryAde`#eiT_CQM*fOqM*q>~f$`5&P_k2xQXY!E(yi zg*m#AlQezKV}Ix=6RDMyZAWgPr4;Jf?WKo5zpg0@l&e_&i`cKM-0~4}xhXi&doj{` z`~#@*ID}0D{x*DXtj|<7ZXl>jp)g(SRv2x4$r7PJhd* zqPsiD|Got>)ye0RV=Z~Mv+ihyc`80y4qk#gUen{l6up{pCpl3}iNu7O)BGq=jN)&n zimPn62n{E;08!~s)fwFA z!v$w7rFgLKr8r^DVJ5237FviWQL6)ZA<@>9t*?hmAIzh>BES znHvgxe)WO;CMh$?DLtq+bH@#p20`3a>NJ4~6yOt*SBK3Ja$QCBFB!U;R1XG=DIkT4S1 zdQe#yE@?tLtFnwc^`Wvg!BE-lM@BM&3Dfek9q0|!^~mO;F|RmgK^GL!9Eajfb?r@b zpF=b!1G#GG0cJfu7=77oLCxnr>PHH=D%=_iB@rsPq$IGepjye7QvI@L{tf#wYFHAe ztuAHqnE&x^FqRH@E#%kdXay}V88kZz1T>^QRmHrNJ4QMq=a4W&#=YGIseht}x#ZF; z&i`GGon=P!lxnS9*_;{YFhwsaC3{Irke_1!*+^9!3?EZn*cR~%?w0VC|Bk>^*=6Ob zmTZNnzZ2bE~3lM3Rm_1XHY+ z$Z$Up6{wxh!GNN=f~8whWXDb9;C9kU!Oe);D6Iu5yohGj^&Dby)N-SwdH&aTZs*RFL7vH8EQ7wa} zO|>-r=<<8r{`!X|nR)JWm$SUjxo4^5Nd|S4_m=1UY*p5G(8<ihsG-ri&u)3TPu^X``Fy2gZq~shIO@x08?R-@{a$jbT+_1nV2M^t7_pgLv7>qXh>EM@D+y=g%+*s`oDr|)j8)h%^pDfa}w z2gho?*A`iYC)rWLrmYxm`tY@e=F4^mySYw9#-RGa+IZq@IuKx@)Y9HUf)W})NMwqB z+0dGc3U;;vb0#N)>WZACqh__2S1Oog;PlMl5;_RKrB8zwj;New;#RQB!(Tv8Lsh|4 zTrR%0@>?=sZnX-J)OxV4Tzlw0DP#}Gfc>To5s#z)pS|q)6DrgOnLv~K>zAO6SF{be zys+5pZj(uV{JGlVrr ztODiw8q4stPzF)94v~EJWN|9D;deRq?{t1jVXD?h#fjQY9xv%DVe{6N%!mxG%C=y= zHjr;WVE^y#G&4DDPQ12m%JZXcf`tLl3^|{oo2M1IA-OV=JrBS-Z*AmVTM0XcyJC~& z3oU_a9SoG|a+~eY-`*rV!)Xa}tpHB=I_7c8@%xB~PgQbz-5|q_@5qv_qt7PEn+uV5 zEzaWT0-$7YfRxU?{Vp5$M7+lWYd8i;FUJI#jGa){N0es=UE? z*Hq48nT-{0+{`PIwO*c92?zZr^kT$eG_GFEq1G+D&dk^ zM;D${MMGu7BBnfeCBlQ?-Xy3r5-0U?5sULw${6qWX6Sf2qiFIqWtIuujSbfWiEaBAtQ_2*IRJ+Oj;dd3Cr-e`Mo(y7%Fw zS^U;hdAk}T^W3`p0M&)?yIue$kG%5$%GJO+`S{qz5E`;8~YGn(dSwy ze1oG?wosH$9j|UN{<^}e)9q~I;cg(H%GG%V{Z|{yv+?lE1ry9urW=0^cO&Tcyc-B( z2?Xaso#hU>IKpa2(b;OyeAzA6-siV|$PX>3OsxPL3illuXsb{=X+^UDxYU;@1AYt? z$>!cby`6cv7!2-50h`=5)okp^5;s?lSVbvKW#!R;Me@0VWB=$d-i9JKDA24&oeG&yR+euu)Z}d*Fo{Q zdhusy!Y0)(n&kj7%6?XfV+PKvX0FVD3#8h2D7*0*aPP{FAcF0+<;bI+knvbHH*z=7 zkUKA%4$dvVw!ATvRnYK#h%PrsFVA6CztJr_ELY!zrJmMVbZNI)PJt?$EHlsW=Dox# z$@@AUwsn2HZ%GzUi99iwwSTLN3Osa#(U%N^GD#(DFL_Jhy4^aw^10=K92Ulv({$|_ zWTS=ymhIIfXEio3R(G0)2?fwW5#CR#vK9z=kmnD&b>g4eI(O*YrF@mq z7#kzA5%kBUbW~-ImOC@q(RXzIR-P(e&P`E44TQXXjttj+!x>y&!iu;!pHjJVjYw*~ zO!)>1e|-zmD^+gB`ocf{GxWnz~v{urYi2dwfuqBM@0w=1ik@HdVA7Y}?11 z0IDnBWQ*7A=X^)%veQX)ke7yc7`B1VDx|PWgr9OI!h>m6ZbeF-fjOKtbX?@qY+HE} z8)Kh>CdZ!J(9;u<{H8cc|9s4J^RnFTn3v&(n7?$co_!8x@2>r!*(zl$^Yk{7aqk1E zh7+D4r+2Xx6)Q8x$CbA?G*nv|xY*VE@v{+S8I*4OQ@E2;b^rTkyi7|yenQm6ARzr0EzKLj(d1mE~b9O=>e*m>SI z#w$rhEYjD{xR`1uM4V>8-$wBgRaKFh8bjLfpvE-th;)^wx&=yPjxIM9EJ3`btFq2o z_0*7;X9P$NcwxO8X@@nC^(>?s(H=9JQLKO}LL+E~7vmIJ_BRi}7Efu)?QLa-9~0h; zztyfFOHwSK*7<$Esg%W_1^{T~eCV)?iaG#7Mtk}ADqGfRjVA}t6YDG4Q%xJB73)Zd zfcIh9IuS)VK-n6B^O6_JjQXZc4!sTiH&BkAyaUPoluDbWE6F;wC!PEFxlmL^YD=Q$ z+|%&``q~>?vR#t(XneBaKZFlF6}j?V0s|R!A(ek>@#LMtG~=P96d<2{BcA<~uDSpr zJmyS69Z=7GvWg9pFMbI)aStS~#xWDDh4%GM&{t@^u(G`!0a<0OrLJy2NnFGTluCG* zw7X{wA#kvA&#Pj$%p(IB9U2*;Z!&F?H6=t~^tG0z z*8ijqY56@rF`dS!>=3S3I)=Zf!Yjb^wk+AvFiZ{9?e9M@Iu|Iqh!UmXLse?P<% z;%>lhM;vDL0aBv(CaQwk_!?=v=B>5fCNCu+A3jq#qm2cHcG7C7wDTK`Hc6F?(o~v^ zZ{D3Cp3hkI589vgsB$%sOp21pM+<=zG!f3w4RL6MnG|K%5))p@sS1S__r*dfsH>f$ z4dmq8?9hu{Aey!+cY$P-AyNCdk?H>4>&s;THAKKRxkCu)3R!VmIh6SOVuRLE{r5e}_4_%tfRcSp#8Vzfli9`i$W|YK7}T@W1J?jS|)e-R{HOX}Qk7CMbYuNv{EJ*IoW=GE1oBD@34g@Ri0C#w1#j5|q1@>LbU-2_$K+xR_?%Iyz4Ir6eD|1pQG7|5Bar(pT%@L%Q7h-1uTdiOUOT;E}$m zEIs5zB7TjfiN3mp$%s)haUa7CCq|CPj^qg@Yv+rpi}6G;v&;c&D6$Eq2FvWunZ zEM&$R+6xzu`56d>-YWM=dD_OIaiotEwn=S)!c?NjMP%J)ID)f8SQ~p&%jCz}tZrwv zw&A=DkfL@aU&k}^6S_jlGL=;y?M)-l{5a02Gx=7s@dq<1m4Ai~W*+f)<&-X3{jZic zA5y^u6C5w_rO!==YR?T|$F|rqd4I6Y9SE*PAwm^>^d(<_cv)Fl4dr?59jGyw4-LmO z@Y(+me17?&S;iA&MK3C1T^ER+$0nm8_npsK&R5E|5Fu@^3Tqd4MBhacWqbwv89N)X zl-;`Q$)wjg_eH1=l#8=9yc(X$^jwcfat0GBSc4+2sob5wZ=vP#*%=hX_s_8A_jMxo zzpA92+`bgZ!VTqJf&@q&AZM1r$yV-I(RB_qN3e-}LqSFzbzV<8-T|g9uLpPj`T*Ux zKf{xS#s-T2%`K{^zI^W`VmM@OFON?eN1)P=D4>DAhUQHF($q`tf3>cP#ULEASWflC zsw$-HE0UqS4L~}@lpouAiU~c)(Z*U5o;_NF&|s#oaUeA+r>a_uHB8mH@%9AWt|dh} zUNS>+<0@l}R>|$;ir?dqM_p64 zA`S~VUOtmKl2rQn`*6*fI)P*)y>qTC_=;V&r29~Fj;8uTjPf%irr}o4Y6e-+!lL#* zk6S+c%FKoJygg9kMm(4@tCjSdqrFa_nhu0SMT0OQtkrSIJ@v$bl~y zGa2`!w7tOoEK){eIe3uy(2dzfy1lcW419|xt|6dh+Z$-Ye!V?DeeIZp*estcPrqR% zW1?lmN$oG8$6wY#o(DCZ;lQ8--M00Vz53I1>Z?*E3&>rCY*Lq8?+ zAE2&AsPgH?^2D+P;u~^gx5bkIU2!$Kwx!3$Q{ETYi?jq|Uuq^_1>ALA!d||rqh|Vi zjwZiC!4wP*?5^d?*NF#68va%V1JbBA@28905GchMDV>TjI6kU`pn0XxtuG0x^Hg+; z#MR(6Q>ieAQ!eI~j=#Y`HC3^PPsm(!$C`n)6q+D~HPI@GQ!u-@Q}4{#ge|}V?+U5e(USr-l(=B< zi4MI%I)}eiw!5 zz}}@Dcz-w2re#qh#8>zF&~UQK^=p#qMg5av*2EtDtfFMiZL+2?S4(S2h=V1kS3m)N zp&Wj##hT0KOE{28@yXCb)_*2@vUFSlI+%!nHob)|9Dfc; zE92!ul*22lr!cG2$~IKHUrgk7a&?(81QHJ^cbYV4&G-HX*@N&yjrEXqWuFdO%jv@b zoxNiE43!jSku<@1oD3VIPhJG$EZ+{FrKQN^5lBhfDOB@(?*3%&C3fq5ZZ9O#+ve3{Ckbs}A}B3hAwDWj-C|*&3>Ny7uljlnQOYmfyr@$POKA zm#P~j(s!vLz$Ce0D%`;TQf20oXwaS>aQ~3tw4nd9czh#yn7ez)ya?IKBtP~~Uz;E;wNc}(1%G~h`)Sz0y2(dcyH7Sj%+Xf2(o($CCHfy!2%=QR;Ea=Uu zq#3ON@^#rnrySO4&mXGdMNFTV$}ZPEA?Y#Iv`8Blc&`0WX8ALyT0ZXYapN}5ZzP@P zSsn871GM+Re_;FFHW+b?2XWb{G{m7oF-uOEE-1g)q3YQMG?%{;V*&q|i`ZSUAEO_b3azO)U!)6;9g531erHOnojpPj8oY@$R$68 z(=XN+npqpMN$hc&b-YLg?pI#D&O#0pG1^rX9%wAcwsSG!%#^|NK<|W;v_TdXw3nwV zvebbpB`6(lAM%Rl0oK)$gBW_X_1&{gr+(05PM2#XUih2}4?*CUH<$5=sGWy$5jCeF zKVwHyu^2berMGwR_p)cP7e3O}F1{Y($WYU&}Xu`nXN ze(~sB_1uLzTBAQWGoy+2N4CMR%BqX%$nS!3D);L=8b1(`0D%SD>MBbHZYW8_9?ARxJ5p#0MXTCb(AD0k>^>X$z^-O_z5 zGUt?mQ21#p+`Wp37Zg>_U@pc(CU+v?K~j5w$0>h|WPhGbM?LM-x8D8IU2IRdCMO$d zhF`mErYv0oUta!@=ew({1IEv#5gkK2jH{_=9T{=@P%)#~+snvq)_hc)zn@Icf~D`o z+l-3oDV%BXO6go2D3gL!$n1wJp?7{%z9}eC8nd=i#p|@gZ;kP1VFvzg+kVP-MWkXj zuiU#ARjYKn&ku$nOStRWz{0&f0YK4(Y;{p*M6!hlgS{N?rNS_r8e1Xe$Dq%_D9S5gep%x~vXi5YaWRnifr|Y@vNLv=;6r7tWzFr>LMsynP|l%^MlTKMfVQ z^%W4xq+$Tn52x2>P~F4~VtHncKn~_MlYv_po?PH=(&HMh9Fs>nymHm&-Q1jgMn5Yi zORmV1l}rm}Z+UI>hra{+ND}=AQS{Y+m5TX&N4?jIyT|4q75>u96U3+3`I<>$KkS|nx_nC=C323!|eZY9pl|c#ky%* zC=>h0T~CT?n{KF!z$P-Uz4Fl*({K2al2teEd}~-?J4$9%$8Lx;lY~Xk?-Kkl^7cfq zLCx%PSx4P3JQ?{ph--U47E!WT@-}*Mk(_Dcr0QIgv@&gIkxSPOaZ(%)+vBGFB@2++ zH)h&o<{!*8rh_nZbjC~HEZ(?T1?OVjrqsti!8`TEa{jgg^IW4!UA!+i1;Q>_E#G|z zXnjv*?3Z0f5Mu*XStDddxUNhaVrD^5pwx+t&_Y3#>%{i8)Ig{cH3y1nAnU3_K8sa; zN7+c57Z7PdYok$m27GTkj$H-`wK>8Xe^;TwVyS2*%fYj(bX#5)+fF`Oo(c~Yg~l+f zh14=GERy%8!K%Y9nO<&$Y}0s2N)sSv{ziPh5uNqxY%RTiH*>%VX(JiKvBKCC?}F8h zvGOS)z_PrvwG0AGdt7dIHgn-JfWiwZnvPB=Pr%hyRE+?S+(e!z*2ziKIiWSR6%!(S z5M@Eti@IL`x*86K6sN1mV7b246O;2VSaoG}A~8hS2zCpN&uc3qOW6;KNufeeQ6mA?4kHyVQ6L zeO@viKJYeyrNh0sWrXIQRT-U73uG&;)JYb1H(bKFS)7h@Ior(sHiHSpxf{sXAs zaKSBFj7>q@B%m*)WBo(P#_OpCQt&+lK-hF2gafy!4&D2>GT0YX8ypI%$w>klr07bi z&YdE^nL661MBkMpZ%5oYMu6ch9gPXuE}V^NUKu@sBP>(iI?#wWof{GVWT?`y0HsaD zcb^hSrmt>|{ItrdCRa_ZJfgP_BsSZ4}4`Bt)#&UZXAmHa$5DZ1l!AspM!`#hPNVmdu z_0j&UTKM0bTV&S;GffUPU+^G}3oo(grB+;LFoY&@OPoI1*QFN_>bbiGm1 z!Qs-b7hGHQFvri8kxLq1fj>J?cZ|D6Z-VSu>d|z8E%_t~p~nBxlVx%u8(mN5$sN=6 zK^u{9C|C0k*B(`p&a6`sF9YEaiIzZnHI#A@B3D ze^+~Qm=p;%Q>J*Luw1Mi!H2PG()k9q!5S4C2VO_3AJLu5x2oWB;Bq9v_^{M2>+i9i z3o2;Cb>vAHzzN} z|K-q5_;E;PX9ED5-^0DLrIEC9pv+o!MP3%G$Rs&=7s&pZ@+az8!>BPGsk&KvOr|t6 z=^`*O7XVpN3V&3He#9xe`eO;C`LX*&XQtlaGBK zd#3r5m8kMRT$Rj}H9B9jum$?<9hLu%9&!>GC-WWl*#Ckw=?dMgT?#s>G?}Hjl>KoX zmYgJauYez(n1^^IP{S(C8<=73_^LzbgEK>xl_RdXRi<3$3?w?8L#xAKHoNw-xY_RF z<`B3s8%ECQEmL>E60?&Gaec;K)JxU(+0hhKKMzrQ@O5eU2o^%C8KB2=c;3hv;daWY z@^|2SDQHO_;G8%xMR<<3I3{zTDSc$7tR$kbnc@EK_lgDNW>dPhjfOJOJy( zopAHuZ5}^w%d}&K1sfs1GS|ttC)lOClteWq#uv#wQt7qT^BAJtd0f-yYm#s|kcRZm znS#hA@BVH2dIBsgSbWO83<>OZF=nwtD(akQgYw#dcdpByPA>q8I@SR=a)}>$VD*Aq z-EE|&6Gh!!IZWA7*NkZbi{n?oZl$ULm-bFr@foMJK%?XmEtMLCq6oEIqMe9EI`yI1 z1prrqvcC(fITzri#V~#)!7P)|CDR2xSbE?s;u5)CgFu0FpA<;JqvoWydM|9pAMS%V zD7M)f-#|2#C@R+?{PZP|etJk!cbmBqUybd!xdUigYnYjaE-7gt`|rT@m;0MeV1&P! zJU`6ifN7mhTt8(lsoE8JicEK?n;k61wP`R-A*o05=?gIZ-+6@bsmwUSU(4r1&jl|+ z329cNv{0LkvQ7dMO;vkREUDu z#2?!+xq7N#FxwRAFN;^1A@ndAw4JiU#^l0?)Ow1Q4phmc6c1s)$z?h!J6}c3?ZquC zgWd@ggRaN{-t=`KM`uY@MULh7Vw$mC^1pg;Poti|p|uf6G+vc2kr&P(RhlWsqf)&$ z93A-$t?^|Wc!Z?NK9V_?ll+%bF!GIsRo7FlzPcq)8r~wpMV{swie1N)#*@1Nm_nVd zxysQUxoqnK&UD0S=W18^xiYfqh7-~KKu74gbS0qf7eTu3HSlPsxlOV7=olPHfX#nY z%xe6enAJ&@jFH35JekZSY>|&oqOof=x5f}Z&KLE8)duurtc4Z5)RX-K&2dX*cIW(a zXUL{9nBr(HPYSFReMw)}TjlWDc$&Z@aIeZe03~?!O=s>hV0SPT|5EGs?06iX%@TL8U#aOJ_!aU#@u2kL?Y)Pb{Z=f7n`p5H}Lx+t3_XCcizbjj3NR zsZ1_BvOa{9ILpH(IIBs~uVymw4#955%Y`l|(|s5j=iOioj}O9}Av^jaJPt65$a*N% zt|6!s=;UlAp}{DmkA61&GG!6g#AxMz4a3?vEL)ypy06kfUW#95Y~ng{jbWcU9Uv=*ZO-A!P4?uncxJ-Y<+`6F5E6;9Hmm z<=Gz*5bi3QmzgCvT2flDHEGQl+-M7~%RLUhi0*|F4kb!coq+ztF1*;jJth|nx^?@@ zFPb?ZimXH|F1Jn#Fgj`9*E(QrmRquPErPd|_DZP8QQliTmQoasb{I^-g|bp-%6(89 zU2XL(;bBYDgoKY2XFIY`%dk;nBrLZ-H z_a?hJklc|{7fb-+FG$s^ecaRXgGXaT@|=xr}vQVA>wGE zE&g5FB&sHMOcVS8MiX|csfqkq!i-h zGTX`VmPo4qVQfnEMP@{2w#PKxv@p6BCbJQdHqv{LNl~91tb|mRF;vFrO1tGrxyA>j zQR%&suTMMwg2iY)#2PYo#m86)kc$X+D%C#6D)!SYT}lV@+7I<*MVu-aDgAY&nlDw# zUKlN9k{k{Oe8%p?oE1hgvT~@7NolX#DaP(F|91@v5mNi1#UDV+rxJ!A$U&@zvv_Z_lu zHAgW{g>@aeq6BH=#w9kNsCz`ZyAIZ#{aNWi6H{a}UhC8Ed)zoO%G@{tyxnAy726~j zK%|#qM@V%0`Z9Qm;JtA>@Y}4O36b5?fFrW+?q&w#iOtE@M$C!CM6 zUUwTEkX%(q9yO*12td7$DjtXQc04Hye?s`D6;hV1G)?;#db4q^{aAashjkEHhP=4O zlOo;UV{gwnAeV{S%pIU|x_KD+ar?!(OT#pMF<7pzNAp#v8GrNFL$u}gB=8R`Ln?O%Bn<}q z8D%`m(Hm0%Il=zoYKwxXq#~`L()`B}P`NE2IG0tH9S>we*Y5f9_e1NylG(T6pq_<6 zS+m6Il#!(S%lSBu9~-UMBXzUSXGx`=9s}}sR+p0}q40y08-g8kbBxCP$#%n7w1FjYiV79JR>=Y%DH7y`WYlSQ4<(uTT2<8+OYJ38Z6y4 zO~xMp57fU3sK4I#P=;}{ut*+V1v%V*+iJ({ffd<`n|uI7ic{=?K(6tTIaVNl$CJjM zykdE-FUS9l%CT^>QqTr2qr$6Y2HFNFk_h~FbQPzL&0nBGbs5RBKuWymr}MgUYVT5n z@*gVpv|Rlc8FN&nJcB^b93!j7pve1X1FNqu;i_@yF*NXJDys)7(@zWv(gE@8v?DdV z??|s_tUBY9TaxCttbfYCfAaFAc#5Ud*Kqqio#B#=A4d9iv^qF4e-dO6?jmhw^S!0t^0v;(IRylfxlCjd zlN|W4X%}hc(Q&lS-LPPwi3BdQ=tou1FRv4fAoe7)T%PJ@>QtJX<}%{|o4Xv+mBPo6 zC@u=`!OW3K#6}j#tlhl!kcLU^$D(SI6=kIMsPdC&dwT+};HLOVvPb3oyN7;0gqijI(vcSA2fr|Y<6`$*$ZUaq99gg<{%`T9C7+(RBT;2^51qCq$oUDb(_ zGANTf+9dK|uBl_x?ezgmmRgv>vD$o(3(4ho*|q$`tbf@wb}duI8sPFF7xZ*>j}y&f zIDrI3@hsLoPeN~abmfasf&8`uk@Bq8{P$Gs8JRU7C|p~??3I<}9a(#ap+46YFM>KQ z58yS-sE|!QtQ*?Na?A2l&hxf z(xsA?cR>ks$c{*8G>(>Wc_A=)@gUhgmyJwqhE{f{qUQ4YZ@iiQXsO(?8Pz27w5fy1 zB5h^PH9i=wT(uA>WjYGTuLWyGZFn@Xfl2umradf-87w9xM>4& ziffO0GUQ93Sh~vTL^xTwM^<{RJ~?n6tN(v_XujiqAeT+$4kafgh06u^L_^ZUNbR&e za0l&o-c}_&(M|uAT1M5D(_-4HMnI^TBbkHmh)=nc3_0p4Lx}j2cr}p% zeUJ&~bm<8oAR=`RvoQ}hWU6M9zw=(8w) zqYoC8vkON_Io;HFuqL%q>oc{Yd@v%_*o;Jb7ZnD{C8iM5zSroPs^j;)XGwc>Pat^D zT~%9BagbqTj9mX7i|P1YE;Yx+vsI+iX^m!UAm_%SEiS2=9c3(bw#=M_Afw<)E!I)e zMq2j^_@tx@$M4nNnu98&uW(6`9;EXJrzDlg#3Hi{ZIcp^b9XVeu(g#CAeJd-7Y4%8 za25*!S}>XM5f@a*~jQXeacb#)9nJ-EYxJuWj?g&Al*~e@AV`zN9Qoac4!Eq zh^zJ=^PF6x+FQt?FL`6U4gh;!$LvJo-F!9c92z_2e^47q^(Dh%mdRB+MDk=uB-QJiVZ)s& zyhCo5SDGaldcG0Hn8FmPa~n-c+N%e}@2|mF`J1rmvYwKSQ%sA@x&#`XrfiQPqA`n} zS7rHVT;oX@bsa`JqAcw-F493FmU-~DH!Oz=5r41THnrMwH3Hl_lmSm8lu(98seqNXu&BsCbI#Qqu+ncqa!$SyNd4%$a$^GI?VwP#@6Z^Y}oF~ zJ4~L6$HsqCmzW<_y531~t0G8&D&2{H@ps(Qjbzq9WW;`|(aMJU5LDzcw2)heN!7q& zMC_+f@<2KCHB!oMY5AVDTt*hN@5JcpYNNq899g~^+1)J0OA{cViIb6+Pc>jKU!*8E zH_sFf7&Z%$;8dYWGN2R0UbAY3CYn>>D2(9+Eo&q91(Cz53Tchd<-#RIx+oW)@+2Z0BlgM>@^ z;o-8)#L49C1gU14l6fRPhsL?^EHN*2wAeqdqZX><`DK9*VBj0UqybM0Z){%jp#;wdTsT2js_+!aJ&qCgn#wpSe@{RSMYjT!&r@d* zxK>_6I-KDwDyXzxFpeW0S_pb%*H7zRI8pJ2{?!6eTcS<=VVi(ICy4HWhW%rtQcaIr zj+dhqKkH*sQ}>EB4F;lJj(>5$t087Cavir#o$I(ls=OI+@D6)jtSrlNNl4(!bScoO z*qPtROPTP=Syk{k&eHsLc-f7G{A$mXYd5S>>ADL8b(`{y(8r0MX$DLw z)>1rkuFgqIVVR*-#)CqAC6(M!&iC?AnN>Ne@+)~)O?A9;;zNNLqfv)xI^n2tNmqHO zxW)9zv~!*mIrkS--a3K;AcAZo=(eP{_LFdUS3P#5P?i;&lG`1-Di6uE1a`NwZsyjf z6X5^r3h-K3B_q|IFx}F0vKd7H5tc=5kXmw|rm?c12R0$HjEbmLc}GuO27_DJQ&Kt; z<@OrRBBX7kyPP1y62Gf|kyO@^ySa(sI&@uUCL{uM%mZet*cBW$1ua0QDEz;a(atzr zn8ONV-K41x(jhjaI%iWdUtTQ;L?Jt0SDqp-w}o<#!lRP00YONeEIJ}|mK~#b9d6hL zm`miW<=!;&VP;od8MRfD=U+^foSS}ZTZ)BkcfTh2UE#G2$~^$RL%iw~`U`s>g*2caumVjq1R(j5P z%3&kC=dV$hj~gChryf$7g|bu!(3WxCNQ(ceND9`KK2JCtB(k3*<6IKsJ9`e&I@IXE#VXRunIR!jTMxjZh5zw$1C-7KC*gYFl5%_L;GO1mAXBy zOn6|I&h#9oQrbav1xKW-u7T1d$D@;19G}2lT>pLe5yJlmimA)CuS}YEC|jI-yA&aW zC&dbd5Wji*9s*`fVG#X9%@(r#--85#i;XYSW4L6(3*|vGo{TB~cwWs-!%K?78TSiy z;af*v<3xNuG>bXCrNk4&d*kbih~`3Uae5iM9_mJDCLy{n=}-f*?3Ui-}q_$;`Yb3th(o%LkO&;h0$g;`|kW_0ys!#OiW zA{Eg{&u3Hf+{vOk7P^gliNUifc0`_yA%mLeU|Js`ksL)W-H#cQ)4e#)X#Y^~R=_9oiqInC+M8Lj;t$S*R1FgrA`x-B zDmx0-x``%#-jfg7^Tw;Iq}z^D#qFf@VUT1WRh|io%IHp5nW-H&#(o_h)D`R#YN_uv zAC{NLTg%iRtt>|N&j9paiNTeTZA~BSpR-g;gBsW;X*#Kh;E5C3TOO}MZ)u0dBICJI z0~$pHR7@>^KZBPgzSM&^ROLL6trx3~1i)fhsQe>j{xnaf@mOIA($i~|2K!7e)6K09 zxN`@PS84OH?wyT!lnO47+(%S}Rr#|-8T*>c+0iB)w0GgH7^2)$P?H%>DXL=zkquPq zbSwLExC$hZM!9e8;lrO&S|X#bafsF8OK=8eHbX{P+HeFhg_%#r(KsCiLoniCu1S(L zLbQ)CKFITfZZp&iBzmD7h?(YC>ulpSREi=Q(-jHe84y1rOX+Xc5o8;dvB z^h5A?r0ddv0d`4S2HK2K?4w6rOAYXG$y~j_p&WccQ>oAu5%JASZ1Z)UoJp`*TN!-< zet1>6cgVBV0(ckpKv5@@BaMXxJK|Gt3^%HSlb%S;K^UYxrLCd1P(bVq)%5r(8N8!d zy0KIbHzDj-u%ZRMob(GkaiO->8M{}hh zsn^XMY5hNbC1j%~0v9jYOch%#hyM?Kye|<4{fgD-jb=sG5Dta#Ov%=zLf+QR7Czwm z4q02)!YD{8*Y-#&z1|sIvgE(S6;S>S5N7!(@f?H$kEw7TqbED%@5`(MW3Uf0J=Ddh zl3KhU7scDHiq2BHCmH2;-4q}HcPODa_UO?#z2mI zZF*evvFD$wV*Tvxx8X?w(K5n(>?#nS~ zE&p@bvK0YJ6m@@GZ=p82WhzBJTu}HZWw>;HcL&2rDu&;khcKkWi?q0jW8lmkMrvUe zn}gJC_BZHY0D-Nr3X#IsOC5l}*eK&#A=R4V9casF7o=u%o6NVE1+-?+Erv4c^VPJuNt%w_C{&r<#9^2Oji$SSMaD4LD5suA zv3>ARxYk4M#Uo_@*O11sF_6JUALiEoIe6fN3dO^R1gibE8)Umc*D7sK>ma1DuZ3h? zgwa}WV0J;}numQ=a0L1Q{LWXg9;l_P$*j3x5n?0~lDQQD&Kb%#8r|tmlrO6v@*^jm z`=s(Zd`;idCjncClUq+2N?8_IZv%t7HJLgcGV)1)#L%~(tC^bVZm3YIEa)4cj9fcr z(6uGX@gL>ji7;m|QpWFQ10bGHamRdjsQYt+IR#g(aWIZ;?M8V*o$)4VBykeU^KP zYn|{X+GuEuA(=Zt%uCjMOmLqrAzS{Yy!xU^E=rn7b7qzAl>Ya?I%6in*{PR@BD9+bljPY*P@5C2;G{-xk3g=>`5UEXs)9#=?ODV5)P4co8ybpM z3{gL7Gz@xNm+l#HiJv}Ip*zN&q+CudgPMu|Sc?G`#jm2N`~$S7-YPe5Loe&zgwT7c z%vqRvxprfByc<#M9)rwr9Fmh&%oqj~5ORfz?t}qc3jr(9>bQseUL&j2Aw#bQav;mh z+Qytzmt3sK-Z%XoQtzX3K0&^v^%3WN4(h$5_?{m1LtBlMu?*dpv07U10{f9glP)I( zZIQ1Nhdoqq#N8NdZmi04!9zZ>iKZfpRw`$2ZD354k%>sfqKmgO$jvkVYy0{#lwzl# z%$rE1$;y8YwIA^#8Nzht9Afpc9D6bxOKF6sS9nVJiQLqyFIaWK+F)Id4_ECY+R?M6 zCc6g2s*3~(e^GW-@v!h&zq11H;i4bctJcj{tq&(^7; z4ZcLC&R++tAjZG0)EWtvYP zr&3!=zx9+zulLQMygrk+ZqeB8AC;y!QWsuJ!p#r?M!fX1!WGevIic2=1iENJOQSp3 z6c>@ULjhvXk2IYGS^9~GN{p^CQUR&I4l$Z97KiQpfHRK9&1gJPx9cprsRaknR=Dl(`~GZHDBIt0)My}+M+w&p0#guE2-{2$5=DaE-+uTd2%*%$Iw2qVI z&G~5D7wqcDYN+duBPs#eA3MsA(5}PURqaZ0FO=D01r2 zBTVZ3fu>KUKEZsISdJ`eEkr;Dqt`UX*`*rZ!F8!RJnmoqqKH`2>QtR^T*+te2%gK&4DrHIb9+R}zdQ2p#h_OP1YD*-; zw8&>~n|?`pmDdjnpf(?WjWxY%WrmrTc-^K2l_t}$Z!;ek?qWk8uD~Cc_G5Abb<*h7 zL=215Ng+x8hj&(Zm#;0DipU>ZgSG!upD7$IWJaBs$ zqg<)O2YYuiK&!20Se|^7SDe^~>E!%~@ho-ub<(#cr9awObEN5dek~fn0Jrb(*w9M+ zwBaCEBH@jEP`w&2`k)y{Kq^jD+RvduV&qPhO8w;E06s4D@$GlF^V60I44pVuANigE z?B13a80^wu^!qyl_{2Yf+0+i0E@_j?3~#Sy5psdAyfvK78~;hX&v=eywr|acAMN3v z!4XYI>HM>wQt>kD8Pg(lwFC5x#_Uzei*WH+ou6EyOXQYx;~0MRaRQw*s@zq|nEj$= znw0vjG5NVQkCj|y7R`3>&w~zVf{~q=c$au^qx~r!o3f23zG>*8<@i)&@bBqqGVN); zVby_{zbRH>sm;Ltmw02Qk7-x@hM!i?=2L4${&}M+ha8G$JL~o_?UFHuqm1jolDFwr z(w+A(RGaS1d%k9oV|AO+6P5YL&>e7^>1V6{$-EOzuv3fA zp+#0D<%{=s(<|>T<{K~h8FP-w=p}?2OJY0{@Orvewq9pC-CjnW)r&8aJ3C;HiMf2= zT+83@;DnUb=XvSby)5GA5oVeU_yI(4F|S+(H-g8ey~=mXuc4|pWc!Tg({0l7E2g(` zIrhh?8`d17D32S}f7!-=DzIhGP1R$S!#snYG2O<4JezDj#aZ0g$J7gGh9QMh81ROJ zpI(Z=UY(z18m>K!JEsurPj%R?9i+p-~ z1@Cxo9)HVR#NTG*^4Rxp!W+|;@uFBiCYf7>E&VkeyH6(#oTg2IWDVn|?a#7XSBs$Y z4?f{i^&n?*gF z4Q89Jd={Cfa}y`%wD+GnazuNOnHUI_DYeM>|bN zSB}Jojr#0!@m(`tHo19#7*CrNTm!2;siU`_>%%LSy<$3LqvqP3S6M{)71o*e1mF8G z$gj`6#aEiN_oT_ZOQub1`(dtTJ~n0gD30Qt?0mV?oOO25ElF3|%mR63IWxYvhnF4h z2g-U``|LKjAdM38OhvuS^M4eZeleSwbnK)ot?_25O$PqPhckO{b{U;`A#vS)S=5{T znBLJOe{&GCo2=J9Tc;JcZt&CHJf0}`ve+kdY0y?e10mSh&#J3kVUR<*9MYtgY|bywGstLqossk(NRi>! zSWY}?yi#Kb?>`-Y3})$qf37#7tjFJF>R+8_fROOLPb#y|o%B3g4&#Z9^EroiI#Hf7Lyc1~CvqiZmlSSVlUellFyPb|P=8@B!<%{XL()}N6rty7xrdX!)=RFB<(eF`) zZC{PoPyfI)@DC9w;_i;^JY4s~{qi1UJh&B4{Zv>W6TV{mH!Xu~J9h?^eo8x5X*vR( z&|S;CSC4WWao?N8Mz=gdiAr;2c01m3?ptOR{}W&6yO>w)?#-qz8_Lv1T;ru@wHW&Q zBer1BB7XY(3zi#6;C=to%-YFBTqS?Z6B~39f*0e-ddBB3za;Xf$F?#DOHzh>_zcss z>00V_-+(~Z$1#sND;a2;W{xpi0){w#;&1CaaIV9?hEyL1=u6ArkO@jh>;GHZES8?j znBtK>oN)94+g*bHjN1A zx6gO*l>bY#&;JqrCY3J*AmAt1EtMWLHgNp;seIw$Eap0mC?a`fKQH}vlgBSTZgIHp zJi@OZ&tsq}FF**%sVTDJIDhImi5Y#9Vfy6rbi5yPh)ILq@ua)ofl9tv!R~&c%hkut z1RD1!B{OpszkM>?lPQm_X0-Aj`SpOtw{=yZgqGh$(p+DH9PQ0)cDUJ_=x`JMZo)b$ zzvgktc@Ot<{Hf!D?%y^KMHhPn--~F!4r`-%OHgoS6ITZc&z?7W;Opl zwDZAm9<2~>I%MBZ?9$hI&1w#78>lNBuU(DSs9fmA=Vh-Yw%6GeXE6`#eTskPJj83l zg{-vYAP(^(8^g^Q&x($9Vx$vS7-a5lmU~wD=si@%-;N|Ry8*wj-+kIM@Ne;6$$y%e z{Hn}kJTwJ$|o|rs(4826wV7x?sXZQU%?8*I7M*1?-w8@Pas~Gtr!_|1# z>dTvDF`j%8f*4+%H#bgU?CTr&>XjaWa_M%9vn*fF(}+zR>)HE!@3k)&An+?2z9R{6?SkBnZPL3wiVydUX6x#1;DJ?h`6n^V=)F=| z+(oO$i5X5n*bFPeVV>?v_WE`{m_CS;yzb^qsue-u2_{ExQMol`#$6WiYE?eXe34nS z`j}nlTb1!bRDzZ2zj892wT*b-5P#cP$QdmhZ2Gx!irE2(zIyWEO@{hXWmT1VyYbUe zxQYs<=}tM|MEG2x+Mwm|Xltc;#>rIpQ8=Ek))eRNd6LCJX(2f=3@LGUCqLEcz&|4& zViu@|5;<0npR#pj&vB(LIj@bx89!T7ELR(A+wf0|9ZSqj}8bj=SfD6Ujc!ob~k-e{3|5zZ=qZEWiWw1 zZn9)7nviUo#TEi9YT-K~>wfrIWHzxkanhbLqOyOEnTwx>DuNv*fj3&kr^#p655>gu zBpdb7%lvb>6-V9jAiA<(52v(zGKaZ1JAzk?ydY8r>&eBRnZxr@(=J~=!66kUG4A8a z1l$@N(xGWQ^_C6~EYhH@ehPv-^&JLTqq%A0W?tdh4_(B%m}Ta*SyIMB z+)<4zeN$Ohe~3XETw(LJ=5sbTo8ezd>}6D{z`b6qnlUF2UqTa3bC<@)gDPS|5*J-d zezmd5?Q~;@H@m|kyJ%`6`@U!Qujg@S5ykGmnamTn{hlZ(uFqsb_7}{s%pb|iJ2hj_`ycROlLHJ8qcpV0cWgjgE)!&%u1I=aSJe7jIU0yD zjHBB;7HQaAksjJoC1VbqN0NL@(I|O1%$%=$%22gVScAldP+>V#la;@wf)=^7h8IoI z#o1n4u|6uFW|lXBe9eXK$AsbJ_WrF_+ogkNXbBaUa5Q--&VLHkWEtY-u?NbIV++DP!ca zM6E>omN6;Kr7V#nEvNe^`aR$M{@KIqoX_X|e%({0l09;&cUa;~f+S|eeHcu$WSKy=X4CN}@?$bdX2Kei`u z-Qk2j<#=;g&o(_c-FTiy+I6Jh-5liHVqG9wlf^i)Y%hF3mstZ^FUo0$t;^$lzd<$Y zw4olI0nn|x_`klBpswjGZb5NmG?b?_iH^aE1RF^fawk}C{ZDbuw=2rzjg<(otI6e3 zZ7V=MT8}F=)pH(qsLW8=UIU!{d<{@G>diCGQ|2d~wwjsJv?jDtpfRxZ7%=(jYfk)l ztkVmJ84R=s!yazNk+8Fd%7oE;!P}i*Ut>Uw?5*UpEA@PiF-1WkQtvXy z+u-D1?`81gqk8*_s3J(MiR?tn1U#Uj;c>ut=|kA_(Nd(#feda?*q8siUydlDQZhoS zv20YXJi#A+`-(T)A~7jxa*S6#Sn^*Ik5N0PYw+FrUJkz!U{H#Ys(X74 zn@q39=k6{w-SR|Jp5lKCRx|6v{(X)+6Xm&)plZoPuA8TGbr<__po$ip`I3oNz4$Kz zqn6(D{Y!Es5qYS4gFd5Kk8kZc4)<+p#e@Ic3We-ufR>DViNDS}#6zY8@saJ3sIxcX zc-({^0slF@4CVG^ZdJVz=dPtY7g{E#O2#46B0FdCW3=8^8KQgW4v&OXs84+Qb#dhVXw?F(b$(7z% z446;;jK_ud=U=tog59sY$LD_T4acc!03~r2Pk!_l|5~WMf$4Sm^;!vN{$`+Ags0(B z(&hs*PJUVe0k$mSyUp*I4vF5w<6Gz;5Nm}C#ik8**AM*l$9Fl)$kQC{^Nkp2M^$R5 zOv3Y&zqA1V_cUkOtBtEZ{rT}iRginfNhIbg6<33F6oSbXn<4EQnHIS$h#&UH)R&=p z`s91c!ET-jWOe9=3zoC5@!Ef~P#f(TA}5xMyuv>N#5K~U)3LYt?nPY~cxwwPyLm;P zRA~(cQakdoTZeh|RYD4qp2Jn1{EK^An{(xQKZ4t@=}?z@A8^jyVO*v4b&gWu(Xc&g z=1EW_8hEa56*$FAdN~;b=FYPOT+*>pBrXIZWD`&e8Ge}0b#tLGzSOqk-bN5Yd|D0( z-^^N|+Fnl@nn5Mmd`$D8B63&(t2{XX@_U)mCV4oJC)Ms;t*6Ck z*boN$NW0o!u7MZ6{1_--odopmD_awpcb?Hto1O^jejf}~pJ}GZgHg!c z6RM!4+|byKaq*PiNggZZ&cJJBUrvAFF(4f|%S@H3CKNp351!ZI&mdwgSNQPiuQ=OW zy+m>Qr~Ep-B{*SYV@+dQT9G7n=KJHn53RKSg`K{PeWbkRt}~e^>9p`zneD4`)mnbvUWb zCFVR#y7f)|FZoq|bnV2I8y$i0t15>byQ9de_sGUk?vNKPD^1SRco2vk$|Ob7-< z%4|fsjoH6@NP8f5>`LY@FmsGnXd>v#qI>d`Xz zIL~}}u+t(}ip*j;qM7qVOK$tqC;YhnBC z$cqb+d}5yNM_#C-Ytb`#?F(93HqS4VS>N->pwUEKc9z7+JT^1QsvVG@c^xcvOs1If z8U(&uFF*0OHN6tFn|E&0Q?(X0hB5E`$_sq2ahT|B{HW*x2~9ueEY^je(Nr*I)RXCF zxI@PUXxe+ZVBz9>rcV~@u+o7?$gz3vJ1xlWq6qnU6{vi36m%`B7|UmnGgD^N;%&{p zMsbZ${!nqAj*k7JD`-j1hR7rvIyFOOgvq6Wri%=%D_Feu;q<>R z@!=+20l}pcNRL%I*7VwwAZg~^eY!VHmu&w9^sLjCz>v>5%6#poZ))qb$>B2i zwoPxYG9;PDU3`a&_Phe~_NN9$Dk>n=n@u_BIuBoq-w9MsE*FmRtF zE)(M4euNV-PGvHW`$H~XU(bCCr8brtiCjH1J*OV6nQps$)0;E)+QfTfFJWlVd*22t z+S*WFWWk>deuTFX%Hl@pavI`XTc49&(_yRZ$L4=lk&Rq#QiPn@1>N*rjTth8@+-Mm z%|jzpMV@qZnUuoYd2m@UWO<%5MFgkFMrQZR)UOeIHFTz?#yZnNwTQ)Tqy^{b&PA#y z98uD}o>?GY>SXl(Zh*9GG4h%CR&ALd%P|h0gbugN3Jw^X06}_3{%dKY`d0oqn)AJK z7FAsTNyzBpK@dGDJ460?5@IfM^APr3Y3v5-$kA7vY{QCo7Rs`3c#5?l2mFB=Mky`> zPtV*31b)5G?T6qvpi&tli(bKqdqKIZ5^)UyH&mDSUM(q?w`w7RK1@KQp5DrhuHOc8 zs}F+1&=0{)&pMp0)%#qc$s2ru*3lMnrxjZv4cHGyI-G=>zne+@wYnafH4@bJ*2EzUe~=_p!@*AWK;n2#^>b|7Xl^bR^) zyw-C6lruzze+)oBX9=@3{hNb-nyp)6LK>oL%NxlUJvRG2JLu00uB7J0+C(}q?${Vx zQUI`BA+qrx=SmJUsix(!#x*RK;{L{Xfbw99D|6_#Rdtuz+;WX1mp;AtBInW`iQo^6-h0MUHD}8Su z#m<;>EyjySI7!FDCUwXsf+8et4PU!@$(bUjFqMsiu>suKoc8h(=}Y5}STABXuBp5? z4}Q(isq(B8Q0KgZID2vo1onBpO*&k2lCam|>3Og5iCt0rE2AqUeOl*&2PdO^>Z`&= zGV4R``*N%s?}hfg)JkWO)ZGM2u2=SUvhpOiIWQ2yIU1}7R$1y{Fc&nGU%K&(Mg2^Y z8;8*`aeY8smmZwv*aXfntqH$Q@bcCQo!`10Y?J?O18&UUq-Q3LOVO((Zyo}9)l^|~ zic2wsHtMFl(SFlMX>fX&u_`}To`?qRt=~26Mp$f-RH1l7s(uOWBy>hbw4J~+ZbdqM za<~B~uQ>{`*>VMmTkuPzG12tNZJoz@Ohf}UoY_&Y{3E$wfng{7#)}*4R@_om*;HOy zvM*1%)io(Mx`9~K{-~_!1iu|hbKyGrgJn}buln>m&eu{GPOKi{WL#J^t~dH0Y>o)M z#oN*awbNT~GhnD(BN=rL>w2+DkCSgVpgYz!h5&w4aqXz7EQ{8y8FgfsHm6^?#*=?_ zn+~}$)MQ4)M<6?m7A4tr-Ymev@kPqdpYx)Ek$kSrN)GRR9stBJrV4ATpp{Jd5~CpZ zNmzCDAMl}N3!l4p6L^g8%5e`p58XbUtaW_`Bi6FZCbV~2WNQh?&sV7p zbwQm)Vs{}6+3nUu7Lmhe`XLivz6|4zM|FUt=3!*gxoLry61nYy7(JScWMVK;d&&=qY+4_?iV91B7CgoZwwXk)W zy`U>$v39~tZlzDm9oE#APIW}L^_jOq9cJGq4^^`hMaIBj#wbvRE~Q+ zUgD?JME70_7QFf8a*UmG(lMEb{IDPXXwM*5DNW-Qdpbg16LlXiO%k1f|3y>siXrRV zA}-PODgN4HmFbq@si2+arYITkGp5788naa%BT0$#w3ehnkW%g#Sf9xh?c|BR9B)n; zKgd#ejmITL5~f#Dj%5gsB>!Y)iup5Vf2eXJi9|U%%115X^+LVpoIpqj@ncN2A4YTb z!)dAV0+9u(vzg@7Fg760oo1?}Y=8mItbr-we?_7N`;nqwmzfS+8-FuNsggwv#8l27 z^%vN0nO{ozU7)#q`X`KD`k5BH6VZsj`~l%KilvIl!hd~gGhH(N8`CZeUPk46^fZYn z9Z>{Zv~gC6Sx2ma7-%YCU}H$KAz_&&`l2~3@#1D@qHMjxXCqX4Q(!|8?yKhz;`h1% zpit#%Dx2D1*#DvOf~i!n$IC4p`L67QppMnv`@-*_z$VI`D95^D&N+@D@s)DbU`Vmv z6zHfUJ&6jSOxfdIeweT3*K%e`U<<7KfF&=|Os`_Km-E#Px&`>(_dxdF?<2?8>sEz> zy+BS4%HjT>$QPpG={<62ag*8{P2uh$RqDbUy<$_1Gx>4SWg1i$8MN7ASUv>8>oUz^ zdGrvC`e!0Q7^S>1vig#VQ2HAA+C_OS2<@br#_zG^lJmYM=}@PQ)~G^B*$!HM)>Rm7 z8uI@y#`CZ3FF+*q*)s=SC-sCg16^h6OEC0j+MhhQ&P*5A3_!=iYX^RQMktl(j{0&{ z``48{cuL=TAo07}{OI;6vb1^>--tuR7zVJHEsLFo1Z!oRm+hBn-QdA_8Sv~|${lX$ zJFs@OEk>KPfHe1;76Nn(VQs4XZN*(5ayqV4{+BO9my8Ox%1b2wCAX22#RGb_$K_$b zk9pLc<)$EQ@9M3#i}cEkrcDZZ8W(`|36DDwK%ynB@qqOTL`X*lgvp#|TaZxwdyO+r&Ta&$_kQ7KGn-gqF?Wg9 zq;!(V3ZzMi@`gc}#RRykdJOs>ibBQvBOk4zD%;AgB4(8Lh4j_STiS%RqB427)ZCi}?6tHs2q*4wUTGW3*?h z#MT(o?pCtxd7e4$5-RGK!JPHZZ=n5D9=BucahRMw!+Yi@nuStVhaCeMfcQ{;nY1Hg zD;o!J#7$&Wq}v>5?E5yb{=do^0wtz)715L0chofV(C>DNv*Hm%MUy!I74}W#Wj)cY zM}@PJgcpNyMkrq}vbWMM?{(na^hkDs_Z-pkg)Yk6ttaDzMe@pVnwC~SfcXOB(&!oB zH^%W=Wm$6$Hdqz5kc1%|W#j88fiX;CkV$7y>(TG<+Uk0dVYUi{OGg7#glf}e+9Mut zIN!9$KU(md?*sCv8jm819EcQ5xNhytFu4x(l%H#+y2ZPrP9p zSrmD^2@ZcjN3mPihORuy*GR5uf9L3W47JI+wdKQ^P;6bMZeotuY`Tf4qcmSk;4Gg3 z3G$d#coLbx$=XZ;aMyRCdXLQJeoV;=1N_C^h#VBQl}!zKn`1Y)j?zm)9-f694*cn? zlt>DfFeKPi^2%jX`?F=ehYiCyQe+X%dgrDMCovU$Go6aBY=z_6&XA}2WrpvzALmn()%Rt~# zrd-Qn>T{6%nGH}Fdd5r_1P<*_L9*h2GY-3ep>g>YJ_N3%C$T2*SG^oIy_O3SAII@H zMzypcVrw5DmYV$E^*bCixh^;vn_n)cZX#P=Yj9W3i?KHbwhX(Rp_|_p{Kfls$0kan zTD-3*3~p@Cc2Te9lcc6*9>HlgEd2F31QB?`%#}}TICG`WVo>(oPo`h0Jz=IApOvHn zO@Ps{7qno!0Ec`q6SyA#9;yfoH^Z0dIMaABm7D1X4I6w1Mu@(qX2s71t^k^@HiO~&PPVKUNeC_O$oQytBi=2BK zB(lt|JyI&SzN~fe>-S7sq~9VlS|`X_OXpS`@~YmXFbaXxO<9^romuF*Nva^87A?0) z8k5U+yoQiD6U{YJ^z;#gOkKI#39=rYz#oRM<9J34JW#w4fe;zWDOzWld9rXU-)N`e znu}{W90xF?Wpfr9zp={b!x;)Ow(pX``L~Oh?ZK442W1hem30H0AyRESuU=vRk7suW6UDmS~y$L{=WQwYz~H%xi%q z(iOcs{`&Gw7Izs!e4d(z@^GC`@e;=@Z%JcPpsu{%-_!%VnY)0k-v%MZDv0%enr=Ok zm@@iqSp8qzgk;`$z}FV5@(>BVi(zQ!%83mhI~hl+8>$(2(q3k>MM**#T_ipmm-|4E z`+lHG8{+9RmnYfPVy4CD`-T~ZI}*}NPCN!OXOnT0*MZqpTt*VtJCh`gv@a;PH#aIP zQiw(-QLB3prdxX6nJBJgcCkK*W(Y*l=5r8V)&iC%8lg25@>3s@-~K}GX_f6c#gr8Y8IMNuEU_$N}_-E{&Ft_ z2wne?i+#d$JNe}a45JO^uZ2p)- zZMnduYN*12vTH5>YN(^Zxk<>H$6d@q z9_6khdAT6xJ$mQy)sxywzbJHixhftc>yCITCB6cd>0c2-)Tz7`QS_vEN=CWgx>nrz zGDX?gn}zIucm*r%QURB2k&c{me9a9i+9JJ^Mgx}aKZ4RI-3{eel|7BS!Rhj^52@$4 z%Kw)~n>K0o6Bw~?fhBeYClM{=glBW`LF%(IOV;U{+#)sw zg|f5-_{Hr{mKWazXZ`kokex~VI1vrQp3yb4<~e?J@i@yQvmFZX!;9Fej_Rf!mp}}( z-czWhubD+455GlBjr_r+{FCAu3A)a&S15py#f|W0oyt`Y#e)LAq9YR3?{i6-zTYmHUFb(4J^jy2b`fa9uA(td}d-HaS0e<-hsNua1foaNpnA{%k9%^uihYpN za%~h1e9_DV*&(fD_e1_LLVE*^^g5Hbl)W46gG7oa>-80bzvHlf41k$_c$I&hUxC6p zc@3T+Bt1;Vf5|&uQLaeodjhnT{R0^1$C;^A`IL^3l%3eocD+Zb(?HjwIiXPwBfu+6DB ztj9x$jlI(R{bcY$DZNrcJjq*3`3KdqTggvV2~7$0K2L5{<{sCk1bvvY+`u za|OE-Bl>@D1%-0&6cbs;89VFD!l!>Ds$#X9b6*wJ24>|05v;Z_ovffp!-B?|@uJf$ z+jLdq($6@-o66N%UU(T%{z#XVbv+iePvYCaP%_?R?BWA6Ad7XpeDuqRpSCebxV+b# zl@g{aPu|(a=f>^eNA%Dv2Xtj`U<^iIz1#(wF;bz>!E_lC)}a*s`8`OELunZdbWUMV z!5s*1fDT3Bc+`>}^#EKH1&Pu?ryZ*j$25Y{$|UL}UuO?!Yd!C+qaKbjR_=X=w%Xr; zKbVRoq``XJ;`iM=Mf4@p&)+4q3Ezx@Fh`+Ygg z<)_C7=rhdvT!j@KXP*E?mEsYRpOAxkV9=FIT=URVm=H`=9x5-!Ab5Uh3^lq~+X{Y| zk=L39R`U2Xpsb;6*hi^mk-X#DXP1~0PTJBBaL{9n)}~L}+K2kWg4*)*4Iba|DnE*9 z2UQw3@{6Rl7T9auyoXVK{pD}H&a9T6Mf&w#SR<+!8}byV5X;j%u7t=V0+(L%nI|Cl zUD_-wc$yzagM$jt7&$zBG+*015GFXOjChICr(;K+VPuDFrnw4!E)NE@ME<8y zxed2tEdOsM;_l>C#9hg4r1U>3E>Y_1+K4xgacr8tyAxV5t&BF$>1_OzoQv7QnAAOb zLh25-v|$++caT@BI`iayBaaWKvEaX5A_EBCv!T-EF!j8VT!r?_sVP=pRt6De4FA@u zAY39}hfNB9Me+`5i_JA>B64g6bDZSmI|%3A3}m6XCNW47c7iYKXOR7#G(*mZnCWP5 z-(X1yB2f5D$ynpp*fOG6NSu9j0UPVHn&VysYD!&QAUX9fGYP=3+u?#X9G+AzJx;Z- z3m9XZ1(RS?P{RcYa`_i$hSc5y*ed8%Ku#2fGgc!*j?_mSw;JcM8VB>sp!xrvQJ%z9 zfIx`c`5T;`qo9bE_f%~ebe7L>uW-@`y42a_Ay2mfywIi5lwyv!XgNyDAi+LHvwW!1 znc|fX6^_#rI?n5+|FwVfYn*=EXwAbcbFT{s{Eg)CU^wNP-f#PGJ~n@h${mf!_SKOW zdcnT`EsieeoesIZA;=;tbZP57ZL(KarI9c)Q`hRJ1!M#% zZYkeO$laZn($0M`~$W84|KfQgZ{4yT|jCt%d(3xI~OvMn8e#5hDvTf0S$W3oABYUH- zKi5j@zV_u9buj>@Ef^!SGWkRJyr2vjc@n3Ejq+pVuug9sVL-i#WasNyP$`dbE#+kc z4yZvdE|ymTbhf}_$5XS`MC6w@#1;3~6wlje!`06q@>yxqov4_jm;CxWjF^?I6ZmDJ zWEIl~BLYhG>dKs2GHMC5dtNVGKJ)<;u(BsY@McfsT=Rk4_WCDgrSwff5nffL$w151 zMB7C5<*St(fGz}%LSTuX;gCHy(ab5i%WXR57_`^_0ve1*fS9J@poV^R!Gso z)01zFwIK=JI^fyfX%_2%e+#0Cfu304DZOOSG-TQt7Jx`-BPexuGN|3J<-%26W&EZd zT7RP-$7@UuT-q--?YQ06;S%!&;GV4t`bmQ)As?46lFC^Tl&(Hh7Nf-+(8v<#p48hT zT-J_6nrW>q`{tvLZ|CbFt;GrAXoov#)uiQCwu!Qm4N~$z`{j)AkC47U;e<864UF1` z-*qDvuFR01zCdM^5v`LUR9BK=aSZ07DI;aKuIZhBgp<@#De;u0r3{d_wqZbx_!}yl zuQC&5Nx^?EW>FJiwO14hspa=DSx&A~<#a@l4 zr0c;?`&G(VBE1Y?IH1bM7?YCSba;43#XKX|H!iB|gSWc*}JG@i*gWo?aZWA`w>G|>oZ zQpsk9WSMx7b0%0kh9q_1*eNFg-t`I>eF|kA=pM-(Da*g(=*#J?=B%zTNpQn|eM%hK zRfyrCN#VG{mZ5|%%7?;hx$PzVNhm+#Uk|mX2|)qrVRx3x%}&l-u^lv7vHZSAuU1Qq z;cINu`u934zKgmWjBZLA7HJ*cE)!}TqgqC?2f!+PZS7`>vnqC{6sr;HN| zSr2(t&sh9LPhbpH-g?sdsA-o#PyX<}UUV75Obm^X}uG9 zD@0Jpbn&#q7oGA2e;C`#3;>9v#xe<1$itsGNOzT}M?t$ikS=4oUtI_hHdMUgy=uv+ zh7O(d)x$DQH`nOFg`&a&${Y#%U@bE2U;Nt z$0?%@IgFx)(yoRxg@0KRWt;BCNK>V4$oOY=BWY@j(dx!W8AxeOQt9dO@?1B_m2uqN z7$-lA;D$f`06c(C^{`j*(_%co&O z210i=?lmcBXMB!a+^9+;WK00?6dr+?rml@+U^X`c5qu+&|L>?*u@6W? z2f0yBQZ&~Lk-24vyDdjB67Mn!Qsx$WtQ1@t|II<9i*uZ*C&HCm0entZTEYL9i#$VfXz{e;aafQ8USeg zRWP^yPK4+->N;H48T4yJA{G)m--Ge-Y(ub?4J#WYkzBj$^QfrrA;elAL$yCL;ZolH zgI|Agpqt0k1bHr1)LX}6>q&22Li)Lmo`g(;0#LtWBzdiqA>)mmdeS8OwwSilMKe%@ z+_=a`zEVlS06WEsUmmc__C(HH^*R*A0K1_`7k{K|$l>~L=ry%#l+jnBbfxAxmd!zp zz6P@4F&e*`-Wc4GvLk6z5nP}g6J_)q9(1+?#|eDRIbV3jnMd7EN-yL2G;Q?;5~b%< z6!#`Qt#`BvOoDp^<6GwB?25b465nHJYmAChq4t?YKa@E3<^C}W$ z>`habHM79>36MFVvtouYeP%G=t_scp+vhEW4QMV@i*^RIWL(iOKx{O5W zb_>W{Y`_6X#y7#RvZcFXbEeBI?f?Gv52B`4GiRoxHS}2JXB|$;J&Twg zrc$R%BOGJmvi?~7zqodiK<917#)kfS#EBXSciNfVW(R zXCvSZ_)Kn&#KPQNjEi-CJr?^hR{2P7Hz%{~e?YkWsMi|McOQ-#N^B>6KLDw}GjNzd zgQJyfoe#36UB(1^Rwp3dIOfbS=Er8rl&)NDKmeo9rDs4+h%E!km}l{XaExWynH;Hn zki2yWo(Lb#r^cog!(0LMqRO2JUl@(?P?&;!S0^O)<>zVr9xs2>U_?cl{l7I!u~PM} zX_JYJQ9vAp41~Y^9e}PQ#Q3M8OndBF(%O>=ar&LoIL7Ic=o0{FiXBF1^SN2T3-g@v z-FPg-Y3w(~aPrQkdps|mbb94*tUh*&KG_M2B0Fav1FB0f^;l9QYa(!vY`UAlMq#jg zp?y<2HkkXc8cJ=~+Ub?0x;x`oKNun5HlX=MRd&~rDVoOzL9XAv56r@H!1H8nL|oD- zz0~JHY;P3^MVyrPA;_CMUtG8Lxqt^b{^K8E@9)6%k(FP7zZI&O#uhzWqVID4ulfkn zCC1N)>&Oqlr29U)uK5lKw0DAu6&Qw4`$KtqU^jT{keIcFW20K*WK+B+4s$us1#||+ z%k5`*$ESJ*)*@zf@VUGW;`|lq5tE!EPvmg;!Dle%8Y)YqG>8Se3w}bwjLS}v=Z88i z5Tzr6P(^tokmp~7>ax_q$Il2Q9>Aost$J_6+uA4|*A*Z|RiJ>4Z%~R%25$oux;n_* z)?8&GUBe&^<;%v_W;`#X3s5Ns2 z$or;+){aaQ(UOw_trXd*zfFvW-u#_3E%t_tef6P!>tZq~eiBR6s2S-cesQk>UvnOs zMQ}j7W&c0qVZ}w*?~72U6}enGRTeBlOMEiJEW@)(iep?~(Fo)LqwPk#;4wIDUK507 zN^R+t408X~`zS}M93`EuVit!EHT~k$S;y{6m9aXw zNJ1X)|4f^Ymh0D?F7kVflC=@SmC7VDNtYY7U~>YbeE(C7J~oi`B%7Ps8ogs}Eg>J7 zNh+bcpxw#q@FSc+i$7H7$ATdH+{V6?B~6l%T086}z^0p86J#1=;qddz zlI1hT2+P?B7=MrUJQo$iuF1(oh-GgLX;KMl2CEz@(izTz>s*YpNid_Hxs*`z1gUwR;K#z0zBM+LqS_9XQ3uHOvgznjK_evSC3z zC_8D8Z2Ok;Jf(6R*i;tM8{3?5np2lSO~pOY@;TGEUubiX?r$jjx9@X+ucW20oIhNG zY6Gs5y4?AZGMqAWZB_xKXPGQBYvE)tCYUjDbhK%?9wdK1i|AOFjvVf$*CY+nOB=^2 zTT7YUmy>SOo&RAvVzraihY~*+X1ezrAHJ*t@%U}7VY2c|Q1V4GpS#P-75QPY$y6(q z(pGlr=>NCM+Dg)fnO-UU9;uOD6@igCiA%fYWI5!aAJI|grdbIo5m#k}R%?{Cy{T+} zia&fdlh4HkJaO{L$DD;;bSu}#^sJ*dXTS2TGfq||0rttEFygOOAfC+%Bm7;JHwMg< zHI|j(n8x&3hogZ~n!-6n6A`PGa<2qq_m8w3`Rxm$E+f-YWzKK}PkU9k5Rk>uI;i6a z$JU|+_m;%p)MZdABsrCXiV9Ia!DLE=`5>{EkEZKL5&;F!`mEKdvAjw<=-smi>2?sJ7 zH&?mm@VQbt6ZL46GhI+gTXM*W+;|I!C#blgvOdn4iM`+(q+8nv2e-r)Vr5>qcz!Sw zar+#LWbhCSoiBCkPr+*t!@i}^!KV5!$rLS|CTP1SL1heIe8o@XBDSubwr_Ck5dV>B7MY(0-`_2$#SVpMq zNkXmdRe_1dlfjzM5=SC6ePPM&!MMw^*EPlh||)T{ja(e~q~p z#=jMY%skE!NJ7=p1D8W&998JXsB8z3tfFPwMs>=Fm+(}ct;Hi=pzI5{7BqmNXz4hp z2b?!dE0(N}kgPTIa=o+k7RVnPK=`f~`6D~M$01pLEAl^NA)_lRMgE=5)m$iU++Z7e^I@ldq#o0%$& zC|kuXDDJ`+i`q#+sTl%M7T1wfaau7D%OgoH1Xf6u6>tg0b9-4+6}|MHvV==SA!Z?q zkz2+L>s!j-`+V5C3*CM&q_ z%+*ny5Q&YgpHW}>nNZUFt7zu-$Dtap#(a21fcg(Yz@v0^>qV9@ORdS65pPx$V8*zD zFwU5gj>1tgax8*t>kQB_Re5#mmoI|rSQf!bxlu+;b{;Xs)C7bQG1>{~kV^VT=>L|w z^ZprFQAYY;N~R7E6JsKW&3er&h1Ss^Jv;GgHsk@Av8Ejc@`aNAVyNtBmD5T7{Ln0r zyGMChBq`$J@2 zrWuEg6&Ge)tjNU@E^UY8FYP4zKY?oK<)|ZlE~3omu~kK4nf`!i^C=t#varG5I(@h& zWJ&fa<0(lUg)k3jNBJAXah8KRk^dI>?LHw?B<|qik7dHYY~*k(&3yRZvgu7uVZa)UW3?g+8gtUq1w@~ zHlpxry^=6nS(HrH+0R>4Aj$o!rjB71rpr6@&*2fWENy}+iq)ycq2fsey0NM}4#bou zq8M|Zk~NE*lrAy>p5fcD(fbvpa`{;^S4!_AEgj2E=K8sLdOwx11~_HfL_#=O-VzK_ zi;5c|MS7B1Tfb>TY87Wu`(^7T{>zBMbUk6`%kCgHPuZW7Q~!f;|5e^L5^^7#>gOhf zF_?LT%2JxhodV8E=F5s!pbVhFNl=`qw;_{-PC?&hzJwIO*Cu~FfanbyimyF}@27r- zB+b>+hb9MkbXTimJW3*Gf^2KJ&&qrZOB!`pmT@v_6cqP*6jYI2k|euEo3v!>@nhfG zO+TcN9wV9LGL0`Q%B4#bPdmB>j*HATbC8uqz2uE#Ohu9egg}z5@}eD$_~dvnw)9PC zJyhkj(OqY2P(_IoWZ(wG=UC-^7NG{mv3HXZ*!h(8WM%1?TzNDBhSc_G+K{FQD=N*X zCZ4Uw-z4gBk`6SsZKgEQ7FZ`$ut>h_=Zur9uRw5Qc{+&y7^j=N6|VnaB3C-okAI!m ziAuQe351@nibF9b7!$Gc4`d8WC1=Zu`euQ4nR`mcubx69mkSKvgO01A{5D!4a@hf| z`yKzA3Vwb%U?(mdMi`Keiycc*!AjrpHx?_%s7?DU5*q0&mu-xD;CK$JT&V}4bexmr zk~ip7!MYAuNB~%-O6(Ps6-5t~#3_m!64+xc!PP@NK3ch7h6)19sKdzXili(GmO;Tt zu+scO9U5ca&j{>4F`I$s_ipLT9$nC*Nk~+z`YjDN8Rd5zt{N%LDsXO?= z)3qwA)(KEws*-x=P#p}1D1x{v!-yUvO+{AVJ&-(DN?}mm)5zIEJ8{~O4zlI~ zjO4t9JwlZL1ErXs1g|2PS-{wgZ@@VZnC~ccdP5Eb21gRPV=YR<3M$7OfIIO;kDD}e zQYccZxcmQ0d~zG3 zQd)9(TG<-p(Yv9D%G5*>?)uZJ?TPCkW!R{w?&Fb&5PG1y?ZIeK@X~(4TFVWpPW98TtKKijrttUl? zpz{(vg#Ia2HWD(Tzq0y0Fm^{3>o8G)UWWx$E!5eL)N0U28@+tBRjgG9F!f^kf-w2( z6%QQW%S@C@TTQ#Xq)TfLs1zkb+c}99{^HbIp5po%Gn3Q7UttTFt+*M~YLAc%JtlmG z##pk-Tq+l51EF#YB74S8IO|vCsG+^1VBQ~6Lw@?&%+v~A7cJXH5dW&Q87jD3X7zx( zXD^1@tLQ-dh#ycJEQ`~D-tsVv0mAeTbj+!xDtk^l(O7W-s;(<+Mx&AxOroTz(qs`v zL4OtKOb#j2itS=)YMe{M5)JbBjra140AAj3`S!b@9GxkOGA5)|!c|zKlUdm(C+j)A z5E)U%cGb*8PC@RvuS%T))wre-&SYZAoe2;NoBkp&81tg!)NDM2_1#b_qm@lBd?>09 z*QGp#OKNF{eV*Hp2o4Zv80r)x`_D2)GFo$xAybh%7S{a92cKfxR;bJ{h$^v#(LP%n zqmFPOT=JHQWhfUZZOVuJjYvV4Hz#OG0KD(Z zW{Pw#L@mDkq0@>Sq+Djq_fXhm;?H{aq#ZmF{VBYOEPd{KMCBdqjryN}rfvFVbga~p zEq@{p7OJv^#=_uYS)_YnU-&f0PZ~CFA_54EbnFA!J);WiW0tuS7`+8o)Azekdnx3i ze+JC2lmzr9w=#J@2etHWEPXn~NV6in%7q%C5DY)Z)6)Jr{~E4Rz2LmCo#g(F1P`hQ zXI_s$`HkDd0a-IPm)H1~%ll7b8(vZ&L~P=niHONW3S--XiZE5W5IXg?kR2;MLVdpulD|ZwksNgwNT5eAx!>mKcgC15;tn5=|Fe45LwU+jhyP3f3v9iWU zokJ+i4Lh;6BbDDFP0jiat!|138yT^sWIlbA?59WKBUSb{x(26004}{tCaJk)wY7S` zC5Y*=uCj6t2wG8*2x%6Mli6uFfB))R|DtjhQ|)1UPP(iHE8nTy7(5-{7OXRd9v$#v zRSa*>1g{v}+Y`RB+Wnp zVQfS+NZqCfvSupVGNk{i z8o5r}2Hl1kjh$(WupBvEvzkd}I)~nZnyCKa%9SJ|X!w=rDoB{cdfd$4dOc^ca&?!D zr;)d>DocVKe+rSn>Z`_b`q^r9UKH*Na$(W3XEeAW5m4)o(B#wWA=rHzos_|MGRutS z**2a+qtA&!@YzLGIaV4j#jkJl4)nGoE#yCQ*I1HolLG;spzLRly|Pq7p8v^sM_)$5F*ET|`tpuiWjl zsX0nI?|}@u>YDhcKQaB#0a`0o6%04luSY;0NQV&@chjP^Z2GCW_%Nj)!BK}K$|(z ztF=>GI6gf@$IqmQ!%mBivec6yw3Hgl^OImoqrLHYb`H)3{On4L(}n-taGT2eF%T@3 z$FqP5eq}n_bqva`^q3M?D4u{lo>2=;liGy3YtG$T#r2V#RS58$w$R2BH_WkeII!=d z*kiL^$8&mzfv10hz)qR+w#JStt$~Q5t9!~$jNpgQdc3&tWdn#$xrXSSJ`p1Ov1{fx z=ct2OHz%)0I}0Vhu33&Rhe>zRfW^n?4tVQiXAuE=e-ka^y1;0*1kCKhaTpS*78&e! z>h3<)BvU9jM4DYds=bbdD%HIxJQ)CpDAt`UqwjL?;IE((3zMx82#Jxn6gd$(oNmL+ z&9xc#D>99paNBo!ge>ua$x^o^7M5MP2FRetkVJQtx*V~dp3aCbRLwN%?Bx1ukY-s~ zkk<$=Da6_G&5`Sk`C}txYlm^_Y9&YOpgv|tV@o8R=5h4cHBiTUW1M9G@@Mqmf+NqH zR--ezy>9i-|j{r18k4ydE&EyFLD2 zYHj)P9j^I~Dq1QNf>8sk`)ewt-ysFlRcRNAe7C;Kg19X0CMWd_gY%s{2IL%Q0MQht zi02J-$O66BAr2lUK!Vnvd&cyMt<2O0Es^q2o$HDx7lMFHwNt+8?k<ypKw%`S;oo%1qPcaZ#Qht z;skKX?8}i9^izvtjh5fjon_cT-e7q#4L_${>nw-N`v-F;RHZE7N8VPWKO8k>9U%Ez z#ieUI)&X#6BKrF;Pk~H%p0G#VgIws@<7f?~J0Pm+9nTB1`M;q}OjgNY{?F9OzprOv zJpJbyfqoZkSr&$bw|BRoqheJ*lQn%BM9Yg8?&99b7=u;Z^0Y9kzHFuq!XMcRoXf$}!By0aZ-_)&-DhO!G;_SOP5WK%=+8VH}R zeru1!EME*8;##p=T)QpQ-JYw24UedR3+px=flWf7hl+b$ZeBt5&$-Qc9*jd~9a>}h zNC>cLd7Y~4$i>MD)=~D+@>IF0qa^OuFd`$qGAQVfgY>5&uJU>kn!!`=c!Q(N%J)hP z`2r%Tu4gIhu0-Rcw!TM&SMxOR{y1JBN0Q=&cuQD;?AUz*p`339P){76YaUYKj9O0&)$3t?s>SZXgt z#?IT%;l^)3vVVIq=p(swnTuUjMy^~s?SwvA+b&nSKwpkLsBfFf?T4f;tfezVweT){ zv2rRM+Whu9PY&q5{`YlpQ&n2l(B*M;#JAPV;lrg~GIRxVv|W>keoooKla0o*dq4i^ zsVENncA8E5vIFIP-O6w*IjB-XgMs!%<(hyQdP)$yYaU z_u`dxF48uqgY5Z?SI>G3`v&rGiYKa~8X%vom(eOQO57WrcA20Hycg=F4}}4o^n)t3 zugpr|+8bi?Wb;F(RXnuMBk*jEwTl^n+iOdcW93MQUp|FVHdCx36Xs%-^i!!@P!p~i z@|hjc{&PFDCt2HU8KP@UULlYn?c)fOS75UAd6IMlnRGP@ zwhW6#sC}3Uwk9d}i(I6m`G{@1F($1SvZ_dDE^?y8OZras%tcmp(A3;zcu=km z@APIQz3xo##mJp5v8>DEAUjpkT)9|`?tc2stV}8U5wTFAY^YdM^5D&bP zS;yzHa%A^RNRHfTAN+}sjS{;Ct&i2wK^$o~*jKkBziFpRk_{V!SiAEjK9V^Asr}$@ zbas&T?SEs4VUW=LW!$RIJ^n3nA& zE@x|jFbXu3);9d=)}e5(uLn0SiY?TtJVK5v@RaM2UTdRqiB*1hv@ZwUpXxTE^DAX| z2?vay&gEWP1`}LV#cd$dGObMwy*@i{AU~4Y;RzMet*3H&O51O-qLg+45~^T3{J=C; ziP?dMs~gE}(kg7)&+09@$);5gr^dlN?B{r{jJKIVVup*&~aHCG?GpSXcq9H2)m!G zn+a{NXuxwH1tYA}3S`Z+pg8RZ58#lQqoq#}M0rhlhsxPEJzm+o!Azu@%sR@LQ<1}7 zKAf$$RAN7ZAcZ^RU5}Y48|#7v9FrDMVOhK^dmAX-)14XP32jPPe~(o<>LUGTYQu*~ zs|t@q#82)qNQ1(K`veuggqb%)d)zSamvN zJiab1RqZRvutP$|$%D_31H>P?%ZyGY<1KGMR#Q~EPA(CORm!<`7!Kx=FlyF;gjp4SUc?6QjipSa+;b$Jw$d)f*pWs%C+3|t947LPG)SbFZM zaE7-(hM%J?F*RnE+z$=%Nwar=!3U~1i5Q(xTi)7>n*B#vLgm-qJUN^q4+J^uZ-(ji zjan+FA68n%C}=3X9iEPTkbF3bZ@j78YvtW#=#Fk$;NMb#;qq7)xmb0*QY?efsaq-T z1N(%G2d3oRZz@}^bWqRp#1L#JWq$3&QmUwi>09IoQRk{YNsI)eIM@ld! z;3Eu^(>t5X_4HFLW9APBtC_d}Yie@Sx6-u3L7pI7hbU(hm^6si&vFxgA%Emo*2dKN z+MH7Tq3I@0lNSu*Gl0;A$**lqwzj5YI|`E+(v}-x+>Gv z`}iCTgRO<}lA9;%8hJ9L_@I=sLa|%aI9_&sdVCS z#-$*M+)X%yr58}lnZh|O>__a68n!V7QhFr>xs8l$pTsQkB*`N^$#{AWj5}F{j4;a5 zD&eg17^5gSlfZ6B6Qf6oSJt*de5}{iF->*;|D0}}dsBIvgT5q)L`a6XEP3)M)3nJ)W!!BcMVflF0NOj(s^vl-$SSX&6gNbojFCx9>|#x4DHKyM1H&0m zGx|rD2hw!gd55!97Cz;4;5a%)$Q5Q!670!z8;SWgq*G=(d5t37K0@X8IhFpBjI(DM z*w*PWI5*S`3|YcOYU-AhmY1Bm@F+J~F0h;xDO=eP`Ir+#=0C#%-&E>Z!R`ZqkF5Dw zb{-+DT(c3_MXA#HYf%4#s-(Vg2a>3_vSg!31A5SkYaynF(G;8%){wF_5W!46w{pJD zLs4ix)re|h?P2JN_|7lPLRtJYbzTj8K^f9_C(P_pw*KU{a+=7wIMXY)JD6qq8f%H! zR5s?yg~V2ZJ%xOv>>!XzoT^jLL(#uRKWwASa0y=vgS~eN>nTqajU}s4)|Su3b%S$> z7F5qd1k;P!7%7BUM)@f0**K@{pf$uu&(D)%y5BajIfu$WVCu?Ex=^p_I2;{!4T!c= z8Sy1^1mx2RNXG*zX)hKp7S=bKM|(FZ>Y#gbyWGRp_NCQ|(MQD?eNvbn|jH(pF@ z6-Ypn6tqVvy86jmE1ZTr^5eTWXt;xYD&Im&mnRV;Djo$(l_tmpio=aTY0P+j=rjm@ zTc0IFWNEKmk2j=X6yUm88YSA2kh!8y3XMQssab}W?Q87Or|-)tG>wBqObyCe1- zxw50XnJ)XIFtoA;nm&1^oO>ivmI|HkTaGP;iB1(xJz2R7{*G0d@v?I`X!x_~f7W3L z6TI1EX=&WPX0X>HQ^E+(in81^EeE=CQ?6oH@WFGaLfU8$e*XFjB64QSG3`2;T&u zt}e-xYEH9=6cTk^@hYSzQIFJjaG)|fsZVZY`S7NhiO7y?AUnSJKa$QnFy{Mx;2%8UE;24g5+6Z~kPpJyXP7u?sZlYTQW`X*rIi*HYJ@fwRK;q4ulM)YKdy53`Mk&TJnv_{pTWvs z=$0)qA}lc&e^y3ydC(EWWVV8XY|-tVWZzjJ)FT~DZvJS#>qs?FIBZ;id^!Q7E>}^4 zTXow!Y^S(Uuxd!KuaZ*p^;FfKgY>q8dj4V=YXKq8RKeV)DvP8=VLk$)*qf13tlF4# zRoo)hkVBti!!;P_Bu3r?VT`(CB;!Li4aUoqQd<5tPL^4F))V~-irJf-I%9F+;uU-OI#A5^5ucCa9>!E$5 z%T(y>;61Yq>Q462c1CFM%FHPQ^O{Dhqn%xzE4reO2(;yn&+Y)%?Vh1FYp zf}ByHi_%zS8pC{6=*6kZ_uT+#Z>cN?*GGrS&n>`hkgko8_Ge)BFa9*#5RA>x@**f~ zT$3F>)HBE6e~ojpo8nzEvjt4;s%}~amBdz4o`Y4=aVb)$j-D|RUDLK+^NEZCCD}pB zQeU77V}BxB3L(Gg9>fF*FJ-YmwAzTR&>oUr-N;6bB8O)3afgI?SR$6@d0wg>T>>XeX}MW(OutaDd!YMRqou zoyB;eg4I1q8rN6LxxRisnJETvaf3gF$~6u4@uykO$H!9!!8`Iaf+efH(;{2P67w@w zNzhOyWR6V}q>`l~5I{xr=xfe%ot;p|7kWZJ*qXy52mg(Z*R-mkW;)TQp_B(#kbrfJ z4igF10&)uE#yV&Z+j*U|qMzx&i53Q}hB4iil?sU@!$}ff4E|DCHXjdRAMlWuZySNw-L|X`Dm~l=Q&^ z86S-WlR1pYJ}$n-MY?enQ38PKRg;t{+YKIKOA>#x;Xud9s>n9A7$}z@v+UZS_>i8w z9*W8T9}LH6CDU!Zny`#hA5PqEbA!aTW~e;p=!qdg4d^Yw>By^$Kc%=nXCk|qy7`!) z)lMqr{KWd=d?N|)NnO0E*Lpw-DPs(pHihx839F4X&J_AgEj_=i92)~GJfYj7L2>a2 zPJsO)ez%k=czWEi^bOBYm(q<@`mqP1)|yKV)#|{i980su6W?{zVm4{7eJSl04R@G~ zXF{@`IJ0Dy!2wB#0`_IG-o^_vLIFLx1%lE?R1Y+Z?z{5y*ptRw2@g`J}HKO zP12L=qb^WMVoFbp?d%@`%ohBfDhxkfGRvB|1~f4bYFu^16xmCqCLwHx>v7%T{rR)x zuNjGM8N3MO<|{9)Us4?59vKo=o2R}|sUNQUP+ns_brBXGS*Ul@lU3Oem5r0z3Liyi zPfi2DL^;X;E~&0RZ;(4EUH1FsBC z$_)dsi`C_gQ_q+Sx6J)k_C;})r_kt=TczMIs5^5G1Ei^&4T~Y6$v-U=k&e-^;%$m~Ou&FjIe}&xS_3;Ra}*H;nKRJ&!W+xLL58Y> z3)8%>JJxfai*1lvayto{jYTg}5aUEWtr`a>;i~+0(6GzyEijlZx^+0`lEAS9-SdG8 zr0b6Sa(87m$5aLgK7qdB2v?kV1mu_8RkoaWmXN1bm}sDGA0QKzr|U~Sxh;&@9tTG9 z+sXk0`@Jt+(y*&YJUHzWt=vYIB+P!zM@AgnW0Sd<04NNYgb7?1p!6rS#j-EXgm0ip zkk1t^S<1@oUH-nEY#zAs3^oGQhGNE&=;$b zWz+P8a!BUT>8M?nNR^r)ogXI_$&>&vGF&h2Cm+U|v&5xpqTU?uB(sGpF-RAWN69K! zZclmx?yr=)ve{g70;6Y#ppb=jRN*%|EQ5>tV|r1%_^boX3zl)bsc%yO2^1fsL`gLi z;W%|Ikl8@0cLxeDBV2Olh1tT9%b3GC2SiKt;PPM;y=`GG%WiUvL%!0NZVeUCfOg)8 zvyg4q1P*U0hmJ&W$MY+k0ul}!I5FJo0V299Y0Bf^f+^*IeeOpx(!yLVRmpQR`fH>6 ziE*-^uHj-`38~t&h-|L%6Pof7RhDs0({LWgFOa-HqC=rlya(I#qDW@7G?$i3ov})T zD96FFwNvtAq_}KsbA;}w%4-=xGPI#VQN>Wy^$U9JIHiCGNK$LgjMpK<#!VX9CYP6f zxDD9F5j}jWn!G1`hyk7}wP0ukZYo;L10iPLG!B`k9s(uR%8y9U$0txNU;hn_h3N&g zv29}e$mDi`j5k|PZHf+=;F1b|K^cSph{)GI;~>l?S{kP($WPIu5^;xE2Fm3R4G)S) zaSL!B`nB9`l$a|AlbkNK`|ik6qVf=5RuJV%=Vs0`l9O$7$t7QKzMIQxsejhYv5O4H zkg}Q}h^$(WL0$w-8Jxn!istd-n`v4N(q=;T2(A#q)9;;)SPy@!ymh( zsE0D(2D8uws~dtntLN^2C6CW~G|TW9smqc~oOUGUfKTQMhGJ+x-&MbhCcZkUY0 zkafBT6i!M{mdj0eezfAs$Mihq#mw!`8nV;mWhWzC^G}bGc~y*p|0$es`q+i=mAvXG z(%}^e;C)HqYc;+^}U>X`}UK^Zynm1Hrw?l>4 zEG_FBRtz6k0Q!E$2eNuTsya~%e&V;7-4E^axLA6hHs@g#XKa)iGmTVWZF0->Q${{! zBKbgF=_V+XUIQh!FEzvT@JURO5F;dsx^Srmx7RxQ+8`oFUpd~AmEd_@sYF`~t!&lY zikb!lCaW8l@;DVBmF#ZNM(hb(AgMKFwYuN2>IK2F@CE>?K)n{3gOfteC_iIEJ)@d7 zHz!5X_Q5i9^yJ#w=MffEt*B1YV=;>Mq%G)_)6;3W*BGI!N#Ff;+mwG>H7*cvots{ zUq(KGpuA&TXxl|0fG}$oN^HR}*?0`N6|FSj=C7fu}6%*(u-^29kX_n!)I80OFeuA zF1GA5sAb7ZphU}AEh#Bq;#?ea%5OViAQpez4Au~lG;^HvDKcD0?c&Dr_ln0U96x)2 zoNfbVGu1se*XylisdBd6B*a8kcY=%1G>On!6}Hkzw_aDAzCMg0tEJpo52x#`QvbZT z!oWZd9U`l%vJHPdqbd|^ZD1NWv~-teP+nGlhITBIhg}jABryO}VC`LCovVb~oFtbN z{0b3l(&OV`Qn`eJSkbfQ?l+XLGI*hM+K5QJ8OX@ldT>wKP{-_n9}wkuV+*{fs~-Ce z&KttTo_~O`GHwN8vZkKa6y%ftl;kSjU4 z()C9`FfKApmVX*eI7t$aS``Kk2kPN%wMM=fXw7yXZ3oAwhQi>rX`Hwz&7E0VN~EfL ziu8a54K}@h;un@6gKop5$0>8{o^DccKbD<;+x_Z>2=Jc zRVod_k15Ah*hZ*}J5Ya^sl^8tV;V-VM*d87D^(jub!a;Yc*N1@kzRxbm|qRM!LuH| zX6u3Shf9?U|9awm1pZoAFY2XLPs&!oA}&Nlgh{vi$>-2gt=21-q*y%bWvxf%uZ%s+Hl!nVv+5LK;UXLc_gt7?#hxHW7uSww#SB+ zKzh<0-O~kNIZlEl+gZM|-NH}BH7C&{Un6PgCa_P}wyTTC^U^JkdH&E{ehAmt>Qb#? zv<=yo))-Fa2!^8bdTakCsme(C7M4hL%@Y~0+6RRhnx38~VX9l(m}{ukq`{4o%G;xh zWnV5V$uX8Ach1`gmIY6YWaL?qIv$oF>GB@L66F)2?$RoY*GEndk}c)1d_Nw_t2Y%B zp*4v%;DuPXon+f^{DT`n4@Rac_A&{cofr#PNgbygR48{|33cU*>TvYex>iws(VSGj zYM+=QICVl%4)VgJWv)3E8JFBm3yR4^X|`46wAni=PtQOOY4SvAK9U*VbB!flSHb)& z#qsJmct2jxo-SvqnX{4Y1>+S{?S=Qi-WBD#ptCe@X|^#1^%~r>7yxymE$MVwlE2Te zDYYj;HivLHC=Q|WH{*TvI}0r$xWQon{E$7rY2aw-cct0#1je@;06=SV&;TgSkn z(Nk5)Wh!$1kO{M2znDFdabUxlAGM6*kz5ZGkmMcejU=O$+YZYEh2F|ap^MPte^54S zb%ck*A+x_n?>jawIsv|s@x6+*t0C9IoOU@q8t@EPcYt?vd$gQyf6ZE`!7@ScJh|@^WdPJm3c2}oI7Tco> zKpT9sogUYZOAFI<2ruqa*ea3VomJ7HNZnhY~4j2@;gTMm4b|x+v%O>wgd}y{I#@f;aw~j^(3{WzUinJTZ;2DA_%jLr7{g-T zA$%kR8+U^XkZbd&Zi|q|_spqMextDst2`@~Umf^8Z#iw46YfLusU08>(vz2AtY$V8 z>c>XO;6p4pNO#PIoa6mC%AP4whjy>xEJpe~Q)+(=J!QI5ww+R|CH&w+)x6}BAyCYd z8=Sju^|T+rdYnJ@R2C($cL4PEs>4i+~Y+{hBCzuAwR%M%PXxr4rMo#VgOx zD1KBOPGjQbRWJZ3p*waTmm8`7GqE=4jgVn~sQL;9oVdq|wWL1&-A4+`?ontY{ytUAMO!+ z;kH=isV~Q!k3B;|Y8wTnSBFZE-6qR$HX<96hs*N@Y>G7G+Gr|Tf6iO52X~CAuFAa9 z#S;vqo<`>5!FK^@PX`GZj6s#s2)*fo3Lb2%C(M^}7r<(Q%4~`=8E!}*e}7k! zLG!P2diH$TSqloUp9n?lwQ+2|(o@1wusr24wvh|1i&543sS9B;w@lce*0Wljs=-^e z_k16tTe?cmdC_UwxEMDtx!jPfqoQ1#gZub~?wKJODt}_8p4vi;&zugaOZ+P)Xi6{i zz?6aVK`ACYt#YO)t=&~x{>9i33LOe4aZfsd7+4VVU=M;mbl#vAv&!r#gdK;yV zhRc6OqDlU_fS`G$nr6I6F;evZUGIh;1nB!E~WR z)ywviqF=%EXx;WL?u9r9AMSD}&ndN#mfC%8{N4T_BXc>j7~cUsgU+GZx@yRo)$A{k zUM(`?5k}9aGt9->7tyfegwH=PX2>mDxGq zK2a~)jYd`c5%a}Sta~@7m#(eT<@9nRK?3W;1qbEi0h;&@97WvO3kh5*I=DO*$TwQ8JaSN4}z^TfwgvWu4`3`#AHra64aLc|X=!5h$ zt1bIl81b-wOAmDCuT+I^g)xhayJqq8%RtK^Kl9N2!}`4kc+DOrsqDpEBaL~{cBk8 z$=j!M)UExQ2k+q${N7@Hh|6jBMHs|Xl)9K$W)po1#R}bN&D*i~55fKwStsxijhbrl!LoE zYFb&}g2B1EWeyp9Z25+s;ed?8L*xi`aKKD)AUc#~4AL|<5_`_wR_l?ZAu;|J|6gQC z2pz)k^aY^y#Fy7bqr-kZmC&fOFqPz8taUl8d%{+#5C<_sVueUtp zS1<}Fq*axJ+mP!iL*YwTmA()*FFF;s6orV_svyan9C|=pF{m8>wX;~7T*3xR7z$vx zPOb=3ww;#vo<_Lzua{_tr)IU0^8uiyiVAJKsN!vipy;Gd^Z+DsLKhgQyS03+qUKLb zgS8J(X-@>qgh`*f9HNGL+88vq_z$$Lk<>9$`M+Zf!yy&N0r6xj?20;xvGUbiB!Ao_ zNN+(-E}plPM6jRM294%_9(&UjnOBI`Q75JtrbH=%4NF*XCzU6+S=}pD())xlczy*L zQJQ0Qw4)IvzIz@!z?N$-RGu{t>d9J-mK^m9k|leQG^hk)CMb{?yb5~?K33cT7ZAx0>z+kB#Pa~6#|D`!dCl;bkmR3+E7=i*C0uk( zcGZi{lg;ZARRJcd3X_N52&AOcmeq5ik8XO@W{wVWr>bHRcVj3o91|;lt7gChxE4Uf zJED{heMo-m!?=4?)gB63P6Ga^29!l?M#eOF=GIM4lYB7RTwvN=%_pppx*2Gu&C79P zNryI)D!E?OOj>SIgr0#YYubL zRzql(uh5I)c1ra>A&P+$oGzKu1CV)9K2RTYLD*yL1X--Q;4V<{c@tGtw!ieergG8G znSfPkZ;A0~tt)TZI14Zo2~2&I6pYfJHB4RYw!`?=YADG(2Ii0*2*ZKm##OagB5d;A zMT}|xK!oBhJ#-cFAx&kq71YpMR4{417b0B>B9-?LO79z^f}t~xX{RHyIclB(DF3cy za6?;t9M&0ClHFXg@(q-+Wf*IwD_!Y z7%AL14c-%-A*=p#=85lCr&YB{O6CM)M$5@sn83CMiD|$le-9d>r$5>Smo@b9_y8($ zhrs;p<*>7Gp3T?YW0n2nhx$c;e9%fGPM%jmAl{p0P)c$inC;-t&EL}ZK>h75SZ}=U zcB2*KZVLI@#Oo=ikjk0tW$z5hPnJ)A* zc?Fw}PWSRMNqba1Q@|Rtm&Nq%T}dyKY0h%tQ$F?Pb81O{KAW0-d9ZOi&c#7<9h25# zso7>KIjyW$sH{9-cA3iVDdA<2ZGUkOzeuHi6}K?Alx|CysP4YfUg4&<0i}4q)jj0V zyf&zCtAoro*^$Rt8!?4{1vbVCvk)t4syW=pi@dvCUaWS~QBLbd70BHA2=7!5M6G=6 zz*bFJ!s91G`C;!f9+}|gBWVh}wN*MI)9(q=;CUmS>s{HP^Ylz6hzjS|)hGEOrx^r! zkYH4~mu8OFel-_d3eh+=IGFgAEr=6RsPv$&X(BX_AI9C_G-Q9x90PhWTiL~cp!)$d$bdA~N)6s4NXu5X7d-xA@F z;Qp+0VjH$s@*Acd9Lh?PRCSEtV#YMK0z;STV-qBNm(vJ$*^%cgYEB8mB$l^MP4>^k zVn3}yXC@|@#Je7t^c@L6Kkmku%P59#dOoIv*KCcUi7%KI`tG7wp-W3JT78xZj2d_0f&-y5lZxVl1Hu9{&TKbLyBJW7j zl5%H2@lxfE{jnK{-S-J_IB}ZA{M;7O_4$+&RM3%;Z#8FG?Y8sAnH4qi(lJBU$E_2k`>UxCE67oD_cp2EB$Ot~nT zr8MvbUr%}B;2d+bTzkTg&K7*s_hRieGWh62hKcmbnJnx^b%XenOmIEl1_Ym` z@W>vFXVSUN*HKdDa~ijd>F?;gOmEQF8WmoVh+ijc;Qbe0r(lgAvTS;O@NHu3E-#*7)s`j>TQV~aL1+#hCO zG^h(xV=dbQ+oi);oShj(=Qor zUJG8>5{V=|6q7BNCi8vaBLETmj%Pa_M?GtnL%d8CgN}_gnY5FMm$v@H6OE_w?5eL> zSH-VU_c9~MPuK%QyCa9CMk|JWe;cm;oo|us`N^Pzp*mI1jzXPHzGDe}*0Q&z35H!9 zw4stNSJ?jA2K?)jT*D@FyczfW6ZNk#0O#ylHhu6i^UHm3{Q#Pj9~K%G`SUn~-X=Lv zZYMEl+9RL_jk=^;IU`l-UIGcLxEq3?nfJ!L?tRdt^Gtr_F}62fbfUNY!o4Tv{rz(lhv z^5J6wG%;kSNx&>1SX`K(%`K!sVo4)d3NrY5NB}P_R&5DxcV&rRd00;13%KRERYt6o z`vx#v#6u)^R~d2AcsSF<4d%J&iwvt)6q75payb1>=s+TlVpg+c5R-;<1PUQNc&jcx zH3?q{t7;x$m-kVO_Qzph(oGC9x8Qvu)w`5A=LGc z*xP`~JfED!o1ZAH)$xrpPS#X5vT16@Bg6(|%a=PKmrv^fp_!40(4#rUlG}j=u2S8o ze5^cnJs8-WR?8o)>adPhhi5?}BSV@!;km1s{934zw<%N}2Q=)emzhgiM#JpO{b~?( z`vohkuE&*=CCzxj_ZX;)Q|WpOHZXdx`n+#D0Z6B~`N14R#r)3D?GSYg(yl{@L1)w= z`WQCJjWtYC`yM1bG>!A$FFi#(1p3H~9H#eCn!=e{9HR&60O3snZ!NtGAcqcSImjNM zqh#ldR3QIdcV=z6k~hqo4AQ0-f~Gr4A*-~KhqI3H+<}A9u`&~dN}k^_VrAQ2SX8M? zyz$XC-rb`3-<{t4dHB77TD%90Z&ZmTG+V|S7?~deq{-g@I6PY)8*y^(Igf152bK#|>g$U!?2ND(ZwZZs z6#CBRW7U1WJH@i)S7j2;`h#CDV+G)HaRtfQ?^vj& zh+&HlgC=YI>yun2diysIH;d)9f7&o|rD435ejgHDX5l;kKl$!d2_D(7l|NtQvB-sW z0HM!dr(L$>1Gu>>c(_$f(6ap(wmLBo$3sp|vSbhoCBDl5-eKKcQHIeA$C;}o|RE~nb@{deV8Bt<>4FmX1rLZ2~%3`~$ zgT$k5@ko^$P|bg;sYq;2xJ3Ix2S1huH|LKSw9rZc3KOrg&+#JwK#3;Ivs>|QzrZ~< zsaPAluAa_!UzS$P=N?dHDexC1h|B4zX6f9RUmdf7(I+>-!}6W%`?AVq8QKrkkZT9p zueAfJp2`egFc&KNCETeh1i8w|uQv>p?t-ZDQIUXAzG>pF$+AmU;05OxptV#5WS4Bh z+~fAKl#eDDZkhI$8T}?8BgW?t7N)u@w|nSJlQA>c_Sy$PEAkgM@SUpRKMGAnjxJ}+ zC94>uMjNLltq%Z!Lu-KGFl8|Wl;Y#WIUcFikzKSn0xYU^WQyF9V6VeHK8AX+p3pOp z=+QseS? zjgc#Nlt>zUg^3z}!SFv^=k$G}3cZ&*&mw~s1KUF}RvL4%iW2L1E9Nl&s{R>oy;DuI zr>8LOoFfoNix^h6KZ!#+N98p&d~8si=?o`hTaHKjGCfoVA4PJLS4JP`xg6fI)A`e+ z+SDdiW5^1_pqcqxjmYUf4D@UeYpt~!$UetyFP3Hod_x6#CaHLz&Dl&%G4f!^r;CgX z8~`Xvxw#6Sr@Or+BZM7)F@$~23^$Tx#m}6T-&Fn6YyUt8Wz`+S;v7~SJ%!iWR7S9V zsCvGv>KkjZkhSOl4(c4Bu`4kcI!y03i3k z!zF$x&P;h6W;o zfgh?G$i6XFI3kO6sB%2Fy;MoL=WxwK-lLz^E}}?6vBBh&(xk?Q^29z>G;9#Q zK@}F{tF7ev=eerRQQjZuOj!&{esCJW@~vtM=hFjl`0C*nai;);x2mf8rk{alz8)H= zqERf8F$>ZO4FoYu2C}Bn)mX{hJ$%IWg<=+Pkt^l5uwzR^8aGUtK{DbwwEbKen_GW| zU|O4a;{1A$GUs0?-&sqIP z|H2>-^b?^WQ0pC)xYtG4j=MKD%P@n9U^r>?*bYb$m*8R6YWr{ z%PnQgj+q2{rk;k6+m^DB$ukTh;bS3)E-LQgnd-!uag74u)0czv7L#w}9YtDilfU0!IGkfMJrEH~8+oYJgG3!4D0` z8fo(SDN}AzPOVa^T5g!0g3+{t@fHl{wJ{_l$njv-m$H`cvZ!B4EOS9sNw{UUisKNY z6#NPo-&4sB%G-a2U4x>0Bwy*e#oWims>!x_iP-Ri|YHt)vztJpt zmLCHT*v$f7v|}$Nhj5ZkPcecC8lV8lPhtGIw>i+hr#os$+odpRb6{n7$><+NAxoI1$Cs6mbw-r5{TpfwUkWUy6oJTpszY0M@x>8VkXng5 zR%>N>pqyMMlzr8wCE)%_-7jd7JeSvnoe~}vCPAw>3NPvzu`y^MBvOOH%Y#8FiXQiqkxOEtV;6Gacb8Dz@ z=THeb?jH(S2A2p4%Vk*&}Fj|0+1Dcxmyu+8B00PBbfFxX$4KkM9NBiIyf?Qo7& zzL{pEX(M8y=*|p&&gAgnRD}mRHy0PwOM5xcwR|?4IMq3~jF_^8<_DsvGMQ*#I>C z3P7&mbjC~N#>nQ8IiZkaoQ2EH_&VHrrqOWOn>9_0VY1V9UTATJ1(4)zmOtAX9&yA2 ziZm7YRSWf8%ZSjV#~{w?hM7=TbK)kkAXv@?e4Xgd_G$zK6qMD|ieUKu4EE9BFdwJ?;F#7W{YYF7 zdH7g6Y$DPIYs#q6MuD{O=8@8SoK|kw)2hf=%8*LaW6(*>Xc&?2Fwl`)5)v7-%7~Zq z^*9()RbtED=Md$zq$0VY`hZwJ0l+CodA9l_*xN32ELCx$f@x6WQeKs0@dH%9b13WvFi#VY?+a0JvDGPK@fE>8U-Y!jro8DW)vG`H>s4E+M)ODNb zQ6Io`KQ3jw)&kAme$QIIHny6y@5UQef)P)&@%kw{CxB4dI+8jN zX>I6MLsIdG$j~DkC_2g2mIq4zx>K9u-#i4g4BKK9v&;e$dD+&gvdyDk z{8%)IoT?Q5rQM8d?QsM;Y)1&;iK=VY|13U>?;G*NkVqAWO&ZyTaQ<3i?q<@Qo%wOk zZAEIKtn*rXE^V8(aXJuj@qyav!YD02DN7FZges?eYoOn3WWA*J_=_bN&hXty$mkDj z7`b3I=V|Mc5(pRok?UK7!j>8EA%$@^d%ds6hRW%8Jkhr;`}b+hz;FL!+eadS zV$Y#Yhn(9FP47_GxF649vOZ(@A>HE)m#SZ(Z=H(BWdEVnrC%#%81@@4l%k!7Odz3B zE;J&3B14_%`(r^_y#>75C5UO)QiFr|HL3`IQwKJ45}_&8zTjb08E&5#%)RQ`Ns#L2knV5Vew3@DiJC%{LyJn zT~Kv{8-gj!yg`R+6zHkV#ZwhxTet{l{H4Or5~_nh`J6?ZdkZ8deF6|3+%{sR=U+!Uy0h*8=?=o9v|Hv;nP!McP%|Q&~9| z01uF4wFdja;lSW}O>|>Zv zbv?cXj60#8bidBhzg7W{wRJo0rcR7Zu!u`GM8Gj7jOO9pzG<>qwIiZ{Xo&n82Q?k2 z1UKE(0Y%lOntNpE*2bhJX|=^mDT6*1n1nS2aBlybZTC=R;|a_TlpPO1(@v^x%Z?@h zAngR}-qfE*>gv>*Sm&!cC}q(va3@nFy(P5&z3vH-CL=(iu3|Scbw?8!_zA%@$38C; z7AM`$p{CSn=nN((E<8*&TagMDl>_yU3OzMlf?94Q1A?ie;J&~5Ab#qo8-=Cn*4lwY z;oyXql`mI1En45gTv>90Ba=ok5c%~DZ&m-5BYZ`P(wFL_Wu0Sd%akK4{Bw@MSGuD< zAONB_su&h&y#Npf>J~$`PJ~GIrvsbM)xGvSRgJ}#!@bEnNb8YDX}6@fNW@9xT~cMh z@aYyFOKSmDH;V>vuFhd2Oq$moz&~y$^KMbD+>`f!e~lPTZoAA5Q}WXiUM=P2d&b%y zQ7j8oxFv-aE6I?WKyLL}cKh*2!=zoWnpt1trbqt#kgcp$nnLLxLCft>XR;ja={3cq zeej);CC`VL!_mqU8W4a_&>MKRO6AK88lFR4s!PD_Ns5t5pjp0IW~+>>fhuJpS=9vz z31prJIoxZ!W&=)~;vyB`GV}&9%a*BzRmN4}{l@ARIQk=~sX(>bTv63pQyUF2RMJBO zWS+&DC#~r&C3gZ~HGOqUOW9GD&0SR8lqj=T4`sf9(xd9S(5S^)sWx9;es~G#lD3>e zNM?9>ne;tu|K2`0jF(DT{TxiPYyRL@VeR8$PPLg5|TVPPe=s#Q6(~DIh5%*+4Z*D=#nqF+vIB%%uCU zFN`EfWn8AZfV1_#XqQCWpa*kAEDD{YifW&bCaqLx(A#3jVJjm1!EjI?wwtw)U)Mwu zIy33E1IXX>!VI}(H|PmdgT4Gri9D_gqw32xF3qtcRgf`d58xee5_I9As8X@`LqM1-x$WN99ar(X{DN2^Fj82wA>e3{Bky+V* zRP`z5ng)4huK_^) z@!%kyf=%QU;Q`!*+Da5Y^{lo=aQUZhhG%w1M%0c;lXXkcHV)~ohAQrWk|DmSlx+&F zE1_=^ahG>us=q&B*s3wWi{vJMIA&Z+sTpf{B(^i^1wD--q(d%Piq=!>qd2)MV@yPa z%R1$3imJzU0+YfebtpiHubSCTQw5}VQ4t!HyF%pDR8GwH>hQo%{G3rT=@cY3Kxr`} z8ltVVBlkw?hOpijVQiJF+e~@na;p5TOfxVwHIvP&QRSMdEP?P_;D4}+3JHUOVhN_y zq#qVkxkz`kllMQtK*_|bBxeSo&n{)_JN__Y<%&v=&%1!=o)wd<&56m7&C8)qnoBj7 zo--J19KE_^pbB;g35PtZsH%py3G$V(up=4u0YCh$QpLVcLN|@!5H%ygMh-#jT_;8u~h2&q$Vs$(-SzHzScgR2O|ERjI2tzDBqV z9*HvYv2N=iom6zv_~y~6s$52#mQskXY!mNm&0!(4mqD&kJ&gqIZDcaeI*(PleFZZb z>CcdzxGIw;e*Bmlk*jpM3~Bq!nFbiF{@RipD;_e0T|WoZ=58nNSE1N!1gGWgpsaF0#uo8;Pgu6xyLghjdS=? zahXzh=Ms&lc_fx($$$CRh#ZUDC%R14W@xCA46C4hIw<9#;s);6VD;;YK^Jkf<-}V} z+kYOz9rxlR`4fDKey7=b`2%cyu|JHADt?`0<`KR-*L07n%Uuvg;!{C@G=ByZZkJ(y zCr2^cd6jbAs|wUmCy<9{j^G3qriaR+v%u-|eb9C%J*t9uZ#Pn3<#~F7kNmq3 zs!PoVH{pdCJ40A;mGZ!7x2|>K8Tw_j!3PGo4L3#G3&uy>Km!0z8Lv2* zIUg{*iiiWN;f*k2;(}-(!C^zQYbDMP!QS2a(aF%2B^;v&X){EnVYPw($V3$?!?cxQ zM{&N=u$D6vwT)725x#Eu*TELy3YGsUMy#5c!@-dJnalFW5WPvevbGguk+KQ+ULDVrk5$p) z@ck_4zG^~KRGyKKG&T1T%SJ<$H_dBBCM{ZEik=(<=~3DGUYL>s5pEovsS&c&0cRYb zhjt=G%+*G`<{89oH-*z*dIr7&vf$g;mNm6WaHI}8896@Xfk!mKZAf`nWo*8@RuWO4 zwa=~&nSVXPh(eQ0h!C@81QUr*nelahm_t=|Q){4_+LefOjq-aARIR4J1Md8%(GIzw z?tiZ8wl?aFw$n7^sbwv@mJAmrFjz|Hdy%GEYi1mty2E&`S4 ze!?+}u(v5XST{Xbk|P1c8s!T5B@Tf7GmC#c`WY}iD-8v_@-@=PR^u?OeQ(6#;VLqt z^JP^d2{ywidGz#A>8OTetB~!w?U~h4$ZJ0|a;mZ(!L`Zg$KIZDJu*$VqD@zI=1SvK zhMWpWq{2xfMJoQtDjQO6MTT7i_oHsY*aG#qs^awzz+D{055IqlEIZwhqY-}yBtSQ9 zln1kpSSBD<8`j|sn<`V$L$`*>LXY69kM%K5w3U;$HqiVl{4aJ$RS?ff~X1;38h zi`t2OYP5>$3a>1Kv(Q@G;^bD<)9b5p7$((I6M=As9}3Jcp~{b)39|eY-;JxtHQQv; zIQ~Dc@DYu;K}~8!<*@0FKIv*8e~#q2(-oe8w{7yoFNQ=vh5c znNzCCi@I!bDQ1_%mNW|FKvTmcm5&45nAZju${!n<$_*7SQ>%leC>3$Pwu=!fwdz2j zyT|a|yk+n<8Rp%vT0%{7ay!p`T7p&WI>!o@9AwHDDq!>P zscgEfN?yEP&WO{R`+8(sI!EH^ME3j{l2aC`SkICV;UzJ+*2MoPtf47?N-DRMz>CtF zv_Whme-J~(hTOHlm7OTA_@o9eCGt3yxr zNJSkKc0I`(&z0K$E)#w2fL_!=0{(U8ayOUNgWF`Si@bhicyNihE5cW)OVYF-`wKY( zfb$muxS#brZv>jD3YkBy2<=2+0xmXmt8dh`z<`n%OT)L$acao-lBwS^N$;e79eTD97FTbf9(&H)yIjlX@^a`edcNq&_N#Cb|>13-!R^XGmuq7>_L!FF9Jdk1e}$zF&dyMYDG3)Mr-;060GI1s&k?1 zcB?E3F|twiV_lk^oo#=!$0U<;jATa3s3522a8k@AV|ujht)@g>DG$ng!jk9x!w&fKj9&i(S`a^ z()JEVuUD1mP-u}oWss&NU5NHXdvKLSwTLw%)r+dgfiiq&i^!IHF)*YW>WZc1d4nop zSUp$%1f@AKR=KLeH$v**S*rIM>fWcvHIeSBc=9fFecKAR7c$e@MBPwV(i-Lzi+@lw zZYz|Fxp&Nj=0-q*Bh$m>bqttWt;aXQB62j9Q_1k;t5naGeFWdJp74RFV+CRIle#H7 zM$=5v0CQ^*z31vF51(KS^-|uT`RSYiW13TWZ|uFf%_@q7t7JEjl~kN1&Oh45!Q`ee zco3ZGfgcpJZo4xD4nTdR=gMKYO|^+_09UzOKap0^HJPV`Dpz$47dwxowUQQIaL&9e zfcaX_YQW7G)YMWBrg6^|G^y_pvNA1zHHO=ON$D!^90Ht7*t)U)GT|q<{fWm&l$W}t zv-YL0NeWfy*ToiWzSvil;aRH zRK>7h$T)X^OjoX63?UroN@P~8;S7Zg=~10gj3Ri{*RIlneI(&6I6a`}m&4U!amdz- zMk=}jMK?eE-5E|oc5#0>s0CVmq+Uf1;ToI`B*ONTzDJ>;IyieE83>Qs4~4TOez%p3+!_8D~k|E9^32zLBRrj?5sSC)SRdOX1aMrKAngGkS{cXK;EUCs)2p zO(fXh0=v)BGb+PAU7f_M8gsc%LBSs7#oDi~B`fI3<>l5wx@bhnG_WAm{&A*A@nOEJUI!kcO>h>-4@sZljdFpYyMVun7NXp z2H01M1y2t(6R|oOq}^1V{}R;A{>ddqmb?u_q68yX`fh;%r6EK&`*<^pM6UkO>2JIIUa*=2egxmyA4FR>eqD_4`_j?-Q6` zdp&cY#BTs5)Jg~gya=B|dQo+`{5hsxfUdQWf?Q}LT#t(6meF2Imiz)Pd+Vl#6jaEp zgJs|fl~F4INUsMPuURJTT?P#DGS>v^mRaFF%BflgBhzDo@xy1elDH3? zG+-JFC#z3dymV{8G|M(RJ#ydDbmF`YqC%A3dVUyY_NRex^8%&W$0@}u9a{kWVoMct1(u8dLvNh43!>Ry-eDNVAvrr(hz_PC#WX0>J9JT5y8w3GkGPs)Oddznv;9 z0KcjIK%|=WmS=sCyjj|+Jm2uV3?d(byw)|@rv+m1a-}I>I zAT?Bw?A3VaVAFh*z%43Zte2jwioRf0-tErnM2~Hao&odQeud>xdZmgiYHHY}ayj^O zMYLpjc8QHrPM{{5U~W0IOk#-?ac~1C>f- zSBK~K0+x$A1S=N4Xtsmw&a#jvKV7FscHTySumPbB>{Zrs4 zxRvyN}c_c*7bjZ5T zSk!_T8!BvQGpVei*{x%M)2TJ)44E(;Uby=tNMEUI1C>Y^F8xWiG+z>?95XAwlkqv!kt%gbs=SZ z_@Smh0HJYR3#lC$ovXbr^hoeKM!N2s006T4$oWNxs(KL_obH4gn9dM**TX<;1}O%R z97WIC-e-s^G(V6N%hl0+!w*c!lreByDyGVmJ#oZa-9{Va)0Bc za0=&a{hR1;Y5jAwTZ(^(%+uOI=~g|n99net2jZuqKtnLcq1pUMSV?vl?S*eXCP}J4 zAolp-v~|MjRRnFG9?0ET%Q+C!q+W2U&R5ac>2f$kPDH`X2DsrM6q;y=*M-h1$4MiA z1`_D2s&t82dU1C|PlgE>0--Bk6voSHWg;y96Rz~UFjQ3rsV&8qpoJ^j;Bn`iaXcPIXE3;DVI1Ft+U)Os1n!rbRNBN^K?1XwzP0*^S*fLeuLZoeF zj(;aAF|i*@Cp15gSsr*`6Jv({Pg)m+NQYfO7TdUr?DgWi@w&$*E!yzN&>MW*@4PFOd>Y^8oLx9tCrD9(+wHsTUE^!k>0adtLVKwLZ zNb7S(F&C40on==IBU#EkV%udctYEL&=_)<7v35Vw1HI=rS65qp^5$m_au?m@VrQ0# zk{OON6#O^ewNQt#Z%(YdBJf_FyCzcQ6>`MLKvwS19bF)x0(`qFYKby&^)ZfcXLVAg zxvoVN$s;$ri^d}XU&)v#b$&qOo1t7e50^lIl&o&YvBLcv#HE+I>m^B`5Qc+%X(#SS z3V2UA{c;oOd@Y&<>`}#zT)ugzVez9u8TD$a>Zz*Iq{D>9!#v-r&ZH@mL3W|H(~g2m zh16O)_Dj&G(Q$Hek>Np1$9j{FQ7{0AIlQGyHwJE?%I8puuAIy%f=&7ArVnJ689y*& zO9c$o$!4kYv%L*;rgFQO*;kVM5zL?KDPee@@x%ZxPW}bOw&YT%} zPiJ?0*@^Ky`UDb1CE>xwk7Zs8`q}Es-mjRMyPe^YVVju&I#b^Ix+w)&IaNpJN%!ZOhfZtTj-k-q8t zvQOnY<*Dmf_uF36BIl3cY25gq8D$=0Q=xsu-E~S&hCFzLi_7)a#rwZ#BR8o)j@JPzCa6da8T%2x)zuqD-@FW0 z^kdeH32{Z*&PgBvcH3je;@rl+s$+v&*wSiJ1Pob6asUA38HteoMddD7qOtrSMdFMby)JZHnQZ? zR7o8|8m{J4BfyL-&ntjQ0cqa)EX{!Id_sAShglD=ZlY`?bLQW4;#yPT-A9R9x&J9F z$%I=z3!-gvplKBHiA=$pct^#^#-9x8q*{5Ck2%L7uYZlPd|DL@2F;zOe3QqFCTk<@ zL^r?x3}aQN5D}e`9FXg}Yv*B?lIgMO=cnIu%np4K4bY%MVb`f}?A`|7I?P&RfpuHqDXFV1af~5I* z`NsEU1<)N=GQ~M_#c_&BC?|BMlTwWIq!_%$P%`~VRN`jkD;Z#br;&s&LOf=^%_DRF z=6aH1ObLH!x^NncXUM0wtwA|G-ADPA#;oWniulRX)xguY2BHri>140TwVC-Uno@tA z?7@tbwx#Npy|?^=+o3t?KsiMDJl2egLvQKa|0h(?C6*t!an4Jon^+|qL=KsxT^JKj~jNT;& zXdPe@r){HAp7s|jRj??5?jsst*m6t06{}j<&`Kpb%Pd8 zJeIPn_ECIwpDIX_BX6OnpC*$tc95rL>g0pT-#VJJ;P|9oFb!3!JgXJxt!Q>ec}SSeks3)s!USRCSW>^@2{yaB@ zxx>Cj%y^`^{FZBS4F5d_9UcKCp|wB)`kqn;Dr>LD4r(Bgz-sYCS+S1MeEv6c;CdFy zSg@AaO*mr~$&Mo~Km7^#Lnj;pTvNoL%o|H?#u{r*v8OAKfLh?DC6k#6WnsRs^P|&HNFO`tBC8)OoXmrV`SOO~Ez66EHetzl> zD%DpN4Ivm-)RJtB09EnRJ=*d6P>a2f`eqv!K8;< z$=9=h$YGjx4W;=!x}Q>43g1HtuIoCQH00?# zGV8N6uQd4Dmx&(|jFX}FqaUf3BZf34kBD7JZ)5rNIgsGbLN7cszq-_E41d!VLKsA} zjm&lMc5iSR4MJTo7TCfKS*MNkHRaJUnbe^81HJ-XJd+ku7g_Sj)v;BzNWG;rvLhX* zfhy1;RlY=86=EvgDqn+d&LgNtc$g-XwIzo%Sv=F!;W~AR0s!M!q?%9oa;5NnpM&S- zjMkxNPU*gf_-$0iP<&j+7&(<-rpnek%uiQk?;=(1u-@G^nK5=t0`=uSy?yq!!E2mR zIYV@X&927Ng~jsHzoz!SVc9P$N1QYpi*QdrfI{w9k?BUIY`c8;I!Ly)i|i4Q)szn=gQdG3WzKwkeHD0gCC%j$btF&)4wUVnB##Uq zgT!B_y0tW1&jbg)fl>Y2+8SYT+yT7QNI%}nGL>-@SCuz=p|WfCBg*NTZ#}KIynCbr zG|ybZ;ocip1oJR@<3t2)(t3@xLaOSTK9|cux7wwq%$M-{wNA@@?4eE3AdC(i+{sYj%fe=0~hiWSOoyga#|l@+>TbLXZa+U zw7>qcz92LZoMDsw9Qmy>>KqWEPg<{^lcV zZ<(oD>KZa0OB1%rycn&F(U4v?nP@@Wg**DYvx3o)($`~ZdByEy zUhlLpHF!-DG!|L4Ls}3}m#12cnR47UAYOkLr zN|UiKQL7wbLrKog1z&GaXB0;?5Ql(eER<=Dz&5rF?VyxDRkBv$KhGeO(fA~tqeA<* z`wH!79MyU?SGOHlr$Wu;tLeTXDb-bV-qy#Lb{=5`^K}~Zo{>0OCVMvA&K(c0EbmD) zoQ*l0l4heWc{9K@(S z0l7wMv)p?g`0|yx(LHwy(732B={Xr~*mkAp_%ZgpjHz)^(57I#?wyirlqcm&N&;B% z6sKd+ddo^)sbVzfTRL54RmX+}tN`ko517jFAMo4e2usn67ShZ;tem163uVALHG5%*@Zqw#Yb@T!h@(8&eEF%MQksa_&Wyd%-wMZ8pk+YvsId0y`j` zhs_@gPV7ED`$swQ-x!-V2}5wA;C}TW(g0g zSS@t1pq4Z&!Ff3K55}~g-oAW~vNcrT%`h>XC7qB0p1N5%JA)?fWcg_u!=ax>|doJj4XQwC8(qi zn?76}@wi7wT8^fgQF$d|wS$GO*#urQtMQ;*u&wvJ#J zTP`C%0l%u~^2|IYGA|C(lrtjHo7 z@G!-B&}i=*Vimyver-i7ng0j$&@$VaD$AAuS@HT@z+j!_GG{INzFJrJ+^D?ub=qm7 zwDbT3o32QN9kW1ilI8j3g+58%n3B-ESy6YBMj zgBTG^ICTfHiEo0L+OJA_f*@o_{1z7`3}JtqWzK0zqK2zJlW6III|P4PoW#b(P*&k< z%DL<6%iZMqGQ@f3MxK9PUb$|PWkbxPZ6&tdm!V^qtvTAF!}6ilP)gTgdH4XJPM5A8 zC|kgfQU(`^&E+d*lN5QsqM3CE<(Q9hKVwWwCUYFWdMif2?0GI)xEDr$a|S=s>dTjps_^p0S*XsFml-4lsXxo;!VmU^^2~0ejC*hkAi#JuyWa;~==>>-=8PcjF+#&KU z`fvqlk5us$Bt| z4!A?&^~0^wzrvE;+{e?FtMUN^k1`Y3#&D|(rdqlQHb(+dBV`#cfBKo^s-Kua zIocO-{*Jg3Y%Zg(Y%|Ovo%LH?dc9y~AW0PyC3y+VCjM8S4$Ij_Nk0!Z?+Z$b=^h!0 zGIOMvgCQa_@Q{<=)+_IPX&+~Gu;E`^o82l`Z{6E#vQDL_#sHnTysfPPETU1z$c(e# z?Zw8^5jIA&A3a;6oB{cJ6ufM&DmA4VO+V2|e>HpsHqZXn=)6KV^*A^Y)3~JuGBj16 z?8zEoEl@uyfo+wTOmsyT4xW)IR^~Fk$#zLbC(|K&A^^w0dc;B21z?3=D2I~vMMpT6 z{3go0P=&k8k9~+U^y-LnI`DZszl$ojH|}3#2*^_u$AC(b#NQdAth&dl%U%&}xMpE) zGM-3_%D9z2L)w?}GIm`Jp)H4yz^~Nha(^b4W1;r{uDF2dwNG+q`eqH5{lpe^86u0^ zYtG;Ls#uaZ_5R)6?=fAE>x`B8x@Xa6%CiRc!J#V-K}MyzK6QAwhp!kQjx~{Icl>A0 zifZEk7jR;C3e2VMaH|Kq8EGeHzB2u6hLjEgX>uGy42#<77-}_!AKi>Bq_iS4o6UG@ zEsP&siWX^t;-Ub9ABvv4Mh31y2Ue=`j5=08sPrhEDfgZ#O*Qi30>}_>epMCeMlS^<8gIlEUH%A@b&#m-D1AtU=8HcKL8Y5tGHeb) zQn}D2+qMC2z-~_hkC?>jgd2%HJLpcVyjvxF-j}QQ>m2gn1Ss(dRX!f0A0g}MK9dBE zy9iUsEk5{A#o}bo8QiO~xxm&r-F)SkDvr~6E_O7lVgweUVjiO{-zK*@VI{`jKyf*X zrL*G0JH+-e*Zl~QlFbwG0Y6{EtzJ(>;#=uH<|lQr_B}~KC^36?c`7zXnwFS3^8E{3 z+viCZ_4V(u0L6*}qU*FT_Ej>a7XP zh1zjywhJjJQTdO^h(=h*Z*;K=N{ST8w&WvGW~HbaaRp~rVi?79j78~e>PL7W>Cf$D z@m|2~+ljoy*`|o`236b_1hJ2n*L2~K)CXDQb8(qSEY6rypU?cm!xu*XiG3QUit0(i z`&bd%i#TqVR7pC%LNG%%cHpb(GuHU8M0YK$rQ|*Zl&l>Aq@PKq(t_@&xSf8ZGA9A{ zJ3wbVB`Qmn?jZD}j*-llKi&qo$MnDAK;puUpscyoj0cj+;U2CjvTrl9cdUfF{i?iU zWKxJ_m7bsDf_Xw{SzkLwFB!o?=yq&exF7`~dnWx@O8| zTUt;vRtuywGbA6^XE4)g*d}!z;1P64NPt%OTY(mivAQ@%0P>t$PX_kSvfyR9Ye~yW zaSo$dR=ym=MrzWjm# z(5<&CL?LF8XD$#OnV_3m|8tA!X;PaY!7b(Y$gnM}(Dky2(I77(tLLHL7bc(~A57$` zk|P?Yhx;-?=7QRCzJpIkKpjJ6jBejZ7L40in^cVBMDqJ#5>_#s6@4XBpPbsQybCa^ zIb-1-ju}j%MH9V}rbfdun|OY+5~oP;Ib`lYGm~pwwK0Q^YEtwiP~U4qS`G#*zb^v` zjn%<;@*Ml)>#_|xMkYPQBn;DGn#L1A%rzXH04F`2jCb=a^20O&AS>VG{g> z&6-K?amk}UVvxJ)bdUKe!vQ*!)RV=N8UG`CU+kLhc--$%s~;rb$YZ6MXvrfL05H@v z8S!=3l`jgd#d;^*l#08)0*0J1NUG}GnHN3-Eb6FICpRsa0NR(2rCn^XFd%4mt# zXZLTa(p6wpN`jo+0t>jg8HptkbCA4x23z*S%jnL$LLHULnT}UOwiGRNACejukgGD? zpyBrZG&eDX)Fo(Aw3|M7kS|*lb27OF;P$2pO+t%GpOS{3;6Z)=h0l#5m$!y}$8>%Z zw6V25v$?Ah;DW?HCEt9`90&A~;HwEhPxvd===1%oTivDHbWnRUCJQ;PBrRL=L8U4n zAxSuk31!&VN#H#O#Exh$`BS?E52bLcmxo%TyeY>nH!0?`r7|*!L(%uIkII?KKTC$< zC_8${x|T@9&pN}llX5>M>vu9s|El76IkhJ)r0h*~lFb8RyA>MqIi{V_*ZMnm}?Q#mb+ zA?c~|#w9ZsuP@Zt^2W^jW{s2|x1<%z-No0C%H#~+Y2`a#T){ZE@-*?$~R%H&uKw#k)$l(o@+W_p2b|SQu>L&1E zER;STZg}Tu)b$@#p|@CYU+&cgq#mW^j(noi=IkWch|>H=i?z84cLryZG|h?*GO`q zFM@0Y66DX5+`y$m(-E!k0J-M$)lqUDEik zZ{XjgAoZ5m|9ITmpBeSDDtEeU9cR^h$R#!9HVVYaM?+d{E?-fNTl(o<5^j7u?;J@V%opCM4Y~Vm z4>S1>8m+fc78rdKT~$@a2;xEkUGvY@gxDW}|95}J$o)cBFd0`5JF@j>B#@*BC#aCI z3}4K(R318;W{B%W9!T%{7DSly8%yFkpW9fSZbJf#;}9oDGih)WP`L3Mf>2vUI02x8 z#CozbCk%5+kByo3Mb)#6*MKd>1+f|s;DKku9)TMsSWO25R`Vix!cYoN9_0#kQH|HA7OAg z2au40u0~;5rp&1ZHqB-MI~A5d?Q#jx-;XBM>kmeDRl%BKqsoZf(MgW<_npqxUWoqM z(^01)3-;g&>-^ZX-<45M^*9!W8}WaWqIY%L8ephBCOTyx`R%A zXbo=K8p}wsu3?!eR-%W&MzXH~Hyx;QUHFp~wT!;$#hR-$l8*-gFyHHbRSOB=$*qpQ zm{irhsuT3)HQg-rHf3m_5soXD=Zkswdb+p$10cApqcWRJyytU~EE|Apc$1{nZph3| zox${8GamnUS-J6DdZyGQq9?QKaqBH-eIfjy0Of0_Ac%dUt!^vb8NRhj8P07T?Jf7$ zt6X`yGzk#XLiUqou1>ZTmFbnGYjE;k=3p8xN2cArT6{1_MO(^qL$VxtN5=_?$gzS< zG?`qOt@mTViXXYX%nI;2Y_iwWM69EXDXhF#V2idoGP4iL^Ub$l%&$j%nb=t8Vok?5 zI9Dw#1u^mTC03AHTi*K`(eZZ0MnCcz(-YGDK%Xegluv)hR_2dDYSybrdsd-j9&{qJ zqt0%fDu4XJwKtS=yxul=OxFCvn%DUe21CME3!N7c&(KQh;rwMBmgD#HDwk5Y)%VJw z%Yaa-VX1C%7F4m}*pig$diSEUoc_?Pkj!UHyPVC(0x_blkX~5Y=9{SFE>-fN)Ej_} z&U-h_Fg`CUK!iBzep#QwesLW^jOx^-ubhBi;BpyH_yefY* zv@q142`LKUi_p)#bO8&Pp2_Qt&<(B5*5p>tyoXWgq`cE)%%HRYno{~0!b!^cFYogz zxu`s;PGcnL6w`1_1=`F19`dDt;UN!*5wmGW7^x8dOBD7@BCbKE~dZ0jkjCI570 zv|$xX)hsk#$ND?UkmWpL+ApBSMV-{eVdZ?8egRHN5A-SW;dba`bro$cA6b~zD|Dnp z&Kyeb_$xWk1yIC`sPy?yt+=Z#;Pyf*qc7-dlE>eIwt5VwRn*#GdH&sE776vxr^|IY`$ZN*?D>t5FM zbKHT^ag7D9-09H1@IL%Z8oRRf-U_*o5aD{9zzi(`Tgij3`f?=S&8X_DFo_i-Fg8;U zn7R^dAPvq_+yoITPeSfJ^}%hw-V-cHf=t_L0Dl$G@*?Y>2SADu0W}pr~ z!mV+fql=V{ zqOS0cb#qa&PK6ec3ZD)eryPWSq6HfV;J+<>hIeY;%nZUL*-jkrz11zN`{0sn-3Ozt z@tGOsEizjhFDIAC&s&Hb{6u*qpr0*NS)ey3lTp#YM_!YL&T!lvQis3;<<@6%?c>}r@R8bxEi6+Q)-FWFxEA*y$1N@k)5A(9;Zi@2_*l~lZ@P}r~)ens~CL7z5Jy=m&Z&9tG zSMf$&ihrf-NBIGn?t}{{*Ky}EIC;#EruNnR3}N z!jnNRvSI%n$oQ7%?eBiya@UKmv$QwEKH4B@4Hk;l!T#C*DADC!?x`Z(-ZN*U(XOxv zAtM0VQ{_G^e`JHz=l;Pr{Eg~;MCJV*u=38Dyf^%7yRukird~ z-e|6o}(Klkgc54#2Fmdwl>anj}_gWOS=D*LDS za=|-7%Ea|nqw<#H#_;SEPmpnx` zbmhOpWU&yz$ZRLRi>4FAW@kMk1`_^&Pbs5y39#+Gvgp$EjE_P{*~K_BDK&J~QUdrA zacH|-O`d!s&27{v(n4kj;J~{I|)7V%=M`>d=NRD2|Yxv_qmbfju z!}5AB3@+_tQ`p4yOq5H{R$d@RpLDm_6EP-Cd_Lh5POG2TWJXyt`%p5j!@W5@O*7=p zD$&wc%RKA?XU0&J&Ax?P#JR@3H7kIYpl-6y04zY)jsr!|G3y_yG z1vZn?RtI&)vi`zh>cZe!*{oI@L>#Z!QOd(PeAx}~&~&r^|`y2>AcgOnZu z4=Untg-U0GIptGCO#tfJCz57T#&h0x@SNo;f3;5iAt+l`L55mzrat`<)NadS$=6rJ z$E9Fqa~orhjTF6Ks*8L%lAuK;R;e`#WoC!kA$z}P;iHdCbyM0z3L6I5}U-qt=u zWH4hAGT?21e21Gd=zLvgtDuiu{Ty{_@jppfpW-Wm5ay)EoZl}pHkcf z(_Vanr6Sn=Q#&gq7qdwoS0$NxXTFat_d}$I)nyJJQej>0DbhtAT+hhK5$S`4aAsiH z?BisNK4nll0ufqSWM;64uKBntv`r1FiW>6xYZ!i-`JUu!9M!t02`<6#4#vH_Fv1jU zCFedxTQ+cR=1#IXy9`DPFmH25OuY#9Gn`&L-1z_4}u!YtMsEPa4BBaU~cQ1IEN)>JerXaUeYoni9+=F)ByCd!uXfcWG$ zl|DMYJzV<_C+nh*8&%E@SVHa`JfMP_@{TShKRhw36vNFKuDtYod96Q^(QcO(n`G*k z*9WKXUh_pI<#QCVmrAWBZ8xR)<==4-mp)rye+BW}m-gO!WcCO+!-+ab;`amjey)ls z8GaZOO{(J|2u?{KS<;J_yP%5JNTYmc)hFGty`LU5DeI}q-TK7pG`s;Sw^b@c+V@nB zi%~nvV{}Z<)ed?K;&Y|_jlYvKWdEB?%e8?R?McD9oX^;${3GOVUG;Zfa)1F=G}Y-V zwJ;||r!+QnBU}}oATrBZYD_J3%9QrL6l#H$qNXf)1?X`I#d zOG9JiF3ywG&LIYJlF~RFxK4?OL6^>!mLKtsmQBFI%PP>9n|M#jp2r#P3{}zrQd#_@ zF)-aFMT?KPAW^oi#;Lqw1kG56_Zny}P3RTFRjzbA&GO*@LW)()gJvet>6}6G-5AjB zv~tiLT*bB$I3?m=p|fJkRcRCX`Lr)nx_;uL^qUGl0X+;?S(kOH3vzpD`0Ox1(#{KM&+Dj@nlr?ir zl_^v!VO1hKWU}%V>%`I-vZf14_~>U0wpW?X0m&Z^j+IO#FPf57ocwM8kl=Wjy?I5- z0JoQoSAlEJ>`#(d3G`)_@*f5#qICgM%d65RndfgU1w}YNF;}*AxgEqB*c!t60yR)w zcUuNv=)S<^^?7YudnsEDGToyLl+Hp}hJ+V4k>dM(x?-?xxK6xifvqhcC0VV|xX)DS zQh6>Pvh?Vdy*Q#uYXdZnK}O?ji>wUr0={_I0_dAID=jgcZx#K)2VHP!{^LB7#bx#MKcyTO){Cju(hX z?JIBe^F_(_i}aIdS5xO3`FF7@UQ;{!!=&4GtiUHDSU7?YOZA@LQ^qX(s$v}urXRyY zxZfC{WXh9ccXO)3r6NtCsJIFv3yY-c8(9?)nsPfEC3a|hU<8usd{C!Kr^vuR5#ANL zZueWiV6gV9@<-r)RDB>yGnZ(9e+jd}loYC9eMt-O4yP&5i5S>x0#>2c`ke0LtXpOV z)Hi1~p%5~*x9DElYgKd&3q>uLA$ci0qWCy`#hu5_RWu#d_ckJ6Q8HZODjm_`v*H?3 zV-El_rYm@|PuU-osICIA=e%i=l)*?K6)O^T+@-m^*VlBg_@%8WGIzv5oFnqMZ2J>b zw|?exOSu}e`VqGfCWxl=LB*rOH(5+|{*E(Ax zQfw4&QTdA;l!#DcPOrgt?;a1TELr~$`)d(aQX;sDWOV1K2*G9@PUJjBF6!mo11SqHFxuqNJzx@+=(D@L*u5W2lwY~V z%l|f8qdLubH;zNeDS7WooYQEXRxHmSF}*V6e$X>K4TRsVJk%oA!JI0}aY?R@20g!< z%4+_x$?7(`Cwt}lOs0IiavcX5iW;HB89n4~k(nvp9i~qBE3XFd1}3qr8CE!swuMU*PU;=Kl7TL`CK#2Z4MA%1mjNNqs5+k<(e z0KA%uQeRM>M^P{T6x1mHWl1)%QWI7FDA{q#49X8Zh)%rp4)fN&0J_ktlozd@=kv&q zlS~SNv*#fxJ-O*mDrA>0_L)A}I}(e0t1k$Dx`xk1HdQzQTqtS@5*NAjmc&x|buVTp zUPbmvgUP-O&6@rz@2_+)vAq;G@!3JR{H{b%VKZHH^m|g0li@l$$nqPIvrJX70jS_4 zG6A$RviCWkSBC$DiU&TessSiTBtIO!KEF?9Ks>z}!F{oQsSR7=w&E^Yaw9{yH%>a=5t`*OFKvd&ZKMJ zww8Yuzl3D?7vqQKuaGD7#`G&aP`oe89C|CL4s0!2oA^mEOsRyZOgdl|U`)v#Csn5Ps}T7po;n1rI*%(Hz$HES4e}D z2$fPmw@8TKv3(g!;20-Ib&^L7-Ad)z4j7BCRk(%BeHK$P??K+{6IGlo2Y=u#Hqv51 z;xkMK(n0Wns8c&QUgk8TPhN#_H`#-VlV9{vVT@m#3F2PJqc*DWaC!GPmOMTe0jr^c zQ?z%X6X!|nBph8+YQ2NiK&7_;PoXsNX}aaB--*Bs(bjvAvOEZZ zW&8ex1t87@Rr)AkPj#iZE}-P3#1QF1>v@8Gj-cub&?|o265e_3(xN~-k#%~xua=IO&8t^iX}ut7e`GIQmfQiNGlq@g@+ z^EtIwx`U!;W;v3P5!1X|>x;|EOBFR^bv$wMl8(iHuly9oZOfZIr93s^!JbjbSfoGy zOs&DxrnUsz*!r_U)Dv;Pw+ zPKNw=4VlPMDT}3V*qXyEEEXN9jvc zAVnrTWR1$f{b{Mv`8Y7Kn5B_|6TX;nHZNaJR!8vYOwdTCa|}x7VJ?tm=d4~J*U=kO zT+xtEBX#u2x4k+RY)U}^E|q5(3*S-Mmg~KVLpZpWTH<)b8k31tv4&?hGx@J8Z=6w( zokJ8grc;kgOj$MzMp8+6YU>0%oxW_N1N&Yx6~5-8s_}bF6#0S4_(3Us@qb^iPYh&hkG)MmrJqQOGs#*&f}h(ZRk2_BV}KVd7D&*G{0JwtG%jva1q;P z`Kv=(n6KLvQf{k@4wSSLYhx=%L(3=*{R^{31ki8T;|h!y5~)Hwi#ro%>`3yLTgKNh zUD`Si=7ur44RHVGOJ}zu@ zF7!wfsF4ngwv{Svx!_ynInA>ws>{ek=-e4Sy`xktUOxUcErX?La!|(Hhe%E^>(U}+ zfr<i}4u#GXi(2$z98OAyC*5s)NnVzjWo_=bfH3Jq|vXtl;*&)~27R8y)ZH8b+ zS}}v9&(u%SseYb#y$cp%79-U#>J`<&9#=Kuhox=sSAtUkRLVrn+rjN%gYz}<*Jpx4 z4Nygkl5OiTlQbD=DIb287M2Ya$VOik=p~O{xRL>*%88eoXOWO}>Ub0Narxm;5sU1a zPh6~3IUMYT(I60wg`CVxbf7%&u9?9sxW;goj3idMpoetHWV7;-fj&2nEM7;dthcYs z90*+fsR~vx=jBZqm8S(%68E}|itLp?>!;Catt?ZVt>9JPJn74&wGYeiyegTC-jbi? zUCMws@lu25l#Z8qz0$mx_ksiRT@lLjiEcFZ)n!0pye@KmDhXWvQDtU}XAV51=od7V zQDCnHzUdXPWo~^&`})UVB`52`j3;M;UcQDXR7L`JBG;|mqJ`L5+f%$i zNfYV5$n+ALjl8O%H~~E@IwNz+;aP?6B9j^DG$);{5{R;=^6&`LjXtHu$@JebiWw?5 znIKPbH%YFQ#i8GhM^cR?aY0sqax#hz2$(lEq2AGFHD98A6{H~0hS*e*4h`G;NMBAg z>seS1o630+35#~;R1Pyun7JvY%=Mrw`otH|9@#{6E>uf~S1=15Im{=;4&)-56p%L$ zLI<;s zsT0^3s2Ip&Q%&HMt((0Rna3(|@~Bv4Y?HB0CVGobJ6Xz(f$>6FDVRz|O;utS5IxM; z7rNcTD>Ka!pfbZF7b(ld`nvX!K8{P26z8LXaxBfldsqY8m4@@3caV@l6uH{Fdp4?c9rLKF5aD^X*Q|(R#pHVX9e#oM)yi} z@-a;jr{MPmSIUe(kfTN_b0kwJ8C1GWl_MT$nut-e*WiPiD%3^p9>OPb=UN?%%an#t%)em;pqrR2UWZq zR1YNpY@sfM)yT|sf8tAJN^>m;D@gb=j1G2CK3aO|w4^u{?Lhc|M3*!bnk@s8O}~0R zfvzAgm{Iw00jz=8;IkYY3p~QQre?v8I0-_3rK@t_Ep+5p6{=&rlSG}`ISj9|?%#Sx zZ~J}`NGp|X9F{>$^T&XsSV?PK5$FKLBQw4Uh?qZ7&OQiY)K)GZo;To}u21Gr=tRlk z2cQ(+e94HZVzS2YW*0!Eb$rRPmHmXwnvBxjPxd;wpt*{kmtJpSV^?xy7=~9q1H@tv zB!3}J$_~nU$oT6(-YFflYE&47Ea$YAizA>v8OqU(Ad%&JK+C#suc9M?Gnr&$-$uK;u-y?_KM9TB9W)>IX+5iPUN zmHeq(YNr*NOnf1WZr1CvbcL3PGET+?WoA4(SmqMuAVEckV`)9Bq%UMfTuV$35lxPN zk6MF`Ijm9}GkjyX?A)CeMR5bE7_*EP1kd5syB<6ek$Yc&zeVXhjH3@^CkyEcDf0Ki!NlxYB_*W*N<4vY~Fkq<0u2Ebx<2YI$1JzAUFKz2MW^ zkI~~|YmI&#eSz@U-hWW=a$QmUp(H~l?ZTI%+XRQYL?Ds!OR`w(dkaYzMN zDy=QkQpku+5YxaY6)k}h1mbk&{W-8H&{95Y0#9D1qVLPSQYPf%QC5reK8$W}80IU$ z!1}AnE?tc7y!?~ksO5?EzO8ozqf3zhS*WxlHZZ*`Z zjjE`LQxd7CRc{nTAT8;rQ+?gQ<;3e#9|9xw^t09ZZL`S>4idxe@{m~>Zv)GZtb ztEb!dx4I13q@Y<2R#5{kpPEiUWa$(t-g5C69ifll0|Xdz6L zJ;SLgznRf8E*}e0{3r{Q+7O#Vs>91FOi>AMmPP9H@cAWg8ruJBUJlCOSj4py)w;RZ z7m`|^`a;l=jCS(-VO(MAPqmj3dvak^rttCQ3<&0?t>;uh8PSp=`H0`r}6K5sY*UB>^iE%>s< z&B?wAa861Y^N?K+6-iZTL#cEU6y{9bS^1Q*$5_*hUJ$Ct*RtVpU}KSrEo9Y6sK2=q zF7&jD#32}j4v*;0ns=44gF9z9r7&cc0&ju(dDBStxXmJ6R-w@vbg5*r&~%DAn&lEt zzFDljsmGaWz!#YN1_Ty*<|>di+K zij}Jt6d>FNpem^&DM=tkM4#0;tGx9jsGGgDIfD4UsY-KMmuNQd5V!*0!d;tXiK|z*jR4685!^!=@EX zWjn+DiA28#hAKfC7Iosq1}SGViK~L}5X-j8g*!evNjU08s>{ctS=19M)=NAWfT0&0 zOx}XLVxxAF1FBOzPDj)oyi!3e`TYyHl3zLPFdy4HXb-!YBb8&G?0?T^LC8s^Iiu69 z&#TgXvS23CaafnIqJjBvl&O4*PH*i2c2G|&uqQ23&hIl_cwo`-@>4BFc12|@;N3lL zf>YinF?8;}SVl(_M&PI~U0?78i8f?bgG&ZlvwM}BAQK928*(d#ec^0e1JEMu0A7lp z!@tUON~4*G%ez@tx-TVyc~{=B77)cD^g^q&08*K`R37*aVfy1YGZiT--3`ABErJSp zYKrm6l@un;JD$}n(0RG!yfeY=0rp%U07bsU2^ysFX$sksWt=-+$5OM`HA}wy3cMxD zWB>tzqBevTAUjhABZ#C-_{Bp{6Zme)Y+2WyF!B!rQQXJliWz*EUJ^+aa%iS6D7|y} z^Fa>06NH95{-#@%jCu^zS+0v^#B`;Fx-TP~6F9;u<9_hjWSjP{CMo|jKxlBfR9VbF zzf$?l5x=4?GUJ^zkGy2FRzR$RX=qsV9a+&cs{s1UN&P0Oc$(DS!nVWy*ARwQDmE8Y z^$y4H$!Ww5z@lYQCb?1}+uFaGY;YG=;3|ZZq-)i*5>PMJ3a=-sj4s(lm__>{*4^0% zx)Y1uO#Vk3Ir@bS= zj6TY)+gz3E1nuxaeZHf&oS@=1Iuv!l0t%jBVKOfmbF+2lSLXmZbJ#2Zu}gN6tLuH3 z15+H2P3j2nN#`qGng62cMYr->$c0Y)HaT54Yx38YE&o{akwv1e8$Jc`i8v(4leJ)u z<~F>p1S*`v38h9GH+Rc><}aAz;M^Bwjg9>nOZjA2mqqC^>W5>oN$ zaio9HPT&C7Bn%Nme5;M=48vmI`X{AjfI> z)YeNUO}0d@+zm*-rHEqRU9901Rq#H4qxb+7eNRj6=Y6D{bhSnir{H$OEXzO|Y)z$n z8*|aCPgc3~_}I*qCg*{4lB|-AoH&<+maxE7Oxh!ds)K@E7Xlr_Ia1oK;5*gw(D}1ZVVs-@264{FZf(gD#4r zqfZ}E%AGGaPNWgxnvKbPR(EQvkxUiXAlzW@II9I9gLq?nf?+lKKW#v$+GIByE~M3FyqX1iT@1*xlrZma&Ue(uaVhZex7X&Qh3G1-lt^< z1C1mi7PREZ)J88}*>=cTml>9|labHIRk1@83Ac>=mFHAa&SBuEr4g1iwVl|?u`dO% zUIv6m)i%w{kXq-`yc&sWVl^}M@yei1({jHJ0Z$o#gwbd9z70M%mfYV_YMe7GwBJ}4 z7_ZQ=ri$ClQt>>A4M47c*>}Vc-yeAGlcAHcc`yY7RL+eA|GX)v zVsIK>QT{^7XwTMVT0UQEr;F7bR^I-`>bx*tMf!FtOMy{T_TLE;i)GE{_qE8vCk&rQ z>^XwxyUXIzKW7b8+P(qv>}Y`GX6%H7<=o{?_L}l;UlOJ;QAM+i7xHvoWUAfxG|vWS zB&bfwm(c)ixG|YqrSijHrYKN1VuQo!h2Gp6F&qq+btgN%r+Lhur^Ki&hrnt${wnxRDqfrZVH< z=B~#OzL-fc&OSgI)`1*w!la&FiLU^i$QW5tBP)R3<(S}{vz^#KX0clT<;wt#B1N*v z1^7=?Mgj8Wd>I|ju!ujG=+fFZtRZn{fNAGA?ID4Q7_J*CqdPhaoUy_9UqhE;a;rV! zA5jinPQSd1OnBCG$$2lkeTAJE)NEzb`?a3t$fmuQk(FL74UDY+HtePtza~eBIoQ{* z0pKJpSp_`O_Bbk^qKdl7UxVOs3u+*m+Z8#h9od2hLP`~&7SU+}sa6>nWl}zT8!!G! zACjmjUhahx=f{)j?b2IAo6wKoAml9dthBhnr2g|hH#wlofw$4l($9-9=*RUb<}c&_ zTNw+rhSLMkoRWQ#LRVKTQ$|z5YlK=V?LoM$0U zS3bQdL4^F1q1Fg{lBrW=$aDC@(_E$(6(T}&KsgQpp>QXAcP_c0g3GymQ9PV8IL@e* zPQM4QsjHYfdO@OWPMrLT7Bc$NG&kB8&Jm*>(^H{}0><#&n)JY?bxY zMbeg5f|a{f*o~c_{2Og5GFbeKhhbBmjp7NKnb5#+Jd{7+U`yJU4+p3iKnNhUv+TQz zd(lDfbvIXmn)0K9Qd`;g08=Y;=oLx5(cR~jR`2l)P8iwc>QV^)gAb%R<$^aY1XuBo zhKRWI*)xO2tZkUWbt-o>VoC^&^S3>D4Yt1COkuh?;hYS#T53blQf3q8NqLx9H~O+) zMQ6wtR+RpSM9Uitj{$@?==`8_@>yduMNaPnKwng@UU<`4vp9))UKdY}S6K?!?1ESc0iDbnaO#<3@Us?EKZFUcp4uE04AHIe2|< zV+1{PM@~$Db>2#{<^u}RHtZF^iYek=Q|8y;Ip5vI8F^P1qC1U)#W1mC^*@Y@shpkC zt&`6MLX>QPH@Ld8&tv;oYl3bhLZo*z4yzhRBu zu>_!wO( zX|KM9{QUunLLr%VwC~*xO~oB11~Ndl`hB|0i(|8VxzD@Rp>3LxWo!mgruH? ziWRYba?47~}WKHHqm3GpZ=h zAQ|!$KMIQ9N&yeO{07*Tx$n{bATQmS#L34s|1OukPx;(g(DzE~1_xOOg z%&JnocUFqL@*LW@OQrsZmCau(Wl2!x4AvBi?vgfdj$i* zw%!Nr`INdm?Il>B_j2k3x!sh18EeT7JSjYlQIyc@uz(_+GE@4q%o+>7p?vB}LZ{{W zut~`%csm^}t9FMO7YKvCSpXjgM%D*gBW>g`(AFe+8DH0>T= zsY-R@_?^m?E%MbWtIcxuTWe*TY|Z7@djGqxvdxlppQW2nq zi_e>E5dJ+{hYo(N#~-`xtX$7x^nQz9A5VqPuC(s57+ZE(tDh*-?wP|A7ErBLy!wF3mG60e!F)bA)`6!U$>6C6 zmhobjTt2Vd)Oh^OCycq%Vj{=ebs@hdx-H%OSxo7bJ;WtoKQ^iU5I>HVFqe0#n?dH%v@k(`ZZzVZQB(bxHXUy)ybD!J6@Me5%#w1=aT2I^s{XTt_M>cBA zB`+W4w;8b{xA8FF?>Kt;F8e$Typg(ldQSx+_UN% zyk@J-T=LKF+~wdK{1|hNXRrQ?i!Iw3@c0pCh)TphkMX0my~Wr*(X`3>2`tUdOV;`_ z{-T*GO{Z~#y<@mv;5Dn=*lyixG1ff8BkPclAO~OJU4CuC95>nqPjF-hWtGXMMq7@v z$T&>|`dwubExJe2*vd@9s#gK9UX0PWkE!ph`xjlA%-yD4;o7`W*4^^=cot#9Gp3(R zLI%+0EUS3uO@4cc1Tnd$%Zz?Bjj!Fmj=zPz@iA{-GTmQMt4?aspG7?LxU*fi$E%e? z%HW!QW|8#$gqM0bky}kX#tiH}&N{sK3)3*F6Bm!rt+6}L;)9nLLV3KGtT~e2kq@4q zg3SD6aAzAm^CbE{zg?@t*CsY$GBXD7+bh{z{MNv@0^=#1$CA7Xd1Em*YVZzUKOAFF zPhH~D$(2~HJyVeUMbB`fubwr7(xWpUT%N!LmW7ys5B4D7uMcAh$DiXdm9kAL=IYdp z?fdm(7I43n&$Bd^QeKu42o6~|i5s*$&OnDA<2`?SlCPcAZIHj}1%X+g7m((snf%Z& z1Zl}_uD!LDJ6#^kq`d6lpQFhFmr%T!FYPFRC0AP_(bII<_b$VDZ2Q@KEolMw-2W?o z>t3lwjpUvMAIJG3FbNJe^P5gMY zFDrlRFrPXq^UvW6xM_o_(CKOwOjXC;7=^XgZWiP6B5w7TaLJ!j^qu*Q%|SLcgDPr~ zwYzws*CwJot48_!P%_$tHm-LiEJJis@9G@=rWz_s>a!Xj_CcTrui-AUZ!xd7H<_G; z%}tw(jV z)rrWZQHe@sJ@&zTs8=Y|5wBt|oKGnPS71 z?2v(*;X$*B)XH}a`0%-8X18`tZeDEg{F_Tnzif?1vjW*g(r+j7+ijM~nXU__S9VP0 z=Fjvrb7jN^zHpXgG3hsmdA6@X0;Xz0oA5O+aYDcP*Eg8dnfloEcZD9AHJmTB$}@wq z`AM#*urrC5>+=2ax<%-qLZ|#akSF{*hvB#7NVJsnV@U6P$C_*z1d}|Nlp)=9L$~s` zt>uRNoiE8JY{=g{YQS}pb`^Fi^l%J)bXU{^!dj3s)vgJ6-IY0gU?ZHJJ~E_BO% z@A77E9$`Z6K4j8{u_|M!eS^82R*lQzyYb_jmS$Mq*JZw2w6m5Hm@?wUJi#+spJaaj z`4EYI|1u9h*q+~dUSJq~wxdX0Q^BAfcv*7jUnV&H4PN)VP6$|^z9!bV$`zW){lecC z6mzR>pYYwIL0)v-L|**XF~0WhOuWE(>3&gkd=XWX891qzWX&4pY?7Xk-4C;TMU*m^ zXaC@pTf(1of|^b0|ITy%v@)3sHlQn;Msm--f1yABR_9e#4PllxHsfK{72jB-i)7Eg z$^DLv;YVw2A7&9?la1Qm<_zG?DmHV2)9X;+C!EaJq>gDA;`5CD$uGICPBUI;*_XU+ zoumRe-jTn%CgKNI(P=XSDlwD|*SP0bC0yHU7C)xahz^&Yyq&w(dGE0+xO?{FzOZr0 z`Vr-H&hebpEx2w$7C%m}W-_U&lqoGFqkSRKdWsjVVCO5nAXl&va`O=>NPy~`wTs$hDS zu-hPeSz98dfm+gEmyLVYW_oaSowcOf0VeN>jYwjTxUeL?%nD}EF;|XHWt?5+^3@M- z5fMFRt@3{)oq1eT)!V>t^_VjtI)Vrs5eD4mswnQ`KJHu<#T|FdjZx8YM@=Jh0S!gX zA~nUdLXD(MBdwoUR#;->ifKhzNhykEiDvnJPwzkTftkCU^X%tYX{UJ1jFr9v%MtTS%|&W%1Q)F$_5u#zS3n>O)$&8}Z<$203& zJpEVztDC%p$7L4s+KY_%kiGcsWWqJx6PjZ><&OY%c1j_wnlf@$2BzS4qv5whKAXVB zLvKI=E1n0Tdpom=Ul(y5)yNK=NL^DdtmKX`t%L7$07(6yMj(*nkmt3AwU>bSRBQr{ zKiU2g)syS`Y~s5e3*h$_w^;MH0pRDM7RkTTjYF=L^Ra0yeB9?7UYW4f)C~y9Qn}ssum{}srf8&Ku?_r7G zKY~IY)7?u>Yen9&D{IQ7sz#cA#^d@91Hhe-iBfR@pzj^Xd&a-a^JdVGLVgV4$Vs9{ z4ciOHY>j@TiIq=Yzrdq{IwI5~^0LJ!s)i@A^jAnH?;3{s=XpiaA&@t|Uz3dux?s9w z{-3-j=1bm9*=MK>-v+MR-{F$pX@IQBE4(pwItx&D%~;v6$#TolO5pLrX5Lfr4AyZ` zUYrcq)bP)XY8ydSrQcEKXm$Qq+Tv`US~;NssqkFIx-uR5-aP5g6u(!_9aSSdoxrTjN*z7 z6}-7eDKxUW8Nj`(WkTKy>}&nc{9wM15UMehSMT@o;I?@@`3$c_ok?F$?}HK`|Rq z@o_SJI3#yOB?RiK34dwVhlf2`1>UZQuuQ*I{5w+5hKc)t8_qM)MBeiv3Jza1U6PZ+ zGKc7jz?}b?$#QiHWLfKb#K?hSPRJi16E^E)m@oG8)-4h2rs&U)#!jSIrI^@uyh)BMtz9l)BqsVd?&bU9G|Wy zkvW(nZKDA-zqR?Cb$r-~zn;_axVPgIj5V2s61~C;bI_eYp48*4Z(}3kj!LMBVj*CE zb`hXCJBMGr?M;t7x(?h9F-Xnu#t^Nn1H`}j2QajH#^Sj1JJLK?<=Es1&IbaOsd!&@ z^S&OOrEwFp$g5$vq84Z}JJZ%IkmkvdL*gf_*>@QbKA_lJ7JAS-=icK1TXk=aHjR1w ztw+F7DG2k)-mK!pVPO0Ahun5lclP%?1Qn;(V+8}-K?l3X@yeI(q0oNK%a&<}`TmZ! z7j_4-_J4HAbEj%1-L@KX*@|p++RHe|ik8TIU zoV>pOk9-rqOjGgDW`LV5_b zIA(%8tw(=8YJyDpNgp~-&c!8KE5L(`F(E=eWE!6FSxkatPKT6jT}_u*{~e(};Zj3K z9^3(Ctc8ekFa&IDsYsNa1KGwm#f)ywVNor+Sj>5u4Qi(hH%VV#1<}rpsg$G~zS>SP zt*nh@ceC{=;WMmn`RX`7^lM{=5~HXJlV1K%9TV)8tUk?iTjV85or9P$ll6~kGGO~g zckjOR5m>9c3;bnGLMRTh@$Z8!h+99E>?gAWxui}nUY5NDw2ey!S@tDJm7D8%Z8Zk7 zL7z4{ZOjID=aaa2up2-(M*R_eqdAQmvqU`e`EHhz?|#$c0u#PAbKz50O%ff5TYhlc z%#oNMxo(wq6nq)LMjND5N%2tr>-|2W{@;Vp!`f+l{6uQ1G1*S*dJ`LWw?Ztck7upe#3$}j!wUg>rSKy9limFd$h zr*S$ZOCm=>wb5<4{-;oo`Zxvnozzo^-xrAG7wuuKyA&GPpwdI-+4BJV?{|@0g(*35 z>k6+QnE`6Aegme}bYNAjE&JW5wO8G<5bBsm1YhZ=SuD2xS0&RKT@6`odXgnBichaomE5=2QrKfWv->n$~h@ZCr>{Dc)$=Q{r8yKZ2x%Dmg&iaH) z-aEpJy5{qg6WWQmy%_X#S&F`!a2;m)Ru2^)TAVM_&j4+@a^@mm;v!^!Yttcvds;Se z*m&^Qx|xQP>AA8^&lLPYxkKg5UPQxTmZDSr>O;qIAtYZG21=_s@Im#@ttuJt5f43j zgbhvAi+bv=HX>9(U1CI`b>+_oW~tOy>?(6E7%9I6l^;;PK)l!mbqJj?zphm$R&eE} z87i;MKvtv^9h8Z#0NOKy;Dx#<>IH_*$Zi+3F;e$QF03#yY(D`$j`L;_Ne90A6lD~J z%3sY*xBM>$Ep?z4ds(25@EJFFdZ0dNcutvV^7lgsCT$ywcj`fLM>~Rz2cMY)^-#)^ zRRi+Rc>xrE8-rJNnn}{qLP%thX;w;&@A&Iordi4fYPW28<_3| zzV$u_g2%6z339Y4Z+yQY+vvNO`}4KI-E*KxBk6c{@Z67}YRyFsEgLC868RVSoZQ8` z_xliXe@`v10#3G&JQV}zGV;?`W-RiA$0 z3f)d4HCo=Hz*fdwGSdjdJA^rh zzB;ErP3))s|$^=}J zV6)`KKe?eRb|BtCW};1N4#C$QYw5oI<R-Zz3BWu%6YOkptUxN`BN> z{@wXIWU*Gux*=20uPszTgvjgMTf#6onN?yL;_A<@xnX8D=`3j>(mkA2Y{`Rf9j0OY zR-CTe6XZzmX>j4vNdC&{2#8*{N7{_KkcV-tf`o>e5bmum{J3#CFWnf3DrlyM49|W~ zzvV?fdf*h2HoBd_iid10n>c_@AJO2=sV$$h28whB4VE#xV2;*WQ9RWbT{}Vfq9y4W zP?n~9H6Q5#JZtqd%fzOS%xMsbjMCjnT@Mgumpf=ZGzwe}i-aH!9OL7;-5{i;N@ykx z-!-F1daA4=+gq@ptS^AO^Nd-pbE^@A_aSx9uL$+t*?{M5BI2_5EiQRl@xxf{(C)t# z$Pl*;kjsPl_wq6JQNACk@QTiPzquB5Gg;-jBwkO<+!e#BK0c3Z*l`2{qvLE4`9%da za?Q*v*`}K{(J89FF)k)n`X{i&*E2D7|K(L@bJ^#FOHgY2&w0Vc=Xh2ARD{5FeS90W zpOgKws-Ur+uH=$6O?Z84-Fj%9c3kbt0s!VC;KmrP7}@hP6#3H$9<^^dIZAEfokm_B z)>4>RB4;NesV`9QA@UXtH%S!TPo<6|oJZ8q*rcH`JEl^0`?1WFi%66o zDk_bISj6*U(&YHI41?qs(r}*DL{|`BqFw7AGD+~&~1KL(FI+rWeUbbzf4=Q8;StixLf^arp0TB)oXJp^EKqy6b zU%uQGlK4`UHkPg}d8Pja9(AWJq`HwjK8euq->3r@4c-JyyXtUnW;Z_W{)|NK)B@m~ z$}1zG*VagG+1LRI)295=$|@v@t(<6F5FqVkhY}x8;UPsES@PDCCeCI(#jS!Co5s|X z0x38P_~$SqfMak$E%~7|6Z6XFYac?lT{^M7`w2f zw^m|BK3k04u3L>{t%iQ_^d-odjUZuACJ6KohalFDz!bPp!Is9gR+hh7d&d9`VVQ-<$ija5RRbP3rzU=umEqi{6*l8?!O)>+r|AiD@tXhj~(~7 zfzkE4eSh*A4nK6M2^U)wIoKWG;Fv8R(`7nD11wxsEaDDt+6j6p%vZrWIO>G zqiqEy=tZFPOB+O1q6%#;7Y|xaV|EIm{M{yT&N^m+@vy=t-5OXqr1Ip($oo@SSj8bE z%_7~eIx&J7o5Z%DlA9rnJmKa0Oq1vTw_63O7xX$aRwbAPu~erVkzgDJ*$`EC`# zQL4;J0yW6n%PkAcKRMWRN%v;_>WqQT#{ZYWNO)F`ZL|`QxHLF;_6;_>IUeHA zGk-Nuc~lpZ=25_ZrL3Xy>eFz?*w4YzD&;0Q#oyTtmo!yUcRy*?q>qye`EbgnK z&ZJ7if-_Rg5&+?9DDV7=;kdcD0L@%#Gsacqa|lhuLvX@t6Ymk=Gi?We^F+e3%t9j;C6MenfN-hoK}Xn!CO(h}H>PGCWWqs&%3l{XC77sQF1g9#WtY;Bmb=K z(_k@ohHOk%9r6hi+<12@+d{Hg(%~v_!G%je;N%B6ccB-fGnw$Rs~%H3I3>hsk>Rn}MnKS7uz>k8JA02ha%)ULX`siyQf z$}`F-n1?>n8%Q>h6S4?%O&RtS$U0NgqSKk4(DLYpNtk6Rq>-!44k(Z+n>_v(DAsR* z4E;-mwlGc>yQK0-=%h!$F#Hj|ju@ZISnmQ(tF3}6B%0+p=ypp2q#%mP{QtXU-8 z2)2?>$>`F5bn)@s<=ji>)UNW^1IY4{^3{+vTJ5~XC||kuy{QwcOJikD8uUNJ1r&C^ z$U@)v5kQ}@p<`&K5Ftv3LOyVSm6R-`<~Pkuw03P?MAp7PY@6O#{QI((PxK&=G2vjj za4!@!NuP-$Kj6098GN*T2r-tcoEC<=B9VePrk*|O!ETi*YACr`FhUQ1Xgec@2VcYl zG|E%5W%vrnVA~H6e#khe>hr!xhYwf(XCkDA>l|%AX_=joYTOSFI(qYpj$XPA4(Q!gZhN_3z3Syf~3Y@Ah|0U%JSJcB*EUiSb1u>h2OfR>64F=SY}9^Ws?D| z%}{x#r@b7FRTV-FPH4FN`iVErXpZn%X~9Y%K8I@p1nY1 zA;TkO(j>rP-8NHkNZNYCE;J(=p%Qg+E|2@V6=YZLMl^NM2liDp>;(`O18_dv!Eq}? zcLTexaymf9?f|idppNMu-xdvJI7BsJKMQJG*)pff9tYWQYTtdH6;!=#Iwhek&^n~g zdz11~36RD$qXHnkErF)}9BzQOaci z+L4%vTsaOJKT<|N8KB1})AyI(^l2;yrOk%ipiz*BDCY6M0#; zx1{&0A4%i{@#;jQaVHZqlhM_9_DBbKyR;S@eR0^-V;X#2j9M|I>3(D-$aAl-=%cJs z&TGXMtIG!F(4j?I42NUqx!GaFB>3wK54U2&h8y9KNSEr`LNv<|jztd*$vNz01d3U`x$Ji8~Wkdy}%H2FPQ~L|xZ8<&Isbz@kMQuT0 zD|vX3z0qM;C$BK4wmgrw_tnRQtu&kBu7pgNh`J! zPA&&|f77N^@;*R9Aqje6u;kYujXb-nM2E7Ze(5mWfe7fZ{0JhOxMG$|sq%-s{-ja zE~E-L?G*8Q(w=5@=W(Rya_^zZt(EmE6be8l^VMVE-DRm@bv9vJZxh^kMDaT>$P%A8#_juu8i$>j8x|Y6Msdl&vrR-ne=) zwi!zc`IrM`s`5k|+ajtO8g-!y#up5fn>s9Uv<~lin8sy0mq4l~v_o~-7W;15mdNt( zjM?OZ4k+Cf=(>M%-WW9-*;Y!(hq~f~hM1K;a!YlnF+i1Bl+Dmz>+pO;LY#6($`5|v z`n5^~)Wi|+-IF>Tnx%3vSB9}DWmMkVQDYfpW7u_bIcL=VE)n!>P~S1e6p z4jaUQ&|CX5wAX^L2?f`wb!F*&NaD6`{C|wD+CY!W>6v;YV`CLtU%tMEblFNNv_uR- zy2Pu}Ae2{H6N*2r0vSEmn{6zPcSy_-m?H3~RmOWt1Eu8;DAlVffh55s3>R}? z;<)6;L+I~r$;n#~het6i>2=maS6qX0G>UG89*I2eu2m@C+jwTHA5ev%%GO!3$Zkgd z=R1tQic1M}d7{Y5@-)TVORgznzT;eVdIYM6pHb4vqR7pJ|12&R24qBxW zLzX$1Mt8k@%`$+yBf^*ykgB_>g~)h}arwB!OxN`yePm1(rql%8wxZW=GgO}YgCClx z+{U`|Tnz~&rx{6?Ra>@?$6 zna0c(38~_e=Ty=VWNlVgSzE$(8DZNI-CQ_CZY)JYUERdeo-eU{lF|k|p3`lrTcuRZZLBw=R(Urh(jhX_cwj8uq#AXM?$~Xhrzhd+(Z3V#XF^Q zEbsp)9Ra-Ts@)|)c`y*fSuSl`v(eelvgOv=@zHGyf^7eOBw>(B3IwdgPoDnD9xJ~e zU^(<5qFufY$$eOhdoN>9%Lls5Zk;krO7B8zXoB@%j%4*WvI7#R-^*<)^V=h3+UfMm zWx6NI=s>$Rz$$yo=r2tMG5icAoQlD`dbSMw2Qf*N;BdMnt%^+IEWdN{9p$!3hR*)$ zvfRodZj}=uMW6&BcX=1S-F$#`AF0TZ;O2nxgdRa?sH*;$;-QqnWMc9T zvb52WUE8=7Zdj2Wupbj6|(A#Z<&o?tXc5ZbmhRAx>vsU{l_CR(Z3 z4l?B%09&>al>f8}upMd7OP^BZb@8*&9U$@_pi}b;q#9W$5z^rlcpI*&nn+BV=|F{h zVhH@v=z1MacS$~s@p`2cxR3hHOqQB1@H4&+P$BBEY5uG+eR>Dr zX{Zuw7)#=-Y4&kxfXz)pw zsb6)&o)zoR1aFc5jA@+fl+q^uouP^v8I6n6Wyi}f;m+snMB`0>yMC6HgTIdXZOwXk z%W(RqMak2#kp+hRs@?&(DMF zTEQz9n}Ol&x|6_pJ&`r!E2~@{-?!4`hw4yh<}7x;a1xlKg^8am(;Z<}s`Mr@=@C?4 z^HUW0Ad*s1{Kdl1YTeQ8zwl`g<9g5MXDO-0rb%ZziLs)2s9EYrX z2(W@xi3hSvtt~(50Lzfh92A!7QSmhy6Ui|o&0ngbA);SZL4de25h#ZW zOkQB-$iQB_dh;pMP1zuvI4H}s>Eb?8Isr!yX!_7PGj;5O?7 zKd_$wtzUvAP_HhcHxGD9W2$WofgqI@Df3>0088`PT_c^My<{)9K3fe1a7>p|tPJ+`Fad6=lW)2zbTn==f>Pg9S2Kl83D6tzpq z6Q)hxIgO6JF^PwaEN0*jff3CAEe=zi%)}_ zb+|R@M_nMb$#l@UD`C>{8X%*~vp)}_joq4lvlJ}JW#tC zUs`G}qo-h56S^H5gTR6-07)C$UA`lC8~m2^lyouyWRF(Rr8TjOYA9oj+!(@2Zm6PQ zoIFKsX@1A3up9Rv@G2c3uGtHr{qZa${K`jOmvJZ)?LCh5&%VfRwy2yaxlJlKXV9Yd z(!D3ZU#hds)i6|Z{-}Mr*FEOXrdYNTOZN~tHV~F#q*<7B8jk_@83QBaTml03ng>Dn zneG(2aw34-qsJ1}%8tobxw@pfa)5N11`D3eidFjKLIIJaHE0?t8dGI|e zOVm9u7^^dk|H?OgGJY3uNq%f4>qE>KnR3WXgOt86h4Nb;xoK@mj{@AJlLJK@<`+Ku=g#{wxl34`zDC@1?)n^_Xw%FLCrsX%vU zZ}3qy3G_sEg)AEB#^popSa{u)BLOQtp)pDbZCIz3H;!%k91_&ro6kwMu&GA$P+8cS z7o{=Wm+(ndKWKq$mU|TdkUohu#7?p*$1NsS-*$tUgJamypSrfcc?Yoa^g=MODWFsq z|IH4Ls`B=nodUK_=&2D7|PGr8eAPL)P=?IZ!TahOz<^<@bq^E#oVvAiE(x+J=U zkB$4_y8OLtI`UN@D1t7q@IQ5dc$sqcGCJmI z=MqyMU()k4>Zz&$dU#zrb=pKlbh4Iw+1T_L7b~>&o-K8t7YCQfu|@+Eh? zSc&;d6lR<>y$Yu@qSaqt1 z<%#lE4n%YJQ`mX(wtaCBLUMf>dx7sZeav^^9>~Mc=`CvZo>B^|W$~Hi{~;WKKbUwec5hNVZ!y@YhICK*(O9pFUX)<3xuGHqi_w-B>a9Ur2zp}D8 zWH^FlzHT^uP35=btmF)rbX~b|;2p%sHl5}3-G);o#>j}&7KR^3<{x!9_f))3J~)C5 z_*zepdN)2zT-VJkgb_e5`Uv8gc4Na7DAc_aC z>tJ13GwTI2&KMhzEQ=pwL4_THA1~G6t0?V^_EY)s2t{WwZmmjx=`kJH-<-m)_#8$f zequ+#aV%nwDj9}2E37Hc7yyi#qf)u^5!Y4dX_gB=Hq)g07p6NVM2wc;prtm2f>jyS zpre8Lx>lT~mIp94vpL&izH=M-EgtX$>FFO&>pY&oKhXeQa%gOyTn=EG9G!v!;A`5O zuBN#WfJlrs+7&0l*S;v3H3H&YtIzgRh^tCU2sYoK1TP_)wb&@;pVg)dm}4-RS)3@p zcR(l&7=j%auBTKC(M`h-Lh90Rh{nS=m`6|MLoqWun!}Ap~G9*)HvZnvn#DzH(V_5f!6a?^D?Sv1$ zf$=tnY0%^ig$8T?cLTYnc+dJi0EG}xZ4zG?lY3FQqOr=79lGC8j@(&l&tX>yk+L}q z5nv-Ai3vuOZZt)k&`xT5+uWwrs&$lZTk#ll(X%9nDWd^+Nc2E+IdneKL*H8Kp)s#T zf~DQ$ta5>ma zDA`xmu-rv?1>$eB2qZ1!`gbc5Cx@wLHgJ+l%3f|N%6m+W$t_szxne<{RD5OD@Lk!{4H5380#Z1tg)x6x1z*& zJ~J6XU)o(J-|^~W3x@cdcGv=Eb;f!{E^PVnCsv8ndje|Sd=A}mQKhz&wm$GUQ#m@z z@QeH~raw^K*BeQ7^LxuC^;`4r@A&v7=N4P}L_5B7e}u4T1{g-3mb0H0&C?`)J7Uel z%m`}r!N>-c>@O?dWT&_PMA^QVm&^&g($c7j(4|y^<)cQp_^f&eE`~JdJa8iIXVzeQ zPtXNa|J#5G_LdFw2llo+2qu2Lvc^gle-nuts1v8o>!VK3=RrC%hq|Nnvb0P|c1eup z$8oiI_k*+8=MQvmmd5xb2wpNU`u1yjthuByUGEfQWa)mAhfL5Cx0x&Ng_i4($&S6MwJ>oh!YPq`;aYv77SSXhIlGjyP0J4j_n zcmqP9PUDTt5*b!urpwX#7(sE#7zGnldIwB7QkH)w!&hgDxsZ{}d-c`l;mwtMAl@>H zwyICyj{9?f&3nuXl7Nu_=YRT4{-tuXkm#Rz)E|0U%IXMLkYV$N>Xg(pIjHltFRQF6 zVqG)weS`tOLte`#vxg%HekQ&sex(q?*lm`Bmu1(0_&gypX8_+#<;-EcpGm%4N~Og7 zZaQtNE?4iv^xuF2eE{nqcgJ$NI;cB6(aqUztWC+1)<1ilx~LK1xEmnHB*5dFbY?j9 zmLBwT>RfbDXbYLX)?NjBFiR-;ZMK<)u?d(f+snck?68Llog>#$svQ)hdto?r9%WM` zKh{c=cVA>pO;p@e9Jt9{I(g#bO@$BQn6P&@m z^uXqCZ$fN*uUs~AND~r~C-gT?)1yKTyUbkUc5#us^@>%9xlCrmGdgK!=Y7ogpUNPB zpHj%lz)?lyEzVCKm9WP>slUg|9~iJ%MtLTqw4P>NRLt5i zLVP`q`6;Eq#y3W8&V#zrdm{pur?Kc%<*?+rTb4t7r=WkT+iS_)Lslw^wX`3u{iG4b z7bzajo`vJ3)*HO^P&}4K{Z`nKNp5Ilq{@#m)>OoCdUhp)7GDT=7R-|4U&7pXHAGAF z@Y>f@k_Qt83QakNAoJ6~u0ON)WlP^gWL+11H0h!W1{+&r;y8q6Hzfd)HPJYj>BM6j zI)N9Y@_Tt#OqBd!qbep;`fX1Qzmdp;7Nbn3 zc>VaRdPB=C*(1#y&Z11MKBrQX*;ELe`glMhhJ49D38%mjW=*VvnVCFHE}yj$VT_#doVyDGjsImBp;iX&iT$`m$|E;_qV6;)h{ZoxyV^F(fB)t z_5&(C7!O(KD84UiBZXsN+wCcCj)K`8nHLJ07SNquI^Q?bICwC_U@E~T{i>$2<6|>X zo*01XR;A+l%jARDsAN6p7_*pZD>j{_PNZd|RV2$=ndF@8M%YfQZ~Ek@ zE(>3vN7Du9f-mM>%rVO1eQ;G#Q{*o>Oeda%8awG3A_PqRrEDiiniWtg8(uMOTGKX> z?=vCr&vbU;%9qSi8Q!LDSuL6WE<8-z%T_!+Ziq~3bt+YMABFoi5vr7f;g%1<>}f5T zf3VpTrJRV!AGQ(vO=No5VF>tf6k6xk#)z)F%Hc**I$Fz-o(Sp_A0l6?bf@P}b(Z7Q z)8Os|Oa4;12jI^o8Xgc(AkSo(4hSK)HtK*y^wbrZF+0s1S+N<^AJV+jQjglsQVu<_ z)MsKk#CjOEgc6w58sc(F#CNo2Pm z1D+pakrrN!U0&B6V~LKpMm|>6WkGaoUXoXBEqEh03IQ3b{YOgzrAi0lm@AbG)7e_c z-#=%#@P$#Xc;_(KsdXGAJg;ZJ=cp4QEFr0`k&>Cuf7!L9-ml=<2S{YS&W#8><8=cZ z#OZ08U|~*H1xZ~4^Zd>^GZtM|x{$MHT6<`z(k)vXK#)yzu1|uV0q|2i_Gks2@}>50 zvxpOLVRNX`S<|>`Ql-;Zw~Gl!)@WbL0ONjUssttRiuu~O%=Te_?x}F-kWDk=4Fn36 zBSaViURbuxUdZ$i>>zYnGRHeFjP>2O0A;Dl8A`ghJ4|Yh zG=0F+7SBU%?ZvZ=&0asmw#T2x6u76SG4)lseWb4g2E+o6lordNfCN(9WWZb47z|;Z zt^+KsF+&icH?H%5ienZD^Aq6Pv1{0$ng~*NeF=CDr{uzwgoi)qgc9kDn@IM05Eo1( zcVSl*CK+D^A{gW;1BydO~&rF>0w zuGS>k(+5FTbdWo6;x(3!D`DuR7unCr`F#BLH2_(p$ASEzg=jrh7$rrype6@L7p%jW zj&fB(wy{ph?ENl!FMl>TU_N0gjucvH()75ggcq^-{tXdO6*2VGJBEbavCKo z9ItUYFHw%I;@~s3%m2HHsHTeLE|2ng#TU%W@;YP?fv$k1BSu&@CZ3cTkuhoHsv%HE z$AK(Df;Ef}Ds&NJTz$y9uqWp<#$hatF&*+XmZ!XXA9{HBJ;G&xa>mJbeN2y3j6$Sb zR=Lkg54vd~sF;1aPq|kU-L?#={te7KHm_7Z_>*N3TEZV*ok9Q>?+)GvAyJm zb{ICUsFWqUfWSw(ONGN{#X(@PPFb~%YfDvJBLIaVvzdGo*n{F_Z=_cr18hCyv(0FQ z)iLE9Vtp+MLAn>n`Loy{4NbdGy6d#Q-8v55TP4KGmOfm|dAJdXGa5pOY15?oLCbB7 z$}2>*rnSUbQW7JDpJC2!p%<$7t%i;sz0Kh(;5v*@Py+QdR?bG)9tkb5EXryb>r;G; z6ZB1$cNdvSR7@xJlbq>hDuGW-kv-cmfIF$gFjBN~S4nQ5SF2`^PTTQ}=VgviJdSa* zQ2bYeuCiOmcj5a| zC1t(kma}hgY3~>}8_aJfLtiy@MqO335nb%WJWE_9n=0681MNSWqzVQ}4ipbqT3`EEz&*nH+X@cq05ST5lwgiQmHW}O3%9i+Gp!#>s^`mJ53=VF1VAf=N;>kr(uN z&rXX0-IW~wP($EMMteV%?x0>L73zs8l4F(dk8cG349Y7&y%cqqUVrmWnlrZsVR4N) z1*K(3y?y|%p?1>Ql&7a`TV$5Y{LQ8zPo*Np8R~3F*<&+NPU{-ylh^pzS{KndcN0Ri zb{MjXVY=%H5R^N4a@I183Z~1yymP7 zf#olYzCtENsYHd+OYVkRcGs4H765t~Bq+B~^dm%m67%u{<2D&g2nMg z5*H3mgDBsY!tUM@M6SD;RBq#$RVqDJQd%Nt%25Brm2J2KJ-xtxX0c77`mJB6gU0IjULEZwjp@_k`OWk9xc zuZD;krm}j$_XXi>ozj!2qh^`(z0Tq;tGM2Lm-ZaXth93sGj@ROIW zgUZ){2bUxvGsuGI83>^NiLer6Vk^*?JrV)ay|x+3u{J3MAkfSIQf8J6*$U4x+jcY` z1tY|92VCAr$${g_8vxDJ!LstIsSg;*^^GH$`gn{%)a#LcqMmxuaS*2S{**LCVrm^d zsbYa_7D%|~hh{$JpeqbNS#nQ)D65NCvZ{^zyn!WDSL9&P3OgN>K5i(ED`qzJtKvz= zl`}xLHG=i(sI+=AE)(J7_bKx67hOVnT#rFW)nj9SP*sC){P+^lr>Xs*;`}h=peC4fWu;2qXE3^{oc`o%C5^@RRlorSlw+%T`7%P{CS)r2p7?UDM*`;{*w zp$qQ>lwo2prKeoa77RO!j%6yezs#R!rt6@|FsZUxIg)Q@In2m$OZzvllz!B4*>NiA z1y)z)FAsIf&qIJwOBYFP)^Mz$O6thK`Ib(FO%IZF<5>Tb+Ungm6G^jCM@4thxLxjQ zTXQFE|7G!ID^>R@Zy-Cgd+#+>vVz6>yjtt6)5JVWf`YNKf>dE`SkoQE+w!Ly6Zah# zc&WUnysrmW-l}kF&tNUtr7PbuGqXXaGZ52&0}hRzG7!q;9wP`Hv`G~WBjeV!2+q>5>87I?x0in96XuM9@>5ShH?NQo{L_&v>U)Lj> zA6NHgNw;7e6aC&n+(v~!RVP(qPdWTOc1967zEbiV64Vn@uEqZr=@RKp#(9(8L2f?* z8`8)j6Rs>DfNO!{#3-nE-8_toP7@%JXga3DCJB9Tr4|KoI4W%q$vOhqe0DvVobOFV z@fEh=afzd4QYb>|wkoQpE#z957=?o+S$6{_U!g==XFw>ENGK5NDUN3Yi%pk2Q;qMA zsq)2A=NmItUh>CFaIg>jcvDNAC*xy*bm>rRl9J9mvuu?to@K^Dnx#S7Nz&Z7S5bz( z^km423HBuEvdk)#^dAsLuVmOgz$wW~NH=wyQ8V5KDsjs0)AJzusHDcyX#hu$WL$?b z?>j(OohEQN5laVXU#^MBq3rz-qnKqVU6aJ}nW@r{5;pA0^f1{~1kU&BT$E!DZ@N^4 z@WyFR0rthJU?$O@uY}B{1?8eIcI zZ$S&{dceRo7`a_J2=lkJuKbjWE-Y7t@kY-S8z&QIten%8tN+l{iPbWkjy4pp$c<;L zN?bV{az=-7!?PsbV+*CVl#g^8R(n14Ib2n~gggj+79&9DFK2Om4V4^jOpb7dsocqku*xv4Xmwu(VoYsm{9?3-FDKQm zRLQHhQwE*E%ydLGiDqqdL)dYpX0rU)9dU9)7bu@rF3edP&*89hHAK}>?m6ZKGePPu zWNbYmtuXwU-5Q=Q*J#sptFN*bOR=As zB{G7N*9}XdfeyNJz{ew?=8kuf!5=8+LewVtD(jj<+%-Oj<_1jU=(+iQq=a{wS&k2b z?5{(AaBqx_hcen?5!97F0)n%xK-lFhlL`CLEx-Sc!1-J`n#jg4?An;jZz(IT0%Qy) z#fi${kV-ul;GbvhIlv5|kxBL?PNAwT_}WFRP0E9?BVs4Z_-7EW$BJF@ zI|J0E=1Yj5_p;Ch$=YYyryD|Sk?z?L)Py&-SFwq5e2=$EX1tm~JDvh4&kFEJxWi1) z5vFW;ud$hpWo7f=R!ABl-+T^p?bI{K8SWT{7oeyGmb7m?aLj2apT7*pra+)Le|!-V z_C9MrGT8D+tJAR2&pHt`K#yI0U%5xi)`k#V%dZiybMq0yHGeb<3|C&IY?%m$jBaO= zFYD&S;#K)+91id&ytIL>?7V^+%KHgkeyGY@8_xJtY4QfE{*I}&k`rvYrEVhn6?3tx zym=YsTB^$XNzg-{S$N+{F!rWsD=>e6ygdqp?xuQGz0y;V!wDN@!7u<;{Ry)ale3_K z{NUwS!bFg0EbfG*#`X9FB0I4!%VzSIjsL|Y!8-|lXqenCPi7j11Q@I>^P4~; zZLaCU_$s#$o#g?1P@P+8w`j!6gbixU3i~3-(_iECogb-BQ+}9+vD4(xJjBHVorPDq z)r_ObNj z`3U*nYslseouLlg#UpS7k+OJ|4&IzRJz>PXsQ;Qtx*X9lSCoveXA-;p40+w1#36YF z=bl_&2|yG7@g~V}-Hp5nL!D*Ww;1CnkvmH*k3O|3_TzNOPkmU)9^FZ5EqT{4 zx~GNIE=6vf%1f74J4*)n1+cSsPDpY-{=PeO|_3xO#H* zq?soBnwdpVUU{rBIg{RO8GvYN0JVPu?H)pC<(XI{w)VU#p{uwP&0M2xfcAouZjzx2 zBXke1K!z`sQ%>6q$vRb~&XAKkOdlr9r z{4ECrloWwaC%c*C>pJZ150L02L8Z#jKu)ccSEa$sY0Xh7$#&UZ&s(Z3RjP+GeJF9K zo%n*UCOf3T;E)~mLKaZIS?otaY}*(Z0@v46K#am$o*4@wN@{UKt|nGzw7mKp%86KL zM_E0`a!PRQ%99y%Pl#akzs1&F74eqULlWs6Z#)-Mh9XGpYfOpBCOqe?DSzF7Zkwxw`7$*bR{cyzTzo2a zxm2U+y_D9#KtGiOLC;d2Ih^R@)=Atwh;)c{AI0jisf)DJ2xE1a9DEl9&@v>ei1R3+8=^9F z2nX%hK1ePPa+#5doFfX>+vcqCtM(fRp!>soCP&~sEz8lVWhwH_b=~Sau^6g-)GxzF zGOCL?MW->r*Z+!6OiMs`<~L<6;mS767>(su#-d_V%CG>9zVeI*`UY38U?16r%1i%p zw1{qMIpj!xv`vx9ewDC9>0)Uc@mmtzxL_#`{;32g3jI03yXIk_Rt+=m*}o9uL|&Qh ztkYPog<`~PJ&JSw)>gooBi=Ifp{j+dS~?ZD*csw7MPUdZ!0 zR3ymOo0ykWxwQs}%%XgBm(6cfWtr&f*nV0fv_#;iDb7BmfhMbjPTI%P4jL-!pbI0~ z$efq3DoL-JK+w~uDQ&&>6837VC+Gh`wmnkLzS!nD{dCxB7_Kg#l7brSGpG}?YLQBB ziIq!c!{S)JYdMjZ+vr_{grQ_(T3!Ij-WfT_?6lfifz}1@Hm}5nU=rD%w`lJWXC=$= zm#tJ|dWuJmb--xbrW|Wzq0h{h=YP+jYF(8_!0{!zs|f9>S|2sFIpSG}gS2!9vNF!b zA;^&=1s+@mH^)H(9TE^p!OR(!rKu>K;&0)T+ZD*anNVy4m1Y`$#87UWj?p-+qEe>x0s6XbpV+vCgtx5^+4mWs|tG)cFJKLRJXoiB5rBT`zF`O5Id8A9_bca}N za_$aAInBiT+*NOUg!W)}2i4;LtG!#H=Yl!bqI4O`-CEU%8^l2;p=i4yn@21(L+c=r4ROh+7Blcn=Gvs5l!x3VO!I;h&( z0;!5w^@c=ibE#?{n^~oDJAkT$&IlX=jg#yM$x7|;bm5+rsu=Xvpw~I3WKhS53D2at zYn;om=F+3&?hVt1ZBtr9)@~B>$%>FzVGP1;~2_Q(%%99#P?fJMs@;Asr7aaeaO0I$RkiAf6 zOVySKUQYOtDt{V5#Mo9W$`K~>53sey%GMkk*HKRzAk?K!C-mi_%tWg1Tuntnq4&Y( zp*s3P^@ar8ov|NnBVZUjV>t5H89!&HA?B)rNu7>u0v(|y9Ln7mtwmNly(+>`jD+oe zW$q`oQB^aU8VXsF7QGrNQW&fKy-o~?vxeU1GV|9!Uh~B^ZRpZAf|iAfdoiY&XLmhG~C|&uvsX-#&h8RNo7aM(*da4TT@WC zFKcn}cROVBsLK$+EoHO8UzMBTYuiweYYRgd*4@{BlBN{3uI%Z5VgET@@~q2YbwH_K`H300$4H zanW>Lz@Sic`KcIZ^xoJA{eww*dOQ;oYg-j2joy{L>r5A}z^niaNoG;iQ?UoMiQR@X zEk~;(DrvR68eOfsKnw0Z#4<|sxZ~{ch{2*Ra_0|wG9D#nU=CN#C}ex;3^!4 z-g`VqJsMutl9$3j`spQB0gAxcL#yr1(s(A;>v+2Jg5jz-2sU&y$jtACPb)S`E>XD& z{>$~mXLkqqF0rG${T9cwcGE3==rAnKkHA>pF{UWOPeQtmKLuWGY-R0b~-LuNN$rlr4 z;W5L9#L-f<^I=H{O21284J)Xs!b#jj63SzDOe8bId^Jx`bgL+Un)2rsmeU^` zmwkao0*(Y`VY%)- zto@y1W2D#$@_I9jmJRgf&g$zZXRa~hVLj~&78}(B;`LzIfN#Wxr6IBqy}#AFmcM{2y#wN>QWz>+67x|oLkZ%e}> zPjlf7szU3s90&E_h)}s>&X&WI+$u6DB1qP}1`{^v`IF_}dEoyHJ*o_MqufQPlRg7` zThs_RW?EzH^S(`-6W=`S_O_%x}aAJ0jUq($gJ9ys{2@WLI zQx01PU?n!opk21iDCH(rPnj8gy3Gm0Nr+H7_CyJI#Vt0n_(eZ>8$L^U?M5@Ye;2A% zLPcmVo5C~o(YBp>LPOxG^~R=0OY9wb_;~GFaCljAr2%fj0O|Aw3_ZMnWm>4$1_i~G zlcG%YspLqfdglhkS@|Kh3u;T^c&;*0vpFUIah0>tToPLtHfdyM{U7KNo4^aLkvKcM zorp&N9(j12AEH%iYfs%(L&i9PBw?=(?#jR)FThEbO^^r`q+$u&)7B$POou=l(@f;I zXAUjSGLb@=C*^NtF`#U9Pl>e$c}?1ZjBHewqUzZHy@R0@wDL^FC`mXms}pYl=gJGi zw&duib;wCoSNZ{RvKa(7NHLW?&sj+=3LK*$#O%`!XXG(jx}C=yK>$0uCqHW48OYHH ztanc3=z-sWpVW+9wZanggNxQlM>R@;?NVPYcnm0va-J0f!s>lC4mpkR>}?e zfJy#pXoa){_t>~B%zg>&F{%*2a94M=m41Ujfb*x5vshSCRU%zdCrg9hgIp%L*TN{2 ze(TXS0#z^<)l@s7$9nroqYpVeL$@B0D|?J+Y8V#QQ#OL(a%Kg#kfPNHm&Uq%4mucF z>527GGruBQkn2ErO%%FxvZMC#Tt)%)oaJikHI#6~{9ZpgPBWn)m=^-+HPqK1nBj}fOK2Gk^239+0NnqVLQUeYc3DW0(?kE)~ z5iHSF3}tN0pN95bG*mnvBx|s7XNZjbgn_E+d$uV+R^f6CGbN~XvK*euT`OBneNImp zj{1w=nhcv=*yZfiP^J@OFy~n#Ye^eecQde_#}OCkM!C_s*R32J-_`tQv|eExM6P81QAAkF;ADuIAp)&9y3)>j2mUk)z99@Iff zZo#iKN)azv&-0}H=*LmWm+&5(NnsQQDfr;edXh=Lzm2tR$ZmHk8k=U)3Q{s5O8*{I zvb^dN@Bu$Iua1IzU0uuT#^)=GObu?YnKtc}MMZkipWw-ER@N~H*>_LX3?P(zKYZSo z#C6Rv(G-zbyvsIj>Av~kL}D{_S8k-ZuM3Dook5t`XJH&o43c^!IoH=JdWza6^VJox zvV33uaTaz8%?{+!R4$2+5?_N*y^nP_(q+efcP7~3&*Z>k8cN~E$yV+Y-p}Ru4D39; zZx|k!u&W7Flifzfl|WWZQgL%6aM_iYXgcq@R8xY?KrXiPl4a^gFxOE^FySq6!ZmWU((QKf*zpd<;2YZOG1;ww^JrfC6iv_(CPa&q5y1vTW z%~7#4Fanf;4X>S4~}dg!gfIpS*z{4y>|Hu>Yo8koy2F6h}oXn?-K9766y zke9L&&4z%>iz4mPp*LHhx(j{h$HD?*bnj7#{IJVzleWfgxTKAlKcdFjBv=K%5m{GD z=`zcu-xgT5Nx$~4il0aZaGue!pn{vmqennm^zzPFS4fU)qI=ap`Gb*Y&gN(tsS#3X zvZ2A!(r2mLy%iDWG4qvbR0~wp<$6mIFA#g<)}v^ugp$H-s;~_>nOTK4HQdebR$wU$ zbl)}=5MM@qN#Qi7=vsei_W;P{>5kE+8iz~QsR9C|w}$5@G?Ds!pmPF0-qZHyC@*kk zL&!{1m9$)8a^qF9moj+hNvV2v8F0e03yRHef=*+L171l(Wpvf#|1MyMSXE(p++(&* zAOQBE=~ED~V~8c@-k59GCKM#X08G=g?_h|Dszm(;l?lFA_u3T)4AIWV7s6O0X2`;x z-0=MqFt|)45K4X;7{`UGB<8UCdUQOJDYh5u^_<6e6%i{BuYrpx^tBZ z=&{M``fRJbp5L92m)Jzk*Pbe)Gvd-Y(I{%?sd56d8yf#9^3@Pb&Z_X!m_68#aT=%S z>7mO0F;W`MG*V?~5x4xfM<5q8(NuCqaKSsSdWn1-)rB;RMT~_38Vq`Q@C1C+(Q|_s+b(>3>KW4%EsQl8n892z5cdS%-%PxwC6~ZjJR|E_+ z!Wjos=E)SV3LLJ5h=G5?H7kJt(POZu+08OXdAvM&Ry}2K4Uqa@7*S{gT*u*H5i0eJ zhI*sTh4OFBI4e;?XtzPSeSw6kT-5Wb%uQ=Odx%_fyVb`0<(MeTB&T1SJ*XII4(2Et zi>i?Dh3p*UPLyIUhWa(o@M=K?nR2+x9-kCzi?T}=P(Acg&xdH;BvPj3xLt5&>~>+> zF@MB-%BeKg9jjSWEEf62qPU@t!)VEdMsAY&*KUi*b&EZ<QV6hCz8_{kOi)Xh6%eH0VB^WBYt9T zk%MWm;@!w%x2(q|l%Z#hM8&|DL_NVS@Q>U<(1mXy5{oL57)SSV-lW`xV;Dj^AnYG~ zu7UJ5b2m-WeW7yip*t4olQ~*8hZ?c)OIxCR-P;J4=WVzuo@ir^mM#6vD%IHAP5TY~ zse3`Lcsm2OXL|Yuy7Z;WthRBQ_RLIjY`+Y<%p77a&k=i7PYth6K7IzfsYl9@_H9@a zV$F^+X#&N6QHAmnRn#8#j>;%A+0*3L*B4!g@zh2z3+y&cpCY)HeF<=8s~YYPGYy|C z@)=f)EiPT!)^+2LRu$PVT}ZYn5%Lji!8K0Owt{yJD3%Dlo9sAY9o6KyijY4( zz(6qaJS>3P$s<(sX&E$ne@7I6EHg@HW;+GpA(BIhWQ2mV66PZGG_blR0wAvyGiJ}0 zXi|aMeN+#b_?4SlU6J7o64MraD|V9<{bA6i;I=tm%M44Ge$;uy>5_I$4yL&2+D`39 zAWP2RHOz1F_;ZlDm!4W%mZ_Fxhx$NE(*!QPj608f^^4(`_X)C4Px4<@3$u>0YE>sQ&leAJ5J_$|fAoY_$@1rV_;x#RswC!Qp zGNUIa>m`FhdVIxWluDe2I7?_I1%GmxK?*h}88J;tEll`upu6;F9?HmOVC+JzrrN$D&D`RFtL?Pq>P&+D%OPZ^rbh!*} z{e$r0@=9i29hDM+F}*pSj$U&?lV%4ic+Ac zP0oS?^FdS?tDpq?HdPa>w60Bp>|+d2fs(Wb({W{)cB4tG>xQZ2!66uRyHiF!gq~JL zYmk$wok($_8x$#Jz!52}?I7Il$Xt|lDvhRpchQA%pB)G>dA|V%eD7j~Nhl2?cC>;K z7gi8AfimLz+MR}3#T0>VlWsxZ%4$b&N?tGdteKnP6tBKP+`YC1Lh5`62%GirbIPf` zQ^oIfmEEOLH?&~(c0sq+!pxueshr!5;R}CLbuKuzYhbaXg_zFsa5ffVdXD*7N<+Or|7~UVM`1-X%*Fl^R!GVo*W^iXBpX;h11)~ z_U^#7Ae!-h)ScCkhnOX+>$&>?Ma*1@4Mk+vjL(%C2H3TDA%dtUrMBflD-PnfjbK-E zJ*lb8=)1=$Q`!X)TRcp~nM_Rafp+=z3F1;cH%at8qfk4Sl8WZ(>j6$$YiOSrxTMN8 z)SCz@kieWsh1{*GQeOJRTqFYqbHV*pHNN<%-s5PAUk5z*tMIyD-P#CjOp6BWsm(x(F=1TghKu7?UzxIhEq37 z-u%LAL-fp9Qu9=tPyYDEh>#ts+|dKlDA@&N!YLxqRhiP-Xgwkm_!d20>TY6Xr<2`Q zvEO$msScee!KOsnGS^U2kgXV(%tcrc{1d@FS2gZSzN&I;xEY=LIrr+sc8KLJigsQW zsH}Fs2h*-;xfYmGBo)>G%{LY&4wablfG{^eUW9>IlXTNMjBwd4K{xqb-nLWWNDA&C>|HVx=6~}cM!@dM);WM2Y`@MPb??bln3Ptx*lR$(LR-?LyR;z^Of1h zRU?L-=n)QVR1qrOle~AQxNW=^RbGj9oAe(6*)L78Y4;1PFznPp=%3R@D#P`runu>R+&Vbk(q@3iNs&@C3J&gShnbDxvw9 z1`+N@cu~JA_(Km3;~M^xo=pOBNkDR-AW584(QrDeit-P#uu=qFW%vEAxbu`*=shf9 zg*UWsQj)adIay+^1d~~&+b3~jjXsVkUsYyKRh|@U7iD7YA#<*Qbf=E zKDe}Lv|Qh3PUi&jtIFb~aheRD2Qh11k>f5sd^=i=qptMMOb*AE`tnV(J{K60B3EC!-{sS%x-}XS%yVjuEH+u? zhW6{}&N=d9JFW$oH(qyq+T3f8tn zs%sUkEnc0DbIFt%i1DRuU=ckdHSP(fd*Y7bLUJa>O|Q@x_})_P7?5n@G1w6|)Cs!J z4Eki!EFeBw7E?B!eJ7REZserN{+Gb_6WmCkbZ&F$xY)4C_xHi;O?n6hWv zW%c||3 z4(3x>&g}v=sk5|=DLy&h3OF{n4halNp{GYT*05uO*)AVd=E(O!LNaJ*Tq5cmp=H#X zG=pVnE#c$@RNHlLH8kPe_R>Gf$mA}V7!9`Pc0=~Lc0eiFlh{dgGEUT@k5mtvG|*4; zfe;u3$;wl9k!-ts=`vtDx~;v8oRS;`ImgCBgHhekgI!0Y-%_k&i?|R-aX%Kt36DYD z-&C>uD~*hFZC03SDO1!)zWg7MtxLs2E~BY|(1o7@$h%;udkGiCJWx1TvNFt0Oi+-> zP8GICmVXmiR$-dFp>DNekAt}fd6ZA1XcSD22xda|gTY+lJ45lOa4pKC&3%DizU&C= zASAXbV6fMb>Jf42QY!$;ovoWTfGzn-54A76d6HOJ{S1(wa zcw*x`>>o9#kE-Rj@(d3T(bEC|Sn4wQ(+Q6KmgD^Yao<)2kYpqcRAy&YywXua+P`Gc zL^_#yE2)c|7|yM_m~I^?gUX=?MW!T2`tt4WF z%OnT1{KW%Qmgmot6H!GH)%-OwiMUM(ZBlXRY8fB4qM6~Up!K5bduS1+X4R?E` z9|6GHuPI)5nT^5&s-N=kEkKUO1+|n`*hO7$daH1b`bw9a4|x~4<FmlawWPo(#W^@XY=T*40|6m3>ud zyyIM}Wo#)o)xyVdis^I;=Q7|o(42`fqdtFqM45dl%*$UzMz#i(Zt0HqwVa$7WKgz~ zK?$d}l_3*#k{gA;5g4D@3M-b&4Q+czNuTdI**&_p z5hXzzp#oe6B0;Y^VrW-^hxmb4US-DgKPL`?XE4vd78?GOV(;&g> z^hj;OVRL)ij4c!cYOaCZIPr7vFk*ll_<>_zXntYR)|TdAW~w4Anx0)73laue>SNE9 z%d=T|QkP_##J7y|!B#wtB;=tHB|8(5+a6Uqwq`Zn-HS<7+6+r3zv_T_o_yFGVs4hA zNuyA=%DbYr_iRs~)jLJWY|q)ANO=dV(jF>7CXaZ$E4a<{ZhrqpJ8@Ye9-H8Vy zWIqDtux>LnJE-{QV7DD07F80P+DR1WK8;U zesGS31fxnTAMbd1|2oS(7v=z!vp)ky{z#yx5#+oPME71Z=W4SHTtFha0eDUXF-aHo zq(w6rJQWc?gZT2x!&cJV+|(m>XitN4(QkG`u&6eOmgQR43>7Y7DlJn>8Y*WQJKwi@ zVQ=7_mcmRp8tP-GgRv4(5~({I@rW~28e9dWlZkMVd1Jxuv+9jpJ+q5)U6zxzs_es8 z_Yv$7+mbV-=f7^d#H`}EcT~X&a*?J24cl(XzZaK)9CFeA@dEhU2E!YCBRdWS+&coU z8nuzjLHYs~ntKtRAAzNiL~d}@I|wZxt+)L70HxqhJ#!#Pl{lFriy9z}o5Q@LR5{Bt zx@)`In}G?;(N6mI<;F9axH@?-C(eonRJ0as^#p<6=?6ea+;i7+sE@G;6_Y6K%`n5f zv0zGJOTd+O0(B4xSx!7F?wPgVxJ0=KlK9l zUtn98Und(XrP`Lqn$mi)3nC^{6vHRkv6Wo!2}K^&^Dir`vLe~@%U}|+c3^Ig3cb%h z1LuhCuL{Pvu^PJO%iUsz39EslE#nnh0Bc^j^xGLnZ;hqqL{#V)Lu%hL`xw-P!LOA; zXPVvSaGChn&=iuDr3>I5MoO(J2U{Y4o9LOz*C>*c*!*+EK}?Q}YYF*3MESio z!zfY_5q+h8AtcaN4-Y`+OROg28={~cd;}BR^VUd$H>IwZVILW}vMm&PX?ZTr1r4Pi zg;lZJKxIn%O(&uEjGj~nbC&&kiW+#(O$*Je+TUGz_j0Ez?Y$precB3X{1*0KU$*2N-886^^{(?jVp2V=V%6YRYi$_4r<%hl$QUx)9?*ATX9dP_cxifx!6Gx zo6CvLZcVv+O;~ifvQN>A#(=D*S(r8wI?CH>TptT^5k@Gl^Hh1`2-F7KERIC$z-;+* zu*7=hm&%3;qc=^)>JaXM<&nofPc_(WrYdz%J*Au0CL>yI_JTUX;}d|Tsg@Xf-O3;n zRRwhMhfx02tzyZkq&iU7sw%>&3W8;t^*;hkTJ3}8G}D2FGS=Xf?c=nk^37I1ikW_&js@u9tv88UOyt4sfIcS0V zx-l-6co~#n<~hO6&=ffRjd^lEIbajSm(&StYH(M{Vd8c?bz>cUh-eknGo^ zQ(+C!KPopvO+_}fVLAy5#oLO5Y^D1r$yZf)j)=Aj=yi_fs7S5=rB9Z_&7+;*hT*TG zT}sIk(;tQdEhAwZra}i)96nV}m3LdEK4wY2##B}kdAydF6HAJl|GK0!jq_rka+ze( zKUh}&C^YQy`Aqa|YLbtX`RCAHdiFt5aY=Ve@vmVKJ$0zDobu$H{t6KivjFX*yPh8$YT)kMt*->sWeFJ*czXRQAvN#NH#W6j1nn)14qkzb#AXHIR z6k9@fHYsy*SyeSJ>neImAhqFCb0M$l3aII)TMJyfuuv}FMDk9GOcM{4+_hhFyvkrs z(?#8Hlc9keN8x92Met7Uu5Wh0(Z|zI1$D#g{r^an_O&>=MLPM&ubaYI|IiagYO~_Q zwf8L~umwQS8#>lSM&VK#jb zSgwRg%9FlhYW~tv!fI;oE6pEDU*Hb|g9A!fWh&PSQ{fUbABTKxjxjw?E5pZp7Yg5q z)8;6XGpTF|n}lZay-7AaYV7YNDhFtN3yU-QujAffuuT*;HzjNrMz3+FB!4LChpgSv zi92G&;jp2fwxHi}wnJHq^O3q< z60Rg8E|i#S(za|jTvFi%x8IqqxDq9(YxK3l{L@?cd7E$I)4B>EOsZ;kwR(ryF8VWm zXzRClHQElp+uqN9p?N7Y-Z$J#~5?>cf7jRVvdl-zk{L^Yx4EYRUXb9Znl|L zeo?AKGVU?{_`-}Pc;PAwEPGgm28?D~{cajg8D7Mwwf-`vi|rGh8yvy%lZPS?9Jhr5w*)PAZv7^y5xnMXCuslyW#w^o_K^iNR zU09VLhOS|u3qEIMbBQ^ZWvc1)uARI#cd22M;|XkS=3;i>`+={cJWP2!m**ls=Jc^{ zM`=ID$H)gym~8(w=3Y11p!^oW(2CA8ZOZeg-A?iOkp9 zTB7Cp6<)tS!5nLH959vWC>P1x5|4#hXBTJLgQhWDd|#Hh;We*LTFY?l7qj*{6^&Gx zm(HgJ-TC$L9#%K~J688)2m{nmVUH!6v*n|u0BFPKJaX}s*{W5Fr}xM%o?RElye0E_ z%-WTCTBU)s$8}y^`4j7`u@FGrTVLXlXbCA1+OZ6)?AyqXL(=(i%W8vy;@kN2`~uqN z^OQsxIe^0&KbqCpzh%6|esLH=NP%lbmtD3B#dm$@Ggd0iFChEEvZ; z=UxPey;arJ;Q(vKb6$HA&evNHIFC9L-O+?z#A;#jS#obHYn!{4-y*I7oKbt2=3lH* zikL|>QoSP^3}wIjRpRi5+8p+wFLM!3hSi16Bj7>3#g%a)3p*S0E@)Q5;r1vV# zTI~`#lE44*39xT5uQ##a{QK;4_XM^$WeKY|*^vQ) z=khhR0dpi4u)DIVWvb^7Mw$IRf15l7^y)H|Rhj27?;V9yYnK9)Mx!~;5g8bU5wucV zNBF!aYG__X8; z!!J2~*u$@>oRGgR|MY0Y*GWIJhh~@f6du9D8Ore3;}-LESPWC>rMe!TKFGcj|1=V0 z-&g!_=n5whG>eC?JmTw2_`UR^RE>Cp44Z6G?D5N5{CcCj*`+0i(7J7>)>=My4DMLQM9K%#u@CYJO!{2-@e)k^$Y}k>_p$=oW4Z zS`>dTqjKkxROrdSAA>)Oeq`sBmxF&N^>q4_Cv)Bbs&Cb@MZlSD=gJQo*jUbD2FdNi z0F(Q$?CuYZba~L0zrEc9bU%Cp%7(53_o9DfU6;~~SQ#@7o}I9a0fwl=Ag5~JF=GY~ zFB#7d5luPsAsqnEySMqZ0Xm9|9nRn43mBl5;tJz%vq}9EEJMn%pmR%@?fwVsugZ7^ z#mj7$e@W$06$hLWwi9sQ%it?H{h6{6+QK`PR$AK4V&3Ds__fgzUYl5s0g7Tcpzk+< zpBKY?vS1cJzEQ6>Qso(EJD`u#yI9%TV@y%}7EAesiV(7GDStanBXKl#qG^IBW4(G{ zNsyR)-Vd0=8GVZ$AR!a@I%*1Eceg_YYn+iS;irrUX{fGItF8j5W8;ANmN1_E^$ttF z{5w4S-0`v{9!uO}rYQP=0lH)xR++q+2TQc!nSG(|D4Fj;iK2d|N$wsj(aog!VtDXo zX+YYb4|v+GAY7^*1GbqH82Y5DZ*WNw{GM-t?c`K;Rp)DlCrd_?;Gg*6^cen@77Y3R zREjeld;uI^rB-m|I8$~S%|CBVd=3AHe7D|-0>^;Mn=KSQ`I-f!jYQa;Od(&VEO39J^7f8giKBVKJyw0VC2uCKWwAN;0dmw5 z#`u@UBQmHAU%Qm#kv~pxW{Z3C&*5?W`mvwC`ARc?xkI2t)H1hCrkHtSq&natD_bdm z6eQh_x;iVKLR&43JZb+F$eUmD#PcaUp;t5@kKcG~-cYc_Q{a*vkxV%445y!P58lyf zE!zz4#+m>0jJIw!M*iK*iB+Xd=(4Glxodf>=933#LK_kQ-CM;!!qxqxaxYf%*E1mA zZytx9GJvV4R5WP)7YG1u)^&U2$6o;KJ08&T1`T4RQn(wR$Rqw%x)-DLJIfnikpwJr z@CJ!>C{u33F(a-;98ZeI53}ZjM2Da7>w@B7SXf55yz@2eeV;Paa*yMh70KE|zXAw* z3bLf$6vHEjsp}?xm1T+kxxDUnGw$V;@PclUsWS5q!>;{M0FqV(K)b8G`WOH{x`335`atQl z(ndAlav+k?A@XI`RcglHG+R>uq==;SZjBdJUewJ`8p- z2#N7*E-<^TnDw|Ju*3fW^B#(B8`XscWksuhe&Tw(kAf@OtDH0}0$HVHdpXW0NSUhc zD2`n3DN??-#VOsW%Zu?WFF@;Uo@~+vJ!iE2s-;5z5zOK|!LkDZ> zw!dbXC;w!)M>LkG{w zWoUaQ+xaO|Bvxm}P6PR8Y#-MBvIJ`!?Prj8e`k#`L1*x@>p}#tm)|moL6Z@G&nIp7c?HM_%|0 zn}l&8(!vWg8L*7G4*d%wrRJqcdJ2@V-OS3*C$gNAW7WxSG@|65!Tjb=fq3dvqfqwY z^_M_(Te{zYX_B6?u&v7VTDZX-BQwycwBU>)nH$NE@5e%K;TcX@p|Io8L+Ii+N_WV( zO>CsXc4+p09eI;0IniAnHXN~nsh@WQZElTX)^>&Lt6?DThN{IERN*FNp|P>7*dW{#H&AE1)p}GHNvH0TcG>jfw|{0 zt?NDH+YMc-Ab|)UZFxbVL}l~$yZ^Y;WY%6VNv<*G%WUjJU#O71vLoD1salptMoxtS zQo6C?wIhva>0{!o--farS)mZRF9Vm+i^|D2+n~w0s)EUPpYUm#o?w!xEAYY)25J+1 zH#)Le6};@#58NrFp_sNO4AxZ~KutQs-k{NNn;;Lol-J{~%mip;nbL?|=e_`|-%{0|3gSjUYqw z8%*>XYp{fof++ivAcXkYKw+EG#!DXopewwLhts42%&D-nyr{=Y-&qe`4pddQ?iP>% zQ3jCf=ODj-?l4l3qMysERJ|*eF~jo!{_8b{U7o!F&?`Igt7|r&F5Tvln{^D6Y-r5a z3TBMhtqF9NFPpLC9w~^Qc&5(MqbxFSF^8NtlxL+no9ck)RvVNNO^jE9MSh&a*{$Bo zg6H*g7s=_Jth&`KZZg#0twsPAsb1w%!sSvGh-qdwfN=CEpH?<@(-NJ$Ct1*x_mg@6 zj{Ez-?+;Xm&gJESXNRpuF0oWBbKqrGU0>akCf+e%0?~1HsWpWoyrvp7g$!cGv^j=} zqK`B@`+`4pQVr?Mx!0$W9*%(z=gRvH8Tgft_r$)*nz<#3< z5PRTb#Lu}Q-atcA{Qu1GZc(a0YPoWPd(H=zBJCA~cAJk{H&&Z0Et<;p10W&zf?z*=U zZH8QO;U1_u`XWGHu3+>=6;*n@6KWdUjIZy018U?JvWh!;cs1=Ec1gu*?K@#c9;7z2=FJSH9Y7d8Tm_i-PET?Kpx@@Tqa?Q%gl`=Vupob(UiTNeq$vV%(K2@afy~reaP|s~r zW>B1{zCdY+DOc97XRilyqE)+$XsI3o<(Itz_&gijCRz28b&kHw5Y6=Tnld(o%{Epx zhi$5IK{pkf8mW?berw5VeH-#=^Bz70eF~zy)suqcVQY7u1l0z39hJ{#4Ash19C~^A z?i6r7UIhqrFxbn1oJ{QFE-O9vSo9}KM|j_5*yQQ2jOpwI@)Vl{Ux^|GQyNV(GKnlD zNHk&^KZJjZ@QTxI0kTsSSGuekkng}63y#3q3r`q`us5LK7F9UqFg{;d_z;*^z}^NE z^4m?DYm_Zv$ai0X5#8q)4lLQ807-ZXbqzYgGbGqmk(<)v-po;3O9 zJm4!IjoR9+IIBGR8VWwq0akm*;=too7%a637ZiI0j}^cl@&87Zmx)RRKWj-NtcVUB zf6fE(=(v?7QzHPwEIq1%46nkOQkSx(9Er4ev_FssV?Shi_iawTdn!tRm(uXkV;hXI z7=_biVLU8?p4`=C@Mr9-i_(9CuGAT5(%M?t_~1AW(Yl-kbk%Ki3BxiiqpAV^+7U(? zjwFJKqf}q5Idhog&x`zUq7jGSQkLYKD%gqUu_g&NC5KC;K>!Jj)-Q9E%IzjNUxrkN zQ*J!Qpsxy9%zNq%>VnnC7Xdu{+bPz2YZmAHU4O$48c{Tf>xolkZaZ--A>T6}GE}mP z--%GQD148h&!}FI@tcy6%3EdJb6;bJG+(VGXKL|7!)-iL`)?zP1SFOlq2heJyV!cn zY6A-R0_X%#7)xjqmmp6w0Y}ywhN|a`%h#S1IOSF$`#bPH;Pk!baO#wT-okZPS=n9A zu*nQX5i}#KBu!%>;t!QMWUIRWE~gf-xIT@uOM!sV1Btko+R93OtAW9F@DMB8GmO0~ zBU(m!(|%a`Z3Z^?!az_;23h25C6|kx=CnK2X3$RS4XaivEL_}?Zkn^-$2B`t&#s0) zG_Q&bJHQibzXMrF1*)fF$0P8}y%p@IovNK* zWip`Xulwpq)LQ_m(~(Jvy<}4rmLk?ddLT0D5{4o5n;Ks2We)WuRqM9kcR7&hXE?3K zfsFJ+b?7P9qVDttbs=jR4w=}S3FDPtB+GfGp*jgF6}H;U1FXlRWTIKBxqecTRvN zhZ!_snG#kcq1dK4jPyE^_lzaHrdX&2iYm&s$CAPB8bD9_m_uKE$xf zgHeW0`#e5DygLzhDY~nob|x&GV8`r=GJPy3lc+~nq~MVpB3Tx(+_h>ybWb ztM0Gyt5|36F!p*!{vOW}w0z#$I>t>qO_#PW;Z@JYXLkn>z}T$21zA7rn-kByd^ z!rH2K=e(j+D#Jrdw0!Xc&(3MjvM4qhEUPYBlH!Y?Vbr46UW3bNShhS-{)I8c|C5Y~ zG*BqW!}*$8O;YF+E#mU->^pmPZhnTP+Iq+9KSVAlbxGrHdD5mXB+M3i46}x zvB8x=?ANvVG)`GudNpL(&B>k+w;w`VlW4f4Ra3a_-~;@+avksfScZr5EU>R8dU|PD zu1uhhlMK5Y(Rl8jx;-yi%|>^rzDF^{`738iro6JaF=IYvfIaiTq@i5Uq<0kXrUg=Y zaaVU|${uuDZBkCE>l`8!;JbKD9?yon z1`Gjgcjtf%r3XMC392CHHZRgKl$ukjJg=2F71H5)k7Yk&6Ai2Eb~FFPB-E&#O0z$S zgGZ`0XQ(>D+}VHg=}<8?&~l%FBku@bt&5CAS$bD(>WC38?bV%U%0$$mB`T}sl6i@s zXOe&Mj>TMF>SGiq6o_gp?%`otj>`FnsV3Y013OnxV!Rx+#M_fqXFlVflI0n!%5ozK z2eq%Vyj;VsSA57(j;P7s$Q2Eio6A_?DrHE2Ylk(q%dyFkzI?mBmsi)-M-nyHBZ6TK zWR%pL$e7w@0KTLugDlz2X-60G+@sO_a5a&Urru}k#8}w0KP+D956Tk2K#lT{n#la_ zYv3yHWY{FS#aV0XQQ*{1 zJ6K6qrIxRZW9eHj8fkLS=T4W;G($B(DymOpL8e7Ulm}Sr4)bu|&MZ4g(F5NA_?8lsMDn*ieJIt8<`L!i=j( zWN&wpJYC5|0e4u&_bIs&U4w%Q;CdqS&VaM`hq~>A6{Oab_bYP_U+C5#`LYaf@NZ`+ zLw@05&lo;!dJMczx8f7oUa3T)p!=LtHK=Q=wsM_A;O|we@Hk5^s4@CiBT9b%5!k#t zfH_tlhw&$<%ko@1J1q!+v25-DJMONs3MQpEq=#~JW@o|c!Yp<<+9t>sAkWf}WOs9L zcH!iO^%Mn3hu;82?@Q)1y=57b_C$HTYN)0IDc`gI2fDqUqO=M!{WEs*SY6Gg-!Ysd zmuRJ>QzanQvNK%o`Az6h+k?(?#bG39qX@}fP>?8PBM`~o>DiSfrZnTup;m~r=nV&{ zx11yLRYQb4Dq&EZYXqAv8I~Yt|KxDqs0hHelJ%0wNqyZnLPlhE+x{jEhyOPffA5=fYzUASCyc z-*pc~lH`-yP;fon8X%7sp>WCAZNI69y*%)=gp z7+R{MC3Z7zOnZtDkIkF#&%VAOR>~w$B0#4q#HePRz#+Y;F)=6BYLfpGiprq#(8tV2 z$X6~2QN-&emY1l02EoK_IS_(W>WIH3+qh-$$mzb2ej!DzWn~L?aJjsjV41bx^4T?z z!7betB=1+?Sl=~48Y>s@TDt4L3=Q z<+4CX@g;I+)eX{2sI8}x7MbH#p8Ql<^Ae(4jZ>aN2~_CJ*a^C9?87O(Owr`aU99`} z3!wkCGs)N-@{B~y0x@5hzXEgacVtnw*|H94R-qvlSz0!trxYC>vh@If}!Rj**TXpx>f@* zx$}TQ=#Ppds;rD#wO6oT3iE`>dL?yd{%EK=5q$moJ0`qdi31Is54T;Xto%I-K&~~q ztB$ak zqXvY^(27PBA)48^6TfHL-nSu^;lM%;b}+0mY$S{SUb$b#{s#0{K4rVbLa?R5$wOpL zfsqaGBIM5%A0?gC&GIeU3GPzlqL}sRe+|5;_(ZNMf@YG#ky`PM5DNZmJHJPIE zJSNC$Wqn?T`%inXAWIJP2D--|^3I8hMwWc|9n;Y@K-poUf(WflY7HW0g~%rl0ZCs~ z#ephVs=}WwRC={{dk77Ut|6tLfqL(qHyqm99Fz2M#i<-KpIu6m11M9JRm`VnR+WDz z#o49l6WB>M?FyyVF?WHSC~0T zl@Ok#d&`Nn4P^QEFTk@xNTBVW8r;#|_SCECn8s`+}Y$U>Z3Ai)U@ zDYv>fNH=5v%e(PCubyj)gs!Y7cEIN1Z7;$9@avoI{2I0g2shGgA<~}?T`07~X#Yk3 zRqX*Ru+IC!!qj z_XL^GDZ_G-J_S{)e*{Z^RvWs~RdV2$da6p@$ZjILJh)Vr>i$Y{B9YA+%B?^~L3_z+ zYuIInvi|>h6e6ml{Be2}5Ve!;Y>zdturB}5kb^$omF4mfMs7eodD&7jj_$0bfWU~K zAr?CxZ#YG4h%|LF%Cu#WXZ&V#Dl985v8biZyA8>|m~P;x!$z#Nh_}o6xp1{W-L6V$ zQ=g!8OE}j%l>gCISt2CkOJFy8KCIlYr*)LRZ}^(o6DW!*m3u}-9dB0l*n*!J{Aojm z9Itz8iGCK0bUS%2R=1YJkZ3g|TQ!jVO10xo_yFKLsp>6XZw8>2fB3pIGL?Eq`Avzp z&L$VEBY2<5pZ=yN26eeorQY{EnTo2b6-1^>jdjrKZHk7=l>cMFj;&Y{dIWU;J0wIv>=8zD6#sY`J?oP69AcGfg44q9?=+m|;^1lit z&g8FckkLPPp_V7{-zKPGGo0-3%rg1 z`-HNbV&9aUC&~4o)^8(0LCRxPS7otusn7ycS#*GrD(XeGxgJGVlbWdvS5;M9Zl-6p zh3(P7YuR&mk{nb1!~R`CspjJlpf1{Bh~*2ACZsaxNbk>z^34d&yCg~j?o`}*g8|dj z66T-f`(&dK8<11#{tXB!s}>YBTk`3B-Qkb{ZDCr2m3V6z%HjNN82CD7K^(h3Hn6=G zxM`8L%pI$2Dh^qH3;?Wumw_K%WqVH5@!go>Msn*#n0D(B;f zzAA`oPL!^_%!rb)v)p;GJg|W}V>J{JH;})TPx!GDf_s)?=s*jYp2c!uis6DWX$@J4y9VIQzM8nQ+q|EUdb! zNq^^&J5#m~V#uG--&CQc#u&141GvN#{udGKZRQ2&Sm8QRw%qgES?!vv9o;V_NW{SeKc^#V4s6#o=YNZca6D8R+{{PLe7L*AzEvYIoT@*PX?Z z@+i$-4++?jtPA=6L)J99fFs{T@f10IpVKCxw=SsydF8a89$QBKF5*ypdVU8ng5g#P zmyIZ7Ao&VZ{RxrZ@57_Kszv6~lvtU+kBuEuCV|ekjWoGrH?nX%K4H#Re!|jP_jr$Vs)qW0I3-%% zeGAAbRvse9YjLQ{3UW<lnVWk+R+uK5P zG^v_u=9$0fxh(hR!*wXTQ%6}yLy1)g?=DmG&<9?vgHJ3^gc>;Na4AjCZ68I55b^;-5vDWX)ml`1MV03y)RA(k?|@!*o+S8MYWL zA~MJ@$?4@rIzKo|%eXhZ^`7#wXKpuah#q?d#CTd;cxiTPxx0YPJPjlCO+`lSf8kD- zJ*N?q^Hpb;U-VR6t`fK}AE@xYoETXCedX!S(6d=i2-LY_dt5e3ajq`;s}O=ubdO6; zWwZHdbvd4=x+heAdY?sYdJhIvvW(jMpH%Byw8Tl zlN2Q1@+$1ndf>6 zn3`}D4)i%ShvjaV;Z+^>E>*7mX1L(R=@n%6 zHghIwQBeh{knZ-%`vQhu)osD@qsPdW6I<9VZ87cmmFPfOApuQi&+m*!+EK7_$yApn z6#Ecw4$zx^l*8@T3F+9Q6)SE3n3LN=#RIAQ(j6`bZ^G72_vCB(yKw{$G&Q1RmN4F- zN=6h)ZebT`rn<+!D9?Cf`;=*NxP2CSnfjI z-&?MbA%YkrzG(8#C`KV+G^MiS%V*~S^-9DKN+=!6)>Z;$cXAZTQS*kvBpp~BPgq(q z-(M8;m44MBm;-;ZiCzB#G8I%I)3tiGTgGcfksNct;I>`k*RB28-9LAXXz8TvdUsR& zDyh__rDr5b#S~Dk|He40T-eQ{>-L2uK<>x%Gz`hdIlhyXVVV#0^t$ru6rb8e=E<+S z!Rx!K3(FHSOQ}{uD8oe~L0)x3HuP4pLe!je%e9@5Nu)lp3AgQL3&?D;Qx~9jQ8l$!}i*D&ML?Wf|lV z%g}bnjR7jt?zp=ChMA#?_dW-v1m9a_Z8Yc@tS5zNT`V?9djgHu*l2jz8p8KgC8LmB z3%6hHJCFgR&bp%{w<*}Tx+GJ~R|fG7M1pD8<1rPPU~nJ(4jG2i??#U7HB{@E*h*?s z?Zjwp;I1hSM(($9SxrMvL}{ny>smj=ci>+Xvu(dJT-o>t(yujDh&;W#ENjeKi||7t z_tJaIM6?2#oB^jgjYm=@eUF6Mr@PAFt<&nt($F}v4YI-L0S-8H%}Ws5p#+00uCL+g z6g3L~v|0aI9O}z^Hvm9L99%ap(a4q2Ib2+ysGb>bRa2wPlx#wLlf&iJOhgp9r;$=W z7^U{kBM|UFJ(%H+H0}_i1Asys4Cov%Q7dNb1o-Y^-<3z?5s{^ts4@a$JE$Bf{O{lJ z55Xr^ZEskDxavYDB})N&1ZWG8y=Z{E+`~QZ6&28Bxw_a);2JObia3nzX^6FI6u9H` z^BaJeejL?{VY;LlmlfAm(6NzjCvX+u?NEKzcj$#RNQ}n3HHj+qk{SgV>u&=xol6-> za!DDkCWfNAlqRYcg)6BYUQs8hmFW$U&d##ZdjJp{sOMJ{{}LlYJ}enW@N8c;+gXLg zKWGhjD{nJWBWzst$I;Je>X_ z>Rrgdk`UTH7>g|sNjG$%B!+iRL{dWe~(3E z5uPd=t{O48o=N2&8fKV~HF#oEe|Bd{LsEccK7M-?a%*Mz>oDBXqB1n8j?xX;=V&AC z4lw2R3(PU*f)OPpb?9Y64mG8oB}d8a0-hz^aIjR?jeI$-qQzr%pIOzmhydpk${-h9 zHtALY44PB|Zu7aGR1Z02CC2V1%bSWtp1u3ZqM}hWykR3l)f2Rq22~5qGIpss8-bBq zTB@}J(o|_EEoHlNKEV-4!Fml?_d(s?Rca6AwOV>%AWP2c1m_Aj%ctYOjqaavqOubl zI{z_C+o8t~kF#74S5=ohwd6rhH>{v6qP>%3Tgb#9uycb7{J3=)idmy7!N;hU5f!PWASa@YSgG=ff$H!U z`7$F9qrik|0ORzRh80>0uP6r(16MOuY@{518=R};BIwcmU_3=l!C2VRJXM+}lw>YS zfY6BWJG&)OqtvYo>H>}5`~kf9Xe)DdC<#$c(2H71Pl%hpwLZBqn3j3ucbIVOsT?A;t`U(+t61 z_dGGNVHpY8-_%Uub1^!nRYgcT{mT*S&xf)d7a>zKbXAwY}9CAR96J3bi(-ku47JLy$_~(M1^hAilSk>n8+w@ zCYgzPa5X|JC9UF&pfgAQ9dsJsmPKXW1k++{$+yeM&rOG1{~9KL)*Q~ayW#@n~fm&I~gBe@e!mi>@0uH;Jsd*&ux}AhnE&m2Lwc6 z3dMPC5^uyJ$WKu;gBjt#i&V9eM<$>Y(|SPgcJhY0B5$-YJ3k~NtFR@*_0FcIdI!g~ zC%hpX0apw8Hz!EpbPXWF4|>ps`&HiNcquGa_7hk9Wzlep5gAG9_MOWh#u>`)rh_6b zhc|dj2n$v@q~dP`wCmp(gh47jUOq7p)%qvBmAsCtuUy@X?qeliGih07b z4!m=V*G)7Y)wi=0y6)E_n$~LX>7CpZ#v(jYw(jS;zt(B#GR+`t@oXzIWAI<`VZLj4 zMN)F!;gxDWg-Sa#RvXG)NKzkVnJeEez&RyC17&hwDH z*UUV5GSf?wZWBZ?fCDU+5#AI%>8y+S>Vnl{oVcQC<(w#o&RRV(^{5$=^>m-Y(!TC8 zegfQFSH&jF;6HerXFmq@>vvI0ObxCt2S@YQf0{zcP%74x^M2dDQzm-6~YZ70cGh2SA9RMs} z-diSa^|}a&S**Zr_O5)COV7+CIQf{Kl)ft;7H6n<2kH7DMt|X*5Olxpe{+##rYsyb zjH${I(zph9v{E^{$#?sJpGP)>Bu;A3eC;`2eCSp*h(E(?(|UFx`8cMvc306wSY8Hd zxGSo)0v`_8V*l>&E}Zy@-yj3!s|_%f77O79zqX+khYV;BVmYjQSu*`O(3g9(I}O(pyRUl&4@gwt0+jyXBg{pbrqq%wLEkqQuUqkkC)Y-L%gTkb7y;0Y8HbE zAn3wtd?VZ0W*F}kms8Om>LGi-g3}W!K1-Io2H$<7l72Z|Wp$8EcMp;mG^PR~jg!{5 zg6x`EO{cLrH%US~>_SLW-Q=ncW)^54w)p|hRckXqs9M=a5TU|F*|!4u*mr(VX$BXN%857Ne%Oel(ZT^*!$u1J{StwA$pwDwy|8jnJyr!~N@2DqH7l zT0lTzf4?jSFqOP!y0G+~9C=oc@Qvyi&}iMJ-s>PIyI#3-;Q8w1((z@pQvQ6@>eNFy z8W}A!SRQ*4f*n-ljhS0Q8SA=@UYev@0ntwCENt}v(=RvFLRnnx&h>a7;#Oy?(n+$r zw8%kh*gc9TmA8@I?*w&Kon${C<}s1nze zYmM5#5uYpHa>;od>vmsui@bWeP-#9KLLPqIUOv)*P`lwS3&{Z1++g#Q6UDj;MSDq= zG$a0P8ztv(EhX@Pi53{cy`)smG}(3z@|_K)sd0Vd#-j77avF2YiHMiQK@VnX3*-r#e#c;H`^Hf3sGiV$&Xfy|0} zBeH!X_qBwSy`I3`2YivFn^v!9ZKOjb|EfSokvkx86XsM(*Trz%>Dz$?cO67;u^FVF zaWX%If9z;L5~nnS7+QQPa1TQ~^gl{eO$kHCj(nuBz%E#Xzeal{k4ff}APp`tNw5C)pBSb%Bx2AjcnY}X{z`9HE-Zzu7_3Yb{S{PXMdzShM(h1Gtb~~< z{xuAL5 zbsA#UmwNAkRDwTw15&32myIR~>RN`W6e7lbvUMO3Zme>*lwAj*)^WY{y8|Pi3D2?c zV3sbxJ#{1S9WSb*G3iPOlwUrNORel8cPzo^E>-12h$Xt}Qwd@jFIzB1a_rnGI=wUT z?pb@*0R+u5_-!iCQimtT%93@^$NxI8>~}pld2R^hJ@t9s#eDtw=e%<0CALh1Ygf5=eLjwp~EuXuFd=cc~igdEXD&zD;w&Nq+YPb-%q za7=w(A}5a^8-rOj18MVmi2W#vF9J3PM5g*l-!*8Eg(}%g{vfT5%5a-{Zcu$#Jv329 z+{fL7XF-9}Yy*=`7eVt3F3yncC-F??D&G)D5T3(^V|uf6RsK00iaVoysb5=p)3T%< zHn8SLTw%N8C@%G-$ygFS>N23WjxyB053*KD>0ei{{`JZXp%EP|Fo(uOtawQ)>9-4{ z^SRbfUG**XU(Bm*tM4tBRu0tK2)(a~n6(ux!0ueubq{BMTT)V zdA)S`*L35RvNYaadKvX%2=9lgu&r)Cogz<$KpDTEfZ1OL!=0Un&MVXVOz4SgLf}0- zR~F8PG&`>($tOpx61v@V>y*ec0+UGm*B}td&BxCet}HDm$7B;9+r4!$=VO_YD&C{A`{B;BBx>D_w^c=M^uU_gAXAG|5E{roT>JfNVB_(LUL7S z4qJ5%rm&%fU8z0{kb^%1tN)|7IbME0SRgC)OpA9mnt9U8K*4QLl}izEOKWIMf!ifA zEVA+?9`L7%&jfE#uGVCn*Cw48n_-#KiLYcZVgQTjStbwYdEMWs%HC4>3`&myJDLn^ zH)6u}7-{q@7~=htn9@&aD3eQC8*ZG(Cw|lcjq>Gvu-V6Y`0hLRBB*({ z!?#0@bLMIlGC@Jn(STHUmT|J8Qra~)9YiL`u)A8p2T0p6%&5B$NubM?-q*i|afe`~EI| z?}cZutN*JGCgnA9z`D4!8;HHi)X{=gvdxbP`CUbvvfxK6B~DF{4O%BwO1ohks8Lx6 zb5*cVX6qXIPR!WXDcG4*zuFs!f0ngeu7%-tmRU9ioU(7IjLCBt(&RYm!;R*$yAl+% z9}6N!C*aCFb~_|op1QIURMz|;wwi(FcbhMzwAuT$C>f&2)vYvVeW^Vi; zNae&BklIca8;m-#4>zKSRk5xt`s-H6P($yDu27c7n0;2#D2;x8%=p%Bypo}pI)ppO zg=1jYF)DSP%z50K0;3UZ-2l)7`*Fi3Q!Ik~;CK_t;UTYc?kX{R~y2r{defYPQ9@Nwv<>@DT**Bpb zmk#on)-?}(%Ne5`AU-o-+hT{}dY3jM_q+b?b@SI)Jx<^rCG~vVS;u?%MDd#t`MBPq z^Y~zt_=t_zY8@ezZj&KvL_>*hDl}HG55!)j`xBa=7~8#x&kxs;sIPQZ@YhuwzV?w* zF?s82R4G+s-SMuO$#_^fdP;dsD7u&{lv-4~B0!b%br)|2E}5$<%kkt*y31u932r-UnD9bY54<1216^GXR4Te%Ui z?+4+k^1h_h+%EK|Ynjx%h@F1pW{!C^hpTy0l`odLAEWNsL)zS^bwy7P4GfSEd{~9f zD}p);67FWaiKWZddpW5JorQ~1s-w1c)+gzO$CDm5bvs2h({+5_u+%MuR+Ck1i40i- zCV%8C=YL-1Y$ct=0{6~8a8}0Yj>I1c_d<){KoPQ_+IP7(E_cp@>3XPuRcc&Af1g)| z)3`A^3~>@PXST+8-KEk)j2m;)GGiguWlJ|Ny>|;trhwiF^Rb9<00VbZ8a)>neRD(7 z<9%yd+W)|NJPD3NM;9`tp@<{gqoMvL6>B7)9mjC^7^Q&&Su%{u?@$BzpLk2<1HFlc z0kbaruk;#RDtAXLN-USZGC};_iV_*4ryxG5;x~|AvwP9baT4*DF)bH$r&VA5zVFuO z4!Kr&-R#%PjuAd zT|K6*ht8;bwU0YQg+_zR82dR^r!eNKP(wM+OfhVncAA)ZJ=Uav42=(k#_gP zX?DnQK?hKKD5gR8QI6hlbomNdItZSr!U9T*?p8{wFT;6fw!s@@2Bx4;@+Qil6(FgV z09o0!AHS-Ta;*jF_*zJtO!*uOMNdz1G;kV6-txo zrFVALV_-Y!A+s&8_qKr;mUwUQILSvdk1Gc~4TR9)i3A+eRt!OGxgI2G$G3sxbM<6=?*p7|&){ z^aOc7HZW8mhj&55O6Bt4dW1Sl)&;W?pi2G82P%{$EBm9lv&)=Pu++;SqxVs&ZTE3I z-SxD3OIZ>xHFSg4DpvU(?Ze(Y#|qgsn)uAp4KVhx;)J9h2hO%y=#BHGsRhOZ6@_FS zJS}-XaCVd`zd`6o#d(aVVt8nTljR+>H=UpGBJArD-}nU zQ8H)=FLc*Y82QAAQ%*Dxxb*1@h~?&W-aP1=sxK{9n?BeAYkPM9ktm=mhF2|c$jwih z3Hh`+|8`byl0MAVXhJ#ErCzHjK$;vwWobEA`%gpg3o9uE*kaDv6wI9f7^^3fJu*?bHv(&paqk<(d0X^Tn zKd;~O0&kheT|BQUdq`#;d_SC?O#*PS9smY11kb3L)A+JXGdp9E=h@A)e722SIaLWe zchHMQzELIp#Ct2}Slq(v!^)O+mxVW)d61t@kSVxVh9}o&Y$`V7{hAPknjpPPAkjR>h2ehc(W0oEcqGp?)rLcj8@Zz#uL>_=r^BR3-lH0RU%U#t0pyS>!>(i)d36)2pSHbVkJ zC=4#qL$5a6Wi7Gvkw&$jLeKhh37auU;gCweq-QXF`-UJtt@S?Qp*khkMkP@ZS+yy6 zJFjz?gLZpUqWYu#o2$|rjBRBp**^y)wbcb2yICiE-XY|SO*VSS!aMNBGGK1qEj?P^ z9K(HleG+c?^ABqj2eWX4wuKFhkC2O!0&7UBTbPEVDVH|3mPMq?N!V~IB^}0dxe58o z>MhmE@<#cEMk@6HBWi7`bgSI`3BoOUr%YfxAN)v{ACIkx`F^z`&5gwR)9Dl2Cd)qN z$d$^SAvblkmde$q;HrNYV!N_?1Rc_Hn^~oO;U=IXUw^5un-mA>D4M6PNfWWof%`3H z4)k~Sl&Tr9%2j3CX!O6LOV51$Ez{3S?H}>8N;4@{_YIewyR0s$`qqr{Z{_Q7eEp5& z)i9#IMI{zX$AhL#ws(gHM^xA@9dAP04AI`&uCXYTh3xc{q20{_pi9WE=l!lR-p(i` zrRMR=;=P=wG^0v0{mmHF!tNlEyq#K5dm{W8hE;T7qN99+W%F6c&wT7XD9}h}dGi8~ zq^Yc@EPM7s8nt9^Eginh>Y@w8L+$YoY(ePD*2Wqc;z`2we0{qZg4he=M&gl&C>4+Bt)LRqADbV{j;%#K}O(x3Gf(U)O3McqU zE4Ha^O&IZXM{G%d{mUImc;YqLdg7&)XB+SLavLvs(P(-;Sl=QuCfmM;DSyz+-%3}x z+^KWuPpfzUB^?K#QuayK2?R#meiT?Prw^j(uZN{zr&>%4=}2oZ0U1ii$w@sJdy4L@ z8#W03+N&y?BkIN|d>*QBepZg4wCrnE{(rpqvz%apVTQ?&lx+5oAUf$z{fMH^OO= zm4t&sVN|q#iPUT#^yp$2w>Vd#D%PrWj*RKZQwWX{q!|QHvpa1#pU9hrJhL#ER;CHa zmfMCf#tWqN1XSo%RnQsEN%@I`D9T_Bw3cp0I4y)wT-M2>}m`t=+;WXC56=*!DYT^#J_B=bN(k>Lyuz9am#fg}2J=RFPe34mF+RB~v}2 zWqE(#1lh<7hj`vTs@kQ#?m{J#>Xv?ey@hhqCY~HrzUH6`jK$52W;b5UEhGR^GEA!P zz^f@7*n(=WvW}cNvmqD!#RJIvA-$(wxvUyP zb$3(fbw#&$*J=%^=3a+3s;&G*dIp#a)SIVtfU^i>z@C}*xY|VW4<)2SqEE|t90%A~OA+uiObZ7L4G75y! z#*X3=V~q6y!Bjcd%lgJvR@tb+*&rL%AmnTTI<>o#SI2zLFl<-;8{2Y|*z~B=p?{)z z8E@Z9J5!S}Q0|6+wzw0wFy7L*Tv3Q(p&4M79vsG==>-J;Bl&EgGeX27;Jgva*$eeh zy%^?>xaGI!O&^z#GfYl3FDj7je_>x<*NBH%femH<9>lDcjtn8<{dr2HzcD8x1y*-A z25D5+(-i4z%q(_Dm(FG>0i%L3ma2NS$F!FZ!=@dl$#RSIo58EU)DsL%cj0twb-_A5 zR^Ay^ug8hpN33li%_;I~GFS@!j#1?I1-#__2X6Ul77Y5j3ZX9ZlPGPL1KBCOM4Hlz z$yyaCd|=;F78uliQh2{eM``ksmyMD`_)2w+&)%-w-Q{75_avXeuOMlUDLr0CFHcgb zLB`~agmHf{Gl{yu7{BT2*`Zk|^HcicJ!$zT-Vt%*yF0D&R25ow#a6WJ7F9Wv;pA0F zqoWWsADbwi+CZ9jAHxLnSCy2je`gIK;?e#x>6F(gjhC5moZEus^2_ZA&a-ZWWB?oF zW&Hwpg}T43GIb(uYnpyT0(tHkSkdYp(keHoW7l~TWXYq!5Jf<7FFqrY4`!OJ#ce3* zE}73jsIMCFl*1W*lxN<^|9Zo^YM4KvCic7wMVzwjlGm*(N;(@Wt~jKf0;c_}2Rr@L z=u}(^-i96jVe^_~okx>)SDqoVJ%ylo_P{|?v^sL?B{aMB@Ug!t{7{+F%rV_jhktc~ z8gG^{sp3T_eL%%l%MPk`Wz(majc@4aMwyZ!PoiZ3ooQV`l(+?g`S<7^oF+2jB(&}p zg2hg$gj?)oCi80lK>ijt^Ty>1ESl`m&A)AeJa3Qktp#j4>cP;_puF)dzxUc;==^iO zjxcx{$>r<4QRrVa7us0%$fq>sp_t=3r{L~~VvW&EdKw-$RpH@jRjWu-R8E%rHV0jD za-X*x$2xoyiBWe~S?l7}pHS}pNS14`j^Sl*YBjm*e)gW@}U%MiXqz6H&mSO6l*v7tY!$03N3NrXTm7Nc8k zsqAkU^aJohMW~MC1lhXMnnKFx5g{}*R10#Ls|wf&wGhi>pLP!FgqDxlx?HV@-9YkZ zJ8bYSmEMg{RNhw-OM@1%>fv`}oiq7?r6(XIX65x0?PdJ|Ug8lIoh{)Z(AAnmIBsZR zMy1;tGXcZ-7c!JQZ0X+D!tnNLJ#FKWV;}>zvM70eHW>BxPip@M&JqLRs?<=~(Aul( z77CBSBk|ki;eKe9Sfw{lKHiJ)&Z;ExK>I|0bQ^)H{ti_3=GCwklIoM4_26MjSo&~E zcT*YlIVR=f*S&7J&Ps6NnyOp_N0jGCo1crKa%Me;xxWNEu%{wH3?toH-l2X6cNV&` z{cky!kvEuTT6FmD6=69z197Q+92LlTpg~6OGA1lvk21fvwYQQ36VLP4LfgDT`!If5 zjuVlKOvKN3nsV1-xUGqY@0%Fm^Dt>oY5owb{dIOJK03(eSg~8oXNz)BF_F2_>lGfv zG>%L-&fQ#Y`WedB83j?}RD$AN`4`CDq!T2d8-6atNn3O&=4l<2D=a(uo{AW=^8H*M zm&`(pjtEM?E6ZrPl$AYAGG-P|^T08$UPV#0PgdOH4NG!5|1qxuCw{nwp$w|@%_I&Z z&E(cqTs*0sdQw;yHa3*24|>n;m(D-(%ZXF?7SRB#aucSWa4VLNocDLS(^dgZrqBAX&+|P%C!^3<>XBB86XXbGo>@eOT@~0UlEFZO_ z*8u4(S-lNKn6pN^F3n|?o_@XNMjYh3HX@BFjk-?g4THmfY^-O{b{+vuKKPm+8Ooj| zXV=3-O<(5xH10Hlaid!?%a!$IjV^qkDtR|YB65J0%qC3|(NU&ABXR+O#q#Ms6UX&o zENl~2|H%J-0cAX*leW`z_42X|nE50k`IO3;3M@%5FXk^ZPm8cs);$A$+OF4Q{-{#B zjft7*Ta2~X=J%bv^o2H@KAV}v#3GF`Xynp0@Qz(ctMT3{3~VWFQ$*%*?g9bvuj1@g zdb{9xRXT*X2(QE{vgFC7%T2ov!b<#{>)>xm&31*yMcoR*O4@=USzh6-HujelO7D)Q zM~?r^g$@505}oLGs+u^xeYf1DyL5k`0zo$F30!1#eNwmrMj18%i~54{9YS#?`f7zU zn4mXq2g#vMa>wVUj$S7%GVEs_@POW-+)br7${S92agTBxl4&l8^`mlh1AjZ~$-l3l zitf{$RAkci__kC#IoZP+#k%0R{q`DMKp_LWg)f>;Wj_5RVu&kYwZzGO#M7PN|GQGMT znOlf!kL6Z9_i78Hkfr1#cQI!gRBWOOhw8#382=6od1IZJe`3G45bIQ!p~uVW5OsMI zN&EmYfBg($enAU52 z`aROPy_NV+OQ_sXCAyN?D($LEb9x(rE0yy56Hw*@RX$(7FGG5c>9H{#RKNk}Iy=g& z&(Lk}z03FH_xF}dC$E=DAT_)?nM)^mU*&JejW|Y8i$av_>Rt|ND%%kGeGMAp$=%rO z^glodVlFedaFVHh<(>+9O7)VvYQhB5RVqTLI=z=*9ezjmvKU25wh6|AWx9vYxtbC; zZz(nTR1?qD8xTkofLE2Z5)a!k6QtlQ;N3%+2tsrj#yf8x9Ip2ysW8%8iyM7Or6%Hj zCJxFYD+r7=)N>kcIgMr-F_;X~dKJGYhdh&>S!Ad5YmETOd?xaFqZrUsqru0uzF30 zsqj!V7lGnw0E)t4vi2`NR!b$f$nB4q1=8y%ftuHK&-~rD7A4V%ls|M(mYMSL8mN9~ zc2F1WroXg&5Szywb#{}i2Cxm%9EYwDttNH_WZ46yp$4^iUN^!~@MN1)G6 z{D^n>>6bA1%H@2=+gFehVpt7Dbd$s_SCWKl5<|ql9QjzXFsM6{oK1B;+bdrj25UY- zAWIfSy>=PjBw7zBV970V{)a)lD6mogq+;P7}n}%5HX# z5cfz9#(jFjv3qWE;_zI~$f*Sl=`PaxZ03_bPG8Hfu8zsZ^Fvo8mG|BV*XV5010U1Nk z0fgfkAc3cf%BAl)U}SPNw{b~D+i<(K1_1x`O#t8O=EnZ~l-zoYSuVFxQz}Cu*p)G3 ztTqIb{^Ol0y-(h0%@sVZ%A3pfOn$IOrAdVq~GK@rc-(q6Mk9+KA~bXe#Nw9MMK`375_|mpv?a#`CaG z)Hk|HcCJSgtjf2@3v<1065s{9<>o~w-@e<;gxoXNT52S+o$`1c&ibLw?BhT!krvc? zOKn}&^-FF<{tlt^O6NfO50!n3oE(YA;lLmexT1t%rK)@YZ>fc|p|R#l0*&i%NXHlW zm8X?^i*(u}WaulJ{g&UMU%AY6)*=t4r-Jp3f#jr@AA3M7as!F{e zGNg!eI$Fr@v*7LKx(w__l^P+tPI}A5Le-m$+-a6zxGWFI`-WMGTaf3M8z1xrWZw6X zsIzi*MHk?CU8gK=S($DAS18*_7fjBSClwCfBE8hku9syDKL-`b;aSmZX;8XMv4^v~4p}%J8Ld+r>-ysyn7eC3 zRP);!6=Y>ZwDxgOU1qD#%`ne}Fltj?vS1|+Q?-Jb=w84!Ws+tq*{E}rx)a-Dl|D0o z`H%c-CdC+OEtK=manQQdB7r&V2c+#QgbY7bdHp1-u{TNur}F^1({~&9!mzY1wF$?3 z-p|ajJh!vfd9O6v!-tvb+JV%Y47zH@<@*LL0`)kRvj=*zs;!aAbu%-NZpLn$|LU*a z|4%o`=jUoKw;0QZf}e+~NU_X&)vSgIs(W&4<@IIbVr={{s`Vg}Z zB@<*EGXy0ylB3i=?@h{=r%YI6AxtP**gj-8tjv1=PTZ{GD~%BuNwO8>o?mRh2Nu|o z{S7MNC#vb2F6U1mqHC&AaC6`U;k0xf*^NSv9FW$`wlywh7b1j?X@s^4qCg6EUfxf)F3_jMW}GahBv9+6IjRsr zp2nq(8UWB6c*Pq>U?V!lT-8i!jS4aWDH}?j&f?W)U8k=@4=6rc%mF>BWT#AIinC0b z&lO$20;FJJ_C1Qtk|{;`5aB+Ci4xRFW$8g}9%KVsZruT+F_LQ&iasZgW7GK9HMbN8 zyVPq;Ez=HTIkVlfkjfuaVk;l=?3HnwNZ0q)vo^P(aHLG{d|=_A!+5=n84%lxXxdTf zfrM5oN2B2*dB)fIA$jm|;3FlXUC4qnYpmn{;31N_Bc#mEtAC|@hcUP5+vT1`Nc$<} zIwlMBu<=9POdlmHr6+-U?g6r6AU@oK_Z5`@$&yy%Oj(YuBpQadmAFpM4Yw8|VL5l8 zDA=+|l6Wv%gZSE^G_D{&mNwF}pBI9Cf`*%~eDnF0@M6LW_PfP#p8Na#dCUQwXko^Q zCbDH@5jS~{O0JbCOMAd6vD+bHXs^8YjMpJ`J`6g=sT;4)QN2@C zA8Yxv11-{773QN@1D$340|Y#nEnbhBt7v!W#%vv~mlpT8U-1^m?`Ba{Zv40?1eI;` zwg@!L9CAZ}uCx56=M_e)2a5y+@Lj^h(%QFnN)B-e-C`2NLMsyE{mB`w) ze-pG8Rf!I=`CH`aY**ZrUV47vzt*b^+qutb)%6%_05MeTljvzFkDVqu_yH3aA*kg# zqji}NA2EEBRP46;MXEHR%Oa?!v_&>2A)rGQG2)YO$mCllnb?Z%EJte(Nr;;}R>&=l`M|da+pnCFiYqMN; zXHJx=z5H7wSMOdgwe#I^i|km;Gsn&ZKwfZq?Zy*V3U&5wsPV>?7!_!x3#3NcnrEbn z3*^eLX0<%jkVkK9hhna?7+-b&GQN0QA8ZN5jXq={oGJCcEGm&7*P`g^s6;n_1*y(l zHxHpXViq2c(MbM2gJ!9IC0K$&t=bBAIYJnfn+x zk_N(3Ir|avb&v875q4=N<*Z)QK1O>TL+LhwsU39b)H7M`%p7FN?;R{a0-~Gb?-)$* zx~f_%tr(AvX<%=$($zs{RLUlk-$rlzo>5|and<~&MF;Ysq=)xUjFE@*@Rk+I-W%s6 zxeR-o&Lo^wa-B{9)~ClU(24twHj*ETVf!Wf;GYh2Rr1q#T-D}`O+q)&PevJW3j*ezY4Xo7D1BM4tl2-$ zERkRT3fhce6+R<7H%Eh&aYRf)w@@7wx=F^69U_Y6VtdGkgjMfV@s9AN(M3)_>aCOt zR!l*JRDZ6R2`%?BQ$a#@yw{zoayvH@VR-y3t^GcS#$*=9B2)!qFwp)1k{gBNe^%vA z$z8!=|C|VW7V6<&uUuj1v<@$Y^R;u}97~4C>6ZKH>XYBHAVL=%&3#JQ`(qz$)A=s4 zswc^b%ZV*PH!$CE8YL5IA%hm6pBUZJ6wMjvFNb~!dU#%CR6ZTWqnlc#&* z_0h*TOc#r6$10{Q{MDlyOQIv?q{Zrnu+_(8_8dH;tYxMR#8erQqff)Buap!Oa*b63 zf7VEosJI!hSNAo$n=yTug(&h#T+O(_+H$6529ZHY% zy~PHhrPN$>Od-S5R`Z~sa)i;c?6~Jbp9h;edZJc+d$FWw-l-}%0<9l*h^-E6l(8SJ z-$BJ@i03AZ$c0x;3w68ES#k%Hh~)D((5-cJcJh+xbz(rGlf>TFj6v?oJG3A+kpoS6 zpT(3%5p`o8N0E(}dR8ThWn3`|>KU);2b4wX!H2kOA1Zer#33Cb)Z>^!03+5(9{v|- ze?o6q?^jWZ(sboX#Z5&XYA+&oey!Rr*CrJ?q*ZHVE1?QU@nz|3nej58x}@B*jN`fG zy6k$IaV;Z_qO!-e(Qc}PxFdmH0h&c0=ADkK!kMTAOA~4EFKWI2S=f~(5377}n;Dj8 zr4(vEPIK^588x@#F1Y91(-_LH9q{ZSjaJD8wZJI4Oh{|% z0i|Oe;tW+yVZ?%L{HyYAx>zYg&aH*%rl>pzbc?`E9eOX55{J2#2#OS=oBpEEO zzcrFd(x#QFsxOfzy#@FRsr^*WSxRJFb2yTgpG>1>nNGI{8mc3EMHkAM^5R)D$qLAr z_}9XxtMtN(W6aM%WssBJoLj}Hk%;Ey)u>UFGd%_U3Iozk#y#k!{n9TtL%EcjTp@ znI!JPHDznb0zBLx&-c1?X*fAQP24*R#26I2!x7j#lGamAzK zmuqB?t{H6vXG-?<*h}j6ud}@HTHP^cp>xcl3wAe>`SIv-BIh8CqkR_lh7(kyA;52C z4jQidB>)mtuXe)Ni3ccTikfoaO|t+oan?t(#iz-^-Cj3r!JxYZnnl zZvoNyduYVa7-2~FKyNd);4|jyh}UNZNFlJ~nygGSc)q$RaS9amohQ&;)zrqx-M``j zUUryyvg^*Qc8vMel^{iW>d>Xh$jUd$asv)%Ne6C|uKzDi7uht5pCR2k7gfUWQJbE_ zc3e*sY_`Ey4$cCk19uF>2lii zOWUdZUZ3x+mBzlzf`hbCNU#K9C-CBeK?6_Ayq92UCrxQEyy24$`w7#1tz!Kona4Q^ z03JovRyM_w1(qOLz93->TQ-;L>1@o%ckxb!E&=w8!B6(=;3W!L%QMH(A`9N&OPMc{ zX{@L)aJ>z?bpOGO10E7SJev|dt~s75Co%vTSqp&egKtItPNVooW%%)XJ=A!sf)U8K zt&=e^KP@#$`0*LbG>lC_6Ry+OfY^ph54(#`mj$nxmE<#xj?&iTPY`kjCZx@7Z=8DxG|_CGDa(hWX8+80gMtF>WqHjOfL6+r+rrU_ z0W#uQu*&D}cw=(?IDozNpQ3UkE7TM|OtO1GPp}d_3~Br)JU;7B9<@MK-XeGJ$NQd^ zg#~D<$9c|B#u6Q;bBgMXX1!7doxZ+@Eb+x}y?V|;VK?zLL>z}G+aX*4Mpgcyk^?!o z(VEm6CC%G(oA!P^1|mb{+#^3dYNq9}^*A(}RctQF1KTlJGSrM*Ii@dI9FZ7nt}y{Q zi-;iEGU(7YscBGVT(ZCC-ou-atkEhk9nrDZ1B90D({I&9)*99-*?$lIaq|iA6U|Id z8C!Gxm}C3%lEZ8!@ls|ee|{Qdg2)J%5>ms}WiD96KT0lyxzvd%q~wCGu>bD=$A%R4 z2JA$#fIT@Ub;X23W-dhIJfaMR&kw}y? zq+}MVFy2VsUhj3Xnw|*Egng*ue^udhc(mF{e91PR=uJG6NppTcTy(Ou*ll|7JA45E zBrS`B(Vg!zu8b?H+CXWed$P=3>h%*XqMCs^fK_}=VXhS30t5eruKG+xTVQQNG@a{8 z)F~FsnW-WT(2JCkooMb2qp|YFk>N^BC47=@k9Yyvo7G2b-KuJ-j(E?)iiA3F6M0hz z!i4)tcitzI_4EiY;MpfrI!ST*Er0;T8%JeC4$(aXvhd;7QB{j|Ju?#4+j5#XLwc+2e z-*H{Tvqetd$>Vn@|6&>RFuc}Dw|HQ?PH_hbz`m;5B4<3NX2P&Sa;tP_s3RB6Vq{^m zZ4+eELO%A<6|H!4?lxX6PNU#`nYutS!9NA|NvjsM&(V|o0}OX~2!dNZp9`N^meM(Z zQt0JwC|#z*NCi5`{+gaK7*KgxvbPBz$XjdmYe(uP@%>@C@CluVP#QTcqHyc>A76Zs)-wA zeGVXT+HdHhL}uQd#yp7IS|{va{Dt!86=Z9Os$41mOb2xg{5reDxQ6sM&@O&cLd<+q zY=Lnp+aILFqx#gC1=&vf3UWy*6%$75MB8LwtV}$(s2TWU#pOBB~;pL4I4@!Eo zHLOzw!%4Novr$2oaX9=A9U63XmXca#HI_DSwcIup#rVm$+}w8M=^g%kL<02U|#8-X>X zvv{6{`4$!3fCdlm7c(EC|E@D<3m2MTud&Iu7SkrP<>3@TodO!80c^D|^8%vivD@N+GR3m;y>{lA47+%MZ(6E_D5dnV6k& zRnSUS-wI>byy8Z`P)YLUB3^p53g^h!Eqw1L<-S2zGRF|HbW7vnl?ZVWYF8Jjcm*GD zjz(2Q$4UM@++>nPa&)13Cd;9#ylJV*X$LCz=OWaOJWlWHCAYsCB)G7ISDU>uSS>B% zrW2h2iw#q zui+`v-3XNo>bhiF#NW@G;FFFn;y=uTMl0J?6qzdr-RW5a3|%U1>4WV5=u>x0oSm_6z}lFIBiu`VDOn$&l$&;ft#(9YL1{=3z!iOzk0G zBlBh#`D8>4;=iPFhGH42-_(|EtjsTh``fIxrupKSj(NG8^3!NN4o>w@wMwt!>q585 z;eSqPc1RA8;n%phKXms{=25)jt5QbGb?O z42@-g?6x5SO^H0nQ-4%KjEr zx3Q^MkrLgH<*Fy|yc{G!rXzcgs=#KXBz+(9lIn+frGlZ;2s0pmedJ9`r}?NlTIsu> zjEq*WVPXiJ%>t+iJNRs{A^^UI#X$1?WTTd-W+dpa0zM~^nuKY%a&>GstQ+z2dTU=~ zppI!YlQmy^{jz*)&?0lXqEjTqm8kutK9U;C#01&47aH^6mx zilvy%90*0~Af}y>E){wN{pEV-s#Y3%4AO1jH>(^Iz)cwQgo-{2bNSaiaziCfsoU6sF(bvDgW7}BZ)#V2COc>1yz+PS-9mx1 z3p4HSFTGzz;WO+$gO*|v&MXLYBMmo!_)48m+=(q^uk!f19;B#!FU8%iQ|g zlFHtX_oXANE9+cP@zwpsv4i7BsO!*)?w7?;g6zMh%WK)Q=EY^Injo(*4?voACXD#; z6V{F7LAdq1&J-o&)h~mL_$>3u$Qc}zj*Bt;&~C@jRc)0~-+2wor(f5t{XRjIs8MzqO^N!V(0iD|<$G2yG5 zDv21sR+LIn6TYyqs%k{!JT??DFML?W&&q0PF}{9@r*s`!^xsgeV;J_r)`X3i*zwGx zJh_54VAt z^DyI*K*IFFel_1)0=qES!>MBPjn{JBGOnH1ZPd<<&~gr&99FJ~T%C(hl&eI2pjv1V zwG_zRU)p8cuS!}FA?4wUJo9Xa`+Y(g@t zl;;wJp|a)+Z~;Kc?v?>P&2njz{dbD(4LA#D|#>Ny2=E~9cfA5SbryO*wmVDV`vPEsd-s8Z(I zSA;RQF&$c=19ZD?rRlD0F8+51XI3-AE~QtpQhrd6 z)RE84pXiisI@s$#Zu0g?=YG}%6&RI+kvXPyj6LqPNd|kpND1cEmuGH<2L`Zb2yN*Y zO00@HzC%wU(w`~MTO_rTc7k&SAIU#|Lqk%{N`AKi-5QS`1m(9q;u6V-X zTZsnYyihI0j>2O>JwYSTR_+^WCP68j;4f8`j}RJodnabf@=>Nu7N01}Bkma)ECZ=* z6aS0eDET1nC81Koq-`rO!81BvdqHncD^X4*E#5A2>4qypt=A*^UY*9N{R!!cE6Oup z-q`^+WvaXruyt%61x}8IY)myR;w$vpF@?sbGWc^dM@BE>$xT$?plm8Yr@K{Pi%dE~ z=yP2f%eY(lP8&Cu6_Qb_zCj)xf(rh6C8}q(ayOF7Rh;o)epEhBflGg&gHL|{6E*jy zo~-tUDyWBD7|~wKMLl`zEu>-D6~sl0(8DUcS-VFE5$oh^ls6N2GA9QaR8kEg=skjfd3h&W~-(rzzd0hTP#2EIpMzFS!gsF@NDr48h0uAMG~+Chul?RCxxUEjfh!m-ouv`UX1#}D zSqSLeQdNC|wjfUYwK7&o#tXZfzGF38H7ftOnJQ-DT&m*u=E+fYw95C_$ z6<7|bBTvZ1Ux___x{gcQQUnkGqT+YJGN=$99Mr&kEu5BY3&bz%LzL4jfh}TVWLk!q z2l8sQT5hiNhA_hM-q<_)Lr6%vwZ=gGji<8h*bdk2aD>ZFh^ABrtSYa=@L7goi9^@R z)mJg>Rm!HTRRgC0$LUV8+zT`MGAU!-AJl6UiM=mS@g8su0DiMZ+;l88)zIyF9qVl< z&P8Sp*vff>Hjqvjv~n(Y>u4kI9e{3+DCZG9<-?E7DCsL7+&^Zr#dxX0>y%FS!y>qR zRs=A$O!8j=hoH5!GNQr~F6xN15>=k5o#R`N5MHdi+9U*)9n$+z{Lnl-5|tB;@@?@^ zvgvLDDdT>CAKxqv$zK;vMR{6z6F^{IUyY{PV2mSEj8LX8+tAZiX7=U#>Z+=@xr@|b zL?>pIrx#dDAyo31d~uAQURL&UPQz5T#~wilKcO>LOgnGOXoiMH+ba?2_Y2~C=}NFv zuKh!dEOiuidF&ypO{TNB7`i8{((D^+T3!#p8qcf50i8Ds!X&=_Mp0S0)a}R@vh{i% z<}x-y)vHH*~L(bJXI>7F}4tDF0ocNPh-gUe(*!m#d4H655{zoD5Qt zt|SU0vt(%$DE8SQlmk_#t7UOpzJJmz^fsm1IdY#~5Vw4WXKsbdD!L(nyGkI}U(AuqiY3>d5P19EsBtoNWwUeumb7_mxaXuqm~ zq}#wcbTPJ-d!~R*Kza1|2M84{plghO^Hb0# zJyP!2i{czW$XF_;A?fXPgX@J1Zw{{?z%&64viAHBw`7Rw!nyS-x zn=xm}ak8NX`syJSSVT~R$)43+OdX3OKlU9NdydZBuw$w)ONQLZ?~%ctEwAmw8247j z2WW-7mIz9+wUm5{k{}c?TjsCBZ5onqwIe8mr6WN;Hdv3ED!Ss91MPX{eX1}9)6-(j zdYk*D?)$+~_%GU2etN=dAvjJI7!vIG~@(RSZ-rF&r zAc3JrP}v+n!pLLzZk6qgb`=RgmPOBlPU$_r<&Og8jT6Lp-AnW{6#6zcUT!7!!YGn4 zy9C9Y&}s34Y`ufrF4gCF6+^{Z(y3W)l{p=(aU_dZS@|2>K4ApxmGxg6hLz&68E)Qo zZ>5XqCe`~EaNFk5v{8Z73y?KQ}FUS}==r>o&0BGuhh zd8L%JHp3cFG$fZ~>tug-bi{Yc zZ9yVvX9bb>U9u4oy~nJF{x1%WzLG{Kfp^mLjz zZ?eZ{z90`C<;(VMFjM%d_FghfSHdpT_+yxgy=zErO6uzEcKZhvrRA^%L~;C0{GC-^ zKESHn9!9vL5QFU>u2mJ0xvdEJ;Cuk_TcfRU+3}X?!Up?yY0c3>Y84{@RQeRliUs`M zkON%JRD$&q)J>91i);WEba`dcK`ZT?x1oPds8p_mhVs*;%6^`fUt>NNI@}#-80(IU zYGm*+M@K!av8lM`dQ;FmHW}`$9&cR9VD89x{%M?IV@!`>2xT`tX#f2_Sn0{n!yDyO zK^B9L@I6aaq@8@At5n$rKSmbP7lWmvcIvWea6oyL4bOmNyt)& znQmHwi?5{RpN_;uJIsXe@9AwpuBFhN%xfD|eoh8 zgSsO!`+XSk+(c^>4mAeIU^guJx}IvbnHCiu;J+70%Pz?Fdi&YANv2co`Ub(jTMv`k zLgrMS@OW9IlwSa){MP~T(Q6#BJ0eTfMvjV3kbl$GYU$b*!auehba8tpjy+UW_2M## zH$8W=83FR-Xs)Twf>Nn@a{7%HtkJoy86&tw0PH=}M z>)}G8F6~r&4BkxWUfnxr%BnJwcBe^S7$4?4DD3b%tn8Z+HFN}xhZ~=1{m@j`#1M&gn`#}!dY!X05170l6uG@~L?xVo)GDv2E ze0l}eKd9@`b|~8qgzusbaTbtDxX{)t;P+gkDFOF&k=_hE6TeRAm*rOCiBxx!*RB*X z+FQlD$X&O29g@xp=Aev>O!;mE9GPA0!otLE!Dr)Z6EQjqr)5i$fbuF^sH%heZ5MBRl|`Rqo!D=vAeqiK_gW4#4ZyCFb&;WylUq1_uo zLLIHp&9sYU+PLRwZ}D~ys_MRy^{(lFmz;f|LbV>~={X^ll^I$+#< zC2ZVX9Kj$YhZ>7A+^Bq}U;cBEZ+TERk{`}4rOJQ}68m4m$UUqFffbK6-E#F5X0f#j zd*r!3Tqv2NVjUiTP2R$vh0BsX2~ODWg#VqmtaZ(#bXt!3{IEAXh#Q*y zcZVU9@iON-skKmT70V=URH##Tl_Q9u<*YaC`3_my9#JRpv7S7Le*#ioxI?Zz%k5k) ztCY=4d74cXT8v-vqY~09E-Z~ri*YtLjd&Pc-~{N&`ti4<^i{0OZJ<^Oul8DGe{Iy& zH5Gdv+C^^T;%x1S)WDdpu+bXbLPZo|9$#;n zpdV_50_jSTs^Ug3OWcec)OrwjPT@&>#}qKgb1%rgh6s>yQgmtab6 z|BRQucQD7iM%|3%#ZC2UH`e~lRJrXpc>fF5G{~=SaP3hmn&qH!->7>H0y1T!o>lr` zCbeZ>QZn`dlVNd^7$G2*@olInkCHDk;Bacw+VjwG zYp86OtsfQz49U02=Y>JdNrB_U1QHe*`Ws57J(Hm1t8rXou8NZfyU6?GQs5|~LZ@7C zD}}=MtRgFMgBBGg&r_+dsz$;#F`rBw$vd{*5lrLNSZ49iyiA$*D^~F__OR&svUc)i zF_(k>YK_-MPuJKCHfE{RB3bdIS;-yPKh}e(vapOaFg?D}OUZsi6k+$inCk&;tu~`s zvD;{oO^2kn@n@I_RF zdyWPeXLNaOmaeegEN%2|iQdX-gLYlTf3^{Zp!_fcRIlRGt|KH%l#bVT`VQJUBZVh6@XFjf7w}EkY zUu$iv+7`>Ve>UX5?~bWgn_b&?SZ$WvB^kACa+F#;OTp?|wQZIo7i#jeTT85V zORWx#_}RdRu{p=Kqv{hRa4n;ruU*?=`DKZP->S6{8h?^Nq4ucD&u7|5o*mBNuWjq` zneX>o%PkxBHK@&IsY+zw1-;bf>?!Q{ouS(8&W48Mr&QN&G8j_8-^1$jc{<=Dw zXI;!U9WtH`F7k?=s?~fi4=ioMPh%ta4=5hoy%*(BA+(om|J`Ck%rGel9yiR zKdp{Xqq8iVKYiDa51+V&*E{|mtTcZM#(s+)UUla)7|Snm_~{UaFv|4qoPX;GNVV(( zzVm~7_`SkX9Nsg#`0JfHeBI#D{BjXfD5Tw=obrb_|Mt(TR)^dhfYYDOXA2*LhosdU z09icSAxkzwjSn?KFJU*3ys_WRk=BL$b*GyEXr`{`dHgkAsPSoD;L=EF^YZ{sJna(1 zcyb3XxucyKl?tR>imUmWI@AqI^b3yIB$KmW24cz5Bm8u9E6!dN`aibLG%%*~Yv4D@ z2{YLRF|I8_n2Qjx#Xj~7!q~@FL6IQDRii_gT*VoaZde_LI&6u4FMNtvAS1{l~D$kzW`zB!KUHImb#; z5VQTY3i$Ny&VPT_o$(TJoL^fk<+%~Rvd7tfpz=BdCrRJq0IKZ`Kyj-(!wtL+QkPy} z+gG?(mHSN?>62wnQ_fD}{a2^5l5#>-Ju^zO%umpYbD z%Lek@hSl(V`Yp9IqWk4vb-?Y4hDXov?x|fY$iJ6SF1g15#8)c8Y-toD|Feh{tn0|1 zz=bT*aURg8zsI1{FMya?7y0z2>K$-vCbPAi42nwP4DNu(L%4It^V-Iqj5qTT*lx0r z)wlkQiF!|C`0PSf(Dq$EHQ#91y?2K|ix$*{-eb)E}S5;UkWkGMzktMMzpCQWBUOXwx4 z6=hO#MyaLsy};I^M7H;MFspcUkng^|&t6(TV~u%*hC>FH^A_&VFgd461JwwJ@!n~| zqPEY4UM|f6M+vieZDk#v+ccUP53j?>Y!~d68WVV;j_NY;`drp@aS0-6P8t6T;H7GB zIe`&@jxsco9lv^kaYv~NFvfpIq}aF|k-o}PQ`(e4=C5JgFv@33$%iSX+@@!ZeD@&( zkM!dX`^ZUteHO=ek8*kBP)nypLL7WI_&q3G>&n91GYoV_b9QlSBNNS93K);L7@(ew z;m-?{5*`h~@2vqbvd3KgIFB5^0XkMF1FOp%Yd4wy z1aP(pF)Z>U^%Z4)A0FAF0c`n=B|Rn=Ps-HDH&$~XKU7BcQXgzuZvBF!`8a+I{F7b( z(*S$#+r%9C?-MpN>mQa`@kD`h1RFfo7$Wr_2SN7S#LYkeoN! zW{o2p0(%G3#NrFSuRPN5ibwHs)Kw!=-nq+TU4G;T--QfvTHWH1ju|ACIqS&Pd4^rq zj_1#@&-g>61I@6%W6a{U{OPcnKTB1c&NxM$TUE9oW#w&hqdKeYSB@r#UV=RPE}!p8 zF7f9V2l#xyKX0sX@!ySNBuLb4BS{v$#WG%{$Er-KhOh2YaE9zP8FS!GgG3-D=DQtZ zXLpN$c4;l9%_?9iZwMi7nd#W&Tzx_P%&m=(6-p-H_U9bS2(vMF*rUs9c z*_gF=82Fg-7AqOBhfQ5s#&G|3hGtt<1%rQ2#0K>wg0@^%({>3bzPV`Jw>z+9AO<*vu;65zkrbV+pTOE`suDk)MlxoPq5L^kz<1&QFs9Fb?63V_ zvBQ?NA;mVoFx;C9osp6dU_{D=SSH(pFHPcJX5c@cAsMz8CQFUy?9x8hh!@K^-n#m# zk%5PX3z7WyS;^CR>?bOL_xCD0q5GQPFs;Epig_ox7)aS#d{{ zglL0ePz^QN`h=OR_8Rx(lH*~=wJj`QXFgl}Baa_i2eRZDU654gBKS0B8&LEFGh7QL z{I)(rJRHijaZbw#mC^^58vOnV0#HmPu_Z9C{RxNC_7?(CIu~fE2 zqK2yL@m1vmi{X&pf8&6alIoJJl!TXFN0_9T>tt;)+g{9DAM(rD_qfl5n3z}2P`>A?DRayHl@5Vgl-KgL+`Q!;dM*H#S z+)!W^^9_IYDRXV%4bal?IX~3QXQaG){P$T1%lV>+PcgcJjjCp}xq?4;&%%+zQqyH@ zW5{GLxjoY9C>(iq7fUw`z%}3tP<`SakSbfsAp3vfyWGGCHR>Ja zue^a-4^?sk-v@k}ql~D>%IzG@BDg?8OkHZlAd$D>mDEwZHUPI21ZP;pVu9Aa8O$4t zu0wx`1cge3mARAOR;d2eSs-`zJ6+OB9j0E)H(Yo~DATc~5)dze{rmM`xFfGX8$(yI zlwtL(Y4YzO#vFXbNxjZ*kWxoEb5-;C9`g*+<5j(|{!7`}%(5a`wwU2YH|G7x%C5V& zoIz$N%K5%GVy>_1I-%*=Rb@N*Qrg1Aa%td#tox@SI^$x2_ zRwi#vqx_+CJ&Gh7mPs_N;)tk+?2AxZhfJKvgp2;-)wlbw^`^N7`Nx0aeA=MfLu8k~ zlQu!KdD!jT)%>t&d_{*b^2 zCqW2>P8{+(Fs)B+mLxtfvSjTEsG^qY7uV(|BUYAB+f%kK z;yYV#hRja}ysed$es%;Dcvacvx5D|e_d060&ElfRg5i?wBZm4u2Dk*;$ z00mV0i$^jx06ZFc2g=U{Jd$vbwdY-9P5-GRrWKP}?86HzcE^79?8so`i6Mqp_SI#v ztKMSQ$%^ZG*x=wwjcNR7yk^#CkgF{v&LMGiQ0wznog5WwUrTOa)z)-ju0Z`~mRecUzf1Z4^&N^F2@aGaq<|CXh|y(wY*%z&XR3M zakPD%Yf5^Qkt}=mFy3oEyqehxH}$PVuhg%{z)yZ*+SzyDu$Ps4c*;OlyzLPi$UkV% z*g%ne-(kW~GkE5M&!e|nwDvEUY(Xq*A9tHI-BmR5-F>k7*B&EM{%Q>gj@Q$wh`+&D)c}@O zm`FfOE&%&71W4GH8DZL^LR!*?p#W42vFzQ+6wUfT-)DLs-- z|MA$GXoJwFW~za4$QDT5lW1Uwe~tM3 zxE-s!JB+P%T>~;(Z{WXQUV%e8cU0T|+i*#|jio;QQ2j80#oSZ3&%M52=-%y}u@dnX zYW-@aSNSN!(jZ2Mz~#eX09I%Wty_?!Sa`8w}nXCrCmIxSQjOR6H5kAaD_8wixb z+l<#N!G9nr-O9SiU120h-Bqm3cR9bLY6$L{R%DegW8;o@g@{iDFCYF2ax0VUO-Kh!y@TC|ZcZ9Uhy{mlmzs9(=G{|F}QeO3`NhjY7F-oP=577Riu@Ku{ zdIZS5@yr@T5~z$X1Q@gF79c)9Bju=3QcLc2fT121hRLxYDC;E)MAd#1&irEZvlbSd zEvJ=n)#h^+jZy2C-<7js#3|6y)Ea8htec!k@^BUmnC9y=<%_=jIx?Njb^RCHeM4rs z)R@BduhxUk3i1H&4=w<)buZJ;Ri*adKh0Ww^T5il=rS%W(&~u4Cl9YG1+u$dvF?xs zpE5ws3LbfSfY)AC@eL1y4SH7}U{RsU48Q-7ZQo#8)B%wI-G&=kvPQK|8B8G~sie%0 z)A5M4AE&_#FU1sa(_pb;M{!?1<12{&)N4#u%L+5s(=%#F(-A-c(*ieZ6rb+B2dNy1 z1D>N#G19+ZJ2km@g8vRmcRHlO4B&Z2W$E4@0C?|20?+>_m?X>XfNR`pp4)tj4SjAm z=y5a{eEuC%EX!-~?3C%eI<`HZdwLil3k0E30 z?9zEnj8+(rd?<+)CpoDS=wswd)9-jPY9|xNsL+thWo|7pv0Tr!FWRi~c_<1b)#S~`b&LY-eqsf8C#6=|+=0g?HD@WM0hm!A?Hw}`G z`ZCC|>ulSqT*A-JgO<>~D2m;BPKc}uM3B5V2sUR2^6QAx?AuVKDF=P)%#s=RAcu(4 zDRznf1SUT`lsOiEfr3Avq9Lcv02TA}>_*CyoFR*DvD1-PnaFm6$!jeGN?TNxTlCvl z4G&{VWcf^3_OoCH+4C&~NV67BRD*bRHcY8{udCWe6+u$t2Bgr)!#0+WKw7NOlXY1c z&q5!L;@zoZSU_t%+b(Oeq4Dw@Rzb0!2yJa@l8U;hCe|3}bo4V`*t5{EsIb{E+1(j^ z(FT)EM*Ici&-|0s+0{wllL0Jy>I>F$Ugcih)QiHoS98}FV=!A;r&^rkC&JseG842v ztye8tpDV0x&kja9N!XuU8qAsGLMCx91~mQJUMouLB=CI84X5{xVy>}!tRv;!LblUr z7_=DqBY$Q~qU@0~S&vJ9KtzV4(uM+O8_e;@ksSTI)eGA`%d&UHvhRoL&xOq0k3Do=l1 z3jkjE3m7#|1V%Sd?38NHk@Z)sM`dppFfr<1BS&m?S*Y(AR#vwJ_&3*MgS1|`Ia0R< zFDe27K9R2p`=}yyxjn%8v608y0|ol&M79}0 zT|4cQOsFx$h?kBnd4i(!hPEZu&0QbH>-f9S!gAfCG6>>>B&s=>8cjobx!8;uKKKB# z?)?v>(fvcCNItp@TyI`8aNw#FTtmfSzD)5pr3(=f&3M|+@l$D|*USqUQLM^e)<&jqk^CigT%#nT_Sjx;2XRKt< zDUm}#8TAD-qXJpsXgR=RxELMi?2g zZ9iz(s#$rX#6u?2;z`|zQ;3ve2)SSZ z`^SNm?21HMzItq(4RSm=t%z(5wn5Xftm2}KdV@Kj$R%nUj(3Q zYHSB1L(ps7_ zNZN+WfQArlm@lT+-M(5^Jg3O7OQb)GR~y;h9YcN#D3t8FdCe2b5idoLU0U=;Fe>PdFH9>jrQ zvx&6f;#)qO!0Jv+W@=L6(})--uO?+Tj7To8D{84Kv~FZfIstwv;WF?l8;D*8;(j^{ zonE8VhHU#^Cxsi#ySu+pE*sQ6Nbt{w8^o2@luFGJXe-75?_H`R@|47U6}=cF^M|v8 z6y0nn<}^ga_jQpdk&(RG;1N>5M-`7~Qfhe63V@=ao4GBjH15^|fIDCjbU!-oCj3YIq04b}fHAxgY+nexL?K;+2~ zh~}kLm2h{BOa2s$bCIB6r+~NyU?qUWA3L|Oq*I+ zvr%X1ar|LQ=ME$TW%RA`;1ZnjuF`JX)OB;7I-wkRA1L)(4TCMvlNu3ST2fas>%u}; z^#rTjO$;V@8-7^&6MIS3i=yR=3^soFHKdYV&qmH4anh(~D%*Xo1EmwHI%mK;?4|d5 zMsFI*Dg1n@CQGL|V?lFlh}`G^z8XD6?tRAhu|1Go`w~ITU8IMsJBM^UtXDKr;fHkD ziYY*hK=96|0DAsOC$8P27+^lRW0OcVywKbl3kWj zT3R+jOq#8*RNj0GK{G{XjBj>GaN@h8ws zQI& z(b=loBWJ66mHy$luir4ir>|7GEcjNY*uc4eeN-e1{}Cruxg&@qso; z|E)vD7Ux(>?x+G;VRq@`&&WMhS^e!FBkrzEPw_~>kIrNXQ@)+M${W0PEtIi&2tuZ# z3P)%+0M1<40m(qltyMa!dT6Jm;)4@XwzrO;+QSF-+Q!QBY>G@E&UAKiHFAH>~4+5l#y4K%CeRG*j z0XgY$gn5T=VADV4BNniJnn^7eP`ep#WO49w1<9Q@*ib8o{gaDF!0_c@-aVPar`f7v zcB^;!m8x{ry|@Ln<@dDRo!XQ}xZs*3%oyY)6U(sKpX0f;*M) zrQ{Yenj13USJ6m*y$IDlXbjU2`-AnZGnnnrBepdp1}EFMjQL1IBu(7TSHz~;Jgcb+Lt^bT9S@8DSEP0ve0xc!DTzujH{~IbyygI4< z()>J-dZ5R)mze5kzK>e)!#I_&+BgX)(>bHASm%L~L9~RF%Rj@4cjmIXT0cS_{o)LQ z8dT4>msGBI<4Q2w3hIo=AD1Btif;$26OC2uT>B7Vl-yKCv^QMjG33;cX9ax9QX#41 z^^#iheOn_;a{hs%cBm7MWgB#}u_Yuu=w&49+y4Qsxq3#3j2#43<(@*>Mym!_EYxB z^_xNh-NF(cwr-cjI7Cjd!IB2sb0ZbWA&VRaIH_xvm>e;8qPa;b3YJ|N&RF?jIy=eugsHCS`8IAuS;rLqcU;d4 zq-;%I1l3V5nsXfx*;Z!gE#bf2#Wy6k!N@^{g4g;!> zCpjVBH-XJd4S@QsG0fLK5eU~`1A`4#;VpB22Y*vilTq?!1L8SRwm#~PZdYTNXPOcv zzA6&Jt`2u&BaLFo=?6okyV&!dKa6yl{jov+x+(y!p3)C{KV^kgaN%;AOR{LcBzolO zCFXj^z-(Nv$H!^+Ga;XUfX_qa+*yU|oCwAYUG>k(A@mGe`LLeThRli#F%LLBDyyQs zB>V_?N0YcNnGadgnJz48fU33|MXa*Cm(G9P;|)`s_xSXFR~|X2N-6A2hhA?JohL_Y zGy1B1OdFG9*kp!_X^7w z#dh7u^3J9vOVAJCFj22)1J9E&I#M?yFyu;`$;)?9E~#auTEAR3eUm-%`)v%pf{Q7U zoHLR0f4#&;;@)4t-}dx9Lp`6GjxIBn#oCk5cA^@d3Q^QheJRFsfmeSlmrCWjoZ4qBgPLiR^h zedV8ZmkJ=wt|n>9yEXGy3?a&u2a4qa06M+ENzzbask-sUFZKpc;HFHL2U$=VJ+G^B z0MRV;l|^toFw$gGTO(GfGlJ;07$xXpvQj5DbB0Oz71ZM})eWeP53h#nMeXE3G4kf6 zenyhY@T-9+b%)B%|A6;&Raxnm4b1b#31IngS1{a7T~b&100^Y$HWdQxjgc3Z;471` zXnFVoQrcCUDXB*pBnP)5>FdV=M}nrqrG7OQ-?bYm$@@O!{Ae-)epfjlq#tE<3Fr)b zw% z_MG^TJr8RN2Nn!rdDC>Q1C&7G19?x4xXI8A$+^TM1?52G-d?2BvlzFGx*U@QOk{8c z0;-dq*A*X(Er=s(K{Z_46!Q3A8Q5P9hJCq)(_`_A_lb5&Y8M1`FID8`PFaqsv==P} zI`~~s#p#lwoAARub)Ns}SHmkC9Vop6y4Df-7ulRd{7BBzG=-GMjljzCVj%m<5F>$d z<1B9I48A+VT12YBZC7+p6=@R#A;#}VzP!F07?1pyPgVE$bJTiOT@-A10CRsk+DMY2 z4^Y{U_2MvDrJAz*t`HPNXb{I3ygzH+VS)Wu@u${&khXLgk1W&U?b^MWw)}{u`L!4t`kvYoTIWKyblq-*$wD>#b>;tW6$6>( z>akX-lZr$kc{+%wX)0b1NiCOMhdGW-p3Hj32|Lbz7piJ<)LDYMBuDZQRnn#OBcnp1 zx*|Q}Cb0K3<)z>A3+Ako1#NMh01K_F7`84=8BhRK#A(8_3wzh64lD zZOD9O6C@E-En9Ik;q|L%2oOplwBkg&Oin~1+jY|-6WX&e;sm^6lw+z*+X;8~kLw%M+U24>o|G00;!K9!Xc?mkvj#dMC&aTC^2Lt&$ZK&>QnZ$K%13XAkUXz z_RLZR7)n&Hp7&K<*Zs)wDmwD(-)L4zUnwf#RB-g9%uW;8cLmlzu1dAguegCMT5k}K zgbHCdWNHrAZ6R`f1As|U0ek6v(c2k`*dpJ8%q7#1sdrvuOHF8ahmoHVDxKZDT20Tc zCuOZ6r4vUD{J>E_cjd<^`O-FxLGs@R;Z2>eQ2@7sBy2bu$Stb6^NG~U7pf22Q>7Bm z{pRG#=xsgyoW=RwRN zW0<_sTK;V5h)n6LXISO=D8nJQoX}V!Jw0A~JD47uQ3e+zJ_Zi+No?rJ6DVk>YEero z>rOIa6x+bMikA=T^TR88X^=|VXd$iEK}>C6718n_!6D!CZX>TXOP=>NlHvTcM)Iy5 zycO&>!c_8HU2SJ#zPz7|5bLE5{^wQ3Q{QwpI%+sjjjfyCL*w2_rbe7|%a+jpA-1G} ze7zgGo28Q5F2od&n3U5?Er|n3v073Kruv_J`7PXuI2~u?$>>)A$B9=M_YW5|_Q!De zrkY+_U)!8pjQr(vv86CsHtv82m#RdVvA|!Om6?XghVY~_O@O?XjfB6lfpJDR0{qFD z8Q8l?@vi5^p&L0J`fQ zJ0yPHh8${}f>O4Mmt7vv@d$S1>{c@zgq>Mq}$b7E1ZLKyVm43slj zoMHX~RWiC3tk?uIUapM+Ogoj%ed|3?vR4&lJFVxmMp2c5>e|wZ9LeFW>Oj`UmdWT97#%?qz7>>yeSEiQ37y#aEm|*oIu$#(piLYkX9EnrnDw5hO|hP_*GNRbJrfPE}_E+JM73Khwc9}qAbjw0}e zS2Z%^dJc5|kscOBeudUve)+&zDqTCJRLDO%OuR$))I=>4`_tqR-1Geyj4aYYqNJt6 z$l)AGi?FyRPPb$%WaNO3SR(!Pf{rpu#V@r^0=B!k&}Z|JDz`c#NPeB{EFd?+)=u`{ zLPZdEZDaGawpj~r;DxkD0xq1NKD`Z1R;vKfx%Z6(`RsioQrfCeg-LTjcGsCuvLD6& zWbZ-1)k`n$BW?B>Hn}hlv0h6R{cnOm3U2_NV7WJt+--(k-sD+9r|W7T5rOmCA6cH&6N?eR6}Wm1Sx8Y53C?m5>!0I zCS2c~{41(qs*%KbM7Px@4K=cfTu{fOWuu|CiC{uD#6s5V0WRL=f@sJChwo=96sBYd zf70(7Ve;)AR@(R)ro{Q&Y!bcG`jE`#*5yO|%ye&Z$oB zzbB%ymCq6h5vF=Elj+@zpk1wUY6fB184T&EFV)m`UQsr2+rfOIG(lBpJgb&6Pjrb< zp*I2V1C!ljQ!LVc5Su+rD>(AM^6SVIUs!XPvI0)41~d0m&kkQbte(VPVI}QOKy^WL zp?fL_1#s3V3YSqgL3*Wkj0)-6CdE`0lwze-B;sU$oDoU!drju10iLGKA+ENRmX>Nt z|B{f~Onx22ccXQ8f6gBj4P@f`&}FeQ0mrN2P(_6_2>i;hA>xuPT4`Y|y;B zr3?mJ3Bj1>Vcd?^XLs3WoT|Ejw=?x8Vh6~ayTE8YSFh5shLI*UqrmFA<2-UO(IKtt zfxz2(Rt##qC_!Z^S3%-N#D1j;ybJT04-Op zffD{fcej+j<3K3Pjdc;e~GV zAdzszEmar6NqN!oZEnt@AP#ttpYRJ%m@Wr%O5MJM^zKdo(4nr$qs9u;3k_MO2uJH6+266%>VX^`|Hjsp4P3EHc`Kt- z_SG~ZWnK$3omF?#mp30m2a)e1h32XjZ>v=1Q^Gg^Kk)^Yz)dH@^G&Qax!sdL6VgCm z0EI!6*${&5ZRsEtnSkiG4V?FW)U`TtY&RRY9GoG`-azEmI0L4h>2_6^H>W?e;;n&E z#NG|13qNrJo{|*9D+e~P$L2fu@1)l-)JNcKkVz^J>F!u?H^{^+dZK64lvq685^~5W zmbT5Nax9S#n=>sf<%8tAzmPWX=&`X-SjKSqYY;Dted073IFif7GLM|O&XJZh zNUIz=z=XTixqF?u<}SL2ko2l%F}pdq;Pp<9(QX&2UMO_S>p_0C%v#S{uRh?}@eSGW zt~XGJg(Epwj@4u9AXKtzz!>%rxx52-Qp0VK?AQf6zMe{G+(aWs&eTU@R8e9_K14(; zMA`*V08+kO0UyY8tl_gN%6!BlXNl}kTl!WRGD)w1_dBPcn}E!GiN1y)3Deyzq_mY$ zAU~dA4HN5vH5w!il{6ySWd8t^?i-acK5mibft3{>A>*H*J@*dfhuBOQ;DPFvF)*`4 z`!X|67M#O`%PK2Hv0^AuNPd`nxDL2=TVkZcLJmK~D58RU$WyF(dnxSr5J zQd+@(K8ZOpbUCYOsi!5#=4lvVAF3iPll1hV)H3qK;C0NZiV~rB%~QQ7SnS;0$;9ax z!Z%beo|c6%KdPaRVSR4g;xJSzNGNySiHCbyP6DwD) z8l{jEeHdS%DJ~|rt*dsuus}}E!^ZnsMXQW&h6O)6*OaBv(7=D4_}|=IJ5tu(RjT^1 zv2u0mes3;vx$rHJ)@ljz-u4gVzMa%1P64HMia>DldaMruYx`@4M`oT!9t`~&=6dAg z3`4>fjFeY##zEy_;V7Zfq4I|>H1WMER~xSlwY= zDy`c=jx(uLD#ay`GcEE)P?>~^7AS>K*{wRNf1u~sw;aSP3$;H0rlnXZi;S5(w z0`q!o@VF~N7FA#e9N7a#=;w#Fo4{tmFJe{Jr1rLSCjX5j=p;hIs-oQOn~XfURu6E_ z*4>Rrd!T+nbWg1PFZD2DD(7JAy`yWR2qSm8mHpgarV~exp;k0pW*xD5;5kkO$Mxhf z+&PCSL$7?4)K=N8*H*Fuu9rfP1EuYFgSyGT$HMQFCu$-wOM&j|8_@|H)pdexG}dBu zJlR~&X1CVPmF3HmEf_)08P05s4Mza7(iSdtXILYHcGtCPK0fkwP1dn5QQbzz4&~Z3 zYdB7WoZ-xt7KFwxts!&UAS_-lV3E2W84i-OBIUITXFiTh8s<$Xbm4Ec_l4(j$b{94 zd#n4$p30lwTj8lOTN~dB!b*>#yi0OF*`0$P3es(LC8?Oj$5gXg%d*N~2)phGiC&VXXM$HsEK}y9js!vq1DdG1?J*b&_$cejig%sr5-Y5kv^!uV16y$UO=agb<*4jH+>5t22+FG~3%D=jK#?_0 zevCk$y|xnDwTK)jyaNS|MTvs?zB$^3d7@m@T=Q#63j#&Ha~M^-@cIqI*WmTFr=*6Txhw ziLzVAbwH?5mw6O4;Tg_6_KsRmVK%gBX~nXOgV2n&V0;QijWB4E#z?i}7%Fkf>C;oS z?ln|+STp62dP%hx#k6k&<=J~n@eP!VMK9=%HlyBN`!ZEKq}+w(N}}`F9+cJ{-PMm4 zH(}AJA9ruA6pDy$dEY3P_V)~;M*hKi?fELmq>yi-%p$yj0|o*n&@_mjeLE#dKD1jC zFlGySs)|lU3{?>UvlX=BswXl~uF=OsVgi6~Y{^;&+t6$>@eLq&=`wVm{;xGsf{E5d z4rxwoz4&3;BIi4T7|3YL?&AkZR8xMfuOA&F$v+}98YdjlcQUxW!FO*+EPkMv}#9vW)VK79n{TF~SQ z%;yEly{&LiMNJ4ZXBotnl_h?&`H(F{UaJ-9oN4LQe$g$-`yGL-y|9 z7)IN3mQO?WLTa5=bBQy0ek^*7`#5?Sw3IC?3>M;H!C$Hlyc8O1tiqS9G$f)U08z2p zu2ioqd%iXvG3;ujEhu!!J5x|Pbz&+c#)?{xN`aBTtd3CsZ$APsr7zzd(U|VA?wKNU zt{d^P@&j}yf_^gXui&NeF-fxKvEh&lH$g^Ke`hINU~47;Dh~PqCts;D2`*^&DzLoy z62hT{UeZ_FQ0SKLt}{cko@geoUNlYX7gLJf<6?&jq9O9;L8An*=4#4`yfhSWW(DIQ z$f^s(u|+)%(5oIKML#>?29;M0BD z$VN*OfmwOIktHLoMmjPFI*i$9m|9$nOBU8*fti33Lpvt|+q<}f%>34fWhLoNGFq!#~z;vP4k#jBfAl!q?AnX>1DJMD-tO=*%vT$43C2v31$v^*(F;8%RR+F^ zzjlV1&gEsu7A^%<489@*e2mAVNh7YSN^-7rM81R&A zkuU?U=t5gjsZ4EJ291R4V-nB;4gbZ(IYGx5;YPsPwyr3K7pm=7clfqTfnzXcQlm~)1l4-db%4a=}KeG z6}lYFhB?dh*#4M&dF1kUHte`Y(zHP^HKV&cP9pYwB4}D zcmE;F{=Dy05vN*`7FXtyJ;6o>TZm3n$hS$hG>wr8`Rqk8FJ zu9MSMZC^?zy`5PuJ#AKpy#1%O6l29++j8xQ`0v9N0BBt8Z<1}*?iL9LggZ{PD*Xmkd%`kX5d#q zZTn~JpiMd2=Kb10ZlbQWl&Pzs3d-OH;b`*KgJ#l4%Xa0g`n(@habC~v1NhvXq|s`? zGFguwjqH!@FJt{tvo-t?-9@U<^ML+JM2~?(e^O3{9}hCylK_@xsKTFZY zJ$j}fE%I7QkCC9qq3+{4Qvv|@L+lv2`7@+4TQ5qbpj6l>>H9Y#r@g8R8gT%TgA=5g zHquxoTyr>D$@l(PIL-B9RUVDfOf*2O$+^?s2p^9(T>e@D=?xlaW=j08)h%#KAMj4I5NFGAS_TC$onz6EBNc@SPScAtlg3Y(!^uYEZlG<1w-M!?(ND^m^ND6JM`j=8`)59gDYfSdM)|;4iY)%6(*#@Z(I4tbuN#CQZk~x zqv6%usF4jHIU^-YhcD`=8pY9iu|w9)2JoYcQ#5HfgH;US_E_E-044mV3d1f}q(t4g z0LG*UYVci9AFHR00nNoDWbH;Y`Kf`13(F_Nu02RhkS$MOQpuih}MtiM#DlrwRp{%`u5vlEqq1JFidB$TgB@j@6*}b*~JNZsUxgONx+i0nY+1jC)a!9^B zP|m4Tn(!nR{VNG$(n>Xo=%f-QyX*okmsRBV7Y)%sr*->82n<1g?H4$TiriKI7bA&i zO0vuSFy0s*Z&;*HJQ{JCo|6K>65?3@kkzJdZ!fGcDM%KTAPFw%p1Cqv1$Y0a4tXh) zocW-%w7;BF=NtdPwQkJfykV$}*q$i){Dv&l5hp*rhczEeZ#>b}oo;_+!p_$d!e#5n zSTCnV$lI`gFAD=;LH zpUU%Od0C9+4CPN|jXD@ww5z$r7$TN%`7Rlz|H;jgX9Xuo*F#nZfrs#}Biibfi1kpi zQ_mYB#qT1|-qw@nF@rrEJI&Pqi5xqPPqvz}aR9uOptRq|b%Bm{X0xbgNZg)V^ zNp;k!G?wv*sE(IU3fG!4^AhCs@nnt>gH(0;-g-h1k}rS_Pb__N< ztUG$pIwXIt=9d{M$3II^1w37YrOt75{W)4P$a}fShLq0`aWCp7kI}uUa^qk2Tdwjd zj!>UgetR9H_5*`l*ykiKU0&3orYS7WJH?V)Sl>J^!rB$NB4ow96%S{}U5RP{i%CR7vPwq5S zepYJ^N7?BBH)AjyP6}}6+wkPZ|4@G(l_XU?9ERzsVwtB-2irM&VEEzK!ZJvuD?B=5 z#N#)0H^-v0>k^lYF+4?&?I!&fu+x>QRPFbw)Amm)XW-%oAm-Q1EVCo9*TOC5Y}%^b(iM`DiZa2 z50?Fro>QHLQCB8gl>zwvP2|GtpW)S}steQ~dUBlny_3oMe*q72Lmw(L9y#Mt=&lJW zi@iwN%x23?_4MjEW$Ab3*@S_5cRANR9};sf{mS@+-To40A3e*i63eeu6eekXVjR*u z9i4tTpVtvnHD&XsJX%>V4_BJWicyqj#ixhspiIvm3V3mllSE!${8UZ@y&!5x&gbZ- z_jG#$Ie9OoR2!36j%`-joZEq%szmD0jIvE`g-qxICZ6c=^1%hrHT& zpIhuF*()YnYf1PDe4f(16Qp%Gd~-vU%-^WInz?lg@EatOms1cf*At+Xtfg?**Rh5v z_Mf4ed%>j~`|xEY55v5Ct*iiNdu$*AIY)7vW+RBIq_^}P?X)3W=yT9roy|Y{3>u!R zM5A}5Q7$uD8zlRxBe)rkAZb_#HLug-dq}$~DQT*zOFUHT9ws5Gukz%xDGu4w2aNnq zvaNP7lS}zU9J$gb7)6`}@@h!n0m!_8?jA@E6X6!0>7`wjiPl(Fc${2$*#Y`DG|kce zit$L~7vOx-9mrz2N`=|~y|n<*LxAGST82vow1dz0=O$=}bJOJK>yTU}no@wWyyYBy zv1rz`Mq6YNtR;&UU;_2aW#vP3?_?Q!o`-YwycuF&#C)r%_b=TqB7bprPt`6a=IdR1 z2u7`=kM;;WUDptWSCor<8|^ctx|9Eg=L6n_deH>gx_fPolubn1zOoZpa72Y+(v5Gp zT(~{jFk>^pg5)EVdVFg__#OCn3COGd)weKKjaGUuOayrLcJ_Rf3CxvkX2c zH6MX(=`AfEGxIIo;9W_xal$F- zt^`$mZg$Oaa;F3&zt9uHWc_zYi)DKLFuAzGsfu7HDB90yg5)fZWoyZk<%=Z1^X!I` zthr7`0+%QW$c+q#T#s-nFB9!g4}QZc+Y_ax_7^Tce}dxj)ARe0>YS6R6$I}Yfxpe9zRb0?9Z|NUv<;A_aDStOFeQl(#F{k>;G1W6g06qtPlM?=N}4dOtCq4}0>ymFv2RIDh&8hZAg5u# zXwuokLX}s9a99;e)cQmUcKNHQ{(QY8UE7XcSVyT7rBUiZOoTa9Dr8Yhb#szb9n51_ zRE5^zstqjtq;l(s z`|Y|rU2-{YW6NQ~&Z0spq~Pd;kcH{3vM(8B6xLRTcW3`C1{*opn_1ydV97WJFA8QF zcbsfjz?z+Q2Zt~s+Aa2oZ3jvd5w2X*E!l0$1Q9)@cpOeG#W4(Ns z?DSf_^2c5zr|yR+wy5Y-uPT;XS$7Q5et>u8 z=xa+6`QE-DJ8dy=%}S8xsv6CW03b9#sb5-R+htz^$gsBV3C0kr7(!iM*BFSp!ogvv zyswneR8>B1=ya*(pB9N;iD0yqneu!w(4mp}VCvgdfGS%r&Qh*G?AG+TV*MD)V`dYc zKcEWJuNj3cy$Rojtg6C^r2bHTIM)EIiyk&yHtu#7$mm;!O@gi1Dm$9-#_v^~8e8;+ z0k-rxN@vANW0Gh%6UR>C)Suj5X7_MfDF&NO9xAq{yh>UADjqukn5 zSh7o*AHTLzSC+$6CQ|HC4ETTbato%nCsFdxVNZOfjtxuyz!aFmWdWYRqEQmN4a~ox z$~3h-(M4F}FNyK8*J0$zZdHNpleIDNN_Zm?3%y%ZpwKa(T4ypOFI2nog@dlLAI(2_Of{t3U{FXBd z=JNKC&eUl_>k%$Nh-m}W45*FN?1OE$=LJ;L=`~QjqRazK+;(h$oQ85nuxxOeIONz_ za71DG8CqIm1%umkY|diJw}u#5vWWy^(kzJfn2$95iQsMmo_U_ArYP;zU<dQy0oo{ zIUE{kUDZa<;2uUVAEK`N1Le;UBaM6RlGc(l16WR0S+R#z#=?-XP*X0c8j1tD$zQWL z3y0|y?Ig4WBZn*Qm$0*FCP3O_l;rM4hxJSX=yTMKcbJNjvy_F&zEkX`%{D0FGC3CN z!nBp@2UA6w_dpmN8jisS9GzV0l+2UY8bP6S746I!IYspeC!4Z)MdX7~$(n*ynn4rB==9b`^hFA7_5G&?nNX?+)6nn>< ze%NrWOKv|#=sfroY4phi$TzJmBmJzhwI|XgNdm{Bjm8yv5FR-#z-jR;BvP6M-BOSs z>-J-XSg5+k@jSnStXPMrzBb62B{giTt#+_c-JZwqtjQ0)?b^1g7>b}CI@rh@w-7J z$V_P7U(aqTR?0VHCAgYO&3=%oN7vLHVo5xoY%U2b0Fj@$oDLau!AOuPs{#6pI8<9( z)zEdZUerfs+mXdvK7@1cQI=c#E*Ec+9`-6n;EWX#PNOWTRt~7PD!P1CPg^K;Q&HEC zR8yC2RA86mDkUnpHIEF_3mQuL38#uzv&>PKpiQw2Xs1&BCeC=eeW^ST9H0NH{>lq1 zxU06taQJz?43Iw3T`$SpNT_|_b0eATl04LMQ4k{99xExO2GR4m#IoplO|VCEYN@-a zI?^-+XyeDIDJRBa;508VxRTIeqQ_MpXj%p1ncG(KsmVjp@)T2H4}WqA|c8i4b_!2k~VLHs41#_STXJafja+1UhVFM*~v%yx3s zp~AF(R?Px0=?SS4HwX3%?~fF3qY|61zs)&ty>i%9YJ+?|-^~b~Y8J;yHakjFMf=}+le zgcCX|8r4;jCimw6tWL^Bd2=w#zuRHBK+fYeW)Y%qF$e+=ULY1Ob4H zdU~YneZE$ua28e3BsUf6&2eo9MVqdSnhdIm^+=-piGJI@)j!K>& zv!>aKRbJ9^eB6HCN**RP+i7XqZLPp3+s{$i6JpjeJ0f8pYo$KLe%{I+R$+G4OwHHF z;!BvX#S>=MH+!(La38y`Tf7zz%QE|DSz44&f^W7ste$W)+Uy5t;(U^Qi^Xg8uugLz zknHZ0;cN4E1bC{MhB*XS4ED+Kb^Ch*JS^NCq2*~4dtU8eM)_c z{k?%6c4CeL{L6hTezr=EAdk&#X-))u8+`13?n>Su4_h;*Xhqsip9H_`N{(O;QrMgZ zMh^NU`xRI627B16Ia4dqPWfc`*(y6iJhjdC<{Th?$tTCpUD+GrVe{sEtz5h9Q{b0f z*%9igYj!plX%*UIpHjc#%HB{9&@dqnA75XK*es4PPo&w+Tmh28eeL45c*8t^$6TX@ z`9%39NVdgM#S>-rGGB)f;(U{(*y63?0bb^tTBJ{R-wd%?9pRparqkRAUI+W;h}-H7 z_W(fi9WB;pf^UIjTOCzBjm$)Is}}Dw+qYDTt=_60AZl)h`j-1z{A~e_Y96~e$lL`f zZ1A=Fy92z{Jb>8TtGRr3`X=~i2RN#GnwZ1P523$#G2#H(-2&^pS_Yh$ZPX}Ci5#T%O}b&p;C5`qoya$oMe6j*~j@M zS1Jzj*7Sfk^CFDZ-7lk(E!d%ZTAL~6chKlyznn_$V6W~0k>*t`&u4;PL8a_qM=eiV zGu6DV<@?O`E3H%+Y@ijGJl4wH~85ryFCm9pb3t>0r(?AHZz~{gNvehj{CFz`glMEActymr>ak>Zt4KWG*nDzzLW9aw@w+ zy>&ehgZWG=_qpv?P&qr)QP0!GTw*@gmR0yX_A9Mi9O|v-fjrC?$PZs?^V-53ksgQn zs_6&!gp1wc4)aEOpck{UPnd6%Bv`V;9Q8fj%~fU~TplOMmf|pPeGdd=hWbSMc9#r` zt%@Vc)6-mQR)ygPOOC}|#T(^;qRbjTvAz?ez>;0X(ZJK&%rtBH#QV;cQcH0aZvzh` zX4XUaEEkK_7Vc>1Nif%&4dBrYVz;`(y$wCkoY~mN<-1c7tl8m?Xir~rli3vEa!~NH zhkK(v5TV(^C&~AeWLRxg9gRHw&CO;j*!q&>Slv~ z%#J?kzK^BUT3prJ*aMlGU42YnUw=!0t(wE`ahW^J9!P?4e|vztn%C}uZp}VES-w&J z2?5#F9ENAG`M%i?nHJ}t98g^C|0p{1$CmFujwku#_~h&->6PA}q(#t@R#94fk`!?V zMI0lkqVA}oDE)krG>tos?_^_qQZ_Y49GlMh=`lKtI7a4dUF$G{jt=Ad>M#2Pte@BG zc|2b4Fdb?5LncSSpw;u-B4s4eiOwVUlA~dGjN(;^%#lnd8ve4$acn;)pVuJrMiQOr zvE&crM0Pc&h}SF%Mlzjg_){mR!0fg2P_Z(KNTto>A#y6rn@v28*c`>A((t!W&R_>P zl{}-^8%3ni6UZOQ*)U2E^9semC?<`DkAa-eZsgSR++t-k(S@Ew{!A`}5p|VUB{oMh zU1<0^$R+F$=K-%l?2RV6(v!(kJPq^NkX(l1QiX$zRAdFu^nUg_59>NvGk1A~&#cM9+6ilrcmG zT|oXyZiYcPieDu$$1oW*d|~8Pwie0fH%PoOL^pam`5U>7orM(fn*rW*~P9Jv!# zkDZT7m9a!7T}WObcfsu3#Mem8u}mfn-ype%Z9*#fMyWTJ=uXcfe<%0B;yBDNlm_AF zJ8AeR$%AYQQp=NWPzgZfLV|vo?*^*~qr*Z_SOc_sP(KfP)tb+-y5NKrPcqWU6@0dKt_8@A3QRa;& zdeKGXWAXy*s|-P*EEv!9qT$0PFT>u|3*0hg0@0f;CjTO@!B8J1sFIlznBFve>Eune zAITRq$h-+eHochqo4n1gMv4T@vS0#}O~a>8-iOU=7ocIvL?VYSAzzUXVGnE)Xu`~i zOb!jRfc%pkKq>{sFmEE!hh9ehLq3Bwd00>w7EEON&@ddxm+VHQR^Sd(CJ}wB&Ht??kT%s)=0MP0FccSZbS4johqea%q@JR0ocUTZtOwUNzC5UQI?)ok0~GMhoRZ zHPfGlAw_lNSh%&QTdqtd2GDEC7^)lW!K-MM+?>n|pkZoJJvlb+1GGWzO(q7?>&XNv zn^VGljW)}J$;?0+#u(L)LvrElN1;q1jI@VTQ3hC@(4Z*HDU6YZxke4-c(`yRrSPT@ zgXm3UJ8B4s5%^sVMKFaKM8lw?Mt}~}3*8E3J7O@sh3rU;hHVSK>8UWcV+PYO`KWOm zKQ~|4pzyXMhR|PM?IqKxsj#3o2{qy7 z_RLTkW+OF&6W~?~jp5$*#4!46vO6^!G|*vTVR*1TGmM5|NzLapav}8yS9TzV)7!~j z)I!+lSA|vK<_^qo8m1?;gcIUE5H^H+I}ju29b_MBIj5ETTG$*O?7)nmVVqKB92Sow zLL-zK!bH=gj>uP>3`UW({MaVZKro9E_(H86&(JVkBKj_NUf>^vVzwMg%p?NE!w% zwSj~4^dfhJvLi8y-c1goHiL2)C8~-rcVtG09wjHO|UQ^zI`Ubju?$nrV?h_Pg2{dd1>L$m}%NI99dDDoA^iSkm>NcmES0rwZ3Z^j=X;=-^eURODF&eGx zLQJBo$pzFy5Kf!KnrL$uW)cm%g8Gva;8lu^(cUh^7xZy*5%mo8?O}0Ybg&Ea1r5uC zddX?z)r#HG%C5v@`XsrOdc!Ge=3Nz6MVq@alWABd)H_ay_dwhb?d?iTp-+=5s1KZ0 z-fMAlbg(Nkg@*k?ede(E90{saY6%O?kQ9Z0K2u0EO0$-+(6DeQ0fO<>5~I?qC9HHU zSx$*TS7%5Hl|e0IrC|e6VF=FGOWaCjI+0J;ldGt3kaVLYRZ4R@lTX7+qM{KkKVQ9gcIDh|ow7fG6x!E|OS4LgcT0^4DipfSn}qJVB7H&V%<0yjxCG3E@WfQF?- zbwEt~N{KPXn?X#Y&y)Y7I)j`)EGdi$W-!xe*kV*y#KNzYxMP&vi0Sl2@++zv2*<0E zsu*)OW;zXPjp~Wm_zxrvG2U*(4EhrJ4V8_Q@Lx-sV}jk78N^v)C%Yd~#)qeWpd{B8 zk^#cUHXs#zOqv4kqxoU+5xdv}kqvxYnhU&wb|!h1_?A5c*~-^S3t|JAe%OS>9`*=i zC)f&SEY@8+i@Z+kWsgSo@J-U)aH8vn)ky4Pk3$afEzT%<+9@;r%kT}Giik#znq$zQx9)4J!#1Zxk>(B%ct!v9}>T1vc5E_&_f|>}=vPdneLQ zK+3oYSZ{4H*-Ttv??MI&JhGGoQ*S>kapD?#4>AI5X>Nk2w{{Wvl=z*!4;d%$%L)<# zz5TG&iJR<$$Q1By&IBx5yO?}V{J}ng%n$@*yAw>=epvIwZT2x_zMxT7pWw;XE+JnK zciAV9C4!LbQ9>Zw4||`u&pv~c30Ps=L@Y;JLWYPYb{$e7z`{}zO*wuL0>nf1Ib?$X z56eyTW1Hmo2qkmCnsK>W$RjO+x<=uE`=XqS;K#9!=d$R2?yYFIViA%)hBxTXqS_3iND#mkz)c|*rUWiA3taZ;wAe&V&kn0VlO6?IV)Hnh9?<8Oj!P9y`i2UmjpY&dbs=ID=hC>|4>hn8-fF2mZSzy7nTpKchn1{ z?4IB>7Mz%Jt^qJ1$EM|HT}!etBX3T}IJI9O&QNEQo^Ew-;k($GoRN$}8^ zgW%UlII-OB=p5`0Xc&)!5Y)&xiQGPDDfSU(=`kmP)?nb8`lD;Hcn~!nCn2m+aN2W+ zq1&+>6{*L{2~kZ1rxSM!x*wYaD#ue!h-;!aUASMMC$Ul$%YeBENsSVW)-?1YwjE@T z$3;kMVmUpybJ084NwDr%1tF`6=k(?-LZ4%IKnHm$h_ISOPG9Z{^dt6B#mdFp1XhEC zH!DXa@^}zN9ycMcQF8{sE4K0S95tDXts)dPDV)LZ6X>b(NuZQGtBCNL_MG9|Z_pfh zshZUvTTMjNXu!^Wiw>1<2kGQlO+?mo;*90~fKHO1gv0<_Lqyf2awc$pMCZuwfR^&C zA);%#a3*t4p{4SVYSsX3EupN@f&=^oT`P|VaphS{#MESPrgMKox65;q$pP3pBDN-z zGmHB>x?esC)Rt!*5m(cLGmm>0Jt;3uW(~yF6Y(`!U>YBz7vQ-q!3a)lxgXjw+m zgR3aLy__o%#wxN>N{pYlxbec?ijtI8BWn|$LK*EF zxhcZIAmOqGk(=;#)L{GnxT(Skit3b-L9ESqdupitD{iK67HGO^Ft2zAYPkIyZjNx7 zqBW&;5Nivrp-grkH&?hCL|)clatq#(8fE{MJ5>0!BCB1=VAlWePE?+KFE>y4J*d8F z@Vj_tYOMVS?j+%HMM=BX!K^RwRLX2W#4Qk>2N{?(g!~duqbAsYuWrdDzsnW?iLOP zDVa5l{2K30&9eW_-7lOFUfsTA7;78egPLRigL_ms3$$f57;(HOHP3#Rds4V8ytRGn zFxGZFi?Z39xb?!-AU3mxliTrLRFVBL_oDFY@T?9c!&%?py{TgRU)-|m!tX(TR)cxR zv#G`QzqxmW$HPlHv<_$Oz;mb)`z!7v;d#)bStH0Dcpqw+{U7*!w_D-W9ZE*9cH(`h zQv1K$7U5G6sMX;2@qW}w`+wYz!uR2=9a=}QXq=!(JDbN9i9oStnMfMfQFc3zClSR) zWNAuFEFZ3?oOU5ELevQ)Y&Ap#xPfxnrM!4i?}!pjtBF;K=TdIFoR=aR44O7;Bw2~~ zr&ik|d8wia5!ITKk*r<#0BWs0hLdhKO?#2gEo9yj)Lq%UlWOXbV#rhT>Ol`4u6ze;DDCM=M^9n@gK^JF@CcndnQD57;^X7*52v=}1OeRk23t zc5?>tHiMMT>*UX~?PZPDea9Kb+a}72EbbJ_3+-j)>8dy*dAmScZ|S5RWBZ;pM)y5u z3~wIFg$I7b% z^&Qb!JJzfKoSD39Ajk7M`^VaTU`^Eh#F@*xEvk+z z?i?B$`hhh`SIt?#dkFe`OK0sk+X2=Wy5pQhyk`)*9PMly7dpV2tUJkB%6kKX{d#Br zINL$i6y0gg3f>1%Yh+93(6|s}kUEA#@eok(5vf|U&CjyxYB}XRF(fsasWx-S&&t=; zb5`-fLDJ`?`pvdOtf{)QoOQf7aaL4uYRDWq#46A=a5nOiLE~>p)sD9vW=+$b=lqY? z8N!{TskZTsALi=3}`-9YqTPxX(t9bwJTUE+Mh%NCbJwWNl|hamdY1voyQ0bD>t zns$QiC~KzfD(73?5Xgfv(`*w$M_IFU*ExH6qrn#BrTHh=eq_zo-QfJdn<%c1DozVc z2>r;KqYH8l@uq@L*pjB5X#0sZS9go^BX2fDNJrCb6GK0-=IQQme&#I%<8VFAKhbuK zHD7m+bBecI+#1!A7Md7>#8ubGso^=mNknwfPO??A7U&*ue&MZw)G4!zZBnS3Rit~w z`IWaBtVLcI|0LVbtcAKKoZooc#97hBT|$#WKeLK;&73Q|UEnpgbkTlcJI-39d&>Ep zw-17z;G|;2i<;alMQG3)>0S65R{VUET?CNpwq>&=(;HdUYXA6R!^3 zNJLldWZOyBQr&CLV_w+>2)8o3+9roivX<#uIDhf3fj!CV>Yr>o#aga=%lVskTU;Go z+%+^gbc$7~YvsJ+Jp{k9rK@&|?KEqJ?mg!p-ZRL5TRviZI%$jvg^2r zkRJ{fCr|6Q*lJh~9UqbM<0M(iVr|G0s$n^GC?e-4gXh_z)mm+}ta6N)a|I& zW)0P{TsjF7!|w(r=(^T#wbijIbTTA?pDig-wrE4v5ailA3{mk7;EE#BwfVMs)+(I> zX~!P|L0x9LEk9JxTCIyfI`T(@P0CC6=i7c^t)Ip3_}e5|F~u37g3zz5EjkS{lD`YQ*Om70vxsim>9&ikuXGv6H2wt$ z9W%SxriU)FzSd=HH?oNT406h&-E1>LmsmS=ImlA}8!&X&yZL9>F0*#(`XVd%A0(|YE!{#h zLYG;zjzB2*|Bc2*fWxx|SU#N|Dd&qJ+RV(f6@~(=N?k6piXSf3#O8tJvt42B(hWe? z@#CafvBjC8!q64gZk-X?$WN9UW4C|@v|VL=s~e2`kKY;6(4(2QnW3w!?{q_vulU`h zg|XMc6xyz__UMKq-|(}gC9y4;p_w7*EOaKs$2Umbu@T@BZP!_Qb)%4P`9mN;&FpTQ z6}pa>vho~Nd@CR|Lm^Q${my(}GX~kuA1^%`JEwc&tibQgzM64}pKk>pxwbn#+jN83 zUo###%AYB{9=pH0Wp>~O^Fz%-@daytw^6EOP_5m>+9qL$h`W%;`ml zdQG>PpK9hJclbX`i{qa6Xq+3k%^a(lk2LbjYQU~adgAj;cbMv$1;``*uhK1XsXZ<8 z0(Y37YZgL@a|QhCP>6?3cbVfgi;(C1Kcq+F=Jafw7r4uus9A!9_)TDF*Y?Eco9;0u zYnCD{{J*5Yp!77)XYSFbsO63}hx;I|p{I8~ai2a-t#Ewcw@P2fHTMk8XYSKys4_>J z<>kbqSxOtxNHY`#o)?uV;x$=j8`DVFP!30%2Uf=$v%EHQG#v6Ni zi-^bcdFp?T&d^~UjxX#LEMgwh7pSd{He+2I@9w2sNIao0QeQc`K^T5DzN(jbA@hX( zjoRjDbJ`E$8+v&c5`WT{sBawEf)d&5_~u^0h0LGyWooCR&3bbZ(B8^oqL~g*K8FGF zb47xtx4D>Urms-D9Bp1)onY+kEhhe=uTtMShCp?ekxSj_xIU!(Ro+RV8=!QETA zhIcU}LA9(Xp}BW(5%Y|` zNgZ^w*>`&anyp++Jg0-yA;(lm={F^4vdxQ`=ky=c5l5S!uS_szdlwUb)3>M}9kZda zJDgCM9bC-(P2Z-DIob?eT*ADh?^9O^CXw}f~_KcIebtbw91Be5_iSi-!bA5!NWZ6Sd^(Ve4QO1!2YQNKDiLwztR zu`0*BlzB}*rY<T6!Xyr=)6 zo;hv`{9*Z!UU*j!ALzH#-;UdYvg)v+q~^ZC70d_v9re<2ADVYN#2Lz!#7DZ7dgXWs z?a8JjO+WKW<|F+t^~Uk1AP`mw`G$8T@gM!3`p5AM+StQMh5dpnng8ey)H}yZL1S1g z1RlyV;uHOm`q%LW3YV)%RsGCm%qRLk>VxB*AQbih5)f}0@tOWa{pa`~XbpRv)Z8yv z#(btfQ=c851uTpMk%*EcSU#4W?L?rXQQ#WFOfoDVbk9x!ieYNVNW3J$_CeR^6hjG| zffo`%l41LxTyln?IHrftL`e}GACw8saHw@g;Z=m0VnBP^#m;C{i{(R#;-v`KC3YB{ zXckt4HxogML3{#xm@^5$13SbjN;|>zp?0}58JeI?xJGBTGhCn09`5Xbny^a9S-f_F z=M&i@ot>e3K8zRYf_8@I6WgPmT~P~Gi@SA72wr>=dyKOilt)+bDxKNEz|gnHIeVfu z>;c}O^Fk8igAdf1jh0}q@n&7n!3cctKg8J&B{7Z))hi*I@nLqA(*S*wLZ#80oeb)e z+moFGQ4gk88TDSsYJ3WNJLeFnyfajV`k<2$`oirUoFf1!(W~5gC4@J=2zy88XsD=0 zsjBqmaz^Bfw0CxnL;YC3szL9CG{+ZZPjgO0tFa~qP>rEIogW7Ry7-f z6-<~f$==UdhO*=wHJYo0h{%WAbxsF#X9~3@*X(96pUQ4eYIlDn1W~?J`*`OO zDD|$YtNNSQFj2lV`$Xq4)FywRZs_lYWXjjY{)O`dS|Wd~Ztfpk!$kYK+NU_r0JOwO zMh7S%s`6>=R%adbfQn?z0P|W#=}Wgyb)G{#a&@wCfEThWUxs~}^8(-u8OenMf@_%= zUpM;<=Vc&_^vUi4N(i%jnf95^Yfv4IO0F7UUdP1xy4zd46)k058N_z8?0u z&f93UyePSOKyV!s=j&;o@4OG#jXfD1sD!-Bmt|kzdprew`P^Li%U*UP@p`6n8X zS0)<=dLaPw^|mi^J_C&6aB|_m;Cd#(mu+9-e2F&7Ym?mrl^cjeA0%SVH_%XCO|BYf z-oPaK`q-B{-=QJ-gXD&R-VH>O42eeiGI=OjZa08R%>t`=>enwdeP6}#NdI;PH znVA!TQd5zlF`7LL?$g;FPJs|ps8ftauZK|iAVhPDf!@eSDKrK>jLK)QS2)9jxI&-e zHYzs~Y9FL(&T!~KN2OF5%^MlDufKhbGg_!sO7MJH3x<5LPI#b}55R zTl^Yoq<$B7pm2i%Zhj_l!LLjLBkvm&> zQ{it{FeLD$ABtlAG46cfePCvsL$Iw{NC@>ma~BH#R0P`X9%9<+??Qc{KLP0NOGRV5 z`XQdJT8Iqwr?|_7?-ZeSkA?)c`k`CapW&7XKPy=6xkE9p7BWPA4c8$Qgk$YfhMK&7 zEoIf$aVvyj;duMpp&qXmLPh;A+%>}JaBcg7p#iTS>S+Br?gn8JpgPW>*jHLe8TG$% zHw!z2o7(RlYWm8bK~2+N;BFOm4Y#zfAL{u^3$dgAH|{oJ&v0A&M?(W&`JutqU*_%< z_6sK=SjWEBLJq0F!rdht818AGGR*X~Ka-lNzsB7o9062X?l8~SS_mffzr!PptAy3 z3(tg;9k?T~9a_jk^{==Oh3CRO9a2V^cKCCsrTRDAKZTcp%F7+$*`bAiRR0h6neb+~ zzeB-@zz#n&`1*I;m%{r1L^(%bJGGFQ>i^}w5&jt-=&*Z)X{Wz0wL<@a`%d^Wys<<5 z2+vL}M5y}zxF3Y?!b2S%jR@@YL+P*o%>69<9L~~kO&G0(OjXb3AtFHprb#iGXg@(w zdW0tsg+<_+T$6{^LddG;@x-F&2(6~T6rlaU3Frm9Fi{e4SWXk>(?aU17xKbI9U@Gc z-6oUIucylOVqUbUYlKBpZ}RxG5X0)Fyf{(M2%F}SDd6)1TA&Z(C5ie)kQ(kttWpcP ztX|Gb77dK>Xi`R+D*d_CDt$PwgJ=XmiMb;^m0AdD^^v^JqHz&^O~J@Or5|VneKfDD zXbRw8&XL$IEhM-47+yEgjEI0{_ej$&{{U*8K91K@G(VzIQ$NzPOAAr1K7p4lS`rb` zJQ^9;@&EEpBo?FZ&T-+?zmv=gW^=P2x3Eu_Etj=a&LJrSmkyGNP6^$({0 zr|--gCps8m=~zF?^Q{)*V0{{IqUczJt>dFnfp7hQN$9)srijh})XE)=eW!(dSf9?D zDmoY8>6kLw^qqew^_9LGZ-(eHFqyfdJ>O{|FxGeH%@*B^@OLa29r(@!uzAc6B5NYbPJI}P&KZ^QRKZLhc z)HTx5sXouMS38>ePTv+w_l&f4dXyK~>j&gRKZ3VY)Gv~RA`bgrn@3gY+j8oGk)BQ| zV@%)s^QgW0(Y!sP5rCxTj`4i29YcMuA5&f`92p{+x^3$S~k)JxT#-jUivlX)L@>-EAQq~zCYurzpt%o>M%g>6|Mu92i665A+1_Cj+kH{sEt8g?zmHmguL*d7T@_HvB+Ov>xNkFTW=`9k~*o+(r-JldKS!mp>4l zja=UuA7?y3PqO~ZSy=u=bSZKN5H9Wm_!m}4(955SZbTmFY#G;Zfd0aIg0rOjh3H=7 zVovEe)PXc0zK~7)0mKQ7T9yKTRj+r^+nW{TiyPlUQ?h{p-`q9ifj6rZ;+Xm-g3qtU^`5PM7P%&L~luazfTovQTx!URmB>yf!K}Eo(vvv;nHC z_TA-!#9N{|rI{yG10DZUm}m6kWr|C4Pl`?mfk-dyqR zsOq$0s2D3N6gr2{JoY{PN#0WNo2cfr z>l6LQZ1dUo^{06=zs(^{Z}2+e0V>KOnFk!Z9_#3XIC&Bkuh*Ye86V&I!Hp_K^P z*bnseyj9}xXib-+9jmG>3z{Bq&X%tg$42*nnxeV7aY55VPD8mztd1Vg#WP9sv!$r% z5$AmQ7ICNOvCvx-{@hs9^q6z8+$+wAp57%esp@CT!loyjOXb_ey`vXHkIT-Njt=XrThd^mb~*TCeeQ}{3H;jV>Kf~T1kz7O_yJZJ6RcLzMYG+5anVD`j4HZJrW5!<6|x+dr3ohT9j87PTlDg7mQ* zY*$-LqtI$BW`-es02a9f5=^Pq8ZBOipnMz$&(&7jWM~U5L587xh(q8Cli*6d)@@PN z5K!Pdgs!$8XOy7^b2y-RD z^?_ZBT9vhg(}z0buC}IUlU8Fj*D_9@&=KzHATcQ`wMMJAmMHg$9FeZJ>gTYw&>E~| z%6(!7{Buey%37`4s;nbiK8Yj7)z$@F)mB-}b&Sg=b;P-PN^Hsp+6Jq)j;Qd-90{&$ zD2HBao2|h*rotEINOJX)kV;NEny;)U+&;{qa<#QZigZoBxt?+Rxor9g`=Ho2-HUz>4o{hdS;a`+|j``0`6P%>F#{xFT`qJgrlRYtxp=2UX^eDg<0*3 zbaZx&llYbS=?(ebUx+onC`X!WqNG|`l-`^l{DoQLi*|H%O_4AVg`iWFXNk2wr6b)n z6&j~a>6)qLv&>pwjH8=th9sb@OgB#To+Z}#VjbOGv!Q}IoL)FJc$Qh`i*xjJ&6hMP zYt!9RmFI}{zIaD3S6e4_HN9%8`5d#}m*B{DEs=zj57HZ^de0FXe2I=euH{fty-shQ z8a&5r@Fh9=xymH07)}OSpll#KKHQ;mwY64?3{8Q#f${iM4uh*gg2kvaj0N5XVxv#( z=sD|BP7!;$GHL%%=?2bo3(bwL*L5s+E+vVr1R$>Bft+*P4z@b)A*0k4XgD(R~sB$_jm& z>%8R4m@eHc(;F_*Us<~$GhCM>J7W5FBd4Rk;a^*!U~^rSd>1nuFn{B3^w-wz$ZXdQ z$$^;307$xj!?#(XadX|0{1h{Zf z>$~AIjF;%`)?Ua$*AvO5m>s}Wx-a41SfP`1J(b*uInd2Aqu~<$jWrut;(8&u7jwEB zIRm|n@32Bi=Xx!95_1Wz3yhcP9o9a`a@SkQi#x!z0O#=Pj(IHTb* zz0(R@rt6dBQ_QDstU@$^(^jbQTpTGsR+fnu8Ur+K)gcZSUn+}D%p?lk0o-SWUe6_x zM#Xl?v=lZ3XrI-9RJdf)#Mr)>WFdM5ue3tZ=Tb=9#}0?12ICdF(%K(c2Iw7jJmo&Cv`fLoQYn;zq3MF z=*o~zkG%x96vk`xch+IZHdhbnyx4o0%uM$+e2*2{LsxI<;@B6NjWZjr(R-{Tke#l+ z(v`8FGFh|G>v)wF>O?SLu2@-je3tP#U1c4K>~al|u8&RZPRw#&$M;&HUvv$Yei_@P zdr4>Jx@WI$blrFS;nHtnbGnz#V*QSNugj~e;*XMk3x(tMS;X(2?{#D9zUPmX{t!E< z`$+&-v3=!hKVSL_RFd(t zi5s5%y76^~`O~Gp#cuDO1FS3dgKk3IQT{CH@6b_Bn$6ts{Ggjy_Y;4f^lt3Q?xnL? zH?ae{Np;oyBI#o&Ew|4mZh8*rzNkCSUo8DQ_D=VcfX8A7b(8B(@|Q{fffn=5Z04rt zpl(XtY5q#-f3Y9Cf1J$aj)6#=2eoGW93trP>#TLPe3w)j7vCcXh+6EBF2Amx zzgijzJ?Er3Owe;kH?{68f4wvzE~iK79M&J$Vc$PwgR8Ch9Nfb@C+iRLuxgs)yz76` z)VK*fs^^sa!8)Rv?zre`YeHxB2+pa7lNQws$0gS{(ww+uJzD3q{=qt`Ds%*3A>_ub z?xCEUb&EWzn(4Ue`c^tL?&}^|b4zZqepJnJTz9o~rQi23&#k^i{-~PmxZ(OiIw|gW zkCM5qw^%=^<~V|`w(|6R5AWQp+vHEGxsF?|AEk5RZuO|1TXLIqOf}DO$JN%RMkub& z_20Gu`)jzzKP4?&9@pBVWp3zp=os5(Xyn(xWEu zan(JG=ZEfvPO(c3t^8Nghfv~fnXkQXJI!8Uc+dYw`YbNk^XPor{m^OlO2bF~ztT6* z>RzAkzi$JO+3<<~pY%gqYtNSXq5GjTY|_9Iuw@9;yb(5Sqm5z1t%v|DOT02G(`IW7 zF>JemD-g=Uq4&+R`5SG(J{$M~sVpu&E34QRY7Et|p~n!&Wyw$kZ?S2cY_;ri1K4O; z=Xi70QJbwPRLh2?E`XU0&G2=bzsUx;v_U3FkY&f0WVP5rO`$rr+kgpFG6Ph_5eu{r zZ1wC_28E!VY)HH}D|3PEL8zX++7Kb=C>sskaoz&|0~-+5hA2UrY+`(MR`G(+gU~PR zwFadiT{acUkP4i?y}kO!K|YTY!5?c+3OAQf?l$P@y%J+7x*9A0LnHb z3i`;F$G2v+EC@Xeonw0pxIia!K;0Zsquiu{jkz;PSe3x>(I#b@;@E($#g{mR~A&b*Jd?U+FDCyl&9QrGCja_9hL91qf>O5kR z_NnbUd#_=X;9J>{1aI%mMYgA*>+J6hd4j#N(a@#mE%HCL{jOTc9$T?rHX-4s-nK=} zPaA)4+J~4c+RF8_y*-OG&n!2Z_9GK2j>={w+yFMd@LA)HrXP?=71gq`qJ$^C1BSaufNalV&EJ+=O-GSg z71w3oCyWIIzwqzITTMSAb1K>@`{~(%#Z`Y>Za4je%&WK~JD#vOJG8j@@5b9r#}HdZ zTZiw;#+GPaSnf2zabCqE+4+Pof!Z&8(RionXQa5It5~JWYpfJ?f z*i?f+j|X5t`y5kARmjrRREvOy4U0|ekz*@q4mCD4)gkVRHd0_fj;BQP%JQJ89$8%x z5!NYjESwV*zG{5X^b4}Kq75CGo)ajkdS!XobQW1(5g*n&adA$lr1@3j!=`hHr=pD| zaOGf2HLop?ni>E;NeLU2xFsibspWOxQO&P)n*lBunYcS==u-N%>2b~Zy8j7M!^S5b z&6%^b@pa&F&4s$HKpR*S>vPsF#ow5o)Lg9lN{|^gGx2)P{-u^TfhRS;)wSUV3lbmY zTwF@OG5uL{sqPy=PT11KmYnBH8{Y)}thrp*#v@Pw;!Bp{EvDw0K%GyJ8@4JjqEG5F zOG}`+=1Scz02DSRrt}%QjBYXgRdcoOTfxwzP#}r>l6)KL`4~#XmI+XrOH+! z8T)wu%upnXd{H~b+u5>_@NDbRQK z3eyMwTkBEw?TVkm<|WNe(YJ9$1MnHvm&pB!(_t%B41YpDZ3VB#@Hf()TVn1S~`y>9Z^)>Ri;(XYbNnQF`RyF{^ zVSR)AS#c?BM^fK@8u*19c;$i%FCAbqJdq$sH%kT2JaFN%9gD)d5Eill zNRoVn3n&(ri1#H(5@oA6`EXs~R$%S%;c$Lq1hgcd#|26m>w-@v7}CvF!KF7Bs6(s= zK96W58-O**7jVPeiCABJCBdRljtWkFxpKD}8-TBeiyb4tC;38dxVt?z9N$3@l$)c1 zV^40RyAw7RKR{Th2B1;$#oTCj7i=;v>c4%27R5kHde;afvP-0G5rY zUS6+8;aUZc*X7tru3V^B)Tf|O=;EomNkFu06zaq4+b)N`!EZBkMb%W3eCbNxGA84lNh!qv})93D}SL9NisYUN%B}Sl=%5kE*>D5jS_uKeFi!m`wic&%W;r=xl|uppNY=Ge#iIgCIJ<*QL2xt z?}5(4?&2qPr4ANoM16dH7FvWo#xLr&117T(Sh)J$=wj?|{EqGm*s>Iu}2@l+ChJb*MC0f(vAqb^LUir42jNmLFz zNPRB48jDn=>L&qNvk`!Z`T^*AEJ2l{FLknnasUGAjp!zwqO8m__B(22Ugfy1d~`c`MFkR73hP7o5ST-9oTak4~I zgc?dQ_qW(k)z|v0@)8j{QVp$;doPx!`X1<|d>L~y`)o_Kb?&_TEJ_Yw# z@Aa+atztH0g-(mR26L)JK=fouCQPql=K zQFn7f(g7!3y$vOE z3Y+W|+==BDqkm!7029gsCAJ`elbyR5-nF}}sy3XQ)+}pEc#zF4akp_pXW^1e1FQH! z4tJTm1$(NxVenTJ!Y+Eyhg<4y1Bsp(0u_yUrli`w^=-Fi|EgLH&np_kt|Zm=t1knx zh^^-4O58X$60ardDfpO$>WJJ_w*|X`*XrsWfEvlwDY--4G&WLItFLc6Pm57!=FV|9 zVpmkPhWa-4NTtrrUF*i>Bh|IJ^{dbbc}I0#?tZsLennl|zrGDdN>>-;UUbv)k;%0K z>f0{bdZ~+ZpSv67SCVT7)^7khNvC$^N><^Dktwyt`i*Fce2{ueZt5zF;z~;Gp!zmk zX{360?$A}VVr0A8!S!uNaO2fSbLXsTR9tCSJEVRq@Jm+si(b154)Y<;ma1EFpRZ~RztW+0MEy=6oG3U2 zl&r=hMrvwJ^*%ILzDga@KXtVw;)> z0~ph>5*R4=2zZ=F-`~Aj8L8`3ndjW=9xbm@kLq8w+6>2dl>kM#$I1Qb{QeEAy^*@k zm1CVhxF^c1)kXcAR|g|&I#&WE<(>l1vDo{gYm`yC)Jn7SkbA1ULA|NJW{nx{_bLIK za?g+l)Rp~>YrIjqw8{z2AKkOzafHMD3)cjrYSJo!qjJxeH^M)^dyO($*QIij^Jn)$ zd9(Uz|Ee`+IQgptu*$te9#TK(->}9Tt?ODj*?G#nT;8gF-M@KFFuJB|C6HF`GC3=m zGXPzyRO+;q7H5syAxD!H12k*Ra3xp?=#{%djwP!H7}t80y7bC?=P&Lx@FqjXfWoyw zWlee|uvqR5ay(f-z`a%(qsyo)aQ^DvEY~ED8c?;?42Ol40B5(!6e24W5Yt(AKj-6`(}r-$5iSZsQxy)gF*x=TJV*)t$zohkMZ;9I%Z z&^_|95%6G5?mACwdUyM*+~3iC@^Q)j0R`&jH6yfcVP2kDig2 z!G$AtJr^_#WNcu{dQpk)5z3fG~kI@VAt;yPf z1?vOxhkyyo{S&<`-w8*NPMEamz3s)hf1%gpdy-89cds`k90Euz_ZfOqelXcGuztNK zAwAo^IQMV#w)|MKZQ!HzfrLXqjpe>X@5|35lLNUMnqyTt`6b*}?g#R-$?FH=8;pRF z(;L7rsbKkjN$nPbehSN(lN!2&M zl>4vywfsr)C3qXd2%uU13hoE@TltIRdjpvb?j%*e{FU7Q-0$UYlV1#M+|U5zS$-M! zv-^|$Q}QRc)jejfBUIt91ExE^n1c5tY)# zXz?@vo0jk3f-6=eru2maPgFIMuXEO;LWQDT%24BWfYmnY_2u;nEywM>y z=w0<~C!pO^<{0k)y|&Sy2VW^nRP;$HHGcH4)bd=tn|qy7DRNU*1J0JErgGJ*oQhTL z6hl+KHfC)sQM3E2S34tCbyVb~d=GqEwVLX$UgL~fm8O`Ka@<(5u~p3;pkC`#u1Z%F zq?`xfEeq~S)$5$GtGX-Zq}(!AZ!Ae>4^*#r#;@w7C{B3_1YC78HBi05nYgNtqBP~b zv2|l>GTW&3IB~e-ai)j>iOWj)KSk#n&|(|M;m+s7R;$*zZK;)1t7ILdc4`t<;!Q$p zA%vC?qFQUMTD2vFP%VVe2qDzk+7K!sgiZ*d5<+BLeV>J;e1XSYy`Ve0V-_UxthNLhNC#U%`5_9_KUjWA`<| z&JKHo>tC?Rtk3!S{EU55F6xW0m0V&0+ickR?)-{S&-k8%SGwwaAOKcC;UFT+)R zG4>cYxL~K*r1OXQz5AMAiHAMT4Jp`THtqat{?xuHS9J!qiW^#xW2QC3AOc-ZglQhO znoBCkGc%fD6G1G`z^g}LYq*qx0yESM_lVf?CfMp>#avoJv6J}QR4J3n{AxY z8_~Ewb>{dojq{Xv{cC1hXN*Mjz$B3UpT-5sD*c;gy8#KF+&?yR{6CEbkk_`tU7g+n z6l?_9u^JLeqQ2a$Xa;(L-+@$*@mP(ElqCH-X8UH~7sSA5kiA^v5@oIaeX}D#5vLy* z^B7;Q(MUeDi-)1O!XX( z*SJne*MD#JXhz+F7#JS1S8Cj#Y}fx}_5xVt^n+ub<100qDH;0XW|2cX zwm?LuSB!a&Gc{xszCO|%=R%JxhczQRQA18C)VDC7=@J{+SdmH?Pt<6q2=uMZeO;0x zdtlzkW@#uWCHnT}K>$%tR*VtGSsF@;P#Ka zNE!spx&Ut;?Q#+Zkaa%HHG_Ax4itEp$GKdIOg%K>Gq7gxo>oPHw|SyVJ1ipczRb0Q z9a@JA{LDAFJc(>LlBt5)A}*yr(I$e)*nuo-L*mbSaue? zYZ91Qb{wvn9ko&VM0QR;-u#+N`oht}{j<9^N}p0Rb5*-b&O*H-X#cR3nqID?fMu?9 zDO-p?66@cZQuB;^sbC!pF>4mmk0jfAZ{j?sT-M)c{?w&mVbYOKf5|4!3yNx)`5J^p z+#`rNgw32j$~FD1=I>lO7uFtO%#i?PPEk!Yzqs_nyP9wgVGHLK<);2_^B*o_3x|(% z&XH{4yr!tOoBv$Y7HL%?h=f#5KcyA!;B{RQi<~PNL*huMmCAtXt%)!IEjVT#;<^`bcfCMNdg6#yS3!+A z&9!8a(@}b0b$TK!LJLlsFLFJ$DD)^l5VI}ugVy7MGv>=&&o4?istl~&mIw>cg7fC9 zTyFs&uNQ>bo;a%Ytl*;gTGt04Y@!EMZ%>5DXu%cp&91K&wH)OKVRj^b)_PfR-F&<2 z=SBTTl|j`z5@ADHaLatJ>)%Cc$Mk|R8Hr1Y63}(I!{VIE_|G@mHYw+UCV`_7eyGWC=5&drS6Rweq zE02ZF6~XLO_Cf!N`8n74#myj&LhdGgm#IMg%dRPl`;N8D6~PWw_F4ax`Ayf%#nZ>s zLXerHDcP9*pt;;tu-NW+Xov`gsj>Dg6oaH?AFvE02fH6Tx&<_Dlb}`6t)D#m(TWLS~Wv z%2YalZ?2<@`;NEF6Ty~M_E-O}`5)Km#na%iL}ruJQC~dqE!>%ecR0QKzxrTub z${26Q2(HS8*L{+PTxAfjzHbf0Y{;0LzV!XEax3haq!GI&zhCU^YI5(O1G$#Vjhn2R@vLe~&sWQ#@SVGF4!0p2ExSd&3fFTk4R z!FvL#9Pb_#Thj!qTdaAWOW{IPB|b1Jqh^YtUWi5I;R~ZswfKmriW(x!aIqG7?uAQH z_4v4`rkV_@dJ)z#&!ccTsu8~-s<);IcDY!qJkP?FsAl}GsHvJMs=5G+&hsv0!f{Js z6k1G#p)S@s&!=z=ssn#GiYU&YsTX5y^85SlTiEv2g)>VB z9kfFVGf}_r-=oIFBjE$3gO1vvh1n<#Hys%J;^|D`An42B<lH7Q3 z{)CTUmJfn9t*{mq;dUU}`$TfYK=~jDBns$ybBtH3wG5Mqs6gs*rlinFca=TShJMIGGUHRUswF zS)gwd!3;U_WL0F;A*qjijX?+MkelC%0u){TedFu-w zq3YbCm(-n%UD$fKX14Zx-KVGvZV5|dCxr_MM>zh}O$IMeSKU&V44>>=C^^EJLsc!8 zZ@XnL(K>}#M5yEtsceHGl)_E8#Q79skpyHIRMnLEA&5LFr-X|LM>&DiZ3Z7vPu&`p zB%SJ9Bst0nqN=vdufZ$FJ%w0IIK~O4?lky{dgqpAS9OZMxcXQkjGYU=qdvR6TG9ev z+%d-!L$vo4{z83s`@E$8lyY(P@kCfZ7yd;NaMMes!LWg>B8AGc4Kys=tPEn%bwrp$ zW5e<|h1wQI?lv*Rx(tSTHI|geg*OhAyL(J*T@&o1vE)1+YzOSz17kAkrWooqSV|th z5NYA;9uZSfM})yNmYP>sXkp>u9v9P8ml35d#?tZxh1Q_$*bviO*8~e{YvR&x~VAjS}Z+JSctKpxEIEtPZMEMjh&xYRye~V(*1A@@pMMC`Uz}AUU{J#m_E+L z#GY<~jWu>b-hskd7V+-aV=_)pMXR60M&?x%5-gJ3@5WS|Cc@|%yD;x?;cN>srZ)!q z7fZ%ZYAm8w8U$Eubsvf8ffY9Ul*VG}F@w1lyWOW^CQpwo83z*|waOsOBG+ASDGFxU z>^hAoYK;NaqR1V+)bC7c%y^wfG_}@XzQsOw{L&cMX|qpjETNt>SZHy?owzjp%vj9$ zX^j|aok5g^*qy$#9ERI$Fpf~q7%a6o>mIwb@l5K{@iQ9BsPzWREiSnyFYP(gu#^o; zZh0Ly-r~CZc5sr6EFCyI_@8z|VTgIFJ9nuOOuM7%rLl5}L6XHC_wuD4XA_on)k~Ml zFM+kB%e{7KBy7E-&Pn6smkm-Zp1C(JO*z}Ytm~X~h5V{Ps>Oi2Y^eao-%;nK@$zd1 z=@#$ZdzRLpP57_tymY1f28d3^-G`TUz#2U2f^?Pqra`8~Pxr~Cqi6g7>$)IaEx!%U zZ1tI1%Zy+i9@QXCkhdCeE%aximU+}C#CA1EnesMpqabHGFN=h|c$7q%D3==uEV8U; z`YoGs?~QGh)Ufgtg{2lwGfA)@M}t14CMoYu;XVue%;;rx^|8xaFV?KdyH|M7!h2>S zOv)>khh3^!n|HtPhy`(G+Om=Q-sP>AYS!gFEIba@S`JLi(O_JuNzUsk6kE_|mMz1d zi;Zh-tXZG;sPL2pV`iNn<(x2%aGA4#`q<#C#ed*rNjldVC%MeoNPTK>!D2Ox(7ERj zD+pIODb!~Mmn_!J>|9oRj(?|{O5e05ie=tu&F}^3X4+Mty9hm;|bR| zY1B6c_bd*A`6cOmXT0PZXDjuc!9$DVFl6VRN30}V=cH5L8$7Z&HIwmQ?RmyZ$#u>) z>PLg878hXIE;}z=Nw~q;PW@!?!s05(VusImu9V#1?4W)zcx`ceX7+ztFm)$1b26yo z216E#nZo~^FECa~nmIeEUk%<_JOr&T1-9>mo19(LZw4PNo`PQ{=|bl!$xY5~>JNi2 z7O%k&$%QdI;T9*8`qSX6#k-lE|J7b#td`v3?4kZP_+jy7X8(UOSjQ7?bM{jI82q;Q z0ZN+T3!SSaw>eqVe+K_7{>@a2)q=S^p@ox8RX5bM)b&8bIyW#9BrTjAs-~f?rKyKg zETuu{j%iKgXai7TY3bn^o6w+4sBTTn)z&RCv2^eVfkC|<6C+LJYU>x7S-N;E0#hZO zSuITj&rOk~rKiU#SlIKKn6^ZowsDb-rN77K*!~73v$`!YU)!|E!7{{SFHG+B5;3ww zzBUp*x@jIIu}%_tVzn$0)Hp>hmWw=&!3Lk7h><74bFRqUa+$~Z*aV3(v09#3q-|B? zX}QYd7L4-sSeW)ifwpy#kL6mA2O!9#v#Q$@!KPE>Z@Jmy6|D66EQ})YWQ(0)pyf7? z(bzso3rnORmC79qLoD}rOvf?~O|jIKSYe(M=uNmD2FuYGiAgz1tSAp#q}uV@EGJ&f zNK(IpEz5H*nh!=E_vNt{o04+wVE5&@6sd;&fy*;4P9>?|#g^yci=r%RJtCG@TqLf^ zxr^PO=U${*`o}GAx|p#>{T}u}o=4Ggkoa7j(Q~n34f~$PL8_de%XSm+-kw@L~m`e?7+4nV$ zQ2h-zS-t=`ApO$V+VT4ul~kf3+j7W5xxD<6_c}J1fvACo+bll9=AsX??%+A3oZ4#VU zWf{z}jPq_xP9EqStk$L!@!^!rC63Wpm(1)MtkI?wsR#pq@C}V54|ENRwdqAdxHqB2 z2^;b2ncai6+7U%6@#a^TxVn5dgi0SliG`lR1Cs)P!x@< zA9yr)N}EwsWl5UFgBvjX24>G7s7#AgP{M&Y@5{*>26_fhYsVDTSw_z~8OOL>w}JV1 z@Qn7dA{C`@1*}FR8wMT^p4E;mYOqY4)gC9jjNiz7GFY!2SERxgp2Rg=PTn~1WbmAJ zd{L8S+N`&6otNu2GM^5frykX9w%iFE!|3Jyja^Tr7v!tKBcC%%YlYDjrSyt zjwveyS9({4J+EoZTVM1D(1++1bys3HwLY)8oVT&)sb#~g#1+ab6`R6d)LhBiRHR}d z(pD%pcW#or;9R9{F??-#dsg-et*eO5gg#CaRRu>VW(ij~UuA5T^l`3Hw;H~)dAz6RGT_bOrw;T5Nus=_DU&FWlHdzG<8 z@``hly4&!F<(FCgD`Z!NTL`Z?x2P(f;s;2bhOc&Rk-X;Ire+!bv-~$pEnce$kxJ<2 zv`|%mg{~(e-nog9D(UC6Qb9UmW$Nh^PiYdS5(YR@s*1R<0uNMDQ)jAVfYV0JH#D_! z1jQ`33Be`|a%5B$g5m1Ph_7v8uqA^WIkgCW!pk!uUe+XJ6NWhL)M7(ht2w}A3^#SM zB|{tqwbanjYMy6yyw){D8ey2Dq>2o2R^gsctevjW)2fFPK`vF~VztQgSbXTUk#&eS zq`PtzwXxK*K0f|hOPc5n>7M+6p{LbK&*u0{&`%-Xk~-unjAM;wM||bA(5<4kr2F#2 zhW=KYJp1CCuT3?1?^k<3tuzd<+UhwH-*c^DEBl?sL+UZZxmLS9r{X8CjcpwV(1TiK z7-p60skah!-8-EP3M*=jAr&57=#_rgQ`5&s5?5-~7SXNBJY7~YuGghA-w$?cpDc>B zI_&8WlB*FwK?Wab*A+3WYCLHxh1c=hm>&jvw9gd9Se@})wzA=RGO!_okG1RJZsnrq z+LfKx>$WjJ4nEO7Uleb3-E%v5u|@zE8GNeUP?TUL_2jKY+`w;Vjt+vQwVFTYhOz@&7IONZnNL!lMMi?T}4?6fm#LOLwJV_0tG<`uK59vGFb3F#ZT zO6f)LN?O&?T(vXmtMsk>zG0YMa~{IOxZ)pD=KRVjeAbbXV4ls`3Wuwr`2 zRtau8?T-2`9hE;bY_wYM)w8M|IG3*P(ogaiAP!@D4X^6BS+zUrhxD`jrD3zxPOr&T zqX52i{g8fO{^Zd-J8|=uLb~|qJLIu-WmydEojiBiPd=O0!_4@w^;(dB{sWNO$xy#cPn4+D$NeTQGG?>-IT0DJ$IU4UBolw^j zo71Wx24+WKh4%JNOi;E|ZX+)mP4D>z|*gz_u|Gr>rGF(J9Nm(vQaI-Zrl3!UIC zO~}51$R&W^jAm`*i(cp5nNZux$dv&1Lj#W#dW&~|f~-}TOVG_Vq1hXO^al)Y!>ygU z65U)=nxhd&UcA+qT2cg;pqGoFVT{1);f-KAOBq}VkU=y~HVs|k?Z6C`^0^p&mYL2B zK?M4Mwg8b!SSTGgK{WcL_aabb(}5jgfi_OC41L~v z6|+Uk&%+q9EOllH;?P&TH#7UCN`Q%20Er1!p>KKbWvaF5@i0a#v<^X#h;H{TVLG+Z zfi7Z!X-=>f{lNPeGqjD*!x*z{bY=@Spr3f3XC|~MVO`F$)tMvMjDF>Pi<#Y~mydxZ zuugy=4gJ>p0T{aJz#p+-wJF$+&id^AirE4;yckoKgAQ!$(BHj3GyB_=KqRpob><27 zqW^mTWva>a_!tDsNheIeLI1`YCECeC`62`v(@rwtp^XW4iNUgLJ`%~=kpq|WV7PM! zW~qsvgT&!@G(j=iju4obA)Df>o8g>!bb$!%Oo#x8i3m4ExEZ_%!G5#{Auh2=mQkP% zr~14|!6A_1Z2+pN3Ebg0SKcDQQFJh2SK^dxsz4ou!}AybCQ%55fH@Hhb5J-qQ4yR# zM-mPv66G0%>J~V6UX0*0c=XNy_taFFV}YB=TP8S%jwf7C%#cqNs$1ebc(DQrI*D)> zKqz8SjwKG>cm$WxDTL>VP4bK)bt{}FFJ5pB)O#O*jA{a9InIj*Pakw9;aB36e5yzt zjq~Ou2wKrxf&rjXL_rQ3N8lw2e6s`7U4VmA2vXd8?kqt6mzS&i+f#p5=b{AAy+q$2WU%LD=Qg7S^s}xR5^1Ns-DbVHBo~)ShQ#WNq!^ zmz1LH7j|LV$h3((Q)?$5QW9T@7KLHO6kdVA+#2r_om8ic6}4i;KzYGul($b}l2TbA z3d4zMykdc^HPI(6X++s8YQ>2G`w}=p-Gq~*cLxnRLouBv65y=qK4nQGt&B1WaA353 zMy}S2p?^ZTBP=7#$c>=wH=1ev9|#YV?sS$(01TrYH1e`u4YGOe9mGC@OKv3Xkdd$T zI-ky@+B=MW5+KEBM~voJZ}I6*lHC#RBe>=+q8&8~vfcrX#Nj)g`y_yl(T*F#QFFk4CvCJV2P08%Mirw9)!0$QzUHbsmt+%3VRbYP7{V>opK++C+F|_#kTuyo;X#5|?kd_%qutg&K=?R(uk)b9D|a>R zwo#V#KVaCjIuI2E@7x4hE5MSvzKFHX9gGTzcP^9GW|VJj3QQZNLs&r|=&X4&O)SYVM9sHnG9Yg;<_ zhcLdZwL13&hpqj6H?QsQP#&uGWv$bBC{ST=d)KPn*E@{yV{YgDNu27$JU14=O4z*W^K@UEI4Dm%=i4-g!{_F)w5X}b)E`Tpxmvs+4uF1VEkDr zI?n_bt=IZKfV63bs4y8@w(P1enM`a(erSQMdI8-^g{94?>TM0?FIjRrK9C-}rUS z53-LUgUH+4M~$9aU-nH|*Y}|1s3?fMqg{pL-So{|H~m2E7&4fg(LQE0Xf5{@tfS}< zkL3j8fX@@WvF`9aypH%VUDLx1C@bLh@}>npLB#oCUGKxD<2mzi zfbYPJ~^hdp^6RAA5F*D*v8h6n4I1@Ry7``$))KTBW-Yg^yKnR?`k$sfHVVR3!9mK zvB`~{snz2oO&-n2*xJU|FFCoVv!R*|SRl>B*xn|{FC%%fbF6xttjVXrsM3b)$4^Fe zdDpOk6Qr3L&#+nGSCQ=3m0B}S(JZ8)jNNRO_|+xHbT!nl0UV@R8qcy>;n$R$-ZfS; zPSq69;FHpZ<)=(8@A4M2fh45a7|*uZ=+~Rv*p(_Cr)ieZ?2H3!w)%}E_jEOg*?<<( z9E|7M?Dm^Vp6nVEkB4gtX->vrHo1Pu;fQWLu!ciI9c(ehrqIuFy?1wV?Erm9q~lyn zw<+^;S%MzSMUhMEE}a?*?Rn=*psc1;zK;&;&nDzoqjbyFjkxlTPQxvn_avS^1T<= zCqL>v*}71Cgg2*n3-p1dfHa~(h$^n+1r%?!8S(30U-u~XRO=$~QC?8-4x2H*L0}v! zPK7NNALGp}-fc7GH?e-?QSYhN#p2_9xQe2TZY z_&B7lI4~7O12`$J<3$yVZRoShHsBw}o@tE{pXMzoJ_W^(nho^F6+lvo&+wKOpM~P^ z#SO`id%@)@KFj;B_<~KsZ0QC*ya0qP6W8;W7hkeTp548n?s4qd)@9;zycNY)ZBl0s zZcsk10M=4`p0~32hE2xoi47xNwVAosGp$Y|hVK1-vHz9A*XUiq59u=QdYnZ{FDdM0u`y z1?#HLmf}}7w`T7JfK%^0CY}Y>+2TQ)_Sq#Hou1OqSI4ui>1-{2Yx7{X`~2Xi+2@ff z$=BP{jX&5to?X8&9=x=omE;@k+l@clyadEE^QqbesdTXdU@s`!VAX+dcl%Dbv8BB9qCT?NzW% z&-1@!j($#T%t^*|@oI{x;O(^mI5neDeLb$5S6e*aw%p%+Q|$Am#+>!IN4%583vDa? z1A$hZYE<8V>*3WEN7>f;M{KHiPQ0A60r!}9rg$l+e&YaN&43OK?g_8Hc)4w(|AtMy z&zmmiY{Wg~oiAQ#+w8v!c-E=Q>M6KhUPCd{HcJNAzvvglD_tAap3yEEud%%cgsb0+ z)GOmDn$KyC#_Me#`PXfVdC_o%y-D*0?TYax+ZTYmroR}wG7bV^S`)0Xhy0bB%3pY2 zWpCDeNxN>m&Gw^z@1{mzV8=IWzM?f7@3j5uKeDOkMZ;D07R}eRTgH2AfBR2ungk|x ze2Zp3tp##snsf9vqx!s?*r}QWG^uf(t>GN>Wi%`trqO>DO2 z5Ut&~*w%IqadSFQv*T>dVVcrdWQ&_a-(243eT|)_`G$7ac)#t;IkB4?0izvH(|k+o zFg|4KJ12Q_WI^3E=GLLNIuD8~Z3E|Q2d;MH+Q8N!2=f$I*^=h)02;twXQmI0=yVs? z+D6Pd0HAI1^?~%E_c}erb+*xSPJ;Tl?mBbZ&w3IOXzt3;%+DSwRzNWb zyM_L$qB(4*c!Kw)SViypY)O9A3&v&fSKhnghfvxiZQ;K{-wfL&p5(nRR^hzSTk2lL z-fZ0^{>J-Q{M5E#PU05jtBRXpyT#vmpNds{Z`ziTSG_k|cZ+}Uz7)TP>>p=~-fQ%& zuuSn3Z@hR2fZwt$_}8(wS~JBzd0&g)L8z!^3;lJ)t*|}fU%YR{DgyZ8mgLvHw_5jz zfAfA6f3fYKBi+J(jlLbWS3J%8S*${WySLQ6j=kNwSNw>DUOR{qR(f%3#v;8+mEmf-@(L%`1RSQ=KXjqqsNOkUKv`DgZAqZun zYiCMyN~QD*TL@5C3s+IZRuD`|>hElk)sfnZAJYsgL)&N3E z;O6RutH9%MqA=BYfFYG|b3wdk;%c`Tx`>nkp_Gu93nDEbk^h5EQ_?`ERFcOE(y=P> zv|B~I1>q*WHVk+^bgfH#?A8(=q*e{k+p2jiV_n-4f4j}ZS75Q`w_);Gz$%vn+HEI( zPVFC1wpHh|Om!VgLhSYu|E8)9>d7#C7Qo9TBs(6_gzYp)msP{6T^9#0HYG#{5MT3U zm;x5i%q0MpkKJ}e;Ux_2^o}mT> z8`4(cAtrn5eiNtIlR#0AmufnNhnnPomR~QeE)lO}3WuC@NhMr6g8)mws*{xi!Xb<< zrG#&17T}V`7_L(?MMGF!T8Y5UCcqy!>k;LEXb7iEhp{;}fR-j4#@}I<4LR#Zl$6`K z2P^~dI{D5(+0YE#$dU>>pMbS#ox^o^nEQrYbQhIW+64w|2NHYa&cMDQP=l9L*^vTx zKplCnW|X(#6;aUKnh$8U$D4$^UFB_ViX|p>c2NPfX^|igkJ{hnu2^bPZ?_y+?UXnD zcf0nt%~bqn(qP97kfjOUINggn(B`37ZqjJC9^mczHwpK;4z$futT1V^V+RbUb-cmr zg&h=o=C3SK(cBYhBX4@|wH_3E<*zQeZI>OOu~qLax`R18$2*(}cz1pPVyp98Mu(&# zmk`b}QP`CRIBli86?PB~<@$uHc<+M{150|_*&zXEdH6b$hjzyULbr0?BJL9o=lX@K z0Ps@*jIFhA8TTcJb7zNdGb;)(#_R_Aw&2a?L%fbgv)QThx-<<{VL*$-s8s210y?%VwT#*fSb?rc-bi0wz zPEi#(Ou;eHu{RF1OAj8&?nDB2uHc#&*jog8q{ol6bc(9UWChQ}#NIwII6ZSjtqTbt zx^%bG(wjyyfK12H@&zT<_I`mI(tAgmx^l$0@cfbz zd;8$PUFlOJQ(fa%)al_u6O27Mke`lv@7>L=)tnz*X5wiV5qJRD_2lk>+M)Tn)bdQw;+wThe4!HZsqk&UHV2TH~E+oJqX>wZQakqrl8}oo)M1als@3Dp#A+^gg1Wgw=~<^CcxLFw~a;L?8R4 zwO+hD|5C|12+Y-NqkpV;5_V1;mw&lrqkV7SMWE_?pR}G6uh5-mvc>*&U<-(X5l;!{ zbK}FWnQXOx7nqh(^^yLx`aEl;?v0Y|_MZb^f#sh66mx;KO7~`o3cLTjt^cF)Y4rs% zTYAf6ul=9E>21>=)q0T)-hO3J^Y+Y7 zYA=vi$SDdH?Y}0dV|yhO_(WI8n-tGYuGnu1>f7E7(<-ito(l^|LoO*H8S+Z(EVX8;_Q6&QcVaJ zn1QGuw!gJSoRbfgLWme*Gw5R#{b8+QPQG@jp#wVD2h0V%{jIH<_My6_rVft5p&8sU z!~j8>%MAyku!Ae83~I+10}^R&UjF(LO9#*3RS-Sk4`A9@JYC~b8*m$J&gdUg4pg_X z@^wv19UMY}_hzV#>kVRLEWR$XR3$qo$#5E{4_3=q1-j;?E)I)=k3lDaKZudD3Uw_? zRrZ7P842Uc!D>0HNY|>=(_vNcEvPH%4Pn|@0$n&j1s%eJjH+?^P**v*SYc!8@31Mj zFQa)pdkCo@mnc+zgl)m28GYj|Lm~yaRN-J6;;<)pI%9Oj)R4Lo2f_p><#2-ycA_VU z!#PTvC?8u&b;uF~+w3GxWDKj{!IkAZm#P#B?mJ^AnxM~s+n4WBy3nCAIB=)h(Ae3yXjJgvruqooJf5$epmBAxJt5cDL8p&&qTu;_C3vm z;hv@|9c~0??3|n!dozAdvm#t&U1$sD??ip|e#`FAJQVI@y2jxiL?8UVroJ8T&^#Qj z(l9&GcV?JQ%w*{D{ zJG>9>*;)TJ;awMq8WcgM84lyY!#g`591-=f?U-V&X{N)^;K`k%U;E#6fw@63&otXX zeXiCnqe-WcsLr-3MVKkqL4Pi4m&atnNEfIa6l9>pk#n7QMM7&Ls;jL=K{XXPSkLv_ zl`=VI)6yjt=ZBY;IylWG0k(>MAJ#3d&7WVo4{8+AyXq!mLG~a%k-wnypo2HCYs$%r z_hFC3C-WDU9&sSfP1`jx*$egu@u~d9rN<$e&Do{*4gDdkM_iX5RVs#nMcFR=w^-0Y zh)?G)DLnO&Hu0TfqgTPz48#OLx?lwNg6ojbTo`K{t(*i-TO{FS9Q95Uuk>>Byj3kC`C zh5Xf}w;i(QYV6kgjvfu`6*uHFOWPomVYVCp9p5hL&6R|+Ocf5LbDefmz6(bQ&vGw@ zuQ9#na1e@}N#K}}Ku9Beo#{h|<8woIbHB$v!#rm->aH*Cb~rhA5lAQKpQ@j;F6(YA zed2I_?kY%a@IPT*u&(HCDt+#7W$tFMP$)lDzhGU}-BS9>;nv)}yVZW^ea7^$nsnJP zd2XLu0$K|C=juMzHQlYHZyg@YJqC3S{%6cf)^*)&r5_xg%smgDiddTPCHaP8yXj|# zmvftUXZ}$8f_z18R;cYptcb8lUo%WQyoX+5WCFa zsVN(^0h~0S3x;u2h&#A0n#Pb1)Y`&%rpAu8A;iq|sj;!~0Zmyr-xTSH1NlYyl=nD$ zP*WaWXlmg&6S5(VQ>o+QgPQH(0#i8q2OCDuRKqxXNK+ABVruUg6q1oSIW;yuKBTD( z7n))m$)LwT{q&w-591Qm%1WIbBSH>jdjCwG7#JS9qg!68@@1UNWc;j~V7?i;t9zi- z!!a)8N@nWMk%@sfL-%wmN>u`l_DtbV{8#4Np$^@{rGAbZLY`zc{7n8j@OJ3FZe^*; zsPQ(l^Jm>x=DQ(CtCj{kX6*|3o;mh&!>eQ_%r8pMC z49@#k^5nqCP?v5kV3=hgE_)ci>L!`*hq`r7mPR@r4)NcU`fFrz;Qi1e-MUhSV@(JR zEFAc6%nw67x@Ssb9EDynd+L8BeCztq_E>Qi`tZk0689*7ReTHkD1MTEu5^WCQ%KsL zkzc*vT0e@P=3gjPc{(_I^nRnihmDGR^ChJ$$flI-!T*l^-a0CNmVc>KZSS9?b=@!S=AoAe;M*JXr&g~1oX1dk!T}bDi+TV;H zlFzv>!*7`GaQp&h57}?w55kw+SK&8JcRT(F8QU}byYq(xQY_)OO|u;Tf$BqR8Zku} z%k2+u1?#2mJj7n-X~vXfEO#Kh%{1T96x<(_Y2g%MJa;f$ZdwFmBk#RQ)16b2@!X+s zg=wjyBgjCwfVmSU#MT;jO3NHw<}KP=HBJ9nJ;8dTd#_Yw2U)e3*x@8r|B8E?-%)zV z(Qn>{y}i>-5PZRbE~NCRWAMCPd#9$SeyUI6M)EtsH%XaSxEKA0_$y};_ddV7^n_#N zyu*8me=>flf5Uyq?RcP>z(|bj+Oh zYwy&bso&~TxUu~H(pE^L7-XUU5~p*faO3%drE*8XJew@y-;8PXpSX$q;nF*f<@4OL zV*fTli3SH!lG6K*mGc6#GX73YtN+4H=8wQdZ0)>=tct%xi1XsWQd0WZv3_1$R@2{% zKkC16-}6UHpE)+p^JMn?ZTQ0mZAtiN)0d7n=4E6}{vG=>{#$b@e9UwJE_3-=sDIvn z+2An=pD=yncn|V6e*aScj!$d;3ZFE6@Azn5T~^G$hQDkOn}mNi{p9#!UQ<^3zp=mL ze;_k7RXXN42p7V_e|RvR3{C6)DxGxvFz-oL!@uNz1AmA9=uVeTIZn)bo7MTR4%8<@ ze|7(sPCNdZ_dRRu-^jm#e?!pB5vn<9gz97?)ZD;@GVCE!M?5gs3pLBet1VZPs@1CV zHHErPh)^uZQ4Xn*)N7&LE;MvPhx%kEt36Ybs@H1rp)=&<6iNbHinTgPqgIQrFEoel zPIPvi+H!TNMy)m<22M`i05&Vt4yltgYjya>LR%+dXj=A&+B0>jW-X+PgpN+3p&an3 zSZk29YW4U?Ar4YKW!ZT3@5BhL0l~^44U%?}7s&zvOXtvr>?HLrjf)WT zp<5xmoK}Nzg{zL#^wHshK@{QZl(jCjGrLwjO7o%)*O(4nZl^8aV3DbdG<|frCUgi> zJM9P^%N|zm(!8k41w#)4gft*!Nw(0_!s;a<^e{q_6Az9ToHXWZ)#xPwxDPi(C7}+G z2rAIR>L;1$%@9U79SHTzNzk~XRimF|uIDOLXMM;JufDb`Ar>E(WCWlpeWj*c0`(UpFP4Ni|k>vQ5YTXo7nf2dTcUtWeb z=VU_g*$i^wN>!@pO=w3>rDm9JnF$5HRw1@KeF7Iuvu2L283eSI!3c0bg^uR*X}0Q? znNr{xhS=-$2jno*n(BIH2nwW+pl+lcX2h}63InML1=>Fd9^9DOae}pS^vsYHh~FRz zoh-sUIPqGodSyro#9t64PWGUW$<$H@D4hcJ69^*B2#e%YYK7^SnNa}#ML^3Jd@{{i zIr?Vi6c=SA0;0NMDV#p7C~}UuvnwC^44;nqQnL z?VpAkXlL*u3tOGIVFtNqod6>atZw*;LY1|~CYPwQ(@4YG8O*rCJ5J?c?zyo#*P!L( z?8{#(RB3Dib2D^)LRH%tt~_D5Q5zPKTcHzR++D9Ro38TOoQ2v`BZLGeY_$C8DTrrI zm%@^Bd!Xo-W~&9~-XOZU5tflV2~x%hTP-3z6)^zg4}LBRtb}QHS^;#G`Q{!3qWnP1 zH({q0NKZ$+cX||7mm8xifySFw5M8Chc@fr>o31+!)-|nQdInqC&?6h3tCuO9a+(Nxo7<^-8p2@1Aj=U>JN*j#o;#-d9s*s%^YpTX zY8VZY4i};42EVbxV9yb1VGKx?TyMSgkgOUG)8m4oz>MU=W$2xTcGNKFbc9A28h9Ni_I_ZctMi2EUFFg>fe><2LB6hpf>sRZk#9V|++!xt)5aAulux z!X2RXGA#|fxF2rM^ zNhi4sK-3eVIHPw!=z+zq9IeFR9Sm=k|w#M`UB?O&M}$IhTPx4V0E3E#aEG!Q z{?`n z{inQuNWip`xOql~7|Uo6S*%iJ0=h#g&+~wJez%8gx$+VMtoo$dyhuoSMbDDODOGBq zXQalw6vF|_?pd-G%BzS}%mCRW>1gGK%M6!~~zWjHzrx z@G(fTGcVGp8pPVNM5RhEWKH(tr5Fu>;abL0DiA^phD_lJj4+^@mL(}wz9Bbq46ojZ zY2EE31EVvd0z)7t@j9R}7VRqovoZ*X0?6sSQKJFtZeJNFjuBOuFfx~C1ha!^KN+}& z!ALZpT+Z__X4-W7$>2g9QHO~l*YYAEOBOv_1~xy z8Bbu0v5oeZr6`{vz%~ocqIzSdZMVN{lkx?k3Bx82^E!-0GehUpZsxxfsw74eyb z8pw&@iacab3aZ`C|0Gn2kkUZf_{?4!RJ()!Mfe)iLgs)y(b|C&T${lkhhuXkxeUxm z%N?Y_wLAG=h3_z(^+N)q*Zg_!~1tp2#0DeFie6TJW|B z|6$Z98hky3wG%0{76fV{P3X3n@$rb|U{|Wm;e)sgi=bfnbi^SiQdljA4+b$Tn&QJx zhNq-7tTvYqDle=Pg~aD0tl=S0%jN5f%pvs_&96f&2mexS9^X)8h4rRvUM53`Fh1~G zKsQF(V&_m&`NJS&x(G2iI($#U&ZA`WwLsnEL*er2NF)v$P7(5*!J%}K!UbUf(iNMv zm{P^3fX&E<$}OY=j)wh@(!fswXVXOp$kAc?ja^M?;d9}KeQ3kEy4E5e>{`kLkT}i9 z)qqw<&sOA*-As7}iMay!>yt|K>_vgt?Uc`8b-IJA2~PsAj3@-Vm-3gdW~T3qr6^Vw&L9pjcR8e=28|(Ii&PKuEpM`RDyjW z$Hi?cZJLHffC*JyJM!0*8>^_4PG&|@xfOJ1fLeAC8pDmKtfZAh3Tkq z{6v&iEnOv~!cpk}924BqqP0%YBar*CGa<^?2%PUkwAM+w%24G?O)l&~N!-!^r=>4K z9>oSxGYThB<8Bj6wCd;#$Oe)@S%tD7xTnQvou)@4ffj|1pPvQWeIiEd3_S*U8oLA> zRxuV5_q3&2XX(q3=ddfNO@-+ILryH!s;9>yC0G_nt;#J3z#wX!qsJjHV>d$juhD`% zb7Gm+d3rqZ8g?sKu6isaGt&UVqpw2V#O|g}yNp@9pE(G4oL+(mZ;?aQDl)Rfctpp_ zfKvmBUjY?WtY=DGJPZRDfR+2p-5)A-o&J;DOm{$}MFk@)!0Sb+o7t2!OKHl-tU^$iLV>)ajyWD|J}jQ(Ba& z@}V}(NI;1u!Y1CimCq4ry_2B9a&G4fMAkSz+6GW~U4xvW zvw~kDvd3lZLuCjiVdFc&(o)ieA})eN;|UPfZZNhTR_c|BoN*Df`dkldrcXDlUX?1# zR}`&Q5D5v!XqcZW_ai-U%V~{*6zc(>ZkU1cdw5>B1e#R9x3=~rt*wPUrpOl}jopGe z>*c=EwYA^@6V1Vy&-|aFI}c>~{{uLVOpbjk!_Zgd*v8x&-8wMG4%ggg%(f^MOGT-W zsUfiqTd7f8OT-)sN$Z+~F>eBSTZ>-m@}=rb;M zBJR!^_J#>XizA-z3TBN*oSK1g^j3z!PI{`GRbQ>^OiajmfP2GE$04lDMxxa7J0Z02e$m;_^vY1S9h zEowb|&sMA^!$=krQewp+7B@U&z=q_lH-}Lu&FjSx7Pme3pPgDQTVJ)832m|B7>iEN z6K7R^)!Ja5#2nXz*&mC3&%(2&zmhjpB{3&7VF<@!*z*!-lDrM(`|% z=mK?s^n9R0_o&6cp0CeN{@M-04y3;aI(1K2{O9@o?Dt<)H=$t2U_eP!BD|1=CMK?% z_;A}l&;y3aFJ2~vE+$!qUbrW7o}mNSX}{>XusAc|{d=m}G56!A0!AknVYl^Ggop&SZHF6_&4Y-O7h7+Q2j2+ZOx0CJQ^( zWNk$qA-x%x)P=hkuh)f>Yr41cp_4Z7T({HWxYzf>?`u@yLWA^fU<$ZRt{3v0$y!$z zei{kx4IrF$*30Ca%i1g#R60p9FsnOgQQ~ELE^2MJ3qPIoen44LyXfU|E(6ZIQAbH1 z29!dl#EW*We61^-HIx1wP^Q(cdqth=SepfhzNC)>uXUeU-115~H@UWZ8~+&T(}1$I zcHb-G-1oJr+ff;$&jU)-Q|eW4&SahIc76uw%YZVz_M2BZoD^nlM;#}99r&R8%HoMv z!?~z+-Eba9`Zl1fvHj)MaV}$>>JAi~gAFJh&^fQ6bLHz?ckoY;{u@x{+5YD>d9Gt! z)(+@f{TTSJ3&%WOug^`c>)ydXN%}dU?6v*p_5EDMgU>tEGBGMV6}~D)-CMiJ$aFKD zVqhTP#z$a~-m8l?nWpbl%fi4}0bdKF?`>TaX4<|pI|~CvH~ude6YniWX{KLxs%2x; zdGNA~!FW@OM5dcvv$HXfc;g#jPTG1$6t$VAyQ-bSXz-AHV~n%+o}zKncGv7v7>K^{ z*I-<{kHTBo7gx0$j3y6gT?`3A8b)TD-Li8qT09hAS%EuWvhF%X>QE5mVLi$0ruaaTKoS;aGjrGOl7?P4SI&7^F2Wa2@0 z5mVs38cK5MBsCreo>cf`jL_SjS@C;KtG&I8 zh2{hgPM*2}*%}p$xdyGdR`V1OMV>mEY>NuVw0I{JkC``laPrj+$?z?Lxd-vN59UiA zihOk=vLgyUy}i#A>st^!IR)y*WFiVC@}WlOY?0!rC{SNbc19&)9(&gn2U#?Ea?Yy5 z2MuZ;=1<7erCKa`D$c5#kl_y;Gvhr_EVLkaaSGMfkl`;4^AGguS}jt%6ou+*$?z+W zdGEbYJZ914#W|`X4h%KQ5HsxtifM}!Fw}`W=*x+AX=wQ zM)MspiQdyivu0W({uvC4jN|)b>b)7ov*uc!sI*naWM_U7=7slYai4|E3k9#RWD>ss z{-yK#FfwKH4o?o^H(=t(qx?RsjEWLr2=D+n1W+tp(1(*LBY2z1sr=WN7vxbvpQX$X zRe^CO=kSqOGYVbUXC+fs@yKK$-xeE385Q>7Wr3(VOaQrtPXk+)UeaeRQ)cqQ$*ue( z>6=%PMHnX;_MBkK!nadEU!Q6E7Tj~c){ zGBFo;{bo+mlW5>TTU37IA+ z$GQ6O&y&{6_Mo0(u9Kq#9T4L~T_A0cC83^SZjqA&lelgl{sq!T*?!b>%zbi(;5*d$ zQ01gevSc_Mkdg}oCYG+g{BqJ}83#3w`HfsIaDiMO>LO{2EEV+%^Mu?Wh_dYVO9A~&q)*q(lC^|;h^NHia@P z9Ku%y(y#RCoxaH5icO;^gbVnIz@95Le!R>49oS5YT8XB$agbYG4YUXO?$|RFREd>! zcu-GWO#rW&?*(VE<|W?NVz7p5phd{1Vnq~UNrLr6P)~hLAnyv_4_iTTD>-9r9PD`#hHsk*Ih7_l)YmaI%5E(azVV;d0G+?W$b z?nRx(uB4ill5C5^h)p>Rva*S2NOde_+Kz-ZH|4~WAEM4-*HL$tl0r1Y&6}AJN#ygf zI4WPK+}1Um-%Mi322rKh`^zFq+icUJnuuZZhWKKvGj&htxNUoQb_-@VZq0$R z`q}miWE3$8yhosZk*F&+>e~?_IIZf50QouUKK2;(bLj_2F)Hq;CzGe(#q|_bvrONf5T)k*B3Wx1 zJ|gp|XlOGgMoqUdkMzu-2C&6cOc~9-GK${b$C1sUhOlCYH741=h#GD0J1BbzN^~W> z{1(`oMbq#0rOFmjW7vAAH#XSEMUUR?J0wdu`v%7E8nF*wpShk5_wD|3#VvIYBz z=$?BuM|f}fe`C9-*=3pz#xZUkHEFyhm?a#b^2@9o!ee?mYSMY{`7g1Ls8wa&4q~9E zYmV~%(Y%TC*acG*oDArLHTh+}3@>;M0x0beD zlG$PBgZZaf)E4^oV@a*+!-+9)Vb%JlWD zaly0(Q52z@!IzTqWoSK!4A44683fgM6r?C+MtVdXi#8-GC%DG*`$=bItMxYE_R%Iq z9YAlRpiL<=(SzwV+H27yp*x=ckaSMARu3+QY2QWP393vK#42T`dTux_4JkGux-$6# zq+*%5o+mEcbhUUBF`cO@fd~oHoE#frd7pB@E26C2y z0PwsiVi9pOD|--A!m}0x;~HoY;x=MBOHGb}zNH`>*GfYo#)wTU4v61md%bAfJzARh z196F^fZ!$BQ7;bHOFJXhcOtMkL+WBOQICa_(?nutrxdmV3YcVPy+qt&TAet^sfo=2 zB%HiKZy)YYTAMi4X^E|XBqn*2UJ7o8HXs%{5q5Ki)i08_=pDlSLmL;jI;HGZKqHgv zqL+?)Pg@X=IW_I(fGbYku6G>wjrLjm!D(r?0-~AZop6q`%t!OQzB3_#^GLmt>;@G( z9Ur6f&dwoPtIi!>9jz(R%z|#?#pbtq4IYZjaAXAkh=|PCuQ6 zRwp!jIX?5}AFZ$7%lKnVIrw#q6YGwg5Pl! zK5iGzY%t#E_N=Crmne9QtMj2>sM`>}ujg6KE#6*%viHe=!!j{Q?98J(_v!tKyW^95 zVRpmFzUDu3?vRxM&<8%57t}ZE?k7&>w2@QvW^fO|YR7CeQ&dhep$ICN#|`^jy0CL2 z4-VXzceM@)Ug4hl+_(@6k33a>F(DBuc!PWH(|I9tqt*fQ=gbbRbb$gl?=yU%Vk7xL z)pI5^Lj@mkZ+xC#=-S9TVE#9=Q|q|kEAHe+pEnn#Hp&iE{mq1msNe_ghtH1-Dx0*D z&8L`MT3G@$OAX%@<))j+$yHNK$chRyELZrhE#J9`mu&um*`vh;u^R1bUmgpOT2(KY z&>0o1uw3iAy*zW1R*LyFvyW`0jk3i0(#p#>xu) zdaEtb#(T=gH?^l^&tRZLD!7l>>YG;nVbfBIVn%&{T%>1a>F#@`Tz@lx!+|s@nXiYn zr22}=oj0d&6tn7sWT773GSIiKJZN(hhXbura;cu3Wu$LgdFtjRj$%%Ih%C}0SjPJf zlnXZ#4ssw~NECJ0;iZ`B+}?-E=W^%TlUhQGJYD2c7+DU+;^;ErdfH=%13W>IGX~^9{S$x+Ue1 z;+6VSa)VyDWs7gZMQ>gm1W{+7YTbZ`)4RThFTUHdcxd{c**~-z1+mcT&%L;6D;^4| zv*TLL0;c7lZ|OzHt%--HU(ZfxwF-zPBfd2kX^>2%|F7?v?3P}V<%DnR#iXq-4v+q? z?@!qsy=2Q5(EBfdhARC{-=wTvFV%9|1Z~Fd{-P236T=Jd0I0R8u`rE$e zvio`&mfwF>TwK^Xair&M&ELESf|HgierixzHBNJTS2LxRsCmkAx!ma3= zL86>fNx6hL&r_Hq~ROkEGYKalK|smfz5&@@=lidZsM?$9pDF#$BIX zYTK56Ozjiq4R2D=Zkg)Wg&Et{bd2*!{ViEpdp+j&`O=4NOUD$S)ZdY(^txfIUGuX3 zc0vZ{v-%QQnSagqGrH`&Jtaf&SzSS%)stBY{H!hqZEwopd{KW-R`y^o_-(qJx_v1_ z@kRXuc>x@TYCrGG!tI3PoUiKtl9hqjYkpyuTeqhiSA13fNPew1VcFuBaCvNd({au> z^-sLlIg^%m{SIG#w|(*W^tahhTJHo?U^(PoUbO>%g7Liu(Y>TM3vZ9Y%eFh>PK$;o+m09AC zobTi>dhab?`aQZ_vm@YS^N*bW$lvrnS-$m~y4(lZS>n%}ALRe^zFB_qdv|$u$H>X% zpE*CtKlOfE{`6C+P=^jJQH85QQPp2&wcLMI1!iYJX0r-cm9k8Kxz$R4Q+R1A&LpaG z)hGyk9VMZneUEH#AP+A_S zL5x1dstIRZ8RpiWldXk??zK=E%-vF%=Jq8=O&hD9XC>UfuG-(bQs_qDabnziIY@*tMcLSu`->`_-BUoGMXaVc>%-=9}8TiXQ!_X_{{%17??rx`2fAj=Cc zyI=i3R=#sz1W#&io%T*)rj=TNM%5}3{tN>~Ahg|}{jVK>u5u(Lo|#@XXQoXOp0Pp& z;Hvyd^=BBmbLQHfLS?vjLsc?q5dere3vF*04KNSztSTbmc?{GXMw=p3mV8618cB(~ zY1AB68>;A5!~k~HBT_w&p*M%q_7y7AzK5&ckrsK=P+Qja7gkug1>{z(^1$aY^yjR! z=|W}ex3tR9BQbAUe-5u5B&@Td2h>*id(`JK4Cbu0Lxjrs@9nB&kHx%c1FpxzQ2j=$ zxPZQ@0uQr%I$CNwsH_1e28>lTc*Nz8qNR3&k@{^`DFL%pLmrj+bVI5ApfV4f9`K>+ zwa1J6QA4T2;4b|x5CGJxk)CD+bR((bU>x*(&>ww`eXqefD~V7&f-RdE2On&ugx z?Pe^2fUr<*B?=%`CwNX2^g!kyZ@2JwtBL@(>NB3kXWdpypeQVSY*iOPudee9Kijif zvOaIG@DHoT07mtIr}(VfuM*fX75-_}7Qm@q@SHf?^Q&ZI-T~p?R$T$v)tX+$u)HRL zoiX8z)j$Bh+R7`uu*XEQIq#tGrPawt0aexBUgAQxHIglPhlI)sadUNo*F<5@8p+nY zBf_^o_>J!>V~@{S2VS-lIGt{(6bpL1I$*`9Y? z_|57woMbF`O`PjlC)tsAQuxzKg|1ek>1_<7U6P%7S;A#_O*#s~#oSz>25V=yp7>5Rzk|-!Q+G#onBMt z9bVjHF7e3AhlH0SoRrA(K*mMo+$3I@VH1ir@s!rgPvKVPS)iU zFBK$%6 zLd_VtNx-p0_)#wDZ^Iv>f3EpJUJ@vvpG>LHcZEudrbM4Y5OS;#0hB6z4}3n|Na9T4 zjZaz41yH6|dgF`f7zvG1DWv12^g)R}4KJoUOOhxrgrj(A;NTT~e|#mKBq^YnmC&uF zL4)=BL2x=0Bx!);@~E{mc<`FOhFK#V7Y$IvC2ls7ki6@{NPHWe0}bYhk{%mLXx>fX zE_@d~TcSw~-{EM>4Wl&a$KxNum=}f`P}*$E4X3o|@5cX5uaJ+s;soijuokWN4OvVoZczPq-vF!+q0sy+l3tbJbf6l&E&EwdsejeJ>g;e8~Sre z7nN6L?!b!CzArqA|44rW9cNitl>=*+_5*w}rj|&^|jKF@#N8>LrHq0eyKNGg%_XIwz zZS+aJFuh@JpY|m1ADqB>=tvsVVqJAaF z$f(?HlO#Fs1uQzA2}E77@(nNV*(6EHn-MCD)8=qzE-rW5EaBwM3FUZEAn{6q??ide zX34?4mqKNN+U?32U*n5zTO_G@i^9kFIyeff^9{e)vqf?!?;oMENzJ%2;48l9wpDUC z?|;HS@h95?Iad~ZCocAEl^n@?D^ymhv#)6S8DDa9k)-7<31{#Ff&430e&LsTTqNmv z?}ab%j{>V8TP?oiwoP(0?_c3R`0>EzD+zuRmwL8Ij^%w4zQs?$iQ^ea0=aFMWaNDj zzQ->Fj$Wzr3%}g6U2;6{oA49anm zs_TF_Hp<-tAbNw<*6V|k>L&fW;cV37^xz7Ewbt8$GU~oVEgMDhI5W5km^hE10@y%x zt>TkBc!MYdoV8C-d7Vo@Ru#&_BX3aOz}h-EsG%+@pu39i;gLUxHn6vj4(h1OfSxwW z)1zR}$be|g3L2^_4{)vKdwQH5Tx|g5f}qK|j)1Icl$S^0pozg2>q9|q$>Ra-)!AOy zb9rk^wpnKceTCw7*4rg71kB3<>UAnevtFN0sNr}cU|OYQTs-wOe zdT;Y^Ne@L*l4nyAYTX#bs2`w_Q5e%yWrg0Jdg$T@RIY^8-n*=oeGlGR|T@JX#161vfdti z;;Krp)>ZQ$7UaZBs;oVO3$L06ldo0bvo$+-_-aKk`5I@+{rX^p!ENh(!IM`z zg0rrnLOpH_DhuR?f?r>q4DN>WcaNKcF$SI1$AiCL{T{5^fC}?y9E>&SvF1XO+$6-c zfgk44G{`XMx4sv&`r4+D^aizXY;zv7MA;~}z7`hJ-jE%RZOLPo{AOJqyyaS2$d?AS z2yAO!Ldl5r$(rD=8-yW*>zoM0Ey^B)C)U@3!>+Z4q+C}-AZ}BV44T$72eYp|3aP)& zh@3n5aG$}Xb$f8?wbvmpu8%_3eDHw5lyx`gbI4G$8}uk?+hB^pEL@QZuce(0gV4F8 zJ?~)2OY29$RoA>j#W&odC3o`OEk@Fj zmtOMTdLel9T3zTD^yd7T+Z9;&xJw8+^0=6#VYmEVR*^cX1z3 zP8$5Q{u!*&pdO~%NQ~unQL+q{*(?uP)qsImdUGtdn{vuvxy{ND(*{ylaU(H~+e6_R z=-3#BI5se$p57eC?WLSHSZT8^WGCEV=r$1<+&&5qfFH{c|ArdKsW&sQK3e%DXdC;G zn1-$}9*iZi`nAuNthU(@vcF*}Oa^&%)#gq_?Ui zVn_38O2{_CkVor+BAQw`iHOIPS_2=O3n81Xr$#KbDiRT6lsba|o9Ynn>q4lybM_#f zP_7yT+gu9?yWSd+a!auX@s!eF5N^{Fl5l+tGVh$dh(9Pd45DrBg`{2o5V3Siu@^B; zX*7tl=>>>WAG+_HB*X-z*??su4-s8=j!e0&NJ2cLv>GJZJPxV59t0tH&OUCv#_f_M zn`a@nuO~+?-k#n!_osFnNLbS${nv{k@pl;e=O(r9mZaLe3VC|HF*5Pa^!~ZOv^z@D zU^;9bQt|b77zgH_Yj>7p*nAE7c>P`E;+^RObAM}hm1Nqeg=#=M9^b}Do}1F{DS5S8 zD^&l6ZB$&_XtMOhV6VX$xQ@cyphZ=-(Nm<;gHlL<8izXHNQ!#VHku-x8GL9^WMc+( z`2rX%rvFUG>-X*4T7bqhJ)iz$CMK>_f0e72Ixr-EKklrt};szdT`vJ*@hK5bfY}l^&TIR z^@Gn0Zrkh&oxITzodq{(9&ZLG4er_;3VnTJGP?U79~$<9&kZ_lj)#7~@jY6#1C{3S zZg9$=$A%kmYUx---fW4?Mi^>+Gc2YZGWXc` zdGjT|*_4Mq-;o-#)S*a6e4s2CjM!9%dfya6_nvbU@h|0-!4sQnp$V^Z!bjv_u% zUK>oj}v{}P;S6t_4hV;wedxHhI1+2MA+f~^~KQ8?`sCVign{l`nOxQKi*>ha-E$>r_ zvYp=p1g7x=w-b`@d0$H2+bo2R-mKdd{-EcCj2-D}&SH6}EH7PQyEbflV$TZ ztFl=L9i38TkUyreE0))7eu|}`^GoS!+YMp+8>eDr-BqVp%XLr|EcC^58M1Oqb6rY zN}u8k0!78Zh>9T0sws%k)W-n_fC}W8?S-&SO{t8fJ_QeazF~8+M}aw?}eo`ePAp}75NCGd<)%7 zOm|og1Sie<=}@K>A-zy)BHxTowOtGwYifv(dpKGsT_fL$P6JABwrME7 z@*y3x0r@s`hV7TI4^6M*UpyRzH9GkYbf&FpxOy{^X*NKIEf~2g@B>=m`pve?xPj3k zshQjzea2Qd9Meo=Ru0gMrRH*_M_?T8+?>RGF)&&zwUB$EVF@Fg)La0&33R>`BUb_i z`0$|S24c2gHKS;i=89nJ;9c0;#3k4_yJ**E!8d zmf0X(D7BI+2?KJtu-TRsH#jPk;^l$pI@^Honr0fSa*$pkwU#SQgYfXy<|NjO!BP0T zmxrPoZR21vwt!_OrCGO4{h8r@-g zG9B21*Q^)v(K4xnd>6XQHY;4c1<5uWqKl-C@;G##Ee{+7TXx*gs7OkX$HVGZG0ZU0 z*p)+cv6Luhp@(e603#%^Ukr_krB3qQ=ts7d;iQ%V_U5FW=OxbhiKR*@f!$&80$aDT=>m5n^#J;B+Xvy9E$X{O+UzpuJ@_X}T?hfin^iSKL;VP}_3A&Gn72KWFEc7zF<=`=35&|AIS8!dar_jso zRz{eD##sD_Sjlyxa?v_=hTtrD>?2}YyApUK=G81uowKD%!C`;vRDx`zs+#4cbFOr)-S&tRttyFH zqvka%Z=K>&CFxMuY6`;&RW&TKj-V80=NoaUb!Q@P)Lg=%=)j{h7!NmEV-sbgRT36e zr>xZ8E+(S0H8WA`v3V_vrXz+U2zJD9YegdYaaAqLN9RK62D|+c&s)0^c`!Q7^3}Om zs#GA}v`!_;9#>sq`RQCP-EMax;zz5>9<4F+I+njqWvQFpE(G$H$sX4+ew{~vybA4U zcQ(T0mdl??nR{C z`mkr|i2`T|Y9l(%t`}%<{k?>zbmEIJo#s-e-C#tr#1^cH(FSR_yakAEUri$RD6H?NlSxZzGe;#_5gH zSa~-($4)C!|F&&X-1sPX74ly689Uub%xzjyBl)+j_Ju#|X_bI^QF_JBKeFn! z_dfA6w_B1u`AV-B0}seb4Sl$!OcD@|vqblR>zvha>64B}LE zaC^HI=y|)}BFpc%9LR#n9FL>&59n8RPa+%cL>=gU&IgS{{t^Ag?yty>I~fO5|3-DP z=FfaapH!QHB{L!L9Gp(Xaq3s}zjpsbj^AlLkn*>p6LEt29sR}bedNNOG4MJ#4-hA* zKhXc#eT)2j=fi=ezZDM_`rnc1Nr71->B8RGJsB3Q#<=rL(;)BzJ;8L{>4eV{A z!rEGsQ(h=~5V=&OA;gcP657VV{NVH=PUmwnP3*Ts9d3J&y0A`4juOImM%?shpOh!pGqkb~02PFmQaMeRO7r9fhPL+MQLSxBDKDl+ zrP6%4p&?}Nfe9%{F`J?HOAF-2hR!g$G}hLT5;rs2FO~9sEmOK7(`^GO;u*JxlEVBo zWn1mjq7-clDHAh24<+aF*Oe(D5w&(rj`6J9fTSqjtjrzOT2bv*obcJ60ZDPb1r$<> zqRc@S5zo5GB>a4A8P#4CMQl&tOw9JkB!Ya)GC%u@D7W@iK#dRwxk4)5Fwp)=lz)2- zCt$95kXu5vF$}f885Ij7RK;^dIk%K*X9&M>QOWJIoRPU^Ik$}JV7SZvK~!eDI$$Kk zA+Cr@FpRf<2J-mL0Z#6ZO2k>mcu?`R^?7 zOUjPge~fz5J_Vy(Rll+CR`!T%Taz5V+^)dduQCi0zz+4hKN1-6U{RMxRKp$wLhrVJ@xjZowh5f9Ox84f1fqR{O;0vAYcb zsf<3AUYAE2wt+i5dw1wiTELlgh>{2{rSzpLzleUF#QJ@v!7 zuZiQ_Mrxd4FBIlh-NPIXc-=hCZKB2-KD0j)vj}} zY~DVJTz=2xNY)$FUmkbmO26evbi=){q4~y&oKOI|1SV9lSjH?aL?m`{Dk2T z`~RZ9-}`<<^&M);qf4G?sOEqG@x>dr`CaxDwmUz&4B>!`S>3TIE&ZL^3v5q*PMH=E zFxDMmY3=W_UtoLlbIX2lFp1gHk(T!5o!T_EFaJ!LE|@Ts4pG|XrR-^}G#?Z?2ius4 zj<&S)CAArBe||xk5{cQBZ4H>04#~xa&JKw&V;v3YaqmX~nvq{H+~iQPY`SA0UHsndrDQn& zV%b)Qv=~LlLiz;AGm_u)FPH6b$ONcb^QiF$w*|?g{K_(SSd>TIw>laQ6pdsgzq-uJ zp$L`-y^o4NxGhRX^Cj?sAc`U0PdGa9p=VL@IR8qSpF;)AKAt&h{IA<9$yk1US)fB* z4E=uH(Qv?OBv10Mm4!Mq#xP*tOZ>0fKa!{U*UKUu+G05O7miMVUL*M<|7O`Phpw3H z`jS~8K}QnuUSQB2i+?_=VRZvT@!%fD5&*I_)S z`F_H&3Gi(sf9BsQJK!)C({q3Fzg3@zZ@81xcEf`XFJm6vuL1C;`3?6k>OI3F4sT1nQD&FaX?_zt*TzJTVCzf>t$b2{>}QV088F`Zo*uKQNK_n4Ot8J>4o4~ouYM)w!~ zdykj$VZ%!fC%5g&==`3MlB@awyO95=OzG<6bs8Pt{5AUnb}@gnOyWS>b-r`c@$|21 z|6*U|kCiE*ovWQ;$J@VV|BL-6|7qEEhp1h5JJXJT`KtC2`#OIdlmX_h!A{Zf&EK*= zV*i)_tn8LU(ynKnZO7BUseQt}$)7A!QarCZ$B(yv%l?FYoBzD*zC*^YubrQdfBB~N z8T$@`Guuhy z=hLz);lbF6rvEtK5TB@T4F7Vtw=390@F(bG(+|#nh;P)7hW~T;b64Ag)RRj;6#pTO+o)2_y>#AP!Y^S{vcscZt2ezt2Ut8!W3^8OV= zYm8hR(_=q$z0P{E?D6vcl|xFaC@WUI8<}mU9;n&BYRJrp1QW^n-L~0r>W?-1b%!hf zFDi!3OuoygLK(jFlb-WoHmJxa4GM>gjr#50Q-W91Y@Zd!ll>H3hn!7DGx^X-%9Ej^fg9-&ty)wmH=4xAtk)DE$%7t zbIup7W%@X9xJ8wYq`17E&sc)?L6DeeFeczgjT7}ab5pe6>uXrkz}0mOjH~Mj;x=g? z1e%E!VszaxGOn#BmAeGyr-m&p%xJGJBd))vh>HiSY2FqYA!>Ep6Zf>Ik(&q(;Jh6& zO4R1aiJR|v#H|OsY2F?QUviE|<39Gh<1XsV7|uH&V`1DpJ5Hl_RW2Shr+G&t*hfHP zqkA256Ol7U^8_Rl_SN|?R_&i#4=~F-5y=(}Ii8Q((3_mQh@3H=cS0tJ9ywNlj#31d z`SGjgosoM)V~$tjLV6o>6MvamJ-;5ABpP>Yj$`*e%B=?gYJLN9zi86&ZrtJCce#td z%=|jP5t$5A?mcn2y{k^+L4=y$gxs=x7RKI%y|$<0pq|?g)mozkM}rkrKtzdGxUZ?* zQgBH0kK=e8-<^4SWJSxG+^w`DMsFSe1~Mw=wC+l$wYe^|bffo&zkH+FP{weS zGV@0CTkyGov@1q_gv*Sb(i~8zoxsDQ)f)v8t}y(iHF*JmEaw7{Y!phk$%vKq!E(40 z09mx_Mu7V=lBKhGBL*!txuLY1Mxfg=GNtPIx@ad5vuI65@q~xa5W)bw+G3j{3G8^} zLs21NZT$BB%mOWdx!6pk45nXj@xJ{P1!UuDBAbPji%JRh@iF~f1w4?t*lgr)qVt3e z@%#Iy3S`FBPVC*t-$j=Q+v894tDMyWQHBk4m#B*18DH3MdX~Jp+L@h*1nY$08-J;P z=ULuri}h@WI#6 z{`C~Bg%ms!-6AB#KkIKhn+_5IZhyg~sGX1+|EhodZ2PaLHsTHxKqQ}#5&yOS^Vu)I zF583y{!8>#|5Uu@L;XU6$-zw;DYR)Ap39Fndgu)O^7orGI5edPCWyCs7*yC~a&WW8 zLE5|#WU=EnJxncJGI_sQBbBxQKv;FW_d{VJVa-9PEz@2ZJt16+SKr%Mn7C$U%RGo- zqH#iVJp19J!umDwTjviW-@v}g-T1=~-xV&dnb|rIgqUcG&=a5gaMd~d+IW}wG^9c_ z3qyRR4;{}XLXvbIOfk^{VKlz>q5rx1wej2Lk0L*cUK5_h-+q{UZgK6*ws}CtMDGaG z@%;~r&f(X^Lt+v6RrG=ID*oxi#&e15X133RIwtxIiGuluk08qzzhnLc@&~NRe2xG3 z@ZGt^bu-YELqZRZsK(S7SXG2Kjo&$+iBuD-6JzlD1GYtRrjNlT8(LiDJ5O*@O2Q-U~&D}}0X9{%1 z7DR8@-?1tVH}56Yf^8FR}2E2>K(C4VlD=-k-5W|?w0|~_w=Di-ZfSrjQh_TF` zfisY7aQCb&C@>Z~5fhlB19ipW7QLRepr4615|f$J0|Uik3wN*D!h$v8t;95@Vql?o z!lKu!7C1EV4q_&332DNPw!3$2QGuD*9pYaonH4`AR*-AKM}yLA5z}1e%@zUJxuoHZVMvPUFRS4PU|T4;B{Zd=c9R(^ z>*E(=ov68`G&t-e-eD%oX89x77HV!8O-ZUfU}nnH1-dvVTCRvjFpeiaga{c1Hj`Rt zxni2qSo@t>AtMQjVeuySJZ-)4Ug96jMj2Buf_uN81?ZZ1Kk+%UQvvb6AJmm@bD z4{OXZhh*gfSJ+nYyg0PQ_%QK*%t={?Aj?wE&-2odi}6w7zs%RNNkKP^r+Z!=+HQP; z_#gAT?7KkKO3&Z3VrZvvHt{5ag&Z^yx>^YUY8!Gh&L#fBG8uFcW?AXMyzLOlIFD$+ zvK@>PcEcW(XZ4ULXbEdrE`u3DRlFYEvu4QK7)Bvkw83(rD_#JU+Ym)uWM<2X7;F=! zo(9&>coq>rvg7v{5 zjd~g#re3Km(V%lliuL;-jjOaE<9cEst8Op|^tXe-8rNtc#@C6FthT|aCkxgy!SmOU zVd6$2gVhfl9Ns2AWWE6zA#Nq^VLcseEJ?JPftz?_6l{EPSo7e+)!W2}&fh@Bh&zZ! zSsw@Al`PuKgwEeY#)`X$*+8kSD#hEzhs`%48R9--9?;m1rHQsPVe?HxMMxE`B9^(_ zyHsrJ9$pK#QDQk!#3ITQN+)c4!)sd#c8isyoSXbisj;1VL~Uz9qWCefjzyQ(m4?F+ zek~loiUG2SVVT{jdYZXzCpA1U%lv8_`C`}R<> zc#$~6nwKL*Vn?Uz@qI%Dz~u$9`-g_?g-%Wh?9rh*akx`2yY{z&3*wDV$?WN&0kPQ0 zeRu6}!A0>_r!=-=XhA&T)VsU(_kzpf9Zs2SH30RDo!t{^9~D%J-61E4`pxQmxN~np z?MOkj*vqMiZ4Sbo*x5a?cCV@1XRNE);KY+L!x`cFTCL z(;w`{-G(NvEG|m0@*|{hX3>r)b^A zr<|7WUNwv<57^X_ocn^-Ykb;iS6mDY?@$sd2uO;cmxaW_kNeMGAWc`A}Tw zw08G)u>Z6+TX5L3NST=Lgxl>qTv1NmT+Lz6A?4yyC;QzofC2J0TO4H1BYzX0ciOOf z|L|0~Y;*NN_DkgN;!953cb^z;-LJYuFV%Bl=#g=ilgI9YVQ0~%EvHg(iv^=%B^P*p zc+z#sjUmXs+*uLcyY-Scb6)XL~yTfT0zie4{827qhT-@Zu+&wrf zy104ksl&Mc6+9FFpQ3w@XL|nwIF2zRW1^1B#zdy;N#?Q{wvo{ZE9$7!Q5?6B`~5!G zei=ob%v^?CN*9+3>Ee=0vCTF2TsMT=L+JPM?|Gd6rtSOreBbZaa}N|8U{_@Y>q7$7 z7w!&aRBQP;&$9cns`Y7s>Oby|Wb6rpTkMUjO?^m^`r_Tuj5@7YXK{{PHjY3GQeV0| z4&Eu>bync$Wd{==!RpI*Co@{L(w&cTY_qEgG_VkQmzUA5#dO94gK(3e8YaAYZW>gA z@|{bQQnQa3IE3(4C1%0O=o9BloR`_I1`{E?HHo>5KF|?z;&8J|3>-pv>k6lqIiOEVFnDl-$k8E%`G81MDv8;P{yFRJg^YmbH|8X(M^zcQHK5z zvvzo?s-y0OuUH2J_T@#>SE7r%)Lj_qQyfk;0nU0bX+{S zqq(i5e43LeT#BKDHFSyK3b4e;sO8t(*vMaF6Hz|7|6vdoS14Em63YW0YTnjVRl2x! z^yO8bp#jQ1`6S~Sw$>%2Vk2R-e0qJoi8H%*fQ}D)4fUskFz9w*H zpePHxfLbsC?VzFYbvYp2ZZkTnYHmS55GcF4tTn}}c6=awh-zz+ne1`Eq zteopHPRO=`U6ALJoe7QG*R|UFaNr z+zL!er$6D~wip}Xy4xvGD0WU2+6t%da}#DuobA4PFisAJ&4VAHSCHi6threIAP(gj81H!t7TVip*c`2owVV% z7`@o%uGhOh7A9G!W;*G>Js1Pn*RGyjqlK-2wRF;j-)4+p-@8V12^5K0sxqDQ;ND>6 zp~;org)TA&&ZQF$?#tj|*{<1L#G)ii)hs7G+@CRv?R9D$N;TVw01sv?V|lLZuG}JXAYnQgz(X11TJx?$UA0BgRx&xJhV5YrTi7+%xvn8V zW2WVp8ns6#>|%FZx4X8B1gSE)ro{Fr1tDz_H;Ha!F$wIgo08gNfE^BZLv|C2qp311 zQ*wK(g1Ghpx8vP*0MATgnHsmpD@bai+z8!a#e(N$@=Q(I6BYJrA9g#}%>@ym=l{pl+6& z3QS!Sw>!Yw#9g2jsGBF>XB^f(=f>#PD+#^;d8~dmIh}Dt+s5q)0642J&>pM*mHd!# zLfg)*raP!)^8%z${ai9QoNGI}eFh>Y&YD)JZjsDjXluK>4RlwP1Y1Lj)Gd>l44k%~ zTLibeq|Dm42x?`Ktze)X>bB9nTC!xlQUs-1$+G0eMs?BmU0Y0$G zMe8R`Hti(}Hel1=rl<7Djf-7Rnl4&gzx%hg$3ONxjZf}f)Os3!$>N#9RqZ!!UZ4fg zaj^oppB66^{?Y#67S$vCRQi%uX}qmPnSz6Mq<}0Pq?XNr1nP7dUIG5}*lyW9#HUG@ zRG&Frfxlw7Y4^H4@3DW{3TRNLtME4rPwf%6_dQWh#r{@(?qmnAX836H+}J(2PtAc9 z>U0hMmJy)6;5O3p@oCcEsxO?b!{0GNwAbAhdPbkN0zA~|ANU7Gg!Zo6ZjV5zn62td zCwq7uBSsr~OR^VTY7R6}ryKAF0J%!uQtBm^CfTZ%Io*VR0(%DsZ=LM5FKxBWFLMIC zD1)x8bjz?es#NT2cMxM6mEqX8dnfaCcN+(x%7h_K( z&F&>WOS+=^%E=Ah!+4^7>DKdJ`)93J@?SaKf`4V~5vA{Yqn?RfRekN`4)14_gBuuj zZ|*bmtNE{;Jm7;04^DX88tScm7JXIbjj3n*kV1{N|E;;+p=Whh)83fgZr?*p!*6Z( zZa)*Wlc_TGY9Cdo*Ny|XILPND&}1<6Zr@W)lW!rv5}rrf$yA&Av`;F0274ICzuG;o zvrDTs^=;o{PP1+izJ@&)ye3m)>eoIE0=W9r{NtI(_c47jJY&!ex& zyfqDIpHmpnes#(KMMYiVyy1KSrAMzr4n2zC3p;B}c=)1dZ6g>mgqx9$StjC4J% z)-99c z-L|jtrT2}MdT6|bgd*^}-Cg>+UzXkQZGa|N?1`-U?m>O4FPCntG(ZzA5Q-|`aX+z7 zwoLP;ZzD9xVvlJ33tWn0%e-%{G(zuK$SNMuv34)+vneaP>H7)#KZ`xH^{RVSpI_P1 z&6Q8kyB2ba;JVMfr7yiq)4{h1nryL$xZZN_>#HpDc35eG(k+mRI2~X2>Avo=G6&yg zC;)(f>Kf|4(YIQ*Z z;(n?BWx4CW6Q6kx;9yQyr`+ACpIct?ufrD}kYO|JbRuiq1Ns-sXa1e|!utlIxz=&qSs=X|=~QEV`oHbb8&N z_uF2tb4qJB&1~0E^wb#!nrv8wptB6yiI}XH>8&&A{<;5A1%-r*4xB53KHk~0{_1t7)X0(p5hxmY8CC-J$ zRcBc+_s8k%^N<^`6LfW%;PO~-6A&OVKBG2pu9D?q-O-fSz9-Vkdte9LE5}^AI-2s^ zDT=8&Djp^ScR|tGy0fXEeUGR;;!#}U9&p~ZqBH)n#b1j5>HO(oKhRit&s7VUwH6kN znL4^2UIUYr9j+B!@kJI^ia9z)9#I3rucX}||NUDGKcAVWW9mU4K)*6~%kOq7fm>%5 z>R5VY4-j7^xvBOzJ%L}$e4=y70~`uM_7XF^GIka!>0sEo;B2*rb!}fgB{fj&_#;FmAh};%6Pw zfun=AudCf@{puCTPT;d*df*3xUT*>^SG_XX1sr7yJkAb2dX4j-4XD3LcFU|qJoT_0 ztORuK$^i7Wg}Y)uIPAI%cE2w3@EwG{vG7zJ)_Lm@G`I>Z-IYOTm4%n$m`;O7;-Kss zO;6t;Xtjlp;*`!8k4J-8fa|UdL2E4h6lZk0Jc2EY|)Bc>Sg@dbod0)E%^_QVM99Xu>^Luc88H^_Zca@*k> z?>#&sb6aQ2V;y+BGq)$c@jk$#G6i&nJfTB~D$Nt;N1HyjM=J{JiUFYac-0NBuFhf$!VDwU8-l^kXdzxkMUID=i~=q!GvW0 za>#3}#V3D?msIyq@ucoG&)T7|YC&Hao@rbAqs%knj-H>v6+h0G##3)kW@KvXx_b@` zRaFQ3Lip;y10mZ7UG92 zsuhb)@jTf8C^iqwUvL_Rzs(HLUGN+k{`fX2Q1yq?2>e}Ui0-=Q0uYN^1M`14eS?3< zjL_Zn+#MFE6$?^bY=XqpJlG?Q#Yg07aY3|2^|9o7FnTU=TVX`6HaG~fq&}Y9n7L;f zA04r+tq!6ssZS&~WhUz)Z{tUTYNIbNE#wVJ`c%g`x{&x#LpkKMK% zsjT%5whrP=!P~&Az0$QrSRvMHO|$Jiilw@jZ&!}EzZ(nb zT5I~=z6T}WxZOB%_nk_p^?K7>d%t3(Zlzr3Nbx(CX~mwLeDmIGUC-MiBOl)-g{uB^ zT7VB{*60S@UKkmD*9s_QryuZdneTN4blCR^4>4O=|nTI*hidzLLz#Y}aMp#(xWXzZnMEQeRD; z&Ww=E2l{gQd(Cj)E$EuXtYVMu)7!S+D&KpDuWUirE#?&a!QGh4x9<04;l971KP?s% zhjriH4*ItGekpwA7j(m7QE^PS0k{LQA2cI;x1pOB%ZgLFUv5A8hW+3jv9b-_vRGA| z(d`0Yv(1OH2;Uv(FN<}>dELR=Rp0zREJdvBK({S66qj@-Znu0(2ZFQjE_BCYOL0y2 z`|ZAOl^?t#S9YPh7Tb!Ox+}M*zjXuB+4nc}H(WG*TX*aB`nSaoGm#U&dB5QTOaVP1 zFDNjck3=~L@C77=n8JGEUI#}{eRPeQ5a0_+3Nv9KzpM_>=aMJ~LB5bAlqsR7;H5L_ z^>HR@LXZ!UgfS6%hrCRI0(~UfL5MFbDbAGDJL+|5RBq5dx=W~8guM?br>6;6=(dln zXlqC_lnqBB^>AK(qjMj}qPrl?qHHN7T8{*rXo)(N7;E8X7+V^t2J%(uqsQxR#B>Qa zi%Ck*_u%NfQM0;?7-Nwlaryz~F+E$a@==$%{+I@lqJ8u~n0u0R{b(Y%fHZ~{Nzmn) zT6!*CoukEdmuU^qA~;=Jw|Lu0vT*)aC~LD>A74l9kXkwi%)}76xn;E^{`j%SXhH#f{Y{u$<{;q z=uLUGjWHWYp!h{WvGNxs0zwmge`pm)sMcD%CDJ85OV zsJi7jWCpk@b{X$(EKBm05ye>U!P)xWLF229OGzs-q8gTBt=iS-ytk z1!jTXS?`DAr#`vfnULilkpwICdKbJ4$Elx6?l>IaA7$etO7$*#SB|@X8oL9LEILO2 zoB2Y|-n)K05nMtVzz{$4jh>hH_;|~w%a>09d)x9F z@~vK=_u}~Er;h(sfDuB=f0)9G5#GDw0!?B7<90qJc?0AY;=N;>kxit#X>t@z_Dy7y zUa~iGf&g@G8F|WSwj=Vh-UILB6Lw8?chlr4T9(kXc0Hyy9&p{8cOeQGaM#D|0O-is ziAPPiWSRm-o9@Q!(JS@70{rgaWQZb0hwjeo*Q@k)o9Jt*PNpehbm^YVVZB=K5CD2_ zCPM(SPWNJt={17vgIqI?PD5gFbRXuFUYj=q2;aeU2nvIz`!Q$qy1k!F*fv+wX()_7 z9ayw_L*6xj{NAKPfO$#}VlL@TdVijH)Qn4^DPaugALtv+xFg?*l6AaC^&{lk75eqgnh&(2xiP1ga$qX^cw% zRFv9hb0e09^Lfd9rYZQil%|NElhRd>IA4%_42lL1eNFQQGK$r9!TT%nI80L{%G4YfE>rz(!nAoAkXhi$$h~+q#egxrk6AS#kKh`ru4o9 zKZKmXfaNUao`L>k%Jxh3LmEi+(O)w?a6>*dQ$b%gA3{!I?C4cYZ-AnIo_h2J_aE&f z<{G_*>5rT98JMd268sN5+J^y>d;e`5Zjzh_pR_~o-Pwfbf0zm-2l z?JYlo7C*!n#*=N+eB^sd^oC_UG7h)TSB{5m^M15)O7x~>BQgmmQc-=r+_XRJkEZIo#lE*iQx7xOpv7#mFV#!6aaK^rgJXt`m`(j01EjvLX=`Y_$ zJZ!r+V+AYfX4#DdmvX+vJR4xK`)Z4VyC38;+*RKyo?rVCV?|rk-Lel^hI8<3;iZEq z6ki=t56c1ME8H#LK3-+Jcjk(Ys3$uwstT8RaFUnmmyv0#TjWU}X1>J*fw{cHsBI{_wzN_mzfQEMc(uY=AP$%h;PQuU>f6!eCSim zFStj(r}-}Ie(?Vm`O^7J4ldW1%ui&?1K?fM$7}}K1^zpgU%)<}Re?|NxBQO$ihJQ} z&u;|cyOw@JfaN@L5ckH{i$BTk$g0p!2(uELGp1N-(0>4FyXZf5<)E3kl%4%eP8k&IUU&*1__~-KaoFh-+VvvV>!|} zT80T>mYZNIhwsbf7jVw!R2U|NTmC}+#QpFc z*K(s-GRBm6wggH6ukGhPJ;beJr5RHa*petEyaAx-x4D9OGA5KnHUf1Bf5z|bG_r$~ zmu5mqV(&+Rh^ilRngDEinKP6-Y+2M%eB?#H(rLSnXdxQNP|^=(oxt1qb=LZIEak17 z5xr|EhtkA5__a)@gE1vv-HE%B3LsSJ=GQy@vcolh!jzvZ34RLkK7PDuZbwPJ1BFkQ zRLa`(?ANCkJ7)4HDEt&jbQTdG;|HBVbRH>iFyp669s<$yJAMZNaPL|$VaC5Fsg`91 z@Cx-AYG+A-gE{}cqv_9ATh^>io1o8Itnd4nI3cJoVXRx(UF8GYwc{65R8HL6cMc_&`>lVJxuYAU( ztG}?p0#i!VMS+KJzmGGqUEYysOJ}AeF3Shc^JCBCc9|FDTRLY+>SqPu7yL$MK6WJ) zsaiQ_(@)5S;Me@-W`?@NBxqKc9J*mvIDXr2Yi8Shsc6MYG?yLrC<-s+51mDHA1QXA z@^dB0S+RI=|AVuqx>?#>YBP&%j7r2y`>V~K>t+>OpKs1%pF!Ql%ll(z-Mhz%yUsV~ zvni-lyb9nx?slt`(4Mgh=;m2zcn$vxv&G$)OByZ|J*NMa^&kF}|4jgbWRw_N7ZuVi zvNG{{ARs#3-CxpRT~tK3%F4kL{bOcfJ@QYCZHkKN=d<$hX8!kQHF_>TX|O3Ov2>Rx z!dv<0&YJYxd!ltQ;fdu%)KmQ5{t_?jds?65Uvz#d`FGZH{B{5Lvr#=_PgO5Dmr7pF zD#ttdvuAUA%%A38qRf51f_jbj@E@A3?TLOW^Ec%=+YVKO_xGQh9qOrjn)Wy41^YVc zJwDujdv?1=uvEsD@{(;2$g((pp$BrkxKf%eri^|w>l2uhRrs#g8(a#xj45ZIRA|Mg z`D=W)=`Abuy)0T`>4a(r*JOI%{d$*5S1yZITDqV*@cI5`-_v_FpZQ)9eP!u}>cKzt zxBXt(>-}uyis)-gcT_*V!r$e4cW>D<->af;EIm=f__zK+-&cE=o~>LJt+Mn&jo}*r zohJKL^SQ5`XtkveY6|}a%fe z-S6wt^Ob9&wU$AsCH#bc3y4=~zVN*+`pz;0wT9P{=>7iktLux2>-=|;VObmaWq&tx z+gH{L>wlWxv%^u__)Y)S?{ib63`b65T29hQ+a7^-~5ps4fIB_0NJ_Y zeK%e@kc#T)v@B75M8Kgrv%ZX%#y5)U>2Y8ePA=f|oJ(K-%Z3|84fKR8DSdPRc`mU} zzRdV$Q6oJmOGaNK;KE#S-{rD~n?;}K|I7MA|5U(D;KO8;89Nj;(UY?j^&|BHeCDS6 z`pX&|ikj&__0=Z^#LU6^<;#s7i(2USvQz;hd4Eo$|8jYQV^J$TEeoSh4al1_>(3}R z{ckajUpo-}O6C@& zmtBgwqW>zueST=5?p4|?%2)Pt)OG!L0b%po1A?z*+$nwRm#CZip91cJA28DEG`IiIeosK@yxm~k>ogC_Ap14SQ-3(1c0O!S@QsWoWr$sc z^3k6PXq#salHR0wQijfT8)?!RR+Kw<#m+T2zSsTEN`=&|ux0wA++#?Dwb$ z{hfgA`Rzf$Dj6@zDElKSMqf>QL2d|FMFZVldOcX;mk3l?&>IS_f_P)b>5W-Q`m%vX z7i@>Bt7zVs33^jjvOY2pzYsLESq1UIOwwDj?&+%qo&^)AxN4dYW{Unf>w*69z$*(? zL&4P$Uks1l21b;yfo=4yR%C4Z33Sx*bZ0M(EKsq>AhLc^z8y`7J`O1Yajub zIeK4KnZ9G-=Y>bZxVN+b%shP{>y^HH;J`xFaPV75AZCF+1O{OI0%sTchO6Jw0x>`6 zBU$hC!veP!HikFfLV_@h^wF$3{n$Y9A95qO+I*V85?v&*5%eSv{x~(_T00TMUy_{6 zY5~}h`VZwAZogZE!Gqn@J{1wUREUvzbgb5h4QF&(_(!9!^L3Qhw z1ls&49l7zYE2Mdi{T2qiuxs&N{Stojs2l(ytEm`!P2%_O2_m`6v4aYE-{2 zF!qPUHpad+YG#E%q8}4&;+7 zf4F}ed*2n_{EPh)wWvQ9*!bh_H4KYO`Df_bjAe;)ixmb$6SEofwfLg(oBoFBXc`lBQ zw$|m-oJFOivXuyiK@p1rV`BBHvCc3l>Fh%U(;)gHdd$2&Kh{}HN+w&KU>TIXNE}N7 z=#;a#)Pd}ygiArs7wyMd>+|EB_euSceUflJ=>1~Um{@~qyt9OqeD-OAQxJPG7xbj_ zJgw=^_S*PNC@QRWybMG$s^wwJai z1Y2ZMDQFH+DTW{tEU}E7B(9$2biNNHrXWvo8TwQwo}!gX%8^RQZ8jbBiIGkEC)?(eug~698a;$ zE+aSwe_nnxh5JH#h&hpRCHob@J$PWbYAW~(G6tIX+@C%+@?whLqLi-Q%XUg^L zcZ9Iut>ulW68Gi*L{C}S1K5lf3|o=qX}0-364kW2sq~5PKd`HV<$1TQJQ6)^<(Pe4 z@;(@;qw-4H95VQ)rJTT+G$UAN#fvx7Hj%;Ck^=KxgnRNg-4l88V2r2;n*zpF2*tq{ zR*HF-+Zz}~+9|iPzY<;s-voSGM!Rt)CC0)-X^`+H*lT5y*U?^)nV@TRTj?9&LvYlJ zFkhOj#Z1t%@&;P~se$wrG~b+^&veE~`DXJ7?BMJbB0mWbY|eNo|Lj>pZ}9UKdk{j; z&vMq63d~+0j0C@5iQoU|N@5hp@vm#`DOy|T?0|liPO9($gJU7KtCiE<9V>aFrdD^A@>Q3a{!R!k>MyJRl>RjMC*Yq0czbLy004a7oa*N@NL=<9mf ze1p@VlxQFws)UgJ12UBq7Yzh%Dm1VNWrBYb(E~ECC~ll? zr6&d#Lrd4~=IcQ9pK^=StMuI9T4?Qh*u3DN%xj7}r%$Qe;NQ@;b>=*2Fzq$PgEOG? z+Q1`pe^u3d@F3(3#xrFoyV}4nbauUOzIu@M26H=QB>SB~Sm@UJ#{A|WqzdDeGMZgy z5F0A~Q*L1?$E!-z+iG0tlfnPMIv{qzduXLf)W>R4snsAYRO6@3LfKG*+_;YvFT2fv z5vudkYhh++qMGl^DU#$GJXRwAOk9v3Hm)i1OPR^;GAItc@UwW~@^C{HZBanVeDANDC%41)W1Oj13dtlv~D4(poV0QV!*u zF}xe5u&K8kJPBz{Fgv5BY;Kqqrm<v&Hr)!RJJ#)3ya;9SW)3ww>3ZFXev7b1}}Z{_{t4lS6g!iM@t#Z5Qi0P zo?Bt@tlOIz9BpNH!~U?!P4^X&X2pipOT)D^B;v{5S87PMu{tR2(oLuSPS8x(?tc;W+4S$D?UKCgr zn^xsI=Sq=sVhl4$5?jbs(sUY^!s3v@LM=QTxkXrw29O3Nk7J^I$M8V-@h!X6y6LnI zNSWvhF&Y{s`sw8+X*Im=K#+_r0Rb!KLwi(e!~&B-++hsSRr){e|Nbn}a) zF60y#o(+Gvb!yFZcA}eKB4v|PYcs_@%afey*Y~q0Gub zx!Eu!9R15={ob5be?qy{zsg?>{|i6$%W=J9uA)Do!pa#uKIViIf5om#&ua}NR9d+z zcNrFiQ-2k#pP#Q7NO)y+OZltei*Wm2jqCU3wFVPjTX`rC8omkl`Zc-UF<&v5@W$%4 z@;Ac|;ZXo5kzUXmN~p5(28)T!;iS*=a>8?enf$!Bpf#LOW96^> z-SBJp%U`xHTNm<&o!?3Y<}4USj)cGe74=i>hw6xPtyFN%GT3fp|H}Po{v&_H`JGf~ z&br}3_{gtx+vp!M-ze`nVai*EYvFUhhJM!lNc%?lz==@aHQWi`{mklTcf=tY?^N#pfq#n%1TBC5n8pT9FcMw}gRvo7KT&X+&THc8`xN~aDb#-Ez-yv0yV+Leh^&RR~$*RK) zzf-CZs4&TkJEWcaoNsf)tlZZ<@FSlIRCT2xb zIZu^sjmjb%ciOgCYu4YJdpXaP?To4;{C4KH#@4#NH-F{4P_{R!i?BUNM)@Z&jh}44zYScT9fWTi2RT7_fS!e9P!-#LJynZ0mXjfjr20 zt?X$u98tRy_Dk@m%mQVIQ>E-Yub9m~ScXbHa_bBer%ner^7QEMi7eKITLl2}g?W%5CE|Xp5M!lzI?D zl!#Q=)!PoESkG!&5wH>?(S;6p9+Q4=gHqvdkZ@YSv2AG``c21TNAu?ol zV|#NGvWl5b;pVW6Od{{>%I)B`WL}BPSakvqjeadBEwWeh^eMj?Gn%2ae)Id(9kz3^W%hH`2 zzq;0&=Q)GQRYu;Cle=v@tY6kYn-@64%C$zpk*m9NJ7d4Pem4K$d{h2t6dkD!h3(33 z8*daXri|q@7$rvT|E;lmdAnhwXenhPr`aeq^3ZQHu)t%yS+txomGi~uQKWdJ<8H@x z#b&~a6gu*)r@kNQBfrIcfs!Yortv% ze>?w{`kk|Dv>UnmTR=cuKuy3!03nzwM1)333aAL26(|sJ5kx?8p+w0jB>|Ga9RW2# z7a@d5t~l{v)JcIG0-ps61YIBqQFOi@DvBTwE+8Z*D`Y0jg`xKo4@aF7s7u2M#tLbO zq>ASrARdppB2X<5A}9>efTl|1$`i3sZUX%RHG;7KRZ5l2MG*;6Ap)BMTY|#E8n9GE zt_sm4>W+ZCAYLd|SVJszf9_%8xhRGJPB27BSOjKoC5t{nw2mqkxF}dIyxmcoGRIfmppc`ZoIt@o4bM=Tm zU>U1J@Cn3GbXpRjluICnM6C-f3C=<$MW>|@=v*RTwV{I2LPv!iVbci2A)r0oi8?5# zDdZ+R37eKisO6dgVp(1AywDS2N3rSs2=&}^L`Ia3ptsPh@TAzZ3_>H9O3aHg6}%^O zRK!snMyfl4wjq{8*$6%px+&5v-g1C@3~fs+i*gif7s?Z{+1GNAdjf4otd8;%oEI7w z>E74!2lr33J+Uq-RuB$31htW9k>hHj!M1i(y5I@OO=!17i#%5g?E=)P0znH%9@GZj zqQKQgyA%7PDg`|t$D2SR!79&$}x$3I?D6U}if99eEk}fEO zer^D9AqtUP3yA@8m6;NkfDR$9Ma>BgL+VA-rOcGMhUf_5PSm#G4nznhi!eiTiRc)j zNb~|mP8bi1MQ9kDBctPp`=aH9w1s_Os|eU3stGy?aE=;67lq4Ve$p^iswp~|s2Hs$ z?2kQn>dU(m-~o#I@(F7 zL->i9qs+8A!ZMdh#774REeX$xP0AFnw^)HsiWD6y1Q$6ZZUbB)?s;?p(L6d`=!D2k z@ow1`fN!9S0lHNnWFe9#ZgZdor1H@q4If=84LYpR-9pbqZtm+o*m8n<6a!U8;4fr5)R;#O`i zG4grzbx0lb4qQ#qMF-)LJ3xFL?Fku!euft)y67Tq=Z+BHM@K*eMH$azkY;*ZFZ4L^ zQ}kU3N|Y>_jx@t@eb78&do&YbAQ~eni!#G={m`?-p6F7@HPL#>bd;GsHvqjr9FDGq zgo_GE$tsxymEA zj0%4eyAO6X6D_G(Xnj&(%%bp=Sf@-SCeezTgEk^r<93Aw#l>aSG+d|%RxX(oA0sKE zBEAf}r(t%Un};?fCC4B|48&t(WsjI$;1;0I0;l7+$TjhL+4Li3*4#q0B`FKsl7@>5 z9gsb0X2UH;Um!h>IVX}OPCk%+)a)Yn3HlQ0S&W^?JMkEx!e?uf|m!-Jf*#^u!V+7FH~BabGz|?82@hjQ(>c4aB?_@!mIkaPq|TRYX;;7im1EN#x$XqklM_ zoVG*Md||>Dc$=563^JuOmQdjDgxiHfW;=dP&?51V5%Y& z=;#*|DzS;KRIma_vZCce&%p01V6_w7s9?j3^f&D#^j~qdN3dA zB)XLpnMTtPy(n3Z^uxh?sXVYxnn}|W^_N^muHs;R)M<1F*bp`Y`p;>UA0FmUokjPM zp3-bZUrG9)R`IX^>KwYCR6%nQ?U5`;`RT&~sSD^~(py@P=$hm*cn<^%qJmc^q&Pyn zsI(OR>jgqf5O)a;?%RQCc0%f=Qa7O`n7e|WCUwvXL@lK9lxz%ILbz+_IZ{8ZQq)6g zT&dfjC6xOUy+|6PHHs!n9a2^^8}%V=qE|?K8dvnO)CJ{A!$feHL;ocGppA-tlDe;q zHA;-2?x4XlGHp?GN~%-25EYO=9MVV4m#=veTVkqTwO$;endEEutc7A9-Nrovd# zWVP6{unYuVB^J9D%#{*v3;;t z(mtxIrZ74cse&W>#!ka}q{~(PD6kZ&l8OO2Gn5o2(8W|_&H~KZ zB?FjLN&tC2c1Y}blsJ+fNdS@8^6{E^rT8jn-YvL29+$6XPx zmJK-~Y~50Cd`=~UtQF@X-XmLn#LpU5M72}_!dP68_?qnUkyUG0F_o&qA{)mgiXQ;> zl^<}cs22bW_!lUqX&>-8x@rS^LbXvTAzQ~4i(foYe$?+G>?!pUD+_id?zOo0f!U*z z7pI>hY*{bKH{+Vb?;SXL%<1}74SAE#oG_$9kcnn zr@ZcAh`;_pECysPBdOE zev^8e^^N>7E@q$D9||W;?24;Wy;&3Fmbm--j{I@uWTRbib*e9mPv*wu?K20N0+VaS zHK`ytOzw*--{<{!sYSI6ap;NN< zW}t1UB4jKQFCl@FBcDpQH>=~us6dV3@zZcUxsX%BH#F)~V?i!KHvXu@Wx1MDvA`@% zjc37)gX8oh{N^mEn-<2=f;!aNnq#P#I3kHwS;ADTpWJ^UZQYC zyV0$;-Pw#H!tQJfArwVon{Bq)or+QsWlk!h6WwcKbGsXN%k6Azx;f?0DW^Im zmFecRS$9r#O6r(XopR_~s&jtV@8ACLsO+=%=Y3tT*Yjz=&^3e#4bjT8EnA}dk16Jm z7bb@EZP!1p{HbMI^u1%6In0IQCH|SkL%eRU@1q|cJ2Pi8+Xy|o^xqwRZrKt2?AYZw zM;0QN;di7zak$Wu9sT;)-8n4_Szu z&Qs{#sGv7<@3Pmy=x@iq&iS^`W;tdTMnmyOuEo!-H7vsgfUaw=(bBf&N zhC|r^14V&*eO|I?ry`%ZNp8K%RauOe_ilKVN4poToO{5{2D0Ofk$e4KifBR+dv1eU zRH!PO@#@}yS9P>+(T2H~+zLW5kOaPW+e;lCT(o`eQ@7qw)o#Y0_wIT%Mu!*epZm?t zCJd9q7`=DTO9RTj(z)*LQDLeahV`D=3qpxSyZ&14o*#zDC5=^DS`5*mqK>%<5TKJl z_^t9m%c_Op0u2g&MtaA*W}&-nVQje4$oRHM&}oOocqCjY{keAqklU<+#|Z6Xy!cBBFak4 znw#(Mz3}RbmKI^=`9cuEvbX=ky_a77(efhvJPvH)MS1<>_g;D3j;<>TnwPm~c%>z; z|DSt*dfBaNE{d8bUE~^0&DT#DEeH{j(IN@4TNj=rT@?2@0MrLmx~V0o9=;xZA|(nhYu~o(Km|9=XrUsBDDL{ zKRdvpAo^}m3PW>0}0ccF*YYmx)oUtxvgpmcryjklREHS< z-E%Jf7X7toYTjp$sa1%>q-pYLn(0(TG19@sGb~bam;?g_jV*O{v8zKUT+=!yNM9T81bf!G%eKqT-8L4t1Vkt0e^_TRBqWMD;07 zbhzM|zZ!9rG*doX<3bH8&Tx3_*|S;#uZ;4!8h7f-;(UkCo>Qw45|W+VLE}ZG7Rwx5 zyuzX+5|X_ftszj^#Ubnh9uEhx=YB2+tQc z;e&%+NzE&saCqxA9BnD;pJ(#GMpBOyBT#d&SgHtqmdOiCrAmu2C|_&})l%G#GU2cc zs=OGF;$St@)Q38>oS=!N))oh%#MmL~#NWq2aF30nHWf#r3bANfkgOQPCu4=w_F^7N zhZWGeWW^w2!>*py6t<<}aC|He?(eY(=+8aSwfJzjfL?z*R>8diu zLQ{w)(&tHW4=N8gMMpfG>Be7<&7!`9e?t{6Feazm&Yd5I&87ZT{0P;J%Z`~Sw_C(t ziOr+_Q#^{gkL!%dsj&0lM_><7zZ8E$P2$F5CMx=u*}&L|Y9pPAM&T)p)XGs$RFpKf^3bTVOprL9T_L7e6^b))n$`Un9Ti?2@iK_zH9saqj#-a=r-YOB-1#-ce#Ek!;- zUm_I5Vrm$ErVZF0YOeGR`YE9|R#n4TV%mtkOg$j|ioPTN6szt2&Y!;-dzE@rYUk)g z3}olj+6C}au-B<2(glve#BBCNtz95L6$`jRDakRO*vZbRvs=ms$Qkvxbh%?LahyF- zXBWgz$C`jg#Bi)426A%LcES7%?0xD9={m=5Vm4<&Z5P7dj(tr1Nt*6>pV-OCInJ17 z2Q^}9k93dY2jUoKDGNJ7HG9F7^%ev9!xE(|ed}fe4}L2kd+53u&KY8GNQijr}W4d2pq$N-d73 zyi?*Vjs4-Keb`Ub57H6GJKmZ&QB!|}=>YaC^|N%s@vZl8+{osZRq{ic|ESYsHW+)7 zE05Z&2PwT~2F*@}#*j&gyuN1rYWY#kcQgkX7Q-Z|dDIp?yz*&g(H6)8F<$FTysnnw zXnqlP9?e4*i75oxc@TWQ_)@GR&0FR+w;43#T1`3)RibgG`N=k7u99S+|4yf)$~3OD z5ZMmQ3(^2z3)XE^g=P^gLUs@{os1OVL4S>sYp^uBtQ@XKEP+;=&O|9SL>gDt2+HWh zb7&odWl~`&w0PM`OcJ?Qpwcm7P1RUGT9WJ{<^b77h-qW6O|{rSTAJ)OrU9;4sx}74 zq{fEOGG$LNm&gS|3^;;K^-yKVmA%0{CHD$d;OaFsVk2k=WM47g$TlL3p20IUW20z= zve{1Vlqk@sGx#PA@QBM~ZcgEp0ujc*5SX-B7AR7EoRTQLA{FRcO>I~%tpNgR2Pii2 zm=1=>q=&B-os8|&K#7W1bui*h9oRLrQ?dF(- zTII9{PG=WqtQ|Q$nuNNj*-IOjy#}{n$J*2%M>nD_YxdKoWFMVAEFN1s0>(bn70qFq zP037Wln-TH>Y35as9uePhAeS*4)DoXH*#h)8P%sLrlCuSpquMhmwI+I1$6_o@~$Nz z&RITVV09bag6h{)(y%3TXSojr^lGE2r~!?NMk!h2eAXueJZGb6sN0%a+M&Rbt=Rvo~D@=RWfXy?Tbwmb@iv4Od8P)Mv20i?3oer{H7986N;;jzzC)mk8m*hxboexFW65Rb(Am>L^~Bh2 zdW7wyrIp-V}54rFg?Nc&~i)OI6w96U9b9?vD5Sn zdzp5iLYxIoI5k7$fE zRS9K&f?v-D$pzAGIap_C^(A5R^ZceZAbLnS@==Y6rY&KC%OxyH(nHFXk7@4HPL?Fj zzu=djg!qL7J`~MkIE*spUynbVl=F+-5Bzsn@Q{@3oj>U}o;2}`-Cq8C>O3uyCS~9kA zg|`9 z>~R=6xl{y%_mpHyZ-1d_7H$DOpfq*C9U!ZVuJs=?&B3|RLrZfOy!Fp$xpGYp0P|K4 zdSq$A0{Z~h6l$NoSdMDN(HW%*pnxZ)^!4eba>rJ0IAA!UEyEtklgVeqwJb0;s-nd@GZ#Dy6twKi+bGi*ZzeBDS*WYQU&u1&*bT7)O<; z$ytE8Ev34Y1!B`g1N|x!8TTHiFBJe@HZe{6Oj5;Pj9W|pv2?S`^*{yi?TV}UzPR=D zpG$YSyb3g@b=@ki;R7s{{%h%B7u%)itwFboYxx1VE%ckEm4F}>Z0)*TT*qIE+eZJb zwAqEWRIxSa&ge$l;MNTKgVNJ3$x8>eYVV{2VY77y{aNV)oBc~|wqfovpy7+lroSq^ z?b5I`YMbgVqrnu0+e3d_`o!hZ(t>T6-x!VjEs{L?MCq8zV*oiyej_!>BU%s8zm`t9 zdm^gU(oTweqa ze6M|&{v&Er>t*`EvVUEthak7(AElo`CAap{kCn{@Y!Yj`_EGv-`Jc8o=w)SY05mGt zj(N=JG^OGO=v8Gt3zL8&qls_kw<{tazc*vcMfVz*gaB<6hBkmDMf`UY5NB;raRkDyManZZ2zE$Ojxr>eJC4 z)SlKi^r5nI3$vDu?HGAF`U`4r>p9emvc837%dk5|&-yQ#@?gqoEwd~?noWh_sONLjxnTiftz7@Vpbot=jA1>^JDzWQ{0Tp)$s1FzhN4my)b53 z`7XCt%gtF`FN<&TD{vk$tI7|%*@mLCgGP$``EndCCZ^nVk!L6?TRW0&L@5EO$Autq z!sa8{m{*JelL|+Pxq|C(O9<`Bmb@a}l7j#+WgN|zI(c4od7$Ba>ylicH>{g`958|1g) zI5Ee|m%HbNjps~Q?JRsfjvv!n&Ty{^3(U=dav#3~7awz?e4TrDSa$BjnB4>ZNnAqA zPvz-Qi0jPF`OEGh|1>Ts=9lul&`cYL-kIGa{ux|y%(e1jcc&GBdvc)e#qY$W#oPj! zL-301Jri&2pksv-#|)P1-Q!nu?#X#;_dEX_4ysJ$=iPHxjPIFvYxjhI9+w4-&4bd) z6_g)R-;F*&^|a>1jFmrdKf5C1hmm)qPf-_J_r{C^h10a613ELK&rp|J_s1NXD*x#I zVZ|7PN=!k;Ob=89Wq<0Y(f6p~ z*1i~7g|kOMM8^J+Pop1DBdz^0MjYOQ6QO~~X8*Y974CLST}6;bX2bxk_p=_NlC8hR zG*?7<=sX+_1WgrB@U6JPnDz>uhYs?FT~o!A{J(GyV}7jI>~TFpaUkeRF{Jo#PBZtr zZuc-`oIfD>LV^Td>(iKv70t8VsE6#YNtui0%`d0jf|3BQ{F@IEC z@hDt{J{0tC@mKz5+&@q;z6Y@-!J)2yi@))|;Qkl$cg1s$>#G!pg8nQ1m%m6e74xy; zoyUt+1BbN#rT>S53{K3y75{onj~t)lIE^{YJPmKdu&tc!=^hz%ST&6~-E4!O$(U2= z<{1tV6U=m`jTwPQGMp-XJd+}OA=<&5VV;Sf%W$t;>3Ja1<_N}yi7?yY(F{T*8>-Gx zkZoYvnvr;ChHvEt&r6X7M=&#(GZk)?E{vec49~}rJx3%n$lodEYTX$tEAu@+M@}6< zAjozK2dx)_S}F5%fql3HLAF<*wFCycQtcVKI==v6OGYX%S~5da+2NV6x~D*5OP-~0 z*7`CwRQ7o0!DbdQlRR6oKpOxPg#pjH)nP{^Gs&>;)CM!QR}OhzSe<_q@f~@t!c7~> z$f+EIong;W$#>*=ibeQ$q5CR7c|KN+9v%74Iu8w-I7UI`OfOUvMUrY~bwI;VfFY}N zhW%ECWW>&jLgTdzhN6<_#gFQcq}p52XxIrc)If*JiW-xQ*jpXZBrOjH6m&0n6s0f~ zX~m!^S`nk8a*fwn*l>&>txjkk?K(zh=Dp@+6|1J$~|5mqQ(kGW?AQ> z{k5AJy_Lsc_D(sLI@`Jc9jHxZJd07mR6OOFWwy~pkSp89xL2w1Vn(ZvNq#MK6$Im{ z!G|7Hp7u(P9yq3*BVLGxF)-s<Ghh*gmD90u7}}(wt#~v`t71^(Yp`dj8PXB7m4IezYZ-y^9{O#nMk;bN63rZZ zJtI_}gMCXKhUtOP+Z+cWr`2)^b`A|I6Jd-bGar64Ot}i{OG}YiFh;Ujh=;qdT#Mz< zG%}Hskz$U=pJXJ+yWq+>EVDQn7n|4O&oGkZeE=!OmWZ55zcOOKpN6xgqv19n^%o?A z-=awFfFsa{zcj;ZIeLrs2BTcQ2)CLpE5*Br1JP;P0Yqfaiwxx|E(OWeqzifrvG#%;M87aEgaj&LUf zbwF!nSmYgW0QQti+{r5xd$eyEPvkwgyqKwS1e_|0z1j~jaT6U^#ROKUn6Fo&p%%m# zlRv=P&n1VX!w6) zhA4L6UoZybS~#E471~A22*pACbS6@PhaW9?kH9i%ic-8gGfJVtGnr-up2*}V>Y)x* zpuiBAEcC+yGE=1Jz$Y+!6cPeCR#A;l4_vP}kI!X}D<%l`Y(XtPfLRfA121P%l&M53 z8?Dv`F*6ho@MoDB%D#Cc=%*1@B|LbIYne9rBbJjW!8fGL(Cdd z^^pxiMY}eR*{p~nyhuS+1yQ6TfdMaMwkvo99ZOKvMUln}PT;^LNF0LT>DM^@Oke2EloJ z7W1V-{h%_IQj_Xu-GJ`V<}k+;4+v*tGipZstV!sL+P(1K`5K0{9W|*-tQ*mnwfmV< zijRa3v12tOORSsFSG0$jHp-bq6q{0;>TlhQ?$t_|NTo9|fSpl0;%`kx_i2lnXyyKW zY_?hz8DL0JT-TN`7bpXXV)jsNUw~nY;-8Xq&#?MC81qDh|1)zXodMkNE9a{jl z5-C_h@L=aqZYEx5E9!!l+W%>DN2_LrD0dKFum|e2OU2vJzd-{nLV1ulor6^4gT(1* zlUBo|E6a&q92WfI#bWfJwvA~yQcn!$6sR%5%nY*y-@%Mmo+Ku5dey36=J)0Y_|wcJ z|LoT(yk3)BFs7nR!6@mH3U*QDU>4yi4(1dzC3sBE4O>VfB*bFOQI z&96sfS|d}Xq>u?ng3J1_kINrQ`ai#l6eq(W@4PadVh^mH?s`^a*HZAQaG;wVtUL1GlGC*IjMX|Nxa@>)oSKZ^Gw2Amb+>t=>X5B1rx=T znC%E?7D2@Z97|M-DvDWXMiRh3ui8Mm#4BjQL^F?>XA@jm!K&?~r@Y=4RW!56JeRPD z6|UM(`o^=-V5rPuvjc1!XsS|@J3k6m7EGxbO(3#3z`F|P7icgvrp$~XP+0M*lcXem zFH9VnC1z)WA1g_9k#vA>(~6-pVUR-zWTmNYlN$I@ttvVbuH}RfR;KC+=@P%76%)fO zH@gwSSh=bMuk(COfj>9@suL>jQ38u6N7Fner>G*JPv&eOV zFr9=&u0qdsiD#Wqts{2}vUL+I`)Yv?VH&AZwUulVVBZL0ji6R1Y+zkb<&fRqKGyZc z8o=YK+r+w}Dj?enUE8Q^L#@EZWee-3s*+r`lhLMSi|fpKkFBgbDxPGyFux7KA*&TZ zIx)+l>L4cwd)g!%@^M9oE|c{{)kDq`PPHMpL2 zHAKE3%x_1;ksB2&b$P4_)foA)u%}%TM{ZI?=nk;Hs;0=Fg;VVa9=TZ&sXM|#R3RxY zqA=?Yo1t6V9eqI^AqPu3`?Iw=cNMWG~!dh`-Lxm7{el>t95jFKnHPkzt0 z*9sT}ISW_Cpj3$h4LJgPoq$ECVlA#(N9h)28zuzyZ34D#^}N8UXFG3;GzO8-*k?j_{OJnSVTsHSqL#uew7v! z2bvC1ys^U^PdLd+fN%ZG_~8yqyzzv2E#V9+xvGy+7LPq4T4OwEP9Su##8no`srZx= zmNmvx=Jka0tgNaL%AI)43DH{PX>$_cA}guGDz@luLL3#ln7KxMlA6Fg$4?{N0{6H7Vk39pq&7kP z6MCEOE~~LBV)2nRojg<`v&)=LFtc>v@oHETbxM`U>^5f*9oYoku9HZU)k zvk9+RcdF_ahp+7jo0dfGQRL{XEK60#;)Jz5rzJ_`UleOdKTxcojQ%!NWQ4p ztNXwjsS0wb0-spUM*B;Ge8ME_uc}9jyVqv_II+?GGTO816KlNc^b|n3sy;6Muy*XnkxkZL(T8=@Vr{Bt`k>ZP&ZKU(UO^Y=5O9z>`vk1ZI5VL?%SI@hz*6hogvs&kAb z!E#c*9wqMIM_qH3Y7{65-m!7j%|5gQMQ2c&bU;u=SRA{y`ixIn zg1NIRO?pdE1Mcii)mMB9;m8l#D!nbJ1I5#}>U%!A1i?>TTMbIoaa~C4j_PMVR}*AE z;kSwJq8oH!v3siD`MgLN_({7>{2RJS7ZH2tVD-N~(-VjODh%+qT6-c*v9Gz-y@00ZhVH= zf;Q*`v2E2{NbRw@wHf03=o7j%v8Ssy`d&?xb>qJmKM?HoN{sEU-tPM}vA0|GJ@cXY zG+|@xrRx2@-x6)kVYV|Lna>bXVy{=1`ns==I;Yysd~EI{Y>mBBUGE#dKD%XFrubQO z7eO35SgrSsU*CByC)56S!8t-^?C;g*eRJ23pPR_Eez)kNrHd)A|Sc zitcc%P0dU{)CS7=)Lqu+=w6*97Fpx$7qB7Y{Kzir3v{2ZI2K()^y6>nIG>tj9Y){K zmBzZ(g!pA`7&||bWqpb6*Hy-1Yv_LR4U`M1+13&CrkSc(a*fFE)P|G`mTcoI^DROx zs5(>q?rhLp5bZX;Hs2xC$A;GA_`TgQe8IBY_^0_dLUZiu8VUGKusxz20$L_3o?&oWk{(}EO43c_XPwbwWcYZID z27b~0AO_>S?sBX$`YQ2G2alr_|7^|!yEs5XQc~RTs8+cN_FEAxpA0j$Itdv;Kn~B_u;$le(k~l6U!sr1>wxqu8_X=6MHX=%RhbkLI_8C$Sf5a+bW^IDE;n&-g#{ z-vscM*GRzkg1s!-Z~SEbKp2TN)~J^FZc4dq*>C)8{)aFcYpT&M;cU`e79B87nI{Qv zVjtIZEy>(8eA#lq_{ID`!u!}4HGNCUHeo^JZTxEfOqhtZ)>xLD+LQuTZsRxe7sBs| z_cg;yjGNTIQV$vaReaNZiTzYF4sXD$E1id={|Wvh{1^LQ%`|`8&FCws-_oZ!PH(ee z&#ay6@3k2@pg7E&K4?Rn$({oy>+sD5S1?DIHiHNvlI;Xq>ZHxRS5!xsGX`f8=d#^v zSNb2=Y;zS;zyyOd5zQvlvi%!2M_pAFFl`5sL}#{d?FRo#n+vXD!0kLZo9N08uHEkc zbaU@jM3&cg%DHXs?3LhU{=9kWDndfGQ#!PHv8lB(f0yL2UWtSZ3gtEesA<*yp~?BZ zh(a<_iD@IVMYSFN3CTUZ5^x(Uo!flb8)|#}^OC1}5yuQ^+y(G9e`pJciK~(WujL%G zpCfc72D85h%W!vc_O*#)_H%`9#8CD@#cO|4a>upQBI{hoMQtnD`L!SYKO~QV?$vs)-AAuJQL)is(2GcAOKUM;=1u9d6dO^4I3j~Bhie-rMFUn{BYKcPWV7qQ znVSiUT#MA`IOt8h;MH6k6(CJw9;suTYZ7J*DLVoZw)EVPl#v%IL)tRgPilJt@<6b8X46b0D33&V)H?7zT~+Px+F=0v&uBH>D69{V2_|29W&Vs=@&Gs5T{=R3`{Dq$Yxk z(?C$hw$*@vJSEVWs)nSBfvDuP9cM4E%MP@r4jKED29RPx>ou~jAkaR|b%3fefZ(#N zmCdMA1d>5t*{3p)mBO}m=&oo2nP5z!Rv9SD__h=5b#6;acL4>`kDJ zEKEy`!-6H!5u}jpt#x+;Thds!wAEr?$MtRJ*gHVzcr{H1w>z<)V^UiWdk@$dU!)D( z($+H+e{M_#G~Um?4`E^KMu)~VYFH91HL@Sl)Yf1}@C33u!Tx00+5r|oYl!2HwkPbKx;;xjY#qBZa@@MiaaY@O zc5mIWrKoKb(DPY`2eODGY-62jsqeOwyB2Wm4eln6vQ2O?VSf`f8dnbH5rOIg!n#x2Qhu{E8p8+o5kIj%fPL=H zHqCFMCS%0l0peHoXHdz#-8TH2rOCKz@DOntXF7P{=A>iqiJFaIeIp_`_G(O!Q&s{v z-HdgSM~P6KPzw)E#hd$qPDr5MIcu@l8!XtHDaox zv<=N6f+=mz50NI7h6$MvqBF-=y&>pQdVvW8o8-Y#qAMp@y*=n@dM_Bun6O?UF5-lP zkL+8zjTxh5f^Ce5D7pq~As>~dz8%ycjv7>fZ);FZ4B=#|p9EbJ7Yt(BncTs8 z0EgzP-vm7s_YSJSgf-YmjNlwle+AE#jRm7;@&=oUQJg~cY>-w(SyXx^e^5iDgB;2& zI6R}kf-x`!gIc)bRjGY~lQMcODg#qE*hb`Xmg2*L^D?F^hz_zysc++Rv}#sxT}Igb ziRhW&bt1-dPN>%fcV}eZpXjh(BRok=;QXXc55Aw#c|Qk)OTyDIQ}{)_H+V8*{QkrV z`*p%I#AMDjb#buM_kj;`K!qgiB&Kn0scVCSzt4U!anhJLtBd#}(WKS}Kh@Vhpq?@$ zD!bdh=R5*Qk@)+e2Ysgupdf17$$75s4VHe7P7XLN-5|U`+|Bt@eGl|Of`?tFrAfkH zh(B=tRzC+9km6y`kJ638OT>Mg|Eb@D*vI^^>qqG(A!vIz|EZ^i*ltHZ3OXa*ECdG+ z=ey$$A%yJ$FziT^h1ZBhfC%yop>0=yE=QUoyiP3P%s=iQk_O|DuCvlD!kfejj>qxH zkizZg$3dOaRG|^RL%cy#quVZc+|?;f6W$`$aF&2_=K6NUox(qezj6+NQp9!#`bp4vDF{P| z*EmJT9hMPx2%dDEmu3lH05p^%Y`e>8 zJRY&^2v|4pJ>nckD_j-Zj`OtjP&84&cxZ5p_<(Z?{1}&Z6gMO-BBRlaX~!Rb3LTjm1R3CTrro^rhH73cPG^|H{N z`Ogqwj8IOrSvi*D9pHoLc_z6;-lzQ7_LlPmwS9oJ;J~svV8co- zlMg7T+9o+~jt?!nurvP;#INLo%CBvoITOdnmOb9t^M~YDW=q_^Ft_@0{L`|@o#TH@ z{Azz#IL&)HcLvxE?%R-_r-IDD(WV{2Mb~~0Wg1Sq2p}t zE&hiB>Qk5B*`;|Qx@J5!ILF(K8(N&BA7`QHBAgJIV!05&>fDNY&rljn_JWMY ztJ4YXEN*qZUubgn0F3&?N=NT@F1N8hBJ@Z$5_slfl_R-bz-_DNhPGt0UTFu&gLf$2 zYq+QCgASet^f=-cxmxMdp1|#@&j@`C9C67la*fijJ&Aju%svSexw{wT; z&xK~~9(z4<$9mimxY^v{`WvD0-IPC5?^^2}!`gGWWAzV0&+g6u#I?1-aiw?Bst@%e zp?7v`{uKRY1R|3+;Hm2;Lf`Hl{?qcCv1xFX_hIhz1_Yp`v7@4U#^%A*-XCVzHK1XR znK;^a&j46x?+}l94Wz=z9N8$|BnIenyNv7H;1`yhGXRrLG2|rL%ejjhBEpX3Agy?y zF+0M%oJ(xrhPC9dtXe=TJI1zGbNw1N0ytS_#SeL-(#jwv?r^XNu$S0NS+Rt)R8wSGaa>M?TJRqM^CboBTw>J!h zUC7P<3-OSATDhV90yn2&EbMV^&tH;<j59Mv@kiEKJ>}}y18bknrO?hh>GIkGs@BN5-wjmXWu9~-^ z-;L)6Grga1FEr!;o^|-G<#*%HgFC&SbFad%xZ#$|JL(hhNj|Vuxi=dsR}g*>yz6=* zy&wcoD)+aB<`py;CkH*1_6T#^-yk0}oL-R(kW=kb@h^@)w7=s%Yq-4P2oOv0&y3Q( zz24*8SHL4}_#x_V)iWm0M!i3BQ^SS;E}H*0;t%p=<^J~1+=+%UAcgk)E%}4|tMJ_1 zZ`>~ppH@u%(6jaBAJ$(T54TT?vuT_OP*2MH)aTYKj)3fpLpD0E4A`6Te&o6Js-vXc zF76N-aGU(S9q&_LSbH6hwa<=o1$JiE-m&*1FRa%bi`yOIumHl8@1=Z59k%v4%Gxn7 z`ChZ~?B0wIBg59~j-~Aj;sQaYY1-TIA@!y8hGTiVTU=PD2z439(Mir1CQ^qYLM&qE;n-s@y)UM2AsmH0W3)Mybr8k;a8uAkV>3eOsqdeZeEN)lh@JeHz z`XB0_hP%p!_7!n^8^>WJKlD%Ep9Y{7wy%mi+-MVSpYJ+B9W?-fusu4ixDg#r&PTud zX;f+wwtC0JRWy2r)AAJ)K~||*sDm+5O{0H!TE2Or3qXIucJH{jrp8FX_@O6*#-tV@ zK>FYj;f3q+1%Tj_-WQ(mUK__t_q4m1UoeUJi}_&iWP1&{t1%<|aemLF1Za86)9p!d z7aQ}#Kj%+PA^?4-Jky>W*ViZuci9*AQSyfTSo#0hy4#KF@X&qv9}#cKz@TdvLzcWF zJYiqYN6A|<0Oi^<Sl$iV4>N*c9E7U6OrY)Udg z2I!Xer)9BCwGqJwvZp2{?8k%_Z$q3AfLZYeI;V0b?f()!@IDo{zKQs3*MYIAkxA=c zj=;8xOKG|hAwNI?a+URs<4}87Tt?G_h_eSW07GSc>-ePod|YkIsiI{w@KHEs%cHy;j-{WtQT^*_gH`e{5H zfY+c7Q>LX&8=Ho)(Ia@s=9zbW4yR1JKW$*TWd_NPhiV>(WB^u%Hoe0}WvidXThJUB zDLys&+~6i0ThdR zI&DVBOw}CyLf-P`>`3e3q3PFWbO1F)@4<^~E{L>0LeU2yWOk}Q>#;m~b9toK5tfY( zkztQ<)DwB!=0?Dm$ZQC<8Ayzip2Azxd>U9J12#I_j9HlZdOzOA=F5>sjv#>}k}(_O zq7US41xiTE5!MXd%#1mhh58WQ4j_J9JtCVy_$~u@9{Moep5}LvFOCe%(0!Kya1MP0 z?_l%4k<$UaL9okkzv)>C<~SZQB|O1gclSboa|Y^z#y@BRn0!D zlL~qfRrV}L3lR)N4b3Z8%Qc@7wnz#_Mbh(mTG*M?6@=LqA}N5H(8u#mHYcvW03Zq5 zSrliLk3NCd)ts^VaY2u5AwVEhe)=Td#pe9gp9`jJZD(g>#`%+ydDog}&qp1l%uJg- zwg3Zo2Hrq(_3D758311xbHN1Z(|P9Rw$=QjHhkfn0ar^fX*=(6bJyz3qr)@r&ly-~ zSw;fURCC|zvZGi)91OT&LS?&oqs`{khNFq!U7y?Gu3Dl0f%m?77-$6Q?`ZQn7O8+d z$@|nizB(by^j-HnnTKc<3BUo()1qu8XuDv*0*HVD!27PnA&MXo*mXO|yhKpx=gn*J zjG{>tP}rAYMKn?gZ+?s0<;{`-I~^(mhl$ab^A@#4L>-YJ?Fmq&$1wG_vk5J15G_R6 zSHVit!Xj1kd|Ng|U6K^oJ3-0a!iFAAaLe{6ROos8LPrWT*7Xg%l`Z27CnV$cla5G| z2!2}n9#Y)fraaA7ub5`(0OL?i=G2wj}`!cJta6LPU=4e1nb zearc%+`@6>q!SVv+oZF+EiL^~RfU1Ga-EUBqC`?R@B5ZVQQc6ko^(d~i8hcf@OHP1 zM%{;|bnblQ646H3weD;A6g62mK5KG*26fJ6{S{cg&WuJuhdFJ*m_H^(-^Y`+I7bH@ z%b5LY!B_w$Rj*khZ^1`%j%lD>JP>F}Bi-iJwFE_H9vglOFNTwxoiaTK#bK!j2FXC@6X7p>JzFHZJpDNDH5f2jidgf~c+ugOe9}kW+mDJ=$=1oA+^Kntnyg~rnsSfJD@xHeBxK0+0&ztl>Mv9=5$)BOIqdFA_ zI^=pHSBs!v$=BnZsR7WZeC0W|8Y9u$@zEM0m0#T9kmfZOg@GP6-&GSr%_<&qc;z(~ zje*uA9}6mOc`*f*h8?3~WO@vrxl=?vRh)vlj~$>{N=VQ!)TDx}TZ4*+7`LU2HYW{HJ7PJq)0~sj7u?f`au3x&0XLgupD*73=ZbF zK7_wR^Ne~`Dsv=wXK*nM`Y`?;%{wTe4LIt&GvY9C4Dt_Z{-qY~o`!LPFB3~MDT-gH znN4$-MPaH)EWSlUqVvl%ZnSV&0mcccT^23;X_dwYid(&yDl$uGX@g5(1Aw;=$ZVXP zC@c}g?60A;8Wyci7Uon)p+JL5AJ0FjNu*tnBM+!q(;VrvzbbBaK(1JSBRVIBap8;ETG87`N2X%;5PJ4PL?20S`M`oMn)q3EiL zK#$xAY5f#4SuyT0 zxe9q$G>tr6FhgqxG2uYZ+(_gRkqy~aFiX3D5nP$=IT?v85J93>;HV`r;ww8nbAgj2 zf_ScAp>{bVw{qNbay3#SvLmAeIEdC(RR((HMp5h#NCQSd(XL^ft<3Ox1+iGnZ1T}r z0g!^d1EJS=h`3tjklh5K+8oB)%8nUVqdSUJ^9&w>NNwjlTR9risIp=aii{J)K-QEX z7ht=oGDwn=y#;aFW(ExcFTpe!#6rnT`!(9rkOCcm1WZN=1~Q(4joQnMBXT4pI5SEy z&`uI;g?wfUL^*Wyj55qZLx^C9_8H?UWGe|V8RZyvLzrNX_8sGed;q6|TqMTB5Ft3I z{g*Lafy5IS8I>5wGzyMs=Q6z%EPUFh)etZx(*(?uCZEM!yaRDtAMi*IRb znZe3z;v^fX76p;TFs{}^WUZ5!%Rzz$oD73b?Rm(bjT0w1$a)c^%LFgA{md$5pm#18 z2`VabuHY~2BWAZU8-ifSM$t-02%S%S%`_=Hywf1ng#kP=M20>xKPbl_qBVA{cNO`t zAbO7t%UX}aU#1b!$R&dLI)4a0 znMvJzS*s`pTn8SyNLHZ=O%4{wHaaj33IS2aWwk&eNhiqAVPeTevwU?MSeGDP!xP6;;XdLS$_MX_B&IjMpuj38Avz^bbX zTU@w?0=X7Lw_v+&h;^YVf3fXa%4y6mUKa$rb)&5NRRw>ftQ|Xz*=V>Z$k%;jeW)5+ z{0j0Un9YVOf&$%4NQ+Q>($>jY!zBlRpdo(T&F5 zujyQpy9o&kbMiOA7u~1WNmz$ZZbDuW9VSl~&StM2@(7TPt*3}9Ex>ID=RiBHUg?_Ngy;>^^AZ*FFVfjjM6$ZC$XFsj&4XA>knx&k) zNEqI>pZ%@YCh+=cT)(Q);3cHC$zWp|7Fd`{F{)r6DrC2**`am$fwpOs0TnDag`&0& zc0yebtS%|HRMiGw;RaYt`euFzoJ>RB7S)migj?GB*;RFcOLMm(?}*goVBz<$GVHGF z?R*IvLQK73xiGu!HSGU7mZohRyNdzK2jt^E!ZL4c>8ovHzhRmUk-~zunH-dw5|oxc zb`PU5P=&HKXHI}RBj{E7mUv?xPY2r4cIBlSb^GN$sYQ(PQbIU zKp%YBk-%5QO?y(WSpd?6_0q1kEH-c$P$IXl;r=c zaCym&z2~nxZGZss{`{fK)j*dLJiyf%8#JJO^SRn3FVMZ@;NI@*cQ!!BI)|TkyIc?4 z01)r7>q8qf$2tL??Q$z{8;~hl6*9`aj|)Ct1!aPPy8ssLQb_hv9T$vTeO6WycxJ~f z04(Qtd9t~Byv9WsNGy@>!KJRlW4`c#Bw-97^ z@#1h}&(rvdz{rvTfX*gvGy_FfS6`OZ1|Bcb@9i(F+sHYA1VCz8V_@>BlIguuh0`0& z04#d-O<8N;*%FI=3qd^)2SA)xhssoey_X#JdESTx{Wul?uXK4Bc%@|ZJ_w+Z>;#Li z_+gjsz-uMz_pt%AXiTt}#D8=Fz#&L2;N9>5{T~*_=bqP}2MS6E`{V%h(^_yf1ggxqed9Oun{@zvv-?u^CQw6iv;*K$puR-APkxgHGOhK$>;O^}I8idZ@99m!Kg?wPZ@b@RKY@r6#!2b_~stuBNgLF)m z85|}+;9W~;`&(~uw>bm20d6I5#IGwo0>FgMZAE8TbKn3fz;7u%z2D@v1poqAbKy1u zC;U#Z<}bgE^Da8e0!zGL6@G8&js4qhbG@B8ED+TuVE90#qW#p{O0aL^U>XF{6;CZ~ z*q?OUbUQp92g19M9(ZPH*Zx8f?#4^UfyEcO4j)zea)0CPG?4g&o2OZf1WRk_=>Eam z%I!R`k!qGAx8RdXfA0Tv+Z3#qIBU%^5J15J%j1$e-aB|1IItQbcjC{NF7{72nYhC& z6KTr>YIqP)L0-Chn_vesQ*R4jCGf)+fTb<}-rF4-u%l`Ihun|9UAn{n_LPH<4_5#d zE``;=_CK!U~DhCzyfwqK^*>5=_~(MFoHW@ zWG#hn79`@om45Pnd$;r7qKmA*;Qt6t&H(w}D9) zw_M|mOvhW5Ee_axFKrk6G7f~aAi+>th7O?KQ|{t{kyEo1c>%wqY*RqeJ=5KAFh6Q` zA#?D~WqtvL_q=!Wa&XR?CuKZ5x{Mr93r0OCf9@&>Om_G+Wn9#gdjjAr(655;660^@Z_?pfS4jZP6y@`I7uMGCo{_MJF<(Md?7yWYHE<60v}%117;>N&{qn8 zHd0WHk1iVucvw{6>ys-0r>8zeE5IV!apq%Ir~me_3T#{!N)OFf+p%9GTorPy+HR z{zKW5z@n0XeTp0U>vK*DzTrQWy$WnC;qG$=hZ#IcFpd9K_9^gfN$0*Ia5BM@1z}dd zz(Mq@#B@LW77iRUAhE?vV2NJ>4iVlhf!oK5FA zAYms#BSuo7TETH>OfUrgwv(ulB1558!CA;k=tO`-og|GMc@T*4uR`LHAc9KNS@~}T zXaj#NxCQwLa|oVdE*VlFBcNVE{Mk_O1B=95*5fMV5ol1*03``ciEv5h#?vOdkqYf>&uFL=fCXERs&9r^DL?8IY}T6~SL5ATi~72E0R% z4LJ(e5n@Do68pc02Q>GQm!Rds9fb2Dry!`j^B`~Y*(=a$VE~~_L=IAw3z$5$AQ!?4 z8Nggn5ad%K2;+4Nu0iXCAFsX#ZUaRH7yQ?eH=*Qzgee3wu`SuNk{b>^McxHw{7VF9 zaR6CS$vq4`LzY1MgclsQfp^DQ!D7KR0s%x2N(gjuC%H($ih%bDBoIy5L`W4|1Upx; zBH{glawtsrh;Uns3ofc+MZsSPs-PpnSArsDt8!p9pWooxw%btfTPP zf+py+@HfFkVnK1PVMW8=3Y1VfsPA1a!BL88STXQ-f)40{a4B({giCR*WyQjE;4|P0 zU5Ru_C#9&CbqxMN@Cdpg+(Jy1SWunoSjXWbz;;nA+)HfTxfOg$IJQQQ)Iwq*h1e(w zsGV7ljN^?8`k_i8i`XxzqjKtzCwQL(gHVGokvJvEnKe;w5f6PqzJ=O^8AQ0$gT`*K zNPs4gI!GX~qVNL}(H@0zDI!5+P|hjooN*68Z)V8ZU%3;G>_Q85=E5LEn)g z+aX~G@v;;dQuM!;Q<^E{HoK3&0#YU=hp3taNxUC||DZ3zA>ipK2=QqaoaX%!{D8iL zZ_9`@jjU+akIeZa_znFQ{wA8pEJB@ISSfH5p($auXdY>~3>R9|!b*kD1mCB5qNSv5 zGH$4ID=Q6dCbS?RM6M*dtTVKzm30O_M>vnLRJ4VZDzl(FD_LjZ=0aP-l(!$LQ07hN zDRCSPbOug9i^!x}St8x64Vlif6fPvJ1>ebjSsk6zhCIicCtO0eWKw${M zq8bugKEW`m^m}X^gjfPY)Iqu|M;>5y=(FLD!nFjJNK2B-SqF3-o#QQwQ0u`r|2?T& ze&+ymzw?4-32GA|K{QDklMfv*BGY&6{R{OEAz3sd$gJGH7w|Y0i}nWHF2^y8I$1n8N=P6`M6{sRaxPQ%W-DKVMv)0)sjQ&>@;W9*jm+h_ z3aNw!QDV?k`83n43z^5m3h9J)5jSXIg;yA-3z^URvY1Iwi}HdtuL=*-b?NirYlY#2 z9#L_SKX^u&-TDIfI$mAB`4C%gXf~rx5vD|wL1Ps|hcu5nZ)*NQr4UTSv&dGJ zPT|lK3#)99+D(`Xo_m){a=7Y=;5Ki&@EpNf>;z0e1>rtV1$THp!YqP=*n@nyav)sw z6clRnXI~^F|0Ui^=2Y4q_Uz%_(@gVn2&=?nrx!5U^zc$mBRgncL+Ph7s$&MI9AbfRvA1{ zSPW!PH^|!*T$XMzPN2b~1O!MdB2yJg7Egl{Y6vI^fhukwCn-!L;98tWLjvh#OmP>v zP~jcH1AZk98C6Ay62BxjD$>9ag_CF~s5$~$JW3u^C?j~iIH`t)Y9b_wf0BPGOe5ia zIGH9Cr6h2~mcdJ^yd!yiIJt&_>LAcE9fLPiB}SU{Bg=UQg%1f=#mL~Gs=7!{KeB=s zChR6$7kdRKR82>k4InFd;lihcd*Iy52c_bi0i=S*0xwo1CI{D6B}SROKvwZ0h5dv| zF)O&gsxFH20$I&FA{-<%h!caSsvO^qzp$u*im$u@&Z_ian`-0{=1YAIJXZLg&?)8z zyMwb+_flUAKQ3Id`>FU|@Sf^B;CAY))38yagnn^la8&is5zSy{z2*e!3picdgU?qx z9fe+XHfR!1lY|lR^I&l`OaC~@ta^(bO_Zb>(-`;=u3eKM^xd)&9P0TsUNM{@q>9HC z!ii`JnNnMm7-RMh*}=;eE+np%uqd8uGMKOvJ!@KHi645-tPA zvUG|~Ei#t*UjG1|1Dw=O=dMyPwTxKJ`_6}&E2z~(f5|<{p4vOHkgl^+!$YBoUHo#2 z?$480g^sI+a#3!?5J?M#Q)_$7bC}x&<)hXR4@sVYyCmS4Vwl?vT|@m%JSurZQPt)g z^ZdYl1c9b-;t9zY%G8+$$BI6%9>Z^f+vc?7H^rpR;<)oi))V+`p*JyIGLO2v4tKoh zBkL&~q&N{TNS0E!)p3tIkFa{+MM4~rFL9;jla$AKBe-XpV$@#HQNM|rRAv7LD zrKkX6k;IQ$Sm({=>2Vs307W2*B@}97T^bv1z-cui6q%@yL{bOqlx&`XrMf5)Qi%ggeN;t#>!k!5ci{*R#U0(KYs9~$ z(KKB{PlDnr_XG48b(8pyG=*l?Xq)Ic$^8gDMcpO-E4@T>2EIbYBo|!as1o8n=`EU1 zV@{%{k*kL^C?Sz3mC=}uJ&6h<*8ugRq{I+u3ysrgd(!h8cNFSJRS*wJpU{dL15PTw zalx&RswN(lzM-iab545x$NdBiqUwn!q+e*d#-5Xk|G1x_*QjPS(EK4&+ptX z&|6d+@to8;#JMTpl;S&g9D0YkPrN8y7UI*CbINmyI|1oXYGSSw8^UbrIi;B5euX}u z9usd$w}x<KU;_x<90-DIkgWl4XP&gj%9RN(*Ui;wCx&V10v+2?vN( z(jy^nn>v$FgQ5kw1Cgh2tRXu`?vU z*(;gz8~KYjEi@8Gr2`?g&56loe~`aDZHGJL4Gg^{W;RhwyH zXa|xzaD;ZZ+)0IIs%LAh(2GeMWXD6tT82_JGu3mn^U=$|By~2_s?{kCnx&qrwLz~W z?Ur2)#k4ZgG_%y^+6AEd#9wwVbWiJ@G{{VCp@pN-q$Ps#(8H|*X(}@zjBhV;BZbP8 zp_#3YXMAQ0E%^&YYe?a;p3t&Z@)@R0suf`o`ft)v*_%*RYt9+ZIa%`vOVArhCuCnj zb*(*T6mzoX6aGT~LrRv-pqnYD^Hnvw3%7*VG@*!DgI?z3p zv1g5QVYY;o=-s3%vek4*S$>vn4qHH21>CLIWb5f{<-}Q|In0i*8Vx8}5VOKldT`hl zz9&tPB0R}H15Xz#861s;8m>j5NhG=KAiZ08hXcXX2rULp0aM{|`j~Qvqk*aIwQlH8 zQj2RE-Mq~)-N#bsz;_oN0D1p8beA@Ax(dvV{55Dd@5i!RAb-@NW~EhxBf=95l3JOB zPH*c>FS3eQg!o%@g!D?*L{DwAIOjYsVliTaD3+8wEPF)1-G)0?G%sQaV)ueL(wOWe zy|FFr9DKfdUd|>_0%;O#Nd0Yf=Q#6GfAP17l1M+nG&x=NJuySoF~^Cp1DJUi$X7F<_PR-ijll`AQXJzVa4bBKZQ69-d9>#duF34mqFswrC z6SK~?fOjJ0>l3hK;T63mZvjpR7ft<&afH*s6i4=&S$u*SX`Z*!3BW_Q*nU2 znBlJyaG7ue5_0o297)70Qx@ZqI}r_pN^bt&xKhY1SVQd%9UfmFy51`SeUhja0vaF zbW^_d0H?$Dyr*NnQeJl;;r{dmv!y7|ZZ8@K;jon?J@mcTzP`b8R__x*tJ z{=`M&U$Cu&6tqdu%<{Ph;SW46v6sQN5z^3RL7p*)gYFMlmvqYv+Yo0#a;|Ops)POy z1eci04c>@!ktMKluR9p?Kz~WM+^`*yA+io~D&KkV@&n{$rjua@0wfFvAG^kMgjCPag;_GglaPA})v)2YHpp9UOn4zpPtf_!n_Wv@FQGJpG`}Lu3wf zrC}E$N3=2sSI$4^4z?BDO2cl%Rna1>fBC(Gdmi4&ft=MiEf0+j3M#KW81--{N8_ya z)#jpsowL0C;Q5D6SD;mDKW#pmU>;uHbFl0o`HBiyY53Pfe+NaE4;_5?u;7Xh*m(Gb zqK!fEcmKt|vZWugFJ z1>DQL-HGEB0gweD6cK_Xf-S1!jDMdjCt*g>P^1I$`=Vy>ARipFmc zB?L{DZz_GRF3)AV!9oa4=%k<@<=>e`^+c}G4HimhL8k`Ite6`H@AAlFV_|fH64>%# z6%JvZU9ovaEQ~>DM`s2ts8}5ab(QC_-C+j^9q98xiz?QKvAZVnjP9_5ga_!$K`SbD zh4H#P^4T6RCZQ9}3qn|7@-TDAGD@I0(;UW$Y-uGgdrY@t_OKl#D$H6 ztx>nea0u~4bSucaB0bEe8(F|yYY5k-uD%zvyW(mXrkhcqS*t#*eFlV;{vaY~PxqYy z$WzVIYS5yfAdnXj)jd?8@l;1>d(pBWdPRHK`EI9c&^mRbwjW&yf+3!ViMtusH0#t+ z+85{=u!_D9>+ZgD4f)hnx~BPCeN_7z-4e9sw&@}BM~>Hhz_QAJ zD{2eM0O1lYkI2_m>xD7=ccS}2mnxhN?S52n-DiU^mah}3gT&!ju+H{ehxZ;hhWH?Q z6m+9v%c0aq7KP4U5yufD0HrFf*n8;qBOC}ah+rcOVBwWi&Z?o@6);G~u(94RtLkW+kZ4>0ppy8YtvG13V&G8iJ|F`H@(9eqL zLsO5ZZ<=jIrSNBnO~|tXkNZnM2%9D)S~PDWLd!WB$v6Lh_o*@NmtO^4sj~uycfYm?h+mmGR-mClj}g+hG|5YY_CXy)q*l{?y|R zdj|}}^J6Uk##Qpe-Ji1V=yn(~5q9F$% z%OHVbPtTou&~7ydr^5J=^DEaLj_Mf#AxP>g+W%qp1K455;qyICMG#JXRqF!es%4e< z!{Q#VlWjO5kB<@)$+Aku;fFm1MNl1wPrzV;$;s7~(T8PepvhoP-m;>bdmE^W%&v3;>ei8YIHR3Sxqe{EGo1Uc=`{b`G;I9=QCO@wf91eO` zSIpUiy2f88jv~LTtUH|WY`WNNFX}oU_{zxd0o;-Q%&Ua6SH0HX3&SQGD*FyAo{{a} z?G>);+=z)Mk5|4w+zrCUp?&Hb+Rd0#zayks?M(f}ObE74Lgb!4_;lr^YP3VB3dsg{Om0%*@q)-R2z2u|{qX+tqe zGDl$03g~F zk)$;h!^yst+Hgz{z_9%y3bo#19*8K>vM^e*SV4(s)TW8y!M;`6NRauiP<)x$uf@%d z4u(||j(`x8dPRJMQ9B_v2E%Fy(U`a7Hbn*inLH$H3apk8i_wwQio6KV-dKr|0;?k& z$BdAlDM|scP%dFpVfBPKjIGyzqADV$S1-{~4fTk4@fY%tq9fvRFH*{+85$6Y;z_bz zp^cFDrrml*Q#Wc)V!o5d74IXud+$i05OtF_3GKB2-EeyVt8@Z75MZ!7x(WQx$NR>B#~oZ$IY%Oc?%uFRPpp+s;1VvneDjimQ= z%8KX_ZHRN?dBLOjO_51`rgAvLw_Tfou?=3j0vDOz=OyPbP%1tbye;S|a%62^qTK8N zs)L^`UKqRI=z6rVU0PF$b5@vbu_NolP3wV^X55gW2 zaxl)|t<8({?2j!s9)xugu3}t*1FK3Sq5kr6HWQ{M@WF!`TvZdv?w=?(GGSeWJdAtr zfvN|Qync@gb{MRiP=N6aj;QL3RP@JI7{g$X2-h(if{#^wjMVg(SFjJk9usbW=$MmL z-y@Cv6BWiouqT9Dm~Fvls^&(OtwvTd!wpXmcf>n_xmAcL_W@R=F5K`GaZkJ}IHzh= zl>dOBl6lzBgD4jJ1t%9&t&55o&{ygX8=fIbfneVH){dz215OHvrGBn0!{CFR=6(xMZ zuNEH-ep984%6#Ei?Q=xlXH=UWF@Nb;;}b1>!*3Oz37%VRd&K1>xkeQ&e9LbW zp9{9Gb~>{AWkHQkjBtpr5@!WFRC^pb{Bod16(fAdzc0QRysX;$Najn&S|5N@B5YS( z2|gaS_sH#+xZ0xF2pvKVfJ@hE+L6|m+*;>j5yOaXaX~Pe8hK>!rLvZH%=d%#5#~nl zd1T^|sh88WX2($<`A;Cs57!*ql7+;JorHMgCo2_k9u|-Y>Y4f)N>KleMc07 zvGv9{*eAkEOl|P7>W=^~Ew9&6hCd-*i5r3wt4k+G2ZsQFs{X8fjcEx^t)6w%>XlOi z6tDiGeGAIvGOO*5VqP&CH1XH?0Iy&^a2Hf9+K225|Tv^w;t>Qzpo=gF*Z zgfYyE;Ogq=qq(%;n>|lw{Ul6b zz6O7&esZ*R>8@rDdD}1Ts_Vw!(dvn7m9Jx)ji+J1`DWL@2TxQFAAR~-(9BFW{6_p1 z{|uh4{&sZywZ2)GZ1|%MLd>9;)Xa*udgIgrrKo50nz+uQnAdDcTm6RIqDm2(HJ2=) ztOf{k*&A}JDotqCdl5Eky=x6Ex)ngq&SxTK+grI-`+3$xMi0JGw(`#S&gq@!x|FiH z=2Y~rH>OJXS>L(6)~-$fl+KR6owr`eIg2vSwUszivajX=lsZ;vJPWfR+PS(=0&7a6 zP3Dy;nH+Gy5CWx*mcM1S z>Cz2W_KRIF_{D%!sP4CS+MsjldA&SXT&C}2dbH;VdR`G9xXcSY|`*9(*-wVPs--kUyz=lHJd-6bhpx3U%& zoB!VHAtwjroQsq2DCk;pZ0-BRhh|q$t8)D$`INP_Tc19EU;dDN1@=E8aFtUw*2c%W zKl}Jlcg65O`~8wz6z|&fqn0|SPUxz7wY{8om$IuiC)Pzr?o?eBy5t5*N+|mQ!LVCb z(CNbyB6A^$5TIn`v4?d7ohqIXl?zZ zu%wz2T{{Fq!3xwqxkA@ms-&J0Upo;yq8m`Fa)oZWA(CcFO0DTJ^I^v>pFANpmo90e zWB}H}WtiNh$`iWh9+2FpT&i_CwtKjs%j|EC2a)NjrsURQk1>aPx)k|YfY)$+Ou1RR z^%!TEC0bDsvDSW@T@N5S{Eih4dw25+d_8+vu3Cz?mU66dIISCg&39dIr0W1hQ5$(| zaM)rr6;L6$Mo3ZW+V97VA0{3f zZ^E_`Q(R4`GwbFahkx{V!oCIDMoe=xqr&POj(dKLePX-?^Cq5k1vRjs9|!ta{)BxS zww;&`a@7{qmF|!JsDGlnZP;#~A+ZKQYdb-M4)Q7Uj$wyAS7Jv+g3(P5SRCCQgO7c- z#DVHj7k2#VN5NC(UBgcM3zEfDue!M7;~(`;b$1Q_+Fz0^qk7k+AGaAn_Au`mcG>4h zR#I_w{NwI`_|e@n?6$uuSxpV7D?aWIlnKlt1J0f=K~s~-byde>M)WHP`jjn7a4JyGQOkgG$tUdkbARQ4{JW zkB^NEJ<|a5ZtqRkt<==IS!^r4({rd)y}$Q1C^*doDsYUR0r(}gfA3w_UDV5U2VuMP z1eIh~NvBrSg#(#IxrQx-5s852qSn{NvyJ)*jZp|A5oNA) zYFk|fV6i;3Y>|*=T`oCD?X2Uo-3=_QPGktOuatySd+LgTK}w)yiVbA@D!_9M)K#%# z40^5R(F1U&xkgjp*0rw26z0s`;E21c(&qNe8JO8NGm>)xyx=z5a+U)>kB4n*Q9q*)=vX4houpSsHTGe#}?oMjQA_N|gM>g@V?amz<>eMPbe zx_z4@ojR|6Y23C^ZlAL}f{~k!;2*iN4@)bvt3ZY2c1iN7YwJ%h(~Oq)vnybSh>u(g zsf#Yf#f^{Z`*jtDL-tQVxw?0KdYsJ|a)4QBD1Q9Z^&VAHmJ{bPMjlXA3J>Q#lLWi$ zt7pvGI>sGvRz$GuH4*`pP%nw2k97_dDI%PoYh5MaZEuK68Z&*Nd9XS%w@)Ic9<1+- zQ@v%XEQd#xdIQbasg|5mFW&hH3o_$RH`?&70J1?MW^^x8|*9K}rz20(U zY~Y2eT6i?~wWOJvQg3>~{FCELpBiCw?psM4b%nY03Fl7%FBLUefGTvoPrX>b?1ay! zoR^-pS+PW&EBGwxu_u_HdR{7OvyKrzxIU)dtlxTq^T~G5vo7m6am4i*wWNOki6Y=d zP}F4scG0z$Dy6GpMM~IzjyG`kML~d-{aQXNy

      l;oaeKvg!Z}L6W`yEtY>FT>q6n^%8&1>>a>YaA|Mjfkvd7|-i+G}{T z@9Ey3uG4_j96d4kS^1jR?3>*C+x0i~SN+cuzdoD3fw%ak^v-ZIrI|HY#xMEe{f5`# zo7!vYHk)SEusDA6m$WzVR^PPVS#B1921UnHzbN1ET7A#tdP=NliyOS+6TVEpF;hk? z+B4V9hPI+%S3K{F$6K}%#vxj`!D*-lQoQ0z>|3J}mQJ*ETS!YDIujQE^o!svvn}iQ zLo2t%v<)DLaqP>`TTPq#TZ)3%N_A8)@+k6Y(SC1MgITvCaKU`VTFo z!8E~q!h6W$LDnVWa<}cYa}CxB&LAOD@gVCmafREzwB_5DCTyGFg2>2-9D8Re4uG$& z3G|81_eBpQuGs%ix|epNVM{{lgoVzzGvcbf3t)YV8}=sLp1|pfIwN@YC@Fy^X`m&v zPH=V3>Il9)MoOmX(jyZFCzOB}_RZ~eb)(T58%`zsnlK%PclqY^V%-=tRYP{dlCR#w zye{ATUJti0T6e?sgw0>mhT+}51-)xQ(U}H-z|^nGVP3cIwcd4ZM`(i$4GBqKO+UaN z`Cji`?-mQV<*tOnuihVck9-Szz1-qxV+}788o#D}fIs%V(Ywhl5y-Zhu?JR6^mEInt!WHT)J&F-u%E$7a!qru)4Uqv62~X?Bf4jX68rst z^!9E{PqZ;2^~~pnQu_dqIge}PC)z~N^_u7EGUETFCA9sGcM^9S3-m@PLqLSwgtXm% zNfPPCPJNLkLTFEvO2E6^l$dI?FgR-?MD{^I3>VgDy}i)rZBSiBiF3(P1ue31ATbFa zV=!uA5+cQ|mUax3$7zh^26it@N~F0p(oQzgOurcQ23@Z~Y99)?_O!;?CvCnVN11&F znLR_=PUAKrPP%_%jq3Uga{Gf)N4v|7D^KnL1$0oqy1bX^)3TW3*t*=Y_B;H%j_~ zR?`@Ja^%~7qho0W$sz2c}TKd z6Y^Bh_qs0}pf$)%ms*7^Zt^;n@EwVrcnfPMX1LjetZ3SGiuc`PoIM0n5xH*g5LDA8 zqWn8+TsLG;*=I`~LOhznPCflD7-zmSbl6{zE)MZ(iaRy_T|cgSXSi>F3FsBPo6=9& zOd%(j?+p*^bEGRna83ME?o+G@-Fw4B`>WE`ApuRrr~Ib`6GfQT&R(7yIwYv6@>JB+ z(1b>(R`=$*VMFLm?WfL9IemqO)m^>$pi%uWNOBcVF}`Ys)!n_<+}4N0G`&C7J#`0& zAk>d~3*9z_Bs5K)8k-vWs`;RP+Z!+Hd6gb;BPC8=>b4M;6~wY?%Ya)_cSGHGyHY2+DvdwV5r zw2;Q8Q%S$3O~1iMefxT4ZVVvw%uZVJ!}}W#gwXbuyM={xH(gKK{38vhGkgbnD?u@$ zrb(1U{h|EE8}oh9Tjh2nWU#3rDd~slfACMfFMDg;VncLIT}g$YVV?KNcd)n4EiPoN z>19&mk2E0Y@O{3N<=k!SLh2Wa`r`>dnIT?A~2if>tC#20SWAxE3voL2qH`QiCr7Er6W)rXvD{&HFef_)YL zWql^Tc54nvZk~~B_S^QS=l85H#J6s3A?KQ{lbwGD{8W6;8YjMUyB~6~d0Dd0@0_2W zQ&|&4otru&w;7wv{N3|YF_ra|_`&UQ$j#=hK=ESx%X2zwk~re_ETp7)e{#|9fM1H~ zEF;n2)*B*i4oy~p_+ZZ;S>K3bZZATro1>F;zk7Zueq{ZZ8_s_f($su9+2oJKZ|9#8 z-|fexLm?f_7m}C%!Tm1!88K!5Ra)cJ-F!WH^PjZe@L#^uy_0VG5KXfP46@4KykEXQ zdcV1S3K?u}NKX1=`Un2o_h;{Sw}}wQt1~(OkJlg0Z`7~cX{j+}w0R)8_D|v;vp=Zc zxj&^-A(PGei_of9%)0{t8L{(fmEx_-Epe@egbUX$ICLbY{!k6!?raGva1g znvhIE(lM;XA;o4x_>AEhqb3fsWOG6E@v0R68NwN1CZnbfvt^c{jxFm_Vt{;S*kp93 z!(5qls8h?%l*=}?LW+Dwgvqe!sF{PM%pugHB`oFXjI4do%&yse zR@lX%8(NO1jLmpwqMg|_r*A%XS?IQwvnf_4D@+Nqy5{!TU{{9jZn>I*F*#tWoz-RD zw*b2uXoBvg>@m4(N-!Jsy$YA1fJCJ{<*>;M(+)EcEYDu%78=^3OvyA^G;=3V4dpGA ztqBcp=}9Rw37*+8TV$2DNcMMVbjwi6Lz8PWcg_*b%UdGb7#iO)kuqZPVrIu2(fqu> zWdDSww3w!vn=YEQ69|p+mdmz>X0+I*x|jyf>X<9C$y&lRkzBBWp(QQ*Q;SUR2HdlVggc;Q zgiuKfEw$A&%WM@a65)W6kwdFmj-ZM{>q`iZij27C|ayr#y=Eh+=OFxG`ENDel zwPdF*nYn#7-_mbkp9eM!=$qz!T0isTY{LCHj-<8N!=cYwN`c^MV;{xjMuzsc zRHjDFd2uS=-I|AR4<5hvfq$7A385)){le0>4##e%OLg&(_Ga z!C7tQ`~`j>1P^;7baU&ev|qDkS|IHFKw1oFiS=pCPFrHO-GXoD=h7E|Ee7JP>uH7Jv6%p+i5Ry&7;cdL*rW~(nic)z&ie4 z5w>g=I0{;4oH3idz;c~KHkL%kwuPQ+wLarKJJ7PqA={mF0DC|5V(T)X8M|V+Zeg|u ziHTK*TA^Lf&}XYHix)<&aX2J<6ndj|%bC>KFsoIL=)Bf&Y!A@*`<*GAz1@oM7+IOk z0&->%keSuaK4~?35qe!-q^v)*61V{RXV+V$1KnBP5!qm9Lu=xhsoALF@kN&FNzvH1 zp>3@hz$dU~UfklY+F7xnBB8UDf5v@I#Ju6fqZ=HKgW~3%*5Wh%z)29cWYo(cPBt2v zJkVNoCT5Od-tdyqjSlfZyf)O@apv+I)cmleqnjKOWs{-$R_z)2oQV0uOGh_5oRWPH zooKybzCl+of9GG3r?)3ze}w+m`sIvn&a?Sde`N!`8}?7=pVk>?2e-|&)_(Nf+LwYg z1p+wBvxE6w*6GX8+w#(6W^_xX!Pg>7jj`q$wD>J9tDUoM`=GI%MFGp|BOP5*E z7Xyi0!rUL$vz^d8@-jf%=}IN;Z2sJhHmj~#_>j0*I31-VomI>|X7kO-5)}4=sPr|; z@Uxn^6*h4zEdM24z%HS0RK^2U-B+7$D=c@BE@45Dn=%6!C)U`;t+d=t%E3C*?{5zc zMw=h7)voN?-FF3~RQZD@q&?<$Z3)g@xIP{hO%GC5o{chpXRCGY^6h&hT4NHTY&pv@ zU$9^uNbx4+W7p6RDW9AzG7nr(wJIA(^{{`_k1F4sRheH|uXTy+(jNkwKL&OHp+$EE^;KZQ=}(pSID24s5d=&Zxle(Or1vW; zIZ?28KxNn!+*gf_roUCTbI!xA+}!FaqU6=ej?q6Vdq7mcGmYFen@R$p0{ye{4MzpL zV!zHUn?`EHf=OHXg`G_r$7p8llnR%Tu9vuzTjldc4K_AL{ zBrBwEYHQhJwEVj8+ZxMo(i7}$`u4Vrbhyt7nc?p4ky3_j?jLt*t3N)e5#~)w;+?hkjW( zJ*@3f`fV%UMaAnPqa0pfZ*Gffdzs#7bq0tm{f_j##MaT-ZKLUfR&9&;fBPNndxdSH zC$;@d|7A6EF=D-6bl)4S5*U0e&n=m^eKCK%UrgT+wu64YZSlFy^UeTWreAE|du%72 z*M>evo!7RQzrpWV-!S$Oy|8W5xukhBmms|Sj`w}U_J9H!zjKB2)CV|TXm*}nrU6<* z^10f1Czs6Lh>pv`Ie^XN&3@3JrSp|%dt-TiL8|CX*(eN)&;`bgXJbK?02{?cygO6vQ8{Z1cm z`|Wt!BImDl|74#g{lfmB|JU~AoNoTJzpDPpPA2`q{-OVAn~`B=yT4W!buI*=Zy*cNoLI9h%6ml5$1AR#;W$c8Mm!{mlu0SavWfC1jDtRmeFdRwS3j~$aDuQpi^Df zek9|qwR(B+_Q-P%^W{q!TiQ=&nAp@;rp;TGK~k2D`+K*iXV}=FoWgdDW;oc&S2A$z z{0w)S2&ds4qnQqNK-wD6UYy}?BXkP$8Ra@4$lw@$C~CBQ`Hq zbo?v2nD>|bA4W>MX{NdDqLn*=UN&#Jd^;nf-8R$3Hh5*nF45(@74ARgUu<8NxuiJ* zwD?8lI5^9341T+7=ELo+EBU+quJo;P-^(a$-;|kTI}`MF`CaW>?H<4=YWK@5wB7E^ z$NBO4knRMAxSf*OXnV#P;p@lmL%RoAR00uhziqv9x-U96&s9!kG_=>OGTMH1{^o0$ zN5Z<(8Ex$uneYW`R>k>Q<^!4T1JllSey00^h*h(7o%wxh+{1xx_g?0n1$S2w_H-5W zdAdh3`r9irqZYgak=y56_J{WxO;bWw;G9QdDhZ&zX8 zM)!Ci^?je&4cgxcAp5j$v->GVEpsAsWWkI7b?g(}%=<^4%$T7%(mcygtmu{zCUF;uu*Wq9J z1;&r!P242AnJ$O`zk7YV+;bStDnD+a-FBDsPpG0ioSer%tH|71yOS=n1JT8Ke)4?A zTA(oQx2tza4@8&bfjD5sCRHML%I=5DY&^O&59H>7o_a2KA$%jSx}!nCm%NDKrz+rX zfS*Lp2BzaYVD@F;fhai$UXM(N(84@|T+E=T>bME;AIRCjBA5rPd5nXqPHsMYBPyMM z7Uz-W3Pz-AfLjYciJA=zJ$V#)E#o-QG55pkQRzgqG>;~4WSmk>bEn`xP_s#BSzf5T zl|jv&n*~R#LB|1wHVG)B8CfcaEKkHS^f!`aISD9V8Oc{vtAX~p0v#7*3DlA9K$WOk zpT$OeMSlyj1nM{UrwsEMJF_kWS78`=w9uat=2_1 zDJ7e0WSf+0iHXfFLJ{V+xf{Ee9lrFmb&hIbi5k8;KIq&!D`Fx#8(io_%WjKj& zD5CO$bA9vmq@`Tk$;q}`gBWkYLdlTUmHOnz1ly<_RZ<8=7%6*oneDIPPbWk z7BIQ8=JKSVwH-F_Tl;HEy8!AY;@LzirS0se6+X?SfTNNIb$7`kHn_w=9U-${SgYHc?-)X-^r!YQkU?CT(OqYHA}j)pa}pHAt5tJ1t1J(Lcc! zf`^G6=w}UQw$BFXHu;110^Uq~fL^!6W9IN_Nz+iG?P|Qvsl)UVL!50f`DU{r?uG&0 zMtp+aV3=eJ$CJf|j7lT?#w%p{uwkQZ7T&96DA{&BewX+X{g0uRok6fy>rkrgMvxZY zas>)?vP%e_D)dPQE*?*DHa)Q|+-@>>stwe;Y{DNB7t%4%0+1({v=3$4TH&!`A7&_2 zX*X+k69mj{x8YBU>*+V4`Un(}F7anpe)A`aJL!2)Ped`1EHz|TqVQDl2)zMB(BUEE zPQ&|^JMc8|G<_V}fgr{@cMav*?#5ph|EB+eY9S3my}E}U+U^4(_0$GKBPV1+s8vsP ze&v2VU2Mc~G>S!PhgtPz7gl=U?~1oDutqh=1Q27-F0RDjbHqCs7mVhR+Tm9HhDwV= z_yX|(My3(Mo))e<;Qs_5@ugya2Hz;gel}cJ=3fRHVR+(D#()veo<`Cg^e?wPj;|4) zXM8i7v7aUB4*6Hu2I0exCNNgQwxWpSsUiRMHo^Ed@m+>B43CnLCGsJjZ79A+T*$z{ z3Q&F&iDHOvOUBE^RSXJDjgnC$$|0d`BuLLUF&MDM{jSt9Rd#h{4F0XSmr(-4p~cj) z;q02qbNEl<2}T=?jdqJD8_BM#jK%*De__0XDbWe9uZ{Y@u)T`^C!UwQ$XMS26-giS zZ?KKWFK$_pY+>x{Ac&-o`#0L&!Y^+zN#1Xq;ed*wgGjz@5`JyV_GE%_pMxNZuJ&)S zO~G$y*_(XRSlK4qw=tioRnZ;>TeYMpkH(IlE7rL71b)$%G?x~{;<@0?_MXxO$E&;wtS zHS5$)vU8G;L!*^9tgf{zNLjibw$tr=*{AIB%0~R{mQ^X6*SiQS&(lBqt8H8GDJ>=` z``2geL|veN@qcaGfxp+XJ%zBoZ>Qh_ea8QdZ5RGQ%U(br=N5S8|2NwY_`#Nn6vBqS z-Ga;XIsfmrAMv9t^?-BG--EhB|LyMH%O{~y~q{P&hmDX&2fk>D!*pZ`DGfB1hb^HLX?=Z4)h4md+VNJ}OkFNC7zua_xk-(4LcG;t zrXIIupHu7k)Ndv;&a?5lORy{KOoLroZ>O#?MeL*9&|QjMWoHrW*_xedZyK|2_J-~< z>}tEs!M?zADZjmB>!F*HW%4!P+>ULnN~M^p_lZ{&=rGrEw*{YSeVO`sY?TWtfv$^% z+985NTDwz=OnqGh33Lb+W`_=rXdO#!HqCHB-J;}6% z!B<=7r7ha1?~1xjUx78Xa|yoHx+2YDquS@;ZSqQ}82~|&Ti2&KZ%lH9Cz4k|EdYa_ z)oPO#w6VjrIFYOmwd8sS=eF)ni`y83)4$`jTE58+2eAR+86RxsBb4;JVgJX@53qxf#&C4CUMHz$dwt zp0>`+#Vw2O1(n;`5rd_zIcdAh;@u1wUPf}HU3l<7YgyWHvqrZph8IkZvZDqMx7Maz zHq&x9NcJ+8JJ>}BziySJ-8XY_&r0@MC*NU53)Zx%((26O-3?N_*2{O=T@3!#I+ZqT z*65y<;*eYwN+a{dbN41hjxY)oC&&`f{_4>k(<|eBxG2Dj$H2j;GJz3(|?-Jd1!;y zSf&p*FW9y1PCC@W>0rWLE6|C`EeiH(W2HM<#2(bnv;s;VZb|UfBW+L8u@*H46Edx~ zGX1!m;J~(L=@%^K4r*svSuy>&mBGPn?dePlCs34VWz7uW@`K53iu7uWSWoSHRyNE) zZgudPw)g3BiyF^_dsblK#&t4?ZTpe_&0@xL_MYx`>?ym3;P|#B8EfDOFIu*)E%vls zx#yiWgABsF(_XS{iLD%O*BYD#+K(~t0xv(Nl`N6K?Fi0pv&#sBE4{?bG6XY(+ZFt< zZC^$r9EO=)eI1Dn1Fct&+I%vK;Jz3Ei*Ao4*$oCiZ94@hA6~dAn~Z`|xQbvw8#QAb z?x~x~_Cm|4b|b-cZI?3sz_q-Ig{BV7NbY!WQ`?;ksHKy4LXMRqGn)H4SOUNxN6T1m z?fX_cm}j`}fGPG#2G+91JK?^S6O+c(1gqMfWn8eF^VZI_+Q~f6{T%$Ntv!Qj>GWSh zuGKE)MG)$p0xVIrW$b_253F`GFL8ece{Fl8A-AmgFX4gJ9_AJ9pWxqZzcPMW&i$wT z&}uL98h1WnVf(VX&`nOjHEsnw;oL<8U4S1sZi@BM&a>Lbyun>cSlvFIc61ZZhnA=7 zf=#f45Mb>Nch7E`@tMujb;aJcTS+i$KX4ZqieW+dQ?AfE+|@ws=XdwG-|0iLe93-! zlAR&JwmtZ+bW_10zXFMyoMC57aBM$wSF=fdNLC^;vor0t#`w*a=>}k{Ko?wJ`{T@`TX}voAXC5!>?j0f zyD0PQ))~K9j&2}UWEVwHx3^|y0^puju6rE&%)RU_|qzM!H5lL8+AGiv+S*6 z{AVk4Ph*=veYk$drYy?hC;r6Bsnbv~0LP3vP+3t{AN(gPr|{5LZYp78hg(*f)mAK# zI~5FV=iVi30pasXD-w2+J4Jv3c8g%wLCos0D#a3cQ$%PdK+Uf1=%8hNw)%jb!O54|f!29kMPBAOfNRD&v+B z0y+d)QPv*sWXXVq+HQCz3?U$L*RxYwYTK);2Tj zCcwNg)!Y_BYRATV9yaVix9YNJ=4);TA-ls4RPZSS#nojoOpvf6JnYzaFVO~e-0fM} z8Rk1~AK@`*o-MJ#9Tx*kj`;zs*eg0t-)pmBA9t%MqcJsHB|+2?eeWFz>WD$ehWU{@ zN~rI+esAeE*a^4Vvh&Q(Ts5J!BmLgyZMYL+5UOC#aNiKRJMuxVJNtxNUD-t@2mla< z0BRSuO?g6GR~F0s&izOj>v(xDaU1NU+w-zZ%%9vD!rPACdnMa&C&ge%%ADhVCw%Id zxYxFgebVhk*%js=?i}F0<-JDc+sHI3k?1cMt}hf_8!_^6O;gk zWlTNZI^w~c^VvUdy$zadqz^SLN0<t^@WNnq)eA92{MXTgWNU4ZPjNE0Vw2 z^X+uN$&8!|^_aI0_-S;R>%eIy3!Fsc8xd|qhQye;%Pv0Hpv?<>Xb}gAnE*;WZr2!` z)#jBUhakm>Z+@RmoP`~!`fvZA8s%pS(bJL_*;)ipRgdeD#K|5x}{h@K(XPZ zBGy^^nef>G-7>5{=)J^CTUnX*2og=E%fSX9_(YO)h{d;$AK$AKcz$DY;B@xM^5U+@B(z|SHG@dF`Nvh>Bh_}QZX(1>hETH-gOP zQX@Zy_(U9+HnACKvk1QtNv-?>;w$l;R6;62<08Z(Wp&J0-gn|B0KB%L*%5A|WzU(H zd2_@c(l6{o?!6I$QThw)Rm4ByKk2-jMGpFrf+|8iG>*3*M7vWj$JD_SRELrqpz%EI z5WUXzInEAAk??WyOXyABvJit#qXWks8UZxx)hJIu=!L|%@5s5}Fc+ykVb#RE&085_ z-gz*G=|Do#CUl#zcMxkrws!jG@Eu~J5+1y2hSGV4A$Fa_oF2g064g^;Xfn?@WJf10 z=d;6yC^4h0MV^8%332Yckppowi}rgZX_coT%mIme7cf)!Xxa8BZOjawWr$B_evXGD zJKF7aSvxb6w>88ceD1;=mC@P`i5=K`h;1P!JL_|9I_k%uCg~C^6M+Z`>Fmxaa`cT6 zOwy%THUb?I(K(jW?3fXQdPDET-bXlvoa_9Q^BVM`3f|DWun!P>LauhsyT53M{u$I; zdN(!?;SzGIa|Pgiea{Hq(tEH42=|bb&ZfRC^fJ2+=RNSw2ian?O(Q1TQ%exTr709~ePBrHeK3a`3pYxX5&vpK| z|IKOU-0Vl)F>E~|F663K*Af6PBhJ%4>5gMxB5neh%`n$~XTibOpRA(W8xgldR(6@^ zVs;js_xmiFkT)afATn%Tm;7r3EoZ|=>V`WH}N=&!JC;2me#<&#^q z)Axel3;i{=192~8Ti2=F=A9WAP&4%Z6VW5*y35T2CC!m!x(+N+Q`=$F0`yMe6QVIkY{M|8^Wy?Q&gHG{=zSphs%?5Q{SPVcw-@)wPgPlWs5BbjJQI%zVWHbgucHP8>7jSuTep*s(1q5jfSoSjyHG9=@GA-$1#d4%csB`y? zhY)A8>wXKROB5@S=AjMWZVElrUHA~=TyWiQk#w124blqW?^O>e z&g$#3MN%EbS|r#OgF<75vst{Kwp3T4a~BaB+THuG1dyC!0LZgoRp`*j?umzO&g^(N zGF~rW9nvZET=%Dkubuni1&bNW12!P{gkJ5QmzTI>>5bymdsiq-ko!V!cI)P?+vft9 zddy12MxwCq+?>^c7zS*wQnGdicEKmebfr)Q z0vQ_G1KOG~E(HmG5UG*E9!UUq0(1;y0Jr79s*uV;O}vuo0A_)5w~qy0b? zYS3eoALQC`yLctV%*eeeIdpx`?)*5{r2JK@Fy;aJiD{wcJqPodu1<*wtE??p2Z0;U zswW`7+BG&&Ti+Va@~UEnB7t>4?pl+Wpl@x-@~*lcy0hnE{!iDrL;-&BrT`ygUZ_jY z?ffk!AReJf6SfP)sQU#N023<3Q=+Xq5Y+Ih-)-kjQpy~LIZjP z`BA`JFu6vv)yTi90_^(4`Dy#NCJ_xZRz?9pWf0yY%g=K>l_WEeS}Sl!QD{`pt9gFyKZM zq)}K^FE}H57v#BlGK!5T_C_RtqnG!bC}?m?V!&Y(lo1737X>}kf(vKr8Cft4T0upQ zgx2+3D){53m26;)aZp4eCxDlcUa-#HB{|C&;~0QTeH|+4VHG&K$0lp9v);ivQ}r&i zADA1l?ls8?>#Utvv?@)gs^?k31^2mR?e*3>S?8-hhuWL87G%03QfTWTy8?JB;#kAY%>mrk>Qcm`x4t;2A=qQbKtGwHJy z5U+r2q-_|!x3w_S1Cc?4Lofj>BIKwNXM{(+mnSo9;F#XOO3Zt zZq?2(*MP5u8V@zlR7n3*JV5RZi|buj1UYDS*Kd>5N0Eng4NC-t=++1EcV(NThZF_K z17T^s=0%u;1$X^Go099Em7Zbude6I`0%nWi%@kjwM^)ZoxxKrK;tuL$!naV47?o5V z23H90qP&BinZ;Wuzz$sH7gpYTqNw3uQYL&W#m|US6%Z!qr524J?8q$MsyWe6jyw@| zwXWBp^2fosOl>P`e^y0RP*_v%og%2GQ&xhNHI@ZN4`CAE%W(9J%_2Tt7!UxI8)1FD zkBg3a^0H{w5L|!&2{80tQPEk?nXFlB2!LjhQDN%d)}l;L#621y>jIu3&xXA#mK9Zc zlI~5~XpS4zR-F&~(o35!^(+9&4e1F*9r9Ax&)%;^8c+2-**57(#S0|JY4$BFhIpA} z`)!w=QZyiMge?Vy@YY`VY}s}x2zVfGhpp^0FUEKk085ATw4xbF4}Ry zPPlz%iZxzprVN}D6kW)NVa|OwiXj*?mLF0Ys^~!$hI#beEw;wsSu&(FOwoq~m0x{@ z#TZNh%gjA9$9$%<_JKOb$}Y#5|+|u@@T(zMh?o65gqUv_%8fyw% zz^Z_xRqkDLKf%fR9BaC2IIOnsBIxd&yRW^|`aJ7X)p%G_-m2FxNgVah%csCS(K<*k%c20yAkg-!H{ zAEo`b^#O6W=91AbU_N=@Cwo-+9|_c%YAzf7uKE%7rEl_44^W{H_h{l<{vdyc{p=H8 zefOX8ff%?^SpTa2h0W_<@OY^Y?4jFU4(Qe7YlScAU-dY#*yo{OFC#8M%O3Pu_L~5E zO2$K36ydtjB7U#wntm%#EZGQjDwueMw!L2Xrj>W`Bl{;Fw;f^^xVdwZS?l;t;ph9mJbnkd#Kb`S!rH*!6Mn6KLCI23>g9HTlN#W@ z#3lSz|B4ce!@h-r1B|o)Gkf=N-;(tu&WDo<;U1K9BMTtl&FZi64LsZkTrpD0H8>v= zel@3mee&7EGoUQWTGs&7L5An|yOpFJ-daRFsJUyjg?|LJ(D;FZB~lSQsy$P&)!rXO zFoH{@hYNsTMw+Ftwhs)i?mttaIjk;{c}njow%MNwZ|J{K0`WB~_VbcvD{Sq9!`ncU zjWuZ4k$Fj(3WPoQM)wz%V0;@Nxnek=$OfQ}L;c*6FkfY{7{g(+(ENz-vHq7OiN3H$ zZlKzQ<;af-e+#5FCBC>vVsFlUmJ|P6_^1Afk~UC&;`Sd0loRt~!+-RDDS79s1f?UK z2dq8(tKsto7Cc#c1oqg?hx3r-%#ROW0z5aHkKi7QeK>h67yhmA6$2YVK?eJ=+aV4p z{^KWw8xGh#2|J>EEIz~my>9%JaFc<3PZE#7O56@}3RxcZ*+|O)pC?5}d_gk<1Jo|q z-wWS1aOz3(k&F_QFL32x?AhVyfygJXkMxxYd>M}d{Lthw#QG1M>S6rx8|3H zV+RTkhaFV{Q97h?-!cCEpmlMzPc9$T0$zCxkn!1b!wCbDC-;xK07Wx~qX@7EZuSAy zle(kvr3S|^<%&T2XW?fDrk>F;O zKnm_umf&w)$vVw%4yO+ol{)&xmT6VqURXyPa0|$Wj2f?8QK==XS{P=xO2K-7Z{Yae201e-W%pVHp4iJG!xfD#T zGy)?kFatdspp}01`@osRX@o|=J{ES5b$pax!*N{H==?8S0iktSjB&ZBoM)6ly z$bzJeid*(8NTxE^GKRky@JmXY6p8k$NtUvXb~~N!w*pOPl>OKp=(z zEJyY*Qp%o@th#{Izqg)_Yx_VJI3yf(W zc#|QTE~AJ1y`<|hZJ;jOT15=ibQ|UKT}gLj1{~V}Qq?37f*BR^50KJjaL%EC(kfz@ zrq`$#2$@(i6ekKOvnIndeMXP%S1rzyxpEi*W_-VJX}_Yx{s^f=c7($T!1D!b$_7}a z{9~jF*=bH&0GprnI1V_RfE`FEi{!ip;w1rzF&OaF{v_#z>>6hgPG5i`Gll{x?eU}* zSt`c@n1=*phCG00A42MqJ>=}iWe8Bf?-RhcCy@qa6&wPtPavQ$fLF&pf;1|t=iJ2U z3sF>tDxez7O5e!3ft7#6n(~Rr81qTioqdNXYFioB2jN>|l#SxTI zqZj-uq`$Ji9IRlz$RH8}Bq;XRNs9(`%hv_Eh_WIv`!tdnkX~ehnJq0E0{j&AnWU{C zmBbH>sh*97Oe#E*vcYbwoQS;4c3giU2gT$vjCrayxv6@+wv4G(9--wBcmZ3;0#aUn78!k!A;{pN^mGcu{;6QquI-{x9jz;P0n@ zPHNQ~T*J&$&O+Il5>qhLRoK4ets zcsjO8JJEU-+g#v5zB{y~5_`I)DIw8XpA8pa$T>iIcH#6~llC3!)$C1zL*znG3cx(= z)SPe!lKbBl)KPNjkUxmM#x&2~ff(SdP*^f=C=}EN@S15!oRDQ8I!b;vM63LKS_j{o zBwefACWu4T4_&WZ3KE)bbd8sbEm)Ma40%nPfb3*3odSg+1VnP@P;R9&K1mE`P>f*U z8x2+|<&{DB4skJv2Exz+DtTn+d1V}Mt-+Hi#xRh;BTo)>R=$3-uO%xPvrf4KMI&p5 zRF!r3_!ff{%zEWc)J0%po2nefH@0M@U^Xasqppzu4E?VBgV$;`NX3}o>KJk4&XOhE zHNj>!gj9_QY#)lJyHviKYaNVl^(%dB$_CLV@(TGzu17FWj7(!}#O+5Vk=M$%a|ywH zt%5X$8O|M*Lf#LAI3pa5*&|ZN=#n3z(-Mi)n-dff)er(&^Uu7C|e>kAzN{NC=MAX7XgvnO#5sm#0uw+sszEx zR&FK%0XFv#Yg_>6I3&r3xO_rP$7~kF1{a6|q8s^J?f`+;LAwV5n`%@YIadAyxaVd% zX7543$`}O}!ty1&HAIAj26nfw73odnJ8}b_Es-Rd%+}b#f(5PQbUB=Nh*%0%TpBwV zQ6M3+_f;t+zr%H zioRk~)lo1PrR70fa0w^~1*UMQIvYCEJ(~w{#ob1MQKI4i7!)FUX!(%+xI3sd6h(($ zRb?otXEI;29|opvl&f|MVpR|5*&r5Z++fK9V+!-q88FpS_s9w`jLa0l2FhN=^{S;| zuwFMH^kak35aoa(y=pTU>xc_E9_$Q(CB;XPU*!?jQL?>=aS)e@0^J9S3NZNS>lG9+ zJaP9>+bAa$^;I{+^!rd?oPcAZ5R?!_cU4iCZ=V3%>v3!pni8QHt7;C*=tDhXc;gN{ z-a(-$rorX7qp$c8aK-5I}FLpq?)Y^{?T4TW6Fq|(BuL0*{%r4Jz)Nx9U;ynLb_{_j; zDa0RFfeNBDDsJ=FkPtFj83c>tq6m}@MK(C@#DJ3uDDM>gd=3deC@Ys9SJt33mY;y| zu#Ln9-(5^iF<+4Tl`n?mp`AZGX$@ooPsq7u2a%Ibp`9l zE<*;<7T_BpxJj9AF#oxoOd6W3&;-E%!a`Z4go8J6DcA{XPQ%0kI>ktd5=4{6C~pWL;E&O-N*b!{K@|cM<6VI@1+S1*Nx>`w1xntPg#rwv zK;Z}OH_8DNuuLnf1Qd!|A>&I)%0W~GB}myMU{K7!KQ1LJ<)|t!HR~5}D0rnzAf+f% zZVM@q$_YUmg{^cGa;WTKK@H`+@{8adMX3}6y(4>6@Pcwp$ueFRf$uRv zsV6k0da8;=lqi^5Af_ZM*9)DgNh)|XB^vfx&`!xx+6aTF9jfALN(}6cpp%lT+%1fw z>I}o5QO>~L0Yzl7(p#8E^&BpKMmY=nAdpeYl_!J^)TCi}4TT2N2o&J8LlurwJBEvE zDCb}w1tXN_$}7V85jrFATFQCYXJC~SE0cw$5uPK(wUi668Nno_Q<*Dtjz}7T*HJFQ zz6sug)qJ@yD57JexQ-GF`z~nF8&=i|FGpyN8a&5bQvN`Fp}Ypm`}+|tqgl@}mzBRz zvlNX|C9I2xA2oP^xuP__^ON#b`CfQ-v~ra80&*4i7xjlSr~D!O7BMqA`vP(eHxE6Z zs-;>Y`mD2cj99O^23sIpNL{8f5ZOkO#wP1Eaj=ELCDf}gdGwJ3k?b+I2F`W%BB2h| zP-Q0yi&Tz@f$x&NSh$>OqS_})1V3N5mz*20V-S6+rOHQC6zL0o;EbC%9rRl2Hq|L$ z#LO5+H8K)#5TT70N<|foN9y;cHDYcl_0a37yHuA%evl`DjsHoK}OyH`gc zCJf3ZZo^gy&8Y`fSt8pg(!^wwCK0w;xQTig{1y*I;THKcOYbPxpslD_RTcQPswZU4 z(j?_t^mb~Hs!7C%GE@7BrF10}jRfA!ev#=GtXeGQFxW65ni{EEW!5abqlULol40wF zJE%0(v}io4LtWfLNr7z;?xtQ*{T0oR)_DbQrKG}4fvPh>rB`hlt;VFaV$zfw(Qe=- zBRJ_89s5eV%{rZJF05Raq1se^G@AE{)&|MI!O<8hTjfxFHaZ^is!fvt+a&a%=BwPQ z)1tS&Cbnzt!nO!cTbHPgRCA*7uVwAhOyyRzKebX7TrG_*cCTS8b zWo(ck(c#nqRay1%n8wL0DTb{?p{c+jT3dZNM(d41CniVffQ|5fjNnhv`@W1KO2=hrJORC=KAQa2ADdB!<|e<$me7AZXeer-D( z{7ibL;GJKev{;Ej=K#6snP-|a>UXj}=_BQT=!aD2;Tz8&XU*RG^-CWs51|XG9>aH^ zS)awfm-S0al)mW4)I-CC&oE~T-un$mpD2%lIt}bF*jAiXz84R0O4-MRPpPMdUp`9& zPq~|nQ^v*$dDPJ1+G|B;eZgbT0LE0bkQy;O_N@7A#s}0Oqa1e}T|+%L{OQ^2vwa@~ zgN&!Rljs-JtHbkZ7SZ&lP(zFgToAgEdTV$^jRnnjN-)IuzhpIRS@Q7u8fRM46kJZ> z!U)24YSyq#O%SbPs#s3p!9s+cRHwe3HJ52x8UqEUN*RgW1z%i&Qf8jmC?csTgmVdTjaTz=~)%z8SV`F9kp-RisW;ybecGF|>nvFX5 zVR~{{Qv*9M{6w7?7T2Vm+xn3>qN#;l6n>>%;Y=j|9h!uXPegZz&wsY){ zZlj#%?90MA>W|?sHSf+TKZ-{=FW6Uw|ETju7St|15Buac#;Iq=3AG}YjI65Nd>;2n zJjQ8Y#|sxntN;pGkMrzLZsVMn?3+TJ2*VM(+OYG=PvUV-Bm0(cd4$QxzS>0aa&?>F zG_ezf`VpH(4%ZfmeZbq6F}fy6Xb@pDa-z23yvyZPYK&M(M;k@hkC1DJ&o_R~Qe#?_ z$>{YFtSc95f1aQFto_Qmm7NM+(5@qQYM~dLz9hV|ZeynlEh4-|ShbE9V!vp=wr*$N z6>g3=0(lI~wY)F1*N_fe7TP)jHzKM%dtv6w>}!YwmyNcKz>l=nW?n$d&?X^L919JG zAR|My{0lKNvy+fcTn^eXB4*?*klpfTXm220xLoudZ`}2-wVDfRV84}iDIcKsM#POQ ztb<%M`|9^r+O5n(y8<2V>N@L-_^-0J(jH|2`ane5hAllCf$0He$tL4r?- z`m5}nv`_g6?GuqV;#$YJX!gzTy|iChf<6Kax<~3b7xCX@@1+CEQqZJR3F7V2iv>Wb zE0rlZ=)j2Tk?72K7nR?{A2@@sAGW{-GjgqNQLO$fYKk$0t3=}?<~JtSnZ|m~7Ee*+ zFrJVY(K(V^=L{^oa1BKP`C|e43(yRSTmd zCPzB!@?t%|7f(}$VKqWp#PrC%irU!t?*<<+Bg$Iz#fWbscU}+0)_hL@CSUe*@D=<$ z@~iG=?A&+lPu64XdSP6|!qH{Vp_iO~Bz&?ShmCBy8KE<32wcB0KW0BcCU8yY#0dS- zP0x>B;{Bj~hNy93G$R5ws=Yb#(uW_DpEYV&E3g4?9Cdr1c4_NR;up;;SiA6U#FkON z=bS62e~KGQUb8!Z$!h!P>E~^i*gxH7IFoFtkR9PL8vXp;CFM`C{1y-hqaQ@<8NK#= z(PjN#sIQE-xE^#tg!^df^Rqn%e-(eFyo2?DA(r=O?sI3L7>0kNyoU`4pG5eLmIIi) z<5%%F$_Ln>usq_#==0}smv!dgvy>^AT*!?ej&?rJyX-kv3_kHNrBDz-9UXbz0IbCD z@04lSu<%&~ZFKti_~nkd;_sA?uuN52s`tY5is$d* zAC%89wNM<9Ji7jcGY}cWe^S1{UJKhJvPNxQ1YJ?*W&Xs>C@0aK5%))TzPNlv>yN=N z%va@GbZ-P_-N6^kD^7nBep!EGzZbpr0ja>%`e!ja4JFpj&e~zpgj3UGY9)2bo3q{ZF<4DYWC0Xuk^R_ zC;Dr|yU~7N^NqDc+L3x=>+7AbCC#&(PhDWV zNVF`{V9cgI2xyugfpe4bVv%0t`mx>hao2R_TP~n3G+qj7{@`QY^?BDk=RaCNU1Y2S zCX_Z~C+ZunCC#_gqG}sML`IRQF>3wzwT}6Zw5W@Xmy6a%?jE~RKR-@qf#pK#662LX z_3Sp5TyGlZx!}=4>QZBU5j@g+EVte{E@^?~BI+{ZZS#I^Ju>#B9vfG);MO7=9TmsY zIubV~sy`byv*7z8J>5V<2f#g!wbp0GA+*kE>p=pI9PGh?F4wv;j->TQdm3VFEOLm9 z1}8ko?gCIq*r}&l=dd&K!q``E^Wq*9#UNTiUrwWdSq@%rS%L(Cf;DEx7ZXtb@iZub`FHdjzo6rGR!GB(-J6JNSGWcl=Zry4_m9(GuDj{7?A~hc&8ou=#b7L+^K#eC_+@L?B%|Vwh)N=lj6c7A z^yVCoux5ITv7e|cG60;_qkzWy&6?@0#{QxTfbWZ6rrq4C6JjuJWgH-?iVPo@y{x=R z(h)wCSgUXjA|U#H^-_AXKE`*elS zWJF+yLs#U3@x6^V6Z9cyqhw@Ym_uLW)kot#jYSDgIr&CZdt(ytiIJU_O7%k4P46%cGn)cLNL3@{mKsRP%Li_s z6Mc*v8-LlDcnh}NeSP^(kT<@!;mvq=W6>?&<-+yJy8>e!zDIr>A8TyBm9ZSXA$fP; zWspezKK`lk^{u|;!VSrL0R#hG(DA7~q1R-3+jGSu6RNXu zyhuArZ(@Cu^X;S+mZsEw#y3UFq86{QYC3+~Y%bo^#w8~~1c@@9*wJ+1_S}la8*N;3 zZabvzF`GEhlzAJm^4vzf{egEJ)bV(LR~be{GzK?GZx;Z0dZ(L; zA(~m{FcIDK?zVDei&?pQPKszl)ZU5fO-mDDtK7}Y59FkY%%Tnez53=v+^QDya*v!0 zk!6(6M1GS;;?@yci{yiWnGT@sa-sqtGkvRs7RjC}bi7T}iHYY;afv$mZ$du=+{^}d zx8RBPCT60O{w=r-CWj@mk0MVfnyM3H^%q;(c<1DZ9HY)myl;{x*67~?&i0&K(XOaV z6Tg~%CeGT{d)4h7~r?2k&CFa%cjnAP7m=^YL%bnuAE znAp^O^bT+Jxy|L9Ruqf8qF58C=BPU#R=?Ri?Q8rPc;E9U+?vzwY+VzwW%`IQC~uE? zGU3-;d8e^_U`yvwRjGr2)Y|9th~_&PYtUPh{Q@~)Oeh3@Gq3OTtr2cbJ{I^Ch!kH; zTx;G{zs$hWit2A%DLNe`o=9#sP4YB&1n!~6JP|Rfb0W9dIVs7&(wZ7z%m*FZvWfEM zprj51biNBtC2*huyL@f)Ep(=z(4sdR0~w&@eb&qbG_eof3ae@^Lm5s+w~#hrIUxTO zqxKFgRf`fjgB4jgi*jKZs1 z#YKwEun^QV*;pZ}iV9cD#FY#Z>G7#kQ^B}?$8)@OTD7SBH7ni=#U&0sCHQ7%>`%@x zMmr`)2flXbh`OiV4h;Q$#zM#Bn7}s#sxaNInzz&Y>^rk=m!FDA^Z` z5R%UZesCC!da6Fv(wv;J&RzR7%~&H+L zJ61L8Fda3lu5G!TqP1b|ZtqK~9}ZtYF@U7yeu~S6d%L|atA07mMrqXfFi}eEhQ)hq zuH^g{{fzpmeh-j_nhm!=bwJKv(VwW_>R&BCQ|2}--fMF$XI}OE=!LJAwL(*!Om6M9 ziOX3~y(n7ul~Jo>YOKj(XPfIe3#*q#uYR?qb^fXflXK2`@qyZokZ9PePt>T?4<>J% zy*IZlc3cr{`pUJHk!ogoY+vUM)l$dR(VJf#Y2~EiO$YaN-c;#08b;f`3T~CA7J!0+ z&IA?2(Ky=i)tOdJs@inWrSq0*xuZ$6^Q#-JkTkQ6$6Pyat5!OiM|-@w+iIPL-#F-+ zEQ;5!wv6_9mEY=-#@^@-It_BxRBw&;f5mML17n$%{pCqHYpb_Kt9I14-b~XsL%Su@ zbEX7#(ZpAst$CpJ;E|inzCk0$Q+vp-6s^^1v1W_iZIW}0s~w}yfJ$U}T8-H)cbk-) z_0_wgFTMKJ`ZI0LZ1Dk`)Etv)=ja=+no?J%Bh1eo&`S&4=(s;R>6Kwy(Rir&n*-Bn z#^%)rqDT77+c4<`=Epob(^VG0|HOQSY@?*B%?CX?GgOw2KGAuvT-zAwW){Z|cHUKO zc03aOKZ?#gkm>)AoLSmzkq>Lo7 zxl*||_gvfD8)F;8Y;*kHzrXvheSALe_v`h1J|5Xa7izz~@!G*U&4}+cwrDt=H*~$W z_D%8*dxzYF6d+si|25=aJNl+)N3}z4Vv1=o+P`AxQLSd&mYw!za^IyKDE9KN8G2K@ z8_b-lfuAA8tk~PXY3O6^skr2w_Gfe7ryMT6>wk)wS9>$Ab!XApnB;5V?&;4Rs;G^O zTWe(Q7?X0%!UE&Z8%l^yi*qo_a`YwZ!NEdYIK-)~jEgoha`L6y)CUu@b3z6mQo33YH9ODlgM%IPD)!J=z-Zw+<6!1js96r{X^44y5 z;(5o+-qRKdzzcP)ZifYachUA6Gb(3_-}xT|^Rv=rA9lOBFh2F3wMg-|7{-Eu+Td=M z3nQ!7$s*0)cKC7Ks(8aaZWkDzdz~#ZfNSDy-M)CQJ**3i>|Pg(&;HKC8Fgpl)AzW! zGQRY>T72~>(oBcC}9qryGnC-(|F~P0`88QDaU|g7=aCaZ?im*@* zIHo`y{Z{?(%d~xNR~bKh?_2cx7Y}3WClUttv92tQ>(qYFS?W&BRudBKH{DHJpXW zpF3Pp|13z;)ClEE*9!vjjgetS{f9&c(=3#4sU87DD1^hD`pU#;QzNu*nO;b-+<$gh zSl^#mZ<>YnE!QIzFZwSIFV-u+)7o$3;ai~>4tyQUN36bY2RMuv?Gadm z?YrpxMxMUEdo8!Ez^otH+u;7L3K%^c0j$kJ8?)ZavLWc5V!t|If~7nyUX9s4a=Ibu z9UNeyO#Y?-v=T<|)TM^PcbEgx>)QXWy|CDXVNz~2Gy@5Ux0mprUQ98=$zlZCF!62> z;6@nLy|ESs7~7G@4Xct254zo80E~mhcAyD)+psUm3(x`>wY_l`yD+yhKQ^37N(PPc z+`5$bVq?suk?#$+lX?zT-^{H~Nh~(Sc#QmRcm`&&_P25yQj&_zFh0O6lAUzbtol}N zV@h(dIp$1XU&G&|I8XtLVO#^XDGY8z+%TNfYF6YO1KcAP))>-=Qsc7sYY&knn!_HEV6 zDt5$BMv#r+@3jsa-SKVfZQF1jlR2`t=G}Yv;ix+%?I~Z1T`^xrt~9RKet1}VN4w)1 zNIPK)MtmCE-xq*jj`o*$$}SLy8NoJAydONwy31hoQY}0%l_QTES0x)BiE>TvOvx|y z!qkksY22NB^+@%-+^&>@VsA{-$j8Q0$;n6T@B4aOD73hXVU0{*^-Qiml6Bv=N3W>Z z7sDATYYb1;GB@(|?bRzT_Q#BjFd9E3JDB6zPqTYVECMkTBfX8K$r0vK{0>eEy*LOn zJu=qVl}s~_^7HjSUuF@4QH;zr&Lm6BrGDCd*D5T+!B|^^u|CD@DDHuF|Fz#1k(k$z zU$uKvypFOSF#3C|EWnAFw;c?=(~r7=!&>iOix-$pyhDupDTCni%^2*hws?iv$~(q* znPLd!EWUqIYK!AAyLjgr*(p~os{M0^Q|gNoF{J&x>x|lz@htDHtR#l(BF@#ZnjR$nR~k_=q_U>U~hM-S049cu8p~&cvMIy=PdE36|;s zCSxgW#o3q(yeatwa)Bi-KwEIF!ve_3c)x%HMq()q&>p|mX#q$^yc$M4s0iW$wI_Np zc|S0BfC((0d>76O6i(>%0!xfPZ-miI&H*Jtp-_)gjN2Z>n`TUq2jMInL)6Q)sKA8t z)R?O%hQK4k5cl?5{J}iqtz+(^cv-RVjLF_XiyF*p-gahq$vZ3ipxmjH;o^GCJ7Ah| z4Q{n63W||j<5@5z%R~!8Gzxn26K}q_^Byo-$oLifQA00fk5!GAhJGkuqC6fvGnB1q3$|q%=f({@R?y zBxaCT!;A-&NtWyIyq>&x8Z*Z0VCJXZwdREi7xd0w%!rK?3nmkk^+OV=n=F&+QLA|5ZokYLRtIOtBvP-%ZaxY_YR{lDm?;sl*KH!*6NI{&3Z< z`bq98@|Kd5%l!C#O@BYc*_lVhXuCr#;aD7B41_;+cAuhrAyEAiBsP@4*rfbX>zL6~ zUmfVylK-$z_}a}|KRO)Cdg{9xY6zlmF??9F>&@_Ey3ZWfu(y|7(tZmZL6<+$jzv8) zSxeqoau%Dyw`>mjXe^k0roGO6m!&f{oqw`9{-fD(+;i>qY=*vU=KPO$kMo`j z*F*P~T*7__edOkkImfG+)f?FREU#gU`Pk-(kAufq(F|R-sig;sm7FvEY_yf5ABguUne#WQA>-dHC8N{(%M^a-PvzMG*Sx{K}L=QbbbR-Q9~b` z#L^eb=F^))(pBwsV;uF^M=kxa!~BM3a=M*;VvHk{ZD|>Zo#6L2m!?P9>&805*jARo z*ctwKGdsP?J~7r&pN+H(#V+vYn-|hmPwKvMG+^6UKEjfgjjn8gX4sufeC4>6{h#F% z?5a_{7DPtGN!{0u+t|k}pJUgLR@7h0pq-3*ZDL410rp^eqbYg;8IqII*V^0NPg=eP zi?Nd}@fl{PaBsACxSzI+2a>-_Erl7FQ_?ruJKfKe@XZd6-fC&i$T{U5C)^2jEJ?vy zjs~_&0*ewaPH4nV+*0ppGy14SGjq#nd!SW<8ZJ-A+K;|!*^%jTy6A1pZg&^UEUeRL zTFddwxYOqGF?-xyEpxDLqj^BQ)OxxoKE~MH&60{mk5;rqX0CNGPl(y;4&2IE@6o1~ z%uE-DqJ)@zY?m)TvA(06mdea%2ctw^6R3Mh2{v$4*wUX_?;!1QFlD1G%dpm^LjRe} z>7^x!gETSbp`J%c1@_VC^48U#20hQeW9(;pTK>U48(r7B?~~UV);q=lwwGlM5Ik*f zJ^LyBj9U`pAp53e1E6gjYQ6ty@C+-7VaE2hY{GsRJ=Xg2li^vn_l!gA+m>zE?9mIY z-vA_y^`3Eaa%o#yL)G6H-^AY%q?$7`RS!$JQ0kNwzkMZo&8y>0HFtIj1#4DHBI zz-Ckyd9=+_xzefa1H+CTWvLUeb?jK%%g=_+ZXX%P*v~B21?(QX(Dv=Kmow`l<2XCo za$~@Ou^V8Sn(pkDu6-r(WyzL+qhtPUqhPyQot|q?jxEs-uo-*QrkTCvynROQN%HFw z!+?`xZ`yWeUp?=g9C?b&{;)H^ag5Stn;m`LDAV^eG`?g{z{RoLHqY$(^I4g`4$#CB zlK|9MSzCCvmW$CR-!sr8fVa6d#%TMH?ckF2$@eT&UF%4|{juJ*((DKq-7H5(c8aBC zK)~2|8#}woB{9pk7>&xfqZ;hH9_;676b~<9iYM4d}go zj`_Ctf2nuP%JFrBQcF+)Wn-c3$~jsWjlTL`awq6s5BNLwq8*xJcQNs+<7M`D%Uc1B zV=3*3oQR9M-yE;7e^}lLAhnNuZU6i1*TtxBCSZ;MVkf<0CG9~uii_&GCfCTnN*)Fb zjn%a$<-pydfHaI;0-T=XW8Lk=IRrO#stJlrF9`~m9vf@#%AvVMQBBa~vXYR1`7wFB zEJxv{o@e4gt|$o)@H?;3vHq*sC0w4ir~7ZqNZ}tzb0b2#fI$nM?yzKU!af4lJ`8Hs;-~w2k zdtGLIXWV4hTLPK8;6_K|*YwM71&mwlM$41{3jwxc;_KjLUil|)GBAJ#*a#kVXnxyr z#U9wq$e?Q(0A2g4W5+j_D@8wIZo2~~bbyl}t>ZXwF`F00086rE4p3j^0pD5cm7>C! zyY4JYY5-bL(GmG=?N#%iG56fNECEVD(A1In&E;y*&zSq}J(fiQeuBP^zu)4(3O~lz zooz`AzzM`1!$2-tR21Xq&b2HLAPJOM%YaPN{8tQcrdd`6LHr?WgyjjWYFXe+3+0tC0w@9`D?Ha>dUc5-_?%d_1Y`<) zVv=&0R7m7t^0ArFRupJ39SZAr}z}?0O zX3toT2XqO3u+U>?yB3V`<3E%^}TQ&$RRF~xfM@oq_ zKr9$wO{Bd735J+ZcX+NmU{z%cE z$)5JWGDluT+Y)$m+`n@)ug9~xDmRJ@q3H+Oj6dqs%r`!JP7mz(eP9 zT{rdSGk3E7&cL(dDV>P?i0it49iOw;!S@8Zj(_cR&#$_k_}4L-y#a0lOm+0mkbG4y z-G7dN_)I$xcw@Y-GbtbL74^^LC3!RLaNxc1?#^NW3{|f-i6QIJECR9PW1U_3G_R;? zlUOp0h6p5%%R6QH3NQ5<#;pqmaNEF$aSdR*GXp?G?bq(x;KxDcV`Eok{OudO8sTdw zp!o&90U;3Q?>RTTYlUy1JAn`H{rHithu;M^c(uYf=o1Y_42#nO}9G6TlQ}F zg~06b3tiv7d);K!G2+?A@JoTD{P7!Ijo;I6y45oh*!$qu0*lA7T@!#L#j0l{vQ6P0 zftBNryH*t#-f{!fBlZFKjlkORw_W=Ryl$}?7)fk1xKCj7ct+RRg7jN%jSSS&!?e4B zoj|*HyP)S*bz^Qa*_`GV*f;*W>luir*fVld$QCqAAb-52OJ?Zot?PW739!9Kbnc(Kt@eg$3_cz2?+yRZ;*SK6VS>kj;>;GWCxZZ9mji(_e1 z-BIw;z@6;b?#V*IT^{H#Ks{&`fsZDZ_pJVD3_|@e!xf(JKY`CC9$B|-cD`5C8I$ks z1+NZ_oiOM*{xj~Lc~=bJ&%r??dt!gj&7ZCJin?M7+`Zv|!!cpg6Zv!Pee>>^AMUr| zt$|q+=Xx@Ky4)}7jwy7%3ugsVCp>!o{**Ls>+$^wy-!QrQ8;n0hyAnaej*4zvVGy4 zz|sj~&%#etU)^5EU+f3)fxy2LFM6OwcD^{qu41x3Z5ZHV-}hJ)5q#CzCM9GnZ8WfL z;!DrvA{xj)nSksPZ6dIDqNFDX1cKB#CUi2MHW@fHQP-1Hlw)vi{|Fr#3|BihI?>ki zt;oxd1u`P;Up`9%#S;TPwIF$64^k845Sk)zZepruw5Z3g8aNioB-&D-l2D~r^VgOK z_I}@b1vN!yf)y}k{M8#!Eb*5PYlBf2`~c2N zSkpTbunXfoB5Z)Zr*Zvm3)_0XfuRcv3{2R`@T0hg!lB;VHSaO@yj+HRjv5??6N-C> zi(4^8ycni?D%=`J5-PEmm8`{@^R-!UiOVFLRAc=zzeeNX-?0BDkCVFvpwEgh&W8Qttcc>3v1 z;dl0JT2Em0L~akch~|gu6aHpDqnYhB6UOwq7lT0>PuRrHq`BaVgfVRLfbLLs}KR*%aP`EsGLk}!B|COi#iD|*bymTe5SpU&lzn`!B|lcG1A-KBB={g{p!b#H}d;haQi zoa3c&!R9kDz}pGW0nX1~mY${c!C5oD0w{}?i$jUZIN_yQ1fyB)nbuDDcic@;1BYB{ zM@XD?oM3muf8y?odO4+~5d>YSqmbPTFTn+f#yRZLDng>vQN-rJ%YgfHp0iM@O4OZm z6tnx_zj2YGm0W0<9Wimvagsd%|A%`i(&Hk^B8a*&$0_y@ybc!+h)wQgRm4P@_MylT zS|ct+WXTOGQxMhVCevg-tr?dgIt`dk@Q^6E$qac6DCED2u5gRX2qEeUlUedOfQ
      UY_!QAZpsSv3`dk!l8a#X8Q^q9M<+%VK_Ubx2Q<=R19wdfUh ztEoe1*1WF_Dxr;l(HDhlTOJ*1wBRd;&d|ni9im*WXL)^S)`G7BDg|dOj;M?qUamzl zTJ)WV%4k!#5z#?>O1T{=anW&sEr-wICPaI3ew7nQ>PsezXjpQ*}Vt>R;SFDneg-Bg%r98Ifr z_>D^E`?4#phG*5hSccNF(#4yKJ^KDu#D$xy#sYn+)fT+D_(9)rMJvcuGMB4B=rBA| z9NMQ`srAT6%})cin!XKxLLAd4yJhz%QO!w{v&L#C=nSUx**X&*sV{@*q?l#y!k-s^ z>T|BldE~uJv;wvPC|F5X#6SBURtg^RmWi}r8|nM;*FjSAb)|8H_j1un*k<}6{2g&e zUw-A?2;OqhDwyh#qxgs7p}ynh?<4HhsoE4M9gfF~r}{=Kdm^gUsSt`j-5MVzR_WLL zz2&jJ233c$)#^SxO03hb2|fKdOT%w9%#eNpA0vkK+y0JzY^3S82DXEK8lNCG>G%9y z|2RvN`r`LaD@Qz8Y}Fs~TQyR5h0{9DF01qSO!1k1@^8CHDRtj^${xBa0F+cHr~Qb>yPkNj z6m5hZq~E~Ti4*z{RNZ~TTPfNEJ4E-vw}?OWJ6GjA@m?j`3_C)A@_4aCS`G(f{qUXB11D3FW}8@c8Yz1O2ri%4!dx!YBwj z0Y4|cl|Nk73bL)S`Y3BF5`IanG_dT?+Nb6^vA|Jl6@gcu)EU_E$ICfO$8Rg==AU^mbc*tfp@oayQKlap@-cK^Nlym}pVH^qgXg+Dj> zap2V7)PmV4(JIaZD$? zD0-dDIJc}Ac=DvxV8}n0jVhZ>4^e#R&G?MT(}PL>;4h;#n;xdzp||6|0*P<&Kf+7( zEv82(_vl@K0CZ=t>mTi9)D~v`DPJo#ILP1!XZ}fEde8Kmqe6G|;VUN}53Z^iU>0s2?1L zqthJOlk~xmYSma>n3EL;Z#9JIsI4asoH;8091SYkW-;U%C%Z=jJEroguSM33pz0sGqknV;Z4?dzf-U{`Zk}lQz=7; z+6bUIb~?>@YqclHb?WPodu`R5#2ro!oCGVAAonTyP)Mz6obFDiGn{u;0D?T#Fhs7k zi%Z<;be8iTxCib}^$wNR+O1VKGIgY+fFL(^YHX;hmKGOfWa>nr&=Enzsf!P1Y9(<} zBWj>}8r>%75r}%Pt}}k?y-Rcs_K|)p==s!!;R8UX%-bb$hGo!C2ECcuIqY1Q^VWN} z=sfHb{R}|592tICCwR--EpmZ!h|WIo=)!nJM4sSA%{{`45lQMHJ;5$GV}Yz|&ejRb#ANua#i5*Y6T6 zpME{)-&D--)_RA8ti67hVE`=}#F(NC+tx=XusoFaM*g7R4r-hF0$7E#gs6R{S1CV1 z(WrN-WH_i^k)Up3dX4gn{xE20s%|)`9-bIwV(LyQu`)j>nCcuZtj8ouO&}-~-6{y= z*!jcl^#zGIQwW+87(on@gJAn)y&#ciDni35=wU%BlI0_-8;sw1?-zN%e$yWZt&nUO zIRGG0y#1o@ZBg|GLfK0DjTAcjLiIQ!5ZoB0e8`Nq`#p) zDa*{y8^)wlf{>EX5#>g$_eO{O07{MiG3Y7ediuum``_aL84J~7l^Nt9c{g&n5%XSp z2yzG2X5|gjxsow*wlV#^+hOJ%PP^6Dpv#gUBljBz-?I)g?{ZjHc|o3%KO-+24U^rD zFhS>&KJDxyX&K3Gyqc^IeRLny1GZp(lD?6@jd985=CMF>ZI$5>C=rhIH`XU-nfv*{ zxb(81P|4y5^te)r?op=)oIX0-@UdhCZxh2z+5ad6WaO;=21QFY@(wdFDbk}5f7Fmw zZBU$K7tfVZ01ki<3~I!xF(_Gblo!B|q)06wSQOu?IVfFn5}32i$T&*~7?)asVIZ(* z6f!VmsU;*3HEz`f>I6Q#c18gi2Z!KLLMwIln%%ONAM8U9G`gl;cAigSrmD0=Tc-%la)fqppsOW6sLAZ$v@PT@N&~SPlqF8` z%5{>VSo zocz)LIQ2PYeW`KqB_JEW-Q4rB`ZzV3qFZVj>@oeD|EzgSy8Q|23(BTavtXa;7Jhc~ z)%5BU)R&YkrRKp8rU&@7&1R5;_OUT2C;}dgn-=qjn_JTpIi9gF{ZeGGEqb26(2Pk^ zJ!$%iVo+)u{CIlB=%yBUMielZQ?`{J2hNnuqZTcM4E0l{Zz$VKPX)i7-ZMIR>3at5 z6eJF{6LB^;dHN`DoJ%sKryy@pyAaNxD{^u)zQrsPcN!9p+JkTn&H>()!WK-X^fV*^ zwHI+YxM136w7sPu6XyU)MA@cZ3oe-s9Gz?tWbzzDiLm{p9>Kq-pNy_qDT#lAh(}#H4T#2x4$! z*wpAKXth*3QBx__rKDh`8I>{3wk@A=?vZIITSP>#`i#!ljy9LiMdxBapzIJ)!H^k) zvEyxVpUs_PKcbEU=&J6_e&B0w{aoZ6n~t(a#0Kln*o;NCt<5$+ADe+Xg@_9_nmIR? z+2)d6bUro{<$y>GHl6Vp``Z?mZSE5L33V2c3@%v@#)jKkvx{6}vrtZm)ZnDb&@tt9 ztuICw{652+f%E+YxQ1NYT!(oeyTGn6sg_UJE0u6|!&uBBgsFV5tSdA8So zDY_lwm2wdja<0vkj0LqTzNlX`{Ytr1nh$g#bz@23Di?Lp^c&^qa)3mc=^QI;$K*&a zLUK{pKq3n>!yjvJF94T42o;4wlm-)K=Ei2)?PdqvL{yjuVue}wjGACohv8SZOUyiu zC*n`=vzc{*eH~t3S(ljk94`cA_p6z0f>RyIU+phbU!J*%s1Ht@*)O=+(fYOMa%=(0 z8^H`tov{%_cC7toekJw?OewnsU}_x&A37YqWnJ+D0G-l~;M^IMpt2+So6%LjpRoHt zSN{`SI{Q27zu_v*6oCTT7k&Cnut3(K_@;i%^cUs&*8bqCnWuvFEVEqPHApeaA2A$U zKl4s-n1#ueUW1gNu!zy%)|t-&R{+DsxkG5EK*R*-BmDx(dr7X;9YROp5tG3roc2G# zYnCz98zrK{f=j1^$7VVN`K-HC9!gXSBbLquPtA-7nprtiZ?vck7Fs$VET5SbOtJ)2 z9$Him3oBJ3sLU=OU)^b(=j|b?fB`WsVa4p{fqk7`c`OfRC1=}vb%M_9w((P)$$9pk z)ZdhS###j3+5O`;J6rRLJY%a+Q3wb@e|Drayi+UR=(^t@7;x?ZZm;9`hfat5tm}S% zVbP@<2qv?r@ygEVd?PQvf3TOOn+Zo|eaHJdB^B{rPSuhwH6e4$1+i=2x|gqR%v2c_gX-Vjcr1GK-dwYW_2cZ zbi4d00_;vyI>Me1J8Lj;ygTlP`R!PMQ3RA?D#2vJvs+M*aNDVs^9f{D$+K1y`4443 z)bE(KQ9hSC6EbE`Pb7820Vm9~odQr=gs-z#KtzO4sD9V9gYvcXGU3PUor$h)T4B^( zQx*k)ZwU0+;0al`qEP*wX(uHQbj$wCJ_jshI3R48c2T~U-Xt{4zMrt@A^cRoZ`w`y zQEIQ%I{SIT70{A#_aQx~pNRW}p4nd$0X>qR()*BJ6o{D<24`y~;(N@BaJ~>Wsss^0 z5X^Q?6!u_1eGvj`gNScRQ?nxz%{@6q-hLtu?7$8pK{h)*G0`(v#PVZuIpv5jg0fUi zxFc)xFMCjGp;UtTiH3BIaCh(3U)2w&{gkTGD1wf3oA6X`@-O>`R1gj?jV5fA9u&R~ zZ246LBF88Y^(E*_ZG@4%Ym3eOV~0?+h&Y0g^qerW*9G+Cfa+@ESpvaCiV{}#Mi(1l z{6OBT^gZE-)K}QwTVGt%eU`^zASeWclqg*2RV~rQI`KJ8h>rwyzvqAr4KInpnvPOh zN;3&(r0<2Y)tC|~7BYrvLu3h$-JI(w`t)EGXdxhy<|C(mcX- z=|ACXwpqAaAak7ZG^~K&BW)38v#-*s!K|3lQ(8oLARQ3avXg1{II57sE~OFh(kbC6 zyN6bdqk@@BX*nTGsv^?lY@yrZsbbE>qg8|`sg7s|r^@;h-Va>N04FF$3KQ9KqUlCK zep9gF(mFzd)I{XTsi$XwgBFYjmRFQ(R-zD&YN>9p(=>;VXdz@u&xptzyHcaY?cmjl zC=AE}F0ew7DUt?5z#ki;3xMZ+MD3h{QXByS{_hZM0*NNYi)J{IQYis4hY}iLro!iH`YlLfKcpi8ILHE!KO?3gcj*1kux`^%sWIRhs~Cb6At?S5Z&hv zma#%i15>1>B0`_^x5#;Pb-8(H>^$XL-V|X($`F0vI+SOH`Ypf|rLzQ~lq0I-MwjbC z>=rrm2svR!Ixb>!tI88ePD`9c#3ErqIxkw_s#fTRIVo|KkSfGwb1TKrKD&y+%3qXM9=C0aczHc6_4;-8x2LmiMTnjc(}i{swgTJg4Rb`6G?MQlgkFy z{xN?VtApMOAm&kXK0<>5hd)_Q{Z{K607zfo&z#n9w!IXM|WVD=aH9I_t02kA=8nM*_E58VCB zdoEh9zqjlX@%!AZ$>xEazuwWJ4f-Z!*N7#cM>;tm_{)nH>FVz<^C147dosCt(D)zj zPu)iD0pty0?cCeReS=>ASTC5HxMoNnV)Icm(leJ&CiN?e$mpIjJJtcOTeeJZ3dkp!UxL%ggc4>%K6ZP8EF!e;)OQN34aw=#@QKSCGbbIPa z(23nHJ3W;&1h1_=p}GTo8W|5tvX`a`hcLC$H;|p^GeA*gCi9tUA1bKD#X*eFjv&PX zm*J;oh9tEt`k)b7s+UHzl|7zXHEdYt_LjMe>x|4Go|3(ta!$WgSN)c{JJqEui`c&> zZR+@NT%CD*>>ji$GKc6U%bU76+*(%@A8U+uLsE%oS;bW3@Y;IwgxJ04%g6$vx2$O@ zbJ(T6C?R$q`YN)B=qKx&`a2v~Z=M)yf_6vJh&Y*e>h_!3`m97hQ+;$<88KA0IHf$I z)nN1v)Opu#`Av)jkbCHeT|?qKrvu#U$bZC_GCc`mB%(n#$&SnGoLLWRd;g`nWP7 zks~XUguhZ@s&CtAm5MK$Bo4{yBuTtz<3I`ofeuE_5CyVM09wN^r4$GfO+?zkreq_M zW?l}{J5_|#4=tM~%4M^XNuGeoOBGq`hm|RXsK_(^uH+jwxur2}xZy~(5KZ~I>3w{! zCRUp0&)bMH^$;ETw&_#+iX#`<>8#S$6%{VfnpjHovMl zG2O{tANmLyVkHlrmhlzM>KUddQ(u*B4LK%%KD}u)dSgHa*R(MSrRITF zIiuz2%zA-Y2ktB6GWt96W=OsK9iZxAI;3AASI|F@w?kUxpJ!YJ1s%9=kgI6GTmmGS zUo!y$Nr&_s`M`w1&xJzeI zUaTj&3Z!@;3WM3><8htl`LWlzpI~?IrU}T|@Num!qwjt< z^lQuBgzQu}&VCqo=*s%;cT>Nny_CDl@!t%e~@v5%G0w-@S z14#)%D2THQn z_^*nuv+fguGZT^?X%XZBx(`_y5~0wL zt{0m1;(kFMq6d(m7pSn>Si$w`W&L74NPKp^+yjQ>qEx$Or(XU zs(KTPoiN-HBm=~DtfV1ARkm)46PC+IwuEFV&VW6c9Xqjv+7~`n)*kW|Fcgc01hzWO zG%$6%tP7y%??}7ACM}9)ic1xiu|pCv@zNQgge|2(@MtlzKcq@A^kcQinBz?s;q|A= zhC=ET3BZ$jm&2org7l}$_#rI-JK!wJ;dqydg7s&SD|EUPKLCedki#lv61Y;NIAmkx zZ|O4tCa^D~5>vq&Dul0Sk!Fjoa;wXzA*qToX^2=cAgvW8bM4Eip{WaHijXul-l2Vt-mO_-~RpvCsTl(xPs9~wf<*K0?^K0gIi?8-oS5U(N)lEIrS!OVITpZVD zUK#rcz06uGRCj*=+)Z(7Us0vVPD8z1I}|pLoC_Cg^&9>6d#tZnz9w|%yyM&lu|t2> zZ@)-=t?~__CiAGdN^x|*QI+2l{Z-|gLyye+&h?8ySLK&Y6j$3C7K)fB&Mk;l2Xy~9 zJ>}|HZwoy>|6&e08QmH2$Mjk1n(`f?LiD>iwu{dI?~mx2{<`vAq2~d0!Fe)g!27T0 zx&DUoy`fj;9Z%h#92{W%Wkz##t@np|&i|P^zIx-J{Xgo9)J^4Pp+561bJ>$u2dn>4 zU#4yWXUYfj19P>L$%FRQ)R-2`Z-THKB{QO>-`&89XVy#mg zcc-;uD0$vW7BZzetXt>wmb=UPd}!wU85w!XZaA^dDW1E>`eNv}`D?P$sfc0SdZz^L zUh6BNh4c4h?5V2Z#CoShu8B1YT&{_-g(=k$R+;g;)cpV{@n`jqJwkOc$z@##3e zo4w~p{p01&LpLm#$UUd)`B}|=>H78{2nSn0%EPC%MvYqhGW1WCzX{#B;3)qv?J%0v z;+LuKP@WKKvVf9TPDhU#wfcS1KU@Aj^iIP)IeWTlG_lnwi|Ys$F^C1Cd||qNhkBdo z=hSoMA3{mT7M{yD&A`X(w`*jh&s%4PIxM`CAD+RCN!uV_&=;(;LtPd=%Ux#*#&GSB z9P~x&Z=qKge#rx7BxBNc$XE0w>-bUrtyDEoj6YS z>B1VtZh-cv?xg0Y0=;tRtA%ZfQ?tqA_FdHPsW-va{N2Jq#qHUi@#-#WL8^B-Gc;}C zKgF}zEfeZT2Pu_Hn&!6-WOYe4!2e%sW0lx@0fED7xhtBXhxJTCuuLj=56Pq#YX*pzx5-_ zSCBR=n#_C7#oc_{?^MNoVy#VrE?Uip%&AW54mkbcKDAy$x;uD!K4}g<88u+~H}$!7 zPly4**Ts)H7IF~-@Hpu+PeQB*nvRK5vcgkG$e|68CbF&gn}Kr_}JB;8zWm?z8ZB2}OtE&>d_1-jop0?$s$P7xutVW8t7~_? zOTVD}7D=TmWdWgxnAM#C!jZp;w@Gb_Ulu%_@0Wr2ODY96oA$=D{_E_O2pAGD<_iV zi?a)p3W1a-5^?n_%EL%1OUoBm&l}Hqi$#6;EZ#N?NnD zbJ2M|XU=<4G@$>l`~``$dFjaF!+F6RZ&EaK z=NGf*jkOAFiRFC-R7t- z-u9PU+ea=b$!xV;jW_AlLBY@IKX#Y)gT zYbH2C*epdZu3a#f$4;Y1ty@W1OXrp{7hM*Ln;l)+}4~mtdMb=zW`BK=@vZb|)QE@(V`jh1Yq<>2>OY!42i;442GVYZ12&oaY z#t=&pi@FO=a_+SC7^!{f>yrCY)nej;lY%>IEhMp*=u06>s!O_yPV?Nk@*g*bmg<(0 zmf%ZKi>3>y^70wd_|l2@g-e(v=^|tiJ#Q@|NtXCaL z;w_1m^pz@%;?ygkYu5?MMNvTiSMlDQJNy}@u(lw6(aQ@W$HJ@|+6D^(A58%UVGlC4q;ZH4kKl?Q4DniXq+4b4gEqtY4W z&ngd=8E64iwuus2>9^7gDX)wJFfIn38s8Z<(=9&&Av4@I{|K+tb|m4tg><0xfOluZ1#k?Dt%KzDgRNy z^G&qUHdO2jyQV}_3Ra$1(OABJW!lDy17SCm>XqIrTd6)>zJFEP=8D5%_mr3!h00jf zqeCOB4D>3FhWRV;lvhVH0MetEQ~2vR%z5 zF_k7iv#69M<%0w>cdaue47YrSeDZ zo#wf-u)toDZA%ep>mxiq*2GEB3GV*t*I!EJx|G@=q1) zvbhyH8$9;fTn;Nxx~<%y@?+U&75)YTlZtDgxPw!kQW;;y*AnX*?62?$8?<|@tgX6z z`K6Uj8~YB}+z6{xidQyKy|KJ=Wz(iUGaH|ld37pPp-<_oOZb4Zdj*Mf$|;I z-sLr`^0s)W?C}fZDpe{!QC+8gRQr{lhlLF$jHlGB{7Ll!093t#_Dw$y3v>x%9>4iW@+9P%&;_NJJsia z3)abuI+Ql~xFzh9vXkn^IU^g+IT%33lD#rVE`LTQaz%M z)t=MS0mVogP-#=nR=ucBr`?2#Ee!%I#>2Xmi&Xv9$F=!TG28%OAr2c*u2y}cz8B&H z6I&SsS4@YEDYI0+tKWn0VPb^Gm&0>mlgh)Y4D}4iCH*F3Ux>|om`r&_Rj57$>C}Jl z^1@9M@UB!=R$H#IPUon>D;tk+8`W@imDOrHH7@A9XXDma z@@J1LG{e`a7^_{;pzA!{y5BDCNyVygJr%fGu*SU3$?vMiJf7OD4mVUerIw&^Xf=MD z&T)_DHtWNURW7Uj)WEKu+op5E^)*^py6D6=2(eNT}Eweg&X${WZte@OOwE4Bl1j55KNbt@cK9?^>Uo`!$#gR_TPrUF2S30bktfhfDyKV|1zTkyhVTPPBgzC; ziFCTR**vQz1jlb6dE`h!V#&Vg5dZ==h8Xf2STJ%dfn6e>PB*Ku@^a+(52TD7Pe>^_ zK7FCtDXT0;yhY2x#uGOdm9$Ogne|yUII;rTQX5Z#uxQ8h4Q6kxh8=qY+7>mQ2G`gd z(~tO=StFcCfpwe=RRXVMN6&PrnU8gfQ%7Lk;*7HiD@*iC51CP{B~JLDyd|FJZse3a zoPOGD-tHStvLN=-k@E>_OP)=?YF2D*>WmC#r;T(ZtS?!m=rh|7GO40q!m`Fo3A+p4 zO@C~*=Q`gRg6my2ayg-}#5nz>*&S=I1GJE~^v0_kkOn*ZdHawp0*GSkc%AFnNI{OU*Aq zi(DXKgzU!t8~aMaO;(#ffnISDg%MVQe&zKAF(zBgzeB&eOoqYu83PIO5~@k1xsy#e zsxTb3D&ti%fo-|FW5Kg^&nGefaq(j?HGC}dSJuI z>x6+4xe48(#@5S?14^`wrY-kNnoSm1wAiM(_0DM9*!b?oqmnigo<*N+gBvTRZByfi z8&69*Og31&1suWNn6}N0pKiP;xnWXfVP%JQXT`P^HX3h?mh_tJvhcS{bMFPIs>Uxj zUY87*I1bY6>fK2*>$Z+8JMk81PfuHL?fTuDXXb75{Bh%B$#av(MMZX|9>_R$(a2=N z=aQE|jC#n9<{^t?Zy))U@V(@N$uo;! z$Y|RE{{1@uf+oM z>x?Pwc|8Gn1s}w9FzaeFJjn5-J4~lpn%g71NUXZL3=ElCT44&e^s!Iz>R{DLGjL>P zX{{;7l3`!(MVeK&GsBxa2PDObmf7}$ULCXQc4hd2MAnXnbu(_8{+RTF%qG+~;yaT{ zn@s0e?zfNi7R@H?Zu|pZSbD;AvE_OD0`JM$uss=pd+1%>#zeEk>oXiJb4jR48QtJ&V@B(kUH%V27sd6=@974pI5i8Gnx!~<)#k+ zn=;FHXx_lVkr;AesnPVMZKac@Uo77`4Eu_pg2Nf{4MD@p*lhU-g)A(!F>|wuLkRH5B(`FNMi!Skngv;`$ZYbS(jc>CR@V>$KjZ@T z(UC;*{?Z7uHCE3N8b8?r_OX$9wYKCKx|Kv%VmW&iK;B1y3wMuqu_2(~~Xz@(#Y%N`4cEM`B<9q+{g%c+{ zIh}2#TtMR7=@=NmPn~G>T+-QInq&6J>WpJ7wAE~>^(CjGAjncea$+v|ap_sJRn`xk>OHfTPMr5#+c{i% z$!xRrXQ%f;<4Y$lc;6zK=&hKi6SncyHVU(uqnWNG4wpLfH=v8 zX)`2bm$ESPRZyRQPx4ZE1KN>lfRl_i-w6GP93u|#26{$n$)06o^K!sY^$MM&ls`z7 z0!uX0TnhDZNeS)9s=Jx7i|kuA-&_u5xYPs6YF%H(ZgN1`B6Afq+hs7cV@2JqjJ@QL zvgP0-Qs{yRBW2h1XY3<~fewfcdekK)tRtK7&gwvCWZ7Et7f_#zWnsvb>^mb3BhZ(Z6`B8r{&JZLgYXFhjg6h;GJ*M@a0DtmypRvmWhlt>vMO^I zn@Cha_#__&9Jb_HWxLG%Y*JC};f1SU_cM+H9pwSQq1%K)9C*2E;=X5dC%a5x&a~Ny z3Is01i9yen&Xlqi^F=miP)%#^=L|d;IYH)>smxc{+(9)(a8?fh>lir=V5T@^kG(T;_b?pa^i@c|vzzYHLsn>1JFZo{-axHs!q=US^?4|i_n?tTN0I_ES zcsytYe=vV$bInx{gn9%7B04t=mOGY9?X!TAy<|;SoqndqTg>_kw0PZ)OJeB zq_e_p*P_QrL-_-vBR%B0GK>Y?wg&AL%>kIWk(=cDGCzxjwx`fp(L?J8UXI)%%gTrr zD{Sweo1!^@;WToa+)x&6vC;M;dMtWq!@%ni;>E+@8ee8><%XWYDrg&T)O9M$SQfi% z{oT?4qN5FXiNGaisnlr6GQ*=D;9&fxT~edqb8^HkZ9_A9o< zZl*EFP3-q0L!BoP2yw6N9-tyG+ywiO@dRikS6e)`J?~Z!Gr0-&F=Lo~v226I8{3C& z?JMeZz1i8Z&gcMpu^7fwug8ag#)Vhd-xQ1{%} z@xlq?$g$wwGO2~P9g4p%mIdPXjW0WImB}o^>~h?PV|%x>eQ5-t<+4T#s-4&!J(IPy z?Q7$k&iiG}77OfJ+|y?Eax1@Oydys*4RCF*UXe{ zWB>4UuYXo{#bTS?FZZdLkRrmQC)z__)@6}t1NR67iI9m&&(ED>WqlTV?Ls}iHVqUF z{2cj09xqcaRoKn;XozENZ~N8wt@B;kki}WM9FJkJ1Z?}=_@nbv*@#7#o!A2%&nj-4 zYW&&xrEJ{dzFmt)TD%Ni4W{elA7!5`^mba0`gqcgy1yBJ$iK^eSQzbIc?`yP?5G2i zF7m&!e-_hV=AMWI(tmaT%bZ3r5m-RrFdxs9gpU8}re&H?EChBC42S~rt*?J6O4u0R^Seahn0Fl!EGfE+!ECG0w2hHn8GuW+zieQjJ4WH5DfA| zYEKCnUzTT%u~DN0k&tyTgJ(avxh&5DW2bf(#6fnzremNKydV#Pfvd^+bVxPK9YdqY z1Z+#aJ;g_`V=m+%2q6h5$a1!o9zpRJEQFkdrDHS{SvlKU??ed}EQ4Hy6=O`PNFf`F zxv`rl$b#I0?Z?DYMM8p&9HovBafvqcjy3|37Oh#_UrU~Cp`NHhS~<0vOw>mWGz zY-|HF@~xKT=h|uJb@n44X?p@F*zAW(DG>Y2EjPwKD-5&#_X+WbCZKgr(gmy z3P1CoGS#~_&mA*Uy;blX@(I5Gu9_*UWqau3D8+(5kU#LhI8zp~j_s*Wpp**CEcMPv zuUM9-j)0L<)a3%GrSwj?R{?9X4u;L7Q7Q!vmVWlBUhS+xDGZm%pi~Q7Eu-!41;1g9 zODAv`mby-WwPf1w^a`BC-#OtWFM!qy{45vRpYqC@HMDcUTR#W?ZijQ`E-=DV8Vfmw-~CXg481o(fDhX_lAmw|Y;_hU_5($~kJ4 zAj|TW{eJJ*IifwV>IF+E=LD-QAKRbzE|@d92Ns-}M!6{1VEM-Wp?CY7!o9GN%yi0C z!8XgE_TRm~&Y9c`BV=Y!It68xwHUaMR~1_}K+tDWG=fUY=qx{}e+s@@zH@lui#WIbFf2Z^nDR&P+w!Nwci*q`ClA9CGQn)G+{DTf;e-!o z7dFDc8$kWS(+Y6SL-D!n@x}=m$D8e7h=szqS^5|f*o@@8RnseQ`l zSsg~WNB@_MJi?x>uc!EzFSI&|NcYnu%Z{+;==Z3@0@AE5BewcYB}0x966G>Av3&pX zUc?^1@CAiOVRJM0QzFV&TRldc_bXV?ab|Qbra?_AUvKpqanG-H0sq*?-z_8o0yMjR40^|T3I;S`v<1*n3_wmSYK&_63>sCghsucWZbrB>dK z^Zc7qIL8N)^+za)<>*_Xjye9rDZR(r7RZlLW?3D!B0CEG5erEzZ8z+XtCPVyVZNin zKXGAmOWs1v33V!f#w~YL`%4z$PvoUyPO6ubpSN1)Xz=e}*nA>y5$3cyz5JTh4#(*M z&{TYD9tWdRbIY|>wT{?;#MI{2yv3Na6nFG(0G^8r5TqhcvX|)FDLLg2tWG+n2WV1d zC)rE&=P7H;pIKdXEDkVTggnJg(|1tTm%p^S?RY4Fwn%o0y-a_JvZ?%o)icLy0fI%y z)9iHp70TA~Z&n{1p9g3b$xgGE>#tFY%l}yYar_%#%0afVGxRqorR8SUHcsw=G>)u| zovBw-gymjmP);)fi#QM!fh*UjE6Wkq0ZvN-uW&>v0#B|5k}9-yjMLV@ukar#SXO2) zMOu!tp5?SBP=e&1naINQsq4%Atrt0+3CvwQerBTO!7Xs`5Up1@-3e@3%sD%djS*Zr zP)@Sm;Pf_dcyaI9ww3a`>ciy;)@4psLFgr{_BOs;r&gA;tamy22c<3PZEss8zpp+9 z4lkwC?4X7vtaEKS@(1b@(R2l@;%le&PV5{(hR zu0-A=v}W?VCWIKxxSB$U_`UUwGC>wfv+G^z< zT}w-LK&QEwha$2_y>)e&7%Ej*0foExgr;P5^wvo;aeygV3&prFLhG|gH|ut0dQ;~J z_d=~xGeY~bns4Uq!uWR06CQ%byOf4PSK#~d>M?#@$-<-1`7Vmk#1+kbdAl(IU8zFz z3u!LbLIo?3x7d61fz&0!bI>&|&qFmUWVhIR^}*D1;ZCe!q6_9>{Oit*^6l$Oq zmmI^wvkUuS`!YkRD}=Y8`+x(tAbYYOwm&nRx(di8j=R{qHfQs1PwdA;cI6760Ncd+ zu=m;Hwo;fx5;_GPO)-VFO2TS#$WZ zoH=qmg(?)<*tntM!UcS!j;+wssA8d`O%N(QT*H^?*h)QvS|fC`nTaY6H(iCi$8OTI zs5^z;Hgi#j!fC5y_t;1Dv#EQ8K{m@!*TMy>koVa~^@-FO+2J;;QBT6JtPSD&bqwXE~JiN$CTcf7dLY@<@$bn)^IA(Lpb$?XsI?;2&RXJ$A3E$g1b-ff-v<~v3 zEp|Z>^{4QQ%^TN;QSIvrU%;+s7E}KSf7|?Y{SM?tlP_REJ4BsUVPb2Eb|Qu66^_6< z^{bmrE3AOgB$Sk!H$F1ai4k-OIdEHVpg3vD9D_KB7x%72ZYz5u68tooUTQ3`9-_-A<`c%xdJ&2Br z7OY3UWcTUoss0rUZBL@pqc!VgFWI;Bd#J$`%WSWri=$09AYZZj^)hN`#R}Wo=tI%8 z4YF74+xq>~$ci<#&(PPR1sgi9VeZHqx}qx%92`eKjBejh_!@RM^AIR?ZnOP~{vQ2x z!{loiK<-k>6=k+J=fh?M0+8**07lV8s}S2dyX9DC6%36J==4e|tHLP;>o$8vLjenr zOyx(q=2XaR!`yOa3>WmiX}d2!)-}JP(U$5ao`DA3$~M59>{?LKY`eg%Wk%Y@-nVTJ z`g9xV@b*ys`IP+aoyuD`ihsfx7`lK&^F6HvY+WMQ8!h5uzlusEk>{z`HB5pe}%fW z;+yRUz!K4HmVF{@KX9#Ud&QLPZ?{sx=gpI!U?Z6!?f+Fwx3h4!j}0v3PfUzp)LnuK zD?4ZR(AeC<@d>txwhO$4;db8cuf|886ODC-%x-E`g|i*eJv(-=u)|mffD}{!ldvPZ z3u6&mNT2J*G67wo!q+az{b+3Lg`=Md&7FNzz#FhTw)SFd(H02k`OEve_Em)2t#*GB zdu5C03*oi=PS?STSi5cRzhb>IzI=g=XAXc{JJqhz-Dzg{*21r_H<|aSM=IvnZCFJB zb>Z=^6K^nsUCkBAcE{XLg*R^Hd>eSHe?V=mSZsF=XhX-g4t*PVr+-9ktH`wL2Id_v z(0&|vuYXK!uUKXG!2Q(BtZhTz2R`VZ0wGhr-Ang7Gn=+?ehhrnKcil$*kbq1{o~BB zZ9_i>KIvajZ&YNAP4}>hLl?0o+a~lfQddQV9o)kwuIaUMGS7$^?b24%*m-zR;v_}* zpLw4#FT46G_Sl`BoffC*D*H+JB7fC&yQ0A^#$#*TR1xGC0W=)CbQL?A?B;mv2gL!= zuZiD_L8QOpgxz9~^Kk{+Cx5}dV{%J|D$dxg^tcz-x}E=f;ydOeb?4lRb{joD#*J;4 zy>0m+|J3!O;)Y$BhgCefm^IZlsegeTt?03ndicbr6n9M3{mlGAeO+fee#c7Htn#us5t{znA) zAk8$9Da_gvl@R-%=zqj%is>3N5ftX?IU}Lyzm68zbgn7QQse;h^Gr=>|F3Ww+=OdR z1K9;wwCARTum4R>gPU?8G`I){o8`F&Y|#s+!_Bx>G)GYYY_aG0gaWVXz$$#5{lUCW;m*S_6CLd5tV6Lt5zEWN1yar~vlP^9lJ%naF}@ zrvQULQ4#F7=P&Y983YW=411@MMP)Ecj1whXPzZs;xmeBTJTc4}6H3Vi6Bi>K>!o3d zb^^UqGNo0(w@gaC?@e<<$zWj^iq`;mw@bmBn>Jr`7)HhjDTs2Cl@!7Cqb(2}gC${( zQc{2sO6tf34<^xRSSChGX-f04&UeBFX_ko211gY#(qG~tQ6c+Jm*|G|ISo6`^?mj7x`ELl@3GKbq? zv9v@{Gb|s2rD}vCTet_7LQCwohbLhTG@4lC2oJ)h(=;L(JPBuD&}u|(@R`_RnrS7{ zKFQ0#qU{uU!{=fT(P))2`y_9}3e|2=AbctI60N8b;z0CKBx?4F!r`m2PiR*vMGiz? z#XQYHQ7n8L_7`oc5`rM&6>N=MbRbKFb)tuh3lVTXZZhqNXbyZIHj-W-og3ah#l4QI0L3EBX;B8e3oE9ht5{B|K*eGjXW>e? z4ttcIQq|!k4dO1PT>!;JJyuJvuOc~1gSpFSvs`v;f`7sq=>1jA&iNsV#guEJ?eIU? zzjRXx5}8CWWYE;2a<~o7oniX96iFm1xSDQ}1n!EP!6=eIT!^8HEX_?(Jv;!ngmFb8 zav_E(vVrca0f11qGNvRD6fs=E*XTq|@Hx2s3;}9A3Le4Dp*;|tfG@_KXB1RVqTrF- zT-syNS@6;jEbG8Wvgw6i=E_ zQ7?QK&Yzi9(~DLC(E)A4;=6D;j=`+2A-PFsa5vK4i5|gKxNPQNO^2H_hP#>eN%R8V zi4!srwIp|GEO!eHP)XoA+)-vqZHK#bCO78rchN`q3!IuMsl|Kb$6>c?eu=&VeAT5} z>ROpcQoNy<_E+=={s;G$X1j{(60^cZ;aDk+ zTS;@M^aX!hM_DP-4y=^UtpZ5<5c^E8(2y!A9tV##NNA|a2>Uf&*$Ev|5st`I)Btv5 zto=5xU#ux9#EZyM)M+r4H2W$q)U4Q@A}``B#ZHZP}~URMyygcvEIe>hXU0$=D;B*_C_jL%g}O z`|F$i@)ux_Y35ZPvXA#JoekZM_s>tk9@ivS9<`tEt(cv-yV*Z~A@+nOwepnxa&Pr) z$!>f=ek%5)W=Z9F`*q%i+5NklfiOx1@et#3EQs?}PIs{Iu}=(?@d?HZtQt&etsIl5G0#)$XiNpJP!&EC3cQuDA z6CBEXtmdNkv%*z;g${t&SPr{<{O6|a?+sV2Qry=ZtK7kMQ2NXUNS!+~EVgQL1$cAY85d&%wmkY96|QMN;J{^qMa4!Rg@Z zKQFDJ7ns=;qnaMD^T7Ge1|~mNv}(QLC2g9?UWe$@8S~QC?~G2`V0cBlQ+d!~rf=~) z(}PH$c`%IA?o}RfnCp9J9_^rPM$$&ZTiV0QlMc&#ugw!2M8+g-GQ6ifsnks5_&%9; z<)8@QPZb|E&njJB75aXk_x0dp417I&LNfveycNFo^8*j@V~vH_&ziByK8L-&q4RSO zjmH|dV83d{EAKfR^G%-LdWb(04!Qe{23Qyl>tgq%I^-Jd=2xbxVz$$ zKofxWxAKzpFW+DDrw&6Bi2o`6*F8;aim=9`*s+bG1Y(I|dbgPvig3lxU>7w)$i!0Y z;cAz)_6T2mGP|{rPd1if&AXvu6e0@0p8dXYoNO$^Lb_pMEP{#O$qtnBDMkUeF9;#_ zLoCFfVrR*RDEVQH*4@rx0)mI{V>ifIRF%;1#L-n8g~-RhVh_qYsL~3q9o<75k0`;L zCm|FhnpDJv({W-ZLW=iEN>Oysq?KF;y03UXLXKx7)dQiTRLpgx2LP6z3ZI=csOX?e ztGLee5b<(ECtjF@P?8u@3D<=lCSHlq;g2SzC_5O^YOX6iO1uuC$7_@7l_aLLhU-R; z5pPBq@vo8wl^slJE!Tq{FW!Ne=4YOaXzD1)t>Z=`DdKX3jh}lmtx3kp-&u|Ari&#A zSHBkL{H943d?(k7K1;j{F>`iOa%&TRmT@Q6r#n%+9}(rZKKXsq_$=ctEWVp9Rv?&u zJCg&C@Q?J>8~o@g;^T;gey5UCYP)8ub}It97m3>tJior=h9j&wsy&Jzz$NTJ=st`k z4<0G>$=Qny=}s44L+tRIz5sd@pO`Ph61%x#Euz*ByCCssb7KBJY*=@;_zvQ*A7z2$ zD1L7Ker!Z{j`$&98*&%)A8nqSe*hcRy;l4j@!PI=f$1^iyrc#LiN0R^5^)>Q4{66_ z^O6o4X3#f@KOmm@U0WbHhMb>t$Pi24D*lG};P-ri=9p}L(qTg!y;%GQ@yGA)0@G$B zJE_r-Kra=WIokNUr_h>Z>?FB?LI*PuM>qet6hSjGDM?|V(Zym%#~^=y%PY;IB%)H0 z9`@8epEcy@gQI)2wIQ}lUfWZ^cL|#$CLi)3pFjWg-It2sld>; z%rP^faN$%7B$e2z-~i)hmg6n|{R?AH;Kv2mpi8>7D_1!_@ISRM>%>s1?xbNUy+fSu z_!2ldn@(^R=}sAz(XWWNIDYg0xNz*m&?4Pw!*co!vEcIb0IO7VD~qFQQ)G5`fxDqY zz>@9-t^DA86_(en71ua=1W-~Xt@y?HXRs@}`^0-3Ljt&|{jJT5^Uq>ecHaTyuK0k` zROm_klKgh;s_uK@qmJ_f6sd_Po0sID!>*=Z&OPb4EZ`d0)gqTBoj2rmUl6uCt`2yT zdgY{ODe;10ZTBzfz}8LehvG3LuyjYaFWrP>W(uiPDG{73+aLwBPYSz>Gyj zry$FSmlOrvZ^VO+Cjyo%x^hai46aPtME@Xu3KWK$7JWT6xeR`VTSzyGM;&hl>{%3k zx-cDnmAjSxP5jRBalrXS1*b)ZH&?NLh?8QY{J6emkHS$w)1yUZ(`6zsG-P|cC3@VtC2wz?Zt73s7t(7<`Wyu2r%M^OVP z+ObaC0)KI)RFEuUuc8jfQ)y0BfvCl?XGB@Vn~I&?-c>{gSzzSiqViW+#+%ssZhTd; z(;Az3i<{1HR_OW+yXk>di=EB^-|E>uwtk&`DMKPA>!REN(i>S*g2YI7kOPLZ@$m9~Y0E9a^coYdA~?Az~-9 zAloHg?Hs;tz#ym7sw$lvgMyc2wGZ)iI)jqVs**Z+2hCg3)XrI@yJt8;PpsPK6ah4^ zW9>t$boUL%=$|_V!?lls@j}*L48Xa z&aqaj9%3V=FRi-ZlppkJ$>6z;)zU}Y)AZ$4*PTj&%$Fk0lX8io|EapURa&RoAnek_ z^Ub;WkFjUFv#ahn9S))_m7K?~QC+&%PS2@&;N%^^S=!WBum=8A;kGBY>WR~Zp!G}N zpC4aie2VR$PowFbx`S$#dR^eG)eRdib~Cq*9~%fdx-{iN$6Dz#?&a<*tB+1Eg3eTx zUO=u(dTzKv-&(cfo70CN^EVyfsX~0AxCXlEQ%=8wel4B4z)G|l;ohK^0Ew+-uv1z% z@MatTL4mKj$_iL*L(_6QCjEQ#ieEb`s$kBz;Mr*n9jttnLDAh^RfTj83(f)F+unTD zsG_I4w#w6)8Z1sj1AC?lSS!1CRpFf%1h=H6UF==2dWk)3wYN&692>imCb)>)ko3yX zPv2h^;k+jJd79>;Y(vs(!yP)1j5%)){+nic30aUdZWy2|s_4$u!S2gwmt+M=Zw&Y7 zN2}&K9}JFLCb)#$nDo{#NC&PK=aa$d%QTl{8f`8-=(=S$SaDEf~a9R81!p-ncT)^M1 z+UER8FtP0Y-6I2LH}|PybkX2AG2HrMF(?Z!>PcS!K^tZPF>x}SzGbU@^AJ`yr- z`Ct%hyXu$XTlc%FA?LFpIm?Hy^=?=FR{ZGxR5b$DO5)|{>#Sncl;UUim#T4RkBX+{ zDc3uSrGL1;=|2D^M<1eHUVojmL;9EdhyJ_jhqE!{)$&150F?gY{_F0D`|JEKWO@el z2L8YN|KX--r%TL{b_8rjA}I9eI9=1VPeY(cS3>fY{2P=1!Kd>~8I}?Uq#q$Qqy0u< z3EYHd&ajbmNE`O$W{lk!D$$vYS}GGo;u5JXYhYrQ3y&oaX2>pALifzfib-3Bj7LLMPdw9|fsgH=4wkr5zSgj5l- zGY8cj&Gb2Q@9N9?_W+J*sLMfEz!U&VBMCu4fGgG=cgi;%xD4yh+-GwZup zx!$iHpcz26gZx1F=y9n_R6(>;x@%`jijcnvzcQz~AR;16>8T}y)u1&I#f{a7L`1j} zj1(jiq$_a-w+K9TiS|k_?JP+>GJv>*dj-5*h~UYfohxZT#t^r1r!?aMaBz!cBuSLW zS;Rfu@a{q}9NY=ClZ+PRBH|ftZudAiYH)#Cj^qpwN8jVNcJr%@PPky8(Y%D*O#IA! z-#uPsbjA@t_H>6DSwXbt1#0;cBN7*?&64yX_Yy;Sx!Q4w(FGR{zJ3pRjF`-8)$*&2 zC|smASMmgTfw-RcUOQfGbj6Xhc@hJ%hq#j$*u$?eqQPV%jr|7sfOv|R)iYG1a~q8T z*(W3NIR7TEp@&tga#zmOZj($RzYyO7gLQAM%0n5i-68p(i%FPNJ8RrIkv&T;_!C&x*Y%+DY_Q&erah1iCB@y##Vx~EIk6LCDPR#M~Q5k|?D^yByChw}R9 zeUjZSL1F3Hnts{7q_ENLyKYMwTuvS?$o|?txep%B<1=&;rOT|aJ=x*63-`k#csYy* zk`pejKiaZ$Z;$UcM&NR_Ly|KtE5q((x8CL-Fh=6mYKJA4TsDV&&VGM;{D3hEm!}<( zs9h?;>{rrq7BuKcqxp<6$xWBtVZkf2?hG~PqDMC{#w9wJBS1CYbcb_LH)C`odi|Ykx~7UA~09T{(QW_pk~Sptb)b|3jICTk+8YtVUIWvP5fA4MDBo zd+}2SIvS;9UKzup+75*YXYlI>NOJhLeMQGLB?weN_!9nCkduH@c|wLwwJRz*d=vlc zz@!{bHyT@@bmlv-J}A};7J%R)nTa1=xHC7I*wAu z7_DZYtD{jH!$0!JbVEuVbF`M>SxrWjhug05y2oiUrtgw!an&r;u5kZVY4>`YRI`-3 zw7%7BlrnraFs-wWsAem7YXhn|sI%cYtA_9O9#PFv?$w4=XP~;meLg$fCmoe0^7b*p zs#l_P;YU}c-0wIloy$AGh^k(P(uZqT)!!!_lg{HEWW-c&Mj69jts1=FaZEa&cbE}h zy#qBZ!aN5tNNSd{ai;qz)k2hardtkeP}ZE3G^$`Qs;hxrF)l|ih&(Rs6*p;TRo9~e zB9`P_87!T(Wd9#*Xtw1p&cIE^=;I|l4a4p)D>K4?Z zh%-6455`-J3vsPl4zR?pjJTK6`hb7Jn2I~4O{=~HwDF&F8qyt_0)Nau%~)Q29aR!x zz8dk6)GFnS{x0KIYf-fkp;5Glveu-sP;jIDQhaSTbz#s`aSuh?>=2k2t4vX`>gxtn>})LBy%m z?Js*ysg@}(1MkHIYBZv6b;BbWrzV|um9ee*2g(@nYW3iwj?>cRqrB$g>c6Oe5z}*_ zL-@A*3|yzSwA$R&E)tuYIMm!GO?jeb2&-*e-6G?11w%+x5_eR?5LY|821TalYKCMg zc-20wwz?Ya8WWlQVq$1g1<%60yHHn+b!A5G%nf|ZKV!_o^=a#?{aqJDp2^L9JbuQw z0@tsVRfoE+jJ%iI`j~&#m_7Q#siC@Q+J?xtxxAxoJNpHJq70xQ)yZb!X!;zFVk|+3c`8l`;+Lr26*VB>QHT_ST&*iVi z{Ux5NUgml=vUrW@Q{?%i+|eP%nd%j;w<8a&p*@wIPg*njgmJ!l4UlVITO)XiypXhZ zbQtI@H@bd^e7>gLQFMX0PWfDWwYtdlcjT`%Q%@ls#60DQwiARzt)ozDV~0f@#C)Ye z+g&Yjb&Z;_wrCiyoW7nn#<*F%%hfL`b#41_A+SF4UNLT0A8?J1+O+oT@Z?2!0WU&5 zP_1xfM(tc1_>2z>jJUVj!Rq$tl&Di{vz`rI(rp}l2ZFIG*A-EB);2xkT-I$G{lFNm zzUaC!>f_q6XG1_vGWv-zQr!u1g45~9=d3HLLZwkVTFu7RMft2tdERkFx<$FT{#ErI z*TYef>i?c2uO@9B{mOV-{lN8PRQfv2a~Xi0j(%r+tbPWN&&BIZU*HAABITskSUu)? zE2`Kr{6*n4IDnBczE!_-eH?XuUAyzkYsT%kDeYwSXV=$J_tq87O1`cG6cff&^-tFq zg7J03FM6-5b}0X8|5g7FZ9=ljLyxd-sQy!e?rseP?LhL+OB?CEp(;_<*;&-sp)n*z zUi}EEQ(DS1V_MfZqlu*Kyup!g30*L9UV_9&4cPWFJ}oxAx!(4L-5(8{duu^ z5eP0St$G}50?|uJm-34AkS?N73GH#I2}iFcJ;}SG7j>!bCfG9FYGTk^NZ<3m>L>FYIPvYSb}Mtzv`faGmvbn$xA zm&m@P`cXVHx#kFZZuFt`!?N-|B2dWp+;l#HUL1WMcw#5};Cpz1%+5t;&?}?wt#5tF zzh&Hm3+_p)xrE*v{dxWSm*cmLdvS!Gj2bn%BHDgK;46NAe%E~>Gppt%dUtg2hOAdZ z{knalVN8CF4t*qg-iD@EoZC9!-Dc+2JVu|3Ub|uJ)zEF-fzc>tUX32z9bL1*>ow<& zu0a`GxuIqpeLuQoL)z*oZ0g`g0zqm`)q^ewd`)v z;n6r|am^p}pXk3EOvm{wzeZ(3&xF9l&2olQLHKy#KvD*b(j%y`a&w*$T97+FK46sN zXg#7DdpF-1$px+Be4SB&WAsRBMuNhDHg{}%NC%IqWcJk7c)C%6&lU}AwW=oNte#yp zc((;JS_;zM^xjh)QO@bvTSIW;&FCv=c*DA{0w&g;17J_IVaD5n;WxeaRmYU`dk%vN z_YYUojfl6TK`Bs?GLY@?7rLQX1|^qi~7b$dGF zd`RBA$%pV$yfo&;nhkDmW<1>3{;m+H7J2E+t2Nu)e$M#5@$0+ENANaY2D7uK%*`^! zX;b+7!Xdbd$7OccRJpmtL;?%oVjvMmt4_7gfwM6$7z}wPf1Eq`f?8&*QBOcWY<#7TRaz?2wzvoju;QiS7qtX8`*LTU2Jv#5evww8TVi7?v5U2Ce)U=n*n!$ z7cdj(21eyfT5YAfsd+XrW zjyF=k$Y;*4J?!p`E7=PDhJQ<()78vOu5~bE<92Ti|5o@G{(#rQT)MW!ebLM_TXO*< z-uM95+QX?m}?I-SszqSl;@ax%OAO`W@c|21Y~XL zQ{DyUy4n}+oil~o5I;yCrNg|7%nh}#-E}jMZcF*m0Z7t3z=^K?o`=%1|5sxjrwo?T$CydbWnDD7wOXVpvPEdbLcc<|!-iW+{h0K-*zyXQbH_HbU@ ztD?c59gldgad&$T)y8@3h?~A0`V0RxKls)FQ&CIzsE%_7L{iz;q&GZ4!ivy29{b`V zx0@au{c3!J8|-PWP4+kju%4~I_}`3gaSwZ1YnOOjh+Dt?{jc$F#&@_O5Uu5UbXQ2W zd;R8o*S#No%xte+XHmhS_!#M=KnOyaGI(NnBR)r3;t(^V_-aESLWPMhkTR2h|{Jw3H`9yq}MYnWpF zRDD*y?70QD=wb0W#lrxxr~0CN-E+6L(SsT%Ho{@?+ETJ3d@l6dnSh`*#M=_Jnx z9xXi{(s-X8DStbrq(6CIn6GQ^ddTA$JL>U2qKYBja zKKHl|8tTx0ctE?t{p|T%`^w{PykbYGI`_Yp|No6U@>OX_Zqd zJ6BuEl8|j#D%w&;&WT9cv9`95Z5avcKvYO*Yb{n{&LRj+b2S&ce_s$jbJt*AXf9Iu|po(5ge9a@nKYM?t0 zl#NGDVuMbH?ud#4z%?v)?L4^OhsfVy|Ds5t08X(i;K zp95>MeJ0eiJEr28``ne9uuTr8$;Me`FD$l#=MGy*1}T4oooXdKh+F#Y^|wD@zW!Ou|g(gJ3I` zQ(<#|xKec}X;SNS{z7^rwy@%h`)eQ#2`6D@@E6gSVs}^kbpN)}dT3x$>kR&4`f@DD zOS+GZo>=TM88egbPG5;VR58h87KkM#O>UjZ_n-q;X~hf=kLb8!;bhD#zNg~Mg<}}PC<@b2n41eToG3Ic%(#Y zinBq4%npNQtf;C8@?b<8iWQDDXCvH9!pbTz9y>tn%Is)zHX_XN*wYm&Jq||?`m`;h zFwfOKO91|oSdSCYiw?(3k+@KhW(rnSLGqACFFRa3#dS8t$DD*cU$MsHVs!f9t|^k) z6kl^PwxMFZhYh#$@SiD;b5MSYwcS@L=pH@M(!)_xwbCj6^i*tnd6viH=wM1MYPo@$zFydvI!tqR%aF?Ll!;1nn1JoXak zCTz3&VUKC67L~*}N#;?4%o*5w6(>9vu3A=7?Bwc7L7C}TU4_WQe^olL;YeH|lWLjW zy%jQ##GD-^-i#(6yhq9^O-c5&;raFuXm}6jKt>|{=MSC8Jft30|SWr6_%I}8q@Sc@1sM2i6Fw~BN3cI~|6wkUCN@+cW1+(L! z;%+})s^`@hcIijpsIe2ERozQ?8J_oJ_@#c%EV!Kr1wIvCmgnObO=R5ujZA)sfvqj@Es zGh#iD5na^Y>_q60Q)FHWgaeO$KPBI zG#lx|PBJ?!pzy>;@UhZkQL{^ZP-_(DyVvmQJ=eu9;<0Cs`aqKD^;jCO(UTQBc&uqQ z*B6pPZ@{kS-SFhb{&Vd6>``CHTKZ+|Cf*&-(%3o2gXeJlAT+uL%i#5RN@7Eg=g%4S zgQS{!m0Ng5&$?L3apBoSe`Bh-8Jo>Bc{a!99yiZ1`5V`nuVT0JUIVW~`Elr6T7WUl z+=|WTZJ%ibtEjZO+5pXZ#f|Pl-dE4JvAW~3xwt@0x}v>%HxD`IYpnJ7z}(hA{s#I@ zY&HF_=YO%|%ACLsgbg+v-3NISy=KIDmJ#Qvm#{ZML0p1&uh}hb8Q2QA22nE1UD#v1 z*1AE>BtUyN-@}&k7J4m>+gs+~>WZS!&D~f&59URRt0{|dm7pjLvknWCWUuvc z9c9I?u4oF=tj9`uAznFgFUq=HC1?r|Td_5~rCxjDte=0mItHV*C^VQeyj5PMan^S} z^I5_6El{9G^7eR$;sQ?4=bM6!EOS5h952yJ8Am>$m`?)|p7{~Bo=5XyByKq|G+!H{ z*{XQleF-$rB#IB8a9Mx>q88nRZRBNoJ&aSGNLoN^-JAn`*4@I(^?DiC34|bI7&{mG zy!$$@!0Sugn-ls4>M-^;=*#X7-d?Z&;>MLbxgo>Z+o7+!J9#BuGgf<+6W!F|>^vwC z4|wHXi&saNm$@M^Y&O)=t>cNk0#|30>)q5CHW;V74ZIq!<*N^tJ1s;;u=Al{YUQcC z;(@V`xKJIz-U0p4J;-bD+OWE_ybMfw>;fp@d-GZV)9_8XexW)Nke+{@sH|x7+P(T0 zn2sEyP&*Z0yI=G6b$K0MUEq@OWHZ1xLhapVp3bZ8kKai;7$7vG#UtG|-UF|*tMMmg zi*VrLuK3aYf%n9#adp{jS>5k_r>e=>U$@Ji!sajd+GmTfAf|meO~?Mq<*n_IeQ=UzwW=hKdygP|2^s8 zF7XK3uNb2p%XffI#m=vYaxVq<5XCs{1ilk=0d^cE&wX?Sb@&Ek7OaaeIhum?99a+p2^H=FMZrC}S!Z1EV5wjYK% zY3K1#P(kkV!&Tro*OVxxX|(}i&^)ZQV!)$y75@l*25vEbIrIS5Q#jdEGJ|l`JQL^1 zkA%Etxja0Rpj`U^B2OF;`a(2 zkggPp0vdCT$A`iQc&RW7SxP}E6?mOtsfR6SSxK(Xb=!jrIb6EvylrB^vS>M)NMvQ*f8~8(_EbB_bDJOftWoPQx|wGhq+$ zDp8VeYcl_$d3(>$)wlL6N3y<>$jb;Gc@xd`nYMmlWyRc774;H~yFC zkFVoe)Mdp+Tru?!%#koh9PGzk3%Nqiz}@8^hq)3$#rb}tYatps9jE2*6TsSw62$d> zESg^@A%s@u2aKjcuG8~z!~A}jnvh)C1lX&P8}tI)C;l^7 zE1{tBd%);=2mmDm&)!>@mLRSS4&~@QNll!<|A)5*l(KOT~A0D10PSEQsuS;w^4ITaR3lr4-MCUbgv>V zQE#*x9v{R_6ikCJBrdBeUgElm(rGTnO%cq7`xDcFRaycfj^+{^h+n~%68Ba)1i5BV zy39wxqXL7Ih&5F)L98~PyNYADKfb~|Yvgimq+lm}g!o=Egw|$i`V^G$DV!`w_0!#?m>!^3WG5$zLBoP5daf6h{Ti*1h=yI{q#Cqyx!FQ8>N&kd6W}}`cE^9Xly5Pr2^Q)smOM}Oo=o%bD&;t)C4yw)$ z9nQ9!piSB>0wcVRM5(S1W#!nPLR+-if=Ff)DYx1jYRWM_GhfAR7rcf`wwG3abO_7k z4?#0q*a9oOk5pSt3{&T_pF?kG3k09xFG-!%ewXKMqr5P;;dTjrz<;-zs@uX!x1nAt zI<$KPf8f7KyYjz<;nKdmgx=Eb7mPzpB12?;;jHcUSI|yvv0w^fE*T}u4j=Zp^V)m| zH-4$XqZOHs4IwYUgfT z`JM={!Z8>Ln_@BRaAJW!Vm-M-R*ccw-K`3}Rw4*S^ra5SnlM}r#6~x0H?N976q4<- zkC`3X)VU?6(Qo$oP`r3I)@ zil^FU!J@V&LH;Z^N16(aU(7FY z04k4|5I?;J8Z|oa{8t4)l?#p{oa21}k4?K%^G)#<=x7xPuXucoEDFcf*cBFSkDv;H zj?bzQ2Gxyh9f8`k20;yCdHlf|r=`eU?C;QF+_G^>1POqbi9yA?P(Kv!wT}e#h;{MT zYl@aS?neEi_^34rnh;y#p8_;<>2B0d1pv$m+7P?rf7Se1>R5#Op90{*1YL;Z@$*kb zEh{ZTjVkO~i$IU4j*mE1w9Ii2>X+iXc33cgI6fu$RMRqbzU>$EAMHoMQ^Yo0?kV#! z6M&bPf8xFhUL)?tm!E8Ttxr@qbS>eHy!;En7DRKUO&2+c9Cz>EIRI z{gAPYargTs zVbgVP!U*re1pDcaD{094lPojv?!uMchZClsfkxAcjgu|2@K7PnTa*xRh8}GyHac3I z@!rBjZ)F1cj3SzL*f_;98}BEqa?~WGetr<$dYC_z0U8&C8@z8Pl$>!{g(=~OPIuLz zgiPaaq-_v=ZdT15=4)CjS_KwL%Q9g>(^YmPAilfMOlPY|B*j!m31 z>qwZd4;asC{JKYpE%OUq4sQ>%|v zA7=v-9zG_j-Fr`>eeVBa9m`O_%B9;R?D9UIIR9)^Txl5!7;|(Ca9ycRj5zCn|5^r_ z&j3CPVZV2ObHZ6S;o1|%`IZoTw$S9=oS1vo9A`RVTwn>qZx_DyzMELHfBtG*xdv?X zbR6M3@BYN|XVbu)CHPMO5|b6bR`r|6B@lO ziFkz&hZIo)lytmFXhd45_;bQUq>@4g_cxlz=wn%fuNTsg&6HfF8E+C9eJv^Y%fd~_ zyI`pbCD6o1KMM`tBxE5UQ5uwVf=R5|BwVMvD%_5IOVKH11YD&iK)GIbL%0(;LV2$o zB4{f$fyxcKxtx8-vDAqw7b2$0-Z*xXt{Hh4IgPqV6+@I%QGzTPcwmh{E~G9~6%$=0 z6qJRoBL)bOzSLAeZZDOf&`PGRS13aUEi6!dCyq)W1&}Se2f|w9YARplM`B6s!LTgd zfbcvrg{o0ylZK`C5LmYEiBN;gq#9NABv!RO6qbuOp_@zC)c2|(lH>K0VT^6~m%^LK z1JsFiE@X_1AI`|bzZKf1m4g@+k*t=nF))tKDl`BNUUXd<87XH+z;@`~3qSp=r>29T zl|)X7v=rb!3!fv~se9`j;$3SfQOf7{zX{FA9;&o1D!#M^wN$xF_XFH`pHZ*Z6~#NA zLM>Ak=|+X$ksqi}>)PT=PoV%J10xToz=uCPv{ni z!hJYN@6Qb-YR_s`D^Kbci&prwUw7=EOhGExSQt;|DT?zs31VeLidtd6^OBE;i%355 zq-E-2iuP&{UMa*sEMR4;k|^qW3QGy$-WB2fL^Pk~q+GR`Vp19jmP*h*yUFKnQn?yR zrKyZWiv%AmV);BuYEaXuCY6z7sm6zk@_Z~w2KCUNRdpJ&QjUK*tI+3Ll2tuGZLQbCd18|K96J$qRu?BK_gTCq`g~auQq88o zlsddf>=U>qp&zsknE*MiRm+%`! zw?XIB^b62rTD@_th1$7U)a@f!<8vV`SzB*p;Tm-d7zUqHYoafdB_l7gQ(?`zEYYCP zr8OBB^vUXr>~*lKx?IrewQyPv4r_3&a$41A9H}hOR*L+2`e& z&I@HJ$V=>WSi5eQXx~3RU)H?2pifa>VsC)m)a@1h_4#kjxO%6x$jj`Fu-m$WqKUpU zl0EB*Yt@(8n_zczC8FuRi<6`4%hn>VurpvFgCUyZ8&s{obsW>8$4f;azB$P+>bq!? zMoOk-R3R5tX75aXy5a+ExY3>od!RcliuOH{47uo+%4)K2ff;oQ5#F~l8Feu`b-2mS zf(_`-i4F@D$>fVs@aSeumhzGAf@rO8V{+ETp|qA}{#M3g{3X!_-`mM07yqPRy}c^i z@&vCDF?@TIbBm(ZmA0UAluvcdqHN!1$=5Fyt#iDJ%2i&%{0n3I+LDbI>(@#CcPH2K z9N#A5`u<4%e9^qlbk(@c@)8eP$9*THOmBdu(XJV{TVCV4M5Vs7Qvw?3X{Kw&Jj+{r zw}|HpOCf_HlGbWuTPz?lDe(2KX4OAg){DkMC8@}9> ze;U58AH4zD$@q$YExH3n)j5}f)46RBF2jzuhQt%Hagx$9aK8>IEr2TeAlL4ZrfOT z8+AxIv1gjt%`b9oXX*EiqqiZ&jLC#qVlO}3+RhyBO{`8-#4Sex=wkCzFFtoUZIc#M zYcV>a=ZXXUHm$t}-b9W+SfFmFXTCVpZ`;~8m-U;}ci2Z@GkT1Nm-_8l`y%>(n;g4P zN0l>sJj5}6$Jfrk5|vTfg(_8o;HjACSG_jkN>PU6UDPq9OAk_<Xo*P(z~eR z$~is$;xxY-Yj<7wli_#|Ri>QR69iPAp0&~|QJYKep-w30_k@VE{hqD8ex+!$<9!sc z8TDX*9`Ir9(<^P8OYfsjDi`%E6&LyaUi<6HpUp_*%nF7(VWs$xpCfIKCYa9chVU4k zgjn%$KUbQee+zxM+s=bQfl-+6hoDh3^>mij&WFK!h+?VV;lO{&hrslr5hxJ=>2}(0 z744YDje*l?gi7#k5ufv;(#~tr7+Rf1r1b513Yo!8r*&$|7|0&B80O!TDsJ)HMth^t zGg|8tDlGwobaA`i9@<}x1JhMcsj@5q5$1h&{Z7ypHO4R{dWyt?A~40hesX|sEoQp* zQlyq(AZ0xAyGToK>|#oKDbo-w)d7M(C~1K1e@xIbomp zKfkf5^KUwCK|Wxgh9RaM6i@JXc@; zEZl(RoHDWJtQhMrNcCw>%hC>L)JkekojAe&Y%0E4kH9?QpJ%KgTo50qXh_Xy)@P|7 zu`j?@jv$spyTC6&-3{ygnz>h zSVm8m_?Z9Pb*Pr??BOT&D=>Obx0vsbSVw88&t{qI8W^)jFP8d;tz)-*%pNw`8(~{| z9)QT@>UI1UzZ}+6dlM|HXFz=3KV_Y!B|B&Msl6GN-Sb4O@y}dmY>BE)dS+~~1x}y(wC^4fXX*I_P77& zx__>I-!}RZa*MH_Ft&1hfMeR6Yr)&OuOPP>2MM5HJis+A^xDuL(I(?<%OS$#%2@%h zH1ajYcG_!Wr{yrgsnRtdBrW%vdAsSg@s8yP=)d;}h)pZM2F;_rF?Lx>2`-h$fHi3i z*XVhsH^#e`;{*_%2w<6$jJ<*jV0W;Q5!Nla_ctH=9wLP(w zWdRG;&nerUKl%=0WGKK*UJ!s>pM1S3pF0fcXQ&9I%6-)VnDqtMzvqt*Lk1Y<2#J+v z190ob*MoO(-$NcT&J)&DUI?JAZ@HeoWAweoVSIhhS}+b|t~Xw<-@*D|A7qq_ORH=R z*s=ceb@L9>2jgSQCBnwa+X2YJ$v4~za33{KlvjE-S9S*o*8AK@E6{#4&e_;V$gJ!O zII}+PhOhwhiT{+*OxRlaIN-{9=8b`Z)=&09hpRogm5-d-*YCaIQ0V%ZGGw_%V1om% zcYV!`m_o^C%5%o@yE`gB20UAT9bl&&zo1?y+j_V_3;nSE>5br+9bX_X86AW@mA`>F z`p=E;g`-~}uNb!o2P(%0I;PKQ3*O273VF@wBpj}s8t9rH+Lpg_^eg0zGUUk7%A1p+ z>Et%WPTDskNQM#0DqRCZ(g$~&c51(A-YS8Er_w!eRr;|uH!cnc#7dyNRQ~o!NLRHb zadBR`7FbV@xbp0uwDgX)Vy^25#cI(Lq?I9oIq5Ijy10@Nij5(;BCm`JOnLIZ1;gCo z5xWicfN-LGW#HlT>Fv;6wC~1umWKpIB`#2u9sp!q(|6;rrJr!FGBHq@PHtE1qWv(w zw>%=$SJDET({tO+yG%cfA1sdvmn%1ccy)O@bhl((?MJ1lr?GNNAT>wPp0vC5AO0uC zGs0EC<$j%huU)ts^OOIX@tkm@vM}&ly0v{^ck5677sgA%O#rVOyJ2F7OA+RO{I87H zggcc-0_%QzbP$Ww|6_lHz3sVIS?RHG!?KRzA~hLiSDJgYmBK*Z4XGV%MWv&t5v8@K zx6(%$zM-Jwd(r49^w=sN1fJcHF{@%P=3o9V#t*`W z%EKN-7;<%~_ zOQvn4w1)359fKaL8mFI7<+NnM#)w-*`yI!k0g{VITw|Y#*hm4etkJgKan=d?zNPb) zpf_gS8rrWN+c;h|Nk60N3VFrGl3OkZFyjOcOh=+~6=F%^M%AsP1Fho(6PQzpbE^VD z$??5g!ULG`f{Dy&M8K(8lDE-%Yv4dDpuaF@5CNBE$$^a%0Wk&RAehXYMf9p#y@a z8=u~8J5&lP-c^hAORI{O{NDKMb{WY&$r)6d6G7MglH>51oqolv$s;r2p8A-oV@u|4 zLUm>r4^JKejm>&&6@Li=tQYEwS&kzRI9yMx3MdKQlneF^CdYnfD}tB+8Yp8om3Pul zVy85^sE~SU)&8o4O{&hM!>wQ=!Sp4jRKZ))H+6QF9Y%t(Sh&AFwW?*wwoPw3^@r6{ zIdkEG`t+*nEk&FDb{1z&cH;adEFo^Hy0fHo)0{g=2QxqiCmi%=R_T^Znig<}USe|U zceSF4pf7TXaufNEqJ#!o7_A|op~SSLc~dUfdzhy6FR+FYw^zMha(7et9rN=?@FQ+a z43Sf1Tk>#|>Q2&;RH8E%gsIwv-RnO#jz(E^h)%2jn8PQ#3N0GBQp72=x zv8p*iff*THG0z3Fs9x4MVtLiVprsjmyBtbgAylXpOXOFOuuA7sT4C*q{>` zi|)o8lekckRtiy7MGBHj516&O&gMv1XOYAOFyK>O*OueD1CTRKQgZG?HAm?*} z;ViweN);5px%M9MgnB+F1fCo{SamUI{pOB)#V1@BP(!WR#HUrwK{=aW-0M0aSwIc5 z<`O~D1VHWo-g78-b)$w`w-et~-3>aidC~ota)}!iV`USqRlPwIQZV<6${iP?BUJhN z;i`e43!78#x0RPJL<3H9z^AHbL9Lq$?td>IUFaNT6`)vYuY>MxF24^wNn6w(W#tmT zR}BX}+T3uTe$uq4f2nmh@qbm{f-IYBBc&&CiyHw)UH`kP`~BC=*82k|TNev}t&{j) z)qg=_=@YwMDlqN>;9?|>mrO#FM2F6%r5XDr*UGx?~}0DZRn{e-)0N=or;e{VWL-727BR79*~wb1ZX&?;`O*rO-9q z*}P%Tky!YaSYUQRG3W+hL!o&!?%fXzB9eXKsO3^~_W-ZeOR$=GQh%yy8EPMWoYskt zgmSQOp59XuhdN0wxw48cfl_f+J`pYVqX&sK;~-SaA)*M@XRI-QbZ2 z^hbkMA)p~fj?~Kak&>OL5&C=WkU)zNfPRp$*_C@xcaHy0`$ypDjZRRV)~}Szai7eX zqYD^+jG>Rtw9UKET$)+vPQZY4#f)Dt9CC^RFwwO`nZ z6i}IU#CQPVUdbrYxri`6f+VJzNR`x~5*aEWLA3(;hzX=0$tBb##=XzSMMz&xGW?=G zRnmgm#(1NXVR^pP6l()9UDA%)!}zOn5WD(O*IF+VGbDFWCm4%*V#E?KFj_T4Fv+3h zjAcE=Vpo4^s^7Q2mVaJ!zHNfX4Nz8${-;FR0gydp*KROrRj0*-qRo`HA|*u!6UD zYoK5Q^Co}?|A!jOoTzuH!Ypxaoe5qWl8NXUOiyrHtCw&#!S8_5*y-rS%xHaC6*7pE z0l%w1CYgf{WM=5~Rq7zlX83)5xnvP~IrE_2NrFUi=&EJonX^~#$1=s>7v!Rx8B84! zNHJhv+oI2xjG~;GOg&K|L8DpBL4A{ii+0Xr8i;a9BpMg@RsT^kj2_8^KhU3+M5B)| zA-#T5R`AFcxKXc=;L&xde!X<5DY&0y?I)fCfm$V#+^dk%Li)3;kMut#u0>2FJl(Ni<$8=_>T;pjZo z8~t5L8G1oxgrP`=IW>{ZG!wzz60P-1Fx1Ofm=QMIs@F@T=&($-;iGIAGs1zt(?5{Z zqE~0~4SsS~#7I8;y?#J)9-We@F=We!BSv<>KkA=IH0aDsqoH2TiX16`f7TC4uA>Vx z?S_x?;mDCf_*eZ`$ZhoD%;|ky-qBHwJ5}HGZ$Sl?Ak(KWt%jR|*{Pevlw)8!|Kc^fl_GoZaxB`p=RV=$n}b`CujF6!znOpg98SBg zpzeoXi)pXiH$K>L%hL|zY1WF7{qQZT9HgbS5Ad)DNvB6^wjF?v@0}=}9=v!<^nL0v*OUHxaGp?(sB~~ZWd}&B<&XyMsy3R;eQIA-slNL!mSMS>L>%pHh zxW*FO!kN7V2UZ3j-ZK3m6!Z%AAGOXRL8Z80!c*^uX|>vz#!{7YFG89Se0B@|p{y1c z+Xz~1dVQs9gB!PGJshgl#x@>T&Fu{YD78CVjy-fci;HV4Q@Qq{rCWmgw=BC)Iold1 zIKf;%3X^UN{`UJe$aUjraphJwQiOCz@TV;=KoyyKDg2~rQSVY|QSk3AzaIWM>xe~H zsN8#3N{fT1vgRA36s1@+PvzMgD=iCFjs_d^6{A>ZJ`+m9Nd>`3RGQuEVl8ZVi-3ffP42QNNa;v1Gj>ol7(+XTalzBsXBNKs{z;-SnX<&%C|QK@Gsi) z?iz(k3_&1f`jgg4uLkF_GM@D))dWr@Jg_%i+7Y~uHLl-Dg#>pV>+VSz(!0SYSd02& zR1$FP!4KYMfOEN;713X$awMUps^H##m-Yv%S;_rPDlW;nni)#Ukv;<^iGu#`s!@`& zj2RC4pWX&*S>pcSIxg8+&Ws@KkbVd@v0D1`>qg1WHOwdyS2_|r%o^-ZqThLNtH!#N zR3se@{=u4WHPvb38&9d0_wJL92^pU?Wx(wmE}`+X>aW`&>7#~ol!mZ zDv?eLS(vqKp!l3?BDK~ULpmm%9payrKF|fu3F=vE9I0HoFl1@g-T?=-D}}1CVo7`{ zEQAEs?UCwI3RZRMI(VVu&(}{77-0 zwx<8Qbq%Q=oL9|RS=6WJwQCwLs8V__NjHXk__z3x%LPobpq@!1HA*u>9%iW?C0(GE zWnYA^>ur(dhP=$`1kC`*6ix$teeZQ?LCBY^H;;xS!W8NyCb_3wS`_j->(`?{7aZ53 zFRM28%F_>pIBuOYC=A-TcH}ZVqqj?XEM$YsZ;)PZTHAlcx*5<dm@``v;vaB2zg{a8|ETstSqUT02O*s7~cH!?*Sh zN*h8pZ0#H@yNF!JX@TeTK9#nFY}@)~P=8Uqj&l{ht@nkrBV^y!agUuEkZGK2*8a#h z(z_uiwk~=c(;!Kswp!UFt8`y)h#5hR`K0&KMz zA%{XXyZLP!ZWvxaas$2-G||5axxbbF*zXc6eWVS(tM|L~eaPdjn#b9fhSNvd;YGdw z1B*u6R^#LPORNne9q_%qf26-cer~lt{&;D4!^lne{@yXw<3cB8pWg|+OxxIh%X)z1 zQ0*8xJ3HVB{jzCe|84j$-lXcV2_D&TPlT5-n*^QA!@Vox=Y$4kXFSngR&V0mfgkCe zUcD%EdG^64PFIi_oGy52FQghC8lPSJgm^`r!MO|EJoo4Ohpx}=cv5`Dbu;yzwG7yM zLPB%0Up(o$BH2v6Z!IS+s$LqpC;RUc2aPM8+HI{Mc~-}Up2$8nD^gQRM{8C5URX6T zv^qP&RHSiapmi!?FS0r*^g?#3sZHZ5oYun>ll-gKg|=oFn7(U98P0kpxY1W{4%KFh zO~H*^CTH$gX>V|KR_NnwjVZfvm^so5m-U8MvqNp!MpJzwE3@&YR!)kl=7#>r-j)2M zQJdM=r#c0o_xnS~=S+F()`Z*A_&{~0cUAS#Q0E+4$9VUgxTnG< z3`<~SDoKRu%Fv)3=F@?uR+gZjSx1VmJ_SINB~M+NFA)2?R8R_bGG11Y__J#{8bs=Gpu=Xm0RTe#WICZ?vBvriXVnS*+k-7=g#VuCk;q6}kbT@K}0muFH=|5Ix- zDZ9G%j3y`R*-(o%r}3HUYVWq{SD|-ujy-d`ipy;rQnmJSs^5k7=bV3*c2%3(_*`|P zx3Kza=-V9KGuc(#Hs>R2+ey2tcTM|}^X8fUs(Raq!(az#Z}q>S|K|LC=5Wn*JN1?I z7U^L1gs^G3i-uyZNw!m8TRTZ5)ziWj<}Mp5zUG=oePiwF1)aoUzPYJGZP!Zk&~H`u zK=9rz?8L(zL*K8B<~f_0-6USMR~Rl=JQUo@WjkA#I+Cc`H;k6sGL+vs%67Ig^(09( zI*gS&IMme2$Lp-ch-j_MKvx= zlpFAze%+McKWyzMovTg^Q|6MNE3VUa^uM=0BGp&Z!kTk)pPR3ncJzO+J|^Y9cjNv%((bHaw@_FctKFvOXhStIi9vKXN}m zH{UQ7_J6UyB;Be$5H?}k^cT=J+Rpy3*4HG^*BLf@Tfhr?n`vkNH|tvfSmK4jwvk^b z+GyNH$zz7fXV_Rov& z?T+(5|Ac?-eO28Vc4QmmrC$fDXykuzJ8%x_fLx1u8FlA}=*p;SgmgQm51^moUJ5%f zdj!9jKS=+}9Spm)E#sxW19<@d+xidbbM^DE_HBD#I^1;KOZ{X0AIV;A3G;Ye^)l*a z>0b1|s$ac7sspi4wza&>zd5?s`7iSiX|#GIYES8M-N{|fuL&HnP^ z&Eb6`|G~!?#>mEnPudQ7<#&s%i((pao|GskQ zbSDzP9_LP<|V&r>f)9< zJ7>blptdy}lUMNOd)H{GvrDEoXrMd`jEmwo!FRdGoM&hHkk`mAgwyg`-sImMJ?1=AIO{d*5w|iOaMBc z$e?N|oRt=ux-Ia6~o}8d&(W`E?;iSwRvx~Psx^JFgwOtoODPb#~YDosFlTGPO?2MM4ej5K_aN+WnmLAD;#&faTr=y4TO{cLN=8W{uA%dWAsl^+(P!CGumvurEI z#BQQQT8#~aF1k27nn)BcT+a?;~36F ztBaoN!vdo5GDl<0?j4qngdnm( zHZp@>|=knB39kYl%g>>aM=?0dY83?5L-Va{|L)ZqB?ba-8!;jOG2 zLuEj1NrqOL0DPEill5WFaN=x218quf1Y(Eby=)M3iQ`$;X^_aNk+uTzXW4U1J7=$r zIJac4V^ro&@;BLAjFuy|1^01loR?1vf#eKzm8}HWb{9r0&ENaZ;h}3S zHO6+743Wbk_BB<#i+WgEOT9)pMxHJI5tN*-d6)fgxOOBKQD&GYM@3`;r*Hkk;im35 z+X=Fp924<-`nz{S54C5V|B_CU7t5DN9LS$I>|(?y1Xu*+gr__%;$*((Fwv-1aBv8L z0WK#;oXU?LE;AyP96Ul~@R27+T*}WF)*IDI4gpbV2#}{o+yv}vr+%c0Lqtfx2Avtv zmtQ+f>{qKeBt*3#RGu60GQV@UtRGp&AtU652zf!om;5)w`u@?E*mzqFd6|4q#J~A} zhaCo7&ruU>r^(Us!x154=D&{`C_RTxtgAJ|%F7}a?1*?@G~lR4Q|c53oc!un?;XkS zn+CY*k+XMWqkt+B5wauqy?MZ-?x))7$O-Z@5wSbUfsL7VzCX#PCMU_&5o>ldyr(}h zo$p^`yFgwmzY@U!e*(oL+J*jP+eLDke8wU6j`#0}9%(N$rqo?BY>?lIIJ9HR2e(07 zJ^B^t3h1f2AMpY0{((5CuIJDYjRq$8I;VC-e<&M7UgV@AnhjafC^5+rlJNABXcAGMRrlq_O?7t+N8~djy$bLjWlwNd8Z+uZ!y{%JIX%5Oy;DxC7g?FFu_acA8(!&`ZGq@d8}OWF%!@0<|tRO1>IvNQLq`IQL(jBS(RXV!Q`#qLy9&U=mPY%H#G>~pU1j-u{7|26Hk zwzKhY-PCyTvcRZKJMVoJzQ)`Ulw?keUr-YkmABLSb>MaD9cR76jJ`!R%cAz}9QV!X z4YG@K6fvvMvnDR;^`5G^RRYAfB-)Cn%bz-^&{CR~k*J@?wKx2StKm59K;^qP(+-kCpk2dgXC3)+^%XVlz{ zI>B8u5@VI|iyCFd?1DYm-yQl++FkeqlFb&*>OYqgg4qVLnem|62Is+C(X z^4&Vx>nzI*kI$=l8>QunM}lo!1J$yAX-t02aMUAi!wB7GGW5%Bk?}igzC~HMh7pC0 z*4JNSTN+X!HIs~NL&v-gDWF7Gf81gA4s#viKLH)-jtUGVQo z?^+)S&LCbKE2){j^k?=F@5vd#!%${QFNYRO=VphRus`Fs6Z@HP*F;P zSQ86M@1dlg06|c35C~#HumljLDN#^qDo9B}+6h60#Dar$uwVgFaLkDFs)LXyHi!iU zzxDl?Uo+P=3dz~)?DahN&1DTM2M%^KoTdeKo~Xrntc6a;`vF7u>I8KVFzb9E-?>kH_(w6sVPJ3DLZJ^V6xU*`{o=v$PMWcYPS zJs6oCC~feXzD$*vLU~qu!DDOYy|3ki>;EacoR~`ar}m1+p3E=l4+drb?7iZ=tMg^; z4Ue2mvv1Qr;9u;$>b$4(b?rTmlbH_R!am4e?9Fdaqr9zs;&Gq0?pq^v_=QbtVmif8 zE0`kAqb$S>eXZW(N@m)(_a6+sL)V=TbbhLR?QuV|^xM*p$uEcgbmn#r z*BU*ZWj24?@zL;dsLeU6^LySlN@&LV$5|0(2E3#d^$ zr`FB!T%E=Fo-e=hs_bs!QOfkX1)e@xh2L#IuYX-G1?pJi|M-giFPX%!Lp{%Bt@)Au<;Q$mDy)!JW8yj zxYfmYUd!6|WBALDKAXphXDKV|tY6>DD*dr^DEaNsW9O<)Vjah`n%MXwKjy*PmM4T*2CXMbHgc^E&ix67H| z>04LmIX&B9By9MhgygJM*}J{ZoG*07)jjwtD|>8Y;@7$F8GrX*q{P>? zdlqNg|MdTQ?mgpqe=}u!UCtAj4n2S7f1NwX(Dy4ToI0JSB0CPG8RrIV2F_omn9ls; zc`bY2&*85>25nv>UZv~-=G?>V%Adq<=Su&3;oRDpUT5@tmfiew$2Y@=p7>e|Va*KR{jU`A5ya{WmFx>&APTAF}$j#wXIQ-)T*l|1e| zS4Z?pIi&yf=Z~VH)%}vEoi%kEy)qAt{QC7{%J6D{sdw`0{JoAJ!i{bmDH>jFkfxcaN;$Lb-;TMCC#?X^B<-`Mc4A3tn{69*{W zbwaO@oUz^Y5I)un_ZukMI++)e(=;}1`Gb)bljL3Jv$~637dG4*D<56|Q?qr}V5h$B zvezy!q&yy-yV&cRB}Yz2rHVOP9tDj@tZ8{7M%m{nN&EeN%rH9i)A@Vnr@Gf( z_j5{rFa4c7HuTGRq;t5===Ch8`S*_BhOwbh=U<)Q>%MsP=cs>gjEVfM8S5VfQ%i5{ zmmK5o7r!6=Zuu?w-TAw2ocDyom~nRF$m5jbTy{*HP;cQq^RUx6WL)dG^6^QhD<;)j zc`rEZ$9lURjvLoLu4+7U!oZ|1b=rLI4TrhoM#o{yh!drgyJpts zjt@CpIgT`*Yi4UcFyWSMJ;6KjaPzp-@kX;@bC+pdtH>*%iSu-vXgqs@cG7?))pGti z@4bhO<9f$;Oi)@#r*|!=_x8>^j2UldMxLmgES=f4xIWl>g7cE`h*|5z@+nC+ROfn% zH~(SU|0aJnTK;2;r&qMEA+%GX`iUeCrFnryp@J3!jKiPM_(rxGS)} z#CyRJU$f$ggemFPE{aTfc z9Qk0@XHKq@%$6?i+FXCjJMRc)g54zYH02!W%C2qo54_8c5GKSz&r&&8y1FZ|z6+Sn zg zOthO!o}pYQ_36q3`;Sd7VPY)QJC%#1eqBdFb-y^5Jh5gnd8X1q8qjsTevZ%TT+YPa z$sIG5i={zbC+ipb_~r5@+D+-$d2~rq2(_%<*@u?<^;GZ_)H>bKCA6!uewoka+&dGC zrx0w?ousE0SJjhzQgdHUY@bqP!*m{qpu%|Jlbt&@(Hw)bWx5Pdse<|dpWxB8WfBIMRJ0!PQRK#%w_~cXOxFPxwW&VVr!Du8x%pJwEauXI zC@7*P_&mw2H4m6tHH+ysz@}cVPxa}|yO9V*-f10km8+$RU7hvy;8^BOva=-Plth<% zR+@T=4=qnT39)R&F=b9ET|M%OzwEOs@Aaf7mgv0nwJy85 zUe@36$;mUbu(Tq~PhaPx(&uJgu7%}vf_?f%mxEnj>ficw<_Rpqr=#}i9xj$rCwgURtX{@_zIV(v)jp1MM z-2-6`9k$iAJTmD5bu<5;WjO_AQ!Qr^mZnF!TFUS6 z)dlWTvu9!4h-|5{E19qLl^3w5w$0+X5o4s6yHfcte6JU9r;g6TEF*4`UhUe$|IhbP zLDf``*}P@MSZQll2LHY9KLxE*vuBSaTE`{*Nj=CP@--CnOjXUs%o&VxxzUxyAMyQG z@MY@YY~%9b%`UgP4)e|YEDEi`S>Mrkf6KsaYCeC8AGUDWw1heO6UaZ}n?0jMy=E9(j#&hZN-C{3Jh3A!->5J#}TO%k1e__`CdG z72dOKD||B&xO3nIwUzHkF&64AP1L<>ZFeQTq_*)h!O8vA(ll>mt?lll|50!A^Zce9 zon?jm+`4XfcTy_lFMg39_UJOJg!%e)+C2krs1E>r<#;p#%J1y;+O&Z_>SIXmkdE%O z>YJ}$uiZP)PwnFK{Q{1bSUK9U-L>fh25_{={E(wfRta``cWuVNJ1Xd+{SuBowG!E} zH)!__3{qe4ulZ#h#iagiH}WZOf6_}#<3l{kM94nlGUF7bYKBVuk zu@*S`haan*5wNJrmsv1?>2BaG^^ZB$IwN}##*bJioz^{X#cKcLV?8rei@1KaN0Y2* zc7mJ@{s)hJoncxu;%5s6U)myppMTM@Su;rvTz}gG4%QHuy8pBM*s_@k;C~&)odDCH zfa)(iMxI&YfX>PTnJW$4t^SveZJ()gXazHC(rnst!Dj!@;}3&&l@OSI!Ua?tg6;mD z#{@IO7o(s7aGBS=MzGWW-(z<`??4Dr4$l>Ktrw(&?o>as4SajVlTv`+3bOov9vcBw zDJGbBN(vxKLB9XgBAm6y|7%81OC7oc1SkCG6}ekyFTsQmOQlP?Lj>ASE=4SB)e>%q zE%-jWU#+O}Ut5%BZHd!|Xv+o~Ybb(xf4?H0wVfjw3iAV4M}|P+Pb(5zBaW@)@`@x^ zTBM*UB(CU=b+IEMG`-Trt(z^l?7yq%we=H6G*mnMU>Pk=(B_|AG-hq?gbQPW-;x$D zX!rkSIoHO+i5EsZCk0t0L`|!U+-ntwv!Z63RA*yEdUodq+5y35|GuK9 zHX>&>MO!=ILCX^S@Hc^IvCmomOj0NHqMgM5_8-riWQzrBVmU|>X;VEW1xyDgVz>)R zP3KFm2NeqDti$o#ZL?i4G+T)e{LKY60eBwER^`G4TcOLE>n8*_kQ1ia4!Rg=!$KDT zNebdD0(d31j(9d*3m!qx9^wNKUXyJCUQgGG2103P1uFv*K>6o5&SmE*aZ)&~S}l7AH&2z_Q z6EFx7l`^|!f^Vr(9t*1?a1mR15{sq~!~|U9?E|6R2x8lq6h*rv*cR{*>~y3FTUf)5 zF6?flfD`bH*NjbdHL`{kE}Ocq3ep1ld1|b}l>?eMm$>df1>Asdyf4^6S7YSx1(z+| zw*(&b{2kbijde##+G$#Fu%dYO1f*S#yl(AXn%W#{RR|j?jU!y&s__+P- zfMr#inAZl=ABMCly!7f8Gogro4$!v`WKn&u>|)W>TjN`Re3;X>5|i9?-(|=QoJE00PE4AMUF8l&q9mXs3WGql=`qUP2d;)h1_jH&bKe-@mB2m4Z$Yrqno|BO=>l!D@Mho4JuR(VZ(Pz!w)FY1rtBc*|g&!T-VXIG4(vpRmKp9v{RBOmDzUW;pckdAD1FsaP zfpWx{I{c5z)$ZNG*MawoOXs<*<$x~4rL{XHLC`Z z2WQvtOP3qnS;CRPZ(!vZTx;Al{IAQc?!!WZ@5BD}(5!q*)OOSrMe))DrkzjguFg>Y8T@{;)ZPe5dm{>J4#r~vGPyh?KCTdpUh zrT^#hu)9>~6htWz%+J^y_{h0W+RnDm zdpqz1QXlJsGD<$o?_00mtL-1?q}2<3f(lBk>>SDqyTZW>q^f+|Xu*(JDR0)z&s zrduMUf>MEOSL063P=YX_TP}Wav(s&8-CzDu>d<;acqr(1 ziJ86SM#2H5e&&$uwlF_vD$xEtHu4S-KTBWJPPiQpntO5$p!-J-*nW}bwLcJ+205Rk z+bcG54i0~DdE5P1SQWJPWSafp#*UQkp@Du{7tD$QCrj)dJ=nm*A25JvRu+VuY_d=A z&~vq>fp@gOh2<`JC)({RJlr$Oza|aRQerLxnEo|HmeI`gZvz)5y%f@z+>;~rqaK(n z;&ci7to*hL3Vo z`gr*_;m07G2e%d!dl7Qd$6Q9czqk*td3RF3pbad!%EZkS5{f6#Wq zsizA?K5W2f8!YMb8`cMBocge^&qohTY{PW=!UmtTzD9DUjO+R28Q^d$`;!4;>L z0j*k}ubpDBp}REDf<>ny7Kwb>1zL;&OJCX$9jrRld|L0@QJ|bEo2_2nuqF8JDIPGO z$%V>kvbpNj4N1Y8Q|O{tzm7tsr3`S*4ZDNioa$Iq12AT#mCR1Pq2WOA&?)1hUcZi` zO5iA~y&4V&PdJTnu=6J$Q_herQu{R=3${5;aESHi7M%jZF+Hf^WbmTXfesb^?nUL+ z$&PfWum-O@z1>0Q-&(}%9`CHiFl&N6P9JhG55V!5wgwjfPKtuVfg2nEm|!N>K%mcF z-WVKny44{&00Ycl*-~|MgEBbfwBDgDfOniYTeeIc+wf;_=IIfK(STOcggMD8=vy1^ z1{a;4wU`vhJz+aHc@;gO;Stnw)-28l964bNSX_E?gCKa2UneD zE;a3~ZO2`Q~yLIS3#El@b=6%DQ-yYenBNewZU4m;w* z)Mp!3g(Q}$mncFwWy4N*fQvS`hwLx?vcxi54QMt4g$|_9kb+VxM@KTdT@7An63sEy{REKq^wi|cLbg?Xx8YdG)GkLSB&_vp z`AQk;A0Jm7VprzrR1k(cXZy755A{PJU%8gCo!Y{9=ZLFi9?PFJM-kEg}b$m$*36*zlTRD5Ben@MNUC$t8iN7ia?v$LTOe zkrT?RE>_XVLM5E=>UpAd6jZJO|Un~kDyWU=i z4~NfAdI%nVx}pejUAa?pE}d+#ge)tE{1lTcFK6T1=sXcIR92~W6Kx`2FX!S%=@>B) zdZg-=z%zSPUWNBy@WjM$S(Q2@I)(gm*Cl)^gMFzx+)zzlC;DsSoAM5P4TCJvvZDap zB?_$k0Alhd3{;X%k$DDtin7V0YP z?vRgGSi8EhIE}+m_-1I@bdb+ftOCfAv2i#WuT&?A)MQBo-PQajO`&BQRP-I9=VVpI zPS-w`UZIUKT%qq4b%x%lAW&^0-J8lcC10cO74?(5p{;{Nwl9RC1S&Mq!$n zFOnY6Pm30ZUJEK+>K5Iqd+>+hA-!CL4@D}QmOe?l->giKJyM?$tq9##*#Tr4a*Gm@ z{pxDby3oCq#-+W{9WBbl@oF^NbU4L|U(w?@4|`zb`r&y68+G zKtbHEm8T{5(E<4py7J8SWx7qR*UI-M_tQH?b)i0h>%hjkx0a^^y{23w4yBwCEDMiC zfwWrfFDtBaXQ_iK7%fg1w4ZiXbkFaqEDf3 z&U7rV*-XBvJR}=c{}g=>9XewKPC~~`WsdB(`nPCY*o3o~6?R+5x0Huv<24h+lYwAB zSP{FWMJUwjD*}xSQTik&@kZjJFDxMv-^6d5%ZRQQP)AR5a8Y{7VnAh3d z6_#5Gchd9mQ#96Mr!dM{!HV#$xb;B=1`K1C*exvf?2Q!#TSv%*LK*b);jxi&R==Wc zEAKAxsLV=ZC*ByAd3I#Q=vK^ML}<=y7K#1Cj-SOrFP`@ou}EgEaTJGzRiAZ-k~^lI z$dlPZW|J8vKg(X(7SDr%x@?xlO}r`W`dRME(Rj=~;tAOt%}QWkKRR2r(qo&x%Az`~Ase&G?ho<<VX5a{t`hw(_Yv`oY?&q&IE=gHgwVgY(+8lbW%=3nInilu}!X813)gzJjln9wqO@{b=*gsH7%}&I05+U8C zISBa2cjxr0+Y)(@=a6}7vc0o6!xKtVDP9)7x#|wI7OP@rdJDre=iuN;snQeMl6g=; zltpN2#5o(oGpk03qsbU8Q7WTq_+rGVsA?98#NldfWyy2~e38S?Rau|h!7*xwWq77W zDyD}^s^}yIhoc)t@hr`&yy$QhwDS5mdYx8oh-6$8Zw+s+DuG5FyGPq-h-N6o^J~>r zC@D6jqelsCIL#ID?(jEN9Z+>6KP!Le4b3+3{_xLLZ%Mk8)@S8VQ)Ao|=Y)^1o&+Vd zj_Ms3@mn=_#D(G0s~y&a??C_7ZhRZhxF;?Sx2yJCQ?LW~oY`#ngV7^7ga|8gg_#}Q6 z-dZW6>y-*IgH=p-{IrWPg;lF?f$>=TggWl6C@T9GtWCgxux}g<+qdb7$Htp5ev@y zLQ{qC3i8>f{V)=2gxmSJb?v*WUNP?)3K>=sy9kf-ht`?z!M$ewCEM{}resM(==rL3 z9(#Du|B&%CSjp0enDedcviD%#5bw!OXy!_Y5h>^O>)Q74pok$W(b!2gMr58JSvR@| z^B?hn?38Aa#6RNrdE9yrs7MeW%7F6@oYLy^?(4JDFnvU*H$YCD86iKdN;UI)dT=9bzY6Tdv-d;KvWxcgoa4U0HDfpSEX|ewwh!yBV1A)u^!^= zSmu9*VGUlYp-MLF@UJOxulO$zvgtC2qD!O^j2hHE7MQC_olLHYlw6G1Hu;`=MTYyk z@}6Xc=4{60h+Q?WArp?iD^Jol0hP5aBD-eH-FzSJJ@aqFMMk{D$#Jc>A1vt~mEB(IfAW-2vF5_NTtyT}`1R$BMF` z)^}s^0mA2Wi05i}l3A4HwecID96&#(zro+vlt}C;UbVR!Eg?9Y{vZA!W7!l(N_cJU z#(;xWUzmM{4#xjyTw@{9T5u3I#C&Ub3=mopP{8;TQ4-%I4N&8WKBiNDfhc*P3}+Li{lO6aJOvj-(L6 zg$|zKS?Krl&&g{&?@5X&$Me>CW@ll3q|504F&;|FAP3mgCClPK><|A|^H@?vSzDLp zIhfU&c5BGc&*+lyDFJmQo{rh<5$&+Sz|cx$6r`@nv)ASJh!VndnrD)WV9f6DwDVc> zv;1rFAmbm&70S^5SDsI@(Vywx@EE5ia=nfVxi8EZ@t15f z;j`oo~~(tMYEq4d|Oy`H2qerq93#rP$W zP5A=hsotE9-^$;z-1VUhj9=|GEQV-q%Nlk^=$99!@Tj(bbmX?QaTUfMEkr=hq)z#69|*FGo%jGp!!m8 zw<8=g(?kMBYbzyCBO%t4dcB^>qCmkI-CnVg^KEv`*3pYF)iA1YHQ`85jaq zOOUdt7wY37wSt;wOyxL)ER)7k+vKrMiD?=E2WKjW z`mDa$CpFJF$z(~G53chxXuzm_6nPvAlNG^UyHU!eeyjfiIS!+RX*yw{)?1oOwcuO( z+6_ic*3B?3V){#qs8~o?B;@NS>t-4kGr@ucEa3=W5u_t@)<#Dr@IN5?u+z6MUq3}> zW5l&kr1exkKF`;#fD9=G6b~6_w+%GD*cT~i#Z+K9uFSEtCP*0E@hvVOOwE`@aMQA- zTj%ZIzw~W~bO17&<9RVw`X@CLcK@RVm}#UrC~Wqne^HO~aef}Kwt{F}0fS z?w1WqcoGgJYLlf}s+`aEYb)eglIEdnw5igZ7u4%~uHR@O#)>o_g=M$&Kk6g?8oQN8 zxmH*^jyn_eHd3Df3Oeuw>PiNbPA}f?R*gB7$|s5<-Z!RO&>d2;$xRimG7!X(Tf%rEauX z!43a{BAgB4WDH|gNlCD&eCgj_RAqxWqnmHlNH@|l1tb2WMHpKWEETnUsXwj#FgAe1 zaIg4TfSet7j)9Ouj?Sufd9cU^jbHY6|x^d2WqFI&#p%)BYhp^X<#3dF)@s$vCaEAy`O zC~cM4ciL@Y}Xa* zIeVFBte(>9gqPpvmEi0UcVjwmzjd_JC!>OjNeAq(8#w!zgHrmxmxXWT|0~hM*1))* z`APbU)();`$CGS(-A3a<=CIU2(+Qh`5>D#vP4v1l=LU4Zzj2$&YYMJ45C`uL(Z^gmO0A zBO8mDpqry34NV||(>v$_jK`UCWGmsy_i3=`G<&fQE_9i2e@oA3_z;{ic5ksN2tBD? zAoHf@HDE&QO36!9!RTr2Vp%Y~tbq^`TiO9qCKPT(WmLMbfgDm(N_JF{(F$!lK9YW^ zVS9+KwAHZ!?kkzgWt-`@8wx|PW$sQDKv!a}mLO^? zno+=nt2Mfv$dg=9j>98#V*`^ctDw7y;>hiA3x)vTp1Fj39`3lfaHWMIkdi~BOqwi5 zNDmQJl1Xqug+(|h6OqZ``sp$`qhf@h`ZT>!8zpO|CyJ_W@GID^I+pRcuXof?@QY6~g4B)CK^cs<8XhEgk z-WBd&w8=6pT`pq7?F!EgdnoQQGgbBv{hDZBsHt+q4ZDeRmAOauhMwRTXzqB14c8vV zYs_?+fvyuZg(jTQFVn>uuQLzGKGXX|PeVm#*voZr|G>vkov}#F%PTrdSeX$|XxH*&vlz?8 z@nKKSqAN4D5$c)Kpru&#baUSRX8F7FPuz)EPzogwhtf761_7| z=BSwf!C{;euK_&G2ocNSXqhnSmWbiSPf^7onK%S^t&_1CSHx-Ig8&;br4atsUX;Z% z?u$#|4U7Zm41%6{GBBB;6Ys2#s_s~$0+@vMiflLIjkp644P?M8pztarJHQwc8^e3y zMMt#@{a1Tab{Jk`FcEg=$?H^5h1cGd9Rt`5AtLsC$2!#>^o{nu>=a|Mgd9NOlI^7Pdp7xOIKSUQR!=Q&z|Dfk__=gFTjGX3(l-Vg^Mr>u>LxD(HtAjm$Kw zMn;UJ6=rbE231S!d**Xl3nN+50|T|5`DPyh=4V+-Xu9MBjLZ5B*wnKhwg1X)!7z-W z*wvCZs$kO9zL7lu%Mk(Q*p7{=gXj=6B)S+v7)onu$sQ^$3L~d1Wc4MR%CO2GQY__GG0mUQ7T|e!zNA~)sD!% zGAQp~P}=LNASZ7eV~)v28KaUh3WA;DO*)K@(~UhfYF%Wx0S`@I&(40I7QKw1JL1>0Ab zZ=8a_z`#VLA0|cfJA73I=v3WobOm#pv;!))WIq*T;dJxRb`;io!^!m*5c z!zqk`02OXt;W5rkWHB1VJPk)`#RI;7MiIeA=Y&Qu>!iN4VgUh;Ljs(BD2sVP8c%y7 zKm#(46K3m{p>c3f<uQAPN55RKIqE+$+B5=QUe#HrW^+ZT$#+d zvNiBj9SO#Q#}9#LCiB`o8eIW99#bg+6k2E%b1m2y1{;harZR$yj*1GHL9$YKcyY+4 zaspT&P?Q-75293&k!-3UEY(G!&CEoZnywIW;F&>yM+Dl++%Ge><^8@Bs)GAQ-DdO_ zGf#$L*on!2h(v*ah(2K2IXE#8aVy+Oap0?ob~5?0Kt_ewJ-h%qOy1Mu4O$0;w^&Bm2-nX0Pl9qdp9*CMADHd=qx4)22m`Z10y0j6`6~! zVP;52s8|8VA1Oj@5Ip?0R8lStE95}*0-cAfNB1$!F|TPRAt6|I3Uxq&&{C$Gj6*jy z5Xic6lz>FS&m>i5gl8ZjOm`Mt1%E*`Qz7FpOkx6jUr~2tKl%m!a7Knnf*v)iM}6Rb zZG||ZY?vgBG?1)@AQk8`Bmvbkb?{#yQj~^>&x3`cOThlN%!u zWyKccF3Lmfi(3YeTb;pXf;IXZc7Z_9ABs65^MOSjeqeZjGEM(?vo&l9^V7VCBnL$eVjn zr1?=FTJNDH$O5@9tGJO6ozYB4&~>1d$Z~l+>k0f3GFk|Ux~FI@;w8^zStrT3;JS}pt$rx+8PB@_ZguX`Z%S$8OE^y*Ze-gNGqh~~($(tinFBs!Y zZGD7^n3svTS-7sb&ztA6uN&W#WBKpm`o5mbuoP0dXywNJk@nXkQ*)4Rt z`2@L@)$~S(sPK#EmWCAGZ&dL`!q$vCgaRE#-tN4#F)ph8V%1jUuCY+} z(RMy-Lt{_(r;8(7v43%nAv5KRSs{&;QKU=Ucx*d|2S0IFaFtw+%D6NVkG;n^fy|Y! zW+gZFM5!)uw_)#dO5m5ifpxI)Yn18I$TsW)4(O`oeypO#S<$3s?jP8PoKnP5o@45$ zY+Mzc+HCy8^pH@lbCJ_o5?F94nmGxk4g$CY}1H@{&m1s6ZD0o|PZZi16VU`Rh|R zp+OfcFJ(C^=xl|OlWbBG#JVtf6>BY+SO%5GWRr#<)luQ&V{`oks9D$?oz^Hrm~t5l zQ8cj=RCxLgGvL1VLyA$rWJR=CF$XPZBH|D2%IC4o^!qM*kpE^|^% zdctKL{^)b~knW7>yR1*u{bRg>?3TY`wJS_gxk7Y zdHwmKauXGLCUijN)KdM*QbXY4_m1&)tQX%bYG@ljNEpiBd(^6oT>VX4(k4q7w~sqcniZd7F*QBO}IP#3ge(X$Yql}9gHZ*{xF$u~`QweDHcNRB#lan;t;JH~v|6j$3GmqvP&=D+WB6sV@kXZI{`+!A&7B5$kRU2>snntX20>PA*#7p!KGyRC&4 zmMQaCYa4e(y}EdBYsFpnqZL*ucJKh+ANBd-+pW60gg>`VcU{oq*?1`G_eHaK%fAT6 zGG@3g>hWzXjGBJQAwK*s^jOAB*Tp@7jU`bFF8Rh6|3xUuuy%FqA;U9!`K5SZETTmj zHm=S+5skG`UYBy?E!zpa3|m(`>tC!WD*RGyd_a2@Jk{S3Sgb}xRP3c2@dfR;FO9{U-@~0i=Db@0@~8Hw;!F110`66v&@lw7 zhTnMhB!CdA`jY!LlJ%DpSX>Gb4u?g7l1ub$ihG=5mA;R}dKdb4RLdpKwqD=n;);1G zYgxM*|BbqRsSu72_mYbFDeGB#8~dZWFNwDy_ghOU>{2$c_BVcvdUfgEwu<}iCspId zdGv7Mfcbdo^|shuWhapZ?>t$D8b_l>FO6+8e}FrMEPUq;9xC%_%VzsO0v=Ss5%kWN zRoG}5ZP)Dihh59K()Q84WlcOuRW#JS{gqTIF1+7fy^e&urzHvPa zA0Phcd#EqdIlrT@K8*l(Lb9<5=I|k&XQKh zxAfd@%#WVhg4^x^ly}lfd3;Yhc)8}axNpyXj5$YIC3nR=Y%Gh$x3IRW9&^uOSEp=; zL8>}>E^p8FK{$X(MfyZmS0nkRZwqg`-4h-ymza{w(!hN`twp>Y0UoI7<)W0Hp2nu= z%`JDf7e66XXRLYWgXkMo(Wxyjw^w-as!40*J9~N?|BTLT8QDJi1aqFWPQJV673jc@ zx8M>zp7PF<*2~jaae0rx4;HtY10*Tade`)xfkt(-q=lZS01lAJ-F08j`$mUv%9fpp zeNXi@x()C4vpzNEyozpbDFHYL8yFn#4ni8=0CQ+lVnV0BR_F09ll8svb97(J)5P8t z{&gx(d3Mjw#_!QXEyl#&&W<`2RF!*vH;!XZP-2qopzx~lmLKVvpqR|IQ4*30e4Fbl zeBK>FPVBN|+bKPh3c7H7#P?l33!}iY-IQ@j?Oj!T#P3}p%SvI#-a74I(r6b(K=PL# z>)B_ym>r_5gn5H2zy_p1n?T{pj#gey%IF>uU;|T5^h}+wik%2a9Yr@sXbN;K>6x!^ zXYW^*-gwom7wUpi#<~_LyxDn5OtPJt+@K1UFQG411hG#84i%XzZODM^8bB)}*mcUD z)I2p#gplQ@^6?5LTdrg$x2bs|61azY+!UMG*OlDlQ8h+Pf^-7w#H4NPhsw%il7=hB z!hV_sz*F{f<>llI&4?Ha+eVh?NE*9esfJ}5M`DU_t?k*U;IhAgg>6t{l$a>4^*!E- zT(*VEn&YPBNKI5%L65(J$DXZP#Yxo~r6!tdLr<`xls@I$+aL8vwx?bpsQ`}(ZsLWC<`FWZsuxBj9@Tz$9asA5J; zV!QtiQM2QvjCj}f9-d-W%<{|eJD&WFUdq_!dcUUx21BpQxjQVM6Ph#ra0M|q0C_2w z1v|o@qs26^_w>*6 znR%Xb&pr3td!OgtdzP(g16!p)*T`m7P%F@DMHKef>Ch?>pszjA5Ht+t6gS!rr^D}i}i%)z|W-F_NA0r zMihJ$_W-aTkzYoO?b8m9L+pBL;jP*#~9OPRD+KcFRV2Aj= z2}=Dhu|P+%guRL6EocpiSih|tYCas@|2JIwYs za8nBOBxtrXi33}SzM<9zZfZf%sf}!RQjH)fRBzb2G5soe8Gb7y_~{blg*pwFHKwPL z4PmokSky154NV(1Yl^srHj;=wu-LcSEtaj^ufsdC zvq(FHqw()y2Qzsa{nzZK%_Gu&VM>_Mh<&TmRyRPGWj`gI5az*wKv`>gIvI@z{6vzc zurVxs#JX+L0)dgZ=yOR_VSiY|NM@T;hSCD>#;zd63jc&@jvBN(-BoVFd$AixX+rGI z?V~B}miG#_NPOAtq%5IzxL)*ByIv-F8@PdXlb#E4;kpSy9g~^p*NZ7^8Hp>Tg^!HN zJM?`2vyud}2T66plyIYW_MMjdop(UK?sueuIXU5u@A5k%ve7mYI&7^5gtg&m@629A zJV4vR3r6{WY+cxm?K+eA-k{4VM|l8XE%jV6Wo~C0-e-0>JyssXUt=4&+LoO^qxr$$mD3aD zVf-z2+LeL2-lCTuer|)PD!4! zkGnoDvyaIBXy3!lFMwkewzDgzEGVMyV`~q$pa3r^WP7;Qm8C=&Dec8vW`T=@!wzum zh9fCgW$Ue5EHV+#X9v5!DQk>KS6WNb*<@F|h#eQ{`nha0qDNUKNiQV32Q5an{_E9#;K?g`)vU6MyiEhy`sv!AfDMSp``aXAc60zyND!IIe zt0d#4>|$46(Uw!$DtiSN_Nx*FTi_Zl`bh6nwJNy$f?&x2yV~`lNITMKT$_AVKn}$Z zvp3DUE;5dE9MA8K5Te5&p>?P0Ly>mEg>i#Er!wUk{6}_=D@)`S*)X2j=Oj|n@#E}1 z*9y`3$XJt<`P${=DEw#kuxpEm5o!4;x?it?9F3o3D_vzGUS!0lpZ$83&9I z@3Zw#dOi6Heo^7-;G^X^QI4?pj%YxqO7sgY+}z7+qtZT`4M!Xpy^1#|v~;7E_eV8+ z&K!1XQeMNaD71y$`JX7wF9vU&nw2;3#)X(98cXl2<(KHUdM)Hzc#}d0w|nK-vk_l@ zzSV0*?|-tP@TA+*@{?y-UxG#^ThR>2`c81;mxrBIefc;t*@nK0zn$#q)=+-!tjSjb z;D9LaNk(@CxOJE3pLPA(I?C-R$dXtUcJ&UHH=a%ZYW*&~lbnsWDU5XcTt0fX=WE%! z^cUoZ_}ztZZWL6zfXTj{tv&u@Uq*2ignSsWl!EmbfF(h4i;W>TUoI$I_;a; zhlp3`T)?{BUT9w77TxeI^Mg~jG7o>W@U*6NMRfG6Yo;U0bp2hGqh-Y*LCYk@ysnE0ZQnV_X4CmMbgZNcQfsIRBtxUrd_XOcfzVb74Uw zb}Ov#is^vUiUjXnw)>LGE$66k1Ty&sT`x&4q|7p@r2V@4-TpHd&O{`sg-hLS2|jSb zTJ|}8nB0WFUbxbIZ)MUsy&u+J(%+I>@V5)syPv4cI~N=E^h?ADx(&|`!no^x5uA(t zAy3vACAZ`66>fD8gw-DRNAy>{cjQhuSFv$FTd6)b`QztTz4v5DGf^0~-~CFZeyrus z=x=%-$gl9Zg%0lbDzUKzzHC@^DDz*RX;yY zPAbt-Nl~GfdsF4DSj=?(kJ>S%3||T^nmv{5Sl?+moYp86ctK&P`$%Oc0Q3^3dd8J~ z`0~QDaC|c!%U#tl#r;$;AgL;hcb{3MectGo{dev}!5c|kVY2(8Dzo#pGs?cFe7myK7bD`QI7Xyv1tr7+zM$aSyB#!y!@hwB8hX9N$+c zaz9(8K0o>U=d|8;@&x`(VV(PxD*ZUiKheMRevrSw!FHSby((;6#2@?hpMEF{G~X9? zx#w0Ljm!N*_}%kUIf)-Dl)A%U-aPsuY{Y_P4T>4+@zv+L-zwy(BQ|_OuM&o+^miNBpPSw(o&M2DYF|V5U{^Z0Z1XhDNL#kOc$HTDNIzASmqk)~lL`qRbi#^cQ zg!l?Xss?uEn2Ek_k)g+qYI^)6Vp>BVNzrm%P_)M5NHrtgQX^&t!(3as7>>8_)g|#% z4do2ACbMMq(jvE=UU1%(t>G|}H;bt&H7wfh5nlZLF+86Zt zHzOHZlm*UfiVk_)tX_A)X@&^NKv5PsuP-|0@n7}93+Xd#G#T0yJ!f7Zw3)@2GbVqqrN|FbALd+eoLMl7u4`^6{kD?o@o;~@K-kRb zS-gt|%ca`^4*Xs9s|&1|4BmYk0k);OhqcS)CzyIrhlhqr$@9QYqdL9US}RA@SES+K{-&qPaXBl&=)W7JZ$Kx9Ni`larba`+dC?yw3JvcQ%7yAHunrFzgk_HzI zdeqgVTr`^PFqgNSxkDOWlzh0WChwxtY|&hX5yi$ivWT^5pr-a>+HCYZ`U-PfX>^g= z<5SJh#g5qp^XMzhcT3L~{qfMb@%y5tmf?KdsC?}jB7aj7vS-URLNm~TK+LTjpxOvR; zzuJR|>Do4W3^NMYGK%n?#Q;3g0~;p>nu2#OEOPg(sZC1M)3MQKm{VMwIYs`SUA1|M zPC6oe28Kd(<`)HfzNu|YOxLkl!myyYI*W=TJwMluCiduv0Ii$i?p#$A=c!RQD@kvT z%~HlDiYJ_yUiMs2w=T(Pj%X=kGsWAvspz`r>biqT>2qwBF}6^Ao!bDHbaS0wQqLUG zGR9Vlzw?Wthn{=ul9KdvZ44OOD1px1;AwiIE-%SRS7gAjq>!EA{psmh*9fPsHiir= zh3c#*;(LbHjVAft&oiW3G3RLAdr~>~|^i5H{XKJ0nW!t%_%dtC{ zVbT$R5zm5yTDwv8a(!#c8Rrj0-GD2OyUd*%W2A3Gp*yRJ6rRF5`sK;Fq6ZT;=26m# zq9MX$+hn2it%vk5f zCsUrE>qam4%$qj6+EsJj`A^Yr&);=_E^E#=T!q^+7VkWxSkr5ucxkfj{M1$0z03ru zX7L;^!+PuF*!eoEvHO^pq^RP>Ug&y4a>e|t9~%29NzS^(hF&}B>B*Dxr&sGApd>pl zC|=`rq@Iy%xgf?^-;R>vtXGWoa;aajfV@CyjGb&rbzWL*-sx4}nw-7BVGZvfGfiq( zyd6MgYQMc+AX>vX#0-<{-C^q$TmL6nbD>U$`C;Y_>FVMGUa9p4S8NxiuEicP4>vI> zcJOjPy7>xqp>nPIi1{4|`akLQwEpB3)g`7<%-cFhjqOFG4DA?8~H;PjJzvOi$v=fjuf<>VYH9c4-6NMwx`$WAEn%P-ccM5 zL1f}9+(j|#^^Y;nChi1-4x_3C5H>UDdHFL-G-%uO*_Oxz$oKK7sUf#NG* z(zHnNU98w3J~8&#`Ec<~FO!C&fcH)`6`vf-1yIR*UX~5Cl##^>Q~I`hPo>9-A9?L> zNJ%l$b1>tbW{eXi72EJ9MN{9<9Opb%}BDL>xiaFMWSt9i%}X*_ckV7)mvt>MeRJdMS81vo%gZEoU4w@3bxRF z%>gOA*uvYrvG!`(vbLKi{l?mzGmGa}2Q|vCRxC^1iuGr9LT>M!-Z70oul`=9vke=- zd48q;A6oGG77q^Z&d%4q?+`6Nk2`1I=GcpMibhht1KnTmzz|IB2ZL znOW@O&26NmjTk5_#pE%Gb5U_4sje|4&9VoJ|d* zyayU<)6xvlR&<(qpOmB>;jL=yPwNOvvBCy3`=u4d=e(yIf2RF5)Y*;=VGc@biZ6NV zG#OlT&Kv%dHWSTQWF`7T#%7l)1wJ2w~K^fqZadM$T3afdi;Y{a=8FqSNv zXxB!TXQ8=Z=a9ZAe&oHsDdn1xk%KkwjQR3o-NjG6Pc-FS%QeWcW@x_r=qxT~dAl}M zUQ0AGw87EG#+>EFTyJVq|Fs6AEHI6XjZ4!*%Dt5+^|eW(X&e10$^_U->b$Qs_0Peq zXp8QM8vE@0wzvh1Cl=RzS15Lh&yIa{b})VE{kX~gdiDwjTV6DCQYyME@h)!iyWX=x zWXp)5Oo4f2-`uLEgzLI1(Yxs9%zpq{{HXVfrkv}O7K~lE*s-6^>SC35UsL7v#Fd7- zap%W=IsYu4^j0F4#|D|Pl@swvU&{)R8SInSH^dx$#E&YMcFx zB+A=%gA%MyaP!cO4)8_bRfy-htSGVdiEaLKL(|ys0R8^$`Lfj|2Ye=T7K3Fd<^Zo* zYoTmyiM>xo^Ol=bW90$$74yZi4PYvI3Q?#mW3rt(#av%zUPAEUH;3I+87uA7spd;% zn@YTVnwrg$)~+FfF==d>%eIn0pPpv+P2V+&gW|L?LznF(p*|zcoi|Mk91iiWF^ybG z*F^e!Zr(T8zNX+1{kr)|**?!$pWn@YZfdSIJdC?Bw#sE+NurN#%ZgieYqJjHZjKqt zVE*|SwpibaU8{2hdyBbNcBJGEoB|VWRjf@tg1yaLCvz;h?_<*vee3sHoekJn)CSq{ zk{q8yEw^rAOvv`?JLYB(s1=v*gKsIhm22N^FHRpryEvB=`}hJlL9PkWLChG#xR6SO zJ`pYBw?<494&scljV>M~H9ms)$sdVE+dZnqoOWjW%q#;`85l0Kh`mM{Ll*6AF@-e&@AOvziH zuAn!&(Cf)Z)%VSU4D_`GX5-}K*l;xX}qFEXdc`sE!?E2i@g*9*nyqv1m~0>(7e5-?>vE|?8oaR8?2hY(w&T42nEPdyOXmBo zX|=zTy}{uGFNdk4om#TQcXO*BIG%`3FdkD5xm+(<>5JKAz(SZBp2R&FJM02U27Db` zAKbAs%{qz89cyY&FERHewl1tXV>*3OKacW17l6a`4Qv&I4KL=D{!_|Pm+TT7-?Odi zJKVOnr_@i)kINpF?DM_cx-cDMMn28>F?zxUQUdssb3PQB zd&ms@FZk~6)nd5DSvoU{D7Iy_B}u-jLq-fc^DJjv@tBuOV@VoV8+S1h%?({}C1XA= zttIKcm^L>?gL#$RUfcDRkfu*Ad=^tqHEmmct^ zaA`BvreKr=HP@U1@7xk!|F%E(Zn_*K@_0HKc%7#iBeXq9}XE<6Ekmy45aM@@{r|-kIeHm#MXjgifd4#O@dXF!w%`KzB zBFhyg8l$_6m-P8owBdIDve0qkxf+IAHR-+qGw8JplZ`|-arxL;mv1E>e0$p18NM48 zZsLlu7?vN9 zoP&UeYz%0}1%PF2ep|MVbJXu)`@VZ=ThPArR`Yb38JKhn+r0qYx4@U)W}YFl;JEo! z0nD!MRdrS1sGY`4$c`cEXnUx zY)$pYzFcYLO}GC9=vABfS}!S&T=sG<`k^}X0RlB9K);LGRd@gl#Y;N2WKy>& z1Jqs2dEKoKZuqU~u+Pli<`Bqx#mtpCaPInT?Fh>3+tvnPoXn@PW1L5R`#VwqCetB^ z2hPs2Q=Dghr#kX8T`k*!cs)${Y&_@T4!4el$rmgQ$vChByAU{?>+a_nHf3dWj=`9UTv{ z?5whAxW2J67p>c4eu|E=tXM0ZU~E6LTy~c8%}?1O&#JIW4aN>ID`l~qUw+dam6{V) z)4}=xrsi^iGt(c{sSj50m=OIplvypQ7>c#liNJnj0vHhIG?*20Hgc_FBJ`CMu?ves z^cQy0!8|`5p+82Ex)gJ~{aZR20BsWkxHS|&rQwkMWu3h22mp}LAE)%X2sz>YAHX!p zwTX$;|3q1tU(SgJ6qYa9DzNCQKba58syPY%vtKNHfZ0ioQcsu<$?7>N{!3nLd7w*w z6(#;WHtYfbAz<=9`XF~F@vQjE*oaFz=bpdi3)+K`or<&KuVe3AAd{%S-HU4vOl*nK z;%{RgTwZgY`)e+J{=n6?4Im?!N|^+9u--3PA7tAqb5hh4m5YMI_YZwB3T^|U7{(N3 zV@^M((m(FSjEAas`sZ-p$0l4tz3cr`0jt1vS1JGxFu%w~IPLyfFRUNN?$U|H{xpAZ z=mV$QKXvT*L)I>Gtoo<9S~kX!`xj489**o%fa}57lnW%h_iuRN*Bzg0d7kl$@&kMk z-uZU{>ZQ|e(RtkA`O_}_(JF9GsDu>ohHFD;GR65#*x)g#tk!}r%OJ=B#OmTnIS zhiuDzd)qGZX0ztWSC;MxnCCU?KRX|jOLQ&F0=aSN!GP;8d7gHVe;9{SEhMZfbqwg< zwEe%>eL9I)ZPsGBY3XS&RTKWJ*q55fxagrTKf)pe@Lz`gr`o4XRPS2%bwBBbdqB<0 zr2q8x+axjOP?r%PMQuPAB%pQLFG^zQQVj`KrNIGjUN-)hzCY%B&|C~8Z7rn-e1Z(D z9gts@J`V%QQcKSV{CzniNB00anLZ!0TE4F|DPUfgagGTlGa0u)WlT6&nigQxwJRqP z5`E$ps@4+hOVa}|U2c$ZGwTWt(r3y!d+dU?chPeu4@_UtUrgOVI8pjI;7AuE$I>n) zMPHBlvhH+gegM9!B!_CJ46cUUj|At^F{4bohMaUen^cBA6+_m^YsF)`gIQN`%T!zBUFI*F?{;AyM;x5Kst>9C z2!Rl@{JabIn0t_1_YGiQ<&@IpOSxUN$4*tPX*@`WCl4+i45))tc1DLBuJIrZojkns zeL#0t{$tleZP$24EE{=b=|sSASL@^KL*%E)E2y@Fn9`|$&t0S7iXytsSV`SYh%5aY zpz&(f6HETO8}wBe$U{~-D{$T`eV@IDvu@y4tM(BfAs9HT?0S-T*zhLK7!!WwO6lT2 z^ee)Xio>apRf}~{o>sa%(B@V2li!DRZeiC_XD_~4x+d_*E5;McBUxT=*HRbi+$l8; zbb7^p;(J7KOAPs)bR)L|fMxXQ?N!>+Vk2xlPP`WEH z?$wN3U3>H$`Ua|_T~4W8VA`t{xpwwhcW|bvlZ3od$H44YyK)mDTMy1mbsD0OoB~;| z+;SW2v(j;B6^_6zbp@xH^SPQ1h76p!%9#MEW&n2hL9U%c76XS-5eU3eYGB{1kBS%v zoeZo6izF99CK=@`IV4g^&A@JCxydU^&jn7uGESIqn9k7OMD-xllwJ%(b?fJ4uk^aB z-h}az*O$h}E$+6+^ZlRVu6VP`n*a$r0!_M)=H>p6cu%}V>4~_t}B&kg*}n5uHWHVyN;8 zTW%n=yFahNF)IsarJ@la%Ryjl_n$n?qlWi!+f^Ziq0)xHwC)v8?I2YEZigz2Fk0FX znBBeWX(EK^y(z*WZWX~WU1Y@CgXP8cuU+EUgX{q*-yod?*RtSI^C(kmO9 zyBQEo9s{xBREPvGRR+qsc~2vbO+V1zMLkFOQTjFTL$~-T_gKtB{oT~_gx{rc(}6#_ z)!;=m{ZM}oHJ$)~1wp9S`p+zn$2`)9H24Hfu1?Uh*Vtzf$EP3ZLsoMFin}Pt^fm4o z7s6xp_fua!o5S5n-S+z0QYl2X;t1{#x5`}^wD)z=GrbcwIgA6;D}+Vd^+6|I=RI>e zAh{dG<*M zh?rxgZA5eNLB)VH+;dWt%YbY>wN`USmso%3APcq0!7 z!CV9g)e7o+UHLrml;Km{QA`(e2P7kUN8V?}I@NZk`KT(J0P#*ilds30k3bZ<7_c-5 zd$<>ZG<&vq8Ju=_#seoR`F^f;&U^^?=_t&3Mn8dpteM%fP&e;laHu59f zJ3;84vT*L{7zl8q=E+ybX9ijIghBL*^10f%<|*Mg_rIXT3s>@uoE-9b5H%uyFz;#5 zp_7mD9i0mD=}wped3FXX$hD_3zi$f9a4od_E4X)VacI~4NVSd-ghxKlx&dq%g13}F^H(}C|?0eHq$5K_U-jU|?|xFquWjS;L# z!ZcT(LX{HW6p7p-W`Jv03{Rg*l@qvJGFc}6bDB(4^3+sJFCpYmD0xKO$;t(1DKU6L z5z4t|!5?m%H9}PI#lflpLN)gSSyM8XZ9*an#38CTgnI53vYy0(?MqS!#G$HTLNoU! z*+g=bol7DL#bK=X+&1oo=1memb`MD;WQ0@S5ngZ~lJ`oIz+J?q3>RniLH>%H2T)l# zY)99EGI|6?DHn6uWOqp|JIxg>qJuMpT+WT-k%J{e><-rg5j|3svAds}ydXyMll|LO zryLu_nvlQY){|2u24LGy1-ugL`xLScnlY6Uii zrIxF>gPR2sI=Jo;gHU-IwZ z-1koM3e25kFlJ)@$bWIYbiYZ)3rE})RpL0+Ujl-sLD7`XEi&;SR*U0R8pP>^T9idn zvm#fI)B}kZs56PPc_Ocs(tSl~9_Si+0!C`C&C{b`q;5qG9$7Vvcb8`=ba~4uHegu# z?V(eP1)Gp!0dFnEL7G`)>zP`MO=M{+^myhJqO`0i)>Eern}m6GY#DDWB@kTlxM0xG zzf7G=G~(G%&PvrqM*m$QR$o@lBd+4@r6fw{7Mpkx>&3~c1;n+yBb0P$r5@3%t)6#< zwMem^cbxKAYG0h~<Yu&1O0k!BiJ~JjfFrxqW-Oe)Dejxg59-Ud zlu&(?&A5!WYlw$=$$N}t_9fXq4lTS}EI6{_-KA`m1;KGsTMO?t^@Y_j-b2b>SyG9f zuT3lC4%L)+iuaUqLY7zJ2# zHkR^ShU0MkVmkC;g9%=k3W`uh=S;%ZPyar3I}w1BDJ?PvxMjw4>St4}i4i;{MJD5M zBK)U2^&e1SBg6Yj`5+T>xc)JKk4D`^jO9&Je#q3Y_@92EpL(&#M(%ps6Cs~fsuy4b zFf!D=L|9@|7s%I@It7ScGXA6PCtl{wqpp@8gmtP7tU{@F#8loA>Snng1QLn57>}uk zh|YIbP`cFO_Cf?$iP#s|jWfz$B&l~jP)pVXYl_)PO?Km>BW-J#2D;3@r zsy}4AV+E4C)p?ksiu*ikYNY&2sVY#}jT7}ABLYARHBmm7YZ63!EqUD)N-;q|(OBZ6ofw+_UP_d7nMnm_}%P$P`&yB%aGKrPt}+N1(L5#ThhBlKn+n2 zs%tQp6k7a+wDr9w1*{PA8+9!vNim0SK(p!%6XeQYz7f}{E)(bTSJM`xT@e_CIt=mZ zSyvQ`_#0@az>(LMUOdETpr#U+@Hf&tdm9Dmp*F*eMrs<-kZ(l`?Hv{Lgo=h4P1Nhe zmHgeb3%#?1dSNzi8O_w2#5Mdww41%_gic|iw~Q9*ZQ}lQM`;gx_X*R&&?EF#OuE91 zkE0d#dI>wi3P$K{m<(ddT2~s^K1%pIOlK6^&bq7E%=e+S^=1lf!&67G9jr`+C7-4v z>*Wa}!ly^|JE`}HJNV(WV>7#itZ?!>vHV>&aVP&QZL)V++`6iTr~RY>&U*U%FC=88<{#4)jy zHO%nlH`8wSnTcHKZDTwci=*)8zob3xvlnI49aKD67%0emDUH(?BMlV=y4*1w{{E;{QQ?t|(TF^?Zd+n5P%+_BL* zlh}`}*NW%-qReM7nYht0lln@kn8*Sp3H#|4lhMfTlyIlmNC;Z)fh&u zDCPf{{korBG5M_)<{wL;;PXR*hx=PAvSS>kc;l=-g@_*&{1pN(`(kX?-1@|t*;C0s zAN;r9FIMB6;dk7GYLHk9Q&x9iMWx-jtnawbsv%+{KP}j3U{_`0Il~{gFRHi1R`8q0 z47gP`oXY}ta@8oYlb^8JdLX*;_c@)P*l(=&klg)Au){!RrEP5LPwXV?qoN0*g^1vA z9viDOja9S86f%A(_}j$qX8jgc5|))(@VnX2p{=re;{o>YXDn2(cOrt5(G;HKt}-X!Xtq zU-?6W*tk^}bYABmEh01)3D$?47|g47x*(d7p-IyrEfH)C@f>WdPQPF?Gh-G_mt-ig z3JDz?t?s!Xnwc@1HV@qEcZX!uBWiRL(8x$Fiv^_rt+NYB8(dLimym_TqsAAK)(IR# zvIlq7BqkVY;I^Nn_`(Ip41$VdLGN&5wtLl(SQSL<{MO$%ROv8MN+;9AJa zH~VVSE}>D83oT4~4eq9c-@jXJ!=)?~e$n_k(osQni0zy6wVH{B+W5ue8%QSwxgm~k z%&c}KrfOU1S=`d{jK~kczbUD;Eskn}fZjxI@UbNI{HTLESCWys?<_I24w zhNkHTG)oA=k%Sb#@vG}e66t0b(yU121N|XYZxZTsFBb$mEoZ;vhYH??v_brV?d1ZV zrV-7W6d_QC$lmbkA}&wQU9y6EoCFI?kS9Pq*?(hb4tP`jaP^(I${3nS4t)}UFr>q3tW<Kb~ zGHy>|z}Mb&sIorsis2&sy7B*!yw+|DB@fB#E3Sw<5bJ42N!bFM(6dA8`pGNPiGTDd!GqT0_gh52387LZ_3O9 z*UJJRl(B zVkl~O-7>SQVUj0z+J{wJqoGV2Ag<12neB!}d+t*Bp%a*6fI0rC`_L;$gq0 zo@*kb3>#V`>62g}v}QP|N$RW>rllrK& zOODWPL72Wn*uA&dmWW%^YnRy5?vPA{wuX6cPqwgbkxi!TEf`?_B82hZhP9|}DNUvv zEbjJh5_*L-y}i|ffk2t5|5;@AZiBoSlD8$WT2ro@am^6W zvq^31qQbsHC{5q(w)OmD?1#O(h4EoCN3>gw?l^4VA7=yHnJ_tQ(TG{A>z%d@{1fcQ zy@!N1!qz~rP4*oJQ$A$n?sX8}4cj^r)Y^Ba&6E$(H@(M%kHYqkq_i5PJDBlLv!C~# z5mmHGCUInLn+bzxE^!_&Ciw_kAf(5l&6lAtm$-~`NCCpGu-uWO zZMh60#%j-XZZBCV4daXiwe>ODFnl7L-y1Bn91N=)Noh06aIoN$*uvg$z*Fjm%ptBB zZ5Di2wx~B!I1x5H(#gD;VY4yAjh)pSEmVhn8X0Qq$SBwt>26WgdmaLX{*KIO*S(A0 z6zO46)0-fi6+Um&xZUw?!KO%0i@M$<;oR_*qx;&O`1+giUgPzo5BZD3(W8X+io2!ZdUj+q79A_FbDd-H@x!!_)8b|hvRTH=GoUz75MPT{Okw~mI)EX&A0 zPayhEND3E?MtA(q)WKR&*wWr&p-*_*=!%|gSuxlpRGJ)470KWZ!0U*}n#L}n(RxWj zVR-lli2mbd#aJx~ruCC5gfZbiM%5jYS<_ZaLTH1e8sWuo)I0sow5@^LBgYSSrqv0P z!xz0X>vX-}ww)i!9`0=tM(D17cd#@4zRiw|Fxm*IO_&kB`JG>9&;1w*Io#r1Z>LZX zsW*}As`s;O$N%)Mb?N9K79tRWdW$(ho;nB(4Xsb@x*D^!5NlWJY9X~XHRo%YCCt`@ z)8oZ6G_{a_PHWDdgXq3GOB03M6NFlYlw_mkYmM}2YcAEQ!lHm;lL^WLN!f^+k6grp z%NPT79JwAdK1R{V4?ArGMO#;sJ z24bVG2Xdn(>M+v08uZX@j*3DWe+P=g2TY$KS4W}tAfL>Hzc)^RZl3nSJ+l>X@ANg) zS!DHo)KaAIhUq5cE@u#?_k-znZjTZm z1IMAf`2|!wQf&oKvn=6W-6i0za}gTjg+MhUzmibzk=Oo*dWo!HgJfejpn*1zfyCL? z^gXiU7W^`0LXBhJp^@M1fD7q2Y7iOsuM@f5pj);fNcK1mRfIG;1SCcFs0rlRL1^DL z1nAba1|Fz6pn;}4@VB%HBpvC7I+9iZ@yk zh?895UL(WQ88Jd#g`9B+?%TZrCAF+Oau=EAfC@mIp$;NP{6L({{lJL%Pvv){P{WBr z(@n_dUs2Ua;Y-kf@p98BBz`Lh`|Upb4f+bI*|)%SKB9@b2YEIL+HN@qM7q8Q3XmtE zn&5As#yKQZGIERy+@EZKHeCyb2jcCZw{<>nFX%CpSa}ov2CW0SN3TKFzh9t=d>hjn zNcl?8+oTG5`I{~%8RZCNQ+rUy5rW&`d=PG)QLvB}MLji}FN@@Nlp2B`E4`j;R0{;DTC(V6!}s3(7!%iM^() zkcnlW3*>XCJ4^&c$t6$)eGQb#!2k(qF^DpJ+jKtim^tX!_YO2eo(ys?ItL|wyn;UC zpiM)Ng%TJrOJd<~V>&QM%K*ClF~I)Dd|>}E34|Yv0tWbQQ2L)WFs9n6z*%w+1pV*= z{+_Qu#UtiW?;`IxnJz`5oI#ZLBw%t|0~8`%4BWkrA8|&e9RYR`B_P0;OQ4nOr-0|q zepCozea8gy*dyqh!%a}iY5|m&i~{Y4Gcd{)g+u)$3Dh-8fTzQ+pk+~>FltvF1m(>W zz%VWO0&V;{A7o#96E510Kxeb~pOP%$Qa*wTK|b9BTtv;#Ri*-{`zQ|TnE41ExHv(H z4GPme2xr~9$aH_`;Jpmc;1m_4JEnuuLaf$pMi!lhl8$XKe+m;puA5i0Dl0%0<8h$2ctfE3{t3j%sf1!&GpG^M z2CS$;=;`poAkOT5sAKaLU=TSMwG=U|Dng!H5BGC5fGh$Hz0i6R>e;seie{DrlRLM8 z@%<5aTC6yNKn?}LSjG54JEw*~l#}{EVsivad}6?_f@Toe>K%;1Rtmg2#eSgLrwCA) zBmypl@u1osI`I2L0|rZkP;WyVbn?h1&=#u?RfH&_-arPm17pp9Q}J0Sl)BanjO$>y zAU>;(BQ3mu^y)d#miia8{ZJby`^DiShY|glWMtugrhAaW1*Rd$WmC}LRhytH-`BwC z@g|Tkcp9o|<)SVvg zAB(s{Gc_+D%LkzC^LB#hvmK$@lnGQIQuDUvdM(7$QjlfkFjUtV4boD#!GK#s0Yb(! zG>UEDp@q!61+7~4A7}-;9O$g8;n%Tg(6^;K)O^_;O7x;Z%O4G(ZFzVY97{Ulk+=Vt zA`q`T1jv{wcy>?;&8U0?CASoUq^~6~a*C~>!rC*yBi|BukjtP}Q5R^oa5-FVPocIW zI?`4lKkNdsTz$~RmpKpMdH-f^ogMrt`uDXw{st=B9s|Tb z5b*2Szj}`U1l{&;H;qSLHV0zfJLr+;Lnvp&56nws$ww%9wH0*(VPLRy4&oj4IPxjZ zGzxjP9Y*UCJ($b`1_*8Baa-Wldk|s%iqcga|Ab--Ty4u^_7IBM8&}7D^WUo6Cj2j!Ym2+(CWE zeW8Wsdw}`AdC=nq_n|KLdZ=2W4^J2U8{Pvmf#lQ!VDMuC?(vOHS0T?WhkNICLW8e% zz-#7zzoA52YM*7Z5$-6Np~1%+wjVq6K>APCFEs9SkF3)Nv5er*cM?K7__( z)`FCNJ4_MC9#2p~fe<*RHbdKnLqXxIZK36z2Ak#~52^oYIN5Y5a@Ja)OB)83^i(L; z%Y|Y8GYVCW_@(_cRiCZrTl{oKC{$aII=7 zGI9=3NE%>rAIOBMNBH-(7Hl&;j5HF1M%xS!y2yl$ppp3-fo{uncuM~Xo%_rX)H3Y? zEl)Rirj1+~2~%3T#ds|| zSm_PzIq(oV!TLVT47apR$id(vhmpYtfYT8l(^W`~e=@#Z4DGbK2<&ja(2A(DKzrZd zkv8(`0eG++rUF7?Ak{*iy$Bk7%>~^rI|5_bFBi23@m2K<8TT1_O&YHoh1dv+QEvmP zWB-;g20qXTQ-enfks1Gk3R?}22#_t&AkvFsO(COr>cQEKE(GA!qJ`%G@g8-tGv{af>DD zCE^ryDYA_SMYDY2g0ckRmc;?Zhd-wCk*?a%qyzpixcfDLT`?V6Q$>Zp8+XA7-R23e z&MFM7%(G25A@#0Ii8@OGr>)MJ-Gz|FnODfCkM; zDS$ERkpn!JZG&c)zXw&`83Ab-Zty-|WpEeu@Grm3RP79dox$BH!VUQ(E|qm zYXhay-@;TZHF%Fg9+N`Z!=}(i4YcWYWcU(LZ?{g6f zRy@-F5yLdCMqUIydL;=Yo2hdQ|Yq;Pjrq#%{SWx}K zGeEq!>_`;SZ0N{#WWh1hoP= zOqz9|0Qpo16J7r{Y64+lFrkCEYY@K#`F9s83K3>-cph?j6wqCPEe>LrstD=X1$8AI zgG;Y13U-QoD5~QFO>8uU^?=2Bc#Eu`55u;w8XgSh!-Ka5)$^faoEIaC4OZzR$7TQt zx(js#8R-t+V!uhyI;r#jB%OPl%;o$4-_?bgJgIYQylUmFPo{>F}a!kdK!t&1%U{;)_pKStCZ7%Bv?|G&G&!e}P{+rsc?o z2v%0=MD3wftoHXLB>rcA9{1!&7;4QXRIa@>)nZJ&YPHJT4|sY_CgioznbaF@G_dbn z9?*CY68D#(Z4YJ1reX3L-qUNgKsLT{pKV&zYrLwx-+ z-xg@U{J-Hm;IUl5?NTP16DEcjSZ0yQFY~m^DbV|J6+b0?z{gLoaeae2rbW6Afr!L2 z{2G0aryWZnhFh!xcB3WIm-EFuy!&kQwRbGs8F>mFervI5H=35^8PBO=yL5WMXGld8 zXkGJy>5`W}fLFU!xULlIRQ9f3-1YoiYl$%}y;2VS0;}qNY|SwyW`83RvW&@Lkt^SUqPiLIFljELKRF+! zjcA1A+AB+4xxJbE`FG`tmnOAg-)|O(dv^+QSg?!tyc6N${k@3FtiVWykAuSfOL_Nw z${!<}@8`NvnossyeDqK9MWt^eB=*!O%px<>FR%B8{^OY*Ii$n2M<0T7Pq)D{T`RLm zLU-^iLscgmmx@A$0j=4gnorvt0esv^c%r~HXB&W9SUPInqFsz8GLw;uezp=@?( zH`%nw-Ls&0h#T4ba0&1HlyWp=U7mQc^oO2%idaD7)hy-MRXD%zF0R|Q0lja$9uIh` zJyM&~gTFm<)(jgTs^X@y=gyXDBdS;XS%>(5VVXo%pP~3!2|VNp6L8)gf*%K~G7O)p ziIbKD2K*GV%VP)lGUps`?YIPm8rp`3x4QhROQ??Kp$v$*cD zrO5JkJHmS{h_byQSVoC*S&U)n<+AY(Zf}rs#7o6FG<@rID4U`Jb>*ccK61||eZ@w* zGM6mr2Pb}W!Kcf=5L!O2T+PT?O6p6e21Gx<7O>z1RS_>Ey7KF;ZU9%M-K>-@{doV; zaR~MLkK|TwYBc!c2^iS5I~Y{`Cp_*j7GfJe$>VQV!qi>bAwRj8J0343a#^6f4UCI; zW#g2J)|HPYn|A4X7!m$a1W@03g*865gqQzamM1y7>N90DA8kHD4c<{DHsebbtS_03 z;oEPgu(%c)Yz{ZZ(6(k4%FG7n^7iTy8RPNG!u(bB;rpu_MYU&z4sFE1TC^U1Va{xtu zx{aA-%v9w`@_Y-ZcvNHGlO*9#7ZTq=nDxc|KEoKQa+=BK@AIDOZ{ebUJo@NMbxFO(t({*-^2urW(%l8;#ukH{u6T>66~1z@-p4}f zR6xgb-|(>@m;Vgj`NmyCc~Sdl?~3hdi) zhA%GY+VQ}lQ1ayqT(Yl-pWa-|7wMPzsg;HIzn~&bkad1DC~?L~-R9`k1r@C?cTL1v zI_r4)zotB;Djg2>(E+MsF28;p$ICxGX$C~}w<}J-lmAA+ia<9rEXMz!p#NWdtRiwHAOlN*FjQ+Dvy(H$^81;6jn2J5|`ZlJZmWYkZ54F za#&?gB}_QK2!&bUMH7b9f#MebphS`1eQxRd3STUFlRMUoM_LU&w7Q7b{jKG%*U{7I z_X7pdD8@2L*_%t=S|oKL1zHN}H^7e&(MonkHt165Fx+L?GGI11yXZL2{CouT-2Wu2 z+E9*ajUR($tkU3r@^OCs@ePO?p9Vb%kMYd1jbK%yD!bHKkF~vc1Fh(*gWX4Wv-r6z zMRuL$nZ;9hb%WKw*=ALcEJHOqf2|4?@mxh)`MQ|vZgl6huQfEwWq0afr#yEZ2{wPr ztQPMOUiQDAa4;iPFo~qACRJwTz_{h_;tK|->gIC$VnzjtMc5%fHGuBZ<5~BEDw0~Y z*kXJx_?W*`NXeLN_`X1etn%Dk(=Hc&fYHa5r_u5K^%H5?am^g>cjA{?R}=-u;zx0 z<7rNW;%+Nx#n3fktLc((AqsMrDsQQ=KWNy}h~Mg>N2eO_(QhdT)~m=Pvu4B2xd*r8 z0N|DN<>p&fuVu@|m|9jmYB1S&MHSkNS<2p2Dt@g+CiBu&ZeH=YX*ZIzmM>_H!~Rnh zw#3W|Y&hneCQ|$Xi1hR}fMmD|HIYZ}WtpF&AkvOUTmuom4np z8m{!^8ROH59hJ>OVceI43hFyUNbBhPFyd|%u5Y}m{7z$z^2Eq5Ix+jJ6$@Pu;A7tw z);L)eJEVb?Xa1s$M0xfPUr5IFh5HslMrmZ2QF$qs{zihzZY{sghNg)WQjMm{USArQ zpc`E};zTT}Fj0Pg5BACfM4yQi4U9LG+bRVF~m9vQ)JBaJpH_x<4 z`a@{rO9NoawRjKSC_6Hf%|FiW}XeUy1+T{sb^>{f|wjFrr-sy5zM?iMm^ z47T;JiZqgNKEhs=8HBHSb!EVC-0Jwuuzb|SbQ>F0X-BE`IheKU9LT#}1zJjtLQm&b zL00`3jHlvNUr;XHL`-c|UVVVokwPdMXd;O)M`C8eu$~uDA`Z9+$he7kk1sGh<9_98 zBhNg629+E`08bNa$?XLM4&6p@n`b<>)o8rWDRpg_+Ch_?^d99+mdA9D&6$8-dN;&% z`{{ua)CJlZE2;vrr3!RUQw0qqt*_~j)(2RaTgUwS4uRM$Uxehn{UE;J1@8J?*_se- z``gR*dL|X}fEg0+ZoU|*BJJceioj?z5Syxk&5YM|1URJ)E7&(5%6{GqChXR$diV{~ zDe*HA#jl^5E_giZ2c&zNJD<;d8`Be&kkSrlOs5 zRgF_F>TL5=3)b=Z)4nhXK`>2P8)l_3qR7kBqpjpb$BcY&9Y%Cxe}dN|l_iy0fstxV zNw>-FKTYkhC~clzf~>aa=H$O40m&u=e0=25kJ6Xyt|~VkQm$6U&nhcMntsoNI;x6P z<74Hdd#z*O8HI@89Tll7HO+l4>HC-&!8>I&7vFa5r)OhwDHkm&q?BWqd3`Xwk8-_o zdMN_-UG#b6$=5)Eak&<yEl1mm`$<N9S_>%J!aQ?l zBF4BX;Hxm+RhEWwP}jru>v@M~bPuJFEDn0A6Vd%}b7?RPG2X8tE##3+X25tSzFNY6 zK+YFY-m~;5y5QICtoz2B=;PZETa%4t4bZLgTbIld<9+3BtY-%3=l#PSFC66J&5Oa5 z?^Iq(K&e7a6vOH>Y+5h;o1Cxr0@(gFG{sz~pI=L0;79>7Si^0qOb$ z+H>zV2!22nc961f%xvRJ9LxeeqL6OkGDmNG6cKqhSslW(7Aa?a+T6ppDjrhy z#9&9IH&|isf1&5j9yBE$L6IbdI!W(O(7v4SVO#5caALzcpI?UAkx{ox*vqFQVgBhX z?ww=syv1%aB$Lz3Qh8|;_EM7TF?yxDsbbdjG(JqTkvd1}8q;)s(AMM;lg?sZRy|d+ zdMzJMEJXtwUpyQaVNNX9x!uO2MTFNl9EuOsPWf~{bbc}&)A~ne-_2G-=spj|JU25chwkAu zSx-Q0^M62}j7>0L&>|~!S3Omde-)#cpgEKGJ09RGm0#buQWPWtvGkR#x^<|&MC_f8 z`|@$BIjzON5N%tO3fzuY_5}IvGt(}yM|uDBbc8bt!Q`pzUZPr)A9l?xm!bPu^s{H+ zVTYn}sr4;a2D72>ux|KOhxuuC1n_Q%yph!tbM!6x5+^}q?{zelAg^>uWWVVcpoYX z$@8bs`nN{HM551FV~Z+ok1CcY%k4Ip-P_7(O8bVaYPX7lNU+RO#o-WHV z997j4)e~aLJ58G`ie+(b z7MGfq3BP1OKY{o`ATdppRumUkT>bV zlN@EG%5Niy%?B%2L*s}F#>n~b%>!K8tre{7u5x(Bm(~cr zBhUqdDXz~fM6k6S{0gPoJOl}EQRN-vp<1}WohqvhIbooKtnG(X9QUGVKT*QL5fhQM z)gI(>5ByW@Z&>KSGPt!lgcbQzmQ$k4UGZdZLkae=lI$!%fmWs-E|k5ecvOc^A!y$b zE*_^sEil&r6HL0K_!cuDMy#?ok-t8{N@mvPqjwh`uo?vs`?s)b{Zkm+p*^tSe&*XHIVi@X#Tn&BrYg1?`&9^&pgnI`RSpB9MF@6Ue3snz zH0JZkV8WGtr>&JnHx+Ir58QPo}tgKL1{;y64{)%p~;;r?eKjtpeVLYr#8pvf% zo|A-`tgg0l4I*kU=quk=aPgmy5F*UgylkZV6z^Gq>AA|&R+@Dt{Q7$Wy!`b)QkZ9S z%F!w_TRz;+`hVHW$C*>y&~yMi88(46J+CT~jK9h%rT1~(wU(U3Xk%`+82z6z%jI?* zRA9qz88*2XZ*3;NohGk%60M_z zh<;J7dJPKdvx`((9aPgQ@FAhqUshUPI>p_^fKvlMt^@@9=%F_+o zp?aCH3ATTIzZsJ88Mx8!uj1%mQjQK%>n)V-)-xbY9~IOYes&{Rp5>8ATdftkscn%j z!hD>6jeEcR5%bLZ0n$b?A3&a2nJ80l!1!igLw$c;&8$#f3z2By0O@*?S3ZYZ1@w&; z#&qR!%l2a!?)VjEiTwC2;N*A(jS37$%C%HZb4hp!sChLlkC@)*E~hWU%!hMnr8hEj zxFFP6ZVkpbmK@_nwvpcI{>b{T!3|D=LU_xi6iChcer-s*J#B~ z9rX`T1*S1Wx$H)9RfQCPiO4FKv947=0%fyw@ye>dUcXg9j;^0Iaj)g66qdE!;xJYIRc@Q-<*yMx5LtOS$4 z=_GP z<;e*6j0$%GH;RYKw+vbt5oPNqJM}=$hkt`Uwq|Cf@tdmgNblSHl&bPlftz4`qghe5 zjEqC;2Bp_XWFAITcPJ`EM$$?Keb4PnTli}5c7etc5e#@-e2qkx!-tf=eTFQ399?>S zJ*bm((#(+*-EHZo>>VlNIPHLGu@flp4mBFa*N`@ZZN@fykTD?B1G(M=dB=dYN4CR; z&rh39BcQVTF`MD=%KZz7I=&902iNm_LFv5&_@9#-HvGAMS-KfJjQx3_Rv7k3fT+X+kL&dcyd44*?Pw;y)NOLv=vU{f$wg2Q;qGb1sI zM^rRfHl72ep1?z+su8zb{Syg(qjEaSm6?P&kB;Wzof!!2#(Ksv)de-`_Xi*CW?{=$-r*zWT?kEl30~)C;?_P_4*ZzTl!shqeQi`$ zqHLQB0fD*@-})BMOt^#fg@*WwjU%}XN4*D3?Y;pKAI&q}#nK|D(2cXW4{it6Cm z-_FQQ%Q0>$o7H$L(?SNAH4KF-A0g-MyrjW#9OTvIpm+nS>vEzIbdRUSEk!>=Rg0g@ zEUcK)WxKsdzfmnSN=9iq<@_qxzvVkLyV~q%F-~6dIgI|Q+y{Xc+;LjD%sK+~UurC# zdX6VCq*Q41R-qVUpQ;%s;|?LY%kQA*KdERt^wUU^?IAeTlUW6#r|iDcU>3Z+ndW25 z!cjw~Xy?}X!t%mYtA)!g!>Q%u3?sO7cafwRM0pp1E{a#;mL(=Yc5(`;Rxr`DNwkL)VO3w7gGDfUsA(Om(bkgZ=+b*WfsH?I^NgcnhK;G?}5u~EYu+hDLm@~!f3!c4_ zUM|UB;fe?8RHkgXnJvEBm%Wml$ZIp7;URtXEala7JZKwLz(U5c5X%xX3b0jLmvZ*X+0!)wKERxv)P#Y;0YNeWb?Kz zQr?#99J*OFwF4{dq3nGnE{2OgF?sJJs=OZ2ZG{PLhQ>%$5}H6nZ7~)U<;znfS@g=h z@p7aFu2h^vL7SxWve8Tq5$aX6KnkuD)ZZCTRW;I0*8fV>&{)}q$=!G32e0kK#{c&# zem(C8Ux+!!z*w0uoK-D9fJb<-D=L)U0a8w}8N#@#D$^xhZ}w=e>kylJuBRQ)%Ehj)bpUSt>*zorgXIreHr^*>^%+D>9jXK2|vw_R**^59N z6S-AXLAo@a0TqAi;%hHP*Ptw~v0!92tk}^TN9-LU=VqA!#&9vSKaLW84zt3j3D3dC zbQ9F3FVLU$UN@71-$mdrhLr3WO_(a0#3>7 zho{)qoI5)9WPR_X@@-?UFDf_ALMS&SqU;#TbsZWa$BC4Gj8VBxxv`%& zdRB5-OuEeoTN?`TRB=5bQ^P0UCvg3d87SCD=7r>72{)ZoMmGwc!N$fxWlJT1C~ygcr=Gl|$DcyB+{t(NMZSYZ}HpayZ@a9%D>Pvq@T6Mh!(&ig^)5672N_{O|ui4D>w;J;uh|YTSHfcdRO>p;YcS1I9;MJzvrRO8GA+ z&|+%NM(Z>0O9BGpJ4%Z#UT#YA5O({-#JIP~l$gj4_ zPa!<9vsvyLynM4IzHF&|3Xok{RH-`pMQ6&xC3iPVN>c6wBa|5+31sAB%rSiId6l0i zlWq{MKEK-x;a3W~7-LlRP~)_!p2(n6w4w2+Nu0Ps*&TYS%_6>6vHOQqK}$SXO-nhr zfM1hUNk55C#xN4j`aCjuFS6-{`x0Xr9FJwh2#(nua?ECyaBrv?GdlUPgyKm_a=H(g z!z<(9v8OeGbxt2D(cZQcDD8}eAJDg3=hHvq4n;N^L`ivDXy%&tj86yg zCJHJ|ll#W8x+Bwg{1oN&;R&M3@Tih06z8ht)e<;1tuMv42UV1biXY8VY@xC#(Bz3B z@OKO_-cM)pT=_>c1yf_?Pv!NEvuQ#3y$@t&U5F1FPh#RxruSfGsF^mvVu|U1aF!;J zfF-MnLC_YxR|;#vDrTMP%c3m=O9#fH+4-;F0?`MOVg8{Z5L+`;8Z*j@BMi8W@%Y9Z zU46T4BX$^~YLaEg6&^xQrM~R`%2z4}R{8>R?FH757{eEkr z3UQ^PWk(2kMSnpf+SZ4- z(iOf^u)v|@=_`EknVuGu)_|%-*WeqFrOU6&`S^AcE-(0OhMjU@c7hz=!M$%Rz@EFS z%F)UH1%{f{vi&1mO1830CrAj7k>ID0G^&-EPwZPcTpl~)W1N)O4PXHs z&*&{I3+TTZ=ZbQq$_0yR4>NV}>7OirW+Sv><&C92<8Lbdw=u9k+nRZD@=@$a-7w2} zy}zq$8f3Z&`&{jfHQ{pj1tgOgzmcMpFRwP!U^AUPWt% z#sVvwb#1YBH8wn6Py4>1tDg%qK#nP@qy@gr7%X?bKx(^`+d`w?p>%=0h4Da|i|{-+ zTdEqPadSTfZ<_T)6FTV4odel5fupTp?{W>yI-)|Y$hotU-)l-$< zD-S&C3(2AjR($_YEcAdKS=L~Ra&!?&{+BZRa)X_RL=Yk7I0g|YXK#g`zm;PKkv|hI zX(LelqdBbL_Bm=8sroueM0?&=#gpXWdfa<8ElXo+BDGC_U*lz!muytVJLI2!rkyB0 zqC=jLTQ+N-{YNg(C{W=MvgB*NeN5SANks>ri)hDVlYVyyp}ddq{&vbck@_#acHMs(kd}`nRvL7I+oj4fn94)6p4`_F zo(KQUaL7Hyu;%quT-RRZk3g8gdQwp8^GIn=+}-@$gy;XLN)s>Th?N;^%iua!JyJJf zruP40uupvp0DLfpMgQ8(WK-(N_-~bBF1-<2$DmoF927`F7)dhEG zLQIkYK8$katEedxw=Vowb_znIalJ0UQ)pe*mu=cR*N!7D>#TBy8@;TzEJoRfyfU?j zSDtQ^Q6jJ1X9lU57P|CO>&h$7S$i5ks{CXbse3hNXcc`LEZWiyUN86vBk#>hDbqONiu*H2W_q54!eN}ZD9 zg%=HO!!3&{o+Yj?Fs7rbCS8`C_p!ZK<@Ls*Jnf~)Su}s<1=FQVWNw+zmxo$aaT^G& z=)r(mR)*x!&H^Tw6+H7a&%3R%=gQtAu)nSH&X6UKf?B;+;>Ih=DhS$)B#M&R4KUhB zH}e0idIZ2YH9!3(^t+;)v>%F@H6tyhIG!_H;s#jlvb_|w^DKcEq`LEXg)Pn4ssfFq zSWmCD`-oU(>RSZ8_b6LkaeRjn2Cw4UPCQ_SrNg-`hLMn#B`rImVYhT-{hglRm_*oS zoGHqZMVTmPS5@d&$7#oO=!)EJGofaH3QU)$BfbI|+|mp~ji({!7T_Y^$+gp^VBl=V z1JL%W-qLvW{R|JGxGkV3u7*o55?G{A*;raVcd61zGW8~0U%~tuaf+?Ed=MeWc&|T;*}sbG(x#XJ$$T;+j~FL1 z59^FlWm*O)nR+ARvMJE>1SeTgBrfg&!ez%v^RHM`*#_hUbd8XS*8tbKWE+yx0#0mK zd9&ow*8sv1RoM}Os@>AghAWP!`~{MF0#Uw^8a#N!WC#nYtcV$&@%hvT~LU5%|SVC$~@y9rgwdJkulsKN^=^xFFa)c!d6=oonN z-azo=0afnCDdnV5Q1WykJ~eu3&(<71R^H1(-7Tni1mmUZd((}r`kS#uuOLmny#s4@ zsJs}y#f5KCHA7I#iUD%%6|+>1bH*D@eSZv{!GdlQnE*mYNGhdseLUS%RnS;=GI1wO zoyeeL3WoiHiguLUZ<5dM(Fw-%WnA}$%I*y!5mVh!_Abgu8h` z`KFK0jqzot%jV}1%w70oS)M}3e7CYFN!g16a~54gUEp*PXp~!Je-ioO$LvO;5@bt} zBd2iB*(&T3quOUPQgh2$N8Uuq(}P1B^*F%?pFm5LQCj1xbib}qj3V|74ws*P0vC_n z_R*oOKzIJFs^nGFJ>OLJmU64kVJ3)TO^^J^t`LH?K$?82aBidYpwE_b0FsGfQDDev zET5*LCzo`K?&Nrrq>N2zMo9O#Kh&n&ql=I7QC(K|n)3DpK1-A3zjp9!s|vRvFU@Wu zCHk0`S$bmcDdiuG;RL$M>_dm0C`z@H{XiANjrBz?z_9H3TV~Qz1Wr23#yu;S{;Wkp_TsVVt~n$Mg~x zGaPwif>|w|S9wa_yFSW|`7Az8&nK{NDMs#2G}--Hv!$APXJy;}KP?us&$RKBP&-1$ z@}6?@eeQT769#ws3sw(rYC4(MDVQuB+ggLLp}>Pa+7d|k+)L#&`<-c(6bz8X!o|KC z>?MV*<|uxCoGdf*cmZB=0cVq^b^Ng~0rBgxB9Bjd}d(0$izt!JYC_M`St)z=Ac!38`l%k?NWxx?$ zxJ%hP5rn&X$m?S>@;I6TBmZ2_99(*fi(8BW6Q5d=mTf$Y8T5Dy`6mAly_t@m)jNDT zpdeAFOtS$xTU2xre+%`M_8ftU>?wW?@f42+KuPX!b+ zSUH*URhq9t9-fRSmoDED=7#Ey#?`VyB51qrPZc!AJBQ}Tvu^{xi~qq~dzbm;*}DlA z`u)aDGwwjr5mi1^Kf8sD-N6;tGucBk4%xQ903&Mc1yH|Id82VU7G;boF63svE4%7_(g?tZ{z?z{V8b~pA$W11G_P#GwgL2gjo znChmdA3`W;EpZ3T0?E!pPa5|lJi4-nYfq@GUKlPx&11?@S0qeid-{8<=pI!#iXx21 z1LB6Q#_f0nP*qK2=S-kAM!)T=R;C5uuw@zc^dJ=tlH;#ZGrUBv{v;yH(y6KOH79%277 zp5A5woFlzt&y%J>HL=15vn$48nc=V=5$H&5IIjf?X|PdZ4(y6KLiDjvV}HASL=TTC z*KD~-QYAZPu=^Hi{#q(*%lW~IDMDHhNMSlVNYT^9k;sbYF3 zNywJrlCiC@*!Qk+Z$OnUC7Q}^PEp*p#F!P&872*UItO<5mjhoR2>)Xy$NjWp&9gyE zN&MBc^Ttq;?9zEb^Ut7mLf)1fgkl>47*}_b*-v5%b?)chO1)#{W)M_utn7niJjdJM z)*_vydObf4R<=Z`{U%EFg(~Slsh5FA#|G}5{1<;qS0z0qWq`F@&bHxEiA#CZbv=FG zNQLj09U3x@>Yc_tU%+jJR92eo)VpWa41%EcR5>J2%sM_wFPAsgfEyWjQ{zfizJwZO z=m3_HjGPCtka~)$cS>fSoPEg`l(*l_2v7x$E@O!Shvnzlch&kq21Tq+JRD;IhvSP; z_JVfu*yluc13x5AdO}%xOH(^`dRHnkOFRsjgY-0DKu4SBs;cFVo@(B3mo+Ty`db+b zRPJQxwak_IEz!Q0nSUdjcTND~+yi8Y2^8N@(S>sGNzDG5au3(a0k`xabNT{ z{j!+m9W77KP)17VlUJ$y4&M7LHY_!y^Ng#Z1c0_%*gKsBa5N+dV3 zKDe}3<=2(F8&RwLM9+dPR-UjVaFPurb4nQ3AIBUkP?Om7iqHk9W!9D?XRb2J}!|jfGjLQm?eVfl`e+?(<7oEWX*F?k1D8 zWJvA15RZ{bD{+}=lTB;E^$`s0NF80XJE8JdOUu_$+YFjxKowKkul!@`FuHt(tS`iy zcs9@dnDdf8!<_HY{ny(cVU3eiHO=nbW|rL1$I`UolsURXIc#FP8z)NNJejaQC)JSn z8dBNHo`)nSj`~Y~ zWt%RxvpC8fWtGN{X-?_(CiF4BHiI&bfBtO zQmZp8l5h`ZnDiWB!rI@^ijP$&85k@K@Syw^bOBvSP*oX{on$?vxY>QA)1M&8B~>$0 z0$P-9mIH`)y@ou#&{?A^$T|~Mbc)nl?$a|GgxHU%@F=4>7P{>Yt}ZwZg^H=-D2GRJ z_2EZQ&DWKmQOiSSP{v2G`xaYKzHa|wAUUdfgw$bg6(X~ZmHQtDJ!YuvM8f%=UG>Hc z7ufjB%lw?c0AC*of9E>O@#k5F)UPH%w9zu34y0{mhsR-2>j&18uI0DY~8 zrC%Bk?0m{lEiTGsAr&ZZXiTXpVWCF0r2XnEpf(?_Cog`?W%tw}C;Q*C80X|q%u0E6 zjL&bhDk>qms98m|pd^{-C5xnk{fTI9s1ah!%a*er!ju~22uu9mc;-zi+*eMmBw>5G z1WEV6KK;KmDf%3 zd%~KHXURFbQxcIY6Sp|gz4L+f{I|OrpRpi8@YVs=_O89Xo8-Q}P)&;^Ge4tp@ZQajLARJ4D9eSVKRC zK*VB&i}(G2l|0f7Z-dSy(BaGOEDz2@E1Rnt2gzyaTpX#hr6hfq5h5OTc9)VC8GhqZ zSr$*KZXs*0Vd;ld{v`Rf7xHLSMC==Xip;2^4#Zp3Bix*)Tn?Qs)J3Sh!*s~bqk0=#wE8)hXF8|5>I@TynI`cI)% zsyJ@;a}@3iFI7}Ok(tbP}9Z#0sehcI((X0J_KZK*QwU!ag zhseU`k&jIoA$qZeEo9vvK9@Al#9O-D(~phjX^o zwc=O`2qxR7lqXN-T+eXHf4$5QOJz6I{zg`taWlO}N;<;Bq_T)yc*&PTRnlmHPY8~Z zk@L7-I)QbBvD0}&w7=XKfk;==17zeQoP;ZJwh~zeP6xd3;(cOxx$QAMm@lJiU+cqN z8_=rfpeLZ?@o#|W-OAsKW;dziUCU9}=_)c@_Fd8*QROGd?aM6rJbPc|w|QXv0_9vO zKV11gi5G8|l?$PGrmAUW#Fx>czZZ2#`UBV`=9*-jPxngyr5TlycEuVb&9XRAr$|pm z6-rvjzSW?vMY;UCM88SK>uGpSMsN^7MV4!@Z%x=iN|33Xy^&OTp6rbR0>8E-uNd4M zD87CUy*HME%-e?oQBTh!cbVH1ZnswzL1{{oYvdO>h(#O|WXnB1J?!92!96irIqyE7 zh1FH|knAr}hJ58+Bik>qxRH6V2`<-#m>|z-_gYT`stGQRO^H4&rT3k*(lC`Cz^;DkF#B?ba#B z0%^P3x`byI*o=CQIF8zok%ltcdXtYa zVKaXx&UEA)bn7pb6@<0+x}--n&1C#rq=svpz8qQT14yiGh;l8z!kbpACM^5T;{rTq zk=YO;7C9Q5p|>n&uae-)n9gNo+b-S*5cSQ4JelB$W9Ul%&t&)2tYE*+k|1x#I0k;x3iQ^);XPji#5IVEu< z5BNjxyNzcA1TJ|RA&G+4@?T>!h{+l8ls_25=y+mFC0xqxOzNIBM64Y4z+kw4iSc1o zIa5jY!E!R$v|%>XE_Wz@i15Vz07_Mzh>GP+mx%7P{kp&`k&O%BnRf%2cpw{UM`hZe zoh*m>?wRuFAeh`$l?>JKvXz~n1s&y{pM750+zms&sUz8L@r5W%d4};orc=%>B#3(K zT@skD{@|lc9b$_1JCMvE&aY%$<=t3A$-|VS%NxqPbF7u=fQdS+O`}>^(LlbYzXvo? zXyHi7Ud-lsQsnqXTrRGynUv>ecr_()=pT+`jWXO^5?UrnOL679eL%x+cQWk6H^q-) zBf%8niIR=f9V$Giccng8A#1ekzrkXMm7~Q?ROME=c?rAwSB2*hs2B8;#oevNw9#$z zWXFgMy^|_nlf!>=3A4WGxE2!L*G41iZKp^I=lzbue|#H$?TV^Sm)2c?s-+CKV>teL zFg~Xju|*^X2H_9ll{Zy}|I5oATQaibaxR`L<8id1X0Fd8Gy9_BZ|{MB@7<3u4UG`H zRm~(RykLz=My$^%_bFcq+C>HZ%P49)xEXJoqX}BBwPvAmc4E-dF`MRDaRcLzc#Cv< z8HGDP6f-|_niafPWS4{bD7J<3c*DcW71rU%cz()XCPiD}{Nt+RF`0JvWj}0qM*2s(xGYl*>s_~0W2t)Ory7yGc3f(4~ zgZ^YWkZx%Md?T&mSKOG)^xPmKb;l5yGlS5o%4gTL#3$uvwu;K)u4v5F6F!&mewp?k z9;>|1{1~Vi&Xv0b!F`EurrO{bVr)vY8Jmj=rD7e398l$J<(`jCFHlb(=G1L4`<%+B z_)%1$W68yGs|3hmRMZK5;MaJ51z<|13d00NbLH+p2$)^p zSXP#=7d1hyiq9sQpFm!ZGzO<``y%9sH>z#*anX zN0m&M6~kHY{mSnr_Ob0CP@|{Os&;lH*ETl5PSGU~=6D_x9 z)PMo_+QE-eba>F#)!d8LNI91JJ!h~oHek`jW zd18%^90K&(UsTNx7JQ~YjK^GV_3PeVeQEv*&p!1VP(NGc?39&0EZ&-c8#|**DpqTT zg?eMz=7>V@Qz#&2cQ% zHre(FA;BIMxL^9#i|06BFyWbp$^M_Ygn#64plms~nbdx)s#vNPrwPVnvtu;IDn9J% zkrBGPRUdIMT_3Uaz;zsX{eBs4d2t(vzSxhB|IU6gQHdX;lMS$j^L9OI-lnOK{&|LqQd(~hqRi+%)o zq}7;=9QjuFlp@NLEOB}ywIqYL?o)pHqH75=3_S^vOO2iUCD>1B>B`+w#&IkKhMzY{7af~pn3Yqd(YK}(7a8d)wHI3psJ$Ye zPrgG7KYto9T*UrWx&M71v)&J5bxk>{gxofNq@Hvd#J^}TIlkx~kh?=U%(+h$&yh)5UN_H7><7Nv1Tz9N!_e zHFgx`)2I%ZfIwb78O%{T^5*wg{i?07Y@~A01%DH@csSR(MOwXuGCij8i1O$C1UQAU z>^w2E$UzDB=INQ?Cjp4Z{~RACkru1)a61 z6G-?<>>SF=9B5h!(-&Q#( zLYOv;bPhCW#Z$4Tkxu`$(P?dv8LDu5?XiPg-Dq4X+Tt! zHHS#evPkMLFqwRgg*Z+pq1~t(?;Gl&i$%)wki6N6dWGX6zxLMC7E|arAtTbHXnixw zGCmN~s;4a3gD#y+3(4se;=LN>A0Ur&|txGTij4x2`L~K9UgNNE1YA>D!6qR>H zAVBa{>M_VC(nW%8G76{>7)r;{gVB;~Q)2&xjZdm@cfA?YDlKAAm6Dx4m$3nz{Z%ZeYOQxRHU8KST~ws@tk5A>5L`3e>v+55VfQ2&W8xu zx@v1{umu2r|3SjnbY*+m@MLl>bP{0lvOe{1q>9uhzzo^M+R^HeDfba7)6sECD92Yp zy`W--jQbaB`7$%dxR^`PgA?Hh0&C!#p3xg~rZ$Q^L6BdaCt_0-OE7MCtln`_NR7E5Rcik9MNlxuGaz#LVyQC; z^{b}JBK0{56P9{A1LPL3EV*j6TMAbv)anEVhnmZe2UzGhRlQA@Tur&y$F#_SJ3uXn zKb0Ck6@-l(gC@(?XGGA?rAu@Paby$uOfu %gm!F7}|}u zcXS1%f+jIoo(p5LwWae+$84FZ@PWa>&XTTo($tFA8z`%LVWoLqs#@7&Af=Qu@{&=H zSY4&QOSzX5+y_R;@f#U-6uo>KqGb$?lHKJLqVTG28HMuu4bsT1>&&3c?BnwjOIBKl z8ND3$dIqIF!v7p0hqa_(=^W^_af8((wZ1a>2O-z+)-zRg2T49oMiBcBSakjan0`l( zRt(h>w?D20QTK4KyxbWyY@o7ZbdEMicW=g#XjQI|Wh;lI3j5>Oi~BrJv2XL0u%42q z$SVvl%MLB)Bice9`x$V_RsOmZNwMJLpCabP@s)a7slAvnWD0lwLZgTO02ubu%5YvE zF!s_Vh>auNnilKS9dp86c-dA(e)&_(F1T9j1H^G{y^h z*ouelvlA?+!m|+r+wOX^K@o$Gc9R#zTRE4p99zGx@?%ix{GLd;Fj1@Y9O>T|ufWMz zo#{DwR?$r_Pr-t`!=&{qghIU>KHaj4JP#n`B+J`_ApC8Bw3jNJr7Ka%^6bmd^TTL9 zHg6-K2(mp$*N{dSFJ3JV@jp?>i;73f#9}U?Q89<0H$0a(+K80|hGYO9<56n4 z!DU85dbYe!>8qr>z&U*}x@xdpIUnRp$7F&d(y5y&s|SoL--xFt#@*U9Yn<}TlCxq~ zOT{CIbB%J0lOCV=>||4}e!z*{iS^MQs!^pAaEK))M(gfQD<_gDDHWzm=OQ`sM+(HP z(Y61cjR2(wRA`-%nr;K)IC8m^30Fv}969qv=J* zJjupwFm)Ou;Kox$y5)`RSLyw#ja1-Sd0ub3IyTT(1G?IJ%Chn;cDa-6WB+*D3@;?v zrt*x^`8vExKwFYr#<{c{La&G(Da|{=pZxvwB=KlDu!D#Erb^mLowY3e3(l+~=*AX5 zKIbb2f1^_+uP!gE_Zy;br}75NuYaJxGeB&@-q1lDc-{mCREpbC+V$^(P>K#O5G>yw zjY}cV93+-mu;gt$rW@9c!{UzCLRrxYG?{kH7uBywMO$-LadJ2|KU9@(La}Oc5PjwS za_cH!EvoW4#2jc$jN^`x)!R%x3l&Pj1JoRlwG$4zIYhjAI)Ah5lvI5**+W`TD^#d~ z^m2oIBUQKwM}j+K&qmx3O}-ZNc(R)kLD}v_34;HRr1K7p@%|t9Q+=yDN#yMFAPyn- zAj3Xl#Lgkas+diU_JR}FR^6oTZX;h_HDQ4MpF(fVb$bO)+A=^c4gmQZ{{)qvRx$J(RVA+F^4lv0 zQCmh-2BCMMcFW-%2_+~K?sWJZY-TO>tt9wwU~?STGh8DgT+Ck>u*@rFAAAa*I4*L^ z_u-miEjE*UTR*|h$k{zm%5wV34>qS=s+>oZT_cH(NF0KMy6G9Zym2F<+Hg1*KRxZn z){Cl&e2HzT+`ndRRhA1t*_z;)EU6&#?;36y*vUwi&e^cY8r)nl!oSVop8z&nm~f;dj2LkUJnwOta~l;P7sepkkue1X5LEO1)CgKxbD|t7aBYmd);G=b0bv&Yk zYR-&hqw}sZZ|=WFEM7QNU4Lrk@GI${=M$s2C4=XFImDcgPVT57y>6m-bg7N(d{PR` zoT(Q~L5IbAs1M_?W6!5}@Lne$l&2I>1fc24$Km?`%O=Oo5m=i;Ww~*92QsTkkMB`m zhQ`FnI;%MsZfJ|+!(9Vn<8e$3-wg7ht%v=LEvxtvjU5Zsx|8ny(D_#jM=t< z#B;}H{BN%)zsr5u9)t*pg zf{x^&*X-=WNl#0JG%G3Ra#=mUfeH<3h1^VT#HNT)uBpdYgMVl-+V?RI>DCvd4bqF} z0rukg%7q#Z1+kL=-9A;`G&U?8g{M9Sc5ZzvNN)gInyG=ha*ULfZP|{6x(-smLP9=z zh|MZ}qEhnF;UbbbOH(W9f7R@gk1shr*eJXQCV!)X>>AI~BVp8>kPFG41FHR^Ttc&oy* z4!1^PE(li~B=f4H!JgNn+Jd6#kzxsiH-+o2f!h86pR}v&w5WR?yFC2_v9?gTHqTQ? zk36RM8uIVoaHUgNaME!&Z+)y6c9ntWKv?lPqfEu;{p&rK9orz()Wl3s7bWvA6$80v zU1_q{t5c(bVWI<(VI|FxQc3+}ayIO1l&(#{Fex0Tl)MB;!@X5j?1USAM4wKe8PyWo zEV{DtsdhjSDeT65cSJ>L*V`Ed;p94%M%N2-BXcMtJ1U`Y7pKdY2aIG?;q0c+X0}!O zlRw+oU$KAVrU_qUoVY;=AX>16}06>q6y(w4pd1PAyJ| zkbo}{R$uDbHmsHS0IYF)C$v<;dE@km88W{ic3fq3P0>S-ZU$Jnk@4s*n@pJquUkh9 zDz}UU<6vaTRQwr%#B`_&`?TlCZc17B$+B%6iJu--4i4uE05#&8b61&GUrO#9c5M?b zjSZ6#O@)h4KZ^CJU^t(zAq1HgDOc{YvH33zyR;2+x@Gf7bGEwK3#5OG4=srhj@_+T zJ0(CmY70YMTA#!;b)nXaV~lPlYo;67@@T8!5HA6&+JVq`ya%QztwWYa>x`9mTGzHo z%rr3JmhSmf6~wj3kGo+SAM4(!9Ei0QdR}1$LF2Bi5N&)b7FAqZR?i0qF6rK?Xi^w= zomv?Ya{VIW^Vv5L*sR%@7XRq^{XhwqKdNa#50;dhhTc~Y#^NdMjBcAw+=8bI(97-w zDtUutKvkzE1NVU5*HYuT{BhE@Z#X`qE~?tISFxHKe;2Kw+GIf@~X-WiFHxvy3*-8_QD+WE7-o!>c~70W()=~^%x#T? zb?k)}s@_jgg*-gtRGEhM67c|E#S<5rT+L75QEBQXWwse}b_V{31;fyGI{joe{eQHr zhxAGLW9crIZNZ0pM0M2}j|kMh2#A2ql8@7=#7I}C+fa6Yp5W4oQ!Qv|q*mO*s*=)2 znTm{)OkAx|Rbbu3WzdE-PrUwj2m zt+aqW*uMb{4qQNOEBF#ldO(lxL!w8Ikj-tXpoYaN=fjV>Z<3@JvjHiC_%yWw*NIII zg$Obyp;WpXX=h^8g`Z`Lcz-lfG-qgpG+u4?Ag<%0&}K^p$%iJerRfMHJ*l}3<<}RG z^lKO{jX(hGw|^T{+pO%&r$kJGs#;P~9YswmC-p@@AiBJ@O69IlFweNJD7JRG!$X?E$3 z0HiTL=+-F8}p|Yc)HO|V%w}eLI%A!5pb=-1)eFfp(rjHC5k5;oZ5K^dl z78w{tz7EJ9ca)2d^v@Bjq)(o6vfgdFW>b}qz2KyGH{;seT4YSJ1XCLi5au_N?opuB z&k696Q!3r0jVc#y;Z%EdEb5AEr5ZMNL ztu}%}W3h{0EyIJWAgU5Vy>KcXEN_I&nVY~>|3wJn;Z^YbZel);w9Fv+bRGye>k!A0 zbQ{RN{Mnff6ycmbR6zVz>H*2W$!3pKsP>-%;)cXJL&babq_GmcE5Rk+ibk$9Q}sTo z>V^A&hpD#KHdIA{yIs<7m>TmNbA}4@`vhAF>-si6bp%jBeVMa~X;&8{WJ;(0PK`0$ zDoemM1*cMGPp&-O)8wPm=u8yY2*z(?p)=28$PHI)CGm_0<$l!*&&XvjXkjU3l*r5E zluW7hHYeRi^|gCnk3B8-RK3$v3P zct(PrtisJ>4LL9AfKpOB5^3 z>4TzJI7>0=6ub$sb>zr(G?7udw;MDa^^w*&Ac`N89NOU+JCe!v-#xj%%(}``LzK6p zbYg*oHbooVtH+1Suh-!SZxzwy)D3#=)8j@f-*Yffq~sAbDQNw%9C0K6gM;}_0EwUt z7(Pytbr9`j26y@lTpGl^zq}ofZd55oWk46smh>Mu#aJ){Rv&vD4r&jRFArjqAD!UL zVVC*UP`=VC$VXG0amuk{X0pN$WSqXM0DhMpD;r&q#rOH}yE}S;F3ZC>sK=`E4!&0# zKqN?Xh8|rBwh-GC=I+_2eNYsKFmN|PZSqADm5<|ah+fbh=4H)LZJ^vxqt=MgGrp5% zC6L8Sm4-~5#|UkCj2GBi2g=XKj41TwTotP0?SPe-)JdY932wNOr8ghZy)|c6vW)-D zD8U9L>}<<&)apsPwIynyMoa6@7`K`p)leR#fxM6Zg7RaF0nu{ZMqxZXq8q0fJwa}# zCuAd0BHojnDIidF-D5%{A)erXvJ#i|0bQHxxs*r6si6(a%7mCb-Pv9CN=&=lDU!LL zX;bv%)8ZP2YiC#+kW!SRZLDr;2TRLrgZ-gpZ{|B^~UDgUq==vCIE7g1OO$ z?0dMLi?5+np>Xtg+M4Fhg<{U0gN;5Hqd@un>&m!Xv#KMC0#s%;Jcb01#%v_NJg0^O z?6?xVV;s(0tU$^q?Nb>Z!>>B?Ranz0Ddf6U)(v9~cYB%1V4M%KRnk2J;Mt}v(rXL) zQO8t=Rz0skaj{xT7Jh(?jGIX1L3nO0)u*^c9UKYQRNa{E2b!;!OV5M`5bPD%8~vn5q^F5j9J+>WK*T+j<3-^OBzUY1GaBzhJZ?54g&h=t_b;Jk&&eB z%__m9%GITPEN_jZpp>cNI#cPdq{E)lWyr9o$noF>sUnY8aEw3ej-kwHZ>t4_I+S3Z zEKhft7-BDHJsv`A>b12lKS8$109F2R!)DQeX6U8sr1idNA4^^EQu_zjI zpzy}p#kiMXm9r1bCV5;eM1I5<`3R)Lw^ak<` z(+f7ibE4}aJ1A64u6bAdQN>&2pVm(5d)+eU!Nc zH0nZA#x<`R~hwC>ZupCwNbH|6PKCR_axQZ{JSrs^Wa2b4f0p4Rvf ziC_C(&#cZtM(vhH7ZWrRK8m%nMteFBm33yq@X6JT*W(lAW@qG4pY8BE(4xOgd1#n8 zEti?ghbTKpn;xLtj}A=1Jt{BlXq;eS*pz(ulsUp7lPF2e*zcwt6n5pA! zSywzdzLc4Ql5@W~>dyDSGfFAFpr5>G2~Sw7M}4V1E~3GGRrCK~`WY4EE9HpXpr_9O zdq^Uqw&e%#aYr7k-sO}AVF|JFuo&=m*C|+%Nw_mq6FnHn(!kYB_KqCyPo%JK^EM{Mut`+wOA(p3+p)73NDQAZlrv@6ERr64h(@Cj*)Wtu{js?v26uh zq#$?KleS?-%-K5CS)(sxC!t4-VYg3(|>i4lFGGC{;;a8O7dTcDzb)%g^yZ z<%VvrBe8W2<+-thqS@HL1;t1zIC2;cbC+mB6;RLz^UOY&fT{wOLtju;xeCWX{ZVsd zU^@7;R5_*_L%f8jTE{~f$&1iF(}QJs98B;gvE&RD)r*vW!v%}-HA8a8`{Ch;t_i#> zq4MWi+*qf7PKl61Cy=&Po}DITE0^^VD$!GGo|ns8I9!O4?Sx&9;ri)&&Rn<*)dy>3 z70L%xCXy>-jBFG-cR6i%KsNF>XCTW=9}WK|TY0PQO@q6~)#kFt9gO^rRW3i&O}lP4 zDN+v7VnPcsQVRCtGRX*%vB$yxxq5miN-Fu($;3Cv-QN%+zVteZ zl^D>-@WG34fYw)((^O9$3Z}(Z`RRJd%CXLP6vv1#Hd;^( zHs z&}5X#_37{?fgCuBSHLvW6G4}_UQ(|-190&XEG;_10#>VDOM%LEZKg=}zABULx&zM` zmg?eo=>yR1W8ROVcN{t!!Fsb)C^7tMGS{oQ?U6G%!=-B{XB6O1s;`|(EEM}N#AW$( zh+~6ZxLo@yAYJY@WG6LsduOj$lGSvG#LTl44j#$lGBV?pp1%}v9RGpT zQGL4W{)7}5M0F&Utk6Ra`k_0`Rn#$1PbP3i^^Mr58Wkr%JCfRjl{c_*I$zA>6sbbe zUS1^P#@a{ieo!m7vNjj6aD`G?>ljMM{@9ThTIH`#=sfEGy z^&3Qb-&?S85@M-x6KVqc9f^>Ush>1dmsEdtfVCi?b|T{}-Q`s|!>g@f7ONgr3q3cp zs?2@coGZznqm@0G0q*uweL|k@hxjJz`NL!xeIOaLxT}QD`Q3^hzki`Tp{{)=;Vxiyt18UHO|dBSYpzv~>iKTuQ2tU?ayVOF z+`?j9LXAn8Snc0U9>}@9xo+U)7``Anb zt7no7@gajI#uVUMamUNS;1mtUR5%QSIBJH}x@Jy-v*3;52EQRLH(S2!tH6oBMxF!P zY`W~_BGx9p1L2LGtWMTYML}mI9`6qcaS`aD{6hoA(OhpKH_dkp@*uA`OR>+~uI#>+b$= z4nha=^tegV;A6Au*Wm81QWX|RTsRcm?-M9tAcU@>20~@6zd2JL-ZanS-NH}B`j@jr z-gMm+B{A_F(6=!WTJ1b#?R%zjc$sxoz+6j61y3|e;!;_j4y{dm$&QEVR+B862BuQ+ zzZP81S6=3I#6s(|3{F{D6;}CJ6{&5hXP$)jd+XvEvrR*lC-55MG!4Cy`wR+#2TApA zF!>FkY23q_M#zip7(?6b4D#(9*wdsQkoe?Qn5v`fRWOO8QqwtJOLJ-aFKGOC3g|;< zXeFg|bcU;>ZH04tSMXih4Q*3iw6-F(K%17TDLMI{!+$tNHSYLXBaH}GY7_$ zoh$cZA&|wo(s5LhZzo-AFYkPgfWM*V?nI(nL*c&B9l_Fq`Qll|(Up6u%XCiRU=bva}$rVw`fV9UKqE{p4()UFlMXF75Cv?-D7gixKFj7q`VMq5Z>x z>oCM8dTet}Fu5(GSSBdvN;QejO(;fD&z~vR-o<8kpIZYJ3hGgTMjIt)9OCw;>TJe+ z$r&wywIS=HloA9&*4D6!k~k?W#unbpB_h~buu0!pPS9dvL($iYM{4T=3gqe7q@hx0Hjv5{A@;Mn=_68&l92$JO{Jp}<@8W!phXs0RrpPyoTHzhwkazO zbck?CpKSAg^)zbA;!9}2RcoP_3>|ga0T^xB5L|jsaYrOrm7CWO)6!E}=Jrj<1*0=V zIp(PL+N;E9t!s=2Q^4*6*ijGhY$gXv;Xj~MGu_)6&RbGZ`#P261l<=ZkM1MZo)dc| zEiw``+Lw~>66%$~Vc-s#`i8nS4?*ZsJ937ewh2ITC0>#YTqmh3vep|A{S7Dvj?t4d z0UIGbTG8fU#0YBes;~sBlqzjRD+yeO_4)ibj)DrfOOl%-sH}z(j@#vI3qymP?29;S z>bxutcjhAq9P}Og4cOGfGl#WqtB-K@`N@DuNVI(ooep`JWTso7UhLu>UmNE090%v@DeIRRLu;-S(Y? zM4rw@2Dzq7!eA)t?~0r#8G%Zn%IzS(a=Xg@qqpggo73t%G{qy|Rz&`iKh{({bc6&w zs9?R}4x}BZTf$Y5*jVoAiDlTUdsBHeDI9ezGgvZ$3@hr8wFXRlR}J}fkJ$^?(ZYq^ z$MG`$Ps2erMd8;NsO#?L8DF;%dIx8w6R&siv zUW$#(JPM5yax#)5!NGWY9N0KQ_aw@j&4!1_j*NEreykp8q&!QpyV&46?U5wmsuAaX z)zb3!YskS!+EpkQR!7-gXrv)Thzy9NW0ZVRmJJ^#h9AEgwt3aoX8KjEc19@BcjTPY zQsf*~2=tjWLWJzwe!2&dO*HEc)I3-!EEingr=^0y~ zpv+&8GrrOAHCGRb_ynoja<~yoT!UksysX4>6IZf*@}_I>hN*&btTkpw8F8RNfzFYa z)P|Quc$6sE246rE|3QxmkTa^tC4Tf8a_l#(q#a~MOWl1C!6@30iO~qA{ZDEk8osG6 zG>|k3P^I;fkljYS)*`V)UL1qP(GMh)B?9l}&6kAg)(-$eb=iKGDnk4fA1EWn@)tBc8XGFMH{ho~>ig;nYx9CSh8(^2;?_un`6B7+f zb*E^h^DNrRA{T~2P96DUXNpBaRP`vYo|GuD4XfmujA|ro9!mO7uwScM_s-Y)61h)U#udc(#G^#Bi!I znt&CL*BZPXxwVjZ!~lP(+wpXk21SG*E3I^B5;`Q(fj*8E2@Q$mdOgH2iHt%^+nNmp zJ<%<@Ss8V1$P1c?WXUKeWryO*sM*CxMrkO#1O5~Yl98!!z+<}iAQuka<`PC_RwY3D zYhj6L+Uue;g^0Z&YM%xD!n73*n|zq)*r>WtQL;hP(+tR_B!qw2YN2`+j#2iDp|vlH zkeAQF)V->_cj(exQ7~LBOSI(BeAzn;Y$OMLqE;>-nv*OUi|gE^s>Ios zp`iKbf*p*sf2`*>K?S#|u+^ykval&Mov%lX0+DINxwHyo`cIx0J3lHNRRShgGnL-5 zpJHxFnh_y?)I)|<2&FFD`_BK0*9B|gSq@p>A;E#Q6%_*6QMT2sQq79=jIT9TA$jR8BrMtGX&Vy|KtFb>;G)6pQv_9_{5C82LcK(N~T; zAdhtcDFtmFE;ecEzZb#s-~au~T`Hr!^_JC^ zAKoraeWx|%SW`kw0PpPgsC!PQ3hB>{8cc{Qj_Sr5k2)*;X-f^=kiK$HQ6I5gQ(lHyh0^aVDDSe3{J%O+%RKT#PwZW+a*?N}=kgt5VY> z19L~VE;d_C!KcloDJy#(bz{ZnczLG!F-*t@n24i)WYos_4A|bofCDe^SKx8OAq!TU zPl(?%#=5FfhxVx69t%{({e`!1@NTTi!(kMSlGSHQv%MM2_-o}9;zzdd#A9< zCvJW?S&9Mj>M`C0)j59XX%=ar*Qe}BWq`biJmTERT1E%+!^(X2vGxl%WimGp(mM;E z&jgAd%DrMn*{D*T9aDJDreV^=#iYrVa_rt(#IrG_U=Q{o{8)A+XI(9t?N+2nlk}|V zw9A}4<~dr+Nm-mT3^(Ls!=n8g;FW#;3|Zj@7+92tKpNVYAD2yrKiCsp5;UH{)8YVG zH3!rG`2~=9P1{)cvXHs5C?O_`d?42x8?#ny$=*v&FtyPaP;~j89S^bs-@SRf@q-qF z2bk~kTQbdpaQ^yV9AgeWXvE3jO|13M8oryNn#cB5A^y|dycMyLg?;!D&!qLDU{C7+5gH3{K)-J ztaQ*hqtIH+_~aSiy{$@FhL*-&F=nxWD=XN_9UA}0g&k_1B7PmymA}f)V^JUe;Izq> zmMo=OutCQ9M?Cya6<(Wl4$^6qmn5DOO4N;D#!>Zn>}4V|u3T+UsV|6sZfwYI--PhO z=m-4pOp)ukdu;jebUxNz1p~%TVo}weG3}bY?9sgtaIX|b|02;NZ+GIYl>2O8^#IVL znyM%JlWMj7_FU8q!^vOmj&mw$ubkQ9cL*dq@dGXgdjT_k-NvBaem%n`Z#81tdYkyJ z+d1I&HwYyYrm;QK9$>jn70R01j*;861dQWxe@WM70Oavw!y-N7ndn-5etj~N~yUd3*bA3x^b+kRjj!5J)} z`4-L|KU9=_?ql|^qj)7D3?OEc*e5R@8qqSwV>skNcNREh4NINy0kAso07zUN#Kga~ zb!syH4F8<{6AwSzz@U>CG5qW{b8z3$9g$S=+BL+)}v%qbDRW$j|L~H7A3$ z(`vXtwiGhs5YHRH{%6M(|`wk+}E&zOJQ2G;p`XO?)Y5Y`4H1z^Kj%_ z0h!tzr%nD>lOJ#U!)4km1gkze%;I*s-#umMAD?8{%N_Zem7O%Uu%ja`(8e!;X|qA(Asr4%6FTx8utJ#4W9 zS>tl>8bbx2GQ2V>3FJI9Ho+>-Re6@0dU9EbQ`d&Rd0=Rkg%g?4kD(%(F#Oqvd^hO~ zdrnwtShe?xisj)Ao;YE~ceFFhChN?GNn&GobZ#xyv`FQLef@o1tlbXr3T=A8=Cs(alV1OCo*6UeZVW$M0ah9}Wby zDqH~)U*BesZ&LYaP1QN+w16Mni45XDj49?^X89@OL5OC@P<|}|D1zZetTgUzc4_Zt z6-xj2*!Z_LcIrSimC=1z$m3B)y7F`f$e7Bk_3(Pe^&RA+g2LM?@AI)c2P_H+O-Pm( z4|v4)7z?wnZViyF#o#hm3`yGH&}6x&y6upLUQWV)Vo83#7$veV(y+?w zmHJ7xYSGx^Pv~x~sw11IM9c?A_zN;jlVhLqL$r-)+pYtyi>cu)&8~5N)^JX8XLFVu z(UR|~^kDCMXx=L`<1vYg6CGMiVzTO>n=7enSy!b`xQD^cF*SGMNA>6LEq=m5?}=*Jw#r%W-w&u(T0AYz85z;d3c3NfF=*kt2ippkDgN0?$;29)k2)t~V1@g^sweN~Ob zou^snBz2RHC9WG6zTmO3Os!bCX^B|39Y7;Od>+3O7z;~P!eqBF#ikgBa zN%NQLNub1*mHc}2o?((L#I4Fis8sIOh3$M949k5?lNOoN3#>U-mkkFkce<2=&PFMy z97*(mNbL^t>Yuw9Y4}hW_@@DGsRqcT!6VM0%L}$Es?zKZ)y3s+gX8}2fZ>r^XV~+C zhM={x1-~AkbvbSUTYy~qiZP?^a|D;>;%4|IFJBsk@~<~0#$0ud?UX3(u)G`p)i;Up zlJ_&9eXZ1$f72N9NDxSIqYY3!6U}w3ebqX|yCauj zdFBMu!vRkd@_2Q)hc^7Mqw>k!}7&%_13F zgGKZzL05h&FGp%B#m%;hCGA%2P^t~%HX>YADM%VftpasWnRy=bg*qPds*IMjhX-&EU%d<+s zt&xdo9k1xMYA8r}?hW59oWpok=kej!fyw`x!5ihC875hGgyCQe`II2)2=NGHOb^l=KEHG5->)JKPs? z_?cD(Qa8+5AnU1wCs#iM|7RDolB{z9F6~F4TDpi2e{K+EUIyQL_k)04Y5>7%t3mr! zhm16Nj(sX+^O?IOo;9VUvF#A}l1x`65Dw{S6}@x1?F3A10)cLb>IreB4l`VO2oi5U$v-p3^Vjm{z;D7X!z-UF$vtrr z7(ban7pb4dPTss>b!!Ll+BZIU#-r3?)gIn1|8T7+BlfV&hYQ(%^eP_fdE7u3d2dpgn{H+-2`~u zdBYCtFX6il$$WQZEw9#Ay*@+#=DW_PjCARL4Q%+RKNORsXIGVas@m<=3(#(xT|nYC zsq*rAyJ3^MO67b*>vHahlPZdr)HXt3Je5_Xk(D>swzE+$d$keEU#>9be*TE z^bKo=C$xKNiTtZn`x{*uXv%2NuhbuG=hiU(*?0gyM4g2w|Goj#e^Z#eVHmSEP_?DE z<@uy+d#6cW4}fzLB~w+FqyhD1uYh{F-}&qM+iap{SN>W2C(!*`83`RT!Oj|_u7WEW zmGHXfRL0YkSwk-LV1><&!7;v1wM+aIo?UXyK!b5JVV$c+g!UlSD}QfwM#~y{uHgFO z_-5ZgU;T~{O!@mBB;PeLNop#kzUoyFp|= ziuBx}F4boTs6b+G1@Yz>ZhVAfJ^W@4G-)m80lqF)_TURLVkM+Lwg)jL)wvqT^pnhd z5cy^!PCYrqX_XJHVEC@yz&djbAD^~2$7;SrkG6|bnoxnQ5)et73gzHvJJ*>@WDJR4 zw`Ukuc{TzBsjr+qW^UWGjah|q;a@h==xr8#@n3`N(X&W=Kiy6c>0Q=POxG3}`mT{9 zS2lxw8Al+sz^_0neXe21l+{KW^1xL=LQgV4#a}tJyu4hg|AajhdW{mi$39)A2OvxX zuO`IF>UhAvp6Cl%u@LwVxC~tXRK@scQc7vo<^X)C(z&YkzM9Q*HHYxrueF%xzh*tp#_(Kh4%0db2UK&^cb4%(@L1m1{*}`uF5$HgD=AV~=U+|JM&0Ik6#_uf~> zk?O_~A5lYoT95YAsx?d2w_{inSm=){J)*qS^D<-a{YHU2O6OP92M_mxWP|rpO#)BX z0;g71W+@$M=qrIW*h|^BQ=+*lpa^8&7#4L_b-ymWY(z5>e&5M;7)lU82K<2KmW*m42gg>Uo1xdS5BUvKt zPPg2C0^}bkJ&$^@Wo3G0K;7dI)9R`_P?yInZ@uaWwzdJMTl-_6hbT^$t&{^1Z(Cl= zR|&b_Di#wypACg5iC5rcd6CC?_g-UXpi0{(nL1GAhHpPir3eY`L>o0yZDu+Pl;iZNL7otgUZmYe3tX9au(1{GBL7Wb%ye*%1Zy)&e=%^9=;X{ zm?pf#R-0ephc)jY?2ZP&aE|d+Vt;nJaW!atMfonyf5MD^>S@)a+lPFY`7Vgto~UgV zXj)GE=YyZ`sm>?wD?jBF70LZb^lIv}-si?yhAQ8Brzf?@caUY$~xv}73u8jAHNez>)QS9fj( zDwjV5wCP{4&ei{NBKKq9kOMk!>M`XBhuA;|3)!_CfJ9s1q5Uax!ZloaIYNXA%1gPW zEIIEn=x=$*_WslpDsZzzK*$kjU#eb$!*wBXSW1&_FQqA|C4Y6~FY8OjTh)L^2LEbM zpl`Wh;f9W??7=r#2AR;AvhTmx=GHv?Qwgf)h8T-XYYL8!9dXwoCE>g;aaNxEhQzsFEEMCz}THNA1uOv0UM)_x^^o&ZpG*DMlruJjCnvT`BxDl(Gg7s|-X{K_>3tdjkj`_?kLyQ(b0k)W;1%aAcTJ3jUk;CNS+Y>h zcJX(Z~#m{_f9t_ennZ}CC>uGgl^XX>Tg0{( z-UOK&*I?PVAD~(Kbe~DieZ{7}yU#$S@=WsmOCw&YDMk6blA67CL3ua7HjsC8*p-V` zHt~b%#u=2s$wW>BvQ2KX;8hja`{#H0c%%xG-#y1`!>e)R@$Wg+Ju0z0cN+r1IoCM6 zCVBC)W4DnZY1>eGuoj9548{;r)AkSXh4=C=zw^zee1&dP5{U zACPy}ld;+Bv39G4P4&|zY|?cAuN~8K z%Bs9MP~|N^wXiGu`}#aWG-!}Hng}d!pxBl3ZqrgD7rJwhwGnC*O4*Sh+|pXS(bW&y zY1RwuzTMpEg&?U(xs~#x(&Gq}RZZPKJYU5lu_~h1y@P+gQ8$!Xn~Z28iima_KZ@l} z{r5k##fi850PC9&$==>4i(RbM_-pWDNM_`>tiEpuAG;RSuOJ1~wfpfQAK}DU`k>t5dv{?h#O@o2%t-OQ`akT)Ak9LH`Tu_Z( z_AFp&&Wnr^uG<=rGDT~bgSSDPPNSfRv(y2Vrz&=}ZDNkx8DXT0QIiePXg)`F%w)CL zu$0e@LrI|n`|uRs#wAS1G|OzKiKN%UDpoY*r!ok%-B z>~nDP4pJngaTISwwc@!SyF24#31iBmuRxuDLijFpFxK#2WWY%&Dv!!drP;+QIp)yq z5k$vPcP?dukt1(41t?ShWY9F_J+af(UHaZOOp^VQC67A+ZLAOQ%IwWh^pcSb;x2+B zhxH3EX{RBI`OAUsK!0btRJzGIH7bghZ3fyU5oYA->1ma)^3eZR>EOtw#X$GB5#Z*K zFo1CQt`RLo!A##`I|u7}VR)244&sgUT2fARd6-ZUUQ|XY@eBW8U7i0iXUe4^z=^2S zvP31=D-nUBLL+1uVk^5A+;C;^^bK+*2Dvw+|ybh?GAc9rC9m7_;jD~rA^p;blV-x#P zPJ?b*jYM@0q&ATBNi?!$+a&hy&Z9?*Nub61KSTr&eWReU7kYMM zVprlS%G%YOCfVKir^kXykA4RApC*~<<)nC=UK$9M!D^J*!Td8xU88*;j(~fRnk`+o zgVO0LPyHm9CJ=dSh}cd6zLPx}vqcCCJ35);+pjA9T%tUagnY&u6q6vcygO(z!dNG& zh*g-%^)X?C4c7I)gPiGjF|6`P z<>8awq6Tl8?)`p7h z7Y_K=NT9bP*J%~Y7HGbFUObu_ozQo#K`b29qjlNa*>I?2$7=GE@(*mR?lemW7h^tH z1GW8Lz`-Rv<}msxV{9;thXCp%EQt+lW#0mJTm31LD2i@Ra!uU~-B7kc2l&>~1 z@-#ia65hK^lboOTpRI(T5ALsyulQC$XDHGDK z2V>q#<7CWNja+$gnSm#J6FkJ-L6~8qVP-q-fWsD5e6r#L7-X?(mv~#HOnX%!84rcUVqur0J^S~Vc3YKBsr#NDdhX$3-%AJ>q+|q3Hj1x z6jt9*%Erp5vaGei6Si~a2?$MDrYcg1g$FzFHIOEloGd%Po{UtQ`*%Nr{eRbm2cC$* z)fT6Fba`vH5hZPY$3$POx(2?J05`l%fmKb*ik58nDqt@NkyJnMq=xynr~*-D39Oos zF4L-lsW1L8EaK@2YSvJN5MJofcBvg>wo3ed09D}#E33SK=LVJmvS%8Z6Ufs^SMB#G zq%=*pwNhjsE|G!kzKLqe5=Mbt@myzD!#_6SWHZIBwGZ-C{=F+eF7yKzXTam+nQB@+ zY9NkadP-D89!UG=Ek=GaXn`C=xaex zTXAJoSjR3SEG(~2EhAsbeV7mnikiVe+_#{H486;r*CL41gu>crRI z>q(WwdzUkQpl$^&y$zNB!tJv7bwPhx5R_J4!&ys;VSaTvIR@mz;aj404UOWuEvXJw zmec)ZnQEX!SE~Aa7u`Z$68dAxn4y(^ONRXeRlOMtB-V|Csg}vhk<(oec6Tm<3)OUM zkn}7jKJKF!HhsfMK*hUY_K2gU-y=urPoq$KT2!d=%9^pvVw*fa#Sw2)X)z0Qw?Sq% z0q=!jh#@~!z%@>l*&3m1m6Ta)$oORD+5IkOGA>ny6s8xq${`29jv zH1ZwgE~>a226d2-NZH>8a*x#AW#yN(ykWBffSI2#$UG%Mtm!cDlYb+w{+GsIDRZEx zx-OK%7-iURL2H-i7YviOG1aC53@xUc!Rk8D@3j#tu0=+ymXf7n5 z*yFSliVyZ)3H1?-=xHWXv%vXJ^@4WdV+O5u0Qq`FRw+Lg0ZOOsXz7?zdbuD|thMAk z&JHvNPgAKe2*%m)Dg(7u<*oMNLF9+*IuyUkZ>x4(TbigVC3|SBeBp+0FSLL`%~DZ5 zJCu8S;A&QxI-kGpcLv&1%q@k`fgC@Q(x`NBU-vpBF2|rdPd4~mN0l-yb<&xR6CpE1 zd|VuXvEFL3`y3c+dd5Q2`C0JLbNQd)0=c*Kf_qeH|K-f-M-OWF5$_D?&& z?y&`J#Jk0CpvCZ3oVvH9K}#F@7VS#X7G@R7jS0?Jm2qEA1)gP~lINIF2&wLOZ7LjU z2r+|%$!1rU-0#@bCoNd#!*jswOPm43+1e|LuZq(nIrBNYvJ;@sT}mSh=>nT=tjE`u zt?F*b)6iHUEhidwoa33TOv0N0A5$@S#Lb)mcTTDaulMW8?hCwoa6jAFo0=?*KEUgBrZ%`_)$>tjIxyL~sVuL>H5Txj>K3^D z5~DBqkaY)C0v1S6hcv8dWU6po3)kZ55&w0hJvj>u+!Tn4c4H7P+<70>pg|0n`du&! zbE>ilp{gU_8kHjPQqQjob<=nI^ED_FM?T}ZdMkJ>yd8+Qtps{H@c?`qspPA_a^6i* zH%1Gh6AC5hA}c!qnaGdTAjh?x_}KEC$^YCB2k%$FY%g?E73uj6NWVxK4QB=bxZhN$ z@#mM6a|R+{w(=JZsfHGkrh8k;q)veA1CsmY z_hQcUd}ZL^7tQ&3srVxMM**)b^Y|Ft85Bu=l#nBl`;iN$bbD=5r7ZsPe56q<$JX&~ zS#@>LsRRnT7!WJVA`K6#r7d&sdO$woIU??}F#f7@0JQl*oY$wVE3)iKzsW@K>(F_lPvp28Q~iSrz6gQ4%4Tyla`?fzg|#a|QQA zm44m+4?he!#SbTyWge*~m6fNRjR;xV$55HV6kzH-hbbPd0>EQbt%~bKUiOq+OFA_K zpm&wGk7||mWOiUmf&BZJgKWvww`@7=%nc-Tb>&RV0i7IH`E!-it-T-WMeT{V%6prEq9-5+k3n&L(SHRJZB<0@8Ol{i z%MU@_;vG0YW>QR6KA8fOG39x*V?`y(^;ku!bOg$1c;&?Oh}rw;`K_=QJf!3MXeI}N z)4e>7Tb;u4^3OG=O*R?8!qfz;F@ zp!WK1J)Oets_%`<;yT`&xnvpuQ-qoyTW+l^w`R>rGNo~RJY-D zDhSArmDtYcC4l_$V7TxV#E^DB0R2jpZQ2?NeV9{Rs-&DDi>d&i=OkYNFRtaXgHTye zxyrG*q);jS5!3v4H#5D6sKOLZt@BY-^^1p5oAl$*@zs@3K50}HB&86LrsFZh@q}kI zizdd>jQauXW_N9hLYBxSb!ACuRhyW$fW2q_0TOwk408540BfHbt2}DuCHn>(I0QXI z{I;>UJX)D)ZvskKkwq2yqfNt&lYpOp>2fbo$&~C;?nZ ztd4g|vHC;Bpy!0)$fe2q!Hym)VKtNWv~XFaB5=>|Fo?&w3jh0`9%Uwkk?6Cwu|~Fh zwuGr?Db<}yKGmgS9t-O~7uoW)9#>N=*`UB>Jv*ACitEZnF?F(?p@p26uQujtZYA`a z2s7e_5>mVL;x^ofrH7+xyUWSaNLHN4Esatx>&jbqj0{;n0_?Byr5OiikwZH4FsxFc zGe^3uEDO7(dNf~Z?o`b|GFx)E=~0RTs*8zQY%=?2m=Lrd5S}( zbu)?)`BXG(>4mfZPxsW2vWwVluZEyqug&bB-w`l4YzFE=wfaUZoDlR44oH_;$5>g{ zq2N^E+c4&b%Fh(D8Xh=uE1do(J*vC>QXd{fyPA%&lQ;y_tb#^d%_O&nu~JH8CN# ze92H{D8^rhuaL{;CwoE=OG^|NYc~vlgk6V}sdm~}_TSYJyPN&(3S#r$s?a`Nh5!67how!{liJC!NkHn7#f+LA z#b0$sqMDG~ZINx=4KCdO2Mx#Jv4SsT61cgKN7j5`ctp5n!Gfoh%aX60CMk299W6U( zj@KuI2{#(l&6)lw;M%Th&16pw#JNb_KmAl6X0~1>AMR?)3#QG^e5qCqh`5zI{HqdH zYw5@Go-hw`pPCDgRY5F^stot8YGQxlHHVY_9v=ypY$*X z%)F?AFhBSMEPSJ<`^m~ir(N)Gs{yDz?XjKGYDkrK3CaT2xOKd5kbt0qY>c^}%_Tj` zq)JQ}vNji)HcrWdDU0Eh6*Adcz6yzW)Eo?MzW^dB^Oa$NqsCe#tI(jLvE{b zr*}r>&&PN9DcFxcsWO=1Ye8BwC=*o#lC>yj5N|dLK*n?c7-_n_Dq2ZlI38lBrmD2m z6qE8De51!&_$N*wx+41%-QudFoz9DuqbFgt2>~!cuah$%zBEaCgd-mt#KV}300(h` zT2-ViCH~r<1&IQ^eDw1N0bA*@L3s4ApDDi8NA?^r(xlrmw)dtf6qRxtJX@n$Rh?3& zzH}O#teG-bJ{ts1m~>A|Ih79tJn;biG3{3|TBDkPVvcT{fwd>Qwb{g>LhJkVG16sz zJBXI*S57c4r!}rE8;S$ua%QpNAlW%xmrvEbT&F55o1!0`@MOe0rP^#HM5bQY31=|r zb)O#ucP%O|yfg`xc)z@TC2z12BC0K`BS7gE8SQC4RV<}uLM+}~BUe21Y-F?eFi-4d zt=-`nv_eQj`LOnp!zvA80GDb~|2-$=Hv?=4K2H?2D;;PY5xwQ2va6=HK-IJWvmIG> zRreuS-9)CznJRz>S8AC2A4%sO-*nae@vnN&q)-SuT%eRfAzz>ncGzJjKq2g~H0%T^ z24zGU1`$ZnLX}mJM;75uHXCZ>*Yslzgr;sMaAvNO5Ub89$r9 zQuW+VDw(^oWF@KWZj!t#*+Py!3L4B?}xsy*rwca;8 z(+4WtRo*T#)8*rsS!uM+j);S7K;uTL)7a9r%y^jZ(5)j=x)tOm$>MW%m2Mqan&eV8 z7l*GorjYH-e-4FWOdy$sezy3es!1h6U>go3*nBeA39?s zl>Fpli+ZbA7g<{b%BC57x~+x_beqMK{@IA9~<&UCc7TUE|jx1?Bz!}x^a4On(TcNvDmnT#i(9^ zeM%_16%NSwrCQIKb||o`p4`KUH4Eg*ASe_c3{q4!%|3@i`>a_C1~w=)&Tpjfx02JQZ8Xco&Ho%2!tEt}r3 z0V@3ukqM$@mL~98qp|$hhx_c^iXe2?&Bvc$lD91C>y5}OF_=VG6?Pg&vh!tL1I!O? z{dHFue;1A}+@mld5t|Nt&cGHkcJhN!Xx$DSiJYOz+LJ@_oV31RI(5(>E?0gqOXb)X zAWTau^!fU5OwciX5n3EfBh}15%4ks@mf0G=HkNy2g{}*E|A$Z+74pIEQrbUMKq^M@ zEP3f5H?ZkGGrw!IT{}A_y^>sPJm-YEyr_-wKd$rUpFGR#PY~eCDw#pNk|CQ9%*TpZ zFL=X7lkB)SYIzHE1M5DxP{8lDo4tOnXwxm}sWfNea($Ux59upk1x8JPF_SV^bvv1Q zB~v(FPAU=%ib*;3qvo_Z8bz)XHg3^vNQ6~qXna1!S|MeZOdk=bg6Z;48?LfT56fJk z;$49y=0kj{g6$+Qo_%FBbQig~23Vorqdt->oXxu02I3f}&y?l05BUky()Vh;o)B7J zg$K#?qfnW9pUU;fCwCy{OvGt|5K=PIKp)-wqLTGFj}y&5FK$6aS>u zM=%1_4r!JH!q9l9J!Y6J-AA__^2-k%-h2HT?A9%hnK2}$coZuYX{Y;bG}F<*22y(* z+jW+)w6ggf{`x@G^b^-yZw)bUX2i^-Z@WBq9SA)(g9TZohdHIt7>Q#Wrs>w2;7!-A zd4E~)U*v#3&Xcj``OS5#ysf04@^S*3K(q7Ov30lY#JQ}V1RO=K^DLv4Z@Tn-5%6EL zAKkuqotwANJ^-nnX4iLv^fOcK=`&uNd@&Eh zTfZ5i9%a-rFp+34na3fR*4Tv(KB8G=c#%29c*#NVsMK*KS6;67pWa@S*!nzitiCLO%Ks533Cf7QEL38xH!xO5q5zSSq z00_xuT1_{sJ3$E`jQIP@blQ?2?T%rJ5`&>fu;4|I=G4m;5k`Tb1W_d4#|X z9ztZ2q#X^Bt*l(#p)3}Y2u;2zBeGz7=XJQV$010&QziRpYU3408g}o!m)QG5y7zh4 z!H1|8FRGGR9{?)!l$?kdSJu-JwiZ(OnzaN7jUmeMVbW@Wm$G6_z?%~;P)|3v7>>^X z?nAmI?$b2OkOv>fm@#s-fjm`)@)bUXsI?#z04FYPEn9jaW_sK<$ADzz4ek!BDZ0mW zx8+s#ZvvoSkCS6!GVYb(=*+MIf7oEA8%A`BU zCMV0al{gSDEkR};)H5reP~kh|^_{%ss5GO!LB2HcP~4cClru#(<>OI(d5nd8RfXHi zn6n`gL2uDEYb4+Ss!W%eB&H%*RlQ{hX{Uf~`h2+~D@5C^-By>h*=OeK1{-#Hk!Y(l zqf3tD=)mUne%^q5`xt)}{v9%8Xf8t3sDRfvhf?umHQhN-C~PQSUgouFbDoM1T+&)D z|HZ3~n+S`pI~$2hQ8jnS>0I!t$>(U`Ey~kG)@YvtlQu#w+Yy7uV_bPN{eR`fD(LIs zwBnY8kV%(eMlxJ7L*?QUe329X+^7RGC*Sv>AF1*&_+A?PX+X{PYAXR->upuhY zPkNokG+!rxtbm=wHsv1vuf>z`H$ezK*V!c=YHJW$S95uGABfas zgXsjNVs@gaVTUXXS&O*?Gc`U}r9tRzg5${9W;(%tf0~{lGg11j#0ZC)PWpD<3EQz)n0!yA%#Z{OyhBvMY!g;=*$F*++J} zS+35ry?W|nFvU2Xm5{BcuxN3W-boyf09wxmc*OnMDJam1f3HkoN9Y7>$4Yy~8rHlj z!^}&U)^p5&blie4-Pny8U)2wB4=c-fnKRg%j;j}*FTXs-3wIoiR(+>jKBKvwqux3P zXPImk%wA<0LHkICgq^7M)4FlybrqSb`=0fNTKU?@%3N=y6s7$uYLO7T9nF%q)h0!b zdNl0KDl%5w&AmFpKx#tHNw&9A_Z7aa59VpoORT{#-QVS;29b2-Tp*iB9fgr%H=E}q zV0-zC^bvjB2v%1av$HFuayrt#T(^E~bI|n5&v%>9QRSO~IK?xF$yAQ!R>5ApO<=Ox zmC3M|&w%K8_k{w|e143~+f)Pl> zKh8viXn?qa6csos4EngTOk)mpI8_#Q29fSkiFEn?N3_z)%tu*kF)jEHw04_QT_zt^ zAcAyf@5Lk5EJlWr+%CHy3oBLG2onrQ~#%|KE}f<64$o<~|SZ z&ZpXx6VGG=Qbt3C5SYkd*hyj#bgOKt+$UzS)NPAzKSxhl674vb<;7&7u4L?}3fsXJ zOIs7@_cWAye#UbftmiY0d<;r4SOpX>mNcbL2!L(q@)pZ2VLq6Ew=BC{z;7SW-}oTQ zB5SUgLFidY7wAx+sjTaWysuHVZv5lOk`miE2O_18j?;;7;G>>|wntD4sPyNCb zZV_Jy;!hi8MellGD01?++;qi!`r|M0dlE4gY4Ys=7TrDM_Y|1p!Ji5Dz26OS7_X9J z0gT8jd3$26g(zHQH+c=-Ew_CbN{9YfTFQIJxgIf&1;~AVFL^W%WaG53nfx`1_1U3` z!1c~vA2!v|QIA5m$>Ukvcd2q@a(RCTm=sDzkG?UOlKfOj!GC}h_&dq|N$|1uD!smZ zJsDv)XuOw`k7Bw8z$)Vza?7uXo3*7$X76wwYk322u}(cACR3MJvgu>er}O!sX9=5< za}gu3?IEurV;)B1U8*!wexvRdbhR`?2&+~Q|1aq!-WeLq&U!5}_k)HMVlEzb82MQW?uUFx4co=KF3-}4`tURxIiLDnl3ec_;B&o=xK71+<4KX^!j<$ zu~bE78?Ti|j4&&>R>xk(Q|c%GA;z!UQ>V+!3UG4cJ*d{RCEU5nlq=k=95nN7p{sZDDOrcq=FYsSe83Z&`;&JDL#?C*KD8(r5gc)QDXA z5twqdZemZ7fxof@8}u37=qJ#WYURIE&TsS*Qs|7hIBsS6K6&1(0b3k7Qu(Lp;f_$# zy{c@YEYktL+lvx>NQ%V4Z0cENqEsiQU@r&jPUe+AT~mY+j){lbmO_csb#PwPjg0(JJ3!%>RZbLGCB{~ zzx8pmP;R`#zB3APHiWIL4hEoRfjsWSEY}~+ZV%C?VBe^umTq@DSU0(^pKcBL<&Sg7 z%rha-OWxr1SF4ItfY~!Zs$U0bKT{P05k60*gDF7AAnQtjtT;#|k749S>y)+i zMVwdB<_S|lH(%yyHIXwRND=YxMYw?YE60`FCF^@2jrKXFP7-xDl}0wqz>&=%Ljpy( z0#B(LCmu)4DkJX$mb-66ezxi7Ii=D)GW~^Azfp;cGEuo^V`BVGFs&rJ-Ms;Pes2qV zMJM!h$=7+DAJHldNoJg%|8B37^w;!Mn1bJq=NSdoKlLU>?&sdW4uT+TQm*@?^$ByD zG-JtWM(yuS2@&_`LZ*(aZVVzFFG9gTW0Hyl-hrB0j@F0uf26NaA=d`Wbd?+2jU(00;BRm?9d&zmuE{>Y^s;xtNr z8Hl8Fa-@*g(%Mo^+!;5mW(7M@xkk4A&OPcc=N{?|S9$GKL@eVa^TCyN#NcGeGa#%h;Etr?{Fcr$lZWgZBTrj$LBtp_Q$&b%oByC$go4an|X{Oat}E zolF()Ez+Ey{-#rI9@aw~>nURZ=gIaKfDrB=hr(@h7_*zIN{30>tz7r-pAiOnZQpNn z)3?$!A?m4rHQwzfL=+i-3!IoHTkph$O=0wpyfK|!{Lgo0u{?GL88%e$7y?~owB!Jv zWI5>$^3T&)h|Xu({K;LsE?j!1p8BWqL5pC>Auo>t(Q2s5pqnQd&=dlU(>c41#2P|F zCa8)!#413Ep0#;@OfQvS^GzQ*6>+gAiF*hjBpV}n9R0a>nN`}0qmAKfS1a^f5WDQC zgXGoU&#V2ul6b=mp+I#>~vMX^(3eK3-XGpdj2@KUi)2w2~}2 zPWF^hJFT>iD$&EBc)1}h5bA&EKEqdIy>y;HmMlHPiVb@U;r~e4vdQx=Xv>oXTj@-W zCU6&5J50XO0@R?-zY)VlB{qQ%u*YQMBAa`aFu+`A|>qCOx=~ z{O3zE#zmRg_3S%H@!SLNCbTU!Wl%o4a6$!X)A6b4mP(Bs&8w^Aui93JROm{V-_y87 z?hey#>D{az4HUnB*c(Nl(o-lmG-PiID@h}xx&(CSx=y{M9#k7$g7!i6Xx3x{nfzU3 z>;#^+G{sv(XaOj3w!>nzWh%1V>_7$Y*~j*D!EQdPeDttg3|jt53wWKsv{K7=HdL>; zo%}wSuQ8^)o;)S|-Y9p&`C_*N)`_O#UCf0pc46P{Rkr!K>XzPyCr{7Kwkv*=tj9xp zT1l%v(A`gJqjmo~3W!DwJ_O&qsu!U>5IHwM3e|o9o9eCP#A! z;H(S06WK{yj6l!I#zOwm%e4QSx+FzUu`WZEZ-o3q?J#^+Q7TG3-iM;K$5xZ5-CF<8YmKR}R3&&qA4p>e=XBb%H@m=c~9?X#~<|ZzD%{!hj#g z&z4_zp*^dVW47!*WY!1^Lqga!vjkDEm?jOgu)}24rfA6@j2L0LW4!^`&dRaC=$EDU z4Ikz7Q)U@ac?6|lo?R~J!j(F0y#?~Bj)UzS>h@-y^p|dUZ4NWxLJP6VT!th%dz5 zP=UghGIxOK$0dT;e5PzOWd4t=+2L&_sa5^ZogIr0*`T!^8&#Wt=6foC2|Wn`WHXgg zsQG54ysguGo2lZ4(svIVdYA4GOSX+pEQ1WcQ|*>*3qib!EH-)*!Zg0T1U3`1N%t#= z`wP3^3PpMYr{tu|7oUNW4=dXM>DLP~`G9gS5p@q->6(dj7(o!G}0 z_5%Lj3SE%0T)FGv(K_eI0h$FExp_9-JGKW35&4(2_QKY9vW*c%g_PmsZ-V&5#>vCg zfT2l#1xClCwDlVVQ!wgsG?G76jN5qShRU@{4-+U6??yJOD|tM|C#g1I-Lr_urKg3S zIFG=an7Xyg5rV6iSkR3s*p02K%#!;WV(WJ0<%@p|&%1dN$eE+v!C!CkQjV?N?Uv(^ z!Zs?}4`E{(JbrQy7?DfQ1h))9u&W-@Mz51B-%Sh+Yd#ARw~gh?>0~EQ;Oh72*&15$$lATp`z7g1=flY?=40GN@8Jd zLg#lmqZ;Wt6Q?9HIwWmya+B$q-r zw*N(yPLONbySodoANQRm_^tGi`(JepZuTxVZS!SzcxfHJLUdr6IB8CZ2`f`5P`IBw zJskxis9*&RTywde(@O={yAG^Z#XxEQ91hAiDm>)hRyJENz&z%uXirJFS?==Nd4s@9 z2+SqT%U{k*LM{Wd#kL*U9eq37?Q~|syIxktecET8Ppt+bo5?XLX9~qj!N>Fo@>o|d z-3h-!E|?%md)?5Wj%3MCAT4W(5EqHCZCgRWTB6<@h&Pzk#Zti7dQ` z#!Z`#9i5l0XLq^A>ipq0a;^|v>6A^&&nkG~vJn9OhpNO+Kq|kERD5%&gy550r=19z}+kG7s3flYA;$vT=-+<=_-|n zdG}XU@o-~gxeFSQ-clO;j3PJQ%5L^52U@$UXlLlGCm^P7fY78b0opHgEa@$JB6Uwz3#LKiG&PFqC7Z^!iYZ*c^faMoSRU zo&k`?o1zwOGSu`aT8d3bN|<2=jI(J4%p_u8haNz6M=pg=?IX}2pOy1hD-|C~`dUn( zR)!57bgz)}|1k^sAb*V9I1O32eKgu#JAm0fJFx3SYNpdL37q;$58OMY>>8hq`V#vLIhvuH^EN+WjT@Qe2|53M z$R?fFoA=7m&Rl7ypZ_1w4&6<6!|Gp9(J4~*bykdupJAkkIBQGqgYbu?V=zR|=)p?P z`!RDQ;%!D#OJ~d9x-gtNIxZI-KZf9RNoz!U<9cASkyb1-WL9$|kP4R7REiv zag(_IF!|g)PBAFcPCVr!qn< zxl`KRk{3|^k$B;y9?rCUIs(U}schxX;ePpd$>X`a{F)2!xi(9A&N01s_IZD?w4-tg zq{B*h>WRr{&$BwZyC6&V<``@e4`4rMwqZ?Hb6_Sdj`GjoyLsk!Uo{J4!XoTeFYRD| zTZ53-KE-vD7VrNj2+O9gd8`p{hNwx(@nMgGJz*FmnA5fEK}-U32N+?UVqDA4hYgkK zGbvYBd15e>=xlSt6?uT}+hyF3I@8&?nCx$qZ!s>vW1O5x@U^=>Fx`3tT{Edz$(0sp zkm)=*Dp2cc&e9Oz;hUr&_>5RDSMW0^RE|9_g7F6W6pOT(sY*EiJw?UjT4^^1ocEg9e~ z(>3tJk$Bm?Q850gNUz^TkOljgmp>^777(+K?oMsx7$_ z`!Kr#-21O8Z6{BiVl9W%I+T=dM^TVzD>VVH#Wvnw6S39J+?nm0>Nsb~Kukund^-{GmL&H&? z&>0PxvT_~v5`S_l@e}(rI(3J1R$Iv)S=+OZ7Id zAwR#%hJH?puh5>^1s^CG1>Iwc{nWe~_PA^~?-}bVAAgI1V?xSm$uW?v4uyx(je^vu zWcA}gpd6iLo1Qj$(-J(^134}^e;nIBU5_JuU)6+=7}7fK(UXW8>a@=G<(yyt;59I< z31*{^vX6;^5BJg46bwxrtEFoWi(_f2St|5C9Apo^V}G5naTyk zTP>Y93`lI?#3tO#2v<9Kg}6obB+W2CKxlqc;b~mBDn))efeEjzkB=T9cT*-gS+#B^ z_Mn4K(@A5B8Xx3Oq2=Bo8v$F#1j~mDZW(@FdlHWmVJX%j#;8q3&cRV}>d7ndcbctEG7exTe7k^ev&_Boo5jmqyPfhh=l{aWlvXJvHeXwMPE zw{SWoaSeII@7pG;zBV1w>SeDUFKnwX>H+|EK~aK0H@HDI?}8h zhCzCvTVC0O`oH}O&g}cNIgxp{5R~~N&(0f=wn@(n9WATrZKUO-%ZB^CL3yW%H;HtOqAUe<{j(x20zfwl8JBumDbrC150w>;FIxIv9hxT7qo4uJ1(3v{_s7{pZFz7urZsq#C^XP7xH+bLopFP#?+_A~w}r`qd9*udaA zd<{wOA-CmXKOiBEIdQ2OE-R^^XL*bjxRL%486hq!YH9DrZEH3||6^fR%dt&rZJxhL z01OMQvaO-8>}hmW*LW5#!%2%QliBq10u(v%YpVP{shEJteYq#=eOVO`WuI+sMsU-h zW34xidmikIG!e}lNB)*9)15hMuQ1as+X|%CQv6|GEBR!Q>EUFo9y@-33q`XITbayw<9JcQQ-sg~V?c}(K0bqNTdl8RPN4u_Wns8(h= z0RrOPGyg#}|E=WP^}K1pew0p!N0-oPlE$=_MMEU>yo|URIc%ybn(AEDjy#ZUuv{O; zdw~19xJ+d)tO}jc7XHN}q~z7emvz|Ki_ZX|UnxK1UYeR=*>Dopo~erOCcs#=#K_Gm z(|rNcxn$uOd31Izf%#`a+TUow3Wunrt-W4 z*TFjoHA?E{l0UTKD*se#av5v~SXojqbT6Z3fDDW z$&1RY4*X!(Scu9I<)9_P1|-S;JpBIZa{l>;x--%c0`o|!35=3pUjVkZiy*81rB)RU-{oRz4AWs^<=){L;^q6~v6nD`#j0wQ)Z2@W?bT*z8-@D#;haKz{TE*Ku;XT#cJ>BHtO_=< z14}2L3`O}Gwc}DVX~lrA;o6673_mGb-t$rjQ=jcyo$ZlP3yJEzsY)`*w&F;Cj5dC; z!Cp8124wT%PBdz)D!Ug|A#RQN4*<8tbBsBuI@QV@mPK25$9`Ec>39Tzx}t{U#T@Yf&Hz<*X0$bCO@l>+U|{G$UqO8Ej1 z2HzoC^B@wJ9tBuxvJ0UFuC)k#ybag%G)A9&ZCHEb*yK<#A^GsHAxJ^Y-5B-TVXsVf-(%oX>QviGo_BX#o@`#x(a zl3Vc~X_v#w&dOxFn(C>^*yLUW@6y);3V{x6TC_h-gLAz0GVT!X5^jD}g&Px!F6;3iEGGBdCw?_e3%fT!}#P?Io~=3Q15?+@ZR*F%ck z8PJSqN2wUa{fTRAAoi8)Dz`Hw!0;F48`pD6K(DIJ09?>1h55W{ALc4R#hqz%2PihS z=EZo>!UfXR2a5fy3&4NUJq1T;u^LzBT_&RqVxbFEAq`>1vzUyHZEBhFTdmBSbUbvH zEY$fmOM0Wb9EEO?KKpsr>CswU1dC(op-;dXVhakE$^F|{g^yL$lMqKg4VsiOTrTXv zCZGEr({QvBK6|bJ>T;*sN@I=}fRxp$*o4LUn<0V45m|JKCv2$lZ)Yv-i;NwqajtIM zL0?(SNU5GbaWsv;)~Nhxn2W;oV*4*5_dW4xnYG9Cuy_a+$HlD?59UBJMYje$QQGBZkwgW=0y2gwwotOa3T~d`^A{#-UVI81t71$^f;;o-NE`=!%W+TiWSQ_-+052xOi=2(T|adPoI zh_yMDhwuF@b|k6_!u%pJfCvQHhA*hZgY1sGpH$9f@153{h|`rjbdQ_8QgqU*Paq0( z@1~LzLTzP}B=;Vq=`D-_t7jP{FCPJ2uB(bEI$y8_=EE{U8q^8d0n?HgS--RoJl2@k z7^>UREZ2OwhqBqFygfiys+`>czi@x~@&4Q>%MoZG+fSJhU5&LwhGfDee=Ukhhqr+e zYjafeCsoy4x>TVj<8I0&DSE!C^9?=YWc>d^VVvGb7V+qOA8ugrXwp*?lb~sZI=O|+ zEpiDqO0U-lvVS=O0#r{GcVlfaBu7ubZvhe?0?PSJ$E`N0hM18!9w`7~ie3(2N zc)3d@yTAadZU?qfup=taSS1VNUWAbQ&a~kiJq?ysO_38Kl(o@xLm0YGOu-KmGq=nA+qM>Msy!MmV ziR0w#fkNA-$aP)E3oV;ZuoAv@SrrVIQZHEZmyT9-7=YvR#2hdSZAHvHLI){$+#+(w(F&R_&Fu-*^*PllXKw zaw4RA{J18`haZ|1Sb>^4a$=G?t zrdnpgX2^M&p-PU)wtE4)m)CHqB`R*0)2(w0b+W>PlKCtU+DEzDvMzYif2y*bGWOSh zuaelqB2x29jdcgagp)Vp?|12K9>;{uRr#Zhs`5fLwY8AyMEcL0D%k><3qA)K4|kJB z2Y^^Jh{wYyfbH6NHI=ivne>1XGbu-AS>rNosaGdE7vE*PoaKbw`ZYJRtddTr@w6v* zfC$!7HS?sjqv==Yx>}5qwpbT3Qfp)F{deJZSUT{U1uf;pjR5XzDt!jvQ?Q?yk=ripl#VUarRV+&uq5Mmt3LcT}4#j*009%Dw_BO9U} z%h9gQBH4YN9o;p>OH%S*-X#CeS%ggLV!JvVH<@L6vCP*j#@G?)>GM>IAb^fu(z_+P zGhd}M)nGPqMn*x393O2REXNbD_xmqhO@}b~w#lVth{>^Y{~A)-0rydsp5P~i5VDhX zczhfdiG%;S7-x$`{HIw~S5LD{z9N|UuV)MS%TI-M`@QfXRO5^u;&7)jJVqd=P{wTa zMx>Dksz0$+s6rPKDB0VG^}4DG*2~^jeBhgc2u!874!S`7_Q^a9W{FA+(UmK0kyA@| znbZ!T|9&pEa_&c_PC+FJuf8f-N=YkWaoR$)-~!2Zyrlm@ov*2558<{t`KBRD?{6%f zelTgqy#gIL(NW)%j2gyOLBlbrqr=A~thN5|uaOJ!SFHg$ZD$jE0{*)t9+y=pN# zCm3gEl;gY1 zoznU%w5L#ciiigmlp>tLuEvGDDt_ZiO8;}Vh#HDdKDRC+zUtM#kd-Z=d4o4~2Q1L&N?Nv-iWIUKdxAE8WOc%tU%%PK~)=BO%t6lDT z1EaL#67N%@)>-APQqc7eRZ8h)yIdQo@Vt*pB_4o^lpMfsj-<#h?UBs)RoOT)K^!5{ z)S@!Vg$9(Ug3&0MS(hao(e-}yIrg&MONxK;PJ)AlP~!qSjLScc zx1+PfZKusj_;^*i1kadGP%XUy2Bp#?gTz`O8?U1DG&pXBZ&-Cw{$9;jZd0*YC`H9m z$@u$FB|;NEzy*?NY5P7`&p)Dl~tU7@q9yOjz zEd`q#ugZ5hEag_^%ml)zf<3F@e;}9Ch@K99CTGgH2LYnX`Z!DKcoFbSer+*&7RG-BDARpL|Q$3n@2odAl$=2W%2bBz&sE*bPXhU|oL z_t9x%DUv+{$vAx*($XQ%hiU}+WB!QM-LG}=d(cf5_*$^4gsvl8<2_G zgINDRc7evuPg&<{UW8{mt^815UuS}{mNBwXM^(sT$wDf_9x2(t_g>wJ|Inxf_IPLl zi0@2jMK1MXx9COfU|ks*WN zR7DCjtz`bcGe}Q*UPL{fOL{#LH5k@fHR;+#2s zAE(B2&y-HTm@auoc*7kd{t*G#$W-{Cx%RxS}jMQn!HH?R<_qFpBOSIocfid0B;N zpa$;)+n^FXUt{ZXRj?ywB3WCyehrX8haZCY`0D7$T`EsjxKSqWH!HxFumj#-#?p6X zvj(2F>fTc}hnyOat2+Rs*Ono=PL-nvH!cikGAA|FbR__xUdq0YplGs{QNNlVf0P2@ z7CFxV=deGM`WSg-R46JJpY%rc+2m+>Wg3|9P>itYA7psY9oiZ<7Fk;f2C;OmG~CI3 zIuPUiQ0GY>_=0y_tD7#~uL5ImzuYZlb*|Mb(Nnoq@|tcMdn>0)qz05`i|bj_iOJ$% zscXwnK7|tTBPzU=9kq2Zp2+sg`R>Tk36S4*fVpQP)~IwjM!m2FJ6-me ze7_kmzoOhl$U%OT*q%R4j*cekLO=9_asv^^sFw|s|9xc|Qcy`i>NuIbIy7I8#ZE5; zWlE2rW4;xf)K&F`0hLaa#rJp%A^o;CfL4KS_8jPrJtKHjU-?JNpqbW)<;Dj$*Lt1B zBU_T9n+{|J2Aad9SX5wll0dmy6KQZes~+E`-$@8LPOKL zlH{!u!Y(U!3Q|BB>K;|NM3>(BFwgGyWX@AK2)7RLmSH0+Cv&=F`H#_{+9NqASk~&CCa*DjU z7Op=1F@h(3RKcC-M|yYTi)trqg-pi93Yl291l8EAEVbneD=V;JM{a>k)qQ-2U=DSY zTpMm>e3Oi-gNjpvno62o$PTW@9%EO5m**PuW%3Y4$&LD{khk7CudxI}Xqhd&&X^A4 zc##bNFU?_I7>Z!3$~!fgOaY-dD!!^LAIXWcNa?StrY;(;)HbIrH z;AVxxj1FnZRw~>SU@z3cnaEyNyCg-5|7Wd1%5WP_>ErLjZ`PoAmwO$sA!kQpbgD&i zCzwHeL_!gv{oqGcPxfLp-5n&2x^Rj@)YPKz%=oMc$jPz z>@WB40M0jTAh6}_6>`XWW|0{avk(ZQLH_eN>f962}Md*6%6tf%W_Dz>>{R)Owm)2i*~FqpEyb!kckp zzcI0wfnF@>M&(|tqlE2c?+v(7bzVT9V(69)6}zNkcPzyA-+|V3=UL363U5@@QL|D$ z8;e5t5Io{#{546%hJjMHJ{T@{s=TzHt2df}LfoY51)puh4j)$~v)Kqsx_s2!OK$A< z-Z&&YI6%6;f(+lLTsn6>{Z8;B`nXY+<$xBGd9izxNl%@F2$n;YKFoch4Vdwq5b+=v z3pbgCa%!wKLa@Nzh`3;3J_eaaXh&4~bbW#vV4#T)i$pC@w|~&B%PR7MSoc}=@Q#FT z1Vlf&Gw+)|Y4{{-5_t%Nm!Vy;F{;!F;3cQAppI@NuseEzn`Lj~A^EdW-HLnJDfb$z zM5GfR6!w$b_W(wJtI8G_!Qeo{l~>7zM!R7NO2+}q$vSwVaYcZzSTjXqFMk!bmGg7F z22e#a)pFf3KALAH;Xa9B2yyymQLlq{O*f!pgo!5VlbZ=#*nwT;75l01IG&D-lE0Nh zM*uB8Y)aJ1QGa1`scON#?Hh*{j?v9iwh>T~pX-ETvcGYN#E6(KY9@)DnIrQ@^ASp)ZTs&FqcnjmB6HMQJig@!(|9T|B<`6jS6 zrD?KTPY-^I$vWI2dLJaWpch93@$pi`5ueoB=Pl+%lg)AROXxqinH}w=iu=QK=nnDf z*W8U9jk=P5kQeCP0KPlrWiJ0mbtN$mE)uncV#cCuABTR1mEaEPOJ6P^K-A$RQM}hK z@>j?4xe21Dg)b5LqIvaUCsx?G$LhfB#%mMgwsgezbiQOcS6^z4M;M~#B%q5pQW;0U zo_T5$rlJ+yVsDU>xVX;vvW<}X8SwfSC|s6CBY2#SD$)T4Y6+8=UFk$r3)8TI4wv5K z45>xZKRa5ok6DXx%s}9A4%VDobu=uKIoMn{MLBFT{U_FFN@}s(u@LPWRjsqeYWgE> zv6k}bdaql886i8wfQ=%IUv>>3X3T%h8eC1a{tjBPJg$Oq?STXkd_c|U?| zN7oW{4b~Rd^9UZDIr|WaNZ6%fx0wLKqH87RF0V^(W(Ra7(A#K}Qy}R-Lk14f+lcUg zdJ9oMVfeC@#etJvnjy=g?DQ{`0Pvrc4PZG5%IYcMkX=_WA{W|t6PSE|Um4fZWJXy- zNN9t-VBQ7gUxEFoI4VEn=B6VItQILKc8!B0<*}LUKIOipa_3dh;exV{we%^LTGKc( ztZ7dQcyP6m%qkQ|Ju}8{{m)6=>81;f4VUN=r(Td1qbt-iFdWh2r2ZQfJr5Nt>rBWZ zHUmOeY}4HZN+B>cX>{rE+KtU=_D!h;pn7l^U*{QaZf&|*c>iukjM-na@U(dA56~8zmv%=1uMeLH*=5EugsZ;_A4bCpHp$l!(*wOW zGG#{p+#mrtqb(~y6U9l#*^DCPsE6zZ3gt8fOER>f8I{Ks+S#LAcMu>u@o}AlyhH`t z<0H|lhx+_3a#6FJCqK0Yc@sy9O!*IL@=+f34%OP}=U91`iqAl!0xbwrI66s}-$Amj z%JL2&4dwtfUB&lrQis}!dkplpQIVa;O0<{W-$Us;uE?##GvsK@g*&%Sk1;H73-sT- z1H*EYZqk@i?L-TFMF>X4Ecr4GrL6edEWpSnhY;I={E@@65Oq%niGK+B<)0Uj9j9(E z*F#l?rL-&*2B$q2z}2!l(BsmzblxwNw^O|~eU>sr+HS(~-=oqWk?bWQ`g-X){x7z^ z%V78Std;WoW-dojcbc?#jLlrF=Vz=YDk9TN9&t{pO$Qjda8%~$2BS6}p`ClaTwKNS zu74LZaY0#*qTQ}fu{;%xWy)}_ak!jozslwv)NP(JbsvM{Dw2-9D%3sIi^oGF?G~eF zY7#n-NP*-*y~!u9Bb{z$7dk0VXIN-q4{VU(1u4(ZmtEd}-C8RAuuE>aL))rRy6N!q zdI-tx)O2b447$u$DZhnZzqFpmXgcFobjo`&*#sW+kt1?+z7nEST6Ne+Rt|gx6+DuPMMVv#f`+fR+QsqCAV` zrNgyqO%vk9ewlC%JU#jpYyV$mZ!WtRvUU*na#=o-hkR6*#O_!R!<^Kd4UDVgdU@g? zJfZ$CR;OA?r3gmOCZj9y1yZyxvMswrz8sG*wA0qRw~C(wnC+tp?ZtM=$PVvDq`^D? zax`kQUpgKNiu*w%f4sw_^Xzi|>hdA?Ck)&Ptw*1*G-J`|Xj|!7g#7)zj-8?aHYWZ< zJY5petqSD2EkAy0&GgaaA6Mg{qbPl;N(3MQmNy_l`Hk@?!W#)c6hE#*zPs=QN)Eyk zcCQfs#{_zxA!C)7=OCx`0@mz(S}RKT8NBUps&qW~XWy&ONauj)oQ5pVKp_(Kc)@)t zahE)w4GO+Qj~!{Ss;0ES7u zaY)e-6`UmA24=u$n^xLdh10=W$1)zG*yN`Ui_tyDf`D0EQZtjsOjEuWWs34zB7|)PuXPCE15}SHi?=YFX_-Rx}q8aMzcnfAB%qcVHV-DMV!S_q^W0u9y`@ z2My&=hn)HyB^bd@D!ojW&oVjo|K1vwB}rIO17Lyjp%nm!F7K-3Y~f^)w@{8PMla9g z1Ub~J3K&Cjd?<{~j%}`LCja}ITQLZ9tbF9OCdju<|BkS=Enm)zLJ3c)fBLx?`u@h)$k$$@71P3gmPL zcKnrAp??$27m*oWvL5@GpKWx>ir^k5N}-X?VN&xPYc*LFJj?c!oCYH-jivV^5Ylmn z0HFtQHe|y=7N+Ir=)i6CGeU4JI|;cKTI7@M+$=Z>GW&*#tuVf+Hjqq92EL)|DXF-b zXYH;}yR*P z1gd@H8LM4B%Hq>u%HzNQyL)0sYUbjKQz|o0JAlLV+?ZQ+QvT}JCW8v^13u_Z8ztfrg7HEy@AaC~MgJBn84->S+GW*&l&osw{b-n40 z8aL9aQjHa8)fp9RfSXgbR+o7U2TTgC;k0nTb0IrltUb(p;5~jfM9%0g1y2&R^+tJY z(x~pk5>ZPR*M|?EZ0B^%Dxrm2LOUcc0x0Mmq zu=Dq7K{tNB7ZQYjkh9d))w9Rwym3TL#mm-O7q1KwigMmtqW5>iaR9LlV(5&WUd z?|@3S)Y|m|i%9+TTPpufA}tkNiD(w)NUCg?(Ec-;O5Ml7dP9vX-h{VB%F zZ+Q}Y|2WI4D`#+*zQmE1MwhJ)vHTH=Y8)S9gmOIM{TRpM9X;!u5kAd1l<;~$-^2p6 zP`7hS&X(9u-1J42$d#`iLBbC9$t@;>D^Mx7=~mf0ch>q{*n8pxE6-*}jcSZXu?{_x zP4f#tk!+31<~PMDDrgJH;iO$tfxd(`{2$2PG2SE^Rd9l>iS!ePx*C)%hgjJQWI2G( z`Pp*lg4af{l$xsTw5^rWDyt8^QQ-nyUFQf&xbcN5CLjXj?=b#zr?jp{Q6?$NayfU! zjK~K!gZ^$30eL-owf?Q=PZK}OlIACooO@N`xS0D)2fW7870tIyg0s~G*qYKAbkQoq zqF7pRU;rInMx#<_fbB3}OP{OGH@kC%tx%!`29QD>9tbs_Z3qJd_)i)ib#q8TLb^ zsx6G@u&(lSXhTq2l?{;Mecl==+2{?BJyv6p@7~5h9b1kvR;zSUD0Gvdm-N&X+Op1p z-Fb$w!4|7bogG3f6fm2YQ_MlO(M)ctgT6Oa&QtQmS4heis$#M+E~`TRf&-9I5x5Bw56J$It~-t`!>ajH^*D_Z8Bz`d ze4Funu+PFxa`MMim$3tJx}+*-)2EM1hwDrbVjC$0U)^~!ryI-atq$w-RulSG(Ev+k zQ^ptg&ugee;>D^u>k@RQc}0@;G|qT9JcCWVxHsrKniWeu+i6LX#{# z4zW@$T=g<}Atxv=Hu9Dl7ppyT_1RE@2lM!3(!xcushL2pUsYX|l3}@w9#=`LJn=rb!WhP0L>#I{urI}F zK>zDL{BMVe0p+;lQc=iB(1_m>Xiw!T#oalysTwcCCzyuZwt!dYGceaCM^g*wgd zZsqDMUuqU{tIpx5uZ+_ zYCNx7yWET-xc=8AG=x<@vw^EmQR%OW?I;)iLKlyFOjBp8l&&EuBdlGR{UAiF0D4f| zhW8;Oe6tF2h;_Ty0ka4Wl+Hbo!|u!o;Ffw41~=NE(^k!94$qw!M?NJfX0E#l2$Ycgouh)>Gt z6P?JiaBPE3GNTWjMQneH!U!0jHBfddF~Ro-(MWE>oN%4Af0*9 z&Ghh+KVr~oGReBh++9dSp7K*5mKDTvE<6GQ^uHxLTJR1;)y5D`pk{2D@=YT4Vo8xN zP9QUAp&ddIZ6(7GnFjLVUL&9ABK;fn&FKG;bl!n6-v0xC@K$$;#1V<}kU0AuBx1+D zn0GY9Z0y)MgxF_?770>8Rcc0+M%xghM6K8{La1tL^c$l@Nt>EMtH0N$zy6V)yXX0g z_x`*;?;$llF(Q$5I6x0^bdajgfG+Z<9l{{$2&eGmDY9>MQk)*$65BnaflAE|mdt#> z#{ujk4B-4oNmSVkfx6`g!XRd&JQxiSs6tu~=7^q$S#Nc5((_c}>c>_Qnke`Qff*A3 z#_+?n=!<=ETqTvZ($g|w^k^zC%~IaC`#Ymy+JqokM{OK*Nv2&JSD1iv!Ltk&j_Dz- zr@13#)m4{8lcWO7{d|8Z$ig(POmL?xJf7&_(mt=5eD)>S*`=2c=FosGf5gWqkRJkJ z?m|6t7YrNLy-ohl+JFDOa+dz@|7B+{n<@5>X=Gr$(AvOP3ng7T<;ak(RCZzdrc7b=Jekg>QQ^cBt6R`wys?+-IdDi zDU4NeE?nwI=OXFdnNPd>vdYA(sQIXL{(1Ey#;WoezgC!MB*?V!40C@rChtQphMDju z&z%^|cW+nm&s!>mzRnE>^>yJ#9VdKOdpM))3pG1Txz19h56J^>K>kxWTkJ6vkz-Hg z=U2-aChK2dG~hAB(s~Q;MTIl!u*N)Gasr=PU1aDd89efE9iUkM4D0;pBsQO+&don; zD8168jjhPbN7pgnp_)dvYze?%ex;~yayIYT#__}Cbe<0S+OSIPANXgNcMNTt$)~ZC zupskCqgAW5g}I*}X1av$m}ylg+c~o zd81_><1JgyG3Z9LVUgpikJ0tPJX|M&9WHZ&jr{I>x33I?PCo03l3|5>N*m20>+WQc zPoo*2YBnqQaTy~I%i_Dk7Tz6J+K85l#~EO@s!kAIi$9L-JpAY;kWQpgwRB2iqKLJy zSDRoCUdqQ{j>Rq5-pG)LI=+I9D#ZGd0_h)3c(nQ*WM3I1-Z$O&ak*+eVC&{0<6;!k zpUFWw{o+Gu+|o4dk(u5{%)!4^H$?r|J~_lw?N$3tHx<%L{RjSdfg8h|7Xr{`!{hd=ptjrxAQNbSn>cp^T;qxi?AB0>1luKT!H=o)IPM z6Mo1b`A zQ+s+t(gO_^d#Da%^P)Fk*bxQ<<|zVt8w9zY-456f-Q~HNuNbP<3Vv98hew_a25pOm zFkZn@eppePA9`J9XC*4SH5v2^OIy~BEz|a%47St!Gq)=bdp!Z7wPx{WdLavXe}mVy zPvM7eTx>Tx3nDF<%Xpt=7!K9yjD<(=$l5F5aAz#BN?uIcLs9F=Q4F%wm%TKd#bY_0 znES>@z~RhAel6$1MUj4rPrZ7AX8Rr>@~}Q5rIlqvr*`wt17nP6SzDgv9gSoGgS??F z&4)JBzX0VWs@V&-Jr?nN$rSyz@R(ON^mgVQ3ih3Ow@ItHf))Qejqh?Q86NrO9s?7& zVUnp^`O{w|2HweVS!KlqvkTQlqH{U?2I9^Ee|1B6xL!Z7ULlj`&i7@ZLqh>b0=-qG z^mQPRTa`ikOk&=Lms!ToIRJCeDn?$jlRw)Iv*f+12-5!Hin_KNNQNzZ_t6B*t&~_z zF4Y85sdbGQncI%Ze)eL^=LRx*t4@r5MEDa^5prxvdkaY;fL5L#G@SAp4wbQVQ1u12l6cpPs8;!X7{g!tiILmB zX9qh*GkiNm6DJLbbb%6j=Sw5jdQJf08_)31orOH|jJqlcoQMfkKF1^;ve@NtAydpN z$u_30XHof?{Bzk4@Yz#WZJsVd?v5`X{meyj#$b z9bdj>*d=x!Z)}f&WmfLsjn$_Z>TORpyK0=lwyA+Dmrl4Ga_kzaV2r~jMf;PjTRvcl zt}_|aah9c*oyDuWzT?kd<#>NXK2+CB-E1~G!^3Y)Ogo<#C@tOG!lb#m2$wJw67}_9 z{&+8CJu`>!$RSl+uMKT=<<~K+`(bN9_OsGTtuGlC86C$qI&1@bmOz72vM0>5U}+u!1@(8pD3liW z#)V-s%i#VTR<#rxsrR?6Vqrx#>ATXf$&1~P*|=4}>`)-a;+%*m`K$y| zzDZ85)TqLCjt4Qst3qVvQVd?jZe}*Svb)N2B_IE&*z5qK+^oxNXI8Qo8sIpkf0`>nMWT@| z^A4~jCG)4g3h5cDZU8dq!}mAlj*6_ak`Nw zi>ES!LzwOg>eoDUP_Gu=H=9Fhj@pP{LB~^d%*bg?t@Lpq8>1 zZcpOJN-Bt^djpm<;T@me{Ra1xZOY=sk@f(MtkvYucm-#;X=Oq>H_111OlSS zU{|J`h~iU?8f^4NOP2Or5y;jLjRg7ko-14EgA_08IMJ=1sD^rN^<0azR<=b)a!uu@OcuK%hs8Gem*o`q z;f0iZBSV^A1{MJW4XgZqmz_oFfFY+bp3}mJVo5b)7==Y7aC-LO>E8tqwkoey;0>no zZNQKNp7X>&s_;tR*(~sUJ_B`m131d4Gv3t0K3Up+=<$CC`EjnW-??qUPD@qi=I2QG zajnJ*OZGDtsi3Y3vZfP+x&@7aw<_98HdW<~=Q{NLk~}0iwS=KI+-Cg=0jxiDB2;wh zHmk1FolV^RnD6e|p_bPUPAOprOLrbFE0aOymZ6|wuNgp(qY$jLZpC8$RP{Ff^~?%V z;tP0c=~O2ExmkeSBoR%Sbb}ADx%($m4Rpfz)n_q*%;sg%ak0R*GQx^I0O3RmYu)bS zkbm^jXy1RnV1+SrPh=P#sSZ29~{Bz@Q1O0i6 z)eP!k#L2T^z@dj~kgy~N%m=P#*6({kuxpor{*?iW-pcTBjfL>eKp#%x3xF9}^3f6# zjL6=ggn*kGJQCQF_5Z2PRJE@1DRe*I{a3l&C}rf!ohv*SJH~8MK^tXd;42t=p2h%g z)-&G16A;U^Gv;i3-h`RFo5^}+FEL_d#!%?`Y&QSwLWqp)_X1+qW;0Yl6*lyJP4+xI znNL&xWMnF?Rg}Dw;4nxz{Oj~#Lq{&~=@IS9k*YX2t{k@crJPValQBM0XEg}8 zZ(e88d@D2V4COtF>kI-)aHlSr7PBFnJE~1!mV-uML+3K1eDuRk(V*R-vXZMhjUw%8giU&G1|6h* zTIA*^z&1UH$41|WQ%m{85ZdC=<((=r&P|4w%<9gYS9&F8$>Q4pA-f*je`63>oUHm1 zH-E#cQy($)u6-=*lwz<+C3vo)s$d+|oQJP{#XnQ07^!k>0)pkuc%*9kpZLxe1P`s6 zh76=*ke8&U^2X1Kf|H)QP_th{sADR?(>K&HuwOV_dq4@1&IGc^0&3LD%WbFtOH#I! zQ3p$ZWn1<zO48&k*8-PG0FzHOms2+TC3@s;Y4-Nt20Oq^N>(=h%n52Cg5dGwH*1u;EkNr>s zul5%tSj#`l0cpZJ@ZE%p-O^((*5qIn5b$;dR^Rg^dIUdb(aThgeR7C>WUvz9mr@Or zq=iHJx0R-tVC7TQfi7wvRd89wK9%ogE&?W^LhIUZ0qJW8G2`Fd#ma&whEwcH0NC=` zirdD@CcQMLiYE(j3~j-~^_Ln2GV*KwxlX-XtQvyf&(Y*DU7_UKUL?ynib%<4=|JsT z1=zVP{p7jPi1d;Mjf^;1@-16=^^mRnrAJqmGA9`6pD6hGxSnhlip3;D$x&6c|HWp@ z92E-C^%e8GHbBy~6;IgbBIAZC&VI2H_8Jl#FVCwPcKK8te7st?NM$VsY}XZut>g|! zntY8i9AK_%(zTz(#Tc2l=YLAUD{H&qWpR=&+I}~n^WyP*I z`Mx46K6f2L$*E>!%P`uSO0RNY$`-)#UJhcEyn0MI_%4rZRZhi#r#y1|r25noVszAl z4f^jwER7t@Bk?Yl(P1W!TvR8-UhVMtQb<2gKB{M!wQI4F(t3|MT7O;Dr0uJLjEatj z9)D`VTr<15L$z>oHIueDlkaxEg{RkS;E{D93{)-&p04(YSx=}|?Xw?X1Qo?*N!2z0 zFnX_Hlhucr-u03nA67E*dflj{^4dW6XE@skYvkeeB}R9(RH?Z>c7pCna2CCrNu zHM+O0*^SOBPa$Y+Z+d|o`Wd(c{>>WJUq>7rh;}DPgGzAVz~fxgJkv7+B;r#@BxNoW ze*Oshez6PsycfzN6DkXuImNQTV<|r-#D3_glNBtNd$*RV8MZP}K zn(7};a8y}U2`&qo=Z^E-P$OOnTVT6vRUJ{%RFj*N3Na5qaTDE*HrLJvr^?Dp$fwp3 zZYev|h!mHP;gsOJQ2U&V0AT8A7M!nKhsWpsRTX&$av(+qy$AP=AJsvIjHEhhvp~S5AhyJ$RyK3H~%uuCL+8KnGP)E#jyK z#-_GX3|yLdW~zhor^Ic^)iN;e&QdUSB5`yo`6ym{LFeugmf^nzNl ze0J%XChf}sOn!;L0P%c}tr4LH)teh(vi}h{{z^?mw69G?$8r(lfHViUMJCSkg^fO* z4=lS`p_EMnF?AcOs?=9?tBtFRJTG}11LNH}faaoPXJxS@0^z~7M2)EJEUe_CAnntm zP)gOmT?LZ27oX~M)xa%r6cS^tZo@R|X>gO!1eBKbiIJQs5M!zj7Eo26ENM3j;@e!s zC=ibw5XLtJdP4{tl>skV-K#*p8=(4&Jy1O0t(x4W)MGzKbihSWTQ#yrH46XYlh8Wn zgb@!BM}C`+5hLr)1Fk6Q!^x%rkn6L;XmNG|qu;D$+>84FUK{fy;UdmbN1k3aQlxSW z)4Z9=G_91^zl^GCoq7ySOeu(J;@ulKol~BHuwe55Nl!LBa#|hJt2Jahp6a}IqzBO< zv@2PN;FC`h*#o5!?9yL3pib|Fa9Z~?xcNtx{8o?U#?g-wYcey4xGP6p?QLwBaos_-ZWx=iBgvv~@csRV!1G*=1=;4|B8<5B*Ee0c>WW%b$EBRb>QnG`ziB+m zZA7+A*#y;(=)mMlQ{g_^1(6<^lS}R9T9jS;^&DqGjCv`p@91pQn z&==w{`P#@t3)4RK!BfnGgNk3Pn|ODh61?;uv&qEgOmor86)I0sTqf?t$cemAieEQ2 z0Cxl&|=#)uf`=NIZQlKLAsX!G?Kyws;g2y8|bDjGpe)p?&=J={|WT@O!eHj zK8;n)KF)WmTJWj+H}LuV*nFup%9W}j9b@RsY>^qtx-|D;o1>Ix=RF0+rNvX^#mg{; z8b3>kKY7cUri@QCV{FmKjy z{1_yDxCwxGtHh=EEyzAN8?AYRI7{*V+N?@`6jv8FS0G4zN04l&1;n=vVvZ%P86fc~ z+t&}X;X=YwBq9zzJ4A(f$v({PlJvNWQnirTO8*Y=2DU+*RVcK`&R(qNobJ%Ic!VS8 zA#9{g2)C9>$~;XEm2h9=cH1XtBmye^(4Yymg<_EnbD zMk~ner{&KMi_QImt=zttIv|8RL z8#A9;ZdHc^NQVvJG|htH8mKIarm81eoj4FfMrc{Nv_2g%q>tJFEqHburSDy!TfrQipnzVo`Dk@Qx)-R^_8gaTn1rc)U)mT z6nYNSQw`bE15kX4=8*}xQ0#U+HGnu`t&yxf!j6sle0N-T*4D11$7sW1Q{|T~@Z)_I z=(C-0XK9#XaDZyXGEH@vI`mbQg>I3 z`1ynFd-Wls%6d_-G{$;94E%+$>55x z&hsRT^bRQ9(4@6f_fdnYzp|e%ymVDPS%w_+8YiTe2JaKLw|c?46CeN4NHh&aAEob1*$CS zW;VEu%c_n_eqvJ&@(hmSBx+d02)y$&QZZrWWyUnDrkWqaF*Eh7Mq()d_#*;A|MowP zWcjiJD)nhjitP9kZ83Zzi%b3oP?XRMs26BO*#)Z1vLl=@A?!GR`Ed!1wLT(^6qM{R z*+J@#+?|d5tfYd+Q&lnZoI<U!hPIWT@F<9VC)O zHI?#D4X6C1T2eM?3RITe0Iva`0j@?@A)1*<*m0k-@h9r+{J&jhw{|d{l%p+5DfkpX z>oGL+2r=jbUy8T4A9YzJd#quRw#k4eDhfmSGdK8X`yO$7{}*?H)Cd6-s|%g-$pj-z zHv6*Uzw3g)U9c19i{i$rs9qiEn+hF1%0y2*)Z@(fLFr#H{WHE>5s09ge}L6&TMKD- zH8X9a$NZD-Lng_HH40?*KakLoxPW#!>{07{T7yT0%rf!RViW9PRZ z>N?U#$}d1aJzoYeKhYEH(l&?zjx0CQSQY&me<}eUSKk3QU48Ni(9CTiCkW<44}_ZK z+8SWx#o0!}n;Lf6HyT~IMo+6Jqb5SLM5#5A8RZadm)1aYMt>H%$!1t3E`;Im(^itH ziNIm+ZNN3UKPb(61&X>U&3RfCP^C>(J*13{l1bxWC7(D3O+w5QtX5oKKK;stN%siN zHa{m41H{={)>94}+~D0@cOTqrKF@ln)KG)lo0NJ$fbNaGUxHQN`ati>{{T|m9}K&; zB|To!egz6$8ne21b$xbo3Dzs!!@M~JWmT40e(a*sMKF42su9O|mx`O)RjZ~cR@l2h z4-ElY*$rim25v6Wj)uder@ol(LG_*bz)sXr^-+OwEx6jUmSgk8ga1J#0PT<1NZI%d z}d1**YnC$!&(B^!^uN78fJPp<3T4E1#h~B!A?}l$=R}Ehv)@;%I zb5bAZbX%-p)$Sx&VLPp!isJ~CzT=Gq`Ti8B9;2KHhc}qx^)mH^WEyPTsf7;Y8S~X{ z1+NZM)p+iQv&=Q>8X-j05h9}4CZ6A6gb)>@VYKHvDzrBzji9KkSkECl2CLMN)P2aF zvAqG{RTq1IS{EfWTaT|R-;{wgnzUqh_jiD#7WmeQ!p>+bb5-fOTe%D{gfp`82iZyZ zjcX`lGmK&>EDuSxM#)o%Cz7RSdP}7=wt2uGP^OM?Wnha(HzR^GyF7_ewrW!6I#U!c zL?F_BkgFhq(jfE5#v~R;c@f`QWS4Yq#*pSzpa1!96hCdle8kzAPw(`!mZ-d<8d9sZ z5i0#~x=ZN<*zTx0@7BEVpQ%J;xa(Jpc+ADz@^bblLw%#i`w`aeuxlrC$lEGRg<cM+Iok0TQua-pu7XpAyOs#+I5$|nf_d-|KE@^%j=lP1b{6_g0F zhIE1T30d}+Aq~Oqx-)FdUpWH~PJ-VS_BAL+bdgWj#v%3Q`l`u-XbeDaMVvasqUO@G zSN`q^*hA~0lS7t3keSQ))Bd_UPg8>b=T3rW%Pu@}zxzg4DusFTI!s2To_&DCB5!z?ofZCh2B zCA~ikIAjFxyw6egkHsR!I=-@egxTb$f@bb-MFDxb1Fy^Ue21nf(6{fn*)3D!jZDcO z3Qanng2ycV3vU^5k2fy2WpCTYLF0AkMXfx;H90(Ztn#%#qUH5#oV-Q!HkPGt3=B_d zqbvVPoRR>QG`}2N3GWaYE;35w_%eh~Q<}%{KEbmes@6Cr-CkZo|AGtdN3s3-GmvB7 zm1N1;3n8@v7c0R5OA<0_UUE}`VFPe96r^kQ;QNM*`VCQg?F zOms1VqpUZ(ozw@At}e&>OTv-*zp5JUc;M{vqdF#fnnOxAD9|B8Zt&^2I^>v6p{_%) z9;G%W8+ii-^z$%)#jd}AeWm^GT&ahtEv1Su5KAW+sq&qlks(pTK+1=zP(9AvHV{Bo zdvVucDRHXt83pVD<;@-jFnwiK<(#9+$@XV3&$-Re$j`N5l-{&Wmz=2xvJVy@9k2ZY z&7LiZQ1{WZ8_WK9=y1HA-&hV5Gs=9oQK0#P<^vJM80)2W7CO3YyH#g8#k zpM1x6{tpdR)WSh;S|Wa>?GU)i+`?s2*C-s1RT(e;Le%43>VwIqjW9RGnk}?-i-wyAaxR17a< z0Z)>VD_2T$W=+teb&>gq&L7nY#1+E2S3ZEa?nPvhM@a4t1>YM=+c$uBggSD~nT1#w zZ{pSc=B2_NN47ta>G@T~bJkp04n(;$C^t=2aifOz?W2q|Lq)o~bHG~~$d-mKf;!u? z;xcG5?rk!>wP8snt$BI^|Ih+S2S9b2bqz9}Ys87q8^a>kuQ5(dsv zWXwc>`uX2bX%!VkFrx$D4QdY!jlqXXlzoDqcE%@zj1f~^iAzHCQJ%n%6>ZqiIX&N7 zULNGx)<44x){7S07HuQM)dJ+1f>T2i7N6a z609D^G`(6HHngKDM0SsamPWkfzeBKAlYgp z&Wu<6XnxhBTXX)7^O9W#+r3QVJF+-Z?KGZXw#=1=%J{eAvd9J8wE}@Ct%(~+^|}Vp zzHz97JUyj0hnmoCIE4$m<=qoVcXlX}bALM)iGMdr6;`sL{_?|RcYDw`o)`$cEXeO$ zE!{CPryf*9Eti_mejKIx-l0ztb^(?rw5OFP&ETxWI7XqatuKeExF-6!>LzV-J!s%m zd;VO77XEZj%1`9`#utr!M&-8U9**9|wsR zp>5@(p**WScV)__>kTdx)D@q$i??WnT?t2RS`TtSi*Zs$himWBHMUvgC($4Cz*LF` z%ZcW!;e8@NT~O!%uvB8z2uFie6Y5I4PmzaIIt-VdCkzgjCbXfh#LR0I!gZ0TA$Cqh z+t7bqp^`ET(Q}xbPOe_!f)SUwUFG?=tjih3%71AEy6ZGIJmkJ*1xj8&U@v-VHFY5E zj59cstcHo`)aB}|GCs>4sR{u!L8ws$ueT5LcF>)5rO9nnWQ&HZ_Ov>EUTtC|@Ts5< z4l=EdG@Rnfl{?doB2^Wkto+cN1r1Q;16KvYUH?-_n74j{EWV3HSF{-g8u1~OMM+Z! zqy&}BRXPqCF2QL)oV$S`s}POtxxK=<7;TeJJ0tM7{{e+QZwqOZ{~F`7dTAq*DNGUa z{xk4)yE=F`shSqJ8U3Y}3VWX1$uN;|mDy9fkV7(>UJ$|MfTf|V7zA_OPcI-r3J;9E zFjHDKFu3k6HfM@&gxiKxv(%8T^|6=Ue8~P6=#EZcCCf(Wj-{4qgAkykOUP_=F1Ld!D+ebrhaEj(566@>ZXYbdG>nyEomJsuH8mUCyR&=BTm zABpI6J%V9c>QR2uWh@|@p@-SzePt~9G4v3UZfDp}$I{q4L5o@CJbymfsu)(Ntf(U# zLhX=EbeqOe7Urk@T^KLbFGD8WE{4g|KTrg7*BeeeGVy5E)5r(f-_%5S#FfKKozh9V zk_E#KoaH2oF5`Z!kBgnob0@EVf6u{y$;YO$)nw?ihs1X1xf z+Z?9F1Z&cIx zt6!SVju!YZe_#zL?TD{ok}bcPvoJ%lL#5#}ejRZO>dRZeJpQp3>VwRxBJs^xYowmv z2S+#6Wqn@(@%G2vVaQF&1($owajNJ?n1-M)PK;eq%>F7=rNd%$Rn{W6itQzuY9n4M z>Hh|hxY4CZM%rNqLsoJv0il0jEpva-3aAZN!4Nk>AeZicA*$VfF7miLpBU#qJ)(UTiN=MLT0 zn7yR9IP#(8oB6Y$@9!F93Vm-xOQo?c2bTk(jqyQ6m)FeZgC?!=S(aC{6n4;lWCtC7 zgM_k3B9x~UTqgPWxw$0*C9VaYPjjgFv}2+>zCfgtY9~NqvJI+_?kLk#Z5&m)4=bgp zfgGsnisOjuY$KmH234E^e6@11PINxUKi)w5#vgD6nccJTA+mQra|6#??$hKzLp(Ye1WjV(@FlXbT4Xr4-hzQ)+ zStf4IBU`EJSaz8*7=epT;|_ zu{`|s0=VD9R=(AvRhzPCYAUL7a^=*GUkP1nE+eNTI<@6F(NunkRJW`&_ulXGU#-`! z%5aUA5fpEc!bVU@t2IWPSRn?o1d2Y=3e!{NR(nHL=<=9l!LLYf8 z5fdt=j%-XuEzmJ8i2G%#YvhbE!nAHVIKw>Xc+B}c8KW|o{uS7t>rfU$!tVWQrze=X^>WBbb zse8IgzyBd{ms0jt&Q^ohgg7xe$(7Uo4la-f#!dp8ml=6V?GKS*%CEI)g)3AFBOhzhaGT2}A;E^aQ>IGv z5^BX@JX-=Oj~Qu~**6iUvC94YTUEB(Hz`*-uQc*xd`GhhOAo@JACMQK%u<&qggn`i zfRH|x?{>-wv^Lh5$4A{PRMP?qIU9y>q%Js`=+Saw$?T~CC^CFPIZoKivE<|^Ns31- zc8TzC?sW8(*VhmrZ%-rR4(2!{sR-&xrf##Anw}{es=%qEDH0<03K{gxMOT_c|AB~$ zC-x4`Cn=n2sI(O~vF$fRO@nzlUd+1&=9>k=rM`b+hxJenH_?np?o+jz;`^hKA?_n! zd76^8w+3TpPdN=fkcS;*_hG!4oBLq|c8*Pxtu+uS$ChAcbbbNlP<=X3Wh%FoSLXrc z?|NQ!i7CYk?Y`rY-pZeNAq(CZ_c!pFSB`hz_C{@5_2LgP4vMR*>}iVx_G2%PR19Jn zy{H?fSWh)Fnxh&J5o7Nm9Zn)O2DNlqWZ@q!hdj6kR({E5ydCw?#fN@^uV*`4m_<%Im32WRgT?4Y%f=F-c!}-vuPgk+E;g4#W4r5mog@s_gH;r{}@Z+EbrAsT>WEx9TxboZYj$waF14X{wU_%TlROGS?&cM=D1emAA`D z64|Y)?n)pwLw2Bi@YqN}dla{m?W9v8w?V;5X*IAu0_bEQIeVh>`eHSD7Uu7cG z=&Dp!5>$rd;Gu}V^h2x_e{>_(nq5mNB0r#SQKw4ECr~5SSL&bCAAx{i=#3!fglCCTx;!$lgz6KDn|qxTE56-lIdj7FmPCNDQJg4NYqWW zB~;biUGM7xCU#MKKnqHvxpq|@J>g?rHM?Jg0{J1$$MOolo zsPYo?DIO0YhaX2HNgDzc?@+C6_UgI4LL|g7tq}0L0hLamrI8dJ=*Qy1R3oI z!yMWLRxeKj1$)s7a^)IJKC}vOHXr7)b8DXFi{!Vd4jjq#5kZb#^0qH5*6=)kdY*7| zJ2}9uN$@v@hjph6!YYaBB)zu4S&u$JqMVH~R7EPP#@x`!O0KJ0>q>nkL#zG(0{_(G zyUI@^5imiN2$0av(E=f(SkTHT5M1NUKxqPJI5GbMMrmtQ7O$WY1J1Lm$n(&h@mOKW zwV)kK3(4K>2Erwv#HloTmUI1hmog4bWZ5`|H>>NxX@5ZXnC*SlHAn$sKv6&xsBT)ZTR|~%m$3}PdLa@~t@3LSar!WMXhh@Cq8TX7`92h{lWx0qH^}3sY4i<>V1`QI zZS@l6;Hc(uN}v8l94346Kso9SUANb>2g&xY4Z9dOa0b-q>e9rmN-W3eSrujZduWc@ zrqz%!#iNh}*_C8#PbRNPmjZss>Mb|t17><()FT+h(?Y_P)iPNx9x6TD96wGOZi1j{ z7~J8Qj|ODB!#VA@lwkKc0bUI1~N!!2>ous)h6ONwCjb$jE!J> z{%zPm#1l|oS5K*em6u|Xzn)=}o!W%h8m&&-8-3g+BwMJr%pSxWADuuy2Lv(M-C2l_ z3#k5L-8xLx%;Bwzci_R9D%_nW?QNv)Io9f&4Epb=D6@!Va3j*Tni9v|WO0Z&4Ra~2 zfsEX5gi7obw3bsZ4waTDZ#nfl3Z%Sl50FIyU9^gJ#bInxl6#1r+eem>af3-4Wy1-P z-&$TKf|2VAg8lWPCb)4D=zH@7;j_~xPaBmJudb7d@!>~$$u%2q(%8P13}_1vhbt4Y z<7fb zth_nRT;*g*BU=Kh{by1-L$xEpsvK&3QyE9~8VEIRgmf6ls=o4xhR*1!^SgPhNyB9y zs=wwEbZ4xFlh@HoR-J~Kr@dkrYN-v7Npq0jNnZoaT|Z=` zNa{!8J4D|h0(^9;g@#Pi^Ucz-5>ow- zq!^@10VPRPndz|fOiWMu$vgq8p6LRtLdwID)Fuw#AgcA#S_c(P(B2whs%uah#a$7>C*jPD_CUFzo;ghV zmw@W3{_1vMY}uSLvLgcKRxJEJ4k<85ck11A2UF24=WAF zZj=8ladG}X=i)Tal?(K3@=gTcvIR z+UK@z^%lzmV4J8H4CiR%7%1bD!DL_E9zu1=Q-*0xlG2DJ zv^WT2!Rz#U8Ki@2HTc@3N44cp5jj|S>so6Slj5cRUJkL1ISLZr7dcAqs{6D|J#-?# z%dQ5&Xoh?++K7<>4a_mJ%)%gCaQfgV(Z+L&RZtcq)iz1~J_%Z@9pzWMs`HzpJ1TMH zDjJUZ%$=$|E==YunAJ{(+%-a>y0Aeq{Vp&an0a24xHd+bj9bdINA%F~GJZM-<7G!& zB%Z4>FiELKwu(LqqlJ}6v~>vl=XG$(Jm~BS)y@}cxWx#8eb5QEJQ$%F59W@Y?ptSt~fT%eqR!~CS+ zERdL~LMJJ=I!T4S)j)V)ki9lsVKTWrLN*{DYp_VqcS-hYM9WS+t)te*Ck5ye15zE|P!+}BA~%rVCdtO{#g@R*^FFx*T(FriS3 z^`cl=N!&ErB~Hn0Vg@V|VYRnQy5W22UF!7SUuI8XyoteZe^!P6v}HzFIY9F# zX+90W3{&N%d#mzx3py~s1U)`PCQx7l(;!bBg%dhR#~TcgwHWJWf*#iuq9;tRPGyY1 z1vZ|$smBeJg60^8o9}V@+1%0y<=9RMw=g|9Nc%V^TRW8$hWL%|C!Q!HRdJq`lOy+F z%?BH)l35i|Pccy%zx>7?14e<9B|39a{j;)M8vs#W-vPr?KD|BNtvvXisd~JS+;UE} zS;#!Cp)4XjQeLSZAs?u4XcyJDC4E0o9uu60^_}3W^5ur0d@_RM{RQ;edTt^yUTARL ztbr?3^ty?5b=6Y^(xppoC?(8rM1++UGnv6sM&J(+sLHup<{wf3_Wj`(?z_0UUzvUvOM}Cl4vMNR>|b^>KHNd z_DnyZ?I|dL#M$^YBvBjX& z%I;qA{USZ4hJ3iy(B#Aj2Ow*g+IBEn6392OZrz;RPXXR-;Mo@;kxOm)s>xGAIX#wrgcF|()X*omXl-G zW)$YC*@=sNkBp_vUt<~79I&okffa{`(=I#a!iR(P1YMTRFdT^Clv?tMA7bG!)dRTn z4V#H{N@<9GwA-NnS&e0Y4-C|q-~@To9`bFd3X?24Ze$}4vqEHz2TV3py_`F?W&PfI zOo#-IH}bXA!aU8HUM!|5_@;7RCFbHMv(!Ry$G22gp+^RhA%*fKYycC$S3KKMw>>$o zsUp8_)$cDte*Xu>+cuZ299BJ(?W)i$chHTrg-KcR!&0bx%Tt!#SvQ5tkCZl0cb#cC zDzaM1xnGR}oI#xb%R5;Ap;oRuseKTD;{^Dx$#Ql2@j;2(c0*G`R8sp-o6FVUovHFdg_#?A=xBNJ7ekd8gaCf1 z=halV`oS`BHwJS_-7*q26*dh0;Ea}+9ias*#VFZ_7eRKchgy3_cxa20EvehkJze(0 zfLe};(^_<>a%{JgTVs&$fBpviFTCSNvaJT<7aHDzi?YZ%(jnhvDcds93G-(g z)EWyzr26W4mE_GE5WP#!tuNi0fuidX7I}$BNa>@Fay`RPWjt}3QKq&ArlJRnLHCT2 zI+e_ndD;LGr@VHDA&Da!NhIz#(4Nv+ovNxzrA3AX?+ult4I9#hIyhGRI0-cLD~9hW zzpq+$6j-G}ckGL3-QHU=NH|B_=7%VAuqtLyd=2Glh>^|&UQ1dhW-HSU&b*^1cws$9 zPLPd1x}tDSg!wa6ltZu7p0&&RMbPR{pyNg>v;7!={eQReDA8)p?k(Ju!v2RR8>071CUWvs_lckjq}Av z9@<)^<8#)`f&|v}fScR2Gwh;wLvVE0iz1M3sm&b@r-WIWD@OLOWZl<-i^vX(j+1w)s$C~lm(4E) z`=Vrsk%o-S4OO>FJz?bHIx^VXa7rYh_82$ubAhk5HGiUOf;-y)fXu~OgsN~8d=Bkx zy$1!{Cs$Ei%5Okpq&cXGs6pE5h)4;YjLg1jb(v(g>i1LcHf&UV1@fkO9Lz}g!8kyh zUtZiN(Q-e|bKBJUz`;#{V?6%uT#J148TfknoQD%tOm-Ae3vx;Ya{9f-fi?dhc5tzd z5h))XgG%ELxKiP%l#$x!L76o)Un2~Gr-jJ!Fm`*eTcQP`$@In&Bg$*nr!alrBz|bq z$NXR3VHilyt*frtf-w8yRRkla*J%-1@{pRBSfzwgwipa9C+nG9e+}T(9x4K|=56q? zvnir*$U0XF?2}wWtLzh{Rq`>(4ArbKfkPuENlXL=aPlCzy$!jsjg&UIH3ASXR{^EL zdem?XjLh=#$67Z0oJ2eL6&Ki7deLmeQvM*>rE#LmR$1yE)jRl*9@dUC7v6($++<0F z9Y`EFXwF5=*cx(KR8&!Z+XOBBamJOw{Y^|fkwvzOV$^k|VYGX~wH<~3H8~BCd{wSx z-a{8&_5x00lT-lGx?GfN2|ZO+MZf<5TW6g?9P0-*PrumOtcc9$XMouSgTFNETvG>R&DUk;2!HUgHm02PGP zQ;I`xj@?bV5N*q3OgmvmDS-aN7Efux`LBE-@r~ZFGk`FXB1$v(3MD^%; zLMsl^#pSr~$sQ>aLgCw8x;0iyi_q`~WrYwIAJ+nv%C*en-(kZY%^74FxwzWq*m#BJ z01jyRzS6!d1n1OKddZ{9tQ5ss3-7Tv1x{C`>69y5Pa$8f2W5`YPO_74Rki$k;Y=2#XVuj91jmW*Kj;m@^~TFtb*=bw z4`{1WFz3dmdS-XjLtYd4=XZXntg5p`1n0@&#Y|vRSx*BsGo{LEvW79o;KA+jsL%QM z-Xw&YmzvEd8^IR6T&_i7jnEk*4o^^8APi#ZC`abHDZ`hKLH_l8SFUt(K>6S5sU0Nv zj*%^A+qx5|Mdb2e3`EzI&dT&q88qVBy3;8i+++c#U!#rj9a25%J^CU-k6%K%OjK>8 zMw(5wkS7l8F^X#H1+%52dbmy?4}aAIvGw@|fMw;5N*25TvTOD18W`2Kq1x9GDe`bU zYfn@MsaH*~2~s22R8!s794!;SR0|1mglS}LU?yl45oh^RY5c($WA*bb=>XxV2PnC_ zexB}-9k;O(?$e_db0w5ORh1;}ugAHdW?P87E;9b>l5Xa#=b3=OaQMqXH@4fjnl9|> z;7CdQ0XcPE&z_}a22-qj3cKv8ZvQJQowT5x;gGNYG}Lh?ssRE&yS;RYa780ch*4i9iNDT}$Sx420ChScQ*2K=WY$&A}a%7Rr&y;VagEkQ22k zyG#<_29RH!XI8fj`8CnCjwH4lgJb7DY#dH-p6sUxEsev|q(K0tV6)8-!|5y7D#?ct zZ=VN1c)d46K3RkfN^&TVT%XS}H9a9w`?oM!wOH|ynDHP4KR~c@TUpRv_VRMZ&#;2V z_=?)V^h|(c3DUlcwXmg_VRB4GMKz(V5oc7=kZ$$C^r9(wkc8}TkzLaOeEy}|g0!na zo?fc6k^4(?Bwa4^3I7cBCD1o6fcw!Rn|PsqP*rqZ?WtR5NdK-7alW2M|34Mm{W;c8 z1-)2xZZGOYbbloNw1fz`7-U4F(s6gFeC>jmD_u5{0S9dXsh;nUHuu0>iNbuD zu*aQ1z%ZOTFIM5w*;brakuN`Ab2(+{`f(SgD$Ly4UM{xS_)3a5aK}gkb zbyfMM_fi&8FDFhSKX%*1u?zG^s9WqjYBEX5ae!}nHUziyEh?(T$LFKud24Lv0eV`T zOjdzftM{OR+v;g{IjExj-rPXlwjT}V{_E@ZNhxHQjq@$?nw;`>dc2P~X0q9G42sat z9fnwSE+CemKxG3*wUB?Z5#4CJxflo5k0kqh42F(vU0IrcgeI>>u#_t`-FYZvny|(E z4kl^CLLonrFtmbLd?_jlNjjU0<;TAu!)EWXgJO^ISc;zEkhY}gN%JEv8<6Eb;!hR$ z^FGKjm~78ftnJ8ZQh>RoytN_Rhh}us_Qa;jmTr*GP-3;{QlNpL+`5uv166fYGfyWC z7@$f{J6Dx1YN#hy*XAaL%4e-$UrIC4oF_ORxMwcLC0cAk%q6GDsIBnf-%Wu251dty zm7WkJUZx=%atuv1`5dX0%E{u8!L{Hy-AO|HHkkAv_c@eo$yDJ7Hq5lVbzD0am6wv6 zp*EVq`=C1Wo0A|=Fiz`Hs3E2@0#tEp=fDJ-?omQl-T%76aKy z8xtwS_yn)VFrBug8N~ZR(3&p2|3$M`Qtb{3b0 zx%?;e6ki#nq8Vty!EJLnBMJZ}Op|3QzaGDZpX^9+Co5_xFAuu8Jd!gMlHNJdWdSH8 zJy&jPIJI=2cr7wQ9e9b0#4>Yootf87`crj7f_?=WZ*@;~t$#$e+*yXgmUtB166H_k zTA@9@vtldE5MxN-8g@6&h1>9k0Z8*tx{6WOgn3-K?v9c(z3~6_{>HF^A`&NBlX-#Q zMpBn6g$(%k16Le#+x(QtSQcX}p&qi%Y9(0H?!r3G^TWtXt1Ssm;4x4y_EJ&!)n%m_ zGUCMD1$!~00p22OO$f@`0^2YyTx@=bm;rcBWcP0fJFA{oU)$_cAYHkIq3BUqD@0PG z8RQ>5N#>f3Xvrcy4Kq5P&PIAh2&Y4fT_K5b0@YnA@3_EE!dHQT87c{aG+8&HJE@Ct z1herTs;g5yav5xuCsbVe@K{3HC4&J zdFtwTnJP+rqozS=4Jw0k*0-)uR|B2IT@Ayu*Ghs0!1ojIf zf>ia)si?(8q)KVL&m4a;nkcWmMY>JopPh9}8@$|kRV06;%O*Si2l-CW3tB6SXPLBr zZ=|ZcQq`j*w;M(Vg*U6{xk^w|y{~IJ!tIu7yEXi&5ehD}mQu0|BweKEEy1spI#~8L zMTgAy4sa@`6j5f`w7!z%DmTptM*PnvCP4M^7>ifbY?Er6T~5#xJxBJZLhQ{ViqRL2 z5#kI7H}oj-$NJ*lX$@9~tT!j-%N2^Y0ZeBAqS0Zcv~sdMtB5RnshDVH3r<4UzeU{9 z@#bSm?7+%KP;-U%9o?Wwts1sPS~sy%1434&xx&Q1DqwM}P0YYK5msL=r8D}mLYx-$ zVPHb@%Q2U!6EZVzl{Bb<&HPq3bw&SYsa)cCvpi^TWOAN!_5w-yb>z$c8Bx-`xj_NX zp@_SadTub%pYE^Q*1MfBKx7MPnZTlE>&erkO(Oa$NVm>}UK}>CE7fc#2CFeSmM?iCMkPwCt6ZZaLG_|6|smDdJxQ(&;sX% z%_GmZ0~w5+vPk~oIGGxWX215Ekt^nL$h>oUHg%G|0Slgbh65VPUWX|W=7nD1$`P*K z!0n4;sZ@FZRBM+t@J((5rbAb^G`0?C~?7jr+Y zvb?43AG(rbu2fp`G*kOJMU#*QXsB^T?o_E;U9qPOk8B-eghO75W8~9iP(Q-2HVcoB z&>F_&!ftWOFIiEvs_Q@L@sK=FjO#?g&QxsE~AebR%tNC z94~?SaL9TKz@ClI(9#MbQ_1y>Rk%>xOx7w6yX*j{O`zB$Zqd?G8P3%b=UjMesni37 z#<$tX*B(aNr22bM?Q&&A+x5g@SUsj7Rm&g{uTNq(I5Z(e?hfWmK*)DFk?~05tPSd3 zr79^9;R7)MO*4sEceIiKm9$0v#yA+cWuA=w70$maAAY^S5A*v;Wpho8N`kSakHYC_ z?+or0ED=kseUzNTY#|eF7L;iX28Ou}kcdvFG!H{Dqv2*lNmPU97p6*w)Zj!tt*I1# zkA9KfESVJdF@|b{UR+CMuvCfYn~V8r9ss=`k>KWy;)OLcn~<&)o6f*L2&V58gg ztnTapBLa7bS=O`%$*liF_Lo#KU+Z*xU0e$`rE5obXIY*NX#S3}%#-cG&4?|Tpvt=b zsA^CAXGIds)@_}|%?*-lz80IVEzC#)KTRW)KedKj2{GqN&~xF>H^ZZ2`a&b96Prv=^+rWt%3Y`gtIT9nzHwMYNR1_Nq7jViIk)-ZYW#NpDVLw z! z5%`X+Yn{!n`TZqnBg-Zy-G-S#Cq+XKU#T5Q$&@ADQGY zJPnpX@cN<1Y|WG6Lmc{47EVzu7_u`Mwnxg&6}@OQvKtugk3$dJJ~Y;<4tO}ih|$WE zohVgLz&nQPju8No%#%y2-B#J`#gNML0@AuB_IU=3rS9i$JBDNS46)ZSlJr+gP1^L|4LYoR&toHWk0Aw{Imie-k5I-)w-;U6 z(@0^cA-lgoZFoxwG|r@S1*wo#`KLNWfa+Q(5vJVhzK$^SxEzUhVuhqNhBPMOfF z-#m&{P3AHyaxg^$xSmTJ1}6HPq7Q&~|Cnu%g`xYmOV?BsQBW^TJy!;Dus25KiSf!- zYix`p{bG=DmJ6_2=&8fyO><;M8)f@FR7BZAd63`9HWEPoqDC@rD4cCgNy8-;>|bqF zp-;knY_`ExR3`fS?jbS|qem}_meWtMq1z`h`1`*ho+Q$N2dcQ5d`FQ^*asZ#_Bpo0 z^?om|)P23AXD_1&=3)uNge`8N0-jbO6Jvr^a8(fQb_~Ky(gWq%QRwVOoL||oacEx7 z9#B(uPgxZJ5)|wCpCGM^ykMMlgz{^sJ0Y(}K2U)NVXt`A%-xgh7=W~?sXVZc^vvlL zc8DLMHLRY8ce9#2{Q!=(j-+@Qu>=~VmwJ%=l%E`pi_p^z&EK^GiqR^-aMBjRstX1n zl%socu}651OQktvnc1*g&q|chHz3DKIexUnbjoW40p%BZ^cHS$y=GKQOAYj4M|osD zy~k&bM~n99{#K~Bj=jp7+sECGs{uxcy&6~(Jq@DFtg7;$DpD8P?ZMd6d`dm7#=#~n zhi(Z5IHaeI0CvT?tgH$ZK2yhjSkGNAB~?=4=C6%BaNM?$_Ei=Xf6WUO`|F|6Bqi~_ zX{@aOi9;z?2?6)%R4wxd@YYx^Zon9;OV+7$-lI){B2Iy-++$?#my$Q#N%8@WNsyu0 zE#U8_HVV;3O4E_Ci5QcK^f)dZo5vMbZWVmpew%z(g+W_(G23A)SySbB6s`>04q&D# zWlNfBX?C!o+a#}U#l|B7((1rHb4~0jZ@HEg<%17KSHftgqQD2b%T6FnVlQ+Byh?Rc z)snq>eh)1X-j=H;t(DfwU*GM-H_>Q^o~rWY9kUe$#o9=A&PFGse^EMwQxK{Qo|?cn zya_1iD<^kTpqcUb-oc)1%Ib$2UYYC#!MI%NEpyhmizU~}P?*${@P(%JljPaKI9dVu z61{kxVgXk9@`Vx4wqqKr$WoJ}?nB-GRnM|W>nA{Pz3z&`0hrZDdLDE;xzk&zx*7LCVmhQo0G{o@88{Z(O>>{1n&7M3$7$dD75frnImroMW3 zD=5{q2t=~Ak%IcM*gk)=3ag&k9Zwb6z>dFho)1>lOjtmej2a$W%#~H~I$UiO{^&lF z9gqZ-pd|+renC-fek3{3z6U+4u@@-x+rax&pGwuX#^pme`IV&YewOsRUfcyk)7A!i z83cXn0=(xFjdUs1%N(zKVS%7|c&z+V7rxNE3U287>Xvwc9y3#g9E^t=^MbUmB5`?r zu+OKiHfzeOp3HRug`RnAUg;h6O#-NpuCl)KdT@;C)np~jPB;|8`N~fM9(PI^exz=C z4e45j6DauBOnz8KmaO*kSiY()=WU48sTYmoa>f<*G_^1Py>12oX+TnwMP$}g=G(in zVI)ZZyRwU-b;m@hpj?XIrlFX4s_^hQtVL>W8L@icedmQpTFd;kGE+0eXCxJo^n!yeZNog@=r6 zsa4t?jb7#i^odSwF}^cOHja^`g&bIhD$vyDA~=a|G61HVprT`QXG-6}0CN9Q6#lxp z$AnqvOCoMXYbKrNa&*Iw8W>7f3ioPE}^;|5p#JE{j7r>xO#XXgOAZ zSW2+IH zL_;X2(#%H5r*n-MPQe?92RmU7UYuy0?J587yJ{+0BuG)9yN+`n6jiv>pFM8;7YGc| zQ%$hBm<2NC0lRvmJ9eW{7B}Zo-_Zhkp{p-di(RTV#&m6cT%yVbQMX<=73Pu5&X{=} zkrX&zI~3;1wTf<~r}}GYr6TiSeAJw5t;hFK+-ol?igPjg7@h!}LvvwXZ)oT)bDZ9d3^_3&J>U{JoLHa((UO zNp~=Gyq<1Gb`kzCN2Qb-R9;o0$00M~V4(}yf=H&@f;hGAmaPwCt#HNs^4hJaOrG_< z%i_|8${07?>gG?_6j){bL>F`~f}x5yS<^7o9@lDx+-s!lndBtcNI_K@^wIyD2lmcN z2)Xc|`A10S=5UoIeL3V#D&y;Ud6*=D{Ed`PVVNXW;8fS^@ivu+Zpgm>cx|}uF{95U zc1Psp4wq5&U~>7oqqR1<(5qbyPQ1j=%b8?^eDCMOKW*0hMcrWRS8(1#Y0SQRipT)w`s>_&N$;Fu32@_Q0rc1id z=8+nqhPp7ej+SYOMgi!X{XT%tZ4Mb&=iG#wKn<_7S!s^`ry5@b&%^zdu2?SR_RB;A*c(V93? zzAJ{M{gOfh&a3VOe5m1@FdX+y2S=UbwY4yZGI~s`%3vXzP*sZPuIGp2Nz1Q>_^}3R zLkg`7rp3$QQ84a`y1kmb{#ZPH-}Z)_R_huN(s^x?$tEX>o>5iI65*%?$^nzdv%AGj|re$+b(H z29q-Mtag$)%k4*}cdD=`UlsOLPy_wWq>|kWYDieT5iXhIpprLwd0kE|MsYPfB@+b$umVregR2?b=D z58Z`8)t(TzwhK3Kb|{9rO%>yDTHsSf)!;6KD@48JK&WOGiP*t4_C)o?enST_p396P z%qqL0_Iq#vY=&}7Ry}`+wk|kJRwP5hiz!0IS81QZQPOJ4fv>=Y!s+|`m@UQQr2ltr zuZ(<+b9Ic{h*v_XBA~-Z*Ivn}CAy=B9BvQ2|I@~8Q;7{GxmXBOUC_!%B`wlZUgDd8 z)mwDu3>jf%d*^=U@LaknR-V(76T~m>#=P-cu~aoxH#EJT1g$|2Y8lMK|6XEK8}+On za0AaFOxwI@P|{Kt5N6g@w?Yw|YHn4)V5%j5blB%bf40P8n&o!Y3R1k%OBLIGbOx49 zTuLSRa3(Tsa!xL5k4fP!$JbTcALWn%U0Db{$m7A_q!q~NJQW|2FcE3t3W23rhRWGP zPi^u}TaNOfa%C;h^HqJw;yAf%%i@nYvG5dnD~yFl-KF=YRz4#i ztjVh&G8E0_pzewQ;HDGMQ^E*Ye$wzuY>U`%{5Rn`BFzr9sS&RIE^>?rZ8MnZqXLdj?y;?ygshm)%7P=yAXREUYhtA`L!netmoKR6GxNX44W>= zE|Z#@{aTq|8+Jk(eb!E+C7$}(@Tf{dRK&?nggb_(B7=+SDUG5P?`N8|D}@euTnuoB&t*-mqT;dd z34JNhGehJ`g?;HTd-S3OR>WhZ9=#0(gFvIvdS+uO*lte8#>xx>)_K*mTu|xQNc4+Z zZt_piKjbbCkr~rCWRTRP87}<5)D3G2>p$R4_u>n3clFsR} zyF+Y|oWGBlov(X3$_GlIk?5Gp$=y*LMbVpU1eH z+nT);>*2$J0PPD*oe(ZPPZ{YriDFhutm=7s)B&1*sb@yZ{>uo103YJL%RBolpi6E! zi7CWCS4y=L>An(Y=-Pa9nzVls>s4~P1I#0#ONm z9pvrJn!-1OVgt?wdhsdY1Fo?BD; zb3K4>-NCI4VI|e|^g#@4DJMVfaVrOdZ8f&IH%c~zfLN(|OgFjmGY)|P~(m zpHYPl&D}Rk+o2j!mAGOyYfDg+mYs%SSzg3O(YlEYJZ95|yJ0PtCpo$+V?~yr;EXW=VbN6hJP&?MuBIW)Z40_4EKY>p1 zPn<>!dN>JiW#s$D?6Jw@Sd-kP%MI#rQZ?x`AG8~yymR0GZ797hzP5B9Xtnt>QF1{-s&lCqQj>mlBaY=9Bd#v9jdfNVA#jBgYdm|P}vd> zLo*2-r1iT9_SKQd+X-QYx`42Dkbf(|Akvjnge&Y5_%d?s+_O?ZX`YQA&Z*L5J5-&C zMY^>i%0qOf^xA@?XjsVo#`DSOP5o#yz#K&@fNg$NsnLrK&O3pn+w2E(0n4D=a2?%I zhjt-`P5zh$U%jI{Ysm9o*wJJ?*C}&)0K-wbw}Ctkg-_H~a^S;QBahy_%<)xI4x%8B zwbWAyR9q63k@c6MwPCtz3^Y(Q6s_4d1D;t-JvOB^{-&F#Tk{!u)iqUcYJ{rcZ(jvw zOjC(TZLb01OS<UJn+T0>4Eag=ON zzytod@)noAZ}_djn8W3c+B~~Z~_w5RIqC7qBwnd(VWdyzY&e9?Tto*%uYrPqwwPV+uz`b@k4n@x;Gb^Ce`z;D%%uB>n^-IS*zv6 zBJ(*p72q~W#$@b=?+aCsW15ay!L2W`kUikRT09|Ekm<}60IydAWY_jHCUT;t)T@bE zut`rW55kZ>b5i%Vk!aGG=(&Of?M(GxQrC*Al6vg6YHfmj5K}^JNwK=Ardkg%{Aw>x zl+%lkDj+nL)6D@EI_fyNz6cfR9tq*Yd6+AWf4eyu1bFmw_i@8 z{c@Ax3YIdh4Z_h&B2+SR(q-cTv(J>|C>Kyuc}YTK@5xw~wmBtHKD`KI_=Af<6gX2$ z307UpX6OzBre;@GLrevA|3-FoP8RoP-kPYeuBPmh20;_`@LMulT`CtYPA-3KkcpC76kGb9DAF@k9qST(!&WSeolb+BVK}HbS$#&)>d2kdmjn{nx@e4TsNn9}` zlr&idO0Pj%$hB(L47>J6a2B^m9-rKL+^w9bn4aM>;&?0`z8gl7OgR&qg^`lfOdD3c zsIfK}z3YIkRYlTff5mAM*Edy1GFfFS8?Vu)Jkln4oME1wB=2H{&Avg$@?|yoRGo zP?-@w=$@`H4X$pf{8UBW3Ks?O?Y+xdO))#UfWE-CsP zbho#Z?>E5+h=WhjTBM|b7zuZz>n_x_mfbnBjP>w~QElBe-b!x(vg8NJAAh@5=&jZr zFla;Bxbq@rr>U2+B%RX6U{}lZf`#%y0iTYNFUO#EB2f$fPl-Ct{fDuOFoWO!{LrhX-Q$GRsrC_L` z!aXv>>5x>URgY6_%Uc<%%My>)L??^iZBWfF-G54wLd+^BVL32eK}i)&8OOReznR#wdinyXq?M|ICIc!|~krNYu$o1LLrS`=r4@xG1n z&qfr8Cop8pMr8lluvkr&Dk%KfRJ(AwesGufyqz6MOSM`~oc*kA6xRosanZv~Q*jgG@2eh5^%wux(06QzzEf3`L z-3Y)oxlrgf)sZ&0Pl{dZh3l)5ihC|M2Ay|#H1tI%aU(h29TLpcGlL05$6Ai^OJq}o z%$K@tD)>MxGqO3nnEtl+|0ygU z8J?q(=11y91eyiMsK7J}RyB8_?6*03pmPp@@C8ZjM41279Ea3631{2$9w#*JXAa`$ z$NYCYp3z#8pMklVRSofNorE0Dsx5=o$67_!LUaWBb^{fxCewp}pn!Aos4ECstox(1 zkvY+*d=8&n*?_*fLzQaWp=ZZK)OiE($YqU_K0^(glrRB5pDLtB2A);UsP<|ps!%aW zJT0Ihu8Es%L1P=Dr_F%-m@2Xi%PqvbKLTWoiBOiRa{MF)Bl;{ZaM%+ngN7pi@Ic>W zeW@+wo*NL7yB4fvN2N1}udR&h4JUq5g2_0s<5F@D41c+9Un%QffzjzX`SQVs9PwXz zu~I7Y`fKBY<0S+K1tQF+!rp8n<(Trn_E4_Zek!r!W^2y$KuQ7eR_P`#xoksN8_imp z>+y#`o&4UCR2wDv>}AMkj-I|;s~?4Blc567V)i4@z`j3K<;HCTaiU?EPmoqukz*MN z?nL?EB`c*QtXVFrK5ZRWxzo{ulKRNfWgut+bv2%@GFs>A#Y-{Fvu8uR+4Hn<8Cfc2 zx)ZCP2fsGb16}kY2MuJ_DY~N z3RMh@Ljr8pO;xb4?S`y4W|$Z>F@a9Nq3MBW6Inr8ok*`1!hR;|X{W?i3-Nn<7m_~h zN1&Fclv09s2g3Zl$1z{E;gXomyuacx&?>5q>do7mlYnHxXJB^BcEntKIaNkF0c9fk zJpyRs7w}rzP5G;V;eeDgs{kZ?X6NAgVv_13}BCLb%;y#ZeJB?j#jHp$&IDuCIn#r7<@p$HmAOQIk4R3$k}h}`*5a!L3g%j+stKIy_##=8 zxfIh>9=PqS+z}zKXR)SWRfw*do>xa|wl&isat|t0)i`x2>9WY*6JVyjbl)*{k~AEx zIK2l7B(9a_G}A$Bd7Hyl^XqY}VAuiOSDR^Zm6p=e-BRKY%%#D4!fv^Hv)g==cE=xT zHz5Y%b=kTL3hSq*w!<9~O+}+=>~u>;t}JW7bm#T7WfBsE%-zX-vCM4ewqwu`FZcDg z@bin5r&WO<-^`)DXoN!n-q|p4`xFo@HA03?H1cKpQ8z*>1|0kA+t_Sq zF?%1CYh%MP{!(p7;+Vs7gf4xk;tB2LRtq$gkd2%?g*jZZsVtn@eaq}ZKt+#{-Jb)O z3_YzDawxrt?4x>sH2#!D5+0^3wyb`rg0?F1?Q%E*0i7nGlloAi-iOuS9|>BM3LT|g zGN`KXLKWhm(C>asYXZ0W5`Zi>&Y2B)+r;g~uS_;wnLVHfhc^UDwufjV;{0F%YEzif zVOFcM-jnrImxPh%g#uDgUbeP`%xb;{tgUs&WVH3P5EZ&nPA>04A!u9qK&87Jz_ttww*r78_qm1EH+!%m7<Jn+DB9>ZZ98T9Rq* z>2y{cKOod%LMbG$VlP$`JEJntkwx%F$4$2ty(7N8^azS|f|Oa5`pPv5kSW>(=H}M@ z&Kh<=3eiliUh}+ijYgPQ$;igzad6}*dgfr21Q#l+2jFDl96cuP0Vk%6m1j|6tBj@$gC| zN$;xE=Fw8>e{eA(s`i0r-jLvaqn?0dB*o@QLSG~{z1b>3Ii62t zu?j@!l?!Vb`~jz1OZSXJCn~lmCAX3c-GOvobC5%b1cBsoJ|w_}Y-PFg1H9I$+jnYz zpcvs_oh}t8GfUc_Q^pF_hXIdn8auh@+KOE|c4r z>#f3=dGec~uEa?(oI}46$c5%4yH$tK{7&v@h&HV*H(~Ks*;&AOeHG1j_@>KphS?Pe zqVP6A0C`>VCoL7gq;Qo)NjdNwGb4NGrUUOVeIhHrG?abe+C%7-r7>E3UM)HL7>zUR zJ1`R=VU^d^+veAT{@k`DmO+wRH$icx5P+UF4yh26j&+~VQNldQs(=yhxyHJytCS&- zQNAh-K86swsw8Jc6kj~>v-HHCGG#P-t4v@cVa9f!^wsydlozzVDhU@XYrX)q&i=6l za<^e@xTHD^FA8qHf%bxXWv1?HsLri}G*T)!>!XEYCTB`1vl} z1?Pk_sHYcg!>y3uz{*GpMOpB)l@(<~~V55=1r-Wuo!`Z@pPyARW zN-8U3;**o0-OQ=n++@yI{@(q}=A6m(O?XpTRRfZ5s_GS8&o$^dP!sz5C(bK-8i!_? zIqcN=avF+2!W3P@GuUmJq6q3OH(FE69q`0sjeRxGxa*L3RT)OKp~14uecwx__BACZ!mK zH0C{w9^wYtm4fBq)a?m!0#+$!cIEnW__0w>Pvwh7ve z6u;DNfInn}QkBD&qX(OGYe$3|eL70KH5}Tc4CR>0i&BEQAKNATVR%_|sCJiPoieU5 zI9c18Y=SAEWK332nV{!Zki=%@OeOgi$&*jD;GJxC5Fp*aoH6Bcnn`P#(F!h1@sR{UB<`6xeAL1T5f)|W9!$yrMH zyJY-X!zCJ_dHm_9&U);vdDp*yGm7r5ZY~=lvoh<*-kjtts@h_mJtJFRmT0h{9@G}% zjr3izjmEUlM_Q2BPoOXjSB?z=St>{ymC62^ux>P#m^pIyA7DE`PwS}F3pRnMS`f-j z5uuTd+yz=;RJu%k*Gv{$AWF|RJuLzy&eQ}PwLicPPFM>;q9;5JaL3?&PPcL76g@e( z9i1{%x2pi+%-Wn+!2;>n8RQQNVkMY29ymw#&)7_s(bz`>%@AJDPR{({Cbs)~Fyfw` zxDlMm{|Xa=8kA%8^!jKj@eX`Bngf`H*AZhJ#teZH75Ni#{URextDEA-GRQiqQVN&K z&SWD?T@7_ceEJ>ozq4j5#&2|W?lPh~NVm-_Cmq#p?`5DM}m>AdeEIdFt8s32X`77&44mlX z_{WgeekC{it2@2bDx{!?p1BrXJaHZ-eE2l^uC3cnT|1b1A9ZK7O84DHG*F$bNEJ;8 zBuA2U`r;u;RG~Nc9RWVJbXyCo_aiD&a|7(FC(jRX%HQ2Y7!QlH;Ena$CGR-=;-V^{ z`->`G;ZTeEOV5qxw!!K`6~eRebQ|2PkWVnegpY8)nPL#H)`pV4fXIj~dQm7d#mD1F|I0Aarzo1*o1GEw)>}IZD%CpwXR?xl&^-&pFQk>kikLJw6B+ zOwH%vTb)_f(cMO>+^fXq50_<6!TWi5%{bOEIRW{zR@H~^8U?cW`g7|s6n~HGNN0eF zH`v2;7tdYY%%52eu-u#0=hvlQu;_DZndn$9ZzSSbG99T_!|)R= znaKF_VwgC5H)~s{S`eLAFw&YCUb#5gTqIR0^6O<4DSxdSLs!QqEB0Wf^Bau+M^sCt=r%&<3Z4lojlJ*4KtjN*`y<{OXj}kNB?c4Bo1` zg@-4@Q{<0%9K+v#;gZ-{m+x2^ePh?K(5X++?_0df!~L%D!?tdG*Lb$srR7B?N!StQ zZ|Y*;#vW!|zr!#|x4pc2tRy422IneG1|~n4l=$SaNpV$@E%h^DeW^XTAR3H`D_tOxL=x8raWhX%T@i~ zl5rlXq{wiB>ibi+ieZ&eomro=x;tEIYrvsnSZs_;4h2e{T}&|h0$V8cHM<;3M73P& z&1;?F0Oar6nXlh%mR0XN*1Y01o0-|1Pm|8@-)D1}v2;s*o%0V{-M)c``@Z3@3PuCE zJTJdq>x(>ClaedlRZW%*7fYGk$8D9cw@g-BfpiM>2j#<4tYV#t;+PX@gv(E7`Qb(a zs||g@He4?_6FR!lxwVMrZmC3#f6g&pu^pqYZ;nX6nv8~wKJ_<-iyOcjp`@$Hi<@k!$u2hV={3l3`fLU+NiPtos7kJvqg=kc ze2=4>NPetTdxv#z`kXEI+6XSX+wk4Hw;;BvPwll-2lGX{y26mb! zPLrP_fXmOXnQ7ZazCXK)N7Jc{A=k@7h41|2c1ri{{CT?r2ves5&-E2;2QtH0%lRwV(~(hqs=K}(kS zdJ_n6$H}{s9)Zk-k-R&yvXLOkAM;$Z^Bj4@&jIfd#Y~G!1!rmlc&k+jzKUzjr+y`j zT-g;0W?EFHPt)4)jR?XQWCfXy^7eB$e~hX@fceEhCgPw$FhXJgGoQ%+#UT;wm-*QSvHMfm#)YR4=ZHvLPm1Xu3^tUN z7f#o~zWiA*hj|(~ILASK43j+D&MwN;i1hL%Ae73i_Q^7*Ej>pdML?ww2O-C|S4ce63hD&TVup#joDj|Vcp;C zV}Gd^ckWVRWVs_ zNx%Z>xsvr1{l`LIZ()GnnlOj8J_C>$kt^vc4y42bfWGsNktF-}yDd^`9go~-$&>@m z0IzH0IY`ayfEhF`-On)JsS&>Ukd-&;yf0Up&S9(;Rruldd3TnosbQ6^bZL;y4|!yM zBg3o0-pa~v7kMP`5&L?oS_}94|?hn+?v4&ri4z;qDo1}u~oLiN}Jdw+dp)Bpfb~jU> zT%N6CCD|YH-?rb`2{qgjwb;#fFMb8s1M?t)U4=-9*PQ^+V(PnS zTA^2x$MM+mhaldGT1-3bq(PRD0~@!GN(+cpEzn;PJ}BB~NdN7-%snoh;ofVl+aK3 z_0w3UTuelVJUYd@tEMq!J5s{sKNS$MybRyXR2A}e_r*^zJuXT9j$z5k-z&_$&RYj- zqFbzpLU|x=SxWxQOCKonrSe+CD??KFGmHo*6~*f1RxmeEBA)V={X^cLlEB&<+*1oH z$9Horu(7`)efTng;NfJT{d_epE4(n3WqD1*B)>%(3DWH)qnp0u^ad?)$4JOb2yR0L zS`F%z?D&+$eXp9rezzXAYonf4N&d!=pyGbKJekDzdtS5i=}J(D=(b%M& zN#xH!s+7qBCAAl*lGf*6u*v-j#uLgLv@$9%cajOe^MmyXa9I959P7Z(d`fx4C@-i@ zBNeY3g!ta$yF3@qmA69@@jKXf#05Y#ag3n~##5evI!QrN@+^bCC^RUAmSdM%VeC6a zrCHc&Gh=Jz>zVT$Pv}+M5+V!V0^l9vz>j7#F_(#XgS~tTT9m6DOK0+0M!t-h3#7(W zWE<)}Od6bG#uXV*N0K7*duf2{SJXdl-{V7-+tMPFh=7sz3GtJOpYmzYOe0Gc^ka^b z!(lJwyKpXrB_P1Dx7fkty?kdQwpE4MyHtYioW4Me|KJA+gj!a19ttzdJJQ*>XPv16YjsJNKo6I}SWTl-Tlp~|yR`FNTz%l%TQ7!Rp^Bs$@GFc&nk_nTm;{Ed{8qK5ps zSk=MkS7?_a)xWMo4dlyDHr{_}<^9QPSyZLt9Fy%o!z(RPnP=U2R`E+iBOB*Abt4yC zV2yt+N3|OMIX?vN!@HBqSrG14oY|Oil zmCx(dMG)Y&>SVf{6a?I-F*W3ID+bt^$hKU)LEj&2Xc2F9D@>v?CloQ9cWcyO$^om) zuch@M$nJ1$4)$<5bHu!4kr&<|5~_1WFAHa~1&up2DWDvHOd_jFQW6c9wBN{33+)_C zRB)nvc?`Vsg+qUdCRRJYJpeLB8WvS_%SqF9_-NP&PVHO|-cLMcRs}oh0+ZaAKf~O- zOQAH2bREH`$-=w)zXB`~Rrh6b;geNG7H6V+F6I_W=Bhp%k6topQ5X+@LGCH`69;?I z&B2z5%!5Gem7!-(m=;s$lhFs=1#-L|m^5`b3m%wf*yU_C+7tETYU07Bn5^O~+&B~n zX^1{Kq>2ELYa1wyI{=v5RLGFh)LYg%#pO$z7Qp`Z^3e7suVIpF<6$R6t0L>hLJl33 z_wC_+=&5av5v~H@Rbl`Y6SZ{M392~d{g;3$4|)OY!Ic2V4~hrmDbBPJ15bu}4X6Cq z5ty9)#>kc%s^m#IAFE4J0oM=8fQg^P@!A;+XzN$JzO4qZ=yRAQrR#8RM`|&A>reT0 zDKLRr&gu0ccP=y%iKemFBE}%yB-M@p%69!3DNBzjC!eTx zPk(1b%Zm><1rm(H<-2SUFryRe+aGL_Z#%G$XE{mo>q#EiOiK@0zK|82`;B#e-PB0X zx)*w6|3bhpc|W9>6c1;;sL1J6mXROgM@133Thdn59GQk@T9jbgn!P8fvGXXH!U zSuFfNYAZ;eHSTaYeHwP3Dt0ALnJ`tis5JHTsxoQ{AXwz$VTojKPYNK+WZhSuxHgjN z=l6&;$<7#av8ntPQ$TC(;k;#$gWgKA;UTbWl9MXkmonZ>RUWLYx*ndgQenp+4($h( zxZ7|mq|xh{!3nsyc6yIIXSq9$4Nd7BFkiTsE3T^jm64<^3C?0YNT~O|V^c3x7lgmN zLK=ZSKtHe*P-{R|uLM+b`{W;bhD-bkrae^X2f^|y6Q`I~LB20W%9e-n`Uw;kxbh`_g_4k29kM1$&iY;J`D9^<9 z#+WcaMpNwsUutZ$uAYNaz=FK7fZ_Q63>&c$Nue@gis6^6sxV8UYU|6>+X(msC8U?^02cdI zm)i;Z_%)phnsPKGmR%`0UVQ*#n4nS?UhCn(%Fk3&iVm>&c^5e1TdK-X`7c;}?F^H= zQe@r$3X$5|*muwSfWcSZ?UczX-Fi)Vj<|r%M-o*BK3YwU`DbvFd^3W@zMn>Cs{aV& z7}J##KB@ZF%s=8;>&7W8X}_xUbYI;%6z*Y$54Z8BQ~992iAJ?9 z3Sbe=Fh+Tzs#d-DpWDj4a3XP4s`|2~=T?{N1jftND;&ez)rN!ePubNZ>NC!9WOKf& z`ZpYXnR&8FJF=5p)>EoM^|WOmLikYj8L!(oTMHNxx7tvZ^Qip2SM{g~n+GLKkgz(##V{wo3vW;?JgEI(BQbv>}HL52h8-d85V2RE<~nF$r5bT^fB{y*vGP4 zs6hB2n(=aOS`E4I2QM^LHq_We1RB@96r(6il8djv!>VIB{JVwDoXRH6_W+60Q`Z{G ztEons_ETiBHaIw4`(Fm7o&II!LHLCU`0^KyB)oMzYk&G*E*U}E$6bBzk-Y9L|M3GWz`Vpelv)uL`ja- zeV>8$`ix?Ew_&nVRx118S}HGG)*TIH?n&&zO1jA+MXH_rif!&Bsos_oouj%pkm6Jh zQoi_@ef$$kzLxK&(b>)yMV>LXN<=yPywo6AkU z6{Ia-qg8cFEjizTRRxX#0S_!;2O+sg;QX_|^w7t=yKe&DEgElx6R#CrSN3*wXLJ2* zk05d(y(%FRndM~59w1d~uiGT)ze66IE&*EyE^=wmkk|gFT0LZb2^(Je^Gz|J4rTljV7#Rc}dV&duh&QC! zL~~8qRURa`^Eq6*nkrbneKNf2scH|FQjAlKppP@fbRnhp=)8}Dq z4w*KTy_BBLDD$ka%|)tM331;hIR$NTlIdmSs#3)V;ano5Cbym(na3N?R2x=jhF^Al z$>wiH;RkQU4DVGp{F)=j2{?Fiyt@3zrwFM`s>^0+Hkl~CERg0tv?xnAxU(cowcC8# zlN~>^^2rt94wuRoIQTO84A-0Z3%Nxqw|qAPq}i%_@VGqV;V;5Kq`OlfxR{RoS~-Cg z_BoHWLv@u}L|aj61Wy89l@tc(zsH>*2cH>UDcZ$?+T3L1PY#0o*6U~;O;ldfzj{KX zxN76Eu&dU7-zW>@Xk%7dBQj5Rw&vN=F`Ol(pu$z2gAMM=Id*#!?5Av^SXB4sOV=^G z>gJ>o%jN_p=0Z6W>qGjRI>FBtMS3K68ra@XwSOBlovjXEZMG0?lu(tK% zE?~@q1j!S^25gS~;0A`}8Dv!<0CJp1^2Jj8PtIO}UPEdExwuyV$b21;j*4T?b2>s$ zt!{Ag(Ry}>`0Y?t_j2DP%8M6lv1dt`%0+5T;SQnBiBC@plb0_TdE8o1$4e%R20K2u zXig@e$QB4~=C&jdDoq}g14fo}jK1bGHcPN%8Trx4VSYTGksRvochNB@l?=C9@{2OY zV$VV4wetDl>PQ1LRzhgE?y(U)N>6Cg20|re8noe_57P)!I$CDF;nd!bgRRtIue2@; z)b^!hOXw@G_J0F8fC@hWwLdoU$->nU`7p%s5MpH3B7$A;Rkv6EHZjC)J=-Bqs&TTv z_vF7n<`~g3y1apV%LYR3Kk3Hr4{8C|n3_@pJd_5<;gTOI_ruPQ;GsYEVy#Q7Csx&V z8y<10c$3T^@PCgI0JFd1)px1uBjc3r5SI_semKK-$$D;({MrGY)ggxc|FAMPRqDS2 z;`N@JlklgYf=>ztSalbJ3eUFiUl)2d9PFk2bXL;!ui8LLXC)F)1PS6=hlhHz!u&sO9WpUbw#*;Ej!EU2BdQ z;ZkQRhkgAnn0hi7eqE2n$&F~va#S2f%LdgRwcS=8?tYi?R;V7sD{6w{9m%tksdZzE zxqWa{k{KnKa{U5>JHv;d*53OZ*O5QrRDTv`t7|l;jQtxJZ$Af!j_Q^OI{Rs~f@9W0 z5BR>zGB98X>2lJr6LW1|3%SPB1Xde$XFF*NcLHF310p4}`lX66T{eJ4Xq5UEliEtZ zznQC?9$!&5>;h~dbQhB6{b4|daipr2-SPAtb+Chw4woO(%2gC;=y)wPA+ahDC0hjGG5+|IH8Wr1q;ppnC z6pQ-*GR?ow*-49RfRp~56@K7gn!|vdjP6(0h6w9aVJr_UuGN^h`R(NyF*j@vn6amAy`TqHDwt(DO1z28TgRX&1qp%RXqj zQ4Tx%h*UA{W{OFMZN{yzO*e(h@H>z{=`=x-_|_dw#v4_s_n7w;;o0+3l4t>k^sVkR z%ljjYe69m%-1Zwv`mH5UIgzeqRjjWS4>hHgJKY9~vvLd@Ve|~QMLAG*-C$1<=SV287N-V9->;D$kX*ma`5UeYIjeJtEj^5gE$j zOn??{*WzTTDBDna*MX5-*RyJ98zTuLdhX_$_aW5xNd+dHr7~TbWN?S~Ei5Bd2bb1` zvBJU?zOYR-NDF(z^!!92p>WVC zpnh{4-?S<>q+ta>T19W-koF2EIeFm*9|P z6ay8a*-F(+9xfnxME34tHN@hoL^xl2G~k3VZHD=hAl-8-O4HTuaJldcto3R#7&iS| zAhC21a7C956H#|0PiKQ+$j>S=paVPUQo$hXqD*YG?8-1tA!gYY3MNp-zw~r>jFip- zim^}GcGFS-uki+8^mjUVHuHaG8!UuHw(kgx2dhdV?;*}*^+iy@^Mjk}b-!U%EY{Nk zC1;7*C3OhvReV`_e{IdBYamog5s^xACf~rl?q!Znf#A~LkBw|OR2m3f{Sck%@}GuX z>UtRT2jYzowDGN^Q498Q`VcFa`74ND#tctrqgvuzjHGb%E509fF_s&c>tJ>FNCGfE zVxxOo1C`*-fdAZwtfb-w{`|_u;SJQ2x@i03{=a%*Wm!@g8L}>cH$Tx+>uDclVBIw0 zu;(6T&UYt+adRUJaEGB z6ZiF`b~1iCsPd1VM2GA53=3G9UycR?yb*iMD26k5EjT!`o5>#@BV2ouLA@Py2RkE= zJ>@Oot0>(STY!T#l4pXvGpJRL$sXCEnF*KTaRm>_wbQ_=BbG;JEg%wBZ zri5}BR?G$4WzR|O=rOv|VMiHOdA|VBmg=-BTmm^S^VbwV7rf zVxr1|=3PEYy6cp@2lv5gcMO8kHVlS`CLHtywP6?2t{PqHfgPR|Hp#3V%$Hr8_`j{n z7;HS94H6&V6%U={Pw&-9GN(V|*P+j_laTJo`wKDKo0GurgXCjrj#_piajrmA#+oau)YHdUah zRI~!vR<~gB>u&^LX6SC>-|mAaX}#cJoP|?iMc|%uUS!=72cHfY#CtPn)+7D5hRUgg zs7com6}vp6^HWN`M&?b;j!CZoJNV#tGooAcbfti>w!S=2r5|z+t9VGaxSL#^kbf5= z``d;%-!IAaMme?gkWV*5NN@bbBB=kMo7q{Obk~tkT#nSDi_F!ObM9k~mee6Xa)rK| z?f)_3^6+;Y7d;(@Q_gdSZ8287Xu72CUuII4 zrhFwFmaPxY@L;AqV+loNicZztFo<2u)?)@Tm=h=64#TIN&$H|f`gjlO-W9e{riWfT zv@XaRZqiPZ)GVl$=*g~kp!F6-Kn6XNx*M6?31y zK@i4%)R%W}gf_CSH^WqMd5I?KaRg44NR#y=%yyFbJ=nZiIfl!C@u-}|olPfXA8xKS zFAKgYrbAxzp^zS(!Qx-|iIc)DRm^HV1LOyojmu29<&n0q(EtV`0uTh68ZkITK6-?m zmZ|6nV`NrJI+F5iw6BiHkC{HZuFy&vJKNCX6QGkVe-XyxgAntk7JT(76*Ot> z!a&wu16lt=8+d)?PWTt_wd!i<3~Ww2W1;imp%?2wcTwGJfoey$^qG%#^EZ9o32)%> z?)fP6^^`J($+Y^&fjjk4Qj?&YGueitYx#nplE10AbS(GPW1R6@?}Gq`b;FKHry${{ z*08?fA)E|j^oAHy!3~C5mdMF5y!*#5KtIN>4Z_lZr{j;A7Wv;J{C!>J$i|3tHI<>Y zNQAMv3ECJR(qL^qLW9|QZHXoj^FMaM>X54+ur-Py8zLgB>f@O*I!a)9XDTxFgdTv)8zhiHubo6ue_}+7Fn*MT{7x$P#S3;_EvY}h3xfWnw$B~!=n zUztT7QhGy%F`^)VLQtt22HM-n<3I71DpYjYI}YZ!to^Fj=_2&!@jggqCR$$2$87f# z$^qz!Tv=Zx7eW>_5#9%^ror+t-QYISNk&>BrrT8U2 zh-wDQB^O8cxXP<9RUi4>q-K%IxLCP;r~vjB(mLPgqRP2+s7&s`k{?nz_<87iBN)Tz}&PsZpR+Y2URlMUzF9%7-;%QP;%3& zn_tiV!mnTSLLfb%Tp(J)VP3NQ~ocJxOZhy|BW=%APGxMu6aS<;8_? zg&Sa;mrD-8{D5giC#nK4L^GZ+OXMz6dFXi%u;Rc8bT!UY*8)4vMv}f6Jh*a?FHf!< z1UwlDIzT6W+POg*8P{9rOX@Ta6p9ik-HUq-%iGdGvLAp!mb=(e=Dz^sz^hjDs@^^w zdyi$vmdhN%*UNo2o{J0>zSDA*iI?boIrG;gCC$6YP8o!Y#&TdM)vatwc{931?1wRMb zHY$q?e}6n3|ElSfIkWlT{$3`jvO1?ZtZI78J%UE}K{HTw`6lj)0eiy;In$K2-657U zcOz6eju4Q6EiIvKtl|76fdA>2*k+sGpoAaw?C?&#LB1nJy6QZyhrICwC;9zX*eFZs zcOXR%@dw*<-uG+s!NGT#q9kQKfrfLsp#$^tl$4ENhoSyxP!Hr^*7f z5>O*uF1=(%Byqx|+R%>#d+T}#yUz8%+qzz?$v@fZuvqS`Lnc8N-OO;;8klxxm`^@n z5UF^En;_yM{%%ziI;yd3PtFxWBhDz+nczb); znSRH1`rXUP`(UI@>Z|3=hk1DuRoPbVT;a1p&aUZjSmkg)!Px|JCaB-&>|@vpqDK9) z5}DoDS1MojWFdJvHFe28PUP>0Igwh<1{0eq?CILIv{MggdHN}^hne4lW!hXbVSJQb zCF5V_%^US7&Ap75mj*BREb>}k#4%m$8XF%Jgc;vzlR3{q4}XmXU&6ZGahH%UEc1(i zzuDbjp4Ah~atgU=1J$G*CyA%kBNm70)*#Vb?n`lgk$*67>5blu9$qw!Z27k@A$R=3 zGHY7EDf@olbieu@2%7k>50P2O89c3=L$s<%j6WL!`AD~!Z%uR?IS?2PA;j84kvSu1 zU+9tRE}{gyq{j+wY7gf(QNi|@bq;RqD`0z<*2`BZcXOWDo4V2$=1ZR(J1H9ZwsKenKXY4utssB9MZ^zzzcT;b^ky?lrVxJD? z{h^Feco*UWWXU*AYwl!DYo?wL&|L-X1Z8o01*sw6-IYDWx5Xs+e+Hm z1;(JlZgS6!P)|eUY9}9VgHsxJ_EEDjANtvw3s(SO&5R9g@)UzRWWg#h;iH?NqnECj zUTI%$67u^H3K+Z@s$?cZJ2C{=XBQtAvRn_4d|uU9W!oumVytpclefEZj5|N#HGPJg zwK5fJszucRGDo7o3qBE5blpmseNZ0wcMtUwaZ|xBbeHo-qmuDXh^>S_U z_H!&Uki=A}DND%dPtXj0Eh{(fOH1fvT-|2AqPg5s0(5*_$vuAgI28UxT9Sr0-XY9{ zjFv^15I-9mZXi-*Qq*)GV@l0~g1U${~fmI)v6yteIq0r{#E za)4fg>GG{s0!C_TSa3V{V^s-L-qq&l%C<1e`aYoC8p?R&VUo~{S!p&{BBdMCSIkC0 z#k7Yig&RV-JJM=}ZhhMr76om1rpf)sd|ocu+e^v|p)iicDedNPN)ui|klpeJI{r_O z;e*&r)({ShWHXt353;_a<}=;N7qcctbmCgmC3{@F+g3!PT^ekcxC-Aldlm+2tGmej zs{EbB`x>`2NV_|k1>Z^57g|f@=Y>{_bUM$M@0LWy0v2~{;RlyoXUkMuTdpg+;_0PyiK$Se_>*btbY@M z>ln=$^}CEg-S#j*eI=VG(GY5|+#Bb5KUR6o*c9%=J5{AqTJ{e)V4zAb8d-3P9JvoL z+CQtXSUy`16CBXJ=dP$)pAI}Lc__hlKRgG@{GlT~Z`DIa$4a2l0Tk8C2ief_-#S0{ zA3dGJqG~$xw@Hh1-tQ|X_K5IrIhNh;_yMvgpTixrPzJ@DBiMZ_x?O9(y#-a;hPAuN zTEnlzrA6mu-lW?#T9l)uyhp%@<{J^0^y;3GvKnK{Jb3kKe_o^q08HNDi^;g7u+LNt zWd}AQ!#ljsP0&_bPeL!91n1+dNM!}F(a@xZsv4Jmum_JJs#C@u4sGC9kYZtO0yC@9 zBA5G^R9>i*$2ebID;+cN0lxnZPuxeRF%?xbNCn>bH znXFvSfeoI8h*_#?b$(g69;fEDSvIVcy6wbeKt9!LLov>T0T_==M|13&W|QM6ZRpe} z;7mpzh|7x6XZ>an%>uf+%g`lQN7u9fNPl$uf^zd3w2|LkzPOy4Ua}O{5;Hii~2^H8*|3K-7#%eC)FJML6Mal`~WxT7@!kM_!azvP7BEDx1FX1 zF;uJ@n&Ea?N|y^UZv?xp)A^4d+-b(RRR0)he{09n7Q_B5g#Med(bzUi_lGYZ22(^Q z$nQVG6ul^`10|?e2=+y@8v7I!mDF>w(TcMO{9^Smjd`g|x{MC-eT0=pr}G!&y|go$ zWtYpZyMcH*8+E``ClLR^b8ztG0kDXVakC5?3aTwH_$#Gp7GkFjPI$Q_qI_SRhZ+-$xTqd7}9AO}6 zd0AABPU0IsQ-S+ms}-~wE2l^k|8G8lWZzY;xd zHXU4J&Pd6v;iZ&=w2_|A@wfy-hGqB5ARUuatlCN3miQ)(CTWr~95W;)&Ne)$lZwxw zxG@PBZ#hr>{~mVu{zoCIWrtcz`JiTo2EI;QRI;pUtK>}74Wqrh zkqNr|d&YE&LO=_GmvdF>@Rm%YGl!CnMV`OuC;flJ0Ymu=A7?IR1=r)m5y$Y5~V)St++|2nFQxN1#I2KCqK< zyG=lJIR3cGnJu2O^Ik0=2Lh@L{qxVZ|C&_MA>< zpZ6}X*<&BNC=#+(p7%*QB#nki!bcP%>{P633!$BRgDFi z8);$OUlit1q_2lmX_lTD6OsIX;q&+D&LD$74h1oDng96vWYdc*_VhA-E6RaDX$nKD z&4$RYnh~-?kCi*|2Eh5FGSIt=O1Un&XGWed>0YskWz;Iq3REz6Z`n4)=anaiqwUwI zik_%MxG7!AK2fIM&n`ymPGTSH0eOeZVp{IBk?ph18tHb#tdfFP_$wUu4UFMg#k#za znxftz7o^G1zUC@19^6gM$NH`%RFj;CFSBHXOx=Rw#YBxRNZG2PvO@QKm@(StH_Rfh zuIFhhcjTcA6ncH7Mx8W#i0?wvSvr?ETRG=R$Is#VO&8e5cDhOc-Oj19{BU6moUwAe?28@ppBl)cb%J-=Tu5W%Lqm;zJX#e{aLb;rZlj0bctlr27tlsy3}Ua9_+LifAydI3_dLm|eK6SQO5bJI}IGzSUUDl1a z>jT+x$nL9^g=vTsp}GtIlxw7%(pX-J1|ffbhb}<~zYz+ntv;(H<>#T5g>GgPa3T{` zO)th0nl4%QJRJS3a{1-A8aVpXjizox6wU$6DtbsZMa|Oc1plmGiD1bbQW)S6W4SW8 zO(-Z&{lf>hJr3i&uWS>vAhVa4BZd_2_*K8)a8B+G??qo0PjK|1a-n%>m z(-VZXzy`4qgr`c!7^iZhTySYanLEQ8l)XnF&so}I_iQPOYe<)Gy}6V-`sg!6aPj?S zP`54|C|SQ+T^Q@hp>iRPp7qsu!0Me2Ea1}voG?!H0W#q+@S~3^9wt*_qnBEYcH`i; z&?UtAaot^eux>TKS~w)%C5{+lD5}p*dNij?S01b_($fg5 zhU2JDjFg|>5(?7FjVFPP*e? z=>CrTA-~(nqyigo`Tb!<+%PSwGgf$wSM2K%7p=E&QZZHK20`NuX-{l4)!j;`$@{xF zrTY{LG%ozka`;!)`GOuQ)=m4Y@10~;X^+!1nvYhD(892%?59-*N`PmG_9b*PT9(Bj zo1CHOOf_xgkEcy1Ze@EWvDd^DprCw?9Qw{@liF*Mt=*NOhvg)1K&vB8d4W1C)`NO; zJu?Dyvpzp?o9H3$pEdos+mg-YPZJz?SkJ4WL2IR)qcILUt+-hhEi!EYM|6@DCVW(n zzK7j;8ckhR)DI zE1kqQYS-LdO(BN%Iz@5KE=Y7P`IPWw)p*FFJe`L+noFze&{}VbwjfS?E<2 z?=E?1Fr!-qm3;HK8KFz~e=KD0iIXa9)8(9W!_IO@#$u@RRu#3$)Cw**^t}%cQJRa5 z6=z6jJg_@YrN%-GmS*y4clc@gHPCeQb|9rt`Pb-##w^QE z8$DB$0-*gfWpoarxRf#Fl3K>Q@A;Synab@E$0APQd5T715xN%FzYO;;Q>E+VPv)MA zbAr_+C#Q#Maca@Gue3Fuk+SW<-&=MR|dG!b^@zUjT z14$q(GG``Y$&ssT5c=~|#3D0w5wEjPzDtf2Ln41@sWQzHB9wKJy)A#6ou~JKFu;IL zG$erL2M#ie$xLyikNZQ3;a6<=PMVe~hqzv|4StW!ESKMN8$_AnA?y+M2WqtyT?@91C~3n4a9y!3kna z?}eTl{uV9m%s0UP7puU_TkhpnBWWqw{W1v`}o8*-eb}?54`^rJxOl8gkAj1_EBwg?>pGSr- z&~J+-;@J=ZMD7#+!N`WUwz4bNIv_g z+6A-*dh1ai2B;Ruz=J`#%I$->fUU~slq+9x``_y7z+oye35R<4RuD9@h719;?Y7w- zRzimTjXKe`oT(QO4kLE{RF$`Z7?yeR$O9-)cj-Py&o>7J^OC+i;nRdOeMqbxN z2@A{eAf(7lxsQwp`QQ)Gt*07EC9WgUl7ahyd7t-aUC&7=Rbuiz~ zr?|P3%5}4tVOVVnu}FEv!FjjS=JqQwRH*@PgXZh_dnkMXN9ZKk?Jv}iwkpz7@)o0p zEi3YhCB$`qH3I@~tnHG)-MCEaAeg9Bg*|d%9Us&v{~S((tkPb}JzLBi2z2JpS_<9@ z?-l8MiQdW>f!0zxUYzv6(DEo8PJ~!v@ zZ6bY$!k|B)R(|&;WVS>58E>@m;NdzsG=osDSPDX6V9Z8RV-I#bU>1kn57)jt`Xbm_ zP;AwmoCAFzf~qW;mJM7qR$jN<(#0%6{~#AUT~+v`oi2cUK?U?2di1-!R5-{(anK{I z5>4gZj`^DHDsp`o}PV9v?ysCPdMGcD5Wd58yp{1zR`ZbN*> zVG_&@c9LU9uwJJrzeW1CL%SjWI0v6yg~C5nLF4g)e4GB&X>81`5%U7Rl96kCwJ<=r z_LT+a;Az4>=%NyHQA8>;$y*^mC)ycQI;M<4usp*@uuB*?_W(rAG~8Q}kI{A%y82O0 ze&Eda)u|R8BP*t49<8}0Ub)mb4Sy% zVDi?F!qL-IZFfk>+lI<9N4lKs;)}@X0dVwRDm+Z9>T!94uAd~Uh37KGpmqf39_viH zwtFgiB2E$J&Wz(^!>uvh^KvmZJX^5e%bUrZI+%FN4q&Vt-p!JK(#cG%^oXqkDv`n4 zEUlmoeEF@FtCh@jApN>1??yqw-7y z8*4XnfA+rQUJ#sXFp8`DO(lmJ2f6EM<#6_1Rcp#O$2f}sC@GsZnif1+#lx_a^U~$* z7ZGxI|6s=CrW-?c?IGzP$NsR!c`#w=gBu{0^Q%qW=sGYQ;2?`Ri@=x8}TmAT3=69V%7QfL9E#r@#rbnip_tDyi!Don2!Ulh}f@2A6u9TDj%qgf8&wDVy?AfRp{W|jplYc=c8ib1sH!bSKw$1!$eH9o;eM|Xh z4kU69KSk4}UA@92qYfQ#jH?%ZuDmVSSGZc~fH9D>$QentF{4o4qApx6j5Na#VY!XC zY49fbi*#XmZ9n@4^qR|r^`_2MGyJ%X$Pi~xU?*9719I;PUH86B+4JN{ou~C;K{;*& zM@Nt+Ncz`#kYbrCzD;^9N6<8(Qw)BW_!=I9>kyGjL-RHWu<4^Z-7RLvfRr1R&4rJ*TyfSBh2Wv0Z^X0J9SUsbT zo~7nj{`$IUp0ok+E{mKTu-tI}nxD^I-N^YGCB`N)Du4xwT6m=LQ(UIRJLC zEaY;G8>Ig(gt%S%ABG>}nhaIi0QPrf!Vtv^WyvGp2OrT)2eK|TzG6zck&by+2H-J8gEDG11oPV}mxMKHtU_c?)) z%F;_VK7`VWt=m=uWGBFa;Ms#J(qFC+ID{h0H-Ke%cG(eThdq_iS6gX8&bWKQ;jYK@GlpqGaWN2KYb4beeXum%Vzi@++M5|f#`TtF82_0wvt(>%^nv=9vv$Ob?WCu-R}J@ zRnbfg`pN(ZIn(5p^-!|X3f>}RH)ToESAlq_B67A2INhnLCd#EQWZ2*v!zEfAng|^-RN_aQ0@$6q9FpU-7peDWIc#l+?Z^R@_LBJnx^93gOqk_aAth%BqX=d z!WdoT#%8aKXf|&E0WvC=1X{c9gQB}5RW^~ok8tdl#vo>>p|Ilo;?6fK_h$L92NLFp z&JOtFCRm_4-51viZYQK~WixF*v_V(TX@pjpTq^%=X8Fw5JwW~VHeMFVk$al!g3sHX0jDiAhQpE#hU9MRy|D5CO z8q$wn-mT?Uo8xYlS9fzM`*p3>CY7+tV=kEaoF039RYkkN(Y$a!344&MK_>SJ2#3(e zWQlIg$5T_o<435x4zll)Z}Q{`i>WgVQ;lI}@@-28aQjnqg7nP=y8NuP1Cpa+ycErZ z!QWR&2NBJxh8Tyo&UiV!^|)2fGI?+Vdc$^_65@>{qWypu&pftbQ+ca9G{YeGdI&d% z9f zI2kQn(IDu!ax9Drx__h@lXvz)eh+9t(!UjqO*VBGbm(ZV{LzueO(tiIHTt)pUskM^ zDTj~^zmQ6)Jy-pa4HcXCz`qw04i)rXT~_1P^BG7Ep<`t)6q)U1x}+?ejYPo+d0oXf z%jOcK(evjxBZi_glPNnUYyO6Fe%GBE{-;NdO<4{ejZmc%TT9K>9_6B=3as-TU-E4fn+_Q|NI$7ZZ zxwA)_G#07{?LMVat#MAIZ14l+h2<)tMRf_{0bEtF~yBVUZPp?0kryw4|qWoyTP-#&dRf!TX3udx#v$Hdnqnf8J{x zL%yt3mgSP};}q`K1H|!B1mw@zrWe^8zC{ywb4&ntrc{&KCyl(Iz*c38z}@cFGB%hW zgzhX$5EaDwkDp={e}2g^jLU%n$zNM5y;cD)GgM$2oEWvqJVz)Ymq#EWt8ie!&yjwz zvI}3H1_UF7QWG)gpoetyFj>7VKSf}|Ru2{pv=PT2>}A_6W>j9?#cfPci4mMq%8Mk3 zY?SxDXOqvU#Be#@(prIaQ!^RVB>%9rPQ*%&!J>@bjh{*~lcM7RtU&THS~&>9{`RkuI-<2*V!&p~`eWyqq~4 z-O!eNdhG%zqlI#}$NkKWjn(Bj_2?ZAAG%t|k=D@X78RMuf{b?Z(=`5K^%m2q6}Yx? z<{h;2+k>!%McEdhqF^0v(yb9I3Sv^;I8-Cuujh>@xs#zA=Knl<#0&x}#Q`~*v?kPh ziy5h;YQ}Ra&Li?qt<|M5Ij-@yHL8@Qp;XmyyFX4vQqJfJ1x}}x6NwRCHcGQf%cP!` zx<{}DEaPR$4d}%5wRP~KVpF?;CZTd9cVoZVH*w&Wb*Kk4rI^~DW4?YNeDNa%Iq1Fa zK636u{;V-~0{thgGKpsR%T*2>i$Xk;u(( z3@s-i5V4X2^DBAYY9~w8%-Ko-}WpUUPz^rcs#87S}p2sP>${J zQh5s$2-g89K9wd@f6VuQYeVqhvH=BWcvyOv#nE z;n#|Me5ofHb|5E8pcS&+TN^*5RRFEVW&k*9@9(s^2TaC<*AQTm5?Y` zd{EqT%nkB275-c&;l_c=bD!+|1_*gp89U{R2u9uyEi*`GTF9s>bsyz$aaFp>m|MUg zijAg6F6jvmkM8I1N2$a;ysS11bvQdARme7}R^Co{o9(k%wKHFizrgzMRq;0HDOH`I z%!)Sht4=l|=e+?2kYj+xt483yr;VlO?bmf|fv9lUb(}w|n5F&S^t7AJxj;u+(T)$4 z>7DDIkpEn_EZEaS-mC+2H?M{Yma1ec$+2@eVdY&9O$C%ZzsS0uJ?!Yg~1MCCgS<;-I)HU3xaR|PMGCIj8+y4Ww zoVZapDx4-F@@wNF?QW1eTmk>&3z?~bA2)5NDp>>3jkz2*e*^@<;--?T@ zYAN0|GP+xXy&rO7a{m{hD7s3{K>0q`ECuKxhv80NLbe{Tx-qKk7S>LX>JLJ-vb*Sf z2{U_qE>TaT1ZJvOUtUqV8sN5!mTR>lFlrz?aJ`~?4?62@;=Ti?Sl@}JhdcsjfDc?Z zNSPbxoQ^64xN&DYRI)qN@7GbY;2a5F;BI@U;yLowR5Kt0iqRSvw%AGBy^(mQRpo41 zz8X{FZ{5{kyK)|t);injfjxXWQq}ZCi*%2W>AFeg9Ty<$XCLAEMyndT+&7FT|CN>k z2m<5e*@1;IXvo`B7G}VNv3qb(-#yW!Ye8qeb^>o3H(+tej$XbR_GfP@mK0ide;tMR zAgfZ^>zwbWv@@Gt!mB0uD>u)a(1&E~1!#kU!DDjje-L(Em251bm$;wLi9yRX%lNZh z$}tS(+Lb9q<8+^`nYh~%cuj2Zg?GLAUsCWivL8rfuYVTphy-dUR#ei7$>^yshvu{O zX?p752*fAyHlVY|gI(Zy`y<$5IfGedtO=)s;{KS{>^`K823Am&o&U?pLM<^)$tQ1{ zF~X%44W#Ku=rvs$aiiOmf0_JmIliev-FU4?m$l4Sk=v!=B9xY;8z6}#I*71cCjf6y zm6HjO1@1m`Ym6M)XfCy8{W^KQJZ^63>TO(T34^^)SZxFFgu%#i{89CcGA!JNbfy5X!qk zarT)q@2csNYaZqS&*+-S3n$DR`D?tXpZ8yiYGp`{Geru=ks}k5Bjz=lodPvLPSqCg4SXiaqWF0BH zP;_3vIGMlAV*GnAFp;hz?NEinHtFZ%)4#EN@buOCQBFu=CcVv+MpKcJvvse}+i8bp z94m-|xi!m~USUs#@vphMQz%^qG)m@e!D#!6q!EjNBH z3(CDhF9rLQcE#!DnM$!&xl0E)@@4F5Yh@8xNN?&MV=t-5 zWcgw>Ppj?*gF&CK^` zYds5JV@Ux-;R?aWks0#GXK1@k4#9>sdPpyzbgQiXkVidD+M#i{NSih(6CZksMO(TW z%3bNGC^-)p2eNB)vF1KVyQH6?(xMn+spTM2X?+a71r4YK{6pJ2QJDl#0^H`)fRmu9@hdZOeCWo$(+Vwu}j>1bKr3M|^Ai$a;35I`at z{Q%K$8(F(QHQE5k@EiX&&SA#9R&&b z-fy(N5&Z?X2eWr_eYa-jnZC4X5ECTK{m>6#S}!tV4XR%NF`1gwb?OApAGU( z(Z0Wu>Pi_r7YHz%`%>@&Vyd#W8{53x8cM$=i?Au-YrmK%qHueM)owB$py<&YpvWI zG|RQ>G1c%_2V_x4*oQ{ekIB#&yo(3DlkD4p&iOTYMqFaeSQ&8$Q}y*@>@ZhH`DduS z0kY}?AM=skF?DHY%}BXU4}bYo1x^vJ!9K0hqjnJs1I4I98Xwjg|1uRBgjVfoiB_1G zDL3qflo@NkLB3gO#%1PHJm8S>hd6CKLP)!zWX}flkOEWz(xdxfW5jmDkEzj;n1vpe z^&+ck;DJdlT4CyqI#?V1uTb6p{3@9*QtfRFVM+NBjx!~4CVANsMlj9vwP)U{%_jO)n`3U$rL##ypl{sx;ml8xI51dFUxd z?Xo0#`wvjYc;njwClFjaA0J_`hpg7lw67QP&)hFi0De>!Y93>-R3n{mSgW0BZ~Vgc z>0Q+TS0||;9nSge(ry&@5zF#uvTJ&F7$LPl~oszIB)9tY~6LXe%WtkfGCdbA(B0+2FQwPt6j?I z|D|S}6|`0VBKXx~N5CgNQlk4@J^6P&Nn-K*N{l#rJycV-20qX25K457yf)6~ms7-$ z<xGB@khb4Fx4{wH();af(nvvio{S;v+qmxw8t~ili#- zBm-8Xyq^l0rC>3%GFtDeL&~fL8Z5hE^^#Vo)}|fajE#}0ZTKlmSA*_Tz|UQ(;uO4v zmiL5?!nRb&z0iGCThzpg#`4~SXd@^Rt>layU1w@0nrSapJ5Aib11WDS_aG=X640tT z)8cZq*VnGS#L=M=+laE&jUqC91E@aBm3sr2=pL@)l}`LBfmNVww2zz_gO%~Au#~AP z)kXT1p}!nbNf*o$yIbcHCsC?!B|WE&o%-6Pwq_LkzeKxy`;b_ODd)@q1UyecB?#fg zbKtLI&Ed_byP8p66mEt&PmE*aMV9=RN=?@xHUc_SM#@W`d{m~zW|*Xjl94>6ybFGC z`jkAc-p-IbUDJ{NEt2RZ73+wGLi3)jx4~9qRb{bfMrU8vn(c(yaLjCChkSs(_~VNq zI{AJdD%TN&RX|fJgwp42sPm?1KB7wca;fDVAX7(En6q31NNEdlu6{3uu>4(l3TZ%Z zkPkFb)5&WI0!2>9UN<}f z@;PAAI3EHopp`h;V8#oidGYvx73X2R|UlI}!n}>L%BYM=-3U zpAxtgJ|X8@gDktYu_0=sW{Fpihy3=U&m|x3=g)ptmX5sEtK_?uK5D^-L#m&sipgBB zJHW5;#aW^kya-|5eYuP-k+)_bX^YvU_~qRkQbF}3|SZ6>9REm@HeA6 z2Y^Zq;Ndi7Dl4mz{N|8bwo1|lt1g$YSo!D6!8sgrtx7IL+sK)Q-ciy{E-XM?;5TS1 z?a60B+Q%mvON*R@#7gyCfZ9F~8a1+ARQM>AUvY!ZKOKPL6JHDhkU8}9FD5;R2Zt)9 zwk2%xuT~T+#K7Tuy1C5g?kkm|c4o1D@D`(2Zk2H%jgcnGGXrN*&JH<07}9$^9Zq$)os^nD!WN86{=qh`~~)#D7Dg49u>TRC$Qk90uYoI>1RfhDZB#^I&9qW8+j-9I@e^0DgOy zbL^!VI9$R}T;i00{G-bIfQ8A^(&0lW;;Ai2`j3_UCRy~l&yYWgIltb@{|9Ef%ZSwqXyiC7f-IP9(`Kan5U0iy^4H&QMYtN zpp0Y%Mthj3YBxp{PWb~mBH!U-=p}C_`?)TyLO%JJ_Z-M72WaC_8PN=;`$y%?mG3s7 ze7$q4wFJ<&4Ax$>`EqqGcTZ_~7FJ-%Y#6Jyy&NkKdF5zlR(I3QkYMYRJiJ32#C={b z?&B*RC_18Sna0X8FUm6YT0cLCVVzLnFr^%3QbLIrs25!_7Cj)zN#E8`!&RWt)e=ZUE4&*dopAOleKgj!4$a5dmrWu zw{xrR`}5tfLf!6~DzSQnzH-&SlkVSIyh5&<2SW|hG0x5^K1e3cV%J0N;9}ORnw6kQ zq6v#LoJMXMlQ&;Rs(r0&R_TAmWJ2=}$g#Bw48d5$o8wjP^Ky1DKs-Ya@^HTlkPKIz z8~F=Ye|>91jOA$o^wE+9+>L*L#Qx>gpJ`k1745s8)Sk1hRKXkN?f?%aiLNQA6K@VZ z@uzes?u4obPmxhG3=agdD?^u>ys83j>6{1I->51lNy+meKZcSmgHy*{PW|0TdG+i3 zBvxQPn%G#0l>bftDG%6~cN#iinf z@>gHT^MI;)gljh@;(v;FmYggou&%1Q6CrDFBgOAyv)z_~^;Y+_d^dBJSR>(`E=U1P z_7&PRk3s>E0q*rjPNK-Yu52fCa!M!S^^id$TAAr^i5(%Ankdmx2<-{E#q!sJ{7Q*$ z=4DCc-J%tw_VUxKRtNe``~;*KUJUh;=uD&ehDM*Xn09bC;s&dFpsZB7IHUMpSUwnG zKjpgg!}%&UqyIj*yGCPFdW4gC4;h7bv9zhtAM5Luc~E<9wxK8LIi;=_q}-?y zugj-8>`UtOGsf-sf};NbK#R)~aznNm*P_3(^t=WMbzlI-p6&Ng4NG%cYF%ci@pw^%Z2cM|sFWHNX_DX@y_EM>8M`;X7+wl^Wcy*xbcAv@ zk*RJ(ZsY8fe4~?Ax2V8+R_vtT5p^eI6gm6J*@tk4ZSw&omSF(JiL0!xeNz-FMltHP zl5~GLuPl?f3eQ8DYTlMFC-BCPRYIYSr$%d?Yk+)S1|nRec5TIDma@s(x||@SkS1e4 z1ax1$1d;!%DtqJ4h&=+Xpkgl0(g5RL0rv)!@oIIgcJzBt{9;oGnv*?LMdt=%QZ8+R z&zly;|3UDF*ZQhBu)L#^e3$p#q-+6sEpEExb=^LFjY@QoVe`@8`W!LK<=QIHqx$tw zj`XetH;@0xSNu9z@}P3q5t4bg%Jtn5NW+t69&)*2Jwvc&~S>Dd{k-mm+UP^{QRqPL*rpUDDS)~aASXHw&U?d+Ly>@7T3 zQ}K7A0=ieB9Q#}1jH@K-$aFWogN;a|SU`?{W!gAT`ysr$#zSCB$ueNYc}Ny|`QCA5 zj6_kdN=W}RJg+&uawBX*)t?ra1(U#p8&9vO|2(qanSiaVnP=T*{OE_O95^c-bB#_h_=&^!It%3@aui^{vU*&$*7V9V%Z9M3ls+a>IIIQAYidNJ>89k(O zZ-&>(2`7f&#CF_84mL?V(}qKF*}6613-oY8RuN=;URCXp-Ae$S`x`;-omFfAvH}10 z1eMsw7Q;*Vhdit1D*lpGW`GE}T)3=!-}InG+MD5jA{nx!s+5sqVlG!T%j9^*Ur6wIM}oJ0gR^et@y2Ogm=g zfuhE88B}eSFm)1-&~UU0xp*`Hq7fZ~+BhqIZh^5*DA zO_g7J`D*3a@i6v?i=nu2E-S*(gk8p_G8gw<<0lxB(4A(Dbi{LF0g?=|x~#m9$@N91 zg_tZJvKkfXhAn7mD_6I|0`IB#Noo2C)HSvnwph!~Sk0tKbi&;7cwp{SI;a_g$IAC z;u&(6&NUf$1&p(;1c08)&XY^puX;v#snTnKrn)=LFEwLAVavrw)9PB~i}7`@w-~d| zK-5Q6bS7Gc{VwBRH9qECn@maP;R9%oj6jV0jOr}f@*?}Lse^FZiQZjb#WHdF*hYfa zWIYZ}t6&%9o00zZccjN_Y=yD9CKhI}P#Z8Ew1cRVJWzuK7@=%OWoc)G<5lIliDSqq zU~4oR?l=xD<2zO1Mby;{0fOxfbp4br$U)AIRdL#)s^X*29|>}QDC3aKz0(ZKDc#4g zNk_AW4=SvXZ7p+xqmXhd&r%nrlgxDNTzYyMdRi z%u)?OA^uA4bdjoRtiddX>%*-8hSDae3#mQ?|9QY~%5j@)ZOTs}-I)2_*Rb<$+gn%x zSJaLqtV}-ePhQQ=O?orrf=cr;eF#5gI33QQPkh@+W(C?SIT?pL0o?vH4@#wH zBGbQX(SoDoSsy^m_0-YC4^;VC-c->5ldqJD7MfFQ6{9&sXR!hjseAAz+qZDWscA5J z1rDLV9_P!7#dRbd5n{=*ant0#iw0Wa4c4Y;m0|W5nrl0Vr9eEDw znX3X}vEE_^ApO`yF2&PZoVS}cS^pcW{vj&{5IZZQT~1%9VM5%z6A74 z@@sSouA#&2538DeI+vpv;@G=G-up7-(y6*kRk{-$CHr-x{fJjFW=LVrX7e>uBzPiZ z2fYFgaof-U>c0W7w$%BucTX~7ve6DG*zf0`FGwlUex@|Zufyf{zXYQ+Q8`nfJa?M> z{Q!6D)-4{cst693v;b#8tD)xbgH;8KxRu~=;}3I~Krs|5RW!*rB>p29PZyDAp@iCV z5_$uAV&ZIjDf-sxlGO*<_pJHQY^y`xgeADnT7d^QPO)xo6j?<`g#ngBO2?_oA(?BW264@tASzNB2y zBVg}Wmg&;Et}iNME*9ElSTo!PORn+7k>5o1-&Db3Y_((&uHKraEF#$%0TXB_BU%*d zjADN}8cy+Op5p9>YcpVjQc63>$gyzf_sY>(9K#^tiOf|ZPEau(MyQxbOfv5+*orRt zZ4dH2(x~i`$%s|if1_E7B5O}*9i}s?Z*enJRexWE80O8d*_G%q_{TGq!z*$XLVjC$ z>Ou6u)rev|2b3Pqkl#*+{oU%Sj8O`{<`WJL+%XX8qu!RPAup_$6ja+gCy< z$T#O@S^XmrF;z!|+G*{Iq6RM`umJx#xxO5^y8_;M`CE`>BrX-Bb6Ff8WJycpej+N( zKL;GHUWOvpt5hCRIDDTmzQ_Wd^MxAAbR)x_157)_72PC{8u@NC=tvv6Fo6S^FaU~wR^?=Ksz_2?;V%HIB-`Qp z?jqMt$JQG*3*@~{xrRx0*sPT6EC|e)?uu|OVI}HyfR%o^AAtRx(H2~|YnrTLW(6Cx ztd$$ugi=!6)T(Q#Q^yQRW8j#|YlYrlx(G^6I#AGaW{8XhOD`(#gR&;y7nUtve!W{o zM~NrEPdBKFVVFr3tB@VF4N#Vy4dl14d<5HQ2yP4k(QZ`UQQF<3+0K8Ft^lu)*#QoZ zK_RGZ4c&U~!H--yN0ZZZV^*6Ahd#k|_gibBp;{6pvr2JF1{%ntg`je1s~MJw%@Jwj zQ;m>%zX7cWRcZ_>rN6mc{?;^P+%rCh4&$!jx^N~^8~?C;bfYzZ4rH+y#c4VWF+-6>o7XV16`r*ay_KhUk4jPz2Wn!u`z+Zoug?!mV(-A??fdg zy{3+`4Asd%tuRJuC3v_Vi>6Uff4+i4_=7swCMUyO$#=@NQ*NXmg^_?Y{;tM3tMm5G zs?w48+Kf!>p}cUaq#wq+@0Yz)`YN26o~9TPXhv z#_L7eGZ2^s1d@L;RnO>{{Qy|hURQ&jS8=a=Ho|lpQ`1}+c!go(ToF@ET3B_@aC()! zSXLtwS^-n3C#_LLONp+I_ivM-5nm-hz*&(ipTW{f?Q*}40e}1}w)?|_*dy(UHvGRA zqr;tX+KJe76@KL=cUwYF z#AVguqhPjFO)$cU_lfktvqz}*->Tv5)4u4Z25D^Gt_03sdh~n@Vonk(5YJNN4jJy# zBPr{oXEo7Jej?~7zx9&T#+faxCj(hL%Ifwir-|kt8v!_I5*k#5Y>Zadnl5^wpHlZe zQpFV+8n3k43q8{pa?Pm^1}s+j<}%C^q8H3lrg<-Eq6%wy)6N9#8E=+_9gQeVg|r1+ zNjO}Aahw5GxGUYHNADyrIA?=4GuNT+WFzInW2g=XT_d@<3m`mI^~cxf+D{w-;<$jm z?hFHYwvkFj__Di#iZ~{0?zoK7e3{IOM2l6i-L^_{O409Jy>Payy5dZc7E@!Q@%M2+ zK~wfdEt;+=)6IrWN8>`IJfAKOkIl{{tYG zRbp5|_yQDKY9yz!l;z0FNHgb<9Eu)Nu3UP}M=X+Y*%^lhfnH<}ok9l2T~tT(xmdLk z(`7)DBpWBIto?HMgAuJI#ijtQ!Wq&&JVw>Argq$l`-X$#N=iRTI*n#KmF!HF$l3gL zsS(?YQe^c=Rn1$`HmMkTi*5>saH3of9ekqPiu-h1fUIt7*d$36NdQ_6F*M_fWWb`B z2&oW)G9Qy%ta*gn5u8*}i83WAWf!DKCa$8?F3&83^spW>^)5XBkMMkk?x+QM3wSRRs<&n ztAM}B4K^%RVPDi@@*f8A&D6Egn!U)RZS+mpt=!(EK@XOxbA>FZoi%^Hs*z5*Hv}s1i?g>KimIwK+|4uaAL)KZ82qOEBaE^ti6js&y86$~Hs1 zN!yawXPrSZHP@*Oc1~cbe4DH!r~?!bQ#IRW#G!$-N;rkIx`2uB>~0x0IT?b=#WY)? zEl5rQ(FGNx*vFj3`8eAhxEJcuzX86FnqMPz7Gzh6u7E1cY>07S?JX_#AcZa$qX)>Q z4#b8H`$P0=NXlhZ%|YCb5ekZ}?t#T&Q&rp2%vu{n|Faj!iOt(%@>r+64QMib3S(J2 z@DX1V$}4#aaHCaFH$EjNc?I})qm%~$-GpqY zh7nUnRXlH{rv+ln2KnH&EOF6FXOu{9Uqh9@wRcdadTU9Hh=~`U>#XELUvm^Z6z?gO zUvcV6(=%&GeC3!NNg8CRjL~R^3~qpA{YLlO#v&5?K(Bcb^7}GpB-qMJM+y}5LzHBn zV?EhXT6k`h&`C?RWy$Z$U2&?)z@t{vB|XbmyO@y<+QK46dUV@kp82e5Nl6klSrhc{X*Xf34WESkGxCX{s!)Nsqp* z!naz;w}WHU={Nlq4!rd0FoC@jHeK3sRA62Te-(1$dAx2FRY`G5D)& z4H0D)o0R*)nTGhc_En@&2Ni3bul?kjC>~!KS$JYN18{ZG5q#LgP!J`Xx#!IHm;4`M zBC*Iye5BwStJq20DY?cmeyW&JHiVDou1hC7LgY|4~nErV2V#rFUvfr0hlnpk|{@T53^=;w=bHPn^Mx1uaFc?nbhWo*7dp zk`T!3V`|+ULn7ETrHvehG?> zHBR6dT51o6rPEW5bTFO&v*b*|)@`c}#5kZD$+zc_@qg$J99{o!wQ}&JzSR*)kLHBZ zIU`KQ)UYYmC;Zx3v+E~YsMrRt{Zuw z>3DEKhrgaT9*x2Kgh$jys71DssZSB8la&>JFhg|=MOtR-IV)A5i3frw@0fVr#(Hb+ z$!M*Vx+SHQR*}gz0yY2;A+sM|tVLAo~xK@y(Qqr5P`$$N37N<9B8 ziPFNt2@QYcOc4KG=6t9EAT??YlC64nNA2HWt2%DGVSI9j(oWAT2EBPI1~s}n7bcvA zGWCQdQf0L>0y8Bg9BVvlMC%>fMN|KM=~D53|F77oZz-)*DbtcoWlELjtGN}*noRA- zmgQ5w>cXdWaCtswnBB{wN*93Upb=)f=}fcIWvy}_znJ`cv#h&G`{q{^L6bpdi>d1( z<=L&V8|Wt_@O(@rtaUXWY4Ee^q2)%=2>J3Z+U|Z>*-(>qrG+`jWYEE}l2l7F`yQ`!@{VT>6Ek$Vi8hxvP2R6p6WJ_(@(_ei+o1*UH@Bhb{N`Sf%b7a_MU*Pd6~i>{+-FT+~Ivx6f!7y{c%yeuprT!7OPMp7^0z_Aox*0Us;U{l9bdpBSKjdJ*iLBICpFvk$3{tm#+J!${+th2deX@ks;^e6$`RbPoA0)4<)6*(8aXme?ay_SIY~Z` zbf$=G2E_Fx*X{JeTiWg^KKqqw}Cmv*JHM~ z;rx}@0(?f*Fq^0fo+;ih0M+t$Oi_CtZ%uy3Qn+%avFR!Zxn~>3^dALY>R)t*$&KxN zcYiKp_MOHfE5BfhE@8YeIfZxsXvs(8gNQ+|jCk=KYuIFsYVrNQ4C~Pv&IR8lyj5P6 zogP(-zg9P4+NFPD!McQ}s?eiMxwVoPV$U+o?hZT=)Y9;i?ccGx*(u=q$P?gEC57em zQZDBy;6Xc?5hQItMVvg%;)w%$SVP}W829zpJXWU_kIm}?v}xOHm*wsGh&l|34!O$@ z6YLoA`@)0dPM|`?RMxe!EXaG=7=+%a%1U0Sb~cX|1B9vmEXRKnZ|%tC;gh!j!j2XE zI!P(Q?><11B^jiWJaAGCJ%}fM;!GlWb68XTC8)oC!A7EdLmyFbEyw6Te#;M=l9_Sz z8^a-SH!vFF0Yb??hDlC#;fcGAUH!CUEcN{`@U>+eTUtb~8@Y29gk|*txIYonXo_v^ zLPw03imDp_#xD&nLPiZ$(?1ifyk$W@{<;-Vg*N2t*D_9hwT&~bz<2&aL`R{kO;-QJ zDpC)+#%a~~?cgdVdbE^5Y^prm`CTy4hZ6u`PjxnRyCq}nP=0EUQv9>qW$T8}on04#55EAbyy5hBwY)f=Yt~BVCHWU{LRF2KD#7 z&=nh8fvumit2h3`50#sNh(6zdxxn`DR7=%A=)ePJ+j)Y$5vx`xEvvAMrgQ<9^$}n_ z-ojsjyCBEkeOOTVk9;>^1TYxvN1SY{-M8< z3~piA#p?qjO|Hj3cXB-<{&bR;H#2y3_X&XUMlBRIks$Y6`RAl>f+MwQYKcz9E$Gi~ z|N01|uBhhawtS}XGWe^`E6{wWI?q|YHfXswgO9it5MiHr;6+=?_CiJjpRaynfS}6! zbN_Vg9XdI-;dh3xh7Z0^m|k zGU&=e@byYn4Jlm4Bl;iw)$%D*-n+qSL)tnMr0@YEXh?FrjJ$2cV{J#7Wd1v6m=r!| z6@A_^+>e6a^{F}08Rp-da* z(z5yC?JCHp&mD$q=fWUFeY#1t2m`A;kmnv18maP~Qb8qD)SPk zkA^Ryu{L}7;bs`L{UYC~$zMrMhqS29M1No8wHB9oBy9m}AtuHuXOwAmIGlHFfdI$T zpN0GU4H?gA3mVY8v^;v|BuaMJN1vt~Sw&C>26g4?S~j9#bsYU7s>Q9v*?0isR?cH%8TFXvmsNb+Q)avOpXMN(Cy_3)KaZhqb!V}A zZ7gj4C>|+mWpyi7;YI7|nk}OT8Fu-+K5L!*h{^6ggNna*@B$qivt{uQIEfdld$`jd zU>u)RejWBTfGH+3TI>rsrgzMUkS&J{n;dw`5>J|9BDA}?1@h?x)-~`BD}18yQJY7l zZ69F0`VvGkI6R&OW_u7gz+hMF7zGkikp-5hT4K5MZkNWYiCrD?iRD%oRzqi|6q&|t zq*z|FhbudH#OA^e!6A?@_o|iThw_F=c3xwiNE?53zix!d2(y8>u>g}flQ1#JTydAx zZ{dzkdqLJY4@Md7%_#3yvfAJ3z(79DjW~HnyFHn|ir2PWWoIsn;GJ1YKt1|{6+GX@ zUxQVfz%|O=E7y}zHuc~Uivxd8tm+J&YUi~U<#>3-Qx-I-1)y0qk%xOfWY94m4CMP| zrd;_D=wGSEBg1d->tmJF-71)e*X(8WSH^&rS~ZxcON^7gzp78?@-?Uc0`WbhPXKR} zQANb>6f*girTq0_4Uc5Gv5$!9{Bw8;3+f(hlV?f_w(&MlRgD;KyaQub%q}ui{nJjr!7X!%zDwLj~+1YSQ}{R-9&8+P~p(a(N(N```gkSI25H z;5Sy_*ANUIYy>p=sEe(WUeNNN+nD&*PnaYu)EOa-N({{QvxY++edIfHDQNykf-_Ef zDY-Z19Yl3Ikae_u2m-v5`SooSqZ}BfAbA)q#Zu~J^Hz5|ygb|DjFuuV)*k#37@VBMt1s8Wy#*R0x%b6{ z-JI-@@4jKyRdewlj8t9dSJSpgzV`rLe^h0-8m-vqHwC;f(8R~_Lgd3&dbW!!Lucbv zNwmndK$z^f4^tc-4=IQ{RJD<|soH>|I0-BRjQS4)xwm5(^L!D5M3hIWf9+bRG7roW zRgtY5syD-a-Bdw*uR8N2{SZTbp2c^E6+^aEnzY|3|!+Pi`{m` zoTrbaa17~?W~phww?~f~g%UT5LDniur=_Z>iRaf{UbWx_-)C$rl~^%J^D&C0#ZLbD z3`0S}Vh|=K*GTcd!ZfjanYGmSPEKGuMLsv0I!d+fkuH=2x-%6!t#7pzd zMgsm(Twei?(fnQ`Sk{h)EV@%Gyjzq7(X}}98o`H??b^28 zGH($(ZmdUmNvUpz31^MTB>!H3s#{GkaoWiJ zUd_Yvn*o|9KInwY>%hgO+x(#K03ku|A>MbYmIKA7s!3=pP`XbEQ?d1qvB{M_Y`2y2 zptsL0lsx6iHiGlz;BkXqKL%|z8oQe0tMx{r-<4Lknul4>$#0?xNk)u-mhGA8P!YU#C&G2elqN-9cHEW=k=0n(@D0-4^_c$XrT z`C2l-AS`k*g6elY+AJ^XF?=Z<$kcn!$C0o3=cUQO{K`WHZLH}1z+JgE%D5U6tKtl{xXP9{fH-tIuV+_>Qx_Xd{bsFFV4w2cAwuHh z)dAKtIuPFKcL9RFP-KyVlbprUQ1$mXav9|~BDg@b%3vt491oC2tIM~gSblkYj57Ha zXfKmG2D#!0s49^0tFPn1+(K z`9C5MNIQW%*YnR_L|MqWd9EgIr$p}r_=og{R-?l+aW6YOq{8nkenT?r9{U%NzV7Wz zAOPx!OAyIa9MZW5O|2^V?)=(7soZ&?{B>QGMxbkemu#KM za=z)$_Vqe!aOM(Z?`3tyc%aHm*IEdC-ITv>{v^=VXBXmVa&C~8laVMn=x(Sz%uTL% z@!Pdufcu_4a8Itgkw?59A>q?hp2-cOi)6!V*!MFo`lab?9=?Cfh?YyYfLT9Zg!2^| zK*`5SP}&Vbe1T{|P08&5&_k?H^O>gnHJF=LVu}ENokm6ho@y9&`Dj?Oi)w#(FWqo( zzhJA31xz9rp?GE)--QKjzGdXepFTi)wK}n`Yr$^M{=)WhJ_U74b8XVD9WLTwWEaVx zwOG7cTvOz307jxiCGOVc`cvvq@)c_+$Rz7Ljd+GxPR{-$oX#R@X zf_@KG1r6txh8lORLks@54zy9r#7q7=0R_x#WwvpLl2}DNe&N^lnXLH7iZOn2{5YZ{ ztrnw<8*libeIs?bHjCwbx}Mi=w{s?H=W|o#xiX##@!E2eN7I{_J%&X-dCI-S>)a0307L^3j5D zNZDiu6J^%%k$;bmVH3d4t5gI}OTAY`xwnt!j`dEjR8bG2 zOuVIPdnPZ*MQ6JoQ%!bS^g}1%yU@Rs=V|qdt*xn=p~iqw#rr5r;LM1 zKZ1~}%h`U{53K4}JG;CrX;EW*JE6P)Qli`rzUckW& zk*mIM`O?{d1%CO$$dTF~*y*6!X!p6Q;?r4g=&FXgSE1yXmoymz(H!@HKsb0*;S!c0 zxU@-#qEX7o-2N5oDILjnTj};{+&L8o$c7vqCN!+Fq!RY7jdqPBRs^VohVblddLBv7 z*$B3CNv!9|X13g^2UA2Nb_mT%tSapuK_SL+{u!c5f@PP$UAy-~lJf$g?t%Ycniv_t zyQUsz5|5vqbmCNz=$lkMA_|k3<;)fqFh{i?s8k6CrsTQ5baaBr0jHfBdFK(8R7^${ zIj6GUwG7%~zJOWYe1vwITxMqr@3CyNs){>tA=3sP2i#XjL5r)@WlPB*fPAeRAZ+>< zygcUz#L$ZIuub1gBUARI0^qn`z!!J29`uw488JHhtL4ckAF3NxnY5ZgZ>F>HM)P@Y zs-ERZR4!uWVK3gTl?xjYi0m)RPGRtlb+u|`icHe3Ib;#>npq>4`;&E+3;6;e>N~@0JM=@$-sa3j2A|J zg1Dnct694iZV@@}Op)JPvDR+K&>YQFQ`46bJo_;lG-S0xXwMDLmtjXyxPNn(CR!0d zY^^eV#^x5vA|wH~#3{ZKIRPa6s;64zs0zhCNR37@ZiCqoZJ1P>3-x(?!mDfWbr6JW zuY{K(+Ee~2g5BRvGa{wOU1y^7?8zIpv%oKS0h>6{#_1^ z0}JzaEtZ-^U_8Q=0uHeeFgKTayY&Q5sr?jIy{8vfS==ef&7l8eQ4h!Kb8$@Z3eop^5dB8a4yx<;+E-3XE=Lm`~Z6GoH_dTu5lGugxq zBT@W4orYcVn!+5}%9-%rLIeQOOullLjS`oT=tkgBfzF`>>Ro=%86+Q-H{UXvflK*7 zOI*IXiu*>!^jDS6o2qE&2Q3-T^E5kH^Eu|scoj6TLIsxwWbg)gnf}sa6acRn|{<&7&=;w28;$cf|Ia;3mZA)bbePggZh%vFqPgM=FV=sb@b4o-y z)!}mDSCCm@rKCvU?x-%)XE{yU(Aa2c>WM1&Rb`ufx&l(YI|ag?OD`e0_BU8~>ce*_ z?Lmr1Fmmx<-Qq3BH-o&oMe*F*X4RI+@sN6c3Nq_Zb@=p{o)s+dnXIWrL&G9x&hhKV zYpA!=dNwDwqu@9yiOGg{WRFYs1A#$$A-A7P47w$J%@1j>IJGoZv3f%zVE*_8EN{<2 z;N4v>Fv$GKca!)(p#2dWSlJrYaj;DovU`H+X!>1sRPtMd>(u!yVE*u!2)P{zQ;hwY z$9`0frlI)82!;ytm-`mxuCLCeeiLC0FO>{lSI@66{$l|n2N{b*>cLi}lqnUNAeA!s zh`I$h59)Ajqa^M6A-fXFIXUc9s_W&z86;g>lffT6l-v$o3@;-NKqeW}I_#L*YdfJuZ8T3;dxJ^8Y?f6HAN;^q& zkxvis{)a<`%8zqYl_!xr(K0S3k_;NaRc#QEDrIG5?@zG!tL-pxh~sxb>{;AoPl?kc zGyXDcvf&J??W zX3R3D;oV)76_uY3^K8ZUhDBbsHnQdE6E^fi8zV()5{wzJibL5pRdROMIHvyJnK$4d z8~yJ&gMX$+S5g(`)5%Z>vPy4tEa|Jx%LLbY$Ij{TnF z1D&OMAxjGDN?moQnx~2oMJp%Smz$y1%Bt3GkFo4v?_T8M#Z3$n%hinZItkBo)Ju@n z@H-)0B2X@_@v81MQOc_0&#f;u^7pI9B(*@pG)U^-(5zrd=UlU_Unb! zMS3&$*$g8`>lPj)<60UPDLa@WwkySmWY`C`xaT{ZMsN7>&^hdoh9mir6k1n#+1bdG z+AE#Jma9Ihn-MpJ;ZSGk<3uo@sRG)+3OB;EnBZ*Kld>gc2O16(QB-9vTr%y-fXu{? zU<#34r7=BvtA_FA4j|E|>v^v7?|gP0-w?dj-Nx>EZyAbtwoBjrPvT69L>7{evR7(_dA+EGaaftaU$E#!O5rI7=KQKv^hfAp{7r7SK-1y2< zd1LyWNHWQXB8U(R3B1oy{3qd8bT&u&2p(Rx!ahzJm6exhHPn`&YlAbI3067dLgKG4~Zl)u1TY z=sOjuO4@TZxib_Y)93*oomA_IK}0|5Ydol4dJgRWl8?|Vx0{7kp3D1f-+-aVdR85> zpHiwwt3lAklxMDy9PhMEMa1Oa$Agfk+qg*ggUl%a(ntm?(RJz;Y`?Mzo5}5HP?qTl z+rNDUNqt?}>r^J!2~8$eJ35x4^6_xkp4=#{S~3RJc{GiWx+-jQQgOeEJsh(iEZ#rs z)TCqGZH4lg>W(@ixtP*%VeZQKX^6p{R$a>d%ZjgmG*UT|qDod1K8FRARU1uZvCFI? z4sZc!a5rbGe23F_w92uB5PozJGV=lrva27P27MBvd0f*!3afy)r$dWj8)GvUcK zVU(o8Y-8rI$Gx?jelo`fx|^HJo3vAzfU}#Ppo(l|)s;Caf6S+j*^r^>Ab5Hq%;2dP zHjxvvAfL$(!Pg#TrhUIM$tsr5Fhi!VgI<~?$?>HT(mPw?1#Yg3PLQT|2rNT;e(*Z(-T0XcWQ`) z&o^wcgWE=#S`|@VbrD8mbJrZ1JQYwpSk9~eEdpiZ)m7EsRe{W(n~XyFwJpP4yJ+Om zEhtScoaSHWzvqx;D@v6SryssxBWdQKe~Qis)4xCUZ&sF3;3kw++zD0fV1 zGQ|El#!l!r=*D-cx|Dm50)LX6F7HcY(07kbl(9jGlb3p-P3|-?tQ;yc0c}{IOdJc~ zKMsM}(lpmNZ5M?4l+ZOKa$+;dI0m{t)1&IkN@DKRZk2n$)(V3)%bRd-tB3qbs3bN{ zWIbv55`14=1n*8#(*GRcPuwEK2f`6SRb(d}pd^O#8YCL+CR5s?pug4AO?b2HHB@kA zr0h7$0Pjhbk-ay;?MYS1j)-+-uBYngUwdusYObB^0Gma;8nEKxCfjf_D3@T(7l4|} zSc|Oa=fttvjs@)RVB{lD6I!BS`L6FQDE=&4Y)i8R*{ED<^iquGt|8J*?%*)j^1`#V zuDM)wjzUD$`46Glj-YPIzh$IjAm@2v#9Nxl$p+}nO|*xU(1zPG<-cKuRnGTxhDrE1 z6z}(Xeidv{d{?dJn>jY;Lec9cAw`Z(27P~TLBs8#9ftI2#vI=-fr*wY7u7#By5NwP zs+#ddhf0^ju-Mfw)JE^N5EBVIT(d6*H4Oz)|2qf4*$?{(F&1B2Yag6V2zzRX$~k#&xpV>cDTt$7X-<1 zSBtb62!UKuC5-;m6DrZTFP{vT9E_NsE_Bb%hIcWXJl5d_ENBFOyCF*Y5sTFSXy<_P`Qj#Nk41gOXNL{tV$ zfJ`^(wwB_36#iJIQejT~!LL0ro}|(!r&U#gkCtOD2($gVUqv#63Jf`wm1LF7kyMSO%4u6t&IX{>;!eE9|1pZ6>MZ@ppW-7HthK2B5TMMH`uhLE1oJrC*ONv zWT~u?AnALYwU5^e+KPD@P=Bl!aLZN`exlA~1?}-EQnL4J7T@tWF#CzJ^~5?b z$Ln>>5vT%Ti*7QC_%pUspNIK1>A7SCun(y0K%@14*c%H$n0oVAkUn z7F)YqjopAfUN!K&RSzIlkJJx{zxeTamQ6Dhi#C4ot~hF_H3Qx zMwSsLMgJO6Qqcs-9IFG^ug0cGOBTp z*C0i_6*!orx^g~ zU}Eoy{AI&?O73A$fQ%X`=YOio@7{#-8e*Eh8hmZa_n$gT`0Q7c`U=NUppT0R78mtR6WHC3ze3wnen z*YYWza#&R(nV@R)g(zp~N$N#PZxye+i<$|tzAQWaM1^@abH`Fm&B)ghGt#xn85Uge z_)ypO1gOQg(UNu&Af#^N|`Z;_2jco2DLW_I}0>_73KZ8YQ9dkr>AUP3yRL@IW^_QB^I?! zZ&8;!Kg|U^QTFi5hKP&Jda*+eMPew;qId_%iqSP?$bamoiSiN+d}dxCnJh@f5SR$C zNDSb5Dy{~*1$oVt0aNLMVF8zvT%GRAc>8oqpfsM~jF!#4Fw0x2EM~4@8_C@TY+=e& z5bBP(M!f+K>gPe(3pO+D8ds}at<4{&m3AM~21Mv@jYK>v1?otK9qvzNKDLvkg;Zz^ zxc*jg4ej;B<`UZ&b7D%okUlbg36IXRnJvh@Kh`lB%C`k2e zx~qg^%bzen7OIrkZY_*V`QZuF(qEln%BWZ_;!mg>*9Y26`57eI-a@!-Toa|bH_LNR zLtg(1kkKeKNO7p5a1E)0hit<&daEL0%^x43b~rXcYJX(@ehrP9R8s&UBUP&2$YGFY z>kmln-M&?@?#=3ku=0C2p9+E<2w9fu|h@aBpJQqOOsB2r5Tojxzx z@nnv5HduU+I$+FTzeX;$>rT7GpJdEmS7RQ=>3N7i zB6+nnv8w3`%`KX{1IBHiv)=m)Vl)}`3gFB4h~n#13X`wSVoTB+!@~8yUt9i)=4NUl z>kRq_8la|CRk`ykTB(yBSsnjUP)*FIg09-I{PA+r{-Z7@@P06HjGisQlIzLMjOllUuahl85RX{La-TkM;oNhl8B73{Hi3 z{`{W(Uf+V6d7@{xg(jft))Cy;*uYNzkazu7~T9R&kADuGu5ZR*uUh zp0emQcK5`QSQ+iV;UHSQC%CGoYla*hh@N|-Yi(uUk4Cy!-eIWMP>HqO4;X&(Vi%&h z5iUL~H3!aJ)%CFXeMWw89T+EWXSFqSzpmQ#WV`Gw1z8fLXyf7|t($cI5sa;XK&5LS zXM~2QAh_T4yo$0t9NF7}E@hIf?7-^wm;?#03I*)1%xa(O*-qqkR&^D{Wuux$mWwK8 z=^~fD1#$bPfvzRRkk)lo)_@A!IPmc=4q=3|GmH-uQNt(+( zl~Z)D4us*%1-5B)l~8J~zQlEg5gMywwndMtDlPtV79+{iYQgkcHRVOaYS;7KrENbW zO(ls($@$qJ_pjZIjMyI_6V@0epd9TZV{YIzXs|rp{oQ2?8p~^_lR;_RM zeg~AoPczbsjYzn>I`C<=E?6r0s@qp@m9mlDl=VDSB|~<*B-!QcWkxxfh?#=tz7^^v zsyb=|1xE>LtGwR|N<;nuO{)el%6q-Ihct);npjS{Of;Ot0Da0Fw*6SeS zH-fdg>K3FEeYIu0H|rkk2X*{Wm+u~itG=aa4bX9vYP+cy)aRd3s`v$+vEN#duB12z zp>3$(6RfKH;-YB6?GJ^j1N5-oDr7TG{z*2hVpq93$FFj(IWr+9NY4If#8J`+754On zQG$RX<^7T>bo1sol=WpKpl+_}!uHY~J#ci#6Z<` zs$4#%cF_(5JET<-FuG=flsQvUjV6sFTdMNi#`Qw=QZ1(!p6IxKvd+R%HbnrfwcGh? zoeC_wSDC*`8aqvJRZxGf?ei_#TOja@9_b@}#v2JTH5enuH3H+M?{Cn+$JV?*Y^cE< zNe_tnzMj~dV@*jr>2jY{T&#l5xOpB9IH4yrV2=rnW!MuWc^?&Y<DqO=;xc}$qtK|AD{CH)RU{$E~) ztSNas1WS~vDX~SFQfV;a1s8b)QL(iCq#0ma>D|YOk>)LuG;O)@r%BrrU?iH-GO1oJ z*}oZuyqluxW(D<=vWvI_R03t_qP> z6LUppkj%|Ob|1V4eipss5Ll#Q947h!Un-mji0cX?URp1J3s0=YM(V65hDy#Sa8e&- zC3@*u?$}pho#pS-2=8Mm!ULRjhq`g|&-emSvrc!kl6ym#cCjjlPIE7BIkOw#{p<)Z zpRE@*mTyPE!KLzGN*Z_6B5Da$(~&F$FidsrcY359eMc9}r|S%pMBFtJ#l8U&{Z=oa zX8R>H!iU^6(f7eN%|cG4qFe2QSrlE3e8?*oujlB$Kieg#T2fR4d>34-Jr3s{`X6ZX z1Wt3(vP&9EO%>Rb*@(SY`vS5a5ovBA$#2>5Z+b){1d_#%qXLzCE02Xr3q!HOL-cGf zz#G&YAzbo_{I-Chu)?YlhL&iNZ_BYAPu*Hu<`p4X&g!;aSR>K(r49G=@^5_(WVwd{ z$}6?8Qk5}@FX-92YEKbPPCeAoYK#%2z0NI=nS0Oz7xZF(dA$TZSe%uSh6Zr^G zP*$iBk1-wDnGl<2q6*vdrcaU+#sfKS>fm}11~7`^%hbrb;*d6wapMZ zM|6vsYEWS{B(6S)d9W3h5o=HIdx02kuV>oi z3kTc2scVh#pk=j`M;!q%{piA}C=y>mo?4NtyB;A)KF}KrH?UuKx#ADp5@Y@F{6u$_ z>#Eva8_HG4c$HoIXF8}WyAQJ4xfxl{BDrw{)Z^;O#ZQem z!c3`yfByjv!b?=Rm6S^`;wSInfXg+(yF=CAYNA?@THc@-uP8S=b@549o&&l69F#=# zn+`<&D#KCl5#i|Elg2g2u8&g(5S&#M@o%j?EQ(-{g}zc!5vIMP0x||Y#uVkkz(vlE zhfC|b7Sqhh0cNcs+ONSH-gsxW#GYa?9V;QsFuUWY0g8$5XC&`4DFEibB#Rl_zvh&%A%q?>5Lp@`a$FkrsJri zOl3>mG(fH|H;T333=+X3;qcN5vCn(3(z43>i}?n$*A%CpG-$OgOBSTUNxeu4LV2dC z+5nbHs2xiytiJ@4t4DN4=h9N8`aG65 z`Vw;RZH^HoMOzIkT2ZSZKb#wI=QK)4NTZuh4dS$P zMr=geNf}G?k`E?Cj(5C~&Lt70skln_VG0QqM2=XSwj=T6HqN*+|;*>UgK zzF9Rho_GhgEO_Hg;YNYFsw+A;^JU&*SU*S43>8Y>VzKDaRixbyEam16r(I51wnd?C zvP0xfdFZrXY>tF3HFNR_j*x<%ovBcGbVs>R0oky=6$JKyreDMba&-;Ws^(`Dft#C= z4lvL}+x3F6svrkmT>|A*6oM$_SaaPH9Lv}}WQ?XwChA(A5hvi< zC_P>cWeGz+=C>qBHQNtjqw2{n;+LSwOm7)_-$}m0E5jr&_95uECm1ToAeveVxK!lf zec;zj_v_786PnBVp++Q>X=^8=N5te%kvprV{Ngm|e0Tw&aPT0+JFrOg^`oP~5xP-} z-+8`zsar$kClAbpp+5n`p?Z32{7HHKQhS#X!IJZ=;$mf|t{M2Ah9Ou2G1ukqLmc<& zI=2u73G2-ClouLPB>oP$A1#D~<4ZfE=tP>2Jm7je z&g8U0Ty}R!^?~*{b<}m3sXpu6UcqcfuVnd4dsW$^zZ(dcl!2DNN5rx$s*2oO*B^M8Rm|C!s<=;C z-O^LWE-^A?;T6Q-92M8G+832Ta8)DCFEPI+5LW4bGN@Um8)ILh~)M zw*|b@>>$$l`L8inj@1cvIhTu2zNeaxF3KnY5;h;r$VinPME+wS+giy@ckp+33=90N zIS^Wt1zbx8vD9i=*n56@Y76PV(FjvqJ_x&-LUc`3j>aWdrWu~i4tr%3$Q+g7KWhg% zYNDPu5yB{ml$M0ZiAJ4lIWz?cbehsonk76<3o6Q^%LgkhrEdOrwT6o**lv@rzXoZ= zdRkvL8CP8btcb$RdQMMiRu^5dERxj(sQA~l51`>v%5^wJc?+(rKqMTU2mPbC-6Z65 zHg-xc>?(Vr4COAiDgV07Q*7@*)2sz-u_Dp(B7Mv`*f=G=xaLY4qipPw@}m{k)1tA{<4FMQ%wNjd_Ao*7KEugw?s56pwTu5-+oLuSdXtK1Gj_GgkJ1t zs-9g(i8Fus{0Uk!d>{fVSWk_R_~R(~bE+2HbfrXln;5y!3^s3niLh&s8_#8ULI;%> zZI|xToqlq-7p#XIZHT7u8>e*%rZ0mBVCKHEndF(Fqc=!Hd1V2Au?q7QssKepr`PH@F4ifQ>@-KE>AcOFfJ8>(iRnW})cD!1=fgXa=jV{{Rs7PY}?kscp_c1aHwaK(|7 zz%hv$VhK%ACZe0#5tN@W8n6XfWYTnS|56WYDv$p&qKRIO50v32j0h<=)oBx>Er6cc zjYm4_(PMED#0``VKY_UH)fjd={aJXYN(k>|fAg@*c@{o)2L{FA8yp5HRoau3NcQnw zw|Xi89Kj(vuaVftFzAbU0Jx4G<3^aKH5g`0sf2c?@okfhAfmmp7JpmH>UYBom=op7 zUXwj<*zjT9eCcWbch|<{=@h-6XObjO_g_)_2`+%;|`B)Zz`*`m87uGvb1-b(dwqz-5oh zc^$X%RD|3QiHYKf3Zd-$gjKwtf~BlgX~b#w`R5|Nq$}z>uM!H8xY5qtooM3wfgvXC zhuWy8__M6#J3WF6eiavP_rXC_MYqL)M@noBbw#yNGAvv=)-zKP<}NFGMg?FDF5KiN zgJbNf*H@g}nh(9*Oy=W22wFW})mx0FtB_nK=oGhCK^2@IwziTR!oGIL21f_yE2pcA zELeti->$&|({x8ulpv6wrl<9mhGvj>;}N#%B?A3%QkBfW2q#)%gbJ}(*%PT4#<3HZ zVO%vWCp=76^yUyWN-yjyji0cD6R}xx#_kM~)t|uMlk`k0(iWo!ZR*B}L>ng+s>76# zjoDU66{iZsF0LqR{)2u-`57sQoAieGIw%r&G&!ETWLi4Z*3+irs<4ja!%f%GZbMWq=5mhcIZInvr(8}>W$k&Z9J1d+A%Mb{ zKw#sqh>Fceor$u(HuB;)MewOGkB7*Azte(*j*kRg#e>;+ezZ&uG4heD*|pKBkqtSK zX7wZ^DZ3H)u{M#bPQymx2eKn+8T7FAQ-iGiU09T^8R%WZHGZ zPcCxa*1n9*ktyw=XPP#5mk)m#sd8)!3z@J8a8AkxT3=D_MEL2A~yjGW~H3DPo0E#LZ zKO9C#45tyG+(~8^3-t8f*omouva<#_e%{GQlPfA^Y)w66`M&LZ7q6DwL(dz_hV0!K zo>ar;O6XAYwl}&vuchR!K|DUw^XEvwO-QSgUffug4KWgsIW*kec@WHXdZN2JaYhnMhs)}UM@+Po{T!zCqaQVdN44ncbQ6*}RS`{zdS-J;CmjqpCQc>P zhscYbAxP2DF9F(+H}F~SuVEgFKN?!UWD4|5H|+A<2@B5-iZMwK9|P6GWv3Px9-$%u zdSMx4575RJCF0mAY$^lV#%LTh{G2!ep%H57R)r3W+iBH`QqaJpjL}5P6Peb|L6yG* zf4MjnQ9bJp22cm046*eKy)nSkf>FqR?WKwji{7MXk3sL`RHD~!34a8&li532k)r`0 zFTVgbBo^0}^G}T&Su@nILT%~M$oqu(n6X*aP=k(Mtd5(o;O_z7jv&maMVC)*f1sqCNH!q8^iVaz!@>=?wJ~QG#8-1_UGmQ~vt~~& z))plzrI+Xe$B~8JF`t7FJwdG;8T|s}y-MRaqnj$@B}fUCXEl=I)Cs9X;;uU*NEqbj}x&~KGPk&#p8g}!6jp2J^9Ta1^1nb=~?9o$G27Y5BK!^8mdid zAX1c~+I!7-D=z0DzokCp>-oZM=TGVGdXu1#tzgcnj&zrmk$>m<2Kt|O8w)f-cxupZ2x z<%Ph^jMBE}7R$Snr}8CcAvXp`^u#*wjAe#&+v1EzRbg)=_J^s95`fbDt?Vd>)Dvo5 zI!5C0DIN{aMz(;+GsgzkF=@?z0*s?$G22@6BrO7FV(3||;5^<2W2dTqs?7O4A-J;e zKO>&YZrFBnE|8_m@@#l7-wPy1w8Rkii&r5aZDq+U2z90k@QJ?#s_z6DIUFDIs>{A! zh96!RZm(Ldg1%bosU0OtC32o|LLhN^#8ALpY>~oZv{ATjYAE_ltf|0lPK$JS3;IT? zfTpR$#mV+S%)pw*WBlY`LnDYiMYt+YpA!%fsWI5fuc5>FwF<@Yu-YSMX+H+rw9df} zbNiV>EmKc+*!Yt~U`k6|1GVtAGPU7T8UC^a^ zWPPZyaI{D?8et-4vNS&d;3iSmnrY%&YVom}iZW`BkjtqiNvaGiMOK@r!n*-dki%Mi zlT{i3lHp}?HqJ0V=xG%~jbgFwfN9zsEjl=d(4S~e5J#+5ttK%<=TAT|nW$*R{^F}? zzl8g#gSxM*PcyQljNRZ?Cp-aV!alHuren5}h5r1Op=@_T9(Ln;mw!`Wxte-jM@gNF z%50@uT~Xc%iefWO91?f4}mKB>V^Zk)Xr)MDa9it#hyKZYL=RX=2X&VJjb~Xj( z$;yiu+TBb9tLj=ZTJ;G%Vnyn`Di6&>Y%>axp*gdV@R}DKkQoeXlyruvi0_EjE%W4M zTO*azeqIIHPt{1qE3PetX$bbKUJRd?k&1>%ohRd>&^m@|x;$?Okm)Ma1J7oVpG@7$ ztQOtUh&LmLVk$-jb5=2_kP1R_zsbSh8>XjEl%wkQcbTe^<{345fFMr_x~pi(no?2~ zqD|6+M##xA?DSqUj8*!CjghubSwUJA$NT7kK(|qERfDmpa4X z&zup`!rQz}+LI0-UC89u&IW5T=nm@c#|8{CSUF;P*ECykN)~&`m=?f$;$sxrXuY5b z0govY<=#{5qI5mJn@m1nCfxmnIZow_7~-~;hua_K-F3Qcket@5WL8k=Q&#EL4X)g{ z5**{0Fv*_bP{vX{jB;NW(1tYMF+=?+J&;-1-R0VTBL(SGTwiP9o6tkKB>KNkvdW)U zpx0baw912$m>_AlnrVx4tBxX1oGjCuv6PnayjfOHuO+@tByp;qT@Qnul+gQ^;D%3v zHN-C+y;_`eTD9MT<7DO-m~EP#(+=J+S>^Yiq4&|;&tf}9kLOLYGOw%B{j^lCfI5ke zMd8xNt%@GxO01cq9#>LiMO^#@80UPBsC%aDm)fe(_Zk}9s7fX#ZEI{K_G6}>G?|D> zt)QE_!AN$elwXU@{z4V7rvp?wuJkekC|ijBKD#r9xAd@uc;s!%x&KS64q;K}p*Q-tWAH^Scn%U%(?Yrej1_Y z^n)QdTm+v&``k$~N~BPg8riRESkT6&lRUiS{NJ5-A8lN$vV%Edyx9OP=PYzuwA5gw z9xXkv`SZQy>?mHUfE|tfm(>L=KtA8OP{Sd04zQr5y0t5SqxwE~i36l#4K&Q%C?iPv zn_0yMfuWF^;n^4m4`gdaHdQ1Rxp7KdY@xaOcm9IAQBsmq=kCybeh^{$y zXnCBT?;*a)=2_xh9%HUT7;>+zDhASg6oSN8xA#Lk*{Y~sOLT_)+6behl0!8oh^hkP z7FU(hQ;ta7%UNESdXcxG|E8DH_@$z zwj|sl-3kziHyY^KM%$duCmN78u*#(4h^ z{Gsn~m&h{WJczUJLF9~bcI>+nLX8+DMz|ovj(v855Id^$%qAsmtO|-~(V)@PYS5Ba zRg6@+NWWTsuTOvfkh^=H&v<{{`|}=;sqILSTr8%vQF4%6Bap4MscdY(Ump_+C;M80 zSlk$qep4ICRe2o`uh4Ckal9-xtILy6>9YkM7N9)5ovZWLELt?6Dq4rZ*SJucqeq0| z1jd&iqhcC?Ko9vVfW4GeZi(WmoYF3x@ z1>N49Yml@$7;9;MV(Dg%lW_qcUM;<_8G$)bv!&V_xOf>o)hg}3F%rer7#047$|Iku z;+(qa*|Q|=0p#6IiO5Lyd{MVn#P*6vl!GS$*PKf5h-2KTksK8lgfD}1MdH7sRM?N| zjYeVq6!M(Z<-D2UO5+h@P6Mw-$x0Q*G7vQ;tv2wsRv~7rM8&@qbP@CKoIy6O7I|

      hA2M@SWx60s0eEcNPNP;<&AYR`N3)&Z%h<+aw$a+0} z*`U)5IR;ggDj%S}+zm#I-Pr*D?b03R!m%1i<7_=Y0fAP~L$dZ6g_3x{6%9FN1v7v( z9-oH2E>fc)6q#~cg$+g3LQ^B{^=5yd+*gkouWrgJVH)6wx%ZTnHK!5<^Zs|ahs$^G zZ=i@pPoItdFTaUgRL1{&q@KDp?g(93nt=~NEs&)r*5lRc--iRSb4pRmoME!n-zS zSmm}LbEfMhBVle%7s{2_PewoBr0JAaPvR5d0`aOi+SOuB!gs|*AmFk_D96n@ZFHoH zKli*NPDYhc(ghuSi`MK6w|$_i1BjZmXUYFQqDJATuln8lmU# zP+c1+&%cUG!coQz>`ayGb6igk7V{`J(L^t-B-IDHC=Qg*MmL%Idbik?d0Udq@4#u|V<{sWOdjBmiDS41xMc9=ktBXNg8rC33jmI>h## z_Yg^)^@3#*I1M=Y>+a!l!vcCm>j_plx*f%HqOwcR6k9M~3!19Cm?~U{6ugHGo81_D zwZvb#yax%y4~1qX=rIE&e`H*dOgPG!&~S+4VqUGGXE&8E(<2?a$%(1A*0%wO)9e_2g*q^X}^VerSeM{aktjs-d9VJ3U8x) zi2j{X%3@Tvz@g19kavs7288@hF;bpD4IhUzDxK{x!Yv>Oj4jMBOmaw!@o*MDUdjeOyw zKRkdeuPPg<1vRe}Go6kS=JlwcvL4#*umprwP-poSML{(kE`YX6>WHz=6@j;$`?OHi zXoPSgi!|*3QjO|>z-q1HG6sIbm|OM47lE-?$#2TMudd?nV=IN&RaK5eY8wm%-zw@6!{97wUEsh8Ue@QNM(c4jyD4W~iP}=d zN8i6>LXulr$m>@qcEx&bEiBud>N1mpPN)$CtT)%K<7I>|>}5@g>g1VQ1vSYs3nF&5 zMdW%cz|m6|N=~xMi&q@?2Vv=ag*h=+xrm1+r4d=9o;m-!HH;Gqns?DLJ&Z~&WvNt19B4U6KSA|`vw@NM`Ax~CLy*ilX8 z@nF=KW+W1$Q6`4M4TD+%`w{{|&M?xF>--a58K(wo<{qj#df6=bCJ)lZoE&~z_ z6^CT)P4Fn4z#A5)XgQU9MX?qM{I$J-1%63GtLJU6AeErBa}^NGltQ)Z{`k+I z-f~5YX|F3+-uoI9SfQe5TX%9%s#b{_+?sXqq(|TlnPa5Fv$LjZIgz>Y?S3wmc6|h+ zZbHEcNg4~n&xR;v^%aox3qjSt@H5Bpqi&%BRcb(cNYAkXe_3-d$9aYRR?Is+Dblt1plbH5q19Gim*+GLXJNtp(g&U ziiw5K0}{6*QTq>1fTRCK=s3DZQXhGJL}~4Nj6^Nfzeu_&`rHCzMOu01OW;CQaYc7q zlHOfjt9e;@M_q(!Ev+%1lc{{ z*3m(@q#VO!`n)(=bNINF$q_MG?(GK{jrD@%;6~9<2tL0L)>+mBd9OE`bxqwiM6#+r zj^^-_`b$3wV4+djHp&HrZsni+nGN%wMdn(AF$UnXr}M?KX| z;55Cc8J@`KrNFDG9R~@Ms6y#g^bAL3?SCnU;O)@{4BGQ9Ac*J=8ES3hrgAQ(>!l0D zG1#EU_b$-+eQtigRKW@y#+v6I)y2&}KzyGW2Gg zRo^icO7_fwD;1V#qH=(2r*d8(iG zOP+GaCoE$(i9O`?O;;|WilU5D3;;L@)dMb?ts`Cu?c=7Zos5fugH+T@)3v2x5%T?( zas~vDEKiz_qK&wW_6K&Kh4l&=36dJYJJZME>=-s3o$b}1y7(d z4T8nDIndqp8?;qa2^lo5Ezr54vMKZzoJV)%9jl@zjb~E@^#L&)dH840+k+jTX+_nl zeUsXOOV9TM+E504UHfW{p@ZgXDyg;Qm)y3gQ%6Q)^<`^;VbM{n=w6L6LE2XUsla}Eh zA^Ep5qA3If8kSr&B9xQ-Z57p71BA2~(yPRk3eqRpu$LmXfSTED5VOunP<_5r+Z*p- z4{3A3E{t#xBLqyhR6wu5iPyNu&YIh1Vd7f#)l*P-Or&>84ph__i${8%SOw6+P zd-2bDOI(cXQoTZqmxhK(Xz3wysj7lUIJ*}AWX+XUi}9J|jW#=FO0u|G1gI=EcvN0OlZQR2+lHm87OQb?8-ldRSfE;}n z66&RTc)nW!91qkTJ*4+-VB4iw)#$cGNOwP1wEU0S{tzyt*-AwKwx7Z(rmbSye-)3v zh}%%sOyYemo*H8_WL0IGkdWkl*HCps~L*ZSqbxcqBaxyWyJ7StK() z!r>A23=MRc(#9Glp_5-81&><)70SUrY*?UaGb6TBNevD}2(BZXLgub>sk;%6x*xV; zC`^a(dHn72b0})oKs~>UOvpCK+&#d?hu*?k8@`MuZt02Ll_g$R?k?uJ0%f<|{+Jc} zXBLU0D@Q^bm?#uMtqlywV@A#7@;0vPGe z6wa*@IU+Hj^i@47P|GgP2hb4Yw9HbTM08D2?N}car}GC99K9Fd=!T#MaWho1P!2qE z<;d)%+~ajV7Uw}k+MTlK6k@ZYUesFJ)-l_OMK9b4lA#fn%{IK!YAwHJW~L)LBD(OH zC0Z&sGCWeQ3*4Ba43KIxzR>&n(J7-15GV|bi&A1nmyMJ+VBgJf-OwY)r&{9c_2 z4(X}wF)Z`zNvoMgF45Xfvh}n&A~skXwqm@bQp*5+m`cx2>KP>@0Hw?6RUFoUS>{~S z$}E*u=pYp@1N5kzS0v>#3=6tXK_5kvpJ=gRHYxgo-$?9Pi^Pcu#5S-;AX;NASeiMl zBqht7h;*@}%i~pXZu!dV4j3!uDXASa z3c|_pqw=ef1amDJg%r$M4uW_cS|rEPO^*qdzmKBqPI`iH+o9yrx0Fso2}tcKZ$1H4 z$|fZT;RP_YM$DlEMC;jWQ0MRjHbvG^X}3B6B;R|pZW)b8^UektbE>Mk{7G{58E%Je zy^>AxQ8`%PvtaaT)I4>$P`FR-seDYNJy&Ua<17+c!us*F%!GAmT?oFU<#fzyghjb% z!~ypKQ$aIY-Y(QjIqmL1tb<-!Tk20_(18vH-9Xqm0I(-x&lWU9s-fLgz>HSPc2#*p z!wK!}NR=1F@ikH9FMUofshKN_DrOX*rI$=qK?#xQJ_xcyzrwRt zAK(za($iEqpy;i*AWJ&P{&^V8gDV<|tSdKGF4n=xbYD*&jn-@FF86vECYWAON5nHa z=vL?_P=)l+q`4V9z+F*Je8kmOTIDz!;x4Em+Cf%r-nZ5v0x|7vv~|-|_*=m;ZnW$$ zCz5XJtA^@#Yr}bN>-ig{CKoguVCrJ!>FO&>eu4AeEVh7K?#=MJoO;^KcnbnaJ0)$~ z1LVCb!D}8qKy`((LV9JGqX2wPGYQ!T9~h=5b|GTJ$?Z?cd-AxCktB!KWDtllgIBwd z-6(H^QV{ zTa|;PfAAK$R|lJKz3xtt@P#fD1eqExclMxl98%YWPgD`5rFS@q(;q{@RDeoHad*^| zj8IpiDQrgty3p02#~wTlxBB(ZP$B1b^QmRh^3tHpW2%v9*TXm#Qz-m}$wAi5L*-9g ztbAa5b!Q{o0oFtCIB!F_@Dn%-y4KU?vFSFt^O%;EmrJy-Rt-<4l`4)whSKfrN$`+T zGujAh#&JDqqjZ1I93xvI9($>epCB8v&84VkS+!-)KM=~#^{|ol>q+f!y_YuRWV45Z zcJ4ZI>qK0%JimqTnf50_VSahTs$w^tTAfHu?kV?mCCqsYvZ5nW;6;()#@`~XD!x~U5^0tL>1hK8HJj70xdaxv+ZjMn=lU#{LoX%H>2nGbM`PMTv@Kf=0UA7ZK@J64P@~<$Gjt>y;HI z7b_zLuQq^xm(lH;MPG>iMeto9s-xYIl{oxlSrznxiL`Z~t5tG@Z0`j+ocSIEKcPD! zQ8o%DqZX%E(_TbIBh(WDIT>Qcngq5AHilyc>lmqSIO7FzF66N7wROReT>?O9%rBb;Z%-pqfM>~pk zWyd`QNn$Wfwx!&7IL$xccDdt488#Y)} zflVbXgfh}5(~l1AGu&l?F0%vFHQ#6%T5KfXx8gpsEBC8Xt}DW&e|WMAA9E*&X+75a z^^c$|8oJlyGAgs@{=dk`)k!M0@v zkj3~WH>|Oq@GhiQU;}UAC-41&2=AbZ52omj+H&qkNNv{{phesgO zw;Sl2zN%_-2YGN7p++`FBNmkYJ`dy0#)Y0HYj>DUGRTBPoqGY*@(@wR_&suFv&j4? z1McO9Nc1rPREHEUZF*P<5vc^dc2rKvWqEe#x(1bQ{Xib@=sA1Any9Hj!Cf1> zJZlv)KYs_-OY{l(X-AwY#1vDXrDs!UBn4gSC%s^_%w%Hwwc|C99th*4Cf7t?6p0EK;D~0@5oB2dBOPPEz>nLG=<`aE zSdQ94yqW(i!;V`vO8J6IPN>|zU>Q#zEs!$CL4-izB3ToGQuCqBpuAx+s|wd6jw!pO zjn+5}BepSI@pn~dtl}klZN7=QlmAhv$r92@}oaLLbhE+^-C%AM} z4S7`qlz+Dx(t&;x!RP^{Pp~+uk%VhdJz-$)XkSJa$==%>eeY>TDe42g7%Nrd2v#T_{g0kL4pvODr48+o zU>Tk0$6PVSqF0C918~)W(Nt41^nMl=|m? zDsA=MP?uFUjzNA0ollNtu?cg;^#JLF+3JfVDyhz?rNv5zv6%L^-pL5UK1EeBg+l8$ zbjN#W23+u-RF%h0FL!w;gXcsN7b-j0y;c04An~H~tZ2?OX)j zinB_tAF}gAIhPe>G&@(>ApYR8r59MKeJIb{0?&Beah4rBby%LKGDPPcE48k(<)^xZ z>W)Pcc0#NAA<`sY zPc{c3u@fusmZt_-OIe9YdQ2NwenK*|lePftE<7NP<*?Hl3cFkBg&vxlC6i(|1lYBG z1}4WV$6SW;5#P~+Jm5lp1MWPdleC+8UOA%LAY9iIT1l^pOmrOi>R;Ct!t<- zuU$w6>nha0#6W52hhDl!FB}M$E8NU?1zi!z2~DI`Co@J*0<8FZJ=ae;MJ_=m#6tY} zwqa9BWKHD&oFHfRCoAU{b?V~#U;%uiYlmRP&L}Z9BIX)bGDvw*uBH_=jU>6>$Lx{H zyBHu`v20CG`W$iWm@R{rfFThoIKRa>SCKS+CoTf;CpcB;ZV2w+oW3&c7%N4|?IGo7 z`ll*4&MQ4JmU}m?lFXx%q3K0fpwAFwLD6azx8#<(G?oRJ8p-%qQ6&d=NA+5)qzoRt zy6icoo}7&G>1)y|7J#>Ysu5Xxy{HpPGkozUJ)*fp_JZWDtTU7^(a#^i)xI`U4e>GOD_#i?X!F$mD+zlQvaym)13j za9M4$%svAZC+gngayp<9VHge=&>9%U}?l0(?NqS;)6%=5F zjg;0>)KM3JXH}4!ZNZtH!8l89>iHqafc#O~H<_h!Wixlnzx*3V%S9`qwfh}cf($!u zBqDQh&SFPYLR(93%3FmM<;P68E%(HAwAzslsqrpk(y}q^v5*pFro!)heVR$98JHER znSiR-Kd>8gK0CH~&}d{(j!N|LP}O-V5^9KU8H{JqR0po$X)do@aI|Ih)SK{Odk5)g zLc#z|rxXlDjR{pQ$G-?LK*_TE@kYWtq`6~0cnz03sC&9|9Hz;DltBKndUT`=-s5xM zq>#_xh02+NN6^DmwJPRtQcU zIQT;bsB(m7b!!M1SJ;_iKTz&|=r803r_@`{+lxHnW20Subt+~%7v#IqEZWlX?wpz49<#7fVL9Ag7LrU#PF8V&RC2C>B?7wpfZGL~}#VeLeic1ShHAYn_wK~ipY z9~T$lPOzyWCaLPaGr|C~(`uoep-<1&OkmJ-z$smVGjB z>0_(h9i5yDK6oogND4w_ZfMYWT~qDoxEVjFrw>8~q}Rf^SQHLx$cYw52hO{(UK+!s z)>H_XQzuGjAI3DH8!h8$Qy*cFu~rB9H+(Ki34Y-Y$brHu61f=_aJVvqcj^&K)tD6^ zOzIRo*Au%*?;=+kQ5-oz65)nHJyl&5e_54b#fG!^vgxAvqAMQpcv_rN{p>LeO4*|z zJ$ksdEX*s-Iwt26E0r)^+Yp*fq*!BFJ{dgjp=xy@!a8FrIKPvz6uAL3oHk1cqlo#(jfFA0K;Ne z(@W@I^N@PRGNS;}pxI|%syl-B*@iYE&mkA+$F8LHIJWR(Bzy+VuTW2%BomgPibD!- zi~C*l!tU+4kW0PCely_9MBRZ@3-p5Fxbo6=;1DV$xh>8W1iy$ z(uD(I*A>dcW4Ykjr8KN~0tiw+!Od(^DSp)!gYNYC>+1oL|Mkz-em33+<`xLQmIpRmE4}KTUujc2aQ0p zj|oP#$uY#GT%w=qmIWAe)L6gQ%bWq31l2|2L5+}Vi;6(w!c*v?xjVGuks1=t z)?5;Q1=Vl#Krw81geNF?8ICB8x&jnSyUP9nF7jNLn%)1o;*Y4l5f4<5#d1AqINTf% zqXsv^<%Fg6w;r=hW-jMs|J5zCxyVUuiU-PF9fq6VIKHY7LWag*-G<1jq~}k-qid?F zqN%#b>tno>r`sBett$9PC*3OW{De9XhqZ>bBP9t}kwfX!+2bU#7TEqmRc^e6paWLk z2&t>Gek-VUT?6x7>1g3bG!}X4UYRhH;XYDQziw;vzO9E`Hd#={P_>G3{NY}uL&0xa z47Uo_V0q; z(HQ)2-#{tT^q9`_cO@vL{|mDnu1oiypi8qNwzhpVnt@z)gA2pNKf~dJsUZm zR{m*tY8;qw`N`@43Z`p|;_(QpG*C*9M$42t2IV52fyN7G7#^@AqMCGdxTwT_02ocz z3&zUOY*(Uc5oeO|Z7zAG%x~Z=4iFRaE6dA2B{o)PA14Fb!6w4>+{zqMlvBmoC2DPp zsRus=WpSozW;-xNwY1L(htTXHsLVlDnGp$g{i54P%c6d+M6}0{o#N+vY|9oncy z6DtfL3So6d^CA`NLobXorxOQp=L@vEwJj*&OpL?R#fFt*bymO(v2Q}*bVjPgk7H0{ zT7p!0gRPaWXS<{&87<0SGqVKm60Qntk@C^UEErC0My<2z;;_4%l5`e>xS1-Vcc!8_ zTk4gAx~8d=`187>t1Rse?Pn;z+c1@RQd{)_Pb=jl0~659mW;%D`E_Bks%z&6g`hlZ zz)pKd(wscKf5c5BdS+IngQy#zFKNYGaDmauVprA)#BY)a_$~zz4rcW z-Q8E)9u_S%4}um~DdMOU^>(sJ730duOo9%G%6h0{;&N-*9;GX-ver0-h`Ku1e7~py zpxaa(s<|`>KwI^6!u~P*hEZ?= z?tBKT2#ZM2BE1Q^W<(u+Eo}-r^mfDsp+1ir+9EZ_3JTfBf@NHLMSB6p^C%-<-SrIx ze72ghFay3gQ7;;ac!GMi>q-8wzXatnAR>98zY~^9QJ@3#m6O3gCzr^m+eWIKrlcMg zQto6~cMrlNYPO4%8*Ek)KgkrSzv_y{p#fk}vgq6CpC*Y{q3wA{5@{U^&7d+VUw8Ug z*%9fA(1sSfrPDug7644|=Y9QMxfmzZRXU~0E+0~{tl#R+{hS6K@;!RAzib(XMtoT> z=p~jekk;BTujbA4ivJo60NQSZ@KjcaR8k!oc7>HFOQ*LqNiZw73=>;3-CVkRRFV$@ z@nbzV8p0}6zA{sFB_0~G?o?IFC3`sz8dD2dM9n5pu4FHcIZ}J5Aihkb0`^T{wGzF=sAW*(%)m{+qi|1 zbUW7J|1dG6_dggCD_W{&<^>g(|J zfM3k{&?%J*8+Jr65L4C^;Z!;SFb0hhryE{hT;w9IBQIZ`-Q^@BshoyRVF`ulButVq zcR}Ut{v~pB#q`G0nVvNorP230XITRNdfH+MWip5 z83UN(etWoMBR#i0C+!R-I3v0fNCqB|=hzETVh2C-)QwJQ4O8V|;&TYylRJg$de&ER zq{y5uC-RV91J1|UrIHIk-oIw#i0QUZC#6$p?a_%GjiAniL6Tb!J-lALx>U@rgN|=G z0}yTNzyNUVo*sQjwthf4=a;@@em%nso4DVlxTYw zd&+Rgntp&8)hB?(msnt?IAlN8vl`$8A#Ad*%J6Ql#|+{mb2@=7S$Rn8=Tk71R! z-#2?@pz1$8lTNG%Gm=S{6jR7mw^qeQ@#revr5a%Y)+Ugn&5Cjo6D#*$aU7J)B3MpO zY=hQeIUskBaMxf#l zCC+^Gn6*;(I6R7^tX>dJX)|Sw)zxf~PI)>QX=+oqoFlkNKpo5ZR{PpKQoy3r#;8a{ zCMNiX;B!2{vB+>XsRkN zlH+ovTn>vot*1AVq6kAZT1!6#4@vEd!_-zm1`I2!6cW-XsUcHS*3m0k z=0v(INcsqsV|aGxF0|!z7E2y5eLeMHgk?c1%9lM{Bkp zl}L^fy@uSnDHMs4=0rB&J6sG$mpA^&>2mf3OCj^(sF*t;DZf-g+fR?FF7uDN(&R}9 z&XL&fK(KqdlONKK$zRWLzizsMbiARL`pV<+s3~s~1|T^D za9|hoG?EDUEmT5qIlRu^h6CWyxBz+*nDr^^HSA|YpR zU~JXXlq=WXkZhr%A#$yJh6$Q+cLxbk|HtY1lcZe(#P@fZR^{E859%d_pjG1UJFlCj zOOJB!zx5>`yi>O~632X4z~SOJ8#~MnV{@hs(4o5o*9<;`>D)ZCO>*Ow>N(rw*<62Z`f()Gme!4`X0@IK)bs=But zbeuI^J~^4J1g!0CdG*LB!Elf91$h!(piE95b$K#aWW#?5Y8s{G=ULDbf@<(%VSOy# z!VbVUXMzM~L20XWXAAsurDMf68&t?E_TZ5yij(`(T^cXYL#FW{w9(P~FiO)@HQ~?m zQ5b9g9+GXAF9Kl;b3zk|XQ(Mn&Tx3adTs+{CoNU6SoG0o*w?tO&Eu+fBk&7^vTqSV(2v)LomHLQ^_NvR9ICpg&&k z$+G%Uv<)$l9VneU8d*$jyN0eusO)k@aYSct(@tcjE3|8;VrRVKxP3j&ZPGdnu}2F_bw?T$CIf7wSnLiy`k zS1LfD41_~Z>i`2th*cNUon@04npRSrDs}?3BzG!PtkG?Ch2OB0M#T$mmd#a3ZrQcjCdfD0yB8~`SuA)j( z^Nram&7K=JvOT%@c{2)s=Ie}PD*~i&k|LU5&fB8y>$#4jR{TJ==cuDOirq?eJSFFw zyWBGCYn*vUdl^(;BtTP4Qy64I9w%HlWLa~>TdVxP` zU(`bs`@$iT9*^n7)oM7-P|8xTPd4X6m|3?`%F~^&7H=YK+)_s|E{D9V4KMvW(n(cJ zL_b%d2gIj2TqOUH*`)kIZD9hI+w$v&z|5)Q{{M{o$fW%=&!zlZcpfCQYuZjcB0rhsA?=@X09S!Y#5t;<>rcF@xySz)pJ~+PQ@_iUaVM+;rcGNO04S36^nf6OcsSIr7owYidrLs+9VRLnh7vFga;emrrf?AMef}iZjg<~Jrqc&;5{<(59AKkI@c_UTU zSB3JQstCy#y`&PNE_DkORrnMHz_CKwqH^^;p&UUt>J#*=geL?t7;g1_lQxyRh_I3T!ry8uVDm9=5C1kS~v@ex0-$W$#CJ9 zSGytiN*B=j=6X;+zbOMFdFLr7|HIGNs#yu}`!B}x+<~$Tc&3!|7<|!Zq)D@0WRJHC zP$NI9ijfxE5_a;wneWEF&x4b?vznGQ;DyWk88&$`jy=6~3aPs&pEpL&LA87w%c#NY zph4p<4^NxPYkx0cl!%jmhumfjD{yJ=+)@4*> z8`t~tk5%Qal zjzvu_1R~!1{MD{L|8#e;jb9t`$f*4c|M+_}%$&%4nf!v$SO3jd@f8^O6K*c#>Q#1| zkj}G%k1$X3)huP;RwGNg1aYn@{n+Es9c*gH3l<+f9wfN&23Vw=XIGqE}K~DF}L5347E@J>E&MDU3!nzw*3r5TE3QF zJL(9Hl1P&bf0y^~d6_Z&E?&A-DJE$?ov)}J9HnJO7Rv3(9M<}Ed{ujc0YYtO#3$b| z#It-hakmzb{d6sVRkXmG&i$8G7s|S3PC>;y`ZljUq4tuPUa-Q{XmH^l3}l)4hRr^> zL zIrua1D69Omr9pG9FiqkVB@74!&4w%PaX1Itv!$NoE2mpA=j|JS>P{C1|GEUo4Iq78 z?5b^Cj?okKDV9Z#QYF1RKVy~UpYaG`ydKGEVJ^}}`@5yQAA{`Bm~|uV-=xb~Ks{wL zt6r+YS>NyBB27y*>{@h`fmiE*b+!^R3oSg7rh35TkL0=A`9Pyw3P7xt%48?2sV&xE zi??6%NaY879A1G(iY(0Wj-LUZT3E&Fr)CO7>PaN;{Ec?`slQzuzI?SSGfApdflKZ` z$y~eDomMI3cg1%&@XP&;srRlo9CB2(w$B`7M9JXtkVE~pC=-+X6J*Z;;B=-pTek)9 z#BYkh{#cEFzBBk3wB0aC@8ejbN0dp|c^HekT@Hxh0r8V?m9BM}_+}Yf0KrhjF-gXD zVa5?Jm}f>`5M#!-2HA1lK)pU!`7WhABY#EWvK&Za_$N!);-d9@?0Fna{OS?l`THBr z=TTkYQAUw+VF|9A=aDgTdnA}#cZ!iHV;`~Pj#B{EPt9GIB)t%%xJPRxnO+KeAgbIU ze_Sx^QcN#fsnrkAeAXLS?55m@{OS+7obJXCullj5q*X?dl&pd}Gfo2I(Y=}D(f=5} zY+WAS;^B}oM>Afx4J?lyrRnme2Lg`OEkG%RANuuTp#|e%vj<vxv>*+YK)radbdH5?Lj=_S5$OVz?(_czZ? z?!v+k9OkdLG@$+o*Aenv7%Odm9vuIIY%0mS$-|j-VR&~B@!iQ2Mw*oIa%{x;S>?U& zj3iV-TJLPZXF~RIfbzpCbxm;atPzF5R}d(1tyt8y#!NOYz@Q9ZYcSx;(d<6NhdDl< z21pA#U>TnYBjK(M=XazpkZ&E9D4|aFl$*r4j%W`k#;NA;redf(Z8hh0<2b8xod!LF zf&p1`Ma;qKY`z=MI<&) z1j5bcF{7mulch!gfauPwV$T2uxzY}_nm!Ngo>qf}YHtC6*JC)gAzVSrKC)}iUcB!2|g|>Et#}&!NsXQHZ&%jL^4EPHu zEhuGfFyG}{5Zze^M}1V?<3A@rR=y|z5G{AGh65)Va3clu#J!pC22WrQbN8~%YUF9i z=(hmQiXl8#HJ6VEM_|hJ(Gx4n9L4mW(#22pB5+8ft&E}-@m)|CR%!o*qZrf2$d-v4 z09(oD&{2P?UrJ@AUrxh+AT<+NRfAZ*3pm2?p$mA<fd(_n|$@$<&;So=}%D*p7=1B zS5v#Xl5hm4hEOLHJi7W0Nc$|$iB$rV{AwlmHLnYg)eK{hUT}R}u%v9nRRhyb4TlEF z<@VD`GfTzT3LQQOaX-PmoC})vBI9_v}hlo1m_y z*Nq2W^JatX#1<9AkfkXyofp*M4 zOHTiYqbAn;5?{t`^*l{^P$vp2$zSEowH2;d<*c$>-xM+ACZ#x$3J|EJ#}`QTtL(U= zYCYTSDD>7g2aw*0}yPN?{oKayG6d&&x&x)bz^9Rpf*Q5xGc-4Y<1<{F8z zVK)zto(T^6WW#R!8Zg(abBbQO0k(nJfO5bhz%%wC<9+czV7DeOP4dZJ(@te(5oYYJ zBqOQwCuaxpREJei@ZWR!%X}MfPD%p=L93Y|EHt@P(v{YfAH+w!1&;)(x?Qa|f+pPk zxh3};hDJtMqINs8kkCk*ueyKtmC02Y=zbf<{Vv(C%cU=Ps#R!KHLWSwLZ5Iq?qIKp z>KcHcp$de~YlA2<(dA5@9p+iOuvC$2VaX=3C-BH6Rl@9!5}Y4e!Ph0*SVIl?4q1*) zi>!=+XnNdag?qm-a^;uvAk6z!L6PqR0K~~T;8fO6e4L?7KvdE=A`sNWh46!)HiZ=L81IoB2kIvr}OL`aPx;{MvRo5$*e@h`)P+W9SXuV zi5|9^q|XyhJ(1p=vZO21?6B~{-t%lTIf%9XsMLYCFe2qnijj?%J)$c9WR$X2XACFC zk%mRHl{L-2$kLyxYHA-&0Qir)7=^G6Y7hRb1jC|gabB&1e=7dun5yy{<#e@?kxuwh zZh(}lf-Li>roCfw0mOl56m6=5o8>}vVlp1pGHDYsSkjV*tRcq7D3prf21aWSHn+SY zGe1{}Rd< z0cJ9Xy`V1;|Lhyan>-u7^lbvz*(jdqzZ{m|HJRfa-WZrff6OWlje=Q@jpyjfHRrMR z{t+S#eLga2%PAo#`+wz!}eB`MQHq#@H6SAYT99#(UlNzu*_wbO}gG;4=vhqB>nZA zy0ZN*g+5;blZRBil#!pSSy~&|eF;W^GDTb2F;aU0w!|)=~1GhX`i}`9NB!2`HxTF;jxn;k-Q-wv9X!qx8&L6nsRkD z`~YZfQYldD^Gc-BdZ4$Jj#ga91xe%WMoeURAqt5)Z6lDJ()%te=+Tz5OFP0pf3)P| zJf&T~h+xVR18g@`yO*GYU~r9g3{4(qfV@&-+b;;@ zzA(aHZp@PBfF6H`a7Ok>{kV@436nx^XVM+?I)u1x?( zQ*j$ad-%(w3IL}5eD)Ss&hW~W`Rx3{B7>fUD72ELeEEUdaOQGEjLUB!kY2`qc)|iR zbknUG`c8)|BT@eHft5JPZ$ztIzxq z`FP|hsQ-951T+6MBE2NCKsMiCzVE+b*s&v_;huR}^5eO<2m-hYLns*ys=?~1^b}vJ zsXFY7w}M%Vs^VTbM9CTPgz($m_ynm-KL_HpiOe7ZJy?F(Y1pLwRzUsM@6d@yHI;az zEQa2vK5DV&kpk2RuX*4tpq&SHoB z_#MDIa1%OubebOzmI92`s`!KdQCFgDq(89awBqPr+=6H19Rp}7$Pmd2WyY6{jZ)cn zn3Zk%iZ{-_1{MilgGv)KU?d-^6RFjce~u3aMOJKOYko^1m@`XxV@(T2Zbc-h)c=fk zhliV0aiySY(pIU2MKyTCU+0ZR&v@?FA&guhk=LfEw3xH1cfqpH_}GCKHe$~OcV=FO z&ioZlH1q{ZU;M+G|5cuuGMzw&Y(1@k1dd_k@#8tdbDcPy2vY3iuj&A3W;%OYeHnls zukLbdk5gi#+%wkwud+z@T6v#l_Ev&i2v~NiB=GxVPz8q#H-bb{y6dn?Ebr81Wcdeo z+5N)xyw>dxzU#jRD7|ROBXz5=lv5X69$Ehwv*1%bB1CrM%qIpZ(Jc0G-UwXHb_f1o zw2@oKfym~$%<<0w7}4R2EN|ZLh*}rH=Q2GUwe+|u)Ky^v=$o~Q*S0j|*hU^>@*zqZ zZa571#y-W#7)Zo`9DEOmj9Y1Tn|d#xa+iE?mE*pPhDqoJdDM?4LanEZaVfT9rDm#n z;c+F$qux5~jc--|>yuYu9DfkvA=%0tzA%Ls{ugN#ZxnOaAH(8jE1T`^84Ry=WKhe$ zQSAQC8cz4s2R_G5+UaJjYW6$KSf0M+a{F`W^uQ~Ajy>WsN#z?LV_1BIc)7PFZYoHH z#8WhUXD^2Du^*H;dS%=#7Ph3gRNcF?>nudkDgw2%Z;RdXQ!LQ<=Y6=)(@Zx$Q$RIckG4_9 z+9HkL=bZBw83}UIAG8h_&&O~7b=g=lmW=M_ic*Piff8~D${s7TfWjHNJ6v`)e@b*N;$r5%>^!JR;dZI^~3;?v%D}r1@fJE4KeYj3eo<~hZ9-v zLY(~e#>kf+scb|g<)A?F4QOA^J+e){`-}zuMF0s=1#XL^Tu^{%1SmJGl4Mv%O6 z1H1m8vGqlIP8C!W#Msv+zDwB3?)v2N+<}Hhu9Tezp6j9?CiRPv3;Q)WH?yW^AN?MS7@&M%VH8|Nm z7dhFPq38;qsjG#GtpMINRbOk8>g=*+F(lj5#Ol7pd!}6rqqN#k7+D*2zp~^3)J3RZ z6WOVOmDR?v;MTqekcjvx?4v#Ow+khORZ>{lVT zRprM@MuK!#9>R5fcs_kABSj)Mar?9d(1}*n~vcacZRq(8l0m)7| z%g2;EFgs%P8%Ti_OxS)19=m80r`zqk;g($=1DFM2kici%f$X7FBTa6;W!S{kgOA;B zva2q&U0iRfvWBPcLGC+97eX)hG?a^WEm}Y~lBZ^RXPw54%i%0osH5p@_HBxA0+aKU^Dw0*zh z1ZSywJ|F10+_ep1J@ZxHxEz%VvHOq#^|XPk!rOs;GZwLf>PoI{{R{qE^%WeyMV=EI zD#0X;2mZ$@iTU|A(=MUqz{dmrQP`F_b+y2}Qu%Epl2%u-ajTv~TwV2|R%lO2EhX;% zNILg8nd|S5f6`Gi>@e1CyxqoSnGcqASq6*2Vn0~cCB`L<`%Wwq%1SaWF-yxvW!$n- zTGg^CwYqM)ShU$nwTc?LEn02qTiZ>)=eyrO`(StG^Lby+d7b+?KU?&&xz48lB-K~O zea7)Uuo-x2-5svCYfEjC?xwH1Gs=#WRuUzAd3M9*$dE@G!awZ~^JDQ)u5RB?_*I%@ zflEKKmS*WO+uOJ5?3O8`0qbpK1d!ft>m&dCjUInd<(cyF3jQ^i0RVETKHs=kMLHR? zsw$=HH2CE1B7SvIPxlE!Oe~JR<+c9c`qbRoWQh)S1mvPt)ZWUq;mj#$CI|eSbmzOx zOcG+N<47SS!HRKa1Xn&|>)&Wk_R%(w;4Nj<(tQ^9^Kw7aiRN1qFGqtg!X`z|`$Xk6 zr@S458@BkZ5@^uYRqlBY8k%s!v`G=2eWZOxC_J`cs!fG$cm&{6O(hGth zUJ;N&7au9?2SKc0NHyUB*yP{G*?;X<`AA@?RU$WEg*LP9!GK7u%9Kk}`2CIwvqZ+~ zAq)xRe@Wki2#j5I;riq#RglslmP+UrKm%dLe$K^b&(zQji!-Y=#eS60GmHL!elI1Nph%=j}F&yiEalF z9hvt5Ccd3KW4+A^={ZE*jStE)bk7VoGe%i2+I2*BJZh#%#ws}DS#l3h=O}eJ`znI$ zyD;CMxSa(SkLMx7f^7IHRnSfM-w43vOo7}qiF^KI6eoGSD*XS5me!AV?NXtx^5#{r zGWD9%omStanm62ar&&ZgWC!z8DweKN2wRnwT z+$Zm}qt&Xk_VUmm(<$#N4l(HnzoS8HlAP(zaaOBXe+jG0LjCl0w@dZ_mbY{ro8PTR|2(Vx*UPkGfB7J*SgT9oj}~S~ofEA1 zauw&>aWPAbpBjuw>i}ChS&U>@tRtX@CtF@p5XnV8L>;KijN>f+iLl4LwC90^OY2In z{(NKn60Y^^ICj?lCrF(DQ@K1g#`MUYor2lqNmq8nTNiMMMPnIRIG9bnqD^$K^7fMn zGkJM~Pr)!!);(Nvab0kwm`-C}IP{Mij^^EDHi;{#r<8_))~CzzCH+=Q&&{A|-=*|S zY48kC{9%~~$%2b*45@D1eUU0jBz4}_L}E9sG>MM0 zZ1UqLeB&|Y>x~&Tb4n-MwP(Q?Vrk;sTS_kc!jU2MAWJy3mhVy28t z<91?Q`9$AdoW|Stz%Nhe<``dXWpycaxTpy)@Y;6D?UDEh|9z%0n#%-S0>+p!-J2ob zWlXOMNK!g0B(*JH>V3kN>e9bd3LfGdcapV7EVbN})-Up)yD5~AuBho6RvXF+9eEq7 zXRy7WXrr>DIz`&8F~bm6MHBf#%lWrd1$^Arbjzr>psv9~EWZpZM06cp&4r(Pnm0c6 zW=@5qodPD_)EM_I&2%H0)TBHdmat>dzaRJ4V}Z35PQ&oe`kQwn+4 zJpr|lY>_Bi%D|s*HUrq5%5=42aT^MkGCOkZ;l?uO0;)xaUrev;S!NcA=QUX7sY0%U zfdzG?Hgi6t$ImdzO3FJVX$_ol@&*V!^bhvk>jL8USyjc9-q>ny&}cJb+gIT z0*KPF!Wz?yqp8NBbCny)s`*@8t)YClAthF_U?QA<>!a{plQ2%Hjq{gqxXUf~{S?EpdZa5HAzp@`s`m3*roct5QeJ-&?dPG@l zma0%vFA+j@yaJ)$sfqW!dNBX+qip-kZ~S=6hPCu~ALwA_(^d_X=SmgxI^U)7s&20s z>Mtj?bM=whoZzU1(AG~Xtr4*6BxPkf*LCp@OMB)#ZRF`xli35WK|^&$at{G!*vc%_ zF&2s~^2T;{5Izr?e)ce{9g|}gfg^d+xE?gz(UPam(gLC!#*^&{uAxLkoyF^z# zMq1-Rh;#thUCm{@ZpO4z&+d3Oo#VMbXRM$jP+WW`aRH8}GM*#L9Av~!b+j2T{lZ6< z(YaebO9cbQ?t@g!L^EHubwae9oCV@o%thL%3l@mz@3I!AcM|9J=gUR`R`Bj!bi4n zp&7X~GUGh&`&?yrlC6*NkZU@odPOD0qh)0~(O9#G$@vQ?&vdX%_tY=Nw=-E*oC;HnObi&m!$6On7-X{R7#8WtQk%p z!8e;Ckj)qrJY+pg*Z45R^I%toyaRzm*f~Cj9Q!+qufe$tBUHYAMFhbVDXVY?PM~MT)7IC4(lKm2|0% zc9Os%h-Zwd9D?gaD;};U(tk3l>MY$ggI=I|zF9_R$vg?!ZdK0FVrUhv`vSII_ZkB8 z5j`-appb50T94^|5{Z9Kw}X3p3RgF414x*q=TVGm6&qqSvLbR~2M-#G1#FvZuRrXH zZnMxwQuh6x4qb1lU%PcDl1L8z660RQJH$GK-k-I?37<0Fcud?z%YG-4~AfzacZ&}sRla@;skUwoaz;W$V zd9CHiK?vhx??5+$RY6Pnu)h`I-09wb`W}w9gqgzBK!U)fwV-3B5Ot&UiGC@k92uG| z+k=?d&5a-II3(9S0x(7Q^7?;2PBRm>U6KHd1bngOttcWYA1LADY<`x`8(fN>B)>;%ueKS|0sQWL z6%xT6*od%*H(6G%;AH<|Vkk9S(LVA8)$hiKS$-;2VA+|<>5~}`!E(ctudng4gE838 zb0DX7LgMrW9Y6@CVyyg>!}T&uj%F`!o5ml_`_1K+wEy`jvL}(fX>P zgi9^$8ML*O^}VfZ)?bq$`|6r8Sc$d_ClbS2<7r4eT+LX{sA#nDdb*yGL}^-IJ#_+2 zFqw0z)ZGqCJw}d$UItZK84C21wih_uRA%l{zekJWnOaCr$-=wPm=~=s%cjiUI~rub z13#d!gbOKeCSzxs6z!Jz2wu~2gA$-N@^yzGSu6*0@ZFE+76x?%Epi?=opR$oh>RlH zc$uW-!xb$@w(GVlBbQr6@=7&_P2Vy*k8-BUqm8(%i@J{S9=^enL#;`fsL@?aoeY`Q zP2O6vcOFm?k4;5M>9FJIfvd(KTawONMTOk@ruJ4I(ck=m^8H`DkcmKbjR{#kvEX+^;ob}>^geemY6AC4g?Bn2C$dD~R>F%J5LL6>V@fsXKHU#h&+V4GbF1>t|jx@A+K6tEU*HHF`%rH<t-lHsmWk*qyD-6_0Vr;gf^i(EX=`mjd!P4d2VM`n zjgtH{6!eVBZ$Sts+*1C0mBZO{!c3!Lw74z8qq2p3`<TRlDLhSN+#2)0nFjUH>-}|P z_&!vz3av{z$FRrloe;a>i|l6+9KvB$GS#9lylXIiMHjOtY7d`Z+C9K&)zfXfQkbxa z_UW-1y$c!FqB17ROYPyL4mzX#)a{7M>-$aJLOiXpu9qJoYmV3ha!*4*{HUHF+g>FN z;+f%eV^LW|NB;YOzzop4OV6CYRk{0u|6#XGB-IuXUeH!tNoKZOZEJG7Z^GES`opYk zRlo+Ghbg@Hg6nb*Lq_kQKvSTY$%vT$>U*2XkRBG%3P%gITnX>z1;3xApPN+EbMU|Y}xle%V2dtPxIOOulp>o)?oW%VpXKb zo#R=+;N|?t+sBV>C!n`qD$QbU(AgBDl}6KdwdFv3Dx)pX8u9~|FEkLxK$iSaZBq{c zj^2UsMF;9vRZ&lZ&Y%#&Aql+7I*3Uy zm2X5ch>esaino6-O{V_J*NUCc)pgx4>Lc9f< zj;LHuIOUSicc>m{yDen_u1frO&;@RU>BMy^oj_i77aa5@bU$E>lFVshrb*84aCNQU zxWA89qqy}ncL*+cdJAH@PQjYX?jkSQFua&|z&!Z_Fc2By14*B%=^!8M0sQJ^d8yH^ zo*^%V?K5RfidiaKbdE=-p^(_(T5JyqYRKs6ocZa0ETY^kkhXN@Y>xla(^ZOFn;xk> z1Iaj%lv1<^KYp;+v6n7X`EV8&$l-GU(jnbQGGKpy;cYG0~Oq=XwNzl{r{uoMAt&tzj!Gu5yYQ={^*@@I>7X#?HwwXF6ATQgyG8=E%XC zJLPWOIz3JsAnRY|P@<~XCUa8kmGbrk-t$K`59zAXn#-T~B!~^W1Si?0+gF&hNzcG3@)U!V4{k8A;&~gs-#1&Te-|_aB zd&5w{K0RKv?S3%#c^#F*LZ{(3DgpMp-z!_XBl=#5v6bP8G~&=B=}6aQEKXt(#*~NDGzmEwZZcEqn1ww_BGeY8$Ds+g(`oh zTzemS$W4WB9{I(}$JQiCsorhe!mb0{`)u8uwYZ$W9vfmZ)%!7vM4=3J@u6-b`sVkj zaG#QqhQ&)mt1YVBEiLOqk@IzcoH7(kHayPbOQv!bzk2K?)XEo410hN~;hz^u{U zewqgj0m9y8qPUES(*mHk6+xBFXaXktnoG>XExxVtn@jmx%PVDnaWFU2_{co%*sSki zDGPL$^G94(C9W={z=dzwXip&i??k7utX1A5qXJUcr3Vi@ z(Ugb%RtJK<>%N>!AUzUie41E_MOi%6m{C0gUhZ2+?qD z7Y}BLV@tWUvy~*HW@)YOPIm7|%tR4%rWy|=M$qVrz^}8M7R|Emko?PB+_t;m9H!LN z)6*B7@)>DOM9k7~zpL1t^4?UowKti=n@|58$OUVakw2|*UN9D&Mp=Xt`k@lOr^212 zJ!6?<;+9}k#(%;?m*V-79G%@prO`v=Y;w0k*>lyCon!+*BC%1>*dM7l4slHmn}BMI20LG#KLW{4)Ujf8`Bt(z-y|7t zq?LzJ?dw3jOTO4%=E2LArwi0T*DdrXh$2pp5i2VJ_KAV zJtF5up?aPxG3nl+(&igK<@)54Jsi;;^Cz&ZmQgMP@7}p`1(M-zB%UI0_bV~0xt!TPGAm-HSD2%P-h%H;TrL{y^ic&WV{<>t#}Xksn& z)Pza8quAFf&=56}(%{d+oaz2*GYPE;PvyADNF$0*(&WlX-aSHVsTouvAYdxmaErD? zUJ2ce+X`=8SDpdTSakq_YwI9uygcd&j4otQaet{l)+E*_{BG6(7-r6ZARQy~Fr@md zg$Hj_ZAPHn;$vB+8_D9Ut}nLckk)fFDMH7f1 zGDpkZjBJqx9g!o~HwQg9dP*pPqZ^Nef3m+ViCZD7^EcoMx@L8gu4~XycK*Vty!;Q8 z>L{>kq(rCjf1L!co>8uO(%};5dHq%>B2yb}bMCN8jVBX}br6*#Dcw+l4muGTL-#eA zI|#d&nVaq8qRy^g@Gk#l?u1X0|6(gsRB3-~1h~(i%(=buguN0{^>k(3Xip#Mx~JWx z2;WW5@U{BN)=_?m2L=y(grt5#CH0g^bXu0NsVwE$ssb3n*&p1>ZijwRV2b^2*8a=A zRsp@@3Hp2*L&zr=AlFXlLf*qqvw%NzpCNiO`D|nB+nN6N*)!12Pk->|A9iu*1aBMQ zLG&z?OebJL+tRvlc9e9S>v%w`$D`ijQ)#NR|8z~ijXJ0fe6FDi4U_M$p*HWk+swfJ z$!LK2ZMe|U(}u`vH8B4{ssrSqLeo#+tER3@-(Y&>Eh;CWCvO`5rbvp+rNYXnNH3T2 zWLWEp3fzOKYe zk`L1%67344Sio=x`Q`BhTzYGVaoKlwWyx=388|*_n9tpVrxOI}D(BBayTz)&F15RG z7NOj1e6!w`m^&HG-P61}L8`~X2?@{Gn;GZJ^r+J6 zR;1_Wca@&m@c#WeZ9D6KAm*0D6651CuQXlAwY!vSfcDsSl^3cILF<&KFO+T@0cyCK zL$fqUuU5sAfjw^*+4@G%C+~j6RZiT=BTJNLwoLm2qFAY_r%BEOheF^%Q3F}^u^E-X zOvv@a22k{ZrI4We2FP$10cLsQDs1-QgkS&#p66v8fFSlLUl%;i%9Q% z^Z6y*2Er>%Za%;@Eyh73%@1-f$5m*c{5b>5j-eecdG)FTB0pkcac&dgs2u+fR zGLE1gzuhXwkjf^O6Ez}x4pX)W47TNxT4%sAd+dDTA)U*X?*^GxjIjH}vlM)7unvss zs(hAlJ<+A1W3ps5;8cImbMZbYtPtxew_vGIJDKp3S%PiOXr}8rW8@a?HfyL;avS`{ z<)78#nm~xIc!*+S9}%38KgT0Q^dOrlqjE%d#!q?EPU4TVZ`&^4r*pIxrzCP)|s|IuC|8p8Ri>r zWTjJCeg_Y(I?B5n(wxp1=qQmXMcmt*GpwW@83THH%q&ib@HKrT8^d>L@QTB<5&9qo z{-lmAwQCDm*V0Az%;{`q)Xgf89=f)_+ZfX+r^vxYVETJufmJG5FqqF?pg57>dc`2= za>%TZ2N;^p)2UP7&DKsfPvgW7Zsy1SVz?A_q9;rkjyKF)xBTjbzJBNx^z-{_ zyUd-yGmm~|mck544UO8lPT9SY`#b#+c$WACV&jCKgEdk&@0|J+=W`I^5Lym3&KV$ikXDqiXhzZK`Rh46vW{K4bdB}D(7?(yDEDjsxsP3_t&g) z=~xZxk(tca-hl+upFrpIs>ni_|1}hiBfF1m{1Y|EiT6-q@ld4I4*%LVl{N|TscfSA z1`d|5#+W|YycCYTsND6@8&C?m)Bus?%B^Q<`t-aLcQS6d?2a<%8vrg_<+tR&)fS|Z zsU<^6d%5uh+Yh4t{Iv;*Mmmxmfy*d`T122`$xnhA zJFCC~iA)8%i?ma(F5ks6$eY1cJ15CyM-bCBw-WK~PC;q+Ps6YbnNAaoo>`T$-)4qrp66>NWk0iKW&-Qv9LCpQ%e6^je=MTf zze10+ql<%{N}B?D6-{N;-DaL~pfHR~D4&6#a!JihuHo~umJ36l@euTOu7gos>XA2Q z1@lnfGJ4DVJ_LE4Wf1X!cc6>2D$-7R&kn}KJuN4SF;meK;G^MpH&xS0W>C}uv%1Dh z;7ypPy#w02p5v%QqS0fGtLnWsR0FbLnb$&y@E9pN&G}BL@#a5`y7U+)% zFpE4Qpun9hJue>0kbxhgVjs}GBCnR|%0HtU{+>Vr5wOzwviJq_k_KC7N#Y=|_ z@=VhPz|TeXwH0BbLNG`P_Jc4_3tjwvs4#+yykZDW*UBa`yDzjv!@q%&R0X<_K+|2f z4hgGny-ED42a5i)$1H-UoMVmAWlovZog?|;1IXiR74^$)`{BAjKIMRaQAMLL1Y-Sg z@D%lzuNnjXKPvA8`F99Xva11A(d^kN=X8Ikk2Zj%pS%IkUww!7Gmt1vCLhOEenMyQ zipm>?Hsrov_WTa>7w9?K(^hiC=T+$lqj|bdT+}QIy@YU^pF;yFNP+q;Q9cT(A*TZ11g=Gn?1I;$q|2^Xs=x@&1HZ!~bW{3IO&3+hla6GV0r~hC&ORi{IO*5pzm0m*a!-o! znS+ekD&9%*1b)SFfDxT+21(5&Q~TSC7o+Hww4^GKN$20(YE=XDq3JTc0epC52_L3k zY;zUb)u|zh2N|xZYHOWhh@!lHkkD0=R{G>}wRA z(Nd|PwRla0<{PR;$#3r>QHCooiMBl;9P1R<%UqNO^7$2q-^j4Qzz?D4FMj6uF01f= zLv&N*QF`%EP!V%#CD@@O>UMc|JlOjh>7ZO&L31`#Fa$p%9lYNxG{ds$2?!#hiihwu z8@;}$VCSMUCrKrCid2Nz5dbm4%K=rqmAQb;$j+9zi!VQUI2sIy|a!u~W`hJTaTt452M(3z~t5z4p9 zI8nt|NLAj2=M;}X8WxZpeuG1~`Y9y1UDs_fvTY1V6kU$dlID@;bgs&7m9#)soa7-N zS5<)RRQ}KVz^s(U0W(S7m<_#zRe=lWK}Ttp7&6XeWg_M)R?2(nmS)^&2f!?(lckI? zviE)@ODaAH0+87qQD_-~_gfZ`lUk4h0h=WhixtD7dDCR}tvo+@OiqTJBUebiY-q-~ zk)+Nj?=cVFK-#yIe!?5taBLA(GDK`3y-fXqAK$)j+T@cVAoXTuC$WZ_6kV3+U~(&& zf}3uKg%;{;tmetAhJk4vP!Xe@j8$2bUIqaR*G_gzkL671-2i-=*1HKgo-0~&7~kt@ zU(J+p7l{tlewu7^a0Txb!uB^wth|?Wx{*K%#<&1CyAGu zC^@wNFp56Mu0omYo7v}1d0Hoau4l*&=0~)bc;kaQ$;!J}_Uk-HI5~{?&X|nsNNWu` z=Fj7@l3K`%|FD8~Oq!FQZei!WBkYaAP#Z9mDrOn$fTV@jprLtDdy@K{x~euRdp^?J zrDrh~D`J9{?y#TXCJ5-UaTeGmt+khu`95v}Y6Ypr8AmyPEQvGR=@F5moJUoJo3U5$ zx~rRm`SKi14lvG2+sfw?5G=J-goZtZIZn{i)&zp7Tqe;sdE`!2G+W;P5#e}3kBas7xj>n zdcp{0L_IhOdLqra)9w=MUCS+%Hq^sM+rwNdMrxZJv^#A zbKC*Dj4nD~C{Jqt026+%T+^|0tLp;WC4K~dbhZ540F7^tuCHfq_^r+eq3zWn~hTiF%E*xqj)~-71k$pMdE*RA`Rw1nHB}Z}6>K^c1X+@;Y&N z7i*uB;g{cj0xcNE9k27`dP~D1hs*dKteg1_485C}DBzv%km!1=7!(cLWc)#>+f=2= zQeDqVVzG?zsJ+N^>M7F2GJg{Aj3%ygq_Y!XEc}v}RoWPdZ;A-*L)?N;mn%-r+jEMg z=>`xeMLQo7^<>Dj^r+mEY*BLg9%nR%pavo&Z@g5uVHJ~dqufeqCwX_+L(kZZy6Fg* z66I_zPdxzuQ(-?1pJdS>v>}Fr=Qics+9!Is*0d>dyN9>_Q&=GD&)Plm(N7>}Y$7M# zRb>Pa3by&21XD0-mZN}%JDB;hQ`fLX?>D3J7+rp)q0b_*Z@Aq_Qe}hdWQ8-k3sM0yI@ncJCq>PzBm^I{U1&!qO36^dJ zmFAFX`+4%J+!{%J+H&d=p$cRu!vfbRTYK;&R2NY5M2sGsPOK$2v-SoDR-^}85{+#n zPhYqEIvz5Ye%9Gun4oepnznnGtoY3g6P}Kx$Wra%jHvP<_*C;d;Qgx@grVuQX}`Qv z`$d|_^LvAN5;xz}egSVc`6U6ip@6cLJZyqUOm>f#g>`T}6&*1Vi{m+!&*!p?fqEDu z24MsQs;(#LUqPT_KR`I2rIQH_Pc9j+6qd{7DJH#&mEndc(%MSR2ljxdH~D%>VL8lM z;E*8+SE`M_94JcN2rp1X9^zt%)ZOfBWy`XE(Yi0%tWxPammd$k&E1Vt#i`QxaVYNd zBiz5K0y)N?g*7CLSViLh79u)$!gOga>TRsZHKcx5o>BH9l5M!o5om34HmxW$Y{{h_bj>_mEFZbkyUrqsp-r8^aG4ODGzs)c{ zX}gE>nyekPm-m=1d76F%h%fIT8SVikOI7hGoH<4#i98N_jOhh=r7i;Q2ZlJv`rklz z@p}P{Xe#GVACyGh?=*_9r<8iu{(6rE*q%>Li0p3?C%IOU*<4w55x z0cG=bJBEVqP!i751`hfN1SI(rJN-h?s-?(o`2@x(=Jebt@*$%v~&Q1 z`5P%H)!A*(F!S`}uxyX4cXHzg4?(S6=%s`4!Pu%vD(nZm)8@+U^GrAHC3g#%`56m% zAK{2G@3YCM`UsD|9^t5_SFy=fOZjVGWyGPj`p3)Q)m+D_ecbiBTaZIob`IHb1|8*x z4-%1ljmca-0o?VqGE|`!{@(kMLOvU26zO@s_-iJI(aI>efyR7A*ppg55GfCm<_9N1D)#F3SBSKMIKG%4{xdPaIM^B zBajLn!R=X{Bpq5>`9?-ni44AkJge#i`OG=O<>hVXra#uv$T_O!5vkt@KD;B{FQ(n3 zTjqmSKt3JB7fRBvlp;2&N6O7+SfQ`cev-vTo^vO!(r2a;J&)}FI(z_J4ZL4b5Q4nfo zGTES5u9urWAd4Xo&r>%|Kq{d9cbM*zm3xc4(GGyz@;55_pDMaZ?%B`542TcXo+q8Q zXusn)`+6qSejD10JT?mkqk>{H=pX1r@TsPqWN3G5N9FE8?^s^_rOrZl^D@8d%rH(7 z?OST6Rs2U@LPREvgFl|V1hIXmY@@kKmwuFUgHB(ara^zK6eq$T2g}?9v@qk(C}W|V zBPjsPV@_4?4wepOVtRjzw?wJyyF zUU^@4R~V^Edti$ZdF!gO`)H$c8mx|LeM47TJ(32nZM&a~Z#V|NdG=ob`&Hf6=)qg~ zU6&6K46O%QHJCv8?Q}FCM=~5hOhdpmas#OF6>67JcqEuj7JG38T5`Lu?J=<$*~i*Puw5sDr# z3rca&6bJnYWOS&!Ip`PRB?v{MfoypK%8gsanmVi4M&lJnSUQ!kry?3$;SBQjgl|h{ z>uC$sGI9y$QLD_xH`AISTSCLp{5>9N^aPiGkE%(K!*wBuO4@^92l|FdW2|CvTt{I1 zt>Xx3O$B(`C+=R#-=z#LZOIb&U$519UVswR;M;V9(2gG$*ET6 zJ#w?5nNP?p>f{Ir3qJmFPKm4=k2{LQgC!uf(=T-}a^$)D?3L2&N4#s>m9v4=@~tGA@%Tv_8jSNS@HtYQ)~Rmywop zr}Ec^I&FB)WGf;qi4e+1k8uPED#?a&OKDZg*AUqjRXJV~AU7SE2+4frC*XmoMhm~q z!q+_j>~d(A%`0tCSo;^20FKOxY?xau_v*DLUSW^S(GH2{B-Bx&D)*ARRPJSA?0;a!Ox*)oF6&+ogg~z9NgHpUh>=b5XizQ{9%H2`hQJqU7r36 zCVSRFrd=P{1_yhhk(KC|55DBr8Ol3A=TL6oU8T$MWTWY&(Go>2oaZ)f8Da$xvw4HT z+k!+{)hEZM1zS^Ds1x%33qPDLtHhji_A%z=Rv2{>YedyY`i<9Vi)Tsd#rSazm1&<@ zmE2P4V@SIE;V2GiBf)ZaL(Z&pH2(4or}1ZcRLbHkx3(bK5z~t{gsuIFxB@6;cRdDi zUQnTjWzySRRf6*G02DJ)bl1B&&_{MR)sX_CR*=V^D%&L~^Mf8)+YExBC2~DutAnoA z!>ke=?AZmRx%O?E~eRZlsn40Pehjm^S&1 z034skAy7>5!qGAaJldc%NtZi zLU0FrB@)_Amb;1sr06CZ>sBql?^H$`ZYm?gxLg>L5xp%s&#LS(k~0n>+P(lS_*w&u zw#NqWhB_z_ST*kYk~G0A;?<>#j22ZT&{f)4Ie5*YEocQxTA{Kt<=b}XB>i>uX!fV* z6*_B7DlR|)PbuF+x~aaC9FUs3C9W-M>R*Y-v9FbPCDO;UL)v|0xe&Onc!NO!j5bZ=KRi=ln;50?A`A&pj< zICZcgiRusu=U&%BtmLndvu1=T^cM(O&39 z)i%fmO`0tE2~wmh35sH=&V)24m`$uR*|P zW&w0*U8H4}-6wbJ7;m0(uaS?Y+bd+w`JfNs>Dg{v&vMGoM>z2U71EGxMFTYmV2gBPbla=^Nn(2|m@RiShL&8aX0?ooT2biJAPY~$wDTapuDmv!lYLlEq$4lZ3#F2} z(LojGX(jbO>E4~Yyl!7?GtPW#w<8j2Qg!^F{tRB^8s=O}d<5=}dYnm2drD@b9DCM~ zv;rEo)j6S+Zb&j&d6vp9GpAfvORd6RQ`1M>oh&!!lHVFPo;My3p0QUptcM66#$U|# z^Bw^PNrJ?(NG85PHT0Lgz$K6Jb(19RQYlmJ-mD;h5F*^01EAFOmiY}#-FhpG=9gP8 zJ4bNq=cf@d_;UlGQmvdGxqLsb_(MhS1#{^Nx=Uqe{g>m%bi64fX84jczK%fOO=>T} zFi6&GgXuBeNi65!^T)_E>KtXqMD&~5Dt|l(?temlYsvXOf=dhMXkfC8h{v#aX)rp% z1G<6m%DV9RCGwtGVMQiJQGwQ>uP`nxGpKtVL@gP*3-a%)+-^D30vR_-JDA^jmlKRv zvE@hvI$V!k1Nhyj3IHeWSsXcYSI?<5n@oAl3}}x?2iV6@NN~?!q^oxeEAwRXFCzh; z0EJ6W_k%j$)b8YawGZc&R;6u?g=UfMhfOl~1@xWb`G|_h0qHf&3`hgGPR7j&(%|M+ zSmu*2K#+co?JlX^!Su_UsBPlIDJ3tap$A!!+R!i!>7>`qe z)5cx$(wnGsx9$$=h?^@)bz6n;T2+B@cbT7{c0qTTJ@O#9WpvQSuM4S`-%O_{4-m)Y z9A>HKprAESt{vC~m5X5eq%OvWbPt;@7X5>F9rkhI%qDhCpTfvaV?eKUE;S}WYrFX1OC<- zk3BA%`amWc zyMk(QpYpYbvZL|ZMzmo;GYt4P9Xxq>A8v_lA48ohTU%jW=-EWFenaz~-O}{2NTa#@ zRSZ3kX=s+p_D|7}7pqWH+4s5WmWJ92e0wN|5>fso96l*Hc-a!<7P}QQjuqy~+kc~K z`juxITk$KI??rfR=*)e8a@o{wT>lv0i=6(vk#;wZq-+kqCF+?^mBPIY*VM@<_sKie zEWq0eG`prV-rnyG(=Q^^6?a1TN%?s{)N!M4uoys28zm1%kof2I>~eZQOc&(9GtM|ZLFLXNHma5%2VujT*@wD3uh2ixQLT>gj|A?}Ed!&K01@@f-U zCSOILV}iMEUPZ6H2UVU|9!N9`Agj{-a^@UL+GEqNGb{5QgRHCr_x{vOCfb& zm5RkdFyW_+_PJq6?!sA|9FJjgUw^bAO~^nMw)4iDbe(?DbkWosde7QjlgRcJBc zu)rvMdET*d{6jE$vo4;j_p@cl?Ao|o%od=GTiJHV?jwk*v8s487B&{m=8u37Y0LAD zshT+UZEFM*m2Z(>{)K06EHFdT?*r)PMiTfy4OXHo(uMn1Re6v)o60(v1kQAzrlOC0 zd4a3XX<_P9$*zMtnL&wUc+UsPa5OBZ|AQoVV)cKn09$BR=R@+i)*JPzqQ>osUU^3M z8~<5lHj?k|L1j3qe4{woyhLcIxR3N{X8AE;ya{r?gmcf+zC9d#EzuIGG9RF`4LZFZ zEJ7Q!>DE7a)8(^EW+`S$`AXE}jNS-WYP%T1T@UTr*+%wmu%{vS{T%=ln;rci+8lvL zv&dK9Hp_AMGLZM>GFo?^w-@mc+du%IZ9TkvjisUv?&;47x{h(+=RkIg*JJt$nJq42pr!1ffX>KiGDqQ(JOYKUgp2a z+kaqGIGASil6(4C<-96?977a6;DUFIwo{tQsgCgGtsTv337f#tcI8jgh}T4WKRatB zyAD#7Oz_KI2zAo!2>AsnvI8kyF~_Hmk;1sndrqsr*%1 z(^?_1UKS;#qw#)y`!&wR_1PTQWL27js+PSQCN8-b9j;&`mr`WvsK#2Lp!z|aR`|hY zvkb{$;vqu2#$rK5J-8V%Rdy8G_aiWDnL6WCCccCmucvG;%TEu&*Wc(l)(`16`&*Uo zKIFP{vFyngN6x%VgENNMDBbRLk&-1SC&Vw7(V?oUGao8my!dC+jRW zW#TG}E@QpS3VA&NTo|k}>Y(s&2Veq2H&?|a$hf2F!?*AQDCYU%A=VNAq4nSpbvDw{>Bi;Q*QyAXVrHH>f z;N01&VuEc*n$$jJtjjHs8&j>Ql)i)L+)aZ}X^bCThR-lVdhK(J#kty7L>s6X0XUyi z^=QCojNRjzicwJdGWv7cHV_XAyNePMh=yFF_jS|1_o9{yTL8{Kp$ZN`js@+Yh)6^5 zx#Bq^nCO(gec`A=6?h)wyygj_;MsQAl4{@cCRlUxOVcH*bZzn$JdrqlV@)7z7p-we zLFY46Ns@SU-#66kp;CR$DuypJH*ng-FqlI-OWy5x$VXT|vWgFT+sTZU!7%QHXgwq= zfuEbCH=q)g`Amk$}wQcW}4}u)|dgaST1@Nzyvb#Xa zJKA8P&NlgxdzkC$tTH=*EQFCio(r8FX=X*_@MTutSA~|y1-krjHDvY;)RmXsbwK)v zAAnWzp8rcQY;3g4b;H)BJu#Mv;Kr+>IN&MVizSnSwcQ%_ieT>BrUD6p)!06atEP8S z9jxNmaI_ZB5P6=NPWV5PW*QZaQkkSnpMTf>{l`>!H(GU@UYV`?n%Chrl}5qc9Q<@^4EV=(gsk<+ZqkfxlcxDOP|D`EGV&LrL4US3;vRhw4?*=5II4J zQ{Fg(5ZI+$8(CC!f2g=prvx%noVtL<2%l0fDvG}jtd`#mx*_y*QM>*YYkG*%S`arf z$phc0ig^;b@-AtcMhyjDwtCCOzP>L>4bS|SlFnwALTuTth=xdBm zXK3Pat}n?^0j*?bBXnp-@qj8%mjhowE3o@0Hvrq`S8;{cRsJ3F+i9~{_WlhKUQ=m3Se|Q@{G@9bE-2?j>9fuB zaBRgCYO3&bZI3aTVIwGW(BVO?$!;$JPf%+fHD2y5FiBedb;YSo#~E^BJ#hH1D$l|T zTKpQ{&z>Qtud=Cswe7iA#X>+@xQqNb3auwyl^oQHa6h?Ax8;wih#MD2XritrXTX_m z>-X}zP=u)y@tg;t#zCsGgFNX^cj``8C)%0X6bXCv` zO5)q7-?&Ll1MoOck?TRo=So!!@m$$ThGzk@_iKH7zDl}Bsz~w1zl^K!jqmtZZxu6* zy=5W!WNwZJR<6EZ>Ss9cL=bqnS995is=O(3I)5nON%hhrTG&_XN!@N$nkTR1qoI8} z)XYX@pqhG7eY=yxcDb}7HCGyCqNZ)A#|6BY9+q7v?P;tbFbHu=62-DEW|(8j4oh4> z>%AS>XM378V67E3kzW&$_MNC@mwORlh?6AUI>0|#Qgs>}l_aAPb`+M%n4bgkW2Pt`LTrkfrC@T zdmT+8u}-jtjoaGD#~YCw|0tuW+>(qI^1!bU)7R(MP@7W4L4eJ4TSr7BFt@!q(+is-vZP_6of`x14pN~asXc=4 zezn6aAT`0+KrMfc`C7d!H)4mN(xE7dK#GpnOjV`vL}}o#OjXm_*j-qGz$|L2)i+b_ zU(F*wuv-RZPEn#Xyoe(4vPxU7-C7jxFAk>3@7En}opt+%uDH)tfrTJ~m!UViF<}nI zL2j7zo#;hXDk~boo@BzbK!#@w2E={KkvY-2m}#Yrp|(=DoOsYI5r^)F=hG8m7|&$` zZ;FS>p#lghb2-vETg4Vb?STQt<#gS%DqtghE6FD763HPL+}zS70^Gsj_c<@S)|xM? z>`Bm*HyP=Q(6dxQykxI5BTzNfJQtq=*lB>fP6GY7gZoy2lXvPVgs--@(om2p>am}Y zPH&i*%6uvhLIC<(OD`p@v(-T-FK8kkkFvtJup?Qrd;x%fCCXiO!+1HMDyMM#fwki6 zo)gwt3)PbH8cJ=8ZqW02RWZ)!)+*Lo6+a2UM)RfZS*#9P0Jq0O5*{p*n0SZZvBGAY z&8kM=W#qDb!y(6VA>q38$w1xA*n?=!UyjSVny917cn(JjYVy4EP~$<}3bS)}IQR%< z6H-{ zRFjo+sjY27t=i+b?V`3=97xsQl`D*Ypm?n6xLk0{g@Vv}d7{Jya~0$#t(N^19viEx ze8!2ga`|jM1j*$lOJY}8VEa)J@Ph7g@srA5Z)`{NKSK2n_BFAIVJdQmY>lFDO&L)8 zEx{pM`j{+OX7_W#rMuJcb^!M%1jP)B0-ELA;3 z>K0oexHA$bIiCRxU6idAT%YtL!aCw2Nju*K6ZxlLghvh{qVa$ZP=WSRaMm(p)5ow~ z_Eqk9BORPLWSFdKmbNzbvP0ox;&BC;+l9+M5d60##rc=`K`qe&Q17;8NEJZ zZ&eMTSkRAz1(W_1x)fpXaUlJW^0&h4O5YM@v`mH$E1K&#S)6fKRRL$}Hl?G+srnPp zuRs6R%GCbq?(&jux)u7It-nfBT9mQyBJ|_psWgTKZL(CPD>iCDF|`SJD7Ia&Y@kv} z51js-$5Xn4f1k1f5V*W2XrDh7wch5FT^)kDM_5u5U?sEz!*zGjM?M~6m>K@Q>0bR{@iKs{A4F z)7#fLms>2kbD{74s$>cd1^-YKt@5d|^A@bq&+cWPy;KF$$gkk)`Z5tnJ=PK1Xww=b zd&l4TSDhZP%D!2sVWO+##(W6s(2}_xnsoL-4L$SG)=OH*fX`S3`5XzlDNqA>vw~IR zs*+@>`H_ncDrc5FIM~jx*MGRjvFUy^)o?S2B&|8O>TJR8oVv0c4YRNAnlV?!`e@}c z#0?y33NE5eZp7k|!#TJ=Al2rw=t2G!)J{|(|&DMPs{mMOAJioHU<*1tXJ!&$l zOxcd2Q3i(V^ngB+Uz6jKx8FjUxlN_jMG4ODjBZ&SFG)xF-S?>9#_sf}BpgNid}m|O z%?c@TA&>PSKowc8b0b>IykQ*vpWAsM7Go1Rb^*pIZEfmW(LkCJN(``akZrz}A5%2X zG*A^iAVGS7KA~#^t5x+x`6~qpOTU3Qz3P_GvAd6vSeB1bP3o;z73`PTNA_}g=3(Sl zZ7p`As%WrS>w>z9!Sw<7!-Y`@DzX=x^tfq}NDGC*XFn z46Ak9*Oo`REH?e}*1vWe>O!$1Z7%ID?Lk|H1dBbag8&P#?fpHaY9<=Z5uN-%H^vlM z{R((?S_P&^FP}-%xfC$&wq<6v4$<5rWpzLn%H&^`{SU!8e<=6ca%5F7Y?zY)CF^A}Nj*rT<^lO;hUwB-)HBdUlIp`60Uzo{!5sN$8~O!ObtZwqia43P zB$!91iJWY-^YT7iSVcEk*OUWXg`Zb?UqW##-C)l)&ZfIKru=pUJF4%J)U^ms+SM{S z@fH3>DEydIL?GA4n9Z~Z&3#xd=fPW4M-Gtl zU$UtONj1l2ESf5bKXS)=R5f!7%Bq35>|{t3Pa@EDAu(>B=P)S1Zcx-@Qs z8#e0PgC4q&{0)^c8kYf0voMcZ$dD1B+X_`tDxcLu6S`kJ3*vS6=LFrNVxY=)%TiYrthkKd@(ri2&Fm-s)r3I|+9~6EN0Hkl(Htj*WPKzPze9r(*w7QejJrFFH35Pi! z)SkfzQtwdd+6LUbNRR}3t#leVZnOp{@>-+O;HmsyJvL7L1PYO=bb8OXhwUoL*;cKeAFD@G@z8mZ28Kb{PA-wGs-rF+01_bd9 zFea2mHMNgrZ!rdF`-u15yeC*KLwsflf~9hf?73*yEiX#!vh;f^CW%!R?wvPL+U9ue z9y2#JxiF3cSd+IFmPq^Irc=6( z<*GKk2gr;#ZMlsR=^ovHp{1Oqc{AeNW0xJe1HeGN%H_(-3=+n%;0a7xgW^;<56SL) z3-RZ6V-3>K;Tj+*S@Eox_I}LrOF<6<^=E4y)q^N0^`da1j=cLV`d+-wd-SQ;3~e2| zWN;q$RIbgI399A=asLDx-&E0LEiqewl+nrHSaoNFKPjAh3te(`D}Z+OxV?zq`})YA zGc7Ort|wLA>7HZbKHS@h-In)}5CJAR+llMsaUHbJH&3oN0Z4yPu{x|L?PF+?6vNFr zQiHr;XFr>Yxq@Rw-;!S@y@<$eOWiHnT>1Q)bZ zx25#mg#u66TRQ4jV4I9?!ey@Ci7hjuzZpXOcyi_Mbysw0md7T441$S<+9_!$(j9$| z<59#kceOSW3JY{PPB$4j3NYEJPJM<>UEM?0(v=TMRD4o9a>7WH?7EzwZ!mCNeOhN{ zdod`W*2E_PopcqR#UIKW$wza|a(N`kzU!)dr_rSSW2u7>`7{(tY^XSnxpb5KlV$fB3v-dUWfk)30?22;eJE^wz6HdWsgmh( z?pre(?JnaCX>{2wWO~pZl{FNzAv|a&m71uVHqeG-=ZKsb0cg6PouOa^Tt#nh`uZX5 zg3Qkc*o#~gtd%}x-0G;5!<`XM9+kfsj>=4s|0C(V17p1Z2mauVI~gwKc|>v{;U0vu z&pu-05MovA*g1r#(b&-%F+$9yMui$tLbOJ#O3fIln2l9YN@$VRXz2RAKHuLz=i;8{ zGv1%~`n+d{kpRG7r?{N|7e+45zT6QiM8G6lqIWq^CQMWDk`|Pc;V+SWtMvRJn6Tq} z_9`Q%{&8UgWtV?vrgY=mE&7sqlN^cAHQ$lY`qe0s%nv zY&Uq2eTm4}#Lh}t3{#1+ifiNz!D5+a2Sk}|P%2H4vhF{WfxRP*6mXKXt*(0EXbz1s zL^byqcw*Jawkx|VoI&PzZOIE@trxf@bS1#6rrR4~P(!?UNNq+6gx4kV%qbf*b3Xef z6Ks!4*>9pdLS^c3aN~&{J_L=#)|ZQ_sV3@6MqSV%vLgF&c9Y**8D#6#F=9;92Kko! zU8O5q;AioDFen%rtSRmWa z5C8MnrA0T+gj)M$W$r|74fo!fos!w&87)OFxSgO~hQFlMHb_aFgud8Qw`|lxBXZGQ z2!4Are6f>LPaG%D_JB3hRfYbZBg}EyrieV*e#J;ts$osBox^&&P-Kwo>Iz(0K)nAKO?kzPa>H?Y3NT(e!qN3)QJ zdK~_`BEzH*ZZ+GnotH0zd9~a)B*H1$t8=w=Rg}_C!GSBftDzKCU={m*H;Uv{8&C(q zRUVF%R~ddgLCJ(pVFDR0 zm#F@Zktf<2mSAEgF{q7)M8H0Jp(iM9Sb&I}s0r=E>GdDvbwa2u*D8V5Po{0 z?B7!fp>`7-k;CQCdV9ia%Ifw&){u6N6Dg`L-t91?27UnfmsJKBRy|>u4D`BS6ZdM! z>a1Rv3{q>IP^q)yW#w&nY+ow&YX2nx^ra{pMXKlr9%vf^*;$fHIn0LZ?nm6K}U+mz&<61T^XVO_;syH89FsCrHhj z{4lw|oF;DvgG4L!?9tkdbO*wN2(EE@luOnmu$*RkS`cg#<95h*z&V*HQ%r3c(Ndgp z53)#c15)vtsTrLry>7Nrrpn8n0*_1jk69e2LfOK^Z!|&>+^Nd&DIL+6ZLP6SB(&ky zNQn176|VV@N|$+-lZv`vSpgU=Rbl?gp)y3J>b6(@)tY+N2$quEfITMY*l>QSAPfvSl# zi2~DCmow9&m3~h0m1?Fycu-?>2fH_GjG4sICTq)*EbNZ@#%4Y8bC5628)HMHLuRam z)$S?oi9|(06FIJ@EMVIhUY_Y`aj=}|p(w}HjM)?n%c2NoB$S=d6?Tp-pVYE1(Pyxf zY}eC#RbWC{MozKG)OH|!Gu@IdbxQ40wW+e)=#yDhK+9~0%x$#TByXm`s>p5QenL%= z5*Go_m~uw47HTdn0ENr+e@QkpOfG+ILo+ZwWyWxR=&z;M!sS!2Da)WPMzYLs!oz8Y z-Ap!C;1F=_l;fgkNt9U?4V%mjLA?HU-k>jd5YXMHr~WoB-Yi~=xMHS?>TORRSQ*(wmSuy++i3x)4NfnFTe(JY-OBUg-b2i2 zv65iHz;Dpl_UPGFkl@xZ)Jszkc5kc`1Jc}S$b$H95t-cg7VD84WW{ofgCp}{tJoxa zgSSy9Wn-q|`Bh*&zADh9r=I95ep`(rs91R$kSN*XQLi(CxG+XG;{sxuEwfLt#=Q+d zwS{_AIVtAh-9Xjdp>}gaIp@jTJD_V}va~)RxIG7a+o0R)bGdT1WPND^%X+33btfv<=KqlnZhLT;?rN+QS}T*^yi$Uj8l)HqLF}jzg$K zU&a$z;L^_em{g^#VKV7w?)^>`*^1*i7Ho=d%(W_Vusm7_XIZAI%k>SVBKkEWhRZ}j zw8XlQNed65PmLara@~`HEWD8YSe~uon@7(a!F|mX#wlg(l=mw^l}aSOp*Y(7(ElUt z5^@PvR!x-?NuXk%eErIu$7vEcg{81BTs~nsyqT&gZ}g1D60C!v)ZMKv!(u?at;0Bo z^TaO7gL_<4`fS~mAf63w)kw&uQic%dPjvfHsipcsS3Hd3l&f3D$fK#dRM&HxLxOGQ z80f~<5=c4Pk=;QS;tV}CK-Cv1RP4M76~8D&;*%h6R)Jh$s{KsNJM`~*a&(RF2- zst8r-x;v3}iWzh;KozL0y9R;wjzcJsMUzmAZG&Zb5PX+lw*m6^bt4UJEwP~lVFJ;< zh_x_Uyg#Z}OkGx<*OglcHqYUDc$mz+w>S$Lw6zD-vciCbv$6_>$dUdFIFAKB(b8)g zHiAx_VDyIw`H&UK2zqbb6^F1+u)`hlol5Bs0UB8yP=PWRF^H?V4E@cpa2?6)sY2}1 zP#zNJfe-XoIIqHfxDIC0g~M%=($S0QE2XLlP-!$qBF-Rx!&I#v3?bd+Zx!RfHONnE zpJalbDRDz&$aJnfMGF8uZsx1mJVD0IVop1CnzqMB`A(==Uuw5G8O5$JUM3N}Cc`@+ zGFIu1H28K(6zXZlPz+yrY4`;4o5;D%2=XmS$+G`*1VfyvT=GRu2DU!dn>-dAKqmcy z*Fifj>mBY?-ptSehjW6YKQ?TTC0%{f+jzpsQSyW%$LW&Nv80}U^C5gz&66WS`DsB|1FyYz%$WMq6p8Rvl@ zU+UrAG)IoAf8qX;dE@PN@q}Z#{MgRpWGGWlD7{c+Rhd3Qs6M9}8z2%K` zxNYuCiqa*+C10;IOmeFQL>8;kpi<@_+B)ckHn_OOk86SRD5`K?E&h+nZtGgWJ}&8T z%VmEa5{P;e$tL^9J|XW}uqj5YDo*)VSb~HOElC8frpMg|bg8;rpMmLg{XEA0OI78y z>sebFmME#JdP*PksHhC=2k?_ix+6eB{x%#iVeBQl^`a&!rtKSk<*YiZRtVQM>@!cqI*H}a6@ zQ6|uyD%555_;Co&?9V}L6qSX_GCNOCJBwW}x(*x7eg(=9-EdBK_G1ABGu4beRDGd( zGJYLgsfr%;sm%S;ohO|sP!Eyjca{BbcUeTY8VN`X!eOZB5Gdd61<63haV#w#7x-J8 zN?DaSip#1c7a`}WC~%A!bc?G7k4b64?&GW~zGt=6NMa&#I$Nr1bUlQ7R-g>7s5r_> zcd`t~;n#b5L1pb=Y${q?+8hM0Z45Fyvn+<7XjPUz+>Y*?SAk1p##wbi3K44$xY`>% z=BVr%iS!#x5fhwv@iJnkCkxt$?v7+C`c11Hp~_b{KQ=A=xOmC^c$(wQXfBx(l>ReO zvNsv=TxVT@5Hq^Lz81SwrVFy?d@^!tz3!YN)`hUdbN>MT>*e76?{XGv6l}7w&*&Ol zuegj+>8Gq5RM(dC6FheKn!{Txo!uEQ_izXD%;AKGx+-Y@Bwc_kTvioIVON0!WhJ<^ zTg8neSisBds=S)!uPsVfsl<-EsvgP?o=s~f<$Eo5$e+{D)f)E(;wxV`W$Zn(RXX+H z5MsB%8&@8Nzg5=LzR*%&mmfdi|5raju0G8%!m)V7uLN7N$|)rCmZe$vF{_a`CXL^4 zrrmX|AB--%l@xu3rgkfg(>S9SMDau99bk|!SuUPnbR3r%aM|!DBAv~untZee1SR<5 zxq~$=(R0gd>s~}7t8+Wb;pC1q9UHA@nBWV}GOChWKIlQ}Xze5L5bV*uaQN^h%F5#A zWK&E^&+a5K`w`#?Yq-5?diGe6C@`{t9(5NT<%Ed5r4>+XQh&*=C-)JfnUTW7)L46}qB6OJOchrNb`+*cg#5VNpooSZ zo~Y#|rEuJt6<{rd(zezkYoNZwl$H5oLDdF&W<&X<7X0Dsl4cXoB+cOu6&<}%cU^#P zQ%*|gWKX_|iY+6lac&@>NO4K`T~67t!jmZrxA7XJ6GXB_jLcW%qXzmEfqey$Skdyk z6Ht}A40}o8S;>xr?@y1g$?4MO7%6rSW;?Tz(SX!uw}T1tmAyXgGCBrX0|}x13HqU6 zm{!Sc=Dad`OZnp<#>}J? z=~@Lz_p6?_Rw^__X`gwK9k0?8J%~$ES?cT2c1|kXU%96Jpl)k#j<9Gkp$B#^>7oGG zL63<78|=Yio5z?G_UZ3l#7<^`xL{v1q#nd`)?t_z>)8S}-rEv%)2FW5=J zhL2=WeXhT_+Y)3{H+Daa@(P@_iZ#RYwNH=UpcLBo@{hhaO5RpBJ7r8S!-C@WYn4HP z%fleoT?(aXGhf&xYk@oU|J%tt90;fNRqcF+eC>`0^~l%pRXt^PDT|UxPiT+Esdk3RS69wz8bilTV5s(6p$6}0;Dg(`>`WZyafdCNl(`250^bun}+RCI2zc<@V^14*XC^??M z-|=_?OlKRFWsyr7Gx+IqMx;De*)r2Rp_s28!Q0d9(DldzOwcNeeg2_(e{cPdH7-u) z&$oN|vsDYeogctLW_`|U%c{e9#`lGV>{YGhYB;$pR4d2UY=Mk- z(Ig)pv+X;7BehaT^XGRJn4{51KK?3ToBQeTEw;6c7Z}ROv#;{QnJ2ulq<~FNENeu` z=1fK|<Zv2kk?)!mXds`SP;4=oONc%Ra7|OTs zmdr8pAHJmzG6-Ty%8)NqWd5uuw8~xzgjf4ATwm2ec3B^Qwzv@s4R6N}VMiJB_A;JJ zRs|$;KIPj-+gRC}KiKZ5SROg;WZibvuY91N64)jqK^3x%mQ!_juJdF*`lkYdtvS3q zg>Z6hY;27D6~Vb&R$k)Lby%HODP~M4;91vQ;Il(jalKQ`?UH*x^XD2xXHQ=MR?kM( zGkz2AE^iBJ9=nHPVe*O5`uW(Xtwk-*Y_@+~wSn5+m$9GIAzQlEGANY&9V0fq$2(ts z4qUDqplPMP{9gtfMC$u6efe?b3~gMFRaPBiZ=Fmm=H>z3ADP1flfPH*|H=!fQ5_<+TXqhU7dNR%y9Gm^rtfMtbZI6Kd>g^dxcx!^wrX?>sJlKufyyDsxq zuhRyJ*k>5?@H$3!s{Y9jY8hElcPIZhYb1~KSC#VKw*Kb=qv)QGE z<)lb?GnjN|6}zb&&+FZ+?hJV|$()77k1~^ypEBW=I0l;S%SYsL7Ph+yfJu4GSvZlpc>5{meb)f4IxA%F@{k+CuU7=tu)KxgY0BdmO7gP^=*L6OQDo zvV41_>MSiSU}R}-#7M#k{%=B@IQ>jvsZyA_sdhUhfO+=;B^C{AvW> zp5Efu_2C@d)+KOzZ_$9=(_;u07e z@5*bJE-^|7`R}r2AQ-b=B{i-xv7S%TqvUl}BTl~F#bg;{83bh?)pZ2`nGu9$HabZY zAKFOE&eh0`RfP;xoRn;-_#5Ajw&bmtBkZF2bUrRzVTLz{m?6^5-b;)C&E^uHA^9_S zxZ2MKwINn9^-y6yseM7ar?IKxH1k$^JomogFYFT42{`Ae2+$QLd8zw*V6v@{Ex}Tp za;qs@pZcEbNO5%j**;ty1C10hU*NZZFy8#9CU~>IKlnvWNRzBlJ9DsD!QHW8PyTxOe5yjSSHlhphTSZD7XbmhvS z7WsBEgADu?NWJe5f=p?-i+=VsdCW~pll-fw;$a0NS;BjO#wcwt51roG{7*Q(ft4LPl3q~b68V4K|S)zeWoa8jyA_eUv{uM~{*LrgrD3RzRGY@hAZRw>dhi?Mt z1REp&8pLUlsTnUDn( zksK>9WnCnrCC`I`3!1Qj-eXvfU1QdzCL>QOtT)u<45ib0oc|m55;Sh2hnwW$ZPwKA zKW=iK^f;-#17x~9iqYR4WtsmvIG^IL8GU+b7BFoCVwYxQzS76w90ohtVw+kXt2`5y z5<8p)98f&tlODVll*Jpx6OBZvA7Z4+gj0+X)yuHTSH67w_#cqYy8;yAV|eb2GJ1cj zbf25o%>~#7aTO5=X_7%h0CKZqpu=%5*vj6uTv8tC_Of#97IS=1y~^&?VyEFpd8B6y zYo64Dk;f%NnY;S)=cBz0dSnEvI{YnN+ z)v$FbX>IaVYt}gb9WUhG0J5nTa3$_51D^E@uZ~t6f8#*5);XI;UN`6eroH9kY$l7E znaX5eD|8G=VJR^BTx_l+asI9vcJ96dF%KvY;Hs_zSI(-Ejtg|QR`~%Bmx=<~Wbr3z z#ewu;+%J;w3#5>!`YO)N=h;`e{9jmS)-(8lVUd>f=2Nu>Y_fbfq}VqAY4biJO-9#X zclj#$#b>1&ycImBZ$yS^RIkzyb*=9J4+-B{+KCr=^3^stWdjvH@OlFf>-(H(Rtz-k za_4ACjM+0N$|~s{*jbfD{IYr%&ooqn_A1X(CUpi3=9gwCmR5ReabL@j{G*VZzaCGq z_Zw_;MUNsiLOmGC+|l8##yk9GfKeHmq{@4FkuZ z3CPsJK&He`EXbsEmPC^|Q|sdsCoi8uQ$er5#s$edo3@49UxHpxQETlJEwj29Su&wE zOaJKvnqEi#YkyrBNX_Y-c$jjZ{V*I&biH24<#{Op@DREqPk#m(TdE7-mSwy;%Adn; zwif7CR?hkp5ixQ`xoUjpv8)GO`B*cLHN3R&f7=Iu3QrYM>|6@@7JCoGZ>s)6X+bIX*fkT_|Y+_;!{(M4(HJr=&Rb;ygA zM?BB?(^$w-9<0RBn;Kdu!yhx@+`Ih$pkpit4a_N?dy;qQ@<4 z0V92IoaI;_aS~^_^_fUjGqz~l=QW4>N0YXD2;2J)!;ft923v+jfga9KryN+o3t6fo z^7u8+1YwX_sV?KZ>!qn`=OaDiwKQ-AT?BbE7Y|HEfPQsQ8TfL8XIr*;F~)=|s7r;)J1lsX&RcvR4(+ zC`ErLjxepF?AXqJ-Y#Oc&xUb6d&UCQdcU9vKv+fM{==}z5|SKZiEnraMgC?@P!^*{fm(f1L|`cB;&j`Y%9c z=_H6TB?s6Iji4o2C^)G{R+V42@FqP>%F2RYnbp>c^=Bo)9e+{mEOk9=4-NfavUEkf zK+Y<%=s)z|dDNDb$rP8Ah;1C&`0s(vnPUv0I!-JddFA^U zHnFlF!(%Bl$&(c-kpu7(HbB(zTk;7qxm@4(NNymC?91%uOPurIB2n0D;^eW zp#0d7b++BetPg|uRwE0P+BAiQ9r~0t&HkLtw+#SN-EXr9pW8h5>BojiMYUEEufxD6 zX&89$6_o^Gekr>2ugMvHuRE*Ccj_*6S2tBwL=vri7ZEL$vlyv<3N-R@AHykk#`6AN zbt`>fW#B`%pu1IsLddpX`12-;i)`%Trb_LvJXd-Y?_SDdJ$075(Zk>Ke;YgDkp5O( zubqnU8mWsC$#UMZLEwnHAMf5L0@P%q#d z3D=OTRLPVH9{_yI3E=MO&ivn-F#a#H&?i?51kclyMXewYD*QruMM^4hx}mnmj=vr8 zabpSy{OS?gJ4H!I+435+-F%Qg7mr0*+JJPDte)V#m+`qt^OOg))Xu1k29ob5Q}zL* zHBWi1)J`KqO#a~Mdz3xG(Gt9+e`ALFo7;}oF&+PGJ?43Nj9sjL!L~c1Y0Di|tNQYG zUVSEpQyTT<|ElfhL@Cf#Mdn2VfNZ^}8i<(TBSWcHEKyVVm{^h5njD9fK2bbl>SDu+ z>xFtt{XRC5CU-%noB-jMQ^|CHM!n@Y4Soxy?SAa(*usfJ5ql zwh~n?RqHcjFRX=JXb7|`uabB3V`IcIk0DF{ZeSp5!TduX8%0uG&_0^qU;}H5v*kxB zwrf-=!zOc9gD>wMb8n{yYDM{+9=X~uA0-ftOgSuYPb6;~v?2huS zH`60)O7SfQMwU<`Mk=c+OFi4Ou&;iAMGp@GGCks0JTxwT}_Y>~Vhn22eW`}v6YId2W zdPHQ3R|spQkZM_com9sEvIz@q^as3-h?@Y37z<9;v;rvqy9{+~2Cw~*0{-UqXMeT2 zkS=?~g@kPPh#kzE!WMPQr(JJ6aKpeik_p|g)CnP*CC{>s{9rMCmFf`xs*5+WErCjc+!k- zYm!08i4@wzW1fWsPUrtte+2U3fltIqPobcn^vqz%>&A2EW3y!56^_fl3bZ-$IrE@1 zB#Qp6+aV)1a!TW7f=N?U%F&k?Cgil672nJ$lHYPT$K{H#HJ!%kQ6i|Cw5soM5V}M? ziz}*w#s)pvUsis}!JX>BVn?|E(V`Y?`}1gb94a62yuYP~%UHtfagcPC+PF@4n~=g zfYuh0lOnYbGr!-~#SZO6l0|}xL-?Uh8FxuZKCWCv^5^R=dIlX}JO5r_hOHA|FiA=3 zo{}G+E-)sP+dM9nk@e9Y<4oGtfczTTTSFrdDBmJ`7lL&^k0G!Fhd7q^+W>RZYQVDQ zXSYcnb$2HcX+ejO)~YW>A9abVb)HA=T>=FfBUE&hskv$q5w`zG*6JEtl*4FsL00J z)GLz(3ym1aB-@}4ra>UMFZ}qR369@Gwdn4rq}P});q1RCfqn742Xq+#=`0w>ncbmX zB9Yfw<)u}BM2l1fX#1XmgH!cPi`dVC8;_Nwnbi(>oK%IH$;_np+&x&0Gbc?Jm1iDW zAk>$J3yoSKgICm=Ud#+pc9(wfibcILJg5h8bCs zp>*Ecd)e6(RbOJ8Qp8$n2*~YSAn`h(F>>${tN7zz7SN1tQPRI8&-NlZP|B8cs~QtA z)urwqU`jecc<5HN`n+77xr=u3F=hjA)G3GJGF;VoqD7=h#$1O{gx%*>^N<`Pd7=UU z8e=_`;+GAa+CE59Qrs%5j=E7q`~v-gU^mWwHk}eZ*`JsNzkb1LS1ey6k><2@qq3iY^avY-nLiRlQBoHos z8r__(_hsbcaP?YwBSAu{BjFlvWpfl_4JJr1(??Fs;@7wcPb8E=WSU0bGZ~u7iQj9^ zn!=7lRj~r-ckTp92^|X%zditB)g`Rje@g%8IdJ6 z2iq<<&0#l+V%n3sjf&wf!I*L@p%lMeoPj8NL~<^#j!_w*6nqYlGW!Af8>}y=9`@?u z+yIRBJms~PhnRb7X_oi;N2ZuT(?I#;V}{xj3=O&sq>@XIHb7T*CPFZ=oNQ8D{f9CD zlrlNBmVdF8Cw5~6llBOIKU4Hhc?plRbdA)$+9h+37 z3Yw*HGb`|t#6)u8>)MTrHEAdOiB9q6ln+@KL zgJ`57WhiVecL}k8NYn7(bVC*S;YAb?ywTL?CoEJFe&p3URDY0HWjL6BqEYzLD|*Os zy#v&)DphOrc1R@g1RJ97auqp7f{WDohu0dYN-a?@&Ee^rOztaKGwqJfTY16dZrHX8+k)f2M$xQ!nUTQEsK zcl79%U{O{hRW1huv&vE;KCPnW?URkC&*_j#jluibdTtAGHsVmW{t4O*CJ~dXP7DQd zC%UcZB8c4bIY4L41h&`W6L?2wRi>`zJ`VKxb}*oQ0QlT}37C1~ECV!5r+E%xWAeHM z0_K7qr+T_)*O7KMG}7#F7z@P^ol;`6N43+-q)%gSBNB6fTXh6UsI9UNy@)SNX(Fjx zA-9~>ym6R%7qV5|EaU5;e=U9hrX4d^Hfe{CGUHMe>=>$qZkd10LE5*WnNs)%Wb`%? zK+Pof0#zKSra25uVqdV%-W|a6CJ%X}>u8Q^a23NUz34lQGLu$Dx+pticim=|f)Mr{ z;0IsJIRfThJL3+QSD*M$9LS&a9m;E`RpRx5T>!$HLSWyaD#+6BF^r?tOy1m#ty(LH zNR}#vaIkDWu_`Pm)0ZrDxYa^~x2~UL*xHvN$De!>rJV|exNwaHJ`Od)wNpMZ^3@o4 z%0?6a|KoX-rL;TjzBeJ&+~4s4%saq7{y4@9YrDF0wdYBA*UuRityKgryl>ob+%xkU z!=@aKuqMRWiKzkU_g812$B$TU=wT>*fL>q|TL24xv(#gi@UMY+zb`$Sd|8=&-+YM1 z|7JJ6^zkQ*Iz)Box_)sS~%K(nuI zv&Bzruna1iG!_q3pgCYP-A1#La{IxxVmnZBf74BV+U^&H*j-Z$8Cnd2+0xF)R>`bX zYEa~Rx-zpajc|cqBr#D1w~pL#@Yv~jfb@)>(o9Z$WKg^8Pv(pKoMqN)PzkBfI~+=g zRJBGfRz;T_uZoiT>6yfPw_@#cR7LR1;~}7(Hh53F*C6b~oNN-2W4u&!qv{Jla?&AX zbl*JGNFb;&+bTE0z{bJbINpor*+s8B6asTEKzS(ww;PRM(0i?IW&R6f?HN^?fRJ84 zIoX^880uuRpQ^re-OPUq0i2`&O?k|@HG9a^WE0ht2$ju@N>mQJW?Vh7c*7_^Y3E6i ztOm%ot?i9M(b|IoVa340Uw1)HBUEdqr>Y|2#w3?q`xh2hPZjGZLy!-I=?avAt&M2; zCYX61g1ZQ>Xp{W)gWD#K1`P7a6~-V$Ph z!|`9G+6l)+e;TRhS|#!y$lxF~6y=Q{-)e0ImO<}0?*4kd3(g)>% zG>8>czLRAG;l9N&Pjcyvt}n4jTJcw|v=!WIBzPgibl6x0%1HI&#<`r zHgmibeT_tWp(oUkYfst#*(k_oR}eEEB}GP3#<13l-Q=SH9EW-uqk0-;)Irg=)RJ$vv7S3$g+?WrWPcTR zvfS}=hpV*1Dw1tsAGxX_1_=h`W!VV`i~6O0vML0WDB2C(K1j-vBf;#$LeeObS!*i0 zpL1wb9Q5Z5F=$Mv!NhCZf^>x{`Z+5fQl6t{mLotfPUMj}T;@jto0m%Yn^?x3C*P>v zZpT#v+K?nrYM!DJ^Hh%HtsY$gvp>}di!7i@EtT5wm!66-tOx51P+1AV#}yD zpiXJ6?ApV=gMdSuxoMBM$-rQ}*iH0ebN=i_FEN?C$jDOpF8@vlhqEhi=+hsU*Z(Lr;dR4t3|tPs(oo z{v=TLcJRW~0v7#uQlXUm4&?nw<(fBbz_ONRbGr`FvrSUxykV0sDAgU}L3*x_HY>t`13R}mV2N)6N>OXFM=f?; zs;3brCGLVvl~jQFrlp|Npmg{j9gP~o!!#cuo}CWLE1c#FYTvj39_8w`3hXi}SdR4tX-1!9aj(8(=T+5> zHapjFQu79M{c!+9PyaxjC@xbiBt%?$X?Ph$V=}#|IPjF(l0y!zjQSa6C`Kg+ZU`mZ zw*v3?EW0aFGVcR~&-WlNDL)#4PM+tG169C=@2SKjzxXWHu2hC( zEe6JLuZWAs_sTCu#tnw|Q&>G-`6RSK=BA^pbQyx zb*ou=KLz;xM!I8^sU=W42lMcXuKY^H^xCqH5IC-?S*pYl{+ErXAyo1XxC``QiphSg z|Ay}DEY4)sH2fTVX3lD45c*jS3G#Q-0?7rn+*{8a8%}||x4J{~HB|G21GC%(^3MoR zf+8H1;nvuw!2vbMNhIl> zq7Oet#9_b6Z!cxPXAO9Si$MSA^3vK1GWn1UyNgq`M}9Fe!}id4oX&-1BrFkn@znrP zb3HFqN{{7k_+JV->t88us~n{t9x<6wl`$$gI!X6~aB9rBb-21jcaclgjXY_5pEa~l z1*s+=I9WR0-o8bg{xRyIshU-O(y9tu+(4i>S!VHFl>Ztpdah5Fb`>xk)C*cj_}^HD zhUq5E=+g{MX1+yA&(LGKpxtLSp?VM%ou_s)5~Kow9Jp#?8cM;(oO_b)Y%e}fz}LsW zvPVA$%%W&NlPJ%Wt--4| z@7&T&6{V)a$rj`kisc+6x1JCZu1Tgqc4K$rhDZrfdT3?XePS(HP;OTw^<+I(`D75J zouWLQ4g`@L_!uB=)bkyzkZWUGJ*p+-ms^`#LR3#4$qSL?qut@s^^}`90oCj4gdR?M z(J~LNCG#Rt7qE)J&R!KYBc`h)cW@VB%+Gd7)*%#$NlG`N&|h1*Xa|E!tCCCubKIhjjG)0nX!=m>eHcVi4vX`8TFmQhMfrNiA z4J6AfW`?rV2uCIr)#VbE--W(G@biYHVDCX+x>diF=qeO|i7X?}KA`n*O|T#*X{OgS zHd|unLcv@0sAjTxDcre-o)W}bGn>oqXm|sewRPm3QmK#30Pr2UbL?l8ei#Gj8W zIWD`e)ZFAD!7Bs0yU_!@siixjB>Oef77|K<>_wm*)2$U`@b5;lygtpo?y+(bsOxCB=gu*SH%ldlgl&EK;l(BtZ3C7j11t`TEiDb^5++j z?-vg_0qUgxugfEK(Q;haI?M(?i0>9M4#S90%sc*SV(>YX?+N7t5!S6 z+7g8-(_JrYN7fgD(^rQMn z#8pswbws2}b+;)`U6dTEqfkhx(9PA&yNilr{;8~?;sXjprK!sA+xI78j0TR)luX5_Q?#v+Ne&hW&%7;WJn_$j7yrqoo%MXWR%!T3_$oXv5 zBO}D>!@7T726c3u4p`5oC&=}Vs6Joo`Q_k51)Y@1H4Ot`W&=D?GzEm#YD>*F?l`<3 zF&3Hln_<$Te2S!`a)xfz^D8ThWe1eYm~L`m8G~=DrTEtiEP;UVWoL1yP`!I1t? z{=yEgJSocU6okT;Sy#5Xz_*=xp}z`S?2r2pOTcEI1ZCPv(F6Zi{sHatoR3p}UTh>_ zS*OSgu=@5gs^QyA6z>V2O#Q-%?l%5WKV5PA5zS7Z79bE7~$*@m#Hs%Lg5y9hnnty{`!XCo33l{99r z!k5WVl%kzS+@xmIV{ttnm`z&E7m1QeN+sJAeI!cH z>w$sFF+>iidPZM8!lgb*&#KS=Wz-V)1rUCbo)9F>4{`BW8bq? zno&b#6p1y|m$G_GQB@?3tu-S#r!vx*fL%@Kor7{_uP;yh;Sx7gCE#~@v|G|+J=qXq zTsetp=`NCTAKZEJ#Z6wCqI!7x>G2Whtx;8F$xl%2+&f0L1gAsT2MFq7anx-tr~&0s zM7)_?o5;pg4KM(d+gSEyfcRQZpj>J`e2UhIUdpLl6Aza?i1VZT7yU0oWXeRf*fB7$ zS6^{eYO%#|Xlv55B;^Ybp%LYbBv5&#+TUFq2^Wj*$&OR%!5FfglAHuF+JjR>$9H?dcH|Ywqmj2=|!*)S8G|?7y{qgo?}I29*91f z+yTRm(=F4V8b#uqfm*a}DHuk9S=CM4Rzqh043=yG9^8BBGt`%vGrRY;=qT~^L3Wchy z(IeH?Z?&H1$5564d9>J^f)fSP_hrlVe(B8+!NShfP#dSTh9?p&gf^gn=9P(LXcxt9@55Y>3 zQjY5eU78xG0@tVY)S7^XYR9!zr{vC!+~IP`?k2>tGIH#EXf~HMnp!ekSvY>YD#qdr zl=00y3G!$yv~W?+Xp5yirUh&&(F;l|s4NLoyAyYaJ6^(XB5FJP8l)W@V#mkQZDNWt zRLK>5jtvbkVCH4RDm^Em11xyprgD;xLz|V9%U!DAe}!UNOUGRJf}845@*>wrkTEk+ zE^BUv>@DPT!mq^mfY2MI)|?oOy0Yan@B~MGHv;+c z8p~a8BV4|v#sSWmc*+#3>W)<6b4%YF92Pxp+DfoW>rBXV=gN&&2%DEL(xj{}s5q`A zJ6WfSK!@5F$IH5R2+%vav$wq81ua$6Z4JrZLA*XDIZf?zuy!@w%&sxIP1_DUpC^Fr zr7%!qH;ij7_s zs%2zK%R)pMWaWl^Is41e@d%hj%3I})0gJmc{=)=yvF$=ZQ+e_)s$77}NRfjLdG~-5 z=o@-oN2x)U1&fRdLPN>!El+ze%Fo2#p`X&@rPNfA@xgU>fi%O33V~5B!&_DO9G8le z{b2>jQMn@byyKv>OtrhCq+b|pZwXs`!;h_%VT)t+X?PCDSwPVf4vL+&(p(4$^jLO?a@>H@Al4I*gIl?uT3U zQT382A>3I~tDtp%q^FM3*2U(@fIA=!E~9#sdV>^*0wE4Hngv{1QEorbWx5;cJ0VM6Go zK~?_jsjUA!*LfWD56cHR^UYj&?dE2czFo zZDfG*%#H2@ZtblCv7OMfYl-;*{3rC3TiI$b{k1dLz=uCsfY&U9d^;83H(&LG_)0Gf z(zYYjYMek^tkqNeWOl59PU$g7y&K43b$SCmBVStqVjN`=3L`&%%#PD^OC?E!z?WwlXQS6*-; zG{WkjYW{WO5txyWZfcA~+0p~u$W#f(x{F5M-4sd>MUa1~hkI*(CS^-8fd!tTO}QNY z!foQZncqex$8)U9t}=RQJ+Z&4mK`HM)ic7yc?z8~ZaV8}(8|bHHGNYcJ@j!q)$%QC8#6_z@DE z<_?#c1UiX*FM4D#io<9>g_2%lhSLY{0K^~k=-&7`qeH|eADqn{jh2GF2cPi9$zHe`=B0vXaN_jKzNV@Tr%g0G~(PB>sqM19>t-K?*K*z z(F#T;s0869q;_Ct(d;s4HzNIDY@8{+M5&T~Fj-q`nKY$jS(7%tu>rpLWSY7-d`ni5 zrQ2EY_$no_Q07rLQWgXvM!FGTie5<5Sw2RigwgZwHT`2`Ko+d-$T~R6#Lx^C8Qu%= z9Yv{#iRfP?euE*cQ<1c}m4Uzq)zoez#YR-&vPvqwT){onBrR8gCZFlyZDd@aVdZj2 zR=;r#bSkZ;b*rXTH_*slM0bU4v_)R{-4i{=B}18Ndl@HPGF$8S!NSya0d`K>du(g zLbK)ke7EXf<7lAerK@Int^v{^lHa=PS>8wpEcjXVjVJ^n_1c0}P2_3na;FwT zZ}Lhkc1hn>97L+_swLOsVft6~qCQfBfFcRBU!S$tQn z0FjksV+=F-`~{xgQJoNR<@JJynA1E@UX|t|%R{ptX$akZIdc zct{6=6+zzo2KW*Z$*Z+O;Ena%I`YFNp6j}Z?RTPXux$Md?)D@L?lw)24#4Q2Wk#?^ zRZ?l%cJNqhCAGd`6+~Az)Gp>E18y2A(MGbHj6KC5Z9)kcu7_ruP?rrh3bo0-Lil&= zZ8JWsIb4vwD8-VtbpZM2*8uO|s{&`o^#+ZlTn<`cesOoEoIi!eV!>{S$c?Nkom3H) zrUAQRTzlMx(Q1| zMk<^7kwL)VvaSsvjwwUA^PE8lr8Hm6X4$G~IfeK!rur+EW1t6Qs6e>fvKV$!Lgac= zF!EtNM)~HwS%sjw`pd5u*ytC!%LTJ1vf3GISY*UyBVJNwaaz?GMQ+q`TexauHkQ;? z?BFEjo8_190LVwGcvJ2{w^a_fVU&OC;SCTkwvIBitlK056)@|oICXsiXgf}Yls;C4 zwi8vk;GU|299=&vN!Qs%A>fZ1A#M9`BtQIR#Gq_gL%7lA*HgB!0AyhF06DMBvDH{fso(tjd+rU$ZuZV```W`H`Sjk&Jmlw*?_>@;!tB z0K|Q$#Za7`oDLTpdSi~Nu0ijyU#U@vy*jZSN9n34m7jpmf2iKSO|Y{`_4)2RIrbiG z`$cz5;JIwl(p3A2=E^MP%^gaHCxG|^DkFX_zGtXB-PIIMR@ebeD65PdzYdr7iHHGP zGAc;QeV#p@;IYc)%3$@EYvLt8XP*g?8Tnllosf#Gd$0S|TgIf>X>y$2AzN6|gG z`YlSwPTl1VInbv>^}1$q8j=04!0l`wU zpV@&{liOLk>EL)*J-3@O74?&mz8rEc^)N9SWlzQ}m0&~ES<1n^vuI|m7lzBI=SBt- z~@*m(H#YbrBw-h&bzug0%?4$3F6WOfJyadtXS$`0 z^eu%nj@Ao1YMHT7NL6w_)~o?X+E??qWU}%j4Y+`kQ8y5n9M+4xw0$pZT9X&KaId%q zgq&n{RyGL=_AE9WTrrD+2z)6Vpe~V-7-BdcHxgqYtjZREk-DQ?dW!a1;mv#LVz>Rj&_yhLOsdMb$#U*i6{9 zQ>LOoh6m!7uncF)!fINDScTCT>xb!a%~=#RWoc6qMv*?ZMG~-j0~QLt9$#Q2$dyZQ z!ILTONEmxs2iZ$b19l<`506pNw%@AozC<$VC1X5GZty#xIWr0%#m#5;eY-Q%NTr4~ z(qlTKXhn6WtdQFd0yuT;`v{lD)knU-#ftmK-b7a9x~-r@ej^;$$vvgtT#kxXPc1O} zq)3}%Bb)UyO^_$DB$D> zWQoxuWOH{TReJavq%{4#%LZ~4Rgxw@!a8Dg=MZ_{5pC#443`KWj?A445Vi%v)r-A| zQic7Li!{79Yoa;&GL@(`*u~v>!gRBf|0{}=l+v)C*-N1ED=O_Wafp$PHj*(Onk6iL z*nYV199=V@P|b>7l-G#sB;um9p4&-wc+5)6%&jgndn5DK#Aah*#xb}17dAlnTpuJZ zX}%?sL7BBanKF`c?(!j#vwC+KP(HhYv^eWxW6MUvQRY2B9+W-~YNrqjt^FTKXC5GP z`9JWFzHgerFz)fV24Ox}JMM9>dj`umVsehFcCakMIve+F+1i8_sU^pXWl>pIR*p7V zt(2xnn~HU|9kQk4_j>yE$24m`pXYc#@B4WleXaE@#-ToDklLJhD=B#0OmdoFhv_ngrACbTF)my+t{P|>IbEgGU7(#z zMNgdk>2=sd;dpESdWpQCToZM&0+j$FrUlvxjME!PfROTL>8cpxQCbSyY89!AbZiI6 z)fAMi8+9F(_3K1{kCQivWoMx17^(b#RnJiFsw|yOHbZrt`MLh<-SK_lcfcQi{}fF8q7KH6)J)tNMZ@KzUZ#VD zcAs0)FPPuSB|E3Su{?BAM@55>h`pbDlxgP3Tzw_iPI+{t==70tZWCaEK`)a6ttrrC zxr%hWi9jA5pDZ5@hsD*WMGxee-ATGn;;;^Hf!A($0!01cG$>LbwNM(meQ~m{5WO<< zCB*7(l|;y~3Op;%{f0h96N<5T1o8$;n~v=Dd2N*+QSM$E)+4x&N>0LJbxuZS2v#(V zd?@Ql;9W*}tDsqgd&!||$dTzPxjC$#KHinv0?B16x16*X29r7(ViD(bnnV0}PQFxz6?N7JW57RgnK35gS9J0^S>0B6tIk-s87=&jc2qnyyUR25cnkw zlRxHxRGoG0e02d`rTnW5fTyqtDuuhM^l4yb(29dl2P*z9kwsb7bVG>t8F94=}4KPdUp9P42D@RRk=ZWzgK7g%v zQRxv%yh1h;lz)YE9*OP|yB;h;RnOBZ{A_vtCjS;8ryEwpRbl63e504N1lp=L0(Dy~+^D?#{+DB>(R_xSJiI_wtBSIoJHfD6>mLM%w z%m~^`X_YohrkF5K95h;T_{SO7t@A9#)AMln{TS@)r$@bb-Q~tOJqS;hI zSvgSN=atdl_;Tg+Q=I$YR6k^xJw&QgAqNvEua>lW5$M#@=0JWNI_pt8jgGk=HUngFORKs4psa1j30nkf)~shPEBx z7nYzIlw-HeWPS5k+xP(n^ZfzzrJ7tlVADh#GL1qBs$L6=QMwJ!AB%x|Vt7rHfB)he zz2oPBy!5dsr^0PHxCF1fR+HVeROvRjx80O(Cw;n83aqlaF4j4cx@$1UU{mROj76yD z;1?MOA%>62vY!cMVOUjl$=8^X{F$N!W>4j9hY1VU^6QwqrPnwW>YvwyLy>a1A&_5Mo-qR!_#s+gp+G^HuIka4gK}R9r1^r{qEMUMDcAy2_XeV|^oNYi4S4HXY36L3JD+F%| zr>Yq=6XYX5gm?ENBZP3CNziwj{Z8355LMiQQpwtOMnnx>@E4 zozL0>l|X!N*|PIp(&KCXmB^{4$mh(eD0kkj#%gH#$CuD$@;8MFxU|TutHU)aNOB3Z zT9$5cQhqVmSM3$k3*2&ITXxUF!*qi3FWoQVAD< ztSTTVcS$CZigy|F&MFpPJ*JpVk)lz$uDu3n3K%Q)X6Qd&a!yx; zilwL4HEyn9Pqhi)#nvODVrX?=m`c$p|D9xQ8B;z~BpsvhI+AUdNG@g~8a$1%{GFkt zYRe#sdMY{`7gMqe9C6QESD z(OT4|`^wXdX7sxad0JQ4O{)iLq)#j`eHsOrZW*gt7 z78t8+VfnxS9=kr>ts7K?lwpl8M$ma?$d=;}dQiv5^;rZOmHNe`?vct2iEXZa;R=L& z8#g8pv)AEVi`t-?d3xg?EE*}cb-w@9ou;y&Ia~i(M@M|LhHqWF!Rr<%&kLGqm6zM6 zuyXL&F2WK^df7OiSfn-BmPFrpD-o06!BsAeFE+g16y!4YWZ zd(oC@R87sqAHu0W8S6_h@)MKf@NIPPL(6;x+B(;yoYxPIkz)T%#=Rm&2+R#@_+mM4j{i!fkj5=`{~9D ziOG<;*VLIy)GcfuqEc{o@cPLtuw|gK43PXYY_baN$goLJ_yWqc7z9mf26s=GAQcXQ zJOnEaME+y#eWBYHZBXIH^4C`&&$%pLp7g4p;*ll88MOW1z9LRJsV}OViyY?|1DNG4 zLXMU^Eg@Z7^zq^pkMa0^6vmWR7Md?=E@!o_%6)zfX^_cEURD{sj34WUxp0|;&j@S9 znc>5?tPP=|#isV4CcCAmn$Vu{)qQEWaEdFTex}#PLxd-Dk#@P2 z{ffh*CtOP?%{hZ@W=@qyt)b@f5e$?e6Oku-zT)5?=~lyeXZhC5Rv`~s1xbjOXNE5fOP=_z)%4^r5#dj|G!rBqMS>pDixBaq#t*y1daY*5* zN8CE%+Np9v==UXkW%VKCz)alZkb;j66mkuX5$3JY>i$=!-K6iqap*@8n<*)`ae&Y5jjeJ0$e($ zE_W@-)n(KXq|)BK%(731(AfUsTZC@)a^@4Llqx5~D9sm5mNO4Hnp$51?n*lOzE#g=?mp6Ks_QC3^_@?F{Id=-viEg((D>RMTDmgEBkSW5Wx^tUol*xf zLZh7ma^xs!nrbF~qKZQ>hvM~U9~ph+Xj`_nQ8_0W$you;*Ro0)FEoo} z`$B5~bIYlN-NW6ar&d4u<|kuWM7n_s=%34CIgY$Dyk1#fWcgHK6XRhjgb_f@{a7AG z)<`ASlTTxzyUyDE-FKCl14lNhvWDDFNXVjTdgk78PiH?u%AJjW^Ml6em`thsJ2**B zN;&z*>MP;k9F?T?bjH1;d%E^%fcDxkgV7G?W-jwp!6XibT!_(`EV{CG)bCS8{i%h_ zq8G&{F?P~O`NnRBC1(-;M7TV7o>pIS5@GgDl%tAN(*5L;RN-^TX72m?-1G(DP>zw1 ze&HbbYZ-uZskoJBftib?dPSL?Gxx~@^0e<@JY5&mS>S#(f;Tn$sMG_Mofww zZjcJktxaZ(@I^!tQ|!`U6A0KPu>`}8UY1c@b?A+OE=DR+x0Z6hh15vv%7?`xkK!Su z>aUnop4GKPia&#Yj8a8IQc8zI@NM0&Y3Dq)bM(;eFoG)bj11q; zTvwF+b@?EKxq9IzGZ)j`H42kGSXM^Gn8mX5L+m+lYp^^s2i!o*oT&q*UV%F4ezZ#^ zx0646U{57?g+W%)MNsamq~4PC9Ljaq!~j+x*L{0oO!DUx(A}^5k^G|Tv_4gvBe;vV zTM4dikvV>QGZH%u#q4t%11JFM0}kY95v83PiXu!CR&nW|Go z^Q(pYU{180zT~dK%}$_!?41htx^$22l~vL5^6?@w$YhZ|sFD79QmKKDmIB{}3OUB2 zO_niJ$~4+7tL~%lJ*BVa6V8}^@$G_jH`4yBEy_Lxy(YR<=4<7tQX#}rN(HNf!9hRk zqc`OD=OM)#l_9FGx@9uml4o#qna!obXAF8w*|j6oHAyDMgW9KX>!O`l21{rId@V(n zZEQabrZP-l-*r>oVIZJqlJ+aNl2U2RwpBSpGOt}Dk^wu?EsNNj;;wYqET}1?{xmJH zpbV>&N;VT9yAn6{NM0XVbqr&#vC8X7$Xxo1m~JJTjaI1{7;?_?5_t!Th}&;c*XVs1 z%r+INs-1oz`TjAg=@v3oq+l(Cw497n<7B=C_F9;PDOy;QGbvgj8wW!)y>Eju|IRiG zjc9&4Oe1d|!alb*oWLC`pH(!Yl2n<^pH;?m?To3)AQl%a!aWX2P8j>vEIUZ#t;Vaa zI=aY(A5tnQFQbOBxUZM{oNO~YEWKmemO5vA z2E0G7ue|go_;W-hwMVvPw?NN`u7&G(`^l+$yxNEEw9@htVm+vptJya~6bC|8HZmdnF z$#fnmTrZPvfy=#YezYT_9t;<8^LKuMj)G_)DPaiUXI<2&r}j6!sS*;T+)SkL?RNqA z@!w!>tCZahJW9&3fk-TZpjMAGdB;^c)9EF(!JFP0AVtp$(~)daT0Ta1BO z$@uW%upzNf*DFVL#wm#d5p%|Zu;Cg~F#`D1cX0hHuOhh)tMGH+U-2Z)DVU~1B4W{B zEnQ@*uEjm)pHNVSzYUXoTUipM_y)+@LOCaLVOh8uYR_#g6{@jqT4q0wDopIfX+#vU z#=$Wf0HZ-PPcNzRA|S7*8_-s$o9kDFei^plKhRtImFCh0L{qe3Ks`M+6eqo!`U-?wa(Wx7v<{?KCc5A- z0qRp)8hm0a{i@uZU;0Ul@tU@xs_5?YSH4C+KZEw5A+i|Ck33~GO)Zf5{Zk@XO_>we zcR^e2W@#mt>cDLFD}Nn~fx@|{Vi}|GFUIwdak_KD-@2B?^H~ghT?Ix;y>yWCi)Uec zvCkkQGJf@CO0T1)U7qyu2)D7-xR*_M#psvl)s000#=QG^FawOJ!bh5iwxZR{RE`#6Z-atY`l)bkx+63Ue9$;d9p)0-7g0m8rKpAu3`m|WT z8ebw~Vi9Two+2*0vF^WnPZf-ZbH_O#ns7B~qEo#uY8CM&4Z+~7nG4_*1+AoNC2;X? zT^^=}j%}$ztzV8LH$`@B4(YD+83EXJ9vQbGv@ujh?e%uGkz6yB>j-w zJ~$I$ZpX>A!O>GZ9^Zd(iiDyjMUM1M%3)o#ZqG=tV7u<4t9o`_pOHw3x0{*8 z1efGKZIsy3r8W`6vg#;6oSLNz1-Tn>`ng?k-&3z*xz2P7vGA5=kvsH zL6vM3+pnl_gjn^$`idkV%8gEPy)0;DTAL*jB6P%gD~WFaY#S<6F^K`APZ0z$r;L5O=o+*#lLK z!L|qrQAsoOLp13a&upa zjI}}}G*Jz3+vOmHTV>!7TKS`itnS6wt4A-m^w!26Mks|8sFm$14(5RsKMP!}b38M&zmXfAnG zaUNwlFhX40rTqjBKVMl!$h&KhR%he={6+1Dw{_R7<|`S}qbvOGtm^Z_GU6;!p!Ksy zNv5u$8PQfm;S|Z=2eAL37l7Z~ zOT%Yp^SIO9cyQ2&ob{#iIMmsA<$np<&S)u(cVM!Xy8{TPtEAQP{<|iI>Jn`O3G8s< zVFvZ%KPF}}*G}%4cMmF6gxXwV0WdUblg}43-c6OB3UXz%Lr13DWCK;Qg4-O|1R?;4 z$h|om8AI8D@=|{QfLw#`gyW}dL{$eGj9oeW(ypvoC|59drS%znh?NheB*@dsnkR7^ z%p}algao9UeKuSu^MFjQ=ZiC*xSuO`;vuTF%HIoh+tLTc(a{>`XrwV6c;L;{hbPZ86PJW;@B?9md_uW znZ~u45;>EDxcyK$`hbve&A^Y`=JE=iy9qrjsE7B!)mcvThH_K2GwI+i*zt8$k_abG zHpO#Nj-Tw;;MThML)B}}) zE1`N1RPHE5S*G^1roFFK2gSc-YpWjhB~CvFpXnxd8vwqBOU)9lXeqMv!%bvimqbda z#Yc>NsR6h@4J3bpKOZ5j(ReXGPI~A&hsG+wA#H~M&ZlX%DfP4YusY?-Yqx78%3QU<;YO+2YGM(cvr0lA*#i^`o4D3rC8ABNnI%Czet ztjQX9t3IKEhWvKH=LU0~b|W@l2eK9PMw6laDgpgVs&_)R*@=| zRV2<1)*>!x@?tS)$Xx8BqVrIob8730=5o^a325n&?uUR@CO`hc<|& zNzzV!s}-Wk+gFI%Tvb#B_GX_dC*d`+_H~wgvlc%zBe_VH%&``N>X`#jPSSjGr3fK& zU8OfbRF$-YzeBjRWZ5V0?}j*v^&S3iDmV$&Y^fvtra>2f=vd|>pK!iqsr4!!W~QXk znBEsBvD;AP##cl(m3rAsLjx?)9k6p6q2YU%$i)w=9?8(zcwb$#1{u^+T@KJV4%u70 z5-PIC8NX#27^GpXS)o#|C%_t2q^P>Af56#(tlV8uo04A!2a}H)3*jM1?eg4AX0Dg; z5Eh$Ce?c4?&T8l$$t~sFMGk)1eYZ@>23mP&L*Oi4npRqYS+AV2Iuxq4-1-H`G~_A} zzh$_Rv}}iV)=U+51AFZ|#B@b>R@Ia|qE%Q@M7Ybr=7(JRxo=2)+F4GpD#BC8* zPB-W-gl~Ftq*hNb^oR;CfzB=cwMy9@QyKLcV%>}x7}T{U@_feftf+hu!3X=?ZuH2q zaj=LnxFyJ;w! z#yrXLUQ1>ip@NO&Gbd9VRE6=dOvi3eJsM-Qh}Y3W$&HNxsTtDZdGxp4%4lNj%gTeL z`^O>#G8-99v!WEIF6s<}3Y4eu>21m;sl1lPFLj+Xm}?<7;(dnPjR{$Z7YjGg?j4=V zU6cUQpd6pd_lqso7dFEAC6eCC^rQI)X!bpe=Mz=Z5G>QUx;U+Zs=oxInJJse0ElbQkJtS+F!&CEI{7?!OZ8>NgD=w*I@+B3Fpkq4Wy_yE{8N*TRmz7B1R z`wj*=>t_H!&`grdqQQV>CuMoNs3+7E=_GaDWahISLvFd;gK=u8oFTFm6$8hqYdMIL zJ_5BmSj|{xOO~4kNdEl>4F2v-;I1x}?Un@|JPg$?SPSJ;iis1&iwb(4;55i8t|DwF zNy*mDC@@2*AR@H}^(MDJKpT0uO4KjQTxPCJ%VIV%6z55mcR0b}WV3<=tga z$uW&}7^p@S)Ekj|(taN7yaK#`ZqCJhg)%zWo_e?~+OwL?xfV3;K#G zMurU4m-AonLu(bDfEFKo4_1`n(V!?t^jy*@IBL3S#M&|tInkfK`oFY1JgU z%(SCO1c%6LR;-3;s-Oc1fvEj1-UB&O-n+pG(?9~`)sPx*Si{m#-+)uJxDjXX(3kVh z8uC#RN<@#>(dkaAxKpx>=n*cvf~hc$puX|X))gAuZo|w9L;A#1tXCzk%E3@5iTk#+ z#$2`p>haBJE0bLu)`u$22g$_6$ZJ6ik#)-J(E)4?jF+viSd7zik?lLP+?WE~`Z=i} zf%i624w8$GhLPq>mi0@)+n*D07`IyunmBZfN|!Y0xy;9C#0(D4)Tu%=)R+Hb-F_!G%LtzyvvzsY6p|0!oYB8+mn_uE3LHDhwb zVfERK`|$yo4v7Z?aznb3LE3a_o)AitE46x4?}lXJ+8~wYYj;4ZLS5c;j8r_0&+I7!!8~b!UbQn3{VTF2qcH+&ehs#)-OSinF`?xrCE$ryG4xxL^p_My}^? z$Dp*Q37SVwqUU>6WvR+tOrArRK=9&APpT=ciD@wQ+44}V3hSed80h-Sb6Peh`=Hb+ zOh}=C-g0A>RGO=k+R6u7VIYn6ndHyL5 z?d(KA^G=KtKXuwf%!1+y`VI*tRe2jrHmz&1;j+`@qmM&D_)k(}SVx{#mYh1nx}+*^ z5c!^1T5C-C76d}38=EHDR$lrMX~oqSjd8zbhGRNH@5c@%ty1Dm3mlv0+URi1>$QAd zEx{Vd=f4B7o+{edsBQ~mkx-|v<~S==Ap2sc?$i-O-mfvCDRe>~#MqF9Z$@(+e-vwM zPfbJX!dFsX_imb>hW^-nHC(|z!Iy?ToL*Tj9D*5)*bKeYQEnSLbFiCy=P=VHV+&&< zF=}!=*yhixXdn0mKF9vb(@82$V%t`gu6?cKb8uSaWt$I-T&^5L0Df*o&dpv;+NNRu z7^$WMiSA8>PT(Oom2pUC9Xh3SY>EqA0}ot~EGHxVemZSPY4h+mq_$0E$Fu)PobR8FK&U>5}jIbBq^sN44_RV3;jc@D28z z*8n7otAcTn9bea1U%VTkREqC(zz`+E71T*>wUp@a2B2J`qH~ZDQEu znS*8mGSo|r(jF)qYkG23f8r+=K?d7u+{y_ajR;h*pc)&#|D0L!9I{gm75Ivccw1B^ z`+d69b3sFRR_+3MQzz^^UcyJxW1m~PeI1G*4GZ%TPDPF7!6~@c^HpP#j5Uz>C5p?) z*4vQHnWx#Jj{>^rSM;X%B^7!M>syGwGpufS3EAPF4+dC73)0xTkk!4T-OWed0N5WY z%WUIOmVs^^I3fEF@&Y%1gXC;wSk^Q8?r5w^m@P5GSW~a5NT+(rvOt%wbIXa&Ft2Vn zWTn>Es1u%UrkB&SOf>xYQTm@}=!#JFA@j7AgkLyw#rPI}I^nP{z8^I1rGcj=8HxG1pke`h;SDq0IGk^h1!a6X4>GO1 zaWG$B#HHKipCr!mbyet+Sl#B0YOvayOoF}R6;!3%Ba9epF$-kO7PDCVI{S3hH8^KY z6|Qb<&Wg&D$`_DzEm7FZw&(CKs*=veu6$Oo7V+zqqJyUQQW|qfEkA?KL3Tu7HJzDPG^`+|&yI5_L zwM341*T6&)hDxA>MW5lmR0m)#m2+{x^B}cKaAbNX%fvk?$ugq@v^3!U1 zOR z+4?70X|muJJez~70>8~33H}sRM|dIRhCWL1%JLeVcS7fsO#B;^pje};C>V&&vO%JX{8`tU~XWs{;Y}ki;~clXw*eIe;p7TCO2krzuyQd|L&I?xZqP{%$RHd zd@5BAkYiiG_>s!JM&8?tl>R`QyB3w)hVdh{uNRBF3jJYS|np~!@aguy!{ag>=S zoo}+zrW(HJNmJ2s1H(~9vpGv>bf-q+Z&cA}5VfS5JWMsUn9rys{`K&J`YLb)>MyJy zzBBMfr@q};uL@Qg)w0rXP8BD~*cLu7xR_B+stmQ}Av6NDL4#y&cPD=ALF=M=Sqo(! zjlaCuiyyDhjzSv_pon;z158IR?J7NqI3jEUwwaEaUeXvu4z!WOdog-mQzfLeh2aIi ztmDipsNmO9beuK*2fR4t?Zri3PE9F0(&u5&g8Ign`B5qTEilg3F0Wy#C<8ZVAO>S4 zH(r)*@+HW+)3E*u=~jz8MO1J6X&a2RiHwol2OK~B zS306u5pO=Fn5}{A?KMQ|aa?)zu$jW|anxeAJ9?2|=ePvfzs_u9hcndn&pzqhmQ}2M)##Mq?lFR8>q+!OFx;bw6fT@FNa_e)M z7VKc$jW6I>y$Kxu?m;+lEODA_bP?d3&Mc>-J^ZY2vUK^MsYBfyR^%N87WbsHvuBiN z9`>HUA_$XPUK$JvdANIscEcNiw3@8^JwZ+qp0HFWL$}_SCAB_EU9%9RP+wVeJPP$G z=$&iV9=2p-L#mUt;}^Zt5`E!^fUi(q%(QyBNTbqTnV2XtKRO2HgAQ|^R7QkdWDO3O zU`=j4(?_CZ75(RH?JB=7__zcQ2_f)@%COFll&c34+SwB$!kA?|h|j?9h9AADmZyPF z|C<4&3sJC%CApkv&aUkcX5fIVUt|W5NrG-95sWUV` zY7UlLHE{KnBG-yM@lb(Oct7NqZd<^xxl=>A^1P3se%6IbM>u^+xP0vKM)-aha)`^6 z>beX^4dvM@v2Vey@TA2_$Dcr^Gzw&}_KcyDr6bM9OobE+RCKnyJQSDjfHX*UQUzHt9})CWB^{I2`%Sy_ z*a=qruAEEwHM0v0B<%uw^lw36lys4!*Wls%bhO7uI_#w6d)Vg=<*gwnn=to@nJL+_ z`4=?MPuHhd-AO3{*<$VJCC9zf~7uYfYLxRZ+aHW&BcT?3j+=-unPnL1ml` z(pEQsC)lc*TpMi$M99*Rc-_MOb?t=zM+cace&j2XPnR=_OWCW)$7=!FFlB!Yb*fPN zAfjb;3nnj|6l-iPHigU_fZK|^m1bBW844;$dkP8a`d(fNA}ZfeIc{Un{S2bV!cC>j zc<8&1GM<)2Ph+B9qP#Zf=@ruYg2c--l-W8&YS9WaTblj@a`jQpcCv4e=@w5-Uqp7=d`?1dB5iaAG_g*) z?%WT50e~cx+n602`=C8{4_sG75#XYkq=R|_GKl~~9*#E0Et@t6Wem^yH_AOjast*g z&dV_clP0I3{QW6>Hb!~Y>!OTa84|G;N{!aOOlkO?8Ic2XP&!A(p%?w6T;s)h3Y99P z3KZ^NN*?&1^I2e3-YW8l7L72_yc@FlAC`9PevS+#XhhzcWA)3K(%cg43vpT0`z5$= ze_slvl#aktZ#Dw6&n_oUa+6L}E2}G^KH-Lz{=CVrmp5YlfW`yGD6rD3&K7}HX3mgj zbfG7E5>Q6zD@(+6Iy)}1VAHy{$ouzDCchcV-goFKr!BBnWlCHqf`&?=gb4G&g1)ZOo|gJqW)L1o{bgr+Upn?twpR=2 zQ83YjbVO=SC-HkYO<2VyPR?v~C!z$wpjuAy z#Tk9>yR-$eR0=FWY}-J9NNosH{%ppESE`b>NXDEQTEBk{yEfO3KrM)cbK1uOoxBQ0 z6I%gVMs7d62;s6gk-X;-cBE+@sOZbYix0%p-pUBc?ipq=!is{)IjX2F;4Dmr*}#)d>lCa$)sfp@5+?Ds7(BQlSK>EZSVAm*MO_(30(~RhSD+8=}O#a?8v>b1&$J#Z3BAt2Q{hRgp(T z7XiCd6C7Dm+8U58I-%dLBO8M{+l>IVYLF2A^tr#o`cEql_be3`JeQB^tM^&tx!U0f8u@AMxLXzxn@K8KQ_y!9&vxh36 zuiT?8fRAI2v`+!aYH0(Mbcv?uo9@=~%|>66YQm+!T$P*01zgzFuA(TInzRrGP~wo) z&%j4FjfJ)f^xeeoeNCOjkWmpC8MjR?ybJHb1@RPu*X=Zh+A_G@Od1Kz7*{||vJZQ! zlA}8Gt&*O#7Yu}SV{+>)IG;V0<c7~{_6t}Ngu#=7AQv@_($QJIOIy2aNL$k z5`P?qQHk2P#viF+d0s&>_s@ z>(pGiG7aSAI-mqvUmtJyrL~mMlS$*7M+StL|_wSRAw7|I>sVQ{ellM zBeP?;ImfnpM&(&>ASRuY*i5V}*RKebZ8M?MQM$t8hVCgA&0h-8qyt~V65~d3xlq_w zn#~PGBfu z92hwj@PbfWbW*;xL{5$1k^e4(AFAjQ*<}Nb13K0I4q-V))ht~~COCj4W-RB9!81&% zJ`c(5R32Ibs^qb_F0$+B7ND8Zy2b8H8Fidv^Xk-#GpVFHJ(&_l5lCAozH??W=v2}T zLnXP7sA{{T*u#O2u%JjMu)tkMi>;bmz!1Sz3I4iE*_Uztxz*)rRd~)$U6kMOB16`E z1w7qaK}k_1OJLgJRxo34V*ryhRGvFvEszdxqhf(}q+EY(>cYeJQOG!dlIEd>`UX^2 z%fk^)MEx?^^DN`dSDA0)lrU^WUnEhljBTf@qHpE(Fox)Gm9`DlzoeX8+50hde9FS5 z=9NMSb=waBZU6ko+yXO2y^4${h(WuWb2iAE2VnYFojiQIG2WhGZ2|O`DtA1>#?!#4 zd_N6#k&9bYXEQ(S2Ihw7)GemV=fHIEH^hKWh(jGustVT(zX3|q)^N*jW(nccj=hLD zx8?xoV|CDNw=4HWdFDJnR#xeQ;AlzDfS;w7%kQM4iFi>o7bDQ+`nS~+ED}vIJs=21 z&Y`+_y>$NEyUIBlyEvJqt2<2xJ9bW#`Ys>7O=VoijLUoxkBa+gd37pkz;4q9;%HkppfF$;yChdtw@cqroF4IN+H z`UDd9OBHz)Gd_7K3UypLNSZL4qY-N^45V-@+I;3r42!^M8NCu@_)f*;NMEa|ix&={qb<>Ka~sH)Q>`u(Z09G`5Jsxe+SP_rEZklukxGl;c1X?! zWZZ*NNLHmBZK*O01N^oM&n76>9GwO>!SMY|pvwTM_BQ&{8~+1dWpzhH*hnQ1lNii` zJ3bHel&0@cA`L0E3yP%au$5D`fSjKPb=jzfrAAz`7|uSvsO&mudZooLa69@mRn&r_ z5y+6U8yB=|qx2ue>%|lm;Dp1qwaK)Wb3mXG&{2-tN5X4 z<()Pb+KPQv5eFK2bbD^x}h0rMAI8~I>4~0c_?P_mj z-SV_VMz;UhfD)+qGx9zPUn`Z_U%RK588c9+{#Vyo*Ql`Df?8&blW)r*BWkMPi?|Z= zdW!d1PK&(GRI~zbAH-?ydVrlV8utjUQ0=pa;ETIR``IBsh6mYNX*z0fTD%*^OuW?h zG(AU`a1D{Pm;Q5qVapN~TgiBoPm)IskZ7hplU_s=$kA4SkMigp^ye#wOX2(dbEw zac!TE44Ve8QfFva1wN2DxJmq{<+mFATN&l}SZXL!m(ZsP$>^m%KMX%xGADzo zy5h6aDS#Lg?RWqkaYP;HakY-fsHHNVXXk-0bV;)IvhI9JSjsMEvp1A|t{iE?`{h)6 zO_r1QjQH0=&u3I#M=2cVE0A5;;Prl$xeSEGWc+3g%FlFN-mkwZ(#7pil&B-xA7pVo zZMoW_LZc3HZ4!#xss*|VYrFWc1Z$zVJy7-`nCvC5OMXLZo{koA8eQ|bPkGJf@*67>~1`GYl# zzLWO|Hd)A>iSolC`jYziI!Rb>jnyqlfh)T=2Go@+Jq)v zG6!5qS}6OfLpSr8`KwPK^BQ zL3l@(!~DGR9^bT+LS9-^Iu=E?1ljM_UCDl=c%%$G$kQJm=Udk&__vBBm~0y+;KaD2 zBRbP2TeC&F!jYfQi3+<>J-4hadyv1RowIr15!^!b>NkM)fAbq%a?bk&km7g=({3an>V4nI86hNaXU4_U5TYWn5p$2@nc zFM4mCc&99_#ws%3Ve~70tk>?sk4>jB_n&Eedl)ob@|xITxwy!0eb)UU*!uWP3n z#VT^;g%yl6{8z@T@+7a;`JP?a&+XwOE$dsaV=7%Zc-m{nN}e`S4FxWF{7f>oLo{uYGL3 zFPV69eL%YY8(7`CSCfo+ET92RUghu(&hdvI07dojOk1%v3wZH*>D3lvcQ1B2&&xgv z53xvR57eV0sW^02ve0@3Z1>O)h@<1Su(ZEz7E0h2D4*7wN81=+-M0H|`Q@kC*~4j^ zPyPemI{g*Uo;HFdtvk*9Q*T&gI%Kn*jTQ7}PnS<)cr=Silm0FAiC+e4 z1M_+I!FGP!r9-@K)MmH?oB0-djaTO%;nioSL*?FBka>9l-|Umwe#Z-Zd@_Mwmmg(u zb3S6hD<-k@vpT}6CB2fQ)o3$8&P17G39fLdyP1_)?}7(e@KdRF2=eYZg;B~No#pEl zrbm{whA=8wcw_1qGc0qc2Pr$2vdQ+;gpp3=(0j5{i{-+%y!FQ~!0y{r_IS_5pBMY{ z>W`XE?+k^eS~mfd6+#^Ls7#i3xrjfzo?rpCwv04yAqz}h3x+U-Z9eC|*~*Xc?b+0? zLwIW+4t>d9$O%4YXP!+9*i>~*gkQAexhh3`5Ep5eQqMEo+uA$TuqLlfT4%yq2nLWU zh-kTgir0SH%%cAA^G2^{cq4_3FG9^(_kyK>;ElGXR}Ou``A^-!S{m2j4Av`AF$O`ukc#4G|=t*zrd=E$pNM`V79vrc`ot^ds}yxkEfpF-MLx( zx{XT#nRlE=Ha+57{!AvScG`5x9w%>v{^1n69}HQf%~^i<(PX7{Yz*+(N%r>3CH{P2 z4bQa<+g!#i;MM$=>6X$|7MIV-IhB$VK)uZcR<@%fllA@_5Oi+Ad0zjMKew*{URMI_ zW%QeD?X~y#^TA~d{~fJ?=Axmz`^#9&37{J_Gwkve+RDv72ABB}<~NskdY$Z*6jbqcLLu&wY&v?`xXrzJ{|wtR=|z zG&L|5Wu?osdO#xh9xn{gG;;e4Pf!^-Njg8p$H*@H`t5wvF1M<(G2c-$TNdjW_KQOp zbJR^1F_$ro^6}~NX$@xBGl<mBM?%SOGW>FaBdD z$uJ#U^!r%A_%N~9sFI%{jj8=4jd@A@4%Rv56Ck@~6Pv2JmA#KS#z)z;fMT)Eex6}{ z&SE?+&2AGOuz~ioq3FuDLtc4^GAezKv9S3+^O0xK%!Q6~#_;@1*=aKRGVGDj3zKJP_Eq6-&PMC;O;=M9l8%D?av9A|@=HENl@fp(g7Vi)1 z1U{YZ%fB_6$y@G)OvwGFMNC}^cwR65toM^icl)~Nykx{%rSd?tK)VLg<-{Y*neJI$ zSn{#qkG}$R!@ULFSmY# zb@aT|DnTtp@3|RrFR7`rsxq?joh-kckLUe_Z$b`PJe?=H%rdF#KcCSPcCnFpR;Krj zW9eoyPW`J`Uxt)>i{~5(Fw?P%z@51?q8H;UrVSakfRPPz`e7h2eP=8Kf2NDor{jf@ zg)|eACug$Z0Wl7F<8AONQH$epuQ69_E7tYwN9@ucQ(rtf(67vAyt#~!JNf5T0L-07 z1$n0s#o@3G$?z1)_=@~=eQ%X6Rr5fS1MwjJgPy!JrVhwg%)|+q>*En-hWz`o&n0<@{9EUZ%s+jsErP}skx%yF z{%oPbR;k+F^h>ZQGj=(KVpgwfC}NDr&y_h_nBVt>X_rf9c=K@@8>z68XOk}i#~Rmx z!*c5WiSG(OJkZ+Ws8Jx(^EUR?FQ0(2TsYx+s*=dLxp3xNHvm^@-FrAa3N0DhUGkEx zK&sKVQ@&ZsG|}^HeJv#pB;^!r`%F5A^r7}a{TWYtst+0aWfw+dr_>B7^A|Ys!WXQg zGQQAx6Ihj&c-fhKOKfJLGx8G;8nWduxwS4*mKZr!!hwn_;n<8KI0 z`2GyvwrpjjePdX_%2YEbM>Ib>oy))7)DrkyEHL=(F@JWh$=S3$4yYPxU40ro*$GKz zJ#*G!?G^lB@a{@1&D9u_1J586A zUe3~9?aQB?6Ikw>O~KCP4SY_Sa34N0Ix$xgXn`%wJBA|i?h^j#?NqlEjNz=0eQS+N zJ!vs6uV&YmU0iAhbr8a@x|ZO>hGs-=>DaMWQKlPw5*GbeVv+1y0jy6o^@ZgOT^S|m zCS0WS-=OjDx|i{8c#@2CKpK=Lsv^C79MSNPIikz!LCcZDOuK}-@o)d?6mXjCTEsdH z3?mx0iY$DeB~4rj2mhb02A2LZuMKjsw`Ws;_~r}1bF!T&^E)$Jj%E!vGeAYsEGkR; zs-S-0S+G7Z4j?bA!6PfQ8CKbCA{W;&l=BDeQOeJil-10=E!8OvG<%-Efu|yEIlH8Z z?Cd~G0Brpg)w-X)WGj9c3P}Ajzys@*b_n0{?Zb<#Wa(0~Sbp!xT&S|e@(GNSODnq- ze|(_2Z1S+WPkm6=)L%Ken_FNwn^i^ynLL=2_-P+|+#6pkpZ;TFH0SfzOjT@?@>@8D z>y3cm>{IOI_%8teWJ{o0G0V(ENKxDM7MT5~w%RsTVzFO+#IkEFXA$2vU=h!}1XfpT z&mi>;KJN90)T%V(*UbYNX;?Mz=N|QVCDI;^H>$lrPo6R@GJhR7OH>|*>E#4!X8@3> z)>HA>+GV4z)boIp8R`20eF<+Zb>7|*2a|N>U=f8j;?(|B!DWe?}pvu@$ zQZ?lXE0Yah&kjyhw&vmYDXxM7oM4j+X`IN(^B^>Fa8xHlBdtH&^O)%-bGhn(!o8<+o(JgQSg8h`Ku>edqANmd3cH)Tu6p@EKt#EQlXC- zkexBWuzP^@jV16$RyF2umQG}Aop12NGkXTR>hU@L9?c`zBlG0Ert*;UEokDWgqM()*mwpgXHP zisjd*e`U(5&zfxE2J+w>ST*BmGm27?T+1kk*Ey|F-LiK8Tdm!YX!ER0xw{LztvZmS z-*^Kgx-_4o`?)UT+@y>j*D7uO)mtB5!0-Q;2J__T62*#<>NbH4`K> z8{m~uxpm|(Dz(bE?^(fN9qbmf6_(-BU4$P02uAO9=*&ZF-=+I>@94XUgTBu4PP}GzMKITCh+UAoq%p)Wzgoc6X4|jzWm$R zJ`B3{Hp`e$4M+~+4o!yFaT#O zKT?%ij%cH2y@!Ff%!3xsQ~RkI=1J zm*L;qUo(qnHDHEi*%9z;&Y!INw{Lt2(s(nwM-z9+n@0iZ2i>8-1YH}&VgtqIk3xi$ zvSmp9ReA&sos&T zHl6|!-A#?iAOGNUA*8U5sK>xPsLRN0yTfr!j^QkNY_?`=!clVYO?I#%%pl)?#Qr8x zI^WoFzgP~O;{6UPKvR$(S@DuLO+Oe>NV_o7L1lO~rv6q-rZS1(P@OrT8TY1@rB6T8 zEk;G=N!_NgHp0LS?y>gHwN2z@H-0!*6;zpT0mC8$pUcMPY`)|Uw;qGzBhq>qe^qPB zm~Cg7ewjdrRQcf~vO?$mNRKLvw526;`##|KdL02Brj206t1&2K+;Ucvs_|gViHZpA z&$6!CErO#NS0lRIZ{cx$X=Wc`o60(R|1uN`ldEqR_<0{ zxP$rW^4EKu>~57@Q%(+pvG*Zzl3QhRsh8&>SY|8$yd!jRP4 ze}BJZrV&h%P+yGhQ1a6MvHpV#z`ph{?1|dU%{(nS9hP=aAo=BnQ{Ne(V53ZRd#XwYR8;GzyVao z$8Fr_s_3|&xR3jcN{(x|kNaSvxRWcI%TS`2rC63zT46?Ey(O6yE)|w2mMNJgmKK>P zmL|67`+fSy56H~D&vMT1?B}qK^kE?Ikd}O57g>;U?QIw;yC$6bpn3)kdxEA#`b_56 z%AQ98K;QiI%?#f2o$Fje8?yMi6^>*RBziU4K-0uWH z52#>^^^3CPSVxe*uRt_ZBUzaK>y~isvdN%Pn>^eEh}=`zcIogZ4=OH#WCU;iwe@*`ft#;S78Y{*0%Z|P*B4q_A)Sfv`##f1c z;?8+7;@DzZF#uI7Li^24=*f4rz)f0yL^vj0S<{5r^aj zFw!pVIwwJDsNJ}ciO!#b)1@o>-$uxdJeG=9^5>J_#XsZt$MHD3-aM8i?N{mHGGqt$ zx@R6Aj@|3^$+aE)HCKDwkNn2|(;Jz7P@FL`k##Z*`QRYeH=n`}8GV?|)}6vS*J)?c zhyFO9aSB4{@;{J#@CyEGvJCcGsBB?U_B^Y27y&)FMv%~aj7p@~em3CC3> zY=LAd{%=BvQ;g@?;lbHZ(2Q7kM@IiLKgci_zj8uUm3WZYFLtr#n-RAUtr5lVmh&oP52B?T?K?0y-gpk=iPI zTB}``x^dfI#F!pg)QH14J0CGZo7)=l^9C;(P$mL;ZBd1MpsziDF}mp;nx)9+%{l(- z;u+R{t%sQCdbYm*@JWl;FMs%$IAQu#Gy5odQugWM->E~|+S8a4fqfAR)m2rzTOuUFSn z;;XT+_y^qGlPUOwFI#h@;7y+UF=JNbFezzrpbdyJ_X?YylAkkIKkLU z5a`+V9R7&j{CYRqYDck5Z6veK@_&>2A$sX>*h~h0&!L#>%`&-+>qp+#UilBvi^w0Z zWK3W-ES962Egn%>HRV=k_UtKzXa?g#qm~PW&kN{ergEbfk~!H{7ogt$-?FkA6<$B; zlfS04Z^o^Br;3?`@+F%*rt;g%?j>eWTBUN_2cP9hIeMP>iFm%Ydk=?M?QQ5SW)~lx zQB;OYCl5N@z=2i{F$uk2Hw#EFA}={z_nBFK6ol*BgLi-33sshG3-N+4&aKng8_T=( zxU_4`%OlmcvW4`0*7WK8UuuB9L0G?B1tj$RhMW#qX}U>fd^B)~V<%-4OL^@d)U%$9 zwv-tuu;ca}rbBLm%5w0O)hpRScKTi&j(=Jnhgqz$I?1C;dD}icvSe&BhjdvxGvCT% zLw(;feX?sgtD=U#wXv+IQZ}CE`;Sb-PjrBSBl)EqieGV?&ql4~kiUDG3;$D3D;$1; z*RBll;djGv3yxr%72I0kkh#71#@Y`2W6-ukjyHgw#=Op5pGU)z3j=}YcNl&vA3Du6 zIq8Hl$H%2f7Ys0YqAicBbg{avar|2P86>t%x75E$9gZv-V!Gtznf#&Tq}L-uKJccI z+DF+c<3vG4OkA?XTy>2>xim323VUAA$!{}MuqJir`0!sCigRTb9awdfbB}m;aW}u; zXYao#Un{vVodL2?WMt6ck*b1Sa!TI3$kD0M5?PlN7&WGxk{ zE6wv+x9tazhjt;9+EWN1z0UEDlgl}{Lk-BmNC~ge{8tQWdsiv>9Rlj77rr3HA+7(u$oJn=8)h+FdteYOr$&;sm?PbKg%|$@Pl`7RDk##m?pxuFRe6wJemcD@8@e@vKu+R%`7yAHViZ}TM8*@HjA8} z%K;rr;rM3i(&Jx}+2hOFuh4kCwX2$k8Pt5vm>?&AC++RcdCq4l?SKsBNzAlHu7Zq~~dsCd4^1f*D z*h<4eL7yGAcGQ;-mOwv$MX~kjkC+v@wyT*@9+xD~bu=kFx7DG+!GJoP$&YIx>hWBS z?gU}V8x8q2QTHp@uajzP{{SBzWmbfoeZs7ihmWy}S3fsHGOa1gYv_jvj;M-o%x!8S zU1b?gF$eIgLkafO+FSW zAM+ODxUIS!%^A-HLs=sN0Z=5yB24$RAe^nYS z;avfygSydU{n?yPZ>qJ_B>=c5lpX=Iy>3os^(;(SK;`oo&l$h z@hFsu;+|6aj~9o}&kzPZEn8FIk<(SC@Al%GoShMnLGwAtHJWW=?s4@St3$H4bzuvA zcRC?#N?XdIbpXpm&$G zRu2;_#^xF9riw0H8l@9DFLdKwW8UMQcz?D`-^JxKZ>#|;HfkGf<3c*-%TUSk(MR6Ep!_z#FVI@9|HIZNsBGK~xk+-a zD&Lg*u-j7Pgz z&sbRd=-%A%4&8Z#whX!pUTIy4d&CSH^HVBiQzx&FN{c|4{JEFce*VzRvW#!|XtgPF zjdB(|laS0%-O}pxebX(!YTsd9?XI8vx#^Gx-y=3Qe1v|}OcnH%zFOv;?1ZGIjd=@M zWZ|xg3336p3FJck&GUBFfK>Jnf5(bUa!Ggt?q!p1n3}KB@N#L+qrYdA^o--^BECT} zT&?m38kOaSygd~y;m%TTnq2#iZ#C13>ZmJdAPRk01gx$pEe>W9Qvg#H>1uuc3jU zNLoLbGS!xQP+)eyX?z;z7iTxflquK{@fB~0ERW)5^;{13?wVEX|i&R^qabmU% zJ7E&!xBx{oc!Q0++?>mssdehpe}j#a=Ubg}+zBRM(-}~24~CKxGt+gOP9@!6<(L`` z;!;oQfxm^DIeVIf_A-tomPncoNYd}#DqY5bHyKazu}ck$Ya1RjB*`85BYnAR%3#9X zoMWiA%nw03BU`|_@74!gJk0r#c1t0Q(>nEF+8VgA+9-Z{y&4Pt7-s~yz$DF154gI? zcOkEf;!V{ala3%!ekXao4*&Pp7;bE;b`}2+G;xsx*jSDI>?!gm2q3IAFS$$@(e2vIJT1=pc=$V^Y2ZkymOX2 zUqF?l*e7wSpC!R@zc1z;=_fgnrrO1S@hEZw7z&p{ENvswI!~bOU`HAB48QJI1^pzh z0kr*#4Z_``t^UV!5ov_x`$QM79CMP-C2DIktR2$iHB}KUHyd;6iO0BphPc{f>t8Ig z-ePZ_^e6*0-?N!MZX(cFm!b9p!wiQ!{TIjbjg@yeBN8NoLne(w|t-2*)> zmG&;t@^u;9c&0o^k6arS8?>AIMF69#^U9mjt>`J<|LA4qNq+d zZl=lW*S!vT{tJvQx=&eUstb~Oa+-Gv4+td!m!5h|z)QlJnOYxB%hppm1`6e7A}Ns6inkrnN&isR|61TtE>PDX?Iq)5*V2qD4j(m?7iG%#gg& ziGb2d<)QIyT?qHz9Z>#k0$gazjjFQam+H^4kwA`06Ft^kFPi`JwW}k%?K+ zR2~dRLv5wpiSnIp{90LrK3tMms=M=ak$jyrak48b`S^PP;_3$QCa6;Dp%~})myhwl z^Rmz|bO=^`)H9?tvp+9@)qE3P~MedbpF+co}OzJFmy(h&>uK!gJGC z92>aew$Q4tp_PR7r&08pJ%Ba8zRkh5@bG_CuEAqRYawAC*Z!~f;xJoSwHsowDd`u< zBKn^Lr%>w%?~n5)NxR|6hOQYJE>l4hqeFQKb;_2H2iSTo!sh=o8sc|MH=^xuf>uyLd}i&0c?@a~()=ImI0y8F;ea=9+V zc=!$mRbHN%Cx=FZ1B_P*lZgI&?WLWt;$UTuHIAZPI*0%yWTZ*2TOYodE*Jr`6XSLd0u9ltxrY){^8>{J)kOSZvu=dHe?ey_9LqI2Bgw!{!SNN9tW?e8pS7V^h#ji8HEQcUshQCQnP zmH%5i7{^w^qj5GpD>(JM8I&LVd^=hXOW$Y*uy1_{bU%~JH*0;w{tn;Z>kJ@p&{_;@ zW%&q5V0U-!;@U$lxzTf6@}3YE`m%D2G?vZ}F-}>gWtgN!U1VhqOm9#eWJSs988Coq zlYtTS0Tk>AgB3)T#!CD-GhaR!$t4cbqtqU#wC?hVZcH*|A5S@@Gr@*8fwi5Dl6?|B zm=%utmJdg(P#C@*XN>&j<1F8MoUwR!&x9+C%+^=x!sB*G4$RkG^`mb-eCqeenk)5LJg62|hz3G! z;^psb2qgFwyVTYdz8 zH&Giv@Jq4;rF1oL65-&2`lJGa3w85{2A$w*qcyiZrzTf*U)j3JbnT;`R+}9azRDjQ z-Pp{L-%W-kPeAspD^0fGgtoLdx-<-i8DdM@{ktuTdPloAh>zBnh0S^T zzgk#T`x@+RuFL$29zrHZx4}e@MDbJGXJODB9XPqX8HMlS0T^;Bx*LaQ$V33wKNU zShTP}J4sIP<{&64+sksDdh<^oc6OsVpPi$f?~8R@H1+Mdo1j=}QpOKf}GTr=Bf6-T->o+Tmu ztPa@NiX8DxNTmdSXHXmf%aEm#~4fEit#ZN zPmqFjrUS2b_E3q^g&~KJ@@u~CyJL5Ai{Wu0`Q&B%GOy_N6g0%Hg;tjrM*9(3;e@w= zCl~+4>N@G1z#qqg_>t5`5R$_aLTB{GGGUijC+$=w5xYqD%BH@i8v`Yaj)>QjZTQSf zqjVdcrXU0DY__U|8k{N~)dAlBM5^7U9jtXFYrB~$RsKP*%sJ0VKlL;#+pl{A|E&Ve zbOVxjdFzNt22vGJd!a(%zkZc9r{U5-yGoBWLb-04YO=h^w5pI+E%@xU`W(g5(=ft0 zoy1L+PPTMII8X=emsXb`XoeI;$c4eyEb09fCvz>5J1fA|MBahhDRrHEBvN;m?N4tc z8MBBL-z~DsWA&`MyKF)$bXpp_ZSS0sgjh-%W*o`Q#>q=7_e%}ax*C%q&jgj!)c7T$ zR1QDE&AmGWvOSq(l1(!(IhCF?>ALqmn_QpiB&!v%&j2wclO3!j`vL^>;ucg?vdJ1r z>F?|yF`lnRoaZ6aY2`18joF%iFZc5AL2nwznqPy*(F^?b4~W4!RZdM~V2VrT`|(pe z3H~#A)l8RYZJ;-z5Xf?7SCwk>jnZ5+<3FK94x=$nHD6twx)CPUHvj;D%V%84Odx+F zwXei>1!x{}p#Kc|M}i%UcDSu9Bjtey&e}jjbU-|<3+)zyBV_UrZ@ygp z5=y>VSq)?MQ{@>hXFX;y{9VzFBJt8%l9~@x-O@3*i(O!??QgJ1=TD%|v0nW2uo-tc zS3wiB%_FOXHbC;%HMZqZ?nX#ZPh;Kst}Rqa-`bbjAOo*90p=Q=<@g@aQTe+bHb_uH z$z!Xy?SY?zvtcjr(2qBOH3y354pT&C&^|CdRylj(sUy(E(9+(pK~_AbRB0C}+rn|L zu89)dP}$;8F7k8}$o!#l=QyvSIm%#`lhsY~_HZNTHo(jqRiy(TCUceOfEz16bw~16 zd&mpvIU6H}Hfj&(s+4CFsWQ22O;xe21iiAXFAqmNzMNYrFYQKqB_*_}Bp`?eSQQ6>{&I6yEy%? zmuk_wyx#tmX@h3ceYly+bPJTU5xDM>JTm>{KFV6Y0Vy`=T&Bo95Hp>o;!*1Y?Jz#_ zJ90Ai8j>=bgRQ5#e%wtgkw=bk5{#5=q&*Q6jJYZFaEr)e_d!!;%t7Y7pwgqIVxPB6 zvUS?v`8ZNLmjKyZT(s;R3yFm5j7#+hgxYu$w%s>>nto8jNzTGjV8_nLCAoA3bbCxy zICV=)Q#SNL^itWDE#-5t8V;zO;bhv=VGwM$azjiye{B;y^VRzpwu7{Xo;u`SMbX z*O10fn*r(64B7fX=c3)!;%j;fnB^=pGvqJ@0+{?2gK!}P6IpJlL+%WR3DOU8jLh{? zu(ZD(|MLQM2t*)bM_nT>xHV4^dKD7dH3NEh1cM$mF zRA>@N?J=eDFN9Mcm18H$SK3`_Z3o2~EP%IdbQhq7sm-o~K5dM*8wPzU7$(71X2|$A zE**fuvG-whvkdEw)7neYiL_}0$-bQ|iGt+`-^tJeWPD~8f%gG`lsMIOQ$uCCynJ6`tu!dWp+eViN` z$5z9_p~V$>5jlER>#t9-woygU^HA;0zB>$biBRqc`KT%2Fi2<0U#n_TPj2@Hx!vqe zvQ+crGg^xr-P2l%sYHv}_v%{poT#ME@=2nZj=oSFA(kP4Cyt~~SuxSTgrOgiF=| zkzM-p)h}M>;YE6o<;~`3nD(lO-Qg-L0grN4AC$f_t4!BXwWhCm)1@7)95LH%LnJbm zrRzD=>S14jmBG(-D%4b(KF8yBCI)4oE{v>E8-?V|M1G35@%b~%wSa0|E#Xs_6Jn$l zOSCFzDBb=_CM2)(1R{&_rTPlQ$|OG~kl^7R{Ly6j)aJ>E}AkO6O@6nFjt3a;G} z3T~(ht&kj{l`n)d-;3oSHe6W7uSI?Y3KI}c4np=mc^u+t(At4(3e6PxCO2wC)?GLJ=K0)>byP8Ei!Yw3q zB7@+-roGWZHj@}7N8^*(`j4ESk3>&IjlVf5ITbdE+y);%u56QycQc(b`BDaozAA9Z zX}!8h$~~LYEgr`tbn__&5Ln#yA>0mAQ=@YkEl5r*F3 z?q_7d_#Kq5gQ#S9u;nwRU9L_>pD~qVj%0oahECD_L*CZqd;jhRFBua%0H!N@OoP{W zo&=F|4KO}kgSB>RB5Z5}Wy=oaCn=aNx!VU- zc#Rvrc$TrB;*W;hs(0W7VzvXOF%)uQ4hC%UTj%6F*`rf(pDijyg(p$2T~PrKPn~a6 zi%2JcP>?ED^ca-I1gUU8LiFczE2Ni?M?a_X=E{g2$uyUp>uv6@9FfTm4bmB@c{*PRiEkzYrOO+$!8#gES0}dou8!%Kyi|mH?4O+1>IbIIjYx};)33pCqwjKF zbw`>d2(#iRjFikiQe@}!+C%`78(WyV*I#;Hd38B>g{~QpzE^nnwoZ^O8fISz(AimT zH-I{RAcTy2AQ(+F!&wWKb9OVvl$XFtsbRXMU}yAF)RAwBT+(5gH&6cc?JJSu_n^jl zD!n5@Hm5gUIr`mF9ak(v4)cbX6hDf9Cro0hvN`&M);G>(=0Uir!}>VYZ^HV&>_Ue7eS?$Vwzjf@bfT?XyX zs*-GH`S#wVW!o*ZD$Q|A)c^SIp0V8CUY+ax-c+dkU*)J{jLN0Q&S>~Ay0Q0KX0y1 zOl~K!ud+@($WKq&chaec7*TkA9#m%8;#vIm?>32Ap z&woZ4eExkik2($hutjC1BEBl>VgfsA%WfNV5-8H@YGs(@nO+L1Px7uc5&5jYv@za^ zRE9X`X5kVpoT{C99$DOu;~SZV98tq5Movj_4GGh_ z>=k8CB^GN;koI%URC%g9U!AXPEoE^4s=q^rY(xU)#AWT0TlYD{^UPhZTgL`W$@hW( zKj{3ttk#I0r*u-v{a08hHffZ>4AB^wIfV;(<}TE;HFKYhWMxdcS>UoXuHaD*%aYN* ztMT0<5hXG%!z_gZ9LbnoAv^w%&=XQ|ErYJIpF@!GKR}fB}8Vu<-V?g$-I zns1EDb;yc6=;7b~4STr>*JG2ES+xG&POc0yUC7z|4l=1RR5>7!>;uFS`je1Hb8CEK zDYUi^%CTr(^TYR$0&e{}TvxEYJP`-+ji!06EZ++o2A*d-ZEs_HT~vihTK!+Bqcgg_ zt1}h_(bkRBE+#z;{h6oGo!f^0oe@AoG`&an5;&7|cSuTI(};8#B-n^XW! z#h!|QrUIk*5m z+mJvbm1n-p-L)^Ec1*Sy9lzt~XERJyR24JTI9jB0+K6a>83FTieBoN>?ujJJvPi}s zZ1Y4j=!cHA{Y0VpVn?}VV$T{gF*?Wt`u;D-vtdLrxaFBG^^TiPu-g~}^Hjv5%oA{( zGQe6&Y})UYPn)plF!H_hNL>dYS29JO`%@Vr+pMV^Q{EHW29LpX zNR8G?Wql0o1l>;;tM8pYT*U$}0M%-)nTJwSJXG9wxu_d!*m}*n+`^m@kjI;Ic+q4% zN+4D-dz0ZUiITe>tggw-9Sl-5AkKKGM4vX(E^q88R86N)n{k*J$fsP?ZG01ruQLyk1bfI}ZD+qMcmokW4|^x17}^?Hye4 zG4?0jLB|jZcl0I@<`2hZ;T|IK1Nlhlccw+s_FL18&{m&H_ZxRoTs({Fs8pSNUZSUc zlZI6rySvyaClySFa^6>#+~MNdsI;lbS$juG{fL7JJPlP&PE3;8FIu$+#?na|)MZ-% zmDGdB+579zYOMU+$t;!8VYsoYsgfl!VFzG~pbV2`ZtktN&ali5fr%%TZ;-af>xp$O zn%pd1tTMOE>(lAsd0bhlh2W~c28yg@yp&(#yEHuxlcdY60>>=FBUP497U>}HMHPyZ zryp{I3vqbLt5ra!rf;KTyzsQ?#_gFlK$>g$K0=qOnJ=@KDmo`+GhG$sq;61>74(wX zQx57d9a$FE&CpppB=*eDH;bi_4s(8;z_y!w(RC}Q{!@^+GIA@NsEp5d$C?7AvG{&MbNF`oT)1jJ$htL-YPHykL>~#>LpDE zB9zDf;!TyI=iu-+Pz9t*FOH79$u(Htp=eg=jFgdsGCYuNYA<7MQ9dFi$BC!RIRKN= zSR!YtaY_R>nH91q25t#|4KN$2IgMnL;c{U$bjdVpCrq0M=p8mor1@vKAcpJ5yf|cU`h2WVISxZhoVs&#k#mit}vef@x==JE#%+>%l z?x`CpwT+x5j3=kI@;OwfrgFPIBz;kLkC^g2a`R19W~a(8&x@S#twOq$O8h1>oppv{ z<#AdWOOL(D1`C5*=Z%51vAT8k5?%&!t%gfc25Wp)cduI1pAR3J!qt&a5~&>)Phcbf zh126t84PWmE{7t)oJXq^_Qc_YWZiNnK@@ z;-U6*!KvVDD=+GL&z^~ei09(^0J6nv?k_v-ysx9PU}p#S9C z+{#)1?Qh`sYZ3yzqA8ZzY!eA{3-4 zFJa=0Qm_kWk6oqLMAA8jp{-Ue-cSVr zNz*;&npFY_7oI>O-q2Z`-%a50Ju)rQ{%zKHQhCN}7svpVhXgXub;{U1U7da8E|9Tm zD+X{E93)y3?}f82P?In~;X*mSpNH&JS#=@kJe%=&TtMdM@z!5eegY0*y5ut&^D$&b zsvYx}y1+Jt5JaTTgtz@@>UzDxMc}VL2=ubYMW?(D&bwfqG`VEjBwP>jo1#hvg8@0? z<%v~jn740Q1ICFI{5@eTF+&BC<)dmSb5@vf&P*kfiCCJQ^}U=dZd1JK$%gb0Yabo?IhAP1@XPI1?0Un~{NEQT0b_Jx3F4uk4LVAwBS(d{ zdG8n(S$qq+!$TTnj7iZg+4Ji|^{{?>l@P}6E8570#%7ZA)ID$(=$cPF9z0DsM}jB# zJ}#=H<`P=X%>fLpj6FqeqfL3GG@i>b9@P~n{uy4o?Eb)XAm94>5~X`JEyVpOALCs_11XIV^&GoTRK@YK=}B)o7D=EpBuU(nZR5tq-J5=JAQNGSMMm93SD<4liyHHZDz(gXqw@Z|9Ywk~ZwYae~y#?fs z_d+8lD7{8O39SdcD>}-sDLmk#{aOXfMhbqb2eF{*t(ST(GtKbFRie4Lo|2f}RyRoK zOx8KBP54CR07;`A5E_xEo48*`a~Q$kYI(By%T;2QPa|<=F$AAFTk( zr0N6o3A;*j697$n3<_cra0#Y0_I_TJ{92pKzNNBeOW~7naMRoDI_^_5U&1zl zl6%Z$O(M?&Jt{j4n=4C4cI|!=3_N<3l+X@Yi^x8tTNkB7fzUx!wv+>Ncamu@qA{G) zXaAsppFAqHJ-seM@@8$s3^}ijAYOZB5{Dl~r(3wdwGz55q(zr*7To_L zr;w89lVP>Fx1(M5Ik4x7H-THsi}mY|`Ch1#76$y}O(9##9&UunOHmpv26Rn82bA&)}bWS?E8lf6PeLhHNvfgSL~yKPQ(!J%RfC z8q(p7oJ2|63A-z0>#nmyV-ZCDNMv4~)^m({M$7UFnf(P^wVxhkfQe_69DEfGXT=A} zG$8xdYm@p70lSFBET!ZC(2H|@F_{tx9-I&*nR4R+r!`yU%#gYbxIJd-4U*q;OqWcY zVfy67-7wB6<)KyH39~@%E-K^uU{0Zkenr_Pos(eBP!sMf3^b;^HiY=JfJjyIYN8gLA=l_v@19|ir`E6D+4L&LOQv=HL* z9dWW}tG9$jl-7~v2e{SjbO>h(%|WDQJRmamFo6G~N&`th;To^0G_TaABNG0;gxUBm z@hkSwdHTbc7X){)#rL!X-=(s=(&r)jY^&1@)-QoszlA$3$IN?Pmta$c_4Do^d3lKu&LiLx(Qq zw}~pjhtq;E$I@bS+-w~*cuUXZTdk|h3boHIT+eS>@H+(h(M0Z}?kzUjya&q2ROG(I zO!L}MA+lp>hd{>Wlcp2>B30+1(2u3KF5kL!1fqCfm)A|J3JLGbb&KUKgw61Fh~Yoh9rLiB2+3$p zi-g`q&^%Y}0Xs>HxTQ*`Ab_2#CApldSo3#ANH+K8!A?E2)uB=oK?kEXghqg?+9ibK zxRes9n3V-TlN4R|C~H2W3TGp0Q=iwup&<}dF_4bc60g_8>Hnl(+wB`PkiH*^79IDO*l9}{m0lzDT z$)80S8>bWf+8OyG7{m0{4|I;j=UOv8qw+H4)4OH>mz>cYrQA0W@=J=6?8C`)+}0gq z&hCIho-DRf)i%v*=W|YnaU?T=OR1bKSx@tc7RkvDnV=)zS?xHznJ9mfrE`F$=DH1Q zVjAZO2HolQYYDtrk!XY(1}$1q_39>lZNAFTcDyuWg7Qf5Z@4fKSjRlyLOvjb(Ngm^_NuIC3XAffr*M(L|zwr9sO_=+4qrc?)FuMZ89|a`1_z zhLr|hKVx4!rh+YSi6TSW#G%>h?kZ`&fCQIDAxK?XpCl!+78`bL7ym0(I#pfZj$eP- zEY&Nf4$SIx07br8T5hzF-Id$|Dc@e?Q+8{4;WevEB31!;8+6Zsy5$z6BqLkK#03yG zp~-aD$+r?}Fj`6SI;`-WjAcTfOPy={5?3szPH+YXw4%9dGPa#F3pgD(0>vnQaz+_T z@{a_UQdi=Fv1~`XVjSPRQ#jIYHRU3{(DpoBK9heuZH%&_e6(ZtBB(5@p6m$Jrbi{+U95|*kPmwucLa4)sWRb9}=sGojFD2oNrVPr_cF>j&a zO>_Zp{ta?C!CENmsZo{-L%83OT6viAE9Xs${7bm%0yfl@;C>`xW!DSb+SnYGG2KyHso_5mc7^U?175%8@qT0)Jzm1xQZ_2rMA z=uZ`W&0<;F42|vkqg;`jf;rTCqlOH>F{6uc6X}nJU$8n$#510|R{|dWo4TeSYI5gY2;M92Kes zZsj$Rr>S9pwj7OMCQEHBqY5v=9q%|<7R4AnV$(so7_q)5R+Wc=wk(ycy1#MV>6S(qH@tu;(V zvrfl0qi3d5I?PTk0Og5{Y$!_5_}L1$cSpl;wyX=^5YV9b&Daox6d`swnKdTMVj)VMP$bV zofUnI)oWL7~oPp)cyC>PW+8=s9oaI<(KU zkKFs0Qzose11Y?rwQ{8#)gfEC_Y!cm*C3`*(MT><+oz`>7f+On3*olNXwxGTb;9fa zYJfGUQiB){m#2m1Ao+VzGVZPRXknZ5bi1!q@m>fhEtBY1!CHCzEvPU@kFc8lKXjMV zs&Y9o(Cpee+slrY;-OIlV$x25bB?*uobm{5M~rXH9Rjw5S{pMWd^+#G zrI7`0rKy4tIy2GjS*l_I?(oz|gnT7=82r(x+$6yiWf|YbIgo8crLUB`*i=C?oi0=Z z{U*OL8_h~WhxXaUJclCr=pv0TOOSMrZ8Tk|6qYFclb(xJ*o(_Z8-~j-s{z_{S`A<$ zyQ9?CX2;S9TE$#T4r(xC{_g6GEcxLRWX4tH??GKqb{p276^WYQij%kKu1V&Wv!=}d z+>5!EYbJ95wuy9I^fEuHNpd;|U^KvSX7>^lNUtZgm|&(yjwQ)&AWK?S5 zszd9=(fngXi@&NjAEz3PBTh^=E$lsiCMH@%Gg;dmS>shX?GbkQouNQra%Qey){M70 z!DXGhqk`Q1F*w4lDwaYX*%P4U;;sOjzqVGjZBT(*Nwub&Sb?gXh+u|{w-uEUsjjqg zr`dzGF|V*kVxbgw>Jz35cN{SG*RqHoMgblqUc#BxsdJ1h5z_&xzxI05p1G1RhgrmmWTz zkw%o=OCz;nE_Tuo{#v4nw-NxTaG`K`Eb{kFw62?*`9oioGl}mOwq#0xP8F$)Ksm^3 zDm%J+vys826Xa^p>k-!{@xU3`=;rb(EnBsg-i2};zr4ML z=eF0XLrq=ynpG93Dqe#AakU&e$!N)6Uxf%hMJ~u+e{&nhRA`*>Xl@ol%C%Gn^oGER z>4b0(WE7y$L@kqFZh`&`lpHAYNFRmW%W~ww2@rJ*m6+_>F+k4fI`d;H?-i;gbDA@n z(`3llF!YpDy52r&Rx-Gn;=#}eW+3}>mcr8oaq>(@7I%Hli6k6G2^}$~^Xj8(S1MVt z3N54D2~9WGSrSinW&uRXV^Bv!Ga#pm2<@4R=CEw-!2?@OnfEoqzCx9>W4cBqCc2?p z98~s}y!B=l0^8%3XU-$$M~yO*P)&+!LVv+-C;@4;j0SOqI$No(beqDtUK#V>yDTFy z9&*Bof&|A&tJ)ZIle(DMd?Cjn-%_onvvInijV3H3lw&l*h&B33`UfV3YC#myRUd=( zEmUZr9CzR%yYLR=U0u(Mx9I+cj`60S<1U?GBp2Cv63k9oiWS8yNy+6%9`dXo`S-F4 z4#p9cmQ5ElGoQP#OhvPFMvyZ>ADW0hu;C!>s3oM@1m;B?x6`9?f7dkyqwaXMf1+Rj z#FzIt){4I!^2kQlvBem1r>K6U8%U)@u0kyAT!n(x;-}9l#*@tOuyo@>Q30>_Yfz^9OM|}Xqr-ONCz$gwk++XFFq>M#E3@FDp0WUda*Q-ZT~4y(*n74szcjOl{Mv)9VPx!foA zdW@X;oC9O>Ya|Lpjzvd*d$D3yA|VBTUCC{N+R#^(ZicDK=Ek5o)J>1pb{`E^RmmuTr)&;jnAJoQu5bax(BhZM@(i?C7$@J> zwH6y~(LI3jvEWe}qv2Kl4z}(~mkGLg_i-2EW!Wf1pyg|CI_iE&q%5z6CXkeyhi0F& zQx57@$OCj^g`bscg7H$3f$PH)qg{BqUr%6%tj#oo+VXEI^^IT zRm{nB`DqRo+ySD=GV~Cn+(s4jkZm0@{2)-EC6H@Gk*$#Kz~kUZ;W+XJbK(94QoS`y z{;GE7eXcCc<<|hTGh#c0T0^&7sNTz(%|mJR@(FdgvifVULoPQ(;b_niY4{l}5)f-f z8>4do&P2F7%K9XB zXDH_ujx#4xhpsyqujiIXvtfYzjR-b8G6e8mtl+k>*Lxxd@~vuVExLXUL-X6EDkt+G zDy_kzKE~DZAhICO3D>)upbYw-MRyI>mT5zi%VbJ@tItR)FBR*LETNt0wS#F!6Wlw& zZQNnm5Nr&{?w=x`RyMlgqA46AhxeL(xgKK;KtukSIFd?-$?E0Y@((JIMgNGvL}Fup zMyO~Td-t*AbUC-hv}2UGqxeH$0al$Y8I2T`;g+(k897MBhuxl7t*azWogD&aUslEd$`=Z z1%tj#cUJf>D_xrPhI0NQ_eOHdK>`BjYaqnZrds!JCZAPF&O+1k43Uak-W++Z-oAj2 zfV?6nUpM_ov|_hWm}^L;1H1F_4}gi6Rb?-E?IB0e|GJqkN$)`khm~ch@n%s#j>ecY z^U{HqGdiuLy(()e^GC4!T`39L_fo25!a&)dhi9w!Wu8`|{Q2OGEmB(&&%$N-<56!s z6O9iOgW`S!=^7*+qz%(p`8o|1;b@YV-Z=D$m8qYwa?&rA)OZqq?8;KKz@^HbiNzEc zWwcH#Ks}~&2(8?m7&;i(O^>hsU`}^GFdX=+A+E&8Mv+tk=PE(fdlD?87 z{aID6XH7Xd@<+B+=Y;xq$%_`#3B;z)kevbO=Nnyp9ivJoLUl;n7Mknl&9Ros@#gFp zBJ2!m7DO2*GYbG$FqG)ClNA$=sY<4sbgJhKz?~%#GM{pPbY{nXQGGa+St_AEVadWq zVtJJFd_Bd5e(#)x#pT`rXppjuh;1tOI`0c1Y+_CcKs24%5ON z&DY7LTjgLOWO_%kiR^d@>%|VQs;p6j6$>JfB#tP)5r~(FANa#u+Kl1yN*^LC5`fOT z%0YR@qny|lm9SAFS9(i~*yPoM%^qr?WmFxGH?J;=7}*NuPP}R<7fl@u%X8tXaI{8@ zCN#nsmD)fyGWblZKWL*CYr6CZ=U1n)*Oqq|KpP)5G!x|AcevK=I&|4VJ9Ou3kFs6& zy*#cAztp<#O+uMK7;I6#A==&BTFR@lwC}a;&`;%U;s40R8FSdQ$R-`tZvQ^+uzQ2e zY}}!E$4b9~;r~*W=VcKi19>tAh?$jKC}~9EI54M9?G7$L{s!8R_hA?B^BER-@Erhk zKC?vp*O8P6;~moN72f(2Eer``O=%&|{ETuDztHNDVO1dR#0~7H%2gDmZx7*h*`$mf z#-7{?d_7na`1+jEg_>ZjbXQm7>vAW~K=|yL3Nuh1Oo)B_rBS2S6x!d8PPG_>jOC0cL zI&_?N0ap{js&I)}htDNu9QsyI5xwro9V`= zOp5(q^Ab-o(R$m*xv=L8ngVRa+vLf`rpEBpahTkN8{}@46Z!n)AB96P>v=$gT7c|9 zuyc&cOTu4aAC642=>9*L2A`_R7%;`T!f4pg6K^PQ0xBl^ysJv0unMzQU}RhB!{MdP zjC6CO=ompAnG?xc3CVhxI-AOK^D{V@3T7JWP&Tc!5^;RGoHEAlE#zPlqNVLEi2pa; zrf2IMj_tx(Kxq72nn4U%!`eYYA4HxuPU>yr*KJxbR#BFE;yHw#GOhz^!X4#k4{hNG z3{%b?5LlW*sVR(+tc_lpsosJ>TC2j1aGiB}z{M2OJu4)KdOhsnQq9 zyo3`5_FHhQBvQ1hd1R|>Kg5nccOyswx}on4ImasN4Hba2lzS@I^LsoQ9P+YNp2g6xQ8n9K_KZZ^<2nG`E3Y?21wd5E}UV|AN5QNsXUinr#b#8 zN<9%o79|s%kmCst;*5x(9<`M{gu#xw0W5`Li5gY6sLE`)`7W454(eV+dPx$xo_m5s zPv$)ruLFA{>0^RUozMnT5E<4&l{}`MX^+dlUBMlcop~&tgiZP?=}r=Xg~&A5;}SO> z+RxJ|gfA88-1y=xD5x$gD-QNWBtoK%bT+-Y3@N@W4^#3a1z<8>w*)36=`-BzL(0*> zI2U2(VhbB_v9=M$twbkxjRM+^nm%dq56nCOkFflzIiW!1>o#1}+H6*?iEu#WTRbW| zLSl9P*LPHq@R95fL1pgY+45|X9;t+m&ajpcI0!y#+|MmT^$$cs^`RaxVev5ZIrn1e z@hRYYqeTW1C2Svk>rpwza*M|p!_ zXHvWz&G4p4nARR(p?0Y8d7UJ;1d_O_TM{*;>IfY0wbK~lVO61xL;?IgWWJu>JYMCb z@Q&d7CqCTT*P7TW9&z+sU7$36p;D30ZV3r9<^5%*oJvxUp_?3cYS zS`#Fp5>7j!{ELAlBU!$0iS}9AIim#j&x%33=A~lDrANw;d&unGV@zH87kCNJw#%YR zMNEC`;ZQNE&nr54;pTa3K2C+;T22xe$ZUwk<2VP(A*n9ZYAk4b@{r3l4BeSK@wfqvrIe3Q&0yPQBohTtv?#- zt{jAT$@>heC%u+NxHjUnv#G(f4nrrWrJ;-3W!>X%mt1ddR)96G!P4+8(`NiH(*x?| zH$bw`HsTTM|4BOUz!>lUfj@ZTE^%=Q;yfbG2zf*}`|Pvt5MswZJ86a3v13)V(nhT) zDt3$zRHfQ#v__1Os*$1<=QG~>^ZvYtISVp3)rI9#0w}Cz zP;9>wFA(BgMP{}$C(5dgo>WjvISA4NMsr3$tL3X`u6R9hyiEJh91Y{!hfC%mMnA7d z{x9z_LN?r8fvWr7Ou4E&kMg(S2YR5}>S<#vUUH{X_{JW{Su1`dR-es@LUeaVA(BPB zO3$(Zfc$1S5%T+^5ZzV`<)TrhFM63uZv_p>)#ShGu;+R`cf73l4_nC^p=#Qi_VEGX z{h_$p`nbU6)mXga+!__0X4IU-|fi5vp!_HsWZ>jp4BBZDcysz*1UO zIdt{ZI#{wH(1HOn=y$OHxvBtrTty-u(Tl<~|3W7w9v4s5^x_a~3hJVKp`7i$x^;~l z+=GbL)P?3+J=G9>WlXB}Z?K6-lir9-Yk4RY+#S-hrpk~;aMWI9{#{U2OrPo2c4*cN zLsB~%s`{6EE%VtL$eV)FF9qi;r623RG+YRfg1Uqbt3oo#eV}byviy+?`)Ru4Fs@oE zNk$L$O&;Ad3XsY!yz9C(NIu!cDR`nP9rOEr&Kv=Hb~(iroh9goCjrrD8YWRx zF~Mz6An!=P|mN4~y`mGXA9ne5^4NO|v9WN$UyWt9c4J*rqK zWjI>usgw2P@2OTp(E%9Y$!3+p=*y9nf=zLH_GTGb!Js>QcOwfbiu1^b_YJGQyP9a9 zr5C6uA`2vtI2_MM++fKGvE&yfAQqCV2wwVxi z7#H7jbyJ+x*WDcN62+k*e zxVyx(Fvw79fE73(*NA5Z`%u;p|0!`Dc_9)=U0DeUd*qM?M{zW-Z);@2@ZKh}K}THQ zO9!l#EX4qU+D|nH#}u(Rl7b3<02%LJGZL{Vk_U6}$Y=<3T(yFD$rBU%H z6*(WDB=zbWE|WdGL}{hSaDQ~y-E-JL^dfaQ8UZ#^16V+&A@h&o_X=o^^>LsuCR#zg zAw&>&fa9UN7G&x?BXb+E?#9a5L}86C^6)z&OHO0ds1XybDT~$yzti|JvtlPJ#7KkC?H~zp0!Rbtp*N#Yk#B($x$7{ zsqU&86_(h~WG(fA*@RAJE|SRo2&w71ZJjjE^%P0lHt6yPM8d1djw5w7e`$P)vqbSh zBa(A+d$AC!WNCAsSafnV)uvK5q8PsA?R9JSAi7vh*MR z>U#y-j+3?W9TOnhu_;KLtmjqHE)=TjD<&%&&z%eyGY7$CMdbluq($mqK|k)*^Sg-8 zY{M=`li2tkWqT#5tjBqJ0*O;EJPGoq>^V0M56321bI?eWxJ71y^TVT&E4Kdldrf9Y zG^#0w--2@TacNA@e{B#)knW8KM7Ax`HXYdR(H)=T$c>yPqg5X0!jE8xe!qdaJ)s`5 zqxFQ&%0y|wB~e;{94m=#ET+^Q1RPxt%ch_va-(RiG>+mF7pzB5!n;%8uN^8ZfF`}8 zx}c!(!#CSW$|H%s<;j^-D9N6)cM$+ zuT>%W-h~(g-ArZ^;>YHLgtNQZV)8KJ>)jUKclL(3h)Ofqi~6ce_DSI-a95&fx7kTC3l+| z>KyB^%Vig;tG=GR4P~5pL2lBqnfoJCTe-Ie_tZC2pq~b95xj%-6klj1@&F)6ttj6v zg{TiIu%Jc*z;cNjQp`0-1;S|!+t_*EW(U5_LMgr^M&M|kM!gl%Kh6wZoY*F-$u~7t~H#> zxuh{Tq(H#;dhtP-@SIIF3H4-2iz`To4tnMQbz=J*3l_8PxN7ft>RwC~(kZoxBwANk zF?ESWIG|_OKpfKBW3C?66>`Y0Da8wMp6!SR3YOA_6#`6Lf{8*f`y*ANfHF2W)akYc z5G12#Ic^euaONOll6{d0+DbWT_Ruz{kYnMuQ)4m}O;FA>DVmc^=vMCc6Dz_WDI?%0 z_aZ`X>14WevfPl4<7bl{rPiAF{_wRrvLQD$lS~ zFL1K#L{iBHuZR=-8c!A=bCg5uJ3DCi(yem!7P4CjriXgUM9I?F(u{x5-WcO#8j5pHb3*m6^!V|BD!QMXOE%)* zudIoXNmM6sFT)CI5}pfcXfIkT8j`-%0B!$Xpphb1528MTJ~teSuA9l>3oudqI>V0A zw6_65E-M-#ZZ_n^iJ(`9z%(CZcjkO9K-@dfc*zyH1&qq%b|}7^^qY!i98%pJjREUe zD1%$EjG;MsvUkp&3{GizU2vapyFSK>=sr{n+v0uQW@pQ}9VBEHY&_%(ka#yK4`(vR z!r42`E(jrQ1ByXAC7qHGJ(?cbA1l+942?UwNb(&I<@S5T*dUCwI`YRwQ2BLRbF#9N zrirZOs(hILw`OW{r6(eboK>XXFwjj1MKZU4nW-qS3_n!1eUPdonyJky z%*RkjtEOG$Ow(PDyG3YbN(kVHY%4cj!0~2P!r+doy*^Q$TF-4YY}`KN-;}>Ugt^-5 z1v}9S`4KWKGA2rHR>nB*Ru~Pnq&0^|pIuG?2=k8@HHdnrfI6F4c&pT=a#s(+1m zEwRvrH#K)Qp>7$dba#N1sR+{%;}il7Sg$H;{fu<^l!KL{w>bfAlKj3ri}B>k!f9Bp zDFcwlTzXHJs9;YqxVB9|D8@II*}kbFsYG40e~@z|bD8duL1SZ(y`(DlB5G za=d-9nIZybiJ~b)2br`!KWALi#l+amEUXAd_-#cIN zRtY^}g#4C;33~q$_S9wF*%REl97w{va>$j?{nDH=QaIcQjhv|}Q+8LCKNbagIH0RE zaO!=iuD+rDWND5;r<_^{;g#xAtzjA~y`-C#;BL>Yj&_RGfrU=CU=)>BM^CqNx@cOKYiu#DNcpQ&KU<-s2oWH0D09AOo=mVJ5OU2oF&~0_yF&X|KD3uF+t3>)hZK>fZ zAt2e{G3!7j)dTVMB(G#EDf5-iX#~|!K6=W1PVBc-#%nbdblq6>1&Y$MtI3@K*nu1V z;0k->c_Tw=Hw7TUs;o#vk^^^=y@~i`!dbgJV(I+z3a%Wp1M!?d=&17D`NIa>B^^@s zR<}N$P`UilU|}mTD^~0@inT*Yp}4;ByG+!zQF128>;?1OpEmB{$&mHSViH8Z3&GIx z-3G*>TWT{oMc_Q$h$PGkle2%Uh+C5`P#qH_T?HKU7kREShXY%AOyL|CKGXb7ba1?4q9AP`l#5V!}J(?#yqcVhh?ZfUP}hlI(-851?De z>)v3QN2^8IcgaKjjvR_^?P0QEs;PCi-^~RW=6DH7U`@k(3JsT$T~I-BKVu9uSCKo2 z8w$K?fbzC(e;`={JSocI)gC>V+e54q`5{eJyR1!yraZr}B1$q!EB66+LSh<>17-CT zu=1`YYKCTgCT&VYGG?7>@Zqh@#j15P9PNx?PsM>SUeq{DT013a@N|gh0%9~;_r@|c zM+7P~1>vME8ZPdyFhAzT0h9(*%H!qK_+^E63v&tR(6g;AZ;#8(C$QJ%ialS0vuuLhT7R2Lp%RR zaXdnLPEv;9y7*lry-bMyqsIWlL*?!1_&Ktp(=%j5TRpBe zJ4p^eNooV6LJ<;ql5(kJ>5+6GQ5}gp^n%ValyLu(Gm(r?HOV*X&i@tKcR-y4YOhtr z!J4$dG)s)Y)H^&IKP{ayHBMl*V7Yq}<%<%Xs z)g=Zw)1w=rp%*LX^Eo}S13Eo(gq*6hLgip*b^xwfYh;jW$czU%MDwwbbMb|dtT`ew zrRr!T@5XG@>Q%4DhHT4dEMIR1aE_#8v5b##X+PwU;IC&;^VtsafU#3P3o&reA;xub z>hO(_$w7-Zw|uf6R_U+}c08;{Z{RKWV9dhAmeT!4zB{G6_hK4E`pZwBgY8(tklB_! z3AV}05$nnS9c5B{)MTAN3PEJ8DFh;KlRaMOD8I8ZhwiAGs4$g1Kx3Rm@Xn_y&pS&` zTPWwLqyxRA&XqqbEN{O$ClbxyK+g0tqUG6TG&uR-0Z2}*0=nE;k7HYWf9ZV`HFbv4 z_Xr~weIoSSgCIV>56U~v1+a+-ZjjC4p{gnw%qd@aSSa4^(i5sE$Bc?pifYMgp`-Df z(3W6{5@h35LmJs&jO8G}+YiZy^NNJ?JO2wGe zR>h=A>LKiTnyZ@SK!`aGtHAD%N$LWN?pLa1M|@3aDYXJ;o9M<`WKy;~Ze&Cw!-{(# zNbPxIZ-mKx?=>K6p@QJ+B}Gcv_7Ei(lnv3Mp)MRmTz7W)8&hun1DN!o{VXhRs%JYS4FH*o=998)~a%z96t!z+Zjoi z{FfxuMn;uTh7$=;!eySeVTcxONQ&e7;xuZwPgxQ7{YeO5~W_DjcZW=Z1%=Qxh`?uWER3EKG2~+pg zmJ(&lXG!7Pd125tbF^u63sY%2(_P_#mEZF5 z{Za=1u5qc-b{Vl8`S-R14~4uDVhAeTMk>@zgawk->G&5FvU$MZue%AQ8=5r#Va&ge z@Ohb_^RcTttX**hf_MwX+f60*lqg+8?0@ph+5leMT4cE8(F)|%*-%I+^(=FX74xL| zPEHhKx|Qx`(*F312kVET0=}eKr=*WIXqMB%@XDU2e01jWMAj9Wv!&uQHhsQ3D+wY3 zQqs2}sDDtgar@fv?wT8%3Ii!EDR&YL0`qq8TI;C@i|jwqjbD{#4vOHqrC=#Ew{tFR z{`)bO_2^f~bjbxA9+fj$+sh3M@7(|dW=&-Ha|V3fK7!}cI&x;K+!<0GG@6fA2l=Q~ ziaENFqE88D9xiu?Io7BG&u0o5bnYQQcrB0rZXuCdW=HeataYrkaUJ%*rz_ue`VEJ)|!<*|HxaFc^=@o1i-AEQaSXY$&rZy2D_J4WB5g4d5k@LlC|jJ$~! z>2mThuN4mmnM)TKkYUf4%s(F&xT0B6cborV8c6kL0 z{bD9fz2YqJ^vDlrHXDF6YZt*C%}|h2_bbG}pLf}4(Nnd9W-*$K`r6z_b}irqpJq(E zcpTe!@r)^geqbryO)T`|IUuaK42W2Gi!psv1EPaDe21qwMP8`_Je}t9@Yu7=@rovu zayAn!KP;a?Q*!xk|3JuScN2Cn=Q`h={*-k;KFb?x2cY2JPY;#D^?_r-H%wdUHB*#~ zhT2^5Joa!KkL_Dy_KI%<=J{m}|LysX=e{AfWAwy{9@tsouC79gK~id&ml z*h32Ph>0X4Ss%`F7CZ(NpS3nX+;)arvXAfj?PJ*j27;$pB zF)It*0+W0+he2PgVe$Lvmn6r|G252eyt{EN%loM>E32`NcfUH#yi&klJ*dK>s;U-= zVFHfVsJuhZL*9L7W42$v=Z%zD449V|JdGVVGAuf#zMsRhPnC&0bsih-Rm5YxKWC(m z%*@#Q4XYSJ=OYPH3GP*6fp1bF*U>fkQCkcpCvP^;J|FPc!g?&M{Womo;d=~nZwSyD zy@UBTPiNq>a~QQ6~@Y2@L* z>n*JDKT09VUp*N4@FxCCmqps{e$Jq2J}myGipDHD47eIyH%NucV}CR5GQjcOSmFZ;(fkKOhOa=56g~IHmm|v&~e0 zj;VA98KW+yC*s$eS(q5kZB%DKd*K|5+I5Eib~q2g_R5cmlrh(M zu6hf;yZX!%Cza;2pnIw(NNN(pVFUR}#ZExZ70$-TQX5uMUNg$%%KVkCq}=H?MuK#{ z%<#XQB@3yV#b3L6877&OkNA%I2@8(sBP@V+ z{A#IgWJry*oJHnXobvk@OmhHLhPmdp$R*;NW$riNXx}(cd#5u*me`ENdA?!7)=Iz5 zv;f(s1+2uV#+qu`4XgN^VskCO1ZV4-v)5mb@JPk7>^0>m-_3fe0JxR^>X-QM`VA zzby-3=k^~0pTHELUF{|dT1;f6-0jJ)k$te@o25rddnEyTj)R-Y%PY?n23NctC|@XR zda&^oqyk8p<1n^OkN4AtM7Xu8>6Dqc$(f;ss_5IXgHa^Qgb}AqXEj$c!D)SeK=I9P zbB0!{Fj58t0u=$Enf_Eb)o3pjQ3ayf&RA+Y*w3J0d>0=k~m+6V%|GSpqk^v_H(mIv< zGx#X1`>j&>Iouj(_0x0Zr)6L_{tq_4OqI4vc+T2qZ)dS76AXvUJqROjT3~SOy9@+2 zoMGS_i&*i2?^x`=Dd7DIE*D8^&a`_bK>@px5@jN-{A6Nfrdg)I=$OF5DwmD1a+64a z@Q#K6<%t4HkE>AI{$QRfo6B>r!+CCmN{cv|&3_l<^U?HUST9Nm>6)Y2%IN#d{ozjl zd9{iEKBDlIyz9x%u9sjNj!i7e_dcU6s>BLj|LSqcO&jbu=B+1_N^R`GmWCKhYs5CX z^Z~xOz&8NJOACcVOFf?RwBoykE%>nb^WB&^AZMkGi5wOA?@nbrTrR|(C_sxvJ@aV%xR z_sr9AJ+CcMwa}e=cy6hZd_Pxb8z;U4J~J&;U+>G09sdIqp_T$|LO4~m4+F5>#GlEs z4(xDA2n4aU6P(}Jk|hr&^7z0DEPUG*uSrt_7jK)uWk6azO^R_UWmk`FD6-aL<$-cDjA z-6rsHr3EXg69_baX$=gn{K)X%bb!DoxmkHwPO;Sgj9vXe2C6*o2QREsLA1AaGg9A7 z#L?B){CetlBU_%v1Lm5;SV}^M@lSz4=0`T*kp(mD`ZC!c%^ETEU3&b z=6?VmEof*&%bNbc@kJZnT5f^+UQpmmT4b@rr!QF3_6X+bM;S_Odr~GYSX!8WF&s#9 zsS+rAf8nu&{z%UIdV-I9`-+w1jRxU+=sZZ_=X`0YH1*6J_IkamhlDbX`TGwxN8>*9 z`pC?FAR$RF_LmLMkz`LM@Z3gaRQ6L%dJ4!MlZZSPI~0Q-M>KbA--Kt%Z_9v5x~i)- zUiCPcLS7c0k36&X#u6!hA2ZOLRgmvf#kaqFYoy7|PV6=831If#M&?8`;NB=e6=EPf z5)Y4oGkZJN|8l<+>fOHsoA~}SHa2~T5lIJRtBgO%hN}~CN34EQ7s$-bB}*SSRz{mxliu3_{@h64u1;e&HSB3_r1VsrXNRm{$4)@Nv|xFcQe?W zJ$;pA3^sCUuVk2HT^1|%-vS7_hT^m%-JpWDAv{gCZvwUB4`4YIslrkEc_y5$LNPLS z@WSb>0JuR<_?Gw>f0>%ccM((;b{ToHt{40>Dbq-n zMPKv&_sY*safJZM4CLVolfdz|Q|2r!CL&uqW4>e3ocnlTdL3rCU5Xc+KSH|u?;{GR z^I(y^{}>MWK^;AYf6k2G`hs?PRg{+{<)Ep9Du*F(Aezft-8@^nY$=j?tuflmE-O{K zisX$nY%+d4T)6TIGw#)+Dyq8{AIX^v1g6|zP`|;*h&rk$NXh4bt9vEHO`F18d2yoGdh*Y(Cr&`Fe^M4g+zz;S5Biqs+8l2)ZK&zZ&hWpiqa4xdynvO zXBPyvAVYD4@=|@Tbo#JQV2e-o^2WJW{IH`2R36@urRn|IXmVA+P5ihoHv-a)A4k-993r5Cyfzjr6_bO#z%~nq53g@i#U=NtJqt;SzvO*Ri-|2(ukM6>cDa` z%j1&1$}H%Tf_R*+JBZvE$-B$G;JX#?Aeo(4V1|TC@WH;Sh(>QXAi1_2X8cZ-NnEcN zR7bR>RFF-3(Ko$xyDB3b9W48|c~Zo26`}D&#nbMJD3I^p2koCxyM1`|SI&tUss#P56KwXv2KM=n6-l-41Gpvn7c9IFjFBeoYn7b1CZ741#PV>-0qpaa z9l$hkgt?K{8Bm`7!^o2B9a-1D`9Q*ap9LTPoWI(h<5evLu)R}`r-UA?dgG;-I4SuW z`gePdyg;uCIX= z=L)I0Tp}>t>#>eniL3v2a)QBlsj_|luj8l0rJD?I$Day&)Iw*4YNGN-1X@;6O5bApq zn<2mN<^4Fdfd(5{&VoP)=SdKNx>c5E`@LhgBUH*a*T0qIwH46Zk1|}cdlLA4JOD@+ z#cD8y)vEQ1P39VrnDW#_l)#d|Sm82@Cr$-LSCpZ7?0=0HDE{*pCjU%luDAPGYc?KX zB8F0}vU3O+Z23FTDN;Ua6-KIrPd7~3wxoFR z)@7YE?^Uf0lLO_|Uho`Hl8@ZE;ADC&;Itz>pI}8QL(Dk`HhnH41*2N>*AhQorS5$> znGyw->*Gwttnp zyZH@UoKe9@lZ*R!VUi_F+Mb4B5>_%~@KUb8K2>Gi%u@mMP>V~FM=?hG3cTL<0Wd8y z8C(bJ#Wm#J1ty(09&&8g8xrjo&T2a4dGJ}rBAT+=vYr2M?8&v>&{L?gT8Fgc=z2pL z?ezxo!!MLJk!9^YnruA}jQVj=DC7Qxo{6^&mVzv(vib)`v}~TnP;{%vl4W@Nh*3$d zOf7nzdzW5>VVG7$vXvjyR~gGM8(^`$c*N-cv|}s#B8pI-i2?E=&WMx)uhe7eUM`$$ zSgyoOb}&<1UOaCZyr2`i=&>EJAz%vR(r*kh`K-ZB_H`pv9DNY0XNez{*iZSjEph$w z`CDK???j8t!7(7^-a!i+XCpJxL%>$L(jPv0vPC>KnRlJa*GN&p`D8m_`7C09v#Sl4 z)M*J;CM@RHxZ?~^sT=cF|BH{KKLXJ|2C=jeQNW_{VYaYq1b?kF^XsXKOg>k4Rg%gY zR##YfyfoGg>h|Rjnnd1=to4Ad=kHe7q<$ah`^p)_#ipCgbtIGdbCv$BKLQl~HjrO^ z_V8E#C0I{2RW;xvFW|bxW;Qh;1?)}bDoYOCL|!eoWD#XcpQnsXjI_hdCc+YUY;O_3 zkNJ+r&QIsR&95oyNypzzkfk;?{C_#bH6^z`SevdI!`1l+g&C%TpJwGCAGM>rx$Fnd zQj)IXhKMG-GSy?#oP`du+C!_!7weHATOx>SQuX@{)mr}kPtb6iY970YvQr}+EQsq2 zyPW+9wVLoBg8GswsqfUIZSrwd9@)Oquo2RW@II@Wf3F|R%zabP1!a)zUsE++zRKal z+$Y8*9s$PV8uQkg>tHlgIblw2g`{jMo~Z|ybn+3j4=a$5Sm#fe@IB?lX-SQ?pBLI?RRttnhRUREX+~~^1)=$gCJiReF1j4o z$qq8r>$8i%hby72YqAlr6`;1HqJv3ljgk=swC| zRfg1D6**;aeXu{)tF5x6aj8v9=%5b`lYBQG{u}%yFb?YvBrfq_htJEBiNa9uOXiR3s0c3SA{&hWi=3+F&03S zxdt!zUxPmGmh)uDeq|FB#U@Ku8xU6505X~|krxI(0#skl1CW1`jV2U&iO}cTcTTM`Xd$C_u7N~2Q3Z%yj=iaw7h=h1DEpSLP!>X5UFrK94+jRbuot>r6_rbFC?V?hap?^}QRX{9 z;d2dX`=Az^syqsk-P6XS8u3{x$)PaRz*2QnIP5DVs3js^D=6e3qw=@vmt$WXwmA9s z1py>_nB+uDqeydEilp;(aJOA|){^wU5Yr9y#OhjZM4qf41g|X8?R6!;3WB25eI|bV zt%pzoq$g*>c#CZD7|~Mh9soaB6Z-LA#9!^Mf%exO&303}5>nbMCGIgO9RVn>5SuR# zL&4-vHQ@rni<-*Czs)viHH+%1SXh{}ny;j~K!?>AD!@Q-$x$xYw19NZlCfZY0b3&Y zvWhhS0t_Af3b}Wd%1oH#iT-kLJYppCB;>HIPwBHJZF&QCHb)iMeohP!@&S7xU;8~EPZ*t$&>?5;v)~9-mfAa%RiqE((X*;KulFxN_9Q7<9xY`0PT___ zbqX(#5Bqqs#J@fmF1dpjzF!PD+QgcOpRxzYNQx-Q#olcH6}HF|m44e)ne`LDWm~IN z7RYQpx1Q`h?TM0@4`B6AElw5E++6&w0CQWqru`C;C2f{6Ozu*!ceF8*0Z&ALJh;h} zZ`vb83OJi0og>Rh57nf4dMzUb(&YTv={~qMI;_kUfCe^?t07+H`cC-*fDXNlBslRTCWCNSlPas^El*@(MmcHI z%E*_Jwb*7Mbv{(L^;809=?!E33BmNPC5-(ZR{{8XkXQ$kJzTbZ0nnT1`Ss-dd+^j4 z5cISVQHSCJv!%zV%zinl4Q+{G` zbIknpKMQB6iq$ChyTMgth7l+4!r6Py5VTdpwHRi4P6jK_3nnT%znXl|-iRj{kPOo< ziGUK*+(bn*l-NavY`fa9DtBV2gcJbmCrg-L-wLd7=!VH?7o=kSQmX8Eh-81R zyIRRV)j;NZe}ZM8Zk2PDSxspb1u^|=v~#4MSL^?MUI)KG<|HgeBDSjn89ph7mMM~C zmiB(&>BnCUFUb{L5BWlnm#GLX{%2w`<<~Zt7`0Xb(eJhzUg6#yJIYpbINssiLa5v`OX6me35r zO3T2_R40<~D@G#0?1@#lOQbDn&tPXmZKq+A53VU8U!523Tf7)N^d*=Z2ps-QWC@RT z2RCOtp^T>-rpitXRsq;PSS<++wBfN)QoV+Gu>3ua z!TkvUK;ajJNH-_sbTJE|z_U6unyK2*Mb^?fg-6Sq{oreSH#8!3uJovV~@JFO)^KYk1#CT?Q%vt-tpPR#cW z*+5jEEl|Gez~p_;!%@*R{3cNy&H<8iz5bh@_$SQhYp8ND1@ad^r zi491EJU+S!7JDoKBE$5ecJg8z(>05UiIaV=p^9&a52m*=ABVd$QEV4nQ>+(kVRG@hc|jMRi{u^9`iylM~gf|yKIpo6Qh0#Ns@ z;@?B7pqf2S*tE264M9$jOi{Zn6q8rTDtf6X&XJXX&7%mnd_xCtBtdp4fuc^8xoK}N zKQ%?fRM?63+_wzbG42aKqI2OlLlpw;*b6SpR@Jap>W?hB*F~F1+Z262jLNxvx5fQiD6RKImi=}~z z6tf?RiI<_}8E&gev#SziSjB%U3z(!ARgzNNmuaIdI5rPLUAEc(19^NU4leOQDjK<> z{4>Z1CA^o82=OYOZ}DwpQr%A&?jO}!1m~-2WEmPDZ&Ex|?oyT5o2A2mWntE4s<~X3 zU`WIL(&G~Alo&VZPPQs?UR}Q)P51IY8Y&Gr1fd+KcAK>2z6beNKz!Q){wiUytBMHt zu%`fAYv-6^*{aa-)h5Fw|CVGmwyGW%B!^eR-WbEWhbo@@8AX7!2H^$bcO7ihuZy4m zgzjyPAhMd}@^-#UQttTeuV7YsKGc*Ac;Zp6scods5HNOY33Ot;3=$6g4qIdt=Hn92 zXsiOcRSzG;`9zMVTHo|6!gCSxfhyv5uI^}s4?mM6cA{a=t2!jdX2Jf8m&4#Q_3R+| zAP(HRNXe5s$%yMQ_Y8;pvmETbm&KNF!DYx7tBq7`bc9vP_Xo*;7QmS|jlg&lDH^FA zBusHejs1pA7L0)QqkTM?+Thqs0{Ihb%JNU(R&Efh$%fXz<-2+)jqgpcL(6dF$Fg(^ z0e_2q`Mm>BQ+$V{!G0d>T5iy^{gxDn-sVfBze(*9Opm8brxYr0<@k6*srfWN3At&| zF-MQ9tYrmfwIXz1cDW(1yP-@>GIj+yhhGibQ)>a2!ft+lpuX1@s;;MwE}XLMChUOR z06a{Q?a(KQArfXmaCS;(*xo9t;rk9oG+sNGuPXfDFExZ8QuVYz>G=pgO#FF8sx)d~ z*ya9hWHW^i8cE+NkkL$4PV1_wSlEuLmNLE+r_@DymQ{|v!$K&~Bb#V9a`MS^OvK8m zZA8k8&me&#k%0eo4qP?(4iaIw9!U~1vOzASqEa?^So^<)DH8fGFhAYblSZiqu+$~U zh>{bL5dTmz_~rADjXYU0l|}TS3A21jt77@G49MNJ9w}en5+&Uqz>iiPURb46=EJw1 zIB9eSHm;`vJKui7vmbs0&yjl;ruhT|o*%$`+a8F&?pgf!&o~%$fa=`d<4=!UYiLpC z0loUleqv=?J@SM&z_SJq@2Olk@|gBfq@UP3TlKro-` zp)F`W;0(jeq5gQnDi8(AD`ING4?)i5&2p)ueYOMnkzNhKa-lw^N4oMB4YG znSrc00U8hKCYKZ`=XBt7XnOpoOipnrGqV+kN!6MjhkUK0;;4yTQJji92NklT^xr|o zHFd$iPYfxzzHvqydA`-KOTSp~ zy~mPBu1;cIf;uSVLiV&(M!Enk<)6xFXTwvWOJ@bu1UO&Y2un@ei2_?)8kS1c6PwGz zq3k?bcl+Tdas^5I{ybJwd6|7qu!Kr)Kyiki7Dlc_YDXy%Vr0vb*+4CsepfQ38t=Z; z>9a9iw^o;W8#40* z1n_n+u=$>IyOj2af*VIe4-54)r}+J66w3hIaNP5k)XJ8TL`6btXFnr02GR#)RXAy=#;a~V3+Ab4dtS@b;6BC zjy|5%p0Z{hc>QZ+OtkzO#F%$2Y2v)2ZpM_q;f!j9aX`;&O6<`J)oPfGne?8}a=f0= zLkc3{=CAaE#wb9N=N26Y_)V5_3LBW?(fFabjf~xfKuf#|QRa2`cqNgmbor_QWRrgn zA?a28vOrzy`M%G)aYPWpSElMn@TlH$tu;`&oyd1NmQWev#SFbzg57Po)70aW{x zK{Ssh3YI&xR5@{%b81`qwab9{d_{hruSB~HH_ifk5AAe#rsQ>Emgck3kf*thit_&X^M6j2A2>~%kJ{Rnr} zY>jO+;ZG!-J0epOR6s_#O8}CVCoSbskq1A>GPG0W9F_e@ReI|ySZJ1t^$NPjq<PO&-y7<%O5)^~-%nv0YII|%MTu~JyYYJ!@yw6YxNJQq$D9mSr-f-D-9ro+* zh;g8fP@7+U4~9tqZUYggHkDuLu8O6W)|$j?yh(@f@sm^OY$0pRhC{;k!&R%w1IUh# zF%f1^#9b~7jjXysLCA7xpjZt6Uw&K;C*tLv!!be>wP`jA~&|1z503}AvHjMvO;E`>#XjPo*f`V{s0NXZ}YzI70#Q-RUMQ| zx}&<}soTuQ*ZK9BY8l&L=N`MvKgkrSiVJ8o-xQ`WMM{Y!hF#T;a;hXZKP<-Viu@Yi zQLG0Ir)b+)`z#eKvHTbkl{BVuGNzZu&E%m?*C_@AoN=7F$uDZL^FZQxxmXhDMGC8(CU}vD>8eTMrfFSyGgAswX9Pv7+mV;BRfXT_%TN4;a59 zI%?^;t-yM8WflLeiov+6a|N7ifqf#wPir;Suz%(bHCW>Nt5tqo-7&n6x#Vh9gL2yg)@U9 zFe4QM-M;}(b`OH~Q+j)%wK}n?M@&H*j1{7Iq$BdrwS{inUj0 zg!p9E!HWgiuGdrAVX>QnWatFU{6}O-;MY!7z5MdRCBq*szSJ6#HzOhuKZCuyyj=`u z#`J|LFO~%Azv`J`@`5xf{Mz2m;up*hA%zgf*e_W?@=(JqAJAY<(k`utqN;ln2h^^@Qd!E`o_7bNFa^9{$*}hwV-pj!5xzWB}X8sHD9o`C-`z$l{y| zNvhEnVfd}?FgVBB%+R;3DTixX&Q#L1))N2BlP~Me!-!FeeOJ8$t7TUr;c4yF1{8YR zt72$=GV(K6_ko@gBp=#X-?|4zE@9uUw$gbR82VWSZ{1OyK=Q(~;iHTQsi4kFG%jz4 za)=C)--8UBD!<)aR-a^^((QoUlD39d>RSveCJ$nkh8cl!cL>ilRYhMehUZIa9{lo8 z48*-#Pr=D?7MSf)i`X=X$24`xUSkPlbz(`33x|1B6&akza934v;TgJ()ph5&IpjD< z#7a0R7h6+iJTY=*v=;|J1oRpioX}N|>M7xUunS^!cXPZvnVn_gG>q_lSc1~8CF^~# zm<|3KkAkf`8i-pa8C1&1v5Dy`BLi)fS{eGvXs((S+hnz2WU{vSs>Cj2b(E5UAm$eN z-^c~&ThZVGfGO*>`bax97RlzMhmjPY!I#AN`8LuDtv?QPJ{<6BsvdtsLRw-*{{ z5eROdOhF2Yo?k`USpdz^(lJ(UR}iT0${JC!`a=X$pII#RP^39k9=%|TvDhgqp}plpirNvi$44&3Ahpn zI%t{+x=)9q?&c|3yi#}7MYLg?{IlK2#;cfL4qD4-B~_{GB>9=Vwxciw>%}^lim@xA zU_K*H4m~hT^4x1=Ngoq}k_t}&@-{Wb3BWSD%ZtINfUk=UQgXt;$vV~3e34!pjC^*;c<^%ZR z-De)FWL5;F?plCobv`R`X25JMRDSmyJ*u$+MN6rb=1I`Lvn0zK8>(ctYL2|o!ns6i~=<-n6_&VF+E4=8sK&4uKcI`f>r z1_r7ILLZGTvyFbWSvgk_|(4+_w z3f7auiG?=O#9geDLt_o#}xwmblp2#mX^Rg2~q8Y`l!^> zRbx>g!}PRzm=~H`et(Gq!EbNkknYChJOI$1=QC)#4#@v{ec{^#TDE8_VnY$JQDIVX zAWw9~Sp>s&JVHLsdT?{xhrEZt+S9VJSGv-1wa%E6HDv;m|R9 z!Eh=17qPZhwK`fnr8M5;QE?#pXD(v^1lM${W3kKs@;J1}bgNXXjmr|R2mA-p6XnD6$fB|lbYAERgi1!im^1V^leRzI zC8ty<^7Wqxg4TdrLV2x&9@RU|d(8tNYXl1 zht?)1lQ;r`C_d|kKKwNmhIZ;vopGLIgyB)o9wl!d^3EGo#rQ-5xcE`GN65na2!ji1 zm2Fo*<}Y;DC~hAqk2rM%4|fbt)}C2XakQtlMD3>a(%u4@fD;JX<);v@>GV;Oma{zs z65KP0Ce5*_5)_RkuQs#$s#ZyBFsOi*cQnkHT3sUlL3IqbcxBiX{0!EbAo#87(AZ7S zHze&ijB5(dl7Aoby;si+kXQY&J-S?lkY=QT)e@@dLCj=+trDEjS{ubK(C-~E1~LP= z;>j2#H)nVfq+2qaiNC9weV2vVV|@d)mGHyeSZ z+D-5{k-4}9x*vd)7i@qx7pOMa-Ke{RNtNA4mVUwUbgdp&Q(ltZq}{P($f+hqoXmRr zKmJN-iS6yxfw5Duu@+-U0GH@C zoT;+A$uc|Jd=+8RN~I?w>a87R(hb8z4Y`!6Quqy8_*|hI1_>RC0Yr(?1Hnc%dL_YE z(y2Trmsfj=0lYO>TCN2_^OUXl_+vy5^~&qYzme!9TPGHtaNe*(!ymufNlo*w!LUqm%7ytzvfNpQQ?9m%amqi_K@H6V$WyD$t~Nb_ckb%8YI3uLk~yk4 zPLitCP&n2w$@{HiT%2C>+oGd#n`#SlTohl+AAn5B2p69Oi2T%Os4C}2lqH&Wq7)sA zv0|%Jmhz6CS62r7W#r>4%5V_oZ?C{zYkVg$&u3@1^o(AZK?M^Dw8MfP{gQ2T)t#ec zjt}H}Fg6|uRr3!PMG14^M63x#16KU$sORs z%4a!4mZ!af4DAq;#mRumiP!M>!zFP8<;x39o^cmNUAmN!z`b6vI>|>@;J}#?!JRA4 z@=~g7-&WOcaJTYsm)6~_q;e|=_ENJLJJQHKj?JcZLRX~K(we3_FS{W5*v`rlXP#8G%tf7El@tzVGTW2DON#$l#*~syNOV`dk7+! zPU+|`den5* z3e)qbMVhS2b&Z6>ZDF!~o+m{f-8V?Vd~T$&rerU4AMK-L+Z1{J#7IQs?4kw6ri!JqIZG-XG>W9-ZyqnI%n>B#Dgjv4wMoAD0=};S=&0J}7$Q~LqhCMK zi<(NqJBXFDdR7R+BsCI9X8OzU5Xi>tj7e4WG#X+lZYcGC0OAu#;lpmrA0%VGMTnKd zJBob}S49F;4qT6C@cf{NVi<_-DlI+-_B$d_ZolY7UR?CawWL}j55g!Q#>8cJa#zXC zHlpB&{7I7k6oQUw>2YunD-MH~sg!*(A`@xtR85Zad!q@s&)k@vEZLpW8PD6J4W8>5_)DA990+?Sl&vzW{sWse(Enr;MtMmea(caLZ!K=^}W;lsSKcUARwYEKHQ_u88?$ z-HXHDolq>JI{$JebH0@mEib(6%ZD}{L)isSTZzQ^N@RXBYc{75iOyDOQI!FH5?kKg&?b(DV6)KCpZPRtzIG=g;Lzc)kg+1l) zxS>%yEsFq#tmkBg2jn|+i7$^s>@0!}4rk*Lax(OKCakjMAB?5V*6-uIr z-ub8k7%|&x+>Z}1HT(<>^PtP|x*TGU56lud@Vcp+qlH@{@iHpO6;f*qZxJEr@1P-a zk`gp1svzAW@Y2=h>>H@I2koE3(znP*y`uy)^)}_T%NsM9PkyGEBmb*uY89DRNn&3> z>3r9UCsxqO9-ntIJ(#q46PZEw0OJ+i$&1qzjk{Q)@;cz_%qNKdB6@B@6GQ~luGSgc;jcj)=a=I!BgoQpXtevHwlZ5QM;qy7 zVOyz?|E%=hXa;aW=2xcBpI@Roz+giznFhB~+%S_)7I^rXP%Bk1h5Jhy>!_c2Jc;2L zV3pBSuKnWkvn^pI4YajIDNaj=-UBseRP|i7$tP3SPw;ZW6K!vNz!v2MFZs^hZS=N;)q_g33D>W7}e1?3!#uw&oG)=v!cS0#|4_yKESu6#6H!0&p-hN8c6j4tB z^eJjvLrO|`%IO<$>8@L-$lp%F{wL?KspD>O@dlM%S8|7<)xX`v_}Vsu>PN+E?^9M4 zJl4qx;5Ekp`ArAC?08~^r2JTf-cF2PqLH1=L>&N%B*$}jx&{fAs>|CAR35wJMoip^ zSd2tXiN0~piNlkDm8O)$7SHU*jp z^(V|7ELZR$J5o*;iYuk8Ar1{nYO{yQ&0$!7JE@}6`$p&k*-H9+KlV*7IiT&JwJL=w z#kg0*H_psbt1A)Gs(g9!iOzzlm6UEwNy_FZKRKY!;B;kS8hmENGu7Tl5ool?Y=uSX z!R4m{w5lfR9Oy(5(N8rQj5&tkv8$NbT%L?ryMVZpH~uxTwx%)sBIUM7@(Q09pkv`( zs$i^~809mJhEYhQA#BOTX=c7GdIw&f+>je6Fy9R0H+QT&cLS~e%w%}^H|4A*q3__N z3d$SQ#=&^`V-~CwuY=IuVbX};LQwXx1oZZXI*I!^|lXbB6hi}32OB}qnpU#%4I1Y`M_c6?utb63%^YQDA zJ*bz;u{rD=2k8x{ae;Y|9V6QbRbzO0$?|&=m#OwXMFQcyNZY zES5i0*{E@?HKTqku}uG%I>S2AD@(g_#rO}Iz#OvErNvuTT}&^(fv)P>UalSF z&p?!pJSpzX?dhf@N23*Twcz?rm5h{WAG56lE~ZGsHNI?)rE}o16sP>$5NF~(9Z%Lu zIXX*+zhR>>R40~GQ`v>H-@yiJ@Dj=0$*efNoX2CJ1*{lZx$*K@FZTSb&bNZB7*v;i8TpHm^- zS-NBme!Oe)(A)M-gy<9qxirTI;CGQ~C zJqWKk+CI9F$CDeY^tqBX96+^zxEvXQ$Uvfno|8MD!`6S0kRjeUpzyOQvl)9&5=|Lp zuO}VrSHtbLUxkxo(RrK}BU>`mjQ3fsC!CtdppB@#XLL9Do+`H@i<&$D!5o+--HU+4 ztq2K}%P#2i-57sG;~~&$;wx4^=bd><-XIMTzYQlKpXsLIROSi;tCGvg@7GKx)6a;* zw2I7-Lr;8e=~T{)$dS|50LK@uX7cA&UqE{GFg>!koiD<7i6fAww$?JYs!1$(kyjox zgv;8j@R4bG$(k>%Db<3YBSvMGm(8h@lxW1JvEwQ*RAxT|%dUt`gbPzBu}7A^*7xP4 z_zDADOCGnZCsY$_*40$rJCI88#!7(my88Hu5Sav-=t4dyWt%4Jb~4o&D$5GBhT2Ny z-&m%%bPKsT&msl_Q3*(Zgh`y)7FNd-o6!^v>?$Wq#-sl_++q-8I6joUU6^YA6jo!t zHljx>n-`@|&%aOZL4Ci)>U6TiCa4v+O?EzlpKtzV+K_n`Eh9==%gIhYyROIGof;pZ z*>HAvoU9(rb#1=zx#U1=Tx&l+;bjQ%iV|JM=Y*?lg8();%jNG=tkvv9XxmP5FKm?3 z3?iXQ`aL^J{@xs(^4EJv&h+C@6bC`w@LYJ7$ov`DX}@U#Xrn@v<&KL#zgF&v(s&*k zw}(m|WXw$}0HozMF-j$s;1#mBl=s>)?Z&Ini|=J4J$`%@G0^{c(=VrgXPckV6%%e& z22gCu14hu|cA#7rkoR?!Jsjj+I1JlAf0%K(C|x#X1*xiZ8S2{1S*tXt2k4ig3TXRW18bsp4(T2ngOzO_8;|E??vFkVIzzqi7TRXZho1yJ zz*GuwyuoRx%>E#lC-!$tyX^2VmetDE9GvAIjc157qT-&&X>}N{=OMVtR3(d~^>3z! zmGw?$6#h!az^D}2auuqgJ6tRIK?|nS_t}|`Qml>(bnR`W3VX3u;Qh~3LQNj+X(sZtzfE0IdKaYkKi3bl4p33cV@hrvwQ za*SQ9sB$YK`wE)NzmphsqbJ~t6FR+UUpGY3YgZY|10CmXTUAGEGb--W7ZY{j~gANm$qVikoieYW!wf9-~d{ttHLc_Bp?mcLL+Wsj#(+Efk0Z?#jc&KdY z1*1LEnGD3#>#`YG?DwM5<-u?ytqWfo&-BI^!(wy9`8KP0P~{Co4+I9_`y!KP-{C^Z z{vQN|i>*6S(xv--=2K3D(MRh}8-4OwGCbL))M$zHdbBFlEm!K9E?Ifn=f+i&^DF`~ zV#WQHQ4jtKw3HT6h^YJv$uhh;L+hygQ@M<)y?d2+2@4%g0_J&oO6ZiC%=sT~8k@?E zl?9wGYwuPISw!t10uXcT^7k?yomg~C+G>^AS>D^g)>OS0w4*`1=?B@7y%p>@do$Sau*$9}b#C#>`CyU3W6(Ag z^l4{XZKx%@{olwOHd#%{D&tyAP(t<{%MvN(Gx;7A|e?tP2@E+UC!kq59z~bDN zOoQ&C=Q_X{l_Q|8()Y~>G-I@o*~5ZVQ!0Zm>{3k*3>-5d{&m`UGPTHSB%xiJ=773S z{=`S9JEPeaACNb$WAS0{`(>>c;v@!rm`+V@sDr91{a0}fUE=5<9g<+*wnYVK>XL@6 zpFLGh?8O}1s{HdILT8K^>w^(ptIUGq(LDs4tDqgM7w#Yx&%#4jwHvkVJXB7Vr~pzS z^gJR1^CKx)x^1_*Wl}AE>Z2mvwHu&;Z0zo{p_ap5HEE3y%aoZ2Rt#5@A1NYfl#j(R zAM22}y{R5qP##3kO1suGjv$!{Z=)>iEmJ4rc~;BvAIw}lYS{O0(Sy3X^m3X56R%K_^-}c}#PgJWNbBJX zfX)P3S>-`5hD2@Z4!9w5+j5FQ`RwOZc(VNO222CxH)SnK666~l5c6an#28o!8`5f^ zg{<5Y%tpUQ>Va;p1btQm$xKsjD^ADk7qRF-TB|H<(D0;m{6YT8IOJUIQ2K?Ep39|Q zsFYQNpwo^1U)?F-Cv8D-X01($;yNt3zbOv$%sSWu&H=cueIB_gY%+?of|TgknY+5c zbvLT5Yg4D39NiiWNozkl{pN)f;;|w+8r{Wfyz8Yy0l(#Z%`V{*X=PZ`3NRHy!{p&> zsKP&=m@duQjbws}6e>^|oi&BW;j_%{F0t+)F~Ln>nN=#wg>g`fRrV$OrHoe6eMfXQur77y#N(S#*9-VQrmDRoyt7lx=(!HJrB;thC%puTVNwGG1n5N2rt~ z8kmkoM>~5+BZ}luYBfdDS25R(DsLfD6mL#WJJ_aRDw?z_8Avi{H_#|iCANVG+?_Qn zUrVS(A&ZIQ{Ej}7y{JMnkRo|)`7ttFl420!PtRf|6iW%>JT>NMkkbwCo-vv=$%&JH zU-KnHqK*mjfy2t_o=V>+4c_r3O1)FOY!Pjo5VP))I_GJQBvod4W{>07JS+*>@Q7U- z8>J743Q?XU2+5r?T+!hN-uRY=_iq>f_w?daWG!b)8(wD>6W(+c(Sixro^qoK zm!ZppDQNQIiZW|%uRZ{dU62mtOt2FU7s!dmK?^W2R~cUvVyKuQ{d2Xi~4keprQiz&WD zJ=F&C`_l~Nk#bHpR9XTw9OzHkN03xXefHqFeyFDFy5#eDou*S5OU%M<)O!7StkhYe zqj0NQ8pxDu$Pkw*p2zagpJ-%k2=5cA8Q0tR3K%S5)X_Uxp@tFfeco zBIJX39e0g=vz=%QsnZt$m8OSZ=2cT?dXyy5Ks_W+mFJ~P+VawIDkX-Mb57G*w~w6C z`M8_vKxx+q`N2x?+EI473xNHUnEy#<{kW)Ej9q50%#lH;wed$(7KdUveV8-nH2b>- zJ>q?yr^o1?1h1q4w;H^SW^O@EL&Qj4b1(qWpc@V+LVSh2WqP#f#3fYhF>1$ZE1F(| zTa~w)j@_us^;tu-)eyk$&kyQu1|77bN-*9iDwcmf1469UPUT^`Pw#yd9*J8d)EAKJ z>I3<3KKk`hoE#OOV$T+-thSQ)d@w=n^N9A3*jPt@bu1v@UjNQPyFBrNZ&#D1tK;w5 z%A5;G;dbb*6Dq~jC!IFA)xr!g`kcB_>b^B1+gl;PmnQ{)srgf73sEQNt;CJuZEvO! z$d1#lp>~#kta4Li(hBIG(43d#3PmF{6z(JbjzIIXs-z=gBS*>KznXEE z&&PGUm8+`JGhUzV0c1kDt_D4xD)4k?TTp2(Z4I(vy^AWNxmdb@9@cILJ?zppZWRq; zHVJ~=KF~~OO|vIxzq>{HIuMe0AS%nQ9s4t3eq%VqSlk0VomF4XeT0sWCX zM{WX>FX<|;zY@%mQYEH#PrAA?AObeuPz56)k-{d%?O3lo*L1%}eyijw1_(No+z9ys z@*NEXNem<lp}x4Pc1b3)}sQ7=8$En%H9T|@T@?dQ$u31 z%1A&1arD(z4=k3zW`5Zt7oNhl-4idxzcD2vwr^%v&1+PujOJ!Z7xWef;8fV z$L@fvS}gJv%aQNR1ex0>)gk8=vKv3pkQ|EV%&Z_O5${l-r$B;bkpGr)=9LcRVA~5xXvo%+htXf!N=hfiVD?%De~6HnS{1lj|TZj#o zF^E6>FrFy6*&ijhQH90#5Wt_s^oAMKrvi!t5Cy)+9T zC%+M8thD>IkgRnr`R3>(qvy3XUP^}qe^=lOaVnY1$!%ID`DYn}QHb7XIX23dg0CU3 zo?QLImnHXfzTQL&A}L18rI^_mH_vW>xc1V`5kN*8yGMV44+ge^{%JAd*D1w?^@+;fpfK_CgPAIjKM^c=tEr{4VSkL2o&flw`Q36hC5#8vsgCE z>FWsL3c5c`MdHE0hG6S1x*N@iA~$y?)&Zp8??UK%aZ;s^z&t4_&yE$Ri7_3)nzhv| z=o5FpnjNw-Z8oa7D4hC)+!jdn!K$vo3fpCOj6^aUN-8G4Q5eD8F*@iUaM zVg>-m)Eyq~X8Ceu4+X9v0QVHOm$uDwX}*bIceIAj3q~U-vdbDx;thFUM?t1fG$XoL zvWIklth#bh*GZm|!s8Z~fwI0;>HRdLM9Y{)NTn6{)}+XRJL1wtHmdL0)Cec3#X)|s zy(EqYxH;Zb$-U5@_TIGTNdMN zX)KFh@g<{^aN#xMBwzg9d=5EIa0zEe;riGs1Vvm~$PN2+f3;K^G9X^UE#%)=Hg&h| zSpUZ&6lh5wF8K(#gUE6%(0f!?R{3EAj)J?%vdw7W2JtykuZX8V)PckA-$p1sukv?E zXghP4i41X^%8Zll6@$5Q84r}Ktmg~MDqR?^FX?qsG8qB(`yM4fQj&kUQ6<0n6RDL`? zObd~)S*>9BtR?h{hu70p;TU~FF->N@1$QT^{8<=3ArignU^k3WDc$)s*#ui=b(F8` zU{y>{L0LSt&@7fee>O8@*O%4=BRjqjEa{nxLWDxUxCal^Qtw#g*NUbcpR1v}X68ks z=uq@QmD2^+N#bDHsTeE{yl!YP+By9~dM>ilH31cqm5w1At|;*}O)8Jk`di+MX0yQp zDpcyDi3wIO$&V>*;i%EHOg4q3wSO!SzhR7tt3}< z$(T?c3#-cyU)>Zk!{C5cIA*dM0) z$jUKOfw6bc8AQlbkc3RA@O_p30*Kh&z_RgNRO$Ix3t@+hdV)&*TLY<4%C$xkAAl$a zG~KTyq_>jX?ibXDy7^;tH}({0+gn+dVspxC%b3;qUPBIB^e>W*9oAA zQw9Cel$H^&PRV9@asnE@p)#|i`7t(s6(x3Hn;Zu_VQgpW$x8aKgBJWc-C)>6(~w3N zeTmYGYFsQO5kjw1FIHm{eLNqkt=n4Jr02GV`-B6W{4R3%f2kqy_TVLt52qt@Qnc-v zG7~u(HfAIRByT!WeeBz;=2|4E)EmYGKhlv5v&uAUy?O}ACfhFI&Vc#}Aa zNUtf+UN8$}+C12M=5*YEBTu6~s_GC~vYHyQLJ^amzx|=7G**O#Y3f8t<(Zs7>@dY-t2ZPC2hjvx*^QykURyF zE4HKcBXz~Qz(Aqd|YBth-)K(Bh3F& zy9l6B2`%B?j7ltamPeXYWP}Tqy@PyC=BeEMf#v*W5oAwC+3MPW94nvS_Blam{uph8 z&6IDeGHn8pVzh&1zI07VrM-d9p4vf~2V;M%SF75+5Ta|9R6B&G>8U*RwO=d>IO?4% ztA_#v$rF!*?up>LuhKh6?s?O$E6#ag`s`Y=Gy`?~)?9>IZ_@19f!tc?AR`%}PlV4- z?I!y#)sS=ebflYo0&9VsS`p0yHmT$(FhFR!#Fk;Z9`{1SsBxwRmG9bZtV+{zGfQLD zNGsgUnX$6|Pr$_l zm2T!xb``|1`#A=`>@=z{-yv%Wwq`8EGAz=Ssf4EFawP_o z@{%1`*huT#(v!?&DNU}dZ0YOEL2u=+G@2&mQBKV-n_h(a9;lRQ^p*=WWf?Q#bMWC8Ubt%;b zSlVwc>pP(@sAAI@ZO7ARRoJANfSJ|~? zS!16c^VQkgI1-x-pri@yk*J8#>q{5#U8Y_Sx|vlV1{<8_=CeqnfUzRg!GPXqh=&08 zj+_+9q`%Cttkr2=%ZqZ6UE$|t^bWlAYsWC3njfN|uPR$@^j_`&^pm3(GLEuwmCn*S?9$J= zf9a2Xkyb?GG$L#CB$ zXdl2*6P2?OX;s(}Bb1((y~d+J`#4#=I2~b^cOeg3Eo=j3r^jheem6Oy)AMXm1+w#r znInFR0UBK{WPvAg6tX2@k`%S~r6cQH3Gy2i)KCnT@$z7#Nf)u^D4Js`G*7zSHX}e7 zR|mjZsGInTAg{T~+0Uqul&&2XI>ye?PCJTq_5zCKetcN=^)YjSGMTR!50Yq;afXLo z<%nGGMRS~ZcB{r7RTxP2Y$g;sCXL=j=#rM)LMFe!oR6yF2^bm03nh9fyYk`&{IuhW ztr25e5oN*N1+6Xk(&~Ybyklkd4orjM|M3bpK}^84+~(X*-K!G%n$LSI3Qy2FO zl9pi7A|8tZd$+KXd{%+U&DU)XN${*MPjy5vmDZWF4V2NHF@_q+;?E(oB7&mi^$Rf1 z_`^=gvRV@uw!ag^$*IsggwR0}#hcMnE(=n_)f)$cA3G)-v`9-GsX~>X=XEjVfb96r zv;#^U-PqU6_EdiLx#ij_=0cUKdbkN()7a~*uCSbEB+Kis4V~oc$2%uW=vgyUCReWJ zF3*heUf1BchdlepERjd!cu8I5cE~r4Kuebs`16#?7^x4snp(OolG0n)4*G3&*alMF zsKNAfU9h+8W3!lpi>%g&p>RWumq!^5iV|^``J;>;Q91HTMS$T9RZP8p-SA-QZK#C! z$yHLX7nAwD2ILbJh3PO2`l?@-4N4~rP8Nq@o-upTTLcxhfC341e8P`TmAh7FvN(x3 zaPP-~lFjwSwKe2uz_A$-F@uZTxC4E7JtA^!)_~4C#u+?HZY(<%UC+ zQ;H48gWJdbIErc?0Cxfcb?P>O3Qai1lRmc@_M0kUH2dvL2a=Sm)%wB;XtE6F;^HbY zV9x$RIirge8ms~$Abl7lmHR4iFmsc%egJBDzayJoo~UJvmf|roe-rPm#p$I^)NqOC zEuSvt=l4p|JfB~zUqQ+lD%6j6*y|cwQcnsjlhB2OgKQ=FKl?loCOGTyO{*ydb~JiD z+=fzF=Qfv8-eYJ=_Va`a571tOsuF34=o+V^3({2fMxY1P88TGKQc>f%w~@-%*d@Uy z-CCN&Z+r`<-O+Ue?jFEm#&on1l#sW9u>=Ol>KW$?WJwqHb*sv+B5ASEK;Ko!$h9ih z4%1m$p~>?58T;dt0e<0fJUyqoR7i$hE~~5n7Dl820Lalrerp6Kc`@Fa!^FMixj&RF zm%5u?!xw`?wz`>&!JR)+7PUsQ-`4?3U3G%#I5Hl;va!S!MFaZt ztIFlNXpeIeK{doa4wU)Shb6oDhDq|&YjLVBNatP>y67n0Em5)Bog<##AsS12Dx zY)njYB_j&qOCDS+C^N6Ul>6AsL{X&Fkq2h!_XS(SGRb(?dQlfM;h=H7vOI^ikiA?^ zFGGM)wx~A{j1Ke(It=U8oHeEh?Auuy5K>^8S(W7oq(-A-kx$sdkr4j-gcX=Pg zP0mZmp%S}XE`{k{ZUxIm7`aPIe+Vveo-*?!=_I;!qyy0zAO;2cDC{Wlk5U~tc3mkj zS;}%0S>|RLveF}pb?l0==q*_A4MvbrH#|FH^flrr9+ z6UiNUhkzE#Tyfp^xx{e?DaURgK_W`G?S8gJgJ3BJV6*14Y;3C8K&%>>PrSzr?1x4 z1j2@jDwe$u*qZ$-u32pObJ! zqn2@_$d9z~u7TmR>d1^&tr?hz&aN;mCRQ6lHTZP~8M0YPN7yMASV=A=j z94JxSg2~|Z#FdaUz8tFe*yP|xT-I0ji0?;|v3BN0iR(`v?Nf#XE$n>p>lh}H{&IE? zqkZEO$ZwSvcS}@3ORSjU7|c|^mR1|OK_PAj-x(FyoRULwMVm514%?yJ;zL3MO$2sY zItFG|3@Rt5mh2vd0d@W|YkpR{V-~2As#57ZOS@W|-hU;5H21~iPF3zU()k;d#;~H0 zgmf5*QU>E zD%?b(V_ro5o_H5;0~t1g7vB7t zC2dCmNr=(yMS{ERGFO5$mNKnbJSp>p+&qhZZ|#L-rgPqAyqRW~Nkj+XQt)g-QKvNI zU7kI%VK2VeN1M5;jh4FasPOwzYZudgv_4oM_9;lls>&5IR4f%pi(#gp%0bn5Vg@iA zvMme!P{J#OJ4wIe5Zhm6ndcJiFdC~$h5%9*0E zbhKREXliAYI8knX00B{0+AbC5!LV0~Xjd|b9n7cuBeYi13>=i(0shEuAx}3ov!V7x zog8l&B<;Rn-R`YnD6p@EJwwK+S-yyua4*oLZ5WK> z86{QrrWR^_8z((xz!J~l@00Pp5#rB0%|2YxZugcmJjQ)S5=b)i>C`EsUYc8WROX$J zl}js3cLhk@u~?dw4!Xc??w5Fe@<`){qJaFf3yINFB{VmV$0x`+1Bft_kWr+S_oS@W zp(l^@NflsYnRs>2o|=OF{OZ~Kt#rdgNF| zp84M~8-_)7IeZBx`&-o*#p^0B5rtRK8deOqM~mVB+?$WGT>qRkMgCdNC0CTUuSTD( zbv=;w#;PPcH)Jo6xE~-Q>a|uN1R%3KByOY1qOzpmNODz1WO@$E!*iV6?8KofM|+xd z2~sH|(B%mUY+CXd`FO97yz*wLy3Sd+lZ~Wrk%sU67D>9@;ixw5-2w zx*)^iS+b0i08Y#@0`m8^pblrSZ9$@COl9a1yRJ+~AJ_of_<|idWbF#AQ$#h9!0vz!Zc%S7wwLpko=~~bq$pPwePi{) zjov67Vp{^CVya#I*5~9nD$-RZH?d~&%)|j8&B7Yk?VdK$_)lKj=Nh;Du5ypcya6WQ zqdBCB6RU%K(HOtdxvf5()tk~AkFKQ-?@q3#{l%5!z;%A@JZ%X>W0GOF1o?Dbka13jag4LPftpi&dnoCFa52I$wdw%L|~TOdz}dl))dT3ryu1&{K*vMaT&wo38MHA_}9ET z9FVCqT5IO`yjZ}6ChV5qk&`o5E7`UNI;^K0Cpex6442k)bVDRKTNv^2USq_CVxngp zc5TSS8_N&dxiX7dJ!5g&cY6hub3?hb=^HttizH8H{jxsRUXr;+^`ZbW%M~rB=R;Tr zRem;-E;0)d%6tbT{nzQIDn2f6M|P=Xg!bI5LRT2ebXXQzZ25m{6R0Ui+WYc|JOe40 zRr&1^^H~F^$%3K!j!NmlJna48f|6-6;BRPcYA^V9jq>!+C!4*b>UXdfIlwe=)(fC zx8g_yz8RgY_TnXWgSy2q^*9R33c}#Zt?I*r>FK4$Y3H;aH;FFoM~-f|^H4YkUFp-6&7a zL4?-#A+C?KN3`htem?}_)E)OiU8T%V%xeZQ#CVDbf!Xl~T5q?qVMWFMfr|vz2EEgpObAqoj~*i-P%{)72?U`2i@7 zMo{FtI?D3VUaSv%O*Ld-4t7K(olafw2|j~wT3c^m2KYy!Fba->MRLZmwVv*h`yx-T zqm1#gvp*Z5PQb3Uba~{{ElizkP+ZT*xZq%YZFQxy!87lugmIkuzFA8O5Bh8n4&eyr z@RwoIHd z4g$CZ5}{~(#=hSWg!`$k&5F5(XGrN!jK(3_yPRH` zYbxN~*ettV*^RaYZf-yf(_nA>0t=Op1{? z=E}eI%q-Niy(-cn_Xrf8M|{qG6DL3lgKtGPlVrDpcNQfT;CiIV1i^S+IE_W(;INyG zKKIhb@MG3=oeomzqOH5<(%fvI@ft70CB07N&yyLA%w(KZnQmFUmIWvs%Ta*en$8vB zUNq-J%4DjN(Y&4lWz$qfT?uO-`sQPk9R0&ts0GJnUAKOQY`?;*7(*Y)t?Tnk-Hkq7 zt;7f*9Fs>g!bmyrakv`PL1w9yJZ8E6hc8inJZiN`jiIQX?3FvKXmV% zWy;+_?ok4tp39V#O1#e!!PAO$a<`L@Ut&3*sO)}tasqwvL?*VDi$A~u7-qG!30hAZ zks08Yykz6WB--32_?%dM`RnAxPNt4s&TI`H%l}HE>Y;R6>x}nX%GL`r$=zB0{LT7~ zj+StD)CwgvAKH9g>D|mELBGlVWa<7tWLs73h@&mL zUwYRLMoNBiu%AJLX%=ODNx~Tm$?c-br$;)5gAy!V! z3I@;#k=F8P8=`9S3D|wF4h-J|3t@w0HiVRkYW5pIk*{tC15#F(rG8uo9C>$G&|<8K zwIeA44tX%h8UR{*7E3S(pwV#)H}sh8D*$K^6n~XKbz?iL^^uFG@x%(0uIIMWzQygYu?K%Us!h6}gHt zz9KY|ZH4)$!1=PuT!*2}Dla7lQAe{iHNt7xEHbt0sN7anj?PAse^32SdHj^=2i{PW zu<#uOQj|7(5^zDv$JoQrXQ(pCOH{GBhPeD)C2Y`fIm2bw%VwsuY;C%vEft34*ifkG zAC*xb_dvE)K3WL({H&5!%EHG%9Y#bc$x*sO0dCw;)NA8R>%XQ^ag-fJi08Kz>jfrt zKoxeD?%BR%)+c$m{-GsGDo{}_tY>XDQbG#Guy z7ZN`~ofkBeo6W7oS|B$?r`u+0wmB@5-(n51s1h6$}o9c1=mf#7HGvk zIa89)KVrFG(50^*s^WooLo=qz<#BA}^UBg5FOstXwnRx=Il11dOVN35imwl&qr~_r z_RHQwZU(I(mZq={woQ(onb~XUrb@GrMTJ9f-sCP8*Kx$>MJu5+oNiRb7w>ve)(?Qz z?JDM|Hcfcz1>6`SG{z|5Q#au z?!Wy-RJ!X3h$!4#D%Xb4mzU#FuPLLgeAe9>F+PpY7f*jQ!XkZ6*FY6m(bAT)utLBj zkt44iEK4<*jaS(I8(q1MqrG zX+ps+7O&u0nLG*Nj5y3?=*vJ$qj^$>uFBA!B_e0JI=V@p`7c7>dUV`VC|dS@$3a7} z9&lLYj53lAQw44cvTTel8pi>_EQG5g07y;O!9T^zS=e-!M5>%mL~#~0=Q<*aq8smz zh^-BCEy9a{FDqk(7;d)g!GA&ldD2>kHB61qlZ)qk)XR!3)QF#?wq;%YG++W!C3?!$$oxs64MMe)>OS z+Y*f*BNXN-aQ)aeIDMXOvp?I&PYTOw-5EJ1o|mN*={$yn-ty^#+jiMJ#!RNeB@%&% zUKfBk-y&NFT0Ig|E@+e4W@<4?Ba#Bfqt+6KD@x9s1aD8&&cqL66Xf#SEZY&iY}B5; z0>mP=&%F!3$%j|*(>Gn{%SPf6LP>`)8*4-`1ODjNQlyKq*X8Nj|NoXcsNd$P2-1Y3}> zSWbq#j?oCFUo0P4*uE2&F#x!&p?rD~2{w!xkC2^nw(M!lS`9zJD|>T5B<>7rF;-*A zY3(}LV|*TkP+D*1F*SENIk184drg-+8#aZfk5fi7+1Uxxm*ZoflzoZ!lqR4W`I6a2 z$JNBa>4qI)O5|*;Dyb#M+Cv08o9(wj0GVysEXOO*RN*x!sYJV6lN%v@3SG#toaTsR z_b;;d6;`QAE0&KEvp%Uh3l2Q0oED^e#wILi`)-)uzCkyUp+xLUL9e_xDwr)T{|BBq zqW!)NRA>z|Ew=Ey^k%%$)snT&og%ky`O@L7gsSrNrzq!-hOu&U5Es$90bQfFO`A~= zl_*OFAq!?`tYPY=)eo0I1YMMSHY{V0LrCOPYD#79kmU!=5;d*B2WnXcHJW0L3)*`J;2@AaDv%>bop#J!u%^V z%5KUDaiK-h#{wVct~9gJU{HS>4)3H>eKSWIZ9@t_R_^8C;*x$ulz0}&-7w^zyc-?5 zQDso6FwKRXfLs8fPKH*}>#|D@pIBC#+MDh88M^fkk(9pda0TrRC%$)L zgP(7UV)>E++F+653@oPn3NTiB6CBW#^Lkn3Ov5AQp~9;Qb&xW;;PeL{!5nvYz#P}{ zQA?%w%#hJLEl~!C%mSF-F-_}$D*8aQ8VhTSHD({qmiP{4BH%8et}!FckD&7EU`9F? z-M@{!F!muo($LYNo;)jvqQYi~_KZ*Ak(_Rl-yBXmt{j)~1{60mUP}wJ1Fi{#xbnJl zI^(ub7nYh=7u5040pESB6B5@YBg&@eQaw)&vzg>b4F|Fjl6fv3F}h!$X!KSEL(vk& zaT0m2TDY84sl+dl=>0jIl9BnJPm2KjLWR+=jO!IS(uuVj5L=*8a9D0N1^(uRQ$Z*e1XJVa-#+W(c|G#d;xX!{O--;DbaT z&JcfJw$)M=T5^mO$7$FRIq|&oO5(Ba>$JjwqEWye0Fv%!3hr` znZ_W4C8CQ+k5%c75f9!BsovJiMP{b=l`DtPXq*3H03;@M!y}u%L7M*s`23PaJ^(P- z{;tY$%E2Q5_RA_KUWSdkoe7)fco7PTqonFHQBI?P?X{#CJlx}hRYNa`toId)Q>Rvk zm1lr#Zv_7=T@RI-+fgE&RiuM7*Cmq=E9W@rF$S@drwadvRn9f~KT_{1> z6`VJY9@fV%gad3-=|zm5oE$1{dNJqh>-izWDjjv5%;$COr)|n}hOHzQuN0jijGIyU zcvV9!S^9)ZcxqTBx`Q`BtO2(;9(xW3PwbZ%z|Esc+aCC>P^49{s z`?L+)L{ep(CYI@lgp$UV4}LATGF6JpfM_Zojv(+NHX|h|SAH6RsP(*4x`nWRp;~B> z!bJIfT`+(N12U&a>RlKz<0QKk&~*zEXULnWrd<*j2aEaDmJQ7&$3Ww*D~ylK$>j3T z>{HsAen7ViIJzuX^1@kH)XyjcszdmZD_ILAvm8|3K{q?1WKBSUvAypk)729Bl4QlG&~r4Xyh` zA(!c)6sJ+5O-39*YTx&mPDW~LirrzPNvoy4Ts+4mOBfpB@{2uS$TOqU zA*k{+l%n|cNm{?j<~Mx_vMUbf$kap`GX4#gq=WJ_N90?cm5deqdPjF& zyo3amkII50>Z_C+^4eM6@r`aDwH5QF^&wU+ULUKk>V&%gPB~v>iwmPrmNiJK0(yquYacTWRkz5waCzO*NKk*9NS| zxo&o2D-DXYf!UU1I*UBo&dQuso^&y6O#9GbwttlJen(sY*&bz@Fq0&e`=;#B5sc$? zcErncA<#QCQilCvb+OdpS2dtW#NTZjA-N4e6%7N>d=Fh+_7bhWiAu~_CX)v+;?MPA zd{tHaEdn!#^PM$7PsJ4wg1j!CqdT(Qvd53nb6Yfym~c;Hf{A zGXt`0h-oq6;xmAES;z73r!RmvJbmC!e+B9BC(86GRnSAu>v%2Vn>#Q;%Le%_4rZ-2 z(C3DSirrE>9mV;(%D*A|%AopgSap3N{c^Q3%v(o?;T$ZfWq8wsmn!+{oJxU&v&^3f(x~C~SM1Qq4Fb{kH&>nN1P$iMwPO)y`$n zYfuDXzO7(D#~?H%XN^;JPlLB(-F#ozPF#oBtLg!(cKH*zpQ0Vc7)h_7??WBHqux}b zVtgimD6x|S>alBc;zRl@agGj6bHX2#{C{H!%Jx%T#kEtt)I6#47;QajIL!1=d4?j{ z(-W}8!agig8Z3TJLkO6rkeXRn!BMjd;FrzUtckj8pQ}>=xug0&r=!$=|1TfBUcq?k zDa(wlQBOTn-uR@#Go>w-KJjJvEbF!)nw}Q^`Szevr7iLXiBgvDr#@BMCiSM2=ITn1 z%hE}`O9z;)+&@11vvq2wCs!8?rYUL37JitO9hG5R25uc9=Rl_GT5gA0@UQT@JhRF! zoYvctkm8Ek{jw^PyDqP>=E#os8Nj;rI{HxOpuKT$v@2F0%b87Q?#jWFe_rArt-nh#5l}95+-kxYu z7+@<)Sfh+-k?y&?=U@QDS!7{6mw#d?gD+c?<(-bcu$33D!YOz6&gqNcDI zJ??SI2cPlKxzFRNFu&lTgLKQ)M^7P|7sVTLp^}*^OF!r7XTIi&|LI!s%iHp=nVWq9 zIdhF`W1F&HE7$Ofk2dl2lb3i8t-oC28pBXVQxsJCR^}C*kFzT7mspEl+WpcciwWKD z#yWmS7OU}A8V%(?_e-Stl;L`tp2ZUxM5vBe{R{z%NFYzI=}l&uN-ml z;1i3O$`4fSk=vyI$&;Iq#IGngsguL-y1dS9#vT6U=Y&@(lxKaXj4~;aq(^qT2Df?N z;2jgcX9jI^5tXCj-BJm~BfDqv{WsZ&nc0WEk}=Qc5RWeJdCJNoFIQyz&y8cALrO92 zy?Pbfy~&@?=rEW)U85|9dKkpm=kQ)~CXa0XE6Cx2VTeQ%0N%Z>=SYRCk>Dr0c z#}%;vcXsn@^@FBc?hWO;GCJY-d@{tjwi2(nvX?PMmFFXIJik6eFB!?4&y&x8$vdup z&xQ{1m^P`xQM#=C4B=I?E)SSUD5PX=;F87@`51PDQFUL$$H@x(*{A`JuW*VbJO4Bz z`0)~x|MdYM9iHX3=ST2>-zxhOjM-@kV%4>=Cg^;CYkNwoAtGfOhM@PZRMZhF&5PLPL_jUr`wGO==F z+3h`~eaYI%ym{qRrg)6zY4U6#cPz-}q1`{?UyEn+@!mgXvGjgy;>ugh4|Fxnm0y11 zmDkgF-qLp&|KemGaQ!AvUilh-Hvf&4u{DJGk1Ryx9QxiGF#dPJkj6y}#r`9gm7U3h zr)*A%HSdAFHQ{4nM-dIZlraGqD(`-*4w)*@`}&usEV9m3xDHt1_zECt*M!lJkv2A0Mx z@zu|#i02Lqxb-b4vRNl?`d~g28|{Hp2H$1IAO6gmOuxY}aiHG5pQBZf^sA z__GF^xw|ps@#10LUo(M6buHup4xHq2xhq0#y|%}et>a(6CgmH;)BLh9&h#2Rt>hR4 zqLL+Y0=IeTqC)BGWZe5Gj3)2pGq#Ek`MA=FRe7|@`hj$x#`Zn)lIfIcukz@EW4vzV zkKl@#NmS{6V$H=ZS5Q{k#6cG=#xbD1FR+FCN||{wy*JE#v` zDJy9np5&m$uB=|ctyj|_-KRV9+NPb@^A<;0g=EU(%1;Jj>{q|(mbJg41Ik8`#JL>a z8T<-A)~LnUttTLtMNgX{`6dJ2E1%?%y$g75lFs#vpTlKqYx1-{y_x&3Q7ruc)~{;^ z{u-rQD)p(x6@!m5x!pSCwMQJp^OA?}#(%@8k8Xk0pTyf``wFf&uaQjaSbn@%6H2f3 zi0}TK%>b<{x$9|N$?Vs2eI> zhZbx?qApBT(%j6E(S1z6^rWne4BWNiy-z{zeiM-sM9r76r;>)qVfV6&|U%${*?90sLHRX%VL@7N0vN?Pm z#4+JCd_3|i{3x4Z?MC)tn`B;O`PNujgP)4I>hxxwdn$ni?{4rQ`=`vNSho;MI0RiR zI%|4lLr;FV(b#m$FS=*XvHSesvB8#?<5`ZMf8?V^C7$xhES94>4LPG*X3&g@#v zV*-C$@K>DV5Wy&4lqK>R#PQxp-dOGn41pRKa;5xmhU34*b06feVb2a>OYYp|il;T? z8^4NKuPT9toJCY#s&A&tKew5FaVf@iei5tjyuq#i{>w{ap5`O0TRTp`=Oy#I@sPj; zuIn<2X^+9qpX$rh|$ga1l4i0 zvS|_fecu1sI5yThlLf0s`k36E%_XEadZ?G1F1OzGIqb599f>@3o>xq-M5gmY^HFf&Sl#ZiwT|jDeq|Nko`eN`K&QNrmsbIg0ZkeP*JeUJu_NeZ zmxG5FW`Dw+`T3?_zSma7$I<-x-2fJ$PH%SconnUh^Ho-8!`nR4`66>7h%_RV*LV1~L=aQdnJpSkd=4o;kC>K9A6J#xw zROId2NU*xv9Qu0=Z>xQRC0l)+?I~=+F#gG4@;yH1ZOAN}Jadj0)$b4Y+UXC4MeD9A zcVJLySDKyvodibt8}wrdA9}j<23Z=*V}7p7&Ts3&@Vk@``n7kVywt4fv&*{uEXJp< z(p@n){9Ln7Z(A7}3CcfH`E$e?7Iy3%)1l7~Dv57`)oWSju$Jy1=1phGHEiEFSjf(9kOi;kL^7N|IUaep>B5PE+Kq{>KssHAoHdEy6U_ig?x6=Ga-JF>hDGGTL~^{w*lVOH$$A!gct zBtING%P#*IV${F*8A|IBAe{-yYn2V7S>7W>EXJFS8P)P3^;GheT^Da*mi=M8pnr;)JU%(aVT*FlS`U*#4&X!woo2SV-CkRLbI)@ z#b)$ODwGWeOqYCq5N?~-(#)3G+xYR{4}5_97^XVvIxN$9&i|2g=J7FIZv(%rM@>kW zgvjJrGC{~ih#il#~%v>H@J+YsGETcmWEst~lAF4`!q z(58(pOgG-|`Mv-6L^5;lInTa796Umm)t9|1xyRW1K;rYv0hiWq!}v2)QMCNK49a}6 zfX{ud*InCZaPrIt`SA8(5KZ5ed^oKuzjGc1E`2+5|AHMH<>x;9^;B$F4gn^L;G_4O zPC4EYa#;2W2&kEFrpsMv7(Dl978*%4Ir6oeSV`B23i-}XwD@sV<&f{fLGwLVu~nI4 zSzn5GMfeOGxM<8Djg-7Jn<*5bdNL> zWq%cy-gb-&UTn?@wop+iW%?`^Uf?5LUN&tqm}WNd4dos!I>GY&=ktw|Phk)+bzRBK za8PpsBafIR<1B_>eYlMfZ+q zsOe^b{BjGnou)iCxuPY-Y)wP=F5n|8%6NHICXZUyjFbNICdm7FpyiOif452{em5}v z^lqr@iI-rHX37^Oeb4i@-*qqPwS&;aC?NPs9p1QNE0ijU3y#1q=h_cQEUELh=b1`LebK! z16Mo!uE{q@uTirivCo+J zfe`?=bFm@Yk~v;STHz!+3eYS}OP0ZFImzE2LpJq?@XdiOt$bP4nR|{1AqVRW}< z^W5AZU%jyc&iXkiOFm8E^#Nv!>Snuu-0K5fU05R^1>LLwO5M{;qFQsieR=uvRY#to zdhvy#dK~}ii+tg$`VdmNPRP=;9jU9NU1B`Qp)aBxbP6q8hJOyCY~|4IO;sC7WsSXs z9L)rjj_!U#Ry@yP&nU#=D|AZ8#~Ub>xU8ouK5!kBQ6Stv-Vd`p((!wUYfcO&cxns+ zQRz+Ezv_9xGe*Ku-?r4AuD(Xhcj(+ORU0EcT&CS zk>SyC_puF#O|!WpoLXh}V*fbX^vdjaFzxq-&8I5yIP`Y>7!DGx$zdX$EU)xk1*D@J z;$-xh1X9NS5H>*f_eN3E!zQN}^-i6MNp6xxLB(xL8p$IsLp?7w#|~LC5qj-81t8NO z7$KV$0loL=X(=I=6gqSjQn@aK+F1m3Z zhi5U8Q_Ai)iC;89dtUztyj*H*QN6%0AX)JT|D+LeBs*0bE6X(pKKvHHdouzc{iQPz zDO5bct{$7wH_sz)_vd&oya;YqxOw_l8+qLGD=fEhuBJpX9!5o+Yj4@5>KhRJ_V3VE z9W8@T=}_it)NqRGX!@kG8UKx|$3u4cky_(-@yuBR`0vUSP%o8xOq#Rv-zFtsblYHX zlj?+!|9BQ4eWzLH_nokJA0~gx{Oz3L^+-PQWj|mMp)Ke(s4&hRvdLpUe)o0)f1TYQ z;WXtFu+)La>nZ~qk$v(Zr|4ssN9VwHv%L5d-IDa&;y^P=o(E??Ih>>vsAKkxZ}yaJT9&S?+M?*PvhJBLo&85FRSN(Jz6M7j7(m}A1-#`BZG7T z?wvZ|@r(t2p3h8{r0?OeJ$H~w(!EX}~GHNm_ zW%ztD2+LI&NkDCZMAvZjeg&pic31GRg#xeLSq)bIJ7ty1!`FD(=YE(lcoj6>(ed4_ zTR`KSX1sRi{d{=#a9FvAuB~gL#f4o3+Q`Uv_}v$pr z^11hTGCmNYkN^~6y@6<$t_ots{VAuqwwA+Zc18juOy<280sv`T883f%AYc1khY1!n zL<>Kuf)2TF9LR{w=TYO2V%YVb1(sf&gvzd4Zr5zz}2ie#B%qQQ}wNQ;ql)AjS9mhaWph5<;+4d2s^Ytb>tE1!`72lu_AMwmb)i-TQ{+ zkS_N_Elcz;qaOw8Cv6=ee|=!OWV>e5)w_Y%FFH1tJ%$%Lx&XyN&+@JpAy?hu6^3vI zX%HqU-lHV~Lkx+t;Gi3h`P;b10%&Mbu zq9nBmh+7}wFxDl7r0Fl3G;csv`G$4N| z$ti#L=d+Wh)n&L#Y{*KKv+T+-b|pDT13Oz#4Ul1%rE@_@X)k2f3-3WB_TGT&yYGpJ zH%O|a5W*WNE1IF=Qp=r^{2y2Etg_lkY8O0;&wl|Nt3Se50dnshT&q_QmA@;L;Hk4|A#-%kDZr=2tyH=E#8V$TH(^s6e+l*Zmd*&8WUm}l!C6jrN7@&HxtDigp{UE;Wp{V;DEDv zWx6z~wG@5Wgds;Chc z%SezXQh3?&FDzCnUbV>ozYGcfG>!M{pTapW4Me8i?rJgoJetpy7sBDIb%J2_Jg)ML zpWhu~Cx-ma;%%~W9%XoVjN7% zl3QpQqel&{bqnrIe||ejd5qI-eFX^8L()`!xD6PZAAw!N9t4(SDD;-XIXq(|q)G}u zD_))r=OYi?fEb-BtF>WdmgwTRHnQt5-zxu!d){~k7&RXN$w2*$T0O2R z>XS{ajFixaJfzhy7-ycIT$t?Q+D)Rk*o~jeT3OHrST?vD?%rKqAjh}!*{c~`xUtKu zg;H|b=x9uiSY05Osk#I0&FO&XV^z{nde-AUp`E+Ffi40A}^S7cTI0inarqriN-mR7hdNOn{I$ljNxh= z{hSW+QL+_~YeRs<$Ckx%U*sWA%J~{SXt_T3QpgF zGkdC%I6WIBAiEFqy9W8DLk|A{Gvw68>AvtHZT_^PHBdS-*k z%#Zz9@=+BKa#8ImU+m@UR!4|BZZ79v+W}Ge%(I|wvd(QDT*-4!2Kn&3862cpoLMR( zzT>~$AK^5Q|G_tgCIO0PlarUr!V}5K@kg@<|8NCwZGuiZAH+`=(|* zZsKUPFH_SaWv!1h-ggJei2j}jzp~$CY{gWwkV;EOq%7OUgQK1Lx{sL2w*Lm-L)>E3 z`+Q~d!#rUrrS)=uF2|jlR!LuDZM^iv3zC*ik>4z`iPnjfS~>nNOnX1I4{0Qj^S-kx zjJt}Nn)Ywql{Vw`k!F=#)hV(~H%;>4vw7@M(jZ|f?Is}#~PsX7f}Rq z9!#y8xRbB#q=;8O>1f&H_YlZ${}=YoSsf$&{w9C*#Xt*wWzZZp6HePSj`Q4b^UC*T z@z={;%m9^S>~%WxHB#cEnrx|K@B=-nZ6xmyM9#$cNcnCeEU>58B+sSi1z?u9mxqFU ze^8TaC6ajr1^CQ&u<%H|nrUHqh4?SPV5qzd$<;HcOP(=v;H-~l4_a{uSmkW9l zI6aV!km8K6+4>MzJ)uK+qZOZRAqqPqxZ+n?D4k~^0Utrwxb+bYSFN1q3 zBZvhsTqn+0m{&;dBdaMW&5xEAhpnIt8q6CG>s6l&B_+H;a$1icXe`PEozz(~S|x_x zv2V7G1oiD)>`S7#iN{^0)K_k9;1%l&Va66wRrgyugSt*_L@Sua^%}Pw}YK8w(7^VYT>9W9Xrptpp z&1~Gb(vEs41k)K*Wqm5rJT2txStQ0$RnbnGcSAC@KLez<9|NPOv^NsXm_UiwyH%gh z^HLR?&fLWnoRO}Q6xnbQ9?De(4(YGsS$jN)ttWJF$fJtd%2#!H?H^B@4%vdkW2~wn z-hT$D_In+j+dWecU(ILJYagCgT8btDv?cko$&e3EGI?W z(DyQvQ`)K#4!-9=`TEUf2}Vz%OO|$z)<8x>_hq zD%&IAKGq{JSLmWhiXR)x2ftWF^5Jolg*&XbmK&X+Zzd@--h7YglFtQp{i+&DnfV@1 zF;sdB-DcedQ<^DW-|NaD47nHz6AM!0r_Hdyd+mtwKCc3;sQfcpQ&A7iy~rp}c^ZYl z3VWX7(tSW1cn8(uj*diGsplCw-}Fd8M*u(j6}sMV4zSVW(oW(Z2YH{@)XL5UVNM9g ztHdz?i52-7U})dLtT5KqcqI3EzYDmPMM-{J(9uZOuD8-X@oSi|$CyVZX#WnvI_bTo zT^O%{tYh9;kll&Kt9bz#J;KbwqQMn<=Scv2?|r~IUzcF5*ERO1`try-i!F!z^tS1y z%B?U;`pn@XDaAf%w*yK%p&YTg`mwcqssmvC90-$XXZdMS08JDl{FE~jARe8~ld8%s zuWWmlYt7Yddq1h72<-oyhSGaDA91kkQ(o629(t!Eu<8u7Dx~vIoaeYso0!^$yVi-% zJ-dusEKjPCj2#y4*6J{aWVW#gAa_6pw%6(Nj&A_){>(U#v#;``7&qGT8k_57&HW~Y zS;c(rM;%`bKg5TZKZ8hF`6Ta|`7-SLadDPh)kC{jx7JoiW8&mdJ;nQh@Z3^4l56^m z-!sX>FNMEb>jtEE7xB$$!#Tt`U69f$$qZ1Zg;-qqC+{B}0qUMn>4ZI7CMGc0Lw9tv zmC1vUb~fz``U+ff?qt}Y)`2n_YK@d-4OG*V_afrx}qY&TOcN z#cCbtu_|P@IG~r?|3Q=$dc*eiqgke|%wsU9|4*LeZH<~aT!{#M^gYuF%xv9hQ$-7Y@)k$F z5x|6Na~OL$ZZ9`|W&v`qR=N7Iw$+?de|!&w@I3R1@(VhoGp(m z;-%+RKuNb9oWH#mH+`3wHhI0PSta&MQ0DB8+{E1Jua>z_LYX%vLVGXjW~qSA;-%WDx)sLEx%Chhr<-=Wt=N6)LDhDva%sSyr4?j$dNCB!KYey z9Zdua@0IfKore*e_g?41FN}s4)1wfE|2~zJ&7LQ2RnblE^apzLzvQDs|KRQOM)Ub0 ziO@Bhyy9d$T3+@)#y7hXB}M9loDytqdbCVyC26Di#({6qN^GicCYNskxzzu_^O)gQ z26>W-2;=FzkhEjfh|w#JAqe#PNqii-^idV)E}svB5&wzs$u}tw@`#W4qI?Yt_IL7$ zZmPBmmT*}YdD8)=TAzSK-`8^RKAr#A@rmi9oe*O6Ar@tn3uLUgz%hI1NcqM_pk-Me ziP+z*N@?{or@Xd}i}XLiPvL}?jE9N~(0smzgiD>#(p482nJugwnZ6i6BcxuH86<-{!I8NH>Skj8$Kz{0%LvN+Yzr zo_l2rkNj8f@g)M=+GrhiRq%`h5g1Q&#eGr2bF+VK^RTskl?_Gh&-J@RUf z76MP3iNrM1n@Q)VtZYD6)|>dd&mk3SdFYawN@ISxQ@qXjdMuk+B=;VeLVXUDevQTu z_>8fXvi&KHgRI8z4O6;$OG67V|9Zvm6sc>LO44w?Q9c)?)rkS`vUM!bY}6Eh`&D`p z_s2$gdlV)?SRusG?^6hTh^iO>(b2uPRV8eK%OdT1nKk zUjgb@ljG#fFT8^yib8mu^=7LVa^^qTjwEj-^WhG(Vamx(kn?s`>oi`s;;3m;?lv%8 zbw7#KxroQ6f|X;AFcTf;94I6I_op@WXdmN&W(>XYvnEj@YOQBPS);*KdP_z zy<&TpSKRXre0uv)o^(qK`DT;(=b)ZU8AH7vmQvwhIqfA{v8kqzY;$c>S@<|8oUj@@ zf4VB_i|tq$EpxX(Z@*ne9yGv~$AYhll^i!u`Y-~FUdt;p&L=sf)`UIg)%aw0Be2z6 z*NVi1^Y_pZ$o81>*1$g4+}~Ujl9Tp zX0^Pwg{SP-zDCnSynfp%7-CI3XydzefPIwK9ariIoqd_j>7G>kpHB#^R(Z zs&q=x6B&Ia3ymewtq=+4RWK16WAn|Se&BF*S_xJD0XCV?Ud>p40F|M$sXIR6loiTm zlVdtpxvV}Haq2N<6IHMu;v>*l?mGu3m%oEK@T2Z~x?5|(@7uy%xhgA0lIK8sJKq4+ zi?!|WwV!{Ucpb3PP}NdithJa)xDQ%BvEO78_8n;SvU2s1e*bXShr*#{jJz;OIbww{ zeyXGBCCO+h`=19`hn3qUlL;3vQMgL`$HYcI$%Ks0Tr{)9{20=4%)u03ZCN+KQQAef zs)Z992u90WOM%mmx<=|~1j4B2AxxzVX0pkGbAWu{&s^2}IXB;|f=+qvZCLt^NPf5G zPtz_>IU&?ts@5)TpRfwBfl5;355186ZX6AClJ!qam^nuwXAm1|EVVR{qhWXWJKqB_cjh zlt)ej(et+uq<@cs9=q#dDR4$hsnA~cj8VwYRFxG6FL~SZ6Ouo`rg`bBK>cCecbKay zKOX63mBQSCPIB=IPub%I^2vJA!&9_ZOYaRx%vv4bx_gghXC1j${?j#N(-$GrmS^g* ze9-6mTWCtQUv-oC=b+EkKcoEz4C7w^);ML~dGz7PB)Zwtvb47!Et?ydi888~6Kwhi zc*m%Wp#V6LBnN)xf`L0w@?w?P2z67@UXu0B#DS_RS})$vA;XXYe?@V|8xbM-s+cF0 z;m;$uvl8JVFX;mnAB$`*bxPdNR*@$67#TDdhbCRKspBV0_pfC)AVuaFBAaL7c|4&) z)K^hDh>5^_eexMD%x#vxrV zXrE|{uqryr(~n&;|F`1Eg9TEZg7< z%wlRx>M0{hH;P5bjqQB+XKh$!Dp!ImE8*S8^$M~>+7l?LMEagzgvD$*iz*0 zMDWk>Jf{qN5HQqXBMizS*j5jN=x$Wy>DeYIjGeD??n~h&^(r?Jp3ggTbst)RTSEJx zy)jq8?aSBs=>Z+U+LvbH6)(lZp$Q$?bBs$)zl*P(?F9jT9|ocPpgi|VVN0_>_S^X9 z-#`2wF^=&E+R{22FBd1ulu3}?`K?w!UYZ4+)~x24KX&E&-<^d{N2Xeda&%j+K~ulI z7n9BklGs;dleJawNITt(yru;&T&%^So`E9vAdnfWM`N=FqA8ZAYml7NEy$0m#3b1{ z2Dx&d%88MwSZDHSg4O}QBTbW(K~2PIICYeEQkiiXCJApM5gP6TT1T}pV!Us9X*W&GiMbAC5kw^z}OTS~8RMQ<5$4$POR>NZ5i3liYt47)r&me=n7-PBQA zbT-|Q89XZ3l?&B|(-KmNZzDSRO;0cd64i~6IAt-0tGEb@fa;{pGqEO~sE5QvXpq|Y z+r*_<1*7(j$VQkXHaMiX>H%N0%@>`{tn~$Vi&F{Qsqw@ zc>8A!AP@8NtzBhiKsVtwf_18g$%<$F8K@d&ARZk6j3X?+p&M>{($8wtm&0GflT30N z%(V$G6|d(}9p8sL4&{Tr@o5!$+(CbtdK2+O=WZ)`d=zB;;a6c5Vy*&2y_AbxVan41 z-Cfv*7TxT4>3$RpY?@)Yk*P)t+5d-`3>v-d2*;)mmS(z=;?;hZM-FrVNiXQNdW;3_ zCCjgwc<#r+f%^~M?#>P{IXk!o@qjz->_K!RVdB18J#PUjzhqeUy;x`1_Ayo#B zTj0}f)s+SD4EvCO`=U45YZ!0zsR@X04|s_zF_UH8_b6~?7R1QeB7{i(6I`O-kASJp z*Jc&L0;4U%s_pW@ue|+@Kg}GO*~i4~{>C(0)KKOBC^O`S}#);q^~=($pmM*L0Zs z+;5z3LwzX!JLO=t=(GH;n07VzpAi-(eu00hRdNEh9Sg|2=)trMZL`Vkdo35hbVtb8 zudQ17;U;isqB6S6LS2@yG7suyK~x8__*r%u7L)rLdukY=LI%|MF3FJdUxL6__JXJQ z8n3ib#OI=!+)epOQ8RI$Ljyge^2Za4qq1mEENHlR(@GY17l5A>4OD+#1fjIQZMx*i zPt0^9C9O>E(Z)Sh+)kLFu#aTxQvbbwTMqe4&;A{zJCy#|1MSf18YHacY9=4N16#kP zGP=oUHxU#61fZv*2l&YEy_+W1m#+Unhszh4dSz5{jBZQqpi5Z>U@cVHb$d~^^sYeK zvGb*Y+%<{MF;%0x%x?jBpVgB2Nmg_FHS1=Sl)*|`=@-l6 zUp|?eD386(h1oI?BXci;hqcP3I}QWwW#%|!bcRZ71yT6=%cmJ;g?!i@3c8|fqx8V5 z_VV@y2yajizV=^IhVe{szPxwMOq3ywOzn9Vj+8U$NbwuGNuHrwP8wDdljR4B;-qU9 zY9vR;P>*fl#MRe1&YK;1(D}R^qXB%isELIG*bxb`Pp542R&jqvF1)tOW)a&WUMe5` zWZEESU%a$@7dyM&G8E94YoP~!3*PW^06dT!wafllfT&tKXumzc@9I>V9&xOOM{Y7q zUJmF|&>+SMr%UcnPi}Rx{Jf8oROV3&{5ZnlJIXJyzd|h7EiDJ9!%34h5Y~Nk4Dp%ZH=ToAnD2OPM;o7pE#(u($?c z@~KK6DJdWEyK;sEh&yOz$iuOg2fL=E5n*5M9MVHq-p~o_ zy3@I1>*Y{dlO70wV+%nGi}#{r=vTle$>FCl=p=IdmuvuX_kPPMOLVUM_;p^nvpiW| z!M#RivqmIS<=iV<*MPY>MBin@MpN%;^+ri!<`qlobu(GArt;zUm1B~gv^EhrUEN+s zI+GEbSuL83AnEm7s`!oe5-jqNM+Xu@&NO*qspW%U_f zhs_#u%*-%07Z)NFeS^sWmc+om+4Ygtp@A~tS@1Jdm5nqW%=7}a(y`c{C#BE;eyBr_WOy)x$u#8`!fOYdkaKtwHZpiz@nNtObtObVe0a_Z|uNQji2r?WgjmW`nvS%akPA%D8A{!=n{XPD1;HX;ky z;Y+-@9q}2ujpUf6%f{TFm|1e|q*Xz@rFyc_AkQZcf5_)=s?ZS9NUV^{c;D}n*B|1! zgwf)mtH1=ga05oE`O~zaZ&kH2s+7-s2WF755T^H=pCDjfL1w^k$_b|sjg89VQ4|c;Hv=~>BiVS*)@wCub zNfV=KQW4>#%6=rsd_B?ho{5wfIC>iC5jMC6zjTVqn1#=kje|BBW}U7oCqe?j9!4M{ zgR()aXQu89rtX)~#zo6f7k9UUSFY-2uzh+##l*X?yiQ+4BF08om2!+cgv^TKpZ1?w zhHbgJfc$8*zd3Ca_5W>q0k1ET60OnDSi!xrv;#Ub?jEZ^ zDi>m-wh(v|6)7gSy?)2-Hlu*5}UKDj2{3rc7@PtDO=Hs)o-D6O(#6|f^Ki@kqxQ_ zv1VCr=<1J2S*A8{eW^xDO}cCx|37;?FkHHm|3~o!bt=g|6F-cp7Vqa;LA(&E3yekl zl?@qPm|DX^q@Vt^Tv9WU2Ono4oZMB&iR&;L8!YAzA-sfXLD5fhV|Y!0TxZplytLcO zmW+95%hfv5wzarS2KVM5bM#`^&a|K7t0(FKelWwwpK$7j7GrtOd7S5-c>_%rOgd*G1=08;IFEoW@^A(&*^H<(m$ZHmGOlb5HCAU=?alDmhY5~naMtsDO<%+R0F~J6g`LJ zP1Q6#Oinkms-&k3Ki6^}+>@M5pj{)=0IZrN`b|X1&7qjmuUx76eNbw?2TV z@!vs0y-aDKu6k{1q?P-mYkf}QQK4ZH-3wx^_a%a#&C7luQr(xpWuZ-$E(bs>iC)R^ zySR(nixyF*oW`TS@04s9yjJk0)58x zzW*Ok$q3R>c;fE1fXdy81fa8l)YVG?7casgT$@y4yjadyu2{<<&%|JM6Y2EJj6kk~ zKyxqw6SIJ23x{}5&tZT;2HfyZ>q_OmFfScO#?8uk01j%h-IpxMv;2$oZ14k#f zl2$sxbw(9TlVcNL+F5(}>zuR-!Yet+#;qg=-?R;qCk|sfjiB=qBZLYQ2Bk(y{u*>j zdll*|zkh&Gs`DXY=ha6L&qNLHlflPLr*vt_ADWv;1jlDcik{4hk$WN$O5d>USx)GQ zGxhau-9Ll;?uv4@m*{(dOCc~;zs56(KoAl%LKMlZwQ%ZZq#q@56C_9|+a-PULf2+D zAg#w^pwhck{vu>1Bj9r@!yI@ufktwrArC&KxAd-1*%{LGH#1H_jA9R*)skCGYdn8ZVVRF+$&WFIpQNTP(CzHbd`w}fcTQGF@EF=Y>5|C z;#9eqN@V2zTF8HT3Kt!$Y{OBF&f%mGvZH1AT+1#I)qvKaqZQGt_K?#oUNAP-IPlq+ zUO0gvd-B{S9(+OtTftYXNK<-8+_lzFL0=vIJM=G#C_Y{KZw(`wEO_WaOcge#gK8rVW*Og$s8^fw=bt*AyBgA_f zHbB<9v_nbdHRod^(#KK#JZ6`7*_)%IflbB2r>fLXGCLX7iO`X#>3U!?9y)RckxJSJLbb% z$S-3kHWTY)S~$5xt9+#1XWCr-!VgDupUXMl=nOZuDtADmMM-+I9<|`Va5z^EQkV6c z9dSCUrpwQx&@5YZlD58qHMn0-e0ZbGbP%UyLgQbm$bq$-HIcxV%_f^tfh=RPTgzcR zp6t65Nah{7=jt)t5c|@PW&z|HY$ElwD?RA5T(3N3hu*7w9O_JsNnxwX0B1{0hm`0b z*fts!jND8^_ML{%VpT&D>H>I&LclGg@3(51K% z43>1(>ByECZlO+^7J4&BsYUm;>wsuELA&WI#yT`(Ut zkUd(K+<+q*9fEx4Zviya(+1V8)$lP-RZE++U2ECh-XYrjx zP7#;#$=zipHvbQp6b-ezz=X>x`7o^S9uo7S{E03c2`&xikYbY zU6ghZG%4vUpRM8L?IIlV>%AQ6PCZWhTLL5<5jF}5p$dD;H%tnXMK6PbZPq8EC5_pWoG4+D zrr#(`bJOWuJ5x5l!>w$3j^Y^Q%SJfcI?Dhj?>>|lluvyK(|Qzj$bN&5v9%D-x-mY+ zJ^Mgk)FDVSEk3X{V$&cg=GWMnw5LpBLB`SZYw84*%-8(S+r||ol zLT(eU>qZ@K0*;O+c+Iawj^!;qb>`Rx6ig$%3$D#Vt5kNJHLI~Q%bLqYJ+Y_h>xh9& zy|1v3j`6O{tC0J00!j-Jzhj8%=2@&d@oRrWY6AtF>|%Doh| zmab>PSNFoysEET;VECY2)utNNf3WO-8ECNYxhY0r z@?vTBhNWdJM%e8_PF+uL4&CyXpMKGvh%FY0c9awCOl_5xj+6)J!#7$)1m(dfUKsZe zPgJf~%RTpp~{Zn@Hd?CuueW#7xjLGfPy_L$c!(&)loqB|oC*f;@Ec z07=xr)14}qgl=LBRzF=D|9vW7^RjPsv=!ZW+6MfaYq6tMd27RB$?Awc;jE%wXdVn+e*c5(=PLP1!kf#Lt?O~H3oFl zgs#YL82VE8738x^)sBIJYO|pcUsDuGHiqyeNcOFIR<>9LP{l=6))E86t9OlNx6=9D z3VC=oy#AQZ*AKk}Sf5s|iCDu`gYi~E!=?Ede-+kSaIXCF92)zI76L7mvyGm@qtkJ< z(;#Vgg7kcY$1hdsjEucXD*GKI0Z$3e;1~hAll8cVKXgy>Yi&Q#omeEwpNhdzV+Z9} zLJXA&a%~EfcvDs03+U4)A%oqcalwn6lD7@0Gikq(Ow(!AeUwb&Y7viFk^$#WQTCQd zjI8kz`#E6x_A&zU8v3vY}cwiP8Fufr{DP*z|+g0FH@VNRwEW*?@*G^S?+3U*>%c%DhXeEs&>KWOYgNN zrqWf5wpKkFdch;y)bl6y!qhC_^^U4~kYIXt2l=ZH+W4~&-aJd)yF%OE0dnONNbqKY zl_Av=A*dIWYawPzAXb-UdhlQK?L?R|8X(1Tyrf>7E{YN}2#^pUzQPU`mh6~!q4j(BSf*a{xjxNq_Z(J;PkY8eHQSoP38BV>5rL}Slmrp)`^bV@z zk#Zo9H||murZip1@5m$6mkz%n2j2aPvqq?_EKL5Arm}*W;6{HulfU%V&3;TV1G?Fh ziM8V>F-CIOyauxsi%e=;J7KwNHp zR73qdJoYsa@H|FX10)YaV&6Z_L+2@XS0)i~6mFf2&|tQIq6)Q_wR+Oe1s%vrPRi$I zt{&ujiaP0_>lE}uReM?fD7T3E{}3I!EgR_yceV+{OH~ne;tMLrWqg&Fj6_Y(M2=AA zpQLKjxq#7(qFpus5Xg#$#hj^_*oAW#;aDV&X*}}#J{H4QcKd5_g5AwUb@FFP!dpE2 zWO1@wcoU1^R}v50zbX#i3U!b-hXJhHbil~jnScWU5KnTC!M=NwN@S}6e~owr*|SK) z8j5W#S8s4P3JK#R+HF>F-vXBmKjROfzO!jXq{@g?gXxxo>oM0R=^~h-E~xLMv;Gn( zO@nWKe;=;0okh8SnP~-Ogziy%U!}Vwx2NTUFl+ncN980LA0*kO(Ua+T;;D~Ae{R5n)b*$IYLoQAU)M(5V?@+ROJ`!0P2XD`(2;?}0&r?gGd zasA2|?EEBiR9*3-xum!qy2{jvlKt7;>0Yd9CHJCkvuDuxngbjL{VF8iuwJ&k+69ow+>i}Z5UbhrjFHb#)cg4v6XnE~T%cd>P`I1?TpxdA~u6(V_ z+)+7?$-~pJEc;*Ju(xgl;}*&YM<8VMfn-XC%Z6cQK(;QiLeeV-6XkxJS%gGzwWTg6 zJqDOn_LO1<0UEy*=V;!GW@u`nv@%N?p3 z0ass=%)7?Dj;buW^iP>ZQhvdzkOz8zO2*@xdQ6R7Pr@E8`(|+0d=(4_39(J|?jp90 zDsLaTSb_*3&pZ_SH9cN_Dlv5_W-^UJHh-lY>;wsK+J(b%{R}KX0kuv3dBJ3!$HQ1B z(b|q*s@#+1y`82{SE4u9zSCeC=(p^UX{eLPT#Qsc*9-b^M#_X?sDmh%RUqdvxgkFW z!$<1v@7ql(pJP;Ls8|O;b$8`$LL!2=^eI)=A7%;8gAmB^jGP5~{GoyxIl%~N8fm5* z6Iup_sh~@)pD>fj40!5Gqg|LOsawJ0c9qi$xAcE1wl;?FUzILWbFo2mG&IXv+Ac6# z)o5a89zA)VI+t5yTXg(8-7ic0SM^&8Q#~ zxt4yfSx7Joi)pET#ag|8D3&SulF*HZ%uNDIO;!Ff`K2Z-$eCE+^uPCy)bW!E zosc!dzrd)V<6tOkA8cu?PYcPWmoR8=zln4(GC;-I>rmQV4KWeAEXqxBq1U~m|ImS z0^#DFOth<{k({`MvT1jeBi^49Rw*e&hvdmtriW-~(P)`O3mK_pcLscw*j@@Rn2e3j zbQ|FjC9-b=482n~H(&b;Xt3-oT=u;Wa6>vQo3APtOVcov#wAr6s}+_D77UHV1FvdA zX{mQUlEzUVL7bx-uaXs|;4YnA3C)x?L-4sOCm@>&RpC04?$w>7aTU1h(b@FMkOh_t zLoz)>{?VNwjc8xvYpi>|`5N|T{}RwpHUX--vJI&xB(4#M_Tf5lZSi5UE#Ymf0Ea9a zCU1{H4UsHuM;-t_@)C7#7*nOu+%{*Rb@$g?||_kgCrsuLLoW`YKNwSlV?T zI-|^~^|%Y0FsGw5N&@ICE9)Sq*KnTss`@G8Xr`0e*jg7RZDqKQz05^2W#5OjUJ)gO zC+6Da+9+ho9aYd+R`rKAd9@!~Ek98Q2%5^3Kg~p0eGqwl`a=-(*b+#DCCpE9fwCpY zH#)fDb^@6_<8g0Rmj~3ZYy&x|Tx+q&vgS%z6U&P$3sP2bF1(6hCn;~k=Qpe{^U<8O zEvX@_?FBbi&&Qt+Es@chkoQfuN~GZn{(v01kAUCH%VCvSD%eF%WLQq=$J!7as507< zUWMS}dLt(Y4UL2xD^sMa1N!T_2{gnbXAo>5dYS$KGPRjSa{UQ-V5-VmOc<#$PIlk& zJ5W-gi758OXjx6ap_DZO##eRb^=@5NCaSED-2WDq!3eVXNXO8lx=pMb+AMtv-d}uz zE^M?>x15!S7_4yOUtsbL(>>McZD7Dzs;nENTcH55J0P7+9%bp^{*PlGj}h!ZUz}Xp z!lgU85GIqSBV5B(bv*peVjbfA2@o|}ZxmAs;d4pU6Suo%3WV?mv~T{wu2=46{|`T`*PyVjgf32J#_vfoj`s{xx7%c ztr>?c?86_N9dwY}smq&Y$>a}EFdryKC#of!4RKJ|e)NxCVRA&}&qplTo9M|XAz7UQ z`%pYR6|yX|k#S3W0B0;Y5w*rBcv`IM%MblbyDaIAN?NDv!!Y0Ma|nr>armAd-5iqq zfZV$o%Gs@qOu5qncBod)(O^FRA$jXN)IqyemJNP(H7CXCYKpH>5iYy;SPtR_Sq`4! zbIWe8N#o5<;BsIAK;N6<)|WMr zM~it}wT^sWR7msmqAyx$Pv)Tu`w81K4bct_`65 zx3N?nFl)J8Q6Ff_O$$6zG>GyP43tkx!`y_Q{RMb7g~{^g` zFsx!k5)wY6JX!wx9%->r8)9P#^mzM=!_@hY$dBdv1(%m~ zi$v09+hpSr(DM(1yW-h0rN zclpg)iP&RS$gSR}>J`PN6W_k7ku+x5m5x8HAaGprm~7FLcn6I@$=+i?cq!jNOtiJ9 z8?l)lYz!p&d{xPc-x25_vh*Y2xa?js{fwCfEVIX;BD0!v?#kBxqZcqx`-oQc5pu28 z%$JML0-n$B%Pqm_BVYWGs%XY(6C-eoE4vYU&zLKV|G{9oeTp;QNwAnTaGV!CqyqQB zp2l2xxe5PeCux*%BS{Bbh(A2q7_Ilr*NBUDlw=qWq*X{vwV5ol&T(|7qTs%<_NmyiW@>Ayg@g`R|uD`J6JS%g%WgiAc1 z!;uS>Z3^yLsIz9v$E5KA)XN#=orPL*TEBTuEzBzG00nT#j4d&j6=1imcUPiuuGJq{S4|CFQkYE<||tGjgdV zcH(%gnS3fSA8}BSgKDRSmDi1ZZ|)TRH~lVpeA z@?cr!kHJ;*HkDu6nv528T%as^219|2Ni0f+*QL9Q?@P$A+m@`Jeuvrsj2j(&& zlU;);_wmzQ?YwMxm^;o_i78t8MoGOV(1H#ItxWmdVrR{Rf|9(<8POV(@+ zs}-9rh*_!Z;WGV6%Vl^o%XB=YDLBh&q<(#oRYrQobZIBkCSQKQ0XJx&1p}?e^eF3x z5DlfGg)rG(t{*5ZnAn}Z%_(D(> zG?PXQNQMCN+d>Z|kIKR7FakJ(O@g$_w-Oc(wZ;xkPL+O4ZQ%W(+0yYg%Il~q>db#@ zyUE`nvkF?TvSFyXy32>Ffyr(a9LODm4dk(xAovr?cn0&;-3>CT-AF*&*8-0)xeecN zT2MwwC-GHVI)$6n8FwQX&(V^b$bDwh!;vzmBepjMa(WGSe!RGg{0^>4s|;fodox}I zz#GcD8)*@Ufh3`guazf7T4D1@!6_|ml34eUist!Vm2EWsD)yl2Dwjw%J=ARC5mXZ+ z6DCR1zu@9os;G&$U%{tR=P-+wPejJl(Q6rwPr@afrSkhBhLcAlN_@SISIa8_CVQ^e zE8jqHp4b9MydX`6^amhV!h@K?maiGRsm$2`_SuiASIv@Qxc|4Xd~Ld<8WEXPJL_ef zHzH`{PxO1_>Qp4>4;naAbuZlYKP<1zdX2lZK9Wl&Y-gBViZ4T$lWIz3Of9e|RfX|7 zl{`is%;Kw`s#2xh-AdV#W2VcHv1T#`WH3&qyn|+=$0$?BfuEF3YY<0%y-gBj6^*r< z(OjAxgAC8Az%X$qas7d-iSm{IjR%*SS3o$(=>%8a>GDiRs{oN6%#$v!TZO3S>KzEu zP<^zecQpqfF*8Ob`s9-|QfbpdFv`16npxOyIZR`1#ns#E%~s=;x0UgZwOC+nV=At8 zRw0&uXpMZJQv~}}podm!(<;ffR^q3}rz1><7_#kcm25eMyyM z@({|8o@))`KM^{c>GF4WJG#+TiN?V3G$a^&a3Yirpvyxz+lwu}wsvHJ_ zY=JdCR*9)laelZA-9$9x)$8b}S9B3xomV+*S6wENUE?7;P~m{Jv$`4kYVvUJYxo!uo4rje&ZV)RL@dl-KN4HjX1^F2r<-?C5Qm-!ZnRp-KY>FNMI9Qc6BwC#x zYZMd*p^VZ7MoOkn-d&3dYNe`Yki{dg7WE`bYg~v|(P7+;Tt#7myfZT_NS?&E49UmJ zx%V=`MW@y#9SU!o4$wxN>Tc!hf*)0=jWI_!*CF=*thV=lBXTQWj)R$AR7DB$-)sA_ z2$YwGlZ4A@F0I@9U9xT!md-4lf%%689>h9G-M*{r(~xCew9(OUAF-rsgWl`fXzAP3^R&E-#4dP!~^`W}yF4xx*T9dhV^Rip!U z4PnBv2O(BREYGVP1aQ)ymUcb-WpZJX=_dZ>z)s8w%TQ<92}0eu2kRv8q;^7b2vFNP zqk^kv8!c<>95!(tFiVbRXyLvb?T@yXPM6SB<5iXgf!G#Ey?nD$etX0U$-t2i2>pFJ zU{;ib`<^@-1EYilY;OWb7(frn($=sOaneb0rJd=JHxD8e|5UY$&_1POv?1c)?7jp& zp&?N|+5l(@!uh8g3tZ&Kxu3Y?wL z-Yr9Q`OKFmp%}v46TnjbIP6x(C{#h!dgF3LNL#DhjSFEj1S`dp%3-TFgR_hdSjL6n zz%JW~_N$mIr7xii25Tezl&+4YKO~M|p{FlSi!V|-r-%9F$h)BMj!L%6nyg&6x{{C*{)Q5^)hv*-0sQxyv?}@2@RxysP$RjlYr1m1}QS zwPBozmDmquX3GIRfe^nw*5}Sx_&vh zAGaW~v+0(%_L(+5jg@;qm4xZYW-O@>-ws`}+DzVgtL`wHG4L6}^WTiKN+{>ce@R}s z1pU6D@)LBLbh?yBA#~qTRZEDMqgYcln^87tH~7*tjtka zYYBzg`beUaM-5R%1IP;0rK)PX{MQ#n{dgM;rjaVxNK&s7M%^=j`%s=7hE~rXfT2`1 zUV5ECM@B8Qie%6fn727eC}T~r7p}_+m%m5wg~8=T#)P~wWF|u?dMmpZ!`G(d)>laC zjngeXmybz!Rc}~{vir#Y5C93^&onu1QqqAWwtE0Cb%&w6l9_|ZB5FA_n`1HtZ8+88 zrQxzS4<2F@YplHfh!%MzR<#toi_Bc13qtDLGOOgJV{rYD`bd$}%Do2VkP%6&q4ojf zQ7BHfT!*gT(lfQ0-}@@?rI>8E-W82$*b)X#Z)WVt%OToCbn@;(zfID^5jM0w^npz) zCmA1=86@%DcH3kadm%Vv6?K$+~#({Fcm?iSZDXRjipf>8&RRGCUMfHvTHEyG@ zWkqko{E%7JS}EQO3$9Z3d{FP~EeG`M7y4B?!trD&tFeDXQNLkw45kt{uIIgCT) zb`1Wisq*4-P{MQyMJs@(pIp>~7Kf}uN^Y2rlpII&hjsCgZ;elmN7a4GW{i6aC~wR$ zoiuKhfP$c3zSY*;<;f6c>tfVG!-K6(%8?Y=z1hjc-u30qVxYie(=ObXpORmiLBL5Z?!sLsVfAh81ghU)1ULtvb_m z>oj&WbCx<7&n0ORhrG7F06^z$!EE|=gyn&LsRm1W&CJJuGop0v%qFd$1HR5dz}Kam z^CT`3gcPXKc&XF__Fqi!XCT5W3yke)?6ygTMcz?HzBE{A*2>uJW`!2j(XwI@@H{pF zK&30AF;K`JBsI}kf$vewBHK%FljNzefV|Na%`rP7i#nYA=F(;mJe8;GL2lr$$m;jt zM{wkjo5iqf|I6qbDo48-&GPKPDwu&4T$X_AUAs^2-GCW?$7^QL;*~X9hJWkN#zx3q zEDH}Ji>|ATkusg6k<948KT|(KU)%^IFK||6Jb@YFUJasydYB4p*qE5 z%79GMZS2l0Fx+W6m_Y5=7*$w+^&V=7#P&4-&9qKryvOx_9ESoqN0+m)EuumqyhzMV zI^IyEi#W=dg(sb;`jD%CaG9($z*?$2+C9zIvomR%s!%?qzaKB_4w)`FIM#9+OCy5F zRKoCQRH&A7bVmh<0yX*en@1kgQ8b&4`x=D%QiumUR<;oI0A!vpfBBAK#!F?5)DU1=e548c*H zH<(8z%{U;`KCzU?Z7zx6EKjVN)sge zHO}py$u;my;`m(#y>80clT;E;&4@wx+LL2)6Xo1^lZxx%5Oo|yF7iP;Y=iZB{q3!q zT3m_jBp9%|iT>j_GQQ31v&jQxjgXr5knA;OPtaY8OLRTtBKgnze**k=wh*4rE2(e?7P~)f&J4Q7qLXFgFv_&YD zYE!8h6u;N!`|BT9xqF_^cz@pe^B%~iMug|#c1#+HgFb7&r0fJavfJW9dvwd}fIB*Z zAiuN@pjf0++mx6FAISEWkTdQWB(G-}>^y(2((b#WTgGEP+RSpKlXtdmDh2#VUp}fk zO5#x=K#mYTJ4=XdE~Sp(-H7#ZC&?`Xg`{L-e)wHAGDaKpG( zuDF1|r4O1;u0vT^=uf(bvQetU#N*aZN-AD)r@*aT+X+mK94Z~^0>jUumD`M(909sZ zB}C-cLO`d@VY^ym@tp;q_(dhBt<-b6$oc*#ZNKM&MU$!B$xdMMxZI|2x0%#mZe3>S z$AfLijnEko6Tzf7QUc`3?+GDVvkuRC5= z73{LX-jYV4G9-CRp(N%yt2>%uAOhu*x>f_yiKK|fLxXxs?Vyk=DptI$9@1LM^>k+v zk!PwVmAi7_luua*l@sGS{*_x(PKF-bwWoE#uFdz8p9pT0L!Tjw8ci`$kPIPTVg#qR zm$luv(YmC(fnhXcmwq{{;Eo}^vgJjbq zoP9OPj7EayQr@ozOSHc;9NG@=0*F0pjrJfl7X-~JflMTcu$P{81WhBCOigOKz!+Q= zrB4ntw@%%`-`@o4PeorN#ww&OGS*cAc4Dn6TZ8v#N5)qxO#d^^Sg+c zM!LPTiZ$&o*A50<6C40s*%Iyfg&ar*1?m?27C?Eb77;Da! zYM&tW!~bvYxaP|bUodJR`je>Otul1TwMjydJ5Az7!{rt%f>WW!m(^B=tHcR%m!8c6 z4+!yWNJ>;lnAP%h9U*Yo97jU0#o>CMB9OdL>$ppfpCtDO>U( z3c=Qi4AhDR$4HKfN9~&z3QvY}y@tX47M#JYYI<9#QyPaz@H%knnyON{s)tjt;E+ptv=CYo>M4mLn${UyfCz(%#guMUH zX;Bw0of#CNynu3W@{b?8BUq}6bT8-3kzZ8Vt!!O8Ctg*-xU1e#GPshFDbuOVqoT96 zOWi((9kfab1YlXdtSrT(LR-oyzw0=eJkXsfZ8RfRMZ{F#2tw&a6!o419?Ebc?b zEA4Tqs)pM^PwvjN>8s?Qx8N0__dUT-9NX<^5eA!gd<$ga<`5ddW$%yBES%M5Abso( zRI$t+()zJeQ$fE?hy^gzJTW2ALvA_X8=>+c3r0!u1zc+xxyT6^_fn~w<90w9+z=7p z708tNWw7qDx=2j$F1xzPass%hQZn7gh=;VD<)q$Ow9$qQFn8bIum@zoL&(SWYAN7#<0|iLYqJbp-G;l`HzsC+tHs*;ygKHp=6ftjwRQne2_x>U~@f=I|L zMH~kghpoqh#tFe_bl%F>K?&zVT~J%QjY6q^)CZy6OqH{^u7~tOaKtp_X3SL|455_6 z%Z-N0%yZS1--F?aiyE6%fh>H@OT#l^TDG3DCW95;(WAO(O@h;;-$5f(ru$LY@!0G1e_zuMV-o*W?!L7*KX1t0t{k6Z(vhV&1@6#AJ42;VWw%AE z5Kh7TsTf0DR6o%VI{`~T12bO5DFz+3^w4R9`?3y7bgCPy$Obq(wK{xcwvxmV zqe|jrG?XtR0Rgqgx-MNR>FIECa+StWG*af>a>hx-eHc?hG^OotFzq2z4y58iZO#*M0))wPs*~$-teBC1?syLCc7BU=LY_$WGEMq37p)NgC{;AWueo8nnzW-;)(g-o5=>@#m|=_bC`L^B^W+gh_$um(G)g52fG{* z9);!~>7|UGYRHh#CJ=8C z4TrG4(6+cq(<`IQM24Z4rtb%x;+x8$isl@-T?vVBO3!G_prL~_XLPJ;Q6I^rVp4t7 zMY|gkG*yJZJ0=o25kCV}5kdAJhB!;H2H})4jgd7!CZuzSHq5IVZikE=1ZYEuX9cH1 z_5oKCVlUF*ADZ=d?ozIXII3rOZP>!E)=0nRStDN0~lgO)=IYT~<#fuMqQ58j0Ll5hTDVtqc(NK&`_j5+dXUduNg&x}#?&t8AM&+Q_zIyf!6xVnI z-fZ!cXUAF2&^%SE!m9$8}*( zX?jLm44HzO5Rq+`_&4%%soOz#XeX6A>aXXI+ND2wBYs&PQ>N&8j}a zk}fcQ(w@9IAqt;W)Kdm&FDhqD(AF85AY+=c<1|A;;{e}aJ%5?R_(NSAhru8R^vA8} zQy$&QUtMXN2q%&)iTwKZ8OrC5zcKjNp9DK&R1U-r-A+NyD@iykJOQ$pGI_A2kfE}% zk26U+o`J=r==uKOYN!o;FOB}fg&@LvdPH9iF`}bXAHq6+P|1wTbo+P=#>}r^7g3$i z?Xk0`A&3EKS`rv%uZU|oCzT&?wbILG5wM|n^Dk!%3nr>Pld}&QidQf0rgRSjy@u-f zgW-U09ZJpnT;88?TMz|ABA2@0B!x*&PRALj)#M~-;=`3$ZMpAG$d>y{6Nq7b2U*;D z3EPjz!`twao8;%bcsx^SRdE|d`pd;9<`}JZVy0~94-;6X$5&>HF|Cw#JyTZPMG)N7 zE$6hJsUF!`8n(Gak2xdJ0nQv$P?sv*66N5-GY9oIZUw?FZVDb8(-<~q8ziBNVZKQ# zoUte_)=b!HT0M13F;3RbLYxsP+YtoGtt#UlV~7oY?6ko`(Tm=yE^l)W^XDB}ULb%i zHK}26Q5B2#H02mWTCsmu1DSFTmCuow4Wh)3#-kK5S$4EB;^m=-yRoai(0{g;6yL);8|^3mDx4syg8063Ysw^ukw^V;ip{Lc7Bz(qRrH(@THD|Vwq$Ao z;O)I(Hu(70>ed8^E3}&o&~ZAnuYh9O_ATn(IX&Man-7>RsD+MhTB9o-)Q60(wA$Vd zym9M>ownNIJc|q$R-LG8%2I+0eEAwF9j-?-6JwS$ z2?4`RP?L{vn)IT|8#T{qlIkllZIbtad3G*DaTg;5=j-v?u|PcIP>C!mtv2je?wF#g zBO&PPa_|nyZ!2OzVCNAkkrVIujhg`UknWi*gAQ}vxAl-7^7wa7fhFp|kujB$E|eI7+o1km$V-D(w5f7JwU9kE%y7xwBJ8`Co^}io5m5pSt)QF4PloM$ z^9RgkRXFWMN;;M4l3i2IUvazOtwd~-h}~Fvo`TPm(F@ifPHfeX-|<7G@&F7Ghi>g6 zUQ3Z@rYjk+!=zs5Kbiir^aM)x^o{J=qzYRU(bII;SxkRvxRsqmlI1MTF-h_6<(AQG z(H8Nm4ELtHSd1i5Q3yMPi2feBR$qDb=3`KJ#)AK`E`S|pm(X{RLk-=wR;pArLL|zY z)s0ivO57opgssOl^d6g^bo8)^zxB*6 zs5dcXrESH8Y(SJ=gIfcld!8!q`T~`v@KFy7)bhi#LAH>}Fv7^jcma|tgHW-xct1p4 z@`a?zxAl!AP%Ew~mK+x30_E-et`;J?gr5Ex;yphRmgt&Eiyvo>6hCaFU>w^Ha1_~b zQg|`~!KG(hmBUrgUGi1gI;uKMlh}ojPhW2roXv9?PE63&B0b40-KM}IxnHS8a6n{h z2_1=ENNR@)S-_>kb$&;jDkwDH>FEua$5m6x#3ExC)c_#3RrGyTJ-fSnw~*v51m$S3{tAg>@)_LsA77L-GaX~q)>^RZ)pFw#&B+;ITr7Aav+#a)>91mkSJrOU`{Z$u@wFWP=1^3RQfjs1eUgkk$F>YoLY*i za~yJf7F|EFk;sr*O-Vjeg3A=NOWpE(Cff+5*Px445G zty>#n4TWrjm&ez~^PGcK8qrp!*Y}Q4iBGTf%#AYYDcg_Gt>2-QW^_kn+E%E*lmG$) zg19vejh3~#MKElB$Fu?K@znUp%7)3r4^XbqT5^I`*-wXWjW;;%94tq6{#AhWmkbQ z;7m=`<3h#qk2y!iE`+n6P;%#kZXK#F)yhI=>E z$(&w~6TYXqpPKy)0z=kyQ{v1|{#>nW(gY)0)?WmMxSX5FoK9GO(_XuiIppLdM#nUm zp%<)_mtVo&O5^8IE}j80vRQU;u@%=}Lp3Ite~GWUcGf6XWvLGBc>bvA4qTWBJT%z9ge;uu&pwqE%0 zVr$5QhRzfSF{^_VJ%I#lg^G#<%!S_cjP)usFJAk?AlV_e8R$0fvNK<{lTP?QM<)3I8HOF+g>meoM{WTS8XdN6=yzHRBVOO!RZyt2w{r-&wX`FN zF1dUJICj!=BDkyKdghbUsnR*Ke5GUs7*woET`Xl{LX&Py1{aCouX)UflLM+V{JmwI z+7UfNRf5KozV4Gd2_79&N)jy)^JS&}RilFyJla(Q8%Lvf<}z~*8ii((v_dF?!SA{H zNL-1~B{g+2NSgVz{5)z`sJON=@AH*Lf!He;8Pc)?hQKR5J)Q87lvIU+LolX7ev_9E z;rYGjdkItJx*BBPOVx!024=>P?zu&Mf{VY#v^4h5<<5zf-71PC5Z2}qSOSyeKQ8tKyS z4|9?XY3`;$rJi3+-4cDJFlHWLNju5uI5x|XmC#v&y4d(q_Z$#YAAr_S)lkS#Ws6AJ z=%-Qzs)^+1~D4qXY+ zp5>9PInz5#&n^WbLHQ1f2)TSYl25v0E)~f*#vNMt3%n;IkKAjt2FE@;xzv-mP?@Ehb<+Z_3$;klJ7YFOrk#fwraw~-H4 zq0MsV-61M1r8=mBYo!Jy?XWV)-ZwXP#>$Vn(<7_qp?WmZO;eQjr?+XOe z{fK9!U~2yciVUJna?AwD^iif;z1m>eWj%BRl$C}A%xQ>k$~F?+vtWzdH6fXnt6~Xm zRbGk~yNYupe~vwyZjt5F-5CV;=k8~kxK*&tv>6a@erJT2WuDlcx-Hy`P#!<9s^tg(0xDv{q7jsM{vDPMSUZ$#GBo|3Z7Fnlh>Z-h`(wU^4B4l}13>vfdZK zzOU}1#BGUo$?QkGR9{d2QSyH>=PHy50&G!Zpl{3?qUzqrmjvsnevJ5<2b+nSQQ(Qz zwaIu$2)`!Sq5{6Us0tt+Y{lc?s#$fNX(p`2=1Q`DT$=`!@O zL12y^VN%Yo8rUEa4HQo=fv}FX!q6xckM>(jp0os7raYp+Vx4wY=4d7X`#5}ZI>XU4 zV5R=Z+s>1fL1Sja8HDq(z$4I{fBOSh(*<7Wn zN6MEWMvB7p4Q$?28s41snt5WIK>`uFy5krvMY5bJV!p9ULn4O_5{v3kQ(3o!qr0V$ z`}P`ZBta#ke62@xl^t(EuuXdAWs0i6SS+czvSSRZbCLls!>%EJh%hwZsxi_bg*|oG za}?ghr=e8S3d46XI9FLu`M@Pak{ju!l|cVXNykTn?{d8gW1Fo!fMs>d1fW0%=YzV* z4beD$(0Y4k%Z?h(7;*VRGf}jFk=;3FC1bJ9ZF)*A{GkPXXuJ&dk|SMHMQk*L^|?ZT zeb$#OkuxW1-6B5ik-!JGxs@pzpNc~eXC&8AZgj%*l>pSLT#)EUuFc?f*Wi3P)dxJn z@~SL1zl1qX7>pFXo~jZosTI&K+$9@RVF9&}J=iQM+u*^uB^CHeA=tAvL7I+j;rj?T zezhwEMakr)tq4ztMOeAD*CRfPUeTS7$(i2-Pf%2@bX^M}0Ap`i)D#5oq>9;r#6I#O z42>>WcMU=n3TrD(cB9lJ+{XfSjWB1mcweo!rvAIL?tlN&O8S4IEnXp$_vPY+ikH+{ z-ziztWD5PeL{XEBnTM*frD4%hCAGeP6)tKvot!EBSP)_2^Hq~lJmQ0RFnu%JyQoE8 z{iV#%V`hix$Dh55T4k!$9BC@`u_!-vc~_M1jQD!kUs?1@l!?EalT3Ab6rvEz!4?SN z3aUrV@C00w-Bk?Xi(il)heuMI&U?X@>amN0{!zHB4#E zfGWe5n{ASJwP;3TW1%~m9Jr>$cEOeqSbwKlO#6;6`RN zcT!u5VYw4~!>?vg^jc>2VbssR0(b7sLnb`r(nd}dVmS{^!*mjVWvijrnR8zfAJacG z>{3DHFaH#7lif3jaDLZ~$BrFh^d&(~ie)7;dh>g%C3|eq3nneb!j%3V-i<#7gwB0n z0bZ6XQY5ttm$xl{x!RZ zI=Fv%uBxiS^OpYhgl41Pw^okBj}X}BKd|k9GCUT)7YS5zJpizJSlHMGJf`LFuR5&( z+|Hs5k`&1M2|0#M0>S{R*Ft_+Glyy4?ck%Ba5lqN0H|GqS^KMpEa0~Ctxj#rC}$r6 zlo_S@FwfxO;^X+?!XEy$_LG5E?=#*#^^uW-nlR{o6Ym~80zz$C$hylE@au7t(c?ai7o!A>2?UQf{0OLL@8;nE zwNUdm9zH%FuylLOPPxdk1i&ZM*PXnaA+!0jcqFXD%<~rm~c) z1&~j#ZA`oD4<0-D-W;J#3D4JRh9^nqPmH)cg7bZ|z-g24<^aLooc#~(#hZIqu&31% zf#Q)9OzQKkVUgd~84mfaKZ93JWag*o{JQ%Lkhyo)NRjJL*ld+hIN7r3e3>1s)ykQ}mKj&jz1iQSuo8XVLIe=h!s#Ug}Waj=ld!GJ= zeJyPQL|@e8)!Q?R-2T8gGzbvMQC~5~s7_s;~Ifvz>pf{nv<* zs{@@DNomgxugzl6+uIo4XB+>jc@>kX&``yU9`SC1YMC(U2Uz!NwX^fp7_?AX93ki}DZ%)1!dJ4IHO^tB>#uU{#Um?NC+JH-wkXYymCFO6JrkK)ZpRX)G- zdA7aq2IID@#B)h6)gH_9{!X&;GJ`5EU2yN%&nWW@w zKsj+V&qmQST9$kS6gB7YuPSHY1D2op;ZN)s`K+~Jl|sdt;yZeU$^jMOS9J%6TKpg& zJMol{n-To*ygA>!QKJN|W!?2vZ8aMajABYKqU4>OAfdX<<_Jhrnb=BgcMC*F!PU% ztgrbTUJIzj@H@vb`L*sG_8^6*i}$fe@CODvf*n;9o~t{beb{2iAZ;|$oS_vB4wbe= zcq~7PMU^_q6W_Mw+3!j~WBG~3Pr(xmilzK{NgWmAC1TdbWRnWZM)n%aj)wsvFLpSrWA2X{FG z>q{QCtNyJNGme!qN{!fcljXe)1S(TX^G5Z*naoRdKN>*OUa3~VDy}?5YLF5}2_QA{ z&mCaFE5cQ?HO)twwd95DR!%G7s-R+1^54X_wPBZGUodUv1%O=5o5_ZU0;uc3Oi@OO zqbm0ekG3y5Nmhq3q>Z)}awQ9(H0H8ide1W)a`FQ&ETF-!h@!MME;!Zt)Li*TMeU$^ z!0pZgvG)J5vDr(RMk~gl7-ED-ZxwZ{sm-2g$E%)>4D#7~&Mj^xk5sC_!_Tf6b}13V zDEeET+p6(~C5az?&F9bTp8QaI16v8H==Mm1aqRK>4n{deA!qG;BDK&1;Fp7`rX*_; zS@xwx0JXlr<&NkklPpy2P8*pF8ZIR;=9(%zcl;*bdFhOK+z;yb*E?6;5qU|UJy?Hl z60_v>UkvGQWvY9nfq$V8BSx;%okseWVvNajzLxdHK**mp&Un)zejtdE>iG14ksD}> zl350Lw9r`Qe5JQ6|4l78)yS5Ld)UsPd%X6vDQ{RSsqqH$oz{xAdr<{WBC7K-Z3%DW zeqf0OE!oe{qj>H~U*Nm!FMc?=8d}LywN^fMG$^||#wd{5!lte(1|EBohx?D=^cE9M zC?kGjAJ&%qFh}QO$Rxuf<(f0)%7vUrgd!)~KRnljQpu7T!SGvE*VDu#jPf(l5faG> za0L}Z7|Ww`WH1$0rF&OqPOr*Wy>xpS z89$>aJZhgD3&6u;oQ%#`?X! zxv6aX4@?Oulc33|9c+8lK*KJn0W6`xEyz4R2|(Xj#J1n4>#|h`dAHvec*oCCKR^~o zVs(Fr&X%?Xp!D+XY&>%ah&)ndcRy>uj7DP+zI7*ttFnO6Ck#Pi%~qK?!QNrwtiWR@ zvJ9K7SjFxa)MEEXz5^lzOnRiyCq62TXZ}J%_}Ds%LGDcCxm)ib>E_jp4B58{ILr!0 z0;py?N6mQ3e}2ugL&mW5v%pQ_ya5WuTjEs7p-81&tLyPNv-EgsX!xfsWpOs(YfGGN zShW3lF*4%|X#0-|EFxwr*AQ2(*tGcIBxzRy7#hdeOwYT3!0(WgOT`(igJ?t=hjlPt z6w5Q11d)=y>bAq1Yt^`t@|g-{8QZ0oUJ6B`J z#k?b=N@%*_B}HuF*FIj4Rgmz8L5 zfzub{2FTt(=3dbO8bJ-C0Dg7gQ793{bd%L(_2z{?>Y+=d-2tWSu?8^~^LgQcmk}o^ zH9#y^YKRnm4gwrXfn&{^&KOqCQ>=rGNaHdhdUYf_L zt@7eWS4VX0eME<7ZdlyTC_umCG#j3^2S@Dx!p4X?DUQd-d3`-Vj}&gIm$pyf#u`t(ZkIJm1{ zfnk%J&3JjLUP+dYTX93J_9dIeOM4z3DUUK(#V5+=%8@iYGKW=Ds=@C7>|c4%Y~|bPom>h-N8)UqgmzUKnKGv2fpXT1}{O3^hwOYUq>C735{51o$kE(T>=Bm zS-a9y>+WX)Rg~BuBKnuWgH!%*sZ;Vx*c!SAxRHk|J zLySYnky$1MF==1bPJET}nSbRA6}P{`bL&SNNQ~|L+F&9_IkN*Jxk%EJC9Qyco}Nzo zqr$AUdK@0;@1n!xtIpt0t=%BjEu{(mTbC`xD(6b{c0OiMm_iy?N(hq=B^a)h0mk=P z$u>VeU_B33^6S~L{NU4shyNo1Q@WmJ!x!7I&;Uxs$>=-IZ1i_mS?SuHEpGf5Tb(VzI6wExsKDM{^>B(R9nMH|(0>A#PeJ~ZT% zpXTMrz>}QMx&1J`IeNUWbZF0luD^BKaNIdert=rPgVu3*kWLvig5f29h7+^vBd=uK zHNv=-@syUF#Yo(XsiCsb$r4tS1ac)*Yv=AWotQZXd27y+qT@{3$=#elL|3{yKE6n#L+5jj4Lhp!F=PbS%%CaLr%(D`My(=M~8 zfOmh}dAEr-2Nm=SCvb2j#2fMuEv8Gr6|?K-$-4^ds&6gu@zzVuXofPRm$wAOX|J7Ga<~^bG~XK_osFh_sV`u- zp>v-4V4(88n-L?ko-u7PRiHjQ2gViU;mlHtW-DLp`prg|jQrX#$uLX@InbUB z7um`Qq)gz~uASLfe>>l;7{Z@HHw};M-pB6FoPtwq8^90s-!s(e&lyxcXX$%;0f1^9 z!PaEOAQxSNHJuN3hRE>o3}5{;=vG;IvaTHDE;B+8sVYa5o8bLrCVEC0CX|^hT`JR~ zp0glBo6hc~B;TL2sMi*KK71qZUQksOrnh3Jqi-8lc|ajPZkDi-zn=oJM@6eJeiJ=* z{dl&}P9t4jD@SPR9>XCIKH~(aXiM2Uluuc|4jH;M+$IM$fCew-ft=0hNF}46Kx9cV zNWI0GAegR)_{(?O3`#p)v2oRs<1HszBLnhUJLAN+FTBW5ku%@icUok21_*lVG{Y@# z%4UDv$y{G|Wbd=qBl^Ci%7H`{@W}PfoN7gKR^;IaU|B!P86hvuGiW${u;st<2H6xP zd98;E3%iGP3_PN$O5wGvhbZvU@~blG7X4tjq=ZV$*&CfJ_2U@$MonJW@eRPQx_}?g zlba}Y_OPBQs(8leDy)_QlN2Nv2!gEX4~(j8bB2hyt6|k5f+>CX7$8wV@Bdr zay8j}&%nFTfYqE)a$#gJpeLxzM~cqnwc}-&A)hRM`DPl2)nqoXOE~9@k>(B_zN|cx zVOW) z8FsGzbIQmc=a_4ba&&#B+85Fo!B=W2YiR#>aEVDL;g%(Smj`M z4M|7b(#8AB^JfstfvN^oMOFLAVe=59Tl%rf(22ZJu%6|vQQY`IBj$a!oAp&tg@7uM zjU^?9vCEb{&;SPpXA-5HRYSIO-9r2)wR2+*GQ~y}Rb2HC@T9F3&Oq9Tk91Y`c;ls9 zw#McqOTRS`(WXh9x9cpU*|$0~DASYRAvzy>p+IXaD}6UmWPMjmIg^s2FZoc)vYH}v>G$r=ypmADE3eCP!tW%I!2 z4fCB=X*JuKDb+%GctLZ1EfxWu%nG*4531ANy+bT4qB8&5T*{z9ud*C(!Sw`IW%@ho@Ur^*?Hu}E-1mf0Xo@7J*$*7 zxelCvv@ylcwf!{N-q)3ccTf&~~4Hj0bZDK%1@l+Ks<1sv4L zx-4MdKM*86+6|&{Qp(GquIwyfGX~C*XjQa3uCf##jl0=P_tX^kb4L44-Idh#pm9-v z9Q&vhFYMh13;#MVO@_M7yHva5NO6rbh$~T1z_i^CG!^$LT(p8@^>reO<}<*f;G*SC z+V6jXqFhkN%EN99v+ye8e{q2$7C^bzI z|Kot@+vz7EJNfa+8K`COIU`OwDi`;Tv&h(dRru>>HU2EMggt*14ZJqr<>TsASgM20 zb~5^c5u&cd{Uu-zu=%|JcHF862e~CYo>(pF0sYVbNc{*26UUMfAjW!Uu2ztjB9YTL z$0$D*d`3kJF098QUeb3|{Hz8>TUllu-H^FPsHL+fn}iMLqk9l|8Fz@amsIUJ8|`QB zp*KOvC%+r%vg#?cJZ&8``u9mANxUg|D5?0ZR1lv@Qa|$Ch;I#td{P&vSJl-3Wn{wS z>tCSs$X|JQbDjUi%EZ)`I%^=~B6?_`Wd94ZB9^r-_u96Ka_cpF(ex-nzKLT~X@jb% zOOon#ne0H50W{Q(_mI9~kpBmgEZiF?}je}yo zPK;E^YBlBG8w!9AVRA?|FWIb6tDYG+S?R#TW%^J4@)_-@<=iut**F}%uALrMMt;3y zU~c{bJ+~_Fj3OvJCrBEcf=(!^Z&t~ie$qwdAT~qOl^>NOZlV+1Dm#=V4v0>Z6T3Ok z%v-#_p}G+zX)b4qIy@@dtYxz8Gg;>bb4dEmUs%GCZo&p{-puVQ1QOUuXQOxM38-Q1t6ADMky1akVK}B%uQlV zzg>m`%c>mIt17yvcrgghHP)FVh0XxC)^+$Z$_k}!z0CF&=H*N8dPv3LdU|;eCxhOaK`vN(s!dZ=AqmM~0`^0$#K*L(0r_A6N8`N5pcC;Vr4p>hJ{i`s~9s;Jix zV->Hx)(gtWy?#J1b}DKub*X&CC~tVUzBR1cAK~-|cnq<9w;A~SV249edB_k=Su8UL z0;FboL3wR-aEc00LkceDiJ7WyC5`k-%aSAreo_f&@+EhlEH=#w1aTluSBN=c(#u-9&xe7FCpks-sz@B@k{b0@=~ zr9CIzuY+NiNy}Mglu9j+nap>q_wdnf1T&k4agJ^88mca}r-bJ5E|4MRjZE#IXdKD0 zEd0OX&KTJ>*PSB!2C>NXswl$%`eZA~7%cbfNT>I@r=hgVQ_J}iNIF9yra;UV7_DUTeyL~Gmy1P!c#ksRdO#J> zr>#ytMWU4h{8Lq~X`-hE%Bl@U1Yy1oGfJaIQI%+Z80Z}#+V||eSQQk)*Q*T`ahxA0 zIdfr~=hA^^jY-UHN=%Zn@kYK}{DdxWPfrh2;o785m1l|VHk!kfYbZ=5c~wHP$5xd7 zF=_+V;Fulq;)ytnszS^eX(&rEbmjXVQ2jvFA-B{frk=MN6xn?x2VQHbE2cRh=+8yj z!vHFy%b8>&O;-QO@$WL%FlmWNOmhQ(pKNU8N?IqwCB2f^rJ>>y9;>7Kg9g^vYXuGE z1KI_aHQDz0_3T=Grxl{26m^=ORZ|=>&REbbw7g{Iz#ub}QFHY#u)#Kzfi@LM$dXGt zVUq#F0gFY^T@O`GF0Y|cfKr}aNfKrtyG~XPrNERy2^w|#dGs|*VUJ3K?XPDANpp{p zudCu&4%$tj3eMo;c*?gO`?WTJg zA<4oj<20nQ%kPoun2sV(BUP3a(f1Z|FeoUYeG9Lom{b;zpyQYO6?nF~3GDXhT0PV# zhfN!t8bke^42!fX0K*G$=K@WEU|9`@ z0zeiy_ygFsP+iOUB_am4(0)ylQKh(wA}+jYOJ|0R=?ed?t!FioW4-v{nC_}0k!?8F zr~M$XIhBnl=qQY;gTXPft}v5r3S){e8mY>8s$9z0bGwqI&sK2m_8KUlSQmFBYDH8D z`K<(7`?d=oF_M6pTO&}2O!J*k;xH;RsO!iV$TzMh9Y6|mZ9^&oQ4zwlm^OI6I~hbq^|@mip7E; zpDdg$@m*F~5)OFj-6MO4CE)tog0Q)vTEmdv$ zd}LY>Z0=Qc<{`hYF$m_Vi>{VmN%qbMLsv(#t(w6R>Z&wTzTvVN{ol%+=T^=r`8(Lk zo`sBLewwqRgto?-})p{Is=_xuXnx@K~Ztem%`B_iv;5e%JuOsNr6p)L>g#6*ZJEOOGOXiIzk1HUqVZ*D z<-_CUe0w8C4yenIseC0B^br+;8^8kA6-U!HV)a{j3 zOlfnt9{u0{)?-?8e8f3~M!VzXk0VByb|zetA_KvzHe;ZblMY^ecNX5{Pz~Fra~~qR zTN)PG76{08j)KFzjLy*Zrh25Gih$bsmEn=H$xM3p3#UiU+%j?yV7WGR)D_6>v7sVe za8ix)3kDDRiY1LyHr58!mwH7IAfELPSo+NoPNriGaQN>j$XhzB1xkNB$J_Bcf3?)=v-rYnCOgEyO{P8ExPEi@f0V)}% zdOE`;tYCD11-466f51CVw(e=%p;BrLzb;&0guozU>&quB4^8vVkiQxN(~jquVbWh< z^QR=vFWj4ejc$PCv%wtrJ3X?c+)9P=P&`7U0`&*ycz;z-c>c>&h2xZ6Y?cgy{7sIu>D97gMK5Be z(7lkNdBiYuFj8cUp}2}l%Bi1?ggyP(NfN5EfhOmeJ+fXgwV(DGA=3J&VU@K{80(x$ zihr%gmKVQGX1nP`P>G`PvUVkBQ@IB0(p?TPT#tpa7tTlj_*(f}4&f-0hQ6$AG&M2g zYGK0$=S&I}w~CjVRSZl`Q&FRnLimXLotXljpm!W24^f37M#=B5cQ?T0Kl10XR86LR#U7@5KolR9 zuYE;z@XKGzt~HfyQCJm*|Fb-UgedaAx*7s|mq$3;WlZ@Xx>f;#qo}Vo~Y+nkP6G-1=n;}CuwmIB#$k<>zH));*mJ| za>;cI;sCd#jnRn_&rtQ4+WgUBnQEjQ(_A_=Lo$pdjYNK*$>RT63c?K6W5TF&5?Nce z)&y{$a`|D#9DvmAA+(w{2+(}gGkqaDChntpq_0&CdJF0CUfLh2E*X) z3L{yUC0AKBg`|Sy_;@G&l-C^muq#mGmwH4U@HLK7vZ=8I^5fuqO&cW{MVoB#JBEIJ zg~}SLI!&Z(pJdpHTZ!v{NQ|wo70xRFWhfvzRV9Mn_`{veX!zg8odS#NA7lO92A~&A z(j5(?*E?9xzvcO^e=(+xJjtq;;x;C{z+OW3RDe5{-C}siy(lmg*YITungU+Z!%C`r zVUv8OF4Y4@IAibxIJ(NJcW#Yb3R7+LsQ5}U0euq3NF-!C5xwInT-s_x$lh>hd(B&ekS0_vx!KOVAK4#Y zQJjRCWBR$uAgw$dCE*G1UA=^lox`2k@_8KlA&77JkWEhVi|-wBtJR9Cc{XKm$+t5yWCm_ zhp3@vw2_>X204acOG2vBVQe)i?VD@jiWg;X-+e};oLF7R9Vt}p5FnY;(Qv_I5Lu{##4(rI%Z7t-(1ja6=+a0n;-DbR5VW=n)5^-MCh0d?(+4ZEsaE^eAKjC00 zR^j?!2mA;SnGTMtb(#9f^x(3sDv^hL>o}L-12)4XRF13>-6X_?rYO|(>WsPh}>kQ6(8qc1( z;mXlY{jb|flI|!ip`XLkSAEGxm(EJZ9svQpDFS6qRCfzIRjH6P6$sH$6{Y;a+u(wk zKqTzBxJoEO@$KY66l~>I|6Nhia}SuAtw$2E5(B&mG~_mI8no~V7ou!_&%)e2kr93b z_Q<)aa0>i8ZNPa)T^YbV4u7Wm$|>>}&~TDNM2|v7A5vu|_I5YB;Mml;=&F+Z&ghwq zibvu5yJx4Hl6KHTV7m&v5mMRj{ zh6Rn-!LNUoG?L}7jT|KPs(fVP|1utaGqSbsqtkG=7dYfl0#sC`nOl>-(QxpFl>p&{ z;}`<_RcX?8?f4gI?E#Qhe1N!C0bBYa2FjyssN@N4V&!;amf})vxg2_28|i(U9HONz zpAO@>M6&ou&R-n_{;hd;OUscz$`+Q&mXPoERpx+BQsL4mB#D24Z)5muGgkPEQ6P#3H)8&FzHW- zfTcKLMfv3vM&B7dT{TXOYc9(gKx_WKjCr*id%EG}jFU;KTt<>^>M0Xyf`tof8L^US zM(lsB(u#I{t?;`P6qNoi7}H&;2gOtlQn5VvX9+zgm`b+ko#jX%c7LhTYVn;69#-i_ zg?~WMR;4qzTq|G)B}#EZo`3o9KFFW8+bl!l22DoL$4y)Ats1S_>&ts}Z``aoSQAHj zI&B9GZsi{FXBR!Hhq}zEr%erxkQ;8dLy=$ywnt0l;wXM5{~g{%kI9aQEWmF&q(K{( zCep123pK)RC`FOvp!S9dPvI;we0b`Q--}`%T~PM~xyi72=eKP1+(2Hn=&n-8)9hd= z-7kSFgXwEnRWmOSZ;$fW9MN^(K(3d`PbklHAg}Q_5Wn! z6!Q&{=?giMiq|-`UlsD=!81$aPKHBLA8=|dLZKeovv!~`C4!-g>IC193~6~h*%=|X zW*TX#4nZIBFKvW!?QQjuh19u{%oMXnE;r`8vwB(p)NSu7<32-b+4N9~jaFh?q>aOJ zR>PhXHUc_w)+>=q8bNI2P7d+VFebm-i-jYJ!_d?$EfwK(A}2Z5kntT1Op%MuEb=C8 z4MmhAX=FHeu`Up3r%DDqSDeG1rnA%enNHP5HHIc42UtLWp6}pLbIA=zaSxH#XCN!5 z>O}Y1c#e}YQzf}>b&-nh36?73IJJk_MvCS%>Z;?+C?Vt8v%Sv|i=rvx>xOrfWDzRD z`cqUlr1jwpZ;awHGK{yaD{AP{rS~Z*-34?*d?Jj}YOKz_0wQBD3O z*hqWg>~7KqHgadedva5U(RbLe{r2 zRC056ZyXBo&7{Nx!-1rsuIAK4RpWueRjXm_IFSkeo#MyA9=irLv_5G*CL8x46gFD^ zCwQXz$kv7+(BV?>moVL&iN`vpp=^C&#L4wfpwdl(6101%VKOeo9HPIeFX}eJkA^-7 zn;I0Omm{ZH@RCE|N^RxT@_fNT^wn*hq}UhC|G!+`(RxNjnX4Nn?LoMm$7t_ViG)wC zU}F)5dR#Zuc1ly71PKt!+}YgXGM8hHFl$HLfM|akpneq7=!>Ta)e8GKh~a&ls>ei% zO}rNxR_$$~MIMk~qr4L(5nnOY`4_ib|C%`s%WiOBX(ISaAM$1r_W=ucBqlUuVzp^r z(FI(qg$8f$X@i1O zHpA^Dof9b%rIjqFKHbN8+RE|0jkDjDd|1TX(D*k)mrx5Gt%VyWgydM z)DPBtQ$x^DqPlXk7U{%(bQj3Dp1j{~Dr}|p8z+ft$#B78dPWmTc#Fei$X0b?>u?U^LlV36XlSBS^%GG z)rvVf6Hr@*>Jq){~@4sQ8Yik8Zl*t zhue;VmNM-ulM|s6Ap=GMtopiENzSiFJ3HN&`U^A2pPdi(fAA0Po*!YbjE5q2(62s`Wm z0=YmT?6Sj7)I!)1R01dlr4UfCfFp=V0jqUVRIu)VRtv7Gh*f@S9o6Ej19?B^^$&zL zx%Z4`e4p(oCEm{?Q)Ei6my!L@nESSQ9>z@nTmUKZVkg!#8 z&df)P`+H0JmnOOBIvH_ihqE^MpbwO}C9eXB;krSl=#X`roB94O!as6ws!0v)RxV=p zZvGbd(;fjad9ox}#^2shj@EP(i}d0Zsm^#olbOcVD)M7mn#Fi#+&v{S8p7`TJN$DY z&vZ(?ew@)YFd~glNhNs_l5LO<`xM!~uo-VZgT6aFpELUI5_{Ut+%=u~3PFblSu~EXCNg#l_q**!C}Oqlbyu#VfkEA+;I}{Fy@4u!Ja8#)H@?ks z>F$qZFnmEP0=ZGyvz_Zq``qr5K^YuwJDr#~u@?$yf*!Flz0iU9OtjNIMZHqikCWeC zSS0Je=K9(nhZ^eKZtBJ$-r>Bn$R$4C>th0omc3jDZhSfFVG_)(9L=(>jVZbOW(&3Mo<358PrvQzS4OlRQR0o zd!$sSiY(R`dgeXQ?k`h8uA9&Ev!j|2KGsJ0tIr}9ZoZL^xPGx>V(#E%&YiW}5Z#GM z(q|LP9jrrwlON#$wKs9Q4Rnp_gS%jZqpPrzuS{k2Zk5vv9a9~YDc|v^-HWVpd94Yg zwDl>7yyH87ikj@Gyy{I$<=HS!V=W-Tnoja{!tzQo$m<%Zq7h_q(qMMpU?lRzz8uW) zd-%oew0|P;#6>pE_3-_(1F*siRZ+SO^Mk`Sp0>wf2|6TxL~kvlM{6cHtTf&7?{!>} zP6el07{*-rm`RB63qaZ;Rl!K%IE!5U0tYru`MVRH2{P$KIcCaFWsvFqW}06%L-b3s z0uWer7s)?s8M1IcJlL6L=`v;zOvBtGB_UlW)r-DhGORU>@Q6ybmjA8hNdLW+1M2xW zh?2l!3t4^&ZkVsLX`a{!X?#S&w((LGa;6WoPQtto$RDIowK4ScR%wp3`uS`O!5`+Z}Ei##!Bh_g@wcWP#oBEfS4#H-dd~Cd#qh zu;i;lvOK!K|66pjk04>w7!!JNGlS^F$U+7rZbRk4_ z(govvvl7qKgtT&5SITJ@B8VuUYni-x zKSIr|`v7J8%s4z=IT4KKpN}jckTKvy4*v3AUzdF+OHS9sN!SKnHI# z<>y!7$zP7LjIBE2G3rAu@lF*QEYB}zgHPQJ7^7p+awsy<3aDAu8i7?{labr`?YQNL zfpcs4NEO|1P(1L!&qeS+OZvJ=;S*t>>{`r=9?|DQpCDQw9JM_1+!&<%v&!&rJ$!C| zvFVUHWKtVj3aPPrRcq5LkRvU(hsj+nkt#>_u@{fBjghn`kU9MvkkQAwI(Wx&yH6s| zK^obrqLr+4z+qc-8_<(ufi~+^Fr8!O{M%{DUYE^652{$EWPi!cGLN(oDHY|-8LCx` zs-T~G?>9K~t$OG%_38OcIAoo*tDMsKlIa!?iSfqmSxMvT(h72XDovfHtfa|LW{uqY zJV$q_9y~upd6pAa@GXV}5;w`TkC8X$qik^EFfO2*a!=3(fm03?fPCqQHbVy^y%^3F z&9JIcZ2+DHHY1e=z+1|;{!t!+v3eE?qb@qh_LE4SXEgs_Q~{ks?pc8}40aIr8&8nCBq~V7kh0z&$5>Bb{^TL%I&lb8#ByyXIeL=hiA6 zMT|G#@FUF)=lLK#$HCpz>5`%AyWW}0n$Pw}>a5j)i571|A5BS2w2jHFgY&0j_pnC~i2rwQOcG8$~e)ZBLI@uV4)uX7;62j4Yq$TtY- zE0yxnh|owkHW%7p$QqaYcmzSU&uiw(-0x~1wHXio1rdHih8VY9W6Fo)Igan2uox>r z2?;SSqZPJK$xQi7H!gUp4|LbEF^Ba+VFCf{Z;Q{MrmGRpt-xE2Tk3(w5VQ*!ETU6* zI?ME%q3g{mv`nsCk9HtR(#NQuT`r!>rbph;QOPH=LM)IT<_mQnzc-S891ow+)61F$=;d)IxU%83FMo0Y~VRRvJAkqIr}Zj2(QkfXjPa_vX6h$oV5Rky80 z_Rd+zx>;e$xU)1$D3Xrc-MjMCHY&c7j)U~YGjY`#yIL2lGg4MU7_n1sJepm=kNqOY_&U{0Ce7p{GnDrxqkExS z#yw*Or1ygWrsA#G(VrDTAh*X^RlS*J0e~bq9LpfTi@ZF5Q_C;(z*Fu%AXv9YQ@FN_ zJQ-LUIc))UwvilBs;zM&2Om(yedW+R%OidOI&Rw7B$;{4UceRRH%2})uy2fZQZE07 zQ#4lsW-nbZop6J#t?axJo*bfT2135v+*Rdh2!)nRmw(zJ>OXg}*~edje_naSjG<6x z(o?gF#!5R8=o3^?4!rLOks;uiE%DQ^#NYQqx3jgqo~^4dE?qM9^w;V!(!G|M11xt> zl9!5sBY&y<`ZBH)qWy_Nh;Q~ z&pwM!k_m)AaC%k_mh>G6bdQc%RPGGBP|M_)3@t68XQpqh{CtjYc2OmEZXgmMP+i?t zUb2PDrR;%YQJlqAK(1wS$KR;x0GK{G7WU1_rZT~kg>4&{f;)z(%Ev$gJY(1r9CntLI z3u!sZ)1=SQ;bb^KMrYp2Om2El@Vu8N6KPv}vHpSh@jEmnusRr2a-&i;YM;WwRS_EJk z@C2}M5D`V38{SEp*c(cP2a~hma(E8AyD5`T%P~IaJ4zSPX3~FYxE$KAY|eu{OXa%@ zK$AxKaA3?2%MDSnPJ}@~XGc^@i#iVtM4AF-)>X0iS$aI_pJ@?kS(h)2KZr<P(g;Ses;>IABLmSsDa&q73MbaIM%eJTPpf`qbi@kgBiokbc11X|BrUvJ=xh- z4nEJ_ET?c+k5!o^-+u{1v>^)^?C0w#6_4|_*B@gQr_LcTywl7I5Q~UU1UDz@N%ka@ zcsD&ExW(ex#*w^AneqAdh;$eOB2_9n*l4L|h$lA4*V|yhGrM{0pOj^Seq&)W zSbjK-oVcVba!vwEdTU@eAxB187;632H&0zFn{C*-^5KsAT2 za3;75J;ueta@3u#G0v^f3@LxejN(!wGb>YhbI8m`Y4*DlWfHGzx}YR)Guc^(Z9J)r zPTWvM1Bfj$L|WD3oEQ8F0J>AWjlG}NvBzWAS=x{9HszMT z_{M*_Gv5`4#>oqt_`_$nbQVx?ADF#*wDe1WaAvtU3-V89Y5(U&4hfB z=ydL;=XhAZ&yP?OV~$-KrMHp6CwagT>}nN<2hPCkmcIko(smbN|0o%ejmch)X~0aq)mr@k5B6cIA-bfdm)so3WC>gZPSVt3qCBU;ZvA~+ zF`4HL0L8{onN$ERPt;w#E~=XT(t5rnrGr9S#&HuH-fCrI1q6=ZwhqadjiAnCKp zbQ;%WvAu^_XmA0McJc>g!*_4lozSaqjlBFGQrDvj?1=oLA)KEh9ddII6z67(bTd^n zNRJe88xOZmrt3Vf?tm(Jy8&v)sT&@4utNzARb^}A_bR$mtV0F=3=v<}W4tCEL#qG% z>2|N&{Ek&&^eW8PxYaF(o-xa?>zQ3J;%lzw^a37{$ko+@(FVpsphUsXrdAs{opC)B zEhSnK=_I{RnIxW7LfVPmrWYzI@Ji(|=)28jwnCESLb6eUL!`^6{H$hFFL@2&Df zwBGSe<0*m8h;nzf3~T{f7>WCi&s8|FtIF&WxzkEYIEms>3{Z#RDn`hF)kuJ$i+LWS z*?UXb25y&mc(=-D`eeST2Ejhil6$$XujSOqZ%w!e%J#m_3+UjwmjG>fV;v2wkNQm2 zGrlc#=yaeN@ItJU+&t0r+t#&f*m1M`GYLxQsuN)v>_B{jfo_#lhMC|2#uQodNH_w? zd2axTrS6nEKUs0&XGXfrxysiz-EUTCSIvB_-0Dg9L3T_JB2(fhW!=nNsV}c+$hcPp zb=^k!I(fX%%$LR+Sp4AQY?qw7!3e>~HrY#;HE4mJ%+2?iak=m!m^KyMS#nYx10m^@8p4ZP;HF z{Y)$^lIM<~=<7U>3K@M6lznJBJ8e;zl9AsbxaLvc0SVG7XESyZ6QR?w7jjz2`u&#f z8bGGZ$IH<>r}n}|-sf8B|`U>ARJ;_25|6Clk>UU#$$Z}QTH-E z1M&zCm1Rr77SrE^jsM%kIpwI5CDN-En?0PJJ(0mA=o0Ns0NB+sLX7>Zm(~d))Z183ESDP5D>XnceO{qJ60; zb?0{61JfViP>Tv6AhIl1!u#cP3%VZ0EE&VG{jMrD0p60mAo!X&a{ewj@m@XW@aZ4G z?YU&qkTDmySvK0ahKqE;Mh#GhQ>qDf!CD^b&kAWQEmOhj>yW z9Q);SIry?0a3c67lRpaQklxcvp6U$YS18{sWK`K$kePqDto?}qz;s{z81JlsD+@8xX^n*% zSq8ZpZj^Mzokr)}0zi7nXxs~x3yohU}p-s1DVA`^&vg>K&gj2Vz#lblre-DMN`NSJR^34C+P#C}{ z&r!I*%BIQDv6jtvzR(Z*`})gd#i!cn+{iDI0E7ZaQ_?q>L6E0+hjB5jOxoXPRuXec z)F-%`$`GU2q~_`Ovkt(7w;^Qg^2^|<$jdLXJfaQ)Au~A&{*^Ds5{UWbv9QjFr&-dV zi55zvJ2!Xfa5#YE2sEZJA}X%Au-74yLBQ--RBd#n%?#q@a{r%5xn+Mr&J4kEa>Uhh zAbZ9vjaTu>$hLeJ04v&uGqsJD*LEYqzsmFBEvoE6s-CL_7$>2FLVh~JQf`HRH8x#Z z{Da!B&;<8Y-7qn7Rd6?whcU1p&D!pib=OQkELYK1E{(*=^wMwaE1eVbD+1kxdjOM1ov=?%MiC=tl?_gD{klndnAt2}hB9>waK47rR` zMU&;x>yUFF^+%jBgtVUAle9-AV<|_oydHvWi7L7m6RoJJwia8{?gurSJp;%JC2f}Ufx$eNV69aat*YVD=Tk<+;b9+``jn$-axw)nxHi%b9G1_yG+l9LgDD!4z140sG!?l_I?gb-5K5(woXL)Oaj6Dwq30 z>Z@}@GHo%7`2Z_MTDt6IP;_|{DI32%1y2{uHyY=r^m*v{LpBS5k{k>_QN7xGn5o@Y zJ|FxTYbr;7f=bA2ZU!87O+&i7r$EFtV>vBHJGtw1F1eF3n#i5HuJ?31G~z&I>q=q_ z!J~1y>%gzfFqEq#4wK448N*R+f*&IZjWkn$MxDzktuzP8#` zYw94MTiWg2?n7c0&5-{NBeGJeV2*s*5SIIwX4klez2L6@!d_W$A4K?JmS4Lt=gQHm zR)hyvw&0lkw*ub1on^i5{P%%wb8$wGR^6m*qfp|Q8OO77WXf{W0nt|XKoN($5M50- z{M-J{#(kwO4y7gszfXXcrVU^T0&LwmK~hr=z>l&vgtqFgrNeJ|F^H6exffOBK6%YB zOSqxpc_5;i26Co0ve0!SG(SC!R4G+X-B(*)YO9L#u-FRigLIR_P@W;F=4MDPxY_RC zSy60P*AQ&L>ef>Hm*oKt6Vl&~^p_{^O$(`1uAHS&KGf{Ul2y7I?-i9#X4yyBB+_sU-c4x|fFM6&fN)8xn2+x7k6EMtydOSrvwIHp{xl{Ac)y8Nr6 z1Fly_Gu2nVrP|j&7f)7dixeNNiyOO#PiR}7%P*9^0pazqRPCqN|bTIY`QR6 z<)Etkg@%>omLZq6dv*O_52-j zVMnJ))HE4rlASQ>mnKm;kqmZIGZK+mv0vs6{7z@?Mt38&v5)A|xfx2W*6wGVh-XxJ z2+WW3wy!Qmp-5B7!kuhxny&0Vr`uloDiF&zziv9^!M#>W4kb;Wba)BvbX>cGwtmB_ zZzk?V=^|t1M`~QqqOb-DQDK*x3hjPWGgB)5*u(v9U5Ib)<6E)P-+mtPNVa3H0x9^z@W&i!rHM(h?P2LP%k&u4sBz74^_5N;?o35gUIC zdUO-05})JsIi~}z!r=FKaA6v;rsA5 zr9hk!8bK=N&K4 ztcB_xR*}U3&eS#_AsMXSN6irD7dW8Jk-^gQBQuGOQ=|jpFj>f1Ve=|{^hm$g-9_tzKa>ti#Mw<*|UmJ}vo0a;FnN`}+7+Ztesp2-N z8o*^S90E#;!>UhIv7m$loh^>^p zDT}xQJ|R?Xz$Yy&?rVffIgnJr8xW%2KC*m+<&x{(h1;l~oQ~4TOru_2 zQXc)1iy{GJl+2+xpV*5BA)^Y0a2_?wh+o+jiu)$;29+fXpXOqOnrW^?zI>R);A z&RiX=Pjr-?A!ZR?HHaAZq^vh@&cIc6)kluwGcselAM^QZAS?BlySK#0u*)JtYfvqA`Akr|Ss-i18*wsLK{|%$I8w0s?sj9|ctc)9D z^9+Os0{Fr#?ukB!O^}hHE%G=Ge(_NhtuSuN%_k=!vQQT2>2OLV2TJ9ANQP;9p%~hm ztd*nvAjR=2XBGTu=mO+|o2A`@Ec(9C*$A2^+C+98;KNN-+c6wmAXlbMH}ly1e&Dy)z{uQNTyQ>784 zyZ;FH?&&HG8?xyVF;lYasu_@@m%?;)ImuR0W4X9waAR-9Vk2$I+e04upA~1pw*T*j z%2SCY^hC9^%Q8L|4+OOlEpUa%j5_>bTM4!FgiX3b zure_9U70-bBP!B$0a)LCEaHehhg+oLj7ixx{HBIY7PdwknYTU2ugR)%l3sZa3A z!kg{Yt!qZG(eh0IQAr*^hgddEj*9T;mUly%7HGU1M1`SSir=@2p*YH>(myj@;ya4c zXuBPhsZ>0VBd6fuQ$0K_#}9D3FE_4TuLI>wTQj9yNgH5G>LMOd?l%9QW-3kJ<*UqG z%EPrxUc_?c9Hm38-GCvnN#H5`1WzbWedJ|jH;7%t&>Baa&^>6P1m#BUD$)RUcga$3B1NgIBc-| z(K5Ikf@h>!S7c{SL_7^l2QoUbM@~G&!XqmYS#ut=DzM?mHd?OoGm*P)2P-Ig2q9lK z6D8}Zhg^saMpe1&h}=XQ=~jjiIIBWyfWg%R02Ao2zpeR#0p#5xfBbEQF#^D2$Tc>(!%kEx^kMGj)SAQNJy zH%QeV_6P)=oF--WgRA?rw{oB!;%K+2Vi)4MsHrrri*5131`Y==RZ!-1gEjyvx5%$l zVj`h@4*7L1_-1ytjh+DUVdyi$MNP4)CH5sdHC0V_jJb+^DB6HiJFM60Rwo4Id)iaD zfx!8u}cHLua<;;wP8um!Pfa6z*pBbJ!vb%IA^~` z4?%iFNQ;h$r^~A#pSMf%jlNZ7#y*rbi4RSrNq^+W$w#$BQ=x9sSy?)k7ko&zInfN4 znM42_E|`Ks$k}Dg%+mReNgF2-$Cx;cVfx7!*k5_Ri^@~~Lj#;qaXo&#Vx@Gx1188( zft#?}QyZo6YRiY0-Q8PyWw8q;y3G(rTZn9gcGl2iq`7=_$jm{H1?J!kw<_i5GvR6( z+5}1&U2R&3Lf=R^^&1D``~{~#4P9gJ(~X3wWpMYdJp|e*a`4ur=|nZbeHCx2K5jssklWC8S$ z`-Z@%Af1prJP1O>@E;;;-sAY{&c+44`Z9>7Wm8;+eV#%={8?HfwdlZuL%}sx8lU4v zp4MLCPv{ISM>22;+jEZ>;YtuFD&(SwQ-B85cHM5!)RWp{rvHs#@AH|xj;!K zK3qpoF1%xwBV|%Ob*yZOOgMsEFH^CHwJ;d2PkyFE+*VWy9wLyE6NZ3}SS{%zC>&dT zIZv<(fR#8V509{iFh^b4V$?v77$qE!%4sZL+->QwTC%64ePp_2x($?+dDtpq2ey2! zgtU|eDzS=(l2Vh!SOF|^BhT?cPm<6{b9F-SB zY~$a1tO#C)ipDadHP-5r%H372z7K`XqR|YZyr7>fzU6Ej92?2h@$zXhjx1g60qu$B z-n$9a>TiufS+p6u-{`@W;lz237I+|a&kkIj1U%YC{~tbtbQU@VW2GvQ#JS-G!F!s^ z^0sEB6!bESB%_B}E&sg;)wpt#sIct)+^c5N5tjex}2x#x>YxZ6Jg*s-l;i zCv$@5c^8nElaNe3P|nl|%NrK+bBC%v1h7st!4no-gRF6{2BTGukc&UTR2@}CH)CWj zX|QQ-d2WZP+gv(Zpm0KaIGmCuGW;4B*n|!XydHPb3x$4d#`cvTa?uj^^g_1&rEC87 zXD7sZ)GER6Vd$2sIVy06GGM`GL7%@oY-o?`bL}9(GDuO-S*h1+8uXeahE(!B48veRlU#s%!4xx9*W9QAFH2o0Lb7!&tY~{$1 z{(H@Q5Mgv2M@Pe{R@!petZ)569VNT_O~m1rZ!E8@$bvZbsFE!jUerf~+h_y)Z#ZFW z$t{uO;|R=`Rpn^(m8&O~UZg8J!yF=<*z_fd}TV9Y6Y9yuYyceIeobvRO+mq0ZXPLNOv6sVVp#xIA z3>FLPVcvhLU`Xc8Gri~0(P+&k+-v_)Yp^h-k3-3nL#JYm} z!yL&$`>CRay=hm{fP^(vV)8{)qHmb&d;`049j+(ITgdTvbkC}xhq%HAnI$ZjehHUp z|4xB%J}(Km`t}(&6$T`5lWC*%4Z>^RMA)fLRk3^+g8(}GZMx(&tUlv(S`w{Qz)aO{ zaNb+W>p+m;ZFx8i>RzYH?>8EkmSYG;7Rcax;jpH(*^}DegaaI(XD~KZPCHhb*oHI- zwII8J^w2@4c~T!$ejM%Zn+(EcR&n>{Rt|3coQ)ta=dE&-dPw$GX2TaqB9SxOjY1&* zq~2A8-~pA>oG0Ow8LR?s@)gJiv_^o7uBgB`xwr2266t>n8nSz>u!j@%PKNd!@!?#ne%^V2S=qqv;4)9pDr_pP@KU%=Xr>pm$jsT!wj>;Q?>(@O6#ReT#d zzVa~Q-W}ysNLtNO$ssJoX-Ux%)Z8JJdJ!SPwCa1aBitO#sIFg)0)I>gd;Lw}u|~VV z4P+0xW$0|HOhb9By zypyd9sGWO|Ca*w7tUp^xG=y@*6-OZP)2A&rt~P?zR4h;7Y32QN+>xsO z)}bY;e70Qu7}2pxRc@756D&rHEN16Jb?Jo9YdVkwi4mNE?|xZcXjRL=k){haarT%0 z=|I(ks?rI%ExJR)f@ZK-zRrj*&){9vdl9lVedQ~%cOmZ75!v<*kg4M!lbKjoP~^l1 zp8&JD`XI|nc3{aCwUVE2L@QdV>?S0rs~%30qAs$iAp*6Wk}j^YI79CE0UYzz3KZH; zv$1$D>txt|T@Xu!Dlk!IZZJJkHOgMhFUm*D@EEq7OC!3;13A=(vgz{v*fGCIHOO(Y zZn{Bljb3uYDtisMzoaFYE81T6Uu0`h<@L*>_uAdM<#h}^M7x6z^?0k!?*l~+hB(?r z9bnHx{~#Z3P?1}>i^N2^aylGCF1zT3{W^-YAdSJQA0X~N)76`g96|aNs^C-|H612D zj5cGE>cm5=Pgu|WDrYa|PtgWEJOKu|=hkquHAj)Rz5_tIWFt(iP(4|Y$?iyJ9x69_ z(68;Y7&xR8zU!&#t}^{acz)o=Nd8kH#Kv7JxYRgQMUChwaA%&wUM&yp=jT)P8b(z` zWa@zJT4$8hlN;wjCy(iL+N0Xxny!LNjILEV#{R+pUj10E(KB0@#nVyo2u{@g;$401u z&1psQ%N)yz79<|}qmJBdR>5J=Gwu5KWcj%n|16_>UPy*-1Ya^dWhxBrXhMKDxg3n> zZz1#7aGocX{{R<=aKSj748kh!ChO8Uk#F@h@@JJRf|R$-lGpu^#wZxj>%ll03!`RHYxi&seBDhMK%>ehnl#6c;f>H8SY=g`!!g+G@r?4YI>*mlx(R|)oz6MDaTMR_=zf3eR)(Fmt zwUiWXUuBGLUC>b{VSa#iGj^nK(Qt5fzVhNt8`1!T6r#@6NOM? z*ISj`No2UJK5yxuKJlIDsyuN_{@sLf^pm?!Aq3tq+X(DSq2A7stLyU@Y;Ll8aHW7seb6PiV38`tZo+ZvTW$k>}3 zWx=7>A>pEScuf*yt}5Sbe1FDxxzTU7K$xWZ;wodHd$aE>j}BoR({&6xSfa;ne3TcI zBi}-&G>Pjjx9b#CnwOQy1y=?=1G>97&I;wvkk_6I`(#!TJoHvtHE(pfQD9VZexmd2 z?}Ar~l{a5|93w#Iq7Kqi_l>elHSVr9o*Yj(`l|s zOfib9ioluJR{trNCI2WXW^+bH-14!9}sXXLxP{8l!u=(^KCa&*Q;%pBfr3a{dFBOGmP5Gb6;=Imx9Lj z0=|NiE1p*)b7^OyKIKz zG)kh`mq>Qf$*i4rk1jbH%n66?!lMrwq{V6*cm@A)Q>tBL*9^;Rbjb>- zmzN5*jbotv_7)Mb@3N85tI6|3^o9=bP47;rwbydX+NTkUAL@GDmryjQ#uRiaAD(YF zw2@PvzqvEyyC=A~<0>)%W3eV6xrKHeqm4|Ls$2li&)Wa6S|#5Am=lEBqkMgU-mWf|m^5Qzs4cwSX@8JIOCvubiVVL~jiUwy{&rzM; zaW35q*`8W2qiR7Jq2|)ShgzysHG^b-SIejFnk_mEzYLLD#PJr~g8FqQHGWv2Bg9=L z@)Z{{PS0;>VzK3wD&AWch{->(<$sh*h!qAbuc&41ab=Alt@}{2u+Vm!?LXPQP0XsfG zHV-W=K>oYOV^HKYlS8#ROowtW=3BUq_Um%_-*Zc())vs>Eft_R^*tOzHfu}qJi*u9 z^s$gxb8&Q4Mq#Km%VR8=M9y&13rc5Imw@Ar&K!+kIc#pj$uO-4L4QiP+wB1 z$034b48kCI9P4hc#G~k00|>XRhJI>hu#H3?E+o217R?1y(I*uh}%qpI~XDVm8ZUlISAq+#fG*1p}9-}C*($wfa*$f*x)gF13--!5&t(Ck# z({dS~Wz)I+0Gpx;w2e_?HfPdde`IZ&){!noj=hB*VCRwBZn1Rgc~Lj1RhSktbo+!W zI*jnVHWddAgp(-Q>|vB;#dH>FKVwdn6H$!rJxf&(7~OST8bmmMoATGgvqh-<1ABo4 z2G|qou4a_`J!h6_FN!I3I-5S^Z)})6zsX9%)HU^h)G?d3Yr24@lf9(;L-zQzs(g-z zrwG4B6@qK>ERMX6Qq6utD;c(9>k%7G)fL<5q@27=vIyypay~j zZ>Y*F9V~Ik>X$K@#{38Bt@?v^i@je?#85 z#7+FrA5Z6&2N2zbU)ue$Y)_bKXx*QP;FB9_CHpX~4t`xT2$6i54o36ouCI^j*7^(5 zJhDC0Vs^nhT>Jg`D5bED!a$b2rI8kLt(A8u_g75>C?{OO*(P^^_L$#v-6ks{@9aS@ z|E1MpALaHLle0Y1>TR=vHSj~-Kjwwfa(vbCckwC(_rT^QrwlW%SibzlJ^->_V|i4NMN0uA9bWHJvjP`&O zM`KyPz{-(zJ-HI!Dl-DHcxQq$Q4)t$`AI2#)yfBHl?`J0l%>Zl6tB|im`2U2{B4v?1e;|VkIFR4|F}-q9SF7>1cJgg|vjh_c<|2G(4CHZ{1_ZEla0l8LRQ{ZHRn7R`o-cETa?&5XMxT$M z?n@B86T_XnldBX6HD6VPDMR4M!y0iD&lD4@y*VmbzO!A{f%J}tT(e8F%DY6WJ> z()QaUfKOjfS$-5NM&YjpCWGT}$EfZ&Q$|Vg40KIR!!7QLUH=9~q4{ExB*3~pLw(X#h$pjzcxBzW;CIr12kHduKdlP(wWW4v|9q_zDk{EQv} z2QeW6d9u4Xu<3)mYRrLDp?qN3y+8n4hLk>G7HbvKSO*GwN-JeLWY{o}!{79VG=8dz z;&z;>}s}ShxmcUDO7|nDWS_jSxaw z4@6`O8=QJyw~jM}Bxkb6x2`p-xvAhtIs7fF3@Y2haDE~wnY#A;6+C@V14hY3Rnk#I z?a>IiM_p`kc3%ANAj^Hzu}%5QGmUFy^(a8ns}zv&Uj!{CCu z((St4N&Z&p`xScpqAI^c(LhPM46S21dB3X-udZ$-W1QQWc&tkHbMmP^;9OUIgb4~6 zv9e4S4MI{O+UcA%V9l*G~M%siSNul!+rdwh~{PZ9l7Y3}SHx?Fp7y zUO7S32#HG5(&GA@RWn_6|5bJZmtJvaar|j%fl6-1oKLxzl+L%$NkNQ^f{Vk5{8kZ&|u!^zY?%uUL~2kC$~L?}^Jy7y8lVlir!Al{U8?I|hwKM@z(G zcT2%PAo=^VT}Yv1N1fYHA4gX4Ecs_Wzx+~{tzF&CWs$e{0g@?oMkih_Y%UQgxB zM!Ol!7$d>$O{QQp?aYGihFe}+7jf0!9 zkdh&Q;AB_K`{Kqdr1};_S!}kv`#!pMs%|6k5@Hi zbqU+Npi}NXR*?tgm1|Z&dj7}@8>Yh#uD*7UH2DLwWaGnj7rMHltNimi55A+L>BGBT zl#h5TYo;?lx687i6_uQhVOrX{kc!@xkD z7rRNnKUpA|g$;~eRdKDdO@cd>acNHFJW!D;k6%i$&$H?x=ikptNS|g_k@|u-Bq_R}F8DQuOa4?=8k#I-nOPwB59EA%uMXR^ zgY01~2^vVfxrm>WDmfYi>T~N*wM!Bw;orO)O{bBaoy7C!>;cjWeS!-mu6upv1m)Z{ zRAf?>-75ZFRt~%pnIuEPsGoDHri1ZqVcZy47$HDhq)Sah+fcKW&Ct=tSd@2lIswE0 zLMPN297Cx1a}@-!orN-b(|Sd|f5v2_l8UxMdxQv~{m)(`%RaE2utzY=l@T>~nie!* zn(`d(&2?C3s`}91_%iRVjSzcFxoI5M9%=GW9qfefJTBEn=G8KIsnN_32)WG}!dXr;v(Bl5())A+%8=Bms4O+j-x!HIj&xM`q z8^psr5qWkC+?Z0nBovCP5c%lUqM!p>p}-06C+1;E_-(MWyA3XD*9jxv0@9B!ESH1X zrc=y;rbphr!z|EF`3yPd#WwkV#j!XJU2kXDfm#w`);h)C z-CR12Mk8NEe*-WHVa%e_t-O6I{Qn7({UjI=q2+-adgkHMKtnRsp|Z1X==_6!am zTHm#1h=$zvtX08{;XBw%8wmhLQCDwt2o65d6d(j*%L);}90Ejwf>U*bzd@m+ynPOjU6Ps7MLJ83rDJ zj&p7W>NvSwz$)}Rl^B^no7II{N#_Ab6}mj1lO4|^q#Ap%WH+kd2>E_Lik%PJ@av~%IZkF26^R}l`l8; zfvw-s2E~V^`LgXRv{VBXx3q%UTE1(K<1Woo9S3yaE~+ToZ1yiMhCf{EGqvP)J%@G-A!>Ha@SVFqEkRw@Y|H7jIEz+R0#?{^s;3Y}vA0AY4P*_xs| zYZ^jOkvWK)vgct{Mxjq{Y?d4E^08~L+iUckX{VfPgS=ln)ha=FMK2L&Ai?DRUpSu@ zDjp&q(>EANiA%4JO3h-O6~kn9-;6{fy?CcA&9)a2TVZy^ZMsWL=^_k+0gOMD-nW_* zU{txJ?=*{15ToJDy@R*wJhA+@5Os;=+C4W2^7l971l&jDOdVv`MU^}$9e(7ieRVt! z^E{i~24@(3N-JThU_Uu~i>W)$d%N?`inrjLYM)%a5mDRiPcuM{Vz8ZzdYD~Z{~n8I z^96KuqvpR;s(PDrz7x8-Tepj$qvA~In8)X~9fXJvs*-uKU>{E=mYjx+^{tnAC&NDJ zwH3wn{$T({XXR=RClm(=R}}Y>!Cye#RJ^u^iYc(mt!JWjp275Z_iLOcD~4DGi;RbX zILSw(M_0IQ^~c;RiEKlW@7X%%Z(Z+vhm)69Uw|)Bg6$0m%eL-u|E7Q zNJ$R?ju#-`ShdyFR`?>DN-z%dkc5HFt!c?Vla(P<#(``IXByNNQ5Ii`zqVfl4bP96X!I*3m8X0bdxnX~#;*@zQvHEW11 zl;z9$`e%!E@M(m6{3qu;mXdfaLBqx$*#$^9`ayKs!7Ex_LXTj{-vM=mni}3xhi&z= zdZP}p9@`AL)(o#8c|Cg0Rb(I%oED^4b;dKow^D{(gtDGiv76+M%Ut~?on!I*0yD%Z zWrWPFh!s% ztHIQ~UKP~i_ESgY@riKm=PG&wl9rUI-D*;2n7gbO*$tF-?mcU#dw;G2WmYpCGR~(l z&jK8*RU6nuqh=tl7OK+E;0ddm??*cWSqqku=juh(1olC9)m^ppuf`0i+{q`BXQ&(@ zzE7vnhT>cTyzZVtUzGI$B0TjlvgY}oW{~w4&z5J?vBr+nK`^dR(Mjlw>Uq-rJ1c+~ zF7U`-+c-dao@mEQV7Xj*2g)S{vL`;NqD~q%+zD9~_>8HgJ_+1yM#OQ5FMO+P-Q~i( zuvdNI7UQL?B&embha{(39%Ep(3zBzhQ55TFc+yfbWk*^;X##0Ty5`TSY`m0CvD2)f zBX=;DwZb_5o-l9O6^ZMTk%%fwlb5sk?Sxvu&x6`$R5&SI1{RCAWx64y>*tQjYri8XdA*q@3$z{5ydBUdla!MqR5 zbk4v#9~T^LlQVUP@m^sl=4lmZZA{Dd4+RaF(j<+yi`c$2D zNZCNk0rR0~Usi!(TvkzYo$N9i(Clo@%YEJCdwM_UXvchLG(HjsmqYgz!3~yBIY&VG z}1)z44*Fi2pz$hP~L))<{Df9 z3^d;Jgx!)00|DLNsI#4nBUvHtFVzZR=wF8s&D0kXlFpMESl&y{?KT5oS>G1TeRtrX z^iGnRkHPi#-5#cN>#SAG6_hssj3=i{qAwq&(M&rT-NWvc;ccKnth9!hT|`yrve1FW zSKo@QNb|MLs%DGvSL<>>Y8zgi)MiTAM<9)y){tenhSro4F`BHa^dn<*@UOo5hW@YW z7W@DIzrp5xX-2JDHnX3-R$ZHMt+0082F6=|*TU#(^;~*wo9%e>I<;*wavQSdy$)RA zEhkWhpI+vl8MpDzpPSj;wxt`Sw%7J*&)WL)?go6m_ZWWnCM7enWIk}GdLf^>S=F!Y zmc;?P%~s2KeQk&A(*4a+57ll+E<}oHOqZ1{M9pZYn zy_u9lKk|%wC>@Z?pR<-uR3b<$!~)(U0YGXec#*bwdNx}1 zWBlyoejdFs$cL}H6Sl28toD3RNNs+Yt>5zIHrDXVDzM|zkF%uIf1E87YpSJ#KI)}k z@|fR#V}(CF#qFgx=4bbu=AUmrV-?7!596l4z7oy5;SLOV85rUho!9V-_eOF6&!1%(xo#9w@zuxYA%_aswEul=)YKB+{2US&f%csCw?~IE%tgs&kwtPJ6Et}Gy=0v zc1)bQ0z7)bbVyN0h^j+E;CA6?IfYjx)7Z(Zhqp3UWR z8-M43TK2RInX#Imy%OddCtld_E&l6SHH2mC#HZYtJCljFilQ|FJX1pNSu`i z9%2`LY#jCRKg=B2L!jL_v^#3_L2X`mi66IJ#pkeIkx%HN#dGiDFSmB4d;_l}c_{C1YSt2iPG+lC9cguck zdRof(m50REvf=wyK*gKa^RXdcvAF63>>;m_-D$kYS$+Q?BsEp{U>*Jif1Fsra?a{L z&s7t7e?R6c>k(7<0sVZn(PgtzigxnP6a851>w4zhxed_r{(Je{@&I%@*3Oy+Ok&04 zjyPo(a~Pz~IWGGCDpq{_F}~6JRWl&BkvJ+{Qk?F;2FLl{1b+KpJO25}c&@yWF1t(E zu(V&zbx8Oje*Wb9Jn7=o?Bdz8yll<;{OrYY{+7fllUi;b9Ph~&-mIb#ee-aMJof}E z>3E&#lsh$Uou5k{Y7}yR^mTqdcd%I_FP-O+cj*BP)16$!DP6HO<>j;-V>1UfSetX( z=ktm6|KlV->Bs_a&0t?YpG4c&&#jO?CcmuTgQuOC#UVBg^0D<4mdbmx*wLvwdC2d5 zSzvyDzFP4+zsUF*;GJ65t5y%y^$0PG81WhkwQ5 zzP}yL`f?+W>oJDgNxz5p{1M^1e`!wmd?No;Kl8_M-NtzC2ELZ9g;5cqCs{t6$KQ-O zBK1t}bI2(ka7K3^u3f+@+y2SQ(wKW-v`j0J6Pc!0?(}nzjR*0g^3s&N@DdNIc))Z> zud@GR>)ZotuK&mXKHGDgvx{xmcJ?;Rb$hch_qiYUb7+iQj{96Gj?%?}bebv(u6W-8X%dV#&|e!`j$+y(T{G;*cO z({s6@=WW>k$ZfvRvhZ~krzER5o-_L9XA=uAa`DxktmKFDpnt6@L^yfR7^-c8tfUib zcr*w29@+uCPWAvwi>g56Co}o1_Fzzzs_^ipN>qC=$HlOUUs%B8BLLG#rvS3Y=Wo%^i0K@!nr2 za>*q>TN^n&iN6-PV8^SB{FC5PcJS794$^!!OHd6J#UXKU2Bg>O#qko&u^?^`6Em?rLtJ;(QxTVRr_(E1+TH5g5caZ7_g(UOFpbv~mQl|y4pp>na zG+YSadtYYRe?0>-zQfNTlc%!d6@P*1gL}CSr-n@mKH~wCj)9wYu>kq6&g|^_!Q2-2 z4eyw7nRm3q(IQP(u#Kk_K)U7dyzQ3Nn#>C~V&&~H)?CMDq{@%Uh9;LTfwY5{*i5vV z>Gwp<2wnR#f359{+~_$>M{k0EzV{m zpI7tLpTN6z!q_6(`4DF5R`OBA=NW0TYp)R|l|#9|{x#mD>G2`*xCd)l>;!o=p5hhF z+wvs8APDl~M@GE#zshDt&gQex8rSW%06)6+bA)g}X|Y>?bKwZS{r4ddy8bK6TcCRR z-hYvcoj=*Wl7&ZI<@3PBsRZ-fnB%cI~Vkl}GfQ%{u>RoU1;Ll27 zm8|`^OWNvXA#$acVU;iL^7N*cj54_%h%JB0w$!RIrra*Qce0n=L%H|U)3!$Pb*5pL zjVt)#m;do&lW8l~Y8SZ0pE>_j zVoyK*9l6X%m&hwT{iw3u@9yD;riST^s^GS=b*$TWpKmLa^(w1b^4Y{Z**qJNt!oUC zeooIsson#~Mm)uXi(0vgiJ{H>x?L@|e&Lc88b2C+47BXA{MGv^Kc3&q zkAdAm?s3JfR~dUKvpaHcuhtOwnwJ3b(zg6{Bbwb#%i_APm$JMTK|I+N;&RFx16f|5 zOs*@NgtK{<>e_IN^em<^S0GU(s*w&4){2kscDJYpdZ1Vrbv zMvkobjkQdCnd=4(hB=OHhjw=F{qxjF_%)pM$7FuQ?PDcpoAd3Rj(6`PpS&Y&dS&RXv*<^2hS%BAjNpmwG%1nMHqKt|6;xA!G3BR=91k~hUb>b%Hp zC8zlJK^pI_+{2F#4g#a!dx4&+l2V!23+jt|1gUpP;maxN}X!oN0nNpe*$v7XSnX=o<^Z0y=#<9zf9h0zY4?0&*D+kXywc6Eevw}erKPq-E2%M zvn2av4*KyaRkAt_iU>OlUEQPmr5u^Wx;{z;%%iHI1S+Df6q&HPmv#7fH7Q*(#t(@~U@k}=mVxKnB=pJWia(bAirn0x*jlU4-rMK8uzfxO z<9UHdlf`;P5MU3p%MsPNdj01{m~8pX@JN^KyyN>nAe5^Afvq(wc|4`7QBpmHFHU|7 zdYMNVBDN9SwjkKB;+rrdB)OT9D$>*CmC7I4%&{2=us7-%N#ciU!BZMbUimr*=gb}! z^wT|<-R~=waXr(mI;s+Q9mX5!)tV+fl6X?5Ua;Fmy*OCfJ_fXJ`PlCKC@#5Q7swSq zz$3aeBuiGWVxuY7T$&`{HYIfLe=>AgqpY2q!gG>_=y$QYR9+XlZ||wCfU)F{Vf0f+Jy&f#b+f$7}p|G z3;3&II|MRQWuAwBh@`E#9pD|G&UF{g7^M<-nMXaJ!Dk<+hO0@bIH?Y*Azn8ogZha_ zxbD*%{P=4#JAEUCmoHn+Ln}8!XUAzu}>kcgj;byWc zTLl7wWp;BubMMAc^7eX5z+u_Cn&+P*TL%wrdZac#vrs<06A)+Bveo(LK@=b%(ntr1 z*Vy{ef(i*}lVO)owTw5%fW!@#Aku6N9$gjUlmA_XDSnSbDm8IH7oS-Sr`%YCad(*} z{-iU+21>`vEcu~c6(f%6CZH^Y=&9~k&Rj&e!e2%h9s=F=q_^UIl@{Mf?)CYh&L z?c#XonJ%ogWTo;t-1-&K5v20PKIz808z{A7d=Dl^!{<0*>D{+bg{hA!X7W1WT2DSN z%3O*hU#*&Q77?Ywfld3WpJ|57qfGjPe`=KB?R5l8=5h${@2Lou_)021*RrT#%-|D? zqN2u|*+#nZ4QvOcWCNzvmG^7#?vMX~$lHEoIB;jj*OF1M0H4?U!_s|A!sMqzMzPHQ zN^LaAAVNKt$Gthmu;WHdZK&NVaLRy=ypPnR2>G)e(3%nrrc3tQ+}fKVMY4$EP6=BL zJf|K2fQLsKG#z=K8{SQ10OxcT7@%=L*@RbFO#?}j(`R|X@VAg~cGcAB_E27$_A;F1 z+y;$(F~e|3-diAxTr8sOnk}0+MQWtXU%(3I>xK%g71=3ofE{Z631&t-y1!FmU&n({V#AhhaSE9$N+LV{|+mrXz9^Oweh7HUx%5{Whg zwKfGwvOE^Fp05Rh^Xu~^&g%wJOBo^hRYcUQM_Dut%}kYU$ZRro0uSGasJ>s2rX`j* z(PEU)J=apk9`3)fYe!M}LB(Yc~ z4+ECRN3e&7RW?)qEzqi6@}k9TGhJ3dS=>;TP!udZ4)bivO=~GWs*t*iINf>*IQ6~( z{hXYDq_&oD*(@^4cwe{3`uFJSD7%wEYGiVheadF2;gf4&~rev+6Y0dqh{;QMgp zt`#74imLscK9ef~>u`fZwHkV1Gfz7H7VGV_h*u12%@@<3hD49P&3in`-|}E9O8uwt z0GPD;XhMqB5#rqpdbP9sIR37S+N#sw_wM&Rxx~&pvT4qP4?HVUu6)j>o~1=PuKUy` z^1^kH*d!hrsJ9b=^;JosbghI)*Z=3cJI5MNehy?bbz*5v4_#S!+hx}~HR90rAxOM! zi=zV{^JHYN8fNzjc0UkH!U5&B8R8#$J5`dp!@*?-`J%2eerB{~Ifoc3A@`p$oGP=k zmP9LG+WdZ?{i924IZpMRcI3dRH|}!V_(YF(CIx5B|6H2XISu-KOpOu0DM&-tO?Eis z0C(=jYb|5zB1;~oKsv8~0}a-j2NW|$v-I}^VC4s@F-ggG_FtbEizHP6&vms7ubi35 z3){csDii;wkP5?8Xek?=3@+1Dq^7Kigt}OE3lB1NyfPL4S^%N7eg~l5QssvK+8SE5 zLuYL?cFRVx(+Z%z{M*GSC0^#i=W~y>EX@V;g&9Vgv@8eEYZ)CO!>&RI!Od7&5B2YS zHMo0UF^GGqwqcdDhj8HJufTFdNtKkgv&Bl6rhN3wVV3fJDGQAu!B)zD;Ej&Va#by5 zCB%{1g!oi;GkGnH%UUks20Ud&a`QVL{$&!EB%~zE@G>KrREuh>Bs>quULOa_Gw5g{ zKd4e1FJ>hIyI7ZTUqS)n?m+>!8v?f<^(<1tUNPdaKq$*iJTOEi+~;izce6k1EY|$lUjX@epNq5cD}d}9 zPJG(QlnpYp0d%zQ38Z7A2Zs19PV~{|r+7M{$>L8%(I0CAM zd0nxDPn~t-twV5i5Bg8yf3QZ#y+wu+kyQ~S+!RO18h;pQ)StX$$4)kwahI|sl| zEmi+6f+6pe^4Zi|E}YCyv*ybEJb>KGAep$xR&4$5Sa5(#`ds6hs7Bx@W){oL{liF? zbDeEgQVPf-8V)o^%a$Ikb;*HR;`lg}$R~*ukR>57}hJp<+ZJXmvHZ zyrVLQAItzYYg&Td=qybC+Ld%B%S;pNm+(y;m3SQWHOMvla#OG?3M-}~$Q`P{?dc7| zd+YJKjM>Pt{!zO6mTE<~FNM03iv~?|vcYoKF-E)$NVH9orbF4>nh37l%UCHAl!+>m z^w?Bpb>Xh}6HDdsap3K|N1QK7rwX4e+AtYsgNcZ+(**Q|6e@Ou<$N{_*7-gX&+{|b z==Gu6-%SG-e_t-8kg$P7jAv`lVm9o@VmuD&Vl@ zS!CbORfZ{@#sTQ9AW-&y2P5rK#pHjhS)NbkNhN%8m^|RLB0jqjD@; z+Mycyef%x_L`tnq-a2li%GOTM12fix)#P~2s>?8K*Q&zhsg-4Nf*B$tqBt5#Xr#e3 z?|fUib~rOl%ZFE|QZfKK%bSVcQ@r5&CIC5NF5>)N6+j-33?_9H?ecSF21!}RdG^yE zvWK;aP{zvU2#QCm0o+ldC#XWRrG&3GlE_zy4{#G>TV@LZ_? zSDkI##&CyN*|!WGK^VKS{7}N~{Bp|`S-7QTANKGYeVk;Z@_Y~d8@2xEhKs=8FIcW8 z^t`6x8qa60CtUo;7S{cHXCqyH$7+Oz^26l&-&tsP-EhcBD z8`$ueGazbjL807C=4l+a!lYmx?^}iwfCLYci~Y%5lItDd+gZB3iPW!c6v^@TvAk&j z6Dq&r3RdX=_0h9rDscrd8^y8d)EJVPoCz|uIs2f@wSm-F?Mju;X0!0~jo9UUN}j}jIa{dz z4MKxrg_fiSlVS(KpB0IJKCIvCs%)HGhs*azZa;I zmOJ;1Lh@a;Aept4ZM~R*7^tJX$T=z!Y|Ta*>TQ6=G&SL<+51SMyUMg5_64|aq!+f5 zD{u1NeNTA&t8W3~m@SCvHT1?*6%`Go_qPDz0zNX*fvZ%lRUy+QhV#m=GsBQoVL@`u z3GKg-SWX&ARRdDq!s^Kf%GEn~7ZPWIjx>E=aeA$b0P~Rw<|lG2mbW7J$7|)biJ}ZG zL$V&hC-aya%h|EGfjrJeWPJ8B_cD65ja;u`_{cRZj)j(rYso4?d7L(pyGOS`bDb|3 zn%q(;TkooJFlH*nVni?=>jNeDhCG6LOTIz0(Y?F26xK~NLn`F_+mLh7EA0GQUssx> zDYE^E@+Fzr2eA=00meRY34mp$7!=;Ffx?=knDhm2BrC(PWx`b-)#z`z>ONdh$&_Ds<*%xMHpv2o*V45(*>K3fH9)jfox)>f+Ik4!)utfULE{xT(cZ{<-3Sec z#&J_vPX>R?eZeW^Qcwg{ZB_%bdzAQO!TUToY7^&}Cwf{lDl@X%NE55eDNzU15OKVH#)j+ba8X9P& z=hsptmMvkSsyZC}@;h+O4Qnb7$Mb;x2V6z+Rb7DhV@i@NIg7fcJb>C^b>>_A2}DPO zp4zg)M$j$oKhd`l)2Dp{w)1B|LOw}zVEmoRW0_>YP7%LBTZxpFgIpGqe zT$1y2Ydv}8U4SjA?9=~U9{hevs=TNs9yLyctkzA07Jkr^?c!?yVj8RX)}GsFhfBEo z!59tf^OO1!xyy>aDG{HsTs)1PIfW!rQT8QTcJhLTMGj~a|x-pnBLVh1aygHdZz(2A1e`N>G)s0n=Wg?gWxz$FhY*Vh< zeQ&Y3+t0zc8cgBLx-3ff7JuT`Q|hqW-I@t9aT0pLQ5|n$Ei0v_=aouu!th7yym4By0 zVGRa=()In(S6b@~t8DxVnz^N#5$#kD$<`fV9E+=(QNNQDA?*gi^_SHo?^S9w8e~&NOVzsx z=i+ZlL;#kxk>|e$qchdSL4yD8<`@XA8@hbRrGORFO_ znsU7pcZo_K?JjdSAW_$=d4f}(Ln8F0Zcx7cluHi12$)u#LRADW0CZ_2SN54)$zUEMhIY=9$S|* zc4q|h)2dima(x_kZY23jKzz2!(6~s~az<6HPD34O?Im1=&tF($_~cj&#C~A9jd)Na zL`N{;G?_RZnRJUFC)Ads=ByN$pN^uDZYa8_VUg!!01)ycx@9(fv#ckGP*A^ zqLPwuDKB@GG4224&klbZZh{90&j00b zW@%rUuUQkTiI`=p##w=KHUO?`shR+}zJy|^a;CWVltEJrH9oa;5Y+ar?rwtq;;&w+ zq9IK8HbU*#gXOCS>}iO~d2BNX+Q$?l&*%ce_S`8SUxfu1(e&`phV?2XWWl4*802q*B%AF1H=1euw`0g`^ynh)uDzgN8n$`}rq$RFt`I|y7`Nd&aWffKq z>Lx7-wN??&!9KNzbnNVMK*V{@UGMeo4`r4<7P8oCu zQn@hRWhLfATcVBr?C4oNKc4ht{3b19048NBssy(*P;T8p#-7Vfl724&(3vjaQ1k~T z!B+H$jQz`~VsFXOR8}}_s8dgONy7gu<;oq#)TpNN?Om+vMqONPPUYGCWRM%Y?hmxt z^&jd(k-MQEt7_3i4NXINybasAXbKpeao?ywWHF??qiUv;q~z{5J1|s9tFD7oL)$z& zf{hRt&Z2H)v-fqkfLLR_C>mN;Me-_S|91qEZcs@c@CzFtqf{)tjhf8Zo5EB1E|=p$ za1e{lIEnN>@(bI(QBWl-KL*QG@>Nq(o^8mp`B2p7L{!84wMZ`pkw8J0R3 zJ$O#n+R2L7;OtK+Yy%DX&5$xEqG%1C+knFfZR93Ic?ZzE=EdHuL8XFJTx3JLs4|qV zw{l&yl6pJ8h3_9Qd4hvgSx@{4$qi)uT?F&jYv8^egW*!zfpA#ul;P1DWtK}oJJ!8W zH~T9`2*KMUtSluOf%&s#b&-6yl{dcsECL*Q3uoijcnSXl+;-GGJ&+4wk+N?x+!Li* zkI-l(QkJfSI8G)S@mMnHJ*2S#hQC~e?XzS#AP7|!#zF=DSC&Cn$$CO4#AI)!;P&|BoV=GFdu<9n^akTWYLw^Un=t;k1ekC6rd&Pkywrs*Aa4 z(0_iUtYLDY?5fTvme2RuRI5X46v+cgT_m;zWSg-RDn7f|rO8VJcu$A{Ds@v=O}ipxMPCRe1_3V58-_X`g!Pt;4lpZ2 zI9xKQl=oCKja1bjMuYl{a5z3aOa?ar78^e{h?ZPLdrVey5;Dgi*g9mxhDS29XS8rzVk4XTK7oUb%uwH_%IvLXsVM-1Tp z`-@@CIhE;h_Xc#?LoaHe3g%2&Zn1ZqSS`@$ISo~7js_}a*iC3LKN{enS!Yxy zURWmwDPDC2%8ONfPkJkf{r5lr{TrrB53=$U-!zq{XCWRLLSIKV7eWVRXIO=+A%f#N zl`2ww6rr_sw_#Owjzbox5dt*E?^-}1^-(q9rK_IYS#nNr)6p;ho2`3d%Q=F>Qcn?%vLdY~GpjQUhG}P0=DI=lnaTh(-E@kT>qxV&SwQvN8+VCQa4K0F5 zF6W`;);`33jcUz`OY|xy6v>lUA9uNBYcE!OdIaY1u{>z=m2bexh?x**O?t`6q`fW` zXDw>0^(=^&XIgPWd!{|$U4D(b&OZd*FRS2XMnkA)Hl<4hjEmzX`Fl7-Q+3V$y4hE1 zz78iPe!v~SuIK4;Gb2p4o&>&quV6YfrGt@duaEp(x85k@P)pa(6q3VH)`d2yJHkbq zPL-GzHXCtVq$=P>{sz9N*F;$au%MMD(C^E5cGQ;{B&)U~KL>kR+K>9s5GkDs`BalO z2AyktRAyQm!nGBBWN*HaB?+~8!1y3e5I?DT1s|dWWswR)W~uoHk5$h}=CIggo2qo~ zz5(I%T%}ig13psF~ZG5hJ)HZ$0@~$PYT4s5)mYQpgY&?&e{MHPZ_1y>}uiXMtJ$?dxM1h_1 zXAEFy+{*9@zG%9sEk;Oq)kK4%Ke5C{l!D5v0hlnoRTr(Xi@4)l4oi7)i%}*=4qzJ8 zqG`U^H@K3;7mpb-?I2?OI;OO29LXzvyCIPxCc(-QmG!a1$I>Nou}&NSMFq~48UM^BUiHnQ7M-qU3rzscWNE;x*6QO{e?-+LZ{vRH= z!2(miqvthI2hrAIor^sAaF3A(ddtHl^Jl2}#1m+czO9swybFl)PO`bme~fgc%m>Q6 z=eeYX%Huwv0>Mq9FgF`k^3Y{^d0iP`<>2|1il;Q7)}1(ReyB8<4hL>wwkk#5)JGNR z<=xqOb}O;Q@u=asNZzANGL|7Kt>}+le3n!L;rf|g7Djy19E89pgmqaj>MTtTXQ)1r zrL8&7Q+2z4h|P(~TGWjy)8g9VuEWcV63aMcI6^qVCHIxH$BcBD!o&j^wUyP*d&Tg` zk}fR_V58w77D=FuKk7Ruk10nb{su{$7$j(TdMwEYRmX`TZ(v9TIM`{IAc*gKYV_&YKnsk2b$6sp+Q$YS zeTstV?g7BlgIN1*J--n#TFez9&CxQBx@1{e?TVEJesKB7|C=nVb(T$uY`MmIR~j1I z-ABfM!Rcr((>?eXl-#Q8N@B@rHp%?}fi=0HTt2lK#l*5o+erz{>SXF*Gm(MYuc0K6^bQzZlG7&TDJqWB`~4L0x_^WupcTN}%Teh5rD3bGnN9=q^ZZ`ad0bKZz|vvn`c#&M&eE$E9+U!8KPZ39me z)Ii6n>)^4Ysu5n^6;|j~(4})*EpiUsJ>`0T&XfI>g-KJgL0VZRC$^Rus={9*N$~g zomF*Z4ioQWU2BByLZ>^o zH|~GsiqdHvtU?c;L8=i%FB$2AEjGOe!=+bdX@gSSSm=daWZaYpj+w~O zd4>wnx7PFeYaMed2(}cpq4u#_Rj1PUIqy$fs zOlPbgen&U?y$*tQ>3P&TY(WSyv^Z9laA=ez8v=0NGXO1AmT~lxZSS-9Yc~K2P5yey zoYR1KZ$S~MeT6ZgC@n$0e-)9i_B~fV61!A&EKiD&21&@}R;tMTg3bV(9ZWrz!6lG# z{#a;yjh-Hab;Xp+CybetedA$&^AlI9^_`Dv5DTsG9OQP@ZE$)~7H!?w)ujX>oT|8% z#eUPpb&jB9YpffLvzB`YDzcl6lN^2aPFihgIR($yNPS#mYxt2;poXlJU@-&OzD zjBtaf-PO9w)|IA;2uM(@`+g*IfcTMz8@swla?yV+U6OpR|P zZ{FbOMW6pbX*P`Yb!!cf+IE2*s~A5aD|0cUHpigH9;r@(2ld#eiY4MEU_4%;ghpv2NVm`>%YFyE6K3$^^?{%&Y6W-F z55K8c-a(7`eF!xS*NbhMKcsjkkxL6Wm;9HfJ@Non*4roG63Z{>>p4n>6prHdGrD~u zJI)T3mu~G(mrJ8uVYHsXi+s`LRC(T-te<6+N*AVH$v$<0g9jT*>I#JO;F3a_H;8qt zKaGHxsB2MDVZ|aitqzaV9|O)yiRn@n#($9(SH2wSXt;S*c}ML`rly@ujFpRDvzE2b zv8qLWZ1!zEY^?M-%%kt9819S|yLKu!AM$th!1YTRp2Q7lbh8H7%IV! zGm^2keT}s*3gT%}P#q*s7P-pN5owcgER`p)#H3)HG)^Zmxw2l|o7YBeyaeFJUCBtJ z2LVYJ8`aTe-4@8|o11Xx4m~SPr9(B5<{l`-vxUo2*1K#nmWEa)y$!vQ&bSaPAYW?P#vsS8>;p2 z7#Uf{5}(WB)-Hpf*rO#dM;gs?h}|a-(7G3dNt=0yne9Konn5eEY@61E=dQch?E4L3 zyJ@Wz6NJX4JJYMw}O-ggE~Ti@f(?wyQm38&gj4z2== zk5wJ?xxc}7!XaKWLT&7tfyLEViVU2}Ut8&O$nmvetX!rBLR(Z4kK;RbjEDoGdzQwv zva<^tT-%(rk&_xMk3QqUi*#>187*Ohu<@>YQPh`@bdWp4j%0S{9m3>=-G7^SKy#4t2UdWfXvf?)a&MT>7H zw%hZ1RzGb`ZnnIABts+r)-Cm3g9^XV9-akZ-2sjyE6z|zWU4N!IxEzOrmF4XJz*>YCcX% z=fa?PVNzsyFZTX-4AeOHXQPln6bH7m$_mN(1bh%c=}tyfyiKO}&QM)%Jao7kj=rcr z01nZ{a>?I%A(MPMMwVKc@e4(+Z3i4Z13_F%HKq>{I7-@eGaM|kv<-q^3slK}ZDoOS z=FB)|%ST2qJ7=CB%5kBdoFSPlv68o|xM3s8kT?tHG?lM+AzD(K1C9BM`SugK@@TzE zeDc-_nEG!o)YP{zuRS^j#yc?AsE~&3p`jn&gL&uZrOXTa6VY8U&6Y-1IPFe6VUY8d zS`cwafhIY9U1buHfK>W|nE(j#><|vd4m&0$f^@a&R#=Cg?y_t%yEoH9mui(X;K(HfYJZGx~e zYUz$yGOXG~pj7uvfLHUI%Hb9+)n(Fw-O~yFqgzccrEe42NnS5`l8;yl>7AiE@M0j}(bKrSHcP7b43D9ag-QHF1{|a3--IL)%59HKGf#Lx_`#qxQctdRAPO5WAv7VZy zbuO?=|1Ws>(Cze*JP-WeqA3Shk!l@7W1jH?=D9X605bdb$8JsX3x zE2KI_N$NM|Ll~pffoXBuw|zk#_#PbqVJjoa9-7GO-X= zzomORKz2pVu(qp*DRnm#Mo#N2<~|qYtDAVh;TdSpAAeva>n}llW8detZK^joJqN9z zh%nlvhdjBS>Oh;WCq>Bj+d<$^3~An*)leebE&`tC;ebw=9=bw=TD4=BjWW)dRTJ?| z60PW0i0o&?U!Yw62NEW0(Izdg!HcrNNYj?0#F$DmnXy^rw6@B5ib4aL1o!g?S8J8A zckydi88R7<;XzgDa#eTKM-wr`r#Uu}ta=p^tJaHp;YhRUbe)VTRuy#nb$@xt8#gnER)H30IpVf}@%@=I@9DyCaiXGw0q zTH{O*J4q5%f{_D|Lgs~8a^@D>CzE=dEIb3d&)37IkbdOW<=rMgbD#=9UT9>a7Jj;s zFJCsrqI=zMmyqmCu1Dj&TqQrF{GE!)-MevfWN< zEZL5dB<#1Mcv>+9_X-9vNR7RSX+%6A$PqwZicKzjKC=+ z0U^%?t76=>(VhKhtZjJIk)$nrLHDR3x}&+w_Q!gFqI)op6w1#ZiiE@xgJuH$&CGxo&whh6 z-4RIpV{f_gWZQK%^mML)qfK`{D|vHx)VR5jL;4i>egZ|EbO;=?7o~=|pvf$dfa`_7*Ren*`JIDJNg|b|ESlZ`05Nz%q zvb~Kf4HOj(mBB5Hcs_HurD0>X^Xd52IdUcjM{En#`K*dDs><=zL2g`CN?wh_S*FML zQgPo-64-%Nj7%xQ4OS6`%PtEg*A&rl?i^It$uH6w!Wy&NayW*b86GoLO~vX4TJvAf z?pHdcof@aJO1BS^Z#O{#aG+bZPvPB+G)%{~@WiObssEiw`(rm2y9gs!hhsfgw8V*< zoF-#5ME#13m`cd(XXVEZ5ZA{IjBI2p^E;kZ;i(@|Xyb4eW4LoP3N~n-%PTDhfkwM3 zh+TCW+3;FDs1c~thWv`D0VcaahNogl{kD$$8 zuNtE*OG%OkkqEEDdbyKWo4vVQnF*))m|IZmJZv+2KJ2l7TRb8VT@og@?p%T-! zsFOOQ1|90CR!^E-fSKx5@SeBm)hRz*b;VQrgQ!3#?YWgMdYCbgr+tBo_ha?eh@eGXt=CPx-iH#1DwdN14#f z3)Cmb`KD0+vmFc{?oCdT4NCzOnIVIr4btWF3ZWy86vJf8xeWR=GzB6LKSyZJNlcfn z>TD6EJRGksK{O@k&V{fA39kKAkEUS-T((J1Uc{M_1d1rVVwPNA4`a8`o!vlLS|kUS z;Fe7?exj3f*`(7&7kk;spH|*q68#}ql*Mx2L}ae;MjL` z;=oH+F<&;7qTKL+svdXg!I&aRt#NNK5No0CZYg7`;qEQ#FbYVls>|_+Gs8JOwyVS? zLch2{5|FcLPfNUt_N`aGVTgFDv>)z@B|psUC^xo4gph^ltrd+z190c0f6K&Bcl+*O-j=H^|Zg{hd*PJ?nwj|%{X^y*0&I2J!WX&05SNtV7 ztYRbFa6wO9f8u{)LAGp8;Vezo+bkJEcRaP&1Wp=` zCHOJ2`pAQqA-6YmUlhu>IDt%fL&-=_C+(@sVr_e-%3doDLZW2NmIupVnkA;qBNC1l z=r<8!iEU75)6*8oiN(;+dwR|@gpN&OEh(%vLQfjYlH)`F&xjl})hNT!>Krf0#kOqi zugZLtx71v&EdZ5WIn7_YYvjnI5BcsHy&_3APc-u6K_B3eqkDoR>FW$99PDlflM@@3 zWqdJEPi-$9qM^+dx~azc(CWR6!OAjyI|`y+fhi4_1B#bbCegNj5adVuSr=olo{^)J zkVw!?9%AT4>a^#HdW zR$YmTTS}APZMYW>@%T%sJm1-u^)NMdIBGK|0y$1j#&`Fy&THsd^E6vZmBhY;N|~)! zwA0?sOqWLsU3N*N#~99qum~AYKO-B9tt>&}m3>H$&(Y9A*fRNj4nRLO0vu1!tEqTa zMIBkMqiX&g%NLI}x!f{hC5$a9U;2*#fZ9#3ULxPTg{@@h)@-@-hAmy1zt0zYGIPYb z6v%W{L8TVTy}Gv*pEK3GF{I*)l<@a6V4TWQ)ptBm?2)Vlk=I{l?tueC)ewcx)JP&u zw#nclYo_$?#cP-u(~&biiEpjXU{n;o$Li*MiBaQMqSB_4Ys&%3cdWRTn?)0UNN+0< z-yjvcb_<9S>H(_eP%uUw>3H4UA0d!lha(8H6^cSq;Uo%Dehh^XF2oonp8o6+gekLz zD5u3h0%-uG!~YP0t5#xdEUjzMuxAs0MNhHi$+7nguSBa;Q6t?mQVutbBrl=sA#_(< zG^W`Px@Q4(bG~@_^Bb;i(%x`LA7NREPlL!y%CECaw}$HHF$Uw)MKP^T2es|2BxHqrP- zKKclnE>%N;d+O$3XdQea!CW-e>Lv{1#W_T(LNLqP(coAvj&>yhPBT``4NH~pX>tdJVuQU( zu8`KFK~|`8&h^)0$I8uyh}M7bfykCKAR@b8Mn2##3YN`ru2NZ~v!bE8Z!QwFq8T|d zNqywe1(Yb04M&P)A{@e)h8QByjzDd4VzGp)BEM&u@s5#EI9M8I!KGK5Lel3`eCk+C zEmsXBdn4V~P+oqFK~kc7Mu@#OK%189m7+1={7QjOn)%}b!G@_Xe=f2qt1Gn|!8+6W z-xM*aKg^X4YgA2^rR(`?f^HuxKYs}g^j9L&+1Zs!l*%3{M}zrnb|s2wt6p92zqYiP zZgKMFbreXwv+Vrm%u+e{r4cVjmT|WGg05enhtXP}{N##-jVcy{edk;@NltxC1D^Kn zd4$>iwTOX@3osCv&eIv6Jb^iAGZl|cA$fIM&^@6Ip^`yU0DKqSF+m>o%%GI?=zlV> zEJ&N4n2Liqt-tK}3%{x|t4;v{?)2P7YwYC%eO&9fI6nP-EQ`6;ZI?iE~npvoXSSJxD_f(nxJi0JaMQw(Z z#KL7T0mCNSu{uh~r%|?r(1Ex=;FzjHnf0JIZL##IV8zU%3Pwg0Mayr~4YxGg1CVHE z(_fpGSg3`Bm>m3JGrMl>F2S2IwHoW?FOcw%O%pHGk8iO$;~@?d&AXqL^-nUA#OsGa z6R+Ec;z%INsp>MPrgduSF->_grSr>dN19>}{%;&|v|ml5TCOjEPv2Gs!C@v&^Sok3 zwpHCAy((RTPeNi01fC-6nG+*NHn8k-iN#b(-g_!rwf41(|x_=&Q-%IaR)LqNxOi& z{(|<>;@QB{pI(sRM6h{gLIJTYuK=9}syOpkz0|4Fd%DUSKO0tdTqb$1ps?qEV8p|= z1Y0Ko^nouyVY3#%tr)XGL~NKs3wsPx@v#<&-Pneb-wv40d zWI=muyYYHhA}nm5E$ujl;N2=-D;rc(woEFtP$=mH%ie*Y%HLqcYK}T9CJcHd*Mq4` znY91Y;PWHh8$?tktvilR`wDrj8w;=`mg*N~3W&(>B(0|!vG{*;!e#Loy!%rdp#ER{ z0nrrw8(-|ueWdw{W-mS5U^xIVTXM5)3Q)>(C*#Hu8|kO z`Xb%hh!ZcPg&wKt^vpib)HbK6V_$4Lt-Qb^kppdqWp#q6^q>5`Rda^2A9DQ z`S*h1z=BH-(QZ|$3DRXcy4}-~@J`uidFB~IlU>_TCf_BjHqnu#6Cp6-$boul9eHh< z%cJF{#LK#_MwQ%&G)!5wA9G_fG6)Jvoqz?1pXvb4Ta=z>39>3VhjT)@RdSBO2a6bc zp!!a!o?y-l8BvJw{Zk5}?%*bXUx!n>MDKRROQ$+4mLBGTIBnxwNas#abF(pA;==rv z$AjVZ-*wL*?DFJ67!3JsuyvS488-^!W{%22nx$9F<+NbU#?_MA8i?6*W%+Hxu1~CM z)oMOOsoaltRcU=fJOnL#6kT*h!hf!twK?@xk0D=~HGM!o4T<;&?XdU#% zI+BCAkh4?}Xmopw9V{m`jWsM9Ay7D3W_(~2YxOFNW$AproT#!1HtSWhWy0GTPPu&7 z<&o=!E|l88?EOkh_SDbzfF9Z^-xoW{slAYCN7Q(`V?y})qB_#YD<{#5xL1ja?RB8j zW7N3S7~L}vj3e+47Vf7v^$rxvF%=Eit(?nodc4X#^!AaAN?W{~zLcTHsJp_jhAW;T zrIKok!+KIHZC`FF9!1{^5_KMuob5A;WnwaX^vrg6l~R}(SS>GFOAqnN+)5)&**0V3 zlg<1!UQL%-RkE(e5X4v_;X$dKH9kn7{+2*3C;WM?-$ z`(CLX(BUBwyxO|wo+9x;8!FhT$5{_(- zWEoTmH8-jB$QXKssrXVudGE2SP`_(G5M>M ze07eJWvXN5KaC-d$yBEzIkQ_6K+YL1mUWQuYP~d_e~ZKMyO+1dR7-kIzMPH9h*dMw znXIZtMQPEP^_ngZCm|A|@Cs<(q*Qahr+_5>6-pToEfJ)}Cdem&hB};MUPtStlQ2oV zedR(FN-9*3?|`5!Y%TT|0QdVY)PJ6;GcayDA?0#as{4i# zHaAo%VdSTvHiIELdd<3`f|DuGOsP$BfAh|bVAvlJgsqw z9fOdXphJ3gH-5N7Wz0O6Roj=DFZVA)F7uHwDs~em3x+{z-SSX~r~iO4>Zp{FFUN!a zclE5=QmMN5S1m^_EL2m*{!wGDhUh+=W6B_E{}z_bfGPvmbBBy9F`Kz63PY=YV7!!(adR;H>Pwy#AGv7K_x{RgP z0g0Ty9g21I#{6aw5~us7s)@Z1RgwN8++WoeQ9%Q_%M)43BP5mt-Z83xaB_kyoQwXq z(+diawpoOaJ*koS9nUBuDb(3R!unyaCh19WG9lYg^lYsS0b8ANmv%R34d$wl>=%d) zncrFd*=X|+t}8Zh0+gpiim7&F5*2J$HzBunKg^)5Y?-SJxm%{(;ITCkFBj$ zblCXq*U{F*MIKvV6kp=x#esvh<~`Qb%3{udvRol)n(N!IKP5(y7l;(B^k2h94SiC99u*%ZeyJN9|q7bljjyqt;BO}f>8 zGx)#JA-3?K|9(ek5ycOqBW^y7PKfYr4UPzls1Z>k#%}G@JM_Qr?bgWuersB!7XSbG zlrL{Q3jX8C6X7o&UZNo4vW{% z3^0SZxUH71m1#bUm+vW1rjeAEquDJEKd+cJvkvzU)ABX9#pmbc1_s+CX_K{L&0}%+ zdux~>CP|6fT&+SYv-teIJi%EP5hYVj7gpHI9z(u~tmwcVQ2 z;_-6?cmvG{vn6YMS4+2)`S}98yu)m*rD>^x@31M;idB_>_);61w8t&2BTGIW>#AoxePICy5>}|>Mvj;c=y>(5)d>U8` zv*i1^1AKvA7H*Ex3bo0WVn0uSBgh+RCYoclB5kgv!mljA7vyCp=0w21++y{&*Kh=T z?PeQu3gFvliT8Kc@CAF>nmJu7)plE){+=3+5HCSJb0!#h*OKmER>K$KWv}M5TDf-0 zlH+f$>8Rz6F*}>{fcQmAzQ4Psua=k1oAWgaku1gjo|=wOZv(Ttxk#(l?pZ4Q%WC>U zy+Ff+JS={GR}Rfl5Zd~s zOPSRd?gd`vdM(z{+b>7#Hb-r5Q`2Q`0z5u-7U+xc0(0|_mS#ESmlI$QazuI)%+cl%sOO?zetuc%QL;;ZKcv*r&l z-*SIzO?xdzj5pbwWBw1a-sm4+(_PCK;|0~`Ev?kD+uvE!Q_E4`+r@m|yaTtr>z`h; ztd_677u=h_Y2}tv{y8=6p^gULZsr2>KAdpTKfk6s)YrfZF_;guD$7s)#Wg*lj)vYI z=3?`qwzS%E&%dH(S*Wj}7xFNlAV2)XYPE+sV!aM?nduMr)RuUwJIojBgmN37@ z;lT;5jFtz}`pMqUWW47J4i^_Co~J>1dQ+t*xe)`8)MNxsz`?rZFY zqRc2ulHX)0wtB)HO}zchT(h1f*>5iC+GXLsCSFL)Y>4n#E>@epwxg-nX>Kr^z@r-} z>v7liHT6PsW}GF}Z?`yYp4yIP-ht+3vn9giT}ii<)%G>>LWE{(OPb#)$+6k%IGTH( zGG8~_!PXZg-{!94Ywm?A%}$mqzn`So=BeX|^A0w5m|ZQ|e)pupR#wLs=Y>qoo)**3 zFTfgTuj`2SrkcCVK1hPv0r7$Ey1sZXbZZW<Y{~EFeG79pP)?g|f|2mO{VD0mXry2uDlrNVC8k zYbo-Z8&DBg7U65@h1AW7$lm1v)*ySNBf;x751UhvHyZ=ugWQq61TVC2PPdf$?GA7T zdHx?o=i=Ye{l@W}bCR5s`$_Njp0q_#l~Pqyom^DhgSr-@~ZI_u2roan4Gv&_soGfU{G%PhvI%lJL{54KmG@AvtJ${ze0WHi1FV<;e#wo>7RN3L#AV1VX4fY;DwKp{)P83 zG7oMNhm1^A=UK|>UwNM(i!s0Okj$9qg|C$UjrSR{5(^01WbQ=uV9N^ncitDsdMqUD zl=&0A@Y<@V-K}VmzELCzI6q7Bd~>okeOe63>tulf3X9(>HnFAUiP%*UQ~W>LHfZ z^q;)%kT)?kUMlw|dEvvRZ}WaY-h~UyAt#g7LoF5b9o|pK2bd8*BsV5|;Y+9Q@qR%* z#u&U!?oL(@v#g{4;{AqvhMDnBxj)$ppFaIJ?+@fFxV!lZGDTfrsiYtBZX(}cZahO_ zO!2~5KtJN$Mt*?1S+8)XsE1oN(*N@AA-`Z=yj0;&@xtLiKjr<6`~ml-LqVph3oTXj zGu|WQHs;665d)kS%x2yT%M)bl1ME6nCazR!1ha+rpXE971bcu76zy<)Fk5-AEw7MQ z*ee(^2#TT^3lFiPsE{uwDAEAu31j79Rv{|qr@;N=q9_ex;|Z){G=`r|Fl0L%FpQmt zTjgj9zmTwygo@HK4xY#wf%fE2fD7CJCk|81lUQTWZ2nxhvRo=k$2fU1YXX|bUqJ-O zb~u8V8lJ+Mf)?_(5IiN3M(G(BPiaj{fr8K~q#MJU4tv%5R`~z^yxzebS%r;)M zwGTRte~e(1?Qlpj+j+6pY;-RF3`{RXI#tBf@#3umQ1}&1m>UdmYB4)_iPk)H1^+r+ zjIMNQ6jRSjwhl$t@gEQYWjh>W%uZgawGiFHe+4rbkwF<4H&12NqE5bCL`4|jTw`|e z(yc{kJwFBRQdb5wn%T|kVI70+g6KhwVH$Y7t&`Ei z{0T6w7~te%_VD^zr=iFAbK&}R^`OQwdwKn=v(S@#*m#a;hog{rlh@xm7d^w@BH~37 zJ*ja_BQM7~A3X=lk5r@q&PL`f-azXjG?u?#R1j(DNsg!YvAM`n^k@E2Q3)KD?(y{7 zY#y=#{hfbWR29kVMNXjiv*A1aPyPkbemFhd6X^Ni}I!I71z6M9KdkXylI{`U>4i{93Uqz1b%Z5jqYJpUe1n0ba27R2JkDNny2$CcPF_tWHCVhe}Mb4vp1X+?2 zIQZQ&=}*{2$VK!WL4l+yhS!grMYpiwDd>H{ILUsv3EZ>jPuZo&74#!PiR3iQrsQn; zBwLPLML!jkORmMR{oHUhuq%-3=$C>j$;%jCHd#Wqvf;t$YeAhPDVEN5!(G9yMsA_s z3-(J2VlCO^9QqVnf!sxZ795q7#Io6LxIEZ($OH6u!D&fVEU!N~mp;vgXRWP&3NA?Y z$I|`XaGkIlktgV1f@_k~FcXvW=re2;@*MqFa9?sQmhJC``-Rq-Eaf3PDG9+3k#%G zal9OIKHbKKN4g%uank*9bdDRYBz7ATgY^@ZNKeBkPA;I&v2{oSHb_`5y%xvjxZ#ds z>yZ>}xUfq462cK#O1HD&DR88)PMQ=?4{*b!#qLIWV&jGTr3LYp0pvpZJllZu!KMn2 zN=xF|0dBa(*u6+LHb;0`S{2V5NG_r;u;IaRq40uqe>^?V4c8jG56Q!p3$IB}LjoWd z(-+zO$WUyp@V@j~JUh@0_Z)iwDa1AlUrI~)-4K5$zr|zKU=E=eo=MAxi;w!z(qtWp z)d>@1`QiQECDbL$A?s+YLD)wI{SDlN)K8Yf*74YW;ZRv6{PreqDRtS>Y@LK15{{8I zCgj7_Nd0U%Vx5W|70!~iCR7gMEu*ejj#_78EyAU;4p6)-%cx&0$ElOCcg}KHaID_>5Ira*f6|O{x~T#Bm_Z>_9F)|9qy7x!6k04 z;N77wA@5^jVO-BmHV+L|@b1!=kq@ycc(c41Zgulo-aYyX@)0&0Z1|-Y#!|Yu>z$_ZNK?`4n3Q6Mjpwd01#2?*V-c`5aq=cgZiqy$?T-`!{_Z`4ZcN zcgr6qhlYh9|Iq=2!|XVrh=M`DT*-S#-$1^`wnHe$O)(dQARg15$hX)Vc&4HlCIj;Z z-Xr=J@;&x8p0B7#2^EAk@E+4$$dA~2xIxhXqk?%O?_c^Z@-y~7$Pq0m=Ha1@yeIU1 zG;f7lDWTk$wGG$I6X zR|ENuoXk+=D2#w9UJ{8dT?JUkIPBlRW~msh-@%HwNXX%nh(qK!m~xEO zteA^Nv-mTK7P4Fw(E6)c2^Wj1`6i-`tb?(~=wzi_Jetd2LR=<~su-Q$$;!AyG@rkU z=p-+|9AvCv*!RTb%#zt|y(PI9G zM6vQ{8Z*+rjg98|0%18$lqxU4EM?rz#&G@668`5zh4Ow{V5EOL8_V@a89qlilu{V3 zjCE`rmxGq_za<)!S?NrXzmARP1_E{XkvOCr2h)~u2b;hRLd*HT5-rN|^gxk+2b;(Z zM$P;n(Wb0}@yl4xCUHa13jQ79vhrv;Gs<7jCUe8kO8!HlQ+WaAG2>1)g&U4m@t+Zo zmG{#Fqx?JBRBi-n;Ugk`gcJreqno8T4eH>FL{Sl08H~a2W>uUHV24ta5it%XHsdZf zjT?#9@#96g5#<>HgMSyB&W%Fde3eKaQ3oTOaW|X6jYb>zy+y?lM>Ckw{@rX3ZY%(% z98qb+1(@lKZ?HYN@#ud35K%?M{fxk9{~K&CZX&=ojmQxpg(1(_z-Drj&_n#uqK1g9 z9?Tej1KXRM0*qyn=upHsnEH%+*go7;^eBI(s3oGjM_`P958IcUj(YhfQCmbEjDg0z zY!){YZQ(BwU5+@~gBk1J%l6}Dqpkc^qRxm5Fc%u%WV5+B=xP22(c_5wJpyC>Z?gTl zc|ho_@IgpnP&78OYR-hV@oPmXO}1=avV&K#<$o3+(PsM|1D8& zWO>iPIR9JhKyESW=N}a5BkOunCLEJLnYsW>UkryC{ z8)3HOmZR7BpNlFY@AnLh_rJ{!=2oHs_|6=W(q7aABMh6|DzuaTt*9X~s~0oDzn>k# ztpTF6^+(a6$Z-(fjWB+4Ytj4sUqvmE<-GzE{O_>CxbbuFV2lB&kPv-2U#6g zgOUQ3SRYlFNlh}sOv}}xQG(v$;;5sU%p`vkJCfTDd@)B{8g&7vf)R#XZU>qq7$UBS zx}O=ClX_XYs8N#a9M;{Ze$ zVGQQ>pjm>M;+ClL-hs*f_t~-Bo2Xh~61PRw^`@p6VJ_z0LURR6#FwLv_GYH|53}RB zx6ypTDsgAj1z;CO7?inp&;r2*@#Coby#rJHAFva-cY)Yi#r$Y#AF9|0lQVY^)eCCH zQPEj_m|}l3JBfP_xZN&sM)Wwq9Yz?Xx%bg=g15xE(dB&t#r_Z3DclFBQE*VKkFM)O zO*O)7&3%X#3qBMVM<4COO!XgOr*i)Tyn0+*8hrr>i4lfv?jy8B@VU4m`hK6lRR90j z>DI^1gv-{*Tz%+-Im+5EQpX*Y%~Q8(}`@K1VAAcf^;YkM?Dz`;W16xG&I3!9#Jg zpuKM+4D6n{)-P?F1l{6)qPzNbOlLp#%(H%F+bZBm&=?{s4kmaHW94jiAT<#&jVj{| zuh(MIoJDH{3JDdXhml@=+%jMDHM(68D^bUkX8C7$f#qnvL3avLfnz#g##f)Plxn_3 z-w^bY7-J4)8E1M=SQcu&L*EqimoPDHF!ZZGu`JSjkG?G!EHTG)X8C7&0sm-zK;IRN zkif1DqyTk`rA+f9`kr8v#2u5-&p6B5Vp*d33H?AY5eT&&VuAWo%TmqH=>G)MCEl3Q ze*RhBPc6$dzn~ur<^eu+Ku%Dfw3KUpML!WN2848|pK-SLq-DA0H}o^XN=YE54T6LE zGs_Ci@8}nT^^#CbXFvaJ?`M{knm^F51Y0HW`T`^gb*sg!38H5OH9(y*vW+F)R?8~Q zP4pYVP6-vOhiIYx+_GBpC;FY>O^G_TG}~X|{oJw!DYRV_9FP=3)*$9krPu!L@~`c#;ELo}EYrVz zuIEbz@R#j@;JV}t%@bd~BHKIm2v5Uf7-p9!Rc3fN{;EzGAi@ z|Jj}ko=6_V2Ku+p^L)hsF|)lAypp_vL_;tX%UBS^jtYfRc^suSFdmk%BA8t$luJ_} z;=DAA(g{GV4JMC|sYWFB z7-6<_E`%VLi8{+T5t%(fm?vEk7f`pGJZBl;a`qHqp>zwRB4R$(#<&orJxy37t&gK} z4D&s0Of3>=?YE|Al{!1E0Q^v_-> z+#-Di$%-hY+8H;ZvTKD-sXU$_&Rn#|ZaIv*Vco z?WLY?832a%3Bm)?LdagkLh3xzfb_Oc79N&P07h?E=sC~qLHgRK36DwVLKt%`q`qVJ zBK_>MgeRpd;sXQP7ka*9fHB(V3eQNlKsqB9Q5Tp-B*#8qcura$PYpCI@?2ovLI&Cw z2`@?yKwNVzqP}PLA%pBog;%7<;+cW%i#*>mfF^o{P*o$T0gl;RES)2z0K+)DO%7WVn5!@QL(6d|+VvV$Tl@kWBj);VbE@cwR06 z36Jq08oNy>l*!@!14Eg|&m2T_cBfD-OM$%z7eqd$2^neMCXA6~Cos9~Wu6}yfS>ky zVT!CU!IDcr24oH)qwTweJ!KQ1kuWUrTw>lw#@hD^vt@G=*jyKcLgp|s-o8(mCxc~^ z-1a4&pBUh!_5;E~*%s(f2}p@dGcw8EBrKBECs2b7OFfsF50NSM!@>!&0}1pX7sN*9 z2r|`vL^w@$3|`Y|U+Vdp0mN!QCY&of0|gcVIg&Yw%(R~nE|Oh@BE+!FbA|Z`nQcEQ zTp_!jzz%Xjuw;%QbL^*t>tqiS0)yI@d46Gl%G%Eex5!>4@bU;qnv54Q*?mH%OrA*P z8Ol8!%yDFa{hY8~mXb*4xgdHnCy<5q^TNHd>_jH7z1;IF10dIaQFuUBm}to(mQz<5 z$fEX3!o#u&(7za#d#*B{BFpSogvVra6WKi1a_Tn*!m0hL@T6=-Vj!=5x#u?q*suM% zaO)Y_mPFoQVg+@Lfw*eFAv`CmPoxGLR(P&4pCfDRw}cmE2NLPQt`*ep4CGh)UEvkk zu|#HY`wGwR4De$61L1Yq87T0GmDF_x0~g8V=dJ;b$|`jdeiZSR9mlOIcBhP1Et{K)|OwrAsW zMMQFJUYdp7@?~%9ddH4$X^(1zPYYlaqfuwC8im#JDND2(0 zrnzs^KX3=ILVUCQWzx$byb2PcH|N2$c(q)boHUfKaCgx^a!ptfzC)grTmV3wge=Y- z!p7ixQj~52e?-@6tbWN3d!5 zNAi;7)4<0HE`ZjkFJAe;YRHeKu;B6!UBXEAK2_LDbOGz3|Z*)JV?{Jr} z!}xf`{*(f!AV}Z}+-2+tK2>ovrDQm}(fu!dkGp~$!{;bYr&JB+Z6X0Qa2?nQe4*k( z%KqW>CifHiFYYRK5?`*kmU0?e3lhi!cMUs*uT|Vnxi*~Ly@oh7t}wsK(oH?I-mrg-*C`TG^9w71O;G<>JMG`%4T?UgGYVU)EYGM% z)?4=P@%@USsg;G@04u1+)-L;x_#wrZ)W*X6&6elXzt+3W7V?^r~%YW1h>l6E5_%+3;)XEXv06eId)^7Vlyi;*L zwQ)rLR?92uKkIY*zxaK{)zsDzl|VqKSJs#IXZT~q-PDc|tLJdL`0E&s2Oyt#e%3X8x&Hrq`Lu{sQ5O>?nDMThB_)!Tg@mU&_X64`jRuK7TRnL0Y%ZmDk?FMETb;dUaJ{H zaIKu5ObjC{smJg)2?18ol6)1RgC;0SqqdnLhHE4EJ&3VnBbBQ$+d}Yz6WTCmnL;*G z#Tu{8EI=e$m~V;MWGhvn3E4sdM5+ygmjzHBHE7g!vk;MK6ZmlXkR4Qu#%vD>5xF)@ zV%Csd)Mbs=ZpIOXHibW&*hF?yk2N8C2nua&7|qzBZ;H~Y9cBViYSZ{5iS1;ZDpzZE zgg}wdhFQ%UWTvWE>vfn#NTjwWe?0LvnXjtQh8!Ug5~U5poA;oiYS5~y&0-{4+lN1u z_#Zh#)uJ_5hr~#XHcWFqAkd>;(+E8@}dUS0V@0^8JD@vz! znx#m*b^w1N@g3Qy%GH^jAt{oe4fCF#$Yxcs&g(SGkVI`Be>w3R*{Z70g`6Q7lB5j- zpqo&HHR#keW;v3q9m-!z+#@?wEjn{eNRFgv!$jy2*`>Oy^VXOZNUFAwznOSJcB>xi zLNy_%@3k}_A`H5-D81TcCJ~iZ%daLR%DA*#z1bB4Lqr>9Mv=|kRAU(BVsx(kpmUdb1 ztu;p?y|fef?-0Y3m1&Rlq1q5w1KKcV(m_KPHB!CJ9EJ4OPUgQ)j8(!bn%DM zvmAb+Nm-QcF3Q}&Bm{HOrH;$Q)+5Sk>E5EU9f5=(;4Mc7aYDH$-CxwagGmfRyXd$^ zoKmh!4;6Lo2qXr9cR2!tPw7l2M`hMCNx^({wWE_buiTri9#vK!ND2-?D;!(G%SW{;*yMx5Z0j9=Lqq9Hca4hMNv9kvux)TWBJ%kD^h1Ei zyeZb<+D-hY#NWz5dfO=XPBnCx+RgkI#8YJ`y%U(3H`O{qyOsZ%c&+4R@C~F}O<8HJ zm5+(=2r?rBAexu5YP5DfE>c8L8G3`;t%llDTg{h!S8Sw z$X)6*t6p2nj}-NaFlHPA%;rtAj?`}F$BOz#Fd1zI_bxRwrP>|*MA6^~b4Dj{H*dOi zly)aSRWu^P0$(S&Tb*GwXm|0`MWZ6z85scLycyQf+Bf*UL=!<&(vNoURzta}-NWxI znjYcJC7A2xW8vyYu$4PSn23A0gTr!Rx^vLpG>0tw!xZzD`sVLH5W1 znCH#3PSU=|A0^ruLG{p&aW|-;!qvXdA1mrfWCPu6Z}9XsO+i0!Ob{K2DC}VwL+qjY zn2OO49g{_eBPIauGwkv7F-=AP=a?os7BRO6JI1w#>T8;Ye&m=XIvKG7_+R@TPhZn? z^kc_d(V2)XJ$Pe@y;PQI2725vUvw^_9)O@>uP4hi6aB=oNOUpcKo5GXYcJK$Gz=8 zG5Uk!fT%EXOHbZ-Vjq=jDnoyCG>M8L>j6v}_IYwmOVFPjheZ=25A>wRyY^9oOi+3| zj))aYD2x@*<#1!`q%b({l7T$4SwO z$m>1X@vgV2!6xX*9j8R=A|C+DY=7G`*t8P;!*NEmCGu5I-UMPlm2WbmL5EM|jFbc1 zH0<}}n^vJW9p^;#ktx0C39kLr5EE4Ej`O0uk=Z~y+xL5hnAV`T9T!Dg4@4IBvP>Y} zp@y0&&^wMxqQj9B0Dl_Z@eDPsMejMTh>k_h?Zr-Ty+aK%K^^b7Dmodt0(fZqJDy>t z_2}P@>!LG}TYB*(5(lUPQziP)aYJ-2vK|1b;ee;Wv;lqOxFxz6d7u|P(RF|tZh{8i zaaVLD@)!`)_5+^brcLNm#{<#z$TJ}F5${rkrYiKA;~&wj$cunb4expiO`FjdjwhlA zk=J{%6J768BTP{GJD!W4L_Pqf+WxL*glQ}K+VM*CD)JR*poE8_O%@cXM#aJ?IY3r} z$3vT}C{`^L%cD{nxW!n#Wbfy|qQJo?#jM@Uf zqDnMTdXo!PR;P)JqUr(E8k#(MQ!N@<-BUav>Odwv$<@Sq>21C!K_Br{Ah#ux*e3T# z&30dmAX_{q>U3t+B;I>uk*3ZUCm0}J2qRRS$FviGHntET7 zV5oR4u-$8u*!SF{G&_AMfRm9|a)=zQ+3m{^j1lhv`dc!YJ>(v(dBfLJFhTrIR6*~m$-MW;F`5QnreL!8eZax{ z!HRH?(d_Z{5lj<*6jjpuG>B^CSj}EvmSC3nQ{cnbCbRFm$7ki@ye7oCG3;dz|Ji-vGfP@%K^tdlyWx945zW_W5!JOT|9}K`xoX9(Iq{ zyzR>qtPuYmb-H)e6y68q1kHY5zF@WZPe981!M$)#(7fXtDp)7}E9zSB)8NOE6Ez2X z1%i#@e}OSyo5FtJo~U`(S18yb{x9lf@0VcPl17ckM+saz^Shlvmd&rXg=^w5F8K>kFM(TvY2;- zEY>vpjDjZdNC4MKAceS#H6QvW3l59NNAK@bFx7H|oT@qED;69PPX&@)GL=2zo~rqu zZ<^qkcuw@`K2=kB|0Abqj{0T@PKXx*+TITyiF=ynBi}5+N%8XNYkf|G4M%KOX{hxcf=40Pn!5Q)9=$Cz7PURgXXK1`WM&J`y1N=?`^~60xbKEyya8A4< zI;n5LG|N$PrsjmNRB&Frbr0b1l4)kGWv1#A+oJ04!A=?4w{lwdQQj<7i><6W%*V&{ zZJd_>k!6NDGl>fb?V+1R&x zTK7l15>>0sTpcFp+xo^$&p&1~v|u&t~93mlkJeJiJTALGqcowik0KNNRHp9jXC|FLDR>Py?k>VL)eqp$XDonHAd zZ=ULmt*ZK&_;K`IfbOkeL#n>AZK?hbw3+99yQg=5%wtrn%>oRRA0q||pYOFWD$ZuB z7D%Kq30e6wD!n|D%4c&_izHDoeX?fEXay{Uc?R%+jom8La^BRo~d^tCK;_*_hQm zqx(3oRMl>CSF0rYm~B~cGxLGutG>1EuI?cj7qc%be`e(g-a^i$Z4mU9Oo?gEDh9M~ zUWk0B-7DxPnH|%bRWUPkBD4^>pluZ7NEXDjXEgxpH!rg0A^WOxB}-$jWO-+neG*s{ z{2tw3oiAA(a|_^q^C!&W;6?O6b%A7K%#*Cp%&t!Yi-SL)o@!cRixEN_klDhN1;Gxj z)=Rd<#Pn0oDr*Up1%E^jRU0I`V|qe8(A>f-30^`ESC5nIi^=O}p4HV7SQ7jRZLT&- znqrEeFUb6qSsJ{I9uf59eHc^R@8~S%Q~y%#XW#!ow>}O9!i8C$PmRmCE55L3^m$B0 zzx%TSpZb?^zxX~DFcJRjJ~OW1uK7L}R7mc`T<&*tHuIT( z1^2t}3qhsiA(RjoW_vy}uH>%!!aCEln8*F@&klU%U&;O9`wCVU5NIW&B~+`?%msWL zxUizwsO+o~rqyreg1)l?mqZCQ#kdkrt8o=~!xuK7;$w5O%S!^S{#D#f-!}rcLzm~h_`$gcDm}1+q>!ABEuH)|eeigJxmc(ApK01f_!oQCD%lDh0 zRkA9!Gy4LRA;$IG17BFp+7SCV`~IB37yk9!-@ZR!Il&ss?=OXR#8}CNd_gdOx7Nl+ z_0O8ioc3384}CWU?UG%w8U4pWU1Hq8{o@PUTyMqZ_Aj3sIPKrSJ@VZa_$3Ep_5JIh zKQV6P9{cVHE=xX)E$)AGF7u^-BloZG9@zHBV@vyAfI`K%iF@J;3t*qeR`kC=H}Iu@ z6Zh2jw;%v&LlHvEi#tPC9qhI}tiCB}kKNrrbKVS4#11~QJ*vJ93UU)vE{$jC%?F>` z{;j?z>4-hizieK|8Fur*7q+L>e@g5l!Uf7iT@uh^{zU)x?+zn1XgP-to5SlV(B)Mh6p zCE_B~nam6j(GDVZplDz|r$KGg$kNt>s9oTcNaNxLsLPlRmbD%P^Tw%=wkm10gCRM! z%?cqv2n}|eQGJLpa$cKNppyt=rTwAUX=B`++HMuaW?n_NDs+64ZavyXII+%Yv{YWI1n zO%g2D8SaA4fRas|rJN=y7VjJ&T@-gw?J*h7dYmR1mgo$ZL)X=;$#s^hF@ZAY94cKG z_dp#mwV(CWm=sv5Gu#%vQuF2$ZIsJIVk)Or>Wq`;Q1cCK9+ydpr8~p*QA!Ry-_=Id zn!vJij*;$-%g$lux3_s}O_5kHXSh!)1QVS2n%ZWH!g@O=OAp6Q$nnfKeC^q0ipKgn z!v)ja9Cp6zYihd*#60IL>B+biIf41@UwgKjVzK_taO1Qkhqr(@N7b3)upH-n>AAT2 z9BP5#oTttdj}3H&E2slG^a9s8YKIA2KIc;DmAGR$%!2lFo*kw{Y_K!jNu2=!o%n{T zHzi?1oU5g`;x6WR78t(q)SHsAVa{+#bv=h&;QEHzX#)Mvxl#Hg?mj7?H;#Dh3TB(8Y^W0UFvG5cA3(!knE31g zOlf<&XO}4*8|4i5T7}@f6W>z1O&Qo|=Wc1w_z43%rG{@kyG=c?vCeSOHFp48>iU*? z!vvC{bDuOXe#L-5Y5TXHH%z^-iOz8IwPgTrA#t8+FlAzsoK4cA`1%3VLc@7agQ+(* z#Tl-`4h*0dy3SL3OyDItk4UG*9~;0dY(MYWW9o}dcZR#MGoafO-%)!_S=dbH3F)Hv zivv6h4c~e8n)+e0o#C?V`T%yJ>pSXA6R3>NQ_^+u4+aDlwtwe&)6^fE=R70bx+VVA z0Nx_v0@Y|zVWr5Uq!t-2cp6PP*aBy`PfHm{FLGU=-ZFvy=sYjo8=pOpS=4^P z^Ok8Kw%ByVmhZ{8 zHF;;tg;%AY#+MJgwut@S{kCTC*>DB-Wqj4Zmy388$^DxAvu5E9>DTdf1Czk1bnn*; zIUDZezK`EOuwb#}BKeMH=-CS4UFpy9M+cTHW-q$m(F{8qF6n-cKRvK&G4BWRfHlWb z>3k^dj6XlHadG|+mIEqi?VSHg@5f&q*t)n9_77Br_A2Ky>Ermj13MPC{$P1m1vQ@Y zKk3W(=L5SJcmKfisA#(dT3CL<^CihEFz(2 zuCc5cioT#0gMUG?goM0Yb6J-^a4@LDTuxG!mQa-IF3W`GFQ~`1IiqBK5~k&P%gTNX zGzG!_bjHaBBrMAHmo-Bb7%al-ok_Bx3F~r0WnDi8-V4HFgp&d+>dYmVWI{I>G+?`( z8L}}6dvn!G$}R;C1wj~fX38cf9L_Z^X@;^eI0oD6%#zJYIGJl+(se2DesC<-=v2$* zC!EW5FUf@VFgOm|=ggHYO}LWlT~hW_;BXKeQ)j+xb;7M&|B_~?6N3}51I_~3#)K!i zp(R~E1wIH)#5_(~W=jwbB9~@DzZf)PO-{XRTSCkr_0qD-f#x7+sZN7zcS6rW#-+_r zI0h$Uhn?eO`x5d7nU{854tyA#f;BshvZjQhLGGoQ&_D)@u_Ml6*^z{4gS<=2ehwT7 zg30QfAv=+(Sth%baCne$Su>QD!CBZzXSwWZ!pTAAWnEVS9|dP)t$<%| zB%B-MUX}?hX0QZ1<*bn1O}H}1yR7V&z_B3svd&7`KMA)6`Ij|Atr?t)opDylo+msR z6k68xOW@<+JdAZ(WN4x=k1WrGo-@c`KBq&R7?YOejKwrPz6AgRD>Dv^;Nl*{^{UL6CBt zjj{oWi}L*C&CrYn7hxBj`(;BD*MX(m^=sghAnd$2Ju)ra2juchs7ixn*d^y7*_gz= zdFth5R|74(WvutwW;XLE=X6R0XOR+1?qq13vCqeA(x*GU2xD4xXdS&wy&*iz7 zXF{18EXS@oTVzWUujF}`m;Dwv8C;HCbGFL1u1>s_=U?6o^=fbhcHMbewlVPu$iQ8{ z1wISHt_>)RwnX7za)oz}v6VBQ4Z`wGZDQ2mtQE{Pe=E1@>`l;xcO_;F9tZua@pEqV z**}FBWN#(r4lZ92xaR+yTXXid&@Vffs2^Mhg{<)uS8?`^@UrZ~#Nxq6S1`Z(Pnnim z?m2&vc@yVDL)-W}{l&p`_P?CJ$yyUv5AIsg@jLs)!S(jPoqxb)o((G7xa;n9mP*}2 z;Z4~$iSEG}D~;E^r>z@wkA$~nKR}18U+KQC{?fWp_pk7t>=&?&@<@T3t6hiV3VA9N!wxeURQs$Bod^f)lw{05WHtu9K5Mm3g2&4H zLsQ&lb_dmGtxlZ`Pm~V^eY(@^4|>m9Yjg@cRXzf$WBw}ghPut_(kbzD`KTm!e#R=} z4R4#ZRu_r)l23##S-;ACL;baNn=TsfE1wR=b?GYq4e!_1?YdaJzkD8)%Z^p#P4ziz zoh}|9C|{i9&p))vc+-2%xei`noxCQA9Fnox*y(Mz?$V{>qvSiGkJhhtcdEa&?$-6d$I9OX zue)@$ztj7z^$lGwe4_kaD5xE)$v@TStqr=~_!RjENyZ_ERvZ8Hp11DN^~I;lKZeG- zZMFMP^>^02x_HW_7rmjCePyQ8D*!(r*E%gO!qb>(uApa)GJtSj| z@s{_3^)1~%e6joo=(P1~+_%)%-Z(5o}W$s!=~39lg~{)17$mL zkGf(yip{J!A>Xa z|DyT$>|Feed^7a%FF~g#J2c+24DORxCrgJVt)uU|J2c17&d1NmcR)d3u+DOy{8e+} zY$<+Tz9%_rSjjr}zWZ0rCubMo7v=9jV_yXZK6zEsa<&Y=B!547+_3$ydFQ?weABkH z=4YtzXANszR|%_ls+0EenqR@uUplN~UF%<#-&CL3SJeCto&LsQ-Rrvl;$2g<+RZgV zd0X?zY2R2A)&Q;!Yh7On8-S`a_Ntm^unBN? zSjYO-zb${LzOrwr33~z0hjp*-{+kz2v33h+VEhzuL0n~i$P!R-b{nXP(v*aP{L0D@ zFR1d_9W^3Y9_Ujrqp~$*398Q8oi$-|U}!;QWp{{oL)B(?)hHDiDPsy6EAt;(Zm7Pt zZ>tHb1hWcSD=Q!JZmQ1N>uTa*-(YD$M`i0n%T3ic_WGKzTd=X9yR!QsuT#}-ch{&C z`jl-2aU1ggv2?1wwePM8%Le-j@;6le!~0Wp-ri8t8&(vK6wKJr`j6#L)pz#2HDUYU zR6*s2?tgfr>hDC@z!)I)4eQdd-`pJH{W;_@SLx)#x?0(F7blrm;`cVn13l6Lu$-4)565`mg1l>KFU5nmOP&Y#iRbvHM@% zeN~6u3xTUGW!vz$P5DnO_f@~zPt=5^hke8IH&s62{iV8UZ>d=V>lR0b&)C%Z#PXNw zH~Yz&umy2yc;%+3sh^r*Aww6|BJQPh3~$+Fej0j!T-Tk( zHz*#ZbPd0}$@|m{>z=x>C-EYsd-&r`p{Jq0k${fHtq}Z(!l){Bw;6UgbUwUVApvnB zx60fd3L!UiZFsFBGBvZXxXRmYeu&)Eox^u162YXXs0ww59wME(c6^s2J=IXyP^Erm zhP4mfd3=weFQ^qQRpw`*e~??c3;0`#fvIJMm#e(b%&=vuyNJJ|7zVDzE_KT_^sn;*HeI5ye})FU`-8N4i`1_lmc{=&0BddKr3#Jl1vLKPui!HH>K3 zqW;eeD=fOZ_|J;}f#T7!#r$9BIr2nzAOBVHNov`M%UitvnO`7Jbr1006<>h!@pwz< zzt9V$TNlEEinFP%5m8&!ugtLZqWcH`Q}G>0Ah}!3uR<@8=eo!E9mP+n%_EApdS99U zLtf~f;D0H811qFrYv@(zKjfvZ8-J*{nc6<0VXOMJ85U)9&+&g1_dpYA*=l|rdWF2w zy~Lj>9;J4TxV+W-+WZ=Mt$T(4r+5M0$m6Y{*P+)4PY>@!kQhbKQ5H3C6%R%9;J%X* z5J__3p~{Cm6qZQ{5gAEk(!~}pZxw9s>4gNWSc6$oVF~dbA}C)^5K1zgGSCecHL?oU zar9y$n(PazNsGmdJVa609w*|-fm9iN+2VyAF%;IPiDYsZxF?S-A=n8*2|dUfq>ggY zQC2m&3O0E3upLW|r5fp6kWn7;QIPEkSpKA%>0+xFT_r$)$r4~SQ>}Cb*eMSMs8k2PB&Q9*eW3^)5Ck%34o9YRRL(Q<9aU+7? z(>0FVN9BRx0xtxKZbb4kT}HBrD$=-ZnPBJNh~oEg6_ZD(X`sB6fu4RNnxEyGL7t!% z!LCuWz$Cd5!_Rh=kf*41;J|bVf|47td@zSeALZ1L_DrEkdLxbx>MVJl+6z)lnJ_55 z5zhyAl)OY8hV`Uop$Vc3ALL2$Ds>VpnJ!^ab|aCW?>Z|FQf-<#JB3e@`;yv*zyi}9 z&}NR>89dl@FxfH8wVn)7H#F_`MqHz4N^uN#g*}>oG+p)%_)wZs9V1*@NuCPT5{@`R z1OHbJjmr*ys|c;P#7G<`Z7RKvs38@w1f_Sli5#UZO&>OGVpVEwDF_nBm9}*KC}Jm> zs-m?H2Pw*lut9YX>P;_|QG3W?6djMS_0U5zP4j2r^7Eg z9%<{PhiZ}>p|XItLW*;uY?=D7oHI)0)@D>2#mA#;z4cRw56G<(Vb@Au?H1=m+xqCk zy3TZ!S6f=`7axzd_0>-&J|^d>SgoU)l;p(Nvh-npXR*q!Jp^3*c#N%|em3zLxf0g1 z+N#}veQnwLu+Xz!71DNA`vH5}`s?QrUy)l?JRRRjN^|0DYJJ%7seymI8BU}0c$_Uq z4@ipK2^(E{r(2p6ZyTTwD?e|l)VflqUwS;=Hc-Eq_6aacD5GCV{6T&NFTC(;NO?|@ z4JLK4guYR^bs05A`SBzhjH<+K@&}bir>}9#bCPXC^fA3X4C7#E>f>FqyCV~s63u#8>!z;#47uzG5R)_ zTbYw?E7FJMq`_%seW%N>Jf3bFrQb=UD(fS*=y|n7geqe`*h;Q6Wl>tao~kuO9L$(M zn*WBYr*cBt0X<#oics~K5Bid;k8)btF?c?pJ>p=G`D6KSy0VpX)6T#fCq$&G=X~&) zTmzJg(k|*fwT8%pJ?D?-zwOFXu1LGCXKP)Ns$TO!aB>Y*u1kBM57f3t9_%%LBL7`i zp>j*wD?M);5v9tU55|*At8}KxM^f7iQ3o^UPvXDlDpJ;`rHrJvxuR6P=Ys;}8l&8s zmJP2Kv_~E6J%0-S1J?v)qpDzJ6+C00?4vDi3p-Kor;QuAe;XaWt54by%MsU9Sk9X@ zvUOW!G}1Q>Bq`TSWlP%9ksaGwqpf|@raO+g<|t36Z5-LXtvedYN&{;O2ya{3wvnB* zI%ZWCI#YjwSg8Cit#M@Tc5}=_(5Un+#1iFCY0V>xw|iq&^+RXtPZGSihG!Ymg?JyUCQ)yLs3JWI)2pvbfNw{u}9fA zeMV7BojLv?$Xfaf#9PXN>19Qi>%8%+2BM4g7m0V2!_q5@9@mB9AA-H5_Y((|TXpHK zqNp9}gjKod68$COedXBn#-iLE=7fjfbLlS=6YIl(mO#!~(j*HPuH^pi#A z9bMpe-B`x&aCw#U)6W&TcVs4-@@|y#uew^4OVh6ud3Th7@O5K3|C*~+xjOw8EI2kN zng-v1_a$7Xl^fHa6oqzlfdO`7B|iX5nznS|D6&2?$&`P?%)jAkQ*KL-8Ktgwo%iSa zR<+%9)yUlGX`>AFjY*mzO{*P$y1r93rVju|tOJ%Lo7OmPyMBPzoV24{^>N9Xp-mNz zJFcIU&FPat8k>>K4Q*QMxaays=}n(Ms=dB3Su?C@o#QXpZ_3v6)uX!VJ7BM}2`n<# zAMjj{4NmL06iq=>rQ@ONCOmwxdsODm87W*r6KG|w+e&|W(?MP<{RbO8JDUlY^riRb3IaarC%D=ymLk>SJ<@K@xm1r&u)xr z-`SX|8PT-G@t^C3vOE2sQC&MbV0*J^tK+rnwUU>C8VGkBrJkm=SS=1}TAIP&ZlpAtCYwW0D~X897+@%Kcfi7DlidMBEx2l0gUcPK(rB9;4pD7n zL}tchL$iB^iqke#J0!KSAg|3gw7VNs8eNmq0bnX3KV!9_%iRGxqD?gpMQtkBY&HY2 zD=tl=Z*n=5wdwH8$8JOBt{G{ZzNyv`S=$TrwkCsNS7VxHWYacBbZy^=85t)GWxG0H zt+Z*oBepi|yZH>RU2*A}qNX}WeC@!9vW!cH=3O(=xuT{Wj>Ou*Ai&))wC`$6*Nkea zcO=&ii>S=_$I!K_12$2cb~;jPM}QHBjwW`;WoQgdZilK?2d|4njLzIWBZD(E?Q*2o zjsit4ZM0!`V}@pQ({4wP+OZLh83RU_?e2i()uuNby=ueGoOZNpcU%w6n5G6t@7gI5 z%^8zNH}9U&gB#Pd$MJuP&ONT>{g30_TlcM2t*qLmvaVJtD=TZ48bT!up;ibY<5mtJa(iHeAns|x$_D)0acd`4{wZXXhd~+u1{eCL`{w?CLQIt zwnbE*%$;Aj6)?N%#q6WyuA$4SPv!a-?y&50zP>o=XdfuD)u(e86l%D;_Qk@Zn7N_L zt1EH?3wK+NIrl8CJIa~cw!HdG?!rP1h&Q}gb+mkL=!)vIxxs}8EoYpk7LOh61FN?B z9E`SQTK?y(wM4HBN%D!zu7piZa35TdOPtFhNsW=&RZI=4X93PD^$%VEgw+!j#6P;blTx@jIa;{6{5?nmntty%eFkj(uOK+Er zOIpi>Zuphl3wp~6Pg!OLxTGu@EK|8vt>j+RTT!UN`wlPBI)-q^$8f>sEv&W-b17be zJH~deis4?;TUn^#{mw3-!J`JhihEgaRpDjJ)h^ed@}hFDTE)Ghx28}706u`=6~Y6* znhOGO;dRR`F3*-!9%Fk{t>#|UTVJRl0^craJtp+Pui;+P+faDda<9whC4D$gh#`VE-|B0S?lU77*48Pqy@lPt-o$MU>O#L|l6G%x8o`c0XJD!l3v zyR_wamKSP0{gy1*sNeFYOY+hI5Rs+p>9=JX%<#TT#?sm2THdHwdb2DQz8=e6#7nU! zXx`FT`W=~uJACR=v6Olu%NwGx$CGVznkz|s~df}u9iAILI{epr5WnOHgiy0df>{h>^Q zD*kerT{?S0i;Rk+x5~1NbgcAUO(U@7G_o{~-X_zqi)OCa2x@s285K`&m*pB6TUon$ zM#O?4Esdvl$TTpcgDW*6qg=}e#ih$+LL*Bn7uSf0igKEdluMV(G<2hvYivYId6o|< zfv%8=jqI)bU6UgQK&qA|(3LU`?ilQv5iwh?<%>$Bt7N4{1S^)SI0Ac;<||F4Kay$q z$7QY+5!91ezNpRg$FedbcdJ#d^%1e)VM{mDpU5;oH;C54p}p%$|(w z&`Kh76*Mmfdz)80nt$*nD1Y*ps!zKnCK*-x;I z08nM&GOG&Lpk@41Gf)yD^ybzTMp@OmE?HK7isCQWP5{cXa25FA>y|a2;`?jwAiT`2 zFI;QYGJQ!43jomxUXwT3in-n>jV(uboWj%WZ%wWtppJ8Rj%4K#)uT4zqAG zDCDP>QBLy%w09EvbDIjcS@pVJUdB1y93a?905G#~r`3S#-DUjKGY~K$4CFQ!@~lQ( zA1^CEO<5pFApogaxCgBB{$5<<&B8-gv#vju z&77WDpq)w>%xx_cSn16*UXH1tPz0$2U^xr(;F{llIi-S6(dH3`b7h5vR_M9z%Q+Rz z6akL_aA#qG&YZcF<>?iZ6g5w0l&Lb3S`p{6mzPzL19?DuG9Mcqv&!_C%URx3ks7E@ z(|OPAG&*TDe{Rz9-in4m9`K*cXGUkN7S81_pRSk;RD=G`>@updV$2mTN1q`F@qhp4GVdIkTTyI z-M4x>=iBnhGh+*f_6`3}ZxE^w*>eq6AkUJ3DIK2Fn=+Ka!w$W|^K2Zoo8PXMjTot{ zq;s8DL;z$OwZDBz_Rgr&>f~I%70G7@soneAzrg%N7kndTt`Gxt8g-z3TBh;zFU^fv zQGYfrxcfl+SJ_AC@-@y)T+s;_YSh8@Z?aENmbx=HeZ|Dt!Qk$L?f=UDXEX?Pirf{Z zz@Wh=x$u0|bhpOtB903YV|uBbm3x2XGY`!Cr~qZupp+|d=CfU-tq zw*Qv>f}qv+xzj5q&J8Z=&TRir_Med!TAQR7X$ov>R93r|TpI!cMkI8kXJs6%JF6X< zEXI)BAUQ`y0PGqiXouL6u_@Y)E|F{Th|st|!+t2i!<8 zk@b~vq21Z-`f?M<68ex5BRc_yjRHqf4jHXLQhMY>{@7ST8yT>{K4v$~Y& zQn_G=!`G~U7EH~qo=!>MtsM_jCR--7} zn&n}<2K@*06l6f<`V?i`u)K`dqi>T^qeiQ{SdB&5wyb%^o6rv-H%dk!!ti1)6cqNB7k>FBTjn%*|Ua((u9uqGxJn7HgLgNO_T>Dm2qg z6pcAgSt2MSxaCEQGyw6bXv%s1676FI_q-Un76FV#Ll@Xm`B-9e|jjYs#LZ(b@m zL711fQKVsxe?-rmpMiWPft(j7YD4G3Zy0ld5+Nuj_~da#8VK2bCFKG?Li;4aH!o52 z2wmppzLImHIYMxfFh6gLNJAwDt>j;rf!-UzFE2^djjnZDva6wg_&jA6$DCNs%Qu;a}%w^ zT%;@)R1gC5(nK1N`P53vMgDT_GlZbLbkPTNuiNF7oQuuN1!o8g^Y)4~Wb@sX{EIV? znIlm1_#!oW)a~)g@{5!eg0qC+yaOT)>fFDw`67RX_BldGUWVv9dfIJ#W#7f-6@qhw zMR|us8ut0e%9)EZE3_*Kw7e{lwzZzSaSW!85-F%8gyv<7G%&P%45f}Asa-{&=Yc(8 zjdpjB;nX!p3aSW!i5$(V z)mhBa0M$#~%VYd5rADi3b(XNojNPnPy4S_TTxy8so!42)(h%0`-J4_5FHJ_Pp#;Jz zhtswMcU4T;CGtuhRQ6aJUDR2@(lFQu z-DhH^FHNpgLp+34X&ee|0>moxWpWI!PA7_`fwGI-F{}J8r^cuuuga=5UTl5TowO?E zazhO7lFmw&hSEOmPFa1EQY_{+n9ZHr>8FT1Z^)o?j! z)zIbPRr>WJj`el-El?z>TNQt07&vZGto2>@y{jfKkF6TIGQ38=0i?5b_xx4JE2P!& z^}_&mixRCLyB}XgzLKH4y2t$jL`>>d$6p-=(p!{l-S2*DRqBi-++C({EV_L2z)pIy)4b|~^9dJvR ziG-scc6~YE;?>+dxv0n*Pb5&V#;MSdRgIygD)Ok?5#3$s>Ao zT|H-Q+x6rEc5t8=d+*ZTm1&U$5h!+NVn*Xr7-!!VYi0p-N43!T@<}+)mwSlUVE>?Pl(x|K%cqRrTR-)vSVIN!9d)06Kt63eZvDce zeoZX&A*J`}2j$<4KU=@?Xjzj1C_L%`JwvYP&%E>KTvPD}ZIko?{gC{J@i*&F9s_Gy zfQ?5zq#u^g8vn5V>M^lq;E&c#(ued+`ETRj*1tSv*USPek7}i7$+aL^qwi_T!8Xw1 zq^)#;T*m~G5}sHN73g_X8$DaDXJTw)?di#hg)F7Cjh-VnFadPHlgh~eoF3Iq&y{N` zH!hwLoQeioytJL3CpR_$zT7jG(*k@ws)L>{H#Gse%QKlX(9jw$?VuOP%}pF_f;}@h zvjE(qWOSh%X+p4Jd5SsMMjBTtqZi68OBVxa zNs!GR&k4>zV{3v`NiUH*nuORK@|@+&0<@1((M#nxP>b_CP1jQau3A2%S zV%JiE;zvEAACWtoEU`J}>A5x*BAn7k^rLcDljSyNJgI9l0P9CRrkBY{Ceb$MJtNju zT&Hc8KBga&yPK@Gx#}6awgnPCDB%3%o+j&TZh9uK9k|}QS^9*2Lhfy{(dNEq#@gBI zT3b*6{>#ZGTpPKkcrEq@ZHu&%ep2phvc=}9XT@47RDV!U>8Iox6X1ns{n}V)e@dUy zPs;;LQf%IMwye##p|utDj9wwvr~&UhJJ(j+ply{tqo0unne4Uslc4jyEXSQ}05+Z^2ELUwYluI8rn)pJBRv*L3}9cv$^iIWqsGJx}EW_hi~bp6b;zi z_1e3B^48eSq1VH=^;1DwZuiPxkGxGvi60nl)=w)MwR!Ay9CA!aDMJIpphgsp*|a;> zu8+7~nG*G;{jPkE$%M@-ucr0Mw+BX#Hb+J<-^j@7z@NW+hERr=B*f^C?0F(?}Bw5n0=BmJXAuC~j) zn`l&MO`+b=AIr;3+-+BZvk?njTj@Lc6ZvryZ`%#tEwLF8MMu4-cgo96d~G*_#8GjF zwp;q1{#1U-B*1p3_dsk5P^}ZbGIR#_d(}5 z>w?KL+tc2Z4e56$)72kzdRZC~qz0N-Wp~MYc;h-RSeH#!*>*Z%NrA1!JRDpL zmGXU{*Ts^X6xn)fd4rIm;#aA;Bl^msVq3>~Ase&c;s`&*9o2tSbi~$m-f}Qh*!!!d zxbO6z6dkwqp0{yhYm0C{{tNfL{?npUwgK}}Kv_}kulmCMNB_^Fv$i4g4sX=Dk2rvz z=8oxi7ggJa%_|0ng?*rEn)^ZjdC>*i<@3%$vsicl|CKwg|Dx!!?do~gKx$DPsQSwN zsQ*{dHQSBz9&F6Ik2r|`#+}fARdn5U%e-e`xv&pbedB)8e_eFTHf7%1jji{E2l4-M zKkL6Kx@)_4-e=HWR0pg6<^E6q@1h5`hv)s?sPzDmfuG?{>JJs|*2?Q}R)h$F|4jd2WgYF-H2G{za~7k)4@G-IVb_>k#S(eOf+d z(qnsmUc{yfFlD4a=wIdICNFKT&WqjD@*oRt*682l6DIw(H|Hg98US@h`jh^zT$3-m zKQCj`>;tXCs9E}qd=iKP`8@F^EVwk%S^9VRl*xPBr}HW{Q6FXL0#Lu{v+@~}Dcg7RIyY5-Z6p0n|0Vxn^3C?syn#(E z53@2+|IvTTXH93{V9^G}#>lo#rfoHc>a*AVQ`2uZSB9zC(b{IeMc2Ibl1p){G=R?5h`6gDvt!}O9b$8$QG^95!^fXl@@F$3hg z@%)aNd~I_gHlHHqVKPU_kK@ZbCo#;i;+=;0wfUxC2=0c)W@Dnk@1DvC?X-B zDK3OVGIuUV)+`jj9dLe_Sc1X&1abMY8KJf%(K(+jmSUWJmT=2ultO_e5fmozF^s3r zI&QO!U#M+Gbj^^m)uJms5%aHbl^%#5Xa?KK_`zma&=_kxm(ivZ35->}!)qRwb1V;YLCYa>WQcOJCQ-NL3ve zTq6)76r0-F`vxSYs>Vu(@FNTZp%`gL@C{2WRgpmH8etlU#ArKr-_?l?swC+UVT5HM z7GvyueYYfbsp`P-8i9797;6{gyEk!CH6|S*j)WVOigD2L$xlQ+B7yWZ!ZtV}cD7sM zdpwc+DCx+M^T=X@GSmAeQNFc_5nzEuxpXX195eN_TkG4DnEYt)NViMJQpE{VvR%Ay z3sv|Cb2QYoCL;f&c)nedZ%<;~BhJw_*P3Pdr^O5GQhkTP6DvO&I=5zd{u%K?JHGE! z;@G3Uqiu6*R^*=(FS5%5{sHkAQx;09iOjD84qxbNxta8sQ`SbRiOR1Lhuf9<62K%Y zFAH_6iOxSSUTT-EN8X(HxT~zuEoWueMbi~_=RhsXeT+Qj3X}{zNwqVi&e*}JQCIz{2|uJXNNayJwY7D zdnRxU8sYF&;ad!)M)vV4&xExGH^dqn?d)dS6X9{ZSHe1jo8s+unJ;~>ZH{}QI$q_K zu-@RdSmUQXfJvsfB9u2HR?%#lX7|pwb92QL+6k#QV}n8?tbOtw*xd3Y>jY{ZW22(Q z^nl%0--*owPg+k%=P@=Z9zg%`m+$Q6S(tS~kr{D{R?}=a1vcG+?WC1U$&7e~#%ME} zkKID;%qmCuFt`etsR&NVJh#L`&|m7qNKj}LH;4JuEg79!CsDqPM1{)qh@H#)h%FVJ zw3AX_#%6`adGnedyQQTw3n&xD7DcD&DLeo9$y)|GTTe>oGqx%;mC0a`;AT6uPNDo5 zNs2DhYCG0^@fPe;+9|0YW1FJK^n%?ou;QpsvreJ>8QT@TrkCwj&9C1Q3p)Z*f5r}l zCQP{jG`S4e%RvP&k`;ZX*X=gX@7z-Hly+Jgz}TtiH@#)I6TG>Wr&*^_3m7Si0n@v7 zd*)AU8F<=yTDpL-OQC659s-eW_Ni6{io!@$44Ssv<;^$UihV|_kWv^t#jvTuP6B2f z^;uR0Dv*(;7&U!tcWl1r)>xP?kOne#D>V7bGoadKJkvUZ3Sy)y#!P$c&d-n7TJel_ zMjFJ}qZl`RX?GReyOw8J0NF71Dke<(?QYIb-a7EC^^9~OW1m7(%)AdW-t052vnVQq zub2dcQ9fV175gXctdz>wub47@Z})V5#a1fVd8lB<0mZcGxZR8S^;=_MH9;E8IH>pr zks_8xwmq^7RaDm6HOVWwtw_VfLcl6t!us(1|Df-rM42m6J7{G{ow$tpESfni89 zygdVuPINc9n#a^F46`(Iv0v&}p5)h^TCLU|EHpq%NcPcw7m^~o>#E~fBM=Ogc-pV? zyOosMJqFs~NRa^sKqjuABB`{SR1+UI0xJ>{Kl|-|FOwR&lfWY!kr-el6#F#4_eovd zbv5zfBhVO?Q0))+eM_3`9s{v(q{ILxp#hL+unpNms*Pullo}8uOnZ@^%{FpR5}1Y~ zQUhlR+y023>$b?Ay4v`~BakJPMA)D53)q(0GY0D6$Wa55B+|axFKkti$a#YZNx8kR|Ca4ty>*x3SB^lIRZ?Le&J=R+`n>NBP%ok9bjeIcinMWyLD9 zdi%BhP1}>fqwHSQaYeDltN}PS*>>>@+?A-+9rcQ}W=;0n{Cl_8gIL+Uy5p*1J#2>X z{71KUzNowswWi~mVuM+W{eJ)H?Gs>HcCYF9L$S%M)n4GQx5M-$u0D#>(O_UB?64R5 zTY`G2mDj+@X$(s+Q`(pK<90A#me%_~FE(tm*%SK{;9@4e?5b~En{y*@WL2{Nr}WUY1_vw z^Er+IA<0>Nh(7^$KwVza^p5;G{%jnm&UDFZ)K- zmV~DUrzHV%QUcy4kG5ppKy76_Q)txFJpmKR1Fu?dNVhWnRA?%_hXQ7kXJ2VGp^_L~ z3XO}J7ht**3llQZBu2NQ+AM62BmldU+Mm^g+Q#TnXynvm0iHW!`&*l&+ZfLk7a;6= zCV;v#qhISLYCEG>p|MrZ2Sn_w=%?M3ZfCquTsB)h=W0Og&X)eHo2VU(mkN!(dNUw- z=Rkk!P3aECUy5tc0?xc2kg;>NU+WesnbD_c0HIY5?=4uEu8}4)UMa4dZJF~lpkgQW zb=ED^PDZ~%W4OKusNWg;y7iWHC*!q3(+_?V(6TcFw&PGKi~&WnS=yX;0i8Q5Uej(% zQy6dHLwL`eiGco{&9C`T7$$-OE7?D1I$(Tf-|Obvf?dSF^B+hK&6y4Ov2*72%x&#d zA~>-U!5qB>#wnNqO0yu9IF#Qm$%ABseF|lO4^3j?aK22Uk!0OdI0Mbi0v>TBUm=mq z!7d0&;SbEf(j;*-UnS94vrAIS2Pk(0X~cK=k0r9O{AtDKi5zceK-qV~RW4RdW~+I!3=C-{qz2ehBL^J3nXXg7Os5 zgzpZekAP?k%xd(43n`Ir>h8wx85uWtDXE{cZo#dT)Hh>yhxUwoH0YCP%v(iD=^N6$ z_`M?&2K|zzIolV!Olf$NbZ=*)cR%Vt$F$-DYz$pmprX{h<=k&OQ1dna zqvS8pdAYk(Z_Dq89<2G6|4H(CPV<7aU1M+i?zbJR`8WT65{=)R4PZUyL1;$JOukw& zGDo%G$S%^~oCj?gHQ)2UNHoH4^)B|`>PszWa1P||0{ayYrG?N(5_+|EU z&c85zH2QbfL&3c`aD*kl=lovqdzaQAq7@IzvW75_hBcs=Lo1ryT9uUm(r~c>)|}#) zN&|ElFG$cagfJG?hT@$XH>hf@5+r~{Tx^bYq=ckq4I zOu&XwilIo&Zm-Hs0P(ok6}y~rHnkKG7*rm^Scx`=juxdpHFl`AT?!_$(gvE<8z?QQ z8ADnfsC1N>?ajpP^8B8N*s~l!$>=lFXyA=P41qieZ{u3VO5B-F!7R^C~5l z*D{jz0Egb?^7~(v%^{nl$e20lFhkTIUIaoM`#Ku zym={o&EX3^rGl5}#E0HC(Z0a1cpJ8ta+${&X$B>lIHw?>cqev%a+k*+nNesL6R`yp zc%dGpJm!^;P{5HUIuryI@4-$``gzSGd`NT?9Sf+%`?1rMabDj@Gf2}!r-G2;L)cl$ z58lkkj8eOlh%2BK3!sT>oQ4^tfJIHj7to9Iu%?0bX_QeuRK1CW0%ma`ute@@oY7{` zs)@vcuwn@o8yJ+vADw|^TB36SyI6{K4qTE}K1u=4n&?unr1%)tGjLs6^JpJp@`$c$ zc!c>$?EJu_v|bRec}I2UhA%figIyTNPn#Z{e55|AOA3!Pufj4Q6^;fIn^&gm79MSW z9=kNKJk9T2>SJ}8u6uZl`6cYiz`C>;P_ub3$`!ua{3>>RU~^jfyUEAuW4fN<9FSKM zpe|kpt~T$uu2=Xv^PAWmfrmXB-X%R5IzHlM7+c(oO$&Sv@^;;m_!A@0A1`jf9tixF zHu-Ms$({wwXW2UJQ)fUHd-r zY2&FJ|8R`}{zqWkZrS_Nr#_m>d8)V{dn+(y_uzZg)2h=6V1yUH#Xbl;3>|Fk}nf-|nv zl+xb>w(QP;VNujs235&7pTfQi?A%@P5A9DWNaV@`=HIZN0ta@t{FC)3>Kr3PnPL6| z`!#T4_rO1`e@ekBR~|P1jr|okyL%Q^hf$Rbnlj5ACKrQD)3IZ;E-7f{%4~B32eTk- zI(00o3suFSD|5{O6$$c8j~#36l2$PoN{x5!5JXMS7}M%TRWq1Mp}D1lOHf35#Tczy zTFqc7H6prKP;7e3SXMWxh7qO|o7+412PLNujJ0-4YZ&25jhP-Cl#xC=rqzS0Ww4c{ z<^+dcXHh!l1Eoh$O9WND*u^0t=u|r810M$0h)W8}irpMyf-a|XJ~a0T&J)31FZP1! z``vW@hnXJj3&ejr6s!mI^i4|<&3%LCm#yc(#tq9Dv3zwy*w zb)9Z>c&+(jhoeEHJu%}Ay*%*W!!MXGbI3d$MA?%*KH00jq#F}nXCCFCg9(G$JPC~W zkr>0v#W4<-gI4cp7*BdJba`ZzVLiwg*Mqj~=^C#CMSf(p;kDvehr2<0_e_qDy%@SO zvc|9h8o%v9`4FflfipkCF}z-!=wwdQT z7%fEa^_+<7>u%_XS9X{S9Z(CM_eM-q_C62}&jG+u|28 z_lhTQucAQAS3ZIzJhz2|x^)wrS8dm8HWxf8J^|Bs+`X!aaxnF4wiG-qKJ7qRn6`Ip zqVHAPjhd|me-@u}pe@Yai}-{Ab-yO5pu4!nfxYm^UeYHH)c0$)6+AD#=nx4$3;R=f zf9TB`aQTa`IB*tT-<$NQufOeP%??AR{vQsTpwFNC3HjOwX#)iN4WKU9TH&=j;mq+x($JCM5m!J|hQw zK>rUPGVgFGScu%`{5f)&&W4E5C{nWVE)|U;=;&%wVxy3G~UhG6aLZsFNZ%C#_f}RE`8$z z#encn=C2*JJyZ4#epbDyx|gue@PEZ`9Ud$^yie#Pn95np5__M>?g%4n1908Gm zgbc&)#ZwON7e3op`9JpGRSyyl8U8H(=J0vp+kLH|1mPbhz#2jE4~K6HKkpm-pX%?b zhY6X6{}ums_`UG=zCO0rAgYy-rPQ*}anz@p^0Ba2AZ=v`lsXnLU_!<6sn8`rwK1}l zdKSix)>Kb^>}2bpw2hIYG_Zh=11goD0igm^J0n+#u&{J=p+@j4CTT;`c1E7k*uuuq ziyF&sfpP(=gORT^wXk>erzZ0UCR>N39gG5{xrL)+Fg1fe3n>GXj3HDaEx@d!iuqVr zMUcuEg-S~cSI1@43O-exH5_#gCn`XfxH-m9FY`I-=3#-HSX5wB;^i1iy$i8}nPF`O zQCxs2@o`L~KIWIJDI)>}QBnY}vyREseyAVtN3@m1;sR_*pkq39oZqKz9uX)zazY&~ zf*lV+4q;k7IiiN8=5U-vsACQlu^&A}9_6WYrQrk%mSbiS1QPtFQb*M)-6P@77K{3+Bo7O9RyRM~#<7u-Kl&pIlUnhe7SYVZE~FLD2LKkGQ7Txh{}R8vRycYdk- zC+g3Rvr0{u;X8GD|HPNUf4cwdIHz1>k>#iztare48aEcz)lsPog;0Z0F#3S!blg~X zS4WkSVIg$XH6$EhPnVB{cGp0;0Z;%;uAtbH?wXo{@RAZoTrmBBa2oR=w5JAA z4kbq&Nx{(v>ZUm`Mp1LVU}?z-7`fygP)(PA2z_1yZHJQ6j+EfE17p*DAKISRTr5~o za?X(!oP7ZC6*C^%TLZC&k{U;L@R0+guN>H~sJT?IvgD#;B-m~2ujS*RFKVFvP;$kQ z6MX$Z($~K6wih*53f7eT;TRX(en9vY^D*>g4dft7Za5|d_Z+DE%7Mv?nyZGs5(kHr zU}Izp^dV4xF|H|NE$%w*37$AG@U``$^e@IA%8eEe91jK09+-t-1gekGpp3IxA?b<%zV-wAT}4I!exvsSacBcFXgjffOw}Mt;FBrNpSf=Kd4Zs zVVom8-Qt7e^WeIJG5Ia)WS|>@$Ux7KCwYUqZQ9y&D4QXxdBeK$aFKS0|)xY)wZX?{pj zM(<36nm3}W3@^5Da9S9`&zOezg?dC+6<%tAcVdJHGtl43Q@l~#N8v{-T%4AMlxO%s z4MRPu`#8MJ!p&)ANL@zE_l7ARtfGb=xA1aW56dIzkjGHJ)9nl|2YEUnM3qtYo&1IO zUiWGEDGPt69U*-gP0-6wzt?>hUSSdFv^!)hqxXBm7v4X*e};AMJP;dAr7H`kf(WLy4~T`K;jpLU=H~~QbRqa+Y?@EvDoQo2XA$L86T&`J_JjPDH?I34{IbO=rz;_xLroCfP><`r46nCX z>vSU|=}_+v#L4)NBQFiFmBc#T4cQA3jaj0@6eH)q>d1csClH}AAa^XVL-l+x@xYjq;ua-$@-ZdZ_bf<-bw?>v*fYZ;|2j z9YP)xKL`Kq{$I!6%7+$NPTGs~4x7&6W}+rL29<3Vxv(C9KI}OgH`6`YF{JE(VSCgf z=fe@Rl`~Q5j$x(TLhNJ*sgLB@!I^G#$B0sCQR;+W#Dt4R-1n%dj#1?!7^iiE_DB6} z-1qLOj(5r@7UfPpixLla&Q^Yp`qJ@U`P8DqDG*{H6SITgyT5e&qx{pN5=_?I!=|wJ z5;fg1rtG$;by^H{5YJz6Kf0$oJ}94C)Hy{hsy!U>tMW(G*N$=J3yXTEwU7%*{x$fc z`)kKX z6YiR$a4KHpm`VFBoW*}nfaFNYW2fVbyffo|t7fadCwwvdr{tN_*+n6lS-%m#@IMlu zK~mD=bYao*%*x;FUsXR6z8ZckdFga*(ZG@UfJ#Zpu#;j@aVG9R_V23Mgztu5O5Qs?TXZ&)_Mh-K{#OEI zOG?I_UM{+p8TX&+ch#?ipN9XIe0F-f=mE^#ApXPuPJoU{$&}OkMb9!T|6~8>vyS#l z`NQIy)2BrPnJxci{fC0EiE!o||=X7RPZLr=xoRER0{!6nijXO(LQ>Sh}_n+fep z{c)MewEnDS?UQk813hy#bkuOWX=7QvFhiYYsAs`;LARVGC zQ$~7dHdsHn8X8+rrsD%ucs*;jJ8~873XLOZf=bJjv7QYZHZXBFXi0)zoksmMU<%nV zIFGwW;|r!C+cIUMhhf9;JFbH!6rgo|4AM;X?AWltj(b8Y7x+QPWeUcd*^s)#J*U+P zVssk~(#-Vcupw@UdrfN=q(j(c%1jT-rXXSEjHVKl>G~L^!7vd!2)PIMk=7?@(&ZUW zLFt7}MefIap^XW8bsG)SEcBpgggk`%Nt+Q&>#7Z>ApOFoAq7x@MP#F4f+!8PjM#K! z9?m2blkEo^JX6quVKb41IIB=nc8p#lOw8y(f&eMO%?YJsr|YQ^Qz$)WHjJO(h@tH4 zGS~-BgZU%&5+rPUgmSW*V5oNrYBB5xB#c{xCS~`+Vs09EhwSA@78d}ed9gRGXTN+xP?FU<~Q_z!P#~?4^R)*GP$H1s+nzfz>do}VZ zZhdHTb~>z^PC;OX&4KtvLZ~Xc4ASFiFtWs6hrEf~5!#pCq|Y;%g5nH27I_D^J9I3& zSHICD4Z_jvjmZ1BgP}9o)B0+Fy_~c3;$S~HJJcWtY2aqc#f*{>+)^1%6l#-0HrNJD zno%DFP!zZ$p{_ZR2A52^cB8(CEv22fQ=tK{-1Gr%ct__Wl1jU9)uCZIr3P-$8y)pS zz%3Z=a_H)u27_%-tr_)4B!jenJ#m%$}Cp&t!Eq?8We?uPD#v8NA^u^C-}fEXgK zJv2WDY3K$gZle?g)BtghLyzZ>4Y$FG$|!VgO2=?Lp%-!@4KJB<9heVV_rOy3tI(#L zWWyoz9tYV%l_qpE6e`OR8{#dhbE@GRap@P_SZGg9o#9&e=2Z*jn^KPU|3U|IWYD|v zb;^aOB=RRNE{I7VfE3OVC*~s6VI=Hxh0f;88fwGG0F$Q5Lh9i4>880@XyY8gG2zFJ zG{BqDvAI-)0BMP5(p9-gFl^|axv_{gDk1!XF7$}p z3PdRC2!Y8`iI6sUFM2F&770+6L}r*ujI_u5)01-tU}XFVQSVr8F}zq)=?8LW5Z_Tc zFwr9{EycsXOq7c;y3?Kt<2mdjNEiH4dU>v&5zlhUS#Jp&Ja_y`dR=Y|1b5P0^k6*( z>4nc+PjAjmH&R;J>&G5?L^fu=zJ|`uD+8}64Mw-v zFvEbqLg(Z)8S|{C-1SzoVGjX+gPxSv3t68u54|<)Ye<+Op!4&ljn&pu9(o*h12P`p zK^NwsP2A?hd5&@r*Gm)ekLkzr$R^uthCN5uBAQB*@ICYk5dOJj!}S_nhqzUmjPIx4 zf+epHHp4(lBY<+o57QM;0&=tEdXL5;?v|$G$LTNg8UU{y_8#4U0IC%~MSl-zpi8#g zd7~Q<4@xueKj`0J{Obdx$wxOKT1&I=It+t+q^TQ*3rTQ9JLDUT88-Q3(`}ex@@PCl zRw~3>GF%}ZbO}>!C4)#d(1~^ozx-riu6lfAP^(75Zx?DpVpF_bw66?sAY>WdjS-Vy zZyImc<12%HGqN1-19_oN(<-~@`LZplr^pI?AR|40!gR>4XTEH!>Q7`PR4#HMMvMoD zK$fKHM%LmNGs>U}91oZ1vTca(S(*S*HS~H<+SB0{S@@##3ZBEb4&mHa_U-<)JA{9g zYKlbdkg&CbnX=ks;j7Xc_#{RTv}D)LX%DE~DSTb3NfZs|t03V@Ur?JOd{cT4&u2{K zkD0xi(+;In;oqg2W|2+-!rTr^hh(X6sI&twWLOrE%-3S4EOU5lO~j~#fh%B|OR>IC z^<=+8KEa=W?XyJlZtQi)YqI|-B_Yo;LJG3XjT{I;iO{($?ZIDQEH9{pGrH=aM93VK zzQkW+Y%FLsFLWR*OoY>;(ti9cMoPh;`6Gwwg^AETDt(K8z&Ko>Wntte6An zf>B(6gO~DZY9fS?O5fw3G0qmyED9Y7!HMv`SUQe>$+%V!XYt6fIye!^NTr|gZy65? zvfwp@5R#aI_+C1Nf6sVUPzhhdt3whYl~no-|C#Z&pcPUg1o#e#_*MD?|BdmvV9?@` zQ}v=mXeX8a#{XvgF3>_6;Rv+EEQGdHhhV@o7vkW-TTM$8Aatb$1aqb%WXTF~gwVun z1Wa)gY?$8A2YUn`tcf`YL#a8zkr^V)LK@)-P~AcpNi7Mk%;myL_^Vb8^(%yl)P~^A z+$d~C7UBu;=LBIUwI>8HQ-p)aNBC-nth&P-1^<)G4B;$n!C6A7Nrgla2rQ;ph((1G zjzEw}Wr=bnWG-V?2&pIm!IH&nPj77B(jzFSGWrOl2Y+$wsGhjcCKqiWX z7^x2-k@;9yjtYcklQVD^aPO-1R|vOz&65Ls+d8QRA!4f<2;B`7d|SsYOln&&nZt#^SPfdT~=Hu-wd zjc#cWq+%~efeXvxi>A?Px2ctSb?itKNU1EL2yN}-o))8biLEJAEoFHcldZS8566sN zLc~bx2+^zyU}au%=dK#Pj94wzB&%+L9{It2c-80?1P5l}xGaUJ)Y=Us#8DXflWH1P zFGUU3+dPI>k6vZ}W6@0Du|`Fm)>R(SYh>3{nyA%&*0g8>rT}}^$o^1mLbbxQoL-Ts z4c;@FBWqA;Dp!Rpbdje`yk`$b)~HHAso>9yb5Vp%m1p!?*>%-sR43siWUrE8J+No3 z?1pM9s*6wse}cvE;Ei4Y2dmyD-m7Pw?51i5s*ljfN(BF>$}4)k?3QXL zYJhMD!dMgF-Sn)N-B#^F4MOrKx5(5M?;Ra01DgdkN_YgNEKl2b@19uM9o23WNF%J; zq6ph6@8}J(yQ)2?3BoH#XC=c%Vb2EHJ=H$cBw>gpD-zq{=S6Rnf!~0dCVYUFR=sWf zyq=9R5Dic>1T|~4sMEG;Ui2o}1JxnaETPkKs%Xsi)x7piwGV}W-V?OKbi@dZ9hn|i z+bR@DHMK2EF$uGl+#XlkCd`o<5|Lp9F&lG|OpmW^7v@Pd`7K{@66O`TJ-)U>SOCFK zTo@hx0^9k}xwSH3A+#+>VbS6`%vzszZmnEcB(;UDM6OtcIq5@Bs8tAoM}RgT&Hw}-9CfSm{<198^vcYRy#bM{5l3gWpi8jxQo zf8Xu7z3fYiYKF7wjvjHN-!{;v_Qoi1P)oiG3Bdo_HHVVWall{?~Wp z0#Jn7M0^;Q1(S!U0AE;!WH+J`h)O7jInRj-xc+~N?meFA{SV-H3CCr>PKL2grxP|d zhHadpR4lh$a+&*`*@jXfr_vQs4T)XYN^;pmsgQ~;VwvocN}WnqghM%nI+aeRe((L$ zKmE}|w(sThd4Ha-^a2>MF-ftx>sxcw`S|gG$Y}-=s%>RlB zOqBR`gBoSBiVQJH|4tJa?EqU$`b+Stwygy({I8kpw0ZA4e6$w^Cdk_gUi#lK*<jRkOI>}B`e+A<4X``3s?&Pe3F@L3rz{bums8qTv z*u4#WGJmy6rH!*+Vreg2;mO+zCj9T1oUmEtmr**l?7n*&fMx!$S821}FSk_pSCiFb zs9gHPrlF0WUs);U*WlG{Aei|-FoDfFznW6-U&~jM|0sCl|EI}Wn+U&_Que2QAkEls z4IY=Dw~6=bD$RwxDeE2VX@jBiHXE+rKxxgdRM-<@|84NB{Ibnfzn7)mz-n2;UW~zu z@~bvG{N9xg|EdJ^j17ArN z1stncPKA|Z_CE%1%OBaC_lrN)y_~m};zIer7g^Iu5`@jCLCZB9xE`NHg z{rN3#mFtSd#F!EOT9v={c9Ht3%aA8e&X=XA51?>Y0_NyNIff{2%5Qf48#i6|GFXu+7Fp&4Im zyfu^sSv66Zs9Q!#b0h&1<=+mE_9K+R!d_#g7JWtn$a0)@l(+nd6r4EN-6`|8F88;VcvE;%BoE7{W5oXC@>J^~Z#hp2mJUA< zQ{zv8#~HK_rATVV!}CKhjs8ryXGZzRi=-gQ@gp!T{;cwTxK~k%r6`^wKL*p`pH)5y zHw}5Q6lghq0;bzPAF{k?xL8Xqc_e-^rr)1mjj+C=%uOQrC&;qS!E z_>Y$N+erK<$E5K2Eb~ACm%4$3K8sK-J+JV9@@QGU*DQ7ylr}fQp1) zMwGw2ObT8d|1hj1S@Zod68M>=aDV0(V=Sp;ekx`N{&6V)cKlDTYVo z3otk;li!GmqRRPF5bgMt7$TL$@5e}>0xPxUZRDT8xKXqClb9i@Tp$JFj(-Z{NzLaQ zU~}+9VF8XBjI77_K_?c24W_jT3-HtsyNgx8|O~w zx=}NbBbZxY@0r<=0=JR#2Ml=+p zcGH4JYsS0`3*eWCq}d9Ca7{t1+A)eDWRX)A>pxT&Dg+OZ9WM_`HLfHln*h(CNV zt8UEOu!wJlwWozco;NLcv~JADumrF}GK~x7UsEuve$3afl)ntSnzlnYgPRH-tsnC< zEaR_$j`bl}9KkbL(lLKSK73Yavnzb`$N9umztMGTJb<(Xfi|hRvc)2=C#W zfDIgjDlp#zyPNh&ID?-G89hA~XjsGdgclQi(Gq(+ID%tAhIQ}}eQjkXB0x8Iu(3B- zS&#IG&n!m709cUO)XP-LkPNJZ<|<0J9|;|7>J3qzMh0PPY2Km|kOrA&dP9{>$WUw( zEmYJDso}vhyX29z5S0r@V&Tg`K61i8R0$GB5PTD{mn8 zV&`ceMbi!=5rY?c6O=cRx!8qt-3kPFhRl}UL}d@M0BcA$s~|uFd9bB7NqGx-1iqn| z3I?GzlG)nJQYzpKCz|eBkxm$i9Bl1PR^EnHbt}4eMG1(9%(h;(Qi&8|@$}G&X2>QF zw)Jw9YNQxTqO&VBgxV-35Qxe<$SSNmJ-cF>FcLL*u{T9|7g+-sWqH!hE!oibV6mUQ%@|v(|fye0*cQkL0;VJv@i~N)jL}`i5$er=#0uLCoi~!dHoEF9-uY2!O&LBsz z3VKOpBF`;8|2igzH^xA+TjK1rt+bA-1`U+M!{Pu(7KXO#T}#^$Y^6=mQ&zvo6UG5enRR@&cBU0##=7XLX%&` zUr6)GI{lb3-U>KVSimrnSd!h@`Tb+|6j&0lEoN9sT*;a2xqe+f6=O=Z6)3ronJ7(O)7YlQ>x_T@h7lQ=Vl+O4`YpoH>OvlOJ;K4n~%WVlRmNAHZCFmw}S*p4##BxNr6 z6xP5P_?Cif+c%6yk`|Y=l+l5)lZGCGU9bi-A-U($l)}0@2H+*!IKMJJNoHK8QbzBN zoig+k;($Cso0jZ zk$qg_TARjv(A%KYqM7a+vyV@^j-(Af=sm3rG_8Rfc)@YR?^c_b4|^Mx!JueTHwupv zpjAHju(wGWV%lgMyit0b@q6ti=AXT1lv-9ZYGeEH^xsD|4gT4CRvBU1Vw<$F_jm~` z&odwOHY>HxX4=Mw$D1K&dHd`Nncmm;_e&2$Qw=;$q!r&6#w1rKZ+H!@E&u8i6ue=5Di+bno& z+p>{$qJO1i3+0&<@J_)q+m4M{Cni@8ZIM5dUg2#Myn@Vn{t1Is=&h9JQqVjFuWkF` z(GIsNYODOYw3D}8@R#j9u!H?qNw!j6NP+hhytRF@vF$|as-dm&7t(9Ior3pJW|yBh zvI@P8@=^-kr{II_9&>56rDTtsj;e5a#09l1|i^`A>OM7?+1wYv>39yD`JV^#+ zL<&Zzz}U_*fLxX8Hk2VBk>27R7W@nWcuLg~H*_XtR0?RQz|;;Gz^rO?i^`ObN)^0f zfrTA0fK}D+Cds6{miFFrMeGg$;YK?UZubpGV}7PBkt(! zlnJSZClP4f(ZQ-l_o(gi3F#f)3BfA6jDXRqes{@s%3sm}-bsNLBAu(6bRXI-|4Vw8 zcS^9{E;m5uWUr%1HhHq(9<>%^(!;P-iCG<--8NZppIUF~XLl@M#Yykg<=Nz^f-QNMvPhMY>w-X5Cl?gjAp)0H zyLsg7(wW7K@_GeYGS$91(xY=%K0Gz^`UFSqmIbb^-sLgBOJ@!<#_NYAFiaq=TI}JT zlRt+U=cxoDJ3?S|b%#ezj?M?n1WzN-qNC36>}p70V|D1-*4s zGCgM(6WCFmy>@o@+&{Y0RIS6B9N1qiTua)M^-1?{>QmE8cAEpARJX6q+%xw{_Z?LW zvu+QZtbVw*bx+o3-S^a&rZ?>N1b(UhxOR5W+-Kbxs#a<}2$>GEb)>ynUvy`wqoxYG z;y}w&uIn=Q&VA9HqiX3^K_Ko_=(^UuS@XIds1v4l>`nx_oyuP4H8V0l29_(lG}Hz9 zo+?|1-^cno_R;W-;Gx}_!0=No>(c!11MaGPYx=~lHIRKuv#xd@^WWZ2%D+vY+jRzJ zpPF7bvTqQySLJ)tVQ|6=P9fG??Pq@L{i2*T9kWvg3QrN%C+!~u{#E(GbkgpAp!5`D zeeHhcf4yIo|CmnO4F$HJN?$*+e-J!ajP@bii#0YN;vn?a!~%;}{!3&KieZalK}**TREHeIJC%(u|DbfK*%M z<#mv+?*p|>vz53lLCgv=Pc%zR{VB(|5OL!1x5R9zDc}E?QFOgXlZ4hdeU2E9mkw8^M1~m_1 zG0rlWT$k!SbVP0%-4SmzkAjq8il{;X4j0UUXVp!55Ao!1?I?op3c$Sibp}4@B8r&|R}?I)$N2~I>u^44C9P(K zV6cMJ8yH+u=j~HoL^dz9r-hjTb{X7K$M(5j)CMjiO-uJC1b5dJ_?Q%v0pg=YnO(%C z1mCNZ`UDrZp$Z*oTFZA^@NnH;>mvYsaRAm5Ud8PQepff_qXfT+1GulS3%5V`d))$G z!xC#t4nR3VIqp#K;`$Z7)RH<&4!|fvt@gXT-qV*?VvXjIjW|LTt|Hi}KG9cMQU_Rq zQHoH@|E{ag^))lqMX+H3SFL*Z5M8)-C1yZnRmb^?>8+yM1}EO~zSiv_<$7 zcR6@#{jjg{Xx%TIRYu!{S|E6T{Q^HjI046TGs+Z>;`)M1>sRN#iwPJ8} zy{8|q)cRM>YNJ1dZ*Y%-&(|mVDNE~q<*YH!AMcp;Z=7{TFr|t66#TA!*iU(^?l;bQqXR;%GW@-Mfxk(c z35<}ba?Nz`dQ60LnZFk-W%zih^37n(2H0T_|GYA!6|5{;ml)#TY{W?)nlzTNtj571 z79#LVnJc9k{!_5!J-)%{2&{fsGkv9H{`hj1^|-eYYzqQR94=kA){bGMe`{b8OX<<++OL5dF zz8u!AiUdMCd=Rrv>Q9yMDOed$#zF^tB(qJLN*#iCSsA^kQs|6NWXh#SsAvJjR>mlj z2wm{0%t2`*HA*12m2E6KAzX#efMuC}szgAslL2-tT!Y`moRdychXisvSzysA;d*>7 zQ%7b%Lksga76j4YJB>dK(^D8)u&@nR7)+}-^TQuwu8?`t%7tX$kZCeAI=+%gl(A{| zg>Cr45ZY<8K>R7Dr>p=rMakeJ)0)ge@TZwUGAS)s)Mj58MmuX3fxo~^l=afeMPvYz zY3Iyh@K=}_usn2M)aFnaLAxMKT-wFlFIzx21VNb-Y1ArY;rp1SFvUf!s3UNqj4lef z_yK0M%o92j){dNLqf5d~_(#n1vP8PFqRx>MV{}Eh75|diCCjB7g0akrHR=>*;U|~_ zvKl(Ivd)PUXLL=t1OJZsQr1o90lLhIH@YF*jsL`aCmW_KE9;y&2}U=C`|;nI-(?FJ zhM+KW5^0-%%E#-4AR3l2ykOnRCrKqYJB0sV$g&0xV40C5AC{`utO&qgLPIR0N6aVr zB&+(&;0RU-(`pxEUaUjLvWxnK<#h3 z;JrP#4fhyL5|+z2*Jwbfm7I4p%rK@Tqb}nqM)!m@cuL42nEk;YXSt528Vw4yv~xuR zabsuveOE=A>Va7!J~%|$z}Q%OocX(AlS*qnM}@REq;DKKKKQ$0v+A)~3qC2Nx1j{) zhnOoB=_)P$oEGx1p?PD{iNTeMEvjc`9r(E?i>lY^@jf2iJ?J;Aqz zuup3OYELrPD0Zl{qI73S_UY+>k&}aK6gyS#&4%$kAq8OZSXDDU6uVSfVp(^~wTeBew9@zZ;Sl-hBLMnP z*2zGj7Jk4_h76u=42(J@UnkpF^sn$Ueg>8t`vWDXDC=dwQVajZe+ij8JsCK3O1@rp zpy-G22Yx}QPNP8(x`yH@1E*TF$lf3n*@z2@s*!uj@`@ITezIQ@Y7G+rk{XJaEWb!k zWNdF4N^VRI8mf_d$qp7l9@2hAD5dd85W1E^krfm{3Bev0%4}>5imH`UWQ9e>A`5#W ztaSDVNopw@WQU6WC;G+SEi|ifGH9q)zCm`l=x5Py_MV~njRwK!I*PaKNRf$1YhCNt zVuFL~+Pn*SG*fds`@qneM(^PAI z>dC%^B{Z#uof3MlQ5qav-{xC*l=h4HD*J7r!?3_wUQhNbETw6A?46-Ajbp+0>)ZSa zkI{ZJUvGaPbU~9Y(?m-4FD#>3ftP(4#xF69wVL8pl+|T}4=!zz9X1SD1E#%X* zfOcglv5C#RFKwe13TSrbf%d0D-CT2-hB9j!2SjZV)4m}zps9vQmDSNWA|nS;xc&Li z_@-_q4~8f>Kx~Vm?JtLJZ5oD!-a0y`(#Tm9Z{HQVziB~;VS_b;1Ln4fW#1QC3S*$u zhB^jEV&o#?+7EX`$wVYn-W8m4RsqiCyZ8!w%Wf8?P|&mF+6P@ zzyXz8lx05=I?z-TLOopY4n zLz#IP;Y@7k?AP32Y<-b0z!S^DR-f4wIuEipR$AmQD!0dk(awlN-J9~6SXmKGB(f)j zMW5*i%>n8g+fc+1N$g$2Hl2AA+S!yJf<0XnAUbL95w`oxm(ckpolq>;-=bP5{uQ0E z40At|4-4N#Op(l<5qA8HTUgE+oiHq5;G!n`(6BRSBEve**~f-mJ+mup z{)|pI7Ibj%uGnF!GvYA!v-#oJb4AgjOZMquPtSCO<($=tzycR8>a@=edvoSVSm)XN z2<(NTc+m~}ylU8VtP_Q8 zQ(ar4v=@Zo&W47!HfKfYU8Hf%@7SLRb32wZ4b{pHwWZ6E#2Hpr!9pZPcY`LK^?rz1u#3??XUs#!f@SYM5I*j=5SO-@F{c% z4?o|W7^MU)ouf9Y6gfG>hj%sSMj2kT=5jPf5_kaMh7UB?L{WiH=iD(mAzJCMHT-3B zcNFhp-6pRARh9W_hd;uno1cTH&QI~VtEx6%=ddsQTl2T5h3&{xpL?nra|%2}AkHm| z_G;$?u&$~TEz&CqUwY0hI;UMH4LevQ6;U08;kFT>Ag*Vn={=w|m~V7A5$<*_J9@T# zE=}(tti&*!7#$?JRvrzV=zN8qPk^%#K9y2bDj}f+rb3+U8OKD zaX?47o==Y*=@`sZyjI;dFN5tT@ADKx2cjH}eLbj#rgY*gdoqpr($$!3dQi+?l5 zV?OGjh$y+REWxXbzt?A8HD^BIa0eKBkA(XioqgD^MIS{qzdeX(ybzku+Lg6W?_ZkM zwSN}TaUnZlwrg&m-Z$D8^I3=25&aj038b4@`}O{#X`%bK5l=3(CuH87+pqVX_RW0W z;X}mag@*~PH?t1t{h(>(`+p<8T=`UUr&#&#qu2@BU>hIQhP@8$MuX66_x~YB)8>WVp9+6;5g{{70U>#BX_jSBu@2= z9vuJ4cuBoWtb%di^eZp~T4Y5F#v(~RSfDUeo1$=p;7Dl;BdJ!-1Ycim zjsk%`vb`leX+%C)s4!Mrpop-?+1pYA1MExy_tj{WDd5gEh>O;643s=tc40> zk&l@ghhh?Bk&MwrV1ZMNj~Yx-8kNk6+@0R+CX&!X0q_J(BbKkVk!%s$Ee4nngCZ z#wPdl@sIlaqF#yGN_YYA|K8+DrorLsVp2J3CW3$Jeu^Gf)zOOH^vxLw2h&OQ2^*4}iSD znkfIw8TOQNR4{I598^(5phO*NTf)JsSi*7W4OB>BsidNf$VpR;3dixrp%slVwA#=X z#A#BoMB@-3fc@*}sLO2`oGI0)XxzaVK3)V?R9{;e2d`#Tj6;#2qMfiQ>QP$@Crv$C zG45y_ThT$tikfJ<$7xctD#syFP;s5GJL*&03};F`S~>1)oLJFK$Ooiz2^X(niN~Q? zP|-^$ib7o^a?>=U;&GBO8$yA6*z619HfdOrafleew)uhMQ7bQIaHlk*l5rQ~)QWqA zx+vd^WnBCn*70%38dN+aoQVp**uqV_GkUx*L%tdHgwPtrzNq2W-eI0l=pEcr@r=+B zm348FJ9I~WLbj@STg59vcU1mGgB0`trAp>joLTXj&>zKzwWFv3d6mq)czeYL!riEv zi{4N|AfGH;P5%S+mhd>LN8g9~m+(1i=Hgh&{kv_a3fI#Qpned3L@j97 zg?IwFrqGj~hg#(LQ}mK{e-rAxx*Cp`@xcmx$HmcA1s( zd;D6T4eCRv#g4y1U14wP!u!ZNA8+*$6w(nJ&1kPm^}5fm^YKv^L0!WUatqf}^X?<- zeSFm=sHKiRk(=6|q;}rVugCfomsV)|3%lFDq|V>hkz)Oe%POoKy`ziTEz{fw^I;aD zm|vj{G8}Jz7??3QC)K0Tg($pZNOWU+Xjv37*Dvkeyzns+(h=$oRF>m-wB}OPX0ONm^FDFvtEe50e@8#RbbWK)? zNU^-4&=C={^s-xe&J&$hEYv0{c#dW<_Ln2mJD=pYVv~#eDvmlXi&=enSNi-Doi;4w zCo0MvjehpOEKDa2Wwq&X=qi-R(IF=Ca(jB_&>ZwA=o*y7(IqDJ^27Ajp{$E~Df9u< zNyjxYyDooBpBkY^NLY+-HhzK{A|%1od&w57XUxlrboCIb-7ziZ z;pJv%S`1!RY*9Z$bvS0mj9u1jseQ)0qS&f_fx7OP6Z7%%G(;{2uPC;uU!l4k59Gz@ zUO{ZNdd}=nWT;0_z0jC7yF!5M#bAdbQ~es%4-+t$D~zqR&zYTyEcG}_<0y%7y^;aAc!jVn>E$4# zGt?hYlaA$U>#q23lf0zdknJx1r{b;SlNi+^HncR_ZWQjJe@4A`d==9Tqd6w8$X$hd z>9XQE$BCE$h-*+^)pc?98UI`H$?;vx%PZa6c(1H)a`qelSMk;HQ_MT4Z75&W-Q*lF z{!#JW@q5hoD+@9VhpoFgxyB1Cb)58L7eju7I$YPy$unM5sqeHnc6o4J(mOwy^;nH5t^AIytA%*tinq z|i{_5lrySw8{#{7s*FZM_=wCbI_ zV~aX0Gu=n?d$GJ?WF^*#5qrGDEi-3SM}aLWHmk%tg~p!gh|KI9%~xQd=~3zE6dQZB zgHbRuI;YSpp<7yzoRVYvJA|2}*I5wxpf9z^LTrkC)X|ce_Ik8${HXD=N;jvh*olsN znN3jr7%w$m4zo+UV?TAwWKO*vy*+-+cm-5r^JDcpmt^6`SpDN=#@3bIPDQb(PGVNt z*l7QFxiJRL>8xCU>103`gsD{U)mRILlO)!)Gd*i$Y*48XsO>C*oN8meJ4>>x#+eWb zQR6K_otk3b88>D{jmuRsQ87FxJGI2JI{ULEP!5q*6gyVNICaEkbxvjtjmy=t%3|lr z1gGxU{7!@IXh@03#Kok_WT$=@s={rLnviQ`l46(26sLQ!b)Ej(CD0a;9WVa9a95#(oh5kxzv>1!Cym!s?r?e+ z`x5FSQ@$p5eX7+S7Q3DPiJk6zzP;x!{#~C_>U9?To&Jmc*7*%`B*=R{HEK_bd?&p) z#MNclUX%QLKDBBJ*#7^ETXxk0`XtEvK6Pquiy|lUIKtJ~?4C*feV=-@uSKcTuW=q% z_hv7gLPD@a?Qg+%!p1SKR%Lrl@dtfmYMMoblVe=$)$7@LQ%ES7s2LW=oqmr?zxq78 zXNv#8=d?P&qS|R~+}^9-vKPKVLefMXWKrkj6IXI|*&kkS_z!)W)J%&8r+-Y3Uv>K< z2O1~XGsU5mO-`Y4XRb#6(fKC-Pwd&^aEmufV&XckW<&I3?oYjDdZb0GQ*vDYRpB3` zw^@($&e5YSE;$|89QWjEJ7iGi9_gK@$69ncZI7G0`tXm|w^@(%F3{sGZaD3U`*QUo zbW!FW>$T7mEqa^|#u;2Q+d-PndZO1#XIUtmisLM=xk4yq?ulNT@y3FFCw`pmwV)kM z)2yNKHe+t3#_4$6N+_pHO^*(ZUo=jwyysLG=X-l(xaaQG+lPa#_8gXaZyV2+4oyOUf+Sp3NwV<6%?^rL!uNv>DoOXH|cllbz&Z&2! zFUGGK@2b?sSNg7%?Zm%ly&S)8yu0$F)0?Ap z#s!rI&d7LM=(RM>utwnX-ikmtqvKt#r|%k>85~jEQXjTh;%o&0my%spv&>ONubO9J z=8TUIz23YlX?Ac_p->lFSUQv9+1E8tbYZ?$^r??pEOT~`&%QprYh-rtwc@t=n8gZb zulR!N2uQpz#}xhQatn+zHC}j~kdrhwIHpjl1r|8x;CSiv=2zl5%D7BbEUI*Hj*M@+ zo|-c>Cm)xoiz_RgofG5b5P?B|piIa##ga-F=hXOoQ)y1{hqj5rJM$OfwEb0hmww7(z4HOc#OUre`AD8DyhpFK*x-B^j=f`c z2Y+mvEWA&zxA1d52HhC%-Q^$2Q-yV0ZN}|YHf_>Mf~*}d-p8-jC|+wT-{-jofnI z&kMD-PV;yC^BdRqpZMt;&-e6v=D+uOrPk(Z{)_*1f_+!y-p()iv)I?g zeU;jv&FZdQd*{FC%wflh`$6KzB+$CVd)?>r=f?MXX$v=hI9nGq?@gK?oKuXe)fQrB zQUbe6v$uAh3CSDv9g8ZM{mkx~-a9fs_(AcP`mRNdvsXev7h<2)SMtBoN%ar2dS}0c zGMEkx{@V7jaEkuG;r#1Ktw`I7kl6RuWkPnzNJ65xZs?S z(A`x4nH=(`!fE=D#YN|ognM1meZl{>eJcE${>nq<+P)Od(qCKLc0QJ{ z;-)v0b;$FDbM$cwHQX_XH`)8|e`}jB{6PQ9;;!?l1kalVklG=CE&NEIvUuQpIw9z$ zbbs)FZC?xjp})0w?0g|15z6J||B?SK{6zoT;+gZ6gp8YG`|tnP_HW^5`g@C4&NmZs zZ|Xvfhy1PZ3w_q&we#(SG8l#p{@(VjaGw6b;xFgB2{kvpq0U48ukb7V9}B^XM+xU| zCLU0JultYlukq)~cg`;px**qM_`~`;=bQ0-<(%_G!oba%1JoaN-#Pyo|6BRV`CYL)Jl zUXe>(P``k?5V1(CPh6b1yxS9oa$hHS>S%N=;YesAp*uFW2c~p=7ish@7ZZO?^yuCT z$sbc4UtNuXC6b6uWOP^MdMy;_`08nlEKx+q#MtiZ&;T-Bm1azH7tSIZXzS`I8=gibQbB^ z8kLxd@x;)?Gu@GSojM2gY>i7$Vndi~SH&xjR} zn9ont?^%+M*G)EdO82$lged|i4kFei`u3E;UTw0`gc$;Qtk5C~?`g?T(|ZkhNdy6=nA;M$J@@jP z^pcGyPzWb+EV0ruyJtF|q}OBIU{T^MP9S!}oU_3}wEhM}14LuQ$;AFdeh=1{S~oByo2~I@nuitQ|+0rPOgbxHf>3knUPn#9E+g2i>j1HMV6@)d5O?oNsF60^8Sa6*XapJ4Q9ZB!x!-Y!Ydbp?~62x7^{Yl^D z3l14VdWlO!K)H!{C~5Jn6^E#ZdP^<|ku2^bmIB)63G?PQXf7GS5vzz5Nlv#C4=EA# zXfD({!~?|Yq; zzb<}CyqvW4*6<wHNwMrptXjG`lQ8 zDooPcsygiTvtYUJTFq|DPsG2Ip5M9-!KV+yR_jXkisy+Rliu8Va=7#74=z^aM*GFu z0@SBlGl!>sehtJEB3HbCq{GthU2+7!BzeVzCn8_0O+ul1iAU0wyk0Tkg(wgkkdQ3f z-k>8*OOmZ7D2PL1Z6j)BZ^n_SC9i>CKpX+f#G2&`wg%oL*?PhoQ6$zzqr!Vzj-;8q zww~}ol)!{OnZ@nBcccl9;ZFD>O2yiW)Q;YnBU2`?Z6^E>W#ScZ`1w%p5*{9zjG6F9 z@WtA!R7Ee5mxg?enV=#<&2Xe(I3CU5)gnW%eKd`>GZn>Z?@i~8ARl1+=$cAPA}NX0 z+grl3G7YiqV`#MDsWjHZ-ez8s=>yxojhYjdZlp}sSg(dxYZ?M`tr~5KDu?y4cbYe1 z`oOLaqV$%Yq&$|c0#TH+%M;fCX)5ss(h(L?fh&qKy9KjMCG}!IQW?uy;SV|R4fqCV zRf*{&F^jB7g&y`Td_zddX>lN_hDA{vfz-_g*m^E$5{Hl)SxiM^QIz>Ds2P`>6-PK- zUg%EkvGyw#6dOX`iW`Y&6|+cvtWw2_ zVk)Yhz>Pv&6mv-ftZIcPoN2IeIF6#aW~Y)_|g>m!qT*mz!h_SZPu3%uPVt z6z?Z}XMI;JfK;3fk(-F<5$BWilNa}`D4|-`!`G3Kq;rV$zvN|o9?;A(CHb;6y_Q9g zs3!EqLQYFS@=ezCS(cKvVOB##Yu<7Z)S^oIA^n3%ruB(S+|dWgwww}`SVSTuM?+OB z2ffI}mRq6`OGpP?lQ;D}Dd|KXbg@k-84#Z&c_i=d`vQ@fMXt7~CHKU&B=6*+KFg!- zOAkUJrDRYnBQcVX_qiR-f$)~?rjm!^CQ@kfnZC%Qol6h?Zo9eUk@y@bHu-AbuA}o% z;<8OIc_MBlv0>;&eANAygDY*flspw*BBduk?dyQ9z@k;QTT7mcJ4xBeZ~C4b?fm87 zDqA4$#WzTMlfU+TIXVxmF58TfVQ~+sAlc})WvTnJgKoB&C8J^msU&&nZ8wm_7rEPJ zm5hm%Bw@1s?a0zh?<{xy?F>M!NhgxsZf8Ty>w~*~He=HAKB+$0|F#fxBv030D_4hKqSk97OgMlMFMgkvEe;?zkTi|0DgI{N?t?W3#_~fMytD0eT^M5!;~O49v9cp89zV9kd?Vm~Gkb zT9yeSn0`J(7i~y3W#jro%UXfN(m%-1NB@u9YP7OHqihPQViN}u29hOYYqoEHSs5OX zq=^EAk;IH_&kpZzDNBRQ*hC=$0cUNtK4HJhDz@?K{vv>5*lubc*Y~nBi z+%J$24)rf7$6F_RPaHuYB^WY|J+p}bA+iVFeLRgR8b=NWD}hm7YaIeGhQ=ICAV;y= z`_rLJ_Q0nP+y^v~oW$NR>s|-12QWS^{ANat3>}zrS2! zv%#;SwB%RG8uBhU^D|jKWOK`};aJIUlJ(?VwvN((kH&0()6yky(wTgOjf9w46y}zH zLwN}t^(B|Ft(E?K31$PefzQa%hqJEihZTu@C8nOr6(H;+0px1-I%O{35NiWxED?AK zlia`#P}cCN*m@clju1-1$>-Ve%5FXnYeVP489GTc`7(Q}a+t5g*3-F_2xmz=xr@DD zxd4u4*)X_p5tp#Yee6=@3OI~Z&)`ZBE)p(zfL*Qh6!2_qHge(AjARq}5&OI{QJ{oE z8utVO4uz0kvb&VI0z*5S0B#im4n>eB*aONM0Tq&I+>?kkk{#rC?3c=Jn8vpW!9tvAkZ+B~6P-`i;~-SaAgU$^I#iKMkrcvh zc&W-wn1fpsY+F|Xv$HTHY!8(+tCY-O{d$Hk8d517H&wQ97H3oSx*p*#DI*`S=J=}0 zgm`>1b3%%s!9gl}PPnQ?n1+ANoRA?H5;2*~;i~QloAAjY6Ag#}NfmiDXNPJ=IE8;5 zGI1IKBUT_|9D*_%-aa{Wq7lKANXax#g^DOjvws~r(S!(O_liL;1ENeh_;&9X8P-XS@Bq8Skl=eIU-9;sSHX%4SpCmszTqp15d{WJbreJS&;sPR3(oN3i=&P4h;Gym|(Sl$}ddWo`l$uzP zMtB`L(TZS8`pJBbtvaZp336`}Z3wPJLq5(~sm`dFBD{{8xQIxV+#}aGc-7}TR5w>9IX{T+yP?^G{z9JRjHxx1 zway{Xl+)}(!>b+Vqk6h>#Q8x&-%ZT{^a7WKTwM(U-a0}cGN;Ky>$(_n%`^mY67fM| zUytS>+Q7wxi_tK|wZssp&S?tK2p2ThRg*3rAwEdzyQMjdUgBcK_12Vt-4y~EIt>qP z28ZNAHO=BA(gRkXLQ{;kbRlus8V%f6LZC~hIf{k=12fv3ACF_CU*?}rq(3{+IE_gXb+bh z?nlkEWW?n`O5a`0DYU0c9#{7c;<%M-2*mC*wPDlDxK4dhZLtIXCgYHO= z2mgL?)1f~ZXVDQZ7r2RcdZCZ!viZ;>#yND1%N1@092&g;`^C+N9y3Od#JhBH_d_tx zaHUN;_X(mE=1=>$rFT}GpsuV>=MEt*O1Lfq+-fN3@mAVw;XXxNl5BE$#65o}@q}_^ z{TA*s#1+X_mzUfwNa`7`vf0Xgj_8zRxlC{e?$n&1uBzY4eSx?p+2Qhz`|?ir2_E1x z+?R+OlHD$!xbN-^pHQx<-^P7~xGCB1@}2wr&VnjKH=7LZFrr72@1mcwcwj{p)vZ2* zJA$|+Ipnf9WjR#%cy2bC+);!=Qsi;~mEtszSfzBU&*Z*F+?JHOEKgYnT|Ps1n=I}a z!^n*9f=yuzBpz~eKbU1ZUZRp{Lw(T$9aTB*rY4If7#g(1#U&+m;9*s(`}Xbne=)S> zzBMVk20m8Jx_{WNKgqaQz7a(2y!o%%D3VRVnn!4!kL zX4NE*?Ys168QL6Raf;<#*Xm4<54-f|7-MLqiy#GeH?+FdV|$MN2Zpu}cp}B^Zg%yo z$A=vKkBmtme(F>F?+U9)Yq#&#|A(Os2A)lcyxU$~o;kAHH?8|E`iV-$&p89n1NlG1y(*pXOim7nj+TG5Dg?t`9lT_g$l{5YDH3ygPkrWc`B!eLplm z&aYdjA7Bj6O;*wjLxrUHih9tM-?)T{r@ObR;uzj!Z_v_~9)?J#AT`F@wS`X~Mws11=1WF;4E|d9NG7RERa|ZAYltLIC zo~e$;c_#A3bCs+JNrmEcxy;|R;a$FNuCrA`QlSK0K2ry6@0kZWN*Pv!K6 z+&csV8Wm6)Ktl}kTmXBxorFpn6=bWCmgqQ7p-#~}IctLyDriIk9+BpW=+w${u&zv@ z%F)#yoIo=`xgM8y!&>u(JMa=SPoeL59__rG*J_=A!(CoRC({?dFNSbib6d=u*!MoiFnujzex~@1q~dbVgTsdUy8cRUfASW}XgS z4xnp2BRkjf){jGOYXf2t9MKv*Zvs;ndxCOX8=Uqtuc1GBW_DuoBThi*+Q3}Q^g{P| zmUN!auRcKm<{I57(+@r5S<@MwzkUJ&$Rs)@GY~!L+1B|WA4{Y#v{mUQnIY&o&%w@@ z`4L1&sy0v=Gb7L|p0l0(`PD>9s493NIH$o0A~ZwuAYMOljJn04#^go zjW+cP?^-EXJh=?4N0L1f>`8g0bSXU4KSfT59wMDVK1BZoo=Bh{2A*05_#?>)S&Vk{ zD(V`ND?a7=-_&85bD5=R4yLNh7if^(|22V;F=QF~idQ2zE1>>zyw`N3-5FU0cB*^2 zN&yeqeXr?HnY;I2paZ~?bqhFe5bhTqrC-c^g%0;x>Y9Ey@K?kA!vE3#&3ucF_Y&zA zFH|~B0FQF$S27#WDPATeT7?d$D<4qxN#G6$oeDa_)`fRY8)Q(Ak*!swom41CGRUN&NnXfqG#3byT01tR{7iR48K2BP^b@a3-R}x-kTkR0 zjb!{Xf0L8`-J@XHJ3q_aSmt`>2>OjzY_||s!d8<-J=6NzpnUIj8|XoJ+Xj>dmL3F# zV?KIic3(akV7m-ZN>VU#2Hovd)a}GAwhG@N2HGvNp%$cQGJ@!S# z_O7{8Rx+m)r45z5(LKIEy6nzvBDB+xpb6`3-E+GL<=~jtWZixPsbF}>+o`7%(3jnL zO~*<68~Y9Qys!4O6lFLN@(WMU>6yxg81EopVGcMnHa z)KErdp=T0!nw4Ct9VrKCX&C1%)GJyd=VVYswI}5w35GOpM6VVAHY=mK-{b>raF?iNISzEZ(MIuiQHL(5~>ra80l!p^|tRV z00d`c3H2=L5%O=tr{0&rBLAJU29K!cNRN^K7{2fh=$!;6XXT?4A0?hAcE z#h*_rJ8`AG9!Uj10hKH?M+CaFZr1DMVAUY z%U7ImYi~i`H9Y8Zx{pJh{m61dSD%)?bl{1SpF{mf zgUDxwK0f_@RgVMzS$+=nCk-L14aq*!eLVmeCBJ}PCygLq8%Fu8^{qT!{Ac+EG=MaU ztTUweh<{ReqVGa}2@NETBi|d|@=^SReiG=i{1O^Onnbo5rupc8asmV?xf)6)O(EM2 zvwSe0f}bq9ELTH=NzZA`7nf_Gp` zIe1-~sGW+CSplVoE}z0r_D_oeV>%Tnvzj?)IOy~IQ^3=iOB1iBqGZ-G7YxUJnn2X| z5O7VWqGdKRR}5!;`aYFC_4;?>%~XucR_2=FcdN-y9|3gg__isweH*!DxbCy^Dc!UG z--fq^arE8HT|*&X5g?x`T_(IMjHe4{L5#$G_w?(QIbUvgSC~MDWWkIOz6bhk$}%n! z>IxI-Vp&o~ioVDCJ zOr}d^?Kjf%y$Z0@j4OnOLMmM*OW6qH8`M8tHgKh(p^!$0XQ> zEC0{vU*M~v6~$dOo7`{9C}kZpa`)x+bIMP;<~O<1Wt6jw0Ju8R{|>Yt@Xh7x&TKyW0bO391RRaihjnUw~1&(91NRXKY#bQM0N|CM#$NYC%;U`y2i{#ZA)kVFFi zJ7d3q!O5ylugY#Jmt?1$V`S+UKPU=(S%V&GQM)@O&*+37V^HJSNpAxmd@-0$-4r?v zPVSgz5#Eqq?Gn0ER_gz~bHz>g2QFAmo$N2sq!QOduJe)x@kZhUd)x&feaNioWs zM!)8j~LQ==@3vkO06SNlwrnj ze~lsQmv;gT0H#YyR+ci}@2@xH_Odgea*WzQqAA0T)%|fpNiXFB4S=pox}hv@tmAJF zax)HrmE+X+q+7~hcEtbkP~*!xfd&)QCK6p)+4z~g|Ip~m*1-G;_huQuNEuuB#|{Zq zO8|e@y+!6u)&b)a{){1w>XSkFlkTlDX<6FFw*KrPT=fkA^18Rl{Fimu_?-Wfq07~+ zLHS?ZKgitAI%<5$|IJW*wFD4)-9O4?WF0eh_va09s!x*hr`+3RvcT8W-+yH29mw8j z3fy@z*;yvWDtqUKhN|m82j@gbJ6qYpxN*-0Fnrg^kZEl`os)$(PV$!=R;L{$ z>3Lbljc@xShcPt~0QJ@Gq8EVY7U}+nhtJnk2UBLWyXl2lBx9Dp$#8hhdN5>GyNAw^ zbTIBbDEZ)a4JyQOwyCGRSlP+A*xzZm6tKVDvrWD2kCe|FtE*ofZmG!#A$%+Bqd(62 z$M~gx5O9D8LK?mmexg6k`qwz_oqyu6_$#GQ!dzj$3|iXNxEaX6TCW^JE9a;Kq)KHE z;|_n$u=T4up$6ZngQRE5XQcc5pA27q)f$@r9qQ2dT-o1v*uQr8bijHjWL|qnMnode zc*4JBIQ^A=71qIzsx52`~Fk5BIz(4!KslwWx5%g{IN=I^`r|iR%g@M{2KxcLQ#?y+Jv}Snj&k z$m!bLaL2`_@%H!1w?Rt?J>pwC7Tyi~;r3=_rZMWe^~mj7RD|PF(`0+AGFV2x?le+b zdnKY9fW++|lph%DUw0e%SeqN+xZE_=-mc6t#$FE|nX4U(=w5~{QaZA7jB(c!N5o$% zMG{sDr|F$p`NlTa?~drcc8+XVDV(8qXK{_~ujheiZ$>2HXW=ZpH|vq{`Rf%Up05WY z8-5l7!#L}y@nz67N_njmMOZDIqYo%|=y_b{kED;=0Mxk9xN%VVxv}5%p^@6x2~qOD zPRzFtDc2Z>T%Q{m0?KjPuM-N-!%xc!0@!<96Ujp9F#4ZOnZMOf$mB+51d71K7DUPsHxJ6&jS~i}( z-u0#Et#e$%PT@9vGwZjuJpQHSZAKhnw{VBPowaFPZnyAd@@;2a_J+}A)!))CT1(8vLExmfXkze@9rcR zK#2Ab85R;_6yQHPUHJ~gr-(4#Z^GF;AbND`op7SOC{dh;P&t4(4&Lvn)Zr7;M2Ql- z-=;GufHi7XmyjqgMwH|!s2s+e4JaLTsjEv&6C+CTek0Ep18PU3>x3!tP@*(XQRNsW zTDJ|nBg9fDP#vH?vkWo50fVD2>mn!+n2sz@UJQ#N2TYIl)Rj}nfPuDEQ!&Rx1+0y( z)Gbn$VK9WPh6)}-2@oGssMk*-0~gx%fXZ>qtpLR_bbVmbvN&vyt(MA3P;Jv4bE+>- zA_FMeR$GOH$qK-X1=lYoEla>a#s$=-y|!Q{81pFL{8&|eV6tYz z*S)ewSm!~I+I@^)U!3eJB>k#|rtQ4kW9k8&Fh7fJ^iA zRopPG0Y)6%24`x6G*^jX!1BQG1M)!BIs<6aTyS^7^1<{6RE&8x3{V?nxXKJ;)^*G% z=y#{$7tiWdCM8!CVS_ z0~BnDTblbkj>tN*DlqPWym8L^lehBsdHgB+59>L`KX3%F*f(xz?)NwdWgT!~%^JW|1$s~PH&xRq$~wjjf7URjHZT%c+v{|QiVm>L zS!0;Sz?&0_%~%FSMF-0WVtvJY49o=hb_4^Ws$;?kX3b!F0!t>&H&-(#sC#$vH|-$tFWxMfvB1UzZB1^q3WLBE(VoXqvBD+?Nq<#o!-Iek z(Ser-Mv~&_K`LL(+7i;`4-wDs3RF&kfWeWkE^T$`X@`i8yh0UQY<5tKcX-=+I^?j9 z6QhXbfGr3*_4Pp;_CLyDowJM*)>&+6kkOkefRB^s$Ir@^s-%gO2f2Oy*p~aB;~&lE zc#lYn3~=JPxs>TjO>?pNIp&i5LQaGe>I zEDtO{C=WF6Gwu=oW#~Tkc^*yf2zGvOU?js+a zy8N*fnCBj@vduukRwG+X#ebB@&^#sx{?MXw7kdy)f3QB{Gt!O`ukqSc?qUBVyG+%7 zOvsQoAiDEDs$^mf$-YxVAL~Fwkm$kVsj#u;+2a1w2mi(&&tJGlW&3v5^N>~ zt>eY$0&9Aw$(d7__J~Y~p^i7BhgFPqB9}~^Z?6V!x{eQ{kM$VqLavz#Z(q-Z80q*j z`oa4uH*(w5gLW*qAJhT&VXS9ZU-ID8%l3#Y2)I^f46&-Q!Q|Pg{`TrDim}dh#t7>* zHk!OKwbs6#1p#MPMxX>Zqmji03Os#q0EY$I{-)M%krf4KULb230}HYRUutX`Sy$l1 zD`%0hFtY7$dp(Pc5d`xVS<6^hu+iY4T8f-;_0_JoNb zM0SqV5BLIi0iRc#?P@9jIy=?OG zaJvJQ<7nZ*rmTa{H%;ym6m>Y~G+1zBcweM`VSkWE1&tkdatthKv9{YPo7i9E1;J!T zXHKOhEzWjVWf!|k7Md0Xi*s^G^S96UY8AIt3~G}Cx+RSgq7 z03W0~QF)HQk>|;&YJu{2`t*^`D|y}4%{O@Rs=6RDSulONGdIuixQ7s9Z}uN1=RtMS zMvI?Ud7O63cE9TXO#Tgi2Qrj$`35I+@DQc!VTQs$rVpvx*}8=1uU2aim@ z>%0M62@i&>YPN|XlHAF{u%33o|>wKNesZ~w>pIjfBiD^7-Ra=wnV6&NMz}3s| zJ($kW&2}&;2tGCQpbPtua!Thv*-f#tCXaw$U)2@(aQPJMp6wA;XOqfc&zYXC@`vQV zVE1joMvloV0NSr~Ek0cS3--WPPxXpPLvYHBLbrY)`7{iS+JL6m2O!*|y8{cCPs6}_ zEmbd*?%<*sr|$AXG6|MtYozLDvJmuqCZKz!aDpVrlEq{Pn2ZNE%`m$Uab0Z%ELrd| zO=g4pX3Dy~xD&R5Y+1AHaFeCrsTqEEG1t{jz?QYhjyBn=vobT+J;v>}YtG?Ws>YiL zhltKf_n?X#?VC9~ylRq(M2N!dk)A6>-S*A7Jc4QpFpRXoTjSg!M~CJ--f`92CQ2db zS>K+qqHc%gd>%17LscWhYBmMDS0B%T2IC2XaByiV@2??5=?ina5xRAtI@m{48f)lrh;gFqgViR(AR=3x= zq``?>%y7)+n%IM+s7-H13E?cagmG52QRwfGGSC?9EU7$8du02!>Qj?{L*C6M0Yt>$ z9Ie#WS+&x{BZNO&(ChH1@*M54?LVq7OsJb%>@lh+~9-?sXMpUArq zU-AM}$3bW6_BWMJ_$O&D#A;rU>Xb=)2{H4mo!5-W?4KrcA%ovueu{Vs`B&!+ zBRYHCWF=(wTmPr(r<8wn-ZEmdw@q5K*1oNLT70_vFYKLdysEG%BvgD(pK|} ztCA*aDjB+OPOsmqY~r$@PBtl9+;mT<_MClxahdBCLA@*$OlToPjphRSXUZn72pVKl zvJt>?IWfls%`Dfef=1b!*$SqIL(k2X^?Q|1Tot^Ry`8OSdMwm^j^AHg?&>ONl4WG8 zn3{x!&MoxMlux({nq`@w4r>)kokI*9s&I7^w8-Ag)-pX6dT&l|z^h`yP0%Wvo~>(o zCbV$QexSI*^_rkf_8wTxkGl~1d@f*Mrefln;DhXgY(3Mfp-pqlfkTz9?t+i9nc&T* zcWB>S*??E&gu9?!mX(b$4GNu_;|~Z+I(jtoc-g9ErV*hlb8`b@mE9iA9lRV>oM}R+ z=y&NsRF$J=GoP2MYHfNWRN?!P!7EkWp3R-Sd{rCMJE2-&h$6Sj(W|+O_fU0ZclWeu zMkxBb@8DQfw^wsFkE?2Lnj32U{q`X0nWJ}e53g9&$@J`BXTBE=IzMah=JqlkW&dqj z7JBu2%V5Scf)BTk@i_Y*)0d$b`^mx1XO%w9ezH%qFPXjxy_B~#DEwUBm)OrMQ@sjW zWVgSoe8xXd^Cb@ODpcJ~+e2C3%|0hQm-izM@~Tw5O?yL2zq@>{d+ykx{h9GR+s||; zwB~#G=k@0hf1M%5OEB><33_G^K4V`{KxCIulO19@7drU;<>!bOkn1`ljN0r7)0NQK z@BN>vUr?^=d||xFjxpT|-T1!tdHn?>KxdTkE<3?gG)!_{aR~d85}-52s0Um86fMPh z^ibf-%%>0O zSCfNaUv1k||1KmPK`kb zZPvC|^^xh{VP*4f!_NQahj@IG{gnO0^itTH`S@XpSDK+7bFu^36{hZCvP!IB{HwH3 z;&T^@Su%Y?d;e=Q6VZ?dfkZO%-NZ8!`&~V+Wv@qfi-iYcO)0nWW`K@8$TKRC| z0&i5c-ZVK(`iII0zBVnKxX2qWQM(S$QB8bbp zDOH|nc9_|Z=n>)9@{z<9-n43$>BF$oKUgF9*J+W&pS)SsKGVlxEz7C82u{9$6Mn0OR8^4e}X4)8b^M~RW>>En7&N^c$ z`>W~4u*@HrFA;AbF*+NJmFyYQp0JW1=f70Hp~UEHGFG#{n+}E5{0RTD{st1Ov&C2g z13;6YclY26_AMn=XPdE+{nKozQsQ)W89UkA zrdwefKi0mizlFr>2&D?Cz0!k(i!UgQ>c1n$!-VZb)WpoB!W9?Lqk-?1<6$CpqQG*J z57%8Nds6()H9;T(hq9&3kl{uP0i!eTCK3b?xHuaDw5Jmb%+V`%&LlRAc1Wlxm>mjt zS|}Zj;MXN`#Zsl%`_1&iuP(HVX4DZVTxhBcTiFZ~9<(q$I#Aa@;lfhkY<06Z9Ed-Q zk15p?lDOija%@dAoAA2}x?|4u4M|*yRC%_JnSFTPg3VY)Js}yqhTg08huPoZWeaX& zo%NN;G%36NYX39)H~if~(wJO>0hK0gr=(_J<`K?cC>V2SsHD(+%ZDAH0zOx`SF43r&;(>sp*j8rLaM?wT@so}D zDIRigP4)>hM!4D{4pgZ%Z+Ps1A7r00yBB_R@$z_UWBv^fdAJVS){Gr)u^2xt@m}+$ zhXVW%2oV;B+b(j(PrlE;>9H672m74alkiK6@5XPu*SzJi5B?|ng4y$M|HaYq42iq9 zi2FN^s$DXB9Ui^7H7?vFf19Y-p{I5g#0)Q^6(_Jw4dp^gcE>>a^h5XqFwB60kKGn1 z!NGuzSyy=BqWwg1lPg_-gd4Ga%?87tF9uA^G)>S2%5V%jz-&CcX^}Z`sM(bvP=TAU zgUx2c`xeV4yqYH%0#!Kp+-6JRQ;YnG;%3)Wfg0R`9c{J|zPh+DG1EMeDo}^x*zsl} z5zr;XYmf8LY zy(PEF&X&qMGysvQ0eC9{x0Ey~*J=PfBfFDoX=XYR_DiQbZCdm1dK`rR#lB~DG{W~& z?PNl$d>T=^E+jm50LvhM^jzc)n>sFvrGM7 zt3Ocg>wpbc_G`1~h>fMSuj?Nm4|McWFSF~+k|HFR6@B$Tk{`hI?XIf5H@g*~xQw0( z{J8u8cFfKVSZ8Swy30;eexD%wn14N7yWB~n1S9sm5W^4ypZQ86eqF_0gmBAx`tG?6_ z_BXTsh>B%T!2quzn`@dH#{OY88d1NTB2emZ?M*PVivY_8f(ZU{fxw}ol1($Wi&Fb# z_9J3+xlwSZ!yt!dVHcyeY4$5(VR=%}*-@E8v$Ts-+cn#b5LywPmg5_6XgIqBHHbMZ z5&?qi4*W_E4R1$LgPFr4HCC*r@9+(BX;ya0YEtI=BlT9?raSqSxio?uO$~0Y9*J8? znwIM{$fH@?-B6P^kJE{?Un!WzYvkv79Eaax?=wFdd1>X{^o>r-Y4y`nOM?@B8mp6s}*rVYk{SSw~T z384H=Jk^n|cG&!EWa)~_OkG#nL*ic@_te0?eq`-R^o($~d?E34#{;!v=GP)WuCQiM zAL0vPhJ;KtFmMsszfv_5*xl_hNrJQ381taWsTKZAaknd1U<+r1FK%Sy>dL~*OgE&p z(XJy$4QHMZDf&}-7S-ce)NJ38t7dI}BU0h#k=ZLf-9^m~9r$9lSpn;kp2YWC*2kv7N_Ft|x5;X1*`L>$ezk@i0eW*vGfOK4~9 z9;y9p{xtIP&&Jt1V6>BV&hD|=KjtqY1Ab1iW_i6aVRW zuI6t(99jF*{-5a0e~VBdKZX5k_eL$od^2+G=gPOmPs>kXm+juEC745^#8(yO^!v$Wuq$@; zYRTqOQHrbRxxjvd)u}7+Ms|w%o+#~A`?=zN*K)yCcoX}!`Q82KRo}U>{_gT-*N$d( z`T>n7tJRb_r2#?(*DbY`oo22Zb!N3_&Uv7rf_p9X1N%PMK)t%Ea`(=FK_$)Iu3at5 z+&C&=b#ktApt6$YVb`ISV{RE0532rhg9cSJPrFXFd~;$Hb5-lR!(e3JU1EMN>dETm@2!LR&pdqKpV&{#FGamsjsGt3S@XGvFMNPqVeTHq zTjhK|`8of&hadbi`?9bczVPse53_5`L!;(bN58jz&VS)?9sY&=#ymD^ zXH{rkVo39)M*w_`U2jf}lKrItbbJtb44n@;$GYak4l#9U3#DlEn(xBcXm?#Rr30ppus{Jl^7aC1%(1>X zsY60t!QxP~(^~1mm9g&n=Hw13bwwaFxUGF$$Q^TRXr^|^sDp>E(ZOqT3u9y54b8L; zxVnY~K00YldJ#45*w~!XA*ZfoaWZl++3Il@elX{h#l7gG>z9{WC-Yl9fX9+!3$6w% z*5j8XzG}9?MCx_b&ww=Y>2=l;{%cwr@xPA4>gOyTN4u=oE+u@G|3JLgaYP+7siJ+? zhnDKTrhOpZ?>MS{$>Mc%^!nD4@Ra;V;)4!7^{W=|qi?UPEaRurJ`yuJj;Xs_v`4en z&6X2@X+g~FK&yLO^hTGiyDZmDrL_~YI*io)Erz3O*Q1vMS4DV&ITA)a&|)IG1tgmF z1!Nv9+s;Hi#KPxL-+I}ym!P|+F1y229bAq@Pl5^bT!AAv6m-0Q5M{9zy|g~PJRoT3 z;O3-S=EPYD#RPPSuEqCK?x zOZ~pZ|6(q0G_Ks4G3cfh+L6GVlW|M{@HskXD!XZ1J3H{7vW$t}5dA4PYtTb0vU5<+ zw;;waH?)2_%vSc$itQZLxfWz+zJl4$gjxAsVo8USdWpr^n9>cGpLMfoy~IZy=hUBA zT#TvRi2f=3O}>v<+HqdJ!s1%Y#|_p`{I|3|;^U4B>d!6wVum(qe^`iO5Ttm@B^`(fpFSJmHJ z+=@}$M6U+UE%(DJ?A+8_Ez)9iH=S0?=g0%FN;`M;c8jbS%x3WF;@t88tjf+)z0)E; z#%42pRsTDA5cbT@TfNufQOxbmUx(&hhXgO-LBK_9i@CR{_seU3Vn|R856+pk=!z-a zwEtB+?>a1~frkRoWH9FWX27qR`H5k{D|mR$g2gykH(>rc^uu*TPz#UDS+SUn>Dw&( z<@IA?MDQ9OowH^Uw-htA$^TXS!}W{c4Lmky%VHyDb#vj@%#VpLg17Ma93e}QSm+jF z?a+ejsNfwuF-O!g6|b;$WbMjA_h@rn2SvRGgNQw_WwVyCKp5lJrzYn}St`aJ+wxo+ zSZEmIHl)&W;FcP(R$D1+N{fVXZe!|=9C=G!V67CbIWIPhbKj@l0-+VX*sFk_%~&K% zaGO%;Im(up*r2WHwSmQk32t+0YL2=kE;eyXe0_aTf0EW>2efEQVk~n@Yu#a~a+20+ z2N)zvyI9Va_4=J9gRjkr(z#j}P$s>KFE2jIG^@UKd`LpCW$j zc%XjF@>=Z2E!H}IIc9Vi_XP;lasDSj%H}Q(OG?;$>HXfCp#i zm{~@~u5K-?&n!;}1RZcr4$d+$7P^htIJDwAE#SlRa;zSE174*VObIw~<#JX?uH;R9{eiQV;pX9h$*2IQxFKo>GocJdA1TV|EY*`;m z-9~I4T6LWh^usH1+$`H-?``XCdaX{(2?pR*Ii8kXv4z{Le{xqHzc&wdJX80z9Eh#j z_T3y??f%~UxuZYvy5(qW{dUTx(l5e1cPOXyNqw6WXDJjXvLn8wv_@Fqj;7Y-P%OpcqP?}Y z9M&oqXk&H_>NHDuoW_pz)*Z0qLL0YxuYSvNf1KWq+t$g`d5azs@a7zbrCOZDPW+a{ zy5^F{B)m1}uBCRI?G9(_<1X#I1JaJ>vd0v>Jtx!BD9(RpbgOke zf7wF-@5o_WTExZf2yIJjXs&on!#i{GEKkHSb~Lt6Zsf0c%)q;I3N3Bp*gLrGfs2Tr z#MzD>^%Bdoaiu#h+jSdhKZ)Nu`qZCTUW}{ViQc|30a?|VOYP4ow{(kZ+j+2!-K4DQ zd`}(Bd1mPwH@Ne1J7N>^OJ_cHD5u&oIBs^Qf4h2<@=NDO>PXIO%jmd`owe=tO~{(g zLh5Kvon=y-*J`qh}}awu3LgN_!5v- z+v1|jj_h99>E3Ey?^sr!w(N{6*gd_QyW_asywUMfea3d%Jnlu7>f$ zaun!ADZ5I$gdOfy>Uz$y<#b%vZqcsuZo>|DJ9RVXm*qm-_-@N?#x7x(yOX+|vuU{& zx3oLGJFwfZ%iT@g&DpgSiWd<&yDKMz7D^Ga|LcJW4jL~jgc3R>R3PLjjDT>UIN5kL zA-vE{AuVA~5rh~=61Ok@sL&OmHlYGxPY43Wk;SRSTL>ixNeXL;c#0wL(+L+Ah3G<=(i|lmI^J7oK&VDI38D*Q%5qe3X7Q0i>p~mC zqN2LuOaxf1vW~wAw!uwAl0Pj;eIr=!a_%@*oVG~FaR9A+H$i`1ExoH(3Nf{2KOC{o-S zQKZ4db0|3RggqjML@$UpB8pH<0=NT5Bpd+Hc&50*i-ZIA$2D%@_9y6xTodgQuaZmA zv?p?!q|_2DL=r?LCA9W<0&E(Py4nf0BDtccBntL;9z^`bxraNNa7pBbTD*jUyp1;h zv_>Y*Fu_-3ShOCXS~fa-TaK!cSwf`9y6A?4sDf^#q~uKw&MJW_A}4lO(oJDn7jcGD zfIF3NPh|VxWyv1kg76(Rif|4I1tKJ|JV~d$Er`M>mFfY(Oa4C^FL8khlVG1OPs(Y3%TfMCjapnp z!ivauv2m%M{Vo6F|Euv1mjGCG8R!9NC&g(~>J?4{E+t_P?46m$D1jbFGQ38Rqru)A>4Ha81I z3}*wknlKL;gSEj6)IBW`ahx68PQngESX>gJ1-_$*1da$EnkXxZ5$tW zHc?F!FMboDh4QpQBy%M3`x1|eUJ-9Y6rel_2pUHgua;;bnjkJIr***78gYZO2d|xI zE1D~QO0M95=W)a>&OZFn#7m-eKupup^gMx}bC7tWM1RpS@jR!ynl?l}LqiR3o)|5< zEiST0LCeO5&(t`8Kc0A7R8_)iPr8;3V8b=E@uWnSsJTSq9)*K8r}*g_hw*0x~jqZ2no*CjUOMF9`TM{>^;Cxrnb# zj1*g!+}JC6ME5+ijB^Fw2!t3VDU*FkM|7Q;6(F7bF)>pND;2p<^iSOj%qor-z9-SF z-$|-sU+|ye|JXm*@WX#jd?^+pwX|>LPjMIfml}ciiNqG5%;a5hJv!}zsNn?TXA}Fx z%B8&bPad7Vh^Xa+e51bPJsdj*D`K8R18C-_U0H_!x0Wi2C5U&H{X0`E@Y zL33qJDHj-d`XN4Zp5y%~BhWgTo61_op8kkoP7OYkG7lY-X;UsR_Pmbx!g+&_rR+e3 zWhGU#FrER3G2r%4!L$cT_LNEi#xoEx!D#{$XlgLLEOJv0YZJtu)M&&1M>zt!C|j?R zjF8yMpi^s!New*KdsRP?4Q$MV#G={_=$E1bC%PKk1M5IfK|&zs*0NG zhBD{Cl5rKq8#W+Yqnc!@3*M}8hViwONZ7jUhN`HUZa8y+Glp-Z+=MB?O~32)N2-aM>qatHI5YSjN(syv?tPeSE*@pSs$n2GNO=wmgty(`o3}*qe`ze> z$0&_pmHB|WlSNB3e_dk*KSSw(J%(RZ@3Cl!;csfJ;TI{Buy%N!x|3x~EPq>L3%^cT zfqjRMtM^#8#PN4Egsg;O>Ml93^+^K!LUA4TOBIwv}xXTO2QU^MKZI&zns1y@#z-VoSuGYt>S;vsTN2Ud=Y-?mpo>07xbwTS;GcT6+*ggBjaUdh_WwiuYxOoMUP4TM4}iH-9@y)HI!|*FQ{tff8SSdmDH#ZT z(3IdMu_f^GH?_4$o|y>1DqHm>J(0K~-=^CVC1%>;l_Q3>;G?jx)7$rX}bdj}6U zo-G3SPwrEz%gGD$q`iCoK%Yw~wRc8UT6rY%B_HlR^GDUW6yU+2URYgE9+iB*_wFC) zzf+#rUqk`SBzZyd>)x(Es{T%SYX2|ltyNsIkd)ZIJx9>zQ_AcCZ(&6Pzk!2*m8d$O zQf}{xYPO0?P1cY)zVGf4v~vm|4^SVh?kDR>y;f{FQsC@aiSWqfS+SEXq!RX7apf-9 zRPjAgU4RoiEtS2`>d*8GHqZFps6MO5$u3f__a%b1y3KRGFKW>0S+cLx@V@##)Bmw~ z!S_djrgw6*)b>7+qY5rIFZlteajW;qx206~TOCbzv8m<56{8-eE)pqiwmbLvG{}YuEe#g(e zD?|{XLZ#&t4}%Kj^gBdit|&p0x=&hP(OYlw@^l>{DOa4ZhpH{@pje{kc4fLALCuvW zAgM;ufgsE`d1bl*k&=rb9H5?%zN2_p-|gyjBjP3i5D!z&NtY{n>rY;tevi1Ft4KIT zb(ii`EYVlsy=&srQ7Qy1HCXz);<$bf2yO9GQ5pn1HAz}V>A*23x0V+E9dK|u3E0R# zm98G^acgPir=fHS4%7nav%Ck#tgjV;vQX|Hg!9x2X)mS0V~y8}J}^Nzhj5u%FMUG^ zX<+SM^pOe5G2j5#C0(p^!JyH-sGXUWYfK2Bj!U9o&B5ecbO4VmM}yJ$2h5ZHry z0O3AWPv#o33ti=v(rwR0*%5N6IGJSR9z(QuN{@Xp3M_+D?PVS!&lpyDr}WxCLj6s6 zO1&)e9;_*&eNy`DAEW*uyr2dEB<{ZoeQt5IZvfT~90D4#Me@b?@>yvhwF%u`2X>XdI9YhCvoLA=h5B&B7#*2vYF(!pzab5(oV}}gK_KhAe*oJ zHWZj)rn$(zR!PJvkZq>;A5ocLE8bUjSmmmD0$KOIEf1AVFr|gd{#04|KSgIA5B2{2 z@p4sWZrdzWikQvJgb+f=GBakz%rJ`-orplby2*op38e`??@e<=PsVa?&#U41|;{`?}sfc;b3Fa?i=Z+4r2ZD@6Od-aUBc zdNuOs!INXN-#QII=%nl42Y-~e5uAPyxlh`>P5JRvW=z@TN zYgeS=Bw-F0*6yN%U84_%T<=C!ob;WO>O2r^IoS2_!9T7Kz|TvYQ|OEf5oLD40@T$K zX*?;KbH+I_#FE)H{@}IiqsR*<#dGdCYePhbx;{O4>-seE%E|OOZ=DB1EQh-Od+@L8 zi^y9ibLY%jj0+VV?)v=TgX>_V>7;s&2kZ?jF!O)#(RDbo=cFDmC)!X^R@cOXao0Cs z0dAd>x_BVelGOzhTvwRBp6r}cxEQxmbfoLs16I|?$l;UbImX5DD?5)gO_qGE|1a|W z$$>e2uN8nwk$#Vzbp0Cn-^urLh8CMwwjP!KFZO?~Frqm*W$rW=G>k^1Q}|PHKhS1K z=guX%2*O&ANvHCEz`>ndDhdd5k%tkoli{YTMbaFko^umiy2JF@$v>cGXlK)$rJ-}P zU6A2~9CBs19S%(+N;z|DTq58C!JO8E#5vGBr0eHibIA$E9%oMPnTd0x{V!cI_pwWR zcq?_Z<#QLf21TgfO+)a}I3g_?pq+lMY4BdP zgLfhh-e=OLxpA&75$Sn$Kk^-L9<+7R%X2f}NjsKj2hTtpuok6Vb2Y9$k?AMwX7L?y zezfh^?-Z%+uP&$l45kR%9l#)<W|uk z8=(!#hA}9zHkPK9GEfQ}l`fVKV^LJCh}JB9c;z%Mk#3Q9+4n5Lt)X3zic$A)TDmB| z%b~{u2Sx&EI_fQMfIhgUex72fb`$NUGzW#iJ2GemP4gAr+AXv$sS-uO$1^$$nieRQ zX}8fHNVO;yzLG&JY;siiX}8lJNn21U_<_*#gBu z7=!mw^>}uaz9`uVwL-g(_D=d7bqg%fgyLjpRERc{_DPzy`6<35N?pu^JqPXxZBqIT zHHo)jVoR7VJz=;U+B6xvVLl;XftJw=s-dy(^f&oDQLL!t{52^9Y<(k zrYr35d>c_gu{&B{ zX4iFpBMyk;vTx`~q74TNAm^UVxDJq!&cry8A~@=Drgu*w?hdU`wg?kK%Hd!um_9vX zTsKVvGN}!u4vxBlxvXa^?mn$X7L3Uv*>JIyOy8bloS9~jaWFNc2(G%4>DME{J*1tH zt;bv=<#4f8O#hx8xPDrjYzO8UsRL}?!%yWqwa;i*Wto^SqzSHVH90^56*pR^OpaOL z7R1Y~CI>2@kV3mFtHcDlW%6um$U%y9t(9hxHDlJgwexZz{i@ifeM@^PYsVaPo8YAf zO%LWDzv!xIHh{jUL@v)tCP2q~; z+8Ok@avYXLu8gJCH$^D&v_I14%Y8tx-VXxxCU`(==g?iiJT4LC1=w5Z)$$ej01)cjQr7eItI*fN}ix~nax`{ZwR=m8;_7<6N(``T)HQ2suL5AGvt2QLn9LxI(c z9+7{-0p&rB7q<>4qs*YHKg!N81zDB^R9M3)C<}b_1o;gPe2?aMaoey2^-z179-n{7 zVUNd1{9xPgcGM&7d3s|09S4O6b)ERq@D5bJwv{f)f9P<*LjrcB;Z)R9?PYpe{wsJ1 zG=rsRcqi()_9{Iy|C7T9kCAnQKM(Ihf%}J^lRteP%9FZYe0g{`YCzjbm*+cxQAe_V z@bd5;)S$MDt^_z4-?MqW7(6woSK7Pu%6$KMdpt+h4_+DGiyGEGplkCP^Aw)c4dV9U zbkvB}LT?7^$^}o!hQapX4AdL#BYI2z)_FbPHWB|aybtwG`;^|Ezi-~X!j~H?zjW>I z`4{(sek(tB-mIlKuzhqL==p#fq?_{9^E{R&ZnRwO0;vXWnBJ4GpT}LQ-6*=&mDw|n z19wG!>%7#Z0~;;Zx(@aHhkH+dncq3Da4Bw+=z7=To-Z(hu;yFlomrZ=$#T65Gz+-T z^bh%i^X|brP;{dUd8Jw41N12cHuDi)j+<#Wn~o`_>V9C@ z6`iD$UMD8%3;NNDYl=JA2n&RANodH!uLWrFXo(i42ZrZbio?4Cd5 z#jdFTHTfiJj?S48T5x3kRDh8aI+LX+ln%#W6coWYg$)q!WEl#hBQbadwe!Qh<%tA< zZ=>vWRK}`;v-1Z25On3`0mgZ2QzEeKnX=Yvl2?Ut(f6rc+?7Q`+v0A04VTdLtN!LMeVE7-i?HegJ@D%oKNq;XdEwIyyw&6~)KcBF8P^I9FNmA?l4N_AT(0oaB`~aHDOk6C09tBS z!S}%f|Fxief#0$;poiL3@&P`_=!Nr5+_ILf>G$la_`u|3JSn)mAY<9+*0Fnb)qH?s zGCcF{zzK-DO?-d28Wp5VXN(j)1O%mI+u;4-8dR|E0AsY^6|f+ix7ohFUE33aKg{@2 zFt*_BvVm=u({-m5D|JU1QwnVy5x$N+%`%3w<0@eEX9-dzy5LC|xnbr7!}BdX6bHbCX`d(y18aLII%daGn>; zQUgC4U&c7Pv~aWIZC_=IZ?Ck04}dtv^1|JYL%!ye)?R5NKL+wjD+`Y~P6NuCuSMF# z=i_yZsKOFQqMtz03NSOi5Z}m%fjtA=Pbu;3lLABx{}W?%;d#d-KeME@Px=#oH6Bo1 zg*P4Z{LtHdA4*U2*W!VhRrtWs;3wGL`cQg?zYhO1V^`q|KvOBV`~E3C%in25O$3a32w z^-tR|_Q+(jjHW`tLZg3tYG;2_n_|1}PsaJeMBvv5cKbY$UgD?XpD=zd+`Vwf-<;YC zm>B*p{9lY4g~t|7TaMo8`&0^e7X06g-wH3kby?2dDSFSi(vt?P{(FVh3qzLY?8H7} zw)dptM;LvDrx$Km-mz2tjQLB?KKwhzv%;%D0kHwb2lHyr0sJUqsPOK>n&lC@)WFv0 z$;5wRye)jX@Y?d6UD&^v*Lx1*Cm7>}Z-7V9u}l3I^G44R{CCD=;kSj8%WZaJUoZjc zfVYX7Ry1>wQ$WOS^$X^$o*ex2s98mefQ*o{8~c*k(US}3Zu=tlMH>P-cB=t5&~pMm zJIbjjcu`h>%^ocD{(Gc&bQH0Ov#2H@Vh`XtZuiJ_E5qH3;ul>BXg@YOV0VXKfL{=` z49@O-0WEvd|F-*$UxasxT2XXh(Kyt}$NskK;+NoI=20YFWFP30mOf}_;w$k`!7eIa zFUklUO&c3Z?wSE@;i$x-JBt*7 z)V<p@RFo)#r5!a8{bC8k@3yLx(bupmXd_ z8Qcva`Q(34f9PtXLW_?$PYpq462_7ti>}i~F^Y?vT|?NJ`mtmysu!S)Jjj`chsYtV zojii-(=|t}Dn9F+5YnBgA5VUb`cro{YGd(r2s9%P5k4iqK|R*BL~VrvZG8y)kRB8D zwx=I|DJr%2sq?jvoI}|EnD2U?;@hJ#ir+xiw&RfcKjyzZ&+*rzvWmYsPlnh)qLul+ z=LOt4^NMFKb_$I+tp3dW&@+I)18~Sii$g+l4r9MCM|%eGkk2f3U%Vl-Eh1Cg)4DKMc=!=^?bpPMOlk2i_fe~JZkygHQDnO|2gVI@!;ZnE44>O|LgkRGl~Be z^`&@h@!ORHM=cPi>G>ai3Nzt=jSC{o@fdB2af;v}{s-oalDRI#Fu}3bDYB`89|$v; zb4pxX=wZrZzEfp3f@y>wne$4#T#~}f5XzAKAecdz!+aSM>XIFX%qG~R*rFl(z$BIM zTdFXN>TylJHx^)&B z7LUxdl2pKe(0sd&l)^fLWcAZgUPhDXXz5N4&o@L4Zm zJ}f!wnh@E2LO&}7roj50%;zQ7U9%&RCkeAt7NLPt!F*M6A4aL{lls{yPUsc-eav?y z&s}dt%3)xc;*1W_XEKj|DtYJnGO`;+hbfEEEA>a1-%Gx`PNgBGgt;j$XxP^?rz>XT zYJ9?^>bWdfju1{TXDJrpLSW2=MX_McK#()-74Eo{nU|%xFkVtJ^o7htiXhxU+Jw{= zO(7_mdIgi9V8ckJ!*MqnK@_kF8s-wk8eAJKO*V!`kOUk8OfeM6xIS8oEFFVz6L`t1 znL&zuFg+NN={0xU%rS&IW=l*C4ncR6)8N*wj;yf;i&i=jLK4l4dwf-z~qvASl^5WEdtOLv4yapyrNs7C;8hT~< zaR=7CaBvGVw<{ju+TcPp=78`NtS5khQ!$L|gPTqIJj7DLM#44bA;o9hIDIsKY#zc( zu$gd+c^sxx_6(nb^!W&H0i4sB`HJ~?KSo-?*nEVKKuj<(RSGgbj?q$(z5uaIu$6F+ zS)~ZTXTTwIYyrYokWA=d>J&`8hT&6~?udYs8=;T+lVT0NjgeM3=7{hY>>xa5UQ{IG z`xq^S=?f9d(KqE!neB>w_z#Sc!oh{s<>=k|7t9XD2|Ox_S|na%4M3;q2btZ9QoJ|3 zhzA#01C`qd!^|Fq9?y-^7KxlpLFNp?8|MAE7I=B}7j-%rS19-E|7E^Vbl_!Cj>R-* zW3UpQ_sn618E=e=FYa_UhA0o||6{&~fW9fJvY58m7^=+De`S7FjKbrrzqoU;aitQ{ zfXpdM8v=sqSVD6#hAFf4Kd|hSXaWU(d^=r?;mYIs8LYWV9O2+dNr^A~*a-3nKeFa2 zy$DH6b4jbKEK&fUPS#>&7$J{|R`}v%G=YqOVYw+|2?nM>(TbD7m6#sysrN%0em$gcSIj^f(qnSwwJUv6Xs43Ja~$ zlUO`uEg_sGR}zRRY_vvCWvx=4B_y!Am3m@IG`dXh$=ayAPRM2c3^ z)U(*|r&GA-D!o5zr}8=B7E7)oxTWyWHToddKIJ>YOIEi^?}k9$tc9LKmhu~6l4YaD zl3B6lIzj|1PdSt5#Ewv_$t=EEOJJ}Hl`Z7~>`b*Sg(6Vu^(@x2Y%(#9-J(vXAcTSj z0+&^#3?OE(N7Z8#ghQth}D9#gbl1;ln038>`~3w62uz8dH6biRZ59g9;H<5Ze4?J(Tm{=FC}_M zOG*dbt!vS(`ec?_*+Aq+HvJ{mM(kJ<%hjgC5p(=*#-utO4a6B3I;E zCh|0`H(w#7v#d%B@l13gNKQ=~%)by0us$dUiT9$lWum2~jpl2F!>ljLG2+|kfilZd z(N2!X}-o}WshFRVycuR3OBy8 z)7O}y?A152VpUoai(6Sq^D|16efpnRYg8@pN$9T}+<0QU;7`Ii)+W_WQXYIXeEnrR z1dj<9S=&?(NCvK;s?}eXDtJQpnYByxf^?g!tQw?j-HCptZ)fdOy(7Kkc30__r|d%i zrN7QPqWVrEMog*32C#OUUl2N2dEm!!;zd-e16X^^1B5$3v|i*E!po_~2C~x3gM@CD zM&<6df!9&34rJ{$zarcRC`Pbb7SE;z3nUctFu}|+s5ovlyoefg5G%txLU_nJqgwBF zjh6#31lB(D8$v&;O|`@A8Ly*8y@Ivh{EqOPbxn2HZIWkGiw$NSFux}Zux_gg+?-+t zT8;-(4k}0WL#(?%)((uxthEiHWGct>R+dH8?6x+hy*4+5a!C0}|CaSs)$VpMW}?;> zN<+%ez<3{2b-Pu>1f9warDQ25^y92Is(!aCF`3XCpd3+t(|=)&sYcu$$F!fyT}e5L z#$hK}hZ&#TKE#Yb{@8jHJ;gAEZKIw}M#WO=#9`KBXd8nq8>x06d&f%Z2E(k`Xj|eg zJCvI6|0H=NE!>!+oM!luJzwoZPKj;Vc0OEoTrh(;hrL)GM$U^x*ZW4uas?3YV7sYf z$%a@#eQSg)&)kvg!1h#cB;Sga*ApUBPM~4K%l23AB)^R9uGhmR5k1G?%nnr_Ay4Ha zwFFv<6pb?A*bLycxboSsOh}QTF$NO*oJ}=3grB3u(phpd7Pbv>>eJ*6{0^8WvhvLi zL{Ihx^;L2f-$sXJunNrciD2VV-v#DNgiZ|)2D2m4pPj0HO1{R=(P5)lMdn3BxXh^E zke~58bm}NpvDujz%Fa@MBTw>e^jIdV#Oy+hVCR893Bd@`=Q1e@CC)%+7pNU6fr3oE zEsLU55)3T1T1};_6}0Pf;Ri(tA5eC+I*4*mFrl|)Q`AbbfzQ^f+2CLbGT1gw*9fTa zJo-t!hSDZTGmNnjr2=>2TJ}YCGNn(@Vn~lhlnKE4%>G4vfRd%N8U~}SWoR$MX7EFD2w-A$D|(!AcTi+P$3_~@Sgi~+04+#urWl1d%tfB8MzDgopZ%};1?9F-+2|W1 zs}+P0FV6a;en)vJR6bC~uuhpniAUIz>Ti@up-mGumQ`mCBSH*OGn48hifB^DS{n)= z^TD2}aij){GMjAq6s;20Cv2>SN?j{zZ_4FUbV`PykiAF~L_H{)XtEVh^h%~d0S6g2 zwL%nR%oR`!O1446cGtvHuZS{@wn7Rxt_&4yUyX$NSk!LJ6;c|NJVOmTSd&TpBAPJT ziYQIWSVKLVrjb(@#053yiYP{Y2F2pDe3PXUyw} zm)UzXgVcL*+Mh(LO=ry;h*#MM;YIN_F7rmmYU4QpVZjY{#evzy-{NdeW7n|Gn>Q0X z*m+>wVtorZox6r|!Mqi9o1L$jzr=4<+Uc=1lr?Cvp^L55ke2XQHJ=u*wYH$Q8t$?y zHU3NXtQt8zxYl|Roosl()@m3_6sxFb#PQZvw8UUxH)~ccxv)xdW-#8`hTdU#1n;M< zOL|r{pAoN1nWi8YK4D+hyty;D%6z7Eo$P19F5+M88=7NFrmaSw^<6K!EZ9T*oBf-n zWC;;oX|3yJR|I>B|FHki)GeW}R-X0UAZr(77}`A_YR)c6SlxYAzaixp^nSxT_H)hk zCE2Ty=Lj29uA&bbM%k}4_m|YKW}nkr+L2Q)@u1V!lsn#=q$qo`;+F~ zl9#Ky&*?X%+&~{Sd}n{xd|xtk4e~r;v+?h)V?>+iX{9sWoz_I0S8rzBGUpJdN6#u{ z&kIXAfmCQRQI)Od`k8vAbu5`AkK=;zVmFT5iBbQ%hI5Usxl#Fqmk)jV>_EkM=3u>fW=a`GRe7 z#7Wj}bg98TI==Lh`<}HUErUtc-_hj;uju2*PWQsKxQn8#rn}|}$i}6VTHMd9O}uE? zYPx5xB3@X$yL7VB(OO1qNbtbXlpFM^e1GwKYZ7D`+If zL?1m>8s?D~535|iBn#ST5JWeW@;vI}*=>4BN+0?s!>Z`Br5inN#mn0W+fyE*&luvP zFO}}}cp2Z_rr)0OC;A+5l6I~1u*YP)%_Zy()+6(IVq)~rPPxZ|bwQVMcTgTHTMS9j zcS|ch0@tAnKkh*E3tHi@*IU};5x1`8QhF-liQp13HTp?u+TrrUFE3eAO;5}}6L&|y zEba6tT!;HvwA1v|e1(`EZ7sEUoLQInvt_60nfVvuf#?r#D7v>!`?F}5>ACqD@o@B) z(lL*>>jr+d>@xjjzCk<|J+;i%bJlv?WzlZa3-c{vE<84|o*wHHFI#q-UYdU;N~2L_ z1W)dI?Pbv((}4Llu^@UunXhN+`hm-qJ*L0SzY$BKUCU^mh3j!wL}{i$v&rz+Y%)B0 zjqBsDbfy`Hl)oFwqP@xz;Vf2pg|^rDN_o#v6&+BP=4oBuf2DJ;@gLvUzaVx*=Yhk|X=B7yHJlpF z10XytEL*fRWMj@%>_HZI-C=32DRW=C0j%xngRGC{S442Kmjy4)+GukPo5>n84-?JN zhBD65nvD_H)S0Yt^9b=_^qI2tORsIrxrRN&`ec4X?2m3M+X1HYj%(^etpCjKh|i<1 zl^q6cxy^O#Vb*8!d*VR!?XrTUPMadG+xpC0KZ2MPOb|iJUG`{c z+orS|1fQWv^f$xj=+UxQOM5mo-w+?Qen)>de2bm{UG|4fBR2+*TK|Wh(lCW%1MX|o zX6jAxG3ykJO@l25S?=KFy;*W|@R)Tf##wIF|c+8PZ>X zQQGuF?~I0-Q(ejTCfE&cJ78JIZ@&YaNlqJnRmZ91^vSni!m!g1*31;CVx=upF1y%$X) zak%BxULgrN9axYb_hLy@PF(qEuMG(u9cnNc_d1X~IUCBadSxX98K<0}%v8;9@aBlY zLR^s$@of)ChlP$LKhE~@M_z3SX}^x0K+F~{BE@6V%3pc)BsBjjK53ne0geeLv;33S zhlJ+#o|C3Iy^Bdl8^%b9-G$G>#0!ydH^(offGH)r%)FISR1saucu%H(`4Lj>0j*J>yJb;%y74YI~QE z)^IMAi@on9YHy3cJKO6)TF<#sp6>lNap1NEB(l9rNt-#h%5y;xi@PH#FwO7vCW+y= zqxSaLl6c2bU|P_-jFb$^emxk2d(j?le4HI$mgJVqRUoEA*jIK3G*4YPlY(hl&ho=RqO^0 z#~sX@SHzhnTNQ<*;xtrnd_kl6Sln*PRZ-kjyoMT1FgP*4h$l?8Y6=)H8|pc<3c1gM zq@eCxHDwuFKr(QmL00LPl-50_Mo@(!l93|>^JH97OLw{ku|&8E?r!TqBbkvj+C8R0 zxC_^iE^xMh6H=4p^Lu(J!b2EOYU6AN!DCxe+V5kf#_7}6H(chVRlEXQWApFgGOH(M zW5ZQWX2mB^FOK{^SY};{+1zlGlT$H$+4yvqyP|RvXcI}D99ac+na9?|yB1K`_HH48 zDz$>JjJs8PS5#pFp&}*C+A$n=4H3HD)0G%6;PN;dcyg+V)wEkxOUnGn=8m)sY{Rw= z(B!dJ^zI|Q<2Z8?8n zPq9EgN1EV_SG-yFY+J`4>QgK*$&tQuCM&)zo7`se01Fkt-eZs;>diy?E=Ue~kXuIy zSApY=JG0W!H!wN#fo(k{LIuJ$$QV(5*Cw|=$gQVDs!lY_;x4KT@;#V5@xWF~p{b;x zx*~u%ts*(7Cs#`WFIs~=*S#{{_X=3gY;_cds-R&3*KK>UZ(njtPr42P+ocBMoS@2m zz8^qPHmI{kVPN#jji?-y&P>69Y|O;$RY269RjKy%NJ#{9n2FV^f+7>YQV&8ctyyF+ zf%S{z&RtvC>YEBqFN?tx-CIWT;wHe&y)XsWD{3%tdMil2+@mR#7T+@|iMR&Wngj`~_t`g=PYjWH^y0CSI3+V~+Pjusk7tW@<4kz8pd z8WdIWmd++4zt?CV#VxM%pB*NVTL{J!0S5F^+|o)yxzfrT>)j z8|rMsdTx8=KEDr=k-ov7tgA8S8#Z%0Dv#l2ZpS?ooi?rMy+9Jf1xgJjq{N4o)26k( z7fH#GC(?t~Nc&K9#uVS%M%uxBT-oZEx_tnYM5cAUKa+NIUsiVd6>i6YU&wk?ex+eA zcNi{VQ;mGE2gx=Fe?Mhq%4ucYZIocmJseuMg%%!x8TH z%I|(tcOV}T&Zlg~fRl$iy=u0<>kjrK{rQvx3Slz-(>g^_#ay_cn`(N9U z1IihSF7Iwb1=qJq;{SL@`{Ue;lx?c}4K>{0s!adKKYn;Tb`g;*d_elRB%Ie!C!Lp!sj4RpKXZ3i4fvZ*v!oNws zaf_>5meY4CpZbF9LpVhGom*DrwLEF3`DrVNJB0s`{@|Xf3R|8B^CI7NS-KFU8{EdK z*yVH_vfnJ%ZGNFL2V)1FZ`GE7xzZh zvE|ctp`ZI+m4TLm^f&jns*>fzU4rMWS7qRsApOJrqpEH>9Sjn_*JPQ(anfta!`FZZvio6GZHOyzrBc324J20+8MJJ;=E!`vz*3-h(%6SsW!)8#e( z2mZ1J>q2jq^B3+%kSz4=YWXYu2I8pjJ86>pwd(Wo@m>9oUf!@C#Y}0O!n3KK9)Q|S z1>1r37{;d2mWQl%2=Lx5c`m!&NgNwMcy_?y%-!kd0U#wR%-RVQAt@ONS}1_eZz+Jnw38z&*${T5Ri1 zsDRYZ3#d*Du=H@%loiOeVy* zx$$DE>jLNi!1ui^lMC(Lmhe_rpASfasa)%AS;3Tfjh?)X)z<^E_aOf!+({|GENJur ziL^sa^`3~o)iAf~O;P&sc2xHVTmg!_?QfJK)uP5gUV8ONz~epbf9C=LUgg{v!aH0& z0ets~ziqoHB`Tn|Lj)5U_|m~=Fue<*5aQhEy!`6g}u9^vabDyE~ZiGe%^m5)$)oTLV($a>;x)FdP zcU#N5Se+c$2cYrv-w|a(PZ%ivQhgwBJZ*Gn>~}=D(911>_iME@(0;Gat8^fK3w_*@ zc)wSd1J66{)!1D`r4VS{yx!`jz_`6FuhQ=!s)T^v<`v>D1?~aT_TW8h6(*o@H*cW& zPM~5h^&jzlYc(dQF&&`T4+AgkmHab!-&%tSZalynt$qbS>*jyNe^_fVp^b-m`Mmi+bWTa{AMWfi>QqsU#XOgqh#(oL5@}{5u$LQEJaUa7$e12)?KB$= zs_4cto>xs`kSV>=O6xT?sNg4#7f_QHWKHk4cJ{J<<;4ICXl2c@plKQC5nqc8c*P(H zkE$sNA_9%L)gm(rg>H?!n3_6>a4JW9`((|+I5z+X*O2yZ&gceCFk)-RDmTCa*IW(C z%CLEjeaK=+O-O+L)^r7FGJIaA1JYL*@7BiKUh@cmy=kw<9wN>P*SlQ-spBxvcUxYk z|A{y!1hy{kP|fF{8qVJKvk&)(N9B>wnYv*mzO3t?>_DT}=eAWgXwr`i-rs zl*V3O6O7x8`{Lhr_8Z$&+Z*Xq&ed!NU4 zyc;#gR!rNE2KK7#vT%>v-@M;yN?>Fs0FbKeig2&nKfFI`>Q>PAE8qD(leG&o++Oqk ztU13T2`133&t$&{_q+Ye`>W%ul!dZpL89Q)i}ZXRP%1dOTdEa|4O-mIokN0_r2!(im3;X?+GtbZep?zhg#3zgah3G&P@3gbFy)Ej5A;{ zvkxFY5C&2@F|tN<3<(&Q^#|A=^aClkG5L)SF-JXXHwNE2Apb!4JLL|h5NwYAwL61f z9_ap{2e>4rxY0Q#wDw5w)Pu-T8jlX7NH<&zy|y6O>0rdDdXQ!6Rl1R4xV6;)b<7#X z4zap>)o#?7xZ2af8xD4is)ty=_m;YO#%!p)8k}{|<|FnM>uzs3q^px^UzL_04EmV+ zigHg?+2|LuqqaZz%E2JnhgS&T9>R=%Z|!h!A5aa`|3UmAtZf{3%&h$s{NdmTAPTL2 zVCovfVsbzOiprz{Z_xSxqiv+c$N?tkohca`9Jcmg^srx5){-E@-8?3?TFsb-Movs+ zt$)a#%#pD{fcRmW8e?O$wG2S?QOCt2Rtp9Id@(q-D8!q;7I=HcKGjc+t7FasP0y5B z2{~{hu<{z$#RwvIhYV$!$6H^^{uG{b+Z1!7_WH^xhmfBLZ&Ds%F1UTe-v)}E6F}?K zZ;TEKVARFjt*s0RJd_CpI!eE)wJ{~e0(`l(ha%SQ1)QAll3QxbliJH68HYwcjlD%Y z73QDV9WwwZIK?6Af8ux6rMSR46o@L%g+n468qVr))L2Ur^Qv-rLB1*W4hHwJmi0mx^P&x7x+FEO2s z(irq95@gz&KZ`$D2QYUU3t}9BNw(+k$mhWi*1s`b(4NErU`%nC`b9iy9mI4us$;09 zR)tBk{Se5JpY}d$_!H?E|Fb^8*pyYZY z6qQ8<Rx#O>#pU4WRnou&Nh z`yZtg^Q^HwX5XoIp)a$#zv=-Gh54)Tdd!hi-$SP!L4G5API=#3OX!HnJ2i8q6M&r5 zpIIMz2i)$&6rNhNG6bkh*e|Tn-a)tS7|kj7l^c$9d{cj6ee8YZb{|0O!7H;_9Wzg9n=1c71h|*j#dTp}dB@OG{cPl)Y!FT8SW+D?to|5#ssYF>*dLo5 zVm<3N0(V6|mH2~XI(Al*W2}GOPJpO%Pc{4?nSljFN^EG|k+7-RNE@Q9#14yU!o^N= zDhP|47hscTOGT*BO~hDE-Gq~GcACw&EfOihk}0vmx~8zW>=v7hX-MFVkUe78)m;wD z$R4#BpN0fz2yEN7KpR5?NRW)_$XOysvR~}>Iyw4$w&aJQ=_9kSK=X)A11WG%cJmKO zGe%}(otuJVGwVKueaIg9VQ7Zr%H74}u-NRn5>G^qqb=R8d9K>EDKb_H&iak@eKKxov&NTZU*$x(W=#8}1?FfIC(*Xzw_B@Lxd1LG~ z06a|Q*i5&d$)0cVA}7XvJuVMla6Aa$4Acc`pQfbPyMSH@Jf1my+AON0MLi!B&d@!H zJKi!q1G!KHw#e8gb(h04j*m_sM=lb9W(fqyT|hYSnUOIY=_CR{LG0_gN8xS9(`Jm% zMmmdtMG*V3ZaBQ}c*~58IY=8+Xw%`?iMmg~5g3^lgYLvu%5v9u;B>@P_X{M=@{B-98F%V;V9dA5JVp4<_;+sitjSx+vB zb*-mG6hd!44&6<#u*lGguNOs}$xTG`p}UEeXfjmX>%|fGau*O|5?lP3oKJa}Sa<}^pDuR6J@Cstq2j;uHl1f5>0pIXw?7kj!sKJvP&+Lpv$UspVc_{Yx`f_Nw zra`F{86-+4TfuhS6d8B2Wp;)W5(=s0x3N#^FGpsa9GyMxgoJV_8GzOGU6GoTK65gh zkqZy&@yO}u0{tzH_&_YOH?18y|nVAMqbL_ zq`e8Oo{PxYu=w-+B6jFhA2_8Wf1stxYNLuWP2S%@>rXc_7z`E|F9DPoVpgkw5A% z?txq{x=h}~{~L-tntY#m8J@@uqIPly|24F7+Va!pje8=Yb3;DJ|EL|N_vN?D%UFtp z0u2NYzG^?y$MZ+$jW0!R7TqLg^MBCUG3*O`=4W^z6GR=o z3n~=jJy2fkA?x^2x)KHv3KDI;@)S`o8QKrJItIN^>FDPtmx%hvP-W1aXCy&pqRmgf zUGyjU9DkGUCL^y9z0l8JzC-kwe38EmIt2!(5VZNrQ$6lTP zA|I!WAY_JUg#36#GbGb56iJ+hf<`j1Z<-$QTXb8adWxEzl2(lD!@g^J%5T^0i~3MB z;xq(_aP0f07m)ur5rrzILb`k80Cu!#klzj2ZSP`<^HA`}LF`!5F#o6-lGyy>X6K}k zkxVQwp85T{OHq4@N1TU3Mh;;=H@)W%=CO;;cN`cg{ zelC+(B5-L7lV@YwOQ#EF>z$e5C2|)c1bng6j7Wil-jkV7((Ph^v@Ujrakju&AIi)w zLAnwF9D_v|(E^g5$E+`5yBZ=Sd02?e3LI9gXI_JptUZKcEi<8o>8Ibpd{)xos)2;7 zWj4iG5TZZKoGh`y+0y|2@@uZEfTou-7eE#@k4BZL(MF^=76~ z5s$ykXg#JCk4c(ss%laq*CNhKCC zZi0vUv#bPFHw1(vRaj8?3Z6qgEn9_jBl0AWJ~O5XUg__%>Q!tv1Ft*mb`WKs;H~~C zB*${x>|@xqmS9SzU|j!(^-R^_rh$u!C6sbRFsc8>npD}4?PJ+>mM}_=V47hj+esZk z*2J>wEfJIxf?0+|>=1Ph*&eb~78(?4?G5gbl}%>AioK8xX(LLdz|XLQJ-+ucrEhXdqljk|EDJL1 zV}DThuYDlwZnVTu>IC72T=pytjv6QGZnE$xdI8IzW_xH7sePhuqeTdZOKpsn&4Ppu zJ+8S~9cTPWu*T5BPSNyJZ^t$Nr2hXf&1S<$M4ko>Q5pGZ(OSwy!8QoO7&L+XsWBG_+u&mPj4m$X3uMYM@>LvYkE1>z-WcfU3A^P&XG zuY!|?xzR)jUA3)|Ul47fGzS+MoTI}_<&eUXv|y8r-GWksXLLeox4U7j>fOedf=+`RG6_MRdF!cH)Q~t3+%;522bN`e z+Owy$i_$4pL9d}HIu6nT85@wli1tw+3t+e$ol!RGIlcjTRdm4k(tf~jCt6WPU7ECU zuEt(x#E?*dxYof(gT?=nrKhONU@P zjy-047(Er1;j_wdUU8edVbx8s6(Sn29FOuuuRhpYTaHuggs27rY^AkcaS7cWmORQ# z;erNVPHOppS6_mhP;$~ZN9fWJ0qbH%?{UdV*bY-L!pF%xSO&Ab4T+N5*nFddFsE`o z=URD=xBV72EOIH=SiTJs&g1fSSkqE}Qx_SXguxA&oRsiU@9{0jF6^8N7hy!h2@VQ2 zs!8IJF09f>fDd0O#~aq8L*fw=Rt@!QK|=$FU(xK7lr++fEj79e;~Oq<_Ee1c48fif zTW<7%ble@-LQQ= zABoOW5`@1tNV)b^KK>c7?Gd$5l7zoElym*6()`DFAp0#j*~vn4Lj#vz)$E^?I?|7Y zwTrO7;SzUG)rkL4>c|r;>`)*ac!#S18+y{tk*8`A9u^{y7CcjxxV&$t1bgfk<9^}$ zh5@drs?y}OOa5GRjRL!ihWFf|D)aKTUGl#~Hz-Gi|7)1Sn^uhu@Y^kaA-Y95E}Q{N z1Y)%yplvssi}}@fLO8q8nHOFy51{+A1}vQv822~2^EOm>1ZY4SYq>)y5;`>o^RlXK z0`1e-gO)CeQb=s%@M@|fV9d@QvUF2Qg&vLTdDlR!X}_2K%5s-dDfDaH!FyKS5vbYA z{>O5kQY-vFiq1Wd>HYuXQB#>NWSew3rPS9pbLr@!%PFN(I3;$UT{UG>p(d1TO>Da9 zY^1A7vAfG$=02uE7iu#~7d0oFE)+%4@BRC)KPt6-KJWMI^?W^Fk7w8kC*N;ZM>dQe zwFPRnTLIt+W$u9it(tetb`aa;Ru~$1FK>^}wE%~A$IOF{&x?yfL%>LJ-!+LZn`bsV z8j5A1BeO(0Co5d@)yvhKA4eHXcr9cLbL1nO-l=bm9t$u$npX3ZZRUlvz| z#@-v)(|Jt-gjVw>$5+L*p^5j#_6%P$Z#03ZB*!O{dP*5U>vzk7_my6ZWceNG6*upixCgjU{*-fOzv zwOM^a@Er@{4?-J&_6E7(u*K)3;0Jct?Oo`7Af&n82mw%i!8rD-8-UpZRm}fJ&K92} z0nohMETP@^RD0RL`KeA4{KNvjdFbnV&-bL>u>H$5nFG|Q;*rpSdyfIiEa0!gWX`1H z&*HJrPxoE{;Yje8YYJz|ak6+KbnM=zy>D-{{Z*L4`Rxcap3td#Q+urdS=cqzcKG=; zB+zS$?1Gov^xG-|YK-0*36Qo$i-R}cOy3Gi3$WJDKmvqo5jvQ9QwwyGfa!W0W}k@H2%B-;Rw!gk-nTfRH^nMiw)i^LzBUF5uP1E9n}vXBmrrlkCK?BAp3Y?tCT z0juy=P=o8iW+U&s?H{-;$RzB~q96N4LHRs--gX&oD>4l`y?9m#?2h*?9w>cren>tR zTI>W0+t^*B7i?W{plro100a*lC`x%3ZNUx^@+@{y@%j+n9hZUri{=%uU6Kn}m*Sm( z@qygU&M~iq1xWtIB0(`&atC?uIjH^gfkFkGA^yPS>;wGD zI2;lI%~XVYvYqeHg%z@QQ~*`UKQ z{UPK_Y<2Om(2w8_(O+o;vuh-{A%KT_&OMhsxT|gJ^)blz*oVdcg43O~M}M_#gZ`)_ zVdK-{TcNw}jM1A&Pxp z+!lKIo-a5dk()({l2PnXac^k#y+m-_A-9OY5CHqF*c{pe#CJXjD-b11Cb7T3$uxd% zVlU(-5}Z2{E8Gl;Ep}m%Z!rHRa+@e!0>RlyoUt2<5`!THNI%hO$!we>c;8(MN`aha zyB(K-oP%2=S&!uvjRdQuAGX6XCG&AElAQnogWSgkFCOgw-!r5Hk1Z+UfVar(56hM~ z;k+dg*oB)4_Tg@~f$exC*anvz!+tF42hVrg-}>{&Ww>p?!ak?i1w5>6yYv?&RnT3) zzU~Iv5ZxUKSWl6x#sy1mVRskjgU3h`AQDMXI3jpRB8z)NbcGV|1V}t^42c$-1-@VZ zyOJQ$Wyu=cK}i?37<^f}yOMuIS0w9j(UN}f$>)dq-;?YSf$IwwFB!oOf=^9%PXeY3 z0MMT*vBu2?AAx_7Bv@1c!do*X4!}xJ#_GTuEdra!06gr1+bzk*`WH(=MBqo`$P`Z; z8~h}7u^<$856s(cNVekm;Exqv;b39lcj4|z_TUO60k}d52M3c%@HmBphHH`taFD_P zyk3SR<&uNAF3DZoKJd4*;p`mBR5}G7izCGBaT<;9uz=U4`E#lMvY@W)`efl8vhl z{1bmj)=$)zw1w;MBQM~VOLyQ0?VU;f+p_lK9wIM+xppwVP!>W`mkPjt44I4bkTUQ( zSq{mkOb~&4guH@VCyfUG(=L*_OmG1A1bGAZmoya*kvoum$^{2;oygm`zoi$zdP4|V zT`q{kJwx8ZrRher^+`0iIO!++TX`F$u#$5G_F7Vp zOO{UItrWIlu2r09*jw-&W=Q7}mMa3n3adCVuy^1iKL=vNwkg`qI8+D3>fa;Vz_*Mh zP!-xRQFXvk{Q&YY?wWKHAxU9P=GxM$KL0!~><h2LM!|XnJsal%9p`!M5e)x{Q<#4WllsN zRV-t)(Ka1712r4J03e}os!|58$@Ub^1~mu2NVc8`Nf>09n#`wQv!wI!F0!2f8ii!C zwdT_>Td5-+DZ>*>R2-&BYv#l3q)vEmSp>0DC1J9g%^9${(#3dRSsZZ~Jfx;(^BLHD z=`y^ZEQ4rYf@HB<%$YC;>HVF%W&aZ0OIR#@OIwzH0bq*nlieZyQ*w)i)3pioFccb3 zk_|v2OL|#4okS=?NImcjnU8%V60@{zf8j&UB^Ypp;IGTJkkU$R*{*Gz ze_?K5)8{UTlrAU@U>CM=L@<;Tk5|ZOq$cowxwdn}Fn1{#Um-h8dRyAYE^O!I!aSrj ze7!80WL0Jx;o8B;gLz3=_*U6Dz}5`_%Q>9Our<;M{A1a5(l+o)x<2CM!+fM1yg{ZQ zG0WN_3LkNuEI7*WAOvB$E1R?fCGh(IoDttrE&OAvR9-g z@D{s1;arDpmL}rIWS>ZH%i0bUKH=Pe{UuGtPsyf8R^_$_U7vDp!nR4%@iy|gSzH2Av7R+BNz{BMjGPAtxU|}ccHtcWdIs6~QP2{9<>quBv zz#aWAR1P>Ag2>m({USwO0fqVilo;DsKq4M!5KJqAX48A&&4^9u! z9_cmwCV3)xynG@O@*G(t3YHe&cgO|gg=$|8|2eW)6aw~1g5>$+4eCS=M30n+u+m~Y zRxTlv)EW*SoHru8RE`gmYsd-e2@b@7l!=Ja68wI78~L)@m&-RGTOCe-f)P*(Vwk!rHfFTnwiTC+T1Z%< zI2gt*D2idfGq=LdNf#4*6>(w1RpuDeJM(?m1?e(^pCTj7z8V?Jes6vN%aN`m1SrH| zUe%mf(|hwn*uMaU7Xr+=`>G|eY?HYSCI%M+Sy2%dQ*Dklnau65JgFyvrDzSiSdBc& z9x!*n@}++gI11u%Sv3n>9c_>FS5WH-F^czLkE(AS#r@azSbq&9`X(s8g?+5E9qN!G*e8lYh*AP|_s= zD?5Zdg*P=)(|b%eD0wEjC*4c9sqh0Ua?U(|v*fv`SQ<)rE^qNI&;;-b%^W>UD!sT) zt_Y*nY0`NxOMpQyLy-w;MHID9qveS#0o{59ibkkYoTN@_tm9xq0Y<$F#Ukhw*&x^6 z58P=1J^E5q1mTh5Ds^*BdK~OyK(D?W#UbbwGAgx38z=f0@IqgKI!x$O)KZgbtjSiN zkS|45(ip;fMH}^Ujqh>(CuE;UBjpi>6us2yn#ALf&&XGzT4@5|8>nx3YBb0BpOLRc z_0lB5FU16Pyk_D!#CO&kQI;r`V6C*F+1DcD*_pmn9gtwa# zOOw~K67(Z&CVe|9kKm=;PkU6`U%TzAWI*&tdWEna+{c5py$QOnw&?*+P&WvFDN|{X zI^WEd-#8y&ozhzXb9Rolye=TI@Ed0k_Dp)0um@y}ZmSa{x{h+pFg+;N@JauXb@ zXf(j0H=q;*sK~;EVnZN(;&WJNlA(zmuXL0mJ%! zR4d`Eaxr~#efmk*c)*DMJ*tf$Qljb9dhJQkc)(Zv0O~Q}nsO7J*D-z)GJ*Ui`XKEj z+yRxt<$B*F{siaT4l_zmkSd9Eb$x8o=!ESUZV1&wC{rG!>+4ICct35wiymKnNzf?c z=)?8qB-2mxci0!{8-iAuLAP%}CbNGbGjuo4ydykN=Fq9R!O7}hf^pne)PIC0%A0im zhMZ)dNdYKFP!>YBQblJsbS0}N1wV1$QJ)E~m5=LE8*Ectr#Qd#e=iyVn88-{%&GF9K$IL%;%o>gmFn_<86*h<~Y48IUH2be|c*S$KQ&3gX|Y z3k=t$kaYD7AtJ#H_Q`@P+ zncR8sKV(~o$*L)amDcvO>n!ek_;Q&aF+(+%xm+7?x^NcP9=-w;aOYG@ncK92)2_3* z4scgl08y;MFqzsmu>Hzi0ADTJL%goq!tC+3;_LUXa1{SfwvTv66~Mf#_2u(zQ7|!5 zh9k;UWM;KCkw02(1;?Yo!fcu9AXBd`<@2CW1RjH?5^GfP%u($Ke-sLJ!h52b#Aa0{ z6V~jV!Lx%d#CxOJ#CBC4)2BH$W7H1njQTD5b=eoipgsp z$rzmjU5sCkjv>BNJz$EOz0dIGwlA^sl^r9RRR(5BGv|zHu4M^)lPn(W=?*YEn0c_MZ)@5B=d<-1wjwgfJ6Su_3QtM)c zqj2e;kT6*lF}7rY)d>n|Ho}5|(`2>8#F8=Au+A(rAuJwnhOCj8USbn&--^s;J6Sy8 zELjVfU%|t@S~=M!CyN(+zw7}qr({+5K2UqJ7h1gG5wZ?qe#zSKm{#Ya&V}u33=AmLR1CzJ1&gz}peGFW*k(j=O75=a_|E&Ka>00q2*$ZM- zNi5ix?mepm^}G0p>@~5mBt3lgedlxji>2$uF|v2W2PHY-ZuiOObc>}M#7AZS5ucXa z3eS3Y@tn^Rp)css6`qC?ML6?*+quFesM^jr^e19p$T9} z%Te3K8L~N~#ihRc6CO-lfViM`h%;sOq?Mr4@Y&&Yk>>&hz+p51?3NPus~^N(9Cd;2 zG@L!_MDi|;*x&g;a*@5lvJ-wzwwUBw8n=J=f%&3oh2?Mf1=%u^UunjE`-jLJ_Dahx zcn$!_1eA*Rdx37nw9>L04xkmJkkY&R_dS&4uw5+yaIp+cB9~U|k9la$F}Yd-;b7g9 z#43IBDf^+zCETj^Aj4&lx$$aq*Z$&%}(4HQ7c| zd}*BN{ll(H>ea%%cwi1CrIb!%L)skv_4%JL7+-(}RR7ZXpeG9XSN%WXK0Gj1kj|Gb zWBa#RpK*2LhQJGD0VHuLhRtki`?t`I8wzK`_K>cXZel03S&LxEK&+t%ebe<0XrnH- z`HJ{R6izIW;YhMlGP}AhQ3OGuKqn?6k;+Rs?4CA_h>t=M#Gqaw)s`l*$J-`E5OBDP zRWc^21$15u+kM4+Gz$Dg8JpBmn$Ok4i;ggL^iJJg!#x0h z0@}zeWyuj%kNooH{~37D@DvRI(Ph~YOCI^5(R%?}Ors4Kg{2xUVXL_h=-4)CU@cH=;{O;bjjZM320$@YX^P;d{|)61U74 zp?(y5Wppj{F#aXFnsgM)T@t@6ZoO7t2;2_y-W1^>vb1u}RiW$}96e5c#k;~0Jr{fu<4EEsgaA=lKtLLT0N?j|YA7zcC!h2XPM z7>ECeenF}%i#}j_+;vU8QFt8x8U32nP?mZC^2Fh~&n96!ei;3ZbieEZXqrQ=t2bHv zXMB|nkRF#^Kd|kI;JWK(ZX$dX$bg12#R2A%w(EtPxhLTF(4RCBj;UGYW zj4a0=ECKDk$4S4Rowxe!kY?Z12<57S z?9Q%RYJVZ1PrG}N>E(?F`JE28eRc}7@eAG8kPm&W8VR|lIq z^PB7e*Ay?lv4MQ7+)7Re zhqnuQ5%1#ePd>dF9_iJ^xntU8xd>k=-vuBRt0If53hv-`x91pK-Gj)N0oCGASN|RT z?)FQD)$YOMg7O`aXS-YqaRKfB8rot;gC{U9vx}w>MB$R3Ul!u?sRfPd1@r&nZsS5AYmRJ?30u4mtTl#q7A;T8Nk_ao%j<(sNgpV{7X-NU^K z-yn}A50pQSEO-`huMq4N!8gj|$e+qzMK(PX000LUtb72;WUTyC6|B9S^W?cje4!mjD>JCEt)OMUD&;5X?5?EmHbI%~pQ7`6fex44@p@FvyJKO;~ zUXA8ZpKFUm`vPwpcDkPXHwO-}}cB5AQWRDKVby>D@L>+>c4IH^R8lS|1&btzzp^h$I% zsZ>mmE6EH%T0gCKkow?-GCaw>jC=s(J{9UifU+Z$<0O&AJ@KpC2 z@^SSSj!EAoRTG3tJl(y4oT8q_g%})UK187k&vb7lpHa`}x*9@cYNCZ~1F9ABIrUQR zHiJOsO5&En*+4=Pt1(=rp-ol@&N%o1`D5~R^%icL!B*}{=9a_3qe8w5HopoC0RVEs zRl~V*16iStOQ|)a%VCtj3PY58FVMlGxP1n#Tto@1G#qw+MXpny1G-dTwISAhfc!{(mAkn+T>+y8Y7EB!XkM?DajD%}g@_thBOWgJNPel- za1**G6c8G!Rva%M1}OSA?&WS@B_Bvx;zao?+#+@G-`!Mj22o z{h?HnB*Kq7W-e zqau5HOLc7MLcd&h4+_1aF^b>gQ05aM>=2*4>`jTNcpO#G6Hrzd!F>e30(82)J6=ZB z^`w`<4g@|nTy^)Q@G6F)`g*iwq62|X4AvWRxCcWxi`HW78%%OxZ@76PZj7x)Lw152=HnQcis0;Qm$2OI+WCF zt%h*|pBswYL%`G{=+HF~P$uF8>J1XP)t|Bo@}cV9L^XtqGKj&2ODP9)k)B?Sn$Jab zixqMzrM4pR(0K2J8WM#vidAwZrKLh}XyFUr3f=JqF;eLoBiK;jh^TJ$VI%IhPuL2rle?{w|i!YFs?8BCqaE<&J z#av-HRPusTX*z7_gV)OADI*mFhdN(KD%nRYui*9alaz^ymd>FU{gwJ7?XL|D?x_^3 z${B~}ymYC;MYq2(G`YWCFsstxu-i*=l`dNPR@^K8&Zjuo2l?e4i0k4na2 z-OHS6pQFM7{3G`(ly#NShfM(SqdqG95C6pd2Ia5H)WeWI2aV4$;Rk%D`)$hKl@1Nd z`T{hC$GC&=XY#w0J(brFZ|f6iTzOnGTrZbU@Rf?g%)T~FA&+Z;cLTkYR#|^ItXoeBc*&@Wm#|=}Oxp3t#!x^5ap%;(ob~l3nS1WW%e(T1WzF z1i*+k!!A|+?+ETyX)P}S`W3%U)bJG}g!Zq7|3Fr7LFJAkXJ6U2?nsn= z6A#LtQHp>+u^5~qx^{ywmKj6dieoAZQxg*Pg)LVFxJ1$n^6HVzugT}9C7~w7-{gao z=arHpq}Q5yeiG`ZcufA0@)C%_39l#WA<3v;;ve#15M9xBWL&98Ea-u))O(LYUUoeNPw!0Af`s(^=Ks%X(7a%vOo zRG>|_9mYA#rz$GC?~S%ebSiLW_gu`_l( zMFg5v6~xJY>jI?jcDwF{m_NfJ0Y*yxmes7!XrI%)2(vycrs{q4qqn!3acA1+b}zyB z&rXR4>iFQ>-e%ny>AYMJULPiOoE8J=cWCj+6wW6s$7~NfQ#C)vwLhdqohbzRIp*)M z^B^wQzdxtNCrju+aK!|Mgw=|(8?IMviAe(ryQ_ffh;UPc01nHZ zn1cQQU7>&rL!cD+FhvzDrm0_`a|Jb<9*rRb4L&NSuV1SZ2?G(`7z{0}uIgmWWWRMQ zEIZJt+Y`eI(^X~1EP3bGDgp?sZf{IP*rTeeF`M6|x5CZ_I(PpGl1ubevKZ<+ZL8>P z;G*uen8RUxRkblm@2u~`&IK;+UXO_lTePSx=JGr9n`7soO9;LgUYMoI7^8j{dw=vC zbSYsICL!!ANI%iPE4|M<5B-C%1(Ou^v+768=(~~oqvxT^2q4}&Y zSHpa(<6?)wfWdUhvKkSrxEbbGoe^tqLbkF0wfqkOtlnV()#6w$6Q|Aeuf+|4RTPDV zRNsx=XOgtBMHVCi%nAV#tRglB%p^=A3kpG0sKQv)t+5wP$ac2af<}-5KaT?{f)W#_ z-6XcSBfz&5R=e|k>?6~yc3f^drkjeX4NIv07W>iE->%PX_vof$8pDby)<fZUxZTugVEvYK&J2jFEsSA=T` zhcGX~DyyT9ng+Tasjmpv5sqM9hc#5E9)#kNEW&hXpSba^nk#HRIIqY?H<5B*9IbAN-xtkCP zijlB^>c>Y508g;+I(IYTgkmi0Q}wH(P5%j=xZdDyK_n?A!p5pU9eoQ%GKDv|e<4y7 zQ(;rpQ%9{n*gkc=$pr!O3duZ(#_rgX4}MQYHv_kIr(%h`gCF*7OJ4I?%wp;W&9P&h+j=^6U=oyjUhxNYt7elU zWYD3@=Z+9?urVvBe`_ura~%xn@~0Nap66gzQUB3gJ+^r;y$e8gWXGt3 zS};5c4D1$TFw_*PruJCUp!GA@-9Uh=!gy2JnvP@F2K}Ci?gswTosU^dJ*0VgtZp#< z8SGx*p6;s{Un);CbgXYs3kFVsd%Le=wosEazm81~T0e&s1qOHD#DJwhjU8`^+3&fi zC~#l*EzC~pSh)6#-?0YSw}oRbXiMT?_yuXg2YZ%+`9CBoNzO zgbAS*YJzyzK!~acxOciG7(7+33FFn7)4>!g5Z^7skf~}-6tB`5q1Rx<%+ckMSA4{xZR0bsz$^q`~nWl(`vy>Wm zawvsRfjLAa%&*|ZSj+~K+(JQADWa+UnpWOL3$mN7u!JEriepr>#=tAFaJo$j3l&kT zh^LNd26&woNjF<*p&{zQ@N7ae#v8VnyG=?99nq*rrCQh6#Muuajck>LfzT>W1K&J6 z&TEKcG^s31M2jMm>QJ*PZr_l^$S$$45UpU|=3KKjE@sGVG?iGw5f2m>s4g`-<1P*% zdwhOA*x&sSgvz=B@4bA81txOs?Cy3<9@VR6f83*?TRph4_K5CBn5)zcHOJyU4mqsT zl}QieK2hADZUr;D*&m&I{mZ2Xb2~v0#;%$i;NK_r>dK{&xz7~$sKGV2;&y+`@AX$p zIk|d;6e#nhalG-Ty=t|POX$WZsq~u0IQ~b67d{ojC_)dWjCufs+ZKWy0Ck1%5a9); zl6s`(O`Psy&I_MPAs7T=YN*F+zQmb6cD+zn3XdRGUa6-h*G$D(eX@P&TE&ee0=k#^p~*ez}W8hm^P}Y#u!Wdq?X6k^5confj*2DUAQw z;k8e_@FZaz^OgEv&C}zBpF>`&>xD^#pP28|&oyt3XI;sB?b^UiMofY!=UC0B<8ME= zy)JCvrgXplV~RRi^XvHJXX`hx#=z9>X&%-zNUdGGj_Uh{--t@fwN^rCcD2s&8@?pI zfi$7gb7v@L(;UHIl=MaOhTnubm20D%Lt9+y8=vrH;tfQLI-NU9X-`{O8xVi_i|<>$ z7RArCRl;cKT5^2#m&CV_W>iM5opK@VPcTdE`J#EtZ$_QTovU0z+f*1HVzF{P zEw*+5gojDqvF}^XA(kpP(GqLN;)lTq*L2@<9%lyDDMdh~z!{p(O{gN*P4UuU(rE1E#*Fhn7o-|0#oMw`z9+ zHTFXEwl?X%xo&{6QdAp~P&^VlSkfjG5l|iknzELWpc~0C`Lqkg1ke`JDr=(?Oe0+; zb-OT^;Nd~1HPogiK)yN*_;d)tjYx@W7O5$^~xC9``WgI%U^x}<3C1S z%k@?AXhUH7T>UliKgbi*_1sO$1lqS+b3)Ho&3}9_j?UepOrrg&ZLIqCb>zR%C(xUO ztsZH#>2^!XP1Ve~2V7GZ}+7HvV@|A57;^aHOGdYiCN zo=tPETbmg3&HTaCX}OKqrMy6MsoR-&5tz`}U6wnD0Oh|lWF0=S1kAWiU6w*bkTQ?v zT^Etq`Ass&erCCg*rU8k^R0_Z9R6k=G(EH2Lj)^t0!(K{qWvh+42izCfWJ?98$?~_ zB)W}~&AR8(^+BN?_h@_TZYKJV=9qo-LJ0x53V{PbmBCv;FAW%PmJlD!XWR0hXr#%ATu=;NM$@L|-3K661r?u9d zJFy&$_zPchs}V<)1GL9=*H3KwF8Jiy$JHQsz~pJDQ=DLaZ~Ii($E`seSAL?ss;fVd z_TBcg>nm<8B0)Jq8>oAHq5!x83SV*S5GRylv`=-fPBeWNe0F`!tw$s&Cun1JpH95} z-uAihHMaqgqMV{l)lHqS`oR&v-UK#wr+Qe^A@z1Am;CViB6<_p)Sd2OL!VQ>_~hmv z>0e-P1GU|!!8S3h9(|JfL;FSaR+uTwP|l$*uJ=8e@MGc&q#xCin+Za~R@Micy!^v= znBR}m!Txsd*6J?CW456v+D9qNAWIpq@iruxK_<3A>b zA@2h{u@^i*bas6v@GE$a@ZLio5^_9T>4EimCw<0aM@HX6+X(-9xY0xFi-6OybcAPu zwiCokybrm);$+OYdBkM0v?KC7s3~fl_qOtH`UMb8xqBiX%Xk6gkoRO*bfgmU~aR zmwvO}FDYxndDP!5eV$va45b&zd{{e>h~z@$|V34s~I;{g3h{Y?;& znKS0|QP@jRdvNL9^{OQH&#p01$^I9JO66httNQw+G+@_o{ltCQy#yIcH`R9}UHj?x zUFWf)FIS`F(TD1Lld69veusQUy~?dsCeXhDLr2d~&3FE1)a%@OWfJ{Y{Y29E&x!94 zg6$hZqemKjdc&+_*e}~>j4zfq2(9uo-L|1WAN`Bc(7!=K5Ae)a#*54XSTzVC5?zPw>a^4VW5NT_^WqZ_f`5|?i1w=`qqYYK;U(r@c$;Y z4ea!|P5-;$Lb5AB@~XcH2MNzS?g8S^&19af05E#FW`tfTq2n7A$;`>NiNaB?1<|ck zfP=g~Ic?JRr|TGZ2+^Y~rAIV8PA-@X_*po{{fKy>te_umc$M4)%qgzlxt|byN)0`( z;ZyS4$+n+`-?^U=ua))mCy&bih3$`a&!VZMi}zwp_err1Sa0nN#0NBcfu!+K6*_dmAcJVt$*aOp}(Mh#$%?^!^6xl(WBGe&eRve|3+0e4`IGt)n?9}4I(7E6Mbidm zW=Z=~nRR zLz~2Qs-=iEjR%3jr_3sDy4_6TT+bzpjg9fCqtm`xjZL?kMV#-soUy$zGZkjF#yW0> z-E5+Rrz<0{F)!7}>ZtYD3_Dw*qo*4q6y%8Ftjet8Aa+n9+|!*wX{<`+S$(x0gV@;- zodB~ryzxP*$ZE~>I2*e;L}yPQ2Di}&Tti2vkJ;GGB`)^dz&P6YAyscxHa%{p-8|w_ z&&`YzjXzRHt-ekln`t-Sxaq=HMtY-7n!Pn@M#QY4`A#5xnjvU}r+Ha(XAI05vUggk z+QrCeT$Q%ZS~??Q_K<_qDpe38zj19^jJ0LP!0e#~PX7Z|q+5+U)7XwK5WH=NqtVS1 z%P4NdrumnVA+5IAE4QLN2@GW;BW+Hg0I~{dxA8t2WTL7Yqtg1UFI`Co?xnnQ%czW+ z#`v^R>#vY8sGSqh)04?)Zp=)BOKSYq7h8P6DQe4q9RWV7Pubu2Y*_B_cLZ~T__ar(QNhIt)-82|E2Wmq-M zNS`xf#Vq{%j%CJeo~IeJn;g^KXN1o(%wQ-N_PhnWTG{DK zAZupv9no%iA*uvMU{fA2ZymKAbF@Pev4F?DIR9??K8Vyd0yczn!mDHe&|Q&Et?IWm zz&cPyf~ShX1jO#MkQGooyaR0{d6qLGo1TL~N;uR2?{GI#JgXQnP4CkmL2d&BSO>;P z^#tW<)3@}Gkati6qQk>T_XJ;T(~MJdY*yIeojN>?OwSfZR+Hl?cbjlKgHwl>G2HV3 z<6_gwQ~%i92A;7FZzJ2YgK-&f!Et>?yVixWHF*bAPZ&3w`~WEYl8cXX_MgN^&u5H# zO~I!M0Xe+FIopTG_3UOSn;55bHkanCT@-XT>lqoV2Xskb)mbM7vRTIX~`^@>s7 zlzhr+CUowq#Zl{xKF}Ba_t(!nG1q1Z+Bff*YLM}~ zNpgxbvu19_67)tVmhDHzi>7JVn3)^xo-OIvXgu!u1$ft5PMrnGy!fRZn~Vvb-xw)_ zU=>_GGkl(5X~$;c3C|yl;imVe9?iTx5C2EU7Gsjb-H7`vyojFVEaN2DaWq#{2*|xk>s+r9BS{HypzcPO(aKz>DRZymr z*86nitQYfJm&>;2ol(tWF4Ov*&YHEzekX{G&&yIRVE#`Va=LgH#lF===AS176{e?_ zb^76~EA~5qi7xM~Y7uk2Hum)3tQYpJD`bD?omc(A+^S7KJ$v>dhn*{ByYeooRxo#g zObNHy6o=N8vfX)?RI8c6+FO9WzQ(-9H9LSP^4vQQ55xoJ>~@E{u2F$bxj^EeY3om? z&4w;mwJIve>9We38KHfAx?pzTg1Z0(OiXrK%RHogdAe@)DPYwL+GD)x>C5D4hfep+ zZeAc>9kkbY-E#{wN&D;c_vL|jYG$_9nZLpI zgrkicIwbFoYB%#=Es{^Nt#Qn7Lx<+wRsF-f3haIfwm%(hkZ5dPk!m0F4u}`IY`YQG zx_TB^0s|}hQZ13Mwmk|PL)zhqz)8d`(;npOZOdSBC_4hsE+aiO+Bp8OtpzrK8X`CW zvm-#uXYlQzD0l>Vi0D+R3Zy**ta*26INX5lAQ{U%4>CKocliH6Z^QBK9b}{0Gm6=x zE$1JCzJsr2NGN%gsw2!dS}i{dx(KloBLgy}>L~Mrwu@g3r65`{GHPCp>NxX@wx9nH z+U~W+Bb!F7^8}T)_6y$xeTEo26+w4uP$e^`v{U?#lU7b}&me}e(KDS1X|~H)Vz=E% z>>0!~YCSWUWpU0KxUg8KF;6=dF{~()>Cn6?W1pS$bf{Mc%c%3rW-e_0GvkomJ12uz zN4W96=LP2S<{cSl?N$JDW5<5uL(hK!0|E<1qTve--W_aXyJsHLt9gIMBfHxRcM@+# zk>)oUx;d8?4cw17?9`{yFyop(WxSo!zUc0{s3T6V zRrSo|=BW&;xzNR{)<;D=`0o0s7~+L#t<1B{i_dJHdulO!Lr|>o zy=NQnWunhe=Qb}EZwNYS9PoV1yw<$wOwwEmWV$cpt}QuQ#)ft9moZq1U6jp*ZfpHzL!+UCSF<8yy5vDt)<&-DoLM+;x=!CoxFlJ8aXwE;gVcvjmz6`!KyXoSz z5-Zk>7Te5)^EWQb*oscevo3+K>{^^NH_Sh=%w`)pJ#R+IY?fmSGLtmFW?9BI^r<|X zk~u70)A~%_{IAQ#w%MH~&hoNnt!UYmDVo1#d7PgepJ?j^W1(7znd4UIcV%B7F8A8bI@2;g%he%tWyP*+5R~ZkH|u=M zvMhgxODosz&IUnvULb$2#WRcT@N8wp?rh+w_1eq2(Xus*@36piEy%(mx_JRaLQ8N~ zp+l%^ML@QQi1H$^l;AqlIb3pG8<;I7x_g0amzL-(69|o{2+Rg1TrWDSp(Qm7vS7ig zwLwEOVP0U*thFWXz~TklSBXKagwfk8g7v87YS!ijr&huL3CcJA>BRw?XtFHof@T0P z54vJp>vfpb*HW96v|##b_@1Dv#`RvYz{uN?b#1}+)#5!t0Q2h=$NJdvGOKREsnzhk zLD!9&yb@WXEkjv-3z}Dp_XgcCmcx%O`PuR#i?5OFe7?8ihVd`2RF;))hG354g2b)C zvYUC^N=~t60W+|hBLxughNOl$w=!6BbxQ@?9FKY~*cWumxB~>r!*pl?)v@`1;(bB4 zjXS;0v6kpI36dPAyTL<(?ihD@o*c7W0e0!d9?VwSruOi<@|3K)um?iz!< z@>xE*C_$fNGvHhV-81g-0%%v=Nx`J!bR--bRAdbHy2Wzv76=x?HX<{y=;FMPl0sIH zE?=+#b^>XGLre0oCB-bPP9h+|YLFQ?v@{Q2A_ucSjR2A}j2y$+$(+uMR3M0>RiHb! z9EHbs$c!Yfa#kcr=8(gL1I>7uJdaXR$%@wXgQ$)xsGS6vB9B^9!;0692nJyb4V znMW^aV5RD;g%3;Sqt_C%RYY)jvd-w{3ti!%Xd6>RiBotw@a^tma**1{@iMUF zx-elK{FFPK5>#P~^6CY4=_p|zyxCn$392+6_Ikyt)14Gf!lz^4VL?^KXs>>-h?p&0 zg4m7`gH#b?tk(eRk?yK+GvX8mP7Tr+k9nC{vDOkH2~mT|prUK?;z~ZUwuDp(d5Eu= zF>1$5tN4;HAc!Q)n2lKBfv0uU8b2t%vOeg#gvAJoM=MQMmv^G%JL|*wH$olalE+$l zc0Do4>nH2{s!zhVh<1;=^r!}>l#(gdlx|9BK0 z7}LFM!soOu&fe^F$`j5E(i%?#_#3PholSLW_7pRNnvEG=bHkUkZpuz_n(hT>1+^G6 zy&S?_TZ6K%Ic*2ePLR$h@PdcCw}xfcIh_K(3TicGdpU>uv_@t3IW>EU!-MV{&v`8k z-`sjKd(vsTH++B41LFlR7qFO_oxNn?c5g8VRWasxtqT9A^=kJ2F?8nfO#g8luapkf zrDPjQvMy8XlvI+Hm=2|EIy9Tj4p-?=NlM9)Bg+gWnZ%ThZIU9)IXTA;w{$SSO0Bzo zpZ_}^YWsfQpU?aCdOn?@F%G6w&*r{6_8TaN<;4ldptTNMs%J|d*WQkDO#VFK64U@? zO7pC|l~Fp|iz06X{rJR?4Nzg$@K?J?^b>BVv9h7MK>_iC68tp@GZSTvqw;|k=FHKwzBctW?;*N~G z(%qC~5K?!C@zDfEac4$;sRt!P?vTiUY0z>;aaTq`={|~j@s7mnFcSqEcEx6y6aYi8W-~sx!)fVxb#US$*M^*jTo$ zCPPyC?}$b&OFRRg#_HL|+M_c2P|B#hF)#j@H=6 zQ06u)PO>rg#!EsHhL6_T-=HjNSOpUJX?SdC!pPBj`*_NVhRsO_wlw1fp{!A}hSDU8 zWy7|lXmcY+L>Oz#tO;cF5e;5Rx#k{@f-u$_vzAgW#jfFaQjK{$@Z+$?%@m-PyJtBo zso%U7r1(AG_O;pPP@EesCjBut*kXFx^Ic!N{R2?kO-Qn^aFxcKwtb(`S(*<%YMDuY z^24_1PaA(Y+HGG*@ospObl5_?CE|?n#8IWakV0;dC7rRD-l9KaJbARwzLY{~Xawb7 zlv6~6amYe-X&EJ`L77x)!E_prnADkROJ$U>hVi5hP!Fb^ot!coD1A=3(4bHHWHI41 zes=Odv!T*zO4QW`!`n+hdw_Os@}t>EX&vQOgXwL1OXk+`bCaLU#!4H3+Xr#Gl>T6= z&-vERM}HN!P%;`EZZj-3Tjl4Ar!(G`zNXx7*a_@EaA(4W;x8HR0m}VR!@=8ymSkr+ zXdq-vl=f1dHk`WMWXX3XL>B8abfxFFNgG0Lzp?CcmP96eJNlpfAmwGltgaa=jkX~! zvc8*rDjlK78zp{2aH_G#0-(UN`ls^r>Z~s|ou*39<=M2>hhgku08jX`JR=I*| zgy&4vEC<5?NTYeO^QxE~rqQ0WRI?ox1T1f~O+L1&b_X}w(?DhDurR=)aclCWRR%7~ zjP)-=B;vaiM?t&hqpkqJixY*lw7eY(#0^w){ti`f(D=)gOXLNDqU#H z4CeYSbg&3;YCMk+7p801EA_nyFR%9YAUh~AT>&oL#-xuj$fZNf|`jo2b^k@C#QhU1}%2d z7zz`i1A-cr$)#3I*YVg%2-IAJ4+v`{l~)jo+MH0tEu=fYe4r%7N8IyJ&HfOQ8sn zXFz5nHpRYSc$YG+)kL+{VShkw3H+5d{S(8%t9Bty_RliUo(_M4W(u#uq8k z)<(M#Ni1`yqbMw3ys<4M*V<#ZAcry0n3|gQ;)5w-NQ}sv{n%u3|!2cR{Ne=GrdQjV!Xz8k3&j;b<>{I^ff48 zZZ-x|Qvj9O*mN|td`-B!KGhiE>j~6Xs3v-9&l<5i*Kq;?^%7+P7gj{-Cs2$TPn%o| z-7mTyu(K&Pb&1W!y)>X^g&qVl<-JYZRC^od-UxBSdX=}sBT$`rm`b$C+v}6wx~}N6^3N@{j;yoiDJ>1w=K?NSlK&@x%f{o3FnEaHlmb zPP0LHdyZcXu!mAa@_?i!%QP26yeERk0!A)?AmBFHrTK%(j)2EPK?6l?0Xa?E(xMSY zMB1rcXcgU|6F>?))3On+z-#1*Q3W|D1B6X}X;p|A;PLSU4ju<}KzS1@tshYfT0WjQ z6~|#9AjRfF+7HApqG6UTo);n-4ybFol?Gkw^dIKr1RfgZfZf>CbSDkDmg1$%YIRhF zJG=|%XnL4N1PTqG?A9%+GY*phs;06uHc&<=vs;~1XB|EU3^&!KC9bvJ=abXARdwD0 zhzOdxL1dS*Pnl!9D=AX+E#PC*NLtfc{yxIJ;%&T3qF({un?9$#U#rN-$RDqKfCAb(Ko#shd=>gMKC zcbe9T^H6!k8fYv4nYQaGfWg`7u1a%QM?KKYBJ5euKS%(rCtkW}BlT#r=nmDp z^PnUj#N-~Q*-?)-zqk{<-pCsvV0l7$A_pq1x$REw`r3{=0^?iBnIa68)vUf#0eEK+JUsFY_3kP*|p#ZqJ-rKtq_G#$D1Fx4_|RS z5?f+>jQ31*hN`vgOE2FTenekleBAfB!+Gj-^Skun{~jNqfr2&kh3F#nXR|*26DTE* zgG~suT6BdvtHqGJ1oXscPbW`8Ux}_!7qpmi?QNMy$DdC6L&1TVy0itsJz!gOlvX-< z3Mv;RP+=`tZm8|V(ectr3bav_OkLeV;HH3T8BH`90Bv?SxM6+EA+FGt;;R(3QdKPu z8C3h00ItF|&(}xXN>eEuvOxLlG}mIM%2zHf4pilu-J?crZ@B{MXeK0VSpwa+!y%8l zyCs=x16+BMvIK^2mjiHTwA|;qA=60M@`NDY9)}|8k(Q@iDzb}Yx{<|%_KHfV$6H=- zqmf2rgoMR{szlosQCnKM*C9^iSP(zwsYT^fR*RZjfs7;@N^Lnjt*C-}y5$4cab+nv zLTViBJK#`7jcobO{fewq4NE73p+ll->eUv*yG!gg`q5;QA0Ns}rr)@1 zG878Zlho7}#N7jSMSir3$uQ_R_?a_Xuy;d2Y;U|G;ZDz6hjwai%dWeLAjpPTEjZ14 zFY2N`YB_kf(2jge{;c>6Z$i{decE#BZj&AV7@@K_f~ON{s4~#3dvB*XCa)|$%ll6> zNUd(UdUyUN_;JGX;&Z%DqETvNOX}VAAi5z3u?ya`=qJ0QrknN5U_MZ`A)< zhVIsHN;`plnQ+c5s58FMz9oWxcqT=4~4A==#5 z#Tho6y-!N26QX?$P;+UET32PbZAPs_)Ud8X=K!XQX{%j^|K^sHf*Mu~bgppIj3lmhKqk7v|uj`v5@vSOhSF^q<7pXAz{=9ER+tqq218VPdO5SA@=X>Jj8rt4gZic-* z^VB%FgMcm*ucIAo6=WO$#X^Bu%`K=?@kZLwR#67SUUN!bZ)~`C1!@!RL~C`%b$b&E z798As!J&&5*xH_v4-PvLa8U6DhYVV9>k#N7rctn9F!u#RHSKKc$BZ}jT@*<}!foG` zsO_|<))~Ax4kiKE#)M>FD-;-!S{L(d!0;kzOh{1)ZWCxpt(H6&hxhUAuL8yFm~J!9@!g6# zPn&Li$Lm3f10!A=-}BvuxWI8?nGUu z87cN8ZALTc&`v0-K}pnviB%?maxVB+4OFye=3~4+JXErT zP_kG;<0u9)YcTOZw#h1iE?xBu)JP{YuQxtp#`fAi;axYXqD3mcgI?)0Q{QX+)b}K+ zh8Cl+JhvFTkwxp9dgh4bJDw5$qf zxOkk_rpV#D;Nm%me%3SS8Sw|2Qc=S9$F*<-{j5spS@9HYK=FbfjWY^H46vR<&x=3P z#uaV+T%1R+V1QKxjRck0Da8Q41{WWU7-YSGUK0PL=@nD_VO&eFV373^8YTWq`>XiN zpNTgLK@72~q0!=5frf3Lw^-m^L%2hpHL9y9!$3%zd6qLiCd72u^OfovYC+)gHruRY z_}UQeuxG6*7PTkX>4;~Y>LzL_z(#pyW#gGgVn%H1dAC3T%C-$FEW?L~ z>PL*_z6oH!M77aD6IUD>F>2i4n*^MvPHpG1zT&4t^`o|r43fpGfCM!*Yl*@t%xA2% zQI&#P6X*`Cs7Oajn0!Y;6E97?HgG>E@47qk!w7GRn|bNtje$qoL|F_+O_&_Cd3kq1 z?B!(JtE@OjcsOCaxRu8fqXOw|omqvBX7GcdMoVYby4kJEy8tPW_Nm=KuM zW|{4>CH^$xJ*yL%4+RC!oPNNY@U8~0TEH5~4sO?pDoD=*kVX9cms}P3=z60ImLMQTB`BZT~ zuTp$A@MGIZc9RqTEa5-Tm7Z0o$iS#?Z6CAWICY(s{Feag&8VorsgS?fGq)O@LwsZn zL2JZU0}Wp{+naB7J;(j%IjnkxiVcLkHqUY18gtI{ljn%44s|PV`D@#pV_R#_aX)#E zs^qAoK#SK~b1rQ)IB)vdb4=BUN(B{Z&z$V7uIITxXsc>QaRZUB{c@_d#+)~u_8j-E zOyC8g0eMWdweo!IwC!77n>agg>+1_SKeqllZ}`Rb9j{&NcFOhjO~58YU5NN%{NA?{ zl@Drn895l|@C*7c#vgpUQH6otuOHOHDEu<-R$ z(IZeHl!N{^Z$jJ)w(_U$HErWxBK#`W^K@cOpzL+nz4zNRm*l^Szw!PPZ`$(m^%c+% zH@O_~E$}<^lXxUh{yOQN)pn1|g5Rtk&}s2_VB71QdoJ7KFC+f2enP*34!IJ9!u_|m zTo(K>c4^V0rULt4KfiZryFrxcU(esF@2Jm#Z(g_F%iit^TIQaAR6kMr!0kTjdllOw zqYVGq{^k7^{|Nl@`olfd_R6T(f42X4f5m?Se}f+SkL|yr3}+x`WX>p?NuS-m;66jV z`AWnL$V}2K^c=cTyV-qRRnV338M>LU*=6(SOWP6m59}zqLYt|Z1v4z;Zx73MxX;+3 zxkCGN&44r)y@sw(q~6zr#CPGAnTo$q2oIzN!7>{rVkg z(X<<<3}KKm7#-Wrxj($4C0by>o(o%4W=Y@He);}P7o)3)+3b0+C1tDWg!ZKSRxTb_ z1+&@nVJ2k=x>tMk>#Z&^S54;-7pRw^H-LhC@%>{il_S^YAdNCj%aC-x_GkCcyZpLp zXoxhNLKBp6_^Gu!*_m%D_==nWwYN#;PS9Nzx!evgYdCSopR z5y=wmOpk2;e*de>bc}v3WHD(K+Jzp|J}-CqPSmxCd5|Tf)o3?*e7jjLW@q>{{XEE0 z(i*foJ*|CR?%|!{YZ3DyCL|y^rt{nJxo37xU(?TrEF-N$AE4*8@5xOEGeN`x$a2yK z^kG1cJDOX*GyJ-K0mPI9Kpk{(J3Y5&r}%n=5o86)4t(1%xdLxJ#X*1fN zUfX^Xp!-m<5ypx0<~fuF(3{)2x%RHi*l}YW6oy9U&g*D5zP87eA4`A;VWfN;COxXR z{VAybcg0E|iEt7Q9ZVl=ug-mMvfu_{A=@10SZ1>0O?zu@wyWz6?n0u4dJ8&&u4~uk zR=LL9FkM8nRBuIJpnnD93jL}8P5ROf#DfC>GC*6RTMZ+C!x5|l`yh1J z#Le*~I&0XTvSj+|4#ERx&FVOxrHVCbcOZsZ-*M=Ha2F*`xm01J_CROQ?K=V^}%{K@-6u? zAt1k(70~y0Jb2*l#=k{aF5HlL81yWTc8DG@+%&i3%Y_>=kCZ(DL8(^{ew_s=?3_KY6UG4){Bt#&kd1#y%5 zIQj+sQU`7Qceh{hhGxjknJ3F?=+`4m@| zAS6%{l`sWH%|!RnOFIJc6ofngUC>R&v&%GeSw~pjdx9oW4j1AwgUbf#)g4#!s^%1Efm@|<_aB$-+ex2R8}KhS$Rit~=`u1(@v z5S`Q!=qY-C$Md{PyA5ueS`xRa&!Iol-*l`&XY6*m9czhn&ZKTWGyg+}5-ekwx5q7Y z&ajJR-|3$_#`8LM7u}|<)NO-ZF8fXY*`d$-w0q+A_)6V&*p;#wj9HzA50?P=18tRV z2kdIuY{r64(}(tZn91X-bS|)KW%C$II}s1-LqwUbR>GZ`*UO9;W&jVt*rQ37TM1n= zZ>?CwSlPMb;q^TxDcIGCyGU_p6UN%ktMk4O?o8pXCc3F_p-macPQQm$dty>dt%(G6 z0vg7^bg~}y@2O4US`&AxlhBq7=gx}{|LieHHC;p8qfSPzX1H}GJhX71phvGkx@V@A z*)TjiGaov-N2VIuAoph8DO=At)cN?KuX|-`tPRp5lMC9}ex1)Ao_GJ1YKTC3W@ePx zGXgrBAO6rgq(vYgL{cWuJuo}_9+ta@r|A(8FH#oTkrCec4sa{PX%TB7`$#!xXGUb_ z_lIBIr_=OnA^S=9(JqXb&UyLE_oD7Ztb-gNJwUrL;ycaqF?+qgj;+%jfITeR!$|E! zpy z>0V~~_y*l!SaI1=MwFW;vkmYrl|bFSq> zJgm7swu+;$pZ6$?#?DmWt{`*ewnE=baakav9S}9#J@{M#Qb@`yFJm(LIz{;m4-Hq2 z6oUS9*{TJDoz?l*JxuOmL0J)~-%04iRv%CZQ?ROdi`4PbB}HnBnP zq3i-f-#L{(?9p;pu!(&F_Ok3U<8SBR{F$Cc8Hmm7ldu{9AvNq;D6sPM$PjF1`@?F> zVi^m&ECnu}@fiqv_9<9BKpLBN*$MnTnI}{1i4?UQoy4%{+A6r@X}~jeAO@%#(WwCJ z;VH=Wbmeg!h*Whmn#(|T`3b5#V|b=0B2C?j&SYS^Sb~1fS{@fg3{j44n07V5IMQBsdK8@(~ya zlcYooL7!hHI76Jy)MFqlQXd-Z3%eQx=|mJi0t;c2)aY^sv#SsAM8f%cECg7~(G`sF zu6KeSaQchDL4rxZ7S4$5`Y!kiOy7DOB!o1CUcSOiG`j%eWeptM3Sihq*D>ONQv~Tn z$x`AKq3Tg|BO|42O#u-|seK$3Vd^*NRtB#NUBLDl1u|$wxcV)+opJo?&Vo2Ecs5~+ z5J;HIx)_hT4gxO;Ia|I(cqVhAte5e$>r_FL7eAZelsNk+un;q(T_FW;yt=X_PKjqT z5swELFS=IU{NrViW4e`iPW=%LM7>=J1s3~UbGTcH=hdIl;|xVtRzc1Db2*02$P1ZY zfEoVl*+&J3_la{NoFS1UJzB@mbjb?N?3)HYSja`vcl0O5XjfxF`o4|E;*Q#j>L2JY zjQ3sL1;Txld&+H!OX^?f?~G4fV+D$RdG~y_D=w@5pno%dbbTq%?Hj$P+^ztMQ1pzT znccG=LH1kUA75T@C39xELD0PJrH|GF&8B>ZFgnwqd~VRB{y!*wY zXdsG15ZDPYXt_fWty{J{`+#d6mq1JezXcP->Q+ChI1rg4vOsl{^;w0={)^zNHS>^28hAB=K)p>>S4qlND65+#w{qm+pG|CF#Mr@ zkFIDbaAyTYEs0xGNIaPL(8pbos%td86eQS2i79FhwyIZ zrt%X()!kPM=X=8i1W#c`rhPdjsIfb>aQNgwpzTQnj2a9$7j)k*bn{LVV2O#DBn*Zb z#C9nu^ao}e0g=sz;mSEd1KlqQqXESR;l<8^0qJ_scz0W2F8C^@coAQw1G#t5WVgDo z!aEX(sF69DK%f%z1#oUu-jxNh`;hlCfypH3p749&SMTWp{XWQj(hdwTt@g}&y!;UA zQN(^oE@>y`dXQ1itVsJq%tzzfPlhcdY8!`8WR4%(}LFZ9DH1Oh+HTKeyB{3^1DHsdrkq%0>6;p zEfl~`EawN|dO{w*In-4s@lGrt?Zey)+TL>ocn3^?BP8(==>R4#Xm?NYW1GX?k0pl^ z0lXhm5VRPX^VsEZ{A0vn_G1`eSOXg^eCj11ky@D@gQV@@d+>1ABBiW0T{sE z2HwEKKH}mCUx=7Q!vF|&55DM(&vdcg7g9!|V*ri1XAj^&qW~reQceoO^aK_490g>c zaG{&Yo{0dye*XjH?_lz#|Pf_|<4k(Ls{F?J;k zG*yEJdR`PoA29+9TlRC<`SQ;}<2`Lfxko&n2#&L>V3FnepsAjLq8b1sim=@MLVXeQ zGw5s2WKsW-+9%u-#Fy&Jn7=`PdVUxEIbr}>5X5Tr70fJvEHW;(IO_V8dy-hAzKStq zLX_sk&PQXOn)(x8sjp!cFqbQBi;n@|CfA=>tB%DiWLhY<7GFAQP-=Qg_wClr@+HhQ z%Du(BqfVuXIyji^57X@~}kS0^y%zEa$XhHD--y#u> zs>3MKF}6%!<>RGvUyVpv9@j#;ivhhRWp(j&UlTEwme@)H+&X5UvK_#cyuoE*qJqT7 z;FwXt%Aw+V-!w5cFcDn3V4Rp|l^=`W_;!gUfr+n4_b}U;QOX&@IRJl#r6;zNaxt#V z8_LB1$mCrnp(l2b@-SeoRIU=bk*q6L0o)fXzkDwM+1UyGNiAgp2D=MZZ401U%B{jn zBm?mDC4#3O<{&d)=_$-6xt4Q-h=4nV@nL=nA_*%!R=_S^m7=LGkLbYTyvWd@b0 zQ^HEh{g}^`5dge2Q9jPp^}?Q(pJcvL#tN5^H%e##X9N?K2QZtJT%kRgDTzJZs#1%= z(XvDNP)H=_NqpD}wYnU`Vycv7LN<96kar+X*QMpb%ppLEiX*3`+H(@MBtX1jzE!ph z^U2;)2`90iREasu{7*R~gcQX|5y9*MSQS8;>y=Z&VRDO95X>Hgy)3`X{Hts|^qXuT zGYugEObX^IfSwzdSopchxFN)0^(#y)6Vhv5;_MeAGYut*yXmpq>>81$O^&+<8fFJ zKZEJeODpN{E2^M{>&9UK^1$5E8&UGfZ=!vs`+FahxF6#`Bb*U_$m}UE0ug6X3FDXsbohl6nZ4ytm?r@S zEAAK^l<9?&nW}Ozliu4|Qh1D9DUT59GC@V18PYpa(sYbpNjNK<%Iq(%WS;B&T=M>y zrc!=Z_+RE=`Ag=N-dUh<4S!BJC;XT>Twcq(*}D`JpUKbV0Hu{ViaERQcJHbuZpYI= z;h5b5eFNB9pElS&Id;7EIrltqT0M?wV`le~TF)K-_1y3R@=NBs@`;0ay?dUdA4h@u zECiHeF+Bk8j&*%{oLMz~LH89lS>DGK_0pbn94`Xb-a0*Os$9!_)*JEU)A5Na!Hvdm z>i;l9%$nY7pp|L;!snvmyZRGmjM>zC=Lzxz<%RO10yG;j@0cCE51$ZEkhQSS@~O+oc+h2H&w|g9EMS@T**zsjysGAc4w1$XyO3qk zxAp0zlLj@WpvGOP=ebN=6#fWn>3qUpw+2{AP>SRofDQL53jIl5lrjPZs z|72|qH%1qiuux*b+S+&F>H5c?YV%=MeY0R^9rq>?M5`sY2?>3$QpZz~b%t@s zrF@WaX5H!AQ0jZCvMx3bX~IWHh^(x>Ev4sA{i-v(g!)eDdF*u6=cBnU{&^=EB$(Ex?Ud-F(Yrr z`m<{LZk8^mpyUw=5GdIJ3pT-h8KoFXxLltA0YOMCgVojd2*B6G@`yyD)C3Dw$m%{x zX((j^ga>tSI7kArM*8HXDU^*3v?QH5d<(#{edtq`mQt7?J*TsPZNid7mH>@`4idq`auKn95` zp%vd%a)Y&4wOZsJz;7g^2v_q#j)`Taau6{BG>vk=eBY%7FfZQZc725DUu=9^>Qk?`;xU?p0S|Wl>#PV4{Y5Eg9WRDR+ruvRnW6TyxQJAAAR3_~hf zr6QJ(#PK?&_h8PnV-lorj zY$2ba|afVyosn z_6zI1s#`1s*Y(OA;g2hkl5eb!su6KhApbSto)FxMOMbDwt3He02Wno+?+Lf_qa^=W zGt{%n=F{Qrg!{rB{Ah^*d!Bk}*?Ky;U4CEa!jF;6WiM8*E^`N0sf1kNPX2X?5!+1d zP{yEZ+6~WMbtT`xE@H1#?CTJ`?2e7bjspp8 z>$bEGY+fP&`hmS6P90MAhThd7$Hj`llu9w(*;3<;TV6&K1c0OZA zC-))IU6YDMu!-udGG_)ncxFCwFaM5YJ^PUQahWfpvNJXx3GO)nf6q_7|HEmVRy4{ zt5=n~1*LUk!3m3;kM&^RRc|h*26c5K&Ykjx10^LpN4>2)8k|ZHz@G;%lpF+GQm^t{ zz$O$tW*>$Zfl;MIeH_f`@jZwlwhvqgV1*Jjr#$;dO%Jz-ctlfz1+%UCV)>sS1Epy( z5kTCrr`YxC1TaFnD!Ii(F#cg_VBE=Gk+-!OOSqinIx1wrfx3hF`as2!Gu6IU2=v!q3$a$XNLFcOCaD?jXlr)Q9WMX z0baATC%WTs01#mRRO`z>F(*I|TXzCpExE#;r7@H^zFOVq^Hc%4h}diF`I==CB#Y9g ze5&x*)M9V4muS|2Cvje%PpRURrXDn=pc=G<%^K}9t$RS>H%M-?S7~-i;#hDMK_m>| zgXao+o#vpVkVRI>MM5gSMRJ$DS#wI##Nw+6=daPo3T)a;tcC-=M=dIWm>mdzEy-bT z(_EI!WE-gwWo$YeeDv4^O_Icl?V%Qwu^I3#iGb~;$&t9QwMQp03Rg%qi)o>+5wnmMWu-O`|WcbTDjiD6D zfnP>du){U)Bt2}gCPE4cCJ$h%*pZq$q)z|$&2sB>OT#iS-;2HZ2d;-k2C7R>X8cuvaqLLi}*GbfDiH0K`=Ct$+ zD%oe@|49beFEp2>GlPu=5YO4?;GZNTY`G>$Y8C7;Ab8F`51$4g$~H}o)Fn8608z!h z0RJlaz*cHXK((!9KmdL$_&3QEdot#^G(GF*fZ+?|MgDimC-#`8S;`A`8jO8`yu|-0 z`O2QusHGLbk%NXWk(c?uB|q3-G#{j@;L5?+m&hpoU&$Z#Z_Q8XkKkW}hSkU`{25Zq z9Z?o7hO#9g8;5Asx-0NmfOok-Ybvu3VGfN~>!RVarSmvTwFudPkfI@4jqWPkP-@J9 zX|b}syH7aRHyiHz- z+s=v7-rqhW%xFySvpZR{3bzxmJB=$W!d%C=jl>kqYN_4a1ntM|n6U6MeIq2592vKV zlL|gO2f~WRXid6QI6~^l$<$&iLc=D;#+!6$@O1$9`9MpkND15chSsdR1K%J$#Cfds zsd#Tsd82Grq-!?fj&e%10Tqg{JOBh$a5YGrAE!clx8 z$mipv!JHxOtBN@AsV1}uv-pnEaLzkG87&Mazm>NMv-wWavz(9Gk&32p{#(LpVGiF} z8p-*t{ao=rT=Q1`T6mAYT^hxi(Ld|i{L}Dvgm&S5zKb-5GY^1C*PkZ8leY_V`L5C% zoW=dCpShprzaw-AAMo7(WXr7I;Thw!=AFDln8)8Oz0FzGzw=of_%abXg%A1e(lpNF zp8bGjO770@Waq;@q+E_&|M6!vr{muvy4V6Z5w~>_2INrsPuITZb`c9SUbq~NbN|I> ze@<)SW_KeW@%Mu-$0XRIfDl!{`FFRqNk=sWuQ;*&3oET6JSGKw>{2*ID(58iTUNS6#7`nrY!RFaJXTylA@h%DnG~qlVtAl* z`;6@Vt(BJ|40NVyBG_W#I)UlRvobrvRmW8m%QZo`cvsW=M=Q%C!gYEzL_!8*AxGR# zuk48s>moD|DVc-o=T!EetNa=F2N zr2yEgqCyjnd&lYMe^^O8n>XdtuXv_8gPY{2`pdv0Y;;Q5uc*|V#eL*N4foeomRIe4 zhaFJVE;}#%%o*=*tIP#}Siu0h3LYudbEf(SDr?Th|A!c4zkpwo{^aQUrz#yky!wwj zNPMZejQa}^vcD_;oHh7pIz+4{TgA)_o;}dIdHFfiN5ba!)$ps**})42OrP7IV}2YT z($&DPN#_MG1&_`H=ZZekhIOytu^{9L8^AsfJvZ@jd{|ctzbRcDyn2A}JOyBAX(PHi z_$}$O;0*&lpftq%G(MuMhbKtQf*l5E&pXbIMjsqi$Tf*L^I#kIu;=g3X+FtEg$?}M z(v`t5$rV7ZWlF8b5>NIe<5mZ|4VZ3%o_G2jJBDoHr%G*tJqI$MJD!jHZ1@J*%)cXD zAAD%w@pIqv;fq7xKw8LL06QlS$N=ni`m_EGq?Me3vk#^WG(Jy1kD87chbYLII7~2e zpznG4`S5A|IHZl7g>wuJAK)~$pD&uGz16kBb8v*9^8;U=>&}k?KCa@m<{oZGaP+|J zD#!)vFFx-S?V4QNuHd+VWmU)vlrPG6iVjU4j^UX+@M7hz3u#}l?-M)u7LvWe+yT2P z{|hZ&1n=2h@B*n0Mg`9pob_VH;{8R<5Baq4E&sLjUhs~= zt1sqXg8w9Z5x(PhNb`dC45q$Ve~J844w%FIE@?sV{=o+?+`;?W)cNWMau2R3_y}mI zQ!jOa7kJ_Xxfk~&_{88(PV^38Dy>qF9-;A?{mUP3QB z{f_;P{KOxT)(0nm%QVN!k-rUpAV2fRq)owhfC$g`a^>&XAINF`xKt6G1=z>uFaP>& z_!Ies|4!Nw{BUsZOWtLtKe0cNU-=&ZCcAi$^s?e|D2eBi#!F08_+R~R z$WQVY-233J!ACC-gLGiTAILAV9;XY|49WmvdHS#Z59BxbJML5P=wRc^^eEK7h`*3O zI2xQ2-+WAV%j1@Dz zW@%^R7lbSyvaLRLrFO>MnO+83L;S)Ji=nO6I>pZ!b7$Gj&YB12s5L`-t9e(PX5N@( zHz#X>YfGVQuWYV)gBdx=%Fh*lBV^I=s#k8;$m6S)a8{eU0R*?{u-z;FYc2B%mT;`icgvDO zEQhzfioRwvfA_78mb^V+tj>S7=T-VO)cmtc7uxvk#ixTg`{=9kYvJ?1Ew!tHdje{= z0~owJt`*G>G?_w}d&#mwFbhtDNqlsEuSwfl?LPed5SQU=uOQd00oQv9%{w5=3)wTA z`fB}k@`8qCZ6TiC_=1pq!}njgT~AwpTb{JR?=Ze7XuxX_0xR9AG(+NkTZo1FveX#~ZC(!2vj6*|U&{;i*@{ z*ISGVR&X|%18!W%<>9}tX2u#BuQlUrHusmk3W*(FSZfvQVO#)?2<8--JR~Y<*s|6o zHr{wGl;dCy3OFF=YgbDp*BIZ0dZDy5d|OEN@YdQ(u?CP8FfX*0j_(Y~2T9=USXanh zm={JHgjWItxnFHnYz$-t+zYE^;ngAK!>ro=*jmV4xED^#!4HH~4PUJN6Fa(crnwzH zD?~ONQa6077J9>J;SF;;$E?Mx-h{LapImK!gSqgn`IMviX;}faWB6e$@kZXlBNlC2 zv}f>>A*$iBTK0|6g}oMSPTI5hk0Haub+w5%tQQ@zY}=|mkN*NV=iT6VLs`^o+2*W` z#D5R@G(1+TxRJN$$jY{D+DrJ~AwPz{)aq`GF6v#`wp|;ApAkB9WOg0oruE_@tJ-#G zqw%vt=Z`F_L*Aq;?p@X9qK(1N3tcj@rjB?s53thPc51KVjYFX$=sNbz(Z#)1ZLZoI z_{E`CBfIJnZ(1)ovbt@THV(fmbp6PoI^j*qlHS#AZrXUfS*ZO;K%L@d9>A-&5wwZ2 ztG@V=u)6m*HA@<-OLk}7maPokF>iIM8M>v1L~I7E`C9}kZT z4IF8&%a8Lmks^|aetbN5^^XkI)yJiQ(_NC6UpC$;^z6vTx;JrMCepQSeJ1w+)&26w z-@2K%jFzok$JuY5E87_wJF>9e>Xyf{f_0n&=6NzgXwryfz00lmWoy@S4w~o7_J(pt z?CSk*wJa-G&+#@d03i6Bk@j~NZy79KvBB$*_7VPIX#R+2efBNa<##uD9o9a^`-BQd z{OYT2#Q;>imyh4)bl=djk)V3jt;*#$HrgG@Dv=!veLixbp5g!T@9@T{Bj!&5=jzo+ zY`u=N%JhhB+fi*PJ|HxsuK-fm@sYOrTmYmmu;UyvuarfEPK^xI*Tl!K zSPLku=2fx_Ai_RXKOEn(q5zzB&0oqchyES;TR$_w$ZYLq&Pnqc+0{_P(S>p=z_2ga z%<(s`mBofG9JQ3YB*dGo1>_I&dfBZ|(@{IQe?p5{fjx&}4oc>smZRI`(FsP-wGNyB z^CnqpC}Px0ex2h2z3bpb)iwiL9dgu9UX>67U4inVX92m)dF{7{e43^^t-9QF;pFvzbPUkJ|S zkL0B=a>%-sVz#QmEf3z9IDXDA#yLl55;DN3SsL7`{0)eblz$SW>O| zT}Q8Q?Kr+IG<$Sw!=)qxixpeEPHW%cJ45qFJsYxGDf~ca)#$~BKS>6bE4F%_(|*K{gw~HHG+5krwY%8_eemqn$n$_TZJI35iyQuwv{}TFsv>RAQC@XunwO!Kw!ha9_G&W{ zon=rlH*E3Pt}AZIX{&H9Nmu>mI2whm7~9-PP3~GHbxDfxo9nnJY~|RF#_P!@R=Ax> z*Zk%?nuM(#+uxX<>}@68nFNdljw`}8jh$?)PfoMKxhBQ>K`L?sv16RZ;p7&p0$0ur z^F+XXWS`c4KUDQ_2~SH3fS(DdyG{gs`MB%O;o9_%&KzT|p1)91}D(eSLs9LTt;_3e%YY zZY64Bq-hWh#1eiMt|B;WXzW!}TpE1sZXkWga;yjs`{B~wl%M9kR_c|M?dRn9-C}BN zps6M;e(l?+)^l&%2_IvqEd++)~OSv zQn69$t5TF<*cg>`TBSm4BZS0s%Elq6aB+>^Qs+$x9(I}O`%B|P77wneG2e*UB=>Ne zTY0ebVaT$M=$+Z&(x)M72KUt50cr`sDN&Z$k<#ZO>jn?kym)21IpDM? z+w55BtB@^&=W0H^q8l`yX61ArcS;G_F?gk>{#DjyFHhEcvm-ClLiP?GxiIjmd2@+p zVy=(7)9ebwU`$Q)u=*C0Gj4g^9!~E<2#@q0*Q|aG-I8#|E!quz2E3%dH7{P<+6J5x zeKfmR`ZeSR8039=9cU{!Cn_}a0VjFL;84vUVy3N^7ptfn%WJp2@!!fh&-&Eu?*!DDgX?QgzRB9^b%FKS zM}x9SW%% z0RvVh5Q(G{OD!?Tp}Tc@sir$!uN3`De^F|UIWqLJZgZ;lPUe-O>hz>i8;r-$ySk&P z8k&w*6Ki~4JAvZ%P;uSW)aIQfSKVG1C70S^&_j&6xKw=yoBx99%u<1jH%lwsg9#eq)%{F;;$RaH#4-b2EKK;2sIE7))*&|_h;2qGMPOovRCSQI`nznd z1vQvumhQ(q7&53|`}X**Trl}H%PKvLNgT4M-v@*yHrIoi%yLSPVcrbc)n9sByDRs4 zP_tQXsT+nogs6{utH0akMo^1ce(7n<`ysdb%(uruJT9o!tf2H9rf3LV{}VV+YyyMY z%s@a0^L;3&zV~hI?%cp2jv2KSjbRMM)I-wr_t@MFYBvMhd(dV}tX~VZ)wwr=I?TR+ zc+6&adObYt9@ss)b#{Msx&a*XAL~(RReO@b8pv!a`4;%5(&{7A6no?j{Z)&90I^CR zm{Gq-vvmvz5_Ox=N<%TDLv8h+(gGa?V4P@H=Jd~RVrZbgK20MP8C=9mFE7PmW`-u~ zN7A|+nZZT;^g|^sfvO=@JtSQpypO@0``h-rK+dw5Wez@@$+z8lyMH-7z$_7Hhd8C< z_UhcR>r4L)CgIBkTUZy;>3b7^z9s!{=`)O(z?l`9uGpLFtv=xM&*=q*U@bVpdY$gz zgbgtm^Z^G2##V41gnuyLkJ@0!=XOd8W~bmkR%3dBQvgOZWLCNz{74YW8c*+c;$T>U zZnjefW}o0bYYEBP*(;Pa+}-GujX5fK#o9*lcjkn$M!K7w@-U|a$OA`6Y0j=;MWgAh zr3IMtg3qk$q%LP>SdlQD`cVgeW^%`e zPm7$m7`h;WEh9_eoRAT@nWVG_qup|!y@X|2z61SguM=$1Q4fD`98`x(XBg|jYe=S890*;^VeWYCcb zadsLcZQ7Dhvtj3k$P5Ki=dPV5Ntd=Xbp7zLhSUt`zJ$AWTBJp^6`@;)FE*5BB=t_+ zHPA*ac3vIoFnq0{HRH=Z>^%b=l!3E(D0DcYL6#xiC%EV#|Co;44d&RRBaaRCFrRA2-4c9Zf_A}#) z^hqW(`%ph1{CSk=U*+&1aWQH&?Pt(`!>oo(U;xT}5VXYHl(r`{XqeaVGxNy-n}i?( zb90(==xY9012?nkKyrecA=kng5gI)_-=NIYIOru_Yecf5?F+p>yr^;YJLtg#APwR! zEIkASb*mb8zr!4qKQvf|TJL-`G--HKwG7)fB0YH>nsOXY@&fV%Go(AR5;w;_$>?LDo-?cr4FM-gia0*H8uj@Q2rqb5zc>q$px zNuis7MJYeq`)G3UlMSfj&dH(nBYsVPvXhQFCM5!`l5<+9?BIb8yFXlI`qBKHww|9T{!v$eCJLpdV+N>e-`!fFl$)GSei>ksc3d9kn<2qg969 z9no%H`rh_LK#FL)`DNPg(8s`PxZ{1`2|sUipSYqW;YcO!?JU%atmNQtej)XRkzwbZENn?RgHEr;4CD2Ie<(j&=rWfrc-J$h|{u_DO zJZ|jiR+66Rgu*xrLfIo3&4=nAx!I6{oXx{%qoHjhMa`FTySiSI+@RcWnu$yI2(39X zSK+2Zwu6x(XcM6WBQ?#bxls26vK^chMU#b&fh=fgF3w$tVuv8%X|thIBcsh7xpemg zik%B725dFwN9LMma~1A78FolgEDaK-Il8E2bsp3sA;WGT=`KwxZ1L!tmcg$f9`cN$ zLy-G4-LPe&8(U806?g<>CdReKJ1-7fJ-VyqR$jjcCzEx6o8W8|ws!Pj%d zxpG+>EW+TU_!ecJ#%Ysmwv>falUxOfAk;lY&m#h-y57n zy>;Fl7C748@+}|ZDSvO^hDvvK3d4*Jx3uPe@xM)8;v} zV_|QAgXz+T+B3OeF>LVxyy zu!7O^t=~RWjb`1hKg%s}J|FgJ^h#^}hpe++1*~)2LT8__($Ub?@eloHIRz{)F4fsD z?C0oxux++J=k<|wp8Lu9N?7&itJZA={y-7Ny1+H~6c9#e7;UpSRFDJ~(uqJC<{TKt z0W{&O1!;5p&+lx$lyBo6W8% zg> z&=SK?LT64GRp`g5e!ZH$g6ooZVCmT3%)lg_?3632PSSan4e|ps~Oqgqn69 z59<>C0RNN}EiX3!iX7*Ouzuk`&TFcJFP32tjhb_og$af2Ab*bWl`{-tQ1i~SVN&4{ zUIVqjHvniv%vH4cusPv0X96hW1QjBJxf)$9TzyQteQB|+UqEG%YYhad8Ll_BynS1- zzaOWP701

      V_{JvvWOKoaX2Hv*<2Ki>@EOYHVlw_2Mo+=Fg&gBpte8xYgMH_D3Km z;R?v{Bt5!u_@=Sb?fJ#tmzckb;z{~+lW-p`zxF@HNtYa}5+9(JKuyCP$IcWEp0S#| zUuBm-0(k9k*jQA1DO?Ju)&>tzM$on42gV+@uK=89>~DidsAbR%AbgS1zWWpAvi!FJ zAPz$}hkK6YwV(a;u>a;e^F#E<;VZ}OIxl^#y_VY;lx6NpPYkyhM|8%4KD13! z5NIj`W8lVdx6aJ3$FJvt+>ALeiig{cqdR|oeRAEVIS4dUp+2j2kKgR%eyzG5KwO(k zas$a4*mzXu>{rEgoff-1k_SCA{NVWG&eh+bHxgRx@=2%ZIpHV9-*!5E)6bb~G5CNw z1I-Wj9M9`K`|ZmOY^y;5>Ku^RdyoGB|9ho3)ChfTV9>4uiWd{cmwPS>L#U&~`v{zS33y zJq!2|S>L(0pxxo6zOLU}#VHPY~@}QequcGM4>=bcfy_{tp-wH-n@R zv%9F21fF!^4dd^+j+Uf>n?ezd6h;?@w*&s!^^&e2Cbx)A!qUazJ>$Q+9+j8|yYh<4 zNRf1D`0#jBSAL0iFq2nQPQuaU;ga#bu4<60aOD> zgfEP%b!+`FzU|nPSc!^*f_1gX2zVMjZOS-%2_&8pf1%=``dAYY z(*F77U$>Kc-M|(QYJfEtor6XGP~6t(v-?eYNMDLwFFMwp`U83=0j!QlkLhV}ThaOM zZ$B`1(U{O!^pVB0Zqacxm zB0-^8tVq&5SgLSa>v)XKCDVWV;em5dxT|T<(1c;TM$$X_ zA?#Dp8m zA#1T|8G#BeG&H!f5q$o#O;wY*WT7dNu7xS)`H%FDo5``!Zl#2BgSMuV^jd70C ziDRe^C}1dw8+bY8o*&!V#Y2~M!}!?D+oJPc8lVA=~dW=z#qDq;f-g`6itx&=}hc% z@jCudMjGCAwrG+x2y~?>;+_2Kj4xWBXA^;86w1a@#7KTT!#LV;E^!Jq0timIVh?@} z!xJO~6J;nNl!K*;{rGx5`mNy%EwlSj}<~H^kZ!1 zgJ#Sp=zZ9~;za)1isLc4^Fg!bQu+{wcOE*2 zm5Imr+=?nf@`Bp}R|Qo8_33(}nFXMh1Sw|d^yzJTFcM4=GiF#;bF!^(rWb_>RnRM4t zi?|5A0%jZGB`NN?`m_0Ni55{GZ3458Aoxlufu+^>o})H#F?u!3AtFH1(3A7i^InNI zaS7TK1_F7Ko}NEHlkPd{7*3d(1CMLCMAXy!v-V!DPOzbcWf>wOMxyG0{L;T~qZ@2w zv9@e~M8eL+z2?8FQsQ;(my*|&9g27=S<~zE3wK{f&wd$sec927SCTEg7k<(2C+OK5 zlQ))~j7SGA*vMas`#Ov4myvpFu#h@mE27fWCMoI(xYf(8q>?#Ats!N1tuNvxoUb{FL19g;nlfxU>Zo;cF? z`nSU)>@vd*=wqslc_O@T z;k8r~x3jQ<2(O9azN>$lAD65k+M>N+)Ck{+%D(tN#!nnq61Q6Tefk{n-vp~K z^Uv`oxhsRWS$LOy2lHlL-_Ji!p4gZK+gSiHHn=N_`g;F(-+5u;ZqGx*7!lDE^L@%c z8c$7Dxo_wB!G1-=Pb}^?|9jwR@+x;Av4H)Jcm`(H@W1z-nyhvQv}agN#OsNz{iwfH zPm@=>JMjL40c70-tUvm%#xs*O?z?!`U`-J@fcQrGd*E5}8u#728?ZLehxO?%|9kJ5 ziK+V@-c5k>`8IL0pZmA!S+c3SBQFTXizuJK_bdOVoqhjVe=qtrtT&=+;$i;^rb8mu z%+LvlfVXMZOuXse!3<0kn2DWuvAc(X892NDB(vH1pg9%1Z(&oeoQYzf3T{p;F(*PT zjI%@$ya`5s9MgE@CuThHg5Cbg52#c7q`G~oR>HZ0z8x~lKT`XeD)FRa4l>;@i6)s%ruhV zIC&dbjhkPTSQA~*M3`lywR+3|q+0)_&HCUY7B9-yMIN3^99Uazyy(Sx@!HgvWg8+- zOuij(s>Z$4*E5B9aQ(HF&G0{}pF5+mn|I^N7rxJUIBQ z#`?9_7WNrlKI{xIG@ltPtnq)%*}^`{D}bGkv}?IMSY4C$+SQhNj$Byg9rowDs>P4oO`9@Yw-W}AdHGSi{m3p2GybY26fYf<22$VCoQZJAJ)i3h-m5jkdwMigh zPP~Zz1`CYj0A1A8+U7SU+lWB>3=59rPgV}b*BU1~+7W%wr7#S*c{B{>)OseD*bz}^ zIt&{rp6nU?Q=63RXir3=%VD@k`J`lUu(mn5#NM5>vZ5>|QZ=a>gw*M$*lZ8>v-nwd zH&R<#^V+QLKuYp<_e;EAum_P#q-I0#x_c=mz-Pq!4SO89Lb`PbRacdgyu;m}_ZOBJ zX)c8gMb~Mhn(TB3z{Y2DQWi5$@;9fUc1=K$V<}SL(l51 z)4cYuZ}K`|Un8$eTZRhj{ek3~eT&xx`w@9ZIxtjUpOxn2$PVIhVP%n#(y5{G`u;SI zBRiPKhgC+#Ni_wBx>u*Wf_x2!7ip>&yGGl=msfft+HU&f4lE?p-$%1TRjb)trl z2Z5IT^~LRitE}eq5+@=CEr78jDN>{$o@GpObS8$PM_^!DEcFoNuslg6V4{W=!Z?vs zsh{8vD~aR?C5EHNLBq5}dRs8aY9^IHiCDB4#*eI!#sb;4G1(DDj6j1^4qPIi3pTJl z$t5siBw7j+M6#tBfivLl=rg_gmDKo4ImxXNxOm%xcQv>Yamg}REZf~g{v z(s_ZBt&w4ZbdTk!!PTR*rWOyIHyp@FM!LuGM)S0y7Ei@C?`ptg$dQJ3(VB3*D1!0S zrs1;LapoxlNSE)VtxMjcM;(4^bbJSCrdEod>e< z>y2ISn1`rO(7iVHQNB}^!|{#AS&oN^PtmIZ%QIlAVK}GJGppn<@fq3_?idw3)iZo2 z^GR0n=>v&8Gx$!2P~a!;XryN)xY`rR7Ul3L{M6j=Y@;Ge=ZO7tvQ_!MsQXikMpidL zvlEWkzaXzGKNR(JYR!mK6E0ilsQpXw`tqYuuYl70LK8hZ;i!EQd1LuW5C(G|iEL72 z>*PRQp*O)#MP*GL8F>wuqS#}GuhCoJXQK+H&X0U+!sN)08NNYpgetj;=#pmZ_g*L1DZCxox@X2Bl*qVxlA`Ii3}~rMoFgnMyi|Ba$Vi2WHP)Q7d10AIWp4RmCJOeQphgl zgeWzc`lw!uL(WbQVg`C2{9crf%xH8&i)UVm2Qd?U04$abWn(I2%ac5tQ^D^n4wXNS zS}C&|z0^{hmwPHW%fhuhG0H+Vd>h}Qk#BO^J)3s~{xWKV?8GRkQUw?cy`oL*{jiQt^OZ4XW02XPk3I`DOvVtM{>#s*R#|Q^Z1hO13|%NbEIGSz6Kj>bom^X|pukncaE?#mVlSGPew zCY-nbL#|N1Y4ypULrys-u!*n}lcEzI?=9aIb>=S6&zOR`z@Mt+xJ; zoD1wPya0GZRHp2paC7SFLf4DbuYlpUK{Hq8AEEM$J_+}AU-WF9PLkoYlJt?&D z4hALh^3JI5vLIn^TWw*kcW{YCFc8`^WHCYrM;|1K4*lTWhJ%Q#>;z^t2U?WiV_!-R zDIbiglf4x>ad1UCD0>fm2m@2W3b{hr`FCx@13uVH|0Z9A#LBj(|@@ z^~?SVUvnI&ShQg|Itng}A_!&e!fzZ;U&xWFa$ahpJW3+#6IOH5fZ&|UAV-%gqGn{1 z!Vym3uP?sD3N!(}5Vasv8`EkxE_U=|kDA7ntK+n%4aU~CA1}`JqrR!X3)jY(-Vv?e z-Hs`i`x*X1--qkrjHfq^oo)XDgqwy{=m&tiW;(rhEUaBxEWc#<8~qTz6lXnsWb8Gd z@?tL={y{%3*IKn@`rO#3_P|eq%i=#4z$u0sslGB+-=6i!%b)$1_Y7`^bD9nv8*lFi zMkF?qM}z|d(DePWB^}nEy{@opq88r-@MFsx^gVO!}yEif5bX;Dxd>LnX$$)JC1+J{V%xQ zBCUK6E@+xJ_Os&&pi~C~I&`@+;0KGwdOK>ryNjMw1hbX7pv$Q0HIEb!=C*$npexg6!Nu`b$B1mGv({PUR+oHkl=F$=jQGjN{ zDL5B-tVoM%Omhq+4x=04SvXhubI}H_C#@uuID&45gT=BuLv)CnL~{f?Pjo9_Lwm`K zMOV4aw30BQkf-&m2z>q-qByQT-6lME%%Y?GGoVMaM48;<^xW{^af`0<@3SjuzZ<8ZKNe67_Ox>A7HjWWg_I;9}$|5rh{gT@FSHydL;3T)cd-*qnEuEIGm* zJnrCsadmcUz=H@^rW0vDNggP##=Vl?y?>S`b-4;|q88u=hwjFM=g0xgG&aeR5DIG%4@;TS`lN6*5+EKJ@Y&f$Aj*jQas zS}4k=aUwZS{FDEr!iEs6A}?6X;bZ`~&E;2BBoo{hcq+IGr#oVyMJ&T_m=TSq^gw?m+_l%DEMly{Z=JaaB2?BTlDmf5zKap7@eVWB zB&|JPeq!$#>i8NU%<<5f2#Ksm`cr<-P}kQ8u@-+|=AmRoufs3weM3FpWrz*<<1;Ce z-MyG!^81F1e3v6O<2`5cBxifS{KCc?>ie!lYy(BgACjzN;A3ahC-nMEf4e@U8d6Ee7)@;G^ZZ@%khs*>8sQ3lfmRcT`jUP- zJ|Zsj-GDfbPoCL6akZ}?B;!%Av84^e4NsmyOvLqB?=yPrzMQ`aaT=d9b7F$jci>O* zWA_#OEr@gY!Wo~5^1ge2OrE%}#DAL^(C+A?|4DdaZ$hzSpzx3J5fie$0|tXn z#3q*88GiU*GY=+~_BVe%^_0DezXNdvUp@0`Vq3reU(QqZYQ6*F8oqHRd*Wn&VAkPh z)HRgdj6i(*%x3`Qd=m5d8PU|&5%JKKH^Z2S>(^)6BnF#VIx#{3Zk;ue*?*jwn;2|v z31x)iMKipKpZ!mmHpE~HOE@D6KRqLw=a;;Y_I?~`{1g*AfLVq44ejAHy{ z(5rnhXj>cbM!ePXBI7Im#%$Z~iV0A^$o~3L8iU5&!TnX4gxr z!_w+pQ>i;CR~fbVl-Zrq>qA|2%v7oaC4j-k6EbFLh4Dki^?(o^x9mEj3IBf9O`17$ zygv7B@NUaM04FN~$3}kWy?T>0_dUR6;eq=)dqdhWM6XXs6W=)*gy7=ovk_9+khET& zX1Le)HUiYkW*yS-}?;mShO^ z4M&WF8u1Tlm_W*slMG?L5r_#qMCY&cg}{~_Ko-M%&upB+kIc4!qOm`lLuSMIIK&Ko zVs=1UFUVqhQP>E6G-4haBp&chKo~|_&LOAb zhm9K@-x1Xe9x;|hZ=7?R${ar4nENjHkmVD`%4oYe^wiH`TdieT?uYqL5o@A%&)uBj z4p%iMXSuub6A>2C@HzaHa#*9uB-{N6|2bk^^r5+@Q|2QFnv%2KkMdt4Hb$SE+jQA! z1lOdKV}FeD3Y4bL%zc=;FhXx~?A18#`v$Qs`pu8;px`2G66A=FTc$9!N3ZkvJ@stF zy4mZ!`{hTe42S4zb4^qEBi_x-_tcY=G;qG(p6i>c2E}XFT&f#|#DGRe%uP-e;u}E| zTYS)*!q^Oro@?1RJECaT$+P#MWH4q;@6GATR*X8dVDk)5`MyIOiheS;O166x(<09^ zJnfr}I2xTaM;<&iS^z8v;?tJz87HFO&h3%i85On&^2MH(c?^%}cXNkjFGg)!13rk) zSbkufiB>B(b2|_8y_p~ES%(ThjFmd)C*y`aYPBf{K4)3TxETF?E=blpTHBfn8sU~y z20EHC_iw}es79N~NB8smV#MX>KXXrInJf0TC498MK>5t5RjZwQ8n9g$*e3W$>|lRE z0P6Z&w(O)ZtIeyBeUbkSaWlGO?yKyUu)mE{$oA%!AZ|zZ&iw{8b8C)Q5!;7f3Id%Y zb1kw$p+ARH#76Py2rzb?8<5otvp8N!{P07ZE{TvRJ?aQw~#709Dn$t_h ztlPbcgI6Z~WZa9^RVQ7-Zh|yMx{nL*?yWRD(;pMZy^%rfU@R-gS^8|Hw;#Epc!JBA11$z4D*pm*M zFQET^a`W5h-HMyj+_9>Tb2vH18H;xx{1(b+` zEP0Hw=qSbXbY^&e7pH_B%;zI2qvI5s@+BhcZm%Eg+msLYs-hn%mdQ7Zyt|n{sCOv+ z3}*Cm#X9*>QChcaDK&&L$f%7@0oJbTpv=!KrD7-o20PlW4k?cp8FL+J#8BT6M00em zLgkhzI?l}nb$?4Cqb<4!TvdLGo^WmG!QqzUj7~rq3zGMWYPq>!QEDk>@S+)t7&%0& z&$B5Dj zyW{xNj8kge3K|#$DR^?mV!SW-*hcp&{>fj99r##=VYKfYLKZDlytOX|U`7GMJ%OV@ z%tTKp2ITePEWTF-JBF`B%ty~ErsU(|emnlnozw$_I#sj(DDg+`3- z{IZ$N67L>nB{hzsQK1uKG{0`_iun(N7>jwt zOq@i&*UR?MeZH>CvX~9?*)LNi(B6b!_VJWO6)R%4&R?7UpW&$ zVchTdm-y6owTnYczyc? z^ye~ziI$cXh?tmp)eK}(f57Hn@N>(x75ifz%#%~hCJziG|8sxAU*~cd&@atq;gk0U zOsd^q^4Gf@i+Mf2brv;QHIQ6Qyc@r<;$%!ZNO46@Dh70F>|aqfRh){+oT>(zL|GY$}-oufWC3%umjaNV^0~HkCqgsUXCtDb?rnrc8%j8>ktSeHHg& zbd}5IHcxpEGaIOxlmitBF-FRDb4LNV!nKk5j&ca>gjXqd&Rw7C8fG?9vnZ|=#272( z{<%j}rX#LR)NIPpilmrL%F}cCQ{E%YCTb4lctvu|cID-{>Z!C5*JkQ_%E^kfn7zu7 zxsj=^5oR+rm*QSQi9ssw&gscaM_pUk$!@1yvSM77&*wJCJV#4fi21&rF1axTcV))h zAz9L>V=M84?^&0E7%ye<+*MgKAZQQ^e7#(#F}}*mxpm5N?pZ@X@@avyWuC_%PxOn03}h;VSIUv)6V|H_jmah^Fp~^adJ9q%&UV9 z#Dp$&F{#R3id)nDV;t}~;Rm=h#AGTDDxOVSk9&2pzw)oUw8Z2q&nOC~{l__->~H)) zm-d)X$}5Wc>8x?DE~+x^7U28*P=+W*rn|DpZtl)286IJ!zxB&+LT3#OY&M#E;pENi3RyXo|2)6 zlk1CZctq9D2$#v2L1mqSBtIZd=DC;iqg`=HG9ed=1$=RZWo(5SL0zRizjVe{63`>Aw7grPNm!&>Ilp5jP$K9N z|Fpbcp-Wh%+Bkn|ra%(VEB5qXx%p2sffIs0@gK{lz-zx#_1}EsOuo z!j^De^%?N|O=CU}5^H^vD?I(sD#m==taYC8kb50J#bqbqs;X|DG01C; zxvA=$FP+6nbp-Y-N_vG8AyhRw-!V&Yu2l#Ggfgc~1q zl`G~Prm(|?jgDESqhggn)0W&WJEj5$herEJ*XMWCwulz--u zWR5~&n{T0uH=#szTRAw_EGrQbIlfdEUqXc{R;i^ho^~8#qbxpETqgWgB`Vh{j!)-~ z1$XdoEdGyBuS!?K75Ao1#@##lUtMkxT2&vFC`HwD^0<2!|GUc*c(;nCj8rJ5bwu{v zlphs$2m`7bWvT)yPY~I2DKx;B8B=vCOBFb|j@X{ZzoQiifHT9&R>c=NR&2=kWw=BW z6sj4eOd*xa#fCk;l`e4vjRjrRig||_ti-U__m@jNL4RSDYWF;5MlLbz^Zo7eh+w?1 zNp*Jq%M5nHu;2Hu%QJ%M!d}&<$k7>Wa!o&<>GGVgcHyAv*}V0v*Ccy@U+waWK-j!+ zMpZcPKg*e95AthWQV2U1uBhtgvu3@d>>+-=OFCi8n-JB=eAg^fN)=Gp6%+z;;jT(g zX*%aRMIELzR%8*5EWA{025cPW6m`UMmvb(dNo1%FDK)y6i;j=-TU}Pm2!pQjIYF{mr%8Ed$rMmttwz%Jmc#F9UxRMyjs|{ z;IHD$vuF8ImtjKVLiWPRg)Ehql0Csr<28xPlf@p!a@N?nGg3Us(%2Me9TQ6G}Ti`BKEhI0v zFYr|^DuQxhenGjQp|(oRL#;;*sUEAPwnWWB?VwtUng^su9jP7rK531bvl?Da7qVSF zgS)eu0bT~R@#+|W7hzPQHa`@+Au_-VWnGX6uVB%RqdtPR>(DViKf+} z%H^?J)V$O_tKEbQt4p-379+QA-l29?jir_ixfbzWJ7YmVo8^E7YkSs4R%wwoG7Z9n9x`X;U0O!ji* zme{jug=%QXZ}k){5B;8%NV`~+S~1TKS9vh@F(5SURKKY?tSeb+wZ8I5>`OHr@k#Y;&GUK<%X&5-PsFCG z?S=%Y4`{aQH5m8UAU$F;)eb=t)z@pCU(~QX!|u6fY@V7Y7^=y<&^i z{2?{!=~_qhi&tiBtMrcjp@xBss&{Ms(=Rs3u&?xutyH@QS)^g6ePnU*s*D{-!A7Rq z3&=)|Gul5EKVR+VfDDLbt7SqCYrN9lyX5&Azuic{Wmo$I`A?%o`(NBGQ!B^Hpx9H7 z${;ZsnmVftJj{CbB12*a)#@Q+je|NV1|H@;&dBiCaWyW4p>bblm7#|PpiS8XO{obX zof>624-L0lW`MUwEMZfFZk2(ojEmKT=&RGNI_O3i8Ljo(kBpDihXi}> z(7dTTY$RE0b+Gbb>@tXr`f1INy4RLU)>$2{+!M7LvRge!b3pfqWs!BpiS5K#OUOR; z2b#=#^)e_VQhi!;N^i#4Xp^4@GBfr7NX)6_PDh=PEzOUWAmWhiHv08eSo>wK`u}9Gm9- zTRl;0z5e-?4O@FIAiu_5hje-7Yx(H^TUop9vcCX zs*h@Q>!+-LWbfyTtcWE*AR5}*%NOrm^?bYEC1h1>0%WPiM(s0;f2?}G!_Obdj3q+W zY8=*nwRn@=t({gt`yKlRVy|&oyKZqFKzR9GF=xe+AaIQsZOtXC)_CmdxrS_t&4HZM zAZs67lCs8Qch3!ETWlc&rNPj?zhsrE$DW>>$j;brkXsr&?cYmMOg$WXf{?t}B{oqS z7sDk>W=xIt`rStM##TWdX{^-QWiYd4c&}ARgB*@+gyd+r z>0}$6H*aw62}6#>b^vjYvu~%tKl5T}2DVZh+XJc6KsPNj+-vb1<`;>S#va|u(s-wH z!Z6R`HrPN_PREKMe2tAAcMVrrdLVl6$hlYQK!-V~NqsTBC!050Og&((I__8{J(?Bi0!m@_US25ofD@LG!zA=+c>W zMu+{LB3H*bs9)D?)BOsvkVdY4iAeJ}sCuO4wC>c>8Ec~>e$SC>;|{1l(p;&xYgxp4 zqoaN=ksIQUtEX!2(F{OF=VbYT3*N zqZ589$Q^P1>eZU)x0{TwZIqm_dRw_G?uI%?^RwPfN;A>7HwUAZIi^qZ^_1kagWqjXxS~gvb=p$ z!>OJuKnA&7G*C#zq#RbPY&`#T&ntRt)N8%%iA|Mc=qHXJ>oLe4{0SXTEF7_ zmWDGu`N%VI`RZr2iWXg2(Y~eOY)=95eB3AXt6Ho@*(=W5Hk|7zMEU?OLYS6l(ZGs! z+Xk;5D$+0Rr+U1Wf&TiH=eIVT@A-th5?8JMTFXxV%F6bw4HtU8Ag{&k;l0y3rk}p@ z$hP8(8Q+k*PZ2okpS1qdZ&~?c+jDQf5&)OvtJ~Ve=`S|fWOv)gsuZv(hSb^rW9Zxi zT7LU5j83c8+G^FRwRY-!+B&X-YHM2uNlRD>wWUI+gb-3rP_|4y{!~ zXoL`I3nAV{2;qJH?yt4&yXX0Q?)$p*WX7>pEuJ~6u-C$;n4cE79PBbq&;H>FU!B?= z4q`A1-s?Rv?zAfKg0D%v5e_CE3x4Vye)ZmJ(yP7i-p%m6#q)La^kGmUYGl#r)W)7$ zCB+aObA4B63Q9?8PiySGRaOkwA?U|I8&FZ?c1~m8t=k){bjbP{&|y>;S;9>n2(K=- z*P-f{LXp-KZ%KOUZ6MEd(qZYJgeC%_on&q5-SC6O8y>FKmqW$Y&OSLASYT`}_Sf00 zuZ9j73GE4iudQe((*cPk;++rOp-oZ6-4|vwraG)RFLxVy@0@{R1Yx zwgvw1jj7MWFBWgmd8U8KB;R({AHFGdH2g~O7M=I{PfR*(3j*MqQ(uK&2dKTD`rl1v zY-a=DIjLjeJ;fzDh6a`}UvvRLcBSq)tSA=gSQ&W31`UTOL%Cy`aAk3gPVPEV#eP8N zljNno0jki0I%I)P_iZUUHAhqDxn&b!jN{9AW44e`|wA_|LL#{PQnuHx`Nsz z<{!gD7oF8vWpD$=w?hTTZyTEme_nhM=tf_`6n6E&@gPkPf`T!HI}LupLY{pKh8Ltx zhmRNc>YOQpo4RANLZ%C_Uv4d7+}5cx@H9=wjD!sB82cJNS*+4IVGw89jA4Ty!PvL( zsp4lq7PiTB0yDC3XxG?}@af`loj!vqQ=&aPbf|FbXZTF_CMzNhiet>4A7I$sSwn2y>j!`h1)|K0jmJWp2#VwP-%rG!iNrp~A8 z@CI1MC!4kl6YyleUK^yk-%mSiGjEH zBRfm?5hUSVJfVvzsZ6z{2JklPE?#pBAtub;3a=VNQ-gS0bw5E~AryrA$oT3p3^kO; zy!;FD3o%E4M3E$o*d8j6uWM{*V-`XbMv-b7@jVQlSl8At$gGeEi6-r9B=$t}YIWTV zlg+w`!f4X|Mn~#x$PwK@!)<0W#Mx;0s3OHZo+s617*?4P9oR8LwPP;SL|)M;-DQSt zX6X(iF++7@Zd5j}LwBv=eY0i<_M)K!V;Y>|q*IhEvjH2}lgE3j`@m3U9_!Q+mva;w+_R1MS@*S}++6JB zyd>usb|Lk+>o?s`hOf*OPW4OTkB@~?_wfGc>Ii1d=bRw%q{c?@o#W~185`LkL!5>2 zBxxh9N5F&X*%}2Q6Hk7Rhs#pw)N-D=o{Q0PWT$fh3*MB!P>Js0}H0>q}@q(YUDRAdjTSHM)araEV&leiD1H z=~TGC-ew~;a?qt^Y0iJxB|XP@p?XC|3oM|n&R{KoW%bB-OpwRNTg1AyB<7sLF6}wV ziv#OjnuUUFzN~%tb%ymR-cr3&M$0XbRorFMr?Ja>T6xKOxo5I1CR|6B4V@WVL2c)y z>Gc^^SrFaW%O(1?Nx+4jrT56_v_*=Wa(VmN#+5ynd4bq5qkfB8H_8f0TPhF-gYJvg z=&i-DTh|K7xzsh(o4g{u`NsN|NOuZb(w>?|?d6r}A&ea@6WzPml8#g^b%0l`hcjkc z*1A)YBXD5zEQj5@k|Y;WGpP^2yq0QQYKioqBujuVllqAFpB~Hjq-7$Y zc}jpsiu#myPA|>4-?G+&vQly>btCl!XdiUo$1MjvT2|&<#%}I;#k-+bWPGe=o+m0L zzH2Ops^<0SRT^Wg=$`c{@mI$3sBd{!aEFb9tO`9Lt4LQH^Lq*{9_gJjUT)R;kF#p} zDt24X6z{p-Wq>N1@|;{XeGLne|2(zcfbl*ncdx9~)7{veJu|!ydQXkdTV;DquAaW$ zsMMR~Y4zS1-?wV^Vy_vxF}9mJ$NQ;wL_BFV>ZM%MezUQt=O1sLz7Eu~0ZyW%N^YeV zQ*}xV^bt@;R3fP>Rnn7MLNzEc)yF}Zs9F*wO(IVfP>oBh^vTc+lsLCBEvFY-)&ncS z=!ZdhC@9&PlcT_v_n4J9>Mw!HP_g6|PEH@T0vLll!7g8p5|b&7)c!Fs)uzNxe;)6qIA7c>G8@j2^C+$(65AItP}4F(n+@)_w_iG#Ofabbdf@D z$l9T-<+VM30uulsHNm>oJ8SLq9qfS~_mX6Q^Ut$xJ*Zkcba(6^6`a!bd!bjG&3z~t zlEKtNKpUN@uY#VmP6SX@$-UGg)PR!B`s2`k>slX5rsRI=QEG6>c6}}Mt@W@^SEl4a zD$sYA6cy>uH_^92`ckqaL#a|KFe&RJOdM?zeY>(Gs?;WGWJ$F?&V*@G>q}WDd6;^F z`q1rwzLyEdX3)1~UCtxye?4&}$Mk<(-fg4nhgu&$Jk~-@C^@0O*yNZE-LHOq{Nu6H z)a4~-^;enPu;KfmHpBy^TF+(sPW=rg&uzN>gd0dt8{4SSh1c|Vn0&XH@vC1w@f6z* z-W$1onTeULyMI>p^a%ERPkPB6{evdnw%PuZK-V%B>@S0 zOEkT2{6coy+scP zc5RWoOMO5+UJ_zu^}muO22vQ)t~Q9W zRq`SAG4)Id@?aZ`V>cMoWgY!->_Eo`DjH0l_o|(-0E1qy0A`E(A)Xw&q#`issN?Hu+)}O#6F6`QwGX)39 z#*#LJHKuZmcwv@)%=gCM(q|=|1{+MDW4b|(- z7+f*kWLhPk4+kDcd~Zl5{2{g#7~!Y& zaWFZW9|YMB|AL*OPL@m?NY~8A@~IF39EKy(*!&^LID$sxM?>o1p15?{2!F_M+??jj zXF*QD<8aNi*?Fgoq!w~Fei0jT0lpqLN>d8jjipv{FMcYd7cRn~Bce*%p;BwP4?hd? z2!0w6l$E9JCQ@5@06zz!f#1P3L`0Rf!=!fdV15DQEBpg)G(uU{ZYs5xhw=A9bPz`P z<_9F9IZr=!Z_&Blqe7=9hZ1F;&fpx2kj zn~yuv;`m1)3lV$px(t*k9y#tzOW>b?EC%H@I-_0`Z!zvlTh2cVS%tWPpP10EAX!S? zc{=|N z3GF!l;+;Ka3ml*^lTBa&#uI2K`HKyYf{+?N#`!JLRhlS2#ZNRmV`g*wd`v+d+%0W6?JR$# z;T5xO4l^;cb#V7IHtiUKV>n>8&%u3B)`4kvT(Z2Azuxev*?EWTMUw}nJ#Z=VE`FZj zJF|xlt&6e_PJ7~3%e(nI0Z?0i@A#sTgF{~9YiPIl#fD{D4IR<3QT6R4X_~y3FElha zcXdpORn`L|-{;T)ezhUa{F1dO*7;D5H*T%`4*vkCE^-_PV_Od8_~0_-_xZ;R!_0Y( z(8bP&b9`~@?eTG6(rUQU{HkN&V#pDazcgF^jNfUv!TdR> zKMIeK0;HSdFZtadoBYdhZZV{RM3Lsm$N37w3UeE$kT_ujDNvdxf5RU%JYpW?R2T<2 zN(z$Z%ir^d4bPY-J9WnikCK9=+W-rE6p%G`I?csFj*&v71#&Ha!tk#7VW*HK!egX` z(w*{e{3*j1=2x8xmq3n_LZyZBpZpoakLJ&vx|ax#lftBX|XG@1@;{qB~ zU~S}w+~hnFKO!B97%!v21vn!HvdWprV#|i;C1Zgy=CE-ca*m+isM=z|f7Ipmt?^0Y=V=*&dqzhsLf!bwQD@_m$2)23 z1vaUtE%fa!EH5|T z3ycjc&D`DDS?$xQIJulJFfz8X^mfl?Pqt5|;S_SAV9bVS8Ry>2W_Jv6#`|bufwi%p z*2dgb}{wbHxtg94H<*Yb{g zLsHa*_6+Gg`4Iudc#Gu+_t7Nfg&fI}2ejjYFyp$D^F5%+&Yd|~xO`Wc;9jiA(#9jC zI;Ru9F6|-hgkX{JAxnRc{N&kA`1-VA+9|bO{wmjhByRzUCd}G>k+C{+z<7bwaJn~o0UV?8*8wFV0EynLHpLlex zECBu*DOA!e*lGO3@{7mR%E`;qn{i|ETLPZ3ft8u3drDTj0p_1TW*lj?&(nQX*462J z+z0uv;6rAjRhwt}s*$ThTgN}rMg$$kYpw2kHm_n|8`?Jh31H=~7;m-u>^ZS&tNr1-FgstUSHaSC4cL?HK<`12rk*6IO9v&8ykhhm4MV zlYbP97+E0Gs^O%w=!N`K0~3MP#oS)KMuS)&BGE9q~L zEuJ>kSl#hzSQB-#9Yhc0zXh|#U#&iPjjmDNY%h}jlm8RUgX*A+Na)lkunCZYL3b$x zYL0RxrKBovwHHfudJRh9Py#B3H0bKnBLNrU2;XA(98T7(RG0-p+ zj|5F~mgn$shP`H`j?g8jgoWX0_40VZxKTv0hdXp7N@kstHYdDOMRg0P)|sm zX$8ITvNTwPU1>0M2kJX%CT+Gi-g_Rr7h6h$mZ8kZ?wl;ev=C?3ODv6n9z=PQvpJKB z_-7;L5#@q-Xp)jnuH{hrB%(Bn2)EJ|&<<1v*?nbmUrq(is@Jo0HS{KmM}~5p`*XxN z>t65DchW)7Qm1q4`{OIeZ6f?jH$q>cZjkw0)IfZdwCP4*X+HEL>N)7K3kOKmQcUl{ z(n9EO)GzWJ7osFdq*&lm;6shAZM;L$g-TM56yM7z6+>;UgS??XW|Z)2*2LbZ(tW_T zkl@{#o^^YAAI`Biw)8MGz&g)+Dt+?y^nRRkZ+xi?8fm@H+kI`;o#|SfYj0xdY3K^; z^WNEOC+|$x;p*`0(hlfa>-*kQBh3p)9V zjQV@=N5=glHkH1FUbo)kqnnAkAKx%eiO4H`1HET0D+tOI-X|TE2K8<&or1owzUot$ z33&kbN(_nEQTi47-uj78XXeP%r(?L#-onyf&>z-ce5NudA50&|f!$B(0uuupGhg?t ztfA>f9IcmMY6RR8-oDvclS9)|9KBaqYG&eOli=H$m8F`N;h4SRQX7*1n>^pCtVz{$ z6E3P(Qi?Z;wAttDzAo$GbTcldx3<*PB$IvCH+|j6!=b}vi+T^1l1#WZcYGVxMLlXi zDP7!qq?BT^#pZ+W=sM-2_Wz_ydXJZeo0QlX`JvZG4Y#*QS-nlAQ6~FsT>VnkD~H=p zNtZ@^Ub)0XYQyxaT~7gW`?Tiz)1}Kz&e?GM2G_Sd&N+iy(c4zK+T^AU&kwr6`AJSI zE~)o?X@<$l-$(uE8|t6LpB+z*xKz5)0|h3#|Slx*cldj}wj z4wOnv2(~f)4cSr8+JQK>_ikyuiI44iAm30vYri1P?0rz$XcA#7@<(rsdfwhCUDx}l zw8dnp?P>p%jmqck7o{6OZJ^zRYkSAPVPn*b_Dj-@y)R0;Ot#p5@E_f%e9?Yc3cNvJ zsZ?TX6fk($2dDwla(k}?_M24O;satgwTzOs59UR@Exl)Q)HXDLzX|m+r_ky<@qOtd zlT)_Kfyb@jB^(&mB0iPAFu4K1k3(XV9kEy~ztY z^`~J>XYJ+B)6w@S&ADvNQ2%&k1iCB`wiW%EGLbu?85$U`j=+{tVFENPkeJ5?)cf(8 z2x3_@tPUtn(g6p42r%3uoXc3S6X>|W<~;VBp*!Ps5$Rnpa0$XZ#8c-OO0H7#61DYr8FrRITpxUiUNzN18+1})`OR&v$Y5?kM z`Iz$*cdmDJ*{q__?r)GT0Mo^fjJHQ{%KBiHc9>u~0I|hC8$TZb_-C-AcA>%iZKzN2 z&&N9>)|WkoowZvXtk_onDgMRyC15Xj3A+xWr1VJr&!kZ)5KNR!!0y@|4i4Ea{7iZ& zz1q9AYzp?m?rL!1cF6xoucV+gUN!^!X!kt08-R33ucbG7cbCn-f-mqlX?G&(%G^woFqcB|cg%i)zfA*`{4yWY zb(klh3Lh0Z^%i%(x1lV^)FWvsWMan%=z@l-Y%IyCAcqY3UdklJQNA?QKFs3WcB&BvtqJm5EJl?Sn!K3n4#A zU!x-W%g~M>AOdM4SgH`7%@`TVY(K`dYX5$e-C{h{{#d(S4_9se-544J@R|#$N2vu z#>$F%Oa=C^FybCIi206dBPPmjo7UNThNbVBHE;eY{o4DkY{<09J|=9!&KqPy)4oM~ zEPHC&VNYvFzF7AMeiX@c{Z#hC^osqqu$euxf8cXzvk_Y01iEd1Ak4R@;4l17+OLRj zWgktS*gfQNkPV_ zY#w}ffNnSvWP57naX{)uXaGlGjXI-2+9h{BUPp0YiwWEg8ye2vJ9*X_ggX?5LL}S` zn;hP~S2&-nBLh(dAsQZx-5EZ&7qWn?E7K1~05_3?R1)q!vwVR{S8Yf~3SHpKuQ7E^R;^f@MUTVVh^7yCARxVS4}y#d}-ff0tni?9o*P@c1H zE(DKI;Dt}sllR}WSRYD>B& z06kvE?xE6~^}ujl1_tB8bogEDVQNT;P@ilfvr+g7v*ByHE`kZ|tUdz9LDK_;dt2UO zA5vROvJA8^JVp^B+zJ1I{X(58nKaOv;=yc8$b%c;Xr_2R8`6<3!YLwzLb#Ssq$TpZ zARW#N2nwdK3T}@(c4jY_g|N)jM0$*{4(@?lO;hme4O!;0jTaUR8vucFC$0I8))0Z@ zIMZ3c)fA08K=Tz87$Gb;uJmQXGw>wbC0f2<)(BzAai=ENBk5Ugd^|^xb+dErAm_yTlEX?&BE949$aYzvWx<&L30A>TZHf6D)3-U zEbD^R*l~jC;9&%RdbuY;T;^<=i@`5c>=b^7{|~2*7%Xcs&9%peDSG3A!+zm@Ma-2! z;AE^UTv064MeHYhsy?K<9InEusdO+TLA==OLQfGY;T@Z;OU{-G{~SQ$L%|_k`BHZxF^lqULK_szfT1R}GqCpsh*pB}~pDLd;*E)e(P@|BBD8rjE z+(lVPtusDB(JYKX9K?GwvPF|ftqVR;(IQMh{D)6qw2HDUw66H&idJDV;xbk8NptlRt5PkS62C;%|sd85*(L06f5RdU~4B`&};LujbE#X*inb@AgpF8#P!xJA9V)(zVIkwAz=?ww-RN;@>OTi9|}(( z76S$py|Uhh<)>axeP8F=)PM;QzBl-wck;G~?MirvoL7x%6M?3}&vBaTXOvggm zF2xVwG(tnT1EgY6_8p+2qxcQp<*$Sfk%_QN_BCOgBKkk!Kg4{ZeiTwd!PbOxis?G# z24)DNV^pGK)*u7ywG{g0Ff)51ElMnL#^uuR0)SOvq*8D-|~7{$`tr>Zrk* z7D8?$zFL7P4>cOgalrJ?qMO+@;xv#(hv50ek?oqzd>s&=-1P808ga)(-wL`L(mohy+eEJsNn~ zTpX6ibk-KQAeM4Y(ifK>F$;9q7Bf>j>w-w+w9r}QO=i&!2V#8d3S1G(IH&2$%Fmc3 zIb4d#ubXv6Ea&8%W|vR2r90e@X|7|tsg|qH(o@Q>m~D0V95Ycj;-*@mK1WXln@fR1 zP#x;PY4)tt%fkJL7QP zxAVXx<#RIrk|MkOIS|I>Et)zo>7iYT?^5KHtIY-+_ATnBF7;HUsISoT%io(lc4%9a zesIK7wMu=BUQoV=q;|NosJX|-t7bLlI(=995AaxEPH^)Mda>51Z_xLY|1taH@OP1J zJ&MFiRo|lX%Js}m95J!<`g#&8O)aN`8;rS~V`wbD9z|wx)C#(&+``<$k+=G4y^u`i z%KCwH9Bm%#xHERH9^y?-mnjuB;5nD;=-K79N85sMbXc z(hrp%)X#K$7~6U%%SW4mzprR04>I5G_$7Ag(4-I9tLFi|v7BlyaD*);9%lQhveYVi zb9uCRoulXCY@c4=j&-s}ivPgwp~>-cwB#_wuVy{xG5t(=k?MJyBWLm8;TFH#4fv;u zw(`~HHywG4p+}tkbF=Z!6z9t`%pW+)7RMfG@z33ef1$WkzRCQxqkOTr#yo(vN&Thx zN_oEdH}>Pjokt1+5Sux#=-114ng4YBzIf)yYycvMGe++zFEKZCvW)X>D4-y6Icj=e zd4;*PlYd-(!z={>ELwCRi86O_S{~QgPyoz2oVWCQBuKSY{gFOW-eJDh>3&@EQFgFuoB9*b6J9aj z>h$*5v!lx3j_tDlDPEV~GABFeFF_uo0G=I3OV^YSnAbQtE=fGr6;iW<^A%_#9+*p= zm`f7xV;AP`#D7zKD1Typ&WW>R@ED~#e3$w={d4(CfcM$61iDf$lw2sARcOm6%!0eV%ys(oMKGkjGc8`+ z=uFM!;XxKdWQnvLSp|NK>Zz;}wLSwTvO_L(-V?7YMbTJ%^&9_15CSsV`9Qp{w19>X zaEurhA~$4`^QHKF=`0OV%7HShMLx)N&QIbyr3Dd)G7h-#h=P$joWIA5UoUB zao)z7ki_(O*KtMYOC?szHku_Y2ad(;NF}SrIq0EeRPosY31Ubklj2P zrLDme`=UhqfTb}Zp|v?HTDuSL*cU50j0|weOPFe&jMnbQJNLzlWXMRDeF^R-vSPHg zc-Ou}(P`ugm-7kPCnjUGb$Itaw&*-E)8%18>xryI+5>pcz7)|_a;gYd*_+;1Onj@S5#%9rLWRFYfQsjS>xS9q|AY+T@9a80Ta%m#)N!A=yFWR?F z^cnfuMZQ%0pYxL3WB7%PxYTdRPcD|QEB^_XkdMnk`*w@wkiT7iEgj9@`*z|uKD=+Q zXn}=+t68FZOIEzL5l`#mi^6szT|E=iTSnqlQgsAFC_-2`07+$Y3!9~qsTmBh$l9X1 zF$-8Lm8^~?S!7?e2x}4HDoRA3ic08cmPPmN7q$I}cRiWd@ZGU4=7c(iaX?h$VX?~f zMk4jVh&aVbV4M&wvN+`GzbyasY$D<`Cy{YVwAA91>+)rtrwf)L&Ty79&WcuATyfpD zY|it|GVK{WyRTiuu^4dOx6J)amrB{HPGWS5)>%AuZCjRpW@NeQta>G|-R4@nalOB+ z`3!r7s!hF$(GB2n-&{X0n>aJFLUm5PhH*<&Y@z1{TTX0cvsLZtG=>5^mMz>om#4Rm zuvHyuE<-7*v2XyA&gRyviKg?iwS9L*^%g#E>z9wVDw8@c$PN-8i0%!By787n&pIdP zcH-CdJrtd^SmGvI9(%SWIrk!dL*El%m09B^UoJlDyfXI^eq-Nr(Itz`ZtCTOXIob0 zUgq$Kz-ex=+wJdi-8NJTt4p23P>cF3D%~(E(5kW&@)cQL-y6}O#Syol6@_h(RphI( z{J!_1VT&_v$t$|sgsaHcWZU{aiAF81x$Rt0OPpPW=*Ayuin1TK=yR)DK|IG^t?E|q zV9bc#TRe7aTakWlWVPzLdKY6>q_ueCc7H|lIrbRTZYiiUi+)&W-QKPkKG(IT2GpJy z|3v>R=DX{&k?oY!np+$YWvwuSnriW9)0Qm$p*%u}=wD<50eUyqk%XjV%lUgri zWoqx^kM)IA?6myh{v~PZ!eplQ0lu*>yn<(G;9-_r*q4x%ixo-xA}WNI<{qxeDV@r! z4wbB_FS0^nN$`kCZs?3!*YQwx0zgUgHjz9sl7~CH*3~@X{Kr^Qaom#XQJRdrNLgPq z%sIta3b3UtkCVxXK;lZaFgwFoKsXCDL^qQ87f~BnPt>i9j}D!d8$6yTcV854P_2t^ z13Hy!mODJYC(m4*-GCV3v@_BwdM(R6c4AyFWo2te@aOx|EAChx^zdGpeQ7eA75L}^ zqiyMsWs^tD%7#l(8#|uMF7|Dx7_ofjk-l>HQrE_s7o5wC%@s`rH#~SN`3KE5u}0Nh zjJ%3BmiIi4fqO{(Ce};!RmQf8PnIt|Zmi^AMr~%jQg<_UR(!Mk%ow-~&NB0Vb;PfQ9On50?b>aB+b6;LZ%&!CjTE(j0>%YdM_ z0%_&unVizyCCnvjWc_`W6=v^lVLti2>|x)Didd^7or|v ztHZo6oC(J5iv3nDUdvaf+~Mrdeo?o14^|wq@&kt6iSCgds;}yIjG+pt6~n7)HSszd zK=jn_8N(H)td@EG%}lzk+}ZI>_Ob72MY|Q(>(1(i>ruNpzRN!K8IUf68{^y6!`HiZ z)d0p31E7VhioNvLAa76#Yi2oG1|Y>))p$9sNxacjsJaFF+GoDxp4CyW&^7!UsNF2U zxnO*#cx-jnYc;ra*6(KhR(}T$)R$J*K_OlDCTb6BPCd)`Qc(iE>vecd$W7rM@*ml+ zzHb#%RxiA+t|~G(niaD#_Ucc7N-7K(x&Ew8v zu5wy{G9a0yy5Gv$tDQ$!(61{tLb>&Oq_!H%_jb&0(&;x4!%+lMOlrfeC{ST&((5-C zqfkDi^{Jz`l*Ju7TyvPI*d9eCm8K$lD7+e7t|8N09FQ4DlBLG>wD5BE2p}9P_C&2A z$y3EW&Lz3}1e1Onu|H}vNu4^_(^8TP(qH`;aVV;Yv>**Ccjl{3E8$GMn1QM!VbbXG zdOpigW5#q8FGd|Dg{JZ4C;`hz0|ss4U-x+L6Vp1ua$1`--?HCboQ%3m%1fJ)PYSfg z1XMo>9ES!-`_kNdvr4s4f=$1#cs=SV>3mvt?_{aggn;g+i1SeINDtFmd$Y=N?e5tz zL&Q5!-$Fg`0Kv;1-nXAOtP&>%qIWv8; z6$liU#7q+J*1Cv!+I|2YR zi=SSbAm8UU547l>z%&68n0OZTmHdG_I-soXux|?MFA~qQ*6}ts8)2()qVh`&o{fSmt4|pY=w7idNjN z-_LU5`qvy4hgkpjdM175_UwMdZo~NgMlsF0%-d|O`<<*>turB^zgZjuEZE*_v+qpS zYF!A4{Vn1I>kE#tYn$(|>r}3qWz1G_lJ!OJO=~CajMS;zG%J|x;xy|%@2a)LyX*rh zcTEzrQ@qalv3J|r^t&SmR34g3hcAnBtToP?;tmE`v=4lYl2Tq2K1@_p&D=Q zTIOAVz3%v(nlU)oawyk_klBA`DxZO z>8Rt@Nj{e{^6$+aM)-5Hna{)*tk?NG$>_XSaD;4$-_$=UzG}VQ=Sv3v(8v*0fMzpu zOx$BF@cH0{x*yfhL21hE*NB~tj7e~rvG-dVasvs#1}eU9ecXqZDZcM~H235EeC7x7 zW9xAjc4jviW08ZKw)KA!k6K>?EY-RD^(P=9+ybUntg*iBb0E|ALBX*OF@9(NjQE4~ zQ=jvh*$*a<0f3v4#jIFs{l@2hX7dB~aaCy3yBoj7v({gIK4gwQP#*6HYbxshC!S}c z<7G^=#5S?qf(i>|7du^1W=)3VaThHBrHrQ55O zA(&h-)4I~fW}WYotWH(S+X*J2svlh$WV7A(OV*TXvPm0Bko03KX*OlPX6w3%lkm2v zrkZ|2Wt7c+UsoVSS2mMfBla_$Dwo(ueVLWz51mis#t`cIT`M7Dt-h<*DIV6JU@g)d zV0u=rvbpZNXPxdN)JayXrk?3txz^^M@3D3CNA)LJi#3Ot{*@bTUi#ix$A5(Sj}@nB zUiWXREuVCkb6c57l|>h9*7-eI-}$)U3}OYh zjk&7wnoYHAdi~Vn$urs&g!cZ_O1Vv$pVBYJDNmJc9jltI^>447w$b?A+0gKGL{zex3ka8$KWwyqZ#N7-?K)SphI^A) zRQbx!+*CqOKd(~X*3@4=E_|fEot37MGfOK?Z0-C*v-u;a4i-nFV2Ua& zY(4x}XDdePJ6K#zKX8iM*)H_oldbz~c9Tnbld`|2l4!fg|8RE5v-&vwwcI<*I^eTU z^1qaw|7`X=B7-}~JXGmpyUzbfcIUH#3y4haedf{1{Bzs0{=Z$`!U9 z{uvvGpLboX*}#3uJO{`#Js>}Vd_lQXlg)j`ya0U3D*uxk6JK;)s@ceW!Mt3#*>>E& ze`D^Ba&FrtNw6zNe-NYY7U18;ECYX0B>ufy&R&P>_)?Z<5(Y$5euRIF4$$K{G zzI5L2l;1Sjuc~aeT@>&T>ix3dDq<`51M_j^8QY|QOPlgv&R#`q1HU{EGtS#)20Yx< z`ZDWU&DDWVOdx8u-5T(D)5Ob>YpU&<|1rlZdu#;(u+79*>~0lc=`$xPZ`;-dcy3OA zHPWrxq4~<3tW?>a2#DL<{EB^D1sJo;smf=z7XvnJo_IBKUA0T|gE?I}Zrc}7wVC*u zeM42K`N;%iG26!hZJX0ykK9o0*1V+{K>i2fxBZ)IUsG<@?BV`l{;vFCs||R&dH8kL z&6*Tj`%H47s3t4z>#l!We`V~`#) zuSs{nunLKGqa^2aj|qFoCEU6N2hwcN0hGL)sjk8jt1TtK2GCVO=tTc$Zb_;^=U;KgHww6Z1e@n z`rJ{q5}d1>tOmTQQqjE>Q7&2|gbT%7>qwufOtgw}GB;7v1wQ6n+eq-=LyuGXb89t} zfto6=U1TtjBxosbbB8ru12xrL`^d1WBJ}(~{k(KHXJxL0fE%Dyl>&_dJ}-8nMVVVe zAPg|8iY!CXfuVW)3Dj-YJ`HhTGqM)#7MPsZJs|{c`zBznuWCRC2ky+9n}FOQ*ETs1 zu&NASMFk$n^LO49#@L^u-o2Z%e^-B>byO1+30%$Smw`97@ZX^xu#Ra$B6n7OLw^cDr-BxpiTJ`Kv%13(nJp2-io!02omL^-=h|kH*=#S_g8)T8yEC^UBvtP zhpZEtn8*WF9zbz>10=~&k7{1tOb)vh;4v=#j^>IuS?kra8Ms_)+YpgWDUkM&Pj zXEn)@m#Q|}y$rgsmH!d-l+~t5i3Hw6yH7!{fCRDrDeIhOb>xkz-FAP1{%+NsLXD{8 z`+-`pif@-hu-N81RWO3+;Bq4Ssw(V`29vgBPfdr~b=?Ri4R zz+ly3yMW-lZBtW|&&VcYnUOC`cIAnKWln(q`l$SMExmzPYRz0=r2+r6({JHC;b^x|LGMMzz z?t1X&J-g)$=iWA+Q4CG5(y2kESFNQjNekgjfDL=+v)&JZphn$VQrVD zW`J8AXnD&qPgj3^f6(K(!EKzR-1N%8YWdL)FwY5Lqq3EPloStp3@d)Q5x<12NSH zFxw?P4cEX9l>DgaSoNMg@=LNMfidIxasu3vg%e$ z@%)QB^S{l0Ks@2LMkZBXz^q&NWM?PvzaXA++agz0U&HKJ_Yc5CTRF7l&7FO*d{$PJrjcTq$0&q0u@xr!U=|4t3t6plZMHWe;$ z-_`ts{Xf+!&2<3c{fYUu@bj*TA0z)$z1G}}+*>{0UJvkziL>l!)tIIy5|knAEkZpD z(`QGfRpXl8NPzIQcL5MwT}SETJ+DT zFC7z2w+HrDI|$-KPZlQrr1+=3A>17}Q0-~ICbaF+-k+$ithbuM$ivls_8UWA6)FHs zmGw?@Kk`_$bK1_(Uxjl&Av5I3rlA38HQl}f07yfA31`Ufn;s6FsE)Ni5*oC-5MVvY zADV^-PE{w`p9xLg-Th1WjSOI!181vK?5~CH+&%XT@|`@@1R#~w>GpR+5AP27E&NXY z)bxDdVs*Cti_ojP3x7j?)G%0p+E~5C{(b0^-JQP+ejxtGeH97d-u6F3zwe&;J^KSO z%>~#(P|-6CvjoV&f?0%?s|K_C3VZ7?|2_G0v$GwhHWT13ywBb#ECE3HvVLm65Z(?9 zRv)$x2+P|uH8=TF`;`DlcGa&C^suTu#6Rp`su|7uNPv{HUl!K3C;iXJFV#2A$H)QHALZ|kA5H%Qgt1%pC1FNIozEx$RL0dtYN`jo z;@z<*@o(2$%}?%EU@CtA8bQpW+P{=PHNUvuA^~>Gz9TH7X!vi}pPJv?ACdp7UcTW* z7_SHlU{<*RRWN@QcnWK+J*=1 zEu3cva6ZktN<&FansfN_y`A&Q-%Kwc>MNlV8*Bi8I>PU+m3ppwZ@F}pW>-EJUI_3J_Ce1cV zKS?(BMfla?LO^}+hBc#=fs%aeN5EsqH{NPs2H*-&3nja-Kf}Kl&*Wr$mF;b^#-DmcQ-QUTK(g`+gvHYh6v1X)%Fm}XXJgHl^ovb4YkX0{O0 zY-!_Io__EB{sEahK0MF&zP_L9x(rY=Jcd{$T;aDSf1laL1Tol}kuh?E& zZMMPAi~>|e8pzeuZ?+%Khdsn|bJ^ zGT-j(s=gKH)}-ujpTPzOTbX}%Ue&;g8*9q;PtJhMM?1&@!SrWdp_=2yUG(SR+{ye6ii3gWmZ%(H?%%e27YXbz7#T(!^<4xD_lS+s=)CvzlV zNbDW`!tyK3vrG#O&gSTtswzjv$gtxTFbEB5fSGYIyegDqVpvNB9by6oF*6~iw#vtm z5!O>7gV1aZ2s0^0QiXLa4*Li^R3=*k(i|Ies%nknfw1`pV6$m<(9UD=pkXV<5z_=e zkT6?i$3fAlF%2NE`*c{;0p4uvA_^#7wJS|ba6A{5a-ecRqGs8VI>E? z&YoBVb*Hb5xm1{PZk}xNDLn4burhfwmJ@l^&F7R zq5Wk5W{sHks#3>qVIL2e=a~L7c$qhJla}vsoEAR&pu=3;5~a5+z58y}VMp6=*Mq5Z z)k~B#dqjGYtO>KvSJ1@rH-NDyAImSvj+^k zc~gv`O72JsuR9nqPq>ugXU>UvQgz;OeR$`=ih1^bQ~b@jG5uAS9rMBm4|dHH{!Ixm z=f@0IjeIE!|9Nn79>f712y=~kU3J%y9lrRG@BE4T9|B>lZgZ8!Q4o$klskXeVGIi^ z>V8*caBK*t9BQ7Q{m)nstfYIis^9TyI18Y(!~cv0!$5Ce)f>kL;haP60AU>qft7WC ztr~HB8m>H)`{(d7w7X$P%)eD%9EZYt4$1zcEjIvUGiI`C!tq=9$3qD#yO%3h$p9%? z^@r26h}nR0#H~=SmhJ7HRz1tf7T|`d3)Cx=Yh?SoXIAfC;N%QAzjmpwBPWc0Af{&e zVkfVNjKjJGjgAJ1?qCd{=bS<#b^#R5-iZ=nJ{$wBhC7iW>JCRN6av7^d?aRZwTILC zh|a?m3+MP?k)*)^hAR9Hhd8T1^jirjZ(?jn>cD*@KnO{rEm z#YXZEM1U})X?Z*b4*J0Wv9Hz!6RLZgl?2%b4yHGb%#@M z!NiftMG$xNTG_=I+?=~kUcVO{oy?}X8`hdH#n7wOPBoF9V2fx1J3@17%%! z)Sk-qu>TfstN!Hl_VO*DKBK%?V4u@nR{h=SQ{J3e&--$U;J;&KT%D=K`$%HpFgWeUxuC{eXMN!zzK(8Om zpfAH#R@*!KMdhsNU!n#=x(tj4)hnFCqmEQM*yGSjx=aoF_Ff-|i&9i3*sIVSAU@ae zt391JMBS;Zv`1lBOqjk~RK2zPVb!zBc6&Akoh9q(mR5&2?}+NzHEutFfo`Hdj5%E$ z?#zzbx8EHomSdY>Kz^*oI}3o8n7ee?cPty$*L}8{;@l8L0UBQE;G-P#Da(hwV!gc>}mIf>W$8Kf$Zh*H_l&~D>HSsR&R1PMl~Hv{hR&vcpm+E%+=}w=hso> zf!cL;0hE06K+N^(V&@N0gU7o576woX=z&)|s>_{!Mg2TB`8OmGy+!t_`*wAObMN;> z$9)}2fkr`p69X2C&i2v%$BP^$0--FrIYwR0bw))e9dC0e#X<||mKc4t*!f4w*5jiN z!`QJx*t_nB)%DKN(VXM%|6~V^6~RWj`>LCqlcSZ#bN?9*8Y_m4c0a3DI%h>2jyM04 z9XwV7``A5DecgFm^cXll4TBn3*yrw7)pwi^MMFVnWp>EecGy_Ax%%$H8gRs*gY$!- z)I1*ZuKJO)Jemv!d0MEU%=|4T$m^MNYjnx@`ot&EU*~{S&jYs$vD|BnrI$ z)ew#y%q=FRs(l4}4SK(9hI$^y0TvunRyDZ-5{3pZK=ncn3`U3%RfRaR!_Wt0v(<|@ z?yz++*Q?4MA>rtQvbkz|jxTJttgougu{0cdh&~_p4<{H_8uPvCtK&pC6adyZN6y>b z2f_2r-6=a_>@Wx90?4EQ*vZ*H349p7+el#0H($%xiTb(%{~Qtd-MRu4vxihcyttw%4vYb z;}o0(xC+myp}XOdITv8NZ`|Tk!chdJ*E4rq3g4TlarEuF3Y!X^5 z^H=AAU+D&}j5~>dkkJxZpn5CE4AbI7+z=$2jF!rR)Ww`p*mK-Qc^E z*n3<*w;jojMW2+dRPW?W!hYd?awn0HIK!HZ)wsQ!X>bUBG0)efG!A;29)<%xQMf(c zpI78E5eKcO14NW#3rFFTcx^7F@yddEQR+(0Qh4e4T%H~McD(Y8ELwenvjQHDKf-fB z*>@h3(Q&ws1Oz-5U(X|=8YwI}jG*RoJmDMgcX*X36qTibk<=oNKfD0{hNnX{QdteK zSals|C44u2YRz0%R08WPxV)Oxk?>>qWi=75!URgAnF6ji2`~5kN zTFqIwhn&;!NWyVG%!8IOj=!G~d zrx%1NLxFHIX*X}jf%Z_i zo=_)<@Dip`I?QFb@0<_t7lckhg_r#XN~gIT+_T@{9|(hjE-xX-xu5l~x#)UPN6Mxs*pzY}tFeu`j zxRHos#FEUV6D4%0n$E$&xdeoe=r1fnPtc(nIuD29D!(d-6k!`Nkzvr7YjEyd8saiB zUuZ^~7zV9b0MgIW5suDmp#uiTRO)0xHHNzhVI(#QQ!#3$QZEy$1Grnj&R8RCv-8d3 z^w6caVD2`=7otVjhf!s5dg&)|tGK%nQ%F+_XZfNwu^z%stHZblK{>6Hi0Iq6iS-C} zMjgdHj#y3#6IJ@6vRMY0T#e`Q5niMWk5W(yz&;Z5(%gyLy?qk#i}X|6T==~J`ig!F zx0k!Qa|RhIaSzPiGWH5~2X`NMt-&AxXfhSey+GS~Z{RiQGWG zBdNrq3RxDIUM=GKBMZoHBsy$kAB6pLgO6LZlidgSpkJR;G3wwg>Bqatl z7O{q5Mzw-VMh-JWq!mH-#bZ^c`qYiw1mrn#inKk5U2OQ0^aOXFyAG)$mq@<`O%y{% z>CbSNxS7b)o`5M>`h{?MRAy4QayKE3g+;RR!Lem^lOd2Y^jFzC^&{}Q zBgBg8LPFVPXi!)10DzA zF3NxApSbVb56Bm>ohK_++3%+OHczTw@BNB=AKQPj9k?LS|C9Yz{|DZ20e;_4eqA-O z8wx_K@l$xyTp)3aPx-Dc-2fUN2G*$eD`(38)GX(%cG({{`?NzCu0lCW zzEHC|A{zKL5T_HuR23YEeCBBckL+?XF6?w=80r9PHhhuBm6zmlA@1OuyJ5lulsQ9- z@t(Z(E*){ZPTPRQ0m|H=C3rNC;i8YLI~@@&JV=>0v{chEKiB1X-1XDt;iXA)O6NFe z0(mSKYnR^MgN!mg9jF&t-c2{CXHLni&=hIRf7Z z+htL_XT6Sh|F9A`d^FL#Di`N?e7)rVr-B*+98cg0Tzul0^?ea4Hs{43jrL@o6x7b{ zs<#0^I>m0t1yAACm*e9_^&ye$Bj`nPS4|?X*=1e)_4@Ki$WioSxx0o2CX3DS{q^mU zsPMUeF+B0>d6v|Y__6xY$l;@7f5E*q8+mtt{TO=29biXeOXTk&={&VdO}ytBIsj1& zfcL;};yrSa$CJ;<07hU~I^>7X<(XVA$LF6hN12Wp{vHazZ{fXmxfg%@Or>WySn)CL z&MM?tT^_|N&Ll*uz>)`^dbEW1(dA|QoimluC?Na8Lp5c*?=GL>-<;7!H=bZEhp*D? z;vHZ3U;OVgHZk^9lodm(@%wnwQS&IvWDzmKDhiNT@CSMGP}Fg(tULx%jdqepXxKbE zlp7^Q)+OCm4RvNj;g9hoDu0R-<86$o+Tc7Cjj!T4q9Q5BWiT9#V}K3e@H`$0l}Kp; zbCii=fDaKszL5`#LFtjnz!Tko7;@|s^8!(Ylo8ngPR&&!<*}NRAkTO|WwzV_kK-x9 zh0vVg(N|Yd5b^}PipN38|GlT+kwO1jn7k5?s$sdp6E#h|B-8~;hFphltYNvqlQk{8 z-D#+slzsBK1Qeg;4yS39JSM7_QZFYG?BB(E45i?&@bXZxUf1R21c(6bDNog?ctxo9 zlzw?Tfh|CL$UbLiJY^ z5hrS)Xa*gx=5fJLn51YUmI|R51{1I6iBVW;iQ+49LJ0L`Y{Ea{aRQ>K9ECe6TQuei zFAeJBHKG!!EeblxBr^C7wcdKlyNJr9Y7`CueqxS4BM<+acNMjjYEkr&)I&J|@B+;s z=n30Ro!T&$jFPYd;afFtc=u81J3Jm%g>92Su?!Y|h^Ivfss0T`NDAQ=LMO%mmScHS=AUC*+VT;_OdT!iFmFvuhT)dL*Qs?TQng zri2e2#Lut!$2BzJW6IAs9AriW0~i= zK=a}4YcQ_a2~WYDH%cftxe_n0(YV#Db$wxGH1<(cat?tZz~Nh(_}=)HI?(`)W1PaTsoC!eNzD1fH6i=#Se$(Al!zL(>!L)@ zCVIj|Y-7Cq41Vw76RwVlQBAxAY$J!lkmCt80#~0zW>a5+s*yuwG~i=Ft&Bn>s=Q)(`|<|Pg`btMYV zQIdyRz`>)#b!X!5CYvPtX3E;3i};M1d#=Y4lblr_ikhJ`h7!-H(SoCi|2YTy*Ue+s zUbf;l0TNK2NIoY^!q&8;$S-R)*F1A=P2`x$e#y_X*1@l8@@ob`-_pKwbCXdmtW>y4 z!>X~k_9oV!BPKVtu)aONjxVYC;QAu5^IS!;{RK+eP&>Y?=9}w>#KCi2$-_-v8|0mu zoi&rLzY>3*n@omWL~oSe((DB%qB%)Rn?u&Bh4a!GcR-rd9B@SPZ!TIpaS@uq=)zwN zhq@ubnW=eg_9gV~JNNLFHTG@?hO?Uo)~YWlGvyC7Cu&x>g(n?pcA(*uO1fOF;nwVS zaibIyqQA2h+36k0>Q&5*#Iq=7trkW(T4@v9K zcclm~Q#KDh!MD_`ck4*nb>3!OV<%c}$X{#j)*N=TO?GWbO;ul0Zk4~)JgBiAUy&Tu!b`=fI4s5xURxt@^GRm5^rfo6 zA(COm_tuZFln23v<&@?X2zaFPb+s_uWn;J6#~AHMX-# z<>OFGP21n1fPQFGkw*ZN1zQ`oUAPf(op9N)#Ay0+;ey`t?7=Ky04;V=FS_q4S=|LjfAE7L;zJfO;p zztR2f+9Q`7GI95nm2!J6hQG<(xVGt1YNq9H}Mq>)cn-_9^EwP#P8wj?z;23inuAy^_di z1WyJyxbOKb?i*-#l$8vWmc@sAXjAz2SLf0Om0b*>mLeGPB&6{>+;`G`D{YweI!f)3 zH?SYyb3aB~)*8VS>d==Cp|uRY&Rt9kYHd&Wp@WJTzJzSP(Y=wD)Y`@@)kDP$e?lJr zx%-anEv=)>Vf~mG9;n^If8+ju#%XoW%I+DHz=O0!{1Nx3G-YdU)^N|56dt18&L4BP z0`;RgE4z2B4!%-b&foph{ReHVbu??ZckCp5wRR7Gs>jR}Xq)>cnn-(UXbo`xLOd3w zc(&0unI0NW4}}vB^A~!+Q^;+yO|(ab`k_cdCEwn|FD1Xtyvg*)aAqi)P{nujh)g*c zyd)cEP|D;uEtl`&K}~6DOU+grlyW&iE8u&3WTt4^B-uElQXwa4#r!~z!jzG=fo!!= z*&vVAo&@C&`%`9LcF4i?DbLE|wP*M-9^4ey%c(i)K4qhvs%_xMd7Mcp%ieh0=`pJb zo~Ui&CwW{*$+)b`X?)B&2T#_v@Y6hQrtG^scQfhw#Le~}2A&2*nf(-fL7t}V01dmpQVw4Hx_RO$^de&; z;WmH22V~viE55m<&!CqW8H9WMqaOC_{I3+{PCQe7Urg6_^En>Kb(Aa3xw!bd${_|p z#}|57=We<(kgGN+Tjg2WUj8YM@O4M7IOO5_m2L8Dt&xA$gSzerFD_5j&$-OlOnAzl z_APDQohy}jsOPLJ@H}ll|B6S!x;IyJd5zCmSK$TPL4K#l$&Zs)Z1U{~DA$Iz5Z(Y+ z*0FVp25dfi0IiZkmk#l@9>R70SBvr|2B6m&MTB9#(W7x)($%*7(ihMhj1s~J{&Nr2 zx{|A3^CwuMfLkS%ey98Q^llk?A#_9_!Pt zcNGbTDaIiMfh>@E&U}7s>R8dlFtm?xmJlyE2dXqLm(BzrCUU@yY zcz9&&F}zv3R-p7$P3^fZE2h0SJQ+GqNEN6&i=PZ%A1GG8S3Z?r09|yqJom4keZ!$- z!iN2fafy&Ac;H#R9&sa~B)fBy3Erw@3U>E+o?IVxqp}3`fz=PcteqbC*z^4Qv>RO| z!Vi?^Lstm-f)}10>v!Fd)Y*JQ56G`+w+hUj+V!FvA=}s=(J$oJwZ(!_&*$r}-zeV( z`Gg*nw`)rUUp?Qi@4wN$jr|F_KedyvQ}D}kV*U3UU$>#4!!P0eOZNz-cuh~6-wxYO z`)qhMbem8ifOsuR^K7SYH+?p|9=c07ELi9TPb0U>w$r{C-VEI%07HeBUs^uUQ%zqC zZ-*WbssxT+k!i=#ym#5BX%lv4GWa!9V#3IyI>nQ5AKNhxkzX_4y* zEfE1;ThlDhg})=E9V{K5#fv=8pIeNIugoMUpd1JBcVYM z>vcLUsv~!e(>K-#{IRx4kmPj%1Rdzg8o#mL!=Gwf1ZiG3)An`D-GTbf8ikv*N&(ZW zH?6*dxTEnq>jV6`_KG0i>t))Vj>;XVAFPk?7uxHB60c8bZ#r~48h@}p!Cz`S1-rcd zmo^oYx}yGNeTKgVIhhB&0=n^ZIAQ%Fc>%`w7a}keQ$nsMB3H zq;$4}mTmO7+@jSBBwoQA$~q^@AwSVy2xLO!mwf`r~P8}K(JyG-k=L6Iimz2K?Wr47uReY;dZ z#b*2HhM3I7Rx0VVwV&9A#B{s;ZTvQF_w?NslX8?VPX@1gx+ z{KGPZXj2REUbN8@Xyv9qj8iPrh_h-Jdc!x8Z{>Q#Olh5}n68@x0w^#Wb8gwJxCYd8 zCg@wHXbbA}870yKbsasaN(tG#Dski6uL)jG0LM+rd)kb-r*ywbdxUbQs5Q4DP zA!^CqCpU)OuH1*3UN{@ENatFcxx ze1NL2-6*;pvY$NzGf%Np=U1ES{e0u~+vWQqGcofO4mxaYk@x$J{kPlqvq99lVwoD0tU=X z9B*Vg+RUmb-wWqwb9S82%9l*}!f`ncowRU#{ z$m~#n;9J#f?joieF}e1FcY6B$JG=wfIo!oe58}GotKM7FEg;xbHD^3$Dby#eR(`H5 z{pX#@1CY6xB?@m{dTp2Y(e$NvLk_a%+D=BGb+*MCZ$UcxZtlV1x#RW-U)`o!gLgwZ zOA98iytw!*5rLTJ^wD)*v)JGTM?ius6G7Wt@ML;?Lf#L*MW)Et=ePp{%4`& zZp0zgzleVnAXKh))cbk*^}FSVAb(<(0d3^rym9a0^e1=Q4we3CyPUa_2wI-KC(^&) z{d#EPPumsDj=g(or~1syfOfeb&R#IS0uiRGsGaTeR|dK(_wew7aYsai?r<&C2bsY? zbLKE@;dsRfkSqiuOE4KZT?2>J3tOEP(Yh10D}2H;j&wP&anM$n0;l8Fy7*8tnz~Zi zYG^B5LC^_my?rt>G$4-zXWNQUkaXhOK%c^lk*)!@+O`#`h}E40y;#FBGw#_P!P?nQ zSrHHX*ik+wGMqp}L!(_G3PIH=YRNt)Gd!2>1Caz&S4#r1sW#E)_A%PMcHq&+xG8{p zzBbk8Mn>7a$s>@(7in;$y|DW@>d)=-cW} zPwVcg5;{^rXeM#K@E>$2eOI?lwY?KH!g81h6~fUZdR=!!wa|$gX*oh%Ec8IHr+0Q& z08oV*WvL`C75bs`=!4x|)k0@#wBw|E~VZRsHxFi#VlQE!>Bm$$)CydD$-Gafo`5+;ar|7Xz)y zfa+Z{twD0K|*iN%Z{aL9t3pO(UGBW4SY=tf48 zwvAuvVY`lbg_tLNj#e>Bv|srX9=55>Ys9U>w*WfeYkk4YXuFy~bha4Kdd6RLBs&GsJO zh`6Jx5YEQ@#YF3JYlpqZ(-B>|!$8wTGAX*|+H9Zk49h@prO+PZ$IRE6YfU~zaDM_2 z)e#fPJg$QYX=sdAtk!XbE*L7aNtY^AqgxpYtxh2H#$+-zI*Aa6X=N((IKcS`m@`0ImWXhE ztvQOvx^u#{;D_bt2SjSW*3F8ix(mXauWx~9VLLI_pPS1x5nF{>m`C8hOAxF4xp~az z#H+#r%xmU-pig51#w+Y!=&lP(FrS!jz%SGoP*{L?sp}N(!u*dpwP&tGbvSy9mDw)9A7BSxu9|_N3a9R8wU#Yt7a}i=h*C%YkBxfmma;3vT*WT1R zbx3QeVq9kxKE#~K!uLw*aG|Ybif_92!e`*(=JXEKsY6?LD1Ly}_E(s@Sx0&uPBt2= z%b63zaiJCS2weVzld6^6oy=dvAHq+V*ICesqbE_T3U?v?)BO~F$9&3~3b}JqxQe>l zg1_At{0lRY^}YA&$%$3AdzgQariiBbLN+aa=zFSkwe4Q!RMK?O9AEoQ{tt^zO{})v z$Fw0qM7CgUzKq&}_L zi9&sMZuY}&WI=jPlm;DIZQZ-AQJfcWHSwY zG8I^QGm)fN(MjJm*#`}Fax9(;wmKxLsKGZrTVY6$tMJ?!rW+|)bip?rIKO#vEP)H8 z3(`8#Ro|`IvybcK*zz!e#gmjK>hRr}{o7D+b1D&2tMJyRi@JP|W-m2{DA+`dP=VGn z0IeX-4lhN83IRooDBCP<<#~0p0A)_ zvT_W4&5haAaRq`59xXwB+j7SGMjMBz;|+*7 z{V7qH-=Q4nO=>Rv93olYB1-eSnX~Wl+;ga;!e#_buM{!;dUNU@6VEj!6`r@Ot7FIH z`aRFN{pSVy&%`Ne6ru*AKNh=}B92Db2Q($t3ATVn6KW zk|)0HX4-fwVw3)%s2&&_IZxcrXQzy}A#(J6q9(uO&B`aa=Z90qFEjh+JriB>V{Gnu zB0Eo8XS`y`Bi-6}&989t$diHd>UFJG6$Sd2qFa9ZH_v|R(1J^Cy=Dm{z7;+2tKN)w zn$V&GtO~P`WEJ)Lo!%Vvl-Gh?&%MqpCcPIu^}Dp0`4pDixW4cPVw?V>XwdH_C){kT@a>GJ1k|S@1hTWFE)2Rt+;@D#87`^F6CMC#q9L)JBWk&x#Fe%0lBPaqZfzM$L}Hz>lcW3JNZZFa-O+g z%FYsB!O?O z|2B_CQ);W4*{WBF^=+5}|JS+qO}tiY7Pp5fB?XJO`G3i^nEF~(fCyrqB&`zf_MehB zqu;I#yNUadd72b1KIFeJPf5hJHEt?=ggB#*5+CqOTrp|kG!=0 zt~Oyd)o4+W$l?^w;JmW_$u>w1rcZHJ9}g%;LY}CZwh{g>N)QUHqW?u|MzJ*;>Z5y^U^>ZqA-v8 zoEhGcCm!&>kyrM7@`~`()B)x-(pK?XfM4>T`(7>0FU*;9UB7C_u>a${7QjE5^0|w|K&{`n0pOZ5c|q< zi?mNXJz!pbOIYwV_7==*#U1?t@!WvF^MeM;uR*qA-YB|2ZlYa)TYkzw!mZ-1wr`mr zRYCkWpsY#;zFwQyYHMb8lLkai0nq@va#v-u#?1%~m`z**lJk`VxvF8-_z*&;uN9*M zvhocBUpwOpTkRBjy;vL=P?$e5FrZQwwptav`jg_-0sHf3zi_yYD{6hGctp|!M+a2r z^KEd~v5$zu76VBkCIv|ISH7sYZeL6tvGkFe#EAjt^V42*T^ANp-&>xLTEy!EI`VhD zu(@GhLLF5+s=6em2WazIxxP0_OKd+d`$?C@n*&VwS6;MtzbYC3fEdtU6SD%W`GyzG zH?p^le?$!G+r?!8Kk~<3jNTaDHvS3mN`FhdFJNW?bkMzBsMCJ7ye4&t*#V0RJO}CR zrtMU=;;sIHxGKQ806!>c$Cb8@DTee~acuymAZIYOT3N~+XIe?U;*+2vRDtT>wXlr( z)$)#H5X%Ed1$7|VQdma)W*H$p5uXoOU(h*N(P6)X`rR^0>K9)Q$SW8e?CKEip#HFY zBn^r?0(KVs9<=GSFQ@)%`3%UldjZD^mc5MV6qZvbEMp{#SRWuQSoyM|(|#xQr)Bw} zVR2u;xq_6J?VaqMm|u$T`VZoPfExv6FDE-8yD*cAfAwF)<^XMh=w--F_AbnSil6## z;@zVG&kL@mzt1wUU--h}MI{L%17 z&lHJG;G8W>LESrh4+a#X_e__}4g7mc(5v!W5YUm`VABJU*af<6NqJSIWbL(`meqe} zu4HLoz?QHQ^DWa}<236`@&butVC0tLuVA-n`;0bL2-!~Jw2QK(=~e1&^}g2W4WQ)` z^htgA*7Yh^>$1OaMwTmnsl+cZZ_D7TuG_->)S1@#lXa%gWaSbjShB>;}6Ylq4bW+?JHr?RVG*FmoCf_jpKB z18;08dp&sv0!maGmh|`l*iXAf^g84&`yggs!_poL4IVw~B-a89w~o9S z09az{UkzSl`&l;w_iml>*6u#G5^##~!VJm%z!O`Y-V*O)zme^&Xfjix50q|Q`L=z` z1|-Ea`1ZW~GoyXBFZr$PKJ8d*?!24+Adn>R(pKi%zWb_UT#$1?E|9zoyuJ0{TZi?q zYlw48|)m`QgcW*y9(pjaB|iG$sDZ5MzlG%d$?-c z5eYJ;0WX7OQOwQV+12At$f%x5iOS2Dm2Wn8o2re@)@X8-#1R|GI&OxkX&fWWiX-zR zC~P9D1wfc4juCDpkZUDAfVt^0%hWWk5n&~fB@!&QnDx}6K|kXd5_J8^3= z`JyBpdk;h_z_c{5(X`S)ByBeKF{@>Wt~K$g;(hCSu1dCG-?B7A5-m>9>e;Zq=Z1to z=L^d+)TdPmxIlm=-<0gePAQyW$(6!t3xOutb4PLz`)8q(g{T|e8|7ooAOkT7`%mFY zONGu}NJU%eJ@o}VtXpAirWH9byafN(~uf9}d z3#Lf&W63!zt?-Ja2n;8-{#m(XljNrzv(R8^)@O^y{gL@S1Cs05ZG~f&QT?!ZJOH_+ z=appl9qge(sMWnETQVMq1Xoj{!3qk|*4&<9$v75S)bmbaz%~?8tP!F_sWHe}Odge( zu$K$-t>zw+lsa3ot>=^E74|O3EOqF`)wPB+l=h5E-eHY}P1aPvzP5%o?C6Q+eZsyj zyl>_8Vo!2cX6+>ZlKcQEz7}g=uj(XsRl{-FZ^>`${|cwRoBI%Ts&F-OZ_iZejG#Y@ zoZb;1Hl8Y6gWTUULpnd`ujr6>6%XxCQ^TwkWY8fNpZDLP;k-5 zm!BU_oVJa~I!sN6@!$DAQV~B#SoMqTq^O|8qLz2`N2YpXr1dCyiPQ&BT0QS% zk7#F%QPyK*2Prlvd*;Zyfk)~yt$y#F?s(YNJF+toS*Wpxy znz;|4&19rB7WCjn4ZG(#%M0jA5OjQCFVHymLUDyU#SCHqVBf?gDL4p$iM8>nQfgd8L-4*F0uINW6vHc(@&b>x-O zok9N<{T{aIvp-9Xvz{V{Ne=|gD_%Ad(I-4hjnCq$Bc+u=)J47{MST-zZ7EqYGET}3 zLKP>CwDpxX+ETL=WRg@Ige@k!z3&@t9H%1B_QXl+gQAN$BMI}{RyQX!H1<%XXM?E4 zO(Us~)lIF54d;52r5A%Ti!~#X$GCH?Nm-vF)=94hZ7sHp^gULc<0fatpH7o@2JJ4M z`hM;cRCD24HVM+#ZK=XmU5e^G^>)#lM5(2e4<_maG^^R}s3*T`Eze2Kc4|K9g$X^ZXptn1`r>H8p4@s;;Q zPbXTqCziJNY?qFKo6+#T`Dylr@ib&-Pr3AG(2wG=_oGjTFN|+M-Xfojn-V;|Wd11Z z8SSERqxCju*n|WxD)9vLwCSQT-Fg=k7A_2imykzg&uEv78PINGCt_~8!s1bMn3Ink){RTEZO&A?sL?Y!d#@OM=50n z_mPU!~?-2gui@#las+JbhKq*;g?I4TC)$((>S6 zB|kq*K8IYxY-xDab6Z*wJZIa|k0Ar>YnZJKZ$M0DW$@qIf27f_c3a8EuLBb*+oG&@GVRE0eF*tEs z%SZYPRR5XchW9;Br5A%Uw`o2~Uf^!DmNb0mc`m&cT)1uI;{ZU?TemfQ>Uk->1)7~_ ze{vYawHp%>zL4Ka9|Tu#LwrgY9L~!sMUMAaq&>kWw}pMG97J^#mLb3Oj7XmZU)YxM zNjKQoQMd#7qvxY^F!<)SeV^vOM0FOHBPRerVhQdAA>YK8jh%%%k-vJj=6nc#v90q{ z#Y_8})Lqs|@=xiv;1AmdK@7O?CUv*wpXRMZTDx*=ykN)xXX8Y z{%7+m6KDap&Wv49=NJ;X{rG3tYuX)Sg%twA#!(@O+gku{Z@ObVV4V{SI@dxN+j~CC zUeiERm33aMLmf7xc>BlC=GUgX#zWRWV^`Fz2|2KR{ukIAT9@&#bzv+FNN_dVJ-^W3 zn7WK?t8J`HU3`dqJNb+34eg%sh;>n{dmSz0^7i~M<~OE$#-rB1V!i9qL+))q{-qBQ zeZRG`!M+z$C!G5b$ao2FRrk3do+~z>t{~*~_WNIWZ?O-!$Fu&44X)c3@@2c_%cN7| zgTfQY<-IHGc7^8S-~&&{(+{ zqQ(G+TW?M|Za$)vGS&jg6Xz)#F^GOK)6VeModEXRJ9QNn_+$ zQ6PN1F(k3HWsE*#(im&3Zn10YE`=~kd&Xo#G(etMJz`VqR3XKsAIHo?CaqCmecidC z?pDbD(%It<7M!lNw!x=2v#vXYTk1NVYEcW=K!c6Vs_P9oT^coR9)#%&MMxlj)IABg z03zOXmPUP{7#Yw@bbb-iQMzl~#%d3yGwaOCt#!C}-KG3-Uu$Vk;qMhey+w5+A=1s~ z$LUs6k5Ot3j@@4OC1eP^?_^e5ud&V=8e3jB5%R6{NTFUu8Q4LHzjm>+5E4t zceIDbQ`R-Hprkc)QJLpg`a9D@<7sO+=tNx@3NIrA=LGGMvECXP3xX6w{mSycn%|io z8P8awW2@>MLnF%+PnQnk46U*TTraoIC6rp$^fh%@ZJ^R(2(f}Xuh4W5WXK!F8o7!r zQmnWx0L0Z>zV;2PjNFE-*xuN@m7%-Krhc0{g6bz2H{D3%uzJXvW%b{P?;(BE^VSsL@yH8(QP%mb;=TP-YKt{B_Ilm!;!yFcCEG8(FMVozAuBDm zqizSVfqeh=_5H+C+lyHnV{g~(4~6Vl{M~o7^qK9Ytc=)ubw@+(clduV8l8A%tIVPU z44NB?+L83VZM4*6+nU9U)z^tbu{%n>e;uXeylX{n>U~&O9~!-b^WFVJcK>)AGN-q% zt|>Hmhw^*whvEM5%gEf`X8@(n+F|(K{2}}K_!VS6P^7PiZrd^Tee}cd^YN?5ExoVm z?t~r!b`((gZxLrC)FUgBWp+OkGfBx zuXo)40S2O%+|I0>vA^nmgtEWB2FmzET-;6TuGrso|Aqdy&hN5NJwqe5sRP?uP0cH~#l=fclhrZRX0-xu;@9RF# z3{+{(iV1 z6~9BjpL$Pw^g$3mUzqLRwd3mV-QOpC@X0-P26l`&{yjSkzps4X_ra%CUeW3&+~NOc zhwb;@?+!+vHhCqu<#+oJ&%@Z3SoRrx+U4TbFyR6JiFv-Z9To?ZPlvp!HA-0K?~%u{ zU9s#l`E<%O=Gngig=1p;vbcF+(xi&Gmq6}An^>L%0DTu-j-wi zZ62%7sE$t(*7~#a^tL+dww|$1GHT*eg_r%e<^8Z(tiOB4KFv56zZ}$a`|@1=5Pod? zIQCh_h4^&g1OHQb^Z(@h`2BILA)_vyE$oDL67x^pk8QnUpJ&v^KM64SH$YwK;}26W zjX5Ij_B!D!|66&yKa+k&eX9LWeyMen(CFWpN7_OCdFj)!u8hm^TZEteU+3NVQ}z>w zU;V3*#>U)b;eFVF7ph+nubn_ju!Xe|^D2^f_>{cp^# zr0c=2KirHj5%v{h@_qmA_~r1`=Z*Y!>po%WbZkC}srtqK`u&ajoz?@w(N;qKTBuDu z``UT*^j-L}rvwD$m;819z3AJqcNzENj|traDEYF#^xv1DSe5Z0{)BK&!1DY%f6IO^ z`hM(v#-n(cngnbpfA#mBet>@72lsig<-)}Q+wy<>wfz45eXKE~EnXxH3D}pPn}D^k zE#Hmq9j#Tu0#d+Ob>0>UZro`I_+$AX_!BV0FLt@1@^kiw1mXeJrd92q-A*AJK=i zZL+wVT04Zi7p&{hRbJs=xNH z{CDejp)sJd0Qr~l?|y%6-v`OupTf@pdrn{fci9yA?9Pe-VFLwrbbBKTiguoXsXMO@C?1GPo;O4>qHO{+@1n8V0h)oigPezE zjwHKQE`}=Jj{%{Wdc}MRu7D=mDuMD$Bt|BsvI}(%Fg_O_Jc4(=p&5d z$=;Rea7R=zFl>-u2;I?$fx=&QVAsy8pe4r)rH?dDBKuZy1ABMA08@)#D1DT1GC9*@ zN8q2Gwt;e+ag?<`?oe3=A zqj2=4+NlR~94t;oJuas5R3LGe{=|QSTO8`fD%{EDmBPTFT_tvCdtX$Ub0&^VtHko- zc8TH26G7#mu^w&A%9_BXyKdT*+T(}kxL`foSd~#>Yj(Y`D;~rfp7T7#i=0|{IdI#q zA9hxI+i>Ucne^k!DsKkv+2u6Ia|maIWjxlqZB=CpWInwIWe+iruuQ-b+A=FU0_D4? zgVaMfjus4de%qQ#Ltw+Mje~lJm>n$>u?ySQSH20n3m5>;p`4N0w{I7bb1F@N%R`Vs zsY4qF?sv^xOn!FwQ{c;86;Zo}Vn)#?8<&uCE3JXOyIu@x9x50`pJMbU=U4Ux_U-B) zWIqfunm*MSNWM0FaM15`^x)>}D@R+VVuRauSE7Q(6?zZO9%deGnT8E*+gn-S6y#n= z9jqS48Dp7_4R1S8i3wUzxN&grF!LD83~XfE;mYYj5rszwdpdBOEHkmBwz5imPy!S$ zvmMM%7B>ZP00ee|(hI|HopK^2x7rLjBMXs;R4(ckr3TAuEIJ%cH zh1^)l4eBinA8>g%ZyaHcBCV~da!1ggLYsZxaMXChT*Wf-uBAmm4n;0Qh$C{xJI~8p zL2jq%{ ztx_BmQFL^Orz2+0ErO9v)>bNm7)3Qh(j5&LEzy`oep-3BW(9P}${k4)>*g!6+n!fm z4BAv=9?}KP)Vc+Vb!{&zuLl(sp@zDSBu%PYsMyfK ztkr@yF)3{jIbG z{Ve)6w0D$wiX{NMvu%LLE_mo}r(vF>Ia4ix*uu8KB2@6W-QL5pN1LZwg0Q>WhKZbl zlLo{Nla6Lh(}G!?JVG=fn7F$!_1Wl_X>}kRZ5t(;8XU5lH_UYmX?k6#Vqe=>5jL2% z8yS{5re%6vm|}n1IMLkTB)8^a{4u^W%EB{C+Xj>72d~@xX4s=K$c&r_?4h>FBLCom z-F6O+PWYKQk=P?`(?#LIhjveQh;>3{=0stSwapU61q*ivJCr)%-Ev4+ej8rI3_iPi zg~KB!#4RTpd!o%tloot#ccFvhSp2M<80@JwZ_(=D8IDJn`;O(!BE%{LZA8(U;ODy= z9rDJa+zD}t!)O~Iz!dI!zeTzBXA%nGufXnXMQ-G)(fpFDNf!rDO&6zvMOFCK|< zb7o<+WTS{2D%ux}F7`sDI~%ZCicvxal25Q(aU4qO%))7@MkzT)bSjuwycX5%Y`|%0 zMmd=*5(S49AB2)I3$LXcm1Md|8O$iIL8YS&crC-IA}5N@2d5W5MoG~u4=vNECZ~ul z1#^o(p}NuLrRNe9XWN#Ft_K$tqlUY>G@9M73+S%$sYwp)Y>8CT*xB73|U#ID%e)e(8u2m26KD;~m7^CXI0q937|kn7`|kqNUp(P9vc{}^GNVDoVX zeK>sTAtCDu#yEOnI0Q>3wxg{|L<%`k;_ax;X(MRa#!fPby+foWagI`qZSDoocDG$Q zkq~kovMa97WyG8;?31<{QTl>N|E-RF7}R`lklz+m@K=P~EjjAw2|{hl8mxhQU^6>J zS5o7cJ`pdKWhC2evi%T?JB~;vh|tZpcd@`!Pu;k3@}WWZOD8}29jpTv_0XV zi}qbqwlTAt3^M4Dd3#om)JkQsauO`MMeVsqH^eW5^aLuJ^i{7hASq zzqGv)-3-~Y$7z)3WDb~&vESO>iCRKV?C~C@IjUQt-Ddm_Pegl&bWa>ehgp8wT%(oz zQS>ZCvuEw7?#Tu}?RMi&@)yzTkmfxHN4bIhSG&Xb8)Rvw5Z#^{Q284CwRwt-XMT#l zguLC;GOBzEDWEQ2(bwjjx1;vco;RZ&O+f;33b6m$`bGUA|Mu98cASb2%-M+@*gnkP zKGbpVOEO1LCWw7P*?aY) zHPblqFC|zIZHs+D%=&F~?=*9WWe*lCtK#6$J$s$TcuwblZWHU&K2uBzJpn@0?CEBw zo8UjCU_o9Wg;~3FIx9@eGomT8#Yv%>y=%vGPd9*x#yE~LSG+v58MLErGg#r;1I9n) zKH|*K&bGd$jmusLwu* zQy2Iy>y8<3hQ^D9p+WmfoX~E($E(V)xOS>o78<`#?9_MV1c}Z!;wenAI`jkfl2e`= zDw+UV$@V01L0xDTSRGGuCewFNSbR z;oY7N5zj&U2xGakY%$J)Oo&K{J42QGm}51wa$}uOXU?Z&i=PAPVDnh(EL&{Z;%N)p z*NfkT-rYw+rn$4?v~YO>Ha%0QZeI=P3k@((H7=%X6>p#Wc3;cb@@;eD%PKPcDBHz9 zL%;5OKUU}77+)sL44@Q<|Ah|Z4RN;87Cw)xGzL-%#r9#2yvfdCSOL6{jlq->u_Flm z{GE%j7z$l%45jeIXt498IX7bk6gs%~+WFp-!WQ##o%^sTDxpdd*?vSkGb|dsd44z^ zl^|7++WF$yVaXuDvqoE~&a%uH$|-T#+zeiavl_>tS!7tal@j}gZQ*?bJ)N0mkz>j2 zmEz#AJv=A0C!RyMD6rJ_DlsXne5x0^Y3`80yOo)si4(_%4a5ebOYwLH-RBjZf`}R5 zd`m-j&n#gOR0=SF6!BwB+CWt>h_0XLD&NDg7wZuG!H=o zMrjidoD;mC=i&;Iuev$~8%B7Fu$3>QE(}lP3P*#@YS)Wv!j|s82`&$Oa!x%qyWJqJ z2f5Z8NcthkIStr#?ZND(u)O{KF7{rS6#7NuddmLT+hP0nW5)S;@lptv6dT*S#SgioG%9l1O=4}BYCmgSlNT?Qa9Odry;uBS*ya7XD9#dmDfIv{K6PFT z$b%I7+s8@fg4?78uHr~5>+UE(ogi5lK4)R~c-=hn_qBJihuSAg{KE?l*iCQ*XHU*O zEPxgz;b7!R`yS*i00EEjC}ox;HvIFYg%k3;QR##hqvH&mgchzmz?`7*&V{mW=5dOr zBsu)jfz1=F-nMjSC}6ZNOjr?q_rTE!o<5w_mWRf%0|}6?)E%gqknUqxt$k!XO#$Id zcsKZEq&}<+?PFs(Wr<{4`0oeC30*!d8Fet3Xb+I=3?EdA!nlIyrLIi@ONFFhcGu|A zSr`VP5pW`z5=w;RQ24Y`l+S)}u;hSprJW=>9zLhE&~%+3VADH|G60U0hx?W0W9-3> zLf0A<6sklT9$SiWTu9_`2s#DQ&X81xCzm#2@`$J`f?iP#mz8zlS)~R{6F5i+PZTxn zsgf&T7_njch^RG$r^b@Hwqic9UA;2qIf z>avNvpmA*(=rJf;B%i}ym)@D^X6KR@`NDXGk}I)>UlTl=*tCGRj_?xuCpJ&=C;WHm zkBQa=wsp?0GC@lr85H4g&}9;FVeWcoa7<8&CBq{o9GpL?{FQ0F_TckdU@#vO;dU@? zl5`Gu@5NDp`H<`2d{!-0ketz-uRFrkPsuXL733I2zB1`gW_>}xnxPi_JcOCAE0sw zMnzk@ND>;c|6oVFk8jJ{FGg%fdzFL~@rXa$H66SJT9dJpqJVExdN9sa>dWG4%|>1O zi_ApG+j3lEt3^qxdazI0Yb0qA*A5oCIxfaU}LfsoJIs{LYoO#%LW#J)iMdCq#joZq;XMUixN`jX8A1a*e2vGSP3)Td^XIVu2A@Sr`KLmjA*q(O1q$XnNp_`K( zZ%^1x|6%+{F-SBK*@s>LyIruI{?qu0@e*sPePR$1Jwq+o0aL2GJ zC%|e`r>X-vyR;2m_SBe+@sU13|}Pm_x3hA167CRQzs+yofuuAr8px}i#dL|1+Iyz}r7 zU_$%uE*oYluAN>L9(m~S^l7od$Zjq2)`>c+DmJqGu>UmshGE47l+w8aS5-idR2?3f zd_Fk0*m*eHh3Z+A9C_*RW*{ruik(N``r7AKt%$sP_~6Qz9ZeId3#&Fnb{~E`O&Y@5qXk|ubxGB>NbBKGKqEHn(T*}rp#p0-(*DTE>29H{ zz1q>HX;e@_N1~5-O-~Os?A4Aj&7ekBIZT>ygfzW8l(etTN$J)RU3D_j=Llzd7vP8L z#wy)A;;SknLyqvKyM~c?Ir&T+l~yH>qySo%5r*7Yg~oYwFso`Jmmaw}y)+EJKgR_J z6x=GXOYSl|-_+afeCvV(=xtSFhMSBf&FD!||m#7#y*KQ>BkQdt}9oN8y;8(GyMcscWmc zBAbrn&#;fc9HdV&Eu?O&>IPzK#f-2B!9hCkSgBymi0nFYbw=rN`$L4uN`R|YSt5Im z7-lp@@D33Gl-iM3^*8c2$}+szFhU?v0u08Ob?pnTiYU*jlHq7Y1IXY*mTTN$8t_v=HnoO zTGbe}<=D4by|HEpHsaQIysEkx^?LD0!1%GwXcw9`P$3Q$#Rn!|dYs`*PR9OC9UrRN zq9n&i?&WdjwMTu~Txw6%6Ts)KcGp2hqHHmH3-$jjyc@?#+|lvA6=h4<+d8Pb-$#Y4 zE_V+Dj2+$2luP|l)eDfitM0||7$M!?w1aA^vPSg+V6GWj5%d64KDEE9FRJfYzq>sd zQ%Mgr?WEdC?MaShld)lBK_xxNR7eGm8_BiIA6rbuh{_1O-5m~6XOc%53m9xX5g|kw zW9KNHNLp038EYlmM9y$9qmGe&T^(L_6zfUhh%KQw*k(%$@T3I5b7fP^uzkZF=ol~c zCS{a$VAT|k#1f7>*fB}!OWIQQ4ciNEUrPk;aK}_>FlkSj6V8*$fggJu@WG@c(up!} zTsF|ZEKxZ4!%`|qUPi^KsT`?=ggf3bTgoCel-(?S4OlK`Lh*@?InrgMv8-Yon&vAj zgI*Za88QH=^?pbs4Uy$QT)bnxbS>#+*-cz24G;7yTzQADbTjF5*&Ey=Kw0I)((gI+Q_0rA2a>6M($Wk#Wf^X+Njln{4?v*M#Xj0}AGH{p}j7Fr4 z!3N5bG;d)V|0Y0{@TwdpPTj$h)|1xp-{2oH5LHfhXALz?+C<9Z_v7t>ZbVNsodcj* zGpUsC{b(VeiU>){3mxe|I6KWpJzLMTRXelTbyT*r6Od<2kM3tH)Rx3H4JgB(ku-e$ zg8!H;KnPMcbgYvKr(Nflc%Tz}0r$hcMBOC)0LLep2P2`erYx0xnYu;#ne-aSJbVE0 zl%<*GAIp{gAbkeR&Lg1eo93$7t8%{2$LVYTvM$0;cxI_tP$c2g3smXM)*+)*xF z621MnZFXN0s*aGU1gwfwd}Z%(7oe!*);Y7;9aLbXM4vi7-;>L-)j4z6TB;8O7_1ZP2XH?*dMC*>%c%~;C z>b2RX=hTbR4bk1lSFbsfOacUmva92=bZhkY<3Bi^$+@}N>u@hSu1fQx{~fpUYQ~Sg zNMCPyMFluVH1YU&FF#;$5H={^fY&!==7baTy|^hPws`~YUB^9uin*VldZ|-5mn<7` z?>in!1O%(yN3$w ziRk4g?s)N2eJ_`7W&=k;+7NxD=e5_PROE8b7To8K7t*WINw@xbsfk0c5Vk4-`5=Aj zvG;@v5FT%N!|j z!L_n{b|3YRwBTp-SHKbImNs51D`5YlmTc>f{&&J|uH!QN^_-o!nz$%|SLc26s~YUA2sJ!%~PF(m6tgiCJ)R9>)I9>!oI8VLyJLo*!a#|ccaLyH2?&4L zftZP>e7zkrf8TRH%AQG!k{ye|onm@xR^{GzKE@ueh>;0m7M$AX-Mh*R&0rk1lPnX* zM4YO)JTsluqAfGwXmpt}hHDV+fUETEOqN}Y*>uY6 z-Id+~ZCd4=&ZV;JF-6dobzMz*Sa(thO<0*N;P9zgK8)3k59!)*KAo#&k7IC_rw!Ej%s$@~Ie1R7+onz2ypKn*#~u?;Im*bJIVmJoYhP=dM7;f$uJOiTborp7i2@`?6Pb)#3C5@{D?=}=^= zAckcL^mH}u>q(7lP3&_)BQXz3XoMPNO6MgR&CVpy6E#`6Pn^%Pp+F|v9{XGHgJ^|% znDaUIGTIGUQLMvhm-(LZ6;Cbaa4R}*%XqQlPJ7SKUSoc0IgeY_c@IG4?x(5q)oVD< zEEjN4v62a5aSqw@yVn?=5$a?aovpG;=u7eDyJnLN&b>@%L&@Z^6evhCvKtL$P;H_; zk)4fQe)`UQezq_4l-N*9k~KgJ={=N=8i8%eUPpT=yB53a^pFK;C=8WdWN)Col@;8L zJ#u>10>;|L|H>}0H+A+ddmJk~9lW4)ExyaL^llE#EYrn)ww@d8Zxq;*k`Bf7jW0HSnr!~J34>KzQukx z-LXKuj`PxT6_*cPPFw8DHJ=uALsLb2%~U`eARicKe`e%DH>jOxubXz!2Fo4d&}Y0B zrmr`=(%!(e-Wn?ZZ8z;q_(J~r#(V2-;)**_@_6*@GieK(q3%JyWh$YKl)J|HoylKl zzX9`xe%rK{2F;AP*fSLi!!`)s(3?#>nv2}-ChJV&!n_Trw}d;&1CYKZ#LZ*s7iu=- zzIDEI?_kGzz z_9>c=JR{C?tL#V6Cc%69Ba;9KqSXRa~I9$@-lX~k7`2FeTLhL$@)B%WimwBf{^v9tHZq07A>-)=B!+f5Si0v!Q3 zW}I)YYnZ7HvgrW-IT`0u&Vl54qg!Do8-nI?VO&so|2GuZ(8EI?Q*=_~RdLbf0$)Ea z&rHxMkxquZIxe}q(KnBa>LKXaC&QBDwQ=lny{`u1+s;qeHMCUux{sS7oZZVc_gJ3b zAbKso8Mmk0X|d;K&PU5r9OS6wEr5gaUYxzz{L%6ZSJ#;-?|=;SCm-=Q}3 z%kM1aZ}I(H_JZ9++b;hZ_Z2e0x-E^L%U-e};Vb_aH?U&J67*KzFJ-UTH)uujq48rX zWE9wz$(YbQ#RE=Ue)kT+iiN#Nm{o)YFWjdCH!r^ z-^<>!AJI)vkosr_tvNxLrJ6>l#b>F>6K^-KH3^qO{CzAqjv^n#$E;g|NS>1`*ARd6_d zhLGf64za(wZ_4h@2lA8gK0*#;_*#C~eOG?yY=vk+*+nn$wLS(K*EPPrnUCPe&G z^I9NGr|ju`B0m?uQrH3w2f@RmKiIvr=kkX54Z`>Sy1d3e^j*_GcfOEcjV}<|1vuv8 z|K|L{eeHZBzY~8*I6c5^ivR*>rf;_6v=tC8w$nn^~z*0ycaqaN-T9m?tJg0I6#EFoB+(G!E+L4OM@3wTl&7p+0X8>MCj(su_h$(FQ1Xk_U+< z`#SCR9mXHcnMQ{|1$m5URxo3CBg7>*Gw2bDL*!`?hT!k^b>KU3-00DY6J)OUa;V@V z4&1SLcWs=af*c?!40bHWqqxp^2q7qB;qJ zYm*dp8<1O-dcRz zeKKF<72-+02Y2p>52Qld$P&0wFE1fEYA^^#eEu?pkc@!f}8$jQw7)Y@fGxI0yC5_fhMS`?NieZ$oVoU&KZ{ukG zG)|_bL@|;w1@2`__u|KJr{fVno&rtr6sN%@t6&UchB=%LSE!V}=B=T9dr?lrnMfo( zSvZ3dB|aWX+?Na2pB%XDRCrL5#Py+ExXMJ&;>6HTDSY4vt`F7h%N>h$=fu&=6^jA< z{3DcmXl&|3cXK?wQV~Qc7LN>bN#a@7hHaP2WoEo`YK?busTYLU8ogGLL1`Du!WjD-UHG0z zU~5HLHl<5^HLQ4l?i(L3Bt?4(fYUu+i;8Dg&yu-5(7;9w|%!LVjA5I1xu5&0sl9F(AsV|04;K0?4LQbJbWZ?|Bc;SQmo&H2od@4Fi+A&-`_+xD%t@y9S8RcELWYcnM?nuct+7k)<$$6G`Q*(o43%v zC|*;VB?lwi4zXaKZ{7xXs3wXIruFHE4B$~UZ>RrKe5G_t9!E$Iv8L$!&3W{{iifq| zB<6^&LoHJ@0Z4)NpJD)YP!%fD^)P9wCJ@=B9i$va9b1KuOg+pamIiT(=tGqanUkx6 zBTEnC!8V95)}oXy)HyJEesmafKQhEz0uXOks$W$;%#ktE8KLIAbeIxTW2-76!;T1G z25jchU6h{G)T*msAj8by;-v?)6O;sMR+Ry!v%DF^2o96vs$5LnR`ny&dc-z^+lo3& zpPm*(Ev_0FZB#fx%1MuJ%*=gll1Wd>X_%c2;^8fWoga7bUETt|IfbqB_R zzV5oVY316(+ql#X!=FVp9YZ>u<2f+nQ*NjJuKE#WJ!W%9lQ}T3Qx;Jjq%NfLLCatq zhX*%-l1Ci}v$pIqGuBGMS7{@G3hoYrGj$mUXQkq0+8CvPxE*M zMp_3+ZhRJA2LmfQU8$rpq%|bzn_GAdOrNv~%5&6}(iT!VpX8xoB5G~2@*)h1%%m>h zple_ZM1Q~b8g-X+NHqGm?`(b|2WC3TyVN7nSSD^(;7kLDlu_2je6 zDOWj4J@u?~Mf9WNh$lB0-=ICau8Z0PlM?$A7%v78N9h}t-PA|Yis-Nt0vKwT8|j;s zz0@ukSrnhZ0AbR6g%0dTYOnMKj2i@V7)#Ap>G{e&YM-<}+8!najAiEQ^g^XQ%~3Wv zChVjDPWk4W0BColxyr(?Yy+ko5qNOgeM)DV2OP+oPV#_|hTPGXDksrCFWwwug~K#@ z73UuPh;jxTm5;{|Pvv@}(>Xx1QhLz38_vZ%ig@O&OE*8DpHvpiqoujLz8 z&MW#2WfAQ|u2Y=n8O}oMdi)#hZ6%L3PVOC-ea5`dx&i-Adk;AB?r>UA!{NZX5&vHM zP${4-0EB&PeA^=3Ci4e+n^HszlOK$0QbhV{MlwwFPNf|360$f(Ip^|fF1|;rSJu#$ z!VbH%9KV>+{pcedDuc9a`HQ&basl-4%%AAckfY_v`{Vpl?3WO?B44zxmA7g89(7H z(z{l@qTQ17;+qzF`}22je$!#AM{AdVn-L8=J8mBSkM@W16YZt^W_+m-A3zLz@Rx2= zT4}xV7xB$PK>(w`+)wXU`W>;!ZSh^49|7o{oB@o1$RN4{Y>$YQxq;|i96N?RG930g z^U2&wTOhg+pDY=MIMKU%W69D=R*(+%DLSIpc)Aa4OS)heqA5a#>P8_`>8-B2$?2uI zVD4`7;K7idq{qSzB1|NJOrjaZa6vrjsfw%QVi6{UQDPp!z#xfCj)K{J4q|KQJ)DsY zS7b3P)K`yv6q!S;d+?)mQ;}f$9)%MuvWCisdy(o>Ghj_UrodBD#VxR6B9OvC$aIN< zL@9^mPs2mdIu9ff);10-Wi4Tv{m3}o93*Ja4MhnBE%6QKAK*;T{aSD|8Ho~5{3N_^ zVkv^r5s@|Y=ZZ#3o&*&^JcvxvEkbhWJqiP*Nx}oIIKD6157|!tt@uH)N^B8mxV~fr zAw_ftcvBOra^XIY^Z&cJ{qzaS`BZL|EfNi;J4O_8jE++>;fg0W3Vn<-ivipXFd7`+ zVyZI3>kg0Akr6RH0$ygGQVt27pv5uhNZM$c5}~HTn@Ph*Jah@jQRXt`9V%by3$GcD zCnFhYpl?vVr|P7QU`61-D+0Mj-=!QvL(6<)_$N8@7%P#x@OaOnF=UM~{8Jns20W^e z3Js_6VMxOlaEOd7ruNShpGZM1QHgNh^gj7PkWLr^`it&_65R00Rk$=fcmKE*~lAr~gyh(H#}6sJKdV zAfphmXE-90>0t^1nIYo5k`yB&7%RsU=v>%%p~aj~27G-C9Kxh)6uA`ZzZ(Jwr%YfF zk+t-0_~>=8j4}=*)8K9>6Rqg5u$0i-k{#E5*au8u5b9;3mCPiKpo- znGcFiAq4mz8t6^%%oE|?uPaBEFt#H$dY=+SCo1s4aUvuH{%RNan{w%BB?lG;B${yw zna|+D??NLSIzx%j;1`pjL2?;r6^F@CBaaj6!7R~>m>E_TK2diLS;4rAc(0hDVlj2q z<|LhMY9^x-p{l5=Mkcpmc(Se@S<85d++>uh@Cn=+e5$SyvB&fxFBr{0iDH~Jr-7X! zkI{$pGwiD|iRd+D%XBwDSJA!NnMth9O+=qZzFfVH@EGG%-puT3bE5S;{^p2#$Putx zd=j6kCM9VuAn5=NKFRQbe_>a3OOmD*$|hvmrkipH*)X9-T#&a!g_JOK8z(XsRc}tT zp0h1S-{$OPj8V;CMpYkAB%aS*fo|sT;2ZZ~CRNuba?fKPnws$kbmQSG$f)i}RG;Up zwBErV)J;J@+lL(;hq@F#R$sx;=c>cS+)TKsD6Lokm_ z;W(`=*<6-Htj%4Ge#8+lh^j2+v+DXJZmn%K`Z4DWW07hTbBaTIlDd|YVSS9R&@EBr zGJjToOX{sPG&i@Jg*xY;oy~ZYIBtBZ8leJ+)xngjsmbbkPPWy6uh*?n z8JG3!2%!KF~L5klcPOnopr-i{+c_bv&)Wg$b+lYS8c?cF7p9FS|K1FlM zHtxfF{9{0f_$F+r`Igdq$-GH(?0qYvesEwyQ4K29wULygF(U0ci7GPT7`#bR8(VTT zCZtm*SCJDWHKf$?MiN&8PoPevN=&G&;iPsows18)$P?XJ)v|IP@2XKj;gq|8hYSU%j7UE~*g|11p zBjHbtEw%45YAf*z@`~|CQIz0t)+LR2C3mZ?YWy4BZ51zJ+*$9m>?`K2*01MUlH?gvhhKs$@1 zr8c$XYOF|)?uqJL!b*5GlsA#KYknZTy638k37g=(+11ptUGo$9tb3`to=|iawY2x* zyd8|QBfje1sO}^jIy-%7>{Z*WXs=!0bls|l38&BcF122??LgZ&7KTaHnV>w&T&lU6 zn}`0x`N8N_4WKog-MF;(syWa42hR)pqI#XseD>f{w`;6??xEekb>CISgid$?IsQJM z&+p^>Vf<8mPDn1lv9uK4NZfvWpU$TGk?{Gflj+7aK>_2Rxu4Om>Ps-qur2MohT2IS zpc_^~0*265Y>Hm#MFFc3JH_ zhUiDYqOjoH#$~1;BJs$%S@5cC+|75$8q0LAj!zVw3t!H^>08W4WuciK)!o#y=Tgq>{ z<>&oZ7T>{?GB7;^CZ}2lD?{4K<~xE?#>AfaOtj>`#LMS%SN7dO9VAXtE!1zQewX;* zykMo@UEV>WtIAi;t^Sz!{Cwleyt}AFT*I*h*;ZIgKAi7ZslMA1&^)=vkGTU-j@I*^ zR(9Vt9MVnc31IH59++f*VdN^ed#uB{sXal=-PI0B<#X_>9#z*J=1 zLDa5$m?MnoJ)sbxKqq-#NL%GrRdR$lLlv&Skufo8(S^;ctoLk3&@;2Z#$G)Gd{DXj z@b@`KtusAH`Z8Fu5Cl*GO8X;t@q z11xGi@yyEVprqmpBh%elSY996dq=!8a9A|)O zg;`&noz!*VYI^Yl%n629PYScKnw!*n;YE7$1HlQ#oSro1)#|*Yz6<^7_75>9xxd;g zy>3(&B?&)`UhVmibJ9B3V}<^9H7{vgt@rBehvt*S)%+o6?p7a8no&zyUH*`CO5?3c z*FUH}ndDQ;S)Dq)?iAlAD}&hzwdbJP5?FzK1$;slo7q_{PomVyRx=(o3ivT2v-JAv znxv(*H&>TF!k^~OSN(_szF|^!?TgjTfZb#)=vm8rQQef3SKGhZ{&8+U7Px8p*VWR9 zeYGwb#K*S3Zx(uN)W3uM+lgB5jO@oPpe%aA^1mFs*}KX3L_KTWEI zb-oLJhi0*Ai~e)U-wOVctZmHi)gO}DYh@XX*2W6HU)FZm34Bg^U0eU` z;vZ)r1FHOxp|Qf&;6+Art3b#I=*egHSNA3L)%IuDw_z$7fjv8!c4~W;W1aVO-!@;F zgTSFsKUn?5(YY?X{oL1cmHeQrBId=Ek*q0o!BDlsi@3oa#d?(5g*B&c1u#Vo%7~Dj z66Q#?E9>9zt(kpoC^0cqwO2nzJ(Cq(C&=_`=ZT47DrW*z16WCQ^_kpuo0vEkcR)X0 z?aj)l>&R5M3xeCjdrFy;)X5_@)tNK9+Ig?`L}VRePEiN2C^rYN9XlGYhDG*til?bV zS^MiSY`+d(6){Q$kvY(iorXVcFSqK_4s<$+qxFEW6fNk zm3hE@n|d?rbKM*EBe)9U#(N0${;)M#T-?0wPbW%2g!ZXENWGgi9QI0nTFjimlq@kb zOue5qK{KDj)!Gzjn9DIiDhq@PqB!awVXcuEf^xGrn$tcg{&Y= z2?wq7MfmhA7@Mf&EQ&_PVZb>8eSL;X&s2~7lA^iFDb`_B3}#O?lcmxO!VPnw3=t zd$vxNQp3#B=yTPKs9E)ZgEg=kG#j&e^=7p-+2f*qz4{I7E_k>-VOM8O@o3cNsLiY= znhROkPs}yeRF5lqs^lxHTk|+e`h;~>m)6t7+@byr`@TbxV7$F?Xs5CfnDC zkHb7Q{8zLr>jtw(Jv4bt{j4>Nr;X?M%d>95#&A^fwEFNh{HMO>`75%Tnfuk_l09md zt$FkmInQ0`aaVs(Jvlj`zHp7>GyDbaDv$g6qv~17@%7?0vCqu2E7&0i_{wR9fa(4lP+(Ki#3)z|mMa9`Nj|S;>sX?CfrXL1Ud% z{)CxQur&EbecGep&++x#H6G9O7u3yr9s-{w?75(xk=xyn`&sXLQ~9P8hGnm zUy`nBcBp>qzpIVOoejvk)R!$+iA52Am_OCq{=I0pxvumj{u(#mqfc*B|49Da@McEy zOTjfpK~F!kU)`77*U-Pt{uSmrV`tBR1iKph6vvB`*N460)lc80vU_4*Ga_Xo?D6wn zp>7ZhRfC^6)HtVjTx6|pdd0f|9OH-?jx_}nQx;v^yx#iCb_2aTYZx2@W~4-2Jieaz zI`<}e-*nUyXE+fjTr~KvdTqF=EAAPdFurD9O8Uje;2dPZL7~SnVNy*vdDBJn`Yw9%ICYRr=vHIXUD;8dOZrscL~pK9zA_Zo7FwI+YtmC5U98v;_EvC* zQQCt^m|sIz8!k3($a{;rOFXEW^kh*@PD;;3!-l4}yt~9h|7-0_z@c9MzO`r}Ejp$o zEsBtmVqOv7r6lp!$#*CR^HWOL0L?mT)i9#8p)u-mRZkC3y+HqG!PBEtNC2ih?>Mf`URu_&b@{4I0C~OGn-B7RKV;J0?8E6}g`hrwBEP}CU%f9&<7>x$X| zb%cdm{|Owvx(8dH@UB9SuS#M4!0jlR66^h8QHS?6^-feOOV`(+a#^g)o5Bw7>uM*U z7Rg+l=(+(le|R$eYg(#4_JR0NNLHw^U+d>}y>~iS2)*00%g=D+KKH(>_CkGM^{a1D zrCnU|kRC8@G7d@a0vSt2B_=MvETPK?m=RD+7Gr>4NsZGC4Rwt&7^hdUS%JO{l{rA@ z5+msERvY~;}I##+b!O)cEGF&M0=BV+gMpm}3sW>7ed00M`pQupm)r{lf}*SGhVA1(?#lF5(;@xQS78Rj9*Jz-i#? zEoRXO8ik^ZrzWCgERHWh+)mTAaa8j746;h?MrQYJ5+K_2NxO z+*LdwBU<{3)$Ds-yguG{q(#LOF=C{HtPbCsV)yvs5%+FhB_mF%X32aP&I^g}8ma8g z7Zc-A?^r#)wc_-6qjz>M^6wK9P@h;SgTBsL*Wx|jMVxi4QYWIcERAnad}Vy{yTo{% zYT^;pSJo%rw_-^=?_FY#(F5X9U=ADi{VLYRkG}iXV^l*thMH0~&2RR72sl|MzBH;O zrlNGq^!<$QBM!&C|Mt?Tj(8FrwHNyRNI!6x`97j|v<{jMVx=$fvjO^|`34=mUG?fy zsQG0p{ao(n0KriVIE<^$pnd_-)V=RpCSX6b^zj-PnbJjN8~uXrw;V41;Qor&$jFwO zlx+p^)`t#ve`tBdYhs+0E-%~ZcjCTYg6T*1*Su!NIq9mhJ$~o!uTSv(*z%h9h;d%J zrfk38&HL^N#UI`Kc`b~5DYOjjR}N12-5*=}d94f}&Q%uRSARbv!Ss{+01q7brJKvb z{GI_@pNnhdKzDPPT%q#2D`oXw%XtC6;jOO?5eqTb{r+b5OPoP(&+slsn z{kT6h(cts?H*p=%>r$t(bia93W{IAk*E0`3Ro_7EE{oW|q{=Qa`Lo_z>rMus6aWx6 z6xb1}K4-kOe#W>ZCHNQl*;dgKb+-+^9eoD9Bi&zi!_T!UFA+W(F*N!d3hdv&(txVs zCXz>ohDN)frBZYm*DtiHGm$wOp&pe$SyEhCrCBW_CRZ-$I7Pq)0{;^6e)J`@ z4n)e=^IufG=?MAD(EHI|=tJr0GDH70)m}%KUm`w?_CXt^S!GN7ZK|n9G+%~3jJ|?4 zOLKwM#I-u_2>ffr$I;i&7U_ku_5P@8?h)pNPGCk+w*oOPD}Mrr^;z;2^Qmh<-G;hc zX6qkUJ#xgMpzafIknu!%wQQ&ViE6!M({Jvdd2bj`r8mp=_@4)MT;FdkpLxLYBE4O< z-+$ejUCH_1>_+p4h+QZ~8Ooni9Re)9c4Dd;DwVRzj>4;|k0$GmFVT7rt7Rx|8Nt7; z`bx6Pcp)&>s1+!F8QH(Px;i;+e94&ih+2iJEQN6 z?}(U;JQuO4|1&6W-s{pHV~LiJAGpFU5EM z=?Uv`hE}R6Q~8S@B&JM!*ZXe$JL8M=L)mNpR$zIuMkjtZnjn5djh3nX9D5+=+?|VX0C(P zk?0IhK~HDT4lt^*JVww_0_zp*8gC|he!!}l8}E1BSv`d|6{aiG1Lmv^HTYv~I*ci_ zX|S2Hx$I>DJ8Dvo8BBprrA>#;mKm}w0`}G1IOaKpF_oqZ)0aWmmI0!I>SJkBOr{}c z49@}F3Tt2idvUB`O3^gr%;CA{Wo(ClIXgYHC zup!!l?HN#9BWX;Ss;`UCEB!^b4)8M=HJIa7Q!{lD`lS#VoQ(?**CZaFn5tZ)Gl#So zZN&}>cvM3_Zapn&2E>501Z~YG2FPpp$K$4r&w$J&EtS1SQ3D2RI*&7_5oc=W!j=Ky zh{S+t>omvvrzvMj<_#}L?_?hj(5;=D>RIeEi$A|~rOcUqI$&Y#`qXY+&Mf)@(kj_a z&zykewN9zo)1k9zhA?xPJDVP`p%$OoHl5+@wQ$%1y_bzUZC9I^Ix$^e5AjRsTG;{i zZ$KorFg3q`rUx-1tw(#a838`E{M0zzB+QTz3?@SZ^Im9eXDU;dsIP^ewow+qt_wI-Yj^@agE)tNxYG)K&3M<*+**qh-ZR8==*FbY=uq~Pfa|rp zPvp!*1vc>~+9Y$vR1xtu&OF z!1#;)&{x?ox0Jf6X?oCKXlr2y zWH;E)xmk6~(p>b4enGAq_Cnucdj=NQZBI+nGciK0A3g;1tGxr6b$)3LdPTteIgCWJ z*uH^Pbw|^5^-Um1*f1K+WfOos^h%nGei4K|bPFTnu}Oi-y2`XoU%TgSgZy5tf zLXtF|e$pby29h7Tik%erwN9Hhsz1I6vXK;ku4SJHoLvt|x1N)<7-B`jqZ`D>mc zCVZPx=LJ3N2Z6`yXPh#ci#4Wfg+&;I2^3E?s+745JVp&^YasBpFX>(0X zk#;b=^ey{&Afq00s%mbgDZ;*#CeyH0f#UkaQ~4i0m`au{J}i6B?h97xst1{`&S)CG)V$5tRI7**NzpTEPTxUF zLH}U?2%Pe8#%ZJZ_ABT+NypLCI5Y7J9$K9Co-bZOcO;zv7UOyNl@E8H&Yy3;lI}!G zL(f4pVGtB|h4Ip}2^2Ym9w@27PQO;#iK4Cl&L z80^AlK3sCf!7y_*!mTt$Q0xa3mhukydg=!0}C+xCpD~F07 zeAs!0xsbR@=w+Dx;<1*$zx2AynR6Py zuwi|M?=P)u`3FjG%3L|Q_*K9fUi^#4I=)wFiOhp@5x*JuI2~-+>*xnbx6%7JSMiPw z=@~{w_Uq|~NO#c(Ik)i#8g2nEsCYfyyYyrRlEcFLH`HZh8$qEoB#bG;aQOJh24q65 zQ4z4H53|tV83&)-@I6BpVgf^=huLU6rvabcuq-n!^%xA{Q_2Mb!H@Apz~-I|(T5{2 zr4_Pp&T~AY0h3t;$%G@Yu%ta?juJ0wpk8QFrh4p(}!u=}#ZoR1)Kcqgz75jSZ4VAZlCoNxHChL4$=Md}S&e^?D#oI52* zw{dQk`C{xwNx*O|`Xpy|kWr&$7Gbe+qa<*+9(|g#AjrINZ&t}-EHL#CH=wgP#zEGN zl&t>6${W{%N}FV7IV*zBn5JhL8Jm!;Lx!5sd7NEU?u~_64#t_A5QNee^ug)SAfHBl zR-EzpCP)bBG5Qi`b5MArB#UR9v>6ggY6tED`=EqIZPuvq_-05LsRP91at=Dv2+6iy zlC%X9PU=LL00resjfm{HB_7tp;jrhjI~=c|+l^k?4Q>alB@x42=u(bP5U-Jv-M>U> zEg=p|(QHm2a5+eF7n^t>_@q*~tele)7WAy~Ube4ED}qlhRieFvB7=Gx8?*CG>}}|j zVN^jShZgj%u`j#LgkeLAg!Ra(IR_ml8oy`jE;ZSzjk)krR?9gVq~Ek8$6;x!rBO7g z58c4Y3^Hy)9{Q<78X+5 z0Yb#MHs$5OO^J5e*wXRk&pCI4yqj1#1XCsOeGIG7GS0ved{awKv8jhWpH`}oDLKNR zn5LneZqrtKethXWSr4Zs=mhYNm@f0!#y?#8LH3H%3^bM2pY>g)tjMS z?z8#J>>cQdq*1h*BMssJ)upawVh4H>X$<|2(-TzNlz!G|x&3zf5z<%mC(dBd)23Tz zy_bu(YhSJ%muZ2ArKhRxZ1(cZ#DhnNuM~add=63rmrK=hEOAB(={tIYGamG{Nqcs5 z`S=dVF_MnY6z;U(+0Bq#>lH~mA;(Emfm235*ti*y8@EEgVcyre6G(ju0wEg^Y>g`Gm~A& zL$~M4S8{g+_wM}VoV8ifF31^@A;2Pd1Yc@KoQpFX-v!Aa{o=EZdl1l2k>_}3NiL8~ z5(GGh)^6rEQ_pG4)Gpdg*kbucE+yV3${eQg&PqpZRVaMuTt;UX2X`s zw{oe$gMiA)T%~`nF=yD+XB#&mIHv2(x$af1uKcs5%jG+`$AYIl(#s>4TJNFf4$rT0 z;-&}Bdt{d9xtg(ub`G{me$_HN*!0o%ytLIOZpggh)jsar{NQLJI@oHt*C-7C2~Z0i%teH+~Mh;^Q@M!8o~JZ$fCnEL@Z%i7PE ztikS+Tp4zdpLTnZ^#0KR@SchH(XWzr_@r=u1W#$1alvS<{eJp2lB3TFZr#FpEoK)y z*E06guEU(=>A=6Xyv6B4_FCuxm);Tcq){i`t;BNUy~TstXZ zv-F_+Hg`YaW{dlUe1L(Z-y?YguL6p|Y4NgYS#R>nlL1SuV{&l>aZ6(U#Cm<;87@W3 zIox1EYfE9i12hwfV3uO!JT8f#Y^lsohUx=5W+^b!abtl2Uy)x0%|szv&a_qB=Oz-y zT0Z7$plX!1tn|+M8t!r6Da^253iCko*`74l3|ztU<@BxIV=+G~5SgiaFAhq9a^(tUm5`DFmFd7hcQxF8L z)B+7qw%3Ybr)R$B_5z<~PJ#Ibte@omZ~_Q`^%fw#SOtU)NqtTW2Bp82SCAtA!PNdE$$Rkpok9oOA$R(if>kA(YrZsh* zm#<0MA9548V;klm2=o?Gz7Mn<6~cKOa;a;R7zo~y3Vb${`-jv%PQRpaUK2ulT)J`{ z5Sk5Xdwk_miT{ZZL|bX0PyP{Ni2O1CQrzb85J)@evX5f|HDnMF?3uu&t8Ir}k#7T_ zs^>F`aTJ>@4h ztZH*FEZ*V~&hIP*B=Pc#A)5g(t9wgpI3Ku$<@?I7Kik=sUT9=(A3=Xky6bbW{C3EJ zwp#$YC61s2{E`o29koO0&U+tbS2?U!G8IbLefG1gNPcMQUrztK~<^Q@(|awS6qo09i9_53ELh ztbA(d%=U<7=3B8*l9$7^J}1j(hZ?n8UM6f+7I1q@>*c4)=ZCIpcfVY`)gzkUSK1)Y zEMFYD89>*%x3)&}UzIk=&z7$Ub#BkNY-;O4MZOOQf>PycLJzdx0^l|=mEKQk1#a;T zp@Hpafhhe+Z+2#1qruMvIxIK}k9fm376~Or}ZRZ9B+P7M8HN*ZV&9T>ytJ?HVk3X4l)o8o@5&9PrP#!363O)A(zrtg?lI8q$_@mE2 zd0S}llkHdG#k$FeZ>68*Z_A&DGM-?r686S}Nc$u$01_xeMNeo~86YZ}QLOL!r_q+^b{&HPBALewTkK{}?(5qO&u15L2|@VIYut`M1!q zCm*kBcBoUdKVUivorf_pJ@L7ua6=q?s9eLN_j>J^$bhw@(cj>w??+zA- z`m9Wq=xX#aiz}?c2p#R$N*u8#Br`Myn594mGp=LgTEC<6gapJe#;mAt3`^^nd)?d# zdr~q>vjDTY!VLf@Ew2+mm|w|k%|guDiUVOK9eb~rIAH+>LSux1RrrMQIw;rsL6}>K zK69dBLq$MXeMiQ1Q)iEK{v4*UVsk}U*t3p%*L|JE`%?|bCK#KF$gtjy#_Rdc_NVA` z$)*_liukY(9Rt_9oW-a3D`tQ&rWNbkeg{EH)54~nM$BifP&ihc4AXzQ45OOnY-BaWZ9*C9+SxB}}++r7o**vA*DB1t?jAWr^ z4d#s5wJ>LZ#ps{fG#2&?e4WCp;#Szcr#EhRx-c?mMsTPCS-}ePe_D4V+Xb3QgOJA; zVJdiGh=*692opS|-qZldht>qXRS{Lu z7uNJN?IAv%ZucxYGxcPC!Zu7!IkeyeZ~FX-G< zLUvbQ(5{0YQrxLHHvPBGy(J|8sx4Ws@djsSpKx9$rKI0onJS0ajV1XWCWBN8W@S ze*D>t+eUlsFVnY@qX819DSW2aCY z;~D05)!xivgguj{&{U|x#m^FNPwdsdg4o79toTszI=uB+;cbU~nO6`F%tXa##c+7} zvI>Ca8NUMAPCkMG^y%=o&m^~b`#jcu-VQ$sByPWjk3IW%TeDApQPU1h3I?bkMCbzk zu=##0AOmZTW2WKe#ENh?stm!dz{H}BBudF(4q*d=OK5x z_Nz04oZ+Vwb9u`mc05nLV{ia^gSHC}Lf!H#BKAGMamVuj;|9$Io~eNFEF=7%*WJlJ z0KG}u4F_Q-0f;E_c^{D4W!$8>!p~xC_Bcc&KmUG5*URKL&oPhk#-c}36d?k0QbmmmOTPO*-62mo0ycdNWIOAsEKPCGacRmFc!1w=G;iPi(2 zzOgetAfoEi+kRO#CPZi$Tu;r zyt5JOmhZfmeF%D&1}HEJcU}rTVnY}HUfUtYUD`qTZN*;Rm53c(srL-Lq4#Kq;CB^X z006bG>jqF3hVEHiUUE<2%`1(-0z_1@cT%iAph;i=2_hoAOLC9roy35k$SjO6PaKiZ zrM)-mopk6e3eHvp@aiH?br~|?NMflL4d*I?c`XqIU7HwW0HV_Rz$+BVZcighy7n@3 zTXr$|096HCd9sM|t_VgqvX#llG6jlA-ph!30JAa$Aa6cEA}C^bZz7&`Ir$wxiCJ`C zvKRwAz7f4$jf{MheHq=4Tm^twV-Y@c`WS7%=|%I0KTsUyeUJFjHMP_L4Q2Bew$xx! zdD$90$&ykBbS4`CEYONH-W;Nd#I7_MtNjBY=BtCWZsp63-_I z;19{*5s25ajwbn7s_{{mYeV53ifcS4;wgzC6Al1j+Aw&h;y0c$fYqCWsGWcxzJG#VUW40wEEq_mIK=4(IKH6Gr7+-s|tm>|`%`YCfCc(cb-t+p1P13wFxIeL4dj$SX@tHS7luEh4-KnnD@?_(fFT4)`Lf2ka;*XtH zepF+)YMeJl`~XC=yMRNGop&uhp!A|4PBrT9xu)TptOgofi zyJ!aAfMhD$&Q1$3sYRxe%Q&<7zmT9Z3|rSOrWSI7JOgXMHzC=|XzbB|@mk19GVsmt zSCNv=53}E}Dhn8_%fZZHbEZvSrTwcx8hG*m$uU`6+><23ZcrJP6y! z&%R=;+{|~o(E&umy@SM!biI`(N(7%m>XFrPvH@a{2JDeaH*OqBBYVMV2r6nqp4BYF zZs#YHCS>0^y1^#R$Xv||tP?+tRGz&ASl2Rv?Tl%rbm3=_OyqXlZ6ew@x<`e8&>2xw2i_8P^^YRzmHo=)z1h5S#efjrEN%C*p34&6f zbCJ9e8_2IGosmPzt-*O3a*4bNOW?PXF3Az)aUtW6A%)~ESa1d<-IXKDc_B${kRmby zFlm$|k(^fEwfuFPwg?W?m-&69CV5^tJe1h3y$rWg#saR5RL(6YhpOAP#qe#)!~Bn= zL3w9+ei!A}1%?+R5)PcONal)qa4alb8$u%v^1U!anc42eZ{X#|=9RiP##2+7Sz;`RN_`%983Q9$P zm=c7))wp7F`76mTiku4baBQapM0~_v;IAWlD_9kTaCJm^3DZMaz~4v?QnXYUtlaX9 zev9l0V3{`LXvIK97x;Fk-zM+FUghr~A6Lxa8AaGXr{B@g3U2Uslg}#5c%I<J* z5c{>YhtP3_?WGNhRj>8Wo zw<-#G4#Z3eg2}`Hd=ZJP1e;YdQ6KC)Oq^20j|HGk1+R*jDMf$}xU!0WggmbJ#+xAO z%Mfg4fU=f?`E6C)n&jBymTgl=RKx-sC4G}BjXVu`9 zEkKRbL&@he$x0<(NRCks@w>^bO1^*@sqEp`kWVP}Jbt0rtLQ>I6mq1>!#Q zu<`@{eOR+HBW69ot!nSX6P2FuS1!p(i56SdAtN|{N^@IFq5)CWg+yA62GL^l1~8a>paRz z)ovm0)#_gD+5VGQJ%JGb!}j|tA49}2EU`+FStQ*?ar%&#}S9L&*Ar!hXnU1sBUg0 zIZi#K?SccBl7LGI?e45(#u3$837n~69cV36NYdb=CN`SJGe=r71s*d)OiId?g~fgE!{Yb!Ubu>rE~pMOeX;$1 z+F_Fq$Ued!(aH_Z#Z1^0r^VPh|Lk0Mv7^P_a_ePf20k$He=5TRz-Tw%)LL+ zf%+(Ut$BpKAXpda{epF$kf8i1>DRPkFAA&x^tt_hNkS5mSRDo{b%|c@>2kgc8xsu!lKGMQT&-H;UlOQ*86#!`$AMyCg|H$lDwhAsr zZtjVw?mp7`mH&x32n7GGMLPFnRFkP2ztKOF-vSu-?Z^W?x2nC9#oy?oWHnYQD2oj2 zX{^pqwjZZ!$r|iYbADuGPhWLgGGm-J27jk~A*hZ_>6!Y#;3)KW+86i-WuKri@@$XU z1J9$3-)UdrpOgcF_QXBU}*i@@3)t>=V$L@pCOrRdxU?|L$d9Z5yD>#?P}{ zuG%g<8a4iOqE`2$$t=|Tkrg;6VOo?y@3K0VlSQ*^Sovlu7hzVENv~a9@=5*KHioR# zDmUSUC}=OHuIgmwY@3BF3zetva+E#5+fSTiEZN(q9!R4=!VoTk_2%i=Inu==R=8l{qp18|%X*jBb#tU<2R8%h^CwZ)dw198r`sD8ETLHAh(c); zr#A$k`o#u_pu%l&L|C~<|7`si$betLf?!} zr)SU$-aUBrtIEJ-OElqH7vLWxEnKvQGTL=nXcryd_wk|TjC$eN8q58vYeJ{!Q+{vue%I;grWbm_d-cSAob18Nk%-qKrjPl${z>%%v+ zWiX85p{(tz0SY_1s_$rnZl(zY1sg%*xI#j7Ti=xim&_sv3O<6tfk%|+?!M{sj7rWz}NM+hKWr5#WovR0k{t_C!%M+f;3uZB`scL zMZx15gqhLCuMmxKS?XE@A8 zT!-*Fpibm8nrCB|jD5Hfu6ibX9Jv40Ex@G^FDbC55OGprS#;p5#>V_?dy@hLg^U9m zX!I11zQ(p}hDp2)W%I%p!YaVF_}-|SqhGmpD=SLXD@=J9o%w1>lS5ACQX5+qRn;#X z+&Jp7v5B0+Soq9#BnJ0J_%yoY)!wF(9IS7N9V{;=ucfyy^ z^?hp|2qM3Ue)j5KlkeHqWda9Ql4?x&KKj+GmZsvf9?J#WS;?w#;g{%7 zuZEhs&$fcc6;_Jshfs$)?X_OBX|Bf#!A{n3)ijX=G5@tivv;m|MS&yb1a78i9(CpG z-Oc&A_A3jVC~1HPvWTj-3TST2wI}X&9!bZ|6`2BJ$kArqb0%h}T_dM)h9YyS=j$uY zF6WBOP%b0iE*ptpRP5`@=Hzqwt88|&vQ);REmR^ngIAr)TxH|R%2An$wo#J+mt^9c z{_1$Qpj_O%1SjgL*M^VafLt=R$1+c~TI5bGc)jTnIZwSBl|OhMw^nq3TJn1Dqmn$V z`PluOeANs+6xAjo@KIYH!#v)@vOr}i@}oX@o%+b&Jk%ooSIbMPO`-(7$FHwEayefF z2r46m!0k$=cE7HEly=@^4QekXuFO^xOI5#CJgPdMxyEK6>x#-jbc8w%C@d4_8QTr^ zQ?B70MJK4U`ynmX7n0U4Ixv!ZdY34Jx~P9s3;BXLu?c8r;P!~lQO*1Jwv=4Ju9JF= zl;D09T>`8ZN=yF*Wx}n4tlO%6qH9#={)`syijC_F4pHvnMv87x_x0as@yutekN2i{ zZ3jFrs&9XFO9^i_)CLea1 zTD$0CcmbBua)KgyL4D9)y+EfxDP4_Y300jLuc(ju>8;iUNtTOzDU~27ff_if`K@sU zf&z@PE}mm>N>oiZ@vvC&2Q zjRgUeTHFcI%$NlO7LUCziZ>PnQcO>!i{{2GA8>k{eG!|`OLDW?e@3)0X59etG4EoM z)uN!0f|FUIB{4Pw)W@2OYOApz%Vt$BApN)w9tTwM|o~wjLu|%2aF-KBj3PuMJ*AM2wLv6{-qRM2vKR+eR)@BgV+AJ6naK z=$L2RmbT&|4;ul6)vXeX4##{N7;5V-YPAtWvU+fVv$j0>J}|Z2;4*Y;e3WIcs!n8` z9y4#ytlbk3mg1u=U#S{JIWfxzo!YZ6Lv7=!mi^!aK#$olh;MJZ%&?7*u^d#ji>}1% z7))(9D2Cd_$6Cs_b&5)2JO{6|yA&4zj?>5xP6Eh;zJt~6X~ib?1y4(dRdP{93~^A= z-hhp@UldOn!F7woF$sg(_R->T`$dN-?{U4N`j|!H`A^_ih}*^vTYgl%7PZ6_3~qWt zzM|eXmSFi=^#<^yZV!5ag9p)JEYUJz>##^3!yBYL=?5o~MQ*8Ms&^u}tY$F%iP2U2 z?FC0DUvZy+Zu`^0TTi@!R(C-%WgIsqdKc3>*!UzLNKqF66=~H~gf`~gVBeFrtBf7- zDV9G}-$mbJehg0SFu0cV)alrWj_=e;-B{<}7k4;Z%iL*moHeyuw{lLb$s4`%g}%W6dDcuIt;zzjsFh1&?cXGJOkC4aDZ_b&YG@t;*l-qRq> z_YPXT6nI;l%vic~@hoend7xc7I*Vsb`T1K%2LgKbpI860YwP~KTmMn#=0EEkNQ;N$ z2oG9}yt^~S;&DYJsMO}KN+Ex%^gpluua!csrDpz|7>iku&f&T#AoJoGQGt};z?j9) zb(74(DZzxp{$PE?kts1|b}O*HuoY&1y!HQ?M)-L($&3<6CV+HG9Ek*SgoG1F!T$e# z?T=FZV-un#Gn&Ow!h<4%31%QSC5{}G@K-`4K7tSwJ9(W6+tNb~=R6<~EB=zSK zAcfSy^th+-L?S#q7_j|DI^M*>aOvvd-c; zj$U5-PtMK`@rwXLUs`+e}VUoj(ZO{gTHLt$fP)oD{>2Z(_R{W z4>A@{@eJP=?u17lj1P4q|9OpM8-*u(kqEBHM7M~@P}Js_a6EZ;Easpa!p+s%3hx>p z3nT|qk^g(%r1-rUTwhy!$8RxUfmKcb# zrcBo5MnZafg>T&AMzON}x3>S;A3}orU+wzYuidtZ!F$|8Je}SCvF-nmC*B8TP4q$U z{d*g&@MKa1ev=o;Eqrpkz*wN2tkIjjtb;(m_JxCfZE}ysp!RP3Bi)J=?7AD2;T87R z`0Uvjb= za*V)uPWCY_7-bbtK>lqG$hP5uWMo9}A7c=U0qp{1Z1nkSUJu*;>O48t)?s+mAN>L4 z#h^El;(~W?P57fvm?#YDFcH1!f64Pt^AFZt@GhS}$K^lA4b1Dty%GO7_n;q#V?x0C z{D(P>!c)9LJQM!cIsJQ{f0|Q}eqhtyKgQ%A#_aFy{-^o03Z7iAKAYW1V4i6~;ovhe zIlr!?@F4Q;1j6LlI=jacP`eX?$hMn*jv)m@3Zm?d!f)F0bIo{0xWsunZ;5xKfVBbg z{CqK@Pjy~YO)1P@g!QYdcfb&-H;utddvg=>_ zg`*>8KiF?3-yU%Lxl`nYI%ae17P6S)f&wVJ~5o|o5%0D)>$xJ`X+k5C=`{6Bu4vEhbePGoZ$+NQ8T*?kO9Q6?9~KoP`SDh?Ph85?c`#neE>Ft3@K zshL=sSsI!b1VyndE308wE6YpT-Rxdw_4}OXVLaO&OjG*)zOVo9@5_hnT%OPSoX!i z342cYNR_hS<53^oKcowv{mzju&G#((Rd{YGXKMD7zrA&x)nokc+t&2R->uCdOpPlJ z??d+oN9Gh1e^!xk`ngZUC#UXqeKe$?%VYaC&kU^9_Yt4_G;3^0mE-w`L&r5rKmIGD z=F`cWmwX&j>gu2K>H9C8c|`ln)Ys;+3*NhQCiUxK`zJ0bJvBD4_5HJVy*b4kx%uL< zGbuIpb*VpG3g}&&ao?GH))Wphn;$>$TF;-0`WNZ$wLkdN;v3u)LJ7SIg zkoMe&D`UPX{`JwWbTLC0^2BLFf0uW!xKi92Hq!BJ?$psDL&1Ba!g|hr?b+RLY%O^rGqUmE z-zUP3mJE>}v_AF9mAnCy27i@t-|77A;~f_^9X{_`edOD3majTpe)UOiPhFup?E!5< zW%HuuIe9yG{#Gfi8Yx0}xV%Y3;}4-IP!t#@;d&}5;d+W8Mv)!FB7_RA<_EyXNrLEd zS5?*Y#7K#tc2QnSaa`Y3wFSeUnW$X9tnkGzXAj-7syKPTW21MhnQ>yqp=tH&t*OtR zu{CV6z4G<%^N(#Q8yVeu<;kZUuUZE^;ojjIwQKV8A3Rs_`nPkAo~xhD2l_C$g`&`O z_`oBZ8wOE`!0|{lK&138JR3(#*f;|e+AD-}XSS7<=!-bR0;DLB2txt_sP40imJLrA z4JXNe7<=_q{C*`oiSB*wBkuj~z3{ie-RN$JmbO;8>)m_Y&8-XE^-_0()O`TbkF+j< z^dnODQAnt6UC_D^zBRyW^{p$V5NL`flgQ*6$WjGQExdPF3PkH$A9U}>Z$bWMDInIn zoAv^@dkvXXFV(6wy5mjK3sq~Rt<~W810pPC)^L&8aTz_K(ZMy7eMu{gu+Y9+%NF1 zAtgabrS6Tb4@0g-;%q1|Wrh2&>@98+6ut{;CiJobPPr7ifcp^eVy*0VcI&c;NQqP* zt24xD3A1ER>qD(80bXUipBT<>28J6)a#y>XBl6G@VkLJYl;@1{Q5MmYd(^!@Di`>{ zv7vg|R0*^ibe27`tP}^Tk{u)_%l2}iXUOLB8{7w_oO@utP)PIawlarhw$f4FKi^(j zDjhV)7|m{OmA%mb619UanHa;+Rf$@IXW9UK7(`bRQob5hB z#1T=PX12T0xIx&uq;=iO(FX&<21(sd0OOAm2id@@v&IHu4Aek{_Gz9no-#gT%n)t? zmK-Ez5h}FBxQq)8Y>Wzh9=gjuH{qO8t#q$yT?7qY>#k4CFj=h1ta9An)Apt~lVOC_ zKL+(mvfC=mHkSf=^C+}`tuF1EQ0NE?T5FBdWFw`{5))J#xI);`LE^bc_d)kQ=rk}` zs;dd(>PYw|9iFYov)fCwdW51)-S`J?BZGqIE&^x2An)W6CAy zHqB8hLaD)K++ZwZyZ39mqZ%U@dY0*4w4L)RT1r?@ow=u}2m6RSVm-8GeM-I*Mm2D^ z&RuVgMn_T(qZf<>vCP;kJ0<%OvuP_(tTM9dW@uNig-|AqnlNxd#-uFfm7rE)vuP5$ zwYILh?n&ZBw4d{7wVrbV)tfJ)2I6V-IMKwZ%lM8DJ*t7XkoW>sKcpazCMB2@$vhL=#CMJAFBFqyeh_vV^2 z0KbR$E!6!2^o%VvLzXO9Kf+{%q3$rvmZn-wr53BjRA7>hu-l8Q=4k0?kW!jBDYTdA zVbm-;0)4_bkPkz*u_d7Hwb1>oRZ{l`!o`QCZHTHys!`t)eCS(y>jdo9ss?6g4^}jt@dmA={}O5V&fzy~JXYW|x`sEheip$?hz(nku8EPJ6z>X?8)=0$(3C zH=$~d8Lg~-TlSRXVGtKh?iZG~qJ4mLq~&=bNQc%3Pw;{3Yf(M#iI$fm>QjoMTbF_~ z(r67JW1d`si~-!%rApCTmt3g9l(%jA5>x==Dyi69;jmlHE@`B(pumz3pTkXdTg1E7 zz0d;Lgt{N!*o~Hs(h*z31JKsGueKSu^-&^U)P2Gw7RK`#)4QzD5`e2`>KErB4SVD#4- zmk~d)7fPe%ard+?G(L-aNQ$y99>h*EXJgBIksz}+y4Sk5x>w1zMYg;KLTQ<~ntvEZ zE9V${fm>Frm9y%`p+?zO&JyU}n~9HzA#0)U?jd@FJivv~x3;dn#h$YM{15MSrI->+t zMGVx&bJ=^PoDFD!X=hzucI$!`^D{OWw0D_ipeF5kPBaYah0xYpL83IaRLjl^o867k zF9|Aw2Jr(!Gb+>V73P*Re3jJwIH8rrn4b_7aE*m%C2{C{2s&_LoADjgXgY5G95o6~ z@%Zk|Ad!X;OSc7tLf=^EKHTpTPq2|2x>S1nS@+tS9Wh@?p)bRzT+q7c(gr@rplI!8 zf^(SQ5cZaPiB0?$+|5l7okV=#NMzkzoK?nPw8!|+2{AhO#&y9rJXE!9RrrF|b$I4( zZhg@BA|E7aYuyR7zh)IWWOxM0&KJJQBb1Nt0trI=j5ci}(EdvirV(h(sH2GOe!-Z= zb|19-0b_Bm;2aOhjFa>m+>g80yB~3{Qo6UepCm$L*99i7U^>sNVM`TJ5vabVQeY1< zW|-b%yZ3%MT(GMvuWMMgz1&tHwHMY!L^>*^<19r*_EPg4!TVhFbk<33^#=AZX-GdF z+G`9%(KVN927-XT#9Mk?LJWO@RTqNAn&5O-leT&n+Iv0=_}N_ZbjZMSyJ%-3MAuj<>PdN8QWWXyf9i(Vumh@g;8S!swJYTM%0sd44en z7*qYW11T$HrBJT@+y8n3A;qf!g4MwSanSPSkep!gr@=K_QA5a4J}*QFbH6nI`~x*> zLOu)LRuApAM_IE!>@66 zW(EsdT4fId{W0D62E8WUq>=R$JQl=;!8xG%?rrnYepy1ucrLFtBwUD^O&+Sd}4rkE0pxnnzI6(5bYhbC`eX@ z_Q?i^#B-O7M@wZy$a?}=F6W`T-qp$IP|Y*Ox-b|Mi$Rms8QG|gFp5*USI@7Qz(RYK ze;~9YB(*yzAXS_{iT6;mczF;W*+h;YA}E9#R5L?(q;+X)jjRzh$-2v`p^0Tnk+OPz zM9Y`=eEB0E2E!u3s-WuMxve$Qna^!|S0Li^L5KRi?xDR6L~jL85XFNryDVfJ4>h-( zsL88N)X$aX*b2;r7MnRrsZ^S_)w~q(vv3ohF;i@UDcsP;DCu0OTX0?=8w%yg<$=Vj z-Wc(bWg0r#{Ugp0&gT)!(0)NLE*rF^+3dsa>W58>*ej*R!^CfVlZZLsB9*Vjx;1*v~Z{l1~p7 z_YcX)M*9>)P~)Y!dw(F5&$Ev23=mei54SFCv2hieXu+Bu;*-$6=XVeh1uhtO+O0@{z{iWAVc^W*alqjUS?ZEv0TS$rHotLxk+~#9#z$2Ih$&;m(5KkwP~rAq zba-1lYB*x5S!!9d--Ql~-xY>51hpVR6W^2{Qr3OI=R)+i@gUlCKCe0mrlcvvJ1LQ< zUhrO5c56*YY3S%4oIWFv(xB{)_6q(CB4%0cj%<*QPMMGmb9cVIx1>i%SvN%p2$JX6 z?t0V5%AN>jpI2WOURbnFGiYsD$h@HHJ>qXd(O%+nL0#AK*FfvpiT1QS4=ir2tw}P5 z5rq+vKs!E2mMpl`)vQ5Z{}n)J0zkWN5WLMRb2s;wshnn$Bfof<%RJka5Rt5jh|@+S zt0RWTaQH|APYOIbg3pdmCIZoLUN}#}Pm0j0d&wZc=f(5x;oXb$oOq<>D1`8uTFDWj zl?ClS>XV?$o!o|g$;7NQ4DcGH*qH(>#=`Yl9U5P==MlloAf(`&LQypz$*zS&cjdw)Am}FwF>yItLtH`Z*5&43fn`bE2CVJIZg8g- zc5Ug~cp9Pob$`JCcA_=a*NLNqTYHs_8bd|~p`$hH&>_Kbf#9_sLmRspa-%RJS|&n`=U)`;0u)tqD*iSe0X)3#JrO^}_i8 z9+qu@)xJ6>b1ozK)V4s!i~hD1I;Bx$CBbvnsmU=3Ea>j44VDB7()I2xAi zPz%hyY7&jK^0Y19F-J**P8Cqb31o|5`wnm)FXPEIe#Z-Wvh>gg_G6|=pvR+(tLbe< ztH8dfr%}p%F;SRelDtwaedXcsSLz~P-x|q(TfgGRA;QoUVR+ix$z$Ihm$NP=l*2? zqManLVeVKL(J^+qXf{dkTtMUm{>8C4--^V~I9&F{6dsp7F%|yI2;xcEUq-^Wchh+i z_QWyp?bjjj=a&pf_-!Ol$h})}k(0@;R6^v>gCI|}0lxL<#p9yTU@rUKM|hDaJ(!D% z;Asq=%}yNzFJAiy@*GWoJa0ktC-7W{y!RVM4l%H%08@V+){o;_!@QfZQIcv>RdoDIVw!QSOSIQs%eBRyM#dVP@ z`lqeiX4!ntL))L(xqIO22X=kktm412C;Ba2#{2tAKaE}S`Jvt40cB`O(!@%23~pG} z;Pw*xbq=m4Nz(oemzNm}RQ)@PL==_|R%UF)ag7}&pFt*G;R8;)9#=8c#$}#c9YwB zVxE4^qld~_WlQcUf9vn)CgmUNviqPvQiGnYocCPdlG0Cmbe*_iQB(h@dw$&qMVc&{ z=2Ik`O~S^BxJYyeilnG-D-xRrw&{KWC^6GqSn=TcXLj!F9Td=X>eMMcUlPo@^7r3= zcZ2Lu-jSm8bWsRd+VEu~;O%=L{scUi;AxT)hy|ZT@VMbQ3J(-0U6kolAe>FY#u;&e z=nzE6*K|-IgL6VnZEauf!~jl?PN%~;pd?#Zq8+SsQ8-zaRm9bm?@L#&9b|a_?94~M z9o4fSjuZUmz0P@c-*M!&h=*ML3|$sFAFZtX{Kvxi7Y3Af+s^KPp&+NR*l=>_7x7!R z{yAw&w)Wd@&X>0It=hKuCC$p={i;5*@wY5^u%>F{h3T7~dSO}B_THDzMc` zRFRKXu}_VEDRAl&RqxNc*2DF8eHA;h>yZ3wFI43Xdj0&|1D{nv&8LW_d)J)nDT0LS z0mGe;84ER3?rY-&2bxCsl7wBAb;=mIZOxj=OZU7txukD!qCnD(BYatszYO;_%n=_M zNyZ_O-!LHe80eh_oQVp-7yoV#T{|6U;*KdCf^*^)!_6HH4^&Pm(t1~pjiYJObO@q* zPHd|Ts|(te^yl9XD2c=X2@w(irH&Lur;CC}n9y{{i@Lyrvvc6VkN`DEG}H$QXQ83! z5JdMWYl8~lAz=jTsp5|e+K0o_`m$i(ghlR~{4z*%y>Gf`Itkg-=L2{NDc~t~xylBq zRK<3yy})9%SxUfmr*f1#ofeZy9W$fciWe{PQhR~fnxGzuM4Cubp~YEjQkH`0m(V8W zn_MQVy-20g=vA}8h+!i{F0(DaGC?CRGZmS2s$rm6+bc{iMVZA0d`YsGmSvh;#Rozs0z!iR{2aYG$&|c;$zfta%XNO?4!&a-{(bQ zCA+LVA~zXY7VjTeD@`t!#cor~G6jDcs)*ObX;tP!=>AbQ#E#I;7>fq?^?JOxJRG1=xaRP=Ho3Uk^EO{xO3Yx(y( zz+w)?Gr4l-Kt7vE6B8FVH33Fkez^lW9*m=`oGev742%*qJcl(WlTenRld5qJb6&aG z<*YQRitJ98O0QKLhUMoa7=Rto@w&K}LY+nzYtU&8F}hfdMrB2R*-@~4i^2scg?5{= zthj8L3mQT6+HEdVzRRhCG2xJ= zIl<)bD6_+Gg<3+np@rvI94bw`ULU7X>kRP*xzn}4fuc(+F!;?5=y7J3@y8){SAnTi z;k1`qO)$Fi;;V@%22G4kZ&2&yWv+xgd09b%0XhdX|1%qgn988uAW~dTh0|m;r_5Kf z3LS{=bXC}iMG_D~HfPDO`Dv)O$ZjuyZ4g(9NmW)1D_f<>Sv*^%QETmUpj(ufDov5| zlTf(Hnpa*5wJEYpE8{#CJk*Xf1ge~GMup!a*(X+LCzZ3j*j}lyI7{p<7%>I8zsXK}H)#H8286ecP?pAeZg4oYK+bqz0u1!eb^F^n<0d4f>3~=Axlwqxb~7nhYpSJ!%Kvc#A{*= zv6|=@t$uJ?lDZ-pv303}fqZF#7Fg{lajL|3bw%=2JvyJLOeK(7JeD_V8X7+Up#()B zAB5$o@zY@Qw4a){`APs9Ml>Pk_|d4hHb~6jxk}1mu$UAkMVVf$i;34q$LnL`4LS{M z!Is-xL~6f1+;m%s$q7T;Rta6hrYwc+D`#a(#*jjbwG?-oSgmSUp~GIfcNV&uUSNW* zpp7%cLcg@xp@`-RYf-}Zq-fpX@#)bTjYb=%kB^b(7boP5OI3g@N=?vcqGO|D)oQtE zbi!yDrun8av)nY^g;2;a!Ju=j@mElvS!{>h!nc-uy8!Gt{^$-00SieMjH%I_JsA)diC_T&~<&ykX~8jeD&h zHSYep8AyR%v^q(2L6I&>Ce@TzWj`6PE`%n*A zDoRWRm00#cPb>o&1R~xFQ@8_mOR2fuVk;|m9f@ixLK?s=N$} zTwubA3iQ+hGxjA=c}6*ixw6^AO3SUTslWvlIRQIWF)+Sj(Om(;xd)y*(+dtV21%!evw6yXR)b3{5e&_a7&U*C}4Duv9ha4POHV~ zBI!{Ih4+*SquNYrs1y^OFqairK%H_aX+C4+G?>RPw5w*myo8nZfF3kFomw5If;p*D z;m{nG$e{?J2pK=J!Z0qh+Gjycu$tndA{g_^Q9aC~fS8-_Fau1!4Kx>BtX>yWWX{Pc zNWFJVUaWrXxbZX6Mobz$D~@GJN(Oq)GDWGW6x2UZMc|7`fD92`iuJ1gct)*Mfx4%P ziwDLUbSjMwCQwVt?$BAqc1?n|d$8W(D$XV6LYPXOF5}5nPOM1M+Hr1~-D+`J@|~b` zTb$B1g_jth3xbhQkS_z>H-ghIVLbJJBMY1u8|L_PnNj|R2)?qafSuQ~j9 zI!>?C>eL2JoIzFSuqw2`G-yfFQ^+zlu6|gdsREV)z;=6Kq1oZ8EHfu$rex)gNli`3 zk~@m93K}afHd~5{UAYn0U0U5>r`egGAh*HVCZWjWw3m>IQW2<`adPE1L@4gYaR%O3 zAXX@q6O@B8J1>Z>RTD(5777!mGibEx_*_k{W=3ZAB*h4`4W`=48474An1&u!O~Gyu z*pvyDVQi{|{cP+_vvOcYbtp7CboY5HrZ|9QP8X~yTnSo(K@A#)JU_2ualnTqEELas zEm5PDmpc>U;E&Y?4Qwklnt(m7K?P8JQU7#m@PkhWz`$@n+PboJ0k10`8-R&dMuOuQ zDE5b27m*_wkL_l2c4)m+FVDBiW^)Y&dG!vqM#mc+KFjW~7AT$Ns$tc=B~Vo?Gf#x0 zVG#%U*(vGilQORk1-~W`WFUz16>)0N#r5hq1q>r|nM;v9(&Vt(U7&KRU?Bp10_%>h zGP^SY)CHK#YzeV3W`|IviHkMx<|TswsmCT@cY^|aOW>6P+0S}V;!TbM5cc`y&fFNY zhzF9iWIvIqd4qxQ?PwX?#yx+mkA&nL3Bc+lBVWr zsYX@@jz;7fR%+k`heD%+QC)Tly!@(-kI61^!JQ%EeIaQWkI)UNg*Mv{a}BmVV@q_M z7JLz8*I>{8P+E5ho{bK;8#x+q1gbYihT~mbxx-{DQ9Q%tV$P(XBo16vw$VYKJYHXh=Z zxD;YOuQ(5MVza|;u7ED!vMV$?m6kFyC{5qTOB2fMWw4ZRBnLZ@bNI8oXeUhWPN&%h z@2Jj~o2CoxHn3Lg->Qb8uG4GuF$O4RPT@%O`|0(0z_tPi!ZI8Ff{@YaV-%U7yv2b$ zF@WHS*Q*VB!}5s|Sj5@wj`Ei$q4Zb{tZlTh@i8D4<3V|i*1#U`pF)`J;$xQ&HJZSy zVM!r&#akRvU%i|V^I^0v%&SEGP6WAZw#t%X)vy_6o5KtfJuEs+7L%gXo@cR|bBj;( zj{~;r)p)5r6TZ3kr3gduhR1_$!PC4@SZc!Fe9HIWu(2n&4||`&6vc&6Y|cgr!4FP; zgEP)tngFw>99HRIa4{#CY?X2wh}7V)1OrGbt7FIwqblutfh!!AZ*{s1J-`Mu zClL**Mr^CK?^)jDy(k=b0srwjeVbel*|814eNd&D3^|GsQeUzPWArMmUfDY!T%PMJ2gb+7PWfsI(!PYIXyQP=6`57~IGv)x zX@*r@sk7Wxlyqe}is%_SEZ^d)Tz+{3ny7`|WXQA30ye<^LTh$mY4#-TSfX2ZNoHYQ zcE*fUOX2M7(WR-@!piK7(yU@mm+5JhqbE+bj2f4w&d8pnADumG_Ly;FV@A(N>3c2s zHCWoH$4;yq=a@9nUTDvpF)6lOGf9y@t{^Tu4WM!|0Y)?a@)a}<9N_XzR%^@1tjXw& z0#qr!KWGRnmQ|#A1(v@cWka)ci0deim@GlvN3b2`-%g$%h0Z?_a>1@7-M8#LLXQd5~z z)gOijtdJe~=7d@1Jb6A$QU=Y0thorJOrL`2VKWoJyNT51_6UN4&V%0pcIaykDd2|g z#m;jo!978oh%j(iaqk0>Hr~dXe;*psjMX%(puvtJ*4a*clB}m}bJ+aRre49yqJy2x z@{Uz+FYU^AD0MVONG%L2h4HbvHr*6<|DfF13meaZi?L87HPbdtm0WAwXisgX^p)zc zwAIp4tECU^jc(soTdmIs&B<9;K2=gN753Dsw&OiDyrI^GB@>cX)f;=KM~l=x)>gcS zMq69y5JZc|wOLyUDe4(Ba97sAdj~4s-Wvq#7l!%xy*PvhL?ToI6o9=$M2S8qIGIMF zLx9rP?fTt%sfCGCGK6S%MG!7kt=upoVH>)oSC8Fu= z1qOY*b0(nZ%S8ix=;I`sJ{^MS9wXa$2lQdLu#AwfLhz{%#&!v?pFCEAv*Vo*sD@T# z@PWZuXc#&K(Y>a(!301PrL7)$*P&8rKq6a$Lf8N{S(NG7n^BPc6I}6u^#}|)O0Bkw zCeO=J2P+zo@AfU6O@Y4q?|zAqZV@#gt=$)T09Pzi@rO0WDNdHhA)_5 zv$6o)SA2jCjg7NO*fS@n~5= zJX+E}5*^sohmILbf<=c{&6w|L=Vdd_*7a!nl#aHQ{gEjqyJyfWNN+&L&~wOt=7a2%_74 z4M6mDP3sqtwl`zrfs7BsBR~v-YOy9fZS|Vs4TcWq42+1;0Z}iC^XZm2njW}x2%>uz z;2@kk1ewm+mYIXXkdR1694ro@kM8Rt+8&q=TlQFPd>C+*+znSk8tAE5hsCzZwyj8P zw%G69gI%w%s~I+;G(Z}-f4L83!yDvTJM5tbW=pIJA8lR88H~@=9f1P^V^9;PYqHjB zM_CGc%&k>~K!V-VV8u;{PPUspfm&%L_Ot|ZGPY~k>`pk_r;dw@87%LwQO9cr%g0b< zB&K!Wl?;AtU?N_DJ>^KBuzhacEP#HMR5ufNuN4kShcI=e)AnzWodw&>_ zjrMZl<5@&b7_KFpmrEj~Q?O?odD;yM)_@P`G?Gekwd@{PULWKXvk$oY+)qe_*jH^q z#C>qAqk%_^kdh}28(SB4BZA}LqzX_w3O;anT^&R882TUzhqWA>$$}Cb30@JpA3M3d zK+FXX=t)8-c5?_3p9tsqfXk%07aJm47eHf=5{>q1Y#3SD9dMd#B+P5lzzkHH$gfO> z12Yb`aU62BTbjvn9 zO?ON>1kv{vY$PDD(j7%*z<`oVSWikdJabDLYTQ~ z;QM{>{SpiftOx!MacuY=0dZ9jhU$$LWqCD)gpIS%)uTgDy?fUC@UkE9vanDZ7ywKA zS~XX~_H$f9E#H$=>Ld|&OB9&lW5sThj8vpNW+o{t9?ttBS7(zvS;8q{TN5I zGay|wkrYDbSkdz9=W(;6#jjr+OK{m;?+5q8d%$Jz)$!n#I6=VUa>tDNj1$TJaXExJ zD=4=|6a_gJJ`5g=;F*Y?f#*Aj;I0PO$5$?ao8e>~{P{o+e|nFCH~K+_f$*r|!S0OM z|8d~G;LbP*0;TYPuOfSAI=C}7!j~0ez>5()7uiF8gD=A&c?>)`ZCM?VaOr1ww;CC1`vO1Pd1!+g;}EM&RSO1xT{wl(tg0} z$C7@nIp#_4*O(^JtM3J-$cHYDTy3^&_uz2b=qD~QmiaLf9SI9eaY zvl<Wh4bRQ}<->9nP}hR`n$+VdH4J>~si<#U$Fv%=q6;?i?Zd z1iWO!NTE~)>>#cdMQpPE@yx(pC4-*&X2RFUyO(C%AGma^J$e3+!lMTakLD+tss^a^ zkyk7DcfYy%^7yFH#K4oA-aKECxT(tvJAztNBTjGszCyum$go`cnN>ccej0cFQ-Q-j z$+A8(@9f}tFFu zAA77XFEI$#YNyVgJ&V_hFpPc?h%R%}MFO&#OXY!J(P^}> zCFI$!gAHzQfg&rw`Dphngg!3ZjVMpMgzgvF>Ig*_WQ^#-`y;9g8s0rT6v_-1_o#k6 z5gN`>M5Xc+Jg8Imdz3{4ZyYjmM#B-OK&a#}lmwNU5Uif-@9O|2Y%qGKrNBp=QXQd- zS@H5Y#B=eMAuVSPFHi_hdum1SL3Vs9r&k%d`3`Oi*h`Y~sY}Sn$sPw;!1ISQkPpSt zaR&0-yzxAKR5xB=ln9EG!ubgfWEpsYyy0m@yzr<%Vl_H{{1~AGoYBa@vGNu{P6uxQ z?=N<|f)l}MLGe*fro*nKk5>#ZE0$m%qa0&0EZ|QCo^URJwvhK8U~dfAe!>m_5!x87 z0el_d6TA;O0+h!QjPv-Y;kbCnU6hK!6d!d(B1cK^gqOJG%x4Gx289psWcA{AF zuaWFJapZTZa}6iIQ(X$0hssPv!e3xgj8|srdyFSrGMPk0QZ#`+AnHBw@Ol#Yog&+o z|7OkF-kV8@E1BX^W_qIln64JVv~)UfRcQ$xQ|DNsFThQl(r4<4tItDsL~DjU8}!X* zf5yJK{rP@r=U>#vTo}{y@X2CDY~vubaLFPxa7tFfitok+tc}azvNo=dNc_$n9rs3N zmy)yJ>pxB3J>a2ls{(f~{F7L$4Ia#Xfx|N0J$>-!v-SP|e*6BOM>ekSd(Ge+cJIh1 z0w#-}jQH#Q9s^EX?`JsnX0YLxSxfH@J~LD@?Y?up)31%`9p0Q4Q=jPU6Myg5yp+}* z(m!f4g13(t9Kq+GP!6eH)N51Kez+Hksjjdw06kUEA*kDbpq~8io*e$4DqWCB;5i4y z5n|OW2KtAulRpf4B1S^QK%&3O1IM0@dmVcUPkozGjX&Q9s`%rLpo;$q>hp*rpvs$0 zL)6D`*oZygIQ*%^XOEu!8vcC$2>e<13;a3rEBq;51b-qPfOWvn!p}If(J#SV?jJoa z!tqmlq^V~pkP_37R z$zD5*H<5il5nkGu3x(SbY4z~zhv!Ggqnr+T`j{XEuO9Hqp&DXO=K;Cz;meFdSRYhC ztR%P=g~6vBo&oSA!7~yvoP{ZvJ?K1;M_<6Xrww0F+&J3y37qx!E|hRbj_@oLpqAv4 z0nh!+Sm3_5dtj%Zd-7Qj0vKsm4uelEJhAXV>HUvQ;AmQE&>@KSIW_@$e&iX0_RN9O zz5$6Sj1XW!%GlSyYZKf90my&D0VmNwbO@r`Hyl_zL?k}EbNt$!eOUm)x-kfr?j3;O zXc~kL`GZtt5yh6AzC=4nrYI|26iO;tk%^F-+^ypS0q#8rr4quCV6u;lMhN!skQ-Dy zTzljg#Q6s$yErY@lmS~jE0wg zcy9ntZ`kzTwnyN*u_9s9y8_S$y+ZIt`<5v_1_R;Z02BgV(-nwgfDno7Z~mQ@oc2=u zSGv(XYr5oA1tgT@#jMZFzrH)?lP?Nc!-pNQpWa;g_k+Sua*kdsaD1K0PTAV?uSwa_ zt#kXvj@0%GzIzw2$lv^nlO=4Nhzm%EpaB2g{Ocp@~n`_LKBJ9R^ND1_~XF+B@?q$WAj%eYX zAHIZDtnneHMeE4T86U1e4m>b?8v%;b={prSN2LZLJsfe-s1SVl$EL?Q8I}0INsb}e zv1pqB&vba&WRbr?2q)2%r$Z3kZV&=fUqH1Q_FV&@6IikA*})RkgKj*E6oqSutB*i- z6bfe658(4@c)kbuo&}$&@U*~l5}q~i?0^T$r|s}yh5Au=PQw!j`8U8*2j92A#?uO5u5E%;J$pHvqY?#K9 zo(m5!V-Y?v?86KUN;88FLH^-!7!mBe9Lx-^8WBVTH!?7+3XKe^I*}M4sOloEO=QKl z1SF0^JJ{z4bxVL~B2orQ7xMKVR4u$hZkoFZ(R;uF*xQGVsp6}+H9;!?84k}^P+C5O zF&8k_dib0NpSWff@E!ZBKz}1oSVE9IC;T>i35KVjCw0mbZ->{zQw9&_4mi`0B7~~# zBH!BLm+0EkA$VuFXWN&oE#eS|i9jhwQic+tyH&vz7lFOYjuF3B1spS?t?cQnb5*#+ zUqDfV2GpNpZ6YiCKQ3!-C@@L8q338j6qNkO6jiuikpFzmQOsC~`S+6WhNTmTLOd&x zE)vYg(Ide=$D9lTAYOw1+!j35(sX;{T2Z0fB6q{u+`QHQm%W-L8szcoh?;`A0!k|~ z!7;ruI3M9CF}q;9RxrE=mKZRhc*zT>M>oHGq!opBXo5XJ_=jK@upf5hBNcx(IrjOW z6;s0*AA|(bzI%*7nvD}kdOmxKx`4Jl#W`@}fv4OFSBGG7xZsXiE=O(!oRPq%FI<)V zAZm`u0mpM);I9p)NjOOD$~ESU8#sT(oxm=-TbwGuC0%@go_78c@$G> zv6VXkkD+-5Qb{CDTJ<_iP&=rk_a4PM%1=-T#7H+{yl>xjsZ2*?h5ngt0B{H0ma!{= zr8A>R|8X~{<0jt17VQKhLo(T8Raqb}H^p|x{xhD*Hqi3%tx*bzgwDi|qmG+KXeWkD{U7Xte zw5s=t=$D^B68FX|@frSs^|b?kJb7l|!}d4eTB-{>pN+2+)zlGVryqGs)8+9C%2k^W zjhgz(iEpRvXgHlQX>dzZx2@fFHVL1cQ?ELV24vMfc6HE2%df-tbF%Kg|D&HDFIxKU z`#UO$H^0wLS8Ip8Js@B5dh7z})sd6Z_b(c}&|3HIE7PLOp8nJ2OrC!E)BOB-mm}v* z8Z~CZc}G=x{;Al%?_ND(uyF8X{id`J(rRCNzI54kW&OvGMy~uob2)d#z2DYf81vXW z$2LE&Jzty&TxRY-dYB;LddP~wu)vIkR=Sai?+$!{s{%z2f$zz5veBvnetLj-W3KL+ zJ5RZEMM#kS!goKI|Evl=*Z;y4$wWo*3x5VqlvW(8Al^8(b4AGp5gcqGnv+8%*F4muE9rR4@!R8NkKU`q^-EgsC)S{TKQD^S! zQoH$A`}Wc;*^90W?G`jj(kJbNbXn=LhkoE4G(5O5Dr`*1McE&(Tkd{w=#{solnt5k zPTtuU-Wn<^Kl#TOPdMYY4@o@yx@Wno&*rCle^*mfSTHPV#LV`PI|ES>4WkgkNyCiglT>9 zFjd0!unl(-W-N3PZ|f5TJNIzv&E?L&nYephpAmy3f%q6{sY!Git6>o=NtX%uxN%xC z60sl@G|E(AO0k*ptmX)5`g1*mc>N*bLkee{#RZWl9cvh5ULIWfTabdUUG~a~N%F|Z zah6gu1wn zrEy|lrpW<^i9Aub2wl@MffI0qvBc9`1>+oWc};#cdANqrO$6EGxxf@#5gcstNyPws zDmAm%0mqenV2lM79lS%8z-e^2EC6Y;dQ}D|b*FAvphj9p9V)~-Lx){TK)sB|HC*ZF zEg$eO?M9_~4SfvTBimxd*KyzzujAoBB=lyDU&%ViqP=+*9VII|cv!d4-Uv4@Nw;2HtG2I*#7k&ocRwd^+B0ggvzN4m*YR1O`(Q(2rQXiFnG=cx2f3eJXWb4=E@ z@$H?$r&5ghSWuTPG;nY%73K1WQM%2E63a+1}w`_%#C?tkS&W zdVstyv)i2JY<#BKI|ZgADka62YS&aVOzhq{MO0BKanvrVyx5JIAoR|JhmNOCxH|-q zJuVa9K1KCxJ{d6v-^2tGtW8#9GQng*Qris(o)pZNZaygtxz__XC%&d4n>sFx+tu?r zCLH2PPcoIlby!6WPbe?-j_nGu9&GRU zARJGvp}nKKK=e4X!-qtdRJe;`oT1@xs1SaS zjo(4ylSHquUe_&h6eeQ#ia~+GUhSHlhy|(Qo!!KZ-Q+*<<=lVsGd{hd!88HA6faqY z@G&GRu$LSxouC6^C{`Pkm)c07?IpyGg#3qY^5fmY{$X)@6Y~5v@|5mLQNBJ~{NcUw z9oh2h!Z5Ie{QtM!;-8|yM;88{xb>FNVZ%CnfK>l|d8T~xOsEU)svX+v zZ?P3%u}qPDnGnB8$gex(haEr`Tb^mMbm&&gO|Q5q>K_u~3xxc%Oa71xsN+32nmQf2 z8FkM*fS2tQ-Qz>WZ}yYdt|jE(l;dgOf9NtY26M&FmHSE`Pfj`s=QDtP)JZ2+Mvv5ycRCgi(c10nh<0j-I}CEc2I=w_|yq0h9V zP&n@p;*SaWwby}JuVWngk^($z>Cmm>_?u(AQQmulkgt0KqtKTVU=%uZt0=za9SY+V zA%2IDAAbupp*JxGeH8)5phLHc;cX@IMp)NM$X|A21o{dBj6jEO6~W)?M4_A`#9t8d zuTB&44W}^*ef&TFkY1*ZD15i10g;}$hW=+ z6?zw=MEG=LI&`ZjUZyk(-~b_RCgd01C*(Wd#{hWejUFd-=wAY0YZwJ^nh<|T$m`D# z@-IHX0Q6M@m``-*W&j>pfRC zrbuikr9gJD#Puxs`g4SQ{ihh@hTee;F8L~w9_D+2&@D`9Tx($JJb9j+FT5`bU=S9s zC)v+jmiZV=sp3CHh@XSLbCHl&UBtzw#Wv;<9lBK>3H|M^6wFpwa1!#5KO^MNebyN; z?dDsGe48$RiIBf{36qC@Fq z3@{4o+sQo$eui)gIirasBAf9Uy5bq~{3nK0?0YA(s4;B^c>W`(Foj()rYBDb#5x zMxqZ^<2uoyJE)VMLZYU*#mk{i%Pv0)@SbTcH6@bcaYqY$Y6_s$BI{Q7bhUYOqK z#5Gg+oa}uA;p4ThMuAYuz+WrD60;IlgFbVOt3iiuRt-;gYul&u=FygiS@Pdj-3gC8 z#J#rryaAe5v*hbmcLsp>uAw)=$B(e&`yROy1dkB#Y(7%dPX&nI43N9mvE--M0QC+I zP(v$p5PP%MXzx^Bo0}Bkz8>No!Sd~OEcur8WT*Q7{OMTc==iE{$D;#p`BGfE#unFd zA*?5ifI_0I!^aMR>~6r!iS<#=>CmkXfdKGApeP?F#9*gf-NcfA*oZ098ZI7Obm&$o zw>dL-6aV-yOa9bhOq^atVB&P>R*BO-5fuG3Eb#`G{NrXASVu5@`p_e$Pls-qe!FW0 zMW0%Dojb;o*B-<4Z`Xw35ew}e8WeeI@woLx&?#CldHO&iENu=QnFXPoE^0AG(3Z;WHFfGz2jJHhY>27kvE3S|W$ zUPs8Uo@B`noFpYg`x!>YU={BU@D&ms%6@J!6yeo`cmr4)USr8my-E_M4>U3<|8xV& zzFsvHVQdju@j6Jf*GRJe$|B<9i$jxJbP9UP>$gtM)30d{9*P##@2&y0)2DEg(F!>3 zcXa5M`yK5Fd5hlLORYCpJL11@37=mr!f+JJ`pzu9lBK{rXv-Fb3H)3h%Nu>1L*4SVI2DC zAI70Ww~FKKm_;GH6CnODK>qT_pr?I^5$NM~7=aGmDniGOUKAG9iBEq5`pzd9i|)4n zt`mEt1H)g8g2BW1>ZhRfe0uBS&ZCuhd6`kfD0hb|;3v?04$6G%NAVb<$X6rr)WOG# zjV}Ll7g+K|7jC`$9&*0EZxp>MLR<@$q>C`6P!#C{dyGPgKVbVxVvj)bcb%h1Q!Tpl zGnRb8XPwca9eLou4x9@XUF?{~-$uWEc zX_7m?0QP?I?@7|0g%qtX$SvVFzXX++qSnDFL1u?|%FoBi)6-}VMT$B#D=+#QMAcVh zSv>~@nItoPyO@Wfw|65&vwEnM)s_KsG;c>g9C^iz}gV?3bIh8_tlPms&D?e5hAYXS4qcM+AVKip!&CopTBb{F8gnj80 zTFll0@@L*?+El@E@lT9FWWREoB_ED6kYy+@Evsx#u;*B|r=H_E zyx0L^+p$KR(ushcbAn7j@03miqz+&*;pliK*ll;3wq_y^crqdVQ<=cGJwV1}GvTPX zP6TN``pCrcN$O0dN}A?Pw$jY1Z}BN1ZYL9&j_+JNuM0Prpni#d$k3;6TOIJd)q{!5OlN}Mwld)B zop9$2{g+eZMKyE z=u|jTt;g4obp+{?*om+{SDP_`8Ofap?YTIOiRzuwiGZFP*_eRdDSm)=I7)mkd}AUr z)BW&a?A*3+0MBqHVp}8==C(%)xQR1i+9Ep;Ta#g$a1u>K=XDAQRFztrJ0Me{?3OZ(1jU`rMkx#APHip>KOU`rXOM z1n(e&AN(CkJ>Q!jZ&(0w``J;6(>oX6=XyydJR|v5(K~eiVYs7E2r-J6PA(OiO`-qNxg0>WftHD znf`-fv|o*S<3jjk>BJ&<-NMQQ_e<&MgC6!b}log8L+OIx2O4bcd*PQm3POx?6|1-ie)$Ox=LpF)*Fh>A3ic?G8ab z37w1Bc7Zn&(JP^I5k1#~Ga&*xNP%ffrxLkX{L$2#0S^ zXI9S>%LMvY`agNR1Ac1_evN=Ubz!ySIZUv8ghLl)l_vOUfLRzCejfw>E(VU(YLsfF zx*r^e@T@Pr6O}rprr$gy;TyBE#!MJQ{g993+)IQC|BD+w%4jr>g$Mt;*J#xLpZ2b0 zJ8`5**JI3_GmGBW`A>mtR~5??lDM=HAcFwy5LAIdV7jomJ|+=|3ot%;LVNRDZ4tc$75}zxxSO!*KZnc4WAo7rf9&hOhP}IaZT2*&E$h>8F^+1F z+evM4Hw}ATYV$Jg?bi2Y@i@BirlVyWpNXx4H=}axYt6E^jWYTy-9pe8^!@IqUs%`5 z^;u}_iec~WC(RM`r_pGS_Rcf#IUPS(L{V7reufL@Ie0+3%3U$6Y-ybB0R0Dj_vg48 zZRL06dNEu?7~f-ji(%iq>)&{HqYB|I9z&!4SI~0{@p_8UxZ+?8Z%S*$x}$L6Zh{Kv zLO7M{zHbkno8@}x2=@cE@7=5)6THgxKE|01?Z}SD4pu>LH}4y~Ch_PSPGMk7>t4C0 z!+#Yn+lCu|4;mM@Km1bRN8gO(Ud*GHu{FrWJzVkj)K&$k%%{B_`zfI~4}YvirQ6@~NC z1P^-CQ87h}rT;SQ(Oz{QiSIDS^Qao7dEqdw)>q+T6o23I3xl3F9arpxe}6t~Y0qEh zoR2r)9cR4oU&iqsoZ@xbCXWwkA1_I(7$?VDk^IqmPvoLpKWQ_nIFN7Wkmi5LT)~~S zbu_D|`0qJ@T%dnpkjUMxbYCXD@ygR)KZwGmXM^{({B_bMZ$F`nUU?fbek8XKX}sUH zm+oeaK0&|E@|LrgCv9|ld1##F`^Nq^_}&UD&}mDr-qrG$yM{cxjP0l`$JNq{=gp^A z<4}7)???40FrrO3*XytD!e0)1rI&|p%s)E*O6EP4N#ZZ2<-7RH@4M0H3{IKvwRB>g znNO!g9)6Fnu#e6GXBX+qbG(&qr{k|n>WWrZi&Lp{o}XBM=hLat8T`)&8uanI^Q}&YCg0`pHuTAAV3g+Sj66L9Ve5>@ri$tRF8+p0EkLJe6zi8$J|ekX z`1_zA%ks7Gdg!|kK`+3(?o+JWa_olVHd^_+BwsN->K2_jfl=E3=$_SCF@5JQ&VGLx z-qi2j)L+c!`TC1x_4oGoefl-V^|-B;u9wvR{zJz;rBRNZ0U6ny(q4<>CHEdUI&Ys- zzF%+r6K56XJdkfQuHYPv#;c&}X=O*-_9}FQzA;McaUb@iaaRj^(Q;Ug&-9%Aet-@j z|F9s>{))c}jA`tHTpw3E=$nP1&TfL-q}G*gdyR9YLiWJWE>4GwlClR+b*8u2O-IJm z9rRWkrSq?e8~mZSaf6TWkKv~5?9+2_9<1T(KXrQ1OL$X@>6^!YAL19@MdLU2wX^UG zou2a-aFe!0-Z@RaS7Jk5;B1NKl=RTqGfnt#?v?k`9~u6>-5ZmSljif&Jx$wxY5L~v ztMm^qdX~J}r*`qIT*N+xvkb6*ivP5G{1G-{+bC^@Mijjj$H{GX4O-4u)A7PN3~uVH zQPp!06MGC*Tqfr{kuj-};qHgf<(V~I1=r}%>6IRby)*H!zUv}RxZ*WjMV1v#-O6+m zF0?Tog_qwRR!T2djMZi5?G_Pt^Vd4Rfax2r#YbN+3ga_7@2q<>W{Vi}9y)9p&zFta zYGRA+K_7AJenY1hGvDX!??=e-gKv^PzJU+&i+zwB`KWhooHy0+X0&+b*i|0gD8|UK zlv7_+#){cLco)sOqlE(v)$ZzCE*jvQe z{TIZmUXr;Nr?ytt@Ta4N`ve_>xYb?6G|$`bOP|d=H)LQByG0ut&25)Q#}Bb3h!})9T6!zU$ZXKl#%$`Ng0qLuMZ7ybqh~H1lkbCP=Hbnl z>MfGbunQT*FR~iqvs`afgRX924aoC z0sHO*(P-(y|G(Yo&hf4ht&t1cBVU2|fY(Fc1H}Lu*UJ&=aMt?QTAbtfs*cW=u|p0*{{P{<(l7*v&voTaK0p++XMKRZDWf112@v^F>{I>gf*-< zd-$dLxYdW8uauwm6S>jtKm4|N;3x8^^)021!a2SC_yP08yJFKgUmFfZoUoEh2Y)6hYD z|GIu+jp>ntSMr^9|1K+wEz~(##5c%OY0o$C*J-;e@qb*$aZWB{yXKlQ*@a%GnxQvn z7W@0i@1M(~O8t7l+#hrqYB=H=lZQ%F(ANrh8^lWEy|<^UaW#%xX!-7fIQm9Z7}_{P zjfpN+(np=v(*xx;^s;$r->QZ=E#%TT50PW@gHE7rSW)?mJQ?_qGaYA>;jC3nlk`n= z=Vzxa6Ib%{BHAH7UgnbiKZI4msekS1KSbc*WW|j+Ml9k(&BU zju$m$jVt+r+P&nst)>o(?T&ZVtChBhOQ44CEN-n)1#!TB{mQsaSBULzM$4!H9GByY z_6)*#;1;NMr$cz&sJaR*{W#8~p&y^0@1kFQj~E~23LB_5Uyggo18U6iT~pIgFJucxXm@r zIOI280$I17)J2}!a>(f*)=ah(V$N~Bz7{vuefj?BZ^b`6`>sRrTG;(6w|E-oI%%WT zIhWMguu~Ry;L|Kka`qH9^4$yFXUVMN8aPs|)r|?i#f`f14$zypQXSbA_#Oh^Eg*;Y zkK&Ac);vSAytpjcKk(t8Pf^p5UEfvs6E#J+FO-ps93H43r#u^GWh_tcTsmLOdYSXZ ziGDs4Q+czGmacqe=gU3EmERSY>Kdilu;;AtcMr5B-bd68 zs2E2Zak81#_J~a6x}E1P?mWE}cf4ebPvM3bo_F@1MsB-2J@p~BMogPF>KaD4+cmhE z>uilUPSkaaqouqHcx9XD7|%!_&sF>Vx~8P9PWD+wE|Hs+{sP_o96Pw5vqUY3;lKJ0 zIl8oSW1=3A^uE`DEo~rIA9dn8ox1ymA7LNA;*DB&+%cOX7n%F(T7+vn-3kC6Wq z?abiJ{nFfa*OIZh^fM2*Pg`KUdgdH>r7P>zb2UUCq7(dX$5%RC=OaZYZSrL6Blr~e zj*D*4!yVXOL8q%u+FXktecyB2hVzX1kl(pIho6O52WS>8Lw)}Jqv%Jrae-hJ){_=0J#M+sFGHYX{$r&8`sE7O=1hKA8AXpWs7tga<*yJ;AMZ2aOsw z#(OdwLR2Pz#S_8bOg#5-FCNhT*2efPf?Hb;?S+l5k0zcCaRgv7A^0mj2y#!ahL{C> zBiizTkTzh;wlLmId`J6s3tx>0{*Jdv#Qmg+_TMdx-zHdW0b5-Qy_TYY_FGrB2Yib+ zNW`CQAZgAmd{`&A*}lhm=Vkml7Hf3f0v0ub3*w2^i#JfrIlX=25-i$)ty&r5MPg3l z(|aisg1^$+CifF8()?b=haG~=Enw@(#CU5l0+Ae(2@)odl zOQhWikh-=7sm(6IRf3!J)=GIoq!|#Y$`ioWoR;!SU}@DhmWFQuTLin92Jfv9Dg%0c zrMv}f6{zcuhN=^l%{jp?!GeaifS3>~`cc6^L^}MTIYrs}g-2rWi(RJHF;mHo1mO<` z5~kFB5;VEJpxp<(7}oWtsZF}qDfO7JxxF9~fF2BUe~F(&g}x>#qd(mjj^JPIrOp9A zeZ|u@@@tR~i(uFkQ~9Y}uUwx5iBLjNU#}qbfc*5Y=H|g$bNgZk0w@)=9^Q0YOdA?#w9@hq#wld!hv1dEk<~!rWg|3qWM05 ze*<5V8Hq33=LFZ9H|RRc;wvXf@t0h*nQODt7{cj1gTAA0KQr_K_h;w@!GL_e`hn7y zq0dB(`j;9<{UPX?{?KWe{;#yd`w*baE%l$!dQ$Y0i%!vxbfmD;zKXveQ(UDh^JC&0 z_5UW&g0PYUnqPU*oN$B&R{+mpiJz(zg|cnSUhdCH}Mq z9R2j;b-bf^xTsj!@&06($CIPHjz?!M^#5@#fLbk@P{M?xAZtJfgJB)L2vyr_c8gR? zkDLwlZ5*=rh{HBRFR66X@KK)Mp%~tiid`tdK`oHF&+vb}3yGj3;|4;OT+|QFdJK-<;(=XB*6!Ca~bo(LC zJE^~M#>?0p#vbteioc#1*729rH<@|~*h8Hi>l=x`%dpPgph3vqP_-bm2H1y05BaO) zFH!prt_7@HkL0iJqV$M*UDQ6f2kQHU@mKn#)w);vqtP#tzIeL|zBZKdAZs1=GsE1UCQZwm)F+6mPQ;AyRrntmR{KFdMB1Jp zn-!kMGH91?6}=QcSPV1$B+@DSr%5B_)VRSOVSLs3VOcP0q<&TQ7fqVRU+(8?&7Yj9 z-2D#K)o(#askx4yXe8XF8ds*V?J8lOPtmW=@LBz7Q@jYZ_*6GRxJZ6@;A)iaPhWo>nxuz3N{+6f*kv#6BS;O!P0U166;D zx>ur%sJA*9{TVQPmOt=jA-1xV(cfi;b@nm&bg=g_HQ=H|^L;zQUpmQYevs?O&566! z2ytQYC%0Gp)MS|Hcg#)LS#fuW>x#Wd8_%bH{u_j{OkGp5?}8=tDp0ud%;q7rB4x&N}EN`_u5T z{S}V~h}V<5|9-VTMV@lTK0NuH@Skrp``=*rJbg3o48VV;^+=Kh^1WeLr!VAsa^F9ZsJw80HQr~2b^1b% zhx#w$c@^-RyprFnZFYTP`&rqyDn6CZFY^7&%-c0E0%l(kZ<%m^6_0yIfBC)<_?h~r z^o<-Zn)V ztF(PYw3hLM)SNI$lKF7a!Q!n*y+Uq%hS@zstjaP1Qd{U$Dh`!nJ zl5^k->leA-CVZ{Td3Q{2slSij@`L}c=2!N?m|>kfk?hdD!tC3IM89FlP?)BtdEMwN1|>Ccey00Uo!g8>?&BN&n2rS>zkAd)c$7+M|Vn|a{Wi5udoKx z{wZmH#$R?)@4gbI^^RJf;(wQ6KEICWCsPGxs|&ogJfGrE(c*ZihCgY(#3!o9Z}Zan zkpIcjOWb~m^%L_$?)k^>4`YBQ)gDYjxo;aRjLp`SL)8qBXqdxuC zH=*{K@uz>z0Dqd0`Iw*1@JISB4O&qdGOyMn7g%BV?0Tvp?$xV1s~E(bDE!&pVwmOO zcs&wf#Ec7g2aHe$z1p$;(|BwCCz>zw_CEMaWap>B+LS1)P-=;CqcnK3DH?qyk5*@l zw`|=f+9&i!@Ml7~&DeCMabi=3eiNVm Date: Tue, 5 May 2026 12:29:52 -0400 Subject: [PATCH 046/115] Update Netty to 4.2.13.Final (#21490) Signed-off-by: Andriy Redko --- gradle/libs.versions.toml | 2 +- libs/netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 | 1 - libs/netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 | 1 + libs/netty4/licenses/netty-common-4.2.12.Final.jar.sha1 | 1 - libs/netty4/licenses/netty-common-4.2.13.Final.jar.sha1 | 1 + libs/netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 | 1 - libs/netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 | 1 + libs/netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 | 1 - libs/netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-buffer-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.2.13.Final.jar.sha1 | 1 + .../transport-netty4/licenses/netty-codec-4.2.12.Final.jar.sha1 | 1 - .../transport-netty4/licenses/netty-codec-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-base-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-base-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-compression-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-compression-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http3-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http3-4.2.13.Final.jar.sha1 | 1 + ...netty-codec-native-quic-4.2.12.Final-linux-aarch_64.jar.sha1 | 1 - .../netty-codec-native-quic-4.2.12.Final-linux-x86_64.jar.sha1 | 1 - .../netty-codec-native-quic-4.2.12.Final-osx-aarch_64.jar.sha1 | 1 - .../netty-codec-native-quic-4.2.12.Final-osx-x86_64.jar.sha1 | 1 - ...netty-codec-native-quic-4.2.12.Final-windows-x86_64.jar.sha1 | 1 - .../licenses/netty-codec-native-quic-4.2.12.Final.jar.sha1 | 1 - ...netty-codec-native-quic-4.2.13.Final-linux-aarch_64.jar.sha1 | 1 + .../netty-codec-native-quic-4.2.13.Final-linux-x86_64.jar.sha1 | 1 + .../netty-codec-native-quic-4.2.13.Final-osx-aarch_64.jar.sha1 | 1 + .../netty-codec-native-quic-4.2.13.Final-osx-x86_64.jar.sha1 | 1 + ...netty-codec-native-quic-4.2.13.Final-windows-x86_64.jar.sha1 | 1 + .../licenses/netty-codec-native-quic-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-common-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-common-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.2.13.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.2.12.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-buffer-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.2.13.Final.jar.sha1 | 1 + .../arrow-flight-rpc/licenses/netty-codec-4.2.12.Final.jar.sha1 | 1 - .../arrow-flight-rpc/licenses/netty-codec-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-base-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-base-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-compression-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-compression-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-common-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-common-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.2.13.Final.jar.sha1 | 1 + .../netty-transport-classes-epoll-4.2.12.Final.jar.sha1 | 1 - .../netty-transport-classes-epoll-4.2.13.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.2.12.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-buffer-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-base-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-base-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-compression-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-compression-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-common-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-common-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.2.13.Final.jar.sha1 | 1 + .../netty-transport-classes-epoll-4.2.12.Final.jar.sha1 | 1 - .../netty-transport-classes-epoll-4.2.13.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.2.12.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-base-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-base-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-socks-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-socks-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-handler-proxy-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-handler-proxy-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.2.13.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.2.12.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.2.13.Final.jar.sha1 | 1 + .../repository-hdfs/licenses/netty-all-4.2.12.Final.jar.sha1 | 1 - .../repository-hdfs/licenses/netty-all-4.2.13.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-buffer-4.2.12.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-buffer-4.2.13.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-codec-4.2.12.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-codec-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-base-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-base-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-compression-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-compression-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.2.13.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-common-4.2.12.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-common-4.2.13.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-handler-4.2.12.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-handler-4.2.13.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-resolver-4.2.12.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-resolver-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.2.13.Final.jar.sha1 | 1 + .../netty-transport-classes-epoll-4.2.12.Final.jar.sha1 | 1 - .../netty-transport-classes-epoll-4.2.13.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.2.12.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-buffer-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-base-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-base-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-compression-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-compression-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-codec-http3-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-codec-http3-4.2.13.Final.jar.sha1 | 1 + ...netty-codec-native-quic-4.2.12.Final-linux-aarch_64.jar.sha1 | 1 - .../netty-codec-native-quic-4.2.12.Final-linux-x86_64.jar.sha1 | 1 - .../netty-codec-native-quic-4.2.12.Final-osx-aarch_64.jar.sha1 | 1 - .../netty-codec-native-quic-4.2.12.Final-osx-x86_64.jar.sha1 | 1 - ...netty-codec-native-quic-4.2.12.Final-windows-x86_64.jar.sha1 | 1 - .../licenses/netty-codec-native-quic-4.2.12.Final.jar.sha1 | 1 - ...netty-codec-native-quic-4.2.13.Final-linux-aarch_64.jar.sha1 | 1 + .../netty-codec-native-quic-4.2.13.Final-linux-x86_64.jar.sha1 | 1 + .../netty-codec-native-quic-4.2.13.Final-osx-aarch_64.jar.sha1 | 1 + .../netty-codec-native-quic-4.2.13.Final-osx-x86_64.jar.sha1 | 1 + ...netty-codec-native-quic-4.2.13.Final-windows-x86_64.jar.sha1 | 1 + .../licenses/netty-codec-native-quic-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-common-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-common-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.2.13.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.2.12.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.2.13.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.2.12.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.2.13.Final.jar.sha1 | 1 + test/framework/licenses/netty-pkitesting-4.2.12.Final.jar.sha1 | 1 - test/framework/licenses/netty-pkitesting-4.2.13.Final.jar.sha1 | 1 + 181 files changed, 91 insertions(+), 91 deletions(-) delete mode 100644 libs/netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 create mode 100644 libs/netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 delete mode 100644 libs/netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 create mode 100644 libs/netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 delete mode 100644 libs/netty4/licenses/netty-common-4.2.12.Final.jar.sha1 create mode 100644 libs/netty4/licenses/netty-common-4.2.13.Final.jar.sha1 delete mode 100644 libs/netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 create mode 100644 libs/netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 delete mode 100644 libs/netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 create mode 100644 libs/netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-base-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-base-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http3-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http3-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-aarch_64.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-x86_64.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-aarch_64.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-x86_64.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-windows-x86_64.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-aarch_64.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-x86_64.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-aarch_64.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-x86_64.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-windows-x86_64.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-common-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-common-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-resolver-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-resolver-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-buffer-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-buffer-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-codec-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-codec-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-codec-base-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-codec-base-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-codec-http-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-codec-http-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-common-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-common-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-handler-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-handler-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-resolver-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-resolver-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-transport-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-transport-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-buffer-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-buffer-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-codec-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-codec-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-codec-base-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-codec-base-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-codec-http-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-codec-http-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-common-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-common-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-handler-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-handler-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-resolver-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-resolver-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-transport-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-transport-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-base-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-base-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/netty-all-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/netty-all-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-buffer-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-buffer-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-base-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-base-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-common-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-common-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-handler-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-handler-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-resolver-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-resolver-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-base-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-base-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http3-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http3-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-aarch_64.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-x86_64.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-aarch_64.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-x86_64.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-windows-x86_64.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-aarch_64.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-x86_64.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-aarch_64.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-x86_64.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-windows-x86_64.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-common-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-common-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 delete mode 100644 test/framework/licenses/netty-pkitesting-4.2.12.Final.jar.sha1 create mode 100644 test/framework/licenses/netty-pkitesting-4.2.13.Final.jar.sha1 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 2355ca5f790e1..9ecbab7250953 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -40,7 +40,7 @@ json_smart = "2.5.2" # when updating the JNA version, also update the version in buildSrc/build.gradle jna = "5.16.0" -netty = "4.2.12.Final" +netty = "4.2.13.Final" joda = "2.12.7" roaringbitmap = "1.3.0" diff --git a/libs/netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 b/libs/netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 deleted file mode 100644 index d8dc651e6d0a7..0000000000000 --- a/libs/netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a1b3a6a4ebaf546860eb119d4e462cd300976ae3 \ No newline at end of file diff --git a/libs/netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 b/libs/netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..723b9fac59b38 --- /dev/null +++ b/libs/netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +177025483d7565afaf4f820139d409bdc0cd7000 \ No newline at end of file diff --git a/libs/netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 b/libs/netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 deleted file mode 100644 index 97f442e1f3f2f..0000000000000 --- a/libs/netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7b393e85c2017ad4f63ac5cc8700babd28934061 \ No newline at end of file diff --git a/libs/netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 b/libs/netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..4063dcfc6685c --- /dev/null +++ b/libs/netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +9b96afed708b58c55ef4c0388f532b48d628d610 \ No newline at end of file diff --git a/libs/netty4/licenses/netty-common-4.2.12.Final.jar.sha1 b/libs/netty4/licenses/netty-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 631d78619a4a4..0000000000000 --- a/libs/netty4/licenses/netty-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d35ffb9bf5cc0e05ae7408cf6a682b62dceceafc \ No newline at end of file diff --git a/libs/netty4/licenses/netty-common-4.2.13.Final.jar.sha1 b/libs/netty4/licenses/netty-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..b1ac1fc1bde8b --- /dev/null +++ b/libs/netty4/licenses/netty-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +f91909ed1b9280cd46d8b0ee260ebff40e1c73d8 \ No newline at end of file diff --git a/libs/netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 b/libs/netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 deleted file mode 100644 index 818090d4302e4..0000000000000 --- a/libs/netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1ccb2b1eed54ce049b3ff39fde225014526ab6a0 \ No newline at end of file diff --git a/libs/netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 b/libs/netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..a3126bb594ff3 --- /dev/null +++ b/libs/netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +920eb7284d62152dfc5cb8ef0f9e0deb47ed5635 \ No newline at end of file diff --git a/libs/netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 b/libs/netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 deleted file mode 100644 index 1d881a45d3290..0000000000000 --- a/libs/netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e9d42074c3d96cf31ce57cc58f6de6f31959b7a8 \ No newline at end of file diff --git a/libs/netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 b/libs/netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..2ada67e7addc5 --- /dev/null +++ b/libs/netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +acec47f1ff71785e090e019920f787e0f7d164e3 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 deleted file mode 100644 index d8dc651e6d0a7..0000000000000 --- a/modules/transport-netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a1b3a6a4ebaf546860eb119d4e462cd300976ae3 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..723b9fac59b38 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +177025483d7565afaf4f820139d409bdc0cd7000 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.2.12.Final.jar.sha1 deleted file mode 100644 index b4a67ffb42f9c..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -067b917da20425d325081eb056883b47e1671430 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..a91736d0ee322 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +99829f1c0fdf0a3f6457bc4fda3325284f8dd47e \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-base-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-base-4.2.12.Final.jar.sha1 deleted file mode 100644 index 12a51f44a7e21..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-base-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -381b47a0cdd126010a7df1c25d25d7bf55c4fddb \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-base-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-base-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..34fbd28571f81 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-base-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +a4476639056149914d7a145ce0bb9f86bb7e3f49 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 deleted file mode 100644 index 97f442e1f3f2f..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7b393e85c2017ad4f63ac5cc8700babd28934061 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..4063dcfc6685c --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +9b96afed708b58c55ef4c0388f532b48d628d610 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 deleted file mode 100644 index 351c6d0feae23..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aa0849118167bc727a8dbdaeccc45d56c1f1e8fb \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..633b40ae21366 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +c2a1fc65daf1a3d5467db37b6e0ce42bbb5b98a8 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.2.12.Final.jar.sha1 deleted file mode 100644 index 1fee91860d10c..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8dbaa045acc60abf333d428dca4339ce36423bd0 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..2096dbd85d87f --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +196f0b6d0779a7a23be4a8bff362741ff0282ce8 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 deleted file mode 100644 index 8f3d42fde9be4..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -383b786cfc2549978390a2881ff3c146cc22bb54 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..51813d949a63b --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +5c8512afb15a0d26a3f1b7b43117aa5d26fac662 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http3-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http3-4.2.12.Final.jar.sha1 deleted file mode 100644 index 5c3d8f6f38f36..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http3-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4c1d110b95a00688f288bc93d11acb6dba3466ca \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http3-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http3-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..afd98f92f481c --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http3-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +9e9d253671a73eabfa84694ed7809b2a3fa42f23 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-aarch_64.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-aarch_64.jar.sha1 deleted file mode 100644 index 6e1ac36b3504c..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-aarch_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -37988fd1ec666656915fd418aded37a01bc65941 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-x86_64.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-x86_64.jar.sha1 deleted file mode 100644 index 69dabfba6fad9..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-x86_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -632cc4feab6a0583e5a879e05c59acb4bef5d8b0 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-aarch_64.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-aarch_64.jar.sha1 deleted file mode 100644 index 44fc97d71ec5b..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-aarch_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ca327d4c0132005fc0bcbe33c110c500083c0740 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-x86_64.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-x86_64.jar.sha1 deleted file mode 100644 index 83778fda79970..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-x86_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -54a84890c0a4ef4b44e5c3919b09f67e229d6233 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-windows-x86_64.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-windows-x86_64.jar.sha1 deleted file mode 100644 index 8f609358a06e0..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final-windows-x86_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e794e36f597a26879225ed839c2ee4687a1f21b7 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final.jar.sha1 deleted file mode 100644 index e7089a2298bea..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7faa5240eaa23383c469b61f2a67ee54013c0fb9 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-aarch_64.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-aarch_64.jar.sha1 new file mode 100644 index 0000000000000..b297b9c6196b0 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-aarch_64.jar.sha1 @@ -0,0 +1 @@ +9f67caefaa7a964b2b7248bbf3414d55c5cdd37b \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-x86_64.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-x86_64.jar.sha1 new file mode 100644 index 0000000000000..a18ef06cbd56f --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-x86_64.jar.sha1 @@ -0,0 +1 @@ +b2f6b62623f17796df2bd4ea1e50174dc9f1dc70 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-aarch_64.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-aarch_64.jar.sha1 new file mode 100644 index 0000000000000..9fa17e216328e --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-aarch_64.jar.sha1 @@ -0,0 +1 @@ +6658ea9d2d15b0dd1339ba323d39d3d22b26af40 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-x86_64.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-x86_64.jar.sha1 new file mode 100644 index 0000000000000..e2932daa0043b --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-x86_64.jar.sha1 @@ -0,0 +1 @@ +6cdc84558d0c09ab47c8a2c38817be89acffc2b5 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-windows-x86_64.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-windows-x86_64.jar.sha1 new file mode 100644 index 0000000000000..95a7e8b7c6047 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final-windows-x86_64.jar.sha1 @@ -0,0 +1 @@ +9baa6c4ceeb5c1b0824ca881ad37858ab77b1b7f \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..4e0c35f6d2c3a --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-native-quic-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +9854dd4789199e79af87f89c98a6d0f039ac0a93 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 631d78619a4a4..0000000000000 --- a/modules/transport-netty4/licenses/netty-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d35ffb9bf5cc0e05ae7408cf6a682b62dceceafc \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..b1ac1fc1bde8b --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +f91909ed1b9280cd46d8b0ee260ebff40e1c73d8 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 deleted file mode 100644 index 818090d4302e4..0000000000000 --- a/modules/transport-netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1ccb2b1eed54ce049b3ff39fde225014526ab6a0 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..a3126bb594ff3 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +920eb7284d62152dfc5cb8ef0f9e0deb47ed5635 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.2.12.Final.jar.sha1 deleted file mode 100644 index cbf4733c23b7a..0000000000000 --- a/modules/transport-netty4/licenses/netty-resolver-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c59aa586a12e62d80207a00f9cf18eedf69d1012 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..bb0791379b05d --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +c68d861f923020f82fea2c99d5921d8142b5c012 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 deleted file mode 100644 index 1d881a45d3290..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e9d42074c3d96cf31ce57cc58f6de6f31959b7a8 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..2ada67e7addc5 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +acec47f1ff71785e090e019920f787e0f7d164e3 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 59a45c78308ad..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8f8e5e39fcf6bebc8ec4c1d855f4f1335756c50e \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..c4354fecd6f89 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +79d5e686999a84552d9b7bbb9589e5b853113bda \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-buffer-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-buffer-4.2.12.Final.jar.sha1 deleted file mode 100644 index d8dc651e6d0a7..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-buffer-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a1b3a6a4ebaf546860eb119d4e462cd300976ae3 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-buffer-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-buffer-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..723b9fac59b38 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-buffer-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +177025483d7565afaf4f820139d409bdc0cd7000 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-4.2.12.Final.jar.sha1 deleted file mode 100644 index b4a67ffb42f9c..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-codec-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -067b917da20425d325081eb056883b47e1671430 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..a91736d0ee322 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-codec-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +99829f1c0fdf0a3f6457bc4fda3325284f8dd47e \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-base-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-base-4.2.12.Final.jar.sha1 deleted file mode 100644 index 12a51f44a7e21..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-codec-base-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -381b47a0cdd126010a7df1c25d25d7bf55c4fddb \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-base-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-base-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..34fbd28571f81 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-codec-base-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +a4476639056149914d7a145ce0bb9f86bb7e3f49 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 deleted file mode 100644 index 351c6d0feae23..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aa0849118167bc727a8dbdaeccc45d56c1f1e8fb \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..633b40ae21366 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +c2a1fc65daf1a3d5467db37b6e0ce42bbb5b98a8 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-http-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-http-4.2.12.Final.jar.sha1 deleted file mode 100644 index 1fee91860d10c..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-codec-http-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8dbaa045acc60abf333d428dca4339ce36423bd0 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-http-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-http-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..2096dbd85d87f --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-codec-http-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +196f0b6d0779a7a23be4a8bff362741ff0282ce8 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 deleted file mode 100644 index 8f3d42fde9be4..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -383b786cfc2549978390a2881ff3c146cc22bb54 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..51813d949a63b --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +5c8512afb15a0d26a3f1b7b43117aa5d26fac662 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-common-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 631d78619a4a4..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d35ffb9bf5cc0e05ae7408cf6a682b62dceceafc \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-common-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..b1ac1fc1bde8b --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +f91909ed1b9280cd46d8b0ee260ebff40e1c73d8 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-handler-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-handler-4.2.12.Final.jar.sha1 deleted file mode 100644 index 818090d4302e4..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-handler-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1ccb2b1eed54ce049b3ff39fde225014526ab6a0 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-handler-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-handler-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..a3126bb594ff3 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-handler-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +920eb7284d62152dfc5cb8ef0f9e0deb47ed5635 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-resolver-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-resolver-4.2.12.Final.jar.sha1 deleted file mode 100644 index cbf4733c23b7a..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-resolver-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c59aa586a12e62d80207a00f9cf18eedf69d1012 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-resolver-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-resolver-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..bb0791379b05d --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-resolver-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +c68d861f923020f82fea2c99d5921d8142b5c012 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-transport-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-transport-4.2.12.Final.jar.sha1 deleted file mode 100644 index 1d881a45d3290..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-transport-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e9d42074c3d96cf31ce57cc58f6de6f31959b7a8 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-transport-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-transport-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..2ada67e7addc5 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-transport-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +acec47f1ff71785e090e019920f787e0f7d164e3 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 deleted file mode 100644 index 5848bd9b96ab7..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -208f99e5eb334344c51eb921563cd04a3458df66 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..4074708aa903c --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +afd19f8ba23aeb6e8db675a4e9642e3cbc0b90c4 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 59a45c78308ad..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8f8e5e39fcf6bebc8ec4c1d855f4f1335756c50e \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..c4354fecd6f89 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +79d5e686999a84552d9b7bbb9589e5b853113bda \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-buffer-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-buffer-4.2.12.Final.jar.sha1 deleted file mode 100644 index d8dc651e6d0a7..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-buffer-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a1b3a6a4ebaf546860eb119d4e462cd300976ae3 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-buffer-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-buffer-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..723b9fac59b38 --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-buffer-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +177025483d7565afaf4f820139d409bdc0cd7000 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-codec-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-codec-4.2.12.Final.jar.sha1 deleted file mode 100644 index b4a67ffb42f9c..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-codec-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -067b917da20425d325081eb056883b47e1671430 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-codec-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-codec-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..a91736d0ee322 --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-codec-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +99829f1c0fdf0a3f6457bc4fda3325284f8dd47e \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-codec-base-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-codec-base-4.2.12.Final.jar.sha1 deleted file mode 100644 index 12a51f44a7e21..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-codec-base-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -381b47a0cdd126010a7df1c25d25d7bf55c4fddb \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-codec-base-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-codec-base-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..34fbd28571f81 --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-codec-base-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +a4476639056149914d7a145ce0bb9f86bb7e3f49 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 deleted file mode 100644 index 351c6d0feae23..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aa0849118167bc727a8dbdaeccc45d56c1f1e8fb \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..633b40ae21366 --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +c2a1fc65daf1a3d5467db37b6e0ce42bbb5b98a8 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-codec-http-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-codec-http-4.2.12.Final.jar.sha1 deleted file mode 100644 index 1fee91860d10c..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-codec-http-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8dbaa045acc60abf333d428dca4339ce36423bd0 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-codec-http-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-codec-http-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..2096dbd85d87f --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-codec-http-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +196f0b6d0779a7a23be4a8bff362741ff0282ce8 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 deleted file mode 100644 index 8f3d42fde9be4..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -383b786cfc2549978390a2881ff3c146cc22bb54 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..51813d949a63b --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +5c8512afb15a0d26a3f1b7b43117aa5d26fac662 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-common-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 631d78619a4a4..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d35ffb9bf5cc0e05ae7408cf6a682b62dceceafc \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-common-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..b1ac1fc1bde8b --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +f91909ed1b9280cd46d8b0ee260ebff40e1c73d8 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-handler-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-handler-4.2.12.Final.jar.sha1 deleted file mode 100644 index 818090d4302e4..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-handler-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1ccb2b1eed54ce049b3ff39fde225014526ab6a0 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-handler-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-handler-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..a3126bb594ff3 --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-handler-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +920eb7284d62152dfc5cb8ef0f9e0deb47ed5635 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-resolver-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-resolver-4.2.12.Final.jar.sha1 deleted file mode 100644 index cbf4733c23b7a..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-resolver-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c59aa586a12e62d80207a00f9cf18eedf69d1012 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-resolver-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-resolver-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..bb0791379b05d --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-resolver-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +c68d861f923020f82fea2c99d5921d8142b5c012 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-transport-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-transport-4.2.12.Final.jar.sha1 deleted file mode 100644 index 1d881a45d3290..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-transport-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e9d42074c3d96cf31ce57cc58f6de6f31959b7a8 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-transport-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-transport-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..2ada67e7addc5 --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-transport-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +acec47f1ff71785e090e019920f787e0f7d164e3 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 deleted file mode 100644 index 5848bd9b96ab7..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -208f99e5eb334344c51eb921563cd04a3458df66 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..4074708aa903c --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +afd19f8ba23aeb6e8db675a4e9642e3cbc0b90c4 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 59a45c78308ad..0000000000000 --- a/plugins/ingestion-kinesis/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8f8e5e39fcf6bebc8ec4c1d855f4f1335756c50e \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 b/plugins/ingestion-kinesis/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..c4354fecd6f89 --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +79d5e686999a84552d9b7bbb9589e5b853113bda \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-base-4.2.12.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-base-4.2.12.Final.jar.sha1 deleted file mode 100644 index 12a51f44a7e21..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-base-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -381b47a0cdd126010a7df1c25d25d7bf55c4fddb \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-base-4.2.13.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-base-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..34fbd28571f81 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-base-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +a4476639056149914d7a145ce0bb9f86bb7e3f49 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.2.12.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.2.12.Final.jar.sha1 deleted file mode 100644 index 02b5eb5499379..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-dns-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d65d2be0cd872c5bb08378b4090232ea3d50793c \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.2.13.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..0425a504d8707 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-dns-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +56b382fec4774601c57e579bc1db9ba83e72669e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 deleted file mode 100644 index 8f3d42fde9be4..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -383b786cfc2549978390a2881ff3c146cc22bb54 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..51813d949a63b --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +5c8512afb15a0d26a3f1b7b43117aa5d26fac662 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.2.12.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.2.12.Final.jar.sha1 deleted file mode 100644 index 754504bddb6d0..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-socks-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e7114ff84cea11086b33367468f5cae16aa727a8 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.2.13.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..5bf8f145a9baa --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-socks-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +bcc142249e2b07f4a07955be27c99d5350b9ba33 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.2.12.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.2.12.Final.jar.sha1 deleted file mode 100644 index 40815c6a62ee0..0000000000000 --- a/plugins/repository-azure/licenses/netty-handler-proxy-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2304d930dcd2c2ba3537318395361944938e3d42 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.2.13.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..4d08f7135a3bc --- /dev/null +++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +1d60f00253dd7db58ae8dcc09914f81bbceb5b80 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.2.12.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.2.12.Final.jar.sha1 deleted file mode 100644 index 3b700a89d2441..0000000000000 --- a/plugins/repository-azure/licenses/netty-resolver-dns-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c7eeea93db8a94947732e318423e5c0d8746e6a9 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.2.13.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..67f80c0dd9a3d --- /dev/null +++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +de63340cfecd51c43569e750e24eb2c6d1f97fa7 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 59a45c78308ad..0000000000000 --- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8f8e5e39fcf6bebc8ec4c1d855f4f1335756c50e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..c4354fecd6f89 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +79d5e686999a84552d9b7bbb9589e5b853113bda \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.2.12.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.2.12.Final.jar.sha1 deleted file mode 100644 index aa0595cc43f54..0000000000000 --- a/plugins/repository-hdfs/licenses/netty-all-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -85f3f6e21f6b11124f693b658187b2d7d173128c \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.2.13.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..e28304ec53443 --- /dev/null +++ b/plugins/repository-hdfs/licenses/netty-all-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +9ec3a5cf8bfef1820d43013216f0302bd2e762e7 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.2.12.Final.jar.sha1 deleted file mode 100644 index d8dc651e6d0a7..0000000000000 --- a/plugins/repository-s3/licenses/netty-buffer-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a1b3a6a4ebaf546860eb119d4e462cd300976ae3 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..723b9fac59b38 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-buffer-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +177025483d7565afaf4f820139d409bdc0cd7000 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.2.12.Final.jar.sha1 deleted file mode 100644 index b4a67ffb42f9c..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -067b917da20425d325081eb056883b47e1671430 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..a91736d0ee322 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +99829f1c0fdf0a3f6457bc4fda3325284f8dd47e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-base-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-base-4.2.12.Final.jar.sha1 deleted file mode 100644 index 12a51f44a7e21..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-base-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -381b47a0cdd126010a7df1c25d25d7bf55c4fddb \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-base-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-base-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..34fbd28571f81 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-base-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +a4476639056149914d7a145ce0bb9f86bb7e3f49 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 deleted file mode 100644 index 351c6d0feae23..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aa0849118167bc727a8dbdaeccc45d56c1f1e8fb \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..633b40ae21366 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +c2a1fc65daf1a3d5467db37b6e0ce42bbb5b98a8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.2.12.Final.jar.sha1 deleted file mode 100644 index 1fee91860d10c..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8dbaa045acc60abf333d428dca4339ce36423bd0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..2096dbd85d87f --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +196f0b6d0779a7a23be4a8bff362741ff0282ce8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 deleted file mode 100644 index 8f3d42fde9be4..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -383b786cfc2549978390a2881ff3c146cc22bb54 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..51813d949a63b --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +5c8512afb15a0d26a3f1b7b43117aa5d26fac662 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 631d78619a4a4..0000000000000 --- a/plugins/repository-s3/licenses/netty-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d35ffb9bf5cc0e05ae7408cf6a682b62dceceafc \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..b1ac1fc1bde8b --- /dev/null +++ b/plugins/repository-s3/licenses/netty-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +f91909ed1b9280cd46d8b0ee260ebff40e1c73d8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.2.12.Final.jar.sha1 deleted file mode 100644 index 818090d4302e4..0000000000000 --- a/plugins/repository-s3/licenses/netty-handler-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1ccb2b1eed54ce049b3ff39fde225014526ab6a0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..a3126bb594ff3 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-handler-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +920eb7284d62152dfc5cb8ef0f9e0deb47ed5635 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.2.12.Final.jar.sha1 deleted file mode 100644 index cbf4733c23b7a..0000000000000 --- a/plugins/repository-s3/licenses/netty-resolver-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c59aa586a12e62d80207a00f9cf18eedf69d1012 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..bb0791379b05d --- /dev/null +++ b/plugins/repository-s3/licenses/netty-resolver-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +c68d861f923020f82fea2c99d5921d8142b5c012 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.2.12.Final.jar.sha1 deleted file mode 100644 index 1d881a45d3290..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e9d42074c3d96cf31ce57cc58f6de6f31959b7a8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..2ada67e7addc5 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +acec47f1ff71785e090e019920f787e0f7d164e3 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 deleted file mode 100644 index 5848bd9b96ab7..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -208f99e5eb334344c51eb921563cd04a3458df66 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..4074708aa903c --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +afd19f8ba23aeb6e8db675a4e9642e3cbc0b90c4 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 59a45c78308ad..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8f8e5e39fcf6bebc8ec4c1d855f4f1335756c50e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..c4354fecd6f89 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +79d5e686999a84552d9b7bbb9589e5b853113bda \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 deleted file mode 100644 index d8dc651e6d0a7..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a1b3a6a4ebaf546860eb119d4e462cd300976ae3 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..723b9fac59b38 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +177025483d7565afaf4f820139d409bdc0cd7000 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.2.12.Final.jar.sha1 deleted file mode 100644 index b4a67ffb42f9c..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -067b917da20425d325081eb056883b47e1671430 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..a91736d0ee322 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +99829f1c0fdf0a3f6457bc4fda3325284f8dd47e \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-base-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-base-4.2.12.Final.jar.sha1 deleted file mode 100644 index 12a51f44a7e21..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-base-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -381b47a0cdd126010a7df1c25d25d7bf55c4fddb \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-base-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-base-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..34fbd28571f81 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-base-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +a4476639056149914d7a145ce0bb9f86bb7e3f49 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 deleted file mode 100644 index 97f442e1f3f2f..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-classes-quic-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7b393e85c2017ad4f63ac5cc8700babd28934061 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..4063dcfc6685c --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-classes-quic-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +9b96afed708b58c55ef4c0388f532b48d628d610 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 deleted file mode 100644 index 351c6d0feae23..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-compression-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aa0849118167bc727a8dbdaeccc45d56c1f1e8fb \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..633b40ae21366 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-compression-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +c2a1fc65daf1a3d5467db37b6e0ce42bbb5b98a8 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.2.12.Final.jar.sha1 deleted file mode 100644 index 02b5eb5499379..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d65d2be0cd872c5bb08378b4090232ea3d50793c \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..0425a504d8707 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +56b382fec4774601c57e579bc1db9ba83e72669e \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.2.12.Final.jar.sha1 deleted file mode 100644 index 1fee91860d10c..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8dbaa045acc60abf333d428dca4339ce36423bd0 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..2096dbd85d87f --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +196f0b6d0779a7a23be4a8bff362741ff0282ce8 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 deleted file mode 100644 index 8f3d42fde9be4..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -383b786cfc2549978390a2881ff3c146cc22bb54 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..51813d949a63b --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +5c8512afb15a0d26a3f1b7b43117aa5d26fac662 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http3-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http3-4.2.12.Final.jar.sha1 deleted file mode 100644 index 5c3d8f6f38f36..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http3-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4c1d110b95a00688f288bc93d11acb6dba3466ca \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http3-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http3-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..afd98f92f481c --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http3-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +9e9d253671a73eabfa84694ed7809b2a3fa42f23 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-aarch_64.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-aarch_64.jar.sha1 deleted file mode 100644 index 6e1ac36b3504c..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-aarch_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -37988fd1ec666656915fd418aded37a01bc65941 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-x86_64.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-x86_64.jar.sha1 deleted file mode 100644 index 69dabfba6fad9..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-linux-x86_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -632cc4feab6a0583e5a879e05c59acb4bef5d8b0 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-aarch_64.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-aarch_64.jar.sha1 deleted file mode 100644 index 44fc97d71ec5b..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-aarch_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ca327d4c0132005fc0bcbe33c110c500083c0740 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-x86_64.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-x86_64.jar.sha1 deleted file mode 100644 index 83778fda79970..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-osx-x86_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -54a84890c0a4ef4b44e5c3919b09f67e229d6233 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-windows-x86_64.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-windows-x86_64.jar.sha1 deleted file mode 100644 index 8f609358a06e0..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final-windows-x86_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e794e36f597a26879225ed839c2ee4687a1f21b7 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final.jar.sha1 deleted file mode 100644 index e7089a2298bea..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7faa5240eaa23383c469b61f2a67ee54013c0fb9 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-aarch_64.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-aarch_64.jar.sha1 new file mode 100644 index 0000000000000..b297b9c6196b0 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-aarch_64.jar.sha1 @@ -0,0 +1 @@ +9f67caefaa7a964b2b7248bbf3414d55c5cdd37b \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-x86_64.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-x86_64.jar.sha1 new file mode 100644 index 0000000000000..a18ef06cbd56f --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-linux-x86_64.jar.sha1 @@ -0,0 +1 @@ +b2f6b62623f17796df2bd4ea1e50174dc9f1dc70 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-aarch_64.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-aarch_64.jar.sha1 new file mode 100644 index 0000000000000..9fa17e216328e --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-aarch_64.jar.sha1 @@ -0,0 +1 @@ +6658ea9d2d15b0dd1339ba323d39d3d22b26af40 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-x86_64.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-x86_64.jar.sha1 new file mode 100644 index 0000000000000..e2932daa0043b --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-osx-x86_64.jar.sha1 @@ -0,0 +1 @@ +6cdc84558d0c09ab47c8a2c38817be89acffc2b5 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-windows-x86_64.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-windows-x86_64.jar.sha1 new file mode 100644 index 0000000000000..95a7e8b7c6047 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final-windows-x86_64.jar.sha1 @@ -0,0 +1 @@ +9baa6c4ceeb5c1b0824ca881ad37858ab77b1b7f \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..4e0c35f6d2c3a --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-native-quic-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +9854dd4789199e79af87f89c98a6d0f039ac0a93 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 631d78619a4a4..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d35ffb9bf5cc0e05ae7408cf6a682b62dceceafc \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..b1ac1fc1bde8b --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +f91909ed1b9280cd46d8b0ee260ebff40e1c73d8 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 deleted file mode 100644 index 818090d4302e4..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-handler-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1ccb2b1eed54ce049b3ff39fde225014526ab6a0 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..a3126bb594ff3 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-handler-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +920eb7284d62152dfc5cb8ef0f9e0deb47ed5635 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.2.12.Final.jar.sha1 deleted file mode 100644 index cbf4733c23b7a..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c59aa586a12e62d80207a00f9cf18eedf69d1012 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..bb0791379b05d --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +c68d861f923020f82fea2c99d5921d8142b5c012 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.2.12.Final.jar.sha1 deleted file mode 100644 index 3b700a89d2441..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c7eeea93db8a94947732e318423e5c0d8746e6a9 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..67f80c0dd9a3d --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +de63340cfecd51c43569e750e24eb2c6d1f97fa7 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 deleted file mode 100644 index 1d881a45d3290..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e9d42074c3d96cf31ce57cc58f6de6f31959b7a8 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..2ada67e7addc5 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +acec47f1ff71785e090e019920f787e0f7d164e3 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 deleted file mode 100644 index 59a45c78308ad..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8f8e5e39fcf6bebc8ec4c1d855f4f1335756c50e \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..c4354fecd6f89 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +79d5e686999a84552d9b7bbb9589e5b853113bda \ No newline at end of file diff --git a/test/framework/licenses/netty-pkitesting-4.2.12.Final.jar.sha1 b/test/framework/licenses/netty-pkitesting-4.2.12.Final.jar.sha1 deleted file mode 100644 index 4ec4efc336176..0000000000000 --- a/test/framework/licenses/netty-pkitesting-4.2.12.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -352305cf4418cbbeec4efa8988361e4324e0666f \ No newline at end of file diff --git a/test/framework/licenses/netty-pkitesting-4.2.13.Final.jar.sha1 b/test/framework/licenses/netty-pkitesting-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..f68dbe308f9f0 --- /dev/null +++ b/test/framework/licenses/netty-pkitesting-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +2a912c171def46e5a9a3ff4ce3726f6f6014a6e5 \ No newline at end of file From c20b3bd04c7353e618b7ba8c4bdebd3da5359d3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 12:33:53 -0400 Subject: [PATCH 047/115] Bump org.jline:jline from 4.0.0 to 4.0.14 in /test/fixtures/hdfs-fixture (#21471) Bumps [org.jline:jline](https://github.com/jline/jline3) from 4.0.0 to 4.0.14. - [Release notes](https://github.com/jline/jline3/releases) - [Commits](https://github.com/jline/jline3/compare/4.0.0...4.0.14) --- updated-dependencies: - dependency-name: org.jline:jline dependency-version: 4.0.14 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test/fixtures/hdfs-fixture/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index e8f1b06bd65e0..cc04d033b5fb2 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -82,7 +82,7 @@ dependencies { api "ch.qos.logback:logback-core:1.5.32" api "ch.qos.logback:logback-classic:1.5.32" api "org.jboss.xnio:xnio-nio:3.8.17.Final" - api 'org.jline:jline:4.0.0' + api 'org.jline:jline:4.0.14' api 'org.apache.commons:commons-configuration2:2.13.0' api 'com.nimbusds:nimbus-jose-jwt:10.9' api ('org.apache.kerby:kerb-admin:2.1.1') { From fbfcabe8013150e445272774fa83647f7a460717 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Tue, 5 May 2026 13:55:15 -0500 Subject: [PATCH 048/115] Fix IndicesRequestCacheCleanupIT flakiness by removing too-short assertBusy timeouts (#21494) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test class passed a custom timeout of `cacheCleanIntervalInMillis * MAX_ITERATIONS` (with MAX_ITERATIONS = 5) to every assertBusy call. With cacheCleanIntervalInMillis set to 1, 10, 50, or 100 ms across the tests, the wall-clock timeout budget ranged from just 5 ms to 500 ms for async cluster operations (flush, force-merge, scheduled cleanup, stats propagation). On loaded CI agents the cleanup thread or stats propagation would regularly miss this window, producing an AssertionError that has no actual race behind it. Drop the custom timeouts and rely on assertBusy's default 10-second budget, matching the rest of the codebase's convention for async cluster assertions. assertBusy returns on the first successful iteration, so tests that assert a steady-state negative condition (cleanup should NOT have happened) still return in constant time — only the failure path gets the larger budget. Also remove the now-unused MAX_ITERATIONS constant and TimeUnit import. Ran locally on the modified tests with -Dtests.iters=5..10: 25 iterations across 4 previously-flaky methods, all green. Relates to #21397 Signed-off-by: Andre Kurait --- .../indices/IndicesRequestCacheCleanupIT.java | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java index 5c6bd27839d23..c153ec64f379b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheCleanupIT.java @@ -55,7 +55,6 @@ import java.util.Arrays; import java.util.Collection; -import java.util.concurrent.TimeUnit; import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING; import static org.opensearch.indices.IndicesService.INDICES_CACHE_CLEANUP_INTERVAL_SETTING_KEY; @@ -66,8 +65,6 @@ @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false) public class IndicesRequestCacheCleanupIT extends OpenSearchIntegTestCase { - private static final long MAX_ITERATIONS = 5; - @Override protected Collection> nodePlugins() { return Arrays.asList(InternalSettingsPlugin.class); @@ -196,7 +193,7 @@ public void testStaleKeysCleanupWithLowThreshold() throws Exception { assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes()); // cache cleaner should NOT have cleaned from index 1 assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes()); - }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS); + }); // sleep until cache cleaner would have cleaned up the stale key from index 2 } @@ -246,7 +243,7 @@ public void testCacheCleanupOnEqualStalenessAndThreshold() throws Exception { assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes()); // cache cleaner should NOT have cleaned from index 1 assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes()); - }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS); + }); } // when staleness threshold is higher than staleness, it should NOT clean the cache @@ -294,7 +291,7 @@ public void testCacheCleanupSkipsWithHighStalenessThreshold() throws Exception { assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0); // cache cleaner should NOT have cleaned from index 1 assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes()); - }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS); + }); } // when staleness threshold is explicitly set to 0, cache cleaner regularly cleans up stale keys. @@ -342,7 +339,7 @@ public void testCacheCleanupOnZeroStalenessThreshold() throws Exception { assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes()); // cache cleaner should NOT have cleaned from index 1 assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes()); - }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS); + }); } // when staleness threshold is not explicitly set, cache cleaner regularly cleans up stale keys @@ -389,7 +386,7 @@ public void testStaleKeysRemovalWithoutExplicitThreshold() throws Exception { assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes()); // cache cleaner should NOT have cleaned from index 1 assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes()); - }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS); + }); } // when cache cleaner interval setting is not set, cache cleaner is configured appropriately with the fall-back setting @@ -433,7 +430,7 @@ public void testCacheCleanupWithDefaultSettings() throws Exception { assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes()); // cache cleaner should NOT have cleaned from index 1 assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes()); - }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS); + }); } // staleness threshold updates flows through to the cache cleaner @@ -476,7 +473,7 @@ public void testDynamicStalenessThresholdUpdate() throws Exception { assertBusy(() -> { // cache cleaner should NOT have cleaned up the stale key from index 2 assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0); - }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS); + }); // Update indices.requests.cache.cleanup.staleness_threshold to "10%" ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); @@ -491,7 +488,7 @@ public void testDynamicStalenessThresholdUpdate() throws Exception { assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes()); // cache cleaner should NOT have cleaned from index 1 assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes()); - }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS); + }); } // staleness threshold dynamic updates should throw exceptions on invalid input @@ -543,7 +540,7 @@ public void testCacheClearanceAfterIndexClosure() throws Exception { assertBusy(() -> { // cache cleaner should have cleaned up the stale keys from index assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes()); - }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS); + }); } // deleting the Index after caching will clean up from Indices Request Cache @@ -584,7 +581,7 @@ public void testCacheCleanupAfterIndexDeletion() throws Exception { assertBusy(() -> { // cache cleaner should have cleaned up the stale keys from index assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes()); - }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS); + }); } // when staleness threshold is lower than staleness, it should clean the cache from all indices having stale keys @@ -629,11 +626,7 @@ public void testStaleKeysCleanupWithMultipleIndices() throws Exception { indexRandom(false, client.prepareIndex(index1).setId("1").setSource("d", "hello")); forceMerge(client, index1); // Assert cache is cleared up - assertBusy( - () -> { assertEquals(0, getRequestCacheStats(client, index1).getMemorySizeInBytes()); }, - cacheCleanIntervalInMillis * MAX_ITERATIONS, - TimeUnit.MILLISECONDS - ); + assertBusy(() -> { assertEquals(0, getRequestCacheStats(client, index1).getMemorySizeInBytes()); }); // invalidate the cache for index2 indexRandom(false, client.prepareIndex(index2).setId("1").setSource("d", "hello")); @@ -653,7 +646,7 @@ public void testStaleKeysCleanupWithMultipleIndices() throws Exception { long currentMemorySizeInBytesForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes(); // assert the memory size of index1 to only contain 1 entry added after flushAndRefresh assertEquals(memorySizeForIndex1With1Entries, currentMemorySizeInBytesForIndex1); - }, cacheCleanIntervalInMillis * MAX_ITERATIONS, TimeUnit.MILLISECONDS); + }); } private void setupIndex(Client client, String index) throws Exception { From c8168ba20bccf6f98038e647966803c0fd84c095 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Tue, 5 May 2026 12:03:56 -0700 Subject: [PATCH 049/115] [Analytics Engine] PPL append: mark Union + multi-child stage runtime (#21474) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Analytics Engine] Mark LogicalUnion for backend assignment Adds the planner-side machinery to recognise LogicalUnion in the analytics-engine route, the first piece of end-to-end PPL `append` support: every PPL `append [ ... ]` lowers to a LogicalUnion (CalciteRelNodeVisitor.visitAppend), which until now caused the analytics-engine planner to throw at the parent rule with "Filter / Sort rule encountered unmarked child [LogicalUnion]" because there was no marking rule for Union. Three things land together because each is useless without the others: 1. EngineCapability.UNION + DataFusion's ENGINE_CAPS — declares that DataFusion can act as the executor for Union operators. 2. OpenSearchUnion (rel) — multi-input Union carrying the viable-backend list and per-column output FieldStorageInfo. Storage is the intersection of branches at each positional column index: when every input reports identical storage we keep it; any divergence (one branch projects a literal NULL, another references a real field) collapses to a derived column so downstream rules see the merged schema correctly. 3. OpenSearchUnionRule — marks LogicalUnion bottom-up. Drops empty Values inputs (the shape produced by `| append [ ]` subsearches with no source — see testAppendEmptySearchCommand), collapses single-input unions to their sole branch, intersects viable backends across inputs, and wraps every input in an OpenSearchExchangeReducer so DAGBuilder cuts a separate child stage per Union branch. The unconditional ER wrap is what guarantees per-branch stage isolation for shapes like `... | append [ source=other_index ... ]` where ShardTargetResolver would otherwise pick one index's shards and try to scan the other branch's table on them. PlannerImpl registers the new rule in the marking HepProgram. RelNodeUtils.copyToCluster learns to copy OpenSearchUnion (multi-input) and OpenSearchExchangeReducer (now appears in HEP output via the rule above, where previously it only existed in CBO output). Multi-input scheduler/runtime support and the filter-rule fallback for derived columns ride in follow-up commits — this commit alone makes the marking phase succeed but downstream forking still throws on multi-input until the next commit lands. Signed-off-by: Kai Huang * [Analytics Engine + Backend / DataFusion] Multi-child stage runtime for Union Lifts the analytics-engine's "single child stage per coordinator-reduce stage" simplification, enabling Union (and future Join) to fan out into one child stage per branch and reduce them all on the coordinator. Pairs with the OpenSearchUnion marking rule from the previous commit: that rule wraps every Union input in an ExchangeReducer, DAGBuilder already cuts at every ER, and PlanWalker already counts down completed children before starting the parent — the missing piece was per-child input partition routing on the backend sink and the matching plumbing along the way. Five layers change, each minimal: * SPI — ExchangeSinkContext now carries a List (childStageId + Arrow schema per child) instead of a single inputSchema. The legacy inputSchema() helper stays for back-compat (single-input case only). New MultiInputExchangeSink interface lets sinks expose a per-child sub-sink via sinkForChild(int). * Scheduler — LocalStageScheduler builds one ChildInput per child stage (no more children.size() == 1 assert). LocalStageExecution.inputSink routes through MultiInputExchangeSink.sinkForChild when the backend sink supports it, so each child writes into its own native input partition; otherwise the existing single-sink path is unchanged. * DataFusion sink — AbstractDatafusionReduceSink stores a Map of per-child schema IPC bytes and exposes an inputIdFor(int) helper ("input-"). DatafusionReduceSink registers one native partition per child up front, kicks off the plan once, and returns ChildSink wrappers from sinkForChild. ChildSink's close() signals EOF on its sender; the parent close() drains the output stream after every wrapper has closed (idempotent). Memtable variant hard-rejects multi-input for now and points users at the streaming sink (which is the default). * Plan forking — PlanForker.resolve handles multi-input operators by taking the first alternative from each child and asserting all children agree on the chosen backend (today always DataFusion). * Fragment conversion — FragmentConversionDriver.convertReduceNode detects multi-input "all children are ER" as the final-fragment boundary (was: single-input "first child is ER"); converts the whole Union+ER subtree as one Substrait plan via convertFinalAggFragment so every branch ends up reading from its own input partition. DataFusionFragmentConvertor.rewriteStageInputScans uses the per-stage input id (was: hardcoded "input-0"), attachFragmentOnTop now also rewrites stage input scans so the isthmus visitor doesn't trip on planner-internal leaves when wrapping operators above a Union, and replaceInput learns about Substrait Fetch (the rel emitted for SystemLimit / `head N`, which sits above many Append shapes). Net effect: 6 of 8 CalcitePPLAppendCommandIT tests pass on the analytics-engine route — testAppend, testAppendDifferentIndex, testAppendWithMergedColumn, testAppendEmptySearchCommand, testAppendEmptySearchWithJoin, testAppendWithConflictTypeColumn. The remaining two (SchemaMergeWithTimestampUDT, SchemaMergeWithIpUDT) hit pre-existing analytics-engine type-support gaps unrelated to Append (native DataFusion timestamp panic; CIDRMATCH not registered as a ScalarFunction enum constant) and are deferred to follow-up PRs. Signed-off-by: Kai Huang * [Analytics Engine] Filter rule fallback for derived columns Filter on a derived column (no doc-value or index format) used to throw unconditionally with "Filter on derived column [...] is not yet supported." That blocks Filter sitting above any operator that materialises columns without physical storage — Project on a literal, Aggregate on a computed result, and most relevantly here Union of branches that disagree on storage at a positional column index (testAppendSchemaMergeWithTimestampUDT and SchemaMergeWithIpUDT both hit this on the column the Union output marks as derived). Replaces the throw with a format-agnostic lookup — when a column is derived we resolve viable backends via filterBackendsAnyFormat (new helper on CapabilityRegistry, mirroring the existing aggregateBackendsAnyFormat). The format-aware Lucene-pushdown path stays as the primary lookup for non-derived columns; format-level delegation isn't applicable here anyway because the column is materialised at the upstream operator and the filter must run wherever that operator's backend executes. Signed-off-by: Kai Huang * [QA] Add AppendCommandIT for the analytics-engine REST path Self-contained integration test for PPL append on the analytics-engine route, mirroring CalcitePPLAppendCommandIT from opensearch-project/sql so the analytics-engine path can be verified inside core without cross-plugin dependencies on the SQL plugin. Each test sends a PPL query through POST /_analytics/ppl (exposed by the test-ppl-frontend plugin) which runs the same UnifiedQueryPlanner → CalciteRelNodeVisitor → Substrait → DataFusion pipeline as the SQL plugin's force-routed analytics path. Covers six surface forms: - testAppend — two stats branches sorted + head - testAppendDifferentIndex — cross-index union via calcs + calcs_alt (same dataset, different index name) - testAppendWithMergedColumn — branches sharing a "sum" column; SchemaUnifier merges by name - testAppendEmptySearchCommand{BareBrackets, StatsWithoutSource, NestedAppend} — `| append [ … ]` shapes that yield no rows from the subsearch; expected output is the first branch unchanged - testAppendEmptySearchWithJoin — subsearch's join has no left source so the union still collapses to the first branch - testAppendWithConflictTypeColumn — branches diverge on column type; SchemaUnifier surfaces an error Provisions the calcs dataset twice — once into `calcs` and once into `calcs_alt` — so the cross-index test doesn't need a second dataset. DatasetProvisioner already uses refresh=true and parquet-backed settings; no per-test infra needed. Result: 8/8 pass against the QA-module's testcluster — no SQL plugin, no force_routing setting, no parquet-IT system properties needed. Signed-off-by: Kai Huang * [QA] Cover all empty-subsearch variants in AppendCommandIT Expands AppendCommandIT from 8 to 14 test methods so the QA-side coverage matches every variant the SQL-plugin's CalcitePPLAppendCommandIT packs into its testAppendEmptySearchCommand and testAppendEmptySearchWithJoin multi-PPL loops: * testAppendEmptySearchCommandLookup — `| lookup INDEX field as alias` inside an empty subsearch (the 4th variant in the original packed loop) * testAppendEmptySearchWithInnerJoin — renamed from testAppendEmptySearchWithJoin for symmetry with the new methods * testAppendEmptySearchWithCrossJoin — `| cross join …` variant * testAppendEmptySearchWithLeftJoin — `| left join …` variant * testAppendEmptySearchWithSemiJoin — `| semi join …` variant * testAppendEmptySearchWithRightJoin — `| right join … [source=… ]` — right side contributes rows even when the left is empty * testAppendEmptySearchWithFullJoin — `| full join … [source=… ]` — same shape as right join here The right/full join methods don't need a CALCITE_SUPPORT_ALL_JOIN_TYPES cluster-setting toggle: AstBuilder.validateJoinType short-circuits when the setting is unregistered (config==null), and the QA testcluster runs test-ppl-frontend rather than opensearch-sql-plugin so that setting key is never registered. Each method is one PPL string per JUnit method so failures pinpoint the exact surface form (the original packed-loop form stops at the first failing variant and only ever reports one). Result: 14/14 pass against the QA-module's testcluster. Signed-off-by: Kai Huang * [Analytics Engine + Backend / DataFusion] Fix sandbox check: spotless + stale tests + javadoc Brings ./gradlew check -p sandbox -Dsandbox.enabled=true back to green on top of the previous Append commits. Three orthogonal fixes: * spotless reflow on six files touched by the earlier commits (ExchangeSinkContext, DataFusionAnalyticsBackendPlugin, DataFusionFragmentConvertor, DatafusionReduceSink, PlanForker, OpenSearchUnionRule). No behavioral change — pure ./gradlew spotlessApply output. * FilterRuleTests.testFilterOnDerivedColumnsAfterAggregateThrows asserted the *old* "filter on derived column must throw" contract. The earlier "Filter rule fallback for derived columns" commit intentionally inverted that contract — derived columns now resolve via filterBackendsAnyFormat. Renamed to testFilterOnDerivedColumnsAfterAggregateResolvesAnyFormat and rewrote the assertion to confirm the resulting OpenSearchFilter contains DataFusion in its viable backends. * DataFusionFragmentConvertorTests.testConvertFinalAggFragment_WithStageInputScanLeaf and testAttachFragmentOnTop_Sort hardcoded the DatafusionReduceSink.INPUT_ID = "input-0" expectation. The earlier "Multi-child stage runtime for Union" commit moved to per-stage input ids ("input-"). Updated the assertions to derive the expected name from each test's StageInputScan childStageId (7 and 3 respectively). * DatafusionReduceSink javadoc had a {@link #INPUT_ID} reference that -Werror flagged as "reference not accessible" once INPUT_ID moved to the abstract parent. Qualified to {@link AbstractDatafusionReduceSink#INPUT_ID}. Signed-off-by: Kai Huang * [QA] Drop unresolved FillNullCommandIT @link in AppendCommandIT The class-doc on the lazy-provisioning method referenced {@link FillNullCommandIT}. That symbol resolves on the dev branch (which is based off pr/fillnull-poc and includes FillNullCommandIT.java) but not on the PR branch (off upstream/main, where FillNullCommandIT hasn't landed yet). CI compileTestJava failed with "reference not found" on this PR. Replaced with a plain prose mention so the doc still conveys "this is the same pattern other QA tests use" without requiring the symbol to exist. Signed-off-by: Kai Huang * [QA] Vendor calcs dataset for AppendCommandIT AppendCommandIT depends on the `calcs` dataset under sandbox/qa/analytics-engine-rest/src/test/resources/datasets/calcs/ (mapping.json + bulk.json). That dataset was first added in the FillNullCommandIT QA commit (opensearch-project/OpenSearch@f4188dd25), which is on the pr/fillnull-poc branch but hasn't landed on main yet. This PR is branched off main and was therefore missing the resource — CI for this PR's :sandbox:qa:analytics-engine-rest:integTest failed all 14 AppendCommandIT methods with "Resource not found: datasets/calcs/mapping.json". Vendored the two files unchanged from f4188dd25 so this PR is self-contained and can land independently of the fillnull PR. When fillnull eventually merges, whichever PR lands second will have a trivial conflict on the dataset directory and one side's copy gets discarded — both copies are byte-identical so the resolution is a no-op. Signed-off-by: Kai Huang * [QA] Use multiset comparison for Union-output tests in AppendCommandIT testAppend (and the other Union-output tests) failed intermittently on CI even though they pass locally and on the runTask cluster. Root cause: the Substrait Set rel preserves order within a child input partition but not between them, so the two child stages of an `| append` pipeline can stream into the coordinator sink in either order depending on shard scheduling timing. Each run picks a stable order, but it's not the same order across machines / CPU counts / DataFusion plan picks. A `| head N` on top of the union therefore produces a stable answer for any single run but a different answer across runs. Added an `assertRowsAnyOrder` helper that normalises each row to a canonical string (numbers → Double, null → "") and compares the sorted lists as multisets. Switched the five tests whose expected output is the concatenation of two child-stage streams: * testAppend * testAppendDifferentIndex * testAppendWithMergedColumn * testAppendEmptySearchWithRightJoin * testAppendEmptySearchWithFullJoin Empty-subsearch tests keep ordered comparison — after the empty branch is dropped by OpenSearchUnionRule the remaining single branch is already deterministically sorted by its own `| sort str0`. The error test (testAppendWithConflictTypeColumn) is unaffected. Signed-off-by: Kai Huang * [Analytics Backend / DataFusion] Make memtable multi-input limitation explicit Addresses @mch2's review comment on DatafusionMemtableReduceSink (https://github.com/opensearch-project/OpenSearch/pull/21474): "this means memTable won't be supported for multi-child stages, lets add a TODO here or throw early, i think we already default to streaming" Three changes in response: 1. Class-level javadoc on DatafusionMemtableReduceSink now spells out the single-input limitation, points at DatafusionReduceSink as the multi-input path, and notes that the user-facing provider auto-falls- back so end-users won't see the constructor's safety-net throw. Adds a concrete TODO sketching how multi-input memtable could be implemented (one MemTable per child stage, per-child input id + buffer accumulation). 2. The constructor's existing throw now closes the parent-allocated DatafusionLocalSession before propagating — super() opens it before the subclass gets a chance to reject the input, and without the cleanup the native session would leak on every multi-input attempt. The throw message also gains an actionable hint pointing at the datafusion.reduce.input_mode setting and a TODO at the close-time registerMemtable site. 3. DataFusionAnalyticsBackendPlugin.getExchangeSinkProvider now checks ctx.childInputs().size() before constructing the memtable sink and transparently falls back to DatafusionReduceSink when there's more than one child input. End-users running with datafusion.reduce.input_mode=memtable now get correct results for Union queries instead of an error — the single-input fast path is still used for the common case. Signed-off-by: Kai Huang --------- Signed-off-by: Kai Huang --- .../analytics/spi/EngineCapability.java | 3 +- .../analytics/spi/ExchangeSinkContext.java | 39 +- .../analytics/spi/MultiInputExchangeSink.java | 33 ++ .../AbstractDatafusionReduceSink.java | 32 +- .../DataFusionAnalyticsBackendPlugin.java | 11 +- .../DataFusionFragmentConvertor.java | 30 +- .../DatafusionMemtableReduceSink.java | 42 +- .../be/datafusion/DatafusionReduceSink.java | 170 +++++-- .../DataFusionFragmentConvertorTests.java | 17 +- .../DatafusionMemtableReduceSinkTests.java | 9 +- .../datafusion/DatafusionReduceSinkTests.java | 18 +- .../exec/stage/LocalStageExecution.java | 12 +- .../exec/stage/LocalStageScheduler.java | 46 +- .../analytics/planner/CapabilityRegistry.java | 10 + .../analytics/planner/PlannerImpl.java | 4 +- .../analytics/planner/RelNodeUtils.java | 6 + .../planner/dag/FragmentConversionDriver.java | 25 +- .../analytics/planner/dag/PlanForker.java | 36 +- .../planner/rel/OpenSearchUnion.java | 119 +++++ .../planner/rules/OpenSearchFilterRule.java | 31 +- .../planner/rules/OpenSearchUnionRule.java | 127 ++++++ .../analytics/planner/FilterRuleTests.java | 35 +- .../analytics/qa/AppendCommandIT.java | 426 ++++++++++++++++++ 23 files changed, 1148 insertions(+), 133 deletions(-) create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/MultiInputExchangeSink.java create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchUnion.java create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchUnionRule.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendCommandIT.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/EngineCapability.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/EngineCapability.java index a0c7fe09a8c97..47838499789c2 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/EngineCapability.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/EngineCapability.java @@ -20,5 +20,6 @@ * @opensearch.internal */ public enum EngineCapability { - SORT + SORT, + UNION } diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java index dcf2be02d52d9..2df1062a60988 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java @@ -11,11 +11,14 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.pojo.Schema; +import java.util.List; + /** * Context passed to {@link ExchangeSinkProvider#createSink} when a * coordinator-reduce stage is being set up. Carries everything the backend * needs to build an {@link ExchangeSink}: serialized plan, buffer allocator, - * input schema, and the downstream sink the backend writes results to. + * one or more child input descriptors, and the downstream sink the backend + * writes results to. * *

      Fields: *

      * - *

      Single-sink simplification: this context assumes exactly one input - * stream. Per-child routing (e.g., joins with multiple inputs of different - * schemas) will require a richer context and is not modeled yet. - * * @opensearch.internal */ -public record ExchangeSinkContext(String queryId, int stageId, byte[] fragmentBytes, BufferAllocator allocator, Schema inputSchema, - ExchangeSink downstream) { +public record ExchangeSinkContext(String queryId, int stageId, byte[] fragmentBytes, BufferAllocator allocator, List< + ChildInput> childInputs, ExchangeSink downstream) { + + /** Per-child input descriptor: the child stage id and the schema of its outgoing batches. */ + public record ChildInput(int childStageId, Schema schema) { + } + + /** + * Convenience for single-input back-compat. Returns the schema of the sole + * child input. Throws when {@link #childInputs} contains more than one entry — + * multi-input callers must inspect {@link #childInputs} directly. + */ + public Schema inputSchema() { + if (childInputs.size() != 1) { + throw new IllegalStateException( + "inputSchema() requires exactly one child input; got " + childInputs.size() + " — use childInputs() instead" + ); + } + return childInputs.get(0).schema(); + } } diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/MultiInputExchangeSink.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/MultiInputExchangeSink.java new file mode 100644 index 0000000000000..8046c20707756 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/MultiInputExchangeSink.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +/** + * Coordinator-side {@link ExchangeSink} that exposes a per-child sub-sink for + * each child stage feeding into it. + * + *

      Used by multi-input shapes (currently {@code UNION}; future {@code JOIN}). + * The orchestrator obtains a wrapper via {@link #sinkForChild(int)} for each + * child stage so that each child feeds into its own input partition on the + * backend's native session. The parent sink's lifecycle ({@link #close()}) is + * still driven by the orchestrator and runs after every child wrapper's + * {@link ExchangeSink#close()} has been called. + * + * @opensearch.internal + */ +public interface MultiInputExchangeSink extends ExchangeSink { + + /** + * Returns the sink that the orchestrator should route the named child + * stage's output into. Implementations bind each returned wrapper to a + * distinct input partition (typically named {@code "input-"}) + * registered on the backend's native session at sink construction. + */ + ExchangeSink sinkForChild(int childStageId); +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java index f40bf225dda50..f4470bb66333d 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java @@ -22,6 +22,8 @@ import org.opensearch.be.datafusion.nativelib.StreamHandle; import org.opensearch.core.action.ActionListener; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.function.Consumer; @@ -46,19 +48,30 @@ * sink is done. * * + *

      Multi-input shapes (Union, future Join) are supported at this base by exposing + * {@link #childInputs} (childStageId → schemaIpc) for subclasses to register one + * native partition per child stage. The {@link #INPUT_ID} constant remains as the + * conventional name for the single-input case (childStageId=0); the per-child id is + * computed via {@link #inputIdFor(int)}. + * * @opensearch.internal */ abstract class AbstractDatafusionReduceSink implements ExchangeSink { - /** Substrait/DataFusion table name for the single registered input partition. */ - // TODO: This will change to represent child ID and taken as input in context - // for now we have a single partition. + /** + * Substrait/DataFusion table name used for the single-input case (childStageId=0). + * For multi-input shapes use {@link #inputIdFor(int)} instead. + */ static final String INPUT_ID = "input-0"; protected final ExchangeSinkContext ctx; protected final NativeRuntimeHandle runtimeHandle; protected final DatafusionLocalSession session; - protected final byte[] schemaIpc; + /** + * Per-child Arrow schema IPC bytes, keyed by childStageId. Iteration order matches + * the order of {@code ctx.childInputs()} so subclasses get deterministic registration. + */ + protected final Map childInputs; /** Guards {@link #closed} and serialises {@link #feed}/{@link #close} against producers. */ protected final Object feedLock = new Object(); @@ -69,8 +82,17 @@ abstract class AbstractDatafusionReduceSink implements ExchangeSink { protected AbstractDatafusionReduceSink(ExchangeSinkContext ctx, NativeRuntimeHandle runtimeHandle) { this.ctx = ctx; this.runtimeHandle = runtimeHandle; - this.schemaIpc = ArrowSchemaIpc.toBytes(ctx.inputSchema()); this.session = new DatafusionLocalSession(runtimeHandle.get()); + Map inputs = new LinkedHashMap<>(ctx.childInputs().size()); + for (ExchangeSinkContext.ChildInput child : ctx.childInputs()) { + inputs.put(child.childStageId(), ArrowSchemaIpc.toBytes(child.schema())); + } + this.childInputs = inputs; + } + + /** DataFusion table name for an input partition associated with the given child stage id. */ + protected static String inputIdFor(int childStageId) { + return "input-" + childStageId; } @Override diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index e226d6b1074b1..90f269991c804 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -40,7 +40,7 @@ */ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendPlugin { - private static final Set ENGINE_CAPS = Set.of(EngineCapability.SORT); + private static final Set ENGINE_CAPS = Set.of(EngineCapability.SORT, EngineCapability.UNION); private static final Set SUPPORTED_FIELD_TYPES = new HashSet<>(); static { @@ -191,7 +191,14 @@ public ExchangeSinkProvider getExchangeSinkProvider() { String mode = plugin.getClusterService() != null ? plugin.getClusterService().getClusterSettings().get(DataFusionPlugin.DATAFUSION_REDUCE_INPUT_MODE) : "streaming"; - if ("memtable".equals(mode)) { + // Memtable mode is single-input only (DatafusionMemtableReduceSink registers + // exactly one MemTable at close time). Multi-input shapes (Union, future Join) + // need per-child input partitions, which only the streaming sink implements via + // MultiInputExchangeSink#sinkForChild. Auto-fall-back to streaming so end users + // don't have to flip the cluster setting per query. + // TODO: lift this fallback once the memtable sink registers one MemTable per + // child stage (see DatafusionMemtableReduceSink class javadoc). + if ("memtable".equals(mode) && ctx.childInputs().size() == 1) { return new DatafusionMemtableReduceSink(ctx, svc.getNativeRuntime()); } return new DatafusionReduceSink(ctx, svc.getNativeRuntime()); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index a555f8f53e0b0..101893acb229e 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -49,6 +49,7 @@ import io.substrait.plan.PlanProtoConverter; import io.substrait.plan.ProtoPlanConverter; import io.substrait.relation.Aggregate; +import io.substrait.relation.Fetch; import io.substrait.relation.Filter; import io.substrait.relation.NamedScan; import io.substrait.relation.Project; @@ -116,7 +117,12 @@ public byte[] convertFinalAggFragment(RelNode fragment) { public byte[] attachFragmentOnTop(RelNode fragment, byte[] innerBytes) { LOGGER.debug("Attaching generic fragment [{}] on top of {} inner bytes", fragment.getClass().getSimpleName(), innerBytes.length); Plan inner = decodePlan(innerBytes); - Rel wrapper = convertStandalone(fragment); + // Rewrite OpenSearchStageInputScans before standalone conversion so the isthmus + // visitor can traverse the fragment without choking on planner-internal leaves. + // The standalone conversion's children are discarded by rewire(...) anyway, but + // the visitor still walks them top-down to build the wrapper rel. + RelNode rewritten = rewriteStageInputScans(fragment); + Rel wrapper = convertStandalone(rewritten); return serializePlan(rewire(inner, wrapper)); } @@ -182,6 +188,12 @@ private static Rel replaceInput(Rel wrapper, Rel newInput) { if (wrapper instanceof Project project) { return Project.builder().from(project).input(newInput).build(); } + if (wrapper instanceof Fetch fetch) { + // SystemLimit + LogicalSort with offset/fetch lower to a Substrait Fetch rel. + // Used by the implicit query-size limit at the top of every analytics-engine plan + // and by user-level `head N` clauses; both arrive here when attached above a Union. + return Fetch.builder().from(fetch).input(newInput).build(); + } throw new UnsupportedOperationException( "Cannot attach-on-top a Substrait Rel of type " + wrapper.getClass().getSimpleName() + " — no single-input rewire defined" ); @@ -212,18 +224,18 @@ private static Rel withAggregationPhase(Rel rel, Expression.AggregationPhase pha /** * Rewrites every {@link OpenSearchStageInputScan} in the RelNode tree to a plain - * Calcite {@link TableScan} whose qualified name matches what - * {@link DatafusionReduceSink#INPUT_ID} registers on the native session. - * The isthmus visitor then emits a {@link NamedScan} with that name. + * Calcite {@link TableScan} whose qualified name matches what the matching + * {@link DatafusionReduceSink} input partition registers on the native session. * - *

      Single-input simplification: every {@link OpenSearchStageInputScan} maps to - * the same {@code "input-0"} table id. Multi-input / join shapes will require - * carrying per-input ids through {@link io.substrait.relation.NamedScan} once - * implemented. + *

      The table id is {@code "input-"}, mirroring + * {@code AbstractDatafusionReduceSink.inputIdFor}. For a single-input fragment the + * sole stage id (typically 0) reproduces the conventional {@code "input-0"} name; for + * multi-input shapes (Union) each branch refers to its own child stage id and the + * isthmus visitor emits one {@link NamedScan} per branch. */ private static RelNode rewriteStageInputScans(RelNode node) { if (node instanceof OpenSearchStageInputScan scan) { - return new StageInputTableScan(scan.getCluster(), scan.getTraitSet(), DatafusionReduceSink.INPUT_ID, scan.getRowType()); + return new StageInputTableScan(scan.getCluster(), scan.getTraitSet(), "input-" + scan.getChildStageId(), scan.getRowType()); } List newInputs = new ArrayList<>(node.getInputs().size()); boolean changed = false; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSink.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSink.java index 52354a6ad77c7..d02fe047057f3 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSink.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSink.java @@ -39,14 +39,50 @@ *

      Lifecycle invariants and {@code feed}/{@code close} skeleton are implemented in * {@link AbstractDatafusionReduceSink}. This subclass owns the buffered FFI structs and the * close-time {@code registerMemtable + executeLocalPlan + drain} sequence. + * + *

      Single-input only. The memtable path registers exactly one {@code MemTable} + * at close time, so multi-input shapes (Union, future Join) are not supported here — + * the constructor rejects them with a clear message. Streaming mode + * ({@link DatafusionReduceSink}) supports multi-input via per-child + * {@link org.opensearch.analytics.spi.MultiInputExchangeSink#sinkForChild(int) sinkForChild} + * partitions; the {@link DataFusionAnalyticsBackendPlugin} provider is the user-facing + * gate that auto-falls-back to streaming when {@code childInputs.size() > 1}, so callers + * shouldn't see this error in practice. The constructor's check remains as a + * direct-instantiation safety net. + * + *

      TODO: support multi-input memtable by registering one {@code MemTable} per child + * stage (each with its own {@code "input-"} table id) and accumulating + * separate buffers per child via a per-child {@link org.opensearch.analytics.spi.ExchangeSink} + * wrapper, mirroring the streaming sink's {@code ChildSink} approach. */ public final class DatafusionMemtableReduceSink extends AbstractDatafusionReduceSink { private final List arrays = new ArrayList<>(); private final List schemas = new ArrayList<>(); + private final byte[] schemaIpc; public DatafusionMemtableReduceSink(ExchangeSinkContext ctx, NativeRuntimeHandle runtimeHandle) { super(ctx, runtimeHandle); + // Fail fast and close the parent-allocated native session before propagating — + // super() opened a DatafusionLocalSession that would otherwise leak on construction failure. + if (childInputs.size() != 1) { + try { + session.close(); + } catch (Throwable ignore) { + // Original IllegalStateException carries the actionable message; suppress cleanup errors. + } + throw new IllegalStateException( + "DatafusionMemtableReduceSink supports a single input only; got " + + childInputs.size() + + " child inputs. Use streaming mode (DatafusionReduceSink) for multi-input shapes," + + " or set " + + DataFusionPlugin.DATAFUSION_REDUCE_INPUT_MODE.getKey() + + "=streaming. The DataFusionAnalyticsBackendPlugin sink provider auto-falls-back" + + " when this limit is hit at request time, so reaching here means the sink was" + + " constructed directly." + ); + } + this.schemaIpc = childInputs.values().iterator().next(); } @Override @@ -81,7 +117,11 @@ protected Throwable closeUnderLock() { arrayPtrs[i] = arrays.get(i).memoryAddress(); schemaPtrs[i] = schemas.get(i).memoryAddress(); } - NativeBridge.registerMemtable(session.getPointer(), INPUT_ID, schemaIpc, arrayPtrs, schemaPtrs); + // Multi-input would need one registerMemtable call per child stage with a + // distinct "input-" table id and separate buffer accumulation + // per child (the constructor enforces single-input today; see class javadoc). + int singleChildStageId = childInputs.keySet().iterator().next(); + NativeBridge.registerMemtable(session.getPointer(), inputIdFor(singleChildStageId), schemaIpc, arrayPtrs, schemaPtrs); streamPtr = NativeBridge.executeLocalPlan(session.getPointer(), ctx.fragmentBytes()); try (StreamHandle outStream = new StreamHandle(streamPtr, runtimeHandle)) { diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java index 538f8ab9358c0..c4610b34d85ab 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java @@ -15,38 +15,55 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.analytics.spi.ExchangeSink; import org.opensearch.analytics.spi.ExchangeSinkContext; +import org.opensearch.analytics.spi.MultiInputExchangeSink; import org.opensearch.be.datafusion.nativelib.NativeBridge; import org.opensearch.be.datafusion.nativelib.StreamHandle; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; /** - * Streaming coordinator-side reduce sink: opens a native partition stream up front, - * pushes each fed batch through a tokio mpsc-backed sender, and on close drains the native - * output stream into {@link ExchangeSinkContext#downstream()}. + * Streaming coordinator-side reduce sink: opens one native partition stream per child + * input, pushes each fed batch through a tokio mpsc-backed sender, and on close drains + * the native output stream into {@link ExchangeSinkContext#downstream()}. + * + *

      Single-input shapes register one partition under {@link AbstractDatafusionReduceSink#INPUT_ID} and accept + * batches via the inherited {@link #feed(VectorSchemaRoot)} method. Multi-input shapes + * (Union) register one partition per child stage and require callers to obtain a + * per-child wrapper via {@link #sinkForChild(int)} — feeds via the bare + * {@link #feed(VectorSchemaRoot)} method are rejected since the routing target is + * ambiguous. * *

      Overrides the base class's {@code synchronized(feedLock)} with a lock-free - * implementation. Multiple shard response handlers call {@link #feed} concurrently; - * backpressure comes from the native Rust mpsc channel (bounded, capacity 4). - * The send-after-close race is handled by catching the native error when the - * receiver has been dropped. + * implementation for the per-sender feed path. Multiple shard response handlers call + * {@link #feed} concurrently; backpressure comes from the native Rust mpsc channel + * (bounded, capacity 4). The send-after-close race is handled by catching the native + * error when the receiver has been dropped. * *

      Lifecycle: *

        - *
      1. Constructor registers the input partition stream and kicks off native execution.
        2. - *
      2. {@link #feed} exports each batch via Arrow C Data and sends it lock-free.
        3. - *
      3. {@link #close} closes the sender (EOF), drains output, releases native resources.
        4. + *
      4. Constructor registers all input partition streams and kicks off native execution.
        5. + *
      5. {@link #feed} (or {@link ChildSink#feed} via {@link #sinkForChild}) exports each + * batch via Arrow C Data and sends it lock-free to the appropriate sender.
        6. + *
      6. {@link #close} signals EOF on every still-open sender, drains output, and releases + * native resources.
        7. *
      */ -public final class DatafusionReduceSink extends AbstractDatafusionReduceSink { +public final class DatafusionReduceSink extends AbstractDatafusionReduceSink implements MultiInputExchangeSink { private static final Logger logger = LogManager.getLogger(DatafusionReduceSink.class); - private final DatafusionPartitionSender sender; + /** + * Per-child senders keyed by childStageId, populated in declaration order so the + * single-input case can pick the sole entry without an explicit lookup. + */ + private final Map sendersByChildStageId; private final StreamHandle outStream; - /** Cumulative batches fed into the native sender. */ + /** Cumulative batches fed into any native sender. */ private final AtomicLong feedCount = new AtomicLong(); /** * Background thread that drains {@link #outStream} into the downstream sink as soon @@ -60,8 +77,8 @@ public final class DatafusionReduceSink extends AbstractDatafusionReduceSink { * or this drain thread, the 2nd send_blocking parks indefinitely). * *

      The thread starts polling immediately at construction. It exits naturally - * when the FINAL plan reaches EOF (after {@link #sender}.close() signals input - * EOF and DataFusion completes the last aggregation). + * when the FINAL plan reaches EOF (after every {@link #sendersByChildStageId} entry + * has been closed and DataFusion completes the last aggregation). */ private final Thread drainThread; /** Captures any throwable from the drain thread for surfacing during close(). */ @@ -69,23 +86,33 @@ public final class DatafusionReduceSink extends AbstractDatafusionReduceSink { public DatafusionReduceSink(ExchangeSinkContext ctx, NativeRuntimeHandle runtimeHandle) { super(ctx, runtimeHandle); - long senderPtr = 0; + Map senders = new LinkedHashMap<>(childInputs.size()); long streamPtr = 0; try { - senderPtr = NativeBridge.registerPartitionStream(session.getPointer(), INPUT_ID, schemaIpc); - this.sender = new DatafusionPartitionSender(senderPtr); + // Register one native partition per child stage. The Substrait plan in + // ctx.fragmentBytes() references each partition by its "input-" name + // (DataFusionFragmentConvertor names them this way during plan conversion). + for (Map.Entry child : childInputs.entrySet()) { + int childStageId = child.getKey(); + byte[] schemaIpc = child.getValue(); + long senderPtr = NativeBridge.registerPartitionStream(session.getPointer(), inputIdFor(childStageId), schemaIpc); + senders.put(childStageId, new DatafusionPartitionSender(senderPtr)); + } streamPtr = NativeBridge.executeLocalPlan(session.getPointer(), ctx.fragmentBytes()); this.outStream = new StreamHandle(streamPtr, runtimeHandle); } catch (RuntimeException e) { if (streamPtr != 0) { NativeBridge.streamClose(streamPtr); } - if (senderPtr != 0) { - NativeBridge.senderClose(senderPtr); + for (DatafusionPartitionSender sender : senders.values()) { + try { + sender.close(); + } catch (Throwable ignore) {} } session.close(); throw e; } + this.sendersByChildStageId = senders; // Spawn the drain thread AFTER the native handles are constructed so the catch-block // doesn't have to deal with thread teardown on construction failure. this.drainThread = new Thread(this::drainLoop, "df-reduce-drain-q" + ctx.queryId() + "-s" + ctx.stageId()); @@ -95,12 +122,7 @@ public DatafusionReduceSink(ExchangeSinkContext ctx, NativeRuntimeHandle runtime /** * Drain loop body. Runs on {@link #drainThread} from sink construction until the - * FINAL plan reaches EOF (which only happens after {@code sender.close()} is called - * by {@link #closeUnderLock}). - * - *

      Polls {@link #outStream} via {@code streamNext} and forwards each emitted batch - * to {@code ctx.downstream()}. Any throwable is captured in {@link #drainFailure} - * and re-surfaced from {@link #closeUnderLock} via the existing accumulate pattern. + * FINAL plan reaches EOF (which only happens after every sender is closed). */ private void drainLoop() { try { @@ -112,20 +134,47 @@ private void drainLoop() { } /** - * Lock-free feed: overrides the base class's synchronized feed. - * Arrow C Data export and native send happen without any Java-side lock. - * Backpressure comes from the Rust mpsc channel; the send-after-close - * race is caught via the native error. + * Lock-free feed for the single-input case: writes to the sole registered sender. + * Multi-input callers must use {@link #sinkForChild(int)} instead — calling this + * method when more than one partition is registered is a programming error because + * the routing target is ambiguous. */ @Override public void feed(VectorSchemaRoot batch) { + if (sendersByChildStageId.size() != 1) { + batch.close(); + throw new IllegalStateException( + "DatafusionReduceSink has " + sendersByChildStageId.size() + " input partitions; use sinkForChild(int) instead of feed()" + ); + } + feedToSender(sendersByChildStageId.values().iterator().next(), batch); + } + + @Override + public ExchangeSink sinkForChild(int childStageId) { + DatafusionPartitionSender sender = sendersByChildStageId.get(childStageId); + if (sender == null) { + throw new IllegalArgumentException( + "No registered partition for childStageId=" + childStageId + "; known ids=" + sendersByChildStageId.keySet() + ); + } + return new ChildSink(sender); + } + + /** + * Lock-free per-sender feed. Exports the batch via Arrow C Data outside any lock + * (the allocator is thread-safe; multiple shard handlers can export concurrently), + * then sends it through the supplied sender. The Rust mpsc::Sender is thread-safe, + * so multiple producers feeding the same sender is safe. If close() raced and + * already ran senderClose, the native side returns an error ("receiver dropped") + * which we catch and discard. + */ + private void feedToSender(DatafusionPartitionSender sender, VectorSchemaRoot batch) { // Best-effort fast path — skip export work if already closed. if (closed) { batch.close(); return; } - // Export Arrow C Data outside any lock. The allocator is thread-safe; - // multiple shard handlers can export concurrently. BufferAllocator alloc = ctx.allocator(); ArrowArray array = ArrowArray.allocateNew(alloc); ArrowSchema arrowSchema = ArrowSchema.allocateNew(alloc); @@ -139,9 +188,6 @@ public void feed(VectorSchemaRoot batch) { } finally { batch.close(); } - // No lock — send directly. The Rust mpsc::Sender is thread-safe. - // If close() raced and already called senderClose(), the native side - // returns an error ("receiver dropped") which we catch and discard. try { NativeBridge.senderSend(sender.getPointer(), array.memoryAddress(), arrowSchema.memoryAddress()); feedCount.incrementAndGet(); @@ -158,7 +204,41 @@ public void feed(VectorSchemaRoot batch) { } /** - * Not used — feed() is overridden directly. Required by the abstract class contract. + * Per-child wrapper returned from {@link #sinkForChild(int)}. The orchestrator + * routes one of these per child stage, and the wrapper's close() signals EOF for + * its specific input partition. Idempotent — duplicate close() calls are no-ops. + */ + private final class ChildSink implements ExchangeSink { + private final DatafusionPartitionSender sender; + private volatile boolean childClosed; + + ChildSink(DatafusionPartitionSender sender) { + this.sender = sender; + } + + @Override + public void feed(VectorSchemaRoot batch) { + feedToSender(sender, batch); + } + + @Override + public void close() { + if (childClosed) { + return; + } + childClosed = true; + try { + sender.close(); + } catch (Throwable t) { + logger.warn("[ReduceSink] error closing child sender", t); + } + } + } + + /** + * Not used — feed() is overridden directly for the single-input path and + * {@link ChildSink#feed} for the multi-input path. Required by the abstract + * class contract. */ @Override protected void feedBatchUnderLock(VectorSchemaRoot batch) { @@ -168,12 +248,16 @@ protected void feedBatchUnderLock(VectorSchemaRoot batch) { @Override protected Throwable closeUnderLock() { Throwable failure = null; - // 1. Signal EOF on input. The drain thread, which is already polling the output - // stream, will receive the final batches and then EOF, then exit cleanly. - try { - sender.close(); - } catch (Throwable t) { - failure = accumulate(failure, t); + // 1. Signal EOF on every still-open sender. The drain thread, which is already + // polling the output stream, will receive the final batches and then EOF, then + // exit cleanly. Senders that were already closed by their ChildSink wrapper are + // no-ops (the underlying senderClose is idempotent on the Rust side). + for (DatafusionPartitionSender sender : sendersByChildStageId.values()) { + try { + sender.close(); + } catch (Throwable t) { + failure = accumulate(failure, t); + } } // 2. Wait for the drain thread to finish processing remaining output. try { @@ -196,7 +280,7 @@ protected Throwable closeUnderLock() { return failure; } - /** Returns the cumulative number of batches fed into the native sender. */ + /** Returns the cumulative number of batches fed into any native sender. */ public long feedCount() { return feedCount.get(); } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java index 1e3e5811fde15..549530c4ee99b 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java @@ -207,11 +207,15 @@ public void testAttachPartialAggOnTop_WrapsInner() throws Exception { /** * A final-agg fragment whose leaf is an {@link OpenSearchStageInputScan} - * converts to {@code AggregateRel(ReadRel(namedTable=[__stage__input__]))}. + * converts to {@code AggregateRel(ReadRel(namedTable=["input-"]))}. + * The stage-input id is per-child so multi-input shapes (Union) get distinct names + * for each registered DataFusion partition; single-input shapes still arrive at + * the conventional {@code "input-0"} when childStageId is 0. */ public void testConvertFinalAggFragment_WithStageInputScanLeaf() throws Exception { RelDataType stageRowType = rowType("A"); - RelNode stageInput = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 7, stageRowType, List.of("datafusion")); + int childStageId = 7; + RelNode stageInput = new OpenSearchStageInputScan(cluster, cluster.traitSet(), childStageId, stageRowType, List.of("datafusion")); LogicalAggregate finalAgg = buildSumAggregate(stageInput, 0); byte[] bytes = newConvertor().convertFinalAggFragment(finalAgg); @@ -227,8 +231,8 @@ public void testConvertFinalAggFragment_WithStageInputScanLeaf() throws Exceptio Rel inner = agg.getInput(); assertTrue("Aggregate input must be a ReadRel", inner.hasRead()); assertEquals( - "StageInputScan must be emitted as a ReadRel with the stage-input id", - List.of(DatafusionReduceSink.INPUT_ID), + "StageInputScan must be emitted as a ReadRel with the per-child stage-input id", + List.of("input-" + childStageId), inner.getRead().getNamedTable().getNamesList() ); } @@ -242,7 +246,8 @@ public void testAttachFragmentOnTop_Sort() throws Exception { // Inner: final-agg over stage-input. RelDataType stageRowType = rowType("A"); - RelNode stageInput = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 3, stageRowType, List.of("datafusion")); + int childStageId = 3; + RelNode stageInput = new OpenSearchStageInputScan(cluster, cluster.traitSet(), childStageId, stageRowType, List.of("datafusion")); LogicalAggregate finalAgg = buildSumAggregate(stageInput, 0); byte[] innerBytes = convertor.convertFinalAggFragment(finalAgg); @@ -264,7 +269,7 @@ public void testAttachFragmentOnTop_Sort() throws Exception { assertTrue("Sort input must be an AggregateRel", inner.hasAggregate()); Rel aggInput = inner.getAggregate().getInput(); assertTrue("Agg input must be a ReadRel", aggInput.hasRead()); - assertEquals(List.of(DatafusionReduceSink.INPUT_ID), aggInput.getRead().getNamedTable().getNamesList()); + assertEquals(List.of("input-" + childStageId), aggInput.getRead().getNamedTable().getNamesList()); } /** diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSinkTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSinkTests.java index f7acdb2c2ed8f..0e6d57134bc41 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSinkTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionMemtableReduceSinkTests.java @@ -63,7 +63,14 @@ public void testFeedDrainsSumToDownstream() throws Exception { byte[] substrait = buildSumSubstraitBytes(DatafusionMemtableReduceSink.INPUT_ID); CapturingSink downstream = new CapturingSink(); - ExchangeSinkContext ctx = new ExchangeSinkContext("q-1", 0, substrait, alloc, inputSchema, downstream); + ExchangeSinkContext ctx = new ExchangeSinkContext( + "q-1", + 0, + substrait, + alloc, + List.of(new ExchangeSinkContext.ChildInput(0, inputSchema)), + downstream + ); DatafusionMemtableReduceSink sink = new DatafusionMemtableReduceSink(ctx, runtimeHandle); try { diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReduceSinkTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReduceSinkTests.java index 63046074286bc..d385f548ff3d3 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReduceSinkTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionReduceSinkTests.java @@ -89,7 +89,14 @@ public void testFeedDrainsSumToDownstream() throws Exception { byte[] substrait = buildSumSubstraitBytes(DatafusionReduceSink.INPUT_ID); CapturingSink downstream = new CapturingSink(); - ExchangeSinkContext ctx = new ExchangeSinkContext("q-1", 0, substrait, alloc, inputSchema, downstream); + ExchangeSinkContext ctx = new ExchangeSinkContext( + "q-1", + 0, + substrait, + alloc, + List.of(new ExchangeSinkContext.ChildInput(0, inputSchema)), + downstream + ); DatafusionReduceSink sink = new DatafusionReduceSink(ctx, runtimeHandle); try { @@ -148,7 +155,14 @@ public void testProducersDoNotWedgePastCapacity() throws Exception { byte[] substrait = buildSumSubstraitBytes(DatafusionReduceSink.INPUT_ID); CapturingSink downstream = new CapturingSink(); - ExchangeSinkContext ctx = new ExchangeSinkContext("q-wedge", 0, substrait, alloc, inputSchema, downstream); + ExchangeSinkContext ctx = new ExchangeSinkContext( + "q-wedge", + 0, + substrait, + alloc, + List.of(new ExchangeSinkContext.ChildInput(0, inputSchema)), + downstream + ); DatafusionReduceSink sink = new DatafusionReduceSink(ctx, runtimeHandle); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageExecution.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageExecution.java index e10f778535139..473c6f568c328 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageExecution.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageExecution.java @@ -14,6 +14,7 @@ import org.opensearch.analytics.backend.ExchangeSource; import org.opensearch.analytics.planner.dag.Stage; import org.opensearch.analytics.spi.ExchangeSink; +import org.opensearch.analytics.spi.MultiInputExchangeSink; /** * {@link StageExecution} implementation for COORDINATOR_REDUCE stages. Holds a @@ -39,9 +40,18 @@ public LocalStageExecution(Stage stage, ExchangeSink backendSink, ExchangeSink d logger.info("[LocalStage] CREATED stageId={} childCount={}", stage.getStageId(), stage.getChildStages().size()); } - // All children feed into the single backend sink. + /** + * Per-child input sink resolution. When the backend sink is a + * {@link MultiInputExchangeSink} (multi-input shapes such as Union), returns the + * sink for the named child stage so each child writes to its own input partition. + * Otherwise returns the backend sink unchanged — the single-input case where every + * child feeds the only registered partition. + */ @Override public ExchangeSink inputSink(int childStageId) { + if (backendSink instanceof MultiInputExchangeSink multi) { + return multi.sinkForChild(childStageId); + } return backendSink; } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java index cac4a396cbdd8..13aa77c61fb62 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java @@ -8,7 +8,6 @@ package org.opensearch.analytics.exec.stage; -import org.apache.arrow.vector.types.pojo.Schema; import org.opensearch.analytics.exec.QueryContext; import org.opensearch.analytics.planner.dag.Stage; import org.opensearch.analytics.planner.dag.StageExecutionType; @@ -16,18 +15,22 @@ import org.opensearch.analytics.spi.ExchangeSinkContext; import org.opensearch.analytics.spi.ExchangeSinkProvider; +import java.util.ArrayList; import java.util.List; /** * Builds executions for {@link StageExecutionType#COORDINATOR_REDUCE} stages — * those that run at the coordinator with a backend-provided {@link ExchangeSink}. * Creates the sink via {@link Stage#getExchangeSinkProvider()} using an - * {@link ExchangeSinkContext} carrying the plan bytes, allocator, input - * schema (derived from the single child stage), and downstream sink. Hands - * the resulting sink to {@link LocalStageExecution}. + * {@link ExchangeSinkContext} carrying the plan bytes, allocator, per-child + * input descriptors (one per child stage, each with its stage id + Arrow + * schema), and the downstream sink. Hands the resulting sink to + * {@link LocalStageExecution}. * - *

      Single-sink simplification: assumes exactly one child stage. Multi-child - * (joins, set ops) will require per-child sink routing in a follow-up. + *

      Multi-child stages (Union, future Join) are routed via + * {@link LocalStageExecution#inputSink(int)}, which returns a per-child + * wrapper that the backend sink uses to register a distinct input partition + * per child stage id. * * @opensearch.internal */ @@ -41,7 +44,7 @@ public StageExecution createExecution(Stage stage, ExchangeSink sink, QueryConte stage.getStageId(), chosenBytes(stage), config.bufferAllocator(), - deriveInputSchema(stage), + buildChildInputs(stage), sink ); ExchangeSink backendSink; @@ -63,16 +66,27 @@ private static byte[] chosenBytes(Stage stage) { } /** - * Derives the backend's input Arrow schema from the single child stage's - * fragment rowtype. Multi-child support (joins, set ops with heterogeneous - * inputs) is deferred. + * Builds one {@link ExchangeSinkContext.ChildInput} per child stage. Each entry + * carries the child's stage id (used by the backend to namespace its registered + * input, e.g. {@code "input-"}) and the Arrow schema derived from the + * child fragment's row type. */ - private static Schema deriveInputSchema(Stage stage) { + private static List buildChildInputs(Stage stage) { List children = stage.getChildStages(); - assert children.size() == 1 : "COORDINATOR_REDUCE stage " - + stage.getStageId() - + " expected exactly one child stage, got " - + children.size(); - return ArrowSchemaFromCalcite.arrowSchemaFromRowType(children.getFirst().getFragment().getRowType()); + if (children.isEmpty()) { + throw new IllegalStateException( + "COORDINATOR_REDUCE stage " + stage.getStageId() + " expected at least one child stage, got zero" + ); + } + List inputs = new ArrayList<>(children.size()); + for (Stage child : children) { + inputs.add( + new ExchangeSinkContext.ChildInput( + child.getStageId(), + ArrowSchemaFromCalcite.arrowSchemaFromRowType(child.getFragment().getRowType()) + ) + ); + } + return inputs; } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java index f41e1c76bd456..5482d7dfe6b80 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java @@ -250,6 +250,16 @@ public List aggregateBackendsAnyFormat(AggregateFunction function, Field return allBackends(aggregateIndex.getOrDefault(new AggregateKey(function, fieldType), Map.of())); } + /** + * All backends declaring filter support for a (function, fieldType) ignoring storage formats. + * Used by the filter rule when the field is derived (e.g. produced by Union or Project) and + * therefore has no doc-value or index format to match against — the filter must run at whichever + * backend executes the producing operator, so format-level pushdown isn't applicable. + */ + public List filterBackendsAnyFormat(ScalarFunction function, FieldType fieldType) { + return allBackends(filterIndex.getOrDefault(new ScalarKey(function, fieldType), Map.of())); + } + public List scalarBackendsAnyFormat(ScalarFunction function, FieldType fieldType) { return allBackends(scalarIndex.getOrDefault(new ScalarKey(function, fieldType), Map.of())); } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java index e5c42dba04494..2d8c332eb2e34 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java @@ -33,6 +33,7 @@ import org.opensearch.analytics.planner.rules.OpenSearchProjectRule; import org.opensearch.analytics.planner.rules.OpenSearchSortRule; import org.opensearch.analytics.planner.rules.OpenSearchTableScanRule; +import org.opensearch.analytics.planner.rules.OpenSearchUnionRule; import java.util.List; @@ -100,7 +101,8 @@ public static RelNode markAndOptimize(RelNode rawRelNode, PlannerContext context new OpenSearchFilterRule(context), new OpenSearchProjectRule(context), new OpenSearchAggregateRule(context), - new OpenSearchSortRule(context) + new OpenSearchSortRule(context), + new OpenSearchUnionRule(context) ) ); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/RelNodeUtils.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/RelNodeUtils.java index 18f9a9dee3fc2..3e93f4eca9ee0 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/RelNodeUtils.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/RelNodeUtils.java @@ -16,11 +16,13 @@ import org.opensearch.analytics.planner.rel.OpenSearchConvention; import org.opensearch.analytics.planner.rel.OpenSearchDistribution; import org.opensearch.analytics.planner.rel.OpenSearchDistributionTraitDef; +import org.opensearch.analytics.planner.rel.OpenSearchExchangeReducer; import org.opensearch.analytics.planner.rel.OpenSearchFilter; import org.opensearch.analytics.planner.rel.OpenSearchProject; import org.opensearch.analytics.planner.rel.OpenSearchRelNode; import org.opensearch.analytics.planner.rel.OpenSearchSort; import org.opensearch.analytics.planner.rel.OpenSearchTableScan; +import org.opensearch.analytics.planner.rel.OpenSearchUnion; import java.util.List; @@ -87,6 +89,10 @@ public static RelNode copyToCluster(RelNode node, RelOptCluster newCluster, Open project.getRowType(), project.getViableBackends() ); + } else if (node instanceof OpenSearchUnion union) { + return new OpenSearchUnion(newCluster, newTraits, newInputs, union.all, union.getViableBackends()); + } else if (node instanceof OpenSearchExchangeReducer exchange) { + return new OpenSearchExchangeReducer(newCluster, newTraits, newInputs.getFirst(), exchange.getViableBackends()); } throw new UnsupportedOperationException("Cannot copy node type: " + node.getClass().getSimpleName()); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java index 5e0983f1218af..3a346a8ff6efa 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java @@ -222,20 +222,27 @@ private static byte[] convertReduceNode( RelNode strippedNode = openSearchNode.stripAnnotations(strippedInputs, resolver); if (!finalAggConverted) { - // First OpenSearchRelNode above ExchangeReducer = final agg - // Check if child is ExchangeReducer — if so, this is the final agg node - boolean childIsExchangeReducer = !node.getInputs().isEmpty() - && node.getInputs().getFirst() instanceof OpenSearchExchangeReducer; - if (childIsExchangeReducer) { - // Strip ExchangeReducer, keep StageInputScan as leaf for schema - RelNode stageInputScan = strip(node.getInputs().getFirst().getInputs().getFirst(), delegationBytes); - List finalAggInputs = List.of(stageInputScan); + // First OpenSearchRelNode whose ALL inputs are ExchangeReducers is treated as the + // boundary between the coordinator-side fragment and the data-node child stages. + // For single-input shapes (Sort/Project/Aggregate over a partial agg) this is the + // final-aggregate operator; for multi-input shapes (Union) every branch is itself + // an ER → StageInputScan, and the entire Union+ER subtree is converted as one + // fragment so all branches end up in the same Substrait plan reading from their + // respective input partitions. + boolean allChildrenAreExchangeReducer = !node.getInputs().isEmpty() + && node.getInputs().stream().allMatch(input -> input instanceof OpenSearchExchangeReducer); + if (allChildrenAreExchangeReducer) { + List finalAggInputs = new ArrayList<>(node.getInputs().size()); + for (RelNode input : node.getInputs()) { + // Skip the ER, keep StageInputScan below it as the leaf for schema inference. + finalAggInputs.add(strip(input.getInputs().getFirst(), delegationBytes)); + } RelNode finalAggFragment = openSearchNode.stripAnnotations(finalAggInputs, resolver); return convertor.convertFinalAggFragment(finalAggFragment); } } - // Operator above final agg — convert child first, then attach + // Operator above the final-fragment boundary — convert child first, then attach. byte[] innerBytes = convertReduceNode(node.getInputs().getFirst(), convertor, false, delegationBytes); return convertor.attachFragmentOnTop(strippedNode, innerBytes); } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PlanForker.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PlanForker.java index 7ff1dcb565340..8a0eae3a41a43 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PlanForker.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/PlanForker.java @@ -73,9 +73,39 @@ private static List resolve(RelNode node, CapabilityRegistry registry) return results; } - // TODO: multi-input operators (joins) — each side is typically a separate stage - // connected via StageInputScan, so this path may not be needed in practice. - throw new UnsupportedOperationException("Multi-input plan forking not yet supported for: " + node.getClass().getSimpleName()); + // Multi-input: take the first alternative from each child. With a single backend + // (pure DataFusion), each child has exactly one alternative anyway. For correctness + // we require all children to agree on the chosen backend — a multi-input operator + // cannot straddle backends within a single stage. + // TODO: when multi-backend pipelines are added, fan out the Cartesian product of + // child alternatives and prune by backend agreement. + List resolvedChildren = new ArrayList<>(childAlternativeSets.size()); + String agreedBackend = null; + for (List childAlts : childAlternativeSets) { + if (childAlts.isEmpty()) { + throw new IllegalStateException( + "Multi-input child of [" + node.getClass().getSimpleName() + "] produced no plan alternatives" + ); + } + Resolved childAlt = childAlts.getFirst(); + resolvedChildren.add(childAlt.node); + if (agreedBackend == null) { + agreedBackend = childAlt.chosenBackend; + } else if (childAlt.chosenBackend != null + && !childAlt.chosenBackend.isEmpty() + && !childAlt.chosenBackend.equals(agreedBackend)) { + throw new IllegalStateException( + "Multi-input operator [" + + node.getClass().getSimpleName() + + "] requires all children to share a backend; got [" + + agreedBackend + + "] vs [" + + childAlt.chosenBackend + + "]" + ); + } + } + return resolveOperator(node, resolvedChildren, agreedBackend); } private static List resolveOperator(RelNode node, List children, String childBackend) { diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchUnion.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchUnion.java new file mode 100644 index 0000000000000..fd9de9e28681f --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchUnion.java @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.planner.rel; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelWriter; +import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.logical.LogicalUnion; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.planner.RelNodeUtils; +import org.opensearch.analytics.spi.FieldStorageInfo; + +import java.util.ArrayList; +import java.util.List; + +/** + * OpenSearch custom Union carrying viable backend list. + * + *

      Per-column output storage is the intersection of inputs' storage at the same + * positional index — when all inputs report identical storage we keep it; any + * divergence (e.g. one branch has a derived literal column, another has a real + * field reference) collapses to a derived column. Downstream rules that push down + * to physical storage (Filter, Aggregate) therefore treat post-Union columns as + * derived unless every branch agrees. + * + * @opensearch.internal + */ +public class OpenSearchUnion extends Union implements OpenSearchRelNode { + + private final List viableBackends; + + public OpenSearchUnion(RelOptCluster cluster, RelTraitSet traitSet, List inputs, boolean all, List viableBackends) { + super(cluster, traitSet, List.of(), inputs, all); + this.viableBackends = viableBackends; + } + + @Override + public List getViableBackends() { + return viableBackends; + } + + @Override + public List getOutputFieldStorage() { + List> perInputStorage = new ArrayList<>(getInputs().size()); + for (RelNode input : getInputs()) { + RelNode unwrapped = RelNodeUtils.unwrapHep(input); + if (!(unwrapped instanceof OpenSearchRelNode openSearchInput)) { + throw new IllegalStateException("Union input is not OpenSearchRelNode: " + unwrapped.getClass().getSimpleName()); + } + perInputStorage.add(openSearchInput.getOutputFieldStorage()); + } + + int columnCount = getRowType().getFieldCount(); + List result = new ArrayList<>(columnCount); + for (int col = 0; col < columnCount; col++) { + String fieldName = getRowType().getFieldList().get(col).getName(); + SqlTypeName sqlType = getRowType().getFieldList().get(col).getType().getSqlTypeName(); + + FieldStorageInfo first = perInputStorage.getFirst().size() > col ? perInputStorage.getFirst().get(col) : null; + boolean allMatch = first != null && !first.isDerived(); + if (allMatch) { + for (int i = 1; i < perInputStorage.size(); i++) { + List branch = perInputStorage.get(i); + if (branch.size() <= col) { + allMatch = false; + break; + } + FieldStorageInfo other = branch.get(col); + if (other.isDerived() + || other.getFieldType() != first.getFieldType() + || !other.getDocValueFormats().equals(first.getDocValueFormats()) + || !other.getIndexFormats().equals(first.getIndexFormats())) { + allMatch = false; + break; + } + } + } + + result.add(allMatch ? first : FieldStorageInfo.derivedColumn(fieldName, sqlType)); + } + return result; + } + + @Override + public Union copy(RelTraitSet traitSet, List inputs, boolean all) { + return new OpenSearchUnion(getCluster(), traitSet, inputs, all, viableBackends); + } + + @Override + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return planner.getCostFactory().makeTinyCost(); + } + + @Override + public RelWriter explainTerms(RelWriter pw) { + return super.explainTerms(pw).item("viableBackends", viableBackends); + } + + @Override + public RelNode copyResolved(String backend, List children, List resolvedAnnotations) { + return new OpenSearchUnion(getCluster(), getTraitSet(), children, all, List.of(backend)); + } + + @Override + public RelNode stripAnnotations(List strippedChildren) { + return LogicalUnion.create(strippedChildren, all); + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java index dc23601f523dd..c50e643282821 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java @@ -170,23 +170,24 @@ private List resolveViableBackends( FieldStorageInfo storageInfo = FieldStorageInfo.resolve(fieldStorageInfos, fieldIndex); FieldType fieldType = storageInfo.getFieldType(); - // TODO: for FULL_TEXT operators, extract required params from RexCall + Set fieldViable; if (storageInfo.isDerived()) { - // Derived column marking is not yet implemented. - // Requires DelegationType split (NATIVE_INDEX vs ARROW_BATCH) and - // DataTransferCapability-based execution model for within-stage delegation. - throw new UnsupportedOperationException( - "Filter on derived column [" - + storageInfo.getFieldName() - + "] is not yet supported. Marking on derived/expression columns requires " - + "a implementation for delegation model." - ); + // Post-Union / post-Project columns have no physical storage formats — the + // column is materialised at the operator that produced it (e.g. Union of two + // branches with divergent storage, or a literal/expression projection). The + // filter still has to run somewhere; resolve viability against any backend + // that supports the function on this field type, ignoring storage formats. + // The format-aware Lucene-pushdown path stays as the primary lookup for + // non-derived columns above. + // TODO: for FULL_TEXT operators, extract required params from RexCall + fieldViable = new HashSet<>(registry.filterBackendsAnyFormat(function, fieldType)); + } else { + // Format-aware: backends that can access this field's storage (doc values + index). + // A backend is viable only if it has the field in its own storage formats — ensuring + // delegation targets are also field-storage-aware (e.g. Lucene is viable for a keyword + // field only when the field has indexFormats=[lucene] set in the mapping). + fieldViable = new HashSet<>(registry.filterBackendsForField(function, storageInfo)); } - // Format-aware: backends that can access this field's storage (doc values + index). - // A backend is viable only if it has the field in its own storage formats — ensuring - // delegation targets are also field-storage-aware (e.g. Lucene is viable for a keyword - // field only when the field has indexFormats=[lucene] set in the mapping). - Set fieldViable = new HashSet<>(registry.filterBackendsForField(function, storageInfo)); viableSet.retainAll(fieldViable); } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchUnionRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchUnionRule.java new file mode 100644 index 0000000000000..e7cb981871156 --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchUnionRule.java @@ -0,0 +1,127 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.planner.rules; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.core.Values; +import org.opensearch.analytics.planner.CapabilityResolutionUtils; +import org.opensearch.analytics.planner.PlannerContext; +import org.opensearch.analytics.planner.RelNodeUtils; +import org.opensearch.analytics.planner.rel.OpenSearchDistributionTraitDef; +import org.opensearch.analytics.planner.rel.OpenSearchExchangeReducer; +import org.opensearch.analytics.planner.rel.OpenSearchRelNode; +import org.opensearch.analytics.planner.rel.OpenSearchUnion; +import org.opensearch.analytics.spi.EngineCapability; + +import java.util.ArrayList; +import java.util.List; + +/** + * Converts {@link Union} → {@link OpenSearchUnion}. + * + *

      Validates that all inputs are marked, intersects their viable backends, and + * filters by {@link EngineCapability#UNION}. Empty {@link Values} inputs (the + * shape produced by an {@code | append [ ]} subsearch with no source) are dropped + * — they contribute zero rows to the result. If only one non-empty input remains + * the Union node is collapsed to that input. + * + * @opensearch.internal + */ +public class OpenSearchUnionRule extends RelOptRule { + + private final PlannerContext context; + + public OpenSearchUnionRule(PlannerContext context) { + super(operand(Union.class, any()), "OpenSearchUnionRule"); + this.context = context; + } + + @Override + public boolean matches(RelOptRuleCall call) { + return !(call.rel(0) instanceof OpenSearchUnion); + } + + @Override + public void onMatch(RelOptRuleCall call) { + Union union = call.rel(0); + + List markedInputs = new ArrayList<>(union.getInputs().size()); + List viableBackends = null; + + for (RelNode input : union.getInputs()) { + RelNode unwrapped = RelNodeUtils.unwrapHep(input); + if (unwrapped instanceof Values values && values.getTuples().isEmpty()) { + // Empty values inputs contribute no rows — drop them. Only meaningful + // for testAppendEmptySearchCommand-style queries where `append [ ]` + // yields a LogicalValues(tuples=[[]]) with the union's output schema. + continue; + } + if (!(unwrapped instanceof OpenSearchRelNode openSearchInput)) { + throw new IllegalStateException( + "Union rule encountered unmarked input [" + + unwrapped.getClass().getSimpleName() + + "]. " + + "All inputs must be converted to OpenSearchRelNode before union." + ); + } + markedInputs.add(unwrapped); + if (viableBackends == null) { + viableBackends = new ArrayList<>(openSearchInput.getViableBackends()); + } else { + viableBackends.retainAll(openSearchInput.getViableBackends()); + } + } + + if (markedInputs.isEmpty()) { + // Defensive — Calcite shouldn't construct a Union with all-empty inputs, but + // surfacing a clear message beats letting downstream rules fail mysteriously. + throw new IllegalStateException("Union rule encountered Union with all-empty inputs"); + } + + if (markedInputs.size() == 1) { + // Single non-empty input — collapse the Union. Row type is preserved by + // construction (Calcite requires every Union input to share the row type). + call.transformTo(markedInputs.getFirst()); + return; + } + + List unionCapable = context.getCapabilityRegistry().operatorBackends(EngineCapability.UNION); + viableBackends.retainAll(unionCapable); + + if (viableBackends.isEmpty()) { + throw new IllegalStateException("No backend supports UNION among viable backends after intersecting inputs"); + } + + // Wrap every input in an OpenSearchExchangeReducer so DAGBuilder cuts a + // separate child stage per Union branch. Each child stage is then routed to + // its own shard set (ShardTargetResolver finds the first OpenSearchTableScan + // in its fragment, which now scans only that branch's index) and produces a + // distinct input partition at the coordinator. + // + // RANDOM inputs need the gather; SINGLETON inputs (single-shard tables, FINAL + // aggregate outputs, etc.) are also wrapped — the ER is logically a no-op for + // SINGLETON but the structural cut is what guarantees per-branch stage isolation, + // which is essential when branches reference different indices. + OpenSearchDistributionTraitDef distTraitDef = context.getDistributionTraitDef(); + List reduceViable = CapabilityResolutionUtils.filterByReduceCapability(context.getCapabilityRegistry(), viableBackends); + + List gatheredInputs = new ArrayList<>(markedInputs.size()); + for (RelNode markedInput : markedInputs) { + RelTraitSet singletonTraits = markedInput.getTraitSet().replace(distTraitDef.singleton()); + gatheredInputs.add(new OpenSearchExchangeReducer(union.getCluster(), singletonTraits, markedInput, reduceViable)); + } + + RelTraitSet unionTraits = gatheredInputs.getFirst().getTraitSet().replace(distTraitDef.singleton()); + call.transformTo(new OpenSearchUnion(union.getCluster(), unionTraits, gatheredInputs, union.all, viableBackends)); + } +} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java index 5c0f5c78021e0..fa41b4fc9d5a7 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java @@ -214,15 +214,17 @@ public void testErrorForUnsupportedFieldTypeOperatorCombo() { // ---- Derived columns ---- /** - * HAVING on derived column must throw — marking on derived/expression columns - * is not yet implemented. Verifies the planner fails fast with a clear message - * rather than silently producing incorrect viableBackends. + * HAVING on a derived column (here, the aggregate's {@code total_size} output) + * resolves via the format-agnostic fallback: {@code filterBackendsAnyFormat} + * looks up backends supporting the function on the field type without requiring + * a doc-value or index format. Any backend with the operator + type capability + * is viable. * - * TODO: add testFilterOnAggregateOutput — Filter(Aggregate(Scan)) where the filter - * is on a non-derived column (e.g. group-by key) should succeed and propagate - * viableBackends correctly through the composed pipeline. + *

      This was previously a fail-fast path because the rule had no way to map a + * derived column to a storage format. The fallback unblocks Filter on Union + * outputs, Project outputs, and HAVING on aggregate outputs alike. */ - public void testFilterOnDerivedColumnsAfterAggregateThrows() { + public void testFilterOnDerivedColumnsAfterAggregateResolvesAnyFormat() { PlannerContext context = buildContext("parquet", 1, Map.of("status", Map.of("type", "integer"), "size", Map.of("type", "integer"))); RelOptTable table = mockTable("test_index", "status", "size"); @@ -251,8 +253,23 @@ public void testFilterOnDerivedColumnsAfterAggregateThrows() { ); LogicalFilter having = LogicalFilter.create(aggregate, havingCondition); - UnsupportedOperationException ex = expectThrows(UnsupportedOperationException.class, () -> runPlanner(having, context)); - assertTrue("Expected message about derived column, got: " + ex.getMessage(), ex.getMessage().contains("derived column")); + RelNode result = unwrapExchange(runPlanner(having, context)); + OpenSearchFilter filter = findOpenSearchFilter(result); + assertNotNull("Expected an OpenSearchFilter somewhere in the planned tree, got:\n" + RelOptUtil.toString(result), filter); + assertTrue( + "DataFusion must be a viable backend for HAVING on derived total_size; got " + filter.getViableBackends(), + filter.getViableBackends().contains(MockDataFusionBackend.NAME) + ); + } + + /** Walks the resolved tree top-down and returns the first {@link OpenSearchFilter}, or null. */ + private static OpenSearchFilter findOpenSearchFilter(RelNode node) { + if (node instanceof OpenSearchFilter f) return f; + for (RelNode input : node.getInputs()) { + OpenSearchFilter found = findOpenSearchFilter(input); + if (found != null) return found; + } + return null; } // ---- Helpers ---- diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendCommandIT.java new file mode 100644 index 0000000000000..1139d840a5de4 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendCommandIT.java @@ -0,0 +1,426 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for PPL {@code append} on the analytics-engine route. + * + *

      Mirrors {@code CalcitePPLAppendCommandIT} from the {@code opensearch-project/sql} + * repository so that the analytics-engine path can be verified inside core without + * cross-plugin dependencies on the SQL plugin. Each test sends a PPL query through + * {@code POST /_analytics/ppl} (exposed by the {@code test-ppl-frontend} plugin), which + * runs the same {@code UnifiedQueryPlanner} → {@code CalciteRelNodeVisitor} → Substrait + * → DataFusion pipeline as the SQL plugin's force-routed analytics path. + * + *

      Covers the Append surface forms that exercise: + *

        + *
      • two stats branches sorted + truncated by {@code head N}
        • + *
      • cross-index union (a second copy of the same dataset under a different index name)
        • + *
      • shared output column name across branches (no auto-rename)
        • + *
      • {@code | append [ ]} with several empty-subsearch shapes that all collapse to + * the first branch (bare brackets, inner stats with no source, nested + * {@code | append [ ]}, inner {@code | lookup})
        • + *
      • {@code | append [ | join … ]} — empty-left-side joins + * that also collapse to the first branch
        • + *
      • {@code | append [ … | join … ]} — joins where the right side + * contributes additional rows under the merged schema even though the left + * side is empty
        • + *
      • type-incompatibility error path raised in {@code SchemaUnifier}
        • + *
      + * + *

      Provisions the {@code calcs} dataset twice — once into the {@code calcs} index and + * once into {@code calcs_alt} — so {@code testAppendDifferentIndex} can union across + * indices without pulling in a second dataset. Both indices are parquet-backed via + * {@link DatasetProvisioner}; {@link AnalyticsRestTestCase#preserveIndicesUponCompletion()} + * keeps them across test methods. + */ +public class AppendCommandIT extends AnalyticsRestTestCase { + + private static final Dataset CALCS = new Dataset("calcs", "calcs"); + private static final Dataset CALCS_ALT = new Dataset("calcs", "calcs_alt"); + + private static boolean dataProvisioned = false; + + /** + * Lazily provision both calcs indices on first invocation. Must be called inside a + * test method (not {@code setUp()}) — {@code OpenSearchRestTestCase}'s static + * {@code client()} is not initialized until after {@code @BeforeClass} but is + * reliably available inside test bodies. + */ + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), CALCS); + DatasetProvisioner.provision(client(), CALCS_ALT); + dataProvisioned = true; + } + } + + // ── two stats branches → sort → head ──────────────────────────────────────── + + public void testAppend() throws IOException { + // Branch 1: sum(int0) grouped by str0 (3 rows). Branch 2: sum(int1) grouped + // by str3 (2 rows). Union all + head 5 keeps every row, but the order + // between the two child-stage streams isn't deterministic, so compare as a + // multiset. + assertRowsAnyOrder( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | append [ source=" + + CALCS.indexName + + " | stats sum(int1) as sum_int1_by_str3 by str3 | sort sum_int1_by_str3 ]" + + " | head 5", + row(1, "FURNITURE", null, null), + row(18, "OFFICE SUPPLIES", null, null), + row(49, "TECHNOLOGY", null, null), + row(null, null, -14, null), + row(null, null, -8, "e") + ); + } + + // ── cross-index union ─────────────────────────────────────────────────────── + + public void testAppendDifferentIndex() throws IOException { + // Branch 1: calcs grouped by str0 (3 rows). Branch 2: calcs_alt total sum(int1) + // (1 row). Each branch is its own data-node stage on its own shard set; the two + // streams arrive at the coordinator's Union in non-deterministic order. + assertRowsAnyOrder( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum by str0 | sort str0" + + " | append [ source=" + + CALCS_ALT.indexName + + " | stats sum(int1) as alt_sum_int1 ]", + row(1, "FURNITURE", null), + row(18, "OFFICE SUPPLIES", null), + row(49, "TECHNOLOGY", null), + row(null, null, -22) + ); + } + + // ── shared output column name across branches (no auto-rename) ────────────── + + public void testAppendWithMergedColumn() throws IOException { + // Both branches produce a column named "sum"; SchemaUnifier merges the column + // by name. Group columns differ (str0 vs str3) so each row populates one and + // leaves the other null. Inter-branch order is non-deterministic; head 5 keeps + // every row (3 + 2 = 5) so multiset comparison is exact. + assertRowsAnyOrder( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum by str0 | sort str0" + + " | append [ source=" + + CALCS.indexName + + " | stats sum(int0) as sum by str3 | sort sum ]" + + " | head 5", + row(1, "FURNITURE", null), + row(18, "OFFICE SUPPLIES", null), + row(49, "TECHNOLOGY", null), + row(32, null, null), + row(36, null, "e") + ); + } + + // ── empty subsearch — collapses to first branch ──────────────────────────── + + public void testAppendEmptySearchCommandBareBrackets() throws IOException { + // `| append [ ]` — fully empty subsearch. + assertEmptyAppendOnlyFirstBranch( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | append [ ]" + ); + } + + public void testAppendEmptySearchCommandStatsWithoutSource() throws IOException { + // `| append [ | stats ... ]` — subsearch starts with a pipe, so the implicit + // source is the empty Values relation; the inner stats produces no rows. + assertEmptyAppendOnlyFirstBranch( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | append [ | stats sum(int1) as alt_sum by bool0 ]" + ); + } + + public void testAppendEmptySearchCommandNestedAppend() throws IOException { + // Nested empty append inside a where-only subsearch. + assertEmptyAppendOnlyFirstBranch( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | append [ | where int0 > 5 | append [ ] ]" + ); + } + + public void testAppendEmptySearchCommandLookup() throws IOException { + // `| append [ | where … | lookup INDEX field as alias ]` — lookup against an + // empty implicit source. EmptySourcePropagateVisitor collapses the subsearch + // to LogicalValues(empty), which OpenSearchUnionRule then drops. + assertEmptyAppendOnlyFirstBranch( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | append [ | where int0 > 5 | lookup " + + CALCS.indexName + + " str0 as istr0 ]" + ); + } + + // ── empty subsearch with join (5 join types; 4 collapse to first branch) ─── + + public void testAppendEmptySearchWithInnerJoin() throws IOException { + assertEmptyAppendOnlyFirstBranch( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | append [ | join left=L right=R on L.str0 = R.str0 " + + CALCS.indexName + + " ]" + ); + } + + public void testAppendEmptySearchWithCrossJoin() throws IOException { + assertEmptyAppendOnlyFirstBranch( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | append [ | cross join left=L right=R on L.str0 = R.str0 " + + CALCS.indexName + + " ]" + ); + } + + public void testAppendEmptySearchWithLeftJoin() throws IOException { + assertEmptyAppendOnlyFirstBranch( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | append [ | left join left=L right=R on L.str0 = R.str0 " + + CALCS.indexName + + " ]" + ); + } + + public void testAppendEmptySearchWithSemiJoin() throws IOException { + assertEmptyAppendOnlyFirstBranch( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | append [ | semi join left=L right=R on L.str0 = R.str0 " + + CALCS.indexName + + " ]" + ); + } + + // ── empty subsearch with right/full join — adds rows from the right side ─── + + public void testAppendEmptySearchWithRightJoin() throws IOException { + // RIGHT JOIN of (empty filtered subset, real subquery) → still emits every + // right-side row with NULL on the left columns. The append therefore yields + // the first branch's rows plus the right-subquery's rows under the merged + // schema (sum_int0_by_str0 / str0 / cnt). Inter-branch order is non-deterministic. + assertRowsAnyOrder( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | append [ | where str0 = 'OFFICE SUPPLIES' | right join on str0 = str0 [source=" + + CALCS.indexName + + " | stats count() as cnt by str0 | sort str0 ] ]", + row(1, "FURNITURE", null), + row(18, "OFFICE SUPPLIES", null), + row(49, "TECHNOLOGY", null), + row(null, "FURNITURE", 2), + row(null, "OFFICE SUPPLIES", 6), + row(null, "TECHNOLOGY", 9) + ); + } + + public void testAppendEmptySearchWithFullJoin() throws IOException { + // Same shape as right join — the empty left side has no rows to match, so + // FULL JOIN reduces to RIGHT JOIN here. Inter-branch order is non-deterministic. + assertRowsAnyOrder( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | append [ | where str0 = 'OFFICE SUPPLIES' | full join on str0 = str0 [source=" + + CALCS.indexName + + " | stats count() as cnt by str0 | sort str0 ] ]", + row(1, "FURNITURE", null), + row(18, "OFFICE SUPPLIES", null), + row(49, "TECHNOLOGY", null), + row(null, "FURNITURE", 2), + row(null, "OFFICE SUPPLIES", 6), + row(null, "TECHNOLOGY", 9) + ); + } + + // ── type-incompatibility error raised in SchemaUnifier ───────────────────── + + public void testAppendWithConflictTypeColumn() { + // Branch 1 produces "sum" as BIGINT; branch 2 casts "sum" to DOUBLE. Schema + // unification refuses to merge the diverging types and surfaces a planner + // error before execution. + assertErrorContains( + "source=" + + CALCS.indexName + + " | stats sum(int0) as sum by str0 | sort str0" + + " | append [ source=" + + CALCS.indexName + + " | stats sum(int0) as sum by str3 | sort sum" + + " | eval sum = cast(sum as double) ]" + + " | head 5", + "Unable to process column 'sum' due to incompatible types" + ); + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + /** Construct an expected row from positional values matching the PPL output column order. */ + private static List row(Object... values) { + return Arrays.asList(values); + } + + /** + * The four empty-subsearch shapes share the same expected first-branch-only output; + * factored to keep the four test methods readable. + */ + private void assertEmptyAppendOnlyFirstBranch(String ppl) throws IOException { + assertRows(ppl, row(1, "FURNITURE"), row(18, "OFFICE SUPPLIES"), row(49, "TECHNOLOGY")); + } + + /** + * Send a PPL query to {@code POST /_analytics/ppl} and assert the response's + * {@code rows} match the expected list element-by-element using a numeric-tolerant + * comparator (Java JSON parsing returns Integer/Long/Double interchangeably). + */ + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRows(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' field for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size()); + for (int i = 0; i < expected.length; i++) { + List want = expected[i]; + List got = actualRows.get(i); + assertEquals("Column count mismatch at row " + i + " for query: " + ppl, want.size(), got.size()); + for (int j = 0; j < want.size(); j++) { + assertCellEquals("Cell mismatch at row " + i + ", col " + j + " for query: " + ppl, want.get(j), got.get(j)); + } + } + } + + /** + * Multiset variant of {@link #assertRows} for queries whose row order is not + * deterministic. Substrait's {@code Set} (Union) rel preserves order within a + * single input partition but not between partitions: the two child stages of + * an {@code | append} pipeline can stream into the coordinator sink in either + * order depending on shard scheduling timing, so a {@code | head N} on top of + * a Union may pick different rows across runs (or the same rows in different + * orders). + * + *

      Cell values are normalised to a canonical string form before comparison — + * numeric types collapse to a {@code Double} so JSON-parsed + * {@code Integer}/{@code Long}/{@code Double} compare equal across the Java side + * even when their boxed types differ. Rows are then compared as a sorted multiset. + */ + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRowsAnyOrder(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' field for query: " + ppl, actualRows); + List expectedNormalized = Arrays.stream(expected).map(AppendCommandIT::normalizeRow).sorted().toList(); + List actualNormalized = actualRows.stream().map(AppendCommandIT::normalizeRow).sorted().toList(); + assertEquals("Row multisets differ for query: " + ppl, expectedNormalized, actualNormalized); + } + + /** Renders one row to a stable canonical string for multiset comparison. */ + private static String normalizeRow(List row) { + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < row.size(); i++) { + if (i > 0) sb.append('|'); + sb.append(normalizeCell(row.get(i))); + } + return sb.append(']').toString(); + } + + private static String normalizeCell(Object cell) { + if (cell == null) return ""; + if (cell instanceof Number) return Double.toString(((Number) cell).doubleValue()); + return cell.toString(); + } + + /** + * Send a PPL query expecting the planner to reject it; assert the resulting HTTP + * error body contains {@code expectedSubstring} (typically the validation message). + */ + private void assertErrorContains(String ppl, String expectedSubstring) { + try { + Map response = executePpl(ppl); + fail("Expected query to fail with [" + expectedSubstring + "] but got response: " + response); + } catch (ResponseException e) { + String body; + try { + body = org.opensearch.test.rest.OpenSearchRestTestCase.entityAsMap(e.getResponse()).toString(); + } catch (IOException ioe) { + body = e.getMessage(); + } + assertTrue( + "Expected response body to contain [" + expectedSubstring + "] but was: " + body, + body.contains(expectedSubstring) + ); + } catch (IOException e) { + fail("Unexpected IOException: " + e); + } + } + + /** Send {@code POST /_analytics/ppl} and return the parsed JSON body. */ + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } + + /** + * Compare two cells with numeric tolerance — JSON parsing produces + * Integer/Long/Double values that may not match {@code .equals()} across types + * even when numerically equal. + */ + private static void assertCellEquals(String message, Object expected, Object actual) { + if (expected == null || actual == null) { + assertEquals(message, expected, actual); + return; + } + if (expected instanceof Number && actual instanceof Number) { + double e = ((Number) expected).doubleValue(); + double a = ((Number) actual).doubleValue(); + if (Double.compare(e, a) != 0) { + fail(message + ": expected <" + expected + "> but was <" + actual + ">"); + } + return; + } + assertEquals(message, expected, actual); + } +} From a70e8ee061c4158a43b7f31d62589952dd425627 Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Tue, 5 May 2026 12:04:19 -0700 Subject: [PATCH 050/115] Refactor WLM settings to use Setting objects and rename field (#21143) Signed-off-by: David Zane --- .../rest/WorkloadManagementRestIT.java | 70 +++++------ .../wlm/WorkloadManagementTestUtils.java | 2 +- .../CreateWorkloadGroupResponseTests.java | 46 +++---- .../action/GetWorkloadGroupResponseTests.java | 10 +- .../UpdateWorkloadGroupResponseTests.java | 46 +++---- .../cluster/metadata/WorkloadGroup.java | 30 ++--- .../opensearch/common/settings/Settings.java | 24 ++++ .../wlm/MutableWorkloadGroupFragment.java | 86 ++++++++------ .../wlm/WorkloadGroupSearchSettings.java | 112 ++++++------------ ...WorkloadGroupRequestOperationListener.java | 30 ++--- .../metadata/WorkloadGroupMetadataTests.java | 12 +- .../cluster/metadata/WorkloadGroupTests.java | 20 +++- .../wlm/WorkloadGroupSearchSettingsTests.java | 100 +++++----------- ...oadGroupRequestOperationListenerTests.java | 11 +- 14 files changed, 276 insertions(+), 323 deletions(-) diff --git a/plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java b/plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java index 47f9c8eb63dad..826faa393f8fc 100644 --- a/plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java +++ b/plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java @@ -148,77 +148,65 @@ public void testOperationWhenWlmDisabled() throws Exception { } public void testSearchSettings() throws Exception { - // Create with search_settings + // Create with settings String createJson = """ { "name": "search_test", "resiliency_mode": "enforced", "resource_limits": {"cpu": 0.3, "memory": 0.3}, - "search_settings": { - "timeout": "30s" + "settings": { + "search.default_search_timeout": "30s" } }"""; Response response = performOperation("PUT", "_wlm/workload_group", createJson); assertEquals(200, response.getStatusLine().getStatusCode()); - // Verify search_settings in GET response + // Verify settings in GET response Response getResponse = performOperation("GET", "_wlm/workload_group/search_test", null); String responseBody = EntityUtils.toString(getResponse.getEntity()); - assertTrue(responseBody.contains("\"search_settings\"")); - assertTrue(responseBody.contains("\"timeout\":\"30s\"")); + assertTrue(responseBody.contains("\"settings\"")); + assertTrue(responseBody.contains("\"search.default_search_timeout\":\"30s\"")); - // Update search_settings + // Update settings String updateJson = """ { - "search_settings": { - "timeout": "1m" + "settings": { + "search.default_search_timeout": "1m" } }"""; Response updateResponse = performOperation("PUT", "_wlm/workload_group/search_test", updateJson); assertEquals(200, updateResponse.getStatusLine().getStatusCode()); - // Verify updated search_settings + // Verify updated settings Response getResponse2 = performOperation("GET", "_wlm/workload_group/search_test", null); String responseBody2 = EntityUtils.toString(getResponse2.getEntity()); - assertTrue(responseBody2.contains("\"timeout\":\"1m\"")); + assertTrue(responseBody2.contains("\"search.default_search_timeout\":\"1m\"")); performOperation("DELETE", "_wlm/workload_group/search_test", null); } static String getCreateJson(String name, String resiliencyMode, double cpu, double memory) { - return "{\n" - + " \"name\": \"" - + name - + "\",\n" - + " \"resiliency_mode\": \"" - + resiliencyMode - + "\",\n" - + " \"resource_limits\": {\n" - + " \"cpu\" : " - + cpu - + ",\n" - + " \"memory\" : " - + memory - + "\n" - + " },\n" - + " \"search_settings\": {}\n" - + "}"; + return String.format(Locale.ROOT, """ + { + "name": "%s", + "resiliency_mode": "%s", + "resource_limits": { + "cpu" : %s, + "memory" : %s + }, + "settings": {} + }""", name, resiliencyMode, cpu, memory); } static String getUpdateJson(String resiliencyMode, double cpu, double memory) { - return "{\n" - + " \"resiliency_mode\": \"" - + resiliencyMode - + "\",\n" - + " \"resource_limits\": {\n" - + " \"cpu\" : " - + cpu - + ",\n" - + " \"memory\" : " - + memory - + "\n" - + " }\n" - + "}"; + return String.format(Locale.ROOT, """ + { + "resiliency_mode": "%s", + "resource_limits": { + "cpu" : %s, + "memory" : %s + } + }""", resiliencyMode, cpu, memory); } Response performOperation(String method, String uriPath, String json) throws IOException { diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementTestUtils.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementTestUtils.java index b39b8e8486def..14159e82368ac 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementTestUtils.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/WorkloadManagementTestUtils.java @@ -51,7 +51,7 @@ public class WorkloadManagementTestUtils { public static final long TIMESTAMP_ONE = 4513232413L; public static final long TIMESTAMP_TWO = 4513232415L; public static final long TIMESTAMP_THREE = 4513232417L; - public static final Map TEST_SEARCH_SETTINGS = Map.of("timeout", "30s"); + public static final Settings TEST_SEARCH_SETTINGS = Settings.builder().put("search.default_search_timeout", "30s").build(); public static final WorkloadGroup workloadGroupOne = builder().name(NAME_ONE) ._id(_ID_ONE) .mutableWorkloadGroupFragment( diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupResponseTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupResponseTests.java index 4055f46edb43e..fc59f7d36d5a1 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupResponseTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateWorkloadGroupResponseTests.java @@ -52,16 +52,17 @@ public void testToXContentCreateWorkloadGroup() throws IOException { XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); CreateWorkloadGroupResponse response = new CreateWorkloadGroupResponse(WorkloadManagementTestUtils.workloadGroupOne, RestStatus.OK); String actual = response.toXContent(builder, mock(ToXContent.Params.class)).toString(); - String expected = "{\n" - + " \"_id\" : \"AgfUO5Ja9yfsYlONlYi3TQ==\",\n" - + " \"name\" : \"workload_group_one\",\n" - + " \"resiliency_mode\" : \"monitor\",\n" - + " \"resource_limits\" : {\n" - + " \"memory\" : 0.3\n" - + " },\n" - + " \"search_settings\" : { },\n" - + " \"updated_at\" : 4513232413\n" - + "}"; + String expected = """ + { + "_id" : "AgfUO5Ja9yfsYlONlYi3TQ==", + "name" : "workload_group_one", + "resiliency_mode" : "monitor", + "resource_limits" : { + "memory" : 0.3 + }, + "settings" : { }, + "updated_at" : 4513232413 + }"""; assertEquals(expected, actual); } @@ -75,18 +76,19 @@ public void testToXContentCreateWorkloadGroupWithSearchSettings() throws IOExcep RestStatus.OK ); String actual = response.toXContent(builder, mock(ToXContent.Params.class)).toString(); - String expected = "{\n" - + " \"_id\" : \"H6jVP6Kb0zgtZmPOmZj4UQ==\",\n" - + " \"name\" : \"workload_group_three\",\n" - + " \"resiliency_mode\" : \"enforced\",\n" - + " \"resource_limits\" : {\n" - + " \"memory\" : 0.5\n" - + " },\n" - + " \"search_settings\" : {\n" - + " \"timeout\" : \"30s\"\n" - + " },\n" - + " \"updated_at\" : 4513232417\n" - + "}"; + String expected = """ + { + "_id" : "H6jVP6Kb0zgtZmPOmZj4UQ==", + "name" : "workload_group_three", + "resiliency_mode" : "enforced", + "resource_limits" : { + "memory" : 0.5 + }, + "settings" : { + "search.default_search_timeout" : "30s" + }, + "updated_at" : 4513232417 + }"""; assertEquals(expected, actual); } } diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupResponseTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupResponseTests.java index d5c4303186983..aa86953239202 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupResponseTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetWorkloadGroupResponseTests.java @@ -97,7 +97,7 @@ public void testToXContentGetSingleWorkloadGroup() throws IOException { "resource_limits" : { "memory" : 0.3 }, - "search_settings" : { }, + "settings" : { }, "updated_at" : 4513232413 } ] @@ -125,7 +125,7 @@ public void testToXContentGetMultipleWorkloadGroup() throws IOException { "resource_limits" : { "memory" : 0.3 }, - "search_settings" : { }, + "settings" : { }, "updated_at" : 4513232413 }, { @@ -135,7 +135,7 @@ public void testToXContentGetMultipleWorkloadGroup() throws IOException { "resource_limits" : { "memory" : 0.6 }, - "search_settings" : { }, + "settings" : { }, "updated_at" : 4513232415 } ] @@ -176,8 +176,8 @@ public void testToXContentGetWorkloadGroupWithSearchSettings() throws IOExceptio "resource_limits" : { "memory" : 0.5 }, - "search_settings" : { - "timeout" : "30s" + "settings" : { + "search.default_search_timeout" : "30s" }, "updated_at" : 4513232417 } diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupResponseTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupResponseTests.java index 7fb46e2128228..a6c4b41c8c8cc 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupResponseTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateWorkloadGroupResponseTests.java @@ -53,16 +53,17 @@ public void testToXContentUpdateSingleWorkloadGroup() throws IOException { XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); UpdateWorkloadGroupResponse otherResponse = new UpdateWorkloadGroupResponse(workloadGroupOne, RestStatus.OK); String actual = otherResponse.toXContent(builder, mock(ToXContent.Params.class)).toString(); - String expected = "{\n" - + " \"_id\" : \"AgfUO5Ja9yfsYlONlYi3TQ==\",\n" - + " \"name\" : \"workload_group_one\",\n" - + " \"resiliency_mode\" : \"monitor\",\n" - + " \"resource_limits\" : {\n" - + " \"memory\" : 0.3\n" - + " },\n" - + " \"search_settings\" : { },\n" - + " \"updated_at\" : 4513232413\n" - + "}"; + String expected = """ + { + "_id" : "AgfUO5Ja9yfsYlONlYi3TQ==", + "name" : "workload_group_one", + "resiliency_mode" : "monitor", + "resource_limits" : { + "memory" : 0.3 + }, + "settings" : { }, + "updated_at" : 4513232413 + }"""; assertEquals(expected, actual); } @@ -76,18 +77,19 @@ public void testToXContentUpdateWorkloadGroupWithSearchSettings() throws IOExcep RestStatus.OK ); String actual = response.toXContent(builder, mock(ToXContent.Params.class)).toString(); - String expected = "{\n" - + " \"_id\" : \"H6jVP6Kb0zgtZmPOmZj4UQ==\",\n" - + " \"name\" : \"workload_group_three\",\n" - + " \"resiliency_mode\" : \"enforced\",\n" - + " \"resource_limits\" : {\n" - + " \"memory\" : 0.5\n" - + " },\n" - + " \"search_settings\" : {\n" - + " \"timeout\" : \"30s\"\n" - + " },\n" - + " \"updated_at\" : 4513232417\n" - + "}"; + String expected = """ + { + "_id" : "H6jVP6Kb0zgtZmPOmZj4UQ==", + "name" : "workload_group_three", + "resiliency_mode" : "enforced", + "resource_limits" : { + "memory" : 0.5 + }, + "settings" : { + "search.default_search_timeout" : "30s" + }, + "updated_at" : 4513232417 + }"""; assertEquals(expected, actual); } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/WorkloadGroup.java b/server/src/main/java/org/opensearch/cluster/metadata/WorkloadGroup.java index 4b7da52631231..ee77f412e4407 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/WorkloadGroup.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/WorkloadGroup.java @@ -13,6 +13,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; @@ -73,12 +74,12 @@ public WorkloadGroup(String name, String _id, MutableWorkloadGroupFragment mutab throw new IllegalArgumentException("WorkloadGroup.updatedAtInMillis is not a valid epoch"); } - // Normalize null searchSettings to empty map for storage - if (mutableWorkloadGroupFragment.getSearchSettings() == null) { + // Normalize null settings to empty Settings for storage + if (mutableWorkloadGroupFragment.getSettings() == null) { mutableWorkloadGroupFragment = new MutableWorkloadGroupFragment( mutableWorkloadGroupFragment.getResiliencyMode(), mutableWorkloadGroupFragment.getResourceLimits(), - new HashMap<>() + Settings.EMPTY ); } @@ -113,23 +114,23 @@ public static WorkloadGroup updateExistingWorkloadGroup( } final ResiliencyMode mode = Optional.ofNullable(mutableWorkloadGroupFragment.getResiliencyMode()) .orElse(existingGroup.getResiliencyMode()); - // Handle search_settings update: + // Handle settings update: // null = not specified (keep existing) - // empty map = explicitly clear (set to empty) - // non-empty map = replace with new values - final Map mutableFragmentSearchSettings = mutableWorkloadGroupFragment.getSearchSettings(); - final Map updatedSearchSettings; - if (mutableFragmentSearchSettings == null) { + // empty Settings = explicitly clear (set to empty) + // non-empty Settings = replace with new values + final Settings mutableFragmentSettings = mutableWorkloadGroupFragment.getSettings(); + final Settings updatedSettings; + if (mutableFragmentSettings == null) { // Not specified - keep existing - updatedSearchSettings = new HashMap<>(existingGroup.getSearchSettings()); + updatedSettings = Settings.builder().put(existingGroup.getSettings()).build(); } else { // Specified (empty or non-empty) - use the new value - updatedSearchSettings = new HashMap<>(mutableFragmentSearchSettings); + updatedSettings = Settings.builder().put(mutableFragmentSettings).build(); } return new WorkloadGroup( existingGroup.getName(), existingGroup.get_id(), - new MutableWorkloadGroupFragment(mode, updatedResourceLimits, updatedSearchSettings), + new MutableWorkloadGroupFragment(mode, updatedResourceLimits, updatedSettings), Instant.now().getMillis() ); } @@ -201,8 +202,9 @@ public Map getResourceLimits() { return getMutableWorkloadGroupFragment().getResourceLimits(); } - public Map getSearchSettings() { - return getMutableWorkloadGroupFragment().getSearchSettings(); + @ExperimentalApi + public Settings getSettings() { + return getMutableWorkloadGroupFragment().getSettings(); } public String get_id() { diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index 9da47ff3aa700..146b0f23c6129 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -589,6 +589,30 @@ public static void writeSettingsToStream(Settings settings, StreamOutput out) th } } + /** + * Reads an optional {@link Settings} from the stream. Returns {@code null} if no settings were written. + * Counterpart to {@link #writeOptionalSettingsToStream(Settings, StreamOutput)}. + */ + public static Settings readOptionalSettingsFromStream(StreamInput in) throws IOException { + if (in.readBoolean()) { + return readSettingsFromStream(in); + } + return null; + } + + /** + * Writes an optional {@link Settings} to the stream. A {@code null} value is permitted. + * Counterpart to {@link #readOptionalSettingsFromStream(StreamInput)}. + */ + public static void writeOptionalSettingsToStream(Settings settings, StreamOutput out) throws IOException { + if (settings != null) { + out.writeBoolean(true); + writeSettingsToStream(settings, out); + } else { + out.writeBoolean(false); + } + } + /** * Returns a builder to be used in order to build settings. */ diff --git a/server/src/main/java/org/opensearch/wlm/MutableWorkloadGroupFragment.java b/server/src/main/java/org/opensearch/wlm/MutableWorkloadGroupFragment.java index f063c58abd9b7..c87658cf72b40 100644 --- a/server/src/main/java/org/opensearch/wlm/MutableWorkloadGroupFragment.java +++ b/server/src/main/java/org/opensearch/wlm/MutableWorkloadGroupFragment.java @@ -11,6 +11,7 @@ import org.opensearch.Version; import org.opensearch.cluster.AbstractDiffable; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; @@ -34,12 +35,12 @@ public class MutableWorkloadGroupFragment extends AbstractDiffable resourceLimits; - private Map searchSettings; + private Settings settings; - public static final List acceptedFieldNames = List.of(RESILIENCY_MODE_STRING, RESOURCE_LIMITS_STRING, SEARCH_SETTINGS_STRING); + public static final List acceptedFieldNames = List.of(RESILIENCY_MODE_STRING, RESOURCE_LIMITS_STRING, SETTINGS_STRING); public MutableWorkloadGroupFragment() {} @@ -47,19 +48,15 @@ public MutableWorkloadGroupFragment() {} * Constructor for tests only. Production code should use the full constructor below. */ public MutableWorkloadGroupFragment(ResiliencyMode resiliencyMode, Map resourceLimits) { - this(resiliencyMode, resourceLimits, new HashMap<>()); + this(resiliencyMode, resourceLimits, Settings.EMPTY); } - public MutableWorkloadGroupFragment( - ResiliencyMode resiliencyMode, - Map resourceLimits, - Map searchSettings - ) { + public MutableWorkloadGroupFragment(ResiliencyMode resiliencyMode, Map resourceLimits, Settings settings) { validateResourceLimits(resourceLimits); - WorkloadGroupSearchSettings.validateSearchSettings(searchSettings); + WorkloadGroupSearchSettings.validate(settings); this.resiliencyMode = resiliencyMode; this.resourceLimits = resourceLimits; - this.searchSettings = searchSettings; + this.settings = settings; } public MutableWorkloadGroupFragment(StreamInput in) throws IOException { @@ -70,12 +67,17 @@ public MutableWorkloadGroupFragment(StreamInput in) throws IOException { } String updatedResiliencyMode = in.readOptionalString(); resiliencyMode = updatedResiliencyMode == null ? null : ResiliencyMode.fromName(updatedResiliencyMode); - if (in.getVersion().onOrAfter(Version.V_3_6_0)) { - // Read null marker: true means searchSettings is null (not specified) + if (in.getVersion().onOrAfter(Version.V_3_7_0)) { + settings = Settings.readOptionalSettingsFromStream(in); + } else if (in.getVersion().onOrAfter(Version.V_3_6_0)) { + // Legacy 3.6 format: read and discard (experimental API, no backward compat guarantee) boolean isNull = in.readBoolean(); - searchSettings = isNull ? null : in.readMap(StreamInput::readString, StreamInput::readString); + if (isNull == false) { + in.readMap(StreamInput::readString, StreamInput::readString); + } + settings = Settings.EMPTY; } else { - searchSettings = new HashMap<>(); + settings = Settings.EMPTY; } } @@ -105,9 +107,11 @@ public Map parseField(XContentParser parser) throws IOExce } } - static class SearchSettingsParser implements FieldParser> { - public Map parseField(XContentParser parser) throws IOException { - return parser.mapStrings(); + static class SearchSettingsParser implements FieldParser { + public Settings parseField(XContentParser parser) throws IOException { + Settings settings = Settings.fromXContent(parser); + WorkloadGroupSearchSettings.validate(settings); + return settings; } } @@ -116,7 +120,7 @@ static Optional> fieldParserFor(String fieldName) { return switch (fieldName) { case RESILIENCY_MODE_STRING -> Optional.of(new ResiliencyModeParser()); case RESOURCE_LIMITS_STRING -> Optional.of(new ResourceLimitsParser()); - case SEARCH_SETTINGS_STRING -> Optional.of(new SearchSettingsParser()); + case SETTINGS_STRING -> Optional.of(new SearchSettingsParser()); default -> Optional.empty(); }; } @@ -142,18 +146,21 @@ static Optional> fieldParserFor(String fieldName) { } catch (IOException e) { throw new IllegalStateException("writing error encountered for the field " + RESOURCE_LIMITS_STRING); } - }, SEARCH_SETTINGS_STRING, (builder) -> { + }, SETTINGS_STRING, (builder) -> { try { - builder.startObject(SEARCH_SETTINGS_STRING); - Map settings = searchSettings != null ? searchSettings : Map.of(); - Map sortedSettingsMap = new TreeMap<>(settings); - for (Map.Entry e : sortedSettingsMap.entrySet()) { + builder.startObject(SETTINGS_STRING); + Settings s = settings != null ? settings : Settings.EMPTY; + Map sortedSettingsMap = new TreeMap<>(); + for (String key : s.keySet()) { + sortedSettingsMap.put(key, s.get(key)); + } + for (Map.Entry e : sortedSettingsMap.entrySet()) { builder.field(e.getKey(), e.getValue()); } builder.endObject(); return null; } catch (IOException e) { - throw new IllegalStateException("writing error encountered for the field " + SEARCH_SETTINGS_STRING); + throw new IllegalStateException("writing error encountered for the field " + SETTINGS_STRING); } }); @@ -169,8 +176,10 @@ public void parseField(XContentParser parser, String field) { switch (field) { case RESILIENCY_MODE_STRING -> setResiliencyMode((ResiliencyMode) value); case RESOURCE_LIMITS_STRING -> setResourceLimits((Map) value); - case SEARCH_SETTINGS_STRING -> setSearchSettings((Map) value); + case SETTINGS_STRING -> setSettings((Settings) value); } + } catch (IllegalArgumentException e) { + throw e; } catch (IOException e) { throw new IllegalArgumentException(String.format(Locale.ROOT, "parsing error encountered for the field '%s'", field)); } @@ -190,11 +199,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(resourceLimits, ResourceType::writeTo, StreamOutput::writeDouble); } out.writeOptionalString(resiliencyMode == null ? null : resiliencyMode.getName()); - if (out.getVersion().onOrAfter(Version.V_3_6_0)) { - out.writeBoolean(searchSettings == null); - if (searchSettings != null) { - out.writeMap(searchSettings, StreamOutput::writeString, StreamOutput::writeString); - } + if (out.getVersion().onOrAfter(Version.V_3_7_0)) { + Settings.writeOptionalSettingsToStream(settings, out); + } else if (out.getVersion().onOrAfter(Version.V_3_6_0)) { + // Legacy 3.6 format: write empty map (experimental API, settings not preserved across versions) + out.writeBoolean(false); + out.writeMap(Map.of(), StreamOutput::writeString, StreamOutput::writeString); } } @@ -220,12 +230,12 @@ public boolean equals(Object o) { MutableWorkloadGroupFragment that = (MutableWorkloadGroupFragment) o; return Objects.equals(resiliencyMode, that.resiliencyMode) && Objects.equals(resourceLimits, that.resourceLimits) - && Objects.equals(searchSettings, that.searchSettings); + && Objects.equals(settings, that.settings); } @Override public int hashCode() { - return Objects.hash(resiliencyMode, resourceLimits, searchSettings); + return Objects.hash(resiliencyMode, resourceLimits, settings); } public ResiliencyMode getResiliencyMode() { @@ -236,8 +246,8 @@ public Map getResourceLimits() { return resourceLimits; } - public Map getSearchSettings() { - return searchSettings; + public Settings getSettings() { + return settings; } /** @@ -280,8 +290,8 @@ void setResourceLimits(Map resourceLimits) { this.resourceLimits = resourceLimits; } - void setSearchSettings(Map searchSettings) { - WorkloadGroupSearchSettings.validateSearchSettings(searchSettings); - this.searchSettings = searchSettings; + void setSettings(Settings settings) { + WorkloadGroupSearchSettings.validate(settings); + this.settings = settings; } } diff --git a/server/src/main/java/org/opensearch/wlm/WorkloadGroupSearchSettings.java b/server/src/main/java/org/opensearch/wlm/WorkloadGroupSearchSettings.java index 5dc66ab8babd0..3974140e21a70 100644 --- a/server/src/main/java/org/opensearch/wlm/WorkloadGroupSearchSettings.java +++ b/server/src/main/java/org/opensearch/wlm/WorkloadGroupSearchSettings.java @@ -8,14 +8,20 @@ package org.opensearch.wlm; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import java.util.Map; -import java.util.function.Function; /** - * Registry of valid workload group search settings with their validators + * Registry of valid workload group settings with their validators. + *

      + * Each WLM setting is defined as a {@link Setting} object with proper type validation, + * default values, and documentation. */ +@ExperimentalApi public class WorkloadGroupSearchSettings { /** @@ -26,92 +32,48 @@ private WorkloadGroupSearchSettings() { } /** - * Enum defining valid workload group search settings with their validation logic. - * Settings are categorized as either query parameters or cluster settings. + * The WLM search timeout setting. Uses the same key as the cluster-level setting + * {@code search.default_search_timeout}. A value of -1 (MINUS_ONE) means no timeout. */ - public enum WlmSearchSetting { - // Query parameters (applied to SearchRequest) - /** Setting for search request timeout */ - TIMEOUT("timeout", WorkloadGroupSearchSettings::validateTimeValue); + public static final Setting WLM_SEARCH_TIMEOUT = Setting.timeSetting("search.default_search_timeout", TimeValue.MINUS_ONE); - private final String settingName; - private final Function validator; - - WlmSearchSetting(String settingName, Function validator) { - this.settingName = settingName; - this.validator = validator; - } - - /** - * Returns the setting name. - * @return the setting name - */ - public String getSettingName() { - return settingName; - } - - /** - * Validates the given value for this setting. - * @param value the value to validate - * @throws IllegalArgumentException if the value is invalid - */ - void validate(String value) { - String error = validator.apply(value); - if (error != null) { - throw new IllegalArgumentException("Invalid value '" + value + "' for " + settingName + ": " + error); - } - } - - /** - * Finds a setting by its name. - * @param settingName the setting name - * @return the setting or null if not found - */ - public static WlmSearchSetting fromKey(String settingName) { - for (WlmSearchSetting setting : values()) { - if (setting.settingName.equals(settingName)) { - return setting; - } - } - return null; - } - } + /** + * All registered WLM settings, keyed by their canonical key name. + */ + private static final Map> REGISTERED_SETTINGS = Map.of("search.default_search_timeout", WLM_SEARCH_TIMEOUT); /** - * Validates all search settings in the provided map. - * @param searchSettings map of setting names to values - * @throws IllegalArgumentException if any setting is unknown or invalid + * Validates a {@link Settings} object against registered WLM settings. + * All keys in the settings must be registered, and all values must pass type validation. + * + * @param settings the settings to validate + * @throws IllegalArgumentException if any key is unknown or any value is invalid */ - public static void validateSearchSettings(Map searchSettings) { - if (searchSettings == null) { + public static void validate(Settings settings) { + if (settings == null) { return; } - for (Map.Entry entry : searchSettings.entrySet()) { - if (entry.getKey() == null) { - throw new IllegalArgumentException("Search setting key cannot be null"); - } - if (entry.getValue() == null) { - throw new IllegalArgumentException("Search setting value cannot be null for key: " + entry.getKey()); - } - WlmSearchSetting setting = WlmSearchSetting.fromKey(entry.getKey()); + for (String key : settings.keySet()) { + String value = settings.get(key); + Setting setting = REGISTERED_SETTINGS.get(key); if (setting == null) { - throw new IllegalArgumentException("Unknown search setting: " + entry.getKey()); + throw new IllegalArgumentException("Unknown WLM setting: " + key); + } + try { + Settings testSettings = Settings.builder().put(key, value).build(); + setting.get(testSettings); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid value '" + value + "' for " + key + ": " + e.getMessage()); } - setting.validate(entry.getValue()); } } /** - * Validates a time value string. - * @param value the string to validate - * @return null if valid, error message if invalid + * Returns an unmodifiable view of the registered settings. + * + * @return map of canonical key names to their {@link Setting} objects */ - private static String validateTimeValue(String value) { - try { - TimeValue.parseTimeValue(value, "validation"); - return null; - } catch (Exception e) { - return e.getMessage(); - } + public static Map> getRegisteredSettings() { + return REGISTERED_SETTINGS; } } diff --git a/server/src/main/java/org/opensearch/wlm/listeners/WorkloadGroupRequestOperationListener.java b/server/src/main/java/org/opensearch/wlm/listeners/WorkloadGroupRequestOperationListener.java index 8c0010f539550..31221f95113eb 100644 --- a/server/src/main/java/org/opensearch/wlm/listeners/WorkloadGroupRequestOperationListener.java +++ b/server/src/main/java/org/opensearch/wlm/listeners/WorkloadGroupRequestOperationListener.java @@ -15,14 +15,13 @@ import org.opensearch.action.search.SearchRequestContext; import org.opensearch.action.search.SearchRequestOperationsListener; import org.opensearch.cluster.metadata.WorkloadGroup; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.threadpool.ThreadPool; import org.opensearch.wlm.WorkloadGroupSearchSettings; import org.opensearch.wlm.WorkloadGroupService; import org.opensearch.wlm.WorkloadGroupTask; -import java.util.Map; - /** * This listener is used to listen for request lifecycle events for a workloadGroup */ @@ -73,30 +72,15 @@ private void applyWorkloadGroupSearchSettings(String workloadGroupId, SearchRequ return; } - // Loop through WLM group search settings and apply them as needed - for (Map.Entry entry : workloadGroup.getSearchSettings().entrySet()) { + Settings wlmSettings = workloadGroup.getSettings(); + if (wlmSettings != null && wlmSettings.hasValue(WorkloadGroupSearchSettings.WLM_SEARCH_TIMEOUT.getKey())) { try { - WorkloadGroupSearchSettings.WlmSearchSetting settingKey = WorkloadGroupSearchSettings.WlmSearchSetting.fromKey( - entry.getKey() - ); - if (settingKey == null) continue; - - switch (settingKey) { - case TIMEOUT: - // Only apply WLM timeout when the request has no explicit timeout - if (searchRequest.source() != null && searchRequest.source().timeout() == null) { - searchRequest.source() - .timeout( - TimeValue.parseTimeValue( - entry.getValue(), - WorkloadGroupSearchSettings.WlmSearchSetting.TIMEOUT.getSettingName() - ) - ); - } - break; + TimeValue timeout = WorkloadGroupSearchSettings.WLM_SEARCH_TIMEOUT.get(wlmSettings); + if (searchRequest.source() != null && searchRequest.source().timeout() == null) { + searchRequest.source().timeout(timeout); } } catch (Exception e) { - logger.error("Failed to apply workload group setting [{}={}]: {}", entry.getKey(), entry.getValue(), e); + logger.error("Failed to apply workload group settings", e); } } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/WorkloadGroupMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/WorkloadGroupMetadataTests.java index 111901e934f7f..c9781de27595b 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/WorkloadGroupMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/WorkloadGroupMetadataTests.java @@ -48,10 +48,14 @@ public void testToXContent() throws IOException { builder.startObject(); workloadGroupMetadata.toXContent(builder, null); builder.endObject(); - String expected = "{\"ajakgakg983r92_4242\":{\"_id\":\"ajakgakg983r92_4242\",\"name\":\"test\"," - + "\"resiliency_mode\":\"enforced\",\"resource_limits\":{\"memory\":0.5}," - + "\"search_settings\":{\"timeout\":\"30s\"}," - + "\"updated_at\":1720047207}}"; + String expected = """ + {"ajakgakg983r92_4242":{\ + "_id":"ajakgakg983r92_4242",\ + "name":"test",\ + "resiliency_mode":"enforced",\ + "resource_limits":{"memory":0.5},\ + "settings":{"search.default_search_timeout":"30s"},\ + "updated_at":1720047207}}"""; assertEquals(expected, builder.toString()); } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/WorkloadGroupTests.java b/server/src/test/java/org/opensearch/cluster/metadata/WorkloadGroupTests.java index b47ac8f28ca0f..a18f57e7667ab 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/WorkloadGroupTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/WorkloadGroupTests.java @@ -9,6 +9,7 @@ package org.opensearch.cluster.metadata; import org.opensearch.common.UUIDs; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; @@ -18,7 +19,6 @@ import org.opensearch.wlm.MutableWorkloadGroupFragment; import org.opensearch.wlm.MutableWorkloadGroupFragment.ResiliencyMode; import org.opensearch.wlm.ResourceType; -import org.opensearch.wlm.WorkloadGroupSearchSettings.WlmSearchSetting; import org.joda.time.Instant; import java.io.IOException; @@ -31,7 +31,7 @@ public class WorkloadGroupTests extends AbstractSerializingTestCase { private static final List allowedModes = List.of(ResiliencyMode.SOFT, ResiliencyMode.ENFORCED, ResiliencyMode.MONITOR); - public static final Map TEST_WLM_SEARCH_SETTINGS = Map.of(WlmSearchSetting.TIMEOUT.getSettingName(), "30s"); + public static final Settings TEST_WLM_SEARCH_SETTINGS = Settings.builder().put("search.default_search_timeout", "30s").build(); static WorkloadGroup createRandomWorkloadGroup(String _id) { String name = randomAlphaOfLength(10); @@ -139,8 +139,8 @@ public void testWorkloadGroupInitiation() { assertEquals(1, workloadGroup.getResourceLimits().size()); assertTrue(allowedModes.contains(workloadGroup.getResiliencyMode())); assertTrue(workloadGroup.getUpdatedAtInMillis() != 0); - assertNotNull(workloadGroup.getSearchSettings()); - assertEquals(TEST_WLM_SEARCH_SETTINGS, workloadGroup.getSearchSettings()); + assertNotNull(workloadGroup.getSettings()); + assertEquals(TEST_WLM_SEARCH_SETTINGS, workloadGroup.getSettings()); } public void testIllegalWorkloadGroupName() { @@ -242,11 +242,21 @@ public void testToXContent() throws IOException { Locale.ROOT, "{\"_id\":\"%s\",\"name\":\"TestWorkloadGroup\",\"resiliency_mode\":\"enforced\"," + "\"resource_limits\":{\"cpu\":0.3,\"memory\":0.4}," - + "\"search_settings\":{\"timeout\":\"30s\"}," + + "\"settings\":{\"search.default_search_timeout\":\"30s\"}," + "\"updated_at\":%d}", workloadGroupId, currentTimeInMillis ); assertEquals(expected, builder.toString()); } + + public void testLegacySearchSettingsFieldRejected() throws IOException { + String json = "{\"_id\":\"test_id\",\"name\":\"test\",\"resiliency_mode\":\"enforced\"," + + "\"resource_limits\":{\"memory\":0.5}," + + "\"search_settings\":{\"timeout\":\"30s\"}," + + "\"updated_at\":1720047207}"; + XContentParser parser = createParser(JsonXContent.jsonXContent, json); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> WorkloadGroup.fromXContent(parser)); + assertTrue(exception.getMessage().contains("search_settings")); + } } diff --git a/server/src/test/java/org/opensearch/wlm/WorkloadGroupSearchSettingsTests.java b/server/src/test/java/org/opensearch/wlm/WorkloadGroupSearchSettingsTests.java index c2212662064d5..9f8d9cbdc843f 100644 --- a/server/src/test/java/org/opensearch/wlm/WorkloadGroupSearchSettingsTests.java +++ b/server/src/test/java/org/opensearch/wlm/WorkloadGroupSearchSettingsTests.java @@ -8,102 +8,66 @@ package org.opensearch.wlm; +import org.opensearch.common.settings.Settings; import org.opensearch.test.OpenSearchTestCase; -import java.util.HashMap; -import java.util.Map; - public class WorkloadGroupSearchSettingsTests extends OpenSearchTestCase { - public void testEnumSettingNames() { - assertEquals("timeout", WorkloadGroupSearchSettings.WlmSearchSetting.TIMEOUT.getSettingName()); - } - - public void testFromKeyValidSettings() { - assertEquals(WorkloadGroupSearchSettings.WlmSearchSetting.TIMEOUT, WorkloadGroupSearchSettings.WlmSearchSetting.fromKey("timeout")); + public void testWlmSearchTimeoutSettingExists() { + assertNotNull(WorkloadGroupSearchSettings.WLM_SEARCH_TIMEOUT); + assertEquals("search.default_search_timeout", WorkloadGroupSearchSettings.WLM_SEARCH_TIMEOUT.getKey()); } - public void testFromKeyInvalidSetting() { - assertNull(WorkloadGroupSearchSettings.WlmSearchSetting.fromKey("invalid_setting")); - assertNull(WorkloadGroupSearchSettings.WlmSearchSetting.fromKey("")); - assertNull(WorkloadGroupSearchSettings.WlmSearchSetting.fromKey(null)); - } - - public void testValidateTimeValue() { - WorkloadGroupSearchSettings.WlmSearchSetting.TIMEOUT.validate("30s"); - WorkloadGroupSearchSettings.WlmSearchSetting.TIMEOUT.validate("5m"); - WorkloadGroupSearchSettings.WlmSearchSetting.TIMEOUT.validate("1h"); - } - - public void testValidateInvalidTimeValue() { - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> WorkloadGroupSearchSettings.WlmSearchSetting.TIMEOUT.validate("invalid") - ); - assertTrue(exception.getMessage().contains("Invalid value")); + public void testValidateSettingsValid() { + Settings settings = Settings.builder().put("search.default_search_timeout", "30s").build(); + WorkloadGroupSearchSettings.validate(settings); } - public void testValidateSearchSettingsValid() { - Map settings = new HashMap<>(); - settings.put("timeout", "30s"); - - // Should not throw exception - WorkloadGroupSearchSettings.validateSearchSettings(settings); + public void testValidateSettingsValidTimeValues() { + for (String timeVal : new String[] { "30s", "5m", "1h", "500ms" }) { + Settings settings = Settings.builder().put("search.default_search_timeout", timeVal).build(); + WorkloadGroupSearchSettings.validate(settings); + } } - public void testValidateSearchSettingsUnknownSetting() { - Map settings = new HashMap<>(); - settings.put("unknown_setting", "true"); - + public void testValidateSettingsUnknownKey() { + Settings settings = Settings.builder().put("unknown_key", "value").build(); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> WorkloadGroupSearchSettings.validateSearchSettings(settings) + () -> WorkloadGroupSearchSettings.validate(settings) ); - assertTrue(exception.getMessage().contains("Unknown search setting: unknown_setting")); + assertTrue(exception.getMessage().contains("Unknown WLM setting: unknown_key")); } - public void testValidateSearchSettingsInvalidValue() { - Map settings = new HashMap<>(); - settings.put("timeout", "invalid_time"); - + public void testValidateSettingsInvalidValue() { + Settings settings = Settings.builder().put("search.default_search_timeout", "not_a_time").build(); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> WorkloadGroupSearchSettings.validateSearchSettings(settings) + () -> WorkloadGroupSearchSettings.validate(settings) ); assertTrue(exception.getMessage().contains("Invalid value")); + assertTrue(exception.getMessage().contains("search.default_search_timeout")); } - public void testValidateSearchSettingsNull() { - // Should not throw exception for null map - WorkloadGroupSearchSettings.validateSearchSettings(null); + public void testValidateSettingsNull() { + WorkloadGroupSearchSettings.validate(null); } - public void testValidateSearchSettingsNullKey() { - Map settings = new HashMap<>(); - settings.put(null, "30s"); - - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> WorkloadGroupSearchSettings.validateSearchSettings(settings) - ); - assertTrue(exception.getMessage().contains("Search setting key cannot be null")); + public void testValidateSettingsEmpty() { + WorkloadGroupSearchSettings.validate(Settings.EMPTY); } - public void testValidateSearchSettingsNullValue() { - Map settings = new HashMap<>(); - settings.put("timeout", null); + public void testGetRegisteredSettings() { + assertNotNull(WorkloadGroupSearchSettings.getRegisteredSettings()); + assertTrue(WorkloadGroupSearchSettings.getRegisteredSettings().containsKey("search.default_search_timeout")); + } + public void testLegacyTimeoutKeyRejected() { + Settings settings = Settings.builder().put("timeout", "30s").build(); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> WorkloadGroupSearchSettings.validateSearchSettings(settings) + () -> WorkloadGroupSearchSettings.validate(settings) ); - assertTrue(exception.getMessage().contains("Search setting value cannot be null")); - } - - public void testValidateSearchSettingsEmpty() { - Map settings = new HashMap<>(); - - // Should not throw exception for empty map - WorkloadGroupSearchSettings.validateSearchSettings(settings); + assertTrue(exception.getMessage().contains("Unknown WLM setting: timeout")); } } diff --git a/server/src/test/java/org/opensearch/wlm/listeners/WorkloadGroupRequestOperationListenerTests.java b/server/src/test/java/org/opensearch/wlm/listeners/WorkloadGroupRequestOperationListenerTests.java index 7863045de2dea..61e9b6e70e9cb 100644 --- a/server/src/test/java/org/opensearch/wlm/listeners/WorkloadGroupRequestOperationListenerTests.java +++ b/server/src/test/java/org/opensearch/wlm/listeners/WorkloadGroupRequestOperationListenerTests.java @@ -14,6 +14,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.WorkloadGroup; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; @@ -308,7 +309,7 @@ public void testApplySearchSettings_EmptySearchSettings() { mockSearchRequest.source(new SearchSourceBuilder()); String wgId = "test-wg"; - WorkloadGroup wg = createWorkloadGroup(wgId, Map.of()); + WorkloadGroup wg = createWorkloadGroup(wgId, Settings.EMPTY); when(workloadGroupService.getWorkloadGroupById(wgId)).thenReturn(wg); testThreadPool.getThreadContext().putHeader(WorkloadGroupTask.WORKLOAD_GROUP_ID_HEADER, wgId); @@ -322,7 +323,7 @@ public void testApplySearchSettings_Timeout_WlmAppliedWhenNull() { assertNull(mockSearchRequest.source().timeout()); String wgId = "test-wg"; - WorkloadGroup wg = createWorkloadGroup(wgId, Map.of("timeout", "1m")); + WorkloadGroup wg = createWorkloadGroup(wgId, Settings.builder().put("search.default_search_timeout", "1m").build()); when(workloadGroupService.getWorkloadGroupById(wgId)).thenReturn(wg); testThreadPool.getThreadContext().putHeader(WorkloadGroupTask.WORKLOAD_GROUP_ID_HEADER, wgId); @@ -335,7 +336,7 @@ public void testApplySearchSettings_Timeout_RequestAlreadySet() { mockSearchRequest.source(new SearchSourceBuilder().timeout(TimeValue.timeValueSeconds(30))); String wgId = "test-wg"; - WorkloadGroup wg = createWorkloadGroup(wgId, Map.of("timeout", "10s")); + WorkloadGroup wg = createWorkloadGroup(wgId, Settings.builder().put("search.default_search_timeout", "10s").build()); when(workloadGroupService.getWorkloadGroupById(wgId)).thenReturn(wg); testThreadPool.getThreadContext().putHeader(WorkloadGroupTask.WORKLOAD_GROUP_ID_HEADER, wgId); @@ -348,7 +349,7 @@ public void testApplySearchSettings_Timeout_NullSource() { assertNull(mockSearchRequest.source()); String wgId = "test-wg"; - WorkloadGroup wg = createWorkloadGroup(wgId, Map.of("timeout", "30s")); + WorkloadGroup wg = createWorkloadGroup(wgId, Settings.builder().put("search.default_search_timeout", "30s").build()); when(workloadGroupService.getWorkloadGroupById(wgId)).thenReturn(wg); testThreadPool.getThreadContext().putHeader(WorkloadGroupTask.WORKLOAD_GROUP_ID_HEADER, wgId); @@ -357,7 +358,7 @@ public void testApplySearchSettings_Timeout_NullSource() { assertNull(mockSearchRequest.source()); // Should not throw, source remains null } - private WorkloadGroup createWorkloadGroup(String id, Map searchSettings) { + private WorkloadGroup createWorkloadGroup(String id, Settings searchSettings) { return new WorkloadGroup( "test-name", id, From a974009ac1cf16bd940530d3a9dc35e3398642e1 Mon Sep 17 00:00:00 2001 From: bowenlan Date: Tue, 5 May 2026 14:41:28 -0700 Subject: [PATCH 051/115] Wire Rust native library build into assemble lifecycle (#21475) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildRustLibrary was only triggered as a test dependency, meaning ./gradlew assemble never produced libopensearch_native. This broke assembled distributions — bin/opensearch crashed with "Failed to load native library" because the .dylib/.so did not exist for opensearch-build to package. Also support -PrustRelease flag passed by opensearch-build's build.sh (previously a no-op since code only checked -PrustDebug). Signed-off-by: bowenlan-amzn --- sandbox/libs/dataformat-native/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sandbox/libs/dataformat-native/build.gradle b/sandbox/libs/dataformat-native/build.gradle index 301208fbfe22d..fa022b6347435 100644 --- a/sandbox/libs/dataformat-native/build.gradle +++ b/sandbox/libs/dataformat-native/build.gradle @@ -87,6 +87,8 @@ task buildRustLibrary(type: Exec) { // Expose the native lib path so plugins can reference it for tests ext.nativeLibPath = nativeLibFile +assemble.dependsOn buildRustLibrary + test { systemProperty 'tests.security.manager', 'false' systemProperty 'native.lib.path', nativeLibFile.absolutePath From c6527fadc24a42e9996358baaea8a282963857b8 Mon Sep 17 00:00:00 2001 From: bowenlan Date: Tue, 5 May 2026 15:55:41 -0700 Subject: [PATCH 052/115] Stream Arrow batches on the data-node fragment execution path (#21418) * [PR 21454 follow-up] Forward skipsDeserialization in TransportService wrapper PR 21454 added TransportResponseHandler#skipsDeserialization() and threaded it through the user-facing wrappers (ContextRestoreResponseHandler, MetricsTrackingResponseHandler, TraceableTransportResponseHandler). It missed the anonymous wrapper in TransportService.sendRequestAsync, which wraps the handler whenever a parent task is set. --------- Signed-off-by: bowenlan-amzn --- .../analytics/backend/EngineResultBatch.java | 16 +- .../analytics/backend/ExecutionContext.java | 24 +- .../StreamingCoordinatorReduceIT.java | 145 ++ .../DataFusionAnalyticsBackendPlugin.java | 2 +- .../be/datafusion/DatafusionContext.java | 8 +- .../be/datafusion/DatafusionResultStream.java | 82 +- .../DatafusionSearchExecEngine.java | 15 +- .../DatafusionResultStreamTests.java | 50 +- .../DatafusionSearchExecEngineTests.java | 48 +- sandbox/plugins/analytics-engine/build.gradle | 133 +- .../licenses/arrow-format-18.1.0.jar.sha1 | 1 - .../arrow-memory-core-18.1.0.jar.sha1 | 1 - .../arrow-memory-unsafe-18.1.0.jar.sha1 | 1 - .../licenses/arrow-memory-unsafe-LICENSE.txt | 2261 ----------------- .../licenses/arrow-memory-unsafe-NOTICE.txt | 84 - .../licenses/arrow-vector-18.1.0.jar.sha1 | 1 - .../licenses/arrow-vector-LICENSE.txt | 2261 ----------------- .../licenses/arrow-vector-NOTICE.txt | 84 - .../licenses/failureaccess-1.0.2.jar.sha1 | 1 - .../licenses/failureaccess-NOTICE.txt | 0 .../licenses/flatbuffers-java-2.0.0.jar.sha1 | 1 - .../licenses/flatbuffers-java-LICENSE.txt | 202 -- .../licenses/flatbuffers-java-NOTICE.txt | 0 .../licenses/guava-33.4.0-jre.jar.sha1 | 1 - .../licenses/guava-LICENSE.txt | 202 -- .../licenses/guava-NOTICE.txt | 0 .../jackson-annotations-2.21.jar.sha1 | 1 - .../licenses/jackson-annotations-LICENSE.txt | 8 - .../licenses/jackson-annotations-NOTICE.txt | 1 - .../licenses/jackson-databind-2.21.2.jar.sha1 | 1 - .../licenses/jackson-databind-LICENSE.txt | 8 - .../licenses/jackson-databind-NOTICE.txt | 20 - .../licenses/slf4j-api-2.0.17.jar.sha1 | 1 - .../licenses/slf4j-api-LICENSE.txt | 21 - .../licenses/slf4j-api-NOTICE.txt | 0 .../opensearch/analytics/AnalyticsPlugin.java | 10 +- .../exec/AnalyticsSearchService.java | 168 +- .../exec/AnalyticsSearchTransportService.java | 168 +- .../analytics/exec/FragmentResources.java | 62 + .../analytics/exec/QueryContext.java | 10 +- .../FragmentExecutionArrowResponse.java | 32 + .../exec/stage/RowResponseCodec.java | 3 +- .../stage/ShardFragmentStageExecution.java | 77 +- .../ShardFragmentStageExecutionTests.java | 243 ++ sandbox/plugins/composite-engine/build.gradle | 5 + .../plugins/parquet-data-format/build.gradle | 59 +- .../arrow-memory-netty-18.1.0.jar.sha1 | 1 + .../licenses/arrow-memory-netty-LICENSE.txt} | 0 .../licenses/arrow-memory-netty-NOTICE.txt} | 0 ...-memory-netty-buffer-patch-18.1.0.jar.sha1 | 1 + ...row-memory-netty-buffer-patch-LICENSE.txt} | 0 ...rrow-memory-netty-buffer-patch-NOTICE.txt} | 0 .../arrow-memory-unsafe-18.1.0.jar.sha1 | 1 - .../licenses/arrow-memory-unsafe-LICENSE.txt | 2261 ----------------- .../licenses/arrow-memory-unsafe-NOTICE.txt | 84 - .../licenses/netty-LICENSE.txt} | 0 .../licenses/netty-NOTICE.txt | 116 + .../netty-buffer-4.2.13.Final.jar.sha1 | 1 + .../netty-common-4.2.13.Final.jar.sha1 | 1 + sandbox/qa/analytics-engine-rest/build.gradle | 9 + .../search/SearchExecutionContext.java | 4 +- .../transport/TransportService.java | 5 + 62 files changed, 1152 insertions(+), 7854 deletions(-) create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/StreamingCoordinatorReduceIT.java delete mode 100644 sandbox/plugins/analytics-engine/licenses/arrow-format-18.1.0.jar.sha1 delete mode 100644 sandbox/plugins/analytics-engine/licenses/arrow-memory-core-18.1.0.jar.sha1 delete mode 100644 sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-18.1.0.jar.sha1 delete mode 100644 sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-LICENSE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-NOTICE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/arrow-vector-18.1.0.jar.sha1 delete mode 100644 sandbox/plugins/analytics-engine/licenses/arrow-vector-LICENSE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/arrow-vector-NOTICE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/failureaccess-1.0.2.jar.sha1 delete mode 100644 sandbox/plugins/analytics-engine/licenses/failureaccess-NOTICE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/flatbuffers-java-2.0.0.jar.sha1 delete mode 100644 sandbox/plugins/analytics-engine/licenses/flatbuffers-java-LICENSE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/flatbuffers-java-NOTICE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/guava-33.4.0-jre.jar.sha1 delete mode 100644 sandbox/plugins/analytics-engine/licenses/guava-LICENSE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/guava-NOTICE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-annotations-2.21.jar.sha1 delete mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-annotations-LICENSE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-annotations-NOTICE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.2.jar.sha1 delete mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-databind-LICENSE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-databind-NOTICE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/slf4j-api-2.0.17.jar.sha1 delete mode 100644 sandbox/plugins/analytics-engine/licenses/slf4j-api-LICENSE.txt delete mode 100644 sandbox/plugins/analytics-engine/licenses/slf4j-api-NOTICE.txt create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionArrowResponse.java create mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java create mode 100644 sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-18.1.0.jar.sha1 rename sandbox/plugins/{analytics-engine/licenses/arrow-format-LICENSE.txt => parquet-data-format/licenses/arrow-memory-netty-LICENSE.txt} (100%) rename sandbox/plugins/{analytics-engine/licenses/arrow-format-NOTICE.txt => parquet-data-format/licenses/arrow-memory-netty-NOTICE.txt} (100%) create mode 100644 sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1 rename sandbox/plugins/{analytics-engine/licenses/arrow-memory-core-LICENSE.txt => parquet-data-format/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt} (100%) rename sandbox/plugins/{analytics-engine/licenses/arrow-memory-core-NOTICE.txt => parquet-data-format/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt} (100%) delete mode 100644 sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-18.1.0.jar.sha1 delete mode 100644 sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-LICENSE.txt delete mode 100644 sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-NOTICE.txt rename sandbox/plugins/{analytics-engine/licenses/failureaccess-LICENSE.txt => parquet-data-format/licenses/netty-LICENSE.txt} (100%) create mode 100644 sandbox/plugins/parquet-data-format/licenses/netty-NOTICE.txt create mode 100644 sandbox/plugins/parquet-data-format/licenses/netty-buffer-4.2.13.Final.jar.sha1 create mode 100644 sandbox/plugins/parquet-data-format/licenses/netty-common-4.2.13.Final.jar.sha1 diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/EngineResultBatch.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/EngineResultBatch.java index c4a9be55af5e4..23743e273dcb0 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/EngineResultBatch.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/EngineResultBatch.java @@ -8,22 +8,34 @@ package org.opensearch.analytics.backend; +import org.apache.arrow.vector.VectorSchemaRoot; + import java.util.List; /** - * Read-only view of a single record batch. Provides field names, row count, - * and positional access to field values. + * Read-only view of a single record batch. *

      * A batch is only valid until the next call to {@link java.util.Iterator#next()} * on the parent stream's iterator. The underlying data buffers may be reused * across batches, so callers must extract all needed values before advancing * the iterator. Accessing a batch after the iterator has advanced may throw * {@link IllegalStateException}. + *

      + * Primary shape is the Arrow {@link VectorSchemaRoot} returned by + * {@link #getArrowRoot()} — the native columnar representation used by the + * streaming transport (zero-copy over gRPC). Row-oriented accessors + * ({@link #getFieldNames()}, {@link #getRowCount()}, {@link #getFieldValue}) + * are a convenience view over the same data. * * @opensearch.internal */ public interface EngineResultBatch { + /** + * The Arrow VSR backing this batch + */ + VectorSchemaRoot getArrowRoot(); + /** * Ordered list of field (column) names in this batch. */ diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java index 78946812c02d0..c4a1ffd52c916 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java @@ -8,8 +8,9 @@ package org.opensearch.analytics.backend; -import org.opensearch.action.search.SearchShardTask; +import org.apache.arrow.memory.BufferAllocator; import org.opensearch.index.engine.exec.IndexReaderProvider.Reader; +import org.opensearch.tasks.Task; /** * Execution context carrying reader and plan state through @@ -21,23 +22,24 @@ public class ExecutionContext { private final String tableName; private final Reader reader; - private final SearchShardTask task; + private final Task task; private byte[] fragmentBytes; + private BufferAllocator allocator; /** * Constructs an execution context. * @param tableName the target table name - * @param task the search shard task + * @param task the transport-created task for this fragment execution * @param reader the data-format aware reader */ - public ExecutionContext(String tableName, SearchShardTask task, Reader reader) { + public ExecutionContext(String tableName, Task task, Reader reader) { this.tableName = tableName; this.task = task; this.reader = reader; } - /** Returns the search shard task. */ - public SearchShardTask getTask() { + /** Returns the transport-created task for this fragment execution. */ + public Task getTask() { return task; } @@ -60,4 +62,14 @@ public byte[] getFragmentBytes() { public void setFragmentBytes(byte[] fragmentBytes) { this.fragmentBytes = fragmentBytes; } + + /** Returns the caller-provided allocator for producing Arrow result buffers. */ + public BufferAllocator getAllocator() { + return allocator; + } + + /** Sets the caller-provided allocator. The caller owns its lifecycle; the engine must not close it. */ + public void setAllocator(BufferAllocator allocator) { + this.allocator = allocator; + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/StreamingCoordinatorReduceIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/StreamingCoordinatorReduceIT.java new file mode 100644 index 0000000000000..6ac06cc18536b --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/StreamingCoordinatorReduceIT.java @@ -0,0 +1,145 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.Version; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.analytics.AnalyticsPlugin; +import org.opensearch.arrow.flight.transport.FlightStreamPlugin; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.composite.CompositeDataFormatPlugin; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.PluginInfo; +import org.opensearch.ppl.TestPPLPlugin; +import org.opensearch.ppl.action.PPLRequest; +import org.opensearch.ppl.action.PPLResponse; +import org.opensearch.ppl.action.UnifiedPPLExecuteAction; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.opensearch.common.util.FeatureFlags.STREAM_TRANSPORT; + +/** + * Streaming variant of {@link CoordinatorReduceIT}: same 2-shard parquet-backed index and deterministic + * dataset, but with Arrow Flight RPC enabled via {@link FeatureFlags#STREAM_TRANSPORT}. Exercises the + * shard-fragment → Flight → {@code DatafusionReduceSink.feed} handoff that previously failed with + * {@code "A buffer can only be associated between two allocators that share the same root"} on + * multi-shard queries. + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 2) +public class StreamingCoordinatorReduceIT extends OpenSearchIntegTestCase { + + private static final String INDEX = "coord_reduce_streaming_e2e"; + private static final int NUM_SHARDS = 2; + private static final int DOCS_PER_SHARD = 10; + /** Constant `value` for every doc — deterministic assertion independent of shard routing. */ + private static final int VALUE = 7; + + @Override + protected Collection> nodePlugins() { + return List.of(TestPPLPlugin.class, FlightStreamPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class); + } + + @Override + protected Collection additionalNodePlugins() { + return List.of( + classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) + ); + } + + private static PluginInfo classpathPlugin(Class pluginClass, List extendedPlugins) { + return new PluginInfo( + pluginClass.getName(), + "classpath plugin", + "NA", + Version.CURRENT, + "1.8", + pluginClass.getName(), + null, + extendedPlugins, + false + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .build(); + } + + /** + * {@code source = T} on a 2-shard parquet-backed index with streaming enabled exercises the + * coordinator reduce sink's cross-plugin VectorSchemaRoot handoff. Before the allocator-root + * unification fix, this failed with an Arrow {@code associate} mismatch. + */ + @LockFeatureFlag(STREAM_TRANSPORT) + public void testBaselineScanAcrossShards() throws Exception { + createParquetBackedIndex(); + indexDeterministicDocs(); + + PPLResponse response = executePPL("source = " + INDEX); + + assertNotNull("PPLResponse must not be null", response); + assertTrue("columns must contain 'value', got " + response.getColumns(), response.getColumns().contains("value")); + + int expectedRows = NUM_SHARDS * DOCS_PER_SHARD; + assertEquals("all docs across shards must be returned", expectedRows, response.getRows().size()); + + int idx = response.getColumns().indexOf("value"); + for (Object[] row : response.getRows()) { + Object cell = row[idx]; + assertNotNull("value cell must not be null", cell); + assertEquals("every doc has value=" + VALUE, (long) VALUE, ((Number) cell).longValue()); + } + } + + private void createParquetBackedIndex() { + Settings indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats") + .build(); + + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(INDEX) + .setSettings(indexSettings) + .setMapping("value", "type=integer") + .get(); + assertTrue("index creation must be acknowledged", response.isAcknowledged()); + ensureGreen(INDEX); + } + + private void indexDeterministicDocs() { + int total = NUM_SHARDS * DOCS_PER_SHARD; + for (int i = 0; i < total; i++) { + client().prepareIndex(INDEX).setId(String.valueOf(i)).setSource("value", VALUE).get(); + } + client().admin().indices().prepareRefresh(INDEX).get(); + client().admin().indices().prepareFlush(INDEX).get(); + } + + private PPLResponse executePPL(String ppl) { + return client().execute(UnifiedPPLExecuteAction.INSTANCE, new PPLRequest(ppl)).actionGet(); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 90f269991c804..f5ed1de1033bd 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -175,7 +175,7 @@ public SearchExecEngineProvider getSearchExecEngineProvider() { throw new IllegalStateException("No DatafusionReader available in the acquired reader"); } DatafusionContext context = new DatafusionContext(ctx.getTask(), dfReader, dataFusionService.getNativeRuntime()); - DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine(context, dataFusionService::newChildAllocator); + DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine(context); engine.prepare(ctx); return engine; }; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java index de20bf3ae0b2a..527ead7df96a2 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java @@ -8,10 +8,10 @@ package org.opensearch.be.datafusion; -import org.opensearch.action.search.SearchShardTask; import org.opensearch.be.datafusion.nativelib.StreamHandle; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.search.SearchExecutionContext; +import org.opensearch.tasks.Task; import java.io.IOException; @@ -30,7 +30,7 @@ public class DatafusionContext implements SearchExecutionContext iterator() { @Override public void close() { try { - if (iteratorInstance != null && iteratorInstance.vectorSchemaRoot != null) { - iteratorInstance.vectorSchemaRoot.close(); + if (iteratorInstance != null) { + iteratorInstance.closeLastBatch(); } } finally { try { streamHandle.close(); } finally { - try { - dictionaryProvider.close(); - } finally { - allocator.close(); - } + dictionaryProvider.close(); } } } - /** - * Iterator that pulls Arrow record batches from the native stream via async JNI. - * Uses one-ahead buffering: the next batch is pre-loaded so hasNext() is side-effect-free. - */ + // Fresh VSR per batch so each can be handed off independently + // Close-on-advance releases the previous VSR (no-op if transport already transferred it). static class BatchIterator implements Iterator { private final StreamHandle streamHandle; private final BufferAllocator allocator; private final CDataDictionaryProvider dictionaryProvider; - VectorSchemaRoot vectorSchemaRoot; + private Schema schema; + private VectorSchemaRoot nextBatch; private Boolean nextAvailable; - /** Incremented each time {@link #next()} is called. Used by {@link ArrowResultBatch} to detect stale access. */ - long generation; BatchIterator(StreamHandle streamHandle, BufferAllocator allocator, CDataDictionaryProvider dictionaryProvider) { this.streamHandle = streamHandle; @@ -108,15 +96,14 @@ static class BatchIterator implements Iterator { } private void ensureSchema() { - if (vectorSchemaRoot != null) return; + if (schema != null) return; long schemaAddr = callNativeFn(listener -> NativeBridge.streamGetSchema(streamHandle.getPointer(), listener)); try (ArrowSchema arrowSchema = ArrowSchema.wrap(schemaAddr)) { Field structField = importField(allocator, arrowSchema, dictionaryProvider); if (structField.getType().getTypeID() != ArrowType.ArrowTypeID.Struct) { throw new IllegalStateException("ArrowSchema describes non-struct type"); } - Schema schema = new Schema(structField.getChildren(), structField.getMetadata()); - vectorSchemaRoot = VectorSchemaRoot.create(schema, allocator); + schema = new Schema(structField.getChildren(), structField.getMetadata()); } } @@ -126,9 +113,11 @@ private boolean loadNextBatch() { listener -> NativeBridge.streamNext(streamHandle.getRuntimeHandle().get(), streamHandle.getPointer(), listener) ); if (arrayAddr == 0) return false; + VectorSchemaRoot freshRoot = VectorSchemaRoot.create(schema, allocator); try (ArrowArray arrowArray = ArrowArray.wrap(arrayAddr)) { - Data.importIntoVectorSchemaRoot(allocator, arrowArray, vectorSchemaRoot, dictionaryProvider); + Data.importIntoVectorSchemaRoot(allocator, arrowArray, freshRoot, dictionaryProvider); } + nextBatch = freshRoot; return true; } @@ -146,8 +135,21 @@ public EngineResultBatch next() { throw new NoSuchElementException(); } nextAvailable = null; - generation++; - return new ArrowResultBatch(vectorSchemaRoot, generation, this); + VectorSchemaRoot batch = nextBatch; + nextBatch = null; + // Caller owns the returned VSR's lifecycle. Streaming handler transfers it to Flight + // (Flight closes after wire write); row-path collector closes after reading. + return new ArrowResultBatch(batch); + } + + void closeLastBatch() { + // Only close batches that were loaded but never handed to the caller. Caller + // owns any batch returned by next(); closing it here would double-close after + // Flight's transferTo or after row-path reads. + if (nextBatch != null) { + nextBatch.close(); + nextBatch = null; + } } private static long callNativeFn(java.util.function.Consumer> fn) { @@ -167,51 +169,33 @@ public void onFailure(Exception e) { } } - /** - * Adapts an Arrow {@link VectorSchemaRoot} to the engine-agnostic {@link EngineResultBatch}. - *

      - * Because the underlying {@code VectorSchemaRoot} is reused across batches, - * this view is only valid until the next call to {@link Iterator#next()} on - * the parent iterator. A generation counter detects stale access at runtime. - */ static class ArrowResultBatch implements EngineResultBatch { private final VectorSchemaRoot root; private final List fieldNames; - private final long createdAtGeneration; - private final BatchIterator owner; - ArrowResultBatch(VectorSchemaRoot root, long generation, BatchIterator owner) { + ArrowResultBatch(VectorSchemaRoot root) { this.root = root; - this.fieldNames = root.getSchema().getFields().stream().map(Field::getName).collect(Collectors.toUnmodifiableList()); - this.createdAtGeneration = generation; - this.owner = owner; + this.fieldNames = root.getSchema().getFields().stream().map(Field::getName).toList(); } - private void checkValid() { - if (owner.generation != createdAtGeneration) { - throw new IllegalStateException( - "Batch is no longer valid — the iterator has advanced past this batch. " - + "Extract all needed values before calling next()." - ); - } + @Override + public VectorSchemaRoot getArrowRoot() { + return root; } @Override public List getFieldNames() { - checkValid(); return fieldNames; } @Override public int getRowCount() { - checkValid(); return root.getRowCount(); } @Override public Object getFieldValue(String fieldName, int rowIndex) { - checkValid(); FieldVector vector = root.getVector(fieldName); if (vector == null) { throw new IllegalArgumentException("Unknown field: " + fieldName); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearchExecEngine.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearchExecEngine.java index eb3b33848c455..40410f8b0f492 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearchExecEngine.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearchExecEngine.java @@ -18,7 +18,6 @@ import org.opensearch.common.annotation.ExperimentalApi; import java.io.IOException; -import java.util.function.Supplier; /** * DataFusion-backed search execution engine. @@ -33,16 +32,9 @@ public class DatafusionSearchExecEngine implements SearchExecEngine allocatorFactory; - /** - * Creates an execution engine backed by the given DataFusion context. - * @param datafusionContext the DataFusion execution context - * @param allocatorFactory factory for creating a child allocator for result stream memory - */ - public DatafusionSearchExecEngine(DatafusionContext datafusionContext, Supplier allocatorFactory) { + public DatafusionSearchExecEngine(DatafusionContext datafusionContext) { this.datafusionContext = datafusionContext; - this.allocatorFactory = allocatorFactory; } @Override @@ -54,10 +46,13 @@ public void prepare(ExecutionContext requestContext) { @Override public EngineResultStream execute(ExecutionContext requestContext) throws IOException { + BufferAllocator allocator = requestContext.getAllocator(); + if (allocator == null) { + throw new IllegalStateException("ExecutionContext.allocator must be set by the caller before execute()"); + } DatafusionSearcher searcher = datafusionContext.getSearcher(); searcher.search(datafusionContext); StreamHandle handle = datafusionContext.takeStreamHandle(); - BufferAllocator allocator = allocatorFactory.get(); return new DatafusionResultStream(handle, allocator); } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java index 3907fd8a69309..574bfb273ea97 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java @@ -31,6 +31,7 @@ public class DatafusionResultStreamTests extends OpenSearchTestCase { private ReaderHandle readerHandle; private NativeRuntimeHandle runtimeHandle; private RootAllocator testRootAllocator; + private final java.util.List allocatorsToClose = new java.util.ArrayList<>(); @Override public void setUp() throws Exception { @@ -51,6 +52,11 @@ public void setUp() throws Exception { public void tearDown() throws Exception { readerHandle.close(); runtimeHandle.close(); + // Caller owns child allocators now (see DatafusionResultStream.close javadoc). + // Close them in reverse registration order so child-before-parent invariants hold. + for (int i = allocatorsToClose.size() - 1; i >= 0; i--) { + allocatorsToClose.get(i).close(); + } testRootAllocator.close(); super.tearDown(); } @@ -69,7 +75,11 @@ public void testCloseAfterPartialIteration() throws Exception { Iterator it = stream.iterator(); assertTrue(it.hasNext()); EngineResultBatch batch = it.next(); - assertTrue(batch.getRowCount() > 0); + try { + assertTrue(batch.getRowCount() > 0); + } finally { + batch.getArrowRoot().close(); + } // close without exhausting the stream } } @@ -79,7 +89,12 @@ public void testCloseAfterFullIteration() throws Exception { Iterator it = stream.iterator(); int totalRows = 0; while (it.hasNext()) { - totalRows += it.next().getRowCount(); + EngineResultBatch batch = it.next(); + try { + totalRows += batch.getRowCount(); + } finally { + batch.getArrowRoot().close(); + } } assertEquals(2, totalRows); } @@ -90,7 +105,11 @@ public void testNextWithoutHasNextWorks() throws Exception { try (DatafusionResultStream stream = createStream("SELECT message FROM test_table")) { Iterator it = stream.iterator(); EngineResultBatch batch = it.next(); - assertTrue(batch.getRowCount() > 0); + try { + assertTrue(batch.getRowCount() > 0); + } finally { + batch.getArrowRoot().close(); + } } } @@ -110,7 +129,11 @@ public void testHasNextIsIdempotent() throws Exception { assertTrue(it.hasNext()); assertTrue(it.hasNext()); EngineResultBatch batch = it.next(); - assertTrue(batch.getRowCount() > 0); + try { + assertTrue(batch.getRowCount() > 0); + } finally { + batch.getArrowRoot().close(); + } } } @@ -127,11 +150,15 @@ public void testBatchFieldAccess() throws Exception { Iterator it = stream.iterator(); assertTrue(it.hasNext()); EngineResultBatch batch = it.next(); - assertEquals(2, batch.getFieldNames().size()); - assertTrue(batch.getFieldNames().contains("message")); - assertTrue(batch.getFieldNames().contains("message2")); - assertNotNull(batch.getFieldValue("message", 0)); - expectThrows(IllegalArgumentException.class, () -> batch.getFieldValue("nonexistent", 0)); + try { + assertEquals(2, batch.getFieldNames().size()); + assertTrue(batch.getFieldNames().contains("message")); + assertTrue(batch.getFieldNames().contains("message2")); + assertNotNull(batch.getFieldValue("message", 0)); + expectThrows(IllegalArgumentException.class, () -> batch.getFieldValue("nonexistent", 0)); + } finally { + batch.getArrowRoot().close(); + } } } @@ -202,9 +229,11 @@ public void onFailure(Exception e) { ); long streamPtr = future.join(); + BufferAllocator failureAlloc = testRootAllocator.newChildAllocator("test-failure", 0, Long.MAX_VALUE); + allocatorsToClose.add(failureAlloc); DatafusionResultStream stream = new DatafusionResultStream( new org.opensearch.be.datafusion.nativelib.StreamHandle(streamPtr, tempRuntime), - testRootAllocator.newChildAllocator("test-failure", 0, Long.MAX_VALUE) + failureAlloc ); // Close runtime — streamNext should now fail with IllegalStateException from NativeRuntimeHandle.get() @@ -253,6 +282,7 @@ public void onFailure(Exception e) { ); long streamPtr = future.join(); BufferAllocator childAllocator = testRootAllocator.newChildAllocator("test-stream", 0, Long.MAX_VALUE); + allocatorsToClose.add(childAllocator); return new DatafusionResultStream( new org.opensearch.be.datafusion.nativelib.StreamHandle(streamPtr, runtimeHandle), childAllocator diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java index a876533548282..7135b988599e8 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java @@ -8,8 +8,10 @@ package org.opensearch.be.datafusion; +import org.apache.arrow.memory.RootAllocator; import org.opensearch.analytics.backend.EngineResultBatch; import org.opensearch.analytics.backend.EngineResultStream; +import org.opensearch.analytics.backend.ExecutionContext; import org.opensearch.be.datafusion.nativelib.NativeBridge; import org.opensearch.be.datafusion.nativelib.ReaderHandle; import org.opensearch.test.OpenSearchTestCase; @@ -66,12 +68,12 @@ public void testEngineExecuteSelectAll() throws Exception { context.setDatafusionQuery(new DatafusionQuery("test_table", substrait, 0L)); try ( - DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine( - context, - () -> new org.apache.arrow.memory.RootAllocator(Long.MAX_VALUE) - ) + RootAllocator alloc = new RootAllocator(Long.MAX_VALUE); + DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine(context) ) { - try (EngineResultStream stream = engine.execute(null)) { + ExecutionContext execCtx = new ExecutionContext("test_table", null, null); + execCtx.setAllocator(alloc); + try (EngineResultStream stream = engine.execute(execCtx)) { List rows = collectRows(stream); assertEquals(2, rows.size()); assertEquals(2L, rows.get(0)[0]); // message @@ -95,12 +97,12 @@ public void testEngineExecuteAggregation() throws Exception { context.setDatafusionQuery(new DatafusionQuery("test_table", substrait, 0L)); try ( - DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine( - context, - () -> new org.apache.arrow.memory.RootAllocator(Long.MAX_VALUE) - ) + RootAllocator alloc = new RootAllocator(Long.MAX_VALUE); + DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine(context) ) { - try (EngineResultStream stream = engine.execute(null)) { + ExecutionContext execCtx = new ExecutionContext("test_table", null, null); + execCtx.setAllocator(alloc); + try (EngineResultStream stream = engine.execute(execCtx)) { List rows = collectRows(stream); assertEquals(1, rows.size()); assertEquals(5L, rows.get(0)[0]); // 2 + 3 @@ -121,12 +123,12 @@ public void testEngineExecuteFilter() throws Exception { context.setDatafusionQuery(new DatafusionQuery("test_table", substrait, 0L)); try ( - DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine( - context, - () -> new org.apache.arrow.memory.RootAllocator(Long.MAX_VALUE) - ) + RootAllocator alloc = new RootAllocator(Long.MAX_VALUE); + DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine(context) ) { - try (EngineResultStream stream = engine.execute(null)) { + ExecutionContext execCtx = new ExecutionContext("test_table", null, null); + execCtx.setAllocator(alloc); + try (EngineResultStream stream = engine.execute(execCtx)) { List rows = collectRows(stream); assertEquals(1, rows.size()); assertEquals(3L, rows.get(0)[0]); @@ -144,13 +146,17 @@ private List collectRows(EngineResultStream stream) { Iterator it = stream.iterator(); while (it.hasNext()) { EngineResultBatch batch = it.next(); - int cols = batch.getFieldNames().size(); - for (int r = 0; r < batch.getRowCount(); r++) { - Object[] row = new Object[cols]; - for (int c = 0; c < cols; c++) { - row[c] = batch.getFieldValue(batch.getFieldNames().get(c), r); + try { + int cols = batch.getFieldNames().size(); + for (int r = 0; r < batch.getRowCount(); r++) { + Object[] row = new Object[cols]; + for (int c = 0; c < cols; c++) { + row[c] = batch.getFieldValue(batch.getFieldNames().get(c), r); + } + rows.add(row); } - rows.add(row); + } finally { + batch.getArrowRoot().close(); } } return rows; diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle index 038550b9cf12a..346aa1385ad70 100644 --- a/sandbox/plugins/analytics-engine/build.gradle +++ b/sandbox/plugins/analytics-engine/build.gradle @@ -20,6 +20,10 @@ def sqlUnifiedQueryVersion = '3.6.0.0-SNAPSHOT' opensearchplugin { description = 'Analytics engine hub: discovers and wires query extensions via ExtensiblePlugin SPI.' classname = 'org.opensearch.analytics.AnalyticsPlugin' + // Extend arrow-flight-rpc so analytics-engine and arrow-flight-rpc share one classloader. + // Cross-plugin Arrow types (VectorSchemaRoot, ArrowBatchResponse) only work when loaded + // by the same classloader, and zero-copy transfer requires identical class identity on both sides. + extendedPlugins = ['arrow-flight-rpc'] } java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } @@ -52,18 +56,16 @@ dependencies { // Also provides calcite-core transitively via api. api project(':sandbox:libs:analytics-framework') - // Arrow — framework's public interfaces (ExchangeSink, LocalStageContext, etc.) expose - // Arrow types. analytics-engine's own code (RowBatchToArrowConverter, ShardFragmentStageExecution) - // uses arrow directly. Bundle runtime here; backend plugins that extend analytics-engine - // declare arrow as compileOnly to avoid jar hell. - implementation "org.apache.arrow:arrow-vector:${versions.arrow}" - implementation "org.apache.arrow:arrow-memory-core:${versions.arrow}" - - // Arrow Flight RPC — compile-only; the arrow-flight-rpc plugin provides it at runtime. - // transitive = false prevents arrow-flight-rpc's transitives (slf4j, jackson, arrow-*, - // guava, netty, grpc, etc.) from landing on resolveableCompileOnly, which bundlePlugin - // subtracts from runtimeClasspath. Without this, jars analytics-engine needs to bundle - // (arrow-vector, arrow-memory-core, guava, slf4j) get stripped from the zip. + // Arrow — provided at runtime by the extended arrow-flight-rpc plugin (same classloader). + // compileOnly here to avoid duplicate bundling; the parent plugin's single copy is what + // zero-copy Arrow transfer requires. + compileOnly "org.apache.arrow:arrow-vector:${versions.arrow}" + compileOnly "org.apache.arrow:arrow-memory-core:${versions.arrow}" + + // Arrow Flight RPC — compile-only; arrow-flight-rpc is our extendedPlugins parent and + // provides these classes at runtime. transitive = false prevents arrow-flight-rpc's + // transitives from landing on resolveableCompileOnly and being subtracted from our + // runtimeClasspath (bundlePlugin does this subtraction). compileOnly(project(':plugins:arrow-flight-rpc')) { transitive = false } @@ -71,10 +73,9 @@ dependencies { transitive = false } - // Required at runtime by arrow-vector (Schema, ArrowType subclasses reference jackson). - // implementation (not compileOnly) so bundlePlugin doesn't strip them. - implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" - implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" + // Provided by arrow-flight-rpc at runtime (api deps in its build.gradle). + compileOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + compileOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" // Guava — required at compile time because Calcite base classes expose guava types. // Uses custom config to bypass forbidden-dependencies.gradle check on compileClasspath. @@ -83,48 +84,50 @@ dependencies { // Guava for test compilation — Calcite API exposes guava types calciteTestCompile "com.google.guava:guava:${versions.guava}" - // Guava — required at runtime because Calcite's SqlOperatorTables and SqlKind - // static initializers use com.google.common classes. The compileClasspath exclude - // above also strips Guava from runtimeClasspath, so add it back explicitly. - runtimeOnly "com.google.guava:guava:${versions.guava}" - runtimeOnly 'com.google.guava:failureaccess:1.0.2' + // Guava — provided at runtime by the arrow-flight-rpc parent plugin (33.3.1-jre). + // Declared compileOnly here only to satisfy the calcite-derived runtime references; + // the runtimeClasspath exclude block below removes it from the bundled zip. - // SLF4J — Arrow's BaseAllocator requires it at runtime. Child plugins - // (analytics-backend-datafusion) see it via the extendedPlugins classloader. - runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}" + // SLF4J — provided by arrow-flight-rpc at runtime (its api dep). + compileOnly "org.slf4j:slf4j-api:${versions.slf4j}" // Calcite code generation (optional in calcite-core POM, needed at runtime for Enumerable pipeline) testRuntimeOnly "org.codehaus.janino:janino:3.1.12" testRuntimeOnly "org.codehaus.janino:commons-compiler:3.1.12" - // jackson-annotations — required at runtime by jackson-databind (transitive via Calcite). - // Without this, child plugins that use Arrow's Schema (which triggers ObjectMapper init) - // fail with NoClassDefFoundError for JsonSerializeAs. - runtimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" + // arrow-memory-netty comes from arrow-flight-rpc (api dep) and provides the + // AllocationManager that arrow-memory-core's BaseAllocator discovers via ServiceLoader. + // We deliberately do NOT bundle arrow-memory-unsafe here — with a single shared classloader, + // arrow-memory-netty's NettyAllocationManager wins the ServiceLoader lookup, which is + // required for gRPC's zero-copy Netty buffer path to work. + + // Provided by arrow-flight-rpc at runtime (api deps). + compileOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" + compileOnly "org.apache.arrow:arrow-format:${versions.arrow}" // commons-math3 — Calcite's TimeFrames. references - // org.apache.commons.math3.fraction.BigFraction. Must live in this parent plugin's - // classloader because calcite-core is loaded here; child plugins (opensearch-sql) - // can't satisfy classloads triggered by parent-loaded classes. + // org.apache.commons.math3.fraction.BigFraction. Not provided by arrow-flight-rpc, + // so bundle it into analytics-engine's own zip. runtimeOnly "org.apache.commons:commons-math3:3.6.1" - // httpcore5 — Avatica's BuiltInConnectionProperty static initializer references - // org.apache.hc.core5.util.Timeout. Must live in this parent plugin's classloader - // because calcite-core/avatica-core are loaded here; child plugins (test-ppl-frontend) - // can't satisfy classloads triggered by parent-loaded classes. + // httpcore5/httpclient5 — Avatica's BuiltInConnectionProperty static initializer references + // org.apache.hc.core5.util.Timeout. Not provided by arrow-flight-rpc, so bundle here. runtimeOnly "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" runtimeOnly "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" runtimeOnly "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" - // arrow-memory-unsafe provides the DefaultAllocationManager that arrow-memory-core - // discovers via ServiceLoader at runtime. Must be in the parent plugin's classloader - // because BaseAllocator (from arrow-memory-core) does the ServiceLoader lookup. - runtimeOnly "org.apache.arrow:arrow-memory-unsafe:${versions.arrow}" - // Required at runtime by arrow-vector (ArrowType subclasses reference flatbuffers and arrow-format). - runtimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" - runtimeOnly "org.apache.arrow:arrow-format:${versions.arrow}" - // Required at runtime by avatica-core (BuiltInConnectionProperty. references Timeout). - runtimeOnly "org.apache.httpcomponents.core5:httpcore5:5.4" + // Unit tests run on a flat classpath (no plugin classloader), so arrow-flight-rpc's + // runtime jars must be pulled back in for tests. The bundled plugin is unaffected. + // arrow-memory-unsafe is used here (not -netty) because unit tests don't exercise + // the Netty allocator path. + testRuntimeOnly "org.apache.arrow:arrow-vector:${versions.arrow}" + testRuntimeOnly "org.apache.arrow:arrow-memory-core:${versions.arrow}" + testRuntimeOnly "org.apache.arrow:arrow-memory-unsafe:${versions.arrow}" + testRuntimeOnly "org.apache.arrow:arrow-format:${versions.arrow}" + testRuntimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" + testRuntimeOnly "org.slf4j:slf4j-api:${versions.slf4j}" + testRuntimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + testRuntimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" // SQL Unified Query API for PPL parsing testImplementation("org.opensearch.query:unified-query-api:${sqlUnifiedQueryVersion}") { @@ -150,26 +153,9 @@ tasks.withType(JavaCompile).configureEach { } tasks.named('thirdPartyAudit').configure { - // arrow-memory-core uses sun.misc.Unsafe via MemoryUtil for off-heap access - ignoreViolations( - 'org.apache.arrow.memory.util.MemoryUtil', - 'org.apache.arrow.memory.util.MemoryUtil$1', - // Guava internal Unsafe usage - 'com.google.common.cache.Striped64', - 'com.google.common.cache.Striped64$1', - 'com.google.common.cache.Striped64$Cell', - 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', - 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', - 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', - 'com.google.common.hash.Striped64', - 'com.google.common.hash.Striped64$1', - 'com.google.common.hash.Striped64$Cell', - 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', - 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', - 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', - 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1' - ) - + // Guava is excluded from runtimeClasspath (see configurations.runtimeClasspath block below) — + // its Unsafe violations no longer apply here and listing them would trip forbiddenApis + // ("All excluded classes seem to have no issues"). ignoreMissingClasses( // Optional brotli compression support pulled in by httpclient5 — not used by analytics 'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status', @@ -184,20 +170,33 @@ tasks.named('thirdPartyAudit').configure { ) } +// Jars provided by the arrow-flight-rpc parent plugin at runtime — strip from the bundled +// zip to avoid jar hell. Calcite drags guava, slf4j, jackson, commons-codec in transitively. +configurations.runtimeClasspath { + exclude group: 'com.google.guava' + exclude group: 'org.slf4j', module: 'slf4j-api' + exclude group: 'commons-codec', module: 'commons-codec' + exclude group: 'com.fasterxml.jackson.core' +} + configurations.all { // okhttp-aws-signer is a transitive dep of unified-query-common (via unified-query-core), // only published on JitPack, not needed for PPL parsing/planning exclude group: 'com.github.babbel', module: 'okhttp-aws-signer' resolutionStrategy { - // Align transitive versions with OpenSearch's managed versions - force 'com.google.guava:guava:33.4.0-jre' - force 'com.google.guava:failureaccess:1.0.2' + // Align transitive versions with OpenSearch's managed versions. + // Guava pinned to 33.3.1-jre to match arrow-flight-rpc (the extended parent) — + // children inherit the parent's loaded Guava at runtime. + force 'com.google.guava:guava:33.3.1-jre' + force 'com.google.guava:failureaccess:1.0.1' force 'com.google.errorprone:error_prone_annotations:2.36.0' force 'org.checkerframework:checker-qual:3.43.0' + force "com.fasterxml.jackson:jackson-bom:${versions.jackson}" force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" force "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" + force "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}" force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" force "org.slf4j:slf4j-api:${versions.slf4j}" diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-format-18.1.0.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/arrow-format-18.1.0.jar.sha1 deleted file mode 100644 index 6372bcd89eefd..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/arrow-format-18.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9d356b6f20620f5619ff85b174f97ae507df4997 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-memory-core-18.1.0.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/arrow-memory-core-18.1.0.jar.sha1 deleted file mode 100644 index 1a4da42973bfe..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/arrow-memory-core-18.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -35f4853d512f06759759b40b53bac850867886f8 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-18.1.0.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-18.1.0.jar.sha1 deleted file mode 100644 index f22c8e1687cb2..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-18.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8b48e832c98695bfd2b50ad0ed324e0d46099898 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-LICENSE.txt deleted file mode 100644 index 7bb1330a1002b..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-LICENSE.txt +++ /dev/null @@ -1,2261 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --------------------------------------------------------------------------------- - -src/arrow/util (some portions): Apache 2.0, and 3-clause BSD - -Some portions of this module are derived from code in the Chromium project, -copyright (c) Google inc and (c) The Chromium Authors and licensed under the -Apache 2.0 License or the under the 3-clause BSD license: - - Copyright (c) 2013 The Chromium Authors. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project includes code from Daniel Lemire's FrameOfReference project. - -https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp -https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py - -Copyright: 2013 Daniel Lemire -Home page: http://lemire.me/en/ -Project page: https://github.com/lemire/FrameOfReference -License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from the TensorFlow project - -Copyright 2015 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -This project includes code from the NumPy project. - -https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910 - -https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c - -Copyright (c) 2005-2017, NumPy Developers. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the NumPy Developers nor the names of any - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project includes code from the Boost project - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -This project includes code from the FlatBuffers project - -Copyright 2014 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -This project includes code from the tslib project - -Copyright 2015 Microsoft Corporation. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -This project includes code from the jemalloc project - -https://github.com/jemalloc/jemalloc - -Copyright (C) 2002-2017 Jason Evans . -All rights reserved. -Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice(s), - this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice(s), - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- - -This project includes code from the Go project, BSD 3-clause license + PATENTS -weak patent termination clause -(https://github.com/golang/go/blob/master/PATENTS). - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project includes code from the hs2client - -https://github.com/cloudera/hs2client - -Copyright 2016 Cloudera Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -The script ci/scripts/util_wait_for_it.sh has the following license - -Copyright (c) 2016 Giles Hall - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The script r/configure has the following license (MIT) - -Copyright (c) 2017, Jeroen Ooms and Jim Hester - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and -cpp/src/arrow/util/logging-test.cc are adapted from -Ray Project (https://github.com/ray-project/ray) (Apache 2.0). - -Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray) - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- -The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h, -cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h, -cpp/src/arrow/vendored/datetime/ios.mm, -cpp/src/arrow/vendored/datetime/tz.cpp are adapted from -Howard Hinnant's date library (https://github.com/HowardHinnant/date) -It is licensed under MIT license. - -The MIT License (MIT) -Copyright (c) 2015, 2016, 2017 Howard Hinnant -Copyright (c) 2016 Adrian Colomitchi -Copyright (c) 2017 Florian Dang -Copyright (c) 2017 Paul Thompson -Copyright (c) 2018 Tomasz Kamiński - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The file cpp/src/arrow/util/utf8.h includes code adapted from the page - https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ -with the following license (MIT) - -Copyright (c) 2008-2009 Bjoern Hoehrmann - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/xxhash/ have the following license -(BSD 2-Clause License) - -xxHash Library -Copyright (c) 2012-2014, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- xxHash homepage: http://www.xxhash.com -- xxHash source repository : https://github.com/Cyan4973/xxHash - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/double-conversion/ have the following license -(BSD 3-Clause License) - -Copyright 2006-2011, the V8 project authors. All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/uriparser/ have the following license -(BSD 3-Clause License) - -uriparser - RFC 3986 URI parsing library - -Copyright (C) 2007, Weijia Song -Copyright (C) 2007, Sebastian Pipping -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - * Redistributions of source code must retain the above - copyright notice, this list of conditions and the following - disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials - provided with the distribution. - - * Neither the name of the nor the names of its - contributors may be used to endorse or promote products - derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The files under dev/tasks/conda-recipes have the following license - -BSD 3-clause license -Copyright (c) 2015-2018, conda-forge -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR -TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/utfcpp/ have the following license - -Copyright 2006-2018 Nemanja Trifunovic - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -This project includes code from Apache Kudu. - - * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake - -Copyright: 2016 The Apache Software Foundation. -Home page: https://kudu.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from Apache Impala (incubating), formerly -Impala. The Impala code and rights were donated to the ASF as part of the -Incubator process after the initial code imports into Apache Parquet. - -Copyright: 2012 Cloudera, Inc. -Copyright: 2016 The Apache Software Foundation. -Home page: http://impala.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from Apache Aurora. - -* dev/release/{release,changelog,release-candidate} are based on the scripts from - Apache Aurora - -Copyright: 2016 The Apache Software Foundation. -Home page: https://aurora.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from the Google styleguide. - -* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide. - -Copyright: 2009 Google Inc. All rights reserved. -Homepage: https://github.com/google/styleguide -License: 3-clause BSD - --------------------------------------------------------------------------------- - -This project includes code from Snappy. - -* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code - from Google's Snappy project. - -Copyright: 2009 Google Inc. All rights reserved. -Homepage: https://github.com/google/snappy -License: 3-clause BSD - --------------------------------------------------------------------------------- - -This project includes code from the manylinux project. - -* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py, - requirements.txt} are based on code from the manylinux project. - -Copyright: 2016 manylinux -Homepage: https://github.com/pypa/manylinux -License: The MIT License (MIT) - --------------------------------------------------------------------------------- - -This project includes code from the cymove project: - -* python/pyarrow/includes/common.pxd includes code from the cymove project - -The MIT License (MIT) -Copyright (c) 2019 Omer Ozarslan - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE -OR OTHER DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -The projects includes code from the Ursabot project under the dev/archery -directory. - -License: BSD 2-Clause - -Copyright 2019 RStudio, Inc. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project include code from mingw-w64. - -* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5 - -Copyright (c) 2009 - 2013 by the mingw-w64 project -Homepage: https://mingw-w64.org -License: Zope Public License (ZPL) Version 2.1. - ---------------------------------------------------------------------------------- - -This project include code from Google's Asylo project. - -* cpp/src/arrow/result.h is based on status_or.h - -Copyright (c) Copyright 2017 Asylo authors -Homepage: https://asylo.dev/ -License: Apache 2.0 - --------------------------------------------------------------------------------- - -This project includes code from Google's protobuf project - -* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN -* cpp/src/arrow/util/bit_stream_utils.h contains code from wire_format_lite.h - -Copyright 2008 Google Inc. All rights reserved. -Homepage: https://developers.google.com/protocol-buffers/ -License: - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - --------------------------------------------------------------------------------- - -3rdparty dependency LLVM is statically linked in certain binary distributions. -Additionally some sections of source code have been derived from sources in LLVM -and have been clearly labeled as such. LLVM has the following license: - -============================================================================== -The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: -============================================================================== - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - ----- LLVM Exceptions to the Apache 2.0 License ---- - -As an exception, if, as a result of your compiling your source code, portions -of this Software are embedded into an Object form of such source code, you -may redistribute such embedded portions in such Object form without complying -with the conditions of Sections 4(a), 4(b) and 4(d) of the License. - -In addition, if you combine or link compiled forms of this Software with -software that is licensed under the GPLv2 ("Combined Software") and if a -court of competent jurisdiction determines that the patent provision (Section -3), the indemnity provision (Section 9) or other Section of the License -conflicts with the conditions of the GPLv2, you may retroactively and -prospectively choose to deem waived or otherwise exclude such Section(s) of -the License, but only in their entirety and only with respect to the Combined -Software. - -============================================================================== -Software from third parties included in the LLVM Project: -============================================================================== -The LLVM Project contains third party software which is under different license -terms. All such code will be identified clearly using at least one of two -mechanisms: -1) It will be in a separate directory tree with its own `LICENSE.txt` or - `LICENSE` file at the top containing the specific license and restrictions - which apply to that software, or -2) It will contain specific license and restriction terms at the top of every - file. - --------------------------------------------------------------------------------- - -3rdparty dependency gRPC is statically linked in certain binary -distributions, like the python wheels. gRPC has the following license: - -Copyright 2014 gRPC authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -3rdparty dependency Apache Thrift is statically linked in certain binary -distributions, like the python wheels. Apache Thrift has the following license: - -Apache Thrift -Copyright (C) 2006 - 2019, The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -3rdparty dependency Apache ORC is statically linked in certain binary -distributions, like the python wheels. Apache ORC has the following license: - -Apache ORC -Copyright 2013-2019 The Apache Software Foundation - -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). - -This product includes software developed by Hewlett-Packard: -(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -3rdparty dependency zstd is statically linked in certain binary -distributions, like the python wheels. ZSTD has the following license: - -BSD License - -For Zstandard software - -Copyright (c) 2016-present, Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency lz4 is statically linked in certain binary -distributions, like the python wheels. lz4 has the following license: - -LZ4 Library -Copyright (c) 2011-2016, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency Brotli is statically linked in certain binary -distributions, like the python wheels. Brotli has the following license: - -Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - --------------------------------------------------------------------------------- - -3rdparty dependency rapidjson is statically linked in certain binary -distributions, like the python wheels. rapidjson and its dependencies have the -following licenses: - -Tencent is pleased to support the open source community by making RapidJSON -available. - -Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. -All rights reserved. - -If you have downloaded a copy of the RapidJSON binary from Tencent, please note -that the RapidJSON binary is licensed under the MIT License. -If you have downloaded a copy of the RapidJSON source code from Tencent, please -note that RapidJSON source code is licensed under the MIT License, except for -the third-party components listed below which are subject to different license -terms. Your integration of RapidJSON into your own projects may require -compliance with the MIT License, as well as the other licenses applicable to -the third-party components included within RapidJSON. To avoid the problematic -JSON license in your own projects, it's sufficient to exclude the -bin/jsonchecker/ directory, as it's the only code under the JSON license. -A copy of the MIT License is included in this file. - -Other dependencies and licenses: - - Open Source Software Licensed Under the BSD License: - -------------------------------------------------------------------- - - The msinttypes r29 - Copyright (c) 2006-2013 Alexander Chemeris - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR - ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH - DAMAGE. - - Terms of the MIT License: - -------------------------------------------------------------------- - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -3rdparty dependency snappy is statically linked in certain binary -distributions, like the python wheels. snappy has the following license: - -Copyright 2011, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -=== - -Some of the benchmark data in testdata/ is licensed differently: - - - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and - is licensed under the Creative Commons Attribution 3.0 license - (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ - for more information. - - - kppkn.gtb is taken from the Gaviota chess tablebase set, and - is licensed under the MIT License. See - https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 - for more information. - - - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper - “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA - Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, - which is licensed under the CC-BY license. See - http://www.ploscompbiol.org/static/license for more ifnormation. - - - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project - Gutenberg. The first three have expired copyrights and are in the public - domain; the latter does not have expired copyright, but is still in the - public domain according to the license information - (http://www.gutenberg.org/ebooks/53). - --------------------------------------------------------------------------------- - -3rdparty dependency gflags is statically linked in certain binary -distributions, like the python wheels. gflags has the following license: - -Copyright (c) 2006, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency glog is statically linked in certain binary -distributions, like the python wheels. glog has the following license: - -Copyright (c) 2008, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -A function gettimeofday in utilities.cc is based on - -http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd - -The license of this code is: - -Copyright (c) 2003-2008, Jouni Malinen and contributors -All Rights Reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name(s) of the above-listed copyright holder(s) nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency re2 is statically linked in certain binary -distributions, like the python wheels. re2 has the following license: - -Copyright (c) 2009 The RE2 Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - * Neither the name of Google Inc. nor the names of its contributors - may be used to endorse or promote products derived from this - software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency c-ares is statically linked in certain binary -distributions, like the python wheels. c-ares has the following license: - -# c-ares license - -Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS -file. - -Copyright 1998 by the Massachusetts Institute of Technology. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, provided that -the above copyright notice appear in all copies and that both that copyright -notice and this permission notice appear in supporting documentation, and that -the name of M.I.T. not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior permission. -M.I.T. makes no representations about the suitability of this software for any -purpose. It is provided "as is" without express or implied warranty. - --------------------------------------------------------------------------------- - -3rdparty dependency zlib is redistributed as a dynamically linked shared -library in certain binary distributions, like the python wheels. In the future -this will likely change to static linkage. zlib has the following license: - -zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.11, January 15th, 2017 - - Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu - --------------------------------------------------------------------------------- - -3rdparty dependency openssl is redistributed as a dynamically linked shared -library in certain binary distributions, like the python wheels. openssl -preceding version 3 has the following license: - - LICENSE ISSUES - ============== - - The OpenSSL toolkit stays under a double license, i.e. both the conditions of - the OpenSSL License and the original SSLeay license apply to the toolkit. - See below for the actual license texts. - - OpenSSL License - --------------- - -/* ==================================================================== - * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * openssl-core@openssl.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.openssl.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - * - */ - - Original SSLeay License - ----------------------- - -/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) - * All rights reserved. - * - * This package is an SSL implementation written - * by Eric Young (eay@cryptsoft.com). - * The implementation was written so as to conform with Netscapes SSL. - * - * This library is free for commercial and non-commercial use as long as - * the following conditions are aheared to. The following conditions - * apply to all code found in this distribution, be it the RC4, RSA, - * lhash, DES, etc., code; not just the SSL code. The SSL documentation - * included with this distribution is covered by the same copyright terms - * except that the holder is Tim Hudson (tjh@cryptsoft.com). - * - * Copyright remains Eric Young's, and as such any Copyright notices in - * the code are not to be removed. - * If this package is used in a product, Eric Young should be given attribution - * as the author of the parts of the library used. - * This can be in the form of a textual message at program startup or - * in documentation (online or textual) provided with the package. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * "This product includes cryptographic software written by - * Eric Young (eay@cryptsoft.com)" - * The word 'cryptographic' can be left out if the rouines from the library - * being used are not cryptographic related :-). - * 4. If you include any Windows specific code (or a derivative thereof) from - * the apps directory (application code) you must include an acknowledgement: - * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" - * - * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * The licence and distribution terms for any publically available version or - * derivative of this code cannot be changed. i.e. this code cannot simply be - * copied and put under another distribution licence - * [including the GNU Public Licence.] - */ - --------------------------------------------------------------------------------- - -This project includes code from the rtools-backports project. - -* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code - from the rtools-backports project. - -Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms. -All rights reserved. -Homepage: https://github.com/r-windows/rtools-backports -License: 3-clause BSD - --------------------------------------------------------------------------------- - -Some code from pandas has been adapted for the pyarrow codebase. pandas is -available under the 3-clause BSD license, which follows: - -pandas license -============== - -Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team -All rights reserved. - -Copyright (c) 2008-2011 AQR Capital Management, LLC -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the copyright holder nor the names of any - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -Some bits from DyND, in particular aspects of the build system, have been -adapted from libdynd and dynd-python under the terms of the BSD 2-clause -license - -The BSD 2-Clause License - - Copyright (C) 2011-12, Dynamic NDArray Developers - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Dynamic NDArray Developers list: - - * Mark Wiebe - * Continuum Analytics - --------------------------------------------------------------------------------- - -Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted -for PyArrow. Ibis is released under the Apache License, Version 2.0. - --------------------------------------------------------------------------------- - -dev/tasks/homebrew-formulae/apache-arrow.rb has the following license: - -BSD 2-Clause License - -Copyright (c) 2009-present, Homebrew contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- - -cpp/src/arrow/vendored/base64.cpp has the following license - -ZLIB License - -Copyright (C) 2004-2017 René Nyffenegger - -This source code is provided 'as-is', without any express or implied -warranty. In no event will the author be held liable for any damages arising -from the use of this software. - -Permission is granted to anyone to use this software for any purpose, including -commercial applications, and to alter it and redistribute it freely, subject to -the following restrictions: - -1. The origin of this source code must not be misrepresented; you must not - claim that you wrote the original source code. If you use this source code - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - -2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original source code. - -3. This notice may not be removed or altered from any source distribution. - -René Nyffenegger rene.nyffenegger@adp-gmbh.ch - --------------------------------------------------------------------------------- - -This project includes code from Folly. - - * cpp/src/arrow/vendored/ProducerConsumerQueue.h - -is based on Folly's - - * folly/Portability.h - * folly/lang/Align.h - * folly/ProducerConsumerQueue.h - -Copyright: Copyright (c) Facebook, Inc. and its affiliates. -Home page: https://github.com/facebook/folly -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -The file cpp/src/arrow/vendored/musl/strptime.c has the following license - -Copyright © 2005-2020 Rich Felker, et al. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -The file cpp/cmake_modules/BuildUtils.cmake contains code from - -https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 - -which is made available under the MIT license - -Copyright (c) 2019 Cristian Adam - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/portable-snippets/ contain code from - -https://github.com/nemequ/portable-snippets - -and have the following copyright notice: - -Each source file contains a preamble explaining the license situation -for that file, which takes priority over this file. With the -exception of some code pulled in from other repositories (such as -µnit, an MIT-licensed project which is used for testing), the code is -public domain, released using the CC0 1.0 Universal dedication (*). - -(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/fast_float/ contain code from - -https://github.com/lemire/fast_float - -which is made available under the Apache License 2.0. - --------------------------------------------------------------------------------- - -The file python/pyarrow/vendored/docscrape.py contains code from - -https://github.com/numpy/numpydoc/ - -which is made available under the BSD 2-clause license. - --------------------------------------------------------------------------------- - -The file python/pyarrow/vendored/version.py contains code from - -https://github.com/pypa/packaging/ - -which is made available under both the Apache license v2.0 and the -BSD 2-clause license. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/pcg contain code from - -https://github.com/imneme/pcg-cpp - -and have the following copyright notice: - -Copyright 2014-2019 Melissa O'Neill , - and the PCG Project contributors. - -SPDX-License-Identifier: (Apache-2.0 OR MIT) - -Licensed under the Apache License, Version 2.0 (provided in -LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0) -or under the MIT license (provided in LICENSE-MIT.txt and at -http://opensource.org/licenses/MIT), at your option. This file may not -be copied, modified, or distributed except according to those terms. - -Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either -express or implied. See your chosen license for details. - --------------------------------------------------------------------------------- -r/R/dplyr-count-tally.R (some portions) - -Some portions of this file are derived from code from - -https://github.com/tidyverse/dplyr/ - -which is made available under the MIT license - -Copyright (c) 2013-2019 RStudio and others. - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the “Software”), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The file src/arrow/util/io_util.cc contains code from the CPython project -which is made available under the Python Software Foundation License Version 2. - --------------------------------------------------------------------------------- - -3rdparty dependency opentelemetry-cpp is statically linked in certain binary -distributions. opentelemetry-cpp is made available under the Apache License 2.0. - -Copyright The OpenTelemetry Authors -SPDX-License-Identifier: Apache-2.0 - --------------------------------------------------------------------------------- - -ci/conan/ is based on code from Conan Package and Dependency Manager. - -Copyright (c) 2019 Conan.io - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -3rdparty dependency UCX is redistributed as a dynamically linked shared -library in certain binary distributions. UCX has the following license: - -Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. -Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. -Copyright (C) 2014-2015 The University of Houston System. All rights reserved. -Copyright (C) 2015 The University of Tennessee and The University - of Tennessee Research Foundation. All rights reserved. -Copyright (C) 2016-2020 ARM Ltd. All rights reserved. -Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. -Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. -Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. -Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. -Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. -Copyright (C) 2016-2020 Stony Brook University. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. -3. Neither the name of the copyright holder nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The file dev/tasks/r/github.packages.yml contains code from - -https://github.com/ursa-labs/arrow-r-nightly - -which is made available under the Apache License 2.0. - --------------------------------------------------------------------------------- -.github/actions/sync-nightlies/action.yml (some portions) - -Some portions of this file are derived from code from - -https://github.com/JoshPiper/rsync-docker - -which is made available under the MIT license - -Copyright (c) 2020 Joshua Piper - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- -.github/actions/sync-nightlies/action.yml (some portions) - -Some portions of this file are derived from code from - -https://github.com/burnett01/rsync-deployments - -which is made available under the MIT license - -Copyright (c) 2019-2022 Contention -Copyright (c) 2019-2022 Burnett01 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- -java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java -java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java - -These file are derived from code from Netty, which is made available under the -Apache License 2.0. diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-NOTICE.txt deleted file mode 100644 index 2089c6fb20358..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/arrow-memory-unsafe-NOTICE.txt +++ /dev/null @@ -1,84 +0,0 @@ -Apache Arrow -Copyright 2016-2024 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This product includes software from the SFrame project (BSD, 3-clause). -* Copyright (C) 2015 Dato, Inc. -* Copyright (c) 2009 Carnegie Mellon University. - -This product includes software from the Feather project (Apache 2.0) -https://github.com/wesm/feather - -This product includes software from the DyND project (BSD 2-clause) -https://github.com/libdynd - -This product includes software from the LLVM project - * distributed under the University of Illinois Open Source - -This product includes software from the google-lint project - * Copyright (c) 2009 Google Inc. All rights reserved. - -This product includes software from the mman-win32 project - * Copyright https://code.google.com/p/mman-win32/ - * Licensed under the MIT License; - -This product includes software from the LevelDB project - * Copyright (c) 2011 The LevelDB Authors. All rights reserved. - * Use of this source code is governed by a BSD-style license that can be - * Moved from Kudu http://github.com/cloudera/kudu - -This product includes software from the CMake project - * Copyright 2001-2009 Kitware, Inc. - * Copyright 2012-2014 Continuum Analytics, Inc. - * All rights reserved. - -This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) - * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. - -This product includes software from the Ibis project (Apache 2.0) - * Copyright (c) 2015 Cloudera, Inc. - * https://github.com/cloudera/ibis - -This product includes software from Dremio (Apache 2.0) - * Copyright (C) 2017-2018 Dremio Corporation - * https://github.com/dremio/dremio-oss - -This product includes software from Google Guava (Apache 2.0) - * Copyright (C) 2007 The Guava Authors - * https://github.com/google/guava - -This product include software from CMake (BSD 3-Clause) - * CMake - Cross Platform Makefile Generator - * Copyright 2000-2019 Kitware, Inc. and Contributors - -The web site includes files generated by Jekyll. - --------------------------------------------------------------------------------- - -This product includes code from Apache Kudu, which includes the following in -its NOTICE file: - - Apache Kudu - Copyright 2016 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - - Portions of this software were developed at - Cloudera, Inc (http://www.cloudera.com/). - --------------------------------------------------------------------------------- - -This product includes code from Apache ORC, which includes the following in -its NOTICE file: - - Apache ORC - Copyright 2013-2019 The Apache Software Foundation - - This product includes software developed by The Apache Software - Foundation (http://www.apache.org/). - - This product includes software developed by Hewlett-Packard: - (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-vector-18.1.0.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/arrow-vector-18.1.0.jar.sha1 deleted file mode 100644 index d526f82b6f06e..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/arrow-vector-18.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b1fb77f4ef36fd52afe480ba12b7da77367eb88c \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-vector-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/arrow-vector-LICENSE.txt deleted file mode 100644 index 7bb1330a1002b..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/arrow-vector-LICENSE.txt +++ /dev/null @@ -1,2261 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --------------------------------------------------------------------------------- - -src/arrow/util (some portions): Apache 2.0, and 3-clause BSD - -Some portions of this module are derived from code in the Chromium project, -copyright (c) Google inc and (c) The Chromium Authors and licensed under the -Apache 2.0 License or the under the 3-clause BSD license: - - Copyright (c) 2013 The Chromium Authors. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project includes code from Daniel Lemire's FrameOfReference project. - -https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp -https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py - -Copyright: 2013 Daniel Lemire -Home page: http://lemire.me/en/ -Project page: https://github.com/lemire/FrameOfReference -License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from the TensorFlow project - -Copyright 2015 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -This project includes code from the NumPy project. - -https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910 - -https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c - -Copyright (c) 2005-2017, NumPy Developers. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the NumPy Developers nor the names of any - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project includes code from the Boost project - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -This project includes code from the FlatBuffers project - -Copyright 2014 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -This project includes code from the tslib project - -Copyright 2015 Microsoft Corporation. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -This project includes code from the jemalloc project - -https://github.com/jemalloc/jemalloc - -Copyright (C) 2002-2017 Jason Evans . -All rights reserved. -Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice(s), - this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice(s), - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- - -This project includes code from the Go project, BSD 3-clause license + PATENTS -weak patent termination clause -(https://github.com/golang/go/blob/master/PATENTS). - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project includes code from the hs2client - -https://github.com/cloudera/hs2client - -Copyright 2016 Cloudera Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -The script ci/scripts/util_wait_for_it.sh has the following license - -Copyright (c) 2016 Giles Hall - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The script r/configure has the following license (MIT) - -Copyright (c) 2017, Jeroen Ooms and Jim Hester - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and -cpp/src/arrow/util/logging-test.cc are adapted from -Ray Project (https://github.com/ray-project/ray) (Apache 2.0). - -Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray) - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- -The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h, -cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h, -cpp/src/arrow/vendored/datetime/ios.mm, -cpp/src/arrow/vendored/datetime/tz.cpp are adapted from -Howard Hinnant's date library (https://github.com/HowardHinnant/date) -It is licensed under MIT license. - -The MIT License (MIT) -Copyright (c) 2015, 2016, 2017 Howard Hinnant -Copyright (c) 2016 Adrian Colomitchi -Copyright (c) 2017 Florian Dang -Copyright (c) 2017 Paul Thompson -Copyright (c) 2018 Tomasz Kamiński - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The file cpp/src/arrow/util/utf8.h includes code adapted from the page - https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ -with the following license (MIT) - -Copyright (c) 2008-2009 Bjoern Hoehrmann - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/xxhash/ have the following license -(BSD 2-Clause License) - -xxHash Library -Copyright (c) 2012-2014, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- xxHash homepage: http://www.xxhash.com -- xxHash source repository : https://github.com/Cyan4973/xxHash - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/double-conversion/ have the following license -(BSD 3-Clause License) - -Copyright 2006-2011, the V8 project authors. All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/uriparser/ have the following license -(BSD 3-Clause License) - -uriparser - RFC 3986 URI parsing library - -Copyright (C) 2007, Weijia Song -Copyright (C) 2007, Sebastian Pipping -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - * Redistributions of source code must retain the above - copyright notice, this list of conditions and the following - disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials - provided with the distribution. - - * Neither the name of the nor the names of its - contributors may be used to endorse or promote products - derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The files under dev/tasks/conda-recipes have the following license - -BSD 3-clause license -Copyright (c) 2015-2018, conda-forge -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR -TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/utfcpp/ have the following license - -Copyright 2006-2018 Nemanja Trifunovic - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -This project includes code from Apache Kudu. - - * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake - -Copyright: 2016 The Apache Software Foundation. -Home page: https://kudu.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from Apache Impala (incubating), formerly -Impala. The Impala code and rights were donated to the ASF as part of the -Incubator process after the initial code imports into Apache Parquet. - -Copyright: 2012 Cloudera, Inc. -Copyright: 2016 The Apache Software Foundation. -Home page: http://impala.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from Apache Aurora. - -* dev/release/{release,changelog,release-candidate} are based on the scripts from - Apache Aurora - -Copyright: 2016 The Apache Software Foundation. -Home page: https://aurora.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from the Google styleguide. - -* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide. - -Copyright: 2009 Google Inc. All rights reserved. -Homepage: https://github.com/google/styleguide -License: 3-clause BSD - --------------------------------------------------------------------------------- - -This project includes code from Snappy. - -* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code - from Google's Snappy project. - -Copyright: 2009 Google Inc. All rights reserved. -Homepage: https://github.com/google/snappy -License: 3-clause BSD - --------------------------------------------------------------------------------- - -This project includes code from the manylinux project. - -* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py, - requirements.txt} are based on code from the manylinux project. - -Copyright: 2016 manylinux -Homepage: https://github.com/pypa/manylinux -License: The MIT License (MIT) - --------------------------------------------------------------------------------- - -This project includes code from the cymove project: - -* python/pyarrow/includes/common.pxd includes code from the cymove project - -The MIT License (MIT) -Copyright (c) 2019 Omer Ozarslan - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE -OR OTHER DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -The projects includes code from the Ursabot project under the dev/archery -directory. - -License: BSD 2-Clause - -Copyright 2019 RStudio, Inc. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project include code from mingw-w64. - -* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5 - -Copyright (c) 2009 - 2013 by the mingw-w64 project -Homepage: https://mingw-w64.org -License: Zope Public License (ZPL) Version 2.1. - ---------------------------------------------------------------------------------- - -This project include code from Google's Asylo project. - -* cpp/src/arrow/result.h is based on status_or.h - -Copyright (c) Copyright 2017 Asylo authors -Homepage: https://asylo.dev/ -License: Apache 2.0 - --------------------------------------------------------------------------------- - -This project includes code from Google's protobuf project - -* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN -* cpp/src/arrow/util/bit_stream_utils.h contains code from wire_format_lite.h - -Copyright 2008 Google Inc. All rights reserved. -Homepage: https://developers.google.com/protocol-buffers/ -License: - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - --------------------------------------------------------------------------------- - -3rdparty dependency LLVM is statically linked in certain binary distributions. -Additionally some sections of source code have been derived from sources in LLVM -and have been clearly labeled as such. LLVM has the following license: - -============================================================================== -The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: -============================================================================== - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - ----- LLVM Exceptions to the Apache 2.0 License ---- - -As an exception, if, as a result of your compiling your source code, portions -of this Software are embedded into an Object form of such source code, you -may redistribute such embedded portions in such Object form without complying -with the conditions of Sections 4(a), 4(b) and 4(d) of the License. - -In addition, if you combine or link compiled forms of this Software with -software that is licensed under the GPLv2 ("Combined Software") and if a -court of competent jurisdiction determines that the patent provision (Section -3), the indemnity provision (Section 9) or other Section of the License -conflicts with the conditions of the GPLv2, you may retroactively and -prospectively choose to deem waived or otherwise exclude such Section(s) of -the License, but only in their entirety and only with respect to the Combined -Software. - -============================================================================== -Software from third parties included in the LLVM Project: -============================================================================== -The LLVM Project contains third party software which is under different license -terms. All such code will be identified clearly using at least one of two -mechanisms: -1) It will be in a separate directory tree with its own `LICENSE.txt` or - `LICENSE` file at the top containing the specific license and restrictions - which apply to that software, or -2) It will contain specific license and restriction terms at the top of every - file. - --------------------------------------------------------------------------------- - -3rdparty dependency gRPC is statically linked in certain binary -distributions, like the python wheels. gRPC has the following license: - -Copyright 2014 gRPC authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -3rdparty dependency Apache Thrift is statically linked in certain binary -distributions, like the python wheels. Apache Thrift has the following license: - -Apache Thrift -Copyright (C) 2006 - 2019, The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -3rdparty dependency Apache ORC is statically linked in certain binary -distributions, like the python wheels. Apache ORC has the following license: - -Apache ORC -Copyright 2013-2019 The Apache Software Foundation - -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). - -This product includes software developed by Hewlett-Packard: -(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -3rdparty dependency zstd is statically linked in certain binary -distributions, like the python wheels. ZSTD has the following license: - -BSD License - -For Zstandard software - -Copyright (c) 2016-present, Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency lz4 is statically linked in certain binary -distributions, like the python wheels. lz4 has the following license: - -LZ4 Library -Copyright (c) 2011-2016, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency Brotli is statically linked in certain binary -distributions, like the python wheels. Brotli has the following license: - -Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - --------------------------------------------------------------------------------- - -3rdparty dependency rapidjson is statically linked in certain binary -distributions, like the python wheels. rapidjson and its dependencies have the -following licenses: - -Tencent is pleased to support the open source community by making RapidJSON -available. - -Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. -All rights reserved. - -If you have downloaded a copy of the RapidJSON binary from Tencent, please note -that the RapidJSON binary is licensed under the MIT License. -If you have downloaded a copy of the RapidJSON source code from Tencent, please -note that RapidJSON source code is licensed under the MIT License, except for -the third-party components listed below which are subject to different license -terms. Your integration of RapidJSON into your own projects may require -compliance with the MIT License, as well as the other licenses applicable to -the third-party components included within RapidJSON. To avoid the problematic -JSON license in your own projects, it's sufficient to exclude the -bin/jsonchecker/ directory, as it's the only code under the JSON license. -A copy of the MIT License is included in this file. - -Other dependencies and licenses: - - Open Source Software Licensed Under the BSD License: - -------------------------------------------------------------------- - - The msinttypes r29 - Copyright (c) 2006-2013 Alexander Chemeris - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR - ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH - DAMAGE. - - Terms of the MIT License: - -------------------------------------------------------------------- - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -3rdparty dependency snappy is statically linked in certain binary -distributions, like the python wheels. snappy has the following license: - -Copyright 2011, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -=== - -Some of the benchmark data in testdata/ is licensed differently: - - - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and - is licensed under the Creative Commons Attribution 3.0 license - (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ - for more information. - - - kppkn.gtb is taken from the Gaviota chess tablebase set, and - is licensed under the MIT License. See - https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 - for more information. - - - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper - “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA - Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, - which is licensed under the CC-BY license. See - http://www.ploscompbiol.org/static/license for more ifnormation. - - - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project - Gutenberg. The first three have expired copyrights and are in the public - domain; the latter does not have expired copyright, but is still in the - public domain according to the license information - (http://www.gutenberg.org/ebooks/53). - --------------------------------------------------------------------------------- - -3rdparty dependency gflags is statically linked in certain binary -distributions, like the python wheels. gflags has the following license: - -Copyright (c) 2006, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency glog is statically linked in certain binary -distributions, like the python wheels. glog has the following license: - -Copyright (c) 2008, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -A function gettimeofday in utilities.cc is based on - -http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd - -The license of this code is: - -Copyright (c) 2003-2008, Jouni Malinen and contributors -All Rights Reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name(s) of the above-listed copyright holder(s) nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency re2 is statically linked in certain binary -distributions, like the python wheels. re2 has the following license: - -Copyright (c) 2009 The RE2 Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - * Neither the name of Google Inc. nor the names of its contributors - may be used to endorse or promote products derived from this - software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency c-ares is statically linked in certain binary -distributions, like the python wheels. c-ares has the following license: - -# c-ares license - -Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS -file. - -Copyright 1998 by the Massachusetts Institute of Technology. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, provided that -the above copyright notice appear in all copies and that both that copyright -notice and this permission notice appear in supporting documentation, and that -the name of M.I.T. not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior permission. -M.I.T. makes no representations about the suitability of this software for any -purpose. It is provided "as is" without express or implied warranty. - --------------------------------------------------------------------------------- - -3rdparty dependency zlib is redistributed as a dynamically linked shared -library in certain binary distributions, like the python wheels. In the future -this will likely change to static linkage. zlib has the following license: - -zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.11, January 15th, 2017 - - Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu - --------------------------------------------------------------------------------- - -3rdparty dependency openssl is redistributed as a dynamically linked shared -library in certain binary distributions, like the python wheels. openssl -preceding version 3 has the following license: - - LICENSE ISSUES - ============== - - The OpenSSL toolkit stays under a double license, i.e. both the conditions of - the OpenSSL License and the original SSLeay license apply to the toolkit. - See below for the actual license texts. - - OpenSSL License - --------------- - -/* ==================================================================== - * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * openssl-core@openssl.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.openssl.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - * - */ - - Original SSLeay License - ----------------------- - -/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) - * All rights reserved. - * - * This package is an SSL implementation written - * by Eric Young (eay@cryptsoft.com). - * The implementation was written so as to conform with Netscapes SSL. - * - * This library is free for commercial and non-commercial use as long as - * the following conditions are aheared to. The following conditions - * apply to all code found in this distribution, be it the RC4, RSA, - * lhash, DES, etc., code; not just the SSL code. The SSL documentation - * included with this distribution is covered by the same copyright terms - * except that the holder is Tim Hudson (tjh@cryptsoft.com). - * - * Copyright remains Eric Young's, and as such any Copyright notices in - * the code are not to be removed. - * If this package is used in a product, Eric Young should be given attribution - * as the author of the parts of the library used. - * This can be in the form of a textual message at program startup or - * in documentation (online or textual) provided with the package. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * "This product includes cryptographic software written by - * Eric Young (eay@cryptsoft.com)" - * The word 'cryptographic' can be left out if the rouines from the library - * being used are not cryptographic related :-). - * 4. If you include any Windows specific code (or a derivative thereof) from - * the apps directory (application code) you must include an acknowledgement: - * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" - * - * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * The licence and distribution terms for any publically available version or - * derivative of this code cannot be changed. i.e. this code cannot simply be - * copied and put under another distribution licence - * [including the GNU Public Licence.] - */ - --------------------------------------------------------------------------------- - -This project includes code from the rtools-backports project. - -* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code - from the rtools-backports project. - -Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms. -All rights reserved. -Homepage: https://github.com/r-windows/rtools-backports -License: 3-clause BSD - --------------------------------------------------------------------------------- - -Some code from pandas has been adapted for the pyarrow codebase. pandas is -available under the 3-clause BSD license, which follows: - -pandas license -============== - -Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team -All rights reserved. - -Copyright (c) 2008-2011 AQR Capital Management, LLC -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the copyright holder nor the names of any - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -Some bits from DyND, in particular aspects of the build system, have been -adapted from libdynd and dynd-python under the terms of the BSD 2-clause -license - -The BSD 2-Clause License - - Copyright (C) 2011-12, Dynamic NDArray Developers - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Dynamic NDArray Developers list: - - * Mark Wiebe - * Continuum Analytics - --------------------------------------------------------------------------------- - -Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted -for PyArrow. Ibis is released under the Apache License, Version 2.0. - --------------------------------------------------------------------------------- - -dev/tasks/homebrew-formulae/apache-arrow.rb has the following license: - -BSD 2-Clause License - -Copyright (c) 2009-present, Homebrew contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- - -cpp/src/arrow/vendored/base64.cpp has the following license - -ZLIB License - -Copyright (C) 2004-2017 René Nyffenegger - -This source code is provided 'as-is', without any express or implied -warranty. In no event will the author be held liable for any damages arising -from the use of this software. - -Permission is granted to anyone to use this software for any purpose, including -commercial applications, and to alter it and redistribute it freely, subject to -the following restrictions: - -1. The origin of this source code must not be misrepresented; you must not - claim that you wrote the original source code. If you use this source code - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - -2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original source code. - -3. This notice may not be removed or altered from any source distribution. - -René Nyffenegger rene.nyffenegger@adp-gmbh.ch - --------------------------------------------------------------------------------- - -This project includes code from Folly. - - * cpp/src/arrow/vendored/ProducerConsumerQueue.h - -is based on Folly's - - * folly/Portability.h - * folly/lang/Align.h - * folly/ProducerConsumerQueue.h - -Copyright: Copyright (c) Facebook, Inc. and its affiliates. -Home page: https://github.com/facebook/folly -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -The file cpp/src/arrow/vendored/musl/strptime.c has the following license - -Copyright © 2005-2020 Rich Felker, et al. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -The file cpp/cmake_modules/BuildUtils.cmake contains code from - -https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 - -which is made available under the MIT license - -Copyright (c) 2019 Cristian Adam - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/portable-snippets/ contain code from - -https://github.com/nemequ/portable-snippets - -and have the following copyright notice: - -Each source file contains a preamble explaining the license situation -for that file, which takes priority over this file. With the -exception of some code pulled in from other repositories (such as -µnit, an MIT-licensed project which is used for testing), the code is -public domain, released using the CC0 1.0 Universal dedication (*). - -(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/fast_float/ contain code from - -https://github.com/lemire/fast_float - -which is made available under the Apache License 2.0. - --------------------------------------------------------------------------------- - -The file python/pyarrow/vendored/docscrape.py contains code from - -https://github.com/numpy/numpydoc/ - -which is made available under the BSD 2-clause license. - --------------------------------------------------------------------------------- - -The file python/pyarrow/vendored/version.py contains code from - -https://github.com/pypa/packaging/ - -which is made available under both the Apache license v2.0 and the -BSD 2-clause license. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/pcg contain code from - -https://github.com/imneme/pcg-cpp - -and have the following copyright notice: - -Copyright 2014-2019 Melissa O'Neill , - and the PCG Project contributors. - -SPDX-License-Identifier: (Apache-2.0 OR MIT) - -Licensed under the Apache License, Version 2.0 (provided in -LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0) -or under the MIT license (provided in LICENSE-MIT.txt and at -http://opensource.org/licenses/MIT), at your option. This file may not -be copied, modified, or distributed except according to those terms. - -Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either -express or implied. See your chosen license for details. - --------------------------------------------------------------------------------- -r/R/dplyr-count-tally.R (some portions) - -Some portions of this file are derived from code from - -https://github.com/tidyverse/dplyr/ - -which is made available under the MIT license - -Copyright (c) 2013-2019 RStudio and others. - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the “Software”), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The file src/arrow/util/io_util.cc contains code from the CPython project -which is made available under the Python Software Foundation License Version 2. - --------------------------------------------------------------------------------- - -3rdparty dependency opentelemetry-cpp is statically linked in certain binary -distributions. opentelemetry-cpp is made available under the Apache License 2.0. - -Copyright The OpenTelemetry Authors -SPDX-License-Identifier: Apache-2.0 - --------------------------------------------------------------------------------- - -ci/conan/ is based on code from Conan Package and Dependency Manager. - -Copyright (c) 2019 Conan.io - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -3rdparty dependency UCX is redistributed as a dynamically linked shared -library in certain binary distributions. UCX has the following license: - -Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. -Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. -Copyright (C) 2014-2015 The University of Houston System. All rights reserved. -Copyright (C) 2015 The University of Tennessee and The University - of Tennessee Research Foundation. All rights reserved. -Copyright (C) 2016-2020 ARM Ltd. All rights reserved. -Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. -Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. -Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. -Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. -Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. -Copyright (C) 2016-2020 Stony Brook University. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. -3. Neither the name of the copyright holder nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The file dev/tasks/r/github.packages.yml contains code from - -https://github.com/ursa-labs/arrow-r-nightly - -which is made available under the Apache License 2.0. - --------------------------------------------------------------------------------- -.github/actions/sync-nightlies/action.yml (some portions) - -Some portions of this file are derived from code from - -https://github.com/JoshPiper/rsync-docker - -which is made available under the MIT license - -Copyright (c) 2020 Joshua Piper - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- -.github/actions/sync-nightlies/action.yml (some portions) - -Some portions of this file are derived from code from - -https://github.com/burnett01/rsync-deployments - -which is made available under the MIT license - -Copyright (c) 2019-2022 Contention -Copyright (c) 2019-2022 Burnett01 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- -java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java -java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java - -These file are derived from code from Netty, which is made available under the -Apache License 2.0. diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-vector-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/arrow-vector-NOTICE.txt deleted file mode 100644 index 2089c6fb20358..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/arrow-vector-NOTICE.txt +++ /dev/null @@ -1,84 +0,0 @@ -Apache Arrow -Copyright 2016-2024 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This product includes software from the SFrame project (BSD, 3-clause). -* Copyright (C) 2015 Dato, Inc. -* Copyright (c) 2009 Carnegie Mellon University. - -This product includes software from the Feather project (Apache 2.0) -https://github.com/wesm/feather - -This product includes software from the DyND project (BSD 2-clause) -https://github.com/libdynd - -This product includes software from the LLVM project - * distributed under the University of Illinois Open Source - -This product includes software from the google-lint project - * Copyright (c) 2009 Google Inc. All rights reserved. - -This product includes software from the mman-win32 project - * Copyright https://code.google.com/p/mman-win32/ - * Licensed under the MIT License; - -This product includes software from the LevelDB project - * Copyright (c) 2011 The LevelDB Authors. All rights reserved. - * Use of this source code is governed by a BSD-style license that can be - * Moved from Kudu http://github.com/cloudera/kudu - -This product includes software from the CMake project - * Copyright 2001-2009 Kitware, Inc. - * Copyright 2012-2014 Continuum Analytics, Inc. - * All rights reserved. - -This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) - * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. - -This product includes software from the Ibis project (Apache 2.0) - * Copyright (c) 2015 Cloudera, Inc. - * https://github.com/cloudera/ibis - -This product includes software from Dremio (Apache 2.0) - * Copyright (C) 2017-2018 Dremio Corporation - * https://github.com/dremio/dremio-oss - -This product includes software from Google Guava (Apache 2.0) - * Copyright (C) 2007 The Guava Authors - * https://github.com/google/guava - -This product include software from CMake (BSD 3-Clause) - * CMake - Cross Platform Makefile Generator - * Copyright 2000-2019 Kitware, Inc. and Contributors - -The web site includes files generated by Jekyll. - --------------------------------------------------------------------------------- - -This product includes code from Apache Kudu, which includes the following in -its NOTICE file: - - Apache Kudu - Copyright 2016 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - - Portions of this software were developed at - Cloudera, Inc (http://www.cloudera.com/). - --------------------------------------------------------------------------------- - -This product includes code from Apache ORC, which includes the following in -its NOTICE file: - - Apache ORC - Copyright 2013-2019 The Apache Software Foundation - - This product includes software developed by The Apache Software - Foundation (http://www.apache.org/). - - This product includes software developed by Hewlett-Packard: - (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P diff --git a/sandbox/plugins/analytics-engine/licenses/failureaccess-1.0.2.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/failureaccess-1.0.2.jar.sha1 deleted file mode 100644 index e1dbdc6bf7320..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/failureaccess-1.0.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c4a06a64e650562f30b7bf9aaec1bfed43aca12b \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/failureaccess-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/failureaccess-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-2.0.0.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-2.0.0.jar.sha1 deleted file mode 100644 index 08a9b7e399e74..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-2.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19da0c1d9f585d2c402057f993f8dea2ff382837 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/flatbuffers-java-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/sandbox/plugins/analytics-engine/licenses/guava-33.4.0-jre.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/guava-33.4.0-jre.jar.sha1 deleted file mode 100644 index 42b66665a578a..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/guava-33.4.0-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -03fcc0a259f724c7de54a6a55ea7e26d3d5c0cac \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/guava-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/guava-LICENSE.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/guava-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/sandbox/plugins/analytics-engine/licenses/guava-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/guava-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-annotations-2.21.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/jackson-annotations-2.21.jar.sha1 deleted file mode 100644 index 081172e4c72c3..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/jackson-annotations-2.21.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b1bc1868bf02dc0bd6c7836257a036a331005309 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-annotations-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/jackson-annotations-LICENSE.txt deleted file mode 100644 index f5f45d26a49d6..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/jackson-annotations-LICENSE.txt +++ /dev/null @@ -1,8 +0,0 @@ -This copy of Jackson JSON processor streaming parser/generator is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-annotations-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/jackson-annotations-NOTICE.txt deleted file mode 100644 index 8b137891791fe..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/jackson-annotations-NOTICE.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.2.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.2.jar.sha1 deleted file mode 100644 index 52686081905c0..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71ab8ff75b4fd74afdee0004173fdd15de1d6a28 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-databind-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/jackson-databind-LICENSE.txt deleted file mode 100644 index f5f45d26a49d6..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/jackson-databind-LICENSE.txt +++ /dev/null @@ -1,8 +0,0 @@ -This copy of Jackson JSON processor streaming parser/generator is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-databind-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/jackson-databind-NOTICE.txt deleted file mode 100644 index 4c976b7b4cc58..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/jackson-databind-NOTICE.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. diff --git a/sandbox/plugins/analytics-engine/licenses/slf4j-api-2.0.17.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/slf4j-api-2.0.17.jar.sha1 deleted file mode 100644 index 435f6c13a28b6..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/slf4j-api-2.0.17.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/slf4j-api-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 8fda22f4d72f6..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2014 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/sandbox/plugins/analytics-engine/licenses/slf4j-api-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java index 6c5786847a4d7..c3a4bbc23c2fe 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java @@ -67,6 +67,7 @@ public AnalyticsPlugin() {} private final List backEnds = new ArrayList<>(); private SqlOperatorTable operatorTable; + private AnalyticsSearchService searchService; @SuppressWarnings("rawtypes") @Override @@ -96,7 +97,7 @@ public Collection createComponents( for (AnalyticsSearchBackendPlugin be : backEnds) { backEndsByName.put(be.name(), be); } - AnalyticsSearchService searchService = new AnalyticsSearchService(backEndsByName); + searchService = new AnalyticsSearchService(backEndsByName); // Returned as components so Guice can inject them into DefaultPlanExecutor // (a HandledTransportAction registered via getActions() — constructed by Guice @@ -120,6 +121,13 @@ public Collection createGuiceModules() { return List.of(new ActionHandler<>(AnalyticsQueryAction.INSTANCE, DefaultPlanExecutor.class)); } + @Override + public void close() { + if (searchService != null) { + searchService.close(); + } + } + private SqlOperatorTable aggregateOperatorTables() { // TODO: re-wire once operatorTable() is added back to AnalyticsSearchBackendPlugin return SqlOperatorTables.of(); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java index 43c6828083f06..252347f2da2cf 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java @@ -8,7 +8,7 @@ package org.opensearch.analytics.exec; -import org.opensearch.action.search.SearchShardTask; +import org.apache.arrow.memory.BufferAllocator; import org.opensearch.analytics.backend.AnalyticsOperationListener; import org.opensearch.analytics.backend.EngineResultBatch; import org.opensearch.analytics.backend.EngineResultStream; @@ -18,13 +18,16 @@ import org.opensearch.analytics.exec.action.FragmentExecutionResponse; import org.opensearch.analytics.exec.task.AnalyticsShardTask; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; +import org.opensearch.arrow.flight.transport.ArrowAllocatorProvider; import org.opensearch.common.Nullable; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.core.tasks.TaskCancelledException; import org.opensearch.index.engine.exec.IndexReaderProvider; import org.opensearch.index.engine.exec.IndexReaderProvider.Reader; import org.opensearch.index.shard.IndexShard; +import org.opensearch.tasks.Task; +import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -39,12 +42,19 @@ *

      Does NOT hold {@code IndicesService} — receives an already-resolved * {@link IndexShard} from the transport action. * + *

      Owns a service-lifetime {@link BufferAllocator} shared by every fragment, obtained as a child of the + * node-level root via {@link ArrowAllocatorProvider}. One allocator per service means memory accounting is + * reported at the service level. For the streaming path, Arrow Flight's outbound handler co-locates its + * transfer target on the same root (see {@code FlightOutboundHandler#processBatchTask}), keeping transfers + * same-root and avoiding the known cross-allocator bug with foreign-backed buffers from the C Data Interface. + * * @opensearch.internal */ -public class AnalyticsSearchService { +public class AnalyticsSearchService implements AutoCloseable { private final Map backends; private final AnalyticsOperationListener listener; + private final BufferAllocator allocator; public AnalyticsSearchService(Map backends) { this(backends, List.of()); @@ -53,29 +63,73 @@ public AnalyticsSearchService(Map backends public AnalyticsSearchService(Map backends, List listeners) { this.backends = backends; this.listener = new AnalyticsOperationListener.CompositeListener(listeners); + this.allocator = ArrowAllocatorProvider.newChildAllocator("analytics-search-service", Long.MAX_VALUE); + } + + @Override + public void close() { + allocator.close(); } - /** - * Executes a plan fragment against the given shard and returns the collected results. - * - * @param request the fragment execution request - * @param shard the already-resolved index shard - * @return a response containing field names and result rows - */ public FragmentExecutionResponse executeFragment(FragmentExecutionRequest request, IndexShard shard) { return executeFragment(request, shard, null); } - /** - * Executes a plan fragment against the given shard and returns the collected results, - * polling the shard task for cancellation between batches. - * - * @param request the fragment execution request - * @param shard the already-resolved index shard - * @param task the shard task to poll for cancellation (nullable) - * @return a response containing field names and result rows - */ public FragmentExecutionResponse executeFragment(FragmentExecutionRequest request, IndexShard shard, AnalyticsShardTask task) { + ResolvedFragment resolved = resolveFragment(request, shard); + long startNanos = System.nanoTime(); + try (FragmentResources ctx = startFragment(request, resolved, task)) { + FragmentExecutionResponse response = collectResponse(ctx.stream(), task); + long tookNanos = System.nanoTime() - startNanos; + listener.onFragmentSuccess(resolved.queryId, resolved.stageId, resolved.shardIdStr, tookNanos, response.getRows().size()); + return response; + } catch (TaskCancelledException | IllegalStateException | IllegalArgumentException e) { + listener.onFragmentFailure(resolved.queryId, resolved.stageId, resolved.shardIdStr, e); + throw e; + } catch (Exception e) { + listener.onFragmentFailure(resolved.queryId, resolved.stageId, resolved.shardIdStr, e); + throw new RuntimeException("Failed to execute fragment on " + shard.shardId(), e); + } + } + + public FragmentResources executeFragmentStreaming(FragmentExecutionRequest request, IndexShard shard, AnalyticsShardTask task) { + ResolvedFragment resolved = resolveFragment(request, shard); + try { + return startFragment(request, resolved, task); + } catch (TaskCancelledException | IllegalStateException | IllegalArgumentException e) { + listener.onFragmentFailure(resolved.queryId, resolved.stageId, resolved.shardIdStr, e); + throw e; + } catch (Exception e) { + listener.onFragmentFailure(resolved.queryId, resolved.stageId, resolved.shardIdStr, e); + throw new RuntimeException("Failed to start streaming fragment on " + shard.shardId(), e); + } + } + + private FragmentResources startFragment(FragmentExecutionRequest request, ResolvedFragment resolved, Task task) throws IOException { + GatedCloseable gatedReader = resolved.readerProvider.acquireReader(); + SearchExecEngine engine = null; + EngineResultStream stream = null; + try { + ExecutionContext ctx = buildContext(request, gatedReader.get(), resolved.plan, task); + AnalyticsSearchBackendPlugin backend = backends.get(resolved.plan.getBackendId()); + engine = backend.getSearchExecEngineProvider().createSearchExecEngine(ctx); + stream = engine.execute(ctx); + return new FragmentResources(gatedReader, engine, stream); + } catch (Exception e) { + try { + new FragmentResources(gatedReader, engine, stream).close(); + } catch (Exception suppressed) { + e.addSuppressed(suppressed); + } + throw e; + } + } + + private record ResolvedFragment(IndexReaderProvider readerProvider, FragmentExecutionRequest.PlanAlternative plan, String queryId, + int stageId, String shardIdStr) { + } + + private ResolvedFragment resolveFragment(FragmentExecutionRequest request, IndexShard shard) { IndexReaderProvider readerProvider = shard.getReaderProvider(); if (readerProvider == null) { throw new IllegalStateException("No ReaderProvider on " + shard.shardId()); @@ -100,58 +154,26 @@ public FragmentExecutionResponse executeFragment(FragmentExecutionRequest reques } String shardIdStr = shard.shardId().toString(); - String queryId = request.getQueryId(); - int stageId = request.getStageId(); - - listener.onPreFragmentExecution(queryId, stageId, shardIdStr); + listener.onPreFragmentExecution(request.getQueryId(), request.getStageId(), shardIdStr); + return new ResolvedFragment(readerProvider, selectedPlan, request.getQueryId(), request.getStageId(), shardIdStr); + } - long startNanos = System.nanoTime(); - try (GatedCloseable gatedReader = readerProvider.acquireReader()) { - SearchShardTask searchShardTask = null; // TODO: real task for cancellation - ExecutionContext ctx = new ExecutionContext(request.getShardId().getIndexName(), searchShardTask, gatedReader.get()); - ctx.setFragmentBytes(selectedPlan.getFragmentBytes()); - - AnalyticsSearchBackendPlugin backend = backends.get(selectedPlan.getBackendId()); - - try ( - SearchExecEngine engine = backend.getSearchExecEngineProvider() - .createSearchExecEngine(ctx) - ) { - try (EngineResultStream stream = engine.execute(ctx)) { - FragmentExecutionResponse response = collectResponse(stream, task); - long tookNanos = System.nanoTime() - startNanos; - listener.onFragmentSuccess(queryId, stageId, shardIdStr, tookNanos, response.getRows().size()); - return response; - } - } - } catch (TaskCancelledException e) { - listener.onFragmentFailure(queryId, stageId, shardIdStr, e); - throw e; // do NOT wrap — preserve type - } catch (IllegalStateException | IllegalArgumentException e) { - listener.onFragmentFailure(queryId, stageId, shardIdStr, e); - throw e; - } catch (Exception e) { - listener.onFragmentFailure(queryId, stageId, shardIdStr, e); - throw new RuntimeException("Failed to execute fragment on " + shard.shardId(), e); - } + private ExecutionContext buildContext( + FragmentExecutionRequest request, + Reader reader, + FragmentExecutionRequest.PlanAlternative plan, + Task task + ) { + ExecutionContext ctx = new ExecutionContext(request.getShardId().getIndexName(), task, reader); + ctx.setFragmentBytes(plan.getFragmentBytes()); + ctx.setAllocator(allocator); + return ctx; } - /** - * Collects all batches from the result stream into a single {@link FragmentExecutionResponse}. - * Field names are captured from the first batch. - */ FragmentExecutionResponse collectResponse(EngineResultStream stream) { return collectResponse(stream, null); } - /** - * Collects all batches from the result stream into a single {@link FragmentExecutionResponse}. - * Field names are captured from the first batch. Polls the shard task for cancellation - * at each batch boundary. - * - * @param stream the result stream to drain - * @param task the shard task to poll for cancellation (nullable) - */ FragmentExecutionResponse collectResponse(EngineResultStream stream, @Nullable AnalyticsShardTask task) { List rows = new ArrayList<>(); List fieldNames = null; @@ -161,15 +183,19 @@ FragmentExecutionResponse collectResponse(EngineResultStream stream, @Nullable A throw new TaskCancelledException("task cancelled: " + task.getReasonCancelled()); } EngineResultBatch batch = it.next(); - if (fieldNames == null) { - fieldNames = batch.getFieldNames(); - } - for (int row = 0; row < batch.getRowCount(); row++) { - Object[] vals = new Object[fieldNames.size()]; - for (int col = 0; col < fieldNames.size(); col++) { - vals[col] = batch.getFieldValue(fieldNames.get(col), row); + try { + if (fieldNames == null) { + fieldNames = batch.getFieldNames(); + } + for (int row = 0; row < batch.getRowCount(); row++) { + Object[] vals = new Object[fieldNames.size()]; + for (int col = 0; col < fieldNames.size(); col++) { + vals[col] = batch.getFieldValue(fieldNames.get(col), row); + } + rows.add(vals); } - rows.add(vals); + } finally { + batch.getArrowRoot().close(); } } return new FragmentExecutionResponse(fieldNames != null ? fieldNames : List.of(), rows); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchTransportService.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchTransportService.java index f80e7b3343b76..d7a933a6a42d1 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchTransportService.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchTransportService.java @@ -8,15 +8,20 @@ package org.opensearch.analytics.exec; +import org.opensearch.analytics.backend.EngineResultBatch; import org.opensearch.analytics.exec.action.FragmentExecutionAction; +import org.opensearch.analytics.exec.action.FragmentExecutionArrowResponse; import org.opensearch.analytics.exec.action.FragmentExecutionRequest; import org.opensearch.analytics.exec.action.FragmentExecutionResponse; +import org.opensearch.analytics.exec.task.AnalyticsShardTask; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; import org.opensearch.common.inject.Singleton; +import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType; @@ -28,36 +33,30 @@ import org.opensearch.transport.TransportRequestOptions; import org.opensearch.transport.TransportResponseHandler; import org.opensearch.transport.TransportService; +import org.opensearch.transport.stream.StreamErrorCode; +import org.opensearch.transport.stream.StreamException; import org.opensearch.transport.stream.StreamTransportResponse; import java.io.IOException; -import java.util.Objects; +import java.util.Iterator; /** * Stateless transport dispatch component for fragment requests. Owns * {@link TransportService} (or {@link StreamTransportService}) and - * connection lookup. Does NOT track per-query or per-node concurrency + * connection lookup. + * + *

      Does NOT track per-query or per-node concurrency * state — callers provide their own {@link PendingExecutions} instance * to gate dispatch concurrency. * - *

      Also registers the server-side fragment request handler at construction - * time (delegating fragment execution to {@link AnalyticsSearchService}). - * - *

      Marked {@link Singleton} because the constructor has a side effect — - * registering the transport request handler — and double-registration throws. - * * @opensearch.internal */ @Singleton public class AnalyticsSearchTransportService { private final TransportService transportService; private final ClusterService clusterService; + private final boolean streamingEnabled; - /** - * Guice-injected constructor. Selects {@link StreamTransportService} when - * available (Arrow Flight configured), otherwise falls back to regular - * {@link TransportService}. Registers the server-side fragment request handler. - */ @Inject public AnalyticsSearchTransportService( TransportService transportService, @@ -66,25 +65,20 @@ public AnalyticsSearchTransportService( AnalyticsSearchService searchService, IndicesService indicesService ) { - this.transportService = streamTransportService != null ? streamTransportService : transportService; + this.streamingEnabled = streamTransportService != null; + this.transportService = this.streamingEnabled ? streamTransportService : transportService; this.clusterService = clusterService; - registerFragmentHandler(this.transportService, searchService, indicesService); + if (this.streamingEnabled) { + registerStreamingFragmentHandler(this.transportService, searchService, indicesService); + } else { + registerFragmentHandler(this.transportService, searchService, indicesService); + } } - /** - * Test-only constructor. Skips handler registration since tests either - * install their own mock handlers or don't exercise the inbound path. - */ - public AnalyticsSearchTransportService(TransportService transportService, ClusterService clusterService) { - this.transportService = Objects.requireNonNull(transportService, "TransportService must not be null"); - this.clusterService = clusterService; + public boolean isStreamingEnabled() { + return streamingEnabled; } - /** - * Registers the server-side handler for {@link FragmentExecutionAction#NAME}. - * Routes {@link FragmentExecutionRequest} to {@link AnalyticsSearchService} - * and responds with a {@link FragmentExecutionResponse}. - */ private static void registerFragmentHandler( TransportService transportService, AnalyticsSearchService searchService, @@ -99,33 +93,69 @@ private static void registerFragmentHandler( FragmentExecutionRequest::new, (request, channel, task) -> { IndexShard shard = indicesService.indexServiceSafe(request.getShardId().getIndex()).getShard(request.getShardId().id()); - FragmentExecutionResponse response = searchService.executeFragment(request, shard); + FragmentExecutionResponse response = searchService.executeFragment(request, shard, (AnalyticsShardTask) task); channel.sendResponse(response); } ); } - /** - * Resolves the connection to the given target node via this class's - * {@link ClusterService} and {@link TransportService}. - */ + private static void registerStreamingFragmentHandler( + TransportService transportService, + AnalyticsSearchService searchService, + IndicesService indicesService + ) { + transportService.registerRequestHandler( + FragmentExecutionAction.NAME, + ThreadPool.Names.SAME, + false, + true, + AdmissionControlActionType.SEARCH, + FragmentExecutionRequest::new, + (request, channel, task) -> { + IndexShard shard = indicesService.indexServiceSafe(request.getShardId().getIndex()).getShard(request.getShardId().id()); + try (FragmentResources ctx = searchService.executeFragmentStreaming(request, shard, (AnalyticsShardTask) task)) { + Iterator it = ctx.stream().iterator(); + while (it.hasNext()) { + EngineResultBatch batch = it.next(); + channel.sendResponseBatch(new FragmentExecutionArrowResponse(batch.getArrowRoot())); + } + channel.completeStream(); + } catch (StreamException e) { + if (e.getErrorCode() != StreamErrorCode.CANCELLED) { + channel.sendResponse(e); + } + // CANCELLED: channel already torn down — exit silently + } catch (Exception e) { + channel.sendResponse(e); + } + } + ); + } + Transport.Connection getConnection(String clusterAlias, String nodeId) { DiscoveryNode node = clusterService.state().nodes().get(nodeId); return transportService.getConnection(node); } - /** - * Dispatches a fragment request to the target data node, gated by the - * caller-provided {@link PendingExecutions}. Uses the typed - * {@link FragmentExecutionAction} and delivers streaming {@link FragmentExecutionResponse} - * batches to the listener. - * - * @param request the fragment execution request - * @param targetNode the node hosting the target shard - * @param listener the streaming response listener for fragment batches - * @param parentTask the parent task for child-request propagation - * @param pending the per-node concurrency gate owned by the caller - */ + public void dispatchFragmentStreaming( + FragmentExecutionRequest request, + DiscoveryNode targetNode, + StreamingResponseListener listener, + Task parentTask, + PendingExecutions pending + ) { + dispatchFragment( + request, + targetNode, + listener, + parentTask, + pending, + in -> new FragmentExecutionArrowResponse(in), + TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STREAM).build(), + true + ); + } + public void dispatchFragment( FragmentExecutionRequest request, DiscoveryNode targetNode, @@ -133,10 +163,37 @@ public void dispatchFragment( Task parentTask, PendingExecutions pending ) { - TransportResponseHandler handler = new TransportResponseHandler<>() { + dispatchFragment( + request, + targetNode, + listener, + parentTask, + pending, + in -> new FragmentExecutionResponse(in), + TransportRequestOptions.EMPTY, + false + ); + } + + private void dispatchFragment( + FragmentExecutionRequest request, + DiscoveryNode targetNode, + StreamingResponseListener listener, + Task parentTask, + PendingExecutions pending, + Writeable.Reader reader, + TransportRequestOptions options, + boolean skipsDeserialization + ) { + TransportResponseHandler handler = new TransportResponseHandler<>() { + @Override + public T read(StreamInput in) throws IOException { + return reader.read(in); + } + @Override - public FragmentExecutionResponse read(StreamInput in) throws IOException { - return new FragmentExecutionResponse(in); + public boolean skipsDeserialization() { + return skipsDeserialization; } @Override @@ -145,10 +202,10 @@ public String executor() { } @Override - public void handleStreamResponse(StreamTransportResponse stream) { + public void handleStreamResponse(StreamTransportResponse stream) { try { - FragmentExecutionResponse current; - FragmentExecutionResponse last = null; + T current; + T last = null; while ((current = stream.nextResponse()) != null) { if (last != null) { listener.onStreamResponse(last, false); @@ -169,7 +226,7 @@ public void handleStreamResponse(StreamTransportResponse { try { Transport.Connection connection = getConnection(null, targetNode.getId()); - transportService.sendChildRequest( - connection, - FragmentExecutionAction.NAME, - request, - parentTask, - TransportRequestOptions.EMPTY, - handler - ); + transportService.sendChildRequest(connection, FragmentExecutionAction.NAME, request, parentTask, options, handler); } catch (Exception e) { try { listener.onFailure(e); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java new file mode 100644 index 0000000000000..9ad9170b4a812 --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.exec; + +import org.opensearch.analytics.backend.EngineResultStream; +import org.opensearch.analytics.backend.ExecutionContext; +import org.opensearch.analytics.backend.SearchExecEngine; +import org.opensearch.common.concurrent.GatedCloseable; +import org.opensearch.index.engine.exec.IndexReaderProvider.Reader; + +/** + * Holds the per-fragment resources (reader, engine, result stream) kept alive for the + * duration of a streaming fragment execution, and releases them in reverse order on close. + * + * @opensearch.internal + */ +public final class FragmentResources implements AutoCloseable { + + private final GatedCloseable gatedReader; + private final SearchExecEngine engine; + private final EngineResultStream stream; + + public FragmentResources( + GatedCloseable gatedReader, + SearchExecEngine engine, + EngineResultStream stream + ) { + this.gatedReader = gatedReader; + this.engine = engine; + this.stream = stream; + } + + public EngineResultStream stream() { + return stream; + } + + @Override + public void close() throws Exception { + Exception first; + first = closeQuietly(stream, null); + first = closeQuietly(engine, first); + first = closeQuietly(gatedReader, first); + if (first != null) throw first; + } + + private static Exception closeQuietly(AutoCloseable resource, Exception prior) { + if (resource == null) return prior; + try { + resource.close(); + } catch (Exception e) { + if (prior == null) return e; + prior.addSuppressed(e); + } + return prior; + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryContext.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryContext.java index b27915c712aa4..cd279a6ba4301 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryContext.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryContext.java @@ -9,10 +9,10 @@ package org.opensearch.analytics.exec; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; import org.opensearch.analytics.backend.AnalyticsOperationListener; import org.opensearch.analytics.exec.task.AnalyticsQueryTask; import org.opensearch.analytics.planner.dag.QueryDAG; +import org.opensearch.arrow.flight.transport.ArrowAllocatorProvider; import java.util.List; import java.util.concurrent.Executor; @@ -36,12 +36,6 @@ public class QueryContext { /** Default per-query memory limit for Arrow allocations (256 MB). */ private static final long DEFAULT_PER_QUERY_MEMORY_LIMIT = 256L * 1024 * 1024; - /** - * Shared root allocator across all queries. Per-query child allocators - * are created from this root with individual limits. - */ - private static final BufferAllocator SHARED_ROOT = new RootAllocator(Long.MAX_VALUE); - private final QueryDAG dag; private final Executor searchExecutor; private final AnalyticsQueryTask parentTask; @@ -125,7 +119,7 @@ public BufferAllocator bufferAllocator() { if (closed) { throw new IllegalStateException("QueryContext closed for query " + dag.queryId()); } - alloc = SHARED_ROOT.newChildAllocator("query-" + dag.queryId(), 0, perQueryMemoryLimit); + alloc = ArrowAllocatorProvider.newChildAllocator("query-" + dag.queryId(), perQueryMemoryLimit); bufferAllocator = alloc; } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionArrowResponse.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionArrowResponse.java new file mode 100644 index 0000000000000..9aac1538db759 --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionArrowResponse.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.exec.action; + +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.arrow.flight.transport.ArrowBatchResponse; +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.IOException; + +/** + * Transport response carrying a single Arrow {@link VectorSchemaRoot} batch produced by + * fragment execution on a data node, streamed back to the coordinator. + * + * @opensearch.internal + */ +public class FragmentExecutionArrowResponse extends ArrowBatchResponse { + + public FragmentExecutionArrowResponse(VectorSchemaRoot root) { + super(root); + } + + public FragmentExecutionArrowResponse(StreamInput in) throws IOException { + super(in); + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java index d18ae5a372850..4e80dfa82e55f 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java @@ -9,7 +9,6 @@ package org.opensearch.analytics.exec.stage; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.FieldVector; @@ -55,7 +54,7 @@ public VectorSchemaRoot decode(FragmentExecutionResponse response, BufferAllocat List rows = response.getRows(); if (allocator == null) { - allocator = new RootAllocator(); + throw new IllegalArgumentException("BufferAllocator must not be null"); } // Infer Arrow type per column from the first non-null value diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java index eda17cf097617..153c6f914edd8 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java @@ -14,14 +14,16 @@ import org.opensearch.analytics.exec.PendingExecutions; import org.opensearch.analytics.exec.QueryContext; import org.opensearch.analytics.exec.StreamingResponseListener; +import org.opensearch.analytics.exec.action.FragmentExecutionArrowResponse; import org.opensearch.analytics.exec.action.FragmentExecutionRequest; import org.opensearch.analytics.exec.action.FragmentExecutionResponse; import org.opensearch.analytics.planner.dag.ExecutionTarget; import org.opensearch.analytics.planner.dag.ShardExecutionTarget; import org.opensearch.analytics.planner.dag.Stage; -import org.opensearch.analytics.spi.DataConsumer; import org.opensearch.analytics.spi.ExchangeSink; +import org.opensearch.arrow.flight.transport.ArrowBatchResponse; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.action.ActionResponse; import java.util.List; import java.util.Map; @@ -30,24 +32,13 @@ import java.util.function.Function; /** - * Per-stage execution for row-producing DATA_NODE stages (scans, filters, - * partial aggregates). Dispatches shard requests via - * {@link AnalyticsSearchTransportService#dispatchFragment}, decodes streaming - * responses through a {@link ResponseCodec}, and feeds the resulting Arrow - * batches into the stage's output {@link ExchangeSink}. + * Leaf stage execution that dispatches fragment work to data-node shards. * - *

      The codec abstracts the wire format: the current {@link RowResponseCodec} - * converts {@code Object[]} rows to Arrow; a future Arrow IPC codec would - * import IPC buffers directly with zero conversion. The stage execution logic - * is format-agnostic. + *

      Handles both Arrow streaming and row (codec-decoded) responses, feeding + * resulting batches into the parent stage's {@link ExchangeSink}. * - *

      Implements {@link DataProducer} because it writes batches into a sink - * owned by its parent stage. Does not implement {@link DataConsumer} because - * it is a leaf stage with no children. - * - *

      Lifecycle: {@code CREATED → RUNNING → SUCCEEDED | FAILED | CANCELLED}. - * Instances are one-shot: constructed, {@link #start()} called once, - * listener signaled once, discarded. + *

      One-shot: constructed, {@link #start()} called once, listener + * signaled on completion, then discarded. * * @opensearch.internal */ @@ -55,7 +46,6 @@ final class ShardFragmentStageExecution extends AbstractStageExecution implement private final AtomicInteger inFlight = new AtomicInteger(0); - // Immutable config private final QueryContext config; private final ExchangeSink outputSink; private final ClusterService clusterService; @@ -82,13 +72,14 @@ final class ShardFragmentStageExecution extends AbstractStageExecution implement this.responseCodec = responseCodec; } + private boolean useArrowStreaming() { + return dispatcher.isStreamingEnabled(); + } + @Override public void start() { - // Resolve targets lazily at dispatch time. For shuffle/broadcast reads this is - // where the child stage's manifest would be passed instead of null. List resolved = stage.getTargetResolver().resolve(clusterService.state(), null); if (resolved.isEmpty()) { - // CREATED → SUCCEEDED directly. transitionTo stamps both start and end. transitionTo(StageExecution.State.SUCCEEDED); return; } @@ -102,13 +93,36 @@ public void start() { private void dispatchShardTask(ShardExecutionTarget target) { FragmentExecutionRequest request = requestBuilder.apply(target); PendingExecutions pending = pendingFor(target); - dispatcher.dispatchFragment(request, target.node(), new StreamingResponseListener<>() { + if (useArrowStreaming()) { + dispatcher.dispatchFragmentStreaming( + request, + target.node(), + responseListener(FragmentExecutionArrowResponse::getRoot), + config.parentTask(), + pending + ); + } else { + dispatcher.dispatchFragment( + request, + target.node(), + responseListener(r -> responseCodec.decode(r, config.bufferAllocator())), + config.parentTask(), + pending + ); + } + } + + private StreamingResponseListener responseListener(Function toVsr) { + return new StreamingResponseListener<>() { @Override - public void onStreamResponse(FragmentExecutionResponse response, boolean isLast) { + public void onStreamResponse(T response, boolean isLast) { config.searchExecutor().execute(() -> { - if (isDone()) return; + if (isDone()) { + releaseResponseResources(response); + return; + } - VectorSchemaRoot vsr = responseCodec.decode(response, config.bufferAllocator()); + VectorSchemaRoot vsr = toVsr.apply(response); outputSink.feed(vsr); metrics.addRowsProcessed(vsr.getRowCount()); @@ -125,7 +139,13 @@ public void onFailure(Exception e) { metrics.incrementTasksFailed(); onShardTerminated(); } - }, config.parentTask(), pending); + }; + } + + private static void releaseResponseResources(T response) { + if (response instanceof ArrowBatchResponse arrowResp && arrowResp.getRoot() != null) { + arrowResp.getRoot().close(); + } } private void onShardTerminated() { @@ -138,10 +158,7 @@ private void onShardTerminated() { @Override public void cancel(String reason) { if (transitionTo(StageExecution.State.CANCELLED) == false) return; - // Bridge to task framework: cancel the parent task so data nodes - // see the cancellation via TaskCancellationService ban propagation. - // AnalyticsQueryTask.shouldCancelChildrenOnCancellation() == true - // ensures child shard tasks on data nodes are cancelled. + // Cancelling the parent task propagates to data-node shard tasks via TaskCancellationService. org.opensearch.tasks.Task parentTask = config.parentTask(); if (parentTask instanceof org.opensearch.tasks.CancellableTask ct && ct.isCancelled() == false) { ct.cancel(reason); diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java new file mode 100644 index 0000000000000..d1715c2ae3de4 --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java @@ -0,0 +1,243 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.exec.stage; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.analytics.exec.AnalyticsSearchTransportService; +import org.opensearch.analytics.exec.QueryContext; +import org.opensearch.analytics.exec.StreamingResponseListener; +import org.opensearch.analytics.exec.action.FragmentExecutionArrowResponse; +import org.opensearch.analytics.exec.action.FragmentExecutionRequest; +import org.opensearch.analytics.exec.action.FragmentExecutionResponse; +import org.opensearch.analytics.exec.task.AnalyticsQueryTask; +import org.opensearch.analytics.planner.dag.ShardExecutionTarget; +import org.opensearch.analytics.planner.dag.Stage; +import org.opensearch.analytics.planner.dag.TargetResolver; +import org.opensearch.analytics.spi.ExchangeSink; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Tests for {@link ShardFragmentStageExecution}, focused on ensuring + * Arrow resource cleanup on cancellation and terminal state transitions. + */ +public class ShardFragmentStageExecutionTests extends OpenSearchTestCase { + + private BufferAllocator allocator; + + @Override + public void setUp() throws Exception { + super.setUp(); + allocator = new RootAllocator(); + } + + @Override + public void tearDown() throws Exception { + allocator.close(); + super.tearDown(); + } + + /** + * Verifies that Arrow batches arriving after the stage is cancelled + * are properly closed (no buffer leak). + */ + public void testArrowResponseClosedWhenStageAlreadyCancelled() { + AtomicReference> capturedListener = new AtomicReference<>(); + CapturingSink sink = new CapturingSink(); + + ShardFragmentStageExecution exec = buildExecution(sink, true, capturedListener); + exec.start(); + + assertNotNull("listener should have been captured by dispatch", capturedListener.get()); + + exec.cancel("test"); + assertEquals(StageExecution.State.CANCELLED, exec.getState()); + + VectorSchemaRoot root = createTestBatch(5); + long allocatedBefore = allocator.getAllocatedMemory(); + assertTrue("batch should have allocated memory", allocatedBefore > 0); + + FragmentExecutionArrowResponse response = new FragmentExecutionArrowResponse(root); + capturedListener.get().onStreamResponse(response, true); + + assertEquals("Arrow buffers must be released after cancellation", 0, allocator.getAllocatedMemory()); + assertTrue("sink should not have received any batch", sink.fed.isEmpty()); + } + + /** + * Verifies that on the happy path, batches are fed into the sink normally. + */ + public void testArrowResponseFedToSinkOnHappyPath() { + AtomicReference> capturedListener = new AtomicReference<>(); + CapturingSink sink = new CapturingSink(); + + ShardFragmentStageExecution exec = buildExecution(sink, true, capturedListener); + exec.start(); + + VectorSchemaRoot root = createTestBatch(3); + FragmentExecutionArrowResponse response = new FragmentExecutionArrowResponse(root); + capturedListener.get().onStreamResponse(response, true); + + assertEquals("sink should have received the batch", 1, sink.fed.size()); + assertEquals(StageExecution.State.SUCCEEDED, exec.getState()); + sink.close(); + } + + /** + * Verifies that non-Arrow (row codec) responses don't crash + * the release path on cancellation (they hold no Arrow resources). + */ + public void testRowResponseSafeOnCancellation() { + AtomicReference> capturedListener = new AtomicReference<>(); + CapturingSink sink = new CapturingSink(); + + ShardFragmentStageExecution exec = buildExecution(sink, false, capturedListener); + exec.start(); + + exec.cancel("test"); + + List rows = new ArrayList<>(); + rows.add(new Object[] { 42 }); + FragmentExecutionResponse response = new FragmentExecutionResponse(List.of("col"), rows); + capturedListener.get().onStreamResponse(response, true); + + assertTrue("sink should not have received anything post-cancel", sink.fed.isEmpty()); + } + + /** + * Verifies that RowResponseCodec rejects null allocator. + */ + public void testRowResponseCodecRejectsNullAllocator() { + List rows = new ArrayList<>(); + rows.add(new Object[] { 1 }); + FragmentExecutionResponse response = new FragmentExecutionResponse(List.of("x"), rows); + expectThrows(IllegalArgumentException.class, () -> RowResponseCodec.INSTANCE.decode(response, null)); + } + + // ── helpers ────────────────────────────────────────────────────────── + + @SuppressWarnings("unchecked") + private ShardFragmentStageExecution buildExecution( + CapturingSink sink, + boolean streaming, + AtomicReference> listenerCapture + ) { + Stage stage = mockStage(); + QueryContext config = mockQueryContext(); + ClusterService clusterService = mockClusterService(); + AnalyticsSearchTransportService dispatcher = mock(AnalyticsSearchTransportService.class); + when(dispatcher.isStreamingEnabled()).thenReturn(streaming); + + if (streaming) { + doAnswer(invocation -> { + StreamingResponseListener listener = (StreamingResponseListener) invocation.getArgument(2); + listenerCapture.set(listener); + return null; + }).when(dispatcher).dispatchFragmentStreaming(any(), any(), any(), any(), any()); + } else { + doAnswer(invocation -> { + StreamingResponseListener listener = (StreamingResponseListener) invocation.getArgument(2); + listenerCapture.set(listener); + return null; + }).when(dispatcher).dispatchFragment(any(), any(), any(), any(), any()); + } + + ResponseCodec codec = (resp, alloc) -> { + VectorSchemaRoot vsr = createTestBatch(resp.getRows().size()); + return vsr; + }; + + Function requestBuilder = target -> new FragmentExecutionRequest( + "test-query", + 0, + target.shardId(), + List.of(new FragmentExecutionRequest.PlanAlternative("test-backend", new byte[0])) + ); + + return new ShardFragmentStageExecution(stage, config, sink, clusterService, requestBuilder, dispatcher, codec); + } + + private VectorSchemaRoot createTestBatch(int rows) { + Schema schema = new Schema(List.of(new Field("value", FieldType.nullable(new ArrowType.Int(32, true)), null))); + VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + root.allocateNew(); + IntVector vec = (IntVector) root.getVector(0); + for (int i = 0; i < rows; i++) { + vec.setSafe(i, i); + } + vec.setValueCount(rows); + root.setRowCount(rows); + return root; + } + + private Stage mockStage() { + Stage stage = mock(Stage.class); + when(stage.getStageId()).thenReturn(0); + TargetResolver resolver = mock(TargetResolver.class); + DiscoveryNode node = mock(DiscoveryNode.class); + when(node.getId()).thenReturn("test-node-1"); + ShardExecutionTarget target = new ShardExecutionTarget(node, new ShardId("idx", "_na_", 0)); + when(resolver.resolve(any(ClusterState.class), any())).thenReturn(List.of(target)); + when(stage.getTargetResolver()).thenReturn(resolver); + return stage; + } + + private QueryContext mockQueryContext() { + QueryContext config = mock(QueryContext.class); + when(config.searchExecutor()).thenReturn(Runnable::run); + when(config.parentTask()).thenReturn(mock(AnalyticsQueryTask.class)); + when(config.maxConcurrentShardRequests()).thenReturn(5); + when(config.bufferAllocator()).thenReturn(allocator); + return config; + } + + private ClusterService mockClusterService() { + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(mock(ClusterState.class)); + return clusterService; + } + + private static final class CapturingSink implements ExchangeSink { + final List fed = new ArrayList<>(); + boolean closed = false; + + @Override + public void feed(VectorSchemaRoot batch) { + fed.add(batch); + } + + @Override + public void close() { + closed = true; + for (VectorSchemaRoot batch : fed) { + batch.close(); + } + } + } +} diff --git a/sandbox/plugins/composite-engine/build.gradle b/sandbox/plugins/composite-engine/build.gradle index ba7c3a12f0b98..2f0f821950c85 100644 --- a/sandbox/plugins/composite-engine/build.gradle +++ b/sandbox/plugins/composite-engine/build.gradle @@ -32,6 +32,11 @@ tasks.named('internalClusterTest').configure { } internalClusterTest { + // arrow-memory-netty initialization requires these Netty Unsafe flags + systemProperty 'io.netty.allocator.numDirectArenas', '1' + systemProperty 'io.netty.noUnsafe', 'false' + systemProperty 'io.netty.tryUnsafe', 'true' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' systemProperty 'native.lib.path', project(':sandbox:libs:dataformat-native').ext.nativeLibPath.absolutePath dependsOn ':sandbox:libs:dataformat-native:buildRustLibrary' } diff --git a/sandbox/plugins/parquet-data-format/build.gradle b/sandbox/plugins/parquet-data-format/build.gradle index 1b8d0f15ffd27..996ab9d2bc5b1 100644 --- a/sandbox/plugins/parquet-data-format/build.gradle +++ b/sandbox/plugins/parquet-data-format/build.gradle @@ -20,7 +20,12 @@ dependencies { // Apache Arrow dependencies implementation "org.apache.arrow:arrow-vector:${versions.arrow}" implementation "org.apache.arrow:arrow-memory-core:${versions.arrow}" - implementation "org.apache.arrow:arrow-memory-unsafe:${versions.arrow}" + // Arrow 18.1's default AllocationManager is Netty. arrow-memory-netty + its buffer-patch + // provide NettyAllocationManager and PooledByteBufAllocatorL. + implementation "org.apache.arrow:arrow-memory-netty:${versions.arrow}" + implementation "org.apache.arrow:arrow-memory-netty-buffer-patch:${versions.arrow}" + implementation "io.netty:netty-buffer:${versions.netty}" + implementation "io.netty:netty-common:${versions.netty}" implementation "org.apache.arrow:arrow-format:${versions.arrow}" implementation "org.apache.arrow:arrow-c-data:${versions.arrow}" @@ -39,15 +44,57 @@ dependencies { tasks.named("dependencyLicenses").configure { mapping from: /jackson-.*/, to: 'jackson' + mapping from: /netty-.*/, to: 'netty' } tasks.named('thirdPartyAudit').configure { ignoreMissingClasses( - 'org.apache.commons.codec.binary.Hex' + 'org.apache.commons.codec.binary.Hex', + // Optional netty runtime deps (not used by arrow-memory-netty) + 'org.apache.commons.logging.Log', + 'org.apache.commons.logging.LogFactory', + 'org.apache.log4j.Level', + 'org.apache.log4j.Logger', + 'reactor.blockhound.BlockHound$Builder', + 'reactor.blockhound.integration.BlockHoundIntegration' ) ignoreViolations( 'org.apache.arrow.memory.util.MemoryUtil', - 'org.apache.arrow.memory.util.MemoryUtil$1' + 'org.apache.arrow.memory.util.MemoryUtil$1', + // Netty internals — standard violations for any module bundling netty-buffer + 'io.netty.buffer.AbstractAllocatorEvent', + 'io.netty.buffer.AbstractBufferEvent', + 'io.netty.buffer.AbstractChunkEvent', + 'io.netty.buffer.AdaptivePoolingAllocator$AdaptiveByteBuf', + 'io.netty.buffer.AdaptivePoolingAllocator$Chunk', + 'io.netty.buffer.AllocateBufferEvent', + 'io.netty.buffer.AllocateChunkEvent', + 'io.netty.buffer.FreeBufferEvent', + 'io.netty.buffer.FreeChunkEvent', + 'io.netty.buffer.PooledByteBufAllocator', + 'io.netty.buffer.ReallocateBufferEvent', + 'io.netty.util.internal.PlatformDependent0', + 'io.netty.util.internal.PlatformDependent0$1', + 'io.netty.util.internal.PlatformDependent0$2', + 'io.netty.util.internal.PlatformDependent0$3', + 'io.netty.util.internal.PlatformDependent0$5', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields', + 'io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess' ) } @@ -61,6 +108,12 @@ test { jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' jvmArgs '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED' jvmArgs '--enable-native-access=ALL-UNNAMED' + // Required by arrow-memory-netty for Unsafe / direct-memory access + jvmArgs += ["--add-opens", "java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED"] + systemProperty 'io.netty.allocator.numDirectArenas', '1' + systemProperty 'io.netty.noUnsafe', 'false' + systemProperty 'io.netty.tryUnsafe', 'true' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' systemProperty 'native.lib.path', project(':sandbox:libs:dataformat-native').ext.nativeLibPath.absolutePath dependsOn ':sandbox:libs:dataformat-native:buildRustLibrary' } diff --git a/sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-18.1.0.jar.sha1 b/sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..291d435138e30 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-18.1.0.jar.sha1 @@ -0,0 +1 @@ +9e9e08d0b548d2c02c632e5daaf176e588810d22 \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-format-LICENSE.txt b/sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-LICENSE.txt similarity index 100% rename from sandbox/plugins/analytics-engine/licenses/arrow-format-LICENSE.txt rename to sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-LICENSE.txt diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-format-NOTICE.txt b/sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-NOTICE.txt similarity index 100% rename from sandbox/plugins/analytics-engine/licenses/arrow-format-NOTICE.txt rename to sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-NOTICE.txt diff --git a/sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1 b/sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..40c7b2992d715 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1 @@ -0,0 +1 @@ +86c8fbdb6ab220603ea3a215f48a7f793ac6a08d \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-memory-core-LICENSE.txt b/sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt similarity index 100% rename from sandbox/plugins/analytics-engine/licenses/arrow-memory-core-LICENSE.txt rename to sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt diff --git a/sandbox/plugins/analytics-engine/licenses/arrow-memory-core-NOTICE.txt b/sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt similarity index 100% rename from sandbox/plugins/analytics-engine/licenses/arrow-memory-core-NOTICE.txt rename to sandbox/plugins/parquet-data-format/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt diff --git a/sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-18.1.0.jar.sha1 b/sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-18.1.0.jar.sha1 deleted file mode 100644 index 281ae8fcc6fbb..0000000000000 --- a/sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-18.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8b48e832c98695bfd2b50ad0ed324e0d46099898 diff --git a/sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-LICENSE.txt b/sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-LICENSE.txt deleted file mode 100644 index 7bb1330a1002b..0000000000000 --- a/sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-LICENSE.txt +++ /dev/null @@ -1,2261 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --------------------------------------------------------------------------------- - -src/arrow/util (some portions): Apache 2.0, and 3-clause BSD - -Some portions of this module are derived from code in the Chromium project, -copyright (c) Google inc and (c) The Chromium Authors and licensed under the -Apache 2.0 License or the under the 3-clause BSD license: - - Copyright (c) 2013 The Chromium Authors. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project includes code from Daniel Lemire's FrameOfReference project. - -https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp -https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py - -Copyright: 2013 Daniel Lemire -Home page: http://lemire.me/en/ -Project page: https://github.com/lemire/FrameOfReference -License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from the TensorFlow project - -Copyright 2015 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -This project includes code from the NumPy project. - -https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910 - -https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c - -Copyright (c) 2005-2017, NumPy Developers. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the NumPy Developers nor the names of any - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project includes code from the Boost project - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -This project includes code from the FlatBuffers project - -Copyright 2014 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -This project includes code from the tslib project - -Copyright 2015 Microsoft Corporation. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -This project includes code from the jemalloc project - -https://github.com/jemalloc/jemalloc - -Copyright (C) 2002-2017 Jason Evans . -All rights reserved. -Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice(s), - this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice(s), - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- - -This project includes code from the Go project, BSD 3-clause license + PATENTS -weak patent termination clause -(https://github.com/golang/go/blob/master/PATENTS). - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project includes code from the hs2client - -https://github.com/cloudera/hs2client - -Copyright 2016 Cloudera Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -The script ci/scripts/util_wait_for_it.sh has the following license - -Copyright (c) 2016 Giles Hall - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The script r/configure has the following license (MIT) - -Copyright (c) 2017, Jeroen Ooms and Jim Hester - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and -cpp/src/arrow/util/logging-test.cc are adapted from -Ray Project (https://github.com/ray-project/ray) (Apache 2.0). - -Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray) - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- -The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h, -cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h, -cpp/src/arrow/vendored/datetime/ios.mm, -cpp/src/arrow/vendored/datetime/tz.cpp are adapted from -Howard Hinnant's date library (https://github.com/HowardHinnant/date) -It is licensed under MIT license. - -The MIT License (MIT) -Copyright (c) 2015, 2016, 2017 Howard Hinnant -Copyright (c) 2016 Adrian Colomitchi -Copyright (c) 2017 Florian Dang -Copyright (c) 2017 Paul Thompson -Copyright (c) 2018 Tomasz Kamiński - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The file cpp/src/arrow/util/utf8.h includes code adapted from the page - https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ -with the following license (MIT) - -Copyright (c) 2008-2009 Bjoern Hoehrmann - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/xxhash/ have the following license -(BSD 2-Clause License) - -xxHash Library -Copyright (c) 2012-2014, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- xxHash homepage: http://www.xxhash.com -- xxHash source repository : https://github.com/Cyan4973/xxHash - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/double-conversion/ have the following license -(BSD 3-Clause License) - -Copyright 2006-2011, the V8 project authors. All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/uriparser/ have the following license -(BSD 3-Clause License) - -uriparser - RFC 3986 URI parsing library - -Copyright (C) 2007, Weijia Song -Copyright (C) 2007, Sebastian Pipping -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - * Redistributions of source code must retain the above - copyright notice, this list of conditions and the following - disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials - provided with the distribution. - - * Neither the name of the nor the names of its - contributors may be used to endorse or promote products - derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The files under dev/tasks/conda-recipes have the following license - -BSD 3-clause license -Copyright (c) 2015-2018, conda-forge -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR -TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/utfcpp/ have the following license - -Copyright 2006-2018 Nemanja Trifunovic - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -This project includes code from Apache Kudu. - - * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake - -Copyright: 2016 The Apache Software Foundation. -Home page: https://kudu.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from Apache Impala (incubating), formerly -Impala. The Impala code and rights were donated to the ASF as part of the -Incubator process after the initial code imports into Apache Parquet. - -Copyright: 2012 Cloudera, Inc. -Copyright: 2016 The Apache Software Foundation. -Home page: http://impala.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from Apache Aurora. - -* dev/release/{release,changelog,release-candidate} are based on the scripts from - Apache Aurora - -Copyright: 2016 The Apache Software Foundation. -Home page: https://aurora.apache.org/ -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -This project includes code from the Google styleguide. - -* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide. - -Copyright: 2009 Google Inc. All rights reserved. -Homepage: https://github.com/google/styleguide -License: 3-clause BSD - --------------------------------------------------------------------------------- - -This project includes code from Snappy. - -* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code - from Google's Snappy project. - -Copyright: 2009 Google Inc. All rights reserved. -Homepage: https://github.com/google/snappy -License: 3-clause BSD - --------------------------------------------------------------------------------- - -This project includes code from the manylinux project. - -* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py, - requirements.txt} are based on code from the manylinux project. - -Copyright: 2016 manylinux -Homepage: https://github.com/pypa/manylinux -License: The MIT License (MIT) - --------------------------------------------------------------------------------- - -This project includes code from the cymove project: - -* python/pyarrow/includes/common.pxd includes code from the cymove project - -The MIT License (MIT) -Copyright (c) 2019 Omer Ozarslan - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE -OR OTHER DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -The projects includes code from the Ursabot project under the dev/archery -directory. - -License: BSD 2-Clause - -Copyright 2019 RStudio, Inc. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -This project include code from mingw-w64. - -* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5 - -Copyright (c) 2009 - 2013 by the mingw-w64 project -Homepage: https://mingw-w64.org -License: Zope Public License (ZPL) Version 2.1. - ---------------------------------------------------------------------------------- - -This project include code from Google's Asylo project. - -* cpp/src/arrow/result.h is based on status_or.h - -Copyright (c) Copyright 2017 Asylo authors -Homepage: https://asylo.dev/ -License: Apache 2.0 - --------------------------------------------------------------------------------- - -This project includes code from Google's protobuf project - -* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN -* cpp/src/arrow/util/bit_stream_utils.h contains code from wire_format_lite.h - -Copyright 2008 Google Inc. All rights reserved. -Homepage: https://developers.google.com/protocol-buffers/ -License: - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - --------------------------------------------------------------------------------- - -3rdparty dependency LLVM is statically linked in certain binary distributions. -Additionally some sections of source code have been derived from sources in LLVM -and have been clearly labeled as such. LLVM has the following license: - -============================================================================== -The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: -============================================================================== - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - ----- LLVM Exceptions to the Apache 2.0 License ---- - -As an exception, if, as a result of your compiling your source code, portions -of this Software are embedded into an Object form of such source code, you -may redistribute such embedded portions in such Object form without complying -with the conditions of Sections 4(a), 4(b) and 4(d) of the License. - -In addition, if you combine or link compiled forms of this Software with -software that is licensed under the GPLv2 ("Combined Software") and if a -court of competent jurisdiction determines that the patent provision (Section -3), the indemnity provision (Section 9) or other Section of the License -conflicts with the conditions of the GPLv2, you may retroactively and -prospectively choose to deem waived or otherwise exclude such Section(s) of -the License, but only in their entirety and only with respect to the Combined -Software. - -============================================================================== -Software from third parties included in the LLVM Project: -============================================================================== -The LLVM Project contains third party software which is under different license -terms. All such code will be identified clearly using at least one of two -mechanisms: -1) It will be in a separate directory tree with its own `LICENSE.txt` or - `LICENSE` file at the top containing the specific license and restrictions - which apply to that software, or -2) It will contain specific license and restriction terms at the top of every - file. - --------------------------------------------------------------------------------- - -3rdparty dependency gRPC is statically linked in certain binary -distributions, like the python wheels. gRPC has the following license: - -Copyright 2014 gRPC authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -3rdparty dependency Apache Thrift is statically linked in certain binary -distributions, like the python wheels. Apache Thrift has the following license: - -Apache Thrift -Copyright (C) 2006 - 2019, The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -3rdparty dependency Apache ORC is statically linked in certain binary -distributions, like the python wheels. Apache ORC has the following license: - -Apache ORC -Copyright 2013-2019 The Apache Software Foundation - -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). - -This product includes software developed by Hewlett-Packard: -(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - --------------------------------------------------------------------------------- - -3rdparty dependency zstd is statically linked in certain binary -distributions, like the python wheels. ZSTD has the following license: - -BSD License - -For Zstandard software - -Copyright (c) 2016-present, Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency lz4 is statically linked in certain binary -distributions, like the python wheels. lz4 has the following license: - -LZ4 Library -Copyright (c) 2011-2016, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency Brotli is statically linked in certain binary -distributions, like the python wheels. Brotli has the following license: - -Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - --------------------------------------------------------------------------------- - -3rdparty dependency rapidjson is statically linked in certain binary -distributions, like the python wheels. rapidjson and its dependencies have the -following licenses: - -Tencent is pleased to support the open source community by making RapidJSON -available. - -Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. -All rights reserved. - -If you have downloaded a copy of the RapidJSON binary from Tencent, please note -that the RapidJSON binary is licensed under the MIT License. -If you have downloaded a copy of the RapidJSON source code from Tencent, please -note that RapidJSON source code is licensed under the MIT License, except for -the third-party components listed below which are subject to different license -terms. Your integration of RapidJSON into your own projects may require -compliance with the MIT License, as well as the other licenses applicable to -the third-party components included within RapidJSON. To avoid the problematic -JSON license in your own projects, it's sufficient to exclude the -bin/jsonchecker/ directory, as it's the only code under the JSON license. -A copy of the MIT License is included in this file. - -Other dependencies and licenses: - - Open Source Software Licensed Under the BSD License: - -------------------------------------------------------------------- - - The msinttypes r29 - Copyright (c) 2006-2013 Alexander Chemeris - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR - ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH - DAMAGE. - - Terms of the MIT License: - -------------------------------------------------------------------- - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -3rdparty dependency snappy is statically linked in certain binary -distributions, like the python wheels. snappy has the following license: - -Copyright 2011, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -=== - -Some of the benchmark data in testdata/ is licensed differently: - - - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and - is licensed under the Creative Commons Attribution 3.0 license - (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ - for more information. - - - kppkn.gtb is taken from the Gaviota chess tablebase set, and - is licensed under the MIT License. See - https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 - for more information. - - - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper - “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA - Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, - which is licensed under the CC-BY license. See - http://www.ploscompbiol.org/static/license for more ifnormation. - - - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project - Gutenberg. The first three have expired copyrights and are in the public - domain; the latter does not have expired copyright, but is still in the - public domain according to the license information - (http://www.gutenberg.org/ebooks/53). - --------------------------------------------------------------------------------- - -3rdparty dependency gflags is statically linked in certain binary -distributions, like the python wheels. gflags has the following license: - -Copyright (c) 2006, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency glog is statically linked in certain binary -distributions, like the python wheels. glog has the following license: - -Copyright (c) 2008, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -A function gettimeofday in utilities.cc is based on - -http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd - -The license of this code is: - -Copyright (c) 2003-2008, Jouni Malinen and contributors -All Rights Reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name(s) of the above-listed copyright holder(s) nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency re2 is statically linked in certain binary -distributions, like the python wheels. re2 has the following license: - -Copyright (c) 2009 The RE2 Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - * Neither the name of Google Inc. nor the names of its contributors - may be used to endorse or promote products derived from this - software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -3rdparty dependency c-ares is statically linked in certain binary -distributions, like the python wheels. c-ares has the following license: - -# c-ares license - -Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS -file. - -Copyright 1998 by the Massachusetts Institute of Technology. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, provided that -the above copyright notice appear in all copies and that both that copyright -notice and this permission notice appear in supporting documentation, and that -the name of M.I.T. not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior permission. -M.I.T. makes no representations about the suitability of this software for any -purpose. It is provided "as is" without express or implied warranty. - --------------------------------------------------------------------------------- - -3rdparty dependency zlib is redistributed as a dynamically linked shared -library in certain binary distributions, like the python wheels. In the future -this will likely change to static linkage. zlib has the following license: - -zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.11, January 15th, 2017 - - Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu - --------------------------------------------------------------------------------- - -3rdparty dependency openssl is redistributed as a dynamically linked shared -library in certain binary distributions, like the python wheels. openssl -preceding version 3 has the following license: - - LICENSE ISSUES - ============== - - The OpenSSL toolkit stays under a double license, i.e. both the conditions of - the OpenSSL License and the original SSLeay license apply to the toolkit. - See below for the actual license texts. - - OpenSSL License - --------------- - -/* ==================================================================== - * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * openssl-core@openssl.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.openssl.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - * - */ - - Original SSLeay License - ----------------------- - -/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) - * All rights reserved. - * - * This package is an SSL implementation written - * by Eric Young (eay@cryptsoft.com). - * The implementation was written so as to conform with Netscapes SSL. - * - * This library is free for commercial and non-commercial use as long as - * the following conditions are aheared to. The following conditions - * apply to all code found in this distribution, be it the RC4, RSA, - * lhash, DES, etc., code; not just the SSL code. The SSL documentation - * included with this distribution is covered by the same copyright terms - * except that the holder is Tim Hudson (tjh@cryptsoft.com). - * - * Copyright remains Eric Young's, and as such any Copyright notices in - * the code are not to be removed. - * If this package is used in a product, Eric Young should be given attribution - * as the author of the parts of the library used. - * This can be in the form of a textual message at program startup or - * in documentation (online or textual) provided with the package. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * "This product includes cryptographic software written by - * Eric Young (eay@cryptsoft.com)" - * The word 'cryptographic' can be left out if the rouines from the library - * being used are not cryptographic related :-). - * 4. If you include any Windows specific code (or a derivative thereof) from - * the apps directory (application code) you must include an acknowledgement: - * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" - * - * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * The licence and distribution terms for any publically available version or - * derivative of this code cannot be changed. i.e. this code cannot simply be - * copied and put under another distribution licence - * [including the GNU Public Licence.] - */ - --------------------------------------------------------------------------------- - -This project includes code from the rtools-backports project. - -* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code - from the rtools-backports project. - -Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms. -All rights reserved. -Homepage: https://github.com/r-windows/rtools-backports -License: 3-clause BSD - --------------------------------------------------------------------------------- - -Some code from pandas has been adapted for the pyarrow codebase. pandas is -available under the 3-clause BSD license, which follows: - -pandas license -============== - -Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team -All rights reserved. - -Copyright (c) 2008-2011 AQR Capital Management, LLC -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the copyright holder nor the names of any - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -Some bits from DyND, in particular aspects of the build system, have been -adapted from libdynd and dynd-python under the terms of the BSD 2-clause -license - -The BSD 2-Clause License - - Copyright (C) 2011-12, Dynamic NDArray Developers - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Dynamic NDArray Developers list: - - * Mark Wiebe - * Continuum Analytics - --------------------------------------------------------------------------------- - -Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted -for PyArrow. Ibis is released under the Apache License, Version 2.0. - --------------------------------------------------------------------------------- - -dev/tasks/homebrew-formulae/apache-arrow.rb has the following license: - -BSD 2-Clause License - -Copyright (c) 2009-present, Homebrew contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- - -cpp/src/arrow/vendored/base64.cpp has the following license - -ZLIB License - -Copyright (C) 2004-2017 René Nyffenegger - -This source code is provided 'as-is', without any express or implied -warranty. In no event will the author be held liable for any damages arising -from the use of this software. - -Permission is granted to anyone to use this software for any purpose, including -commercial applications, and to alter it and redistribute it freely, subject to -the following restrictions: - -1. The origin of this source code must not be misrepresented; you must not - claim that you wrote the original source code. If you use this source code - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - -2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original source code. - -3. This notice may not be removed or altered from any source distribution. - -René Nyffenegger rene.nyffenegger@adp-gmbh.ch - --------------------------------------------------------------------------------- - -This project includes code from Folly. - - * cpp/src/arrow/vendored/ProducerConsumerQueue.h - -is based on Folly's - - * folly/Portability.h - * folly/lang/Align.h - * folly/ProducerConsumerQueue.h - -Copyright: Copyright (c) Facebook, Inc. and its affiliates. -Home page: https://github.com/facebook/folly -License: http://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - -The file cpp/src/arrow/vendored/musl/strptime.c has the following license - -Copyright © 2005-2020 Rich Felker, et al. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -The file cpp/cmake_modules/BuildUtils.cmake contains code from - -https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 - -which is made available under the MIT license - -Copyright (c) 2019 Cristian Adam - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/portable-snippets/ contain code from - -https://github.com/nemequ/portable-snippets - -and have the following copyright notice: - -Each source file contains a preamble explaining the license situation -for that file, which takes priority over this file. With the -exception of some code pulled in from other repositories (such as -µnit, an MIT-licensed project which is used for testing), the code is -public domain, released using the CC0 1.0 Universal dedication (*). - -(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/fast_float/ contain code from - -https://github.com/lemire/fast_float - -which is made available under the Apache License 2.0. - --------------------------------------------------------------------------------- - -The file python/pyarrow/vendored/docscrape.py contains code from - -https://github.com/numpy/numpydoc/ - -which is made available under the BSD 2-clause license. - --------------------------------------------------------------------------------- - -The file python/pyarrow/vendored/version.py contains code from - -https://github.com/pypa/packaging/ - -which is made available under both the Apache license v2.0 and the -BSD 2-clause license. - --------------------------------------------------------------------------------- - -The files in cpp/src/arrow/vendored/pcg contain code from - -https://github.com/imneme/pcg-cpp - -and have the following copyright notice: - -Copyright 2014-2019 Melissa O'Neill , - and the PCG Project contributors. - -SPDX-License-Identifier: (Apache-2.0 OR MIT) - -Licensed under the Apache License, Version 2.0 (provided in -LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0) -or under the MIT license (provided in LICENSE-MIT.txt and at -http://opensource.org/licenses/MIT), at your option. This file may not -be copied, modified, or distributed except according to those terms. - -Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either -express or implied. See your chosen license for details. - --------------------------------------------------------------------------------- -r/R/dplyr-count-tally.R (some portions) - -Some portions of this file are derived from code from - -https://github.com/tidyverse/dplyr/ - -which is made available under the MIT license - -Copyright (c) 2013-2019 RStudio and others. - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the “Software”), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -The file src/arrow/util/io_util.cc contains code from the CPython project -which is made available under the Python Software Foundation License Version 2. - --------------------------------------------------------------------------------- - -3rdparty dependency opentelemetry-cpp is statically linked in certain binary -distributions. opentelemetry-cpp is made available under the Apache License 2.0. - -Copyright The OpenTelemetry Authors -SPDX-License-Identifier: Apache-2.0 - --------------------------------------------------------------------------------- - -ci/conan/ is based on code from Conan Package and Dependency Manager. - -Copyright (c) 2019 Conan.io - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -3rdparty dependency UCX is redistributed as a dynamically linked shared -library in certain binary distributions. UCX has the following license: - -Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. -Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. -Copyright (C) 2014-2015 The University of Houston System. All rights reserved. -Copyright (C) 2015 The University of Tennessee and The University - of Tennessee Research Foundation. All rights reserved. -Copyright (C) 2016-2020 ARM Ltd. All rights reserved. -Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. -Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. -Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. -Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. -Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. -Copyright (C) 2016-2020 Stony Brook University. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. -3. Neither the name of the copyright holder nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - -The file dev/tasks/r/github.packages.yml contains code from - -https://github.com/ursa-labs/arrow-r-nightly - -which is made available under the Apache License 2.0. - --------------------------------------------------------------------------------- -.github/actions/sync-nightlies/action.yml (some portions) - -Some portions of this file are derived from code from - -https://github.com/JoshPiper/rsync-docker - -which is made available under the MIT license - -Copyright (c) 2020 Joshua Piper - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- -.github/actions/sync-nightlies/action.yml (some portions) - -Some portions of this file are derived from code from - -https://github.com/burnett01/rsync-deployments - -which is made available under the MIT license - -Copyright (c) 2019-2022 Contention -Copyright (c) 2019-2022 Burnett01 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- -java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java -java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java - -These file are derived from code from Netty, which is made available under the -Apache License 2.0. diff --git a/sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-NOTICE.txt b/sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-NOTICE.txt deleted file mode 100644 index 2089c6fb20358..0000000000000 --- a/sandbox/plugins/parquet-data-format/licenses/arrow-memory-unsafe-NOTICE.txt +++ /dev/null @@ -1,84 +0,0 @@ -Apache Arrow -Copyright 2016-2024 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This product includes software from the SFrame project (BSD, 3-clause). -* Copyright (C) 2015 Dato, Inc. -* Copyright (c) 2009 Carnegie Mellon University. - -This product includes software from the Feather project (Apache 2.0) -https://github.com/wesm/feather - -This product includes software from the DyND project (BSD 2-clause) -https://github.com/libdynd - -This product includes software from the LLVM project - * distributed under the University of Illinois Open Source - -This product includes software from the google-lint project - * Copyright (c) 2009 Google Inc. All rights reserved. - -This product includes software from the mman-win32 project - * Copyright https://code.google.com/p/mman-win32/ - * Licensed under the MIT License; - -This product includes software from the LevelDB project - * Copyright (c) 2011 The LevelDB Authors. All rights reserved. - * Use of this source code is governed by a BSD-style license that can be - * Moved from Kudu http://github.com/cloudera/kudu - -This product includes software from the CMake project - * Copyright 2001-2009 Kitware, Inc. - * Copyright 2012-2014 Continuum Analytics, Inc. - * All rights reserved. - -This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) - * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. - -This product includes software from the Ibis project (Apache 2.0) - * Copyright (c) 2015 Cloudera, Inc. - * https://github.com/cloudera/ibis - -This product includes software from Dremio (Apache 2.0) - * Copyright (C) 2017-2018 Dremio Corporation - * https://github.com/dremio/dremio-oss - -This product includes software from Google Guava (Apache 2.0) - * Copyright (C) 2007 The Guava Authors - * https://github.com/google/guava - -This product include software from CMake (BSD 3-Clause) - * CMake - Cross Platform Makefile Generator - * Copyright 2000-2019 Kitware, Inc. and Contributors - -The web site includes files generated by Jekyll. - --------------------------------------------------------------------------------- - -This product includes code from Apache Kudu, which includes the following in -its NOTICE file: - - Apache Kudu - Copyright 2016 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - - Portions of this software were developed at - Cloudera, Inc (http://www.cloudera.com/). - --------------------------------------------------------------------------------- - -This product includes code from Apache ORC, which includes the following in -its NOTICE file: - - Apache ORC - Copyright 2013-2019 The Apache Software Foundation - - This product includes software developed by The Apache Software - Foundation (http://www.apache.org/). - - This product includes software developed by Hewlett-Packard: - (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P diff --git a/sandbox/plugins/analytics-engine/licenses/failureaccess-LICENSE.txt b/sandbox/plugins/parquet-data-format/licenses/netty-LICENSE.txt similarity index 100% rename from sandbox/plugins/analytics-engine/licenses/failureaccess-LICENSE.txt rename to sandbox/plugins/parquet-data-format/licenses/netty-LICENSE.txt diff --git a/sandbox/plugins/parquet-data-format/licenses/netty-NOTICE.txt b/sandbox/plugins/parquet-data-format/licenses/netty-NOTICE.txt new file mode 100644 index 0000000000000..5bbf91a14de23 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/licenses/netty-NOTICE.txt @@ -0,0 +1,116 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2011 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product contains a modified version of 'Webbit', a Java event based +WebSocket and HTTP server: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ diff --git a/sandbox/plugins/parquet-data-format/licenses/netty-buffer-4.2.13.Final.jar.sha1 b/sandbox/plugins/parquet-data-format/licenses/netty-buffer-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..723b9fac59b38 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/licenses/netty-buffer-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +177025483d7565afaf4f820139d409bdc0cd7000 \ No newline at end of file diff --git a/sandbox/plugins/parquet-data-format/licenses/netty-common-4.2.13.Final.jar.sha1 b/sandbox/plugins/parquet-data-format/licenses/netty-common-4.2.13.Final.jar.sha1 new file mode 100644 index 0000000000000..b1ac1fc1bde8b --- /dev/null +++ b/sandbox/plugins/parquet-data-format/licenses/netty-common-4.2.13.Final.jar.sha1 @@ -0,0 +1 @@ +f91909ed1b9280cd46d8b0ee260ebff40e1c73d8 \ No newline at end of file diff --git a/sandbox/qa/analytics-engine-rest/build.gradle b/sandbox/qa/analytics-engine-rest/build.gradle index e96065e65a88a..d636582c7b911 100644 --- a/sandbox/qa/analytics-engine-rest/build.gradle +++ b/sandbox/qa/analytics-engine-rest/build.gradle @@ -31,6 +31,7 @@ dependencies { } testClusters.integTest { + plugin ':plugins:arrow-flight-rpc' plugin ':sandbox:plugins:analytics-engine' plugin ':sandbox:plugins:analytics-backend-datafusion' plugin ':sandbox:plugins:analytics-backend-lucene' @@ -43,6 +44,14 @@ testClusters.integTest { jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' jvmArgs '--enable-native-access=ALL-UNNAMED' + // Arrow memory allocator needs Netty unsafe access on JDK 25; mirrors + // gradle/run.gradle's arrow-flight-rpc overrides so AnalyticsSearchService + // can construct its RootAllocator at node start. + systemProperty 'io.netty.allocator.numDirectArenas', '1' + systemProperty 'io.netty.noUnsafe', 'false' + systemProperty 'io.netty.tryUnsafe', 'true' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' + // Native library path for DataFusion systemProperty 'java.library.path', "${project(':sandbox:libs:dataformat-native').ext.nativeLibPath.parent}" diff --git a/server/src/main/java/org/opensearch/search/SearchExecutionContext.java b/server/src/main/java/org/opensearch/search/SearchExecutionContext.java index 025effc3833cb..770949560c392 100644 --- a/server/src/main/java/org/opensearch/search/SearchExecutionContext.java +++ b/server/src/main/java/org/opensearch/search/SearchExecutionContext.java @@ -8,8 +8,8 @@ package org.opensearch.search; -import org.opensearch.action.search.SearchShardTask; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.tasks.Task; import java.io.Closeable; @@ -21,7 +21,7 @@ @ExperimentalApi public interface SearchExecutionContext extends Closeable { - SearchShardTask task(); + Task task(); S getSearcher(); diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index 37dbb8e3a69da..f5eb5b081ce88 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -1841,6 +1841,11 @@ public T read(StreamInput in) throws IOException { return handler.read(in); } + @Override + public boolean skipsDeserialization() { + return handler.skipsDeserialization(); + } + @Override public String toString() { return getClass().getName() + "/[" + action + "]:" + handler.toString(); From b22852333a545a99bf0c0e4cfadf5835e9baba28 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 5 May 2026 17:45:36 -0700 Subject: [PATCH 053/115] Add support for PPL SEARCH function (#21499) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add SEARCH adapter: expand Calcite Sarg into OR/AND before DataFusion Calcite folds IN-lists, BETWEEN, and unions of ranges into a compact SEARCH(field, Sarg[...]) RexCall that DataFusion's substrait consumer can't decode — Sarg has no wire representation. SearchAdapter runs during the backend-adapt pass and calls RexUtil.expandSearch to invert the fold before substrait serialization. Adds ScalarFunction.SEARCH (SqlKind.SEARCH) to the SPI enum, registers SearchAdapter with the DataFusion backend, and declares SEARCH as a filter capability so the filter rule accepts the predicate and lets the adapter handle it. SearchAdapterTests covers IN-list, BETWEEN, and pass-through for non-SEARCH. SearchOperatorIT exercises the full PPL -> Calcite -> adapter -> substrait -> DataFusion path against the calcs dataset with three Sarg shapes. Signed-off-by: Marc Handalian * Rename scalar declaration and add more tests. Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../analytics/spi/ScalarFunction.java | 2 + .../analytics/spi/ScalarFunctionTests.java | 37 +++++ .../DataFusionAnalyticsBackendPlugin.java | 11 +- .../opensearch/be/datafusion/SargAdapter.java | 33 +++++ .../be/datafusion/SargAdapterTests.java | 138 ++++++++++++++++++ .../analytics/qa/SearchOperatorIT.java | 101 +++++++++++++ 6 files changed, 319 insertions(+), 3 deletions(-) create mode 100644 sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SargAdapter.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/SargAdapterTests.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/SearchOperatorIT.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java index 7185a3e8cb630..c8dd9b085961e 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java @@ -37,6 +37,8 @@ public enum ScalarFunction { IN(Category.COMPARISON, SqlKind.IN), LIKE(Category.COMPARISON, SqlKind.LIKE), PREFIX(Category.COMPARISON, SqlKind.OTHER_FUNCTION), + /** Calcite's Sarg fold for IN / NOT IN / BETWEEN / range-union. Backends expand it before substrait. */ + SARG_PREDICATE(Category.SCALAR, SqlKind.SEARCH), // ── Full-text search ───────────────────────────────────────────── MATCH(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), diff --git a/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java new file mode 100644 index 0000000000000..077c52a3103cc --- /dev/null +++ b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.apache.calcite.sql.SqlKind; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.EnumMap; +import java.util.Map; + +public class ScalarFunctionTests extends OpenSearchTestCase { + + /** Non-OTHER_FUNCTION SqlKinds must be unique: fromSqlKind picks the first match and would shadow later entries. */ + public void testNoDuplicateSqlKindBindings() { + Map claimedBy = new EnumMap<>(SqlKind.class); + for (ScalarFunction func : ScalarFunction.values()) { + SqlKind kind = func.getSqlKind(); + if (kind == SqlKind.OTHER_FUNCTION) { + continue; + } + ScalarFunction existing = claimedBy.put(kind, func); + if (existing != null) { + fail("SqlKind." + kind + " claimed by both " + existing + " and " + func); + } + } + } + + public void testSargPredicateIsBoundToSqlKindSearch() { + assertSame(ScalarFunction.SARG_PREDICATE, ScalarFunction.fromSqlKind(SqlKind.SEARCH)); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index f5ed1de1033bd..58c705ab9812d 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -61,7 +61,8 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP ScalarFunction.IS_NULL, ScalarFunction.IS_NOT_NULL, ScalarFunction.IN, - ScalarFunction.LIKE + ScalarFunction.LIKE, + ScalarFunction.SARG_PREDICATE ); // Project-side scalar functions DataFusion can evaluate natively. Each entry corresponds to a @@ -69,7 +70,11 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP // here only after verifying the function deserializes through Substrait isthmus into a plan // DataFusion's native runtime can execute (see DataFusionFragmentConvertor for the conversion // path). COALESCE is the lowering target of PPL `fillnull`. - private static final Set STANDARD_PROJECT_OPS = Set.of(ScalarFunction.COALESCE, ScalarFunction.CEIL); + private static final Set STANDARD_PROJECT_OPS = Set.of( + ScalarFunction.COALESCE, + ScalarFunction.CEIL, + ScalarFunction.SARG_PREDICATE + ); private static final Set AGG_FUNCTIONS = Set.of( AggregateFunction.SUM, @@ -141,7 +146,7 @@ public Set aggregateCapabilities() { @Override public Map scalarFunctionAdapters() { - return Map.of(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter()); + return Map.of(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter(), ScalarFunction.SARG_PREDICATE, new SargAdapter()); } }; } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SargAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SargAdapter.java new file mode 100644 index 0000000000000..e0b9ff84d5b57 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SargAdapter.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlKind; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; + +import java.util.List; + +/** Expands Calcite's {@code SEARCH(field, Sarg[...])} fold so substrait/DataFusion can consume the predicate. */ +class SargAdapter implements ScalarFunctionAdapter { + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + if (original.getKind() != SqlKind.SEARCH) { + return original; + } + RexBuilder rexBuilder = cluster.getRexBuilder(); + return RexUtil.expandSearch(rexBuilder, null, original); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/SargAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/SargAdapterTests.java new file mode 100644 index 0000000000000..f3d251f0bd91b --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/SargAdapterTests.java @@ -0,0 +1,138 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import com.google.common.collect.ImmutableRangeSet; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.Sarg; +import org.opensearch.test.OpenSearchTestCase; + +import java.math.BigDecimal; +import java.util.List; + +/** + * Unit tests for {@link SargAdapter}. Calcite's {@code SEARCH(x, Sarg[...])} is a + * compact, expanded form for {@code IN}-lists, {@code BETWEEN}, and unions of + * ranges; DataFusion's substrait consumer doesn't recognize {@code Sarg} as a + * literal, so the adapter expands it back into native comparison/OR trees + * before the plan is serialized. + */ +public class SargAdapterTests extends OpenSearchTestCase { + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private RelOptCluster cluster; + private RelDataType intType; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + cluster = RelOptCluster.create(planner, rexBuilder); + intType = typeFactory.createSqlType(SqlTypeName.INTEGER); + } + + /** Builds SEARCH(x, Sarg[{1, 2, 3}]) — the IN-list shape. */ + private RexCall buildInListSearch() { + RangeSet points = TreeRangeSet.create(); + points.add(Range.singleton(BigDecimal.valueOf(1))); + points.add(Range.singleton(BigDecimal.valueOf(2))); + points.add(Range.singleton(BigDecimal.valueOf(3))); + Sarg sarg = Sarg.of(org.apache.calcite.rex.RexUnknownAs.UNKNOWN, ImmutableRangeSet.copyOf(points)); + RexNode xRef = rexBuilder.makeInputRef(intType, 0); + RexNode sargLit = rexBuilder.makeSearchArgumentLiteral(sarg, intType); + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.SEARCH, List.of(xRef, sargLit)); + } + + /** Builds SEARCH(x, Sarg[[1..10]]) — the BETWEEN shape. */ + private RexCall buildBetweenSearch() { + RangeSet rangeSet = TreeRangeSet.create(); + rangeSet.add(Range.closed(BigDecimal.valueOf(1), BigDecimal.valueOf(10))); + Sarg sarg = Sarg.of(org.apache.calcite.rex.RexUnknownAs.UNKNOWN, ImmutableRangeSet.copyOf(rangeSet)); + RexNode xRef = rexBuilder.makeInputRef(intType, 0); + RexNode sargLit = rexBuilder.makeSearchArgumentLiteral(sarg, intType); + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.SEARCH, List.of(xRef, sargLit)); + } + + /** + * The core contract: SEARCH is expanded. The resulting RexNode must not + * contain any Sarg literal or SEARCH call — it has to be something the + * downstream substrait consumer knows. + */ + public void testAdaptExpandsInListSearchAwayFromSearchOperator() { + RexCall original = buildInListSearch(); + RexNode adapted = new SargAdapter().adapt(original, List.of(), cluster); + + assertFalse("expansion must not leave a SEARCH call at the root", isSearch(adapted)); + assertTrue("expansion must not leave any nested SEARCH call", containsNoSearchOrSarg(adapted)); + } + + /** + * BETWEEN-style Sargs expand to AND(ge, le). Same acceptance criterion as + * the IN-list case: no SEARCH and no Sarg literals in the output. + */ + public void testAdaptExpandsBetweenSearchAwayFromSearchOperator() { + RexCall original = buildBetweenSearch(); + RexNode adapted = new SargAdapter().adapt(original, List.of(), cluster); + + assertFalse(isSearch(adapted)); + assertTrue(containsNoSearchOrSarg(adapted)); + } + + /** + * Non-SEARCH calls must pass through untouched — the adapter is a no-op for + * anything that isn't SEARCH. Guards against collateral damage if the + * adapter gets registered against a different ScalarFunction by mistake. + */ + public void testAdaptPassesThroughNonSearchCall() { + RexNode xRef = rexBuilder.makeInputRef(intType, 0); + RexNode tenLit = rexBuilder.makeLiteral(10, intType, false); + RexCall greaterThan = (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN, List.of(xRef, tenLit)); + + RexNode adapted = new SargAdapter().adapt(greaterThan, List.of(), cluster); + + assertSame("non-SEARCH input must pass through unmodified", greaterThan, adapted); + } + + // ── helpers ──────────────────────────────────────────────────────────── + + private static boolean isSearch(RexNode node) { + return node instanceof RexCall call && call.getKind() == SqlKind.SEARCH; + } + + /** Returns false if the tree still carries a SEARCH call or a Sarg literal at any depth. */ + private static boolean containsNoSearchOrSarg(RexNode node) { + if (isSearch(node)) return false; + if (node instanceof org.apache.calcite.rex.RexLiteral lit && lit.getValue() instanceof Sarg) { + return false; + } + if (node instanceof RexCall call) { + for (RexNode operand : call.getOperands()) { + if (!containsNoSearchOrSarg(operand)) return false; + } + } + return true; + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/SearchOperatorIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/SearchOperatorIT.java new file mode 100644 index 0000000000000..e8434f38ee9e6 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/SearchOperatorIT.java @@ -0,0 +1,101 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** End-to-end coverage for PPL queries that fold into {@code SEARCH(field, Sarg[...])}. */ +public class SearchOperatorIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + public void testInListFoldsToSearchAndReturnsMatchingRows() throws IOException { + assertInt0Values( + "source=" + DATASET.indexName + " | where int0 in (1, 8, 10) | fields int0 | sort int0", + 1, 8, 8, 8, 10 + ); + } + + public void testNotInListFoldsToSearchAndReturnsMatchingRows() throws IOException { + assertInt0Values( + "source=" + DATASET.indexName + " | where int0 not in (1, 8, 10) | fields int0 | sort int0", + 3, 4, 4, 4, 7, 11 + ); + } + + public void testBetweenFoldsToSearchAndReturnsRangeRows() throws IOException { + assertInt0Values( + "source=" + DATASET.indexName + " | where int0 >= 4 and int0 <= 8 | fields int0 | sort int0", + 4, 4, 4, 7, 8, 8, 8 + ); + } + + public void testRangeUnionFoldsToSearchAndReturnsAllMatchingRows() throws IOException { + assertInt0Values( + "source=" + DATASET.indexName + " | where int0 < 4 or int0 > 10 | fields int0 | sort int0", + 1, 3, 11 + ); + } + + /** Project-side Sarg: eval produces SEARCH in a projection expression, not a filter. */ + public void testSargFoldInEvalProjectionReturnsMatchingRows() throws IOException { + assertInt0Values( + "source=" + + DATASET.indexName + + " | eval is_match = int0 in (1, 8, 10)" + + " | where is_match = true" + + " | fields int0" + + " | sort int0", + 1, 8, 8, 8, 10 + ); + } + + private void assertInt0Values(String ppl, long... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> rows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, rows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, rows.size()); + long[] actual = new long[rows.size()]; + for (int i = 0; i < rows.size(); i++) { + Object cell = rows.get(i).get(0); + assertNotNull("null int0 cell at row " + i + " for query: " + ppl, cell); + actual[i] = ((Number) cell).longValue(); + } + assertEquals( + "int0 values mismatch for query: " + ppl + " expected=" + + Arrays.toString(expected) + " actual=" + Arrays.toString(actual), + Arrays.toString(expected), + Arrays.toString(actual) + ); + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } +} From cbad6c668d1dc625605017d6c2cde81ef95450ed Mon Sep 17 00:00:00 2001 From: expani1729 <110471048+expani@users.noreply.github.com> Date: Tue, 5 May 2026 17:55:09 -0700 Subject: [PATCH 054/115] Introduce Instructions for the Backend to use during execution (#21479) --- ...xt.java => ShardScanExecutionContext.java} | 5 +- .../spi/AnalyticsSearchBackendPlugin.java | 11 ++ .../spi/BackendExecutionContext.java | 22 +++ .../analytics/spi/BackendExecutionState.java | 22 +++ .../analytics/spi/CommonExecutionContext.java | 21 +++ .../analytics/spi/DelegatedExpression.java | 60 +++++++ .../analytics/spi/ExchangeSinkContext.java | 2 +- .../spi/FilterDelegationInstructionNode.java | 68 ++++++++ .../analytics/spi/FilterTreeShape.java | 32 ++++ .../spi/FinalAggregateInstructionNode.java | 41 +++++ .../spi/FragmentInstructionHandler.java | 31 ++++ .../FragmentInstructionHandlerFactory.java | 52 ++++++ .../analytics/spi/InstructionNode.java | 27 ++++ .../analytics/spi/InstructionType.java | 48 ++++++ .../spi/PartialAggregateInstructionNode.java | 40 +++++ .../spi/SearchExecEngineProvider.java | 8 +- .../spi/ShardScanInstructionNode.java | 39 +++++ .../rust/src/ffm.rs | 47 ++++++ .../rust/src/lib.rs | 1 + .../rust/src/query_executor.rs | 38 +++++ .../rust/src/session_context.rs | 149 ++++++++++++++++++ .../DataFusionAnalyticsBackendPlugin.java | 12 +- .../DataFusionInstructionHandlerFactory.java | 78 +++++++++ .../be/datafusion/DataFusionSessionState.java | 19 +++ .../be/datafusion/DatafusionContext.java | 10 ++ .../DatafusionSearchExecEngine.java | 12 +- .../be/datafusion/DatafusionSearcher.java | 38 +++++ .../FinalAggregateInstructionHandler.java | 32 ++++ .../ShardScanInstructionHandler.java | 59 +++++++ .../be/datafusion/nativelib/NativeBridge.java | 60 +++++++ .../nativelib/SessionContextHandle.java | 38 +++++ .../DataFusionNativeBridgeTests.java | 54 +++++++ .../DatafusionSearchExecEngineTests.java | 30 ++-- ...DelegationForIndexFullConversionTests.java | 75 +++++++-- .../LuceneAnalyticsBackendPluginTests.java | 48 +++++- .../exec/AnalyticsSearchService.java | 30 +++- .../analytics/exec/FragmentResources.java | 6 +- .../exec/action/FragmentExecutionRequest.java | 25 ++- .../exec/stage/LocalStageScheduler.java | 20 +++ .../stage/ShardFragmentStageScheduler.java | 2 +- .../analytics/planner/CapabilityRegistry.java | 15 ++ .../analytics/planner/RelNodeUtils.java | 32 ++-- .../planner/dag/FilterTreeShapeDeriver.java | 76 +++++++++ .../planner/dag/FragmentConversionDriver.java | 72 +++++++-- .../analytics/planner/dag/Stage.java | 10 ++ .../analytics/planner/dag/StagePlan.java | 29 ++-- .../PlanAlternativeSerializationTests.java | 107 +++++++++++++ .../ShardFragmentStageExecutionTests.java | 2 +- .../analytics/planner/FilterRuleTests.java | 33 ++++ .../analytics/planner/MockBackend.java | 45 ++++++ .../dag/FilterTreeShapeDeriverTests.java | 115 ++++++++++++++ .../dag/FragmentConversionDriverTests.java | 70 +++++++- 52 files changed, 1916 insertions(+), 102 deletions(-) rename sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/{ExecutionContext.java => ShardScanExecutionContext.java} (90%) create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionContext.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionState.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/CommonExecutionContext.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedExpression.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationInstructionNode.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterTreeShape.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FinalAggregateInstructionNode.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandler.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandlerFactory.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionNode.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionType.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/PartialAggregateInstructionNode.java create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ShardScanInstructionNode.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionSessionState.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/FinalAggregateInstructionHandler.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextHandle.java create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriver.java create mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/action/PlanAlternativeSerializationTests.java create mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriverTests.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ShardScanExecutionContext.java similarity index 90% rename from sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java rename to sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ShardScanExecutionContext.java index c4a1ffd52c916..fb0df3f1301d3 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ExecutionContext.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ShardScanExecutionContext.java @@ -9,6 +9,7 @@ package org.opensearch.analytics.backend; import org.apache.arrow.memory.BufferAllocator; +import org.opensearch.analytics.spi.CommonExecutionContext; import org.opensearch.index.engine.exec.IndexReaderProvider.Reader; import org.opensearch.tasks.Task; @@ -18,7 +19,7 @@ * * @opensearch.internal */ -public class ExecutionContext { +public class ShardScanExecutionContext implements CommonExecutionContext { private final String tableName; private final Reader reader; @@ -32,7 +33,7 @@ public class ExecutionContext { * @param task the transport-created task for this fragment execution * @param reader the data-format aware reader */ - public ExecutionContext(String tableName, Task task, Reader reader) { + public ShardScanExecutionContext(String tableName, Task task, Reader reader) { this.tableName = tableName; this.task = task; this.reader = reader; diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java index e580b9824e36d..e4722784197f6 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java @@ -70,4 +70,15 @@ default ExchangeSinkProvider getExchangeSinkProvider() { return null; } + /** + * Returns the instruction handler factory for this backend. Used at the coordinator + * to create instruction nodes (backend attaches custom config) and at the data node + * to create handlers that apply instructions to the execution context. + * + *

      Backends that declare {@code supportedDelegations} or participate in multi-stage + * execution MUST implement this. Validation at startup ensures consistency. + */ + default FragmentInstructionHandlerFactory getInstructionHandlerFactory() { + throw new UnsupportedOperationException("getInstructionHandlerFactory not implemented for [" + name() + "]"); + } } diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionContext.java new file mode 100644 index 0000000000000..ac3ca2508a2c7 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionContext.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +/** + * Marker interface for backend-specific execution context that flows between + * successive instruction handler calls. The first handler in the chain receives + * {@code null} and bootstraps the context; subsequent handlers receive and build + * upon the previous handler's output. + * + *

      Each backend defines its own concrete implementation (e.g., + * {@code DataFusionSessionState} holding a native SessionContext handle). + * + * @opensearch.internal + */ +public interface BackendExecutionContext {} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionState.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionState.java new file mode 100644 index 0000000000000..f5ae62ce81424 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionState.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +/** + * Marker interface for backend-specific execution state that flows between + * successive instruction handler calls. The first handler in the chain receives + * {@code null} and bootstraps the state; subsequent handlers receive and build + * upon the previous handler's output. + * + *

      Each backend defines its own concrete implementation (e.g., + * {@code DataFusionSessionState} holding a native SessionContext handle). + * + * @opensearch.internal + */ +public interface BackendExecutionState {} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/CommonExecutionContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/CommonExecutionContext.java new file mode 100644 index 0000000000000..db68ec841e11e --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/CommonExecutionContext.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +/** + * Marker interface for execution contexts provided by Core to instruction handlers. + * Concrete implementations carry the information relevant to their execution path: + *

        + *
      • {@code ShardScanExecutionContext} — shard fragment execution (reader, task, tableName)
        • + *
      • {@code ExchangeSinkContext} — coordinator reduce execution (planBytes, allocator, schema)
        • + *
      + * + * @opensearch.internal + */ +public interface CommonExecutionContext {} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedExpression.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedExpression.java new file mode 100644 index 0000000000000..d914642ede6fd --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegatedExpression.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; + +import java.io.IOException; + +/** + * A single delegated predicate — carries the annotation ID, the accepting backend, + * and the serialized bytes produced by the accepting backend's + * {@link DelegatedPredicateSerializer} or anything similar. + * + * @opensearch.internal + */ +public class DelegatedExpression implements Writeable { + + private final int annotationId; + private final String acceptingBackendId; + private final byte[] expressionBytes; + + public DelegatedExpression(int annotationId, String acceptingBackendId, byte[] expressionBytes) { + this.annotationId = annotationId; + this.acceptingBackendId = acceptingBackendId; + this.expressionBytes = expressionBytes; + } + + public DelegatedExpression(StreamInput in) throws IOException { + this.annotationId = in.readInt(); + this.acceptingBackendId = in.readString(); + this.expressionBytes = in.readByteArray(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(annotationId); + out.writeString(acceptingBackendId); + out.writeByteArray(expressionBytes); + } + + public int getAnnotationId() { + return annotationId; + } + + public String getAcceptingBackendId() { + return acceptingBackendId; + } + + public byte[] getExpressionBytes() { + return expressionBytes; + } +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java index 2df1062a60988..22b755a73772a 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkContext.java @@ -43,7 +43,7 @@ * @opensearch.internal */ public record ExchangeSinkContext(String queryId, int stageId, byte[] fragmentBytes, BufferAllocator allocator, List< - ChildInput> childInputs, ExchangeSink downstream) { + ChildInput> childInputs, ExchangeSink downstream) implements CommonExecutionContext { /** Per-child input descriptor: the child stage id and the schema of its outgoing batches. */ public record ChildInput(int childStageId, Schema schema) { diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationInstructionNode.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationInstructionNode.java new file mode 100644 index 0000000000000..d56a5c5bed775 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationInstructionNode.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.List; + +/** + * Instruction node for filter delegation to an index backend. + * Carries the tree shape, predicate count, and serialized delegated queries. + * + * @opensearch.internal + */ +public class FilterDelegationInstructionNode implements InstructionNode { + + private final FilterTreeShape treeShape; + private final int delegatedPredicateCount; + private final List delegatedQueries; + + public FilterDelegationInstructionNode( + FilterTreeShape treeShape, + int delegatedPredicateCount, + List delegatedQueries + ) { + this.treeShape = treeShape; + this.delegatedPredicateCount = delegatedPredicateCount; + this.delegatedQueries = delegatedQueries; + } + + public FilterDelegationInstructionNode(StreamInput in) throws IOException { + this.treeShape = in.readEnum(FilterTreeShape.class); + this.delegatedPredicateCount = in.readInt(); + this.delegatedQueries = in.readList(DelegatedExpression::new); + } + + @Override + public InstructionType type() { + return InstructionType.SETUP_FILTER_DELEGATION_FOR_INDEX; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(treeShape); + out.writeInt(delegatedPredicateCount); + out.writeCollection(delegatedQueries); + } + + public FilterTreeShape getTreeShape() { + return treeShape; + } + + public int getDelegatedPredicateCount() { + return delegatedPredicateCount; + } + + public List getDelegatedQueries() { + return delegatedQueries; + } +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterTreeShape.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterTreeShape.java new file mode 100644 index 0000000000000..8081ba7d63cb6 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterTreeShape.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +/** + * Backend-agnostic description of the boolean tree shape when filter delegation is active. + * Provided by the planner so backends can choose their execution strategy without + * re-inspecting the Substrait plan. + * + * @opensearch.internal + */ +public enum FilterTreeShape { + /** No delegation — all predicates handled natively by the driving backend. */ + NO_DELEGATION, + /** + * All predicates (delegated + native) are under a single AND — no interleaving + * under OR/NOT. Backend can handle delegated bitsets and native predicates independently. + */ + CONJUNCTIVE, + /** + * Delegated and native predicates are interleaved under OR/NOT — the boolean tree + * mixes predicates from different backends under non-AND operators. Backend needs a + * tree evaluator to combine bitsets from both backends per the boolean structure. + */ + INTERLEAVED_BOOLEAN_EXPRESSION +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FinalAggregateInstructionNode.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FinalAggregateInstructionNode.java new file mode 100644 index 0000000000000..87bfc2c5081d8 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FinalAggregateInstructionNode.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + * Instruction node for final aggregate in coordinator reduce — ExchangeSink path, + * remove partial agg, preserve final-only for the driving backend's reduce execution. + * + *

      TODO: add backend-specific config fields as final aggregate implementation is built out. + * + * @opensearch.internal + */ +public class FinalAggregateInstructionNode implements InstructionNode { + + public FinalAggregateInstructionNode() {} + + public FinalAggregateInstructionNode(StreamInput in) throws IOException { + // TODO: read config fields when added + } + + @Override + public InstructionType type() { + return InstructionType.SETUP_FINAL_AGGREGATE; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + // TODO: write config fields when added + } +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandler.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandler.java new file mode 100644 index 0000000000000..db70c1c9fdd33 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandler.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +/** + * Applies an {@link InstructionNode} to the execution context at the data node. + * Each handler is created per-execution by the backend's + * {@link FragmentInstructionHandlerFactory#createHandler(InstructionNode)}. + * + * @param the concrete instruction node type this handler processes + * @opensearch.internal + */ +public interface FragmentInstructionHandler { + + /** + * Applies the instruction, reading from Core's context and building upon the + * backend's accumulated execution context from previous handlers. + * + * @param node the instruction node + * @param commonContext Core-provided context (shard info or reduce info) + * @param backendContext backend state from previous handler, or {@code null} for the first handler + * @return updated backend execution context for the next handler or final consumer + */ + BackendExecutionContext apply(N node, CommonExecutionContext commonContext, BackendExecutionContext backendContext); +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandlerFactory.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandlerFactory.java new file mode 100644 index 0000000000000..6e62fa10f012e --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandlerFactory.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import java.util.List; +import java.util.Optional; + +/** + * Factory for creating {@link InstructionNode}s at the coordinator and + * {@link FragmentInstructionHandler}s at the data node. One factory per backend, + * accessed via {@code AnalyticsSearchBackendPlugin.getInstructionHandlerFactory()}. + * + *

      Coordinator-side creation methods return {@link Optional#empty()} if the backend + * does not support the instruction type. Core logs and skips unsupported instructions. + * + * @opensearch.internal + */ +public interface FragmentInstructionHandlerFactory { + + // ── Coordinator-side: create instruction nodes ── + + /** Creates a shard scan instruction node. */ + Optional createShardScanNode(); + + /** Creates a filter delegation instruction node with the given delegation metadata. */ + Optional createFilterDelegationNode( + FilterTreeShape treeShape, + int delegatedPredicateCount, + List delegatedQueries + ); + + /** Creates a partial aggregate instruction node. */ + Optional createPartialAggregateNode(); + + /** Creates a final aggregate instruction node for coordinator reduce. */ + Optional createFinalAggregateNode(); + + // ── Data-node-side: create handler for an instruction node ── + + /** + * Creates a handler for the given instruction node. The handler's + * {@link FragmentInstructionHandler#apply} will be called with the node + * and the execution context. + */ + FragmentInstructionHandler createHandler(InstructionNode node); +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionNode.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionNode.java new file mode 100644 index 0000000000000..e52e545d0384b --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionNode.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.opensearch.core.common.io.stream.Writeable; + +/** + * Metadata node produced by the planner (via backend's factory) at the coordinator + * and consumed by the backend's handler at the data node. Carries typed configuration + * that the handler uses to configure the execution environment. + * + *

      Generic parent interface — backends extend with concrete classes if they need + * additional coordinator-side context beyond what the framework provides. + * + * @opensearch.internal + */ +public interface InstructionNode extends Writeable { + + /** The instruction type — used to look up the handler factory at the data node. */ + InstructionType type(); +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionType.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionType.java new file mode 100644 index 0000000000000..490f60a967707 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionType.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.IOException; + +/** + * Types of instructions that the planner can produce for backend execution. + * Each type corresponds to a specific execution concern that the backend + * must handle during the prepare phase on the data node. + * + * @opensearch.internal + */ +public enum InstructionType { + /** Base scan setup — reader acquisition, SessionContext creation, default table provider. */ + SETUP_SHARD_SCAN, + /** + * Filter delegation to an index backend — bridge setup, UDF registration, IndexedTableProvider. + * + *

      TODO: add a DelegationStrategy field (BACKEND_DRIVEN vs CENTRALLY_DRIVEN) to the + * instruction node when centrally-driven delegation is implemented. Currently only + * BACKEND_DRIVEN exists — derived from the backend declaring + * {@code supportedDelegations(DelegationType.FILTER)}. + */ + SETUP_FILTER_DELEGATION_FOR_INDEX, + /** Partial aggregate mode — disable combine optimizer, cut plan to partial-only. */ + SETUP_PARTIAL_AGGREGATE, + /** Final aggregate for coordinator reduce — ExchangeSink path, final-only agg. */ + SETUP_FINAL_AGGREGATE; + + /** Deserializes an {@link InstructionNode} from the stream based on this type. */ + public InstructionNode readNode(StreamInput in) throws IOException { + return switch (this) { + case SETUP_SHARD_SCAN -> new ShardScanInstructionNode(in); + case SETUP_FILTER_DELEGATION_FOR_INDEX -> new FilterDelegationInstructionNode(in); + case SETUP_PARTIAL_AGGREGATE -> new PartialAggregateInstructionNode(in); + case SETUP_FINAL_AGGREGATE -> new FinalAggregateInstructionNode(in); + }; + } +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/PartialAggregateInstructionNode.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/PartialAggregateInstructionNode.java new file mode 100644 index 0000000000000..2f94d08f3ef0f --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/PartialAggregateInstructionNode.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + * Instruction node for partial aggregate mode — disable combine optimizer, cut plan to partial-only. + * + *

      TODO: add backend-specific config fields as partial aggregate implementation is built out. + * + * @opensearch.internal + */ +public class PartialAggregateInstructionNode implements InstructionNode { + + public PartialAggregateInstructionNode() {} + + public PartialAggregateInstructionNode(StreamInput in) throws IOException { + // TODO: read config fields when added + } + + @Override + public InstructionType type() { + return InstructionType.SETUP_PARTIAL_AGGREGATE; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + // TODO: write config fields when added + } +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/SearchExecEngineProvider.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/SearchExecEngineProvider.java index f16b8f36d9021..8edd8d0a71dc6 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/SearchExecEngineProvider.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/SearchExecEngineProvider.java @@ -9,8 +9,8 @@ package org.opensearch.analytics.spi; import org.opensearch.analytics.backend.EngineResultStream; -import org.opensearch.analytics.backend.ExecutionContext; import org.opensearch.analytics.backend.SearchExecEngine; +import org.opensearch.analytics.backend.ShardScanExecutionContext; /** * Execution engine factory for backend plugins. @@ -23,6 +23,10 @@ public interface SearchExecEngineProvider { /** * Creates a search execution engine bound to the given execution context. * The context carries the reader snapshot and task metadata. + * The backendContext carries backend-specific state produced by instruction handlers. */ - SearchExecEngine createSearchExecEngine(ExecutionContext ctx); + SearchExecEngine createSearchExecEngine( + ShardScanExecutionContext ctx, + BackendExecutionContext backendContext + ); } diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ShardScanInstructionNode.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ShardScanInstructionNode.java new file mode 100644 index 0000000000000..8000d34f68844 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ShardScanInstructionNode.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + * Instruction node for base shard scan setup — reader acquisition, SessionContext creation, + * default table provider registration. + * + * @opensearch.internal + */ +public class ShardScanInstructionNode implements InstructionNode { + + public ShardScanInstructionNode() {} + + public ShardScanInstructionNode(StreamInput in) throws IOException { + // No fields to read + } + + @Override + public InstructionType type() { + return InstructionType.SETUP_SHARD_SCAN; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + // No fields to write + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index 7c9a67827f326..ce180c060f563 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -412,6 +412,29 @@ pub unsafe extern "C" fn df_cache_manager_add_files( Ok(0) } +// --------------------------------------------------------------------------- +// SessionContext decomposition — instruction-based execution +// --------------------------------------------------------------------------- + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_create_session_context( + shard_view_ptr: i64, + runtime_ptr: i64, + table_name_ptr: *const u8, + table_name_len: i64, + context_id: i64, +) -> i64 { + let table_name = str_from_raw(table_name_ptr, table_name_len) + .map_err(|e| format!("df_create_session_context: {}", e))?; + let mgr = get_rt_manager()?; + mgr.io_runtime + .block_on(crate::session_context::create_session_context( + runtime_ptr, shard_view_ptr, table_name, context_id, + )) + .map_err(|e| e.to_string()) +} + #[ffm_safe] #[no_mangle] pub unsafe extern "C" fn df_cache_manager_remove_files( @@ -526,3 +549,27 @@ pub unsafe extern "C" fn df_cache_manager_contains_by_type( .ok_or_else(|| "df_cache_manager_contains_by_type: no cache manager configured".to_string())?; Ok(if manager.contains_file_by_type(file_path, cache_type) { 1 } else { 0 }) } + +#[no_mangle] +pub unsafe extern "C" fn df_close_session_context(ptr: i64) { + crate::session_context::close_session_context(ptr); +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_execute_with_context( + session_ctx_ptr: i64, + plan_ptr: *const u8, + plan_len: i64, +) -> i64 { + let mgr = get_rt_manager()?; + let plan_bytes = slice::from_raw_parts(plan_ptr, plan_len as usize); + let cpu_executor = mgr.cpu_executor(); + mgr.io_runtime + .block_on(crate::query_executor::execute_with_context( + session_ctx_ptr, + plan_bytes, + cpu_executor, + )) + .map_err(|e| e.to_string()) +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs index d12fa24fa3acc..ee876888450c9 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs @@ -27,4 +27,5 @@ pub mod partition_stream; pub mod query_executor; pub mod query_memory_pool_tracker; pub mod runtime_manager; +pub mod session_context; pub mod statistics_cache; diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs index 766baffcc7afe..8ba9c93b3caea 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs @@ -29,9 +29,15 @@ use substrait::proto::Plan; use crate::cross_rt_stream::CrossRtStream; use crate::executor::DedicatedExecutor; use crate::api::DataFusionRuntime; +use crate::session_context::SessionContextHandle; /// Execute a vanilla parquet query: substrait plan → DataFusion → CrossRtStream. /// File access goes through DataFusion's registered object store. +/// +/// Deprecated: Production now uses the decomposed `create_session_context` + +/// `execute_with_context` path (via `api::execute_query`). +/// TODO: Remove this function and migrate benchmarks to the decomposed path. +/// Retained only for benchmarks. TODO: migrate benchmarks and remove. pub async fn execute_query( table_path: ListingTableUrl, object_metas: Arc>, @@ -147,3 +153,35 @@ pub async fn execute_query( Ok(Box::into_raw(Box::new(wrapped)) as i64) } + +/// Executes a Substrait plan against a pre-configured SessionContext. +/// Consumes the handle — SessionContext lifetime is tied to the returned stream. +pub async unsafe fn execute_with_context( + session_ctx_ptr: i64, + plan_bytes: &[u8], + cpu_executor: DedicatedExecutor, +) -> Result { + let handle = *Box::from_raw(session_ctx_ptr as *mut SessionContextHandle); + + let substrait_plan = Plan::decode(plan_bytes).map_err(|e| { + DataFusionError::Execution(format!("Failed to decode Substrait: {}", e)) + })?; + + let logical_plan = from_substrait_plan(&handle.ctx.state(), &substrait_plan).await?; + let dataframe = handle.ctx.execute_logical_plan(logical_plan).await?; + let physical_plan = dataframe.create_physical_plan().await?; + + let df_stream = execute_stream(physical_plan, handle.ctx.task_ctx()).map_err(|e| { + error!("execute_with_context: failed to create stream: {}", e); + e + })?; + + let cross_rt_stream = CrossRtStream::new_with_df_error_stream(df_stream, cpu_executor); + let wrapped = datafusion::physical_plan::stream::RecordBatchStreamAdapter::new( + cross_rt_stream.schema(), + cross_rt_stream, + ); + + let stream_handle = crate::api::QueryStreamHandle::new(wrapped, handle.query_context); + Ok(Box::into_raw(Box::new(stream_handle)) as i64) +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs new file mode 100644 index 0000000000000..9de9caaa968a5 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs @@ -0,0 +1,149 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! SessionContext lifecycle for instruction-based execution. +//! +//! `create_session_context` creates a fully configured SessionContext with +//! the default ListingTable registered. Called by ShardScanInstruction handler. + +use std::sync::Arc; + +use datafusion::{ + common::DataFusionError, + datasource::file_format::parquet::ParquetFormat, + datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl}, + execution::cache::cache_manager::CacheManagerConfig, + execution::cache::{CacheAccessor, DefaultListFilesCache}, + execution::context::SessionContext, + execution::memory_pool::MemoryPool, + execution::runtime_env::RuntimeEnvBuilder, + execution::SessionStateBuilder, + prelude::*, +}; +use log::error; +use object_store::ObjectMeta; + +use crate::api::{DataFusionRuntime, ShardView}; +use crate::query_memory_pool_tracker::QueryTrackingContext; + +/// Opaque handle holding a configured SessionContext between FFM calls. +pub struct SessionContextHandle { + pub ctx: SessionContext, + pub table_path: ListingTableUrl, + pub object_metas: Arc>, + pub query_context: QueryTrackingContext, +} + +/// Creates a SessionContext with per-query RuntimeEnv and registers the default +/// ListingTable provider for parquet scans. +pub async unsafe fn create_session_context( + runtime_ptr: i64, + shard_view_ptr: i64, + table_name: &str, + context_id: i64, +) -> Result { + let runtime = &*(runtime_ptr as *const DataFusionRuntime); + let shard_view = &*(shard_view_ptr as *const ShardView); + + let global_pool = runtime.runtime_env.memory_pool.clone(); + let query_context = QueryTrackingContext::new(context_id, global_pool); + let query_memory_pool = query_context + .memory_pool() + .map(|p| p as Arc); + + let list_file_cache = Arc::new(DefaultListFilesCache::default()); + list_file_cache.put( + &datafusion::execution::cache::TableScopedPath { + table: None, + path: shard_view.table_path.prefix().clone(), + }, + shard_view.object_metas.clone(), + ); + + let mut runtime_env_builder = RuntimeEnvBuilder::from_runtime_env(&runtime.runtime_env) + .with_cache_manager( + CacheManagerConfig::default() + .with_list_files_cache(Some(list_file_cache)) + .with_file_metadata_cache(Some( + runtime.runtime_env.cache_manager.get_file_metadata_cache(), + )) + .with_files_statistics_cache( + runtime.runtime_env.cache_manager.get_file_statistic_cache(), + ), + ); + + if let Some(pool) = query_memory_pool { + runtime_env_builder = runtime_env_builder.with_memory_pool(pool); + } + + let runtime_env = runtime_env_builder.build().map_err(|e| { + error!("create_session_context: failed to build runtime env: {}", e); + e + })?; + + let query_config = crate::datafusion_query_config::DatafusionQueryConfig::default(); + let mut config = SessionConfig::new(); + config.options_mut().execution.parquet.pushdown_filters = query_config.parquet_pushdown_filters; + config.options_mut().execution.target_partitions = query_config.target_partitions; + config.options_mut().execution.batch_size = query_config.batch_size; + + let state = SessionStateBuilder::new() + .with_config(config) + .with_runtime_env(Arc::from(runtime_env)) + .with_default_features() + .build(); + + let ctx = SessionContext::new_with_state(state); + + // Register default ListingTable for parquet scans + let listing_options = ListingOptions::new(Arc::new(ParquetFormat::new())) + .with_file_extension(".parquet") + .with_collect_stat(true); + + let resolved_schema = listing_options + .infer_schema(&ctx.state(), &shard_view.table_path) + .await + .map_err(|e| { + error!("create_session_context: failed to infer schema: {}", e); + e + })?; + + let table_config = ListingTableConfig::new(shard_view.table_path.clone()) + .with_listing_options(listing_options) + .with_schema(resolved_schema); + + let provider = Arc::new(ListingTable::try_new(table_config).map_err(|e| { + error!("create_session_context: failed to create listing table: {}", e); + e + })?); + + ctx.register_table(table_name, provider).map_err(|e| { + error!("create_session_context: failed to register table '{}': {}", table_name, e); + e + })?; + + error!("create_session_context: successfully registered table '{}', table_name_len={}", table_name, table_name.len()); + + let handle = SessionContextHandle { + ctx, + table_path: shard_view.table_path.clone(), + object_metas: shard_view.object_metas.clone(), + query_context, + }; + Ok(Box::into_raw(Box::new(handle)) as i64) +} + +/// Closes a SessionContext handle without executing. Used for cleanup on failure. +/// +/// # Safety +/// `ptr` must be 0 or a valid pointer returned by `create_session_context`. +pub unsafe fn close_session_context(ptr: i64) { + if ptr != 0 { + let _ = Box::from_raw(ptr as *mut SessionContextHandle); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 58c705ab9812d..93b79e17c9acf 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -17,6 +17,7 @@ import org.opensearch.analytics.spi.FieldType; import org.opensearch.analytics.spi.FilterCapability; import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory; import org.opensearch.analytics.spi.ProjectCapability; import org.opensearch.analytics.spi.ScalarFunction; import org.opensearch.analytics.spi.ScalarFunctionAdapter; @@ -158,7 +159,7 @@ public FragmentConvertor getFragmentConvertor() { @Override public SearchExecEngineProvider getSearchExecEngineProvider() { - return ctx -> { + return (ctx, backendContext) -> { DataFusionService dataFusionService = plugin.getDataFusionService(); if (dataFusionService == null) { throw new IllegalStateException("DataFusionService not initialized — createComponents() may not have been called"); @@ -180,12 +181,21 @@ public SearchExecEngineProvider getSearchExecEngineProvider() { throw new IllegalStateException("No DatafusionReader available in the acquired reader"); } DatafusionContext context = new DatafusionContext(ctx.getTask(), dfReader, dataFusionService.getNativeRuntime()); + if (backendContext != null) { + DataFusionSessionState sessionState = (DataFusionSessionState) backendContext; + context.setSessionContextHandle(sessionState.sessionContextHandle()); + } DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine(context); engine.prepare(ctx); return engine; }; } + @Override + public FragmentInstructionHandlerFactory getInstructionHandlerFactory() { + return new DataFusionInstructionHandlerFactory(plugin); + } + @Override public ExchangeSinkProvider getExchangeSinkProvider() { return ctx -> { diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java new file mode 100644 index 0000000000000..88ec47e2da9f9 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.analytics.spi.DelegatedExpression; +import org.opensearch.analytics.spi.FilterDelegationInstructionNode; +import org.opensearch.analytics.spi.FilterTreeShape; +import org.opensearch.analytics.spi.FinalAggregateInstructionNode; +import org.opensearch.analytics.spi.FragmentInstructionHandler; +import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory; +import org.opensearch.analytics.spi.InstructionNode; +import org.opensearch.analytics.spi.ShardScanInstructionNode; + +import java.util.List; +import java.util.Optional; + +/** + * DataFusion backend's instruction handler factory. + * + *

      Coordinator side: creates typed instruction nodes for wire transport. + *

      Data node side: creates handlers that call into Rust via FFM to configure the SessionContext. + */ +public class DataFusionInstructionHandlerFactory implements FragmentInstructionHandlerFactory { + + private final DataFusionPlugin plugin; + + public DataFusionInstructionHandlerFactory(DataFusionPlugin plugin) { + this.plugin = plugin; + } + + // ── Coordinator: create instruction nodes ── + + @Override + public Optional createShardScanNode() { + return Optional.of(new ShardScanInstructionNode()); + } + + @Override + public Optional createFilterDelegationNode( + FilterTreeShape treeShape, + int delegatedPredicateCount, + List delegatedExpressions + ) { + return Optional.of(new FilterDelegationInstructionNode(treeShape, delegatedPredicateCount, delegatedExpressions)); + } + + @Override + public Optional createPartialAggregateNode() { + // TODO: return Optional.of(...) once PartialAggregateInstructionHandler is implemented + return Optional.empty(); + } + + @Override + public Optional createFinalAggregateNode() { + return Optional.of(new FinalAggregateInstructionNode()); + } + + // ── Data node: create handlers ── + + @SuppressWarnings("unchecked") + @Override + public FragmentInstructionHandler createHandler(InstructionNode node) { + if (node instanceof ShardScanInstructionNode) { + return new ShardScanInstructionHandler(plugin); + } + if (node instanceof FinalAggregateInstructionNode) { + return new FinalAggregateInstructionHandler(); + } + // TODO: FilterDelegationInstructionHandler, PartialAggregateInstructionHandler + throw new UnsupportedOperationException("No handler for instruction type: " + node.type()); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionSessionState.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionSessionState.java new file mode 100644 index 0000000000000..edd48ddb11f22 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionSessionState.java @@ -0,0 +1,19 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.analytics.spi.BackendExecutionContext; +import org.opensearch.be.datafusion.nativelib.SessionContextHandle; + +/** + * Backend-specific execution context produced by ShardScanInstructionHandler, + * consumed by DatafusionSearcher at execute time. + */ +public record DataFusionSessionState(SessionContextHandle sessionContextHandle) implements BackendExecutionContext { +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java index 527ead7df96a2..cd30c64e0758e 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java @@ -8,6 +8,7 @@ package org.opensearch.be.datafusion; +import org.opensearch.be.datafusion.nativelib.SessionContextHandle; import org.opensearch.be.datafusion.nativelib.StreamHandle; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.search.SearchExecutionContext; @@ -31,6 +32,7 @@ public class DatafusionContext implements SearchExecutionContext { - - private static final Logger logger = LogManager.getLogger(DatafusionSearchExecEngine.class); +public class DatafusionSearchExecEngine implements SearchExecEngine { private final DatafusionContext datafusionContext; @@ -38,14 +34,14 @@ public DatafusionSearchExecEngine(DatafusionContext datafusionContext) { } @Override - public void prepare(ExecutionContext requestContext) { + public void prepare(ShardScanExecutionContext requestContext) { byte[] substraitBytes = requestContext.getFragmentBytes(); long contextId = datafusionContext.task() != null ? datafusionContext.task().getId() : 0L; datafusionContext.setDatafusionQuery(new DatafusionQuery(requestContext.getTableName(), substraitBytes, contextId)); } @Override - public EngineResultStream execute(ExecutionContext requestContext) throws IOException { + public EngineResultStream execute(ShardScanExecutionContext requestContext) throws IOException { BufferAllocator allocator = requestContext.getAllocator(); if (allocator == null) { throw new IllegalStateException("ExecutionContext.allocator must be set by the caller before execute()"); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearcher.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearcher.java index 627ea17dd05e9..c8bb98991f10e 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearcher.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearcher.java @@ -10,6 +10,7 @@ import org.opensearch.be.datafusion.nativelib.NativeBridge; import org.opensearch.be.datafusion.nativelib.ReaderHandle; +import org.opensearch.be.datafusion.nativelib.SessionContextHandle; import org.opensearch.be.datafusion.nativelib.StreamHandle; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.action.ActionListener; @@ -45,6 +46,43 @@ public DatafusionSearcher(ReaderHandle readerHandle) { @Override public void search(DatafusionContext context) throws IOException { + SessionContextHandle sessionCtx = context.getSessionContextHandle(); + if (sessionCtx != null) { + searchWithSessionContext(context, sessionCtx); + } else { + searchVanilla(context); + } + } + + private void searchWithSessionContext(DatafusionContext context, SessionContextHandle sessionCtx) throws IOException { + DatafusionQuery query = context.getDatafusionQuery(); + NativeRuntimeHandle runtimeHandle = context.getNativeRuntime(); + CompletableFuture future = new CompletableFuture<>(); + NativeBridge.executeWithContextAsync(sessionCtx.getPointer(), query.getSubstraitBytes(), new ActionListener<>() { + @Override + public void onResponse(Long streamPtr) { + future.complete(streamPtr); + } + + @Override + public void onFailure(Exception exception) { + future.completeExceptionally(exception); + } + }); + long streamPtr; + try { + streamPtr = future.join(); + } catch (Exception exception) { + throw new IOException("Query execution with session context failed", exception); + } + // Rust consumed the session context — unregister from live handle set + sessionCtx.close(); + context.setStreamHandle(new StreamHandle(streamPtr, runtimeHandle)); + } + + // TODO: Remove searchVanilla once all execution paths go through instruction handlers. + // Deprecated — retained only for tests that bypass AnalyticsSearchService. + private void searchVanilla(DatafusionContext context) throws IOException { DatafusionQuery query = context.getDatafusionQuery(); if (query == null) { throw new IllegalStateException("DatafusionQuery must be set before search"); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/FinalAggregateInstructionHandler.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/FinalAggregateInstructionHandler.java new file mode 100644 index 0000000000000..c3a9be1813c67 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/FinalAggregateInstructionHandler.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.analytics.spi.BackendExecutionContext; +import org.opensearch.analytics.spi.CommonExecutionContext; +import org.opensearch.analytics.spi.FinalAggregateInstructionNode; +import org.opensearch.analytics.spi.FragmentInstructionHandler; + +/** + * Handles FinalAggregate instruction for coordinator-reduce stages. + * TODO: Configure SessionContext optimizer (disable CombinePartialFinalAggregate) for multi-shard aggregates. + */ +public class FinalAggregateInstructionHandler implements FragmentInstructionHandler { + + @Override + public BackendExecutionContext apply( + FinalAggregateInstructionNode node, + CommonExecutionContext commonContext, + BackendExecutionContext backendContext + ) { + // TODO: Configure LocalSession optimizer settings for final aggregate execution. + // For now, the reduce path works without explicit configuration. + return backendContext; + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java new file mode 100644 index 0000000000000..2910499103bd5 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.analytics.backend.ShardScanExecutionContext; +import org.opensearch.analytics.spi.BackendExecutionContext; +import org.opensearch.analytics.spi.CommonExecutionContext; +import org.opensearch.analytics.spi.FragmentInstructionHandler; +import org.opensearch.analytics.spi.ShardScanInstructionNode; +import org.opensearch.be.datafusion.nativelib.NativeBridge; +import org.opensearch.be.datafusion.nativelib.SessionContextHandle; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; + +/** + * Handles ShardScan instruction: creates a SessionContext via FFM and registers + * the default ListingTable provider for parquet scans. + */ +public class ShardScanInstructionHandler implements FragmentInstructionHandler { + + private final DataFusionPlugin plugin; + + ShardScanInstructionHandler(DataFusionPlugin plugin) { + this.plugin = plugin; + } + + @Override + public BackendExecutionContext apply( + ShardScanInstructionNode node, + CommonExecutionContext commonContext, + BackendExecutionContext backendContext + ) { + ShardScanExecutionContext context = (ShardScanExecutionContext) commonContext; + DataFusionService dataFusionService = plugin.getDataFusionService(); + DataFormatRegistry registry = plugin.getDataFormatRegistry(); + + DatafusionReader dfReader = null; + for (String formatName : plugin.getSupportedFormats()) { + dfReader = context.getReader().getReader(registry.format(formatName), DatafusionReader.class); + if (dfReader != null) break; + } + if (dfReader == null) { + throw new IllegalStateException("No DatafusionReader available in the acquired reader"); + } + + long readerPtr = dfReader.getReaderHandle().getPointer(); + long runtimePtr = dataFusionService.getNativeRuntime().get(); + long contextId = context.getTask() != null ? context.getTask().getId() : 0L; + + SessionContextHandle sessionCtxHandle = NativeBridge.createSessionContext(readerPtr, runtimePtr, context.getTableName(), contextId); + + return new DataFusionSessionState(sessionCtxHandle); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index 387d18376cead..7afa5eb24feff 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -69,6 +69,9 @@ public final class NativeBridge { private static final MethodHandle CACHE_MANAGER_GET_MEMORY_BY_TYPE; private static final MethodHandle CACHE_MANAGER_GET_TOTAL_MEMORY; private static final MethodHandle CACHE_MANAGER_CONTAINS_BY_TYPE; + private static final MethodHandle CREATE_SESSION_CONTEXT; + private static final MethodHandle CLOSE_SESSION_CONTEXT; + private static final MethodHandle EXECUTE_WITH_CONTEXT; static { SymbolLookup lib = NativeLibraryLoader.symbolLookup(); @@ -253,6 +256,19 @@ public final class NativeBridge { ) ); + // ── SessionContext decomposition bindings ── + CREATE_SESSION_CONTEXT = linker.downcallHandle( + lib.find("df_create_session_context").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG + ) + ); + // i64 df_cache_manager_add_files(runtime_ptr, files_ptr, files_len_ptr, files_count) CACHE_MANAGER_ADD_FILES = linker.downcallHandle( lib.find("df_cache_manager_add_files").orElseThrow(), @@ -314,6 +330,16 @@ public final class NativeBridge { // caller step required — as soon as this class is loaded, callbacks // are installed and `df_execute_indexed_query` can dispatch into Java. installFilterTreeCallbacks(linker); + + CLOSE_SESSION_CONTEXT = linker.downcallHandle( + lib.find("df_close_session_context").orElseThrow(), + FunctionDescriptor.ofVoid(ValueLayout.JAVA_LONG) + ); + + EXECUTE_WITH_CONTEXT = linker.downcallHandle( + lib.find("df_execute_with_context").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) + ); } private NativeBridge() {} @@ -650,6 +676,40 @@ public static long createCustomCacheManager() { throw t instanceof RuntimeException ? (RuntimeException) t : new RuntimeException(t); } } + // ---- SessionContext decomposition ---- + + /** + * Creates a SessionContext with the default ListingTable registered. + * Returns a tracked handle consumed by {@link #executeWithContextAsync}. + */ + public static SessionContextHandle createSessionContext(long readerPtr, long runtimePtr, String tableName, long contextId) { + NativeHandle.validatePointer(readerPtr, "reader"); + NativeHandle.validatePointer(runtimePtr, "runtime"); + try (var call = new NativeCall()) { + var table = call.str(tableName); + long ptr = call.invoke(CREATE_SESSION_CONTEXT, readerPtr, runtimePtr, table.segment(), table.len(), contextId); + return new SessionContextHandle(ptr); + } + } + + /** + * Executes a Substrait plan against the configured SessionContext. + * Consumes the session context handle (freed internally when stream closes). + */ + /** Frees a native SessionContext handle. Safe to call once. */ + public static void closeSessionContext(long ptr) { + NativeCall.invokeVoid(CLOSE_SESSION_CONTEXT, ptr); + } + + public static void executeWithContextAsync(long sessionCtxPtr, byte[] substraitPlan, ActionListener listener) { + NativeHandle.validatePointer(sessionCtxPtr, "sessionContext"); + try (var call = new NativeCall()) { + long result = call.invoke(EXECUTE_WITH_CONTEXT, sessionCtxPtr, call.bytes(substraitPlan), (long) substraitPlan.length); + listener.onResponse(result); + } catch (Throwable throwable) { + listener.onFailure(throwable instanceof Exception ? (Exception) throwable : new RuntimeException(throwable)); + } + } public static void destroyCustomCacheManager(long ptr) { NativeCall.invokeVoid(DESTROY_CUSTOM_CACHE_MANAGER, ptr); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextHandle.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextHandle.java new file mode 100644 index 0000000000000..c26d1799611eb --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextHandle.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion.nativelib; + +import org.opensearch.analytics.backend.jni.NativeHandle; + +/** + * Type-safe wrapper for a native SessionContext pointer returned by + * {@link NativeBridge#createSessionContext}. The Rust side consumes this + * handle when {@link NativeBridge#executeWithContextAsync} is called, + * so {@link #doClose()} is a no-op — the pointer is freed by Rust internally. + * + *

      This handle exists to participate in the {@link NativeHandle} live-pointer + * registry so that {@link NativeHandle#validatePointer} passes for FFM calls. + */ +public class SessionContextHandle extends NativeHandle { + + public SessionContextHandle(long ptr) { + super(ptr); + } + + @Override + protected void doClose() { + // TODO: Handle error-path cleanup. Currently Rust consumes the handle in + // execute_with_context (moves QueryTrackingContext into the stream). If execute + // fails or is never called, this handle leaks on the Rust side. + // Options: (a) AtomicBool 'consumed' flag on Rust handle — close_session_context + // checks flag before freeing, (b) don't consume in Rust and use no-op tracking + // on the stream, (c) markConsumed() on NativeHandle to skip doClose on happy path. + // See df_close_session_context FFM entry which exists but is not yet wired here. + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java index 3ec318a4e2ae6..0e5c78087986f 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java @@ -10,10 +10,13 @@ import org.opensearch.be.datafusion.nativelib.NativeBridge; import org.opensearch.be.datafusion.nativelib.ReaderHandle; +import org.opensearch.be.datafusion.nativelib.SessionContextHandle; +import org.opensearch.core.action.ActionListener; import org.opensearch.test.OpenSearchTestCase; import java.nio.file.Files; import java.nio.file.Path; +import java.util.concurrent.CompletableFuture; /** * Smoke test for the DataFusion JNI bridge. @@ -63,4 +66,55 @@ public void testReaderLifecycle() throws Exception { NativeBridge.closeGlobalRuntime(runtimePtr); } + + public void testSessionContextCreationAndTableRegistration() throws Exception { + NativeBridge.initTokioRuntimeManager(2); + Path spillDir = createTempDir("datafusion-spill"); + long runtimePtr = NativeBridge.createGlobalRuntime(64 * 1024 * 1024, 0L, spillDir.toString(), 32 * 1024 * 1024); + NativeRuntimeHandle runtimeHandle = new NativeRuntimeHandle(runtimePtr); + + Path dataDir = createTempDir("datafusion-data"); + Path testParquet = Path.of(getClass().getClassLoader().getResource("test.parquet").toURI()); + Files.copy(testParquet, dataDir.resolve("test.parquet")); + + ReaderHandle readerHandle = new ReaderHandle(dataDir.toString(), new String[] { "test.parquet" }); + + // Create session context with table registered + SessionContextHandle sessionCtx = NativeBridge.createSessionContext( + readerHandle.getPointer(), + runtimeHandle.get(), + "test_table", + 0L + ); + assertTrue("SessionContext pointer should be non-zero", sessionCtx.getPointer() != 0); + + // Execute a simple query to verify the session context is properly configured + byte[] substrait = NativeBridge.sqlToSubstrait( + readerHandle.getPointer(), + "test_table", + "SELECT message FROM test_table", + runtimeHandle.get() + ); + CompletableFuture future = new CompletableFuture<>(); + NativeBridge.executeWithContextAsync(sessionCtx.getPointer(), substrait, new ActionListener<>() { + @Override + public void onResponse(Long streamPtr) { + future.complete(streamPtr); + } + + @Override + public void onFailure(Exception exception) { + future.completeExceptionally(exception); + } + }); + long streamPtr = future.join(); + assertTrue("Stream pointer should be non-zero", streamPtr != 0); + + // Session context is consumed by execute — close the Java handle + sessionCtx.close(); + + NativeBridge.streamClose(streamPtr); + readerHandle.close(); + runtimeHandle.close(); + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java index 7135b988599e8..21fea0bc58494 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java @@ -11,9 +11,10 @@ import org.apache.arrow.memory.RootAllocator; import org.opensearch.analytics.backend.EngineResultBatch; import org.opensearch.analytics.backend.EngineResultStream; -import org.opensearch.analytics.backend.ExecutionContext; +import org.opensearch.analytics.backend.ShardScanExecutionContext; import org.opensearch.be.datafusion.nativelib.NativeBridge; import org.opensearch.be.datafusion.nativelib.ReaderHandle; +import org.opensearch.be.datafusion.nativelib.SessionContextHandle; import org.opensearch.test.OpenSearchTestCase; import java.nio.file.Files; @@ -62,17 +63,16 @@ public void testEngineExecuteSelectAll() throws Exception { runtimeHandle.get() ); - // Build the plugin-level objects DatafusionReader reader = createReader(); DatafusionContext context = new DatafusionContext(null, reader, runtimeHandle); - context.setDatafusionQuery(new DatafusionQuery("test_table", substrait, 0L)); try ( RootAllocator alloc = new RootAllocator(Long.MAX_VALUE); DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine(context) ) { - ExecutionContext execCtx = new ExecutionContext("test_table", null, null); + ShardScanExecutionContext execCtx = createExecutionContext("test_table", substrait, context); execCtx.setAllocator(alloc); + engine.prepare(execCtx); try (EngineResultStream stream = engine.execute(execCtx)) { List rows = collectRows(stream); assertEquals(2, rows.size()); @@ -94,14 +94,14 @@ public void testEngineExecuteAggregation() throws Exception { DatafusionReader reader = createReader(); DatafusionContext context = new DatafusionContext(null, reader, runtimeHandle); - context.setDatafusionQuery(new DatafusionQuery("test_table", substrait, 0L)); try ( RootAllocator alloc = new RootAllocator(Long.MAX_VALUE); DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine(context) ) { - ExecutionContext execCtx = new ExecutionContext("test_table", null, null); + ShardScanExecutionContext execCtx = createExecutionContext("test_table", substrait, context); execCtx.setAllocator(alloc); + engine.prepare(execCtx); try (EngineResultStream stream = engine.execute(execCtx)) { List rows = collectRows(stream); assertEquals(1, rows.size()); @@ -120,14 +120,14 @@ public void testEngineExecuteFilter() throws Exception { DatafusionReader reader = createReader(); DatafusionContext context = new DatafusionContext(null, reader, runtimeHandle); - context.setDatafusionQuery(new DatafusionQuery("test_table", substrait, 0L)); try ( RootAllocator alloc = new RootAllocator(Long.MAX_VALUE); DatafusionSearchExecEngine engine = new DatafusionSearchExecEngine(context) ) { - ExecutionContext execCtx = new ExecutionContext("test_table", null, null); + ShardScanExecutionContext execCtx = createExecutionContext("test_table", substrait, context); execCtx.setAllocator(alloc); + engine.prepare(execCtx); try (EngineResultStream stream = engine.execute(execCtx)) { List rows = collectRows(stream); assertEquals(1, rows.size()); @@ -137,10 +137,22 @@ public void testEngineExecuteFilter() throws Exception { } private DatafusionReader createReader() { - // Wrap the raw pointer in a ReaderHandle via the existing native pointer return new DatafusionReader(readerHandle.getPointer()); } + private ShardScanExecutionContext createExecutionContext(String tableName, byte[] substrait, DatafusionContext dfContext) { + ShardScanExecutionContext execCtx = new ShardScanExecutionContext(tableName, null, null); + execCtx.setFragmentBytes(substrait); + SessionContextHandle sessionCtxHandle = NativeBridge.createSessionContext( + readerHandle.getPointer(), + runtimeHandle.get(), + tableName, + 0L + ); + dfContext.setSessionContextHandle(sessionCtxHandle); + return execCtx; + } + private List collectRows(EngineResultStream stream) { List rows = new ArrayList<>(); Iterator it = stream.iterator(); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java index 953f50afa1d09..bcbe2c3ef7c92 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java @@ -38,15 +38,22 @@ import org.opensearch.analytics.planner.dag.StagePlan; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; import org.opensearch.analytics.spi.BackendCapabilityProvider; +import org.opensearch.analytics.spi.DelegatedExpression; import org.opensearch.analytics.spi.DelegatedPredicateFunction; import org.opensearch.analytics.spi.DelegationType; import org.opensearch.analytics.spi.EngineCapability; import org.opensearch.analytics.spi.ExchangeSinkProvider; import org.opensearch.analytics.spi.FieldType; import org.opensearch.analytics.spi.FilterCapability; +import org.opensearch.analytics.spi.FilterDelegationInstructionNode; +import org.opensearch.analytics.spi.FilterTreeShape; import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.FragmentInstructionHandler; +import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory; +import org.opensearch.analytics.spi.InstructionNode; import org.opensearch.analytics.spi.ScalarFunction; import org.opensearch.analytics.spi.ScanCapability; +import org.opensearch.analytics.spi.ShardScanInstructionNode; import org.opensearch.be.lucene.LuceneAnalyticsBackendPlugin; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; @@ -69,6 +76,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -154,8 +162,8 @@ public void testMixedNativeAndDelegated() throws Exception { ); StagePlan plan = runPipeline(condition); - assertEquals("should have 1 delegated query", 1, plan.delegatedQueries().size()); - assertMatchQueryBuilder(plan.delegatedQueries(), "message", "hello world"); + assertEquals("should have 1 delegated query", 1, plan.delegatedExpressions().size()); + assertMatchQueryBuilder(plan.delegatedExpressions(), "message", "hello world"); SubstraitResult substrait = substraitResult(plan.convertedBytes()); logger.info("Substrait plan (mixed E2E):\n{}", substrait.plan()); @@ -165,7 +173,7 @@ public void testMixedNativeAndDelegated() throws Exception { assertEquals("AND must have 2 arguments", 2, andFunc.getArgumentsCount()); // arg[1]: delegated_predicate(1) — annotation id=1 maps to MATCH 'hello world' assertDelegatedPredicate(substrait.plan(), andFunc.getArguments(1).getValue(), 1); - assertMatchQueryForAnnotation(plan.delegatedQueries(), 1, "message", "hello world"); + assertMatchQueryForAnnotation(plan.delegatedExpressions(), 1, "message", "hello world"); } /** @@ -184,7 +192,7 @@ public void testComplexBooleanTree() throws Exception { ); StagePlan plan = runPipeline(condition); - assertEquals("should have 2 delegated queries", 2, plan.delegatedQueries().size()); + assertEquals("should have 2 delegated queries", 2, plan.delegatedExpressions().size()); SubstraitResult substrait = substraitResult(plan.convertedBytes()); logger.info("Substrait plan (complex E2E):\n{}", substrait.plan()); @@ -203,14 +211,14 @@ public void testComplexBooleanTree() throws Exception { // OR arg[0]: delegated_predicate(1) → MATCH 'hello' assertDelegatedPredicate(substrait.plan(), orFunc.getArguments(0).getValue(), 1); - assertMatchQueryForAnnotation(plan.delegatedQueries(), 1, "message", "hello"); + assertMatchQueryForAnnotation(plan.delegatedExpressions(), 1, "message", "hello"); // OR arg[1]: NOT(delegated_predicate(2)) → MATCH 'goodbye' Expression notExpr = orFunc.getArguments(1).getValue(); assertTrue("OR second arg must be scalar function", notExpr.hasScalarFunction()); assertEquals("not", resolveFunctionName(substrait.plan(), notExpr.getScalarFunction().getFunctionReference())); assertDelegatedPredicate(substrait.plan(), notExpr.getScalarFunction().getArguments(0).getValue(), 2); - assertMatchQueryForAnnotation(plan.delegatedQueries(), 2, "message", "goodbye"); + assertMatchQueryForAnnotation(plan.delegatedExpressions(), 2, "message", "goodbye"); } // ---- Pipeline ---- @@ -261,10 +269,10 @@ private RexNode makeMatch(int fieldIndex, String query) { ); } - private void assertMatchQueryBuilder(Map delegatedQueries, String expectedField, String expectedQuery) + private void assertMatchQueryBuilder(List delegatedExpressions, String expectedField, String expectedQuery) throws IOException { - for (byte[] queryBytes : delegatedQueries.values()) { - try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(queryBytes), WRITEABLE_REGISTRY)) { + for (DelegatedExpression expr : delegatedExpressions) { + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(expr.getExpressionBytes()), WRITEABLE_REGISTRY)) { MatchQueryBuilder matchQuery = (MatchQueryBuilder) input.readNamedWriteable(QueryBuilder.class); if (matchQuery.fieldName().equals(expectedField) && matchQuery.value().equals(expectedQuery)) { return; @@ -311,15 +319,20 @@ private void assertDelegatedPredicate(Plan plan, Expression expr, int expectedAn /** Asserts the delegated query bytes for a specific annotation ID deserialize to the expected MatchQueryBuilder. */ private void assertMatchQueryForAnnotation( - Map delegatedQueries, + List delegatedExpressions, int annotationId, String expectedField, String expectedQuery ) throws IOException { - assertTrue("annotation ID " + annotationId + " must be in delegatedQueries", delegatedQueries.containsKey(annotationId)); - try ( - StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(delegatedQueries.get(annotationId)), WRITEABLE_REGISTRY) - ) { + DelegatedExpression found = null; + for (DelegatedExpression expr : delegatedExpressions) { + if (expr.getAnnotationId() == annotationId) { + found = expr; + break; + } + } + assertNotNull("annotation ID " + annotationId + " must be in delegatedExpressions", found); + try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(found.getExpressionBytes()), WRITEABLE_REGISTRY)) { MatchQueryBuilder matchQuery = (MatchQueryBuilder) input.readNamedWriteable(QueryBuilder.class); assertEquals("field name for annotation " + annotationId, expectedField, matchQuery.fieldName()); assertEquals("query text for annotation " + annotationId, expectedQuery, matchQuery.value()); @@ -437,5 +450,39 @@ public ExchangeSinkProvider getExchangeSinkProvider() { public FragmentConvertor getFragmentConvertor() { return convertor; } + + @Override + public FragmentInstructionHandlerFactory getInstructionHandlerFactory() { + return new FragmentInstructionHandlerFactory() { + @Override + public Optional createShardScanNode() { + return Optional.of(new ShardScanInstructionNode()); + } + + @Override + public Optional createFilterDelegationNode( + FilterTreeShape treeShape, + int delegatedPredicateCount, + List delegatedExpressions + ) { + return Optional.of(new FilterDelegationInstructionNode(treeShape, delegatedPredicateCount, delegatedExpressions)); + } + + @Override + public Optional createPartialAggregateNode() { + return Optional.empty(); + } + + @Override + public Optional createFinalAggregateNode() { + return Optional.empty(); + } + + @Override + public FragmentInstructionHandler createHandler(InstructionNode node) { + throw new UnsupportedOperationException("stub"); + } + }; + } } } diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java index 9f441489365a6..735ce496ba481 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java @@ -37,14 +37,21 @@ import org.opensearch.analytics.planner.dag.StagePlan; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; import org.opensearch.analytics.spi.BackendCapabilityProvider; +import org.opensearch.analytics.spi.DelegatedExpression; import org.opensearch.analytics.spi.DelegationType; import org.opensearch.analytics.spi.EngineCapability; import org.opensearch.analytics.spi.ExchangeSinkProvider; import org.opensearch.analytics.spi.FieldType; import org.opensearch.analytics.spi.FilterCapability; +import org.opensearch.analytics.spi.FilterDelegationInstructionNode; +import org.opensearch.analytics.spi.FilterTreeShape; import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.FragmentInstructionHandler; +import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory; +import org.opensearch.analytics.spi.InstructionNode; import org.opensearch.analytics.spi.ScalarFunction; import org.opensearch.analytics.spi.ScanCapability; +import org.opensearch.analytics.spi.ShardScanInstructionNode; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; @@ -67,6 +74,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -140,11 +148,11 @@ public void testMatchPredicateDelegationEndToEnd() throws IOException { StagePlan plan = leaf.getPlanAlternatives().getFirst(); // Verify delegation happened - assertFalse("delegatedQueries should not be empty", plan.delegatedQueries().isEmpty()); - assertEquals("should have exactly one delegated query", 1, plan.delegatedQueries().size()); + assertFalse("delegatedExpressions should not be empty", plan.delegatedExpressions().isEmpty()); + assertEquals("should have exactly one delegated expression", 1, plan.delegatedExpressions().size()); // Deserialize and verify the MatchQueryBuilder - byte[] queryBytes = plan.delegatedQueries().values().iterator().next(); + byte[] queryBytes = plan.delegatedExpressions().getFirst().getExpressionBytes(); try (StreamInput input = new NamedWriteableAwareStreamInput(StreamInput.wrap(queryBytes), WRITEABLE_REGISTRY)) { QueryBuilder deserialized = input.readNamedWriteable(QueryBuilder.class); assertTrue("Should be MatchQueryBuilder", deserialized instanceof MatchQueryBuilder); @@ -282,5 +290,39 @@ public byte[] attachPartialAggOnTop(RelNode partialAggFragment, byte[] innerByte } }; } + + @Override + public FragmentInstructionHandlerFactory getInstructionHandlerFactory() { + return new FragmentInstructionHandlerFactory() { + @Override + public Optional createShardScanNode() { + return Optional.of(new ShardScanInstructionNode()); + } + + @Override + public Optional createFilterDelegationNode( + FilterTreeShape treeShape, + int delegatedPredicateCount, + List delegatedExpressions + ) { + return Optional.of(new FilterDelegationInstructionNode(treeShape, delegatedPredicateCount, delegatedExpressions)); + } + + @Override + public Optional createPartialAggregateNode() { + return Optional.empty(); + } + + @Override + public Optional createFinalAggregateNode() { + return Optional.empty(); + } + + @Override + public FragmentInstructionHandler createHandler(InstructionNode node) { + throw new UnsupportedOperationException("mock"); + } + }; + } } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java index 252347f2da2cf..804635187cf4d 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java @@ -12,12 +12,16 @@ import org.opensearch.analytics.backend.AnalyticsOperationListener; import org.opensearch.analytics.backend.EngineResultBatch; import org.opensearch.analytics.backend.EngineResultStream; -import org.opensearch.analytics.backend.ExecutionContext; import org.opensearch.analytics.backend.SearchExecEngine; +import org.opensearch.analytics.backend.ShardScanExecutionContext; import org.opensearch.analytics.exec.action.FragmentExecutionRequest; import org.opensearch.analytics.exec.action.FragmentExecutionResponse; import org.opensearch.analytics.exec.task.AnalyticsShardTask; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; +import org.opensearch.analytics.spi.BackendExecutionContext; +import org.opensearch.analytics.spi.FragmentInstructionHandler; +import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory; +import org.opensearch.analytics.spi.InstructionNode; import org.opensearch.arrow.flight.transport.ArrowAllocatorProvider; import org.opensearch.common.Nullable; import org.opensearch.common.concurrent.GatedCloseable; @@ -36,7 +40,7 @@ /** * Data-node service that executes plan fragments against local shards. * Acquires a reader from the shard's composite engine, builds an - * {@link ExecutionContext}, and invokes the backend's {@link SearchExecEngine} + * {@link ShardScanExecutionContext}, and invokes the backend's {@link SearchExecEngine} * to produce results. * *

      Does NOT hold {@code IndicesService} — receives an already-resolved @@ -107,12 +111,24 @@ public FragmentResources executeFragmentStreaming(FragmentExecutionRequest reque private FragmentResources startFragment(FragmentExecutionRequest request, ResolvedFragment resolved, Task task) throws IOException { GatedCloseable gatedReader = resolved.readerProvider.acquireReader(); - SearchExecEngine engine = null; + SearchExecEngine engine = null; EngineResultStream stream = null; try { - ExecutionContext ctx = buildContext(request, gatedReader.get(), resolved.plan, task); + ShardScanExecutionContext ctx = buildContext(request, gatedReader.get(), resolved.plan, task); AnalyticsSearchBackendPlugin backend = backends.get(resolved.plan.getBackendId()); - engine = backend.getSearchExecEngineProvider().createSearchExecEngine(ctx); + + // Apply instruction handlers in order — each builds upon the previous handler's backend context + BackendExecutionContext backendContext = null; + List instructions = resolved.plan.getInstructions(); + if (!instructions.isEmpty()) { + FragmentInstructionHandlerFactory factory = backend.getInstructionHandlerFactory(); + for (InstructionNode node : instructions) { + FragmentInstructionHandler handler = factory.createHandler(node); + backendContext = handler.apply(node, ctx, backendContext); + } + } + + engine = backend.getSearchExecEngineProvider().createSearchExecEngine(ctx, backendContext); stream = engine.execute(ctx); return new FragmentResources(gatedReader, engine, stream); } catch (Exception e) { @@ -158,13 +174,13 @@ private ResolvedFragment resolveFragment(FragmentExecutionRequest request, Index return new ResolvedFragment(readerProvider, selectedPlan, request.getQueryId(), request.getStageId(), shardIdStr); } - private ExecutionContext buildContext( + private ShardScanExecutionContext buildContext( FragmentExecutionRequest request, Reader reader, FragmentExecutionRequest.PlanAlternative plan, Task task ) { - ExecutionContext ctx = new ExecutionContext(request.getShardId().getIndexName(), task, reader); + ShardScanExecutionContext ctx = new ShardScanExecutionContext(request.getShardId().getIndexName(), task, reader); ctx.setFragmentBytes(plan.getFragmentBytes()); ctx.setAllocator(allocator); return ctx; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java index 9ad9170b4a812..0c61c3b85f79c 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/FragmentResources.java @@ -9,8 +9,8 @@ package org.opensearch.analytics.exec; import org.opensearch.analytics.backend.EngineResultStream; -import org.opensearch.analytics.backend.ExecutionContext; import org.opensearch.analytics.backend.SearchExecEngine; +import org.opensearch.analytics.backend.ShardScanExecutionContext; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.index.engine.exec.IndexReaderProvider.Reader; @@ -23,12 +23,12 @@ public final class FragmentResources implements AutoCloseable { private final GatedCloseable gatedReader; - private final SearchExecEngine engine; + private final SearchExecEngine engine; private final EngineResultStream stream; public FragmentResources( GatedCloseable gatedReader, - SearchExecEngine engine, + SearchExecEngine engine, EngineResultStream stream ) { this.gatedReader = gatedReader; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java index ffc20d47fdb16..6bfb308081310 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java @@ -11,6 +11,8 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.analytics.exec.task.AnalyticsShardTask; +import org.opensearch.analytics.spi.InstructionNode; +import org.opensearch.analytics.spi.InstructionType; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.index.shard.ShardId; @@ -97,28 +99,43 @@ public ActionRequestValidationException validate() { } /** - * A single plan alternative: a backend ID paired with its serialized fragment bytes. + * A single plan alternative: a backend ID paired with its serialized fragment bytes + * and ordered instruction nodes for data-node execution. * Produced by {@code FragmentConversionDriver.convertAll()} using the backend's * {@code FragmentConvertor}. */ public static class PlanAlternative { private final String backendId; private final byte[] fragmentBytes; + private final List instructions; - public PlanAlternative(String backendId, byte[] fragmentBytes) { + public PlanAlternative(String backendId, byte[] fragmentBytes, List instructions) { this.backendId = backendId; this.fragmentBytes = fragmentBytes; + this.instructions = instructions; } public PlanAlternative(StreamInput in) throws IOException { this.backendId = in.readString(); byte[] bytes = in.readByteArray(); this.fragmentBytes = (bytes.length == 0) ? null : bytes; + int instructionCount = in.readVInt(); + List nodes = new ArrayList<>(instructionCount); + for (int i = 0; i < instructionCount; i++) { + InstructionType type = in.readEnum(InstructionType.class); + nodes.add(type.readNode(in)); + } + this.instructions = nodes; } public void writeTo(StreamOutput out) throws IOException { out.writeString(backendId); out.writeByteArray(fragmentBytes != null ? fragmentBytes : new byte[0]); + out.writeVInt(instructions.size()); + for (InstructionNode node : instructions) { + out.writeEnum(node.type()); + node.writeTo(out); + } } public String getBackendId() { @@ -128,5 +145,9 @@ public String getBackendId() { public byte[] getFragmentBytes() { return fragmentBytes; } + + public List getInstructions() { + return instructions; + } } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java index 13aa77c61fb62..d9205260000ef 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java @@ -11,9 +11,13 @@ import org.opensearch.analytics.exec.QueryContext; import org.opensearch.analytics.planner.dag.Stage; import org.opensearch.analytics.planner.dag.StageExecutionType; +import org.opensearch.analytics.spi.BackendExecutionContext; import org.opensearch.analytics.spi.ExchangeSink; import org.opensearch.analytics.spi.ExchangeSinkContext; import org.opensearch.analytics.spi.ExchangeSinkProvider; +import org.opensearch.analytics.spi.FragmentInstructionHandler; +import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory; +import org.opensearch.analytics.spi.InstructionNode; import java.util.ArrayList; import java.util.List; @@ -47,6 +51,22 @@ public StageExecution createExecution(Stage stage, ExchangeSink sink, QueryConte buildChildInputs(stage), sink ); + + // Apply instruction handlers for the reduce stage. + // Unlike AnalyticsSearchService (shard path) which resolves the factory from its + // local backends map, the coordinator-reduce path has no backends map — the factory + // is stored on the Stage during FragmentConversionDriver.convertAll (root stage only, + // no serialization needed since reduce executes locally at the coordinator). + // TODO: find a cleaner way to provide the factory without storing it on Stage. + FragmentInstructionHandlerFactory factory = stage.getInstructionHandlerFactory(); + if (factory != null) { + BackendExecutionContext backendContext = null; + for (InstructionNode node : stage.getPlanAlternatives().getFirst().instructions()) { + FragmentInstructionHandler handler = factory.createHandler(node); + backendContext = handler.apply(node, context, backendContext); + } + } + ExchangeSink backendSink; try { backendSink = provider.createSink(context); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java index 701f0d2871e54..fdabe2581f423 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java @@ -76,7 +76,7 @@ public StageExecution createExecution(Stage stage, ExchangeSink sink, QueryConte private static List buildPlanAlternatives(Stage stage) { List alternatives = new ArrayList<>(); for (StagePlan plan : stage.getPlanAlternatives()) { - alternatives.add(new FragmentExecutionRequest.PlanAlternative(plan.backendId(), plan.convertedBytes())); + alternatives.add(new FragmentExecutionRequest.PlanAlternative(plan.backendId(), plan.convertedBytes(), plan.instructions())); } return alternatives; } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java index 5482d7dfe6b80..01474ba800efe 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/CapabilityRegistry.java @@ -98,6 +98,21 @@ public CapabilityRegistry( for (DelegationType type : caps.supportedDelegations()) { delegationSupporters.computeIfAbsent(type, k -> new ArrayList<>()).add(name); } + // Validate: if a backend supports FILTER delegation (i.e., it drives the tree walk), + // it must provide a FragmentInstructionHandlerFactory for instruction-based execution. + if (caps.supportedDelegations().contains(DelegationType.FILTER)) { + try { + backend.getInstructionHandlerFactory(); + } catch (UnsupportedOperationException exception) { + throw new IllegalStateException( + "Backend [" + + name + + "] declares supportedDelegations(FILTER) but does not implement" + + " getInstructionHandlerFactory(). A driving backend must provide an instruction" + + " handler factory to configure delegation at the data node." + ); + } + } for (DelegationType type : caps.acceptedDelegations()) { delegationAcceptors.computeIfAbsent(type, k -> new ArrayList<>()).add(name); } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/RelNodeUtils.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/RelNodeUtils.java index 3e93f4eca9ee0..06cb3e725caa8 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/RelNodeUtils.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/RelNodeUtils.java @@ -19,7 +19,6 @@ import org.opensearch.analytics.planner.rel.OpenSearchExchangeReducer; import org.opensearch.analytics.planner.rel.OpenSearchFilter; import org.opensearch.analytics.planner.rel.OpenSearchProject; -import org.opensearch.analytics.planner.rel.OpenSearchRelNode; import org.opensearch.analytics.planner.rel.OpenSearchSort; import org.opensearch.analytics.planner.rel.OpenSearchTableScan; import org.opensearch.analytics.planner.rel.OpenSearchUnion; @@ -112,29 +111,20 @@ private static RelTraitSet rebuildTraits(RelNode node, RelOptCluster newCluster, } /** - * Extracts the single backend from the leaf operator in a resolved fragment. - * After resolution, every operator has exactly one viable backend. Throws if - * the leaf has more than one (indicates resolution didn't complete). + * Finds the first node of the given type in the fragment's single-input chain. + * Returns {@code null} if not found. + * + *

      TODO: migrate existing findLeaf/findFilter usages in FragmentConversionDriver to use this. */ - public static String extractLeafBackendFromResolvedFragment(RelNode node) { - if (node.getInputs().isEmpty()) { - if (node instanceof OpenSearchRelNode leafNode) { - List backends = leafNode.getViableBackends(); - if (backends.size() != 1) { - throw new IllegalStateException( - "Expected exactly 1 viable backend on resolved leaf [" + node.getClass().getSimpleName() + "], got " + backends - ); - } - return backends.getFirst(); - } - throw new IllegalStateException("Leaf node [" + node.getClass().getSimpleName() + "] is not an OpenSearchRelNode"); + @SuppressWarnings("unchecked") + public static T findNode(RelNode node, Class type) { + if (type.isInstance(node)) { + return (T) node; } - for (RelNode input : node.getInputs()) { - String backend = extractLeafBackendFromResolvedFragment(input); - if (backend != null) { - return backend; - } + if (!node.getInputs().isEmpty()) { + return findNode(node.getInputs().getFirst(), type); } return null; } + } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriver.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriver.java new file mode 100644 index 0000000000000..55123d261f56c --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriver.java @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.planner.dag; + +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.opensearch.analytics.planner.rel.AnnotatedPredicate; +import org.opensearch.analytics.planner.rel.OpenSearchFilter; +import org.opensearch.analytics.spi.FilterTreeShape; + +/** + * Derives {@link FilterTreeShape} from a filter condition while annotations are intact. + * Must be called before stripping removes the annotations. + * + *

      Single-pass walk: determines both whether delegation exists and whether the tree + * is mixed (delegated + driving-backend predicates interleaved under OR/NOT). + * + * @opensearch.internal + */ +final class FilterTreeShapeDeriver { + + private FilterTreeShapeDeriver() {} + + /** + * Derives the filter tree shape from the filter's condition. + * + * @param filter the OpenSearchFilter with annotations intact + * @param drivingBackendId the filter operator's resolved backend + * @return the tree shape, or {@code null} if no delegated annotations exist + */ + static FilterTreeShape derive(OpenSearchFilter filter, String drivingBackendId) { + Result result = walk(filter.getCondition(), drivingBackendId); + if (!result.hasDelegated) { + return FilterTreeShape.NO_DELEGATION; + } + return result.hasMixed ? FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION : FilterTreeShape.CONJUNCTIVE; + } + + private static Result walk(RexNode node, String drivingBackendId) { + if (node instanceof AnnotatedPredicate predicate) { + boolean isDelegated = !predicate.getViableBackends().getFirst().equals(drivingBackendId); + return new Result(isDelegated, false, !isDelegated); + } + if (node instanceof RexCall call) { + boolean isOrNot = call.getKind() == SqlKind.OR || call.getKind() == SqlKind.NOT; + + boolean hasDelegated = false; + boolean hasDrivingBackend = false; + boolean hasMixed = false; + + for (RexNode operand : call.getOperands()) { + Result childResult = walk(operand, drivingBackendId); + hasDelegated |= childResult.hasDelegated; + hasDrivingBackend |= childResult.hasDrivingBackend; + hasMixed |= childResult.hasMixed; + } + + if (isOrNot && hasDelegated && hasDrivingBackend) { + hasMixed = true; + } + + return new Result(hasDelegated, hasMixed, hasDrivingBackend); + } + return new Result(false, false, false); + } + + private record Result(boolean hasDelegated, boolean hasMixed, boolean hasDrivingBackend) { + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java index 3a346a8ff6efa..c97489e9d239b 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java @@ -16,23 +16,28 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.analytics.planner.CapabilityRegistry; +import org.opensearch.analytics.planner.RelNodeUtils; import org.opensearch.analytics.planner.rel.AggregateMode; import org.opensearch.analytics.planner.rel.OpenSearchAggregate; import org.opensearch.analytics.planner.rel.OpenSearchExchangeReducer; +import org.opensearch.analytics.planner.rel.OpenSearchFilter; import org.opensearch.analytics.planner.rel.OpenSearchRelNode; import org.opensearch.analytics.planner.rel.OpenSearchStageInputScan; import org.opensearch.analytics.planner.rel.OpenSearchTableScan; import org.opensearch.analytics.planner.rel.OperatorAnnotation; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; +import org.opensearch.analytics.spi.DelegatedExpression; import org.opensearch.analytics.spi.DelegatedPredicateSerializer; import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.FilterTreeShape; import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory; +import org.opensearch.analytics.spi.InstructionNode; import org.opensearch.analytics.spi.ScalarFunction; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.function.Function; /** @@ -70,6 +75,12 @@ private FragmentConversionDriver() {} */ public static void convertAll(QueryDAG dag, CapabilityRegistry registry) { convertStage(dag.rootStage(), registry); + // Root stage executes locally at coordinator — store factory for instruction dispatch. + Stage root = dag.rootStage(); + if (root.getExchangeSinkProvider() != null && !root.getPlanAlternatives().isEmpty()) { + AnalyticsSearchBackendPlugin backend = registry.getBackend(root.getPlanAlternatives().getFirst().backendId()); + root.setInstructionHandlerFactory(backend.getInstructionHandlerFactory()); + } } private static void convertStage(Stage stage, CapabilityRegistry registry) { @@ -80,11 +91,54 @@ private static void convertStage(Stage stage, CapabilityRegistry registry) { for (StagePlan plan : stage.getPlanAlternatives()) { AnalyticsSearchBackendPlugin backend = registry.getBackend(plan.backendId()); FragmentConvertor convertor = backend.getFragmentConvertor(); + + // Derive filter tree shape BEFORE stripping (annotations must be intact) + OpenSearchFilter filter = RelNodeUtils.findNode(plan.resolvedFragment(), OpenSearchFilter.class); + FilterTreeShape treeShape = filter != null + ? FilterTreeShapeDeriver.derive(filter, plan.backendId()) + : FilterTreeShape.NO_DELEGATION; + IntraOperatorDelegationBytes delegationBytes = new IntraOperatorDelegationBytes(registry); byte[] bytes = convert(plan.resolvedFragment(), convertor, delegationBytes); - converted.add(plan.withConvertedBytes(bytes, delegationBytes.getResult())); + + // Assemble instruction list + List instructions = assembleInstructions(backend, plan, treeShape, delegationBytes); + + converted.add(plan.withConvertedBytes(bytes, delegationBytes.getResult()).withInstructions(instructions)); } stage.setPlanAlternatives(converted); + // Store factory on coordinator-reduce stages (local execution, no serialization needed). + // Shard stages get the factory from the local backend plugin at the data node. + if (stage.getExchangeSinkProvider() != null && !converted.isEmpty()) { + AnalyticsSearchBackendPlugin backend = registry.getBackend(converted.getFirst().backendId()); + stage.setInstructionHandlerFactory(backend.getInstructionHandlerFactory()); + } + } + + private static List assembleInstructions( + AnalyticsSearchBackendPlugin backend, + StagePlan plan, + FilterTreeShape treeShape, + IntraOperatorDelegationBytes delegationBytes + ) { + FragmentInstructionHandlerFactory factory = backend.getInstructionHandlerFactory(); + LinkedList instructions = new LinkedList<>(); + RelNode leaf = findLeaf(plan.resolvedFragment()); + + if (leaf instanceof OpenSearchTableScan) { + factory.createShardScanNode().ifPresent(instructions::add); + List delegated = delegationBytes.getResult(); + if (!delegated.isEmpty()) { + factory.createFilterDelegationNode(treeShape, delegated.size(), delegated).ifPresent(instructions::add); + } + if (plan.resolvedFragment() instanceof OpenSearchAggregate agg && agg.getMode() == AggregateMode.PARTIAL) { + factory.createPartialAggregateNode().ifPresent(instructions::add); + } + } else if (leaf instanceof OpenSearchStageInputScan) { + factory.createFinalAggregateNode().ifPresent(instructions::add); + } + + return instructions; } /** @@ -93,7 +147,7 @@ private static void convertStage(Stage stage, CapabilityRegistry registry) { */ static final class IntraOperatorDelegationBytes { private final CapabilityRegistry registry; - private Map result; + private List delegatedExpressions; IntraOperatorDelegationBytes(CapabilityRegistry registry) { this.registry = registry; @@ -140,16 +194,16 @@ Function resolverFor(OpenSearchRelNode operator, Re annotationBackend, serialized.length ); - if (result == null) { - result = new HashMap<>(); + if (delegatedExpressions == null) { + delegatedExpressions = new ArrayList<>(); } - result.put(annotation.getAnnotationId(), serialized); + delegatedExpressions.add(new DelegatedExpression(annotation.getAnnotationId(), annotationBackend, serialized)); return annotation.makePlaceholder(rexBuilder); }; } - Map getResult() { - return result != null ? result : Map.of(); + List getResult() { + return delegatedExpressions != null ? delegatedExpressions : List.of(); } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/Stage.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/Stage.java index 410d657a691af..61e5668b5dda9 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/Stage.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/Stage.java @@ -10,6 +10,7 @@ import org.apache.calcite.rel.RelNode; import org.opensearch.analytics.spi.ExchangeSinkProvider; +import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory; import org.opensearch.common.Nullable; import java.util.List; @@ -45,6 +46,7 @@ public class Stage { private final TargetResolver targetResolver; private final StageExecutionType executionType; private List planAlternatives; + private FragmentInstructionHandlerFactory instructionHandlerFactory; public Stage( int stageId, @@ -118,6 +120,14 @@ public void setPlanAlternatives(List planAlternatives) { this.planAlternatives = planAlternatives; } + public FragmentInstructionHandlerFactory getInstructionHandlerFactory() { + return instructionHandlerFactory; + } + + public void setInstructionHandlerFactory(FragmentInstructionHandlerFactory instructionHandlerFactory) { + this.instructionHandlerFactory = instructionHandlerFactory; + } + private StageExecutionType setStageExecutionType(ExchangeSinkProvider exchangeSinkProvider, TargetResolver targetResolver) { if (targetResolver != null) { return StageExecutionType.SHARD_FRAGMENT; diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/StagePlan.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/StagePlan.java index 515a1d0537231..afa941ccaa5c3 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/StagePlan.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/StagePlan.java @@ -9,9 +9,11 @@ package org.opensearch.analytics.planner.dag; import org.apache.calcite.rel.RelNode; +import org.opensearch.analytics.spi.DelegatedExpression; import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.InstructionNode; -import java.util.Map; +import java.util.List; /** * A single plan alternative for a {@link Stage}. Contains a resolved fragment @@ -19,21 +21,28 @@ * are narrowed to exactly one backend, plus the converted bytes produced by * the backend's {@link FragmentConvertor}. * - * @param resolvedFragment fragment with all viableBackends narrowed to single choices - * @param backendId the primary backend for this plan - * @param convertedBytes backend-specific serialized plan bytes (null before conversion) - * @param delegatedQueries serialized delegated queries keyed by annotationId (empty if no delegation) + * @param resolvedFragment fragment with all viableBackends narrowed to single choices + * @param backendId the primary backend for this plan + * @param convertedBytes backend-specific serialized plan bytes (null before conversion) + * @param delegatedExpressions serialized delegated expressions (empty if no delegation) + * @param instructions ordered instruction nodes for data-node execution (empty before resolution) * @opensearch.internal */ -public record StagePlan(RelNode resolvedFragment, String backendId, byte[] convertedBytes, Map delegatedQueries) { +public record StagePlan(RelNode resolvedFragment, String backendId, byte[] convertedBytes, List delegatedExpressions, + List instructions) { /** Creates a StagePlan before conversion (bytes not yet available). */ public StagePlan(RelNode resolvedFragment, String backendId) { - this(resolvedFragment, backendId, null, Map.of()); + this(resolvedFragment, backendId, null, List.of(), List.of()); } - /** Returns a copy with converted bytes and delegated queries populated. */ - public StagePlan withConvertedBytes(byte[] bytes, Map delegatedQueries) { - return new StagePlan(resolvedFragment, backendId, bytes, delegatedQueries); + /** Returns a copy with converted bytes and delegated expressions populated. */ + public StagePlan withConvertedBytes(byte[] bytes, List delegatedExpressions) { + return new StagePlan(resolvedFragment, backendId, bytes, delegatedExpressions, List.of()); + } + + /** Returns a copy with instructions populated. */ + public StagePlan withInstructions(List instructions) { + return new StagePlan(resolvedFragment, backendId, convertedBytes, delegatedExpressions, instructions); } } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/action/PlanAlternativeSerializationTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/action/PlanAlternativeSerializationTests.java new file mode 100644 index 0000000000000..bc6d36bd8a62d --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/action/PlanAlternativeSerializationTests.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.exec.action; + +import org.opensearch.analytics.spi.DelegatedExpression; +import org.opensearch.analytics.spi.FilterDelegationInstructionNode; +import org.opensearch.analytics.spi.FilterTreeShape; +import org.opensearch.analytics.spi.FinalAggregateInstructionNode; +import org.opensearch.analytics.spi.InstructionNode; +import org.opensearch.analytics.spi.InstructionType; +import org.opensearch.analytics.spi.PartialAggregateInstructionNode; +import org.opensearch.analytics.spi.ShardScanInstructionNode; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.List; + +/** + * Wire serialization round-trip tests for {@link FragmentExecutionRequest.PlanAlternative}. + */ +public class PlanAlternativeSerializationTests extends OpenSearchTestCase { + + public void testRoundTripWithShardScanOnly() throws IOException { + List instructions = List.of(new ShardScanInstructionNode()); + FragmentExecutionRequest.PlanAlternative original = new FragmentExecutionRequest.PlanAlternative( + "datafusion", + new byte[] { 1, 2, 3 }, + instructions + ); + + FragmentExecutionRequest.PlanAlternative deserialized = roundTrip(original); + + assertEquals("datafusion", deserialized.getBackendId()); + assertArrayEquals(new byte[] { 1, 2, 3 }, deserialized.getFragmentBytes()); + assertEquals(1, deserialized.getInstructions().size()); + assertEquals(InstructionType.SETUP_SHARD_SCAN, deserialized.getInstructions().get(0).type()); + } + + public void testRoundTripWithFilterDelegation() throws IOException { + List expressions = List.of( + new DelegatedExpression(1, "lucene", new byte[] { 10, 20 }), + new DelegatedExpression(2, "lucene", new byte[] { 30, 40 }) + ); + FilterDelegationInstructionNode filterNode = new FilterDelegationInstructionNode(FilterTreeShape.CONJUNCTIVE, 2, expressions); + List instructions = List.of(new ShardScanInstructionNode(), filterNode); + FragmentExecutionRequest.PlanAlternative original = new FragmentExecutionRequest.PlanAlternative( + "datafusion", + new byte[] { 5, 6 }, + instructions + ); + + FragmentExecutionRequest.PlanAlternative deserialized = roundTrip(original); + + assertEquals(2, deserialized.getInstructions().size()); + assertEquals(InstructionType.SETUP_SHARD_SCAN, deserialized.getInstructions().get(0).type()); + assertEquals(InstructionType.SETUP_FILTER_DELEGATION_FOR_INDEX, deserialized.getInstructions().get(1).type()); + + FilterDelegationInstructionNode deserializedFilter = (FilterDelegationInstructionNode) deserialized.getInstructions().get(1); + assertEquals(FilterTreeShape.CONJUNCTIVE, deserializedFilter.getTreeShape()); + assertEquals(2, deserializedFilter.getDelegatedPredicateCount()); + assertEquals(2, deserializedFilter.getDelegatedQueries().size()); + assertEquals(1, deserializedFilter.getDelegatedQueries().get(0).getAnnotationId()); + assertEquals("lucene", deserializedFilter.getDelegatedQueries().get(0).getAcceptingBackendId()); + assertArrayEquals(new byte[] { 10, 20 }, deserializedFilter.getDelegatedQueries().get(0).getExpressionBytes()); + } + + public void testRoundTripWithAllTypes() throws IOException { + List instructions = List.of( + new ShardScanInstructionNode(), + new FilterDelegationInstructionNode( + FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION, + 1, + List.of(new DelegatedExpression(3, "lucene", new byte[] { 99 })) + ), + new PartialAggregateInstructionNode(), + new FinalAggregateInstructionNode() + ); + FragmentExecutionRequest.PlanAlternative original = new FragmentExecutionRequest.PlanAlternative( + "datafusion", + new byte[] { 7 }, + instructions + ); + + FragmentExecutionRequest.PlanAlternative deserialized = roundTrip(original); + + assertEquals(4, deserialized.getInstructions().size()); + assertEquals(InstructionType.SETUP_SHARD_SCAN, deserialized.getInstructions().get(0).type()); + assertEquals(InstructionType.SETUP_FILTER_DELEGATION_FOR_INDEX, deserialized.getInstructions().get(1).type()); + assertEquals(InstructionType.SETUP_PARTIAL_AGGREGATE, deserialized.getInstructions().get(2).type()); + assertEquals(InstructionType.SETUP_FINAL_AGGREGATE, deserialized.getInstructions().get(3).type()); + } + + private FragmentExecutionRequest.PlanAlternative roundTrip(FragmentExecutionRequest.PlanAlternative original) throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + original.writeTo(out); + StreamInput in = out.bytes().streamInput(); + return new FragmentExecutionRequest.PlanAlternative(in); + } +} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java index d1715c2ae3de4..aa8b9a054faf2 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java @@ -177,7 +177,7 @@ private ShardFragmentStage "test-query", 0, target.shardId(), - List.of(new FragmentExecutionRequest.PlanAlternative("test-backend", new byte[0])) + List.of(new FragmentExecutionRequest.PlanAlternative("test-backend", new byte[0], List.of())) ); return new ShardFragmentStageExecution(stage, config, sink, clusterService, requestBuilder, dispatcher, codec); diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java index fa41b4fc9d5a7..a26f054ff34d0 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/FilterRuleTests.java @@ -28,7 +28,9 @@ import org.opensearch.analytics.planner.rel.OpenSearchFilter; import org.opensearch.analytics.planner.rel.OpenSearchTableScan; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; +import org.opensearch.analytics.spi.BackendCapabilityProvider; import org.opensearch.analytics.spi.DelegationType; +import org.opensearch.analytics.spi.EngineCapability; import java.util.List; import java.util.Map; @@ -341,4 +343,35 @@ protected Set acceptedDelegations() { }; return List.of(df, lucene); } + + public void testBackendWithFilterDelegationButNoFactory_throws() { + AnalyticsSearchBackendPlugin badBackend = new AnalyticsSearchBackendPlugin() { + @Override + public String name() { + return "bad-backend"; + } + + @Override + public BackendCapabilityProvider getCapabilityProvider() { + return new BackendCapabilityProvider() { + @Override + public Set supportedEngineCapabilities() { + return Set.of(); + } + + @Override + public Set supportedDelegations() { + return Set.of(DelegationType.FILTER); + } + }; + } + }; + + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> new CapabilityRegistry(List.of(badBackend), idx -> null) + ); + assertTrue(exception.getMessage().contains("bad-backend")); + assertTrue(exception.getMessage().contains("getInstructionHandlerFactory")); + } } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java index 33b70bd35ca5f..c378b00c6e254 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java @@ -11,16 +11,27 @@ import org.opensearch.analytics.spi.AggregateCapability; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; import org.opensearch.analytics.spi.BackendCapabilityProvider; +import org.opensearch.analytics.spi.DelegatedExpression; import org.opensearch.analytics.spi.DelegatedPredicateSerializer; import org.opensearch.analytics.spi.DelegationType; import org.opensearch.analytics.spi.EngineCapability; import org.opensearch.analytics.spi.FilterCapability; +import org.opensearch.analytics.spi.FilterDelegationInstructionNode; +import org.opensearch.analytics.spi.FilterTreeShape; +import org.opensearch.analytics.spi.FinalAggregateInstructionNode; +import org.opensearch.analytics.spi.FragmentInstructionHandler; +import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory; +import org.opensearch.analytics.spi.InstructionNode; +import org.opensearch.analytics.spi.PartialAggregateInstructionNode; import org.opensearch.analytics.spi.ProjectCapability; import org.opensearch.analytics.spi.ScalarFunction; import org.opensearch.analytics.spi.ScalarFunctionAdapter; import org.opensearch.analytics.spi.ScanCapability; +import org.opensearch.analytics.spi.ShardScanInstructionNode; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -119,4 +130,38 @@ protected Map scalarFunctionAdapters() { protected Map delegatedPredicateSerializers() { return Map.of(); } + + @Override + public FragmentInstructionHandlerFactory getInstructionHandlerFactory() { + return new FragmentInstructionHandlerFactory() { + @Override + public Optional createShardScanNode() { + return Optional.of(new ShardScanInstructionNode()); + } + + @Override + public Optional createFilterDelegationNode( + FilterTreeShape treeShape, + int delegatedPredicateCount, + List delegatedExpressions + ) { + return Optional.of(new FilterDelegationInstructionNode(treeShape, delegatedPredicateCount, delegatedExpressions)); + } + + @Override + public Optional createPartialAggregateNode() { + return Optional.of(new PartialAggregateInstructionNode()); + } + + @Override + public Optional createFinalAggregateNode() { + return Optional.of(new FinalAggregateInstructionNode()); + } + + @Override + public FragmentInstructionHandler createHandler(InstructionNode node) { + throw new UnsupportedOperationException("Mock backend does not execute instructions"); + } + }; + } } diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriverTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriverTests.java new file mode 100644 index 0000000000000..8930b1043b9a7 --- /dev/null +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FilterTreeShapeDeriverTests.java @@ -0,0 +1,115 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.planner.dag; + +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.opensearch.analytics.planner.BasePlannerRulesTests; +import org.opensearch.analytics.planner.rel.AnnotatedPredicate; +import org.opensearch.analytics.planner.rel.OpenSearchFilter; +import org.opensearch.analytics.spi.FilterTreeShape; + +import java.util.List; + +/** + * Unit tests for {@link FilterTreeShapeDeriver}. + */ +public class FilterTreeShapeDeriverTests extends BasePlannerRulesTests { + + private static final String DRIVING = "datafusion"; + private static final String ACCEPTING = "lucene"; + + public void testNoDelegation() { + // Single native predicate — no delegation + RexNode nativePred = annotated(DRIVING); + OpenSearchFilter filter = buildFilter(nativePred); + + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); + assertEquals("No delegation should return PLAIN", FilterTreeShape.NO_DELEGATION, shape); + } + + public void testSingleDelegatedPredicate() { + // Single delegated predicate under implicit AND + RexNode delegated = annotated(ACCEPTING); + RexNode nativePred = annotated(DRIVING); + RexNode andNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, nativePred, delegated); + OpenSearchFilter filter = buildFilter(andNode); + + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); + assertEquals(FilterTreeShape.CONJUNCTIVE, shape); + } + + public void testMultipleDelegatedUnderAnd() { + // Multiple delegated predicates under AND — still SINGLE_AND + RexNode delegated1 = annotated(ACCEPTING); + RexNode delegated2 = annotated(ACCEPTING); + RexNode nativePred = annotated(DRIVING); + RexNode andNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, nativePred, delegated1, delegated2); + OpenSearchFilter filter = buildFilter(andNode); + + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); + assertEquals(FilterTreeShape.CONJUNCTIVE, shape); + } + + public void testOrWithDelegatedAndNative() { + // OR mixing delegated and native → MIXED_BOOLEAN + RexNode delegated = annotated(ACCEPTING); + RexNode nativePred = annotated(DRIVING); + RexNode orNode = rexBuilder.makeCall(SqlStdOperatorTable.OR, nativePred, delegated); + OpenSearchFilter filter = buildFilter(orNode); + + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); + assertEquals(FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION, shape); + } + + public void testNotWithDelegated() { + // NOT wrapping delegated + native → MIXED_BOOLEAN + RexNode delegated = annotated(ACCEPTING); + RexNode nativePred = annotated(DRIVING); + RexNode andNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, nativePred, delegated); + RexNode notNode = rexBuilder.makeCall(SqlStdOperatorTable.NOT, andNode); + OpenSearchFilter filter = buildFilter(notNode); + + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); + assertEquals(FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION, shape); + } + + public void testOrWithOnlyDelegated() { + // OR with only delegated predicates (no driving backend) — SINGLE_AND (no mixing) + RexNode delegated1 = annotated(ACCEPTING); + RexNode delegated2 = annotated(ACCEPTING); + RexNode orNode = rexBuilder.makeCall(SqlStdOperatorTable.OR, delegated1, delegated2); + RexNode nativePred = annotated(DRIVING); + RexNode andNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, nativePred, orNode); + OpenSearchFilter filter = buildFilter(andNode); + + FilterTreeShape shape = FilterTreeShapeDeriver.derive(filter, DRIVING); + assertEquals(FilterTreeShape.CONJUNCTIVE, shape); + } + + // ---- Helpers ---- + + private AnnotatedPredicate annotated(String backendId) { + RelDataType boolType = typeFactory.createJavaType(boolean.class); + RexNode literal = rexBuilder.makeLiteral(true); + return new AnnotatedPredicate(boolType, literal, List.of(backendId), 0); + } + + private OpenSearchFilter buildFilter(RexNode condition) { + return new OpenSearchFilter( + cluster, + RelTraitSet.createEmpty(), + stubScan(mockTable("test_index", "col")), + condition, + List.of(DRIVING) + ); + } +} diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java index 1d2c4f81f16ba..1d1d6de31f427 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java @@ -38,7 +38,10 @@ import org.opensearch.analytics.spi.DelegatedPredicateSerializer; import org.opensearch.analytics.spi.DelegationType; import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.FilterDelegationInstructionNode; +import org.opensearch.analytics.spi.FilterTreeShape; import org.opensearch.analytics.spi.FragmentConvertor; +import org.opensearch.analytics.spi.InstructionType; import org.opensearch.analytics.spi.ScalarFunction; import java.nio.charset.StandardCharsets; @@ -109,6 +112,10 @@ private void assertShardScanConverted(RecordingConvertor convertor, Stage stage) assertEquals("test_index", convertor.shardScanTableName); assertDoesntContainOperators(convertor.shardScanFragment, OPENSEARCH_OPERATORS); assertDoesntContainOperators(convertor.shardScanFragment, ANNOTATION_MARKERS); + // Instruction assertions + StagePlan plan = stage.getPlanAlternatives().getFirst(); + assertFalse("instructions must not be empty", plan.instructions().isEmpty()); + assertEquals("first instruction must be SHARD_SCAN", InstructionType.SETUP_SHARD_SCAN, plan.instructions().getFirst().type()); } private void assertReduceStageConverted(RecordingConvertor convertor, Stage stage) { @@ -117,6 +124,14 @@ private void assertReduceStageConverted(RecordingConvertor convertor, Stage stag assertTrue("convertFinalAggFragment must be called", convertor.finalAggCalled); assertDoesntContainOperators(convertor.reduceFragment, OPENSEARCH_OPERATORS); assertDoesntContainOperators(convertor.reduceFragment, ANNOTATION_MARKERS); + // Instruction assertions + StagePlan plan = stage.getPlanAlternatives().getFirst(); + assertFalse("instructions must not be empty", plan.instructions().isEmpty()); + assertEquals( + "reduce stage must have FINAL_AGGREGATE", + InstructionType.SETUP_FINAL_AGGREGATE, + plan.instructions().getFirst().type() + ); } // ---- Single-stage query shapes ---- @@ -350,9 +365,10 @@ private void assertDelegationResult( int expectedDelegatedCount, boolean expectPlaceholder, boolean expectNativeEquals, - List expectedFunctions + List expectedFunctions, + FilterTreeShape expectedTreeShape ) { - assertEquals("delegatedQueries count", expectedDelegatedCount, plan.delegatedQueries().size()); + assertEquals("delegatedQueries count", expectedDelegatedCount, plan.delegatedExpressions().size()); assertEquals("serializer call count", expectedDelegatedCount, serializer.callCount); assertEquals("serialized functions", expectedFunctions, serializer.serializedFunctions); @@ -379,6 +395,26 @@ private void assertDelegationResult( // No annotation markers should survive stripping assertDoesntContainOperators(dfConvertor.shardScanFragment, ANNOTATION_MARKERS); + + // Instruction assertions: delegation plans must have SHARD_SCAN + FILTER_DELEGATION_FOR_INDEX + if (expectedDelegatedCount > 0) { + assertTrue( + "delegation plan must have FILTER_DELEGATION_FOR_INDEX instruction", + plan.instructions().stream().anyMatch(node -> node.type() == InstructionType.SETUP_FILTER_DELEGATION_FOR_INDEX) + ); + FilterDelegationInstructionNode filterInstruction = (FilterDelegationInstructionNode) plan.instructions() + .stream() + .filter(node -> node.type() == InstructionType.SETUP_FILTER_DELEGATION_FOR_INDEX) + .findFirst() + .orElseThrow(); + assertEquals("delegatedPredicateCount in instruction", expectedDelegatedCount, filterInstruction.getDelegatedPredicateCount()); + assertEquals( + "delegatedExpressions in instruction must match plan", + plan.delegatedExpressions().size(), + filterInstruction.getDelegatedQueries().size() + ); + assertEquals("treeShape in instruction", expectedTreeShape, filterInstruction.getTreeShape()); + } } // ---- Single predicate ---- @@ -389,7 +425,7 @@ public void testSingleDelegatedPredicate() { RecordingSerializer serializer = new RecordingSerializer(); QueryDAG dag = buildSingleFieldDelegationDag(makeFullTextCall(MATCH_PHRASE_FUNCTION, 0, "hello world"), dfConvertor, serializer); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); - assertDelegationResult(plan, dfConvertor, serializer, 1, true, false, List.of("MATCH_PHRASE")); + assertDelegationResult(plan, dfConvertor, serializer, 1, true, false, List.of("MATCH_PHRASE"), FilterTreeShape.CONJUNCTIVE); } /** Single native equals — no delegation, empty delegatedQueries. */ @@ -398,7 +434,7 @@ public void testSingleNativePredicate() { RecordingSerializer serializer = new RecordingSerializer(); QueryDAG dag = buildTwoFieldDelegationDag(makeEquals(0, SqlTypeName.INTEGER, 200), dfConvertor, serializer); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); - assertDelegationResult(plan, dfConvertor, serializer, 0, false, true, List.of()); + assertDelegationResult(plan, dfConvertor, serializer, 0, false, true, List.of(), FilterTreeShape.NO_DELEGATION); } // ---- AND conditions ---- @@ -413,7 +449,7 @@ public void testAndNativeAndDelegated() { serializer ); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); - assertDelegationResult(plan, dfConvertor, serializer, 1, true, true, List.of("MATCH_PHRASE")); + assertDelegationResult(plan, dfConvertor, serializer, 1, true, true, List.of("MATCH_PHRASE"), FilterTreeShape.CONJUNCTIVE); } /** AND(delegated, delegated) — both replaced, two entries in delegatedQueries. */ @@ -426,7 +462,16 @@ public void testAndTwoDelegated() { serializer ); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); - assertDelegationResult(plan, dfConvertor, serializer, 2, true, false, List.of("MATCH_PHRASE", "FUZZY")); + assertDelegationResult( + plan, + dfConvertor, + serializer, + 2, + true, + false, + List.of("MATCH_PHRASE", "FUZZY"), + FilterTreeShape.CONJUNCTIVE + ); } // ---- OR conditions ---- @@ -445,7 +490,16 @@ public void testOrNativeAndDelegated() { serializer ); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); - assertDelegationResult(plan, dfConvertor, serializer, 1, true, true, List.of("MATCH_PHRASE")); + assertDelegationResult( + plan, + dfConvertor, + serializer, + 1, + true, + true, + List.of("MATCH_PHRASE"), + FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION + ); assertTrue("OR structure should be preserved", RelOptUtil.toString(dfConvertor.shardScanFragment).contains("OR")); } @@ -467,7 +521,7 @@ public void testInterleavedAndOrNot() { RexNode condition = makeAnd(makeEquals(0, SqlTypeName.INTEGER, 200), orClause); QueryDAG dag = buildTwoFieldDelegationDag(condition, dfConvertor, serializer); StagePlan plan = leafStage(dag).getPlanAlternatives().getFirst(); - assertDelegationResult(plan, dfConvertor, serializer, 2, true, true, List.of("MATCH_PHRASE", "FUZZY")); + assertDelegationResult(plan, dfConvertor, serializer, 2, true, true, List.of("MATCH_PHRASE", "FUZZY"), FilterTreeShape.CONJUNCTIVE); String strippedPlan = RelOptUtil.toString(dfConvertor.shardScanFragment); assertTrue("AND structure should be preserved", strippedPlan.contains("AND")); assertTrue("OR structure should be preserved", strippedPlan.contains("OR")); From 680cee8d1accaf692a456d902c55c1add3406398 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 5 May 2026 21:20:21 -0400 Subject: [PATCH 055/115] Update Jackson to 2.21.3 / 3.1.3 (#21493) Signed-off-by: Andriy Redko --- client/sniffer/licenses/jackson-core-3.1.2.jar.sha1 | 1 - client/sniffer/licenses/jackson-core-3.1.3.jar.sha1 | 1 + gradle/libs.versions.toml | 8 ++++---- libs/core/licenses/jackson-core-3.1.2.jar.sha1 | 1 - libs/core/licenses/jackson-core-3.1.3.jar.sha1 | 1 + libs/x-content/licenses/jackson-core-3.1.2.jar.sha1 | 1 - libs/x-content/licenses/jackson-core-3.1.3.jar.sha1 | 1 + .../licenses/jackson-dataformat-cbor-3.1.2.jar.sha1 | 1 - .../licenses/jackson-dataformat-cbor-3.1.3.jar.sha1 | 1 + .../licenses/jackson-dataformat-smile-3.1.2.jar.sha1 | 1 - .../licenses/jackson-dataformat-smile-3.1.3.jar.sha1 | 1 + .../licenses/jackson-dataformat-yaml-3.1.2.jar.sha1 | 1 - .../licenses/jackson-dataformat-yaml-3.1.3.jar.sha1 | 1 + .../licenses/jackson-databind-2.21.2.jar.sha1 | 1 - .../licenses/jackson-databind-2.21.3.jar.sha1 | 1 + .../licenses/jackson-datatype-jsr310-2.21.2.jar.sha1 | 1 - .../licenses/jackson-datatype-jsr310-2.21.3.jar.sha1 | 1 + .../licenses/jackson-databind-2.21.2.jar.sha1 | 1 - .../licenses/jackson-databind-2.21.3.jar.sha1 | 1 + .../crypto-kms/licenses/jackson-databind-2.21.2.jar.sha1 | 1 - .../crypto-kms/licenses/jackson-databind-2.21.3.jar.sha1 | 1 + .../licenses/jackson-databind-2.21.2.jar.sha1 | 1 - .../licenses/jackson-databind-2.21.3.jar.sha1 | 1 + .../licenses/jackson-databind-2.21.2.jar.sha1 | 1 - .../licenses/jackson-databind-2.21.3.jar.sha1 | 1 + .../licenses/jackson-databind-2.21.2.jar.sha1 | 1 - .../licenses/jackson-databind-2.21.3.jar.sha1 | 1 + .../licenses/jackson-dataformat-xml-2.21.2.jar.sha1 | 1 - .../licenses/jackson-dataformat-xml-2.21.3.jar.sha1 | 1 + .../licenses/jackson-datatype-jsr310-2.21.2.jar.sha1 | 1 - .../licenses/jackson-datatype-jsr310-2.21.3.jar.sha1 | 1 + .../jackson-module-jaxb-annotations-2.21.2.jar.sha1 | 1 - .../jackson-module-jaxb-annotations-2.21.3.jar.sha1 | 1 + .../licenses/jackson-databind-2.21.2.jar.sha1 | 1 - .../licenses/jackson-databind-2.21.3.jar.sha1 | 1 + .../licenses/jackson-databind-2.21.2.jar.sha1 | 1 - .../licenses/jackson-databind-2.21.3.jar.sha1 | 1 + .../licenses/jackson-datatype-jdk8-2.21.2.jar.sha1 | 1 - .../licenses/jackson-datatype-jdk8-2.21.3.jar.sha1 | 1 + .../licenses/jackson-databind-2.21.3.jar.sha1 | 1 + .../licenses/jackson-databind-2.21.2.jar.sha1 | 1 - .../licenses/jackson-databind-2.21.3.jar.sha1 | 1 + server/licenses/jackson-core-2.21.2.jar.sha1 | 1 - server/licenses/jackson-core-2.21.3.jar.sha1 | 1 + server/licenses/jackson-core-3.1.2.jar.sha1 | 1 - server/licenses/jackson-core-3.1.3.jar.sha1 | 1 + server/licenses/jackson-dataformat-cbor-2.21.2.jar.sha1 | 1 - server/licenses/jackson-dataformat-cbor-2.21.3.jar.sha1 | 1 + server/licenses/jackson-dataformat-cbor-3.1.2.jar.sha1 | 1 - server/licenses/jackson-dataformat-cbor-3.1.3.jar.sha1 | 1 + server/licenses/jackson-dataformat-smile-2.21.2.jar.sha1 | 1 - server/licenses/jackson-dataformat-smile-2.21.3.jar.sha1 | 1 + server/licenses/jackson-dataformat-smile-3.1.2.jar.sha1 | 1 - server/licenses/jackson-dataformat-smile-3.1.3.jar.sha1 | 1 + server/licenses/jackson-dataformat-yaml-2.21.2.jar.sha1 | 1 - server/licenses/jackson-dataformat-yaml-2.21.3.jar.sha1 | 1 + server/licenses/jackson-dataformat-yaml-3.1.2.jar.sha1 | 1 - server/licenses/jackson-dataformat-yaml-3.1.3.jar.sha1 | 1 + 58 files changed, 33 insertions(+), 32 deletions(-) delete mode 100644 client/sniffer/licenses/jackson-core-3.1.2.jar.sha1 create mode 100644 client/sniffer/licenses/jackson-core-3.1.3.jar.sha1 delete mode 100644 libs/core/licenses/jackson-core-3.1.2.jar.sha1 create mode 100644 libs/core/licenses/jackson-core-3.1.3.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-core-3.1.2.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-core-3.1.3.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-3.1.2.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-3.1.3.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-smile-3.1.2.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-smile-3.1.3.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-3.1.2.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-3.1.3.jar.sha1 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.21.2.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.21.3.jar.sha1 delete mode 100644 modules/ingest-geoip/licenses/jackson-datatype-jsr310-2.21.2.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1 delete mode 100644 plugins/arrow-flight-rpc/licenses/jackson-databind-2.21.2.jar.sha1 create mode 100644 plugins/arrow-flight-rpc/licenses/jackson-databind-2.21.3.jar.sha1 delete mode 100644 plugins/crypto-kms/licenses/jackson-databind-2.21.2.jar.sha1 create mode 100644 plugins/crypto-kms/licenses/jackson-databind-2.21.3.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.21.2.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.21.3.jar.sha1 delete mode 100644 plugins/ingestion-kinesis/licenses/jackson-databind-2.21.2.jar.sha1 create mode 100644 plugins/ingestion-kinesis/licenses/jackson-databind-2.21.3.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-databind-2.21.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-databind-2.21.3.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-dataformat-xml-2.21.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-dataformat-xml-2.21.3.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-datatype-jsr310-2.21.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.21.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.21.3.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.21.2.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.21.3.jar.sha1 delete mode 100644 sandbox/libs/analytics-framework/licenses/jackson-databind-2.21.2.jar.sha1 create mode 100644 sandbox/libs/analytics-framework/licenses/jackson-databind-2.21.3.jar.sha1 delete mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.2.jar.sha1 create mode 100644 sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.3.jar.sha1 create mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.3.jar.sha1 delete mode 100644 sandbox/plugins/parquet-data-format/licenses/jackson-databind-2.21.2.jar.sha1 create mode 100644 sandbox/plugins/parquet-data-format/licenses/jackson-databind-2.21.3.jar.sha1 delete mode 100644 server/licenses/jackson-core-2.21.2.jar.sha1 create mode 100644 server/licenses/jackson-core-2.21.3.jar.sha1 delete mode 100644 server/licenses/jackson-core-3.1.2.jar.sha1 create mode 100644 server/licenses/jackson-core-3.1.3.jar.sha1 delete mode 100644 server/licenses/jackson-dataformat-cbor-2.21.2.jar.sha1 create mode 100644 server/licenses/jackson-dataformat-cbor-2.21.3.jar.sha1 delete mode 100644 server/licenses/jackson-dataformat-cbor-3.1.2.jar.sha1 create mode 100644 server/licenses/jackson-dataformat-cbor-3.1.3.jar.sha1 delete mode 100644 server/licenses/jackson-dataformat-smile-2.21.2.jar.sha1 create mode 100644 server/licenses/jackson-dataformat-smile-2.21.3.jar.sha1 delete mode 100644 server/licenses/jackson-dataformat-smile-3.1.2.jar.sha1 create mode 100644 server/licenses/jackson-dataformat-smile-3.1.3.jar.sha1 delete mode 100644 server/licenses/jackson-dataformat-yaml-2.21.2.jar.sha1 create mode 100644 server/licenses/jackson-dataformat-yaml-2.21.3.jar.sha1 delete mode 100644 server/licenses/jackson-dataformat-yaml-3.1.2.jar.sha1 create mode 100644 server/licenses/jackson-dataformat-yaml-3.1.3.jar.sha1 diff --git a/client/sniffer/licenses/jackson-core-3.1.2.jar.sha1 b/client/sniffer/licenses/jackson-core-3.1.2.jar.sha1 deleted file mode 100644 index 3a47314d227c2..0000000000000 --- a/client/sniffer/licenses/jackson-core-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d0da2e67ffb0b7cf5aba0436b315aa3eb3eb37ca \ No newline at end of file diff --git a/client/sniffer/licenses/jackson-core-3.1.3.jar.sha1 b/client/sniffer/licenses/jackson-core-3.1.3.jar.sha1 new file mode 100644 index 0000000000000..640b22d8ce4d3 --- /dev/null +++ b/client/sniffer/licenses/jackson-core-3.1.3.jar.sha1 @@ -0,0 +1 @@ +2f1dbeb81fe57c51e660534d3678003e514c1eb7 \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9ecbab7250953..1767c3567e958 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -9,10 +9,10 @@ bundled_jdk = "25.0.3+9" spatial4j = "0.7" jts = "1.15.0" jackson_annotations = "2.21" -jackson = "2.21.2" -jackson_databind = "2.21.2" -jackson3 = "3.1.2" -jackson3_databind = "3.1.2" +jackson = "2.21.3" +jackson_databind = "2.21.3" +jackson3 = "3.1.3" +jackson3_databind = "3.1.3" snakeyaml = "2.6" snakeyaml_engine = "3.0.1" icu4j = "77.1" diff --git a/libs/core/licenses/jackson-core-3.1.2.jar.sha1 b/libs/core/licenses/jackson-core-3.1.2.jar.sha1 deleted file mode 100644 index 3a47314d227c2..0000000000000 --- a/libs/core/licenses/jackson-core-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d0da2e67ffb0b7cf5aba0436b315aa3eb3eb37ca \ No newline at end of file diff --git a/libs/core/licenses/jackson-core-3.1.3.jar.sha1 b/libs/core/licenses/jackson-core-3.1.3.jar.sha1 new file mode 100644 index 0000000000000..640b22d8ce4d3 --- /dev/null +++ b/libs/core/licenses/jackson-core-3.1.3.jar.sha1 @@ -0,0 +1 @@ +2f1dbeb81fe57c51e660534d3678003e514c1eb7 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-core-3.1.2.jar.sha1 b/libs/x-content/licenses/jackson-core-3.1.2.jar.sha1 deleted file mode 100644 index 3a47314d227c2..0000000000000 --- a/libs/x-content/licenses/jackson-core-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d0da2e67ffb0b7cf5aba0436b315aa3eb3eb37ca \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-core-3.1.3.jar.sha1 b/libs/x-content/licenses/jackson-core-3.1.3.jar.sha1 new file mode 100644 index 0000000000000..640b22d8ce4d3 --- /dev/null +++ b/libs/x-content/licenses/jackson-core-3.1.3.jar.sha1 @@ -0,0 +1 @@ +2f1dbeb81fe57c51e660534d3678003e514c1eb7 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-3.1.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-3.1.2.jar.sha1 deleted file mode 100644 index 4904926655c44..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-cbor-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -29d243064344c3ff89510c4f652e84980a468315 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-3.1.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-3.1.3.jar.sha1 new file mode 100644 index 0000000000000..6923a099bade7 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-cbor-3.1.3.jar.sha1 @@ -0,0 +1 @@ +d782414b2c8d2d1dee03bf841fe7d44d65cc03f0 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-3.1.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-3.1.2.jar.sha1 deleted file mode 100644 index 55fce143a09e6..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-smile-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6b6c5b24eb9a1f1e2cbc24130003f47e31a35c0a \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-3.1.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-3.1.3.jar.sha1 new file mode 100644 index 0000000000000..bc5f98db973a3 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-smile-3.1.3.jar.sha1 @@ -0,0 +1 @@ +af978473a4123fc8f31a3945e8324ae1d8f85057 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-3.1.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-3.1.2.jar.sha1 deleted file mode 100644 index 7feb58a4d7574..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-yaml-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d782286464620deeed1f1733a960e7fd4c179df \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-3.1.3.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-3.1.3.jar.sha1 new file mode 100644 index 0000000000000..1ab423427d0be --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-yaml-3.1.3.jar.sha1 @@ -0,0 +1 @@ +6b63a5a53c5e5f0db77e8ba2e3eb6942635e81b7 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.21.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.21.2.jar.sha1 deleted file mode 100644 index 52686081905c0..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-databind-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71ab8ff75b4fd74afdee0004173fdd15de1d6a28 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.21.3.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..0f1ca8bfdace0 --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-databind-2.21.3.jar.sha1 @@ -0,0 +1 @@ +aa7ccec161c275f3e6332666ab758916f3120714 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-datatype-jsr310-2.21.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-datatype-jsr310-2.21.2.jar.sha1 deleted file mode 100644 index bff6df2dc56c2..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-datatype-jsr310-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -65b0cef8d997561541b7db6bbb1f6d42913b60e0 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1 b/modules/ingest-geoip/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..2d820120f91fb --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1 @@ -0,0 +1 @@ +a0958ebdaba836d31e5462ebc37b6349a0725ff9 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/jackson-databind-2.21.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/jackson-databind-2.21.2.jar.sha1 deleted file mode 100644 index 52686081905c0..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/jackson-databind-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71ab8ff75b4fd74afdee0004173fdd15de1d6a28 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/jackson-databind-2.21.3.jar.sha1 b/plugins/arrow-flight-rpc/licenses/jackson-databind-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..0f1ca8bfdace0 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/jackson-databind-2.21.3.jar.sha1 @@ -0,0 +1 @@ +aa7ccec161c275f3e6332666ab758916f3120714 \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/jackson-databind-2.21.2.jar.sha1 b/plugins/crypto-kms/licenses/jackson-databind-2.21.2.jar.sha1 deleted file mode 100644 index 52686081905c0..0000000000000 --- a/plugins/crypto-kms/licenses/jackson-databind-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71ab8ff75b4fd74afdee0004173fdd15de1d6a28 \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/jackson-databind-2.21.3.jar.sha1 b/plugins/crypto-kms/licenses/jackson-databind-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..0f1ca8bfdace0 --- /dev/null +++ b/plugins/crypto-kms/licenses/jackson-databind-2.21.3.jar.sha1 @@ -0,0 +1 @@ +aa7ccec161c275f3e6332666ab758916f3120714 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.21.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.21.2.jar.sha1 deleted file mode 100644 index 52686081905c0..0000000000000 --- a/plugins/discovery-ec2/licenses/jackson-databind-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71ab8ff75b4fd74afdee0004173fdd15de1d6a28 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.21.3.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..0f1ca8bfdace0 --- /dev/null +++ b/plugins/discovery-ec2/licenses/jackson-databind-2.21.3.jar.sha1 @@ -0,0 +1 @@ +aa7ccec161c275f3e6332666ab758916f3120714 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/jackson-databind-2.21.2.jar.sha1 b/plugins/ingestion-kinesis/licenses/jackson-databind-2.21.2.jar.sha1 deleted file mode 100644 index 52686081905c0..0000000000000 --- a/plugins/ingestion-kinesis/licenses/jackson-databind-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71ab8ff75b4fd74afdee0004173fdd15de1d6a28 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/jackson-databind-2.21.3.jar.sha1 b/plugins/ingestion-kinesis/licenses/jackson-databind-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..0f1ca8bfdace0 --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/jackson-databind-2.21.3.jar.sha1 @@ -0,0 +1 @@ +aa7ccec161c275f3e6332666ab758916f3120714 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-databind-2.21.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.21.2.jar.sha1 deleted file mode 100644 index 52686081905c0..0000000000000 --- a/plugins/repository-azure/licenses/jackson-databind-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71ab8ff75b4fd74afdee0004173fdd15de1d6a28 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-databind-2.21.3.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..0f1ca8bfdace0 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-databind-2.21.3.jar.sha1 @@ -0,0 +1 @@ +aa7ccec161c275f3e6332666ab758916f3120714 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.21.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.21.2.jar.sha1 deleted file mode 100644 index 262cd88437fba..0000000000000 --- a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7aeeda1cef4980f51e0985e83519ec965c7a3fa6 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.21.3.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..002ed2c4b0cb2 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.21.3.jar.sha1 @@ -0,0 +1 @@ +e3bdcc80b645f1c8780b3b3583787f6019540fee \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.21.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.21.2.jar.sha1 deleted file mode 100644 index bff6df2dc56c2..0000000000000 --- a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -65b0cef8d997561541b7db6bbb1f6d42913b60e0 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..2d820120f91fb --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1 @@ -0,0 +1 @@ +a0958ebdaba836d31e5462ebc37b6349a0725ff9 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.21.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.21.2.jar.sha1 deleted file mode 100644 index 8432ed172c520..0000000000000 --- a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e432148db21fc721cfa2c4fa3af9f5edf5160353 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.21.3.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..6a5e6082726a9 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.21.3.jar.sha1 @@ -0,0 +1 @@ +97cfa86183734f8001d724a49dc8f03318c8179b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.21.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.21.2.jar.sha1 deleted file mode 100644 index 52686081905c0..0000000000000 --- a/plugins/repository-s3/licenses/jackson-databind-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71ab8ff75b4fd74afdee0004173fdd15de1d6a28 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.21.3.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..0f1ca8bfdace0 --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-databind-2.21.3.jar.sha1 @@ -0,0 +1 @@ +aa7ccec161c275f3e6332666ab758916f3120714 \ No newline at end of file diff --git a/sandbox/libs/analytics-framework/licenses/jackson-databind-2.21.2.jar.sha1 b/sandbox/libs/analytics-framework/licenses/jackson-databind-2.21.2.jar.sha1 deleted file mode 100644 index 52686081905c0..0000000000000 --- a/sandbox/libs/analytics-framework/licenses/jackson-databind-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71ab8ff75b4fd74afdee0004173fdd15de1d6a28 \ No newline at end of file diff --git a/sandbox/libs/analytics-framework/licenses/jackson-databind-2.21.3.jar.sha1 b/sandbox/libs/analytics-framework/licenses/jackson-databind-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..0f1ca8bfdace0 --- /dev/null +++ b/sandbox/libs/analytics-framework/licenses/jackson-databind-2.21.3.jar.sha1 @@ -0,0 +1 @@ +aa7ccec161c275f3e6332666ab758916f3120714 \ No newline at end of file diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.2.jar.sha1 b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.2.jar.sha1 deleted file mode 100644 index ee9b80f634c10..0000000000000 --- a/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0222d829578930f27afa8d9b6842393824e2eb38 \ No newline at end of file diff --git a/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.3.jar.sha1 b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..eaa58d13290e8 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/licenses/jackson-datatype-jdk8-2.21.3.jar.sha1 @@ -0,0 +1 @@ +d43500553adcacf036f24eeb8c91f2a222b7176c \ No newline at end of file diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.3.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..0f1ca8bfdace0 --- /dev/null +++ b/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.3.jar.sha1 @@ -0,0 +1 @@ +aa7ccec161c275f3e6332666ab758916f3120714 \ No newline at end of file diff --git a/sandbox/plugins/parquet-data-format/licenses/jackson-databind-2.21.2.jar.sha1 b/sandbox/plugins/parquet-data-format/licenses/jackson-databind-2.21.2.jar.sha1 deleted file mode 100644 index 52686081905c0..0000000000000 --- a/sandbox/plugins/parquet-data-format/licenses/jackson-databind-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71ab8ff75b4fd74afdee0004173fdd15de1d6a28 \ No newline at end of file diff --git a/sandbox/plugins/parquet-data-format/licenses/jackson-databind-2.21.3.jar.sha1 b/sandbox/plugins/parquet-data-format/licenses/jackson-databind-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..0f1ca8bfdace0 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/licenses/jackson-databind-2.21.3.jar.sha1 @@ -0,0 +1 @@ +aa7ccec161c275f3e6332666ab758916f3120714 \ No newline at end of file diff --git a/server/licenses/jackson-core-2.21.2.jar.sha1 b/server/licenses/jackson-core-2.21.2.jar.sha1 deleted file mode 100644 index b7afc1b02a505..0000000000000 --- a/server/licenses/jackson-core-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7d11eac823392f28d8ee7bda77eaadfccbab83e5 \ No newline at end of file diff --git a/server/licenses/jackson-core-2.21.3.jar.sha1 b/server/licenses/jackson-core-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..5f13f1a28c200 --- /dev/null +++ b/server/licenses/jackson-core-2.21.3.jar.sha1 @@ -0,0 +1 @@ +3358e9345dd0f2537c47bee152c0377df6c81ad5 \ No newline at end of file diff --git a/server/licenses/jackson-core-3.1.2.jar.sha1 b/server/licenses/jackson-core-3.1.2.jar.sha1 deleted file mode 100644 index 3a47314d227c2..0000000000000 --- a/server/licenses/jackson-core-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d0da2e67ffb0b7cf5aba0436b315aa3eb3eb37ca \ No newline at end of file diff --git a/server/licenses/jackson-core-3.1.3.jar.sha1 b/server/licenses/jackson-core-3.1.3.jar.sha1 new file mode 100644 index 0000000000000..640b22d8ce4d3 --- /dev/null +++ b/server/licenses/jackson-core-3.1.3.jar.sha1 @@ -0,0 +1 @@ +2f1dbeb81fe57c51e660534d3678003e514c1eb7 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-cbor-2.21.2.jar.sha1 b/server/licenses/jackson-dataformat-cbor-2.21.2.jar.sha1 deleted file mode 100644 index d3c2ccfb308f6..0000000000000 --- a/server/licenses/jackson-dataformat-cbor-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0dd1621412ece3c25b7293e707a18ab49ed4e8cf \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-cbor-2.21.3.jar.sha1 b/server/licenses/jackson-dataformat-cbor-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..1d2ad6b0d678e --- /dev/null +++ b/server/licenses/jackson-dataformat-cbor-2.21.3.jar.sha1 @@ -0,0 +1 @@ +418e133c66e74a1a8b4b1b50eb2560918064c040 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-cbor-3.1.2.jar.sha1 b/server/licenses/jackson-dataformat-cbor-3.1.2.jar.sha1 deleted file mode 100644 index 4904926655c44..0000000000000 --- a/server/licenses/jackson-dataformat-cbor-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -29d243064344c3ff89510c4f652e84980a468315 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-cbor-3.1.3.jar.sha1 b/server/licenses/jackson-dataformat-cbor-3.1.3.jar.sha1 new file mode 100644 index 0000000000000..6923a099bade7 --- /dev/null +++ b/server/licenses/jackson-dataformat-cbor-3.1.3.jar.sha1 @@ -0,0 +1 @@ +d782414b2c8d2d1dee03bf841fe7d44d65cc03f0 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-smile-2.21.2.jar.sha1 b/server/licenses/jackson-dataformat-smile-2.21.2.jar.sha1 deleted file mode 100644 index 745ed24ff6f32..0000000000000 --- a/server/licenses/jackson-dataformat-smile-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -25ac9abdb48555a92ee0a0be0188d5d6f9acc5d5 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-smile-2.21.3.jar.sha1 b/server/licenses/jackson-dataformat-smile-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..52ccbfa235688 --- /dev/null +++ b/server/licenses/jackson-dataformat-smile-2.21.3.jar.sha1 @@ -0,0 +1 @@ +eeede5d065d36d315cc709867af414fe60a70653 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-smile-3.1.2.jar.sha1 b/server/licenses/jackson-dataformat-smile-3.1.2.jar.sha1 deleted file mode 100644 index 55fce143a09e6..0000000000000 --- a/server/licenses/jackson-dataformat-smile-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6b6c5b24eb9a1f1e2cbc24130003f47e31a35c0a \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-smile-3.1.3.jar.sha1 b/server/licenses/jackson-dataformat-smile-3.1.3.jar.sha1 new file mode 100644 index 0000000000000..bc5f98db973a3 --- /dev/null +++ b/server/licenses/jackson-dataformat-smile-3.1.3.jar.sha1 @@ -0,0 +1 @@ +af978473a4123fc8f31a3945e8324ae1d8f85057 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-yaml-2.21.2.jar.sha1 b/server/licenses/jackson-dataformat-yaml-2.21.2.jar.sha1 deleted file mode 100644 index fb6e6c57c2656..0000000000000 --- a/server/licenses/jackson-dataformat-yaml-2.21.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -61694c28cd9661c97cf160c9858ec9658360ae71 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-yaml-2.21.3.jar.sha1 b/server/licenses/jackson-dataformat-yaml-2.21.3.jar.sha1 new file mode 100644 index 0000000000000..1437db26cf0cb --- /dev/null +++ b/server/licenses/jackson-dataformat-yaml-2.21.3.jar.sha1 @@ -0,0 +1 @@ +400fe3e019f87353512e1fec1c4cd61653456676 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-yaml-3.1.2.jar.sha1 b/server/licenses/jackson-dataformat-yaml-3.1.2.jar.sha1 deleted file mode 100644 index 7feb58a4d7574..0000000000000 --- a/server/licenses/jackson-dataformat-yaml-3.1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d782286464620deeed1f1733a960e7fd4c179df \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-yaml-3.1.3.jar.sha1 b/server/licenses/jackson-dataformat-yaml-3.1.3.jar.sha1 new file mode 100644 index 0000000000000..1ab423427d0be --- /dev/null +++ b/server/licenses/jackson-dataformat-yaml-3.1.3.jar.sha1 @@ -0,0 +1 @@ +6b63a5a53c5e5f0db77e8ba2e3eb6942635e81b7 \ No newline at end of file From e93b56d8faf50b8eb16f6cade71920a5fe704e1e Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 6 May 2026 01:17:53 -0700 Subject: [PATCH 056/115] Add PPL operator support: 19 comparison / arithmetic / logical operators (#21504) Wires the following PPL operators to DataFusion via the analytics-engine route: Comparison: =, !=, <, <=, >, >=, between, in, like, ilike Logical: and, or, not, xor Arithmetic: +, -, *, /, % Signed-off-by: Marc Handalian --- sandbox/libs/analytics-framework/build.gradle | 1 + .../spi/StdOperatorRewriteAdapter.java | 74 +++++ .../spi/StdOperatorRewriteAdapterTests.java | 108 ++++++ .../DataFusionAnalyticsBackendPlugin.java | 38 ++- .../DataFusionFragmentConvertor.java | 14 +- .../be/datafusion/DataFusionPlugin.java | 3 +- .../opensearch/be/datafusion/LikeAdapter.java | 41 +++ .../opensearch_scalar_functions.yaml | 22 ++ .../be/datafusion/LikeAdapterTests.java | 73 ++++ .../licenses/jackson-databind-2.21.3.jar.sha1 | 1 - .../analytics/qa/OperatorCommandIT.java | 311 ++++++++++++++++++ 11 files changed, 679 insertions(+), 7 deletions(-) create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/StdOperatorRewriteAdapter.java create mode 100644 sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/StdOperatorRewriteAdapterTests.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/LikeAdapter.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/LikeAdapterTests.java delete mode 100644 sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.3.jar.sha1 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/OperatorCommandIT.java diff --git a/sandbox/libs/analytics-framework/build.gradle b/sandbox/libs/analytics-framework/build.gradle index 3e9c18d648ecf..af29a627a1b0f 100644 --- a/sandbox/libs/analytics-framework/build.gradle +++ b/sandbox/libs/analytics-framework/build.gradle @@ -21,6 +21,7 @@ configurations { compileClasspath { exclude group: 'com.google.guava' } } sourceSets.main.compileClasspath += configurations.calciteCompile +sourceSets.test.compileClasspath += configurations.calciteCompile dependencies { // Guava — required at compile time because Calcite base classes expose guava types. diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/StdOperatorRewriteAdapter.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/StdOperatorRewriteAdapter.java new file mode 100644 index 0000000000000..c421db5ffa465 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/StdOperatorRewriteAdapter.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlOperator; + +import java.util.List; + +/** + * Rewrites a {@link RexCall} whose operator is a PPL / library-specific UDF to the equivalent + * {@link org.apache.calcite.sql.fun.SqlStdOperatorTable SqlStdOperatorTable} operator. Used to + * normalize PPL emissions so Isthmus's built-in {@code FunctionMappings.SCALAR_SIGS} can resolve + * them to the Substrait default extension catalog. + * + *

      Examples: + *

        + *
      • PPL's {@code DIVIDE} UDF ({@code PPLBuiltinOperators.DIVIDE}, a {@code SqlFunction} + * named "DIVIDE") → {@code SqlStdOperatorTable.DIVIDE} → substrait {@code divide}.
        • + *
      • PPL's {@code MOD} UDF → {@code SqlStdOperatorTable.MOD} → substrait {@code modulus}.
        • + *
      + * + *

      Adapter-level rewriting (rather than extending Isthmus's {@code ADDITIONAL_SCALAR_SIGS}) + * keeps the rewrite scoped to a single backend registration and avoids cross-cutting changes + * to Isthmus. The rewrite preserves operand order and result type. + * + * @opensearch.internal + */ +public class StdOperatorRewriteAdapter implements ScalarFunctionAdapter { + + /** Canonical Calcite operator this adapter substitutes in. */ + private final SqlOperator target; + + /** + * Operator name we expect to rewrite. Matching on name (case-insensitive) guards against + * applying the rewrite when the call already uses the target operator — an adapter is + * keyed by {@link org.opensearch.analytics.spi.ScalarFunction} which can map to either + * the PPL UDF or the std operator depending on how the call was constructed upstream. + */ + private final String expectedName; + + /** + * @param expectedName case-insensitive match against {@code call.getOperator().getName()}; + * if the call already uses {@code target}, the rewrite is a no-op. + * @param target the {@code SqlStdOperatorTable} (or other Isthmus-mapped) operator + * to substitute in. + */ + public StdOperatorRewriteAdapter(String expectedName, SqlOperator target) { + this.expectedName = expectedName; + this.target = target; + } + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + // Already the target operator — e.g. PLUS arrived via SqlStdOperatorTable.PLUS. No-op. + if (original.getOperator() == target) { + return original; + } + String actualName = original.getOperator().getName(); + if (actualName == null || !actualName.equalsIgnoreCase(expectedName)) { + return original; + } + // Re-construct with the standard operator, preserving operands and result type. + return cluster.getRexBuilder().makeCall(original.getType(), target, original.getOperands()); + } +} diff --git a/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/StdOperatorRewriteAdapterTests.java b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/StdOperatorRewriteAdapterTests.java new file mode 100644 index 0000000000000..ed4775ea814c8 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/StdOperatorRewriteAdapterTests.java @@ -0,0 +1,108 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.volcano.VolcanoPlanner; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +/** + * Unit tests for {@link StdOperatorRewriteAdapter} — verifies that PPL-emitted UDF calls + * (e.g. a {@code SqlFunction} named "DIVIDE") are rewritten to the matching + * {@link SqlStdOperatorTable} operator so Isthmus's {@code FunctionMappings.SCALAR_SIGS} + * can map them to the Substrait default extension catalog. + */ +public class StdOperatorRewriteAdapterTests extends OpenSearchTestCase { + + private final RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + private final RexBuilder rexBuilder = new RexBuilder(typeFactory); + private final RelOptCluster cluster = RelOptCluster.create(new VolcanoPlanner(), rexBuilder); + + private SqlFunction pplUdf(String name) { + return new SqlFunction( + name, + SqlKind.OTHER_FUNCTION, + ReturnTypes.ARG0_NULLABLE, + null, + OperandTypes.NUMERIC_NUMERIC, + SqlFunctionCategory.USER_DEFINED_FUNCTION + ); + } + + public void testRewritesPplDivideToSqlStdDivide() { + SqlFunction pplDivide = pplUdf("DIVIDE"); + RexNode a = rexBuilder.makeLiteral(2L, typeFactory.createSqlType(SqlTypeName.BIGINT), false); + RexNode b = rexBuilder.makeLiteral(4L, typeFactory.createSqlType(SqlTypeName.BIGINT), false); + RexCall original = (RexCall) rexBuilder.makeCall(pplDivide, List.of(a, b)); + + StdOperatorRewriteAdapter adapter = new StdOperatorRewriteAdapter("DIVIDE", SqlStdOperatorTable.DIVIDE); + RexNode adapted = adapter.adapt(original, List.of(), cluster); + + assertTrue("Adapter should return a RexCall", adapted instanceof RexCall); + RexCall rewrite = (RexCall) adapted; + assertSame("Operator should be SqlStdOperatorTable.DIVIDE", SqlStdOperatorTable.DIVIDE, rewrite.getOperator()); + assertEquals("Operand count preserved", 2, rewrite.getOperands().size()); + assertEquals("First operand preserved", 2L, ((RexLiteral) rewrite.getOperands().get(0)).getValueAs(Long.class).longValue()); + assertEquals("Second operand preserved", 4L, ((RexLiteral) rewrite.getOperands().get(1)).getValueAs(Long.class).longValue()); + } + + public void testRewritesPplModToSqlStdMod() { + SqlFunction pplMod = pplUdf("MOD"); + RexNode a = rexBuilder.makeLiteral(10L, typeFactory.createSqlType(SqlTypeName.BIGINT), false); + RexNode b = rexBuilder.makeLiteral(3L, typeFactory.createSqlType(SqlTypeName.BIGINT), false); + RexCall original = (RexCall) rexBuilder.makeCall(pplMod, List.of(a, b)); + + StdOperatorRewriteAdapter adapter = new StdOperatorRewriteAdapter("MOD", SqlStdOperatorTable.MOD); + RexNode adapted = adapter.adapt(original, List.of(), cluster); + + assertTrue("Adapter should return a RexCall", adapted instanceof RexCall); + assertSame("Operator should be SqlStdOperatorTable.MOD", SqlStdOperatorTable.MOD, ((RexCall) adapted).getOperator()); + } + + public void testNoRewriteWhenAlreadyStdOperator() { + RexNode a = rexBuilder.makeLiteral(2L, typeFactory.createSqlType(SqlTypeName.BIGINT), false); + RexNode b = rexBuilder.makeLiteral(4L, typeFactory.createSqlType(SqlTypeName.BIGINT), false); + RexCall original = (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.DIVIDE, List.of(a, b)); + + StdOperatorRewriteAdapter adapter = new StdOperatorRewriteAdapter("DIVIDE", SqlStdOperatorTable.DIVIDE); + RexNode adapted = adapter.adapt(original, List.of(), cluster); + + assertSame("Already-std call should be returned unchanged", original, adapted); + } + + public void testNoRewriteWhenOperatorNameMismatches() { + // Adapter registered for DIVIDE; call is for a differently-named UDF. + SqlFunction other = pplUdf("SOMETHING_ELSE"); + RexNode a = rexBuilder.makeLiteral(2L, typeFactory.createSqlType(SqlTypeName.BIGINT), false); + RexNode b = rexBuilder.makeLiteral(4L, typeFactory.createSqlType(SqlTypeName.BIGINT), false); + RexCall original = (RexCall) rexBuilder.makeCall(other, List.of(a, b)); + + SqlOperator target = SqlStdOperatorTable.DIVIDE; + StdOperatorRewriteAdapter adapter = new StdOperatorRewriteAdapter("DIVIDE", target); + RexNode adapted = adapter.adapt(original, List.of(), cluster); + + assertSame("Non-matching names should be returned unchanged", original, adapted); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 93b79e17c9acf..221838a98dece 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -8,6 +8,7 @@ package org.opensearch.be.datafusion; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.opensearch.analytics.spi.AggregateCapability; import org.opensearch.analytics.spi.AggregateFunction; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; @@ -23,6 +24,7 @@ import org.opensearch.analytics.spi.ScalarFunctionAdapter; import org.opensearch.analytics.spi.ScanCapability; import org.opensearch.analytics.spi.SearchExecEngineProvider; +import org.opensearch.analytics.spi.StdOperatorRewriteAdapter; import org.opensearch.index.engine.dataformat.DataFormatRegistry; import java.util.HashSet; @@ -52,6 +54,11 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP SUPPORTED_FIELD_TYPES.add(FieldType.TEXT); } + // Filter-side scalar functions DataFusion can evaluate natively. Comparisons, arithmetic + // (for `where x + y > 0`-style predicates), and Calcite's SARG fold (IN/BETWEEN/range-union) + // are all supported via the Substrait default extension catalog. AND/OR/NOT are recursed into + // by {@link OpenSearchFilterRule} structurally and never looked up here, but registering them + // keeps the capability declaration complete for auditing and symmetric with PROJECT_OPS. private static final Set STANDARD_FILTER_OPS = Set.of( ScalarFunction.EQUALS, ScalarFunction.NOT_EQUALS, @@ -63,7 +70,12 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP ScalarFunction.IS_NOT_NULL, ScalarFunction.IN, ScalarFunction.LIKE, - ScalarFunction.SARG_PREDICATE + ScalarFunction.SARG_PREDICATE, + ScalarFunction.PLUS, + ScalarFunction.MINUS, + ScalarFunction.TIMES, + ScalarFunction.DIVIDE, + ScalarFunction.MOD ); // Project-side scalar functions DataFusion can evaluate natively. Each entry corresponds to a @@ -74,7 +86,21 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP private static final Set STANDARD_PROJECT_OPS = Set.of( ScalarFunction.COALESCE, ScalarFunction.CEIL, - ScalarFunction.SARG_PREDICATE + ScalarFunction.SARG_PREDICATE, + // comparison / arithmetic / logical operators in eval-style projections. + ScalarFunction.EQUALS, + ScalarFunction.NOT_EQUALS, + ScalarFunction.GREATER_THAN, + ScalarFunction.GREATER_THAN_OR_EQUAL, + ScalarFunction.LESS_THAN, + ScalarFunction.LESS_THAN_OR_EQUAL, + ScalarFunction.IN, + ScalarFunction.LIKE, + ScalarFunction.PLUS, + ScalarFunction.MINUS, + ScalarFunction.TIMES, + ScalarFunction.DIVIDE, + ScalarFunction.MOD ); private static final Set AGG_FUNCTIONS = Set.of( @@ -147,7 +173,13 @@ public Set aggregateCapabilities() { @Override public Map scalarFunctionAdapters() { - return Map.of(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter(), ScalarFunction.SARG_PREDICATE, new SargAdapter()); + return Map.ofEntries( + Map.entry(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter()), + Map.entry(ScalarFunction.SARG_PREDICATE, new SargAdapter()), + Map.entry(ScalarFunction.DIVIDE, new StdOperatorRewriteAdapter("DIVIDE", SqlStdOperatorTable.DIVIDE)), + Map.entry(ScalarFunction.MOD, new StdOperatorRewriteAdapter("MOD", SqlStdOperatorTable.MOD)), + Map.entry(ScalarFunction.LIKE, new LikeAdapter()) + ); } }; } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index 101893acb229e..fe072c85a2e99 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -25,6 +25,7 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.schema.ColumnStrategy; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlLibraryOperators; import org.apache.calcite.util.ImmutableBitSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -77,9 +78,18 @@ public class DataFusionFragmentConvertor implements FragmentConvertor { private static final Logger LOGGER = LogManager.getLogger(DataFusionFragmentConvertor.class); - /** Maps DelegatedPredicateFunction to its Substrait extension name for Isthmus conversion. */ + /** + * Maps backend-specific Calcite operators to their Substrait extension names so Isthmus + * serializes them through our {@code SimpleExtension} catalog: + *

        + *
      • {@link DelegatedPredicateFunction} → {@code delegated_predicate} (delegation to a peer backend).
        • + *
      • {@link SqlLibraryOperators#ILIKE} → {@code ilike} (case-insensitive LIKE; resolved by + * DataFusion's substrait consumer to a case-insensitive {@code LikeExpr}).
        • + *
      + */ private static final List ADDITIONAL_SCALAR_SIGS = List.of( - FunctionMappings.s(DelegatedPredicateFunction.FUNCTION, DelegatedPredicateFunction.NAME) + FunctionMappings.s(DelegatedPredicateFunction.FUNCTION, DelegatedPredicateFunction.NAME), + FunctionMappings.s(SqlLibraryOperators.ILIKE, "ilike") ); private final SimpleExtension.ExtensionCollection extensions; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java index a9e9ecfe27d0e..fde6909f06513 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java @@ -149,7 +149,8 @@ private static SimpleExtension.ExtensionCollection loadSubstraitExtensions() { try { t.setContextClassLoader(DataFusionPlugin.class.getClassLoader()); SimpleExtension.ExtensionCollection delegationExtensions = SimpleExtension.load(List.of("/delegation_functions.yaml")); - return DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions); + SimpleExtension.ExtensionCollection scalarExtensions = SimpleExtension.load(List.of("/opensearch_scalar_functions.yaml")); + return DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions).merge(scalarExtensions); } finally { t.setContextClassLoader(previous); } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/LikeAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/LikeAdapter.java new file mode 100644 index 0000000000000..582aada5863d3 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/LikeAdapter.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; + +import java.util.List; + +/** + * Drops the 3rd escape operand from LIKE/ILIKE calls so Isthmus can serialize them via the + * 2-arg {@code like} / {@code ilike} Substrait signatures. Calcite's grammar always emits + * {@code LIKE(value, pattern, escape)} — the escape is almost always the default {@code '\'} + * and is not expressible in either signature. + * + *

      Case-insensitive semantics are preserved: PPL's default {@code like} lowers to + * {@link org.apache.calcite.sql.fun.SqlLibraryOperators#ILIKE}, which {@link DataFusionFragmentConvertor} + * maps to the custom {@code ilike} extension declared in {@code opensearch_scalar_functions.yaml}. + * The adapter therefore leaves the operator untouched and only normalizes arity. + * + * @opensearch.internal + */ +class LikeAdapter implements ScalarFunctionAdapter { + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + if (original.getOperands().size() != 3) { + return original; + } + return original.clone(original.getType(), original.getOperands().subList(0, 2)); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml new file mode 100644 index 0000000000000..4889f35c11b6a --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml @@ -0,0 +1,22 @@ +%YAML 1.2 +--- +urn: extension:org.opensearch:scalar_functions +scalar_functions: + - name: ilike + description: >- + Case-insensitive LIKE. Semantics match DataFusion's ILikeExpr + (datafusion-substrait maps the extension name "ilike" to a case-insensitive + LikeExpr). + impls: + - args: + - value: "varchar" + name: "input" + - value: "varchar" + name: "match" + return: boolean + - args: + - value: "string" + name: "input" + - value: "string" + name: "match" + return: boolean diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/LikeAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/LikeAdapterTests.java new file mode 100644 index 0000000000000..91519e7e15637 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/LikeAdapterTests.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.volcano.VolcanoPlanner; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +/** + * Unit tests for {@link LikeAdapter} — verifies the adapter drops Calcite's default + * 3rd (escape) operand so the call shape matches Substrait's 2-arg {@code like} / + * {@code ilike} signatures, while leaving the operator (LIKE vs ILIKE) unchanged. + */ +public class LikeAdapterTests extends OpenSearchTestCase { + + private final RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + private final RexBuilder rexBuilder = new RexBuilder(typeFactory); + private final RelOptCluster cluster = RelOptCluster.create(new VolcanoPlanner(), rexBuilder); + + public void testIlikeWithEscapeDropsEscapeAndKeepsIlikeOperator() { + RexNode field = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 0); + RexNode pattern = rexBuilder.makeLiteral("%e%"); + RexNode escape = rexBuilder.makeLiteral("\\"); + RexCall original = (RexCall) rexBuilder.makeCall(SqlLibraryOperators.ILIKE, List.of(field, pattern, escape)); + + RexCall adapted = (RexCall) new LikeAdapter().adapt(original, List.of(), cluster); + + assertSame( + "ILIKE operator must be preserved so Isthmus can serialize it as ilike", + SqlLibraryOperators.ILIKE, + adapted.getOperator() + ); + assertEquals("3rd (escape) operand must be dropped", 2, adapted.getOperands().size()); + } + + public void testLikeWithEscapeDropsEscapeAndKeepsLikeOperator() { + RexNode field = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 0); + RexNode pattern = rexBuilder.makeLiteral("%e%"); + RexNode escape = rexBuilder.makeLiteral("\\"); + RexCall original = (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LIKE, List.of(field, pattern, escape)); + + RexCall adapted = (RexCall) new LikeAdapter().adapt(original, List.of(), cluster); + + assertSame("LIKE operator must be preserved", SqlStdOperatorTable.LIKE, adapted.getOperator()); + assertEquals("3rd (escape) operand must be dropped", 2, adapted.getOperands().size()); + } + + public void testTwoArgLikeIsReturnedUnchanged() { + RexNode field = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 0); + RexNode pattern = rexBuilder.makeLiteral("%e%"); + RexCall original = (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LIKE, List.of(field, pattern)); + + RexNode adapted = new LikeAdapter().adapt(original, List.of(), cluster); + + assertSame("2-arg LIKE should pass through unchanged", original, adapted); + } +} diff --git a/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.3.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.3.jar.sha1 deleted file mode 100644 index 0f1ca8bfdace0..0000000000000 --- a/sandbox/plugins/analytics-engine/licenses/jackson-databind-2.21.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aa7ccec161c275f3e6332666ab758916f3120714 \ No newline at end of file diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/OperatorCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/OperatorCommandIT.java new file mode 100644 index 0000000000000..6f7816d3d26fb --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/OperatorCommandIT.java @@ -0,0 +1,311 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * (comparison / arithmetic / logical / concat) routed through the analytics-engine PPL path to DataFusion. + * + *

      Each test exercises one operator on the {@code calcs} dataset in both a filter + * ({@code where}) and a project ({@code eval}) position where applicable. Per-operator + * inputs are hand-picked so that filter row counts and eval cell values are small and + * stable under the dataset's current 17 rows. + * + *

      Covers: {@code =, !=, <, <=, >, >=, and, or, not, in, between (via >= AND <=), + * like, +, -, *, /, %, concat (||)}. XOR is the PPL {@code xor} function which + * lowers to {@code NOT_EQUALS} on booleans — validated in {@link #testXorViaNotEquals()}. + * ILIKE is deliberately omitted: Substrait's default extension catalog does not declare + * an {@code ilike} function, so Isthmus cannot serialize it to the shape DataFusion's + * Rust substrait consumer expects; see the Group F tracker for status. + */ +public class OperatorCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + // ── Comparisons (filter-side) ─────────────────────────────────────────────── + + public void testEqualsFilter() throws IOException { + assertRowCount("source=" + DATASET.indexName + " | where int0 = 8", 3); + } + + public void testNotEqualsFilter() throws IOException { + // 17 total rows. int0 = 8 matches 3 rows → 14 != 8 rows (nulls excluded by the operator). + assertRowCount("source=" + DATASET.indexName + " | where int0 != 8 | fields int0", 8); + } + + public void testLessThanFilter() throws IOException { + assertRowCount("source=" + DATASET.indexName + " | where int0 < 4 | fields int0", 2); + } + + public void testLessThanOrEqualFilter() throws IOException { + assertRowCount("source=" + DATASET.indexName + " | where int0 <= 4 | fields int0", 5); + } + + public void testGreaterThanFilter() throws IOException { + // int0 distribution in calcs: 1,3,4,4,4,7,8,8,8,10,11 (+6 nulls). int0 > 8 → 10,11. + assertRowCount("source=" + DATASET.indexName + " | where int0 > 8 | fields int0", 2); + } + + public void testGreaterThanOrEqualFilter() throws IOException { + // int0 >= 8 → 8,8,8,10,11 + assertRowCount("source=" + DATASET.indexName + " | where int0 >= 8 | fields int0", 5); + } + + // ── IN / BETWEEN (Sarg fold) ─────────────────────────────────────────────── + + public void testInListFilter() throws IOException { + // IN folds to SEARCH(Sarg[...]); SargAdapter expands before substrait. + assertRowCount("source=" + DATASET.indexName + " | where int0 in (1, 8) | fields int0", 4); + } + + public void testBetweenAsRangeFilter() throws IOException { + // PPL's between desugars to `>= AND <=`; Calcite folds contiguous ranges into a Sarg. + assertRowCount( + "source=" + DATASET.indexName + " | where int0 >= 4 and int0 <= 8 | fields int0", + 7 + ); + } + + // ── LIKE ───────────────────────────────────────────────────────────────── + + public void testLikeFilter() throws IOException { + // PPL's `like(field, pattern)` emits SqlLibraryOperators.ILIKE (PPL treats like as + // case-insensitive by default). Isthmus serializes ILIKE via the custom `ilike` + // extension declared in opensearch_scalar_functions.yaml; DataFusion's substrait + // consumer routes it to a case-insensitive LikeExpr. + // Pattern "%e%" matches every str2 containing an 'e'. + // str2 values: one,two,three,five,six,eight,nine,ten,eleven,twelve,fourteen,fifteen,sixteen + // Contains 'e': one(yes),three(yes),five(yes),eight(yes),nine(yes),ten(yes),eleven(yes), + // twelve(yes),fourteen(yes),fifteen(yes),sixteen(yes) → 11 rows (two,six exclude). + assertRowCount("source=" + DATASET.indexName + " | where like(str2, '%e%') | fields str2", 11); + } + + public void testLikeFilterIsCaseInsensitive() throws IOException { + // Guards against regression to the previous ILIKE→LIKE rewrite that silently dropped + // case-insensitivity. str0 values are all uppercase ("FURNITURE", "OFFICE SUPPLIES", + // "TECHNOLOGY"); a lowercase pattern would match 0 rows under case-sensitive LIKE. + // Under PPL's case-insensitive `like` (→ substrait `ilike`) it matches both FURNITURE rows. + assertRowCount("source=" + DATASET.indexName + " | where like(str0, '%furniture%') | fields str0", 2); + } + + // ── Logical (filter-side) ────────────────────────────────────────────────── + + public void testLogicalAndFilter() throws IOException { + // int0 > 4 AND int0 < 10 → 7,8,8,8 = 4 rows. + assertRowCount( + "source=" + DATASET.indexName + " | where int0 > 4 and int0 < 10 | fields int0", + 4 + ); + } + + public void testLogicalOrFilter() throws IOException { + assertRowCount( + "source=" + DATASET.indexName + " | where int0 = 1 or int0 = 10 | fields int0", + 2 + ); + } + + public void testLogicalNotFilter() throws IOException { + // NOT in PPL — `where not (x > y)` syntax. Negates the inner predicate structurally. + // int0 values: 1,3,4,4,4,7,8,8,8,10,11 (+6 nulls). NOT (int0 > 4) keeps 1,3,4,4,4 = 5 rows + // (SQL three-valued logic excludes NULLs — Calcite's NOT on a NULL stays NULL, which is + // truthy-equivalent to false for filtering). + assertRowCount( + "source=" + DATASET.indexName + " | where not (int0 > 4) | fields int0", + 5 + ); + } + + // ── XOR (PPL xor → NOT_EQUALS on BOOLEAN) ────────────────────────────────── + + public void testXorViaNotEquals() throws IOException { + // PPL's XOR is an infix boolean operator: `a XOR b`. It lowers to `a != b` on booleans + // (PPLFuncImpTable maps XOR → SqlStdOperatorTable.NOT_EQUALS with BOOLEAN type checker), + // so the same not_equal Substrait extension that powers `!=` handles this. Rows survive + // the filter only when bool0 and bool1 differ. + Map response = executePpl( + "source=" + DATASET.indexName + " | where bool0 xor bool1 | fields bool0, bool1" + ); + @SuppressWarnings("unchecked") + List> rows = (List>) response.get("rows"); + assertNotNull("xor query returned no rows block", rows); + // The calcs dataset contains rows where bool0 != bool1; assert the filter surfaces them. + assertTrue("xor should return at least 1 row, got " + rows.size(), !rows.isEmpty()); + for (List row : rows) { + assertFalse("bool0 xor bool1 row has equal values: " + row, row.get(0).equals(row.get(1))); + } + } + + // ── Arithmetic (project-side via eval + filter for verification) ─────────── + + public void testArithmeticPlusInEval() throws IOException { + // num0=12.3, num1=8.42 → sum=20.72. Select one row by key to keep expected values stable. + assertSingleRowField( + "source=" + DATASET.indexName + " | where key = 'key00' | eval s = num0 + num1 | fields s", + 20.72 + ); + } + + public void testArithmeticMinusInEval() throws IOException { + assertSingleRowField( + "source=" + DATASET.indexName + " | where key = 'key00' | eval d = num0 - num1 | fields d", + 3.88 + ); + } + + public void testArithmeticTimesInEval() throws IOException { + // 12.3 * 8.42 = 103.566 + assertSingleRowField( + "source=" + DATASET.indexName + " | where key = 'key00' | eval p = num0 * num1 | fields p", + 103.566 + ); + } + + // DIVIDE / MOD / CONCAT: PPL emits custom UDFs rather than the SqlStdOperatorTable entries + // that Isthmus's default SCALAR_SIGS covers. {@link StdOperatorRewriteAdapter} rewrites them + // to the standard Calcite operators before substrait serialisation so the default extension + // catalog's {@code divide} / {@code modulus} / {@code concat} entries resolve. + + public void testArithmeticDivideInEval() throws IOException { + // 12.3 / 8.42 ≈ 1.4608 — StdOperatorRewriteAdapter maps PPL DIVIDE UDF to + // SqlStdOperatorTable.DIVIDE, which Isthmus serialises via substrait `divide`. + assertSingleRowApprox( + "source=" + DATASET.indexName + " | where key = 'key00' | eval q = num0 / num1 | fields q", + 1.4608, + 1e-3 + ); + } + + public void testArithmeticModInEval() throws IOException { + // int3=8 for key00; 8 % 3 = 2 — MOD adapter → SqlStdOperatorTable.MOD → substrait `modulus`. + assertSingleRowField( + "source=" + DATASET.indexName + " | where key = 'key00' | eval r = int3 % 3 | fields r", + 2 + ); + } + + // ── Project-side comparisons: eval boolean result, filter by it ─────────── + + public void testEqualsInEvalProjection() throws IOException { + // eval produces a boolean, filter selects rows where it's true. + assertRowCount( + "source=" + DATASET.indexName + " | eval m = (int0 = 8) | where m = true | fields int0", + 3 + ); + } + + public void testAndInEvalProjection() throws IOException { + assertRowCount( + "source=" + DATASET.indexName + " | eval m = (int0 > 4) and (int0 < 10) | where m = true | fields int0", + 4 + ); + } + + public void testOrInEvalProjection() throws IOException { + assertRowCount( + "source=" + DATASET.indexName + " | eval m = (int0 = 1) or (int0 = 10) | where m = true | fields int0", + 2 + ); + } + + public void testNotInEvalProjection() throws IOException { + assertRowCount( + "source=" + DATASET.indexName + " | eval m = not (int0 > 4) | where m = true | fields int0", + 5 + ); + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + private void assertRowCount(String ppl, int expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> rows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, rows); + assertEquals("Row count mismatch for query: " + ppl, expected, rows.size()); + } + + private void assertSingleRowField(String ppl, Object expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> rows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, rows); + assertEquals("Expected exactly 1 row for query: " + ppl, 1, rows.size()); + Object actual = rows.get(0).get(0); + assertCellEquals("Cell value mismatch for query: " + ppl, expected, actual); + } + + private void assertSingleRowApprox(String ppl, double expected, double tolerance) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> rows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, rows); + assertEquals("Expected exactly 1 row for query: " + ppl, 1, rows.size()); + Object actual = rows.get(0).get(0); + assertNotNull("Cell is null for query: " + ppl, actual); + double actualD = ((Number) actual).doubleValue(); + if (Math.abs(actualD - expected) > tolerance) { + fail("Expected ~" + expected + " (tolerance " + tolerance + ") but got " + actualD + " for query: " + ppl); + } + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } + + /** + * Numeric-tolerant cell comparison: Integer/Long/Double arriving from JSON parsing + * may differ by concrete boxed type even when numerically equal. + */ + private static void assertCellEquals(String message, Object expected, Object actual) { + if (expected == null || actual == null) { + assertEquals(message, expected, actual); + return; + } + if (expected instanceof Number && actual instanceof Number) { + double e = ((Number) expected).doubleValue(); + double a = ((Number) actual).doubleValue(); + if (Double.compare(e, a) != 0) { + // Fall back to tolerance for floating-point arithmetic residue. + if (Math.abs(e - a) > 1e-9) { + fail(message + ": expected <" + expected + "> but was <" + actual + ">"); + } + } + return; + } + assertEquals(message, expected, actual); + } + + // Suppress the "unused" warning — Arrays.toString is retained for debug parity with + // other QA ITs in this package that dump row arrays on assertion failures. + @SuppressWarnings("unused") + private static String debugRows(List> rows) { + return Arrays.toString(rows.toArray()); + } +} From 144386c2f5ccce9f529c9ab9acf658be14687cc0 Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Wed, 6 May 2026 03:49:41 -0700 Subject: [PATCH 057/115] Add back getSearchSettings to fix japicmp (#21500) Signed-off-by: David Zane Co-authored-by: Andrew Ross --- .../opensearch/cluster/metadata/WorkloadGroup.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/WorkloadGroup.java b/server/src/main/java/org/opensearch/cluster/metadata/WorkloadGroup.java index ee77f412e4407..294c05ff17701 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/WorkloadGroup.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/WorkloadGroup.java @@ -207,6 +207,20 @@ public Settings getSettings() { return getMutableWorkloadGroupFragment().getSettings(); } + /** + * @deprecated Use {@link #getSettings()} instead. This method exists only for binary compatibility + * with 3.6.x clients and will be removed in a future major version. + */ + @Deprecated + public Map getSearchSettings() { + Settings s = getSettings(); + Map map = new HashMap<>(); + for (String key : s.keySet()) { + map.put(key, s.get(key)); + } + return map; + } + public String get_id() { return _id; } From 7961b6c669384641835ab3b97c40c9b12c89c1fb Mon Sep 17 00:00:00 2001 From: gaurav-amz Date: Wed, 6 May 2026 21:44:41 +0530 Subject: [PATCH 058/115] Add DynamicLimitPool for runtime memory pool limit changes (#21286) * Add DynamicLimitPool for runtime memory pool limit changes Replace GreedyMemoryPool with DynamicLimitPool in the DataFusion runtime. The pool stores its limit in an AtomicUsize shared with a DynamicLimitHandle, allowing the limit to be changed at runtime without restarting the process. Rust changes: - New memory.rs: DynamicLimitPool + DynamicLimitHandle with AtomicUsize limit - api.rs: Use DynamicLimitPool, store handle in DataFusionRuntime, add get_memory_pool_usage/limit and set_memory_pool_limit functions - ffm.rs: Export df_get_memory_pool_usage, df_get_memory_pool_limit, df_set_memory_pool_limit as extern C functions Java changes: - NativeBridge.java: Add FFM method handles and static methods for the 3 new memory pool functions - DataFusionService.java: Expose getMemoryPoolUsage(), getMemoryPoolLimit(), setMemoryPoolLimit() methods Runtime behavior: - Increasing limit takes effect immediately for new allocations - Decreasing limit only affects new allocations; existing reservations drain naturally as queries complete - Without set_limit calls, behavior is identical to GreedyMemoryPool Signed-off-by: Gaurav Singh * Wire memory pool limit as a dynamic cluster setting; fix review feedback - Mark datafusion.memory_pool_limit_bytes Dynamic and apply updates to the native DynamicLimitPool via a cluster settings consumer. - Load the limit inside try_grow's fetch_update closure; use saturating_add in grow; validate non-negative inputs in create_global_runtime. - Guard startup and shutdown races in the settings listener; drop the duplicate log line in DataFusionService. Signed-off-by: Gaurav Singh * Fix duplicate getSettings() from merge conflict Signed-off-by: Gaurav Singh * Apply spotless formatting to DataFusionPluginSettingsTests Signed-off-by: Gaurav Singh * Fixing build failures Signed-off-by: Gaurav Singh --------- Signed-off-by: Gaurav Singh --- .../rust/src/api.rs | 59 +++- .../rust/src/ffm.rs | 36 +++ .../rust/src/lib.rs | 1 + .../rust/src/memory.rs | 301 ++++++++++++++++++ .../be/datafusion/DataFusionPlugin.java | 43 ++- .../be/datafusion/DataFusionService.java | 26 ++ .../be/datafusion/nativelib/NativeBridge.java | 41 +++ .../DataFusionPluginSettingsTests.java | 63 ++++ .../be/datafusion/DynamicMemoryPoolTests.java | 105 ++++++ 9 files changed, 669 insertions(+), 6 deletions(-) create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/memory.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DynamicMemoryPoolTests.java diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs index d6bfab0c9fd37..815232ff4749d 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs @@ -44,7 +44,7 @@ use arrow_schema::ffi::FFI_ArrowSchema; use datafusion::common::DataFusionError; use datafusion::datasource::listing::ListingTableUrl; use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; -use datafusion::execution::memory_pool::{GreedyMemoryPool, TrackConsumersPool}; +use datafusion::execution::memory_pool::TrackConsumersPool; use datafusion::execution::runtime_env::RuntimeEnvBuilder; use datafusion::execution::cache::cache_manager::CacheManagerConfig; use datafusion::execution::RecordBatchStream; @@ -56,6 +56,7 @@ use futures::TryStreamExt; use crate::cross_rt_stream::CrossRtStream; use crate::custom_cache_manager::CustomCacheManager; use crate::local_executor::LocalSession; +use crate::memory::{DynamicLimitHandle, DynamicLimitPool}; use crate::partition_stream::PartitionStreamSender; use crate::query_memory_pool_tracker::QueryTrackingContext; use crate::runtime_manager::RuntimeManager; @@ -107,10 +108,12 @@ pub async fn create_object_metas( } /// Opaque runtime handle returned to the caller. -/// Contains the DataFusion RuntimeEnv (memory pool, disk spill, cache). +/// Contains the DataFusion RuntimeEnv (memory pool, disk spill, cache) +/// and a handle to change the memory pool limit at runtime. pub struct DataFusionRuntime { pub runtime_env: datafusion::execution::runtime_env::RuntimeEnv, pub custom_cache_manager: Option, + pub(crate) dynamic_limit_handle: DynamicLimitHandle, } /// Opaque shard view handle returned to the caller. @@ -129,12 +132,26 @@ pub fn create_global_runtime( spill_dir: &str, spill_limit: i64, ) -> Result { + if memory_pool_limit < 0 { + return Err(DataFusionError::Configuration(format!( + "memory_pool_limit must be non-negative, got {}", + memory_pool_limit + ))); + } + if spill_limit < 0 { + return Err(DataFusionError::Configuration(format!( + "spill_limit must be non-negative, got {}", + spill_limit + ))); + } + let disk_manager = DiskManagerBuilder::default() .with_max_temp_directory_size(spill_limit as u64) .with_mode(DiskManagerMode::Directories(vec![PathBuf::from(spill_dir)])); + let (dynamic_pool, dynamic_limit_handle) = DynamicLimitPool::new(memory_pool_limit as usize); let memory_pool = Arc::new(TrackConsumersPool::new( - GreedyMemoryPool::new(memory_pool_limit as usize), + dynamic_pool, NonZeroUsize::new(5).unwrap(), )); @@ -151,7 +168,7 @@ pub fn create_global_runtime( .with_cache_manager(cache_manager_config) .build()?; - let runtime = DataFusionRuntime { runtime_env, custom_cache_manager }; + let runtime = DataFusionRuntime { runtime_env, custom_cache_manager, dynamic_limit_handle }; Ok(Box::into_raw(Box::new(runtime)) as i64) } @@ -165,6 +182,40 @@ pub unsafe fn close_global_runtime(ptr: i64) { } } +// ---- Memory pool observability and dynamic limit ---- + +/// Returns the current memory pool usage in bytes. +/// +/// # Safety +/// `ptr` must be a valid pointer returned by `create_global_runtime`. +pub unsafe fn get_memory_pool_usage(ptr: i64) -> i64 { + let runtime = &*(ptr as *const DataFusionRuntime); + runtime.runtime_env.memory_pool.reserved() as i64 +} + +/// Returns the current memory pool limit in bytes. +/// +/// # Safety +/// `ptr` must be a valid pointer returned by `create_global_runtime`. +pub unsafe fn get_memory_pool_limit(ptr: i64) -> i64 { + let runtime = &*(ptr as *const DataFusionRuntime); + runtime.dynamic_limit_handle.limit() as i64 +} + +/// Sets the memory pool limit at runtime. Takes effect for new allocations only. +/// Returns an error if `new_limit` is negative. +/// +/// # Safety +/// `ptr` must be a valid pointer returned by `create_global_runtime`. +pub unsafe fn set_memory_pool_limit(ptr: i64, new_limit: i64) -> Result<(), String> { + if new_limit < 0 { + return Err(format!("Memory pool limit must be non-negative, got {}", new_limit)); + } + let runtime = &*(ptr as *const DataFusionRuntime); + runtime.dynamic_limit_handle.set_limit(new_limit as usize); + Ok(()) +} + /// Creates a native reader (ShardView) for the given path and files. /// /// Returns a heap-allocated pointer (as i64) to `ShardView`. diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index ce180c060f563..85404ff2f49ce 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -79,6 +79,42 @@ pub unsafe extern "C" fn df_close_global_runtime(ptr: i64) { api::close_global_runtime(ptr); } +// ---- Memory pool observability and dynamic limit ---- + +/// Returns current memory pool usage in bytes. +/// Java: MethodHandle(JAVA_LONG → JAVA_LONG) +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_get_memory_pool_usage(runtime_ptr: i64) -> i64 { + if runtime_ptr == 0 { + return Err("null runtime pointer".to_string()); + } + Ok(api::get_memory_pool_usage(runtime_ptr)) +} + +/// Returns current memory pool limit in bytes. +/// Java: MethodHandle(JAVA_LONG → JAVA_LONG) +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_get_memory_pool_limit(runtime_ptr: i64) -> i64 { + if runtime_ptr == 0 { + return Err("null runtime pointer".to_string()); + } + Ok(api::get_memory_pool_limit(runtime_ptr)) +} + +/// Sets the memory pool limit at runtime. Takes effect for new allocations only. +/// Java: MethodHandle(JAVA_LONG, JAVA_LONG → JAVA_LONG) +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_set_memory_pool_limit(runtime_ptr: i64, new_limit: i64) -> i64 { + if runtime_ptr == 0 { + return Err("null runtime pointer".to_string()); + } + api::set_memory_pool_limit(runtime_ptr, new_limit)?; + Ok(0) +} + #[ffm_safe] #[no_mangle] pub unsafe extern "C" fn df_create_reader( diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs index ee876888450c9..5a8d8f36355ad 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs @@ -23,6 +23,7 @@ pub mod indexed_executor; pub mod indexed_table; pub mod io; pub mod local_executor; +pub mod memory; pub mod partition_stream; pub mod query_executor; pub mod query_memory_pool_tracker; diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/memory.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/memory.rs new file mode 100644 index 0000000000000..134adf7e67c7c --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/memory.rs @@ -0,0 +1,301 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Dynamic memory pool for DataFusion query execution. +//! +//! Replaces DataFusion's `GreedyMemoryPool` with a pool whose limit can be +//! changed at runtime via a shared `DynamicLimitHandle`. The pool and handle +//! share an `Arc` for the limit, so `set_limit` is lock-free +//! and takes effect on the next `try_grow` call. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use datafusion::common::DataFusionError; +use datafusion::execution::memory_pool::{MemoryPool, MemoryReservation}; + +/// A `MemoryPool` whose limit can be changed at runtime. +/// +/// Behaviour matches `GreedyMemoryPool` exactly, except the limit is stored +/// in an `AtomicUsize` shared with a [`DynamicLimitHandle`]. +/// +/// - Increasing the limit takes effect immediately for new allocations. +/// - Decreasing the limit takes effect for new allocations only. +/// Existing reservations that exceed the new limit are NOT reclaimed. +#[derive(Debug)] +pub struct DynamicLimitPool { + used: AtomicUsize, + dynamic_limit: Arc, +} + +/// Handle to change the pool limit at runtime. +/// +/// Can be stored separately from the pool (which is consumed by +/// `TrackConsumersPool::new`). Both the pool and handle point to the +/// same `Arc`. +#[derive(Debug, Clone)] +pub struct DynamicLimitHandle { + limit: Arc, +} + +impl DynamicLimitHandle { + /// Atomically set a new limit. Takes effect on the next `try_grow`. + pub fn set_limit(&self, new_limit: usize) { + self.limit.store(new_limit, Ordering::Release); + } + + /// Read the current limit. + pub fn limit(&self) -> usize { + self.limit.load(Ordering::Acquire) + } +} + +impl DynamicLimitPool { + /// Create a new pool with the given initial limit. + /// Returns the pool and a handle to change the limit later. + pub fn new(initial_limit: usize) -> (Self, DynamicLimitHandle) { + let limit = Arc::new(AtomicUsize::new(initial_limit)); + let handle = DynamicLimitHandle { + limit: limit.clone(), + }; + let pool = Self { + used: AtomicUsize::new(0), + dynamic_limit: limit, + }; + (pool, handle) + } + + /// Read the current limit. + pub fn limit(&self) -> usize { + self.dynamic_limit.load(Ordering::Acquire) + } +} + +impl MemoryPool for DynamicLimitPool { + fn grow(&self, _reservation: &MemoryReservation, additional: usize) { + // `grow` is an infallible accounting call; the caller is responsible + // for pairing it with a successful `try_grow`, so under well-behaved + // callers `used + additional` cannot overflow `usize`. Use a saturating + // CAS loop so that a buggy caller (or a malicious `additional == usize::MAX`) + // cannot wrap the counter. + let _ = self.used.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| { + Some(used.saturating_add(additional)) + }); + } + + fn shrink(&self, _reservation: &MemoryReservation, shrink: usize) { + self.used.fetch_sub(shrink, Ordering::Relaxed); + } + + fn try_grow( + &self, + reservation: &MemoryReservation, + additional: usize, + ) -> Result<(), DataFusionError> { + // Load the limit inside the closure so every CAS retry sees the current + // value. A concurrent `set_limit` that raises the limit while we are + // spinning here should be honoured; loading once outside the closure + // would miss that update. + let dynamic_limit = &self.dynamic_limit; + self.used + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| { + let limit = dynamic_limit.load(Ordering::Acquire); + let new_used = used.checked_add(additional)?; + (new_used <= limit).then_some(new_used) + }) + .map_err(|used| { + // Re-load the limit for the error message. This can be a slightly newer + // value than the limit used in the decision above under a concurrent + // `set_limit`, but we prefer "most-recent" for operator visibility. The + // allocation decision itself was already made against a consistent + // snapshot inside the closure. + let limit = dynamic_limit.load(Ordering::Acquire); + DataFusionError::ResourcesExhausted(format!( + "Failed to allocate {} bytes for {} ({} already reserved) \ + — {} available out of {} (dynamic limit)", + additional, + reservation.consumer().name(), + reservation.size(), + limit.saturating_sub(used), + limit, + )) + })?; + Ok(()) + } + + fn reserved(&self) -> usize { + self.used.load(Ordering::Relaxed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::execution::memory_pool::MemoryConsumer; + + /// Build an `Arc` + handle for tests. + /// DataFusion 52+ `MemoryConsumer::register` takes `&Arc`, + /// so tests wrap the concrete pool once rather than repeating the cast + /// at every call site. + fn new_pool(limit: usize) -> (Arc, DynamicLimitHandle) { + let (pool, handle) = DynamicLimitPool::new(limit); + (Arc::new(pool), handle) + } + + #[test] + fn test_initial_limit() { + let (pool, handle) = new_pool(1024); + assert_eq!(handle.limit(), 1024); + assert_eq!(pool.reserved(), 0); + } + + #[test] + fn test_set_limit() { + let (_pool, handle) = new_pool(1024); + handle.set_limit(2048); + assert_eq!(handle.limit(), 2048); + } + + #[test] + fn test_try_grow_within_limit() { + let (pool, _handle) = new_pool(1024); + let consumer = MemoryConsumer::new("test"); + let mut reservation = consumer.register(&pool); + assert!(reservation.try_grow(512).is_ok()); + assert_eq!(pool.reserved(), 512); + } + + #[test] + fn test_try_grow_exceeds_limit() { + let (pool, _handle) = new_pool(1024); + let consumer = MemoryConsumer::new("test"); + let mut reservation = consumer.register(&pool); + assert!(reservation.try_grow(2048).is_err()); + assert_eq!(pool.reserved(), 0); + } + + #[test] + fn test_dynamic_limit_increase() { + let (pool, handle) = new_pool(1024); + let consumer = MemoryConsumer::new("test"); + let mut reservation = consumer.register(&pool); + + // Fails at 1024 limit + assert!(reservation.try_grow(2048).is_err()); + + // Increase limit + handle.set_limit(4096); + + // Now succeeds + assert!(reservation.try_grow(2048).is_ok()); + assert_eq!(pool.reserved(), 2048); + } + + #[test] + fn test_dynamic_limit_decrease_existing_reservations_kept() { + let (pool, handle) = new_pool(4096); + let consumer = MemoryConsumer::new("test"); + let mut reservation = consumer.register(&pool); + + // Reserve 2048 + assert!(reservation.try_grow(2048).is_ok()); + + // Decrease limit below current usage + handle.set_limit(1024); + + // Existing reservation is NOT reclaimed + assert_eq!(pool.reserved(), 2048); + + // But new allocations fail + let consumer2 = MemoryConsumer::new("test2"); + let mut reservation2 = consumer2.register(&pool); + assert!(reservation2.try_grow(1).is_err()); + } + + #[test] + fn test_try_grow_overflow_protection() { + let (pool, _handle) = new_pool(usize::MAX); + let consumer = MemoryConsumer::new("test"); + let mut reservation = consumer.register(&pool); + assert!(reservation.try_grow(1024).is_ok()); + // `additional == usize::MAX` would overflow `used + additional`. + // checked_add inside fetch_update must reject it cleanly. + assert!(reservation.try_grow(usize::MAX).is_err()); + assert_eq!(pool.reserved(), 1024); + } + + #[test] + fn test_grow_saturates_instead_of_wrapping() { + let (pool, _handle) = new_pool(1024); + // `grow` is infallible accounting — a buggy caller must not be able to + // wrap `used` back to zero by passing `usize::MAX`. `saturating_add` + // pins `used` at `usize::MAX` instead. + let consumer = MemoryConsumer::new("test"); + let reservation = consumer.register(&pool); + pool.grow(&reservation, usize::MAX); + assert_eq!(pool.reserved(), usize::MAX); + pool.grow(&reservation, 1); + assert_eq!(pool.reserved(), usize::MAX); + } + + #[test] + fn test_concurrent_set_limit_observed_by_try_grow() { + use std::sync::Barrier; + use std::thread; + + // Repeat to give the race a chance to surface. + for _ in 0..64 { + let (pool, handle) = new_pool(1024); + let barrier = Arc::new(Barrier::new(2)); + + let raiser = { + let handle = handle.clone(); + let barrier = barrier.clone(); + thread::spawn(move || { + barrier.wait(); + handle.set_limit(1 << 30); + }) + }; + + let allocator = { + let pool = pool.clone(); + let handle = handle.clone(); + let barrier = barrier.clone(); + thread::spawn(move || { + barrier.wait(); + let consumer = MemoryConsumer::new("race"); + let mut reservation = consumer.register(&pool); + // Retry until either allocation succeeds OR the handle reports + // the new limit is visible. The previous fixed-iteration loop + // flaked on fast runners because the allocator could exhaust + // its retries before the raiser thread's store completed. + // + // The atomic invariant under test: once `handle.limit() >= 2048` + // is observable, the very next `try_grow(2048)` MUST succeed + // (that's the Release/Acquire happens-before contract). If that + // final try_grow fails, that IS a real pool bug. + loop { + if reservation.try_grow(2048).is_ok() { + break; + } + if handle.limit() >= 2048 { + reservation + .try_grow(2048) + .expect("once handle.limit() reflects the raise, try_grow must succeed"); + break; + } + std::hint::spin_loop(); + } + }) + }; + + raiser.join().unwrap(); + allocator.join().unwrap(); + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java index fde6909f06513..d1d36c4a0b7b1 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java @@ -50,12 +50,18 @@ public class DataFusionPlugin extends Plugin implements SearchBackEndPlugin + * Dynamic: changes take effect for new allocations only. Existing reservations + * that exceed the new limit are not reclaimed — they drain naturally as queries complete. + */ public static final Setting DATAFUSION_MEMORY_POOL_LIMIT = Setting.longSetting( "datafusion.memory_pool_limit_bytes", Runtime.getRuntime().maxMemory() / 4, 0L, - Setting.Property.NodeScope + Setting.Property.NodeScope, + Setting.Property.Dynamic ); /** Spill memory limit — when exceeded, DataFusion spills to disk. */ @@ -131,6 +137,10 @@ public Collection createComponents( dataFusionService.start(); logger.debug("DataFusion plugin initialized — memory pool {}B, spill limit {}B", memoryPoolLimit, spillMemoryLimit); + // Wire the dynamic memory pool limit setting to the native runtime so updates via the + // cluster settings API take effect without restarting the node. + clusterService.getClusterSettings().addSettingsUpdateConsumer(DATAFUSION_MEMORY_POOL_LIMIT, this::updateMemoryPoolLimit); + this.substraitExtensions = loadSubstraitExtensions(); return Collections.singletonList(dataFusionService); @@ -187,6 +197,35 @@ public List> getSettings() { ); } + /** + * Applies a new memory pool limit to the running DataFusion runtime. + *

      + * Takes effect for new allocations only. In-flight reservations that already + * exceed the new limit are not reclaimed and drain as queries complete. + *

      + * Safe to call during plugin startup before {@link #createComponents} returns + * (service is null, ignored) and during shutdown after the native runtime has + * been released (service throws {@link IllegalStateException}, caught and logged). + *

      + * Package-private for testing. + */ + void updateMemoryPoolLimit(long newLimitBytes) { + DataFusionService service = dataFusionService; + if (service == null) { + logger.debug("DataFusion service not yet initialized; ignoring memory pool limit update to {}B", newLimitBytes); + return; + } + try { + service.setMemoryPoolLimit(newLimitBytes); + logger.info("Updated DataFusion memory pool limit to {}B", newLimitBytes); + } catch (IllegalStateException e) { + // Service has been stopped/closed (e.g., during node shutdown). The listener is + // still registered on ClusterSettings because there is no removeSettingsUpdateConsumer + // API; swallow the race so cluster-state application does not log a spurious failure. + logger.warn("Ignoring memory pool limit update to {}B; service is not running", newLimitBytes); + } + } + @Override public String name() { return "datafusion"; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java index b626ff15592c3..a8cf655aa9e7e 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java @@ -124,6 +124,32 @@ public NativeRuntimeHandle getNativeRuntime() { return handle; } + /** + * Returns the current memory pool usage in bytes. + */ + public long getMemoryPoolUsage() { + return NativeBridge.getMemoryPoolUsage(getNativeRuntime().get()); + } + + /** + * Returns the current memory pool limit in bytes. + */ + public long getMemoryPoolLimit() { + return NativeBridge.getMemoryPoolLimit(getNativeRuntime().get()); + } + + /** + * Sets the memory pool limit at runtime. Takes effect for new allocations only. + * Existing reservations that exceed the new limit are NOT reclaimed. + *

      + * The user-visible info-level log line is emitted by the caller in + * {@code DataFusionPlugin.updateMemoryPoolLimit}; this method is silent to avoid + * duplicate log entries. + */ + public void setMemoryPoolLimit(long newLimitBytes) { + NativeBridge.setMemoryPoolLimit(getNativeRuntime().get(), newLimitBytes); + } + // Cache management (node-level, delegates to native runtime) /** diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index 7afa5eb24feff..558f1e76bdb64 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -44,6 +44,9 @@ public final class NativeBridge { private static final MethodHandle SHUTDOWN_RUNTIME_MANAGER; private static final MethodHandle CREATE_GLOBAL_RUNTIME; private static final MethodHandle CLOSE_GLOBAL_RUNTIME; + private static final MethodHandle GET_MEMORY_POOL_USAGE; + private static final MethodHandle GET_MEMORY_POOL_LIMIT; + private static final MethodHandle SET_MEMORY_POOL_LIMIT; private static final MethodHandle CREATE_READER; private static final MethodHandle CLOSE_READER; private static final MethodHandle EXECUTE_QUERY; @@ -104,6 +107,21 @@ public final class NativeBridge { FunctionDescriptor.ofVoid(ValueLayout.JAVA_LONG) ); + GET_MEMORY_POOL_USAGE = linker.downcallHandle( + lib.find("df_get_memory_pool_usage").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + + GET_MEMORY_POOL_LIMIT = linker.downcallHandle( + lib.find("df_get_memory_pool_limit").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + + SET_MEMORY_POOL_LIMIT = linker.downcallHandle( + lib.find("df_set_memory_pool_limit").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + CREATE_READER = linker.downcallHandle( lib.find("df_create_reader").orElseThrow(), FunctionDescriptor.of( @@ -463,6 +481,29 @@ public static void closeGlobalRuntime(long ptr) { NativeCall.invokeVoid(CLOSE_GLOBAL_RUNTIME, ptr); } + // ---- Memory pool observability and dynamic limit ---- + + /** Returns current memory pool usage in bytes. */ + public static long getMemoryPoolUsage(long runtimePtr) { + try (var call = new NativeCall()) { + return call.invoke(GET_MEMORY_POOL_USAGE, runtimePtr); + } + } + + /** Returns current memory pool limit in bytes. */ + public static long getMemoryPoolLimit(long runtimePtr) { + try (var call = new NativeCall()) { + return call.invoke(GET_MEMORY_POOL_LIMIT, runtimePtr); + } + } + + /** Sets the memory pool limit at runtime. Takes effect for new allocations only. */ + public static void setMemoryPoolLimit(long runtimePtr, long newLimitBytes) { + try (var call = new NativeCall()) { + call.invoke(SET_MEMORY_POOL_LIMIT, runtimePtr, newLimitBytes); + } + } + // ---- Reader management (confined Arena for path + file strings) ---- /** diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java new file mode 100644 index 0000000000000..a9ad8834fe6e5 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.common.settings.Setting; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +/** + * Verifies the settings declared by {@link DataFusionPlugin} — in particular that + * {@code datafusion.memory_pool_limit_bytes} is registered and marked dynamic so + * the cluster settings API can update it at runtime. + */ +public class DataFusionPluginSettingsTests extends OpenSearchTestCase { + + public void testMemoryPoolLimitIsDynamic() { + assertTrue( + "datafusion.memory_pool_limit_bytes must be dynamic to support runtime updates", + DataFusionPlugin.DATAFUSION_MEMORY_POOL_LIMIT.isDynamic() + ); + } + + public void testMemoryPoolLimitHasNodeScope() { + assertTrue("datafusion.memory_pool_limit_bytes must have node scope", DataFusionPlugin.DATAFUSION_MEMORY_POOL_LIMIT.hasNodeScope()); + } + + public void testPluginRegistersMemoryPoolLimitSetting() { + try (DataFusionPlugin plugin = new DataFusionPlugin()) { + List> settings = plugin.getSettings(); + assertTrue( + "Plugin must register DATAFUSION_MEMORY_POOL_LIMIT via getSettings()", + settings.contains(DataFusionPlugin.DATAFUSION_MEMORY_POOL_LIMIT) + ); + assertTrue( + "Plugin must register DATAFUSION_SPILL_MEMORY_LIMIT via getSettings()", + settings.contains(DataFusionPlugin.DATAFUSION_SPILL_MEMORY_LIMIT) + ); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + /** + * H1 — the cluster-settings listener can fire before {@link DataFusionPlugin#createComponents} + * is called (service field still null). {@code updateMemoryPoolLimit} must swallow this quietly + * so the cluster-state update does not log a failure during node startup. + */ + public void testUpdateMemoryPoolLimitBeforeServiceStartDoesNotThrow() { + try (DataFusionPlugin plugin = new DataFusionPlugin()) { + // Service field is null — should be a no-op, not an NPE. + plugin.updateMemoryPoolLimit(64L * 1024 * 1024); + } catch (Exception e) { + throw new AssertionError(e); + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DynamicMemoryPoolTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DynamicMemoryPoolTests.java new file mode 100644 index 0000000000000..2062b0ae49029 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DynamicMemoryPoolTests.java @@ -0,0 +1,105 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.be.datafusion.nativelib.NativeBridge; +import org.opensearch.test.OpenSearchTestCase; + +import java.nio.file.Path; + +/** + * Tests for the DynamicLimitPool — verifies that the memory pool limit + * can be read and changed at runtime via the FFM bridge. + */ +public class DynamicMemoryPoolTests extends OpenSearchTestCase { + + private DataFusionService service; + + @Override + public void setUp() throws Exception { + super.setUp(); + NativeBridge.initTokioRuntimeManager(2); + Path spillDir = createTempDir("datafusion-spill"); + service = DataFusionService.builder() + .memoryPoolLimit(64 * 1024 * 1024) // 64MB + .spillMemoryLimit(32 * 1024 * 1024) + .spillDirectory(spillDir.toString()) + .cpuThreads(2) + .build(); + service.start(); + } + + @Override + public void tearDown() throws Exception { + if (service != null) { + service.stop(); + } + NativeBridge.shutdownTokioRuntimeManager(); + super.tearDown(); + } + + public void testGetInitialPoolLimit() { + long limit = service.getMemoryPoolLimit(); + assertEquals("Initial pool limit should be 64 MB", 64 * 1024 * 1024, limit); + } + + public void testGetInitialPoolUsage() { + long usage = service.getMemoryPoolUsage(); + assertEquals("Initial pool usage should be 0", 0, usage); + } + + public void testSetPoolLimitIncrease() { + long newLimit = 128L * 1024 * 1024; // 128MB + service.setMemoryPoolLimit(newLimit); + assertEquals("Pool limit should be updated to 128 MB", newLimit, service.getMemoryPoolLimit()); + } + + public void testSetPoolLimitDecrease() { + long newLimit = 32L * 1024 * 1024; // 32MB + service.setMemoryPoolLimit(newLimit); + assertEquals("Pool limit should be updated to 32 MB", newLimit, service.getMemoryPoolLimit()); + } + + public void testSetPoolLimitMultipleTimes() { + service.setMemoryPoolLimit(100L * 1024 * 1024); + assertEquals(100L * 1024 * 1024, service.getMemoryPoolLimit()); + + service.setMemoryPoolLimit(50L * 1024 * 1024); + assertEquals(50L * 1024 * 1024, service.getMemoryPoolLimit()); + + service.setMemoryPoolLimit(200L * 1024 * 1024); + assertEquals(200L * 1024 * 1024, service.getMemoryPoolLimit()); + } + + public void testDirectNativeBridgeCalls() { + long runtimePtr = service.getNativeRuntime().get(); + + long limit = NativeBridge.getMemoryPoolLimit(runtimePtr); + assertEquals(64 * 1024 * 1024, limit); + + NativeBridge.setMemoryPoolLimit(runtimePtr, 256L * 1024 * 1024); + assertEquals(256L * 1024 * 1024, NativeBridge.getMemoryPoolLimit(runtimePtr)); + + long usage = NativeBridge.getMemoryPoolUsage(runtimePtr); + assertTrue("Usage should be >= 0", usage >= 0); + } + + /** + * H1 — after the service has been stopped, {@link DataFusionService#setMemoryPoolLimit} + * must surface an {@link IllegalStateException} rather than dereferencing a closed runtime + * handle. The plugin-level listener catches this to keep cluster-state updates quiet during + * node shutdown. + */ + public void testSetMemoryPoolLimitAfterStopThrowsIllegalState() { + service.stop(); + expectThrows(IllegalStateException.class, () -> service.setMemoryPoolLimit(128L * 1024 * 1024)); + // Null out so tearDown does not try to stop again. + service = null; + } +} From 84777877aa3591a8b0130bf80fa3ae5433585e6e Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 6 May 2026 23:46:10 +0530 Subject: [PATCH 059/115] Replace mimalloc with jemalloc as global allocator for native sandbox plugins (#21497) * jemalloc as global allocator Signed-off-by: Himshikha Gupta * make jemalloc settings tunable through cluster settings Signed-off-by: Himshikha Gupta * address comments Signed-off-by: Himshikha Gupta * address comments Signed-off-by: Himshikha Gupta --------- Signed-off-by: Himshikha Gupta --- .../libs/dataformat-native/rust/Cargo.toml | 7 +- .../dataformat-native/rust/common/Cargo.toml | 4 + .../rust/common/src/allocator.rs | 165 ++++++++++++++++++ .../rust/common/src/error.rs | 23 +++ .../dataformat-native/rust/common/src/lib.rs | 1 + .../dataformat-native/rust/lib/Cargo.toml | 2 +- .../dataformat-native/rust/lib/src/lib.rs | 21 ++- .../dataformat-native/rust/macros/src/lib.rs | 23 +-- .../spi/NativeAllocatorConfig.java | 72 ++++++++ .../spi/NativeAllocatorConfigTests.java | 35 ++++ .../spi/NativeMemoryMetricsTests.java | 45 +++++ sandbox/modules/build.gradle | 6 +- sandbox/modules/native-bridge/build.gradle | 20 +++ .../nativebridge/NativeBridgeModule.java | 88 ++++++++++ .../opensearch/nativebridge/package-info.java | 12 ++ .../nativebridge/NativeBridgeModuleTests.java | 25 +++ .../rust/Cargo.toml | 1 - 17 files changed, 520 insertions(+), 30 deletions(-) create mode 100644 sandbox/libs/dataformat-native/rust/common/src/allocator.rs create mode 100644 sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeAllocatorConfig.java create mode 100644 sandbox/libs/dataformat-native/src/test/java/org/opensearch/nativebridge/spi/NativeAllocatorConfigTests.java create mode 100644 sandbox/libs/dataformat-native/src/test/java/org/opensearch/nativebridge/spi/NativeMemoryMetricsTests.java create mode 100644 sandbox/modules/native-bridge/build.gradle create mode 100644 sandbox/modules/native-bridge/src/main/java/org/opensearch/nativebridge/NativeBridgeModule.java create mode 100644 sandbox/modules/native-bridge/src/main/java/org/opensearch/nativebridge/package-info.java create mode 100644 sandbox/modules/native-bridge/src/test/java/org/opensearch/nativebridge/NativeBridgeModuleTests.java diff --git a/sandbox/libs/dataformat-native/rust/Cargo.toml b/sandbox/libs/dataformat-native/rust/Cargo.toml index 6adb44c9a62b4..2413680eb8710 100644 --- a/sandbox/libs/dataformat-native/rust/Cargo.toml +++ b/sandbox/libs/dataformat-native/rust/Cargo.toml @@ -46,7 +46,12 @@ serde_json = "1.0" log = "0.4" # Allocator -mimalloc = { version = "0.1.48", default-features = false } +# disable_initial_exec_tls: Required because this library is loaded at runtime via dlopen/JVM FFM. +# Without it, jemalloc uses initial-exec TLS which fails on aarch64 Linux with: +# "cannot allocate memory in static TLS block" +# The feature switches to global-dynamic TLS model, compatible with runtime loading. +tikv-jemallocator = { version = "0.6", features = ["disable_initial_exec_tls"] } +tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] } # Misc dashmap = "5.5" diff --git a/sandbox/libs/dataformat-native/rust/common/Cargo.toml b/sandbox/libs/dataformat-native/rust/common/Cargo.toml index 64b2370a5ddaa..a6eb4f679540a 100644 --- a/sandbox/libs/dataformat-native/rust/common/Cargo.toml +++ b/sandbox/libs/dataformat-native/rust/common/Cargo.toml @@ -10,3 +10,7 @@ crate-type = ["rlib"] [dependencies] native-bridge-macros = { path = "../macros" } +tikv-jemalloc-ctl = { workspace = true } + +[dev-dependencies] +tikv-jemallocator = { workspace = true } diff --git a/sandbox/libs/dataformat-native/rust/common/src/allocator.rs b/sandbox/libs/dataformat-native/rust/common/src/allocator.rs new file mode 100644 index 0000000000000..b20585c8765a5 --- /dev/null +++ b/sandbox/libs/dataformat-native/rust/common/src/allocator.rs @@ -0,0 +1,165 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! jemalloc allocator interface: memory stats and runtime tuning. +//! +//! FFI convention (same as all other native bridge functions): +//! - `>= 0` → success (the stat value in bytes, or 0 for setters) +//! - `< 0` → error pointer. Negate and pass to `native_error_message` / `native_error_free`. + +use crate::error::{ffm_wrap, into_error_ptr}; +use std::sync::OnceLock; +use tikv_jemalloc_ctl::{epoch, epoch_mib, stats, stats::allocated_mib, stats::resident_mib}; + +struct StatsMib { + epoch: epoch_mib, + allocated: allocated_mib, + resident: resident_mib, +} + +static MIB: OnceLock = OnceLock::new(); + +fn mib() -> &'static StatsMib { + MIB.get_or_init(|| StatsMib { + epoch: epoch::mib().unwrap(), + allocated: stats::allocated::mib().unwrap(), + resident: stats::resident::mib().unwrap(), + }) +} + +/// Advances the jemalloc epoch and reads both stats atomically. +fn refresh_stats() -> Result<(i64, i64), String> { + let m = mib(); + m.epoch.advance().map_err(|e| format!("jemalloc epoch advance failed: {}", e))?; + let alloc = m.allocated.read().map_err(|e| format!("jemalloc allocated read failed: {}", e))? as i64; + let res = m.resident.read().map_err(|e| format!("jemalloc resident read failed: {}", e))? as i64; + Ok((alloc, res)) +} + +/// Returns current jemalloc allocated bytes (live malloc'd objects). +/// Useful for application-level memory accounting and DataFusion memory pool budgeting. +/// On error: returns negative error pointer (use `native_error_message` to read). +/// +/// TODO: integrate with node/stats +pub fn allocated_bytes() -> i64 { + match refresh_stats() { + Ok((alloc, _)) => alloc, + Err(msg) => into_error_ptr(msg), + } +} + +/// Returns current jemalloc resident bytes (physical RAM used by native layer only). +/// Excludes JVM heap, metaspace, and other non-jemalloc allocations. +/// On error: returns negative error pointer (use `native_error_message` to read). +/// +/// TODO: integrate with node/stats +pub fn resident_bytes() -> i64 { + match refresh_stats() { + Ok((_, res)) => res, + Err(msg) => into_error_ptr(msg), + } +} + +/// FFI: Returns current jemalloc allocated bytes, or negative error pointer. +#[no_mangle] +pub extern "C" fn native_jemalloc_allocated_bytes() -> i64 { + ffm_wrap("native_jemalloc_allocated_bytes", || refresh_stats().map(|(alloc, _)| alloc)) +} + +/// FFI: Returns current jemalloc resident bytes, or negative error pointer. +#[no_mangle] +pub extern "C" fn native_jemalloc_resident_bytes() -> i64 { + ffm_wrap("native_jemalloc_resident_bytes", || refresh_stats().map(|(_, res)| res)) +} + +/// FFI: Sets dirty_decay_ms for all arenas at runtime. Returns 0 on success, negative error pointer on failure. +/// Called from Java when the cluster setting `native.jemalloc.dirty_decay_ms` changes. +#[no_mangle] +pub extern "C" fn native_jemalloc_set_dirty_decay_ms(ms: i64) -> i64 { + ffm_wrap("native_jemalloc_set_dirty_decay_ms", || set_all_arenas(b"dirty_decay_ms\0", ms)) +} + +/// FFI: Sets muzzy_decay_ms for all arenas at runtime. Returns 0 on success, negative error pointer on failure. +/// Called from Java when the cluster setting `native.jemalloc.muzzy_decay_ms` changes. +#[no_mangle] +pub extern "C" fn native_jemalloc_set_muzzy_decay_ms(ms: i64) -> i64 { + ffm_wrap("native_jemalloc_set_muzzy_decay_ms", || set_all_arenas(b"muzzy_decay_ms\0", ms)) +} + +/// Applies a setting to all existing jemalloc arenas. +/// Skips arenas that are not available (destroyed or internal). +fn set_all_arenas(suffix: &[u8], ms: i64) -> Result { + let narenas: u32 = unsafe { tikv_jemalloc_ctl::raw::read(b"arenas.narenas\0") } + .map_err(|e| format!("failed to read arenas.narenas: {}", e))?; + let suffix_str = std::str::from_utf8(&suffix[..suffix.len() - 1]).unwrap(); + let mut any_success = false; + for i in 0..narenas { + let key = format!("arena.{}.{}\0", i, suffix_str); + if unsafe { tikv_jemalloc_ctl::raw::write(key.as_bytes(), ms as isize) }.is_ok() { + any_success = true; + } + } + if any_success { + Ok(0) + } else { + Err(format!("failed to set {} on any arena", suffix_str)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[global_allocator] + static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + + #[test] + fn allocated_bytes_is_positive() { + assert!(allocated_bytes() > 0); + } + + #[test] + fn resident_bytes_is_positive() { + assert!(resident_bytes() > 0); + } + + #[test] + fn allocated_increases_after_allocation() { + let before = allocated_bytes(); + let _data: Vec = vec![42u8; 1024 * 1024]; + let after = allocated_bytes(); + assert!(after > before, "expected {after} > {before}"); + } + + #[test] + fn set_dirty_decay_ms_applies_at_runtime() { + let rc = native_jemalloc_set_dirty_decay_ms(5000); + assert_eq!(rc, 0, "setter should succeed, got {}", rc); + + // Read back from arena 0 to verify it took effect + let actual: isize = + unsafe { tikv_jemalloc_ctl::raw::read(b"arena.0.dirty_decay_ms\0") }.unwrap(); + assert_eq!(actual, 5000); + + // Restore default + native_jemalloc_set_dirty_decay_ms(30000); + } + + #[test] + fn set_muzzy_decay_ms_applies_at_runtime() { + let rc = native_jemalloc_set_muzzy_decay_ms(10000); + assert_eq!(rc, 0, "setter should succeed, got {}", rc); + + let actual: isize = + unsafe { tikv_jemalloc_ctl::raw::read(b"arena.0.muzzy_decay_ms\0") }.unwrap(); + assert_eq!(actual, 10000); + + // Restore default + native_jemalloc_set_muzzy_decay_ms(30000); + } +} diff --git a/sandbox/libs/dataformat-native/rust/common/src/error.rs b/sandbox/libs/dataformat-native/rust/common/src/error.rs index fc43129f2d1d1..ec30f053654a0 100644 --- a/sandbox/libs/dataformat-native/rust/common/src/error.rs +++ b/sandbox/libs/dataformat-native/rust/common/src/error.rs @@ -23,6 +23,29 @@ pub fn into_error_ptr(msg: String) -> i64 { -(ptr as i64) } +/// Wraps a closure with `catch_unwind` and error-pointer conversion. +/// Same contract as `#[ffm_safe]` — the canonical implementation used by both +/// this crate's FFI functions and the `#[ffm_safe]` proc macro. +pub fn ffm_wrap(name: &str, f: F) -> i64 +where + F: FnOnce() -> Result + std::panic::UnwindSafe, +{ + match std::panic::catch_unwind(f) { + Ok(Ok(v)) => v, + Ok(Err(msg)) => into_error_ptr(msg), + Err(panic) => { + let msg = if let Some(s) = panic.downcast_ref::() { + s.clone() + } else if let Some(s) = panic.downcast_ref::<&str>() { + s.to_string() + } else { + format!("unknown panic in {}", name) + }; + into_error_ptr(msg) + } + } +} + /// Returns a pointer to the null-terminated error message. #[no_mangle] pub unsafe extern "C" fn native_error_message(ptr: i64) -> *const c_char { diff --git a/sandbox/libs/dataformat-native/rust/common/src/lib.rs b/sandbox/libs/dataformat-native/rust/common/src/lib.rs index 88302f600a4d9..0f4b8c132407f 100644 --- a/sandbox/libs/dataformat-native/rust/common/src/lib.rs +++ b/sandbox/libs/dataformat-native/rust/common/src/lib.rs @@ -10,6 +10,7 @@ pub mod error; pub mod logger; +pub mod allocator; // Re-export the proc macro so plugins use `#[native_bridge_common::ffm_safe]` pub use native_bridge_macros::ffm_safe; diff --git a/sandbox/libs/dataformat-native/rust/lib/Cargo.toml b/sandbox/libs/dataformat-native/rust/lib/Cargo.toml index 354c8f5019a43..6eadb23e82a21 100644 --- a/sandbox/libs/dataformat-native/rust/lib/Cargo.toml +++ b/sandbox/libs/dataformat-native/rust/lib/Cargo.toml @@ -19,4 +19,4 @@ opensearch-repository-fs = { workspace = true } opensearch-block-cache = { path = "../../../../plugins/block-cache-foyer/src/main/rust" } native-bridge-common = { workspace = true } opensearch-tiered-storage = { path = "../../../../libs/tiered-storage/src/main/rust" } -mimalloc = { workspace = true } +tikv-jemallocator = { workspace = true } diff --git a/sandbox/libs/dataformat-native/rust/lib/src/lib.rs b/sandbox/libs/dataformat-native/rust/lib/src/lib.rs index ecaf66b1ecfee..3a53b77a9e721 100644 --- a/sandbox/libs/dataformat-native/rust/lib/src/lib.rs +++ b/sandbox/libs/dataformat-native/rust/lib/src/lib.rs @@ -9,20 +9,25 @@ // ═══════════════════════════════════════════════════════════════════════════════ // Single cdylib for JDK FFM (Foreign Function & Memory API). // -// Unlike the JNI approach (RegisterNatives, classloader workarounds), FFM calls -// extern "C" functions directly via SymbolLookup + Linker.downcallHandle(). -// No JNIEnv, no JClass, no classloader binding — just plain C ABI. -// // This crate: -// 1. Sets the global mimalloc allocator (shared across all plugin rlibs) +// 1. Sets the global jemalloc allocator (shared across all plugin rlibs) // 2. Pulls in plugin rlibs via extern crate (forces linker to include symbols) // 3. All #[no_mangle] extern "C" functions from the plugin crates are // automatically available for dlsym/SymbolLookup // ═══════════════════════════════════════════════════════════════════════════════ -//TODO: AwaitsFix: Fix mimalloc lifecycle issue -// #[global_allocator] -// static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; +/// jemalloc tuning applied at process start (before JVM/OpenSearch boots): +/// - dirty_decay_ms and muzzy_decay_ms: also dynamically tunable at runtime via cluster settings +/// (see NativeBridgeModule). The values here serve as defaults for the brief window between +/// process start and OpenSearch initialization. On restart, the persisted cluster setting +/// is re-applied by NativeBridgeModule.createComponents() — these compile-time values are +/// only used until that point. +/// - lg_tcache_max: NOT dynamically tunable by jemalloc — init-time only, requires process restart to change. +#[export_name = "malloc_conf"] +pub static MALLOC_CONF: &[u8] = b"dirty_decay_ms:30000,muzzy_decay_ms:30000,lg_tcache_max:16\0"; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; // Pull in plugin rlibs — forces linker to include all #[no_mangle] symbols. extern crate native_bridge_common; diff --git a/sandbox/libs/dataformat-native/rust/macros/src/lib.rs b/sandbox/libs/dataformat-native/rust/macros/src/lib.rs index 3883358fc4963..ccb3eab9d6c0f 100644 --- a/sandbox/libs/dataformat-native/rust/macros/src/lib.rs +++ b/sandbox/libs/dataformat-native/rust/macros/src/lib.rs @@ -38,25 +38,16 @@ pub fn ffm_safe(_attr: TokenStream, item: TokenStream) -> TokenStream { let sig = &input.sig; let body = &input.block; + let fn_name = input.sig.ident.to_string(); let expanded = quote! { #(#attrs)* #vis #sig { - match ::std::panic::catch_unwind(::std::panic::AssertUnwindSafe( - || -> ::std::result::Result #body - )) { - Ok(Ok(v)) => v, - Ok(Err(msg)) => native_bridge_common::error::into_error_ptr(msg), - Err(panic) => { - let msg = if let Some(s) = panic.downcast_ref::() { - s.clone() - } else if let Some(s) = panic.downcast_ref::<&str>() { - s.to_string() - } else { - "unknown panic".to_string() - }; - native_bridge_common::error::into_error_ptr(msg) - } - } + native_bridge_common::error::ffm_wrap( + #fn_name, + ::std::panic::AssertUnwindSafe( + || -> ::std::result::Result #body + ), + ) } }; diff --git a/sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeAllocatorConfig.java b/sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeAllocatorConfig.java new file mode 100644 index 0000000000000..4c425702d1be6 --- /dev/null +++ b/sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeAllocatorConfig.java @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.nativebridge.spi; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; + +/** + * Dynamic jemalloc tuning via FFM. + *

      + * Provides methods to adjust jemalloc's {@code dirty_decay_ms} and {@code muzzy_decay_ms} + * at runtime for all arenas. These are called by plugin-level cluster settings listeners. + *

      + * Note: {@code lg_tcache_max} is NOT dynamically tunable by jemalloc (init-time only). + */ +public final class NativeAllocatorConfig { + + private static final Logger logger = LogManager.getLogger(NativeAllocatorConfig.class); + + private static final MethodHandle SET_DIRTY; + private static final MethodHandle SET_MUZZY; + + static { + SymbolLookup lookup = NativeLibraryLoader.symbolLookup(); + Linker linker = Linker.nativeLinker(); + FunctionDescriptor desc = FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG); + SET_DIRTY = linker.downcallHandle(lookup.find("native_jemalloc_set_dirty_decay_ms").orElseThrow(), desc); + SET_MUZZY = linker.downcallHandle(lookup.find("native_jemalloc_set_muzzy_decay_ms").orElseThrow(), desc); + } + + private NativeAllocatorConfig() {} + + /** + * Sets dirty_decay_ms for all jemalloc arenas. No restart required. + * + * @param ms decay time in milliseconds (-1 to disable decay) + */ + public static void setDirtyDecayMs(long ms) { + applyDecay(SET_DIRTY, "dirty_decay_ms", ms); + } + + /** + * Sets muzzy_decay_ms for all jemalloc arenas. No restart required. + * + * @param ms decay time in milliseconds (-1 to disable decay) + */ + public static void setMuzzyDecayMs(long ms) { + applyDecay(SET_MUZZY, "muzzy_decay_ms", ms); + } + + private static void applyDecay(MethodHandle handle, String name, long ms) { + try { + long rc = (long) handle.invokeExact(ms); + NativeLibraryLoader.checkResult(rc); + logger.info("jemalloc {} updated to {}", name, ms); + } catch (Throwable t) { + logger.warn("Error setting jemalloc " + name, t); + } + } +} diff --git a/sandbox/libs/dataformat-native/src/test/java/org/opensearch/nativebridge/spi/NativeAllocatorConfigTests.java b/sandbox/libs/dataformat-native/src/test/java/org/opensearch/nativebridge/spi/NativeAllocatorConfigTests.java new file mode 100644 index 0000000000000..eea59a29eec4e --- /dev/null +++ b/sandbox/libs/dataformat-native/src/test/java/org/opensearch/nativebridge/spi/NativeAllocatorConfigTests.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.nativebridge.spi; + +import org.opensearch.test.OpenSearchTestCase; + +/** + * Tests that the native jemalloc decay setters work at runtime. + */ +public class NativeAllocatorConfigTests extends OpenSearchTestCase { + + public void testSetDirtyDecayMsSucceeds() { + // Should not throw — applies to all jemalloc arenas + NativeAllocatorConfig.setDirtyDecayMs(5000); + // Restore default + NativeAllocatorConfig.setDirtyDecayMs(30000); + } + + public void testSetMuzzyDecayMsSucceeds() { + NativeAllocatorConfig.setMuzzyDecayMs(10000); + NativeAllocatorConfig.setMuzzyDecayMs(30000); + } + + public void testDisableDecayWithNegativeOne() { + // -1 disables decay (pages retained indefinitely) + NativeAllocatorConfig.setDirtyDecayMs(-1); + NativeAllocatorConfig.setDirtyDecayMs(30000); + } +} diff --git a/sandbox/libs/dataformat-native/src/test/java/org/opensearch/nativebridge/spi/NativeMemoryMetricsTests.java b/sandbox/libs/dataformat-native/src/test/java/org/opensearch/nativebridge/spi/NativeMemoryMetricsTests.java new file mode 100644 index 0000000000000..335f348ad486a --- /dev/null +++ b/sandbox/libs/dataformat-native/src/test/java/org/opensearch/nativebridge/spi/NativeMemoryMetricsTests.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.nativebridge.spi; + +import org.opensearch.test.OpenSearchTestCase; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; + +/** + * Tests that the native jemalloc metrics functions are available and return valid data. + */ +public class NativeMemoryMetricsTests extends OpenSearchTestCase { + + public void testAllocatedBytesIsPositive() throws Throwable { + SymbolLookup lookup = NativeLibraryLoader.symbolLookup(); + Linker linker = Linker.nativeLinker(); + MethodHandle allocated = linker.downcallHandle( + lookup.find("native_jemalloc_allocated_bytes").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG) + ); + long bytes = (long) allocated.invokeExact(); + assertTrue("allocated bytes should be positive, got " + bytes, bytes > 0); + } + + public void testResidentBytesIsPositive() throws Throwable { + SymbolLookup lookup = NativeLibraryLoader.symbolLookup(); + Linker linker = Linker.nativeLinker(); + MethodHandle resident = linker.downcallHandle( + lookup.find("native_jemalloc_resident_bytes").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG) + ); + long bytes = (long) resident.invokeExact(); + assertTrue("resident bytes should be positive, got " + bytes, bytes > 0); + } +} diff --git a/sandbox/modules/build.gradle b/sandbox/modules/build.gradle index 61afb2c568e1b..1b7b6889972fd 100644 --- a/sandbox/modules/build.gradle +++ b/sandbox/modules/build.gradle @@ -12,8 +12,8 @@ configure(subprojects.findAll { it.parent.path == project.path }) { apply plugin: 'opensearch.opensearchplugin' opensearchplugin { - name project.name - licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') + name = project.name + licenseFile = rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile = rootProject.file('NOTICE.txt') } } diff --git a/sandbox/modules/native-bridge/build.gradle b/sandbox/modules/native-bridge/build.gradle new file mode 100644 index 0000000000000..24cb121a3feff --- /dev/null +++ b/sandbox/modules/native-bridge/build.gradle @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +opensearchplugin { + description = 'Native bridge module: manages runtime tuning for the native (Rust/FFM) layer.' + classname = 'org.opensearch.nativebridge.NativeBridgeModule' +} + +java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } + +dependencies { + implementation project(':sandbox:libs:dataformat-native') + compileOnly project(':server') + testImplementation project(':test:framework') +} diff --git a/sandbox/modules/native-bridge/src/main/java/org/opensearch/nativebridge/NativeBridgeModule.java b/sandbox/modules/native-bridge/src/main/java/org/opensearch/nativebridge/NativeBridgeModule.java new file mode 100644 index 0000000000000..44f669b68ca7d --- /dev/null +++ b/sandbox/modules/native-bridge/src/main/java/org/opensearch/nativebridge/NativeBridgeModule.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.nativebridge; + +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.nativebridge.spi.NativeAllocatorConfig; +import org.opensearch.plugins.Plugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.script.ScriptService; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.client.Client; +import org.opensearch.watcher.ResourceWatcherService; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +/** + * Always-loaded module that manages runtime tuning for the native (Rust/FFM) layer. + *

      + * Registers dynamic cluster settings and applies changes at runtime via the FFM bridge. + */ +public class NativeBridgeModule extends Plugin { + + /** jemalloc dirty page decay time (ms). Dynamically tunable — applied to all arenas at runtime. */ + public static final Setting JEMALLOC_DIRTY_DECAY_MS = Setting.longSetting( + "native.jemalloc.dirty_decay_ms", + 30_000L, + -1L, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** jemalloc muzzy page decay time (ms). Dynamically tunable — applied to all arenas at runtime. */ + public static final Setting JEMALLOC_MUZZY_DECAY_MS = Setting.longSetting( + "native.jemalloc.muzzy_decay_ms", + 30_000L, + -1L, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + Settings settings = environment.settings(); + + // Apply initial values (handles opensearch.yml overrides of the compile-time malloc_conf defaults) + NativeAllocatorConfig.setDirtyDecayMs(JEMALLOC_DIRTY_DECAY_MS.get(settings)); + NativeAllocatorConfig.setMuzzyDecayMs(JEMALLOC_MUZZY_DECAY_MS.get(settings)); + + // Register dynamic update listeners + clusterService.getClusterSettings().addSettingsUpdateConsumer(JEMALLOC_DIRTY_DECAY_MS, NativeAllocatorConfig::setDirtyDecayMs); + clusterService.getClusterSettings().addSettingsUpdateConsumer(JEMALLOC_MUZZY_DECAY_MS, NativeAllocatorConfig::setMuzzyDecayMs); + + return Collections.emptyList(); + } + + @Override + public List> getSettings() { + return List.of(JEMALLOC_DIRTY_DECAY_MS, JEMALLOC_MUZZY_DECAY_MS); + } +} diff --git a/sandbox/modules/native-bridge/src/main/java/org/opensearch/nativebridge/package-info.java b/sandbox/modules/native-bridge/src/main/java/org/opensearch/nativebridge/package-info.java new file mode 100644 index 0000000000000..e0de17ee4d5f8 --- /dev/null +++ b/sandbox/modules/native-bridge/src/main/java/org/opensearch/nativebridge/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Native bridge module — owns jemalloc cluster settings and applies runtime tuning. + */ +package org.opensearch.nativebridge; diff --git a/sandbox/modules/native-bridge/src/test/java/org/opensearch/nativebridge/NativeBridgeModuleTests.java b/sandbox/modules/native-bridge/src/test/java/org/opensearch/nativebridge/NativeBridgeModuleTests.java new file mode 100644 index 0000000000000..13d02dcccb4dd --- /dev/null +++ b/sandbox/modules/native-bridge/src/test/java/org/opensearch/nativebridge/NativeBridgeModuleTests.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.nativebridge; + +import org.opensearch.common.settings.Setting; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +public class NativeBridgeModuleTests extends OpenSearchTestCase { + + public void testGetSettingsReturnsBothDecaySettings() { + NativeBridgeModule module = new NativeBridgeModule(); + List> settings = module.getSettings(); + assertEquals(2, settings.size()); + assertEquals("native.jemalloc.dirty_decay_ms", settings.get(0).getKey()); + assertEquals("native.jemalloc.muzzy_decay_ms", settings.get(1).getKey()); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml index 1c5e72ea4e2ed..1f80c7685c408 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml +++ b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml @@ -36,7 +36,6 @@ once_cell = { workspace = true } dashmap = { workspace = true } log = { workspace = true } num_cpus = { workspace = true } -mimalloc = { workspace = true } native-bridge-common = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } From 916ef741090c3408f8b2f22f03bddf6aa0b51f0c Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Wed, 6 May 2026 14:31:15 -0500 Subject: [PATCH 060/115] Fix waitForReplication race with INITIALIZING replicas (#21514) Previously the waitForReplication() test helper would only check if STARTED shards had caught up. However, we should wait for INITIALIZING replicas to catch up as well since they will be stale immediately after they transition to STARTED. This change also pulls a fresh cluster state in each assertBusy iteration to ensure it gets the latest shard state. Signed-off-by: Andrew Ross --- .../test/OpenSearchIntegTestCase.java | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index b64843eb6161f..d97823aa059d6 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -83,6 +83,7 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.routing.allocation.DiskThresholdSettings; @@ -2552,38 +2553,35 @@ protected void waitForReplication(String... indices) { try { for (String index : indices) { if (isSegmentReplicationEnabledForIndex(index)) { - if (isInternalCluster()) { + assertTrue("Unable to wait for replication with external test cluster", isInternalCluster()); + assertBusy(() -> { IndexRoutingTable indexRoutingTable = getClusterState().routingTable().index(index); if (indexRoutingTable != null) { - assertBusy(() -> { - for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) { - final ShardRouting primaryRouting = shardRoutingTable.primaryShard(); - if (primaryRouting.state().toString().equals("STARTED")) { - if (isSegmentReplicationEnabledForIndex(index)) { - final List replicaRouting = shardRoutingTable.replicaShards(); - final IndexShard primaryShard = getIndexShard(primaryRouting, index); - for (ShardRouting replica : replicaRouting) { - if (replica.state().toString().equals("STARTED")) { - IndexShard replicaShard = getIndexShard(replica, index); - if (replicaShard.indexSettings().isSegRepEnabledOrRemoteNode()) { - assertEquals( - "replica shards haven't caught up with primary", - getLatestSegmentInfoVersion(primaryShard), - getLatestSegmentInfoVersion(replicaShard) - ); - } + for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) { + final ShardRouting primaryRouting = shardRoutingTable.primaryShard(); + if (primaryRouting.state() == ShardRoutingState.STARTED) { + if (isSegmentReplicationEnabledForIndex(index)) { + final List replicaRouting = shardRoutingTable.replicaShards(); + final IndexShard primaryShard = getIndexShard(primaryRouting, index); + for (ShardRouting replica : replicaRouting) { + if (replica.state() == ShardRoutingState.STARTED) { + IndexShard replicaShard = getIndexShard(replica, index); + if (replicaShard.indexSettings().isSegRepEnabledOrRemoteNode()) { + assertEquals( + "replica shards haven't caught up with primary", + getLatestSegmentInfoVersion(primaryShard), + getLatestSegmentInfoVersion(replicaShard) + ); } + } else if (replica.state() == ShardRoutingState.INITIALIZING) { + fail("replica shard still INITIALIZING, not caught up with primary"); } } } } - }, 30, TimeUnit.SECONDS); + } } - } else { - throw new IllegalStateException( - "Segment Replication is not supported for testing tests using External Test Cluster" - ); - } + }, 30, TimeUnit.SECONDS); } } } catch (Exception e) { From bd8e81027a62750d51e35e4d612f05ce6dca5c94 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 6 May 2026 13:20:09 -0700 Subject: [PATCH 061/115] Add 3 different types of PPL scalar functions to analytics-engine - prove wiring based on DataFusion capabilities (#21476) * This PR adds three functions to illustrate implementation for different types of scalar functions and supporting them with backend-datafusion. 1. year(ts) - rewrite to date_part using an adapter. 2. convert_tz - rewrites to rust side UDF 3. unix_timestamp - rewrite to DataFusion native to_unixtime(ts) Signed-off-by: Marc Handalian * Consolidate and clean up scalar_functions yaml. Signed-off-by: Marc Handalian * add missing udf registration sites Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../spi/AbstractNameMappingAdapter.java | 96 ++++ .../analytics/spi/ScalarFunction.java | 13 +- .../rust/Cargo.toml | 3 + .../rust/src/api.rs | 1 + .../rust/src/indexed_executor.rs | 1 + .../rust/src/lib.rs | 1 + .../rust/src/local_executor.rs | 6 +- .../rust/src/query_executor.rs | 1 + .../rust/src/session_context.rs | 1 + .../rust/src/udf/convert_tz.rs | 519 ++++++++++++++++++ .../rust/src/udf/mod.rs | 272 +++++++++ .../be/datafusion/BaseScalarFunctionIT.java | 204 +++++++ .../datafusion/ScalarDateTimeFunctionIT.java | 42 ++ .../be/datafusion/ConvertTzAdapter.java | 191 +++++++ .../DataFusionAnalyticsBackendPlugin.java | 10 +- .../DataFusionFragmentConvertor.java | 6 +- .../be/datafusion/UnixTimestampAdapter.java | 60 ++ .../opensearch/be/datafusion/YearAdapter.java | 33 ++ .../opensearch_scalar_functions.yaml | 18 + .../be/datafusion/ConvertTzAdapterTests.java | 228 ++++++++ .../datafusion/UnixTimestampAdapterTests.java | 112 ++++ .../be/datafusion/YearAdapterTests.java | 116 ++++ 22 files changed, 1925 insertions(+), 9 deletions(-) create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AbstractNameMappingAdapter.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/convert_tz.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConvertTzAdapter.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UnixTimestampAdapter.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/YearAdapter.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ConvertTzAdapterTests.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/UnixTimestampAdapterTests.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/YearAdapterTests.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AbstractNameMappingAdapter.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AbstractNameMappingAdapter.java new file mode 100644 index 0000000000000..b093b434e2ce3 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AbstractNameMappingAdapter.java @@ -0,0 +1,96 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.SqlTypeName; + +import java.util.ArrayList; +import java.util.List; + +/** + * Reusable base for {@link ScalarFunctionAdapter}s that rewrite a Calcite call + * to a different named target, optionally prepending or appending literal + * operands. Pure shape rewriting — no decomposition into a different semantic + * function. For that use case (e.g. {@code ILIKE → LIKE(LOWER(a), LOWER(b))}) + * write a dedicated adapter instead. + * + *

      Example use: + * 

      + *   class YearAdapter extends AbstractNameMappingAdapter {
+ *       YearAdapter() {
+ *           super(SqlLibraryOperators.DATE_PART, List.of("year"), List.of());
+ *       }
+ *   }
+ *
      + * rewrites {@code YEAR(ts)} to {@code date_part('year', ts)}. Paired with the + * {@code date_part} signature in a backend's extension catalog so the isthmus + * visitor resolves it against the backend's native date_part. + * + * @opensearch.internal + */ +public abstract class AbstractNameMappingAdapter implements ScalarFunctionAdapter { + + private final SqlOperator targetOperator; + private final List prependLiterals; + private final List appendLiterals; + + /** + * @param targetOperator the Calcite {@link SqlOperator} the rewritten call + * will use. The isthmus visitor resolves this to a + * Substrait invocation against the backend's loaded + * extension catalog. + * @param prependLiterals literals to prepend to the operand list (e.g. + * {@code List.of("year")} to prepend a string literal). + * Currently supports {@link String}, {@link Integer}, + * {@link Long}, {@link Double}, {@link Boolean}. + * @param appendLiterals literals to append to the operand list. + */ + protected AbstractNameMappingAdapter(SqlOperator targetOperator, List prependLiterals, List appendLiterals) { + this.targetOperator = targetOperator; + this.prependLiterals = List.copyOf(prependLiterals); + this.appendLiterals = List.copyOf(appendLiterals); + } + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + RexBuilder rexBuilder = cluster.getRexBuilder(); + List operands = new ArrayList<>(original.getOperands().size() + prependLiterals.size() + appendLiterals.size()); + for (Object literal : prependLiterals) { + operands.add(rexBuilder.makeLiteral(literal, inferLiteralType(rexBuilder, literal), true)); + } + operands.addAll(original.getOperands()); + for (Object literal : appendLiterals) { + operands.add(rexBuilder.makeLiteral(literal, inferLiteralType(rexBuilder, literal), true)); + } + // Preserve the original call's return type. The enclosing operator (Project + // / Filter) caches its rowType from the pre-adaptation expression; if the + // rewritten call's Calcite-inferred type differs (e.g. PPL YEAR returns + // INTEGER but SqlLibraryOperators.DATE_PART is SqlExtractFunction → BIGINT), + // the downstream stripAnnotations path feeds the adapted expr into + // LogicalProject.create together with the cached rowType, and + // Project.isValid's compatibleTypes check throws an AssertionError that + // breaks fragment conversion. + return rexBuilder.makeCall(original.getType(), targetOperator, operands); + } + + private static org.apache.calcite.rel.type.RelDataType inferLiteralType(RexBuilder rexBuilder, Object literal) { + var typeFactory = rexBuilder.getTypeFactory(); + if (literal instanceof String) return typeFactory.createSqlType(SqlTypeName.VARCHAR); + if (literal instanceof Integer) return typeFactory.createSqlType(SqlTypeName.INTEGER); + if (literal instanceof Long) return typeFactory.createSqlType(SqlTypeName.BIGINT); + if (literal instanceof Double) return typeFactory.createSqlType(SqlTypeName.DOUBLE); + if (literal instanceof Boolean) return typeFactory.createSqlType(SqlTypeName.BOOLEAN); + throw new IllegalArgumentException("Unsupported literal type: " + literal.getClass()); + } +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java index c8dd9b085961e..13cbc837a8056 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java @@ -77,7 +77,10 @@ public enum ScalarFunction { EXTRACT(Category.SCALAR, SqlKind.EXTRACT), // ── Datetime ──────────────────────────────────────────────────── - TIMESTAMP(Category.SCALAR, SqlKind.OTHER_FUNCTION); + TIMESTAMP(Category.SCALAR, SqlKind.OTHER_FUNCTION), + YEAR(Category.SCALAR, SqlKind.OTHER_FUNCTION), + CONVERT_TZ(Category.SCALAR, SqlKind.OTHER_FUNCTION), + UNIX_TIMESTAMP(Category.SCALAR, SqlKind.OTHER_FUNCTION); /** * Category of scalar function. @@ -87,7 +90,9 @@ public enum Category { FULL_TEXT, STRING, MATH, - /** Catch-all for functions that don't fit other categories (CAST, CASE, COALESCE, EXTRACT, etc.). */ + /** + * Catch-all for functions that don't fit other categories (CAST, CASE, COALESCE, EXTRACT, etc.). + */ SCALAR } @@ -121,7 +126,9 @@ public static ScalarFunction fromSqlKind(SqlKind kind) { return null; } - /** Maps a Calcite SqlFunction to a ScalarFunction by name, or throws if not recognized. */ + /** + * Maps a Calcite SqlFunction to a ScalarFunction by name, or null if not recognized. + */ public static ScalarFunction fromSqlFunction(SqlFunction function) { // TODO: Add an explicit functionName field per enum constant instead of relying on // valueOf(toUpperCase). This couples enum constant naming to SQL function naming convention. diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml index 1f80c7685c408..afa0f1d9ddf1a 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml +++ b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml @@ -42,6 +42,9 @@ chrono = { workspace = true } roaring = "0.10" thiserror = { workspace = true } +# convert_tz UDF +chrono-tz = "0.10" + [dev-dependencies] criterion = { workspace = true } tempfile = { workspace = true } diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs index 815232ff4749d..4197f6355719d 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs @@ -443,6 +443,7 @@ pub unsafe fn sql_to_substrait( .with_default_features() .build(); let ctx = datafusion::prelude::SessionContext::new_with_state(state); + crate::udf::register_all(&ctx); let listing_options = ListingOptions::new(Arc::new(ParquetFormat::new())) .with_file_extension(".parquet") diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs index 12ce9342f5989..f22735cb06f0c 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs @@ -133,6 +133,7 @@ pub async fn execute_indexed_query( .build(); let ctx = SessionContext::new_with_state(state); ctx.register_udf(create_index_filter_udf()); + crate::udf::register_all(&ctx); // Resolve the object store for this shard's table URL (file://, s3://, // gs://, ... whatever the global runtime has registered). We pass this diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs index 5a8d8f36355ad..c883e8d5992d3 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs @@ -30,3 +30,4 @@ pub mod query_memory_pool_tracker; pub mod runtime_manager; pub mod session_context; pub mod statistics_cache; +pub mod udf; diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs index d62ec523b477f..90d267d71a771 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs @@ -70,9 +70,9 @@ impl LocalSession { .with_runtime_env(runtime_env) .with_default_features() .build(); - Self { - ctx: SessionContext::new_with_state(state), - } + let ctx = SessionContext::new_with_state(state); + crate::udf::register_all(&ctx); + Self { ctx } } /// Registers a streaming input on the session under `name` and returns the diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs index 8ba9c93b3caea..14c8d172add9a 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs @@ -100,6 +100,7 @@ pub async fn execute_query( .build(); let ctx = SessionContext::new_with_state(state); + crate::udf::register_all(&ctx); // Register table via ListingTable — all IO goes through object store let file_format = ParquetFormat::new(); diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs index 9de9caaa968a5..d10887be4c1a9 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs @@ -99,6 +99,7 @@ pub async unsafe fn create_session_context( .build(); let ctx = SessionContext::new_with_state(state); + crate::udf::register_all(&ctx); // Register default ListingTable for parquet scans let listing_options = ListingOptions::new(Arc::new(ParquetFormat::new())) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/convert_tz.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/convert_tz.rs new file mode 100644 index 0000000000000..6ae7c4199640d --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/convert_tz.rs @@ -0,0 +1,519 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! `convert_tz(ts, from_tz, to_tz)` — shift a timestamp from one timezone to another. +//! +//! # Division of labor with the Java adapter +//! +//! Literal validation + canonicalization happens Java-side in +//! `ConvertTzAdapter` (see `.../be/datafusion/ConvertTzAdapter.java`), which +//! runs at plan time: +//! * bad literals (unknown IANA, malformed offset) surface as +//! `IllegalArgumentException` at plan time — users see the error instantly. +//! * literal tz operands arrive here already canonicalized (`+05:00`, not +//! `+5:00`; JDK-normalized IANA ids). +//! * identity cases (`from == to`) are short-circuited plan-side and never +//! reach this UDF. +//! +//! What stays here: +//! * **Per-row DST-correct shifting.** IANA offsets vary per instant; can't +//! be folded at plan time. +//! * **Column-valued tz operands.** Values aren't known until runtime; +//! unparseable entries yield NULL rows (matches MySQL's lenient +//! `CONVERT_TZ` behavior). +//! +//! Semantics (MySQL-compatible): +//! * `ts` is interpreted as a wall-clock time in `from_tz`. +//! * The return is the wall-clock time in `to_tz` for the same instant. +//! * Timezone strings may be IANA names (`'America/New_York'`) or ISO offsets +//! of the form `±HH:MM` with hours ∈ [0,14], minutes ∈ [0,59]. +//! * Any null input → null output (null propagation). +//! * Unparseable column-valued timezone → null output. + +use std::any::Any; +use std::sync::Arc; + +use chrono::{DateTime, NaiveDateTime, Offset, TimeZone, Utc}; +use chrono_tz::Tz; +use datafusion::arrow::array::{ + Array, ArrayRef, StringArray, TimestampMillisecondArray, TimestampMillisecondBuilder, +}; +use datafusion::arrow::datatypes::{DataType, TimeUnit}; +use datafusion::common::{plan_err, ScalarValue}; +use datafusion::error::{DataFusionError, Result}; +use datafusion::execution::context::SessionContext; +use datafusion::logical_expr::{ + ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility, +}; + +use super::{coerce_args, CoerceMode}; + +pub fn register_all(ctx: &SessionContext) { + ctx.register_udf(ScalarUDF::from(ConvertTzUdf::new())); +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct ConvertTzUdf { + signature: Signature, +} + +impl ConvertTzUdf { + pub fn new() -> Self { + // PPL emits `convert_tz(ts, from, to)` with ts typed as Utf8 (string + // literal), Date32, or Timestamp(any precision, any tz). Signature::exact + // only let through the Timestamp(Ms, None) variant; user_defined + + // coerce_types lets DF insert the right casts. + Self { + signature: Signature::user_defined(Volatility::Immutable), + } + } +} + +impl Default for ConvertTzUdf { + fn default() -> Self { + Self::new() + } +} + +impl ScalarUDFImpl for ConvertTzUdf { + fn as_any(&self) -> &dyn Any { + self + } + fn name(&self) -> &str { + "convert_tz" + } + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + if arg_types.len() != 3 { + return plan_err!("convert_tz expects 3 arguments, got {}", arg_types.len()); + } + Ok(DataType::Timestamp(TimeUnit::Millisecond, None)) + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + coerce_args( + "convert_tz", + arg_types, + &[CoerceMode::TimestampMs, CoerceMode::Utf8, CoerceMode::Utf8], + ) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + if args.args.len() != 3 { + return plan_err!("convert_tz expects 3 arguments, got {}", args.args.len()); + } + let n = args.number_rows; + + // Fast-path: scalar tz operands are parsed once up front, not per row. + // The Java adapter canonicalizes literal tz strings at plan time so + // bad-literal input can't reach this UDF — a None from parse_tz on a + // scalar therefore means the scalar was SQL NULL. + let from_scalar = scalar_tz(&args.args[1]); + let to_scalar = scalar_tz(&args.args[2]); + + let ts = args.args[0].clone().into_array(n)?; + let ts = ts + .as_any() + .downcast_ref::() + .ok_or_else(|| { + DataFusionError::Internal(format!( + "convert_tz: expected TimestampMillisecond, got {:?}", + ts.data_type() + )) + })?; + + // Only materialize column-valued tz operands; for scalars the parsed + // TzSpec is already in hand. Keep the ArrayRef alive alongside the + // downcast reference — StringArray borrows from the underlying buffer. + let from_arr_ref: Option = if from_scalar.is_none() && matches!(&args.args[1], ColumnarValue::Array(_)) { + Some(materialize_string_array(&args.args[1], n, "from_tz")?) + } else { + None + }; + let to_arr_ref: Option = if to_scalar.is_none() && matches!(&args.args[2], ColumnarValue::Array(_)) { + Some(materialize_string_array(&args.args[2], n, "to_tz")?) + } else { + None + }; + let from_array: Option<&StringArray> = from_arr_ref.as_ref().and_then(|a| a.as_any().downcast_ref::()); + let to_array: Option<&StringArray> = to_arr_ref.as_ref().and_then(|a| a.as_any().downcast_ref::()); + + let mut builder = TimestampMillisecondBuilder::with_capacity(n); + for i in 0..n { + if ts.is_null(i) { + builder.append_null(); + continue; + } + let from = match (&from_scalar, from_array) { + (Some(tz), _) => tz.clone(), + (None, Some(arr)) if !arr.is_null(i) => match parse_tz(arr.value(i)) { + Some(tz) => tz, + None => { + builder.append_null(); + continue; + } + }, + _ => { + builder.append_null(); + continue; + } + }; + let to = match (&to_scalar, to_array) { + (Some(tz), _) => tz.clone(), + (None, Some(arr)) if !arr.is_null(i) => match parse_tz(arr.value(i)) { + Some(tz) => tz, + None => { + builder.append_null(); + continue; + } + }, + _ => { + builder.append_null(); + continue; + } + }; + match shift_millis_parsed(ts.value(i), &from, &to) { + Some(v) => builder.append_value(v), + None => builder.append_null(), + } + } + Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef)) + } +} + +/// If `cv` is a non-NULL string scalar, parse it once. Returns None for NULL, +/// non-scalar, or an unparseable string (the latter unreachable from literal +/// paths — Java canonicalizes — but defensive for degenerate scalar inputs). +fn scalar_tz(cv: &ColumnarValue) -> Option { + if let ColumnarValue::Scalar(sv) = cv { + let s = match sv { + ScalarValue::Utf8(opt) | ScalarValue::LargeUtf8(opt) | ScalarValue::Utf8View(opt) => opt.as_deref(), + _ => None, + }; + return s.and_then(parse_tz); + } + None +} + +fn materialize_string_array(cv: &ColumnarValue, n: usize, label: &'static str) -> Result { + let arr = cv.clone().into_array(n)?; + if arr.as_any().downcast_ref::().is_none() { + return Err(DataFusionError::Internal(format!( + "convert_tz: {} expected Utf8, got {:?}", + label, + arr.data_type() + ))); + } + Ok(arr) +} + +/// Parse timezone string (IANA name or `±HH:MM` offset). +#[derive(Clone)] +enum TzSpec { + Iana(Tz), + /// Fixed offset in seconds east of UTC. + Offset(i32), +} + +fn parse_tz(s: &str) -> Option { + if let Some(off) = parse_offset_seconds(s) { + return Some(TzSpec::Offset(off)); + } + s.parse::().ok().map(TzSpec::Iana) +} + +/// Parse `±HH:MM` → seconds east of UTC; None if not an offset literal. +/// +/// Bounds ({@code hours ∈ [0,14], minutes ∈ [0,59]}) match the Java adapter's +/// {@code canonicalizeTz}. For literal-path inputs the Java side has already +/// validated and canonicalized, so the defensive checks here only fire for +/// column-valued tz — where a malformed entry yields a NULL row, matching the +/// documented lenient behavior. +fn parse_offset_seconds(s: &str) -> Option { + let bytes = s.as_bytes(); + if bytes.len() != 6 { + return None; + } + let sign = match bytes[0] { + b'+' => 1, + b'-' => -1, + _ => return None, + }; + if bytes[3] != b':' { + return None; + } + let hours: i32 = s.get(1..3)?.parse().ok()?; + let minutes: i32 = s.get(4..6)?.parse().ok()?; + if hours > 14 || minutes > 59 { + return None; + } + Some(sign * (hours * 3600 + minutes * 60)) +} + +/// Shift `ts_millis` from `from` to `to` using pre-parsed [`TzSpec`]s. The +/// stored timestamp has no tz attached — interpret its wall clock in `from`, +/// render that instant in `to`, then return the shifted millis as a tz-free +/// value the caller can continue to treat as naive. The shift is exactly +/// `to_offset(ts) - from_offset(ts)` milliseconds. +fn shift_millis_parsed(ts_millis: i64, from: &TzSpec, to: &TzSpec) -> Option { + let naive = DateTime::::from_timestamp_millis(ts_millis)?.naive_utc(); + let from_off = offset_seconds_at(from, &naive)?; + let to_off = offset_seconds_at_instant(to, ts_millis, from_off)?; + let delta_millis = (to_off - from_off) as i64 * 1_000; + ts_millis.checked_add(delta_millis) +} + +/// String-operand wrapper retained for direct-invocation tests that exercise +/// the full parse + shift flow in one call. +#[cfg(test)] +fn shift_millis(ts_millis: i64, from_tz: &str, to_tz: &str) -> Option { + let from = parse_tz(from_tz)?; + let to = parse_tz(to_tz)?; + shift_millis_parsed(ts_millis, &from, &to) +} + +/// Offset (seconds east of UTC) for `from_tz` at wall-clock `naive`. +fn offset_seconds_at(tz: &TzSpec, naive: &NaiveDateTime) -> Option { + match tz { + TzSpec::Offset(o) => Some(*o), + TzSpec::Iana(z) => { + // Use .from_local_datetime → pick the earliest resolution for ambiguous + // (DST-fall-back) wall times, which matches MySQL's behaviour. + match z.from_local_datetime(naive) { + chrono::LocalResult::Single(dt) => Some(dt.offset().fix().local_minus_utc()), + chrono::LocalResult::Ambiguous(dt, _) => Some(dt.offset().fix().local_minus_utc()), + chrono::LocalResult::None => None, // wall time in the DST "spring-forward" gap + } + } + } +} + +/// Offset (seconds east of UTC) for `to_tz` at the UTC *instant* represented by +/// the input. We reconstruct the instant from `ts_millis` + `from_offset` (since +/// `ts_millis` is a wall clock in from_tz), then look up to_tz's offset at that +/// instant — DST-correct even across transitions. +fn offset_seconds_at_instant( + tz: &TzSpec, + ts_millis: i64, + from_offset_seconds: i32, +) -> Option { + match tz { + TzSpec::Offset(o) => Some(*o), + TzSpec::Iana(z) => { + // instant_utc_millis = wall_millis - from_offset_millis + let instant_millis = + ts_millis.checked_sub((from_offset_seconds as i64) * 1_000)?; + let instant = DateTime::::from_timestamp_millis(instant_millis)?; + Some(z.offset_from_utc_datetime(&instant.naive_utc()).fix().local_minus_utc()) + } + } +} + +// ─── tests ────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + // ±HH:MM offsets parse to the expected second counts. + #[test] + fn parse_offset_accepts_positive_and_negative() { + assert_eq!(parse_offset_seconds("+00:00"), Some(0)); + assert_eq!(parse_offset_seconds("+05:30"), Some(5 * 3600 + 30 * 60)); + assert_eq!(parse_offset_seconds("-08:00"), Some(-8 * 3600)); + assert_eq!(parse_offset_seconds("+14:00"), Some(14 * 3600)); + } + + #[test] + fn parse_offset_rejects_malformed() { + assert_eq!(parse_offset_seconds("bogus"), None); + assert_eq!(parse_offset_seconds("0500"), None); + // Hour >14 is beyond canonicalization bounds — Java rejects at plan time, + // we reject at runtime for column-valued paths. + assert_eq!(parse_offset_seconds("+15:00"), None); + assert_eq!(parse_offset_seconds("+05:60"), None); + } + + // Offset → offset: simple wall-clock delta, no calendar. + #[test] + fn fixed_offset_to_fixed_offset_shifts_by_delta() { + // 2024-01-05T12:00:00 in +00:00 → same wall clock in +05:30 means + // +5h30m = +19_800_000 ms added. + let ts = 1_704_456_000_000; // 2024-01-05T12:00:00Z (stored naive) + let out = shift_millis(ts, "+00:00", "+05:30").unwrap(); + assert_eq!(out - ts, 5 * 3600 * 1000 + 30 * 60 * 1000); + } + + // IANA ↔ IANA: DST-correct jump across a transition. + #[test] + fn iana_new_york_to_london_applies_correct_offset() { + // 2024-01-05T12:00:00 wall-clock in America/New_York (UTC-5 in winter) + // → 17:00 UTC → London (UTC+0 in winter) = 17:00 local. Delta = +5h. + let ts = 1_704_456_000_000; // treat as 2024-01-05T12:00:00 naive + let out = shift_millis(ts, "America/New_York", "Europe/London").unwrap(); + assert_eq!((out - ts) / 1000, 5 * 3600); + } + + #[test] + fn iana_dst_summer_offset_differs_from_winter() { + // Summer: NY is UTC-4, winter: NY is UTC-5. Pull data at both dates, + // confirm the two shifts to UTC (London+0 in winter, +1 in summer) produce + // the expected distinct deltas. + // 2024-01-05T12:00:00 (winter): NY→London → +5h. + let winter_ts = 1_704_456_000_000; + let winter_out = shift_millis(winter_ts, "America/New_York", "Europe/London").unwrap(); + assert_eq!((winter_out - winter_ts) / 1000, 5 * 3600); + // 2024-07-05T12:00:00 (summer): NY (UTC-4) → London (UTC+1) → +5h. + // Same delta because both shift to/from their summer offsets in lockstep. + let summer_ts = 1_720_180_800_000; // 2024-07-05T12:00:00Z naive + let summer_out = shift_millis(summer_ts, "America/New_York", "Europe/London").unwrap(); + assert_eq!((summer_out - summer_ts) / 1000, 5 * 3600); + } + + // When from_tz crosses DST boundary but to_tz doesn't, the delta changes. + #[test] + fn iana_to_utc_crosses_dst_in_source_tz() { + // 2024-01-05 in UTC (no DST there): NY winter = UTC-5, shift = +5h. + let winter_ts = 1_704_456_000_000; + let winter_out = shift_millis(winter_ts, "America/New_York", "UTC").unwrap(); + assert_eq!((winter_out - winter_ts) / 1000, 5 * 3600); + + // 2024-07-05: NY summer = UTC-4, shift = +4h. + let summer_ts = 1_720_180_800_000; + let summer_out = shift_millis(summer_ts, "America/New_York", "UTC").unwrap(); + assert_eq!((summer_out - summer_ts) / 1000, 4 * 3600); + } + + #[test] + fn unknown_tz_returns_none() { + assert_eq!(shift_millis(0, "Not/AZone", "UTC"), None); + assert_eq!(shift_millis(0, "UTC", "Not/AZone"), None); + } + + // Coercion: PPL may emit the ts arg as Utf8 (string literal), Date32, + // or Timestamp with a different precision/tz. coerce_types should + // normalize them all to Timestamp(Millisecond, None) + Utf8 + Utf8. + #[test] + fn coerce_types_accepts_utf8_ts() { + let udf = ConvertTzUdf::new(); + let out = udf + .coerce_types(&[DataType::Utf8, DataType::Utf8, DataType::Utf8]) + .unwrap(); + assert_eq!( + out, + vec![ + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Utf8, + DataType::Utf8, + ] + ); + } + + #[test] + fn coerce_types_accepts_date32_ts() { + let udf = ConvertTzUdf::new(); + let out = udf + .coerce_types(&[DataType::Date32, DataType::Utf8, DataType::Utf8]) + .unwrap(); + assert_eq!(out[0], DataType::Timestamp(TimeUnit::Millisecond, None)); + } + + #[test] + fn coerce_types_accepts_other_ts_precisions() { + let udf = ConvertTzUdf::new(); + // Nanosecond with tz → should coerce down to Millisecond, None. + let out = udf + .coerce_types(&[ + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + DataType::Utf8, + DataType::Utf8, + ]) + .unwrap(); + assert_eq!(out[0], DataType::Timestamp(TimeUnit::Millisecond, None)); + } + + #[test] + fn coerce_types_passes_through_exact_match() { + let udf = ConvertTzUdf::new(); + let ts = DataType::Timestamp(TimeUnit::Millisecond, None); + let out = udf + .coerce_types(&[ts.clone(), DataType::Utf8, DataType::Utf8]) + .unwrap(); + assert_eq!(out, vec![ts, DataType::Utf8, DataType::Utf8]); + } + + #[test] + fn coerce_types_rejects_unsupported_ts_type() { + let udf = ConvertTzUdf::new(); + // A boolean in the ts slot is clearly wrong — must error explicitly. + let err = udf + .coerce_types(&[DataType::Boolean, DataType::Utf8, DataType::Utf8]) + .unwrap_err(); + let msg = format!("{err}"); + assert!( + msg.contains("convert_tz") && msg.contains("Boolean"), + "unexpected error: {msg}" + ); + } + + #[test] + fn coerce_types_rejects_wrong_arity() { + let udf = ConvertTzUdf::new(); + assert!(udf.coerce_types(&[DataType::Utf8]).is_err()); + assert!(udf + .coerce_types(&[DataType::Utf8, DataType::Utf8, DataType::Utf8, DataType::Utf8]) + .is_err()); + } + + // Batch / null handling through the full UDF. + #[test] + fn invoke_nulls_and_bad_tz_propagate() { + let udf = ConvertTzUdf::new(); + let ts = TimestampMillisecondArray::from(vec![ + Some(1_704_456_000_000), + None, + Some(0), + ]); + let from = StringArray::from(vec![ + Some("+00:00"), + Some("UTC"), + Some("Mars/Olympus"), // unknown column-valued entry → null + ]); + let to = StringArray::from(vec![Some("+05:30"), Some("UTC"), Some("UTC")]); + let args = ScalarFunctionArgs { + args: vec![ + ColumnarValue::Array(Arc::new(ts)), + ColumnarValue::Array(Arc::new(from)), + ColumnarValue::Array(Arc::new(to)), + ], + number_rows: 3, + arg_fields: vec![], + return_field: Arc::new(datafusion::arrow::datatypes::Field::new( + "out", + DataType::Timestamp(TimeUnit::Millisecond, None), + true, + )), + config_options: Arc::new(datafusion::config::ConfigOptions::new()), + }; + let out = udf.invoke_with_args(args).unwrap(); + let arr = match out { + ColumnarValue::Array(a) => a, + _ => panic!("expected array"), + }; + let arr = arr + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!arr.is_null(0)); + assert!(arr.is_null(1)); + assert!(arr.is_null(2)); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs new file mode 100644 index 0000000000000..3d13de51967b6 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs @@ -0,0 +1,272 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! OpenSearch scalar UDFs that aren't in DataFusion's built-in registry. Each +//! must have a matching YAML entry in `extensions/opensearch_scalar.yaml` so +//! the substrait converter on the Java side can route to it by name. +//! +//! Functions registered here: +//! - `convert_tz(ts, from_tz, to_tz)` — DST-aware timezone shift (chrono-tz) + +use datafusion::arrow::datatypes::{DataType, TimeUnit}; +use datafusion::common::plan_err; +use datafusion::error::Result; +use datafusion::execution::context::SessionContext; + +/// Categories of input type a UDF slot can accept. Each mode declares a +/// canonical target arrow type plus the set of sources that coerce to it. +/// UDFs use `Signature::user_defined()` and call [`coerce_slot`] per +/// argument position to produce the `coerce_types` output. +/// +/// Invalid sources produce an explicit `plan_err!` — no silent fallback. The +/// failure message names the UDF, the slot index, the observed type and the +/// expected canonical type so planning errors are actionable. +#[derive(Clone, Copy, Debug)] +#[allow(dead_code)] +pub(crate) enum CoerceMode { + /// Accept Utf8 / Date32 / Timestamp(any precision, any tz) → canonicalize + /// to Timestamp(Millisecond, None). DF has built-in casts for each source. + TimestampMs, + /// Accept Utf8 / Date32 / Timestamp(any, any) → canonicalize to Date32. + Date32, + /// Accept any integer or float → Int64. + Int64, + /// Accept any integer or float → Float64. + Float64, + /// Accept Utf8 / LargeUtf8 / Utf8View → Utf8. + Utf8, +} + +/// Coerce a single argument slot. Returns the canonical target type for this +/// slot when the input is compatible, or a planning error otherwise. +pub(crate) fn coerce_slot( + udf_name: &str, + slot_index: usize, + observed: &DataType, + mode: CoerceMode, +) -> Result { + use DataType::*; + match mode { + CoerceMode::TimestampMs => match observed { + Timestamp(_, _) | Date32 | Date64 | Utf8 | LargeUtf8 | Utf8View => { + Ok(Timestamp(TimeUnit::Millisecond, None)) + } + other => plan_err!( + "{udf_name}: arg {slot_index} expected timestamp/date/string, got {other:?}" + ), + }, + CoerceMode::Date32 => match observed { + Date32 | Date64 | Timestamp(_, _) | Utf8 | LargeUtf8 | Utf8View => Ok(Date32), + other => plan_err!( + "{udf_name}: arg {slot_index} expected date/timestamp/string, got {other:?}" + ), + }, + CoerceMode::Int64 => match observed { + Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 | Float32 | Float64 => { + Ok(Int64) + } + other => plan_err!( + "{udf_name}: arg {slot_index} expected integer or float, got {other:?}" + ), + }, + CoerceMode::Float64 => match observed { + Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 | Float32 | Float64 => { + Ok(Float64) + } + other => plan_err!( + "{udf_name}: arg {slot_index} expected integer or float, got {other:?}" + ), + }, + CoerceMode::Utf8 => match observed { + Utf8 | LargeUtf8 | Utf8View => Ok(Utf8), + other => plan_err!( + "{udf_name}: arg {slot_index} expected string, got {other:?}" + ), + }, + } +} + +/// Coerce an entire argument vector against a fixed template. Enforces arity +/// and delegates per-slot coercion to [`coerce_slot`]. +pub(crate) fn coerce_args( + udf_name: &str, + observed: &[DataType], + template: &[CoerceMode], +) -> Result> { + if observed.len() != template.len() { + return plan_err!( + "{udf_name} expects {} arguments, got {}", + template.len(), + observed.len() + ); + } + template + .iter() + .enumerate() + .map(|(i, mode)| coerce_slot(udf_name, i, &observed[i], *mode)) + .collect() +} + +pub mod convert_tz; + +pub fn register_all(ctx: &SessionContext) { + convert_tz::register_all(ctx); + log::info!("OpenSearch UDF register_all: convert_tz registered"); +} + +#[cfg(test)] +mod tests { + //! Direct tests for the [`CoerceMode`] helper library. `convert_tz` exercises + //! `TimestampMs` and `Utf8` through its public `coerce_types`; these tests + //! cover every mode's accept + reject paths so future UDFs that pick up + //! `Date32`, `Int64`, or `Float64` inherit a proven helper rather than being + //! the first caller. + use super::{coerce_args, coerce_slot, CoerceMode}; + use datafusion::arrow::datatypes::{DataType, TimeUnit}; + + fn ts_ms() -> DataType { + DataType::Timestamp(TimeUnit::Millisecond, None) + } + + // ── TimestampMs ──────────────────────────────────────────────────────── + #[test] + fn timestampms_accepts_every_temporal_source() { + for observed in [ + DataType::Utf8, + DataType::LargeUtf8, + DataType::Utf8View, + DataType::Date32, + DataType::Date64, + DataType::Timestamp(TimeUnit::Second, None), + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + ] { + let result = coerce_slot("t", 0, &observed, CoerceMode::TimestampMs).unwrap(); + assert_eq!(result, ts_ms(), "TimestampMs should canonicalize {observed:?}"); + } + } + + #[test] + fn timestampms_rejects_numeric() { + let err = coerce_slot("t", 0, &DataType::Int64, CoerceMode::TimestampMs).unwrap_err(); + assert!(err.to_string().contains("expected timestamp/date/string")); + } + + // ── Date32 ───────────────────────────────────────────────────────────── + #[test] + fn date32_accepts_date_and_string_sources() { + for observed in [ + DataType::Date32, + DataType::Date64, + DataType::Utf8, + DataType::LargeUtf8, + DataType::Utf8View, + DataType::Timestamp(TimeUnit::Millisecond, None), + ] { + let result = coerce_slot("d", 0, &observed, CoerceMode::Date32).unwrap(); + assert_eq!(result, DataType::Date32); + } + } + + #[test] + fn date32_rejects_numeric() { + let err = coerce_slot("d", 0, &DataType::Float64, CoerceMode::Date32).unwrap_err(); + assert!(err.to_string().contains("expected date/timestamp/string")); + } + + // ── Int64 ────────────────────────────────────────────────────────────── + #[test] + fn int64_accepts_every_number() { + for observed in [ + DataType::Int8, + DataType::Int16, + DataType::Int32, + DataType::Int64, + DataType::UInt8, + DataType::UInt16, + DataType::UInt32, + DataType::UInt64, + DataType::Float32, + DataType::Float64, + ] { + let result = coerce_slot("i", 0, &observed, CoerceMode::Int64).unwrap(); + assert_eq!(result, DataType::Int64); + } + } + + #[test] + fn int64_rejects_strings() { + let err = coerce_slot("i", 0, &DataType::Utf8, CoerceMode::Int64).unwrap_err(); + assert!(err.to_string().contains("expected integer or float")); + } + + // ── Float64 ──────────────────────────────────────────────────────────── + #[test] + fn float64_accepts_every_number() { + for observed in [ + DataType::Int32, + DataType::Int64, + DataType::UInt32, + DataType::Float32, + DataType::Float64, + ] { + let result = coerce_slot("f", 0, &observed, CoerceMode::Float64).unwrap(); + assert_eq!(result, DataType::Float64); + } + } + + #[test] + fn float64_rejects_strings() { + let err = coerce_slot("f", 0, &DataType::Utf8, CoerceMode::Float64).unwrap_err(); + assert!(err.to_string().contains("expected integer or float")); + } + + // ── Utf8 ─────────────────────────────────────────────────────────────── + #[test] + fn utf8_accepts_every_string_variant() { + for observed in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] { + let result = coerce_slot("s", 0, &observed, CoerceMode::Utf8).unwrap(); + assert_eq!(result, DataType::Utf8); + } + } + + #[test] + fn utf8_rejects_numeric_and_temporal() { + for observed in [DataType::Int64, DataType::Float64, DataType::Date32] { + let err = coerce_slot("s", 0, &observed, CoerceMode::Utf8).unwrap_err(); + assert!(err.to_string().contains("expected string")); + } + } + + // ── coerce_args ──────────────────────────────────────────────────────── + #[test] + fn coerce_args_maps_each_slot_through_its_mode() { + let observed = [DataType::Utf8, DataType::Int32]; + let template = [CoerceMode::TimestampMs, CoerceMode::Int64]; + let result = coerce_args("multi", &observed, &template).unwrap(); + assert_eq!(result, vec![ts_ms(), DataType::Int64]); + } + + #[test] + fn coerce_args_rejects_arity_mismatch() { + let observed = [DataType::Utf8]; + let template = [CoerceMode::Utf8, CoerceMode::Utf8]; + let err = coerce_args("arity", &observed, &template).unwrap_err(); + assert!(err.to_string().contains("expects 2 arguments, got 1")); + } + + #[test] + fn coerce_args_propagates_slot_errors() { + let observed = [DataType::Utf8, DataType::Utf8]; + let template = [CoerceMode::Utf8, CoerceMode::Int64]; + let err = coerce_args("slot", &observed, &template).unwrap_err(); + assert!( + err.to_string().contains("arg 1"), + "error must name the failing slot index, got: {err}" + ); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java new file mode 100644 index 0000000000000..6dd4a9116aaee --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java @@ -0,0 +1,204 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.opensearch.Version; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.analytics.AnalyticsPlugin; +import org.opensearch.arrow.flight.transport.FlightStreamPlugin; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.composite.CompositeDataFormatPlugin; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.PluginInfo; +import org.opensearch.ppl.TestPPLPlugin; +import org.opensearch.ppl.action.PPLRequest; +import org.opensearch.ppl.action.PPLResponse; +import org.opensearch.ppl.action.UnifiedPPLExecuteAction; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * Shared fixture + scalar-result assert helpers for end-to-end PPL → Calcite → + * Substrait → DataFusion scalar-function tests. + * + *

      Each subclass declares its functions as test methods using the + * {@code assertScalarXxx(expr, expected)} helpers. The query template is fixed: + * {@code source=bank | eval x = | fields x | head 1}. Inputs are + * literals so assertions don't depend on the bank fixture's data — the test + * exercises the function's name lookup, type inference, and runtime, not + * arithmetic on rows. + * + * @opensearch.internal + */ +// TEST-scope cluster per method — slower but eliminates cluster-reuse degradation that +// surfaces as cascading NodeDisconnectedException when many test methods share a SUITE cluster. +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 1) +public abstract class BaseScalarFunctionIT extends OpenSearchIntegTestCase { + + protected static final String BANK_INDEX = "bank"; + + @Override + protected Collection> nodePlugins() { + return List.of(TestPPLPlugin.class, FlightStreamPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class); + } + + @Override + protected Collection additionalNodePlugins() { + return List.of( + classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) + ); + } + + private static PluginInfo classpathPlugin(Class pluginClass, List extendedPlugins) { + return new PluginInfo( + pluginClass.getName(), + "classpath plugin", + "NA", + Version.CURRENT, + "1.8", + pluginClass.getName(), + null, + extendedPlugins, + false + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .build(); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + // SUITE-scoped cluster is reused across test methods — only create/index once. + if (!indexExists(BANK_INDEX)) { + createBankIndex(); + indexBankDocs(); + ensureGreen(BANK_INDEX); + refresh(BANK_INDEX); + } + } + + private void createBankIndex() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("account_number") + .field("type", "long") + .endObject() + .startObject("firstname") + .field("type", "keyword") + .endObject() + .startObject("balance") + .field("type", "long") + .endObject() + .startObject("created_at") + .field("type", "date") + .endObject() + .endObject() + .endObject(); + + Settings indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats") + .build(); + + CreateIndexResponse response = client().admin() + .indices() + .prepareCreate(BANK_INDEX) + .setSettings(indexSettings) + .setMapping(mapping) + .get(); + assertTrue("bank index creation must be acknowledged", response.isAcknowledged()); + } + + private void indexBankDocs() { + client().prepareIndex(BANK_INDEX) + .setId("1") + .setSource("account_number", 1, "firstname", "Amber", "balance", 39225L, "created_at", "2024-06-15T10:30:00Z") + .get(); + client().prepareIndex(BANK_INDEX) + .setId("6") + .setSource("account_number", 6, "firstname", "Hattie", "balance", 5686L, "created_at", "2024-01-20T14:45:30Z") + .get(); + } + + // ---- Assert helpers ---- + + /** + * Runs the given expression against the single bank row with + * {@code account_number=1} (firstname='Amber', balance=39225) and returns + * the resulting cell. Pinning the row makes assertions deterministic and + * lets tests reference {@code firstname} / {@code balance} as fields — + * which prevents Calcite's constant-folding from optimizing the function + * away at plan time. Tests must therefore use field references to truly + * exercise the Substrait + DataFusion runtime path. + */ + protected Object evalScalar(String expr) { + PPLRequest request = new PPLRequest( + "source=" + BANK_INDEX + " | where account_number = 1 | eval x = " + expr + " | fields x | head 1" + ); + PPLResponse response = client().execute(UnifiedPPLExecuteAction.INSTANCE, request).actionGet(); + assertNotNull("PPLResponse must not be null", response); + assertEquals("schema columns", List.of("x"), response.getColumns()); + assertEquals("head 1 → exactly 1 row", 1, response.getRows().size()); + return response.getRows().get(0)[0]; + } + + protected void assertScalarLong(String expr, long expected) { + Object cell = evalScalar(expr); + assertNotNull(expr + " result must not be null", cell); + assertTrue(expr + " result must be Number, got " + cell.getClass(), cell instanceof Number); + assertEquals(expr, expected, ((Number) cell).longValue()); + } + + protected void assertScalarDouble(String expr, double expected, double delta) { + Object cell = evalScalar(expr); + assertNotNull(expr + " result must not be null", cell); + assertTrue(expr + " result must be Number, got " + cell.getClass(), cell instanceof Number); + assertEquals(expr, expected, ((Number) cell).doubleValue(), delta); + } + + protected void assertScalarString(String expr, String expected) { + Object cell = evalScalar(expr); + assertNotNull(expr + " result must not be null", cell); + assertEquals(expr, expected, cell.toString()); + } + + protected void assertScalarBoolean(String expr, boolean expected) { + Object cell = evalScalar(expr); + assertNotNull(expr + " result must not be null", cell); + assertTrue(expr + " result must be Boolean, got " + cell.getClass(), cell instanceof Boolean); + assertEquals(expr, expected, cell); + } + + protected void assertScalarNull(String expr) { + Object cell = evalScalar(expr); + assertNull(expr + " result must be null but was " + cell, cell); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java new file mode 100644 index 0000000000000..4729a89663b58 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +/** + * End-to-end smoke tests for scalar date/time functions routed through PPL → Calcite → + * Substrait → DataFusion. Bank fixture row 1: created_at='2024-06-15T10:30:00Z'. + * + *

      Two representative cases: + *

        + *
      • {@link #testYear()} — YAML alias with literal-arg injection + * ({@code YEAR(ts) → date_part('year', ts)}).
        • + *
      • {@link #testConvertTz()} — custom Rust UDF registered with DataFusion + * ({@code convert_tz(ts, from_tz, to_tz)}).
        • + *
      + */ +public class ScalarDateTimeFunctionIT extends BaseScalarFunctionIT { + + public void testYear() { + Object cell = evalScalar("year(created_at)"); + assertNotNull("year() must not be null", cell); + assertEquals(2024L, ((Number) cell).longValue()); + } + + public void testConvertTz() { + // row 1: created_at = 2024-06-15T10:30:00Z (UTC). + // Shifted UTC → +10:00 = 2024-06-15T20:30:00Z, unix seconds = 1718483400. + // ConvertTzAdapter rewrites PPL's bespoke CONVERT_TZ to our locally-declared + // SqlFunction("convert_tz") whose Sig is in ADDITIONAL_SCALAR_SIGS; + // UnixTimestampAdapter does the same to to_unixtime. Isthmus resolves both, + // DataFusion runs convert_tz via the Rust UDF and to_unixtime natively. + Object cell = evalScalar("unix_timestamp(convert_tz(created_at, '+00:00', '+10:00'))"); + assertNotNull("convert_tz must not be null", cell); + assertEquals(1718483400L, ((Number) cell).longValue()); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConvertTzAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConvertTzAdapter.java new file mode 100644 index 0000000000000..cb460333dbdaa --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConvertTzAdapter.java @@ -0,0 +1,191 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.AbstractNameMappingAdapter; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; + +import java.time.DateTimeException; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Cat-3b adapter for PPL's {@code CONVERT_TZ(ts, from_tz, to_tz)}. Two jobs in + * priority order: + * + *
        + *
      1. Identity short-circuit: when both tz operands are string + * literals and canonicalize to the same value, the call reduces to its + * timestamp operand. No UDF invocation, no wire traffic.
        2. + *
      2. UDF fallback with canonicalized literal operands: every other + * case rewrites to {@link #LOCAL_CONVERT_TZ_OP} whose + * {@code FunctionMappings.Sig} in {@link DataFusionFragmentConvertor} + * resolves to the {@code convert_tz} Rust UDF. Literal tz operands are + * validated + canonicalized via {@link #canonicalizeTz(String)} at plan + * time so bad literals surface with a clear error rather than silent + * per-row NULL at runtime.
        3. + *
      + * + *

      Why no offset+offset → interval fold: building an interval literal at + * Calcite's level requires {@code org.apache.calcite.avatica.util.TimeUnit}, + * which lives in avatica and is a {@code runtimeOnly} dep of this module. + * Pulling it in just for the fixed-offset case doesn't pay for itself; IANA + * pairs dominate real-world {@code CONVERT_TZ} usage and must go through the + * UDF anyway (per-row DST lookup). + * + *

      The fallback preserves the original call's return type via + * {@code rexBuilder.makeCall(original.getType(), ...)} so the enclosing + * {@code Project} / {@code Filter} rowType cache stays consistent (see + * {@link AbstractNameMappingAdapter} javadoc for background). + * + * @opensearch.internal + */ +class ConvertTzAdapter implements ScalarFunctionAdapter { + + /** + * Locally-declared target operator for the rewrite. {@link SqlKind#OTHER_FUNCTION} + * so it doesn't collide with any Calcite built-in. + * {@link OperandTypes#ANY_STRING_STRING} keeps validation permissive on the + * timestamp slot — real argument vetting happens inside the UDF's + * {@code coerce_types} and {@code invoke_with_args}. + */ + static final SqlOperator LOCAL_CONVERT_TZ_OP = new SqlFunction( + "convert_tz", + SqlKind.OTHER_FUNCTION, + ReturnTypes.ARG0_NULLABLE, + null, + OperandTypes.ANY_STRING_STRING, + SqlFunctionCategory.TIMEDATE + ); + + /** Matches {@code ±H:MM} / {@code ±HH:MM} with hours [0,14] and minutes [0,59]. */ + private static final Pattern OFFSET_PATTERN = Pattern.compile("^([+-])(\\d{1,2}):(\\d{2})$"); + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + RexBuilder rexBuilder = cluster.getRexBuilder(); + List operands = new ArrayList<>(original.getOperands()); + // Slot 0 is the timestamp; slots 1 and 2 are from_tz / to_tz. + for (int slot : new int[] { 1, 2 }) { + operands.set(slot, canonicalizeTzOperand(operands.get(slot), rexBuilder)); + } + + // Identity short-circuit: both operands resolve to the same canonical + // string → the conversion is a no-op. + String fromLiteral = tzLiteralValue(operands.get(1)); + String toLiteral = tzLiteralValue(operands.get(2)); + if (fromLiteral != null && toLiteral != null && fromLiteral.equals(toLiteral)) { + return operands.get(0); + } + + // UDF fallback. Preserve the original call's return type — see + // AbstractNameMappingAdapter for why (Project.isValid compatibleTypes check). + return rexBuilder.makeCall(original.getType(), LOCAL_CONVERT_TZ_OP, operands); + } + + /** + * Returns the string value of a canonicalized tz literal operand, or null + * when the operand is not a VARCHAR/CHAR {@link RexLiteral} (column refs, + * NULL literals, other expressions). + */ + private static String tzLiteralValue(RexNode operand) { + if (!(operand instanceof RexLiteral literal)) return null; + SqlTypeName typeName = literal.getType().getSqlTypeName(); + if (typeName != SqlTypeName.CHAR && typeName != SqlTypeName.VARCHAR) return null; + return literal.getValueAs(String.class); + } + + /** + * If {@code operand} is a string {@link RexLiteral}, canonicalize it and + * return a new literal with the canonical form (or the original if already + * canonical). Non-literal operands (column references, function results) + * pass through untouched — their runtime values can't be validated until + * the UDF runs. + * + *

      Throws {@link IllegalArgumentException} for literals that don't match + * either the {@code ±HH:MM} offset pattern or a known IANA zone id. + */ + private static RexNode canonicalizeTzOperand(RexNode operand, RexBuilder rexBuilder) { + if (!(operand instanceof RexLiteral literal)) { + return operand; + } + SqlTypeName typeName = literal.getType().getSqlTypeName(); + if (typeName != SqlTypeName.CHAR && typeName != SqlTypeName.VARCHAR) { + return operand; + } + String raw = literal.getValueAs(String.class); + if (raw == null) { + // NULL literal — UDF handles null operand at runtime. + return operand; + } + String canonical = canonicalizeTz(raw); + if (canonical.equals(raw)) { + return operand; + } + return rexBuilder.makeLiteral( + canonical, + rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), + literal.getType().isNullable() + ); + } + + /** + * Canonicalize a timezone string. Accepts either: + *

        + *
      • {@code ±H:MM} / {@code ±HH:MM} where hours ∈ [0,14] and minutes ∈ [0,59]; + * returned zero-padded as {@code ±HH:MM}.
        • + *
      • IANA zone id recognized by {@link ZoneId#of(String)}; returned as the + * JDK-normalized form. {@code ZoneId.of} rejects unknown ids, so invalid + * IANA names surface here as {@link IllegalArgumentException}.
        • + *
      + * + *

      The {@code ±HH:MM} bounds match the Rust UDF's {@code parse_offset_seconds} + * (rust/src/udf/convert_tz.rs) — `+14:59` is the maximum offset anywhere on + * Earth (Kiribati is +14:00; the extra minute tolerance matches existing + * UDF behavior). + */ + static String canonicalizeTz(String raw) { + Matcher offset = OFFSET_PATTERN.matcher(raw); + if (offset.matches()) { + String sign = offset.group(1); + int hours = Integer.parseInt(offset.group(2)); + int minutes = Integer.parseInt(offset.group(3)); + if (hours > 14 || minutes > 59) { + throw new IllegalArgumentException( + "convert_tz: invalid offset [" + raw + "] — hours must be in [0, 14] and minutes in [0, 59]" + ); + } + return String.format(Locale.ROOT, "%s%02d:%02d", sign, hours, minutes); + } + try { + // ZoneId.of() throws for unknown ids; the returned ZoneId.getId() + // is the JDK's canonical form (same id for equivalent inputs). + return ZoneId.of(raw).getId(); + } catch (DateTimeException e) { + throw new IllegalArgumentException("convert_tz: invalid timezone [" + raw + "] — expected IANA zone id or ±HH:MM offset", e); + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 221838a98dece..1119f8d8fa17a 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -100,7 +100,10 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP ScalarFunction.MINUS, ScalarFunction.TIMES, ScalarFunction.DIVIDE, - ScalarFunction.MOD + ScalarFunction.MOD, + ScalarFunction.YEAR, + ScalarFunction.CONVERT_TZ, + ScalarFunction.UNIX_TIMESTAMP ); private static final Set AGG_FUNCTIONS = Set.of( @@ -178,7 +181,10 @@ public Map scalarFunctionAdapters() { Map.entry(ScalarFunction.SARG_PREDICATE, new SargAdapter()), Map.entry(ScalarFunction.DIVIDE, new StdOperatorRewriteAdapter("DIVIDE", SqlStdOperatorTable.DIVIDE)), Map.entry(ScalarFunction.MOD, new StdOperatorRewriteAdapter("MOD", SqlStdOperatorTable.MOD)), - Map.entry(ScalarFunction.LIKE, new LikeAdapter()) + Map.entry(ScalarFunction.LIKE, new LikeAdapter()), + Map.entry(ScalarFunction.YEAR, new YearAdapter()), + Map.entry(ScalarFunction.CONVERT_TZ, new ConvertTzAdapter()), + Map.entry(ScalarFunction.UNIX_TIMESTAMP, new UnixTimestampAdapter()) ); } }; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index fe072c85a2e99..e7d67bb5879cf 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -89,7 +89,11 @@ public class DataFusionFragmentConvertor implements FragmentConvertor { */ private static final List ADDITIONAL_SCALAR_SIGS = List.of( FunctionMappings.s(DelegatedPredicateFunction.FUNCTION, DelegatedPredicateFunction.NAME), - FunctionMappings.s(SqlLibraryOperators.ILIKE, "ilike") + FunctionMappings.s(SqlLibraryOperators.ILIKE, "ilike"), + FunctionMappings.s(DelegatedPredicateFunction.FUNCTION, DelegatedPredicateFunction.NAME), + FunctionMappings.s(SqlLibraryOperators.DATE_PART, "date_part"), + FunctionMappings.s(ConvertTzAdapter.LOCAL_CONVERT_TZ_OP, "convert_tz"), + FunctionMappings.s(UnixTimestampAdapter.LOCAL_TO_UNIXTIME_OP, "to_unixtime") ); private final SimpleExtension.ExtensionCollection extensions; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UnixTimestampAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UnixTimestampAdapter.java new file mode 100644 index 0000000000000..2f7056ac92c55 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UnixTimestampAdapter.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.opensearch.analytics.spi.AbstractNameMappingAdapter; + +import java.util.List; + +/** + * Cat-3a rename adapter for PPL's {@code UNIX_TIMESTAMP(ts)}. Rewrites to a + * locally-declared {@link SqlFunction} named {@code to_unixtime} — the name + * DataFusion's substrait consumer recognizes for its native + * {@code ToUnixtimeFunc} (no UDF registration required on the Rust side). + * + *

      Same machinery as {@link ConvertTzAdapter}: locally-declared operator is + * the referent of the {@link io.substrait.isthmus.expression.FunctionMappings.Sig} + * in {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS}. + * + *

      Type note. PPL's {@code UNIX_TIMESTAMP} returns + * {@code DOUBLE_FORCE_NULLABLE}; DataFusion's {@code to_unixtime} returns + * {@code Int64}. {@link AbstractNameMappingAdapter} preserves the PPL-declared + * return type on the rewritten call so Calcite's {@code Project.isValid} + * assertion holds. The downstream substrait consumer (DataFusion) re-resolves + * {@code to_unixtime} by name and applies its own {@code coerce_types}, so the + * Calcite-inferred type is purely plan-validity bookkeeping. + * + * @opensearch.internal + */ +class UnixTimestampAdapter extends AbstractNameMappingAdapter { + + /** + * Locally-declared target operator. Name matches DataFusion's native + * {@code to_unixtime}. Return-type inference is irrelevant — the adapter + * clones with the original PPL return type. + */ + static final SqlOperator LOCAL_TO_UNIXTIME_OP = new SqlFunction( + "to_unixtime", + SqlKind.OTHER_FUNCTION, + ReturnTypes.BIGINT_NULLABLE, + null, + OperandTypes.ANY, + SqlFunctionCategory.TIMEDATE + ); + + UnixTimestampAdapter() { + super(LOCAL_TO_UNIXTIME_OP, List.of(), List.of()); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/YearAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/YearAdapter.java new file mode 100644 index 0000000000000..5ad28fc0ba13a --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/YearAdapter.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.opensearch.analytics.spi.AbstractNameMappingAdapter; + +import java.util.List; + +/** + * Representative {@link AbstractNameMappingAdapter} for Calcite {@code YEAR(ts)}. + * Rewrites to {@code date_part('year', ts)} so isthmus resolves it against + * DataFusion's native date_part (see the {@code date_part} signature in + * {@code opensearch_scalar.yaml}). Demonstrates the reusable rename + + * literal-arg-injection adapter pattern for cat-3 PPL functions. + * + *

      Follow-up PRs extend the pattern to MONTH/DAY/HOUR/etc. each as a + * one-line concrete subclass — identical shape, different unit literal. + * + * @opensearch.internal + */ +class YearAdapter extends AbstractNameMappingAdapter { + + YearAdapter() { + super(SqlLibraryOperators.DATE_PART, List.of("year"), List.of()); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml index 4889f35c11b6a..8f5d778e543b3 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml @@ -20,3 +20,21 @@ scalar_functions: - value: "string" name: "match" return: boolean + - name: "date_part" + impls: + - args: [{ value: string, name: "part" }, { value: "any1", name: "value" }] + return: any1 + + - name: "convert_tz" + description: "Shift a timestamp from one timezone to another. IANA names and +/-HH:MM offsets." + impls: + - args: + - { value: "any1", name: "ts" } + - { value: string, name: "from_tz" } + - { value: string, name: "to_tz" } + return: any1 + - name: "to_unixtime" + description: "Return a timestamp as Unix epoch seconds." + impls: + - args: [{ value: "any1", name: "ts" }] + return: any1 diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ConvertTzAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ConvertTzAdapterTests.java new file mode 100644 index 0000000000000..19eb0df9ad578 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ConvertTzAdapterTests.java @@ -0,0 +1,228 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +/** + * Unit tests for {@link ConvertTzAdapter}. The adapter has three jobs in + * priority order: identity short-circuit when both tz operands canonicalize to + * the same value, plan-time validation/canonicalization of literal tz operands, + * and rewrite to the locally-declared UDF operator otherwise. DST-correct + * per-row shifting stays in the Rust UDF since IANA offsets vary per instant. + */ +public class ConvertTzAdapterTests extends OpenSearchTestCase { + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private RelOptCluster cluster; + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + cluster = RelOptCluster.create(planner, rexBuilder); + } + + private SqlFunction convertTzOp(RelDataType returnType) { + return new SqlFunction( + "CONVERT_TZ", + SqlKind.OTHER_FUNCTION, + ReturnTypes.explicit(returnType), + null, + OperandTypes.ANY_STRING_STRING, + SqlFunctionCategory.TIMEDATE + ); + } + + private RexCall buildConvertTz(String fromLit, String toLit) { + RelDataType tsType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP), true); + RexNode tsRef = rexBuilder.makeInputRef(tsType, 0); + // 2-arg makeLiteral returns a bare RexLiteral; the 3-arg form with a + // nullable type wraps in a CAST, which the adapter must then peel back + // to inspect the string value. PPL's frontend emits the 2-arg form, so + // we match that here. + RexNode fromNode = rexBuilder.makeLiteral(fromLit); + RexNode toNode = rexBuilder.makeLiteral(toLit); + return (RexCall) rexBuilder.makeCall(convertTzOp(tsType), List.of(tsRef, fromNode, toNode)); + } + + // ── Canonicalization (unit tests on the static helper) ──────────────── + + public void testCanonicalizeTzPadsOffsetDigits() { + assertEquals("+05:30", ConvertTzAdapter.canonicalizeTz("+5:30")); + assertEquals("-08:00", ConvertTzAdapter.canonicalizeTz("-8:00")); + assertEquals("+14:00", ConvertTzAdapter.canonicalizeTz("+14:00")); + } + + public void testCanonicalizeTzAcceptsIanaNames() { + // ZoneId.of passes through canonical ids unchanged. + assertEquals("America/New_York", ConvertTzAdapter.canonicalizeTz("America/New_York")); + assertEquals("Europe/London", ConvertTzAdapter.canonicalizeTz("Europe/London")); + assertEquals("UTC", ConvertTzAdapter.canonicalizeTz("UTC")); + } + + public void testCanonicalizeTzRejectsInvalidOffsetBounds() { + // Hours > 14 is beyond any real-world zone. + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> ConvertTzAdapter.canonicalizeTz("+15:00")); + assertTrue("error must include the bad value: " + ex.getMessage(), ex.getMessage().contains("+15:00")); + + // Minutes > 59 is malformed. + expectThrows(IllegalArgumentException.class, () -> ConvertTzAdapter.canonicalizeTz("+05:60")); + } + + public void testCanonicalizeTzRejectsUnknownIana() { + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> ConvertTzAdapter.canonicalizeTz("Mars/Olympus")); + assertTrue("error must include the bad value for UX: " + ex.getMessage(), ex.getMessage().contains("Mars/Olympus")); + } + + // ── adapt() behavior ────────────────────────────────────────────────── + + /** + * Identity fold: when both tz literals canonicalize to the same value, the + * call reduces to its timestamp operand. No UDF invocation. + */ + public void testAdaptIdentityFoldReturnsTimestampUnchanged() { + RexCall original = buildConvertTz("UTC", "UTC"); + RexNode adapted = new ConvertTzAdapter().adapt(original, List.of(), cluster); + + assertSame("identity fold must return the original timestamp operand", original.getOperands().get(0), adapted); + } + + /** + * Identity fold must apply *after* canonicalization — `+5:00` and `+05:00` + * are the same zone but different strings; the adapter must canonicalize + * first, then compare. + */ + public void testAdaptIdentityFoldAppliesAfterCanonicalization() { + RexCall original = buildConvertTz("+5:00", "+05:00"); + RexNode adapted = new ConvertTzAdapter().adapt(original, List.of(), cluster); + + assertSame("identity fold must compare canonical forms", original.getOperands().get(0), adapted); + } + + /** + * When literals can't be collapsed (IANA pairs, mixed IANA + offset), the + * call rewrites to the local UDF operator with canonicalized string + * operands. The tz strings passed to the UDF are the canonical form. + */ + public void testAdaptIanaPairRoutesThroughUdfWithCanonicalLiterals() { + RexCall original = buildConvertTz("America/New_York", "Europe/London"); + RexNode adapted = new ConvertTzAdapter().adapt(original, List.of(), cluster); + + assertTrue("adapted node must be a RexCall, got " + adapted.getClass(), adapted instanceof RexCall); + RexCall call = (RexCall) adapted; + assertSame( + "adapted call must target LOCAL_CONVERT_TZ_OP so FunctionMappings.Sig binds", + ConvertTzAdapter.LOCAL_CONVERT_TZ_OP, + call.getOperator() + ); + assertEquals(3, call.getOperands().size()); + assertEquals("America/New_York", ((RexLiteral) call.getOperands().get(1)).getValueAs(String.class)); + assertEquals("Europe/London", ((RexLiteral) call.getOperands().get(2)).getValueAs(String.class)); + } + + /** + * When literal operands need canonicalization (e.g. `+5:00` → `+05:00`), + * the UDF-bound call sees the canonical form so the Rust side doesn't need + * to do the padding. + */ + public void testAdaptPassesCanonicalizedLiteralsToUdf() { + // Pair of distinct-canonical offsets so the fold path doesn't fire. + RexCall original = buildConvertTz("+5:00", "+10:00"); + RexNode adapted = new ConvertTzAdapter().adapt(original, List.of(), cluster); + + assertTrue(adapted instanceof RexCall); + RexCall call = (RexCall) adapted; + assertSame(ConvertTzAdapter.LOCAL_CONVERT_TZ_OP, call.getOperator()); + assertEquals("+05:00", ((RexLiteral) call.getOperands().get(1)).getValueAs(String.class)); + assertEquals("+10:00", ((RexLiteral) call.getOperands().get(2)).getValueAs(String.class)); + } + + /** + * Adapter preserves the original call's return type — matches the + * {@code AbstractNameMappingAdapter} regression guard. If the rewritten + * call's Calcite-inferred type differs from the original, the enclosing + * {@code Project.isValid} compatibleTypes check breaks at fragment + * conversion. + */ + public void testAdaptedCallPreservesOriginalReturnType() { + RelDataType originalType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 0), true); + RexNode tsRef = rexBuilder.makeInputRef(originalType, 0); + RexNode fromLit = rexBuilder.makeLiteral("America/New_York"); + RexNode toLit = rexBuilder.makeLiteral("Europe/London"); + RexCall original = (RexCall) rexBuilder.makeCall(convertTzOp(originalType), List.of(tsRef, fromLit, toLit)); + assertEquals(originalType, original.getType()); + + RexNode adapted = new ConvertTzAdapter().adapt(original, List.of(), cluster); + + assertEquals( + "adapted call's return type must equal the original — otherwise Project.rowType assertion fails", + original.getType(), + adapted.getType() + ); + } + + /** + * Invalid literal tz operand surfaces at plan time as + * {@link IllegalArgumentException} with the offending value in the message, + * rather than silently producing per-row NULL at runtime. + */ + public void testAdaptInvalidLiteralErrorsAtPlanTime() { + RexCall original = buildConvertTz("Mars/Olympus", "UTC"); + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> new ConvertTzAdapter().adapt(original, List.of(), cluster) + ); + assertTrue("error must name the offending literal for user UX: " + ex.getMessage(), ex.getMessage().contains("Mars/Olympus")); + } + + /** + * Column-valued tz operands are not validated at plan time — per-row + * values can't be inspected until runtime, so they pass through into the + * UDF which handles them leniently (unparseable → NULL row). + */ + public void testAdaptColumnValuedTzOperandsPassThroughToUdf() { + RelDataType tsType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP), true); + RelDataType stringType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + RexNode tsRef = rexBuilder.makeInputRef(tsType, 0); + // Column refs for the tz slots — not literals, so no canonicalization. + RexNode fromCol = rexBuilder.makeInputRef(stringType, 1); + RexNode toCol = rexBuilder.makeInputRef(stringType, 2); + RexCall original = (RexCall) rexBuilder.makeCall(convertTzOp(tsType), List.of(tsRef, fromCol, toCol)); + + RexNode adapted = new ConvertTzAdapter().adapt(original, List.of(), cluster); + + assertTrue(adapted instanceof RexCall); + RexCall call = (RexCall) adapted; + assertSame(ConvertTzAdapter.LOCAL_CONVERT_TZ_OP, call.getOperator()); + assertSame("column-valued from_tz must pass through unmodified", fromCol, call.getOperands().get(1)); + assertSame("column-valued to_tz must pass through unmodified", toCol, call.getOperands().get(2)); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/UnixTimestampAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/UnixTimestampAdapterTests.java new file mode 100644 index 0000000000000..e27216f8ee28d --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/UnixTimestampAdapterTests.java @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +/** + * Unit tests for {@link UnixTimestampAdapter} — the cat-3a rename adapter that + * rewrites PPL's bespoke {@code UNIX_TIMESTAMP} operator to a locally-declared + * {@code to_unixtime} {@link SqlFunction} whose {@code FunctionMappings.Sig} we + * own. Target name {@code to_unixtime} matches DataFusion's native function; no + * UDF registration required on the Rust side. + */ +public class UnixTimestampAdapterTests extends OpenSearchTestCase { + + public void testUnixTimestampRewritesToLocalToUnixtimeOperator() { + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + RexBuilder rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); + + // Synthesize UNIX_TIMESTAMP(ts) with PPL's return type (DOUBLE_FORCE_NULLABLE). + RelDataType tsType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP), true); + RelDataType doubleNullable = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.DOUBLE), true); + SqlFunction unixTimestampOp = new SqlFunction( + "UNIX_TIMESTAMP", + SqlKind.OTHER_FUNCTION, + ReturnTypes.explicit(doubleNullable), + null, + OperandTypes.ANY, + SqlFunctionCategory.TIMEDATE + ); + RexNode tsRef = rexBuilder.makeInputRef(tsType, 0); + RexCall original = (RexCall) rexBuilder.makeCall(unixTimestampOp, List.of(tsRef)); + + RexNode adapted = new UnixTimestampAdapter().adapt(original, List.of(), cluster); + + assertTrue("adapted node must be a RexCall, got " + adapted.getClass(), adapted instanceof RexCall); + RexCall call = (RexCall) adapted; + assertSame( + "adapted call must target UnixTimestampAdapter.LOCAL_TO_UNIXTIME_OP so the " + + "FunctionMappings.Sig in DataFusionFragmentConvertor can bind by reference", + UnixTimestampAdapter.LOCAL_TO_UNIXTIME_OP, + call.getOperator() + ); + assertEquals("to_unixtime is a pure rename — 1 operand preserved", 1, call.getOperands().size()); + assertSame("arg 0 must be the original timestamp operand", tsRef, call.getOperands().get(0)); + } + + /** + * Regression guard mirroring {@code YearAdapterTests.testAdaptedCallPreservesOriginalReturnType}. + * PPL's {@code UNIX_TIMESTAMP} is typed {@code DOUBLE_FORCE_NULLABLE}; DF's + * {@code to_unixtime} is typed {@code Int64}. The adapter must preserve the + * original DOUBLE type so the enclosing Project / Filter's cached rowType + * doesn't mismatch during fragment conversion. (DataFusion's substrait + * consumer re-resolves {@code to_unixtime} by name at plan time and applies + * its own coerce_types pass — the Calcite-inferred return type at isthmus + * time is purely a plan-validity artifact.) + */ + public void testAdaptedCallPreservesOriginalReturnType() { + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + RexBuilder rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); + + RelDataType tsType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP), true); + RelDataType doubleNullable = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.DOUBLE), true); + SqlFunction unixTimestampOp = new SqlFunction( + "UNIX_TIMESTAMP", + SqlKind.OTHER_FUNCTION, + ReturnTypes.explicit(doubleNullable), + null, + OperandTypes.ANY, + SqlFunctionCategory.TIMEDATE + ); + RexNode tsRef = rexBuilder.makeInputRef(tsType, 0); + RexCall original = (RexCall) rexBuilder.makeCall(unixTimestampOp, List.of(tsRef)); + assertEquals(doubleNullable, original.getType()); + + RexNode adapted = new UnixTimestampAdapter().adapt(original, List.of(), cluster); + + assertEquals( + "adapted call's return type must equal the original — otherwise the enclosing Project.rowType " + + "assertion fails during fragment conversion", + original.getType(), + adapted.getType() + ); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/YearAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/YearAdapterTests.java new file mode 100644 index 0000000000000..a101f74994151 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/YearAdapterTests.java @@ -0,0 +1,116 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.analytics.spi.AbstractNameMappingAdapter; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +/** + * Unit tests for {@link YearAdapter} exercising the reusable rename + + * literal-arg injection adapter pattern via {@link AbstractNameMappingAdapter}. + */ +public class YearAdapterTests extends OpenSearchTestCase { + + public void testYearRewritesToDatePartWithYearLiteral() { + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + RexBuilder rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); + + // Synthesize YEAR(ts) — a one-arg Calcite call of our own SqlFunction + // so the test doesn't depend on any specific builtin. + RelDataType tsType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP), true); + SqlFunction yearOp = new SqlFunction( + "YEAR", + SqlKind.OTHER_FUNCTION, + ReturnTypes.explicit(typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BIGINT), true)), + null, + OperandTypes.ANY, + SqlFunctionCategory.TIMEDATE + ); + RexNode tsRef = rexBuilder.makeInputRef(tsType, 0); + RexCall original = (RexCall) rexBuilder.makeCall(yearOp, List.of(tsRef)); + + RexNode adapted = new YearAdapter().adapt(original, List.of(), cluster); + + assertTrue("adapted node must be a RexCall, got " + adapted.getClass(), adapted instanceof RexCall); + RexCall call = (RexCall) adapted; + assertEquals("adapted call must target DATE_PART", SqlLibraryOperators.DATE_PART, call.getOperator()); + assertEquals("date_part(unit, value) must have 2 operands after year-literal prepend", 2, call.getOperands().size()); + assertTrue( + "arg 0 must be a string literal, got " + call.getOperands().get(0).getClass(), + call.getOperands().get(0) instanceof RexLiteral + ); + RexLiteral unitLit = (RexLiteral) call.getOperands().get(0); + assertEquals("year", unitLit.getValueAs(String.class)); + assertSame("arg 1 must be the original operand", tsRef, call.getOperands().get(1)); + } + + /** + * The adapter MUST preserve the Calcite {@link RelDataType} of the original call. + * Otherwise the enclosing Project's cached {@code rowType} (derived from the pre- + * adaptation expression) mismatches the adapted expression's type, tripping + * {@code Project.isValid}'s {@code RexUtil.compatibleTypes} assertion during + * fragment conversion. Regression guard for the PR10 IT hang where + * {@code DATE_PART} produced a different Calcite-inferred type than {@code YEAR}. + */ + public void testAdaptedCallPreservesOriginalReturnType() { + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + RexBuilder rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); + + RelDataType tsType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.TIMESTAMP), true); + // PPL's YEAR operator is registered with INTEGER_FORCE_NULLABLE — distinct + // from Calcite's SqlLibraryOperators.DATE_PART (which returns BIGINT via + // SqlExtractFunction). If the adapter didn't clone with the original's type, + // the Project's cached rowType (derived from INTEGER) would clash with the + // adapted DATE_PART's inferred BIGINT, tripping Project.isValid. + RelDataType integerNullable = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.INTEGER), true); + SqlFunction yearOp = new SqlFunction( + "YEAR", + SqlKind.OTHER_FUNCTION, + ReturnTypes.explicit(integerNullable), + null, + OperandTypes.ANY, + SqlFunctionCategory.TIMEDATE + ); + RexNode tsRef = rexBuilder.makeInputRef(tsType, 0); + RexCall original = (RexCall) rexBuilder.makeCall(yearOp, List.of(tsRef)); + assertEquals(integerNullable, original.getType()); + + RexNode adapted = new YearAdapter().adapt(original, List.of(), cluster); + + assertEquals( + "adapted call's return type must equal the original call's return type, " + + "otherwise the enclosing Project.rowType assertion fails in fragment conversion", + original.getType(), + adapted.getType() + ); + } +} From dcb68f94131858b4e57d96bcea31ffd7d8250ffd Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Wed, 6 May 2026 13:51:48 -0700 Subject: [PATCH 062/115] Wire PPL where command through the analytics-engine path (#21502) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Wire DataFusion backend for the PPL where command Three orthogonal compatibility gaps surface when CalciteWhereCommandIT runs through the analytics-engine route. Address them in the DataFusion backend: 1. CAST as a project capability. ReduceExpressionsRule.ProjectReduceExpressionsRule constant-folds field references through equality filters into typed literals — `where str0 = 'FURNITURE' | fields str0` becomes `Project[CAST('FURNITURE' AS VARCHAR)]`. Without CAST in STANDARD_PROJECT_OPS, OpenSearchProjectRule throws "No backend supports scalar function [CAST]". 5 where-command tests previously broken; substrait isthmus and DataFusion handle CAST natively. 2. Comparison ops as project capabilities. PPL `eval x = (a == b)` produces a LogicalProject whose projected expression is the comparison itself (returning boolean). EQUALS/NOT_EQUALS/GT/GE/LT/LE were declared filter-only; add them to STANDARD_PROJECT_OPS so projections of boolean expressions are accepted. Fixes testDoubleEqualInEvalCommand. 3. ILIKE adapter. Substrait isthmus only knows the standard SQL LIKE operator; the Calcite-specific ILIKE has no extension. PPL emits ILIKE for the `contains` operator and for `LIKE` when plugins.ppl.syntax.legacy.preferred is true (its default), affecting six where-command tests with "Unable to convert call ILIKE(...)". Additionally, PPLFuncImpTable always passes an explicit '\\' escape literal of type CHAR(1), which substrait's like signature rejects as "Unable to convert call LIKE(string?, char, char<1>)". The new IlikeFunctionAdapter handles both: ILIKE rewrites to LIKE(LOWER(field), LOWER(pattern)) — wildcards and the escape character survive Character.toLowerCase unchanged, preserving case-insensitive semantics — and the 3-arg form is reduced to 2-arg by dropping the default-only escape, since PPL never emits a non-default escape (the contains operator pre-escapes user-provided literals before composing the wildcard pattern). Both LOWER and 2-arg LIKE are natively supported by DataFusion. Registered for ScalarFunction.LIKE because Calcite's SqlLibraryOperators.ILIKE shares SqlKind.LIKE with the standard LIKE operator and resolves to the same adapter slot; the adapter checks the operator name to discriminate. Signed-off-by: Kai Huang * Add WhereCommandIT QA test for the analytics-engine route Self-contained REST integration test under sandbox/qa/analytics-engine-rest mirroring the surface exercised by CalciteWhereCommandIT in the SQL plugin, adapted to the calcs dataset already shipped under src/test/resources/datasets/calcs/. Each test posts a PPL query through POST /_analytics/ppl (exposed by the test-ppl-frontend plugin), exercising the same UnifiedQueryPlanner → CalciteRelNodeVisitor → analytics-engine planner → Substrait → DataFusion pipeline as the SQL plugin's force-routed analytics path. CI for OpenSearch core can verify the where command end-to-end without checking out the SQL plugin or relying on its IT classpath. Coverage (26 cases, all passing): - Comparison operators (=, ==, !=, <, >, <=, >=) - Boolean connectives AND, OR, NOT - IS NULL / IS NOT NULL via isnull() / isnotnull() - IN / NOT IN against keyword and numeric columns - LIKE function and operator (with % and _ wildcards) - contains (lowers to ILIKE — case-insensitive) - Sub-expression scalar calls inside predicates: length, abs, + Signed-off-by: Kai Huang --------- Signed-off-by: Kai Huang --- .../DataFusionAnalyticsBackendPlugin.java | 8 +- .../analytics/qa/WhereCommandIT.java | 346 ++++++++++++++++++ 2 files changed, 352 insertions(+), 2 deletions(-) create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/WhereCommandIT.java diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 1119f8d8fa17a..08913c55615a3 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -82,12 +82,16 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP // PPL command/function we want the analytics-engine planner to route through DataFusion. Add // here only after verifying the function deserializes through Substrait isthmus into a plan // DataFusion's native runtime can execute (see DataFusionFragmentConvertor for the conversion - // path). COALESCE is the lowering target of PPL `fillnull`. + // path). COALESCE is the lowering target of PPL `fillnull`. CAST is required because + // ReduceExpressionsRule.ProjectReduceExpressionsRule (in PlannerImpl) constant-folds field + // references through equality filters into typed literals — e.g. after `where str0 = 'FURNITURE'`, + // the projection `fields str0` is rewritten to `CAST('FURNITURE' AS VARCHAR)`. The remaining + // comparison / arithmetic / logical operators are project-capable for eval-style projections. private static final Set STANDARD_PROJECT_OPS = Set.of( ScalarFunction.COALESCE, ScalarFunction.CEIL, + ScalarFunction.CAST, ScalarFunction.SARG_PREDICATE, - // comparison / arithmetic / logical operators in eval-style projections. ScalarFunction.EQUALS, ScalarFunction.NOT_EQUALS, ScalarFunction.GREATER_THAN, diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/WhereCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/WhereCommandIT.java new file mode 100644 index 0000000000000..1b03f175b5409 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/WhereCommandIT.java @@ -0,0 +1,346 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for PPL {@code where} on the analytics-engine route. + * + *

      Mirrors the surface exercised by {@code CalciteWhereCommandIT} from the + * {@code opensearch-project/sql} repository, adapted to the {@code calcs} dataset + * shipped under {@code sandbox/qa/analytics-engine-rest/src/test/resources/datasets/calcs/}. + * Each test sends a PPL query through {@code POST /_analytics/ppl} (exposed by the + * {@code test-ppl-frontend} plugin), exercising the same {@code UnifiedQueryPlanner} → + * {@code CalciteRelNodeVisitor} → analytics-engine planner → Substrait → DataFusion + * pipeline as the SQL plugin's force-routed analytics path. + * + *

      Top-level filter operators covered (see + * {@link org.opensearch.analytics.spi.ScalarFunction} → {@code STANDARD_FILTER_OPS} in + * {@code DataFusionAnalyticsBackendPlugin}): + *

        + *
      • {@code = / == / != / < / > / <= / >=}
        • + *
      • Boolean connectives {@code AND / OR / NOT}
        • + *
      • {@code IS NULL} / {@code IS NOT NULL} via {@code isnull()} / {@code isnotnull()}
        • + *
      • {@code IN} / {@code NOT IN}
        • + *
      • {@code LIKE} (operator + function) and {@code contains} (lowers to {@code ILIKE})
        • + *
      + * + *

      Sub-expression coverage (passed through to DataFusion via Substrait without + * appearing as the leaf-predicate operator): {@code length()}, {@code abs()}, + * arithmetic {@code +}. + */ +public class WhereCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + /** + * Lazily provision the calcs dataset on first invocation. Same lazy-provision pattern + * as {@link FillNullCommandIT} — {@code client()} is only reliably available inside a + * test body, not in {@code @BeforeClass} / {@code setUp()}. + */ + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + // ── Comparison operators ──────────────────────────────────────────────── + + public void testWhereEqualOnKeyword() throws IOException { + // 2 rows have str0='FURNITURE'. + assertRowCount("source=" + DATASET.indexName + " | where str0 = 'FURNITURE' | fields str0", 2); + } + + public void testWhereEqualOnDouble() throws IOException { + assertRows( + "source=" + DATASET.indexName + " | where num0 = 12.3 | fields str2, num0", + row("one", 12.3) + ); + } + + public void testWhereDoubleEqualOperator() throws IOException { + // == is parsed as = at the AstExpressionBuilder layer; same plan, same result. + assertRows( + "source=" + DATASET.indexName + " | where num0 == 12.3 | fields str2, num0", + row("one", 12.3) + ); + } + + public void testWhereNotEqual() throws IOException { + // 8 non-null distinct num0 values; != 0 keeps 7 rows (drops the single num0=0). + assertRowCount("source=" + DATASET.indexName + " | where num0 != 0 | fields num0", 7); + } + + public void testWhereGreaterThan() throws IOException { + // num0 > 0 → {12.3, 15.7, 3.5, 10}. + assertRowCount("source=" + DATASET.indexName + " | where num0 > 0 | fields num0", 4); + } + + public void testWhereGreaterEqual() throws IOException { + // num0 >= 0 → adds the row with num0=0 → 5 rows. + assertRowCount("source=" + DATASET.indexName + " | where num0 >= 0 | fields num0", 5); + } + + public void testWhereLessThan() throws IOException { + // num0 < 0 → {-12.3, -15.7, -3.5}. + assertRowCount("source=" + DATASET.indexName + " | where num0 < 0 | fields num0", 3); + } + + public void testWhereLessEqual() throws IOException { + // num0 <= 0 → adds num0=0 → 4 rows. + assertRowCount("source=" + DATASET.indexName + " | where num0 <= 0 | fields num0", 4); + } + + // ── Boolean connectives ───────────────────────────────────────────────── + + public void testWhereAnd() throws IOException { + // FURNITURE rows are key00 (num0=12.3) and key01 (num0=-12.3); AND num0>0 keeps key00. + assertRows( + "source=" + DATASET.indexName + " | where str0 = 'FURNITURE' and num0 > 0 | fields str2, num0", + row("one", 12.3) + ); + } + + public void testWhereOr() throws IOException { + // num0 == 12.3 OR num0 == -12.3 → key00, key01. + assertRowCount( + "source=" + DATASET.indexName + " | where num0 == 12.3 OR num0 == -12.3 | fields num0", + 2 + ); + } + + public void testWhereNot() throws IOException { + // NOT (str0 = 'FURNITURE') → 17 - 2 = 15 rows. (str0 has no nulls in calcs.) + assertRowCount( + "source=" + DATASET.indexName + " | where not str0 = 'FURNITURE' | fields str0", + 15 + ); + } + + public void testWhereMultipleChained() throws IOException { + // Three filter steps: FURNITURE → num0>0 → str2='one'. Should leave one row. + assertRows( + "source=" + DATASET.indexName + + " | where str0 = 'FURNITURE'" + + " | where num0 > 0" + + " | where str2 = 'one'" + + " | fields str0, num0, str2", + row("FURNITURE", 12.3, "one") + ); + } + + // ── NULL handling via isnull() / isnotnull() ──────────────────────────── + + public void testWhereIsNull() throws IOException { + // str2 has 4 null rows in calcs. + assertRowCount( + "source=" + DATASET.indexName + " | where isnull(str2) | fields str2", + 4 + ); + } + + public void testWhereIsNotNull() throws IOException { + // str2 has 13 non-null rows in calcs. + assertRowCount( + "source=" + DATASET.indexName + " | where isnotnull(str2) | fields str2", + 13 + ); + } + + // ── IN / NOT IN ───────────────────────────────────────────────────────── + + public void testWhereInOnKeyword() throws IOException { + // FURNITURE (2) + OFFICE SUPPLIES (6) = 8. + assertRowCount( + "source=" + DATASET.indexName + " | where str0 in ('FURNITURE', 'OFFICE SUPPLIES') | fields str0", + 8 + ); + } + + public void testWhereInOnNumeric() throws IOException { + // num0 IN (12.3, -12.3) → key00, key01 = 2 rows. + assertRowCount( + "source=" + DATASET.indexName + " | where num0 in (12.3, -12.3) | fields num0", + 2 + ); + } + + public void testWhereNotIn() throws IOException { + // Complement of (FURNITURE, OFFICE SUPPLIES): 9 TECHNOLOGY rows. + assertRowCount( + "source=" + DATASET.indexName + " | where not str0 in ('FURNITURE', 'OFFICE SUPPLIES') | fields str0", + 9 + ); + } + + // ── LIKE function and operator ────────────────────────────────────────── + + public void testWhereLikeFunction() throws IOException { + // like(str0, 'FURN%') → 2 FURNITURE rows. + assertRowCount( + "source=" + DATASET.indexName + " | where like(str0, 'FURN%') | fields str0", + 2 + ); + } + + public void testWhereLikeOperator() throws IOException { + // str0 LIKE 'OFF%' → 6 OFFICE SUPPLIES rows. + assertRowCount( + "source=" + DATASET.indexName + " | where str0 LIKE 'OFF%' | fields str0", + 6 + ); + } + + public void testWhereLikeUnderscoreWildcard() throws IOException { + // 'on_' matches 'one' only (3 chars starting with "on"). + assertRows( + "source=" + DATASET.indexName + " | where str2 LIKE 'on_' | fields str2", + row("one") + ); + } + + public void testWhereLikeNoMatch() throws IOException { + assertRowCount( + "source=" + DATASET.indexName + " | where like(str0, 'XYZ%') | fields str0", + 0 + ); + } + + // ── CONTAINS (lowers to ILIKE — case-insensitive) ─────────────────────── + + public void testWhereContains() throws IOException { + // 'URN' inside FURNITURE → 2 rows. + assertRowCount( + "source=" + DATASET.indexName + " | where str0 contains 'URN' | fields str0", + 2 + ); + } + + public void testWhereContainsCaseInsensitive() throws IOException { + // Lowercase pattern still hits FURNITURE because contains uses ILIKE. + assertRowCount( + "source=" + DATASET.indexName + " | where str0 contains 'urn' | fields str0", + 2 + ); + } + + // ── Sub-expression scalar calls (pass through to DataFusion) ──────────── + + public void testWhereInnerLength() throws IOException { + // length('FURNITURE') = 9 → 2 rows. + assertRowCount( + "source=" + DATASET.indexName + " | where length(str0) = 9 | fields str0", + 2 + ); + } + + public void testWhereInnerAbs() throws IOException { + // abs(num0) > 10 → {-15.7, -12.3, 12.3, 15.7} = 4 rows. + assertRowCount( + "source=" + DATASET.indexName + " | where abs(num0) > 10 | fields num0", + 4 + ); + } + + public void testWhereInnerArithmetic() throws IOException { + // num0 + 100 > 105 ⇔ num0 > 5 → {12.3, 15.7, 10} = 3 rows. + assertRowCount( + "source=" + DATASET.indexName + " | where num0 + 100 > 105 | fields num0", + 3 + ); + } + + // ── Helpers ───────────────────────────────────────────────────────────── + + private static List row(Object... values) { + return Arrays.asList(values); + } + + /** + * Assert that the PPL query returns exactly {@code expectedCount} rows. Used when the + * exact row contents would be brittle (e.g. set membership tests where row order is not + * guaranteed by the engine). + */ + private void assertRowCount(String ppl, int expectedCount) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' field for query: " + ppl, actualRows); + assertEquals( + "Row count mismatch for query: " + ppl + " — got rows: " + actualRows, + expectedCount, + actualRows.size() + ); + } + + /** + * Assert exact row contents. Mirrors {@link FillNullCommandIT#assertRows} including the + * numeric-tolerant cell comparator (Jackson parsing returns Integer/Long/Double per JSON + * shape, but PPL doesn't preserve that distinction at the API surface). + */ + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRows(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' field for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size()); + for (int i = 0; i < expected.length; i++) { + List want = expected[i]; + List got = actualRows.get(i); + assertEquals( + "Column count mismatch at row " + i + " for query: " + ppl, + want.size(), + got.size() + ); + for (int j = 0; j < want.size(); j++) { + assertCellEquals( + "Cell mismatch at row " + i + ", col " + j + " for query: " + ppl, + want.get(j), + got.get(j) + ); + } + } + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } + + private static void assertCellEquals(String message, Object expected, Object actual) { + if (expected == null || actual == null) { + assertEquals(message, expected, actual); + return; + } + if (expected instanceof Number && actual instanceof Number) { + double e = ((Number) expected).doubleValue(); + double a = ((Number) actual).doubleValue(); + if (Double.compare(e, a) != 0) { + fail(message + ": expected <" + expected + "> but was <" + actual + ">"); + } + return; + } + assertEquals(message, expected, actual); + } +} From ed78d0374d21fe6552da98ee4da07fac28436d03 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Wed, 6 May 2026 14:59:45 -0700 Subject: [PATCH 063/115] Enable PPL eval string concat on the analytics-engine route via DataFusion CONCAT/CAST (#21498) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Analytics Framework] Resolve symbolic operators and add SAFE_CAST Calcite emits SqlBinaryOperators (e.g. `||`, the lowering target of PPL string `+`) with SqlKind.OTHER and a non-identifier name. The existing ScalarFunction.fromSqlKind / fromSqlFunction pair fails to resolve these: fromSqlKind misses (OTHER is shared), fromSqlFunction throws because `||` is not a SqlFunction (it's a SqlBinaryOperator). The planner-side fallout is "No backend supports scalar function [null] among [datafusion]" with no useful name in the error. Introduce ScalarFunction.fromSqlOperator(SqlOperator) — the unified entry point used by OpenSearchProjectRule, OpenSearchFilterRule, and BackendPlanAdapter in subsequent commits. Resolution order: 1. SqlKind via fromSqlKind (covers PLUS, CAST, COALESCE, etc.) 2. Symbolic-name lookup (handles `||` -> CONCAT) 3. Identifier-name valueOf fallback (covers UPPER, LOWER, etc.) The symbolic-name table currently has one entry (`||` -> CONCAT) but is the documented extension point for future SqlBinaryOperators with non- identifier names. Also adds SAFE_CAST as a sibling enum constant to CAST. PPL emits explicit `CAST(... AS ...)` lowered to Calcite's SqlKind.SAFE_CAST when the source value may be NULL or the conversion may fail. SAFE_CAST and CAST share the same backend semantics (DataFusion's native cast already returns NULL on conversion failure) but resolve through distinct SqlKinds, so they need distinct enum entries. Unit test pins all three resolution branches plus the unknown-operator return-null contract — a regression that drops a branch surfaces here rather than as an opaque "[null]" IT failure. Signed-off-by: Kai Huang * [Analytics Engine] Migrate rules and adapter dispatch to fromSqlOperator Three call sites resolved a RexCall's operator using the same two-step pattern (SqlKind first, SqlFunction-cast second) and all three failed identically on `||` (a SqlBinaryOperator with SqlKind.OTHER): - OpenSearchProjectRule.resolveScalarViableBackends - OpenSearchFilterRule (predicate operator resolution) - BackendPlanAdapter.resolveFunction (per-function adapter dispatch) Migrate all three to ScalarFunction.fromSqlOperator, the unified resolver added in the previous commit. Behavior for previously-resolved operators is unchanged — fromSqlOperator delegates to fromSqlKind first, so anything that resolved through SqlKind continues to. New behavior: `||` now resolves to CONCAT, and unrecognized operators return null (catching the IllegalArgumentException that fromSqlFunction's valueOf threw before; the call sites already handled null and now produce a better-formed error message that includes the operator name). Also drop the unused SqlFunction import in OpenSearchFilterRule and BackendPlanAdapter, and tighten the OpenSearchProjectRule error message to fall back to operator.getName() when the resolver returns null — "[null]" was unactionable for triage; "[||]" or "[]" points directly at the missing capability. Signed-off-by: Kai Huang * [Analytics Backend / DataFusion] Wire CONCAT/CAST/SAFE_CAST + concat null adapter Three new ScalarFunctions in STANDARD_PROJECT_OPS: - CONCAT — lowering target of PPL `eval`'s `+` for strings (Calcite emits `||`, resolved to CONCAT through the symbolic-name branch of ScalarFunction.fromSqlOperator) - CAST — covers PPL's explicit `CAST(... AS ...)` over non-null source types (Calcite emits SqlKind.CAST) - SAFE_CAST — same surface, but emitted by Calcite when the source value is nullable (SqlKind.SAFE_CAST) CONCAT additionally needs a ScalarFunctionAdapter to preserve null semantics. Calcite's `||` follows the SQL standard: if any operand is NULL, the result is NULL. Substrait's default `concat` extension is documented with the same semantics, but DataFusion's substrait reader maps it to the DataFusion `concat()` function — which deviates from the standard and treats NULL operands as empty strings. PPL queries like `'Age: ' + CAST(null AS STRING)` expect NULL, not 'Age: '. ConcatFunctionAdapter rewrites `||(a, b, ...)` into CASE WHEN a IS NULL OR b IS NULL OR ... THEN NULL ELSE ||(a, b, ...) END The inner `||` survives unchanged and serializes through the same Substrait conversion path; the surrounding CASE/IS_NULL short-circuits the DataFusion `concat()` call whenever any operand is NULL, restoring SQL-standard null propagation without a custom DataFusion UDF. Trade-off: the rewrite double-evaluates each operand (once in IS_NULL, once in the inner `||`). For RexInputRef and RexLiteral operands — the only shapes PPL emits today for string concat — this is free; for nested calls the cost is proportional to operand count, not operand depth, since each `||` adapter wraps one CASE around its direct call. A custom null-propagating concat UDF (Bucket-3 work in sandbox/plugins/analytics-backend-datafusion/rust) is the alternative but disproportionate for a Bucket-1 surface. Signed-off-by: Kai Huang * [QA] Add EvalCommandIT for the analytics-engine REST path Self-contained integration test for PPL `eval` on the analytics-engine route. Mirrors CalciteEvalCommandIT in opensearch-project/sql so the analytics-engine path can be verified inside core without cross-plugin dependencies on the SQL plugin. Each test sends a PPL query through POST /_analytics/ppl (exposed by test-ppl-frontend) which runs the same UnifiedQueryPlanner -> CalciteRelNodeVisitor -> Substrait -> DataFusion pipeline as the SQL plugin's force-routed analytics path. Four tests on the calcs dataset cover the eval surface this PR enables: - testEvalStringConcatLiteralPlusField — `'literal' + str_field` exercises the symbolic-name resolution for `||` and the CONCAT capability; null str field rows assert null propagation through the CASE adapter. - testEvalStringConcatWithCastIntField — `'literal' + CAST(int AS STRING)` exercises both CAST/SAFE_CAST and CONCAT in the same projection; null int rows confirm CAST(NULL) -> NULL propagates through the surrounding concat. - testEvalStringConcatMultipleLiteralsAndFields — chained four-arg concat exercises the recursive AnnotatedProjectExpression strip for nested project calls. - testEvalStringConcatTwoFields — pure field-to-field concat with no literal operands; planner takes the hasFieldRef=true path in resolveScalarViableBackends. Reuses the existing calcs dataset (no new fixtures). Once this lands, the SQL-plugin's CalciteEvalCommandIT is verification-only — this QA IT is the source of truth for the analytics-engine path. Signed-off-by: Kai Huang * [Analytics Framework] Rename fromSqlOperator to fromSqlOperatorWithFallback Per @expani's PR feedback: the method walks three resolution paths (SqlKind, symbolic-name table, identifier-name valueOf) before returning null, so the name should advertise the fallback behavior at the call site rather than only in the javadoc. Mechanical rename across all callers — `ScalarFunction.fromSqlOperator` -> `ScalarFunction.fromSqlOperatorWithFallback` in: - the resolver itself plus its 7 unit tests - OpenSearchProjectRule (2 call sites) - OpenSearchFilterRule (1 call site) - BackendPlanAdapter.resolveFunction (1 call site) - EvalCommandIT javadoc cross-reference No behavioral change. Signed-off-by: Kai Huang * [Analytics Framework + Backend] Address @expani review on PR #21498 Three feedback items in one commit: 1. Co-locate symbolic operator name with the enum constant. The static SYMBOLIC_OPERATOR_NAMES map duplicated a property that belongs on the enum itself. Moved to a nullable `symbolicOperatorName` field on each ScalarFunction constant — currently set only on CONCAT ("||"). The reverse-index map is now built from the enum at class-init time, so adding a new symbolic operator is a single-site edit on the constant rather than a separate map entry. 2. Inline the OR/IS_NULL fold in ConcatFunctionAdapter. Drop the temporary List nullChecks and accumulate the OR-of-IS_NULLs directly in the loop body. Same generated tree, fewer allocations, less to read. 3. Note the Map.of single-line constraint on scalarFunctionAdapters. Per-pair formatting is rejected by spotless; left a comment pointing future contributors at alphabetical ordering instead, and reordered the entries (CONCAT before TIMESTAMP) to make the convention concrete. No behavioral change. CalciteEvalCommandIT 4/4 still passes against the analytics-engine route; sandbox per-module check (excluding the unrelated commons-text dependencyLicenses task) remains green. Signed-off-by: Kai Huang * [Analytics Framework] Resolve symbolic operators by Calcite-operator reference Per @expani's PR follow-up: the symbolic-name string ("||") was a runtime-coupled identifier that could silently drift if Calcite renamed the operator. Replace it with a direct reference to the Calcite operator constant (SqlStdOperatorTable.CONCAT), so the link is enforced at compile time and a Calcite-side rename surfaces as a build failure here. - String symbolicOperatorName -> SqlOperator referenceOperator on the enum constructor. - CONCAT now points at SqlStdOperatorTable.CONCAT instead of "||". - Reverse index switches from Map keyed by operator name to Map keyed by operator identity. Calcite's standard operators are singletons, so identity lookup is exact. - Unit test renamed (testFromSqlOperatorResolvesPipeConcatViaReferenceOperator) and its comment updated; the assertions on `getName()` / `getKind()` are kept as documentation of WHY this branch is needed at all. No behavioral change in the resolution logic — same three-step chain (SqlKind, then this branch, then identifier-name valueOf), with the middle branch now identity-comparing rather than name-comparing. CalciteEvalCommandIT 4/4 still passes; ScalarFunctionTests 7/7. Signed-off-by: Kai Huang * [Analytics Framework + Backend] Address @expani follow-up on PR #21498 Two feedback items in one commit: 1. Drop the redundant Map.copyOf on BY_REFERENCE_OPERATOR. The HashMap built in the static initializer is private static final and is only read via the resolver's get() — never returned, never iterated. The immutability wrapper added an allocation without conferring any external safety guarantee. Comment explains the reasoning so future readers don't reintroduce the wrap. 2. Add ConcatFunctionAdapterTests with seven structural assertions on the CASE rewrite contract: - testAdaptBinaryConcatProducesCaseWrapper: rewritten root is a three-operand CASE (condition, then, else). - testAdaptedCaseElseBranchIsOriginalConcat: else branch is the original RexCall by reference (assertSame, not assertEquals) — downstream substrait conversion expects the same object the resolver annotated. - testAdaptedCaseThenBranchIsNullLiteralOfMatchingSqlType: then branch is a NULL literal whose SQL type name matches the original CONCAT's. Comment explains why we compare type name rather than full RelDataType (RexBuilder.makeNullLiteral promotes nullability, so the full types differ harmlessly). - testAdaptedCaseConditionIsOrOfIsNullChecks: condition is OR with each disjunct an IS_NULL wrapping the corresponding original operand at matching index — null-propagation contract is per operand. - testAdaptPreservesReturnType: full RelDataType identity between adapted CASE and original CONCAT — locks the type-preserving argument of rexBuilder.makeCall(originalType, CASE, ...). - testAdaptNaryConcatChainsIsNullChecksLeftAssociative: builds a ternary CONCAT via SqlLibraryOperators.CONCAT_FUNCTION and verifies the left-fold structure OR(OR(IS_NULL(a), IS_NULL(b)), IS_NULL(c)) — the binary `||` only ever appears with arity 2 in production, but the loop's correctness for arbitrary N is now a test invariant. - testAdaptSingleOperandConcatPassesThroughUnchanged: 1-operand call returns input by reference; documents the early-out branch. Each test pins one structural property in isolation, so a regression that drops any one piece of the contract surfaces with a focused failure rather than at IT-level row-mismatch noise. Signed-off-by: Kai Huang --------- Signed-off-by: Kai Huang --- .../analytics/spi/ScalarFunction.java | 84 ++++++- .../analytics/spi/ScalarFunctionTests.java | 67 ++++++ .../be/datafusion/ConcatFunctionAdapter.java | 69 ++++++ .../DataFusionAnalyticsBackendPlugin.java | 25 +- .../ConcatFunctionAdapterTests.java | 187 +++++++++++++++ .../planner/dag/BackendPlanAdapter.java | 10 +- .../planner/rules/OpenSearchFilterRule.java | 13 +- .../planner/rules/OpenSearchProjectRule.java | 11 +- .../analytics/qa/EvalCommandIT.java | 224 ++++++++++++++++++ 9 files changed, 656 insertions(+), 34 deletions(-) create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConcatFunctionAdapter.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ConcatFunctionAdapterTests.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/EvalCommandIT.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java index 13cbc837a8056..2bc65b658f3d9 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java @@ -10,8 +10,12 @@ import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import java.util.HashMap; import java.util.Locale; +import java.util.Map; /** * All scalar functions a backend may support — comparisons, full-text search, @@ -52,7 +56,15 @@ public enum ScalarFunction { LOWER(Category.STRING, SqlKind.OTHER_FUNCTION), TRIM(Category.STRING, SqlKind.TRIM), SUBSTRING(Category.STRING, SqlKind.OTHER_FUNCTION), - CONCAT(Category.STRING, SqlKind.OTHER_FUNCTION), + /** + * String concatenation. Calcite's {@code SqlStdOperatorTable.CONCAT} is a + * {@link org.apache.calcite.sql.SqlBinaryOperator} named {@code "||"} (not {@code "CONCAT"}) + * with {@link SqlKind#OTHER}, so neither {@link #fromSqlKind(SqlKind)} nor identifier-name + * {@link #valueOf(String)} resolves it. The {@code referenceOperator} hook below pins the + * concrete Calcite operator constant so resolution is a singleton-identity match — a Calcite + * rename surfaces as a compile error rather than as a silent string mismatch at runtime. + */ + CONCAT(Category.STRING, SqlKind.OTHER_FUNCTION, SqlStdOperatorTable.CONCAT), CHAR_LENGTH(Category.STRING, SqlKind.OTHER_FUNCTION), // ── Math ───────────────────────────────────────────────────────── @@ -68,6 +80,14 @@ public enum ScalarFunction { // ── Cast / type ────────────────────────────────────────────────── CAST(Category.SCALAR, SqlKind.CAST), + /** + * Calcite's {@code SAFE_CAST} — emitted by PPL's explicit {@code CAST(... AS ...)} when the + * source value may be NULL or the conversion may fail; returns NULL on failure rather than + * throwing. Resolves through {@link SqlKind#SAFE_CAST}, distinct from {@link #CAST} which + * uses {@link SqlKind#CAST}. DataFusion's native cast already returns NULL on conversion + * failure, so SAFE_CAST and CAST share the same backend semantics. + */ + SAFE_CAST(Category.SCALAR, SqlKind.SAFE_CAST), // ── Conditional ────────────────────────────────────────────────── CASE(Category.SCALAR, SqlKind.CASE), @@ -98,10 +118,24 @@ public enum Category { private final Category category; private final SqlKind sqlKind; + /** + * Optional Calcite operator that this constant maps to when the operator cannot be resolved + * via {@link SqlKind} or via identifier-name {@link #valueOf(String)} — typically operators + * whose {@code getName()} returns a non-identifier token (e.g. {@code SqlStdOperatorTable.CONCAT} + * is named {@code "||"}). Null for the common case where SqlKind or name resolution suffices. + * Stored as a reference (not a string) so a Calcite-side rename of the operator surfaces as a + * compile error here. + */ + private final SqlOperator referenceOperator; ScalarFunction(Category category, SqlKind sqlKind) { + this(category, sqlKind, null); + } + + ScalarFunction(Category category, SqlKind sqlKind, SqlOperator referenceOperator) { this.category = category; this.sqlKind = sqlKind; + this.referenceOperator = referenceOperator; } public Category getCategory() { @@ -134,4 +168,52 @@ public static ScalarFunction fromSqlFunction(SqlFunction function) { // valueOf(toUpperCase). This couples enum constant naming to SQL function naming convention. return ScalarFunction.valueOf(function.getName().toUpperCase(Locale.ROOT)); } + + /** + * Reverse index from {@link #referenceOperator} to enum constant. Built from the enum itself + * at class init — adding a new symbolic operator is a single-site change on the enum constant, + * no separate map to maintain. Lookup is identity-keyed because Calcite's standard operators + * are singletons (e.g. {@code SqlStdOperatorTable.CONCAT}). Empty in the common case (most + * constants resolve by SqlKind or identifier-name valueOf). + */ + private static final Map BY_REFERENCE_OPERATOR; + + static { + Map byOperator = new HashMap<>(); + for (ScalarFunction func : values()) { + if (func.referenceOperator != null) { + byOperator.put(func.referenceOperator, func); + } + } + // The HashMap is private static final and never exposed beyond the get() in the resolver + // below — wrapping it in Map.copyOf adds an allocation without any external safety guarantee. + BY_REFERENCE_OPERATOR = byOperator; + } + + /** + * Maps any Calcite {@link SqlOperator} to a {@link ScalarFunction}, or returns null if + * unrecognized. Resolution order: {@link SqlKind} match, then {@link #referenceOperator} + * identity match (handles {@code SqlStdOperatorTable.CONCAT} a.k.a. {@code ||}), then + * identifier-name {@link #valueOf(String)} match. + * + *

      Prefer this entry point over {@link #fromSqlKind(SqlKind)} / + * {@link #fromSqlFunction(SqlFunction)} when resolving an arbitrary {@code RexCall}'s + * operator: a {@code RexCall} may be backed by a {@code SqlBinaryOperator} (e.g. {@code ||}) + * which is neither covered by {@code OTHER} {@code SqlKind} nor by {@code SqlFunction}. + */ + public static ScalarFunction fromSqlOperatorWithFallback(SqlOperator operator) { + ScalarFunction byKind = fromSqlKind(operator.getKind()); + if (byKind != null) { + return byKind; + } + ScalarFunction byReference = BY_REFERENCE_OPERATOR.get(operator); + if (byReference != null) { + return byReference; + } + try { + return ScalarFunction.valueOf(operator.getName().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException ignored) { + return null; + } + } } diff --git a/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java index 077c52a3103cc..e0db1cd82ab8e 100644 --- a/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java +++ b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java @@ -9,13 +9,42 @@ package org.opensearch.analytics.spi; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.opensearch.test.OpenSearchTestCase; import java.util.EnumMap; import java.util.Map; +/** + * Unit coverage for {@link ScalarFunction}'s three resolution paths used by the analytics-engine + * planner ({@code OpenSearchProjectRule}, {@code OpenSearchFilterRule}, {@code BackendPlanAdapter}). + * + *

      Each test pins one of the resolver's branches so a regression that drops a branch surfaces + * here rather than in IT-level "No backend supports scalar function [null]" errors. + */ public class ScalarFunctionTests extends OpenSearchTestCase { + // ── fromSqlKind ───────────────────────────────────────────────────────────── + + public void testFromSqlKindResolvesDedicatedKind() { + assertEquals(ScalarFunction.EQUALS, ScalarFunction.fromSqlKind(SqlKind.EQUALS)); + assertEquals(ScalarFunction.PLUS, ScalarFunction.fromSqlKind(SqlKind.PLUS)); + assertEquals(ScalarFunction.CAST, ScalarFunction.fromSqlKind(SqlKind.CAST)); + assertEquals(ScalarFunction.SAFE_CAST, ScalarFunction.fromSqlKind(SqlKind.SAFE_CAST)); + assertEquals(ScalarFunction.COALESCE, ScalarFunction.fromSqlKind(SqlKind.COALESCE)); + } + + public void testFromSqlKindReturnsNullForOtherKind() { + // SqlKind.OTHER is shared by many SqlBinaryOperators — must NOT resolve via SqlKind. + assertNull(ScalarFunction.fromSqlKind(SqlKind.OTHER)); + } + + public void testFromSqlKindReturnsNullForOtherFunctionKind() { + // SqlKind.OTHER_FUNCTION is shared by many name-distinguished SqlFunctions — must NOT + // resolve via SqlKind even though several enum entries declare it. + assertNull(ScalarFunction.fromSqlKind(SqlKind.OTHER_FUNCTION)); + } + /** Non-OTHER_FUNCTION SqlKinds must be unique: fromSqlKind picks the first match and would shadow later entries. */ public void testNoDuplicateSqlKindBindings() { Map claimedBy = new EnumMap<>(SqlKind.class); @@ -34,4 +63,42 @@ public void testNoDuplicateSqlKindBindings() { public void testSargPredicateIsBoundToSqlKindSearch() { assertSame(ScalarFunction.SARG_PREDICATE, ScalarFunction.fromSqlKind(SqlKind.SEARCH)); } + + // ── fromSqlOperatorWithFallback: SqlKind branch ──────────────────────────────────────── + + public void testFromSqlOperatorResolvesViaSqlKind() { + // Calcite's CAST has a dedicated SqlKind.CAST — short-circuit before name lookup. + assertEquals(ScalarFunction.CAST, ScalarFunction.fromSqlOperatorWithFallback(SqlStdOperatorTable.CAST)); + assertEquals(ScalarFunction.PLUS, ScalarFunction.fromSqlOperatorWithFallback(SqlStdOperatorTable.PLUS)); + assertEquals(ScalarFunction.GREATER_THAN, ScalarFunction.fromSqlOperatorWithFallback(SqlStdOperatorTable.GREATER_THAN)); + assertEquals(ScalarFunction.COALESCE, ScalarFunction.fromSqlOperatorWithFallback(SqlStdOperatorTable.COALESCE)); + } + + // ── fromSqlOperatorWithFallback: reference-operator branch ───────────────────────────── + + public void testFromSqlOperatorResolvesPipeConcatViaReferenceOperator() { + // The original "no backend supports scalar function [null]" symptom for PPL string `+`. + // SqlStdOperatorTable.CONCAT is a SqlBinaryOperator named "||" with SqlKind.OTHER — + // neither fromSqlKind nor fromSqlFunction(SqlFunction) resolves it. CONCAT's + // referenceOperator field points at the singleton, so the resolver matches by identity. + assertEquals("||", SqlStdOperatorTable.CONCAT.getName()); + assertEquals(SqlKind.OTHER, SqlStdOperatorTable.CONCAT.getKind()); + assertEquals(ScalarFunction.CONCAT, ScalarFunction.fromSqlOperatorWithFallback(SqlStdOperatorTable.CONCAT)); + } + + // ── fromSqlOperatorWithFallback: identifier-name branch ──────────────────────────────── + + public void testFromSqlOperatorResolvesViaIdentifierName() { + // SqlStdOperatorTable.UPPER is a SqlFunction named "UPPER" with SqlKind.OTHER_FUNCTION; + // resolves through the valueOf(name.toUpperCase()) fallback after SqlKind misses. + assertEquals(ScalarFunction.UPPER, ScalarFunction.fromSqlOperatorWithFallback(SqlStdOperatorTable.UPPER)); + assertEquals(ScalarFunction.LOWER, ScalarFunction.fromSqlOperatorWithFallback(SqlStdOperatorTable.LOWER)); + assertEquals(ScalarFunction.ABS, ScalarFunction.fromSqlOperatorWithFallback(SqlStdOperatorTable.ABS)); + } + + public void testFromSqlOperatorReturnsNullForUnknownFunction() { + // UNARY_MINUS has SqlKind.MINUS_PREFIX (no enum) and name "-" (not a valid valueOf input); + // both resolution paths miss and the resolver returns null instead of throwing. + assertNull(ScalarFunction.fromSqlOperatorWithFallback(SqlStdOperatorTable.UNARY_MINUS)); + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConcatFunctionAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConcatFunctionAdapter.java new file mode 100644 index 0000000000000..04887359d884f --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConcatFunctionAdapter.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; + +import java.util.List; + +/** + * Adapts {@code ||(a, b, ...)} (Calcite {@code SqlStdOperatorTable.CONCAT}) into a + * null-propagating form for the DataFusion backend. + * + *

      Calcite's {@code ||} operator follows the SQL standard: if any operand is NULL, the result + * is NULL. Substrait's default {@code concat} extension is documented with the same semantics, + * but DataFusion's substrait reader maps it to the DataFusion {@code concat()} function — which + * deviates from the standard and treats NULL operands as empty strings. To preserve Calcite's + * semantics on the analytics-engine path, this adapter rewrites + * + * 

      {@code
+ *   ||(a, b)
+ *     →
+ *   CASE WHEN a IS NULL OR b IS NULL THEN NULL ELSE ||(a, b) END
+ * }
      + * + * The inner {@code ||} is left intact and serializes through the same Substrait conversion path, + * but with the surrounding CASE/IS_NULL the DataFusion {@code concat()} call is short-circuited + * for any input that contains a NULL — restoring SQL-standard null-propagation without requiring + * a custom DataFusion UDF. + * + *

      Single-operand calls fall through unchanged (the result equals the operand, so no + * null-handling rewrite is needed). + */ +class ConcatFunctionAdapter implements ScalarFunctionAdapter { + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + List operands = original.getOperands(); + if (operands.size() < 2) { + return original; + } + RexBuilder rexBuilder = cluster.getRexBuilder(); + // Fold operands into a single OR(IS_NULL(o0), IS_NULL(o1), ...) predicate. IS_NULL on a + // non-null literal reduces to constant-false, so the OR collapses cleanly through the + // optimizer for cases where some operands are statically non-null. + RexNode anyNull = rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, operands.get(0)); + for (int i = 1; i < operands.size(); i++) { + anyNull = rexBuilder.makeCall( + SqlStdOperatorTable.OR, + anyNull, + rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, operands.get(i)) + ); + } + // Result type stays the same as the original CONCAT — nullable VARCHAR. + RexNode nullLiteral = rexBuilder.makeNullLiteral(original.getType()); + return rexBuilder.makeCall(original.getType(), SqlStdOperatorTable.CASE, List.of(anyNull, nullLiteral, original)); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 08913c55615a3..9e34579447db2 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -85,12 +85,17 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP // path). COALESCE is the lowering target of PPL `fillnull`. CAST is required because // ReduceExpressionsRule.ProjectReduceExpressionsRule (in PlannerImpl) constant-folds field // references through equality filters into typed literals — e.g. after `where str0 = 'FURNITURE'`, - // the projection `fields str0` is rewritten to `CAST('FURNITURE' AS VARCHAR)`. The remaining - // comparison / arithmetic / logical operators are project-capable for eval-style projections. + // the projection `fields str0` is rewritten to `CAST('FURNITURE' AS VARCHAR)`. CONCAT is the + // lowering target of PPL `eval`'s `+` for strings (Calcite emits `||`, resolved to CONCAT in + // ScalarFunction); SAFE_CAST covers PPL `eval`'s explicit nullable `CAST(... AS ...)` + // expressions. The remaining comparison / arithmetic / logical operators are project-capable + // for eval-style projections. private static final Set STANDARD_PROJECT_OPS = Set.of( ScalarFunction.COALESCE, ScalarFunction.CEIL, ScalarFunction.CAST, + ScalarFunction.CONCAT, + ScalarFunction.SAFE_CAST, ScalarFunction.SARG_PREDICATE, ScalarFunction.EQUALS, ScalarFunction.NOT_EQUALS, @@ -180,15 +185,19 @@ public Set aggregateCapabilities() { @Override public Map scalarFunctionAdapters() { + // Add new (ScalarFunction, ScalarFunctionAdapter) pairs in alphabetical order for + // readability — the Map.ofEntries form keeps spotless happy past the 5-pair point + // where Map.of becomes single-line and unreadable. return Map.ofEntries( - Map.entry(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter()), - Map.entry(ScalarFunction.SARG_PREDICATE, new SargAdapter()), + Map.entry(ScalarFunction.CONCAT, new ConcatFunctionAdapter()), + Map.entry(ScalarFunction.CONVERT_TZ, new ConvertTzAdapter()), Map.entry(ScalarFunction.DIVIDE, new StdOperatorRewriteAdapter("DIVIDE", SqlStdOperatorTable.DIVIDE)), - Map.entry(ScalarFunction.MOD, new StdOperatorRewriteAdapter("MOD", SqlStdOperatorTable.MOD)), Map.entry(ScalarFunction.LIKE, new LikeAdapter()), - Map.entry(ScalarFunction.YEAR, new YearAdapter()), - Map.entry(ScalarFunction.CONVERT_TZ, new ConvertTzAdapter()), - Map.entry(ScalarFunction.UNIX_TIMESTAMP, new UnixTimestampAdapter()) + Map.entry(ScalarFunction.MOD, new StdOperatorRewriteAdapter("MOD", SqlStdOperatorTable.MOD)), + Map.entry(ScalarFunction.SARG_PREDICATE, new SargAdapter()), + Map.entry(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter()), + Map.entry(ScalarFunction.UNIX_TIMESTAMP, new UnixTimestampAdapter()), + Map.entry(ScalarFunction.YEAR, new YearAdapter()) ); } }; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ConcatFunctionAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ConcatFunctionAdapterTests.java new file mode 100644 index 0000000000000..e8123a3446c14 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ConcatFunctionAdapterTests.java @@ -0,0 +1,187 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +/** + * Unit tests for {@link ConcatFunctionAdapter}. The adapter rewrites Calcite's binary + * {@code ||(a, b)} (a.k.a. {@code SqlStdOperatorTable.CONCAT}) into a null-propagating + * {@code CASE WHEN IS_NULL(a) OR IS_NULL(b) THEN NULL ELSE ||(a, b) END}, restoring + * SQL-standard null semantics that DataFusion's substrait-mapped {@code concat()} + * function deviates from. + * + *

      Each test pins one structural invariant of the rewrite — a regression that drops + * the CASE wrapper, mis-orders the IS_NULL operands, or swaps the THEN/ELSE branches + * surfaces here rather than at IT-level row-mismatch failures. + */ +public class ConcatFunctionAdapterTests extends OpenSearchTestCase { + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private RelOptCluster cluster; + private RelDataType varcharType; + + private final ConcatFunctionAdapter adapter = new ConcatFunctionAdapter(); + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + cluster = RelOptCluster.create(planner, rexBuilder); + varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + } + + /** Builds {@code ||(field0, field1)} — Calcite's binary string concat operator. */ + private RexCall buildBinaryConcat() { + RexNode field0 = rexBuilder.makeInputRef(varcharType, 0); + RexNode field1 = rexBuilder.makeInputRef(varcharType, 1); + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.CONCAT, field0, field1); + } + + /** + * Builds an n-ary {@code CONCAT(field0, field1, field2)} via {@code SqlLibraryOperators.CONCAT_FUNCTION} + * to exercise the multi-operand IS_NULL chain path. The binary {@code ||} only ever appears with + * arity 2 in production, but the adapter's loop handles N — this test guards that path. + */ + private RexCall buildTernaryConcat() { + RexNode field0 = rexBuilder.makeInputRef(varcharType, 0); + RexNode field1 = rexBuilder.makeInputRef(varcharType, 1); + RexNode field2 = rexBuilder.makeInputRef(varcharType, 2); + return (RexCall) rexBuilder.makeCall(SqlLibraryOperators.CONCAT_FUNCTION, field0, field1, field2); + } + + // ── core rewrite shape ────────────────────────────────────────────────── + + public void testAdaptBinaryConcatProducesCaseWrapper() { + RexCall concat = buildBinaryConcat(); + RexNode adapted = adapter.adapt(concat, List.of(), cluster); + + assertTrue("expected RexCall, got " + adapted.getClass().getSimpleName(), adapted instanceof RexCall); + RexCall caseCall = (RexCall) adapted; + assertEquals("rewritten root must be CASE", SqlKind.CASE, caseCall.getKind()); + assertEquals("CASE must have exactly three operands [condition, then, else]", 3, caseCall.getOperands().size()); + } + + public void testAdaptedCaseElseBranchIsOriginalConcat() { + RexCall concat = buildBinaryConcat(); + RexCall caseCall = (RexCall) adapter.adapt(concat, List.of(), cluster); + + // Else branch must be the original RexCall, untouched — by reference, not just equal. + // Substrait conversion downstream relies on seeing the same object the resolver annotated. + assertSame("else branch must be the original CONCAT call", concat, caseCall.getOperands().get(2)); + } + + public void testAdaptedCaseThenBranchIsNullLiteralOfMatchingSqlType() { + RexCall concat = buildBinaryConcat(); + RexCall caseCall = (RexCall) adapter.adapt(concat, List.of(), cluster); + + RexNode thenBranch = caseCall.getOperands().get(1); + assertTrue("then branch must be a literal", thenBranch instanceof RexLiteral); + RexLiteral literal = (RexLiteral) thenBranch; + assertNull("then branch literal must be NULL-valued", literal.getValue()); + // RexBuilder.makeNullLiteral promotes nullability on the literal's type even when the + // original isn't nullable, so the full RelDataType objects differ. The SQL type name + // (VARCHAR vs INTEGER vs ...) is the load-bearing invariant — overall CASE return type + // identity to the original is asserted in testAdaptPreservesReturnType. + assertEquals( + "NULL literal SQL type must match the original CONCAT's SQL type", + concat.getType().getSqlTypeName(), + literal.getType().getSqlTypeName() + ); + } + + public void testAdaptedCaseConditionIsOrOfIsNullChecks() { + RexCall concat = buildBinaryConcat(); + RexCall caseCall = (RexCall) adapter.adapt(concat, List.of(), cluster); + + RexNode condition = caseCall.getOperands().get(0); + assertEquals("condition must be OR(IS_NULL(a), IS_NULL(b))", SqlKind.OR, condition.getKind()); + + RexCall orCall = (RexCall) condition; + assertEquals(2, orCall.getOperands().size()); + for (int i = 0; i < orCall.getOperands().size(); i++) { + RexNode disjunct = orCall.getOperands().get(i); + assertEquals("OR operand " + i + " must be IS_NULL", SqlKind.IS_NULL, disjunct.getKind()); + // Each IS_NULL must wrap the corresponding original operand — order matters for the + // null-propagation contract. + assertSame( + "IS_NULL operand " + i + " must reference the original CONCAT operand " + i, + concat.getOperands().get(i), + ((RexCall) disjunct).getOperands().get(0) + ); + } + } + + public void testAdaptPreservesReturnType() { + RexCall concat = buildBinaryConcat(); + RexNode adapted = adapter.adapt(concat, List.of(), cluster); + + assertEquals("CASE return type must equal the original CONCAT return type", concat.getType(), adapted.getType()); + } + + // ── n-ary path ────────────────────────────────────────────────────────── + + public void testAdaptNaryConcatChainsIsNullChecksLeftAssociative() { + RexCall concat = buildTernaryConcat(); + RexCall caseCall = (RexCall) adapter.adapt(concat, List.of(), cluster); + + // Condition shape: OR(OR(IS_NULL(a), IS_NULL(b)), IS_NULL(c)) — left-fold. + RexNode condition = caseCall.getOperands().get(0); + assertEquals(SqlKind.OR, condition.getKind()); + + // Right child is IS_NULL(c) — the most recently appended operand in the fold. + RexCall outerOr = (RexCall) condition; + assertEquals(2, outerOr.getOperands().size()); + RexNode rightChild = outerOr.getOperands().get(1); + assertEquals(SqlKind.IS_NULL, rightChild.getKind()); + assertSame(concat.getOperands().get(2), ((RexCall) rightChild).getOperands().get(0)); + + // Left child is OR(IS_NULL(a), IS_NULL(b)) — the previously folded prefix. + RexNode leftChild = outerOr.getOperands().get(0); + assertEquals(SqlKind.OR, leftChild.getKind()); + RexCall innerOr = (RexCall) leftChild; + assertEquals(SqlKind.IS_NULL, innerOr.getOperands().get(0).getKind()); + assertEquals(SqlKind.IS_NULL, innerOr.getOperands().get(1).getKind()); + assertSame(concat.getOperands().get(0), ((RexCall) innerOr.getOperands().get(0)).getOperands().get(0)); + assertSame(concat.getOperands().get(1), ((RexCall) innerOr.getOperands().get(1)).getOperands().get(0)); + } + + // ── pass-through guard ───────────────────────────────────────────────── + + public void testAdaptSingleOperandConcatPassesThroughUnchanged() { + // Built via the variadic CONCAT_FUNCTION since SqlStdOperatorTable.CONCAT is binary and + // can't represent a single-operand call. The adapter's contract is that a 1-operand call + // is a no-op — concat with one input equals that input, no null handling needed. + RexNode field0 = rexBuilder.makeInputRef(varcharType, 0); + RexCall singleOperand = (RexCall) rexBuilder.makeCall(SqlLibraryOperators.CONCAT_FUNCTION, field0); + + RexNode adapted = adapter.adapt(singleOperand, List.of(), cluster); + + assertSame("single-operand call must pass through unmodified", singleOperand, adapted); + } +} diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/BackendPlanAdapter.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/BackendPlanAdapter.java index a3bd02fa811dd..e65cff3b8b686 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/BackendPlanAdapter.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/BackendPlanAdapter.java @@ -13,7 +13,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.SqlFunction; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.analytics.planner.CapabilityRegistry; @@ -196,13 +195,6 @@ private static RexNode adaptRex( } private static ScalarFunction resolveFunction(RexCall call) { - if (call.getOperator() instanceof SqlFunction sqlFunction) { - try { - return ScalarFunction.fromSqlFunction(sqlFunction); - } catch (IllegalArgumentException ignored) { - // Not in our enum — fall through to SqlKind resolution - } - } - return ScalarFunction.fromSqlKind(call.getKind()); + return ScalarFunction.fromSqlOperatorWithFallback(call.getOperator()); } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java index c50e643282821..379240c44ee81 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchFilterRule.java @@ -15,7 +15,6 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlKind; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -153,15 +152,11 @@ private List resolveViableBackends( ); } - ScalarFunction function = null; - if (predicate.getOperator() instanceof SqlFunction sqlFunction) { - function = ScalarFunction.fromSqlFunction(sqlFunction); - } - if (function == null) { - function = ScalarFunction.fromSqlKind(predicate.getKind()); - } + ScalarFunction function = ScalarFunction.fromSqlOperatorWithFallback(predicate.getOperator()); if (function == null) { - throw new IllegalStateException("Unrecognized filter operator [" + predicate.getKind() + "]"); + throw new IllegalStateException( + "Unrecognized filter operator [" + predicate.getOperator().getName() + " / " + predicate.getKind() + "]" + ); } Set viableSet = new HashSet<>(registry.filterCapableBackends()); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java index 8d98096e21557..711bb3a5c8e1b 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java @@ -119,9 +119,9 @@ private RexNode annotateExpr(RexNode expr, List childViableBackends) { // Standard scalar function List scalarViable = resolveScalarViableBackends(rexCall, childViableBackends); if (scalarViable.isEmpty()) { - throw new IllegalStateException( - "No backend supports scalar function [" + ScalarFunction.fromSqlKind(rexCall.getKind()) + "] among " + childViableBackends - ); + ScalarFunction resolved = ScalarFunction.fromSqlOperatorWithFallback(rexCall.getOperator()); + String label = resolved != null ? resolved.name() : rexCall.getOperator().getName(); + throw new IllegalStateException("No backend supports scalar function [" + label + "] among " + childViableBackends); } // Recurse into operands @@ -158,10 +158,7 @@ private List resolveOpaqueViableBackends(String funcName, List c } private List resolveScalarViableBackends(RexCall rexCall, List childViableBackends) { - ScalarFunction scalarFunc = ScalarFunction.fromSqlKind(rexCall.getKind()); - if (scalarFunc == null && rexCall.getOperator() instanceof SqlFunction sqlFunction) { - scalarFunc = ScalarFunction.fromSqlFunction(sqlFunction); - } + ScalarFunction scalarFunc = ScalarFunction.fromSqlOperatorWithFallback(rexCall.getOperator()); if (scalarFunc == null) { return List.of(); } diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/EvalCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/EvalCommandIT.java new file mode 100644 index 0000000000000..285f3a771df89 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/EvalCommandIT.java @@ -0,0 +1,224 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for PPL {@code eval} on the analytics-engine route. + * + *

      Mirrors {@code CalciteEvalCommandIT} from the {@code opensearch-project/sql} + * repository so that the analytics-engine path can be verified inside core without + * cross-plugin dependencies on the SQL plugin. Each test sends a PPL query through + * {@code POST /_analytics/ppl} (exposed by the {@code test-ppl-frontend} plugin), + * which runs the same {@code UnifiedQueryPlanner} → {@code CalciteRelNodeVisitor} → + * Substrait → DataFusion pipeline as the SQL plugin's force-routed analytics path. + * + *

      The eval surface this test exercises is string concatenation via PPL's {@code +} + * operator (lowered to Calcite's {@code SqlStdOperatorTable.CONCAT}, i.e. the {@code ||} + * binary operator) and {@code CAST(... AS STRING)}, both routed through the + * {@link org.opensearch.analytics.spi.ScalarFunction#CONCAT} and + * {@link org.opensearch.analytics.spi.ScalarFunction#CAST} entries in the DataFusion + * backend's {@code STANDARD_PROJECT_OPS}. {@code ||} resolves through the symbolic-name + * branch of {@link org.opensearch.analytics.spi.ScalarFunction#fromSqlOperatorWithFallback} since it + * is a {@code SqlBinaryOperator} (not a {@code SqlFunction}) with {@code SqlKind.OTHER}. + * + *

      Provisions the {@code calcs} dataset (parquet-backed) once per class via + * {@link DatasetProvisioner}; {@link AnalyticsRestTestCase#preserveIndicesUponCompletion()} + * keeps it across test methods. + */ +public class EvalCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + /** + * Lazily provision the calcs dataset on first invocation. Must be called inside a test + * method (not {@code setUp()}) — {@link org.opensearch.test.rest.OpenSearchRestTestCase}'s + * static {@code client()} is not initialized until after {@code @BeforeClass}, but is + * reliably available inside test bodies. + */ + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + // ── string concat: 'literal' + str_field ────────────────────────────────── + + public void testEvalStringConcatLiteralPlusField() throws IOException { + // 'Hello ' + str2 — Calcite emits || (CONCAT). Null str2 propagates through CONCAT, + // producing a null greeting (e.g. row index 3 has str2 = null → greeting = null). + assertRows( + "source=" + DATASET.indexName + " | fields str2 | eval greeting = 'Hello ' + str2", + row("one", "Hello one"), + row("two", "Hello two"), + row("three", "Hello three"), + row(null, null), + row("five", "Hello five"), + row("six", "Hello six"), + row(null, null), + row("eight", "Hello eight"), + row("nine", "Hello nine"), + row("ten", "Hello ten"), + row("eleven", "Hello eleven"), + row("twelve", "Hello twelve"), + row(null, null), + row("fourteen", "Hello fourteen"), + row("fifteen", "Hello fifteen"), + row("sixteen", "Hello sixteen"), + row(null, null) + ); + } + + // ── CAST + concat: 'literal' + CAST(int AS STRING) ──────────────────────── + + public void testEvalStringConcatWithCastIntField() throws IOException { + // CAST(null AS STRING) is null; concat with null propagates → label is null. + // int0 has nulls at rows 1, 2, 3, 7, 8, 12 (per FillNullCommandIT row data). + assertRows( + "source=" + DATASET.indexName + " | eval label = 'Int: ' + CAST(int0 AS STRING) | fields str2, int0, label", + row("one", 1, "Int: 1"), + row("two", null, null), + row("three", null, null), + row(null, null, null), + row("five", 7, "Int: 7"), + row("six", 3, "Int: 3"), + row(null, 8, "Int: 8"), + row("eight", null, null), + row("nine", null, null), + row("ten", 8, "Int: 8"), + row("eleven", 4, "Int: 4"), + row("twelve", 10, "Int: 10"), + row(null, null, null), + row("fourteen", 4, "Int: 4"), + row("fifteen", 11, "Int: 11"), + row("sixteen", 4, "Int: 4"), + row(null, 8, "Int: 8") + ); + } + + // ── chained concat: 'a' + str + 'b' + str' ──────────────────────────────── + + public void testEvalStringConcatMultipleLiteralsAndFields() throws IOException { + // Chains four CONCAT calls — exercises the recursive AnnotatedProjectExpression strip + // for nested project calls (same pattern that fillnull surfaced for ceil(num1)). + // str0 ("FURNITURE"-style) is non-null in calcs; str2 has nulls — null str2 + // propagates through the chain to make the whole row's full_label null. + assertRows( + "source=" + DATASET.indexName + " | eval full_label = 'A=' + str0 + ', B=' + str2 | fields str0, str2, full_label", + row("FURNITURE", "one", "A=FURNITURE, B=one"), + row("FURNITURE", "two", "A=FURNITURE, B=two"), + row("OFFICE SUPPLIES", "three", "A=OFFICE SUPPLIES, B=three"), + row("OFFICE SUPPLIES", null, null), + row("OFFICE SUPPLIES", "five", "A=OFFICE SUPPLIES, B=five"), + row("OFFICE SUPPLIES", "six", "A=OFFICE SUPPLIES, B=six"), + row("OFFICE SUPPLIES", null, null), + row("OFFICE SUPPLIES", "eight", "A=OFFICE SUPPLIES, B=eight"), + row("TECHNOLOGY", "nine", "A=TECHNOLOGY, B=nine"), + row("TECHNOLOGY", "ten", "A=TECHNOLOGY, B=ten"), + row("TECHNOLOGY", "eleven", "A=TECHNOLOGY, B=eleven"), + row("TECHNOLOGY", "twelve", "A=TECHNOLOGY, B=twelve"), + row("TECHNOLOGY", null, null), + row("TECHNOLOGY", "fourteen", "A=TECHNOLOGY, B=fourteen"), + row("TECHNOLOGY", "fifteen", "A=TECHNOLOGY, B=fifteen"), + row("TECHNOLOGY", "sixteen", "A=TECHNOLOGY, B=sixteen"), + row("TECHNOLOGY", null, null) + ); + } + + // ── concat between two field references ─────────────────────────────────── + + public void testEvalStringConcatTwoFields() throws IOException { + // Pure field-to-field concat through two || calls (str0 + ' ' + str2). + // No literal-only operands — the planner must accept CONCAT with both + // RexInputRef inputs (hasFieldRef=true path in resolveScalarViableBackends). + assertRows( + "source=" + DATASET.indexName + " | eval combo = str0 + ' ' + str2 | fields str0, str2, combo", + row("FURNITURE", "one", "FURNITURE one"), + row("FURNITURE", "two", "FURNITURE two"), + row("OFFICE SUPPLIES", "three", "OFFICE SUPPLIES three"), + row("OFFICE SUPPLIES", null, null), + row("OFFICE SUPPLIES", "five", "OFFICE SUPPLIES five"), + row("OFFICE SUPPLIES", "six", "OFFICE SUPPLIES six"), + row("OFFICE SUPPLIES", null, null), + row("OFFICE SUPPLIES", "eight", "OFFICE SUPPLIES eight"), + row("TECHNOLOGY", "nine", "TECHNOLOGY nine"), + row("TECHNOLOGY", "ten", "TECHNOLOGY ten"), + row("TECHNOLOGY", "eleven", "TECHNOLOGY eleven"), + row("TECHNOLOGY", "twelve", "TECHNOLOGY twelve"), + row("TECHNOLOGY", null, null), + row("TECHNOLOGY", "fourteen", "TECHNOLOGY fourteen"), + row("TECHNOLOGY", "fifteen", "TECHNOLOGY fifteen"), + row("TECHNOLOGY", "sixteen", "TECHNOLOGY sixteen"), + row("TECHNOLOGY", null, null) + ); + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + private static List row(Object... values) { + return Arrays.asList(values); + } + + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRows(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' field for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size()); + for (int i = 0; i < expected.length; i++) { + List want = expected[i]; + List got = actualRows.get(i); + assertEquals("Column count mismatch at row " + i + " for query: " + ppl, want.size(), got.size()); + for (int j = 0; j < want.size(); j++) { + assertCellEquals("Cell mismatch at row " + i + ", col " + j + " for query: " + ppl, want.get(j), got.get(j)); + } + } + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } + + /** + * Numeric-tolerant cell comparison — JSON parsing returns {@code Integer}/{@code Long}/{@code Double} + * interchangeably. PPL doesn't preserve the distinction at the API surface, so cross-type numeric + * equality must be measured by {@code double} values rather than {@link Object#equals(Object)}. + */ + private static void assertCellEquals(String message, Object expected, Object actual) { + if (expected == null || actual == null) { + assertEquals(message, expected, actual); + return; + } + if (expected instanceof Number && actual instanceof Number) { + double e = ((Number) expected).doubleValue(); + double a = ((Number) actual).doubleValue(); + if (Double.compare(e, a) != 0) { + fail(message + ": expected <" + expected + "> but was <" + actual + ">"); + } + return; + } + assertEquals(message, expected, actual); + } +} From 50125c57ce46f267d4d368deddc0ea911a361d4f Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Wed, 6 May 2026 16:36:51 -0700 Subject: [PATCH 064/115] [Analytics Backend / DataFusion] Wire ABS / SUBSTRING + QA ITs for fields/rename/head/sort (#21521) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bucket-1 capability-registry expansion for the analytics-engine route — narrow scope: only the two scalar functions PPL sort push-down materialises into a projection (ABS, SUBSTRING). Fields / rename / head don't add scalar surface; they're covered here purely by new QA ITs that lock in the routing-and-shape behavior end-to-end through `POST /_analytics/ppl`. After this PR (with the eval-side surface from #21498 already on main): * `CalciteFieldsCommandIT`, `CalciteRenameCommandIT`, `CalciteHeadCommandIT` 100% green on the analytics path under `tests.analytics.force_routing=true`. * `CalciteSortCommandIT` and `CalcitePPLSortIT` pick up the cast / abs / substring push-down failures (CAST is already in `STANDARD_PROJECT_OPS` upstream from #21476; ABS and SUBSTRING are this PR's contribution). ## Changes **1. `DataFusionAnalyticsBackendPlugin` — `STANDARD_PROJECT_OPS` += ABS, SUBSTRING.** PPL sort push-down lifts an expression like `abs(num0)` or `substring(str0, 1, 3)` into a `LogicalProject` child of the sort, which is what the project rule's capability check sees. DataFusion has both natively; isthmus' default extension catalog already binds them. Without this, the analytics planner rejects the projection with `No backend supports scalar function [ABS] among [datafusion]`. **2. QA ITs in `sandbox/qa/analytics-engine-rest`** — one per command, each self-contained and provisioning the existing `calcs` parquet-backed dataset via `DatasetProvisioner`. Tests fire through `POST /_analytics/ppl` so the core build can validate the analytics-engine path without the SQL plugin. Mirror the failing surface in `CalciteFieldsCommandIT` / `CalciteRenameCommandIT` / `CalciteHeadCommandIT` / `CalciteSortCommandIT` one query at a time: * `FieldsCommandIT` (5 tests) — basic projection, single-column, explicit order, suffix-wildcard `*0` (set-equality, since wildcard expansion order isn't part of the contract), and `fields - num*` exclusion. * `RenameCommandIT` (4 tests) — single rename, multi-rename, post-rename reference fails with "not found", backtick-quoted target names. * `HeadCommandIT` (5 tests) — default-10 cap, explicit count, count > total rows, `head N from M` offset, and value-equality on the first 5 rows (parquet preserves insertion order, so this is deterministic). * `SortCommandIT` (5 tests) — plain ASC/DESC by integer (with calcs' 6 null int0 entries placed at the head/tail per Calcite's nulls-first/last defaults), `eval n = abs(num0) | sort n` covering the 9 null + 8 non-null abs values, and `eval s = substring(str2, 1, 3) | sort s` validating the SUBSTRING capability end-to-end against the 17-row calcs dataset. ## Test plan * `./gradlew :sandbox:qa:analytics-engine-rest:integTest -Dsandbox.enabled=true --tests "*FieldsCommandIT" --tests "*RenameCommandIT" --tests "*HeadCommandIT" --tests "*SortCommandIT"` — 19/19 green. * `./gradlew check -p sandbox -Dsandbox.enabled=true` — green (the unrelated `ScalarDateTimeFunctionIT.testConvertTz` flake from a stale local `libopensearch_native.dylib` resolved by rebuilding the Rust crate; not caused by this PR). * SQL-plugin Calcite ITs against this branch + companion https://github.com/opensearch-project/sql/pull/5413, with `-Dtests.analytics.force_routing=true -Dtests.analytics.parquet_indices=true`: `CalciteFieldsCommandIT` 39/39, `CalciteRenameCommandIT` 2/2, `CalciteHeadCommandIT` 4/4, plus +5 sort-push-down wins in `CalciteSortCommandIT` and +1 in `CalcitePPLSortIT` from the ABS / SUBSTRING capability additions. ## Out of scope (separate follow-ups) * `Unable to find binding for call AVG($N)` Substrait-isthmus issue — needs an `AggregateSig`-style additional-mappings hook in `DataFusionFragmentConvertor`. * Window functions (`dedup` lowers to `ROW_NUMBER OVER`). * Advanced aggregates (`first`, `last`, `take`, `arg_max`, `percentile_approx`, `distinct_count_approx`) and PPL `span`. * The eval-predicate surface (`AND`/`OR`/`NOT` in CASE projections, `IS_NULL`/`IS_NOT_NULL`, broader string/conditional ops) and STDDEV/VAR aggregates — kept out of this PR to keep the scope focused on the four commands the QA ITs cover. Signed-off-by: Kai Huang --- .../DataFusionAnalyticsBackendPlugin.java | 7 + .../analytics/qa/FieldsCommandIT.java | 162 ++++++++++++++ .../analytics/qa/HeadCommandIT.java | 111 ++++++++++ .../analytics/qa/RenameCommandIT.java | 124 +++++++++++ .../analytics/qa/SortCommandIT.java | 198 ++++++++++++++++++ 5 files changed, 602 insertions(+) create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldsCommandIT.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/HeadCommandIT.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/RenameCommandIT.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/SortCommandIT.java diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 9e34579447db2..cd7ada3134752 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -96,6 +96,13 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP ScalarFunction.CAST, ScalarFunction.CONCAT, ScalarFunction.SAFE_CAST, + // ABS / SUBSTRING — `eval x = abs(...)` and `eval s = substring(...)` projections that PPL + // sort-pushdown moves into the project tree (see CalciteSortCommandIT + // testPushdownSortExpressionContainsNull and CalcitePPLSortIT + // testPushdownSortStringExpression). DataFusion has both natively; isthmus default catalog + // already binds them. + ScalarFunction.ABS, + ScalarFunction.SUBSTRING, ScalarFunction.SARG_PREDICATE, ScalarFunction.EQUALS, ScalarFunction.NOT_EQUALS, diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldsCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldsCommandIT.java new file mode 100644 index 0000000000000..6a315b287480b --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldsCommandIT.java @@ -0,0 +1,162 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for PPL {@code fields} on the analytics-engine route. + * + *

      Mirrors {@code CalciteFieldsCommandIT} from the {@code opensearch-project/sql} + * repository so the analytics-engine path can be verified inside core without cross-plugin + * dependencies. Each test sends a PPL query through {@code POST /_analytics/ppl}, which + * runs the same {@code UnifiedQueryPlanner} → {@code CalciteRelNodeVisitor} → Substrait + * → DataFusion pipeline as the SQL plugin's force-routed analytics path. + * + *

      Covers the field-projection surface this PR cares about: explicit single/multi-field + * lists, wildcard include patterns, and field exclusion. Wildcard suffix/prefix patterns + * delegate to {@code CalciteRelNodeVisitor.visitProject} which expands them at plan time; + * the exclusion form (`fields - x, y`) goes through the same code path with `exclude=true`. + */ +public class FieldsCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + public void testFieldsBasic() throws IOException { + // Two-column projection. Row order is the document insertion order; the analytics + // path reads from parquet which preserves that. + assertColumns("source=" + DATASET.indexName + " | fields str2, num0 | head 3", "str2", "num0"); + } + + public void testFieldsSingleColumn() throws IOException { + assertRowsEqual( + "source=" + DATASET.indexName + " | fields str2 | head 5", + row("one"), + row("two"), + row("three"), + row((Object) null), + row("five") + ); + } + + public void testFieldsExplicitOrder() throws IOException { + // Column order must match the | fields list, not the document/storage order. + assertColumns( + "source=" + DATASET.indexName + " | fields num0, str2 | head 1", + "num0", + "str2" + ); + } + + public void testFieldsSuffixWildcard() throws IOException { + // *0 expands to all columns ending in '0' — {num0, str0, int0, bool0, date0, time0, + // datetime0}. Order isn't guaranteed (analyzer resolves wildcards by mapping iteration + // order, which is alphabetical here). Verify the set rather than the sequence. + Map response = executePpl( + "source=" + DATASET.indexName + " | fields *0 | head 1" + ); + @SuppressWarnings("unchecked") + List columns = (List) response.get("columns"); + assertNotNull("Response missing 'columns'", columns); + java.util.Set actual = new java.util.HashSet<>(columns); + java.util.Set expected = new java.util.HashSet<>( + Arrays.asList("num0", "str0", "int0", "bool0", "date0", "time0", "datetime0") + ); + assertEquals("Wildcard *0 column set", expected, actual); + } + + public void testFieldsExclusion() throws IOException { + // `fields - num0, num1, num2, num3, num4` removes those five columns from the + // projection. Validate the result no longer contains num*. + Map response = executePpl( + "source=" + DATASET.indexName + " | fields - num0, num1, num2, num3, num4 | head 1" + ); + @SuppressWarnings("unchecked") + List columns = (List) response.get("columns"); + assertNotNull("Response missing 'columns'", columns); + for (String name : columns) { + assertFalse("Excluded column should not appear: " + name, name.startsWith("num")); + } + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + private static List row(Object... values) { + return Arrays.asList(values); + } + + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRowsEqual(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size()); + for (int i = 0; i < expected.length; i++) { + List want = expected[i]; + List got = actualRows.get(i); + assertEquals( + "Column count mismatch at row " + i + " for query: " + ppl, + want.size(), + got.size() + ); + for (int j = 0; j < want.size(); j++) { + assertEquals( + "Cell mismatch at row " + i + ", col " + j + " for query: " + ppl, + want.get(j), + got.get(j) + ); + } + } + } + + /** Assert the response has the expected column names in order. */ + private void assertColumns(String ppl, String... expectedColumns) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List columns = (List) response.get("columns"); + assertNotNull("Response missing 'columns' for query: " + ppl, columns); + assertEquals( + "Column count for query: " + ppl, + expectedColumns.length, + columns.size() + ); + for (int i = 0; i < expectedColumns.length; i++) { + assertEquals( + "Column at position " + i + " for query: " + ppl, + expectedColumns[i], + columns.get(i) + ); + } + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/HeadCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/HeadCommandIT.java new file mode 100644 index 0000000000000..2681e72fb7dab --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/HeadCommandIT.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for PPL {@code head} on the analytics-engine route. + * + *

      Mirrors {@code CalciteHeadCommandIT}. {@code head N} lowers to {@code LogicalSort} + * with {@code fetch=N} (no sort key); {@code head N from M} adds {@code offset=M}. + * Pure relational op, no scalar surface — exercises the row-cap path through + * {@code OpenSearchSort} and the DataFusion fragment driver's limit propagation. + */ +public class HeadCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + public void testHeadDefault() throws IOException { + // `head` without a count defaults to 10. + assertRowCount("source=" + DATASET.indexName + " | fields str2 | head", 10); + } + + public void testHeadWithCount() throws IOException { + assertRowCount("source=" + DATASET.indexName + " | fields str2 | head 3", 3); + } + + public void testHeadWithCountLargerThanData() throws IOException { + // Calcs has 17 rows. Asking for more should cap at 17, not error. + assertRowCount("source=" + DATASET.indexName + " | fields str2 | head 100", 17); + } + + public void testHeadFromOffset() throws IOException { + // `head N from M` skips M rows and returns the next N. With 17 rows total, + // `head 5 from 14` returns rows 14, 15, 16 (only 3 left). + assertRowCount("source=" + DATASET.indexName + " | fields str2 | head 5 from 14", 3); + } + + public void testHeadValuesMatchInsertionOrder() throws IOException { + // Parquet returns rows in storage / insertion order. The first 5 calcs rows + // (key00..key04) have str2 = one, two, three, null, five. + assertRowsEqual( + "source=" + DATASET.indexName + " | fields str2 | head 5", + row("one"), + row("two"), + row("three"), + row((Object) null), + row("five") + ); + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + private static List row(Object... values) { + return Arrays.asList(values); + } + + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRowsEqual(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, actualRows); + assertEquals("Row count for query: " + ppl, expected.length, actualRows.size()); + for (int i = 0; i < expected.length; i++) { + assertEquals( + "Cell mismatch at row " + i + " for query: " + ppl, + expected[i], + actualRows.get(i) + ); + } + } + + private void assertRowCount(String ppl, int expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> rows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, rows); + assertEquals("Row count for query: " + ppl, expected, rows.size()); + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/RenameCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/RenameCommandIT.java new file mode 100644 index 0000000000000..97d61b9aadc39 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/RenameCommandIT.java @@ -0,0 +1,124 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for PPL {@code rename} on the analytics-engine route. + * + *

      Mirrors {@code CalciteRenameCommandIT} from the {@code opensearch-project/sql} + * repository so the analytics-engine path can be verified inside core. The {@code rename} + * command lowers to a Calcite {@code LogicalProject} with renamed output column names — + * pure projection, no scalar functions, no capability-registry dependencies. The IT here + * is a smoke test for the full pipeline: PPL parse → AstBuilder → CalciteRelNodeVisitor + * → analytics-engine planner → DataFusion execution → JSON response. + */ +public class RenameCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + public void testRenameSingleField() throws IOException { + // The output column name must be the rename target ("label"), not "str2". + Map response = executePpl( + "source=" + DATASET.indexName + " | rename str2 as label | fields label | head 3" + ); + assertSingletonColumn(response, "label"); + + @SuppressWarnings("unchecked") + List> rows = (List>) response.get("rows"); + assertEquals("Row count", 3, rows.size()); + } + + public void testRenameMultipleFields() throws IOException { + // Two renames in one command, then explicit projection in the renamed names. + Map response = executePpl( + "source=" + + DATASET.indexName + + " | rename str2 as label, num0 as value | fields label, value | head 5" + ); + @SuppressWarnings("unchecked") + List columns = (List) response.get("columns"); + assertNotNull("Response missing 'columns'", columns); + assertEquals("Column count", 2, columns.size()); + assertEquals("First renamed column", "label", columns.get(0)); + assertEquals("Second renamed column", "value", columns.get(1)); + } + + public void testRenameThenReferenceOriginalFails() { + // After renaming, the original name is no longer addressable. Mirrors + // CalcitePPLRenameIT.testRefRenamedField — analytics path should surface + // the same "Field [...] not found" error from the analyzer. + assertErrorContains( + "source=" + DATASET.indexName + " | rename str2 as label | fields str2", + "not found" + ); + } + + public void testRenameWithBackticks() throws IOException { + Map response = executePpl( + "source=" + + DATASET.indexName + + " | rename str2 as `renamed_label` | fields `renamed_label` | head 1" + ); + assertSingletonColumn(response, "renamed_label"); + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + private void assertSingletonColumn(Map response, String expectedName) { + @SuppressWarnings("unchecked") + List columns = (List) response.get("columns"); + assertNotNull("Response missing 'columns'", columns); + assertEquals("Column count", 1, columns.size()); + assertEquals("Column name", expectedName, columns.get(0)); + } + + private void assertErrorContains(String ppl, String expectedSubstring) { + try { + Map response = executePpl(ppl); + fail("Expected query to fail with [" + expectedSubstring + "] but got response: " + response); + } catch (org.opensearch.client.ResponseException e) { + String body; + try { + body = org.opensearch.test.rest.OpenSearchRestTestCase.entityAsMap(e.getResponse()).toString(); + } catch (IOException ioe) { + body = e.getMessage(); + } + assertTrue( + "Expected response body to contain [" + expectedSubstring + "] but was: " + body, + body.contains(expectedSubstring) + ); + } catch (IOException e) { + fail("Unexpected IOException: " + e); + } + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/SortCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/SortCommandIT.java new file mode 100644 index 0000000000000..259a02e4355a5 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/SortCommandIT.java @@ -0,0 +1,198 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for PPL {@code sort} on the analytics-engine route. + * + *

      Mirrors {@code CalciteSortCommandIT} / {@code CalcitePPLSortIT}. {@code sort} lowers + * to {@code LogicalSort}; the asc / desc / nulls-first / nulls-last variants set the + * collation field on the same RelNode. Push-down sort by an expression (`sort abs(num0)`) + * lifts the expression into a {@code LogicalProject} child of the sort, which is what + * exercises the new project-side capabilities for {@link org.opensearch.analytics.spi.ScalarFunction#ABS} + * and {@link org.opensearch.analytics.spi.ScalarFunction#SUBSTRING} added in this PR. + */ +public class SortCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + // ── plain field sort ─────────────────────────────────────────────────────── + + public void testSortAscByInt() throws IOException { + // int0 across the 17 calcs rows: [1, null, null, null, 7, 3, 8, null, null, 8, 4, 10, + // null, 4, 11, 4, 8] — 6 nulls and 11 integers. Default sort is ASC nulls-first. + assertRowsEqual( + "source=" + DATASET.indexName + " | sort int0 | fields int0", + row((Object) null), row((Object) null), row((Object) null), + row((Object) null), row((Object) null), row((Object) null), + row(1), row(3), row(4), row(4), row(4), row(7), row(8), row(8), row(8), row(10), row(11) + ); + } + + public void testSortDescByInt() throws IOException { + // DESC nulls-last (the analytics path follows Calcite's default DESC = NULLS LAST). + assertRowsEqual( + "source=" + DATASET.indexName + " | sort -int0 | fields int0", + row(11), row(10), row(8), row(8), row(8), row(7), row(4), row(4), row(4), + row(3), row(1), + row((Object) null), row((Object) null), row((Object) null), + row((Object) null), row((Object) null), row((Object) null) + ); + } + + // ── push-down sort by scalar expression — exercises ABS / SUBSTRING capabilities ── + + public void testSortByAbsExpression() throws IOException { + // `abs(num0)` lowers to ABS($N) inside a LogicalProject child of the sort. Without + // ABS in STANDARD_PROJECT_OPS, the analytics planner rejects the projection with + // "No backend supports scalar function [ABS] among [datafusion]". + // + // Calcs num0: [12.3, -12.3, 15.7, -15.7, 3.5, -3.5, 0, null, 10, null x8] — 9 nulls + // and 8 non-nulls. abs(num0) preserves null and yields {0, 3.5, 3.5, 10, 12.3, 12.3, + // 15.7, 15.7} for the non-null tail. Sorted ASC nulls-first puts the 9 nulls first. + Map response = executePpl( + "source=" + DATASET.indexName + " | eval n = abs(num0) | sort n | fields n | head 9" + ); + @SuppressWarnings("unchecked") + List> rows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows'", rows); + assertEquals("Row count", 9, rows.size()); + for (int i = 0; i < 9; i++) { + assertNull("Row " + i + " should be null", rows.get(i).get(0)); + } + } + + public void testSortByAbsTakesNonNullsFromTail() throws IOException { + // Skip past the 9 nulls and verify the 8 non-null abs values appear in ASC order. + Map response = executePpl( + "source=" + + DATASET.indexName + + " | eval n = abs(num0) | sort n | fields n | head 8 from 9" + ); + @SuppressWarnings("unchecked") + List> rows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows'", rows); + assertEquals("Row count after 9 nulls", 8, rows.size()); + double[] expectedSorted = { 0, 3.5, 3.5, 10, 12.3, 12.3, 15.7, 15.7 }; + for (int i = 0; i < expectedSorted.length; i++) { + Object v = rows.get(i).get(0); + assertNotNull("Row " + i + " unexpectedly null", v); + assertEquals( + "abs(num0) sorted value at row " + i, + expectedSorted[i], + ((Number) v).doubleValue(), + 1e-9 + ); + } + } + + public void testSortBySubstringExpression() throws IOException { + // `substring(str2, 1, 3)` lowers to SUBSTRING($N, 1, 3) inside a LogicalProject child of + // the sort. Without SUBSTRING in STANDARD_PROJECT_OPS, the planner rejects it with + // "No backend supports scalar function [SUBSTRING] among [datafusion]". + // + // Calcs str2 first 3 chars (where non-null): one, two, thr, fiv, six, eig, nin, ten, + // ele, twe, fou, fif, six. Sort ASC nulls-first puts the 4 nulls first. + Map response = executePpl( + "source=" + DATASET.indexName + " | eval s = substring(str2, 1, 3) | sort s | fields s" + ); + @SuppressWarnings("unchecked") + List> rows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows'", rows); + assertEquals("Row count == calcs row count", 17, rows.size()); + // First 4 rows must be nulls (4 null str2 values in calcs). + for (int i = 0; i < 4; i++) { + assertNull("Expected null at row " + i + " (sorted ASC nulls-first)", rows.get(i).get(0)); + } + // The remaining 13 must be sorted alphabetically. + for (int i = 5; i < rows.size(); i++) { + String prev = (String) rows.get(i - 1).get(0); + String curr = (String) rows.get(i).get(0); + assertNotNull("Non-null after null block", curr); + assertTrue( + "Sort order violation at row " + i + ": " + prev + " > " + curr, + prev.compareTo(curr) <= 0 + ); + } + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + private static List row(Object... values) { + return Arrays.asList(values); + } + + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRowsEqual(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size()); + for (int i = 0; i < expected.length; i++) { + List want = expected[i]; + List got = actualRows.get(i); + assertEquals( + "Column count mismatch at row " + i + " for query: " + ppl, + want.size(), + got.size() + ); + for (int j = 0; j < want.size(); j++) { + assertCellEquals( + "Cell mismatch at row " + i + ", col " + j + " for query: " + ppl, + want.get(j), + got.get(j) + ); + } + } + } + + /** Numeric-tolerant cell comparator (Jackson returns Integer/Long/Double interchangeably). */ + private static void assertCellEquals(String message, Object expected, Object actual) { + if (expected == null || actual == null) { + assertEquals(message, expected, actual); + return; + } + if (expected instanceof Number && actual instanceof Number) { + double e = ((Number) expected).doubleValue(); + double a = ((Number) actual).doubleValue(); + if (Double.compare(e, a) != 0) { + fail(message + ": expected <" + expected + "> but was <" + actual + ">"); + } + return; + } + assertEquals(message, expected, actual); + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } +} From 6c7d6ffdcde72a7914816261222dbabc27abf488 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Wed, 6 May 2026 18:31:12 -0700 Subject: [PATCH 065/115] [QA] Add AppendPipeCommandIT and TableCommandIT for the analytics-engine REST path (#21526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [QA] Add AppendPipeCommandIT for the analytics-engine REST path PPL `appendpipe` already passes 4/4 of its v2-side `CalcitePPLAppendPipeCommandIT` cases on the analytics-engine route under `tests.analytics.force_routing=true` — the existing capability surface (LogicalUnion + LogicalAggregate over SUM, plus the SchemaUnifier type-conflict path) is sufficient. No code changes in core; this PR just lands a self-contained QA IT so the analytics-engine path can be verified inside core without cross-plugin dependencies. Three tests, mirroring the v2-side surface that exercises the three distinct shapes of `appendpipe`: * `testAppendPipeSort` — duplicate the post-stats stream and re-sort the duplicate inline. Exercises Union over identical schemas. 5 / 6 rows kept by `head 5` — multiset overlap is fine because the outer `sort str0` pins the original branch's order, and the duplicate's `sort -sum_int0_by_str0` pins the inner branch's order. * `testAppendPipeWithMergedColumn` — duplicate the post-stats stream and collapse it via an inner `stats sum(sum) as sum`. Exercises SchemaUnifier merging the str0-bearing original branch with the inner-branch single-row that has only `sum`. The two branches arrive at the coordinator's Union in non-deterministic order, so multiset comparison. * `testAppendPipeWithConflictTypeColumn` — inner pipeline rewrites the same-named column to a different type. Exercises the SchemaUnifier validation path that surfaces "due to incompatible types" before execution. Reuses the existing `calcs` parquet-backed dataset via `DatasetProvisioner` (no new fixtures). `testAppendDifferentIndex` from the v2-side IT is intentionally not ported — it exercises `append` (separate `source=...` sub-search), already covered by `AppendCommandIT`. ## Test plan * `./gradlew :sandbox:qa:analytics-engine-rest:integTest -Dsandbox.enabled=true --tests "*AppendPipeCommandIT"` — 3/3 green * `./gradlew :sandbox:qa:analytics-engine-rest:check -Dsandbox.enabled=true` — green * Verified end-to-end: 56 force_routing transitions, 63 analytics-engine PlannerImpl entries, 0 v2 PPLService.Incoming entries during a sweep including this IT. ## Out of scope * PPL `multisearch` was originally bundled with this PR. Triage showed ~12 of its 15 analytics-route failures are blocked on a Substrait-side issue: DataFusion's substrait consumer rejects the Plan emitted for `LogicalUnion(StageInputScan, StageInputScan)` with `Names list must match exactly to nested schema, but found N uses for M names`. Root cause not diagnosed yet (the registered child-stage schema width vs. `Plan.Root.names` width disagree somewhere between `LocalStageScheduler.buildChildInputs` and DataFusion's `make_renamed_schema`). Out of scope here; tracked for a follow-up PR after deeper investigation. * Aggregation, TIMESTAMP/DATE type-system, eval-predicate scalars, and PPL `span` follow the same scope discipline as #21521 — each is its own work track and not addressed here. Signed-off-by: Kai Huang * [QA] Add TableCommandIT for the analytics-engine REST path `table` is a syntactic alias of `fields` — the v2 AstBuilder dispatches both through `buildProjectCommand` once `plugins.calcite.enabled=true` is visible to the AstBuilder via the UnifiedQueryContext. The mirror fix landed in opensearch-project/sql#5413; this commit closes the gap on the test-ppl-frontend side, where the UnifiedQueryContext is constructed locally rather than coming from the SQL plugin. Two changes: - `UnifiedQueryService` now sets `plugins.calcite.enabled=true` on the context. The unified path is Calcite-based by definition; without this flag the AstBuilder rejects table/regex/rex/convert. - New `TableCommandIT` covering the surfaces specific to the `table` keyword: comma-delimited, space-delimited, suffix wildcard, leading-`-` exclusion, and `fields` ↔ `table` equivalence on identical inputs. Plain projection semantics already covered by FieldsCommandIT are not duplicated. Validates: 5/5 TableCommandIT pass, 3/3 AppendPipeCommandIT still pass. Signed-off-by: Kai Huang --------- Signed-off-by: Kai Huang --- .../ppl/action/UnifiedQueryService.java | 5 + .../analytics/qa/AppendPipeCommandIT.java | 232 ++++++++++++++++++ .../analytics/qa/TableCommandIT.java | 143 +++++++++++ 3 files changed, 380 insertions(+) create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/TableCommandIT.java diff --git a/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/UnifiedQueryService.java b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/UnifiedQueryService.java index aad9e5b4c19bb..a3a0bc277cbba 100644 --- a/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/UnifiedQueryService.java +++ b/sandbox/plugins/test-ppl-frontend/src/main/java/org/opensearch/ppl/action/UnifiedQueryService.java @@ -80,6 +80,11 @@ protected Map getTableMap() { .language(QueryType.PPL) .catalog(DEFAULT_CATALOG, flatSchema) .defaultNamespace(DEFAULT_CATALOG) + // The unified PPL parser reuses the v2 AstBuilder, which gates Calcite-only + // commands (table, regex, rex, convert) on plugins.calcite.enabled. The unified + // path is by definition Calcite-based — flag it on so those commands lower + // through the same Project/Filter RelNodes as their non-aliased counterparts. + .setting("plugins.calcite.enabled", true) .build() ) { diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java new file mode 100644 index 0000000000000..322c6957810f1 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java @@ -0,0 +1,232 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for PPL {@code appendpipe} on the analytics-engine route. + * + *

      Mirrors {@code CalcitePPLAppendPipeCommandIT} from the {@code opensearch-project/sql} + * repository so the analytics-engine path can be verified inside core without cross-plugin + * dependencies. Each test sends a PPL query through {@code POST /_analytics/ppl} (exposed + * by the {@code test-ppl-frontend} plugin), which runs the same {@code UnifiedQueryPlanner} + * → {@code CalciteRelNodeVisitor} → Substrait → DataFusion pipeline as the SQL plugin's + * force-routed analytics path. + * + *

      {@code appendpipe} differs from {@code append} (covered by {@link AppendCommandIT}): + * {@code appendpipe [pipeline]} duplicates the current intermediate result, applies the + * inline {@code [pipeline]} to the duplicate, and appends the duplicate's output to the + * original. {@code append [search]} runs an entirely separate sub-query and unions its + * output. Both lower to a Calcite {@code LogicalUnion} but the upper-stage shape differs + * because {@code appendpipe} reuses the original's row stream as its input rather than + * starting a fresh {@code source=...}. + * + *

      Provisions the {@code calcs} dataset once. {@link AnalyticsRestTestCase#preserveIndicesUponCompletion()} + * keeps it across test methods. + */ +public class AppendPipeCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + // ── duplicate + inline sort, then head ────────────────────────────────────── + + public void testAppendPipeSort() throws IOException { + // Branch: stats sum(int0) by str0 → 3 rows (FURNITURE=1, OFFICE SUPPLIES=18, TECHNOLOGY=49). + // Outer `sort str0` pins the original to alphabetical order. `appendpipe [sort -sum_int0_by_str0]` + // duplicates the 3 rows and re-sorts them descending, then appends. `head 5` keeps the first + // 5 of the 6 total rows: original 3 + first 2 of the descending duplicate. + assertRows( + "source=" + + DATASET.indexName + + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0" + + " | appendpipe [ sort -sum_int0_by_str0 ]" + + " | head 5", + row(1, "FURNITURE"), + row(18, "OFFICE SUPPLIES"), + row(49, "TECHNOLOGY"), + row(49, "TECHNOLOGY"), + row(18, "OFFICE SUPPLIES") + ); + } + + // ── duplicate + inline stats producing a smaller schema (merged column) ───── + + public void testAppendPipeWithMergedColumn() throws IOException { + // Outer stats: sum(int0) by str0 → 3 rows. `appendpipe [stats sum(sum) as sum]` runs an inner + // stats over the duplicate, collapsing it to a single row carrying only the `sum` column. + // Schema unification keeps both the original branch's `str0` and the inner branch's + // `sum` column; the inner row is null-padded for the missing `str0`. The two branches + // arrive at the coordinator's union in non-deterministic order (each is its own data-node + // stage), so compare as a multiset rather than positionally. + assertRowsAnyOrder( + "source=" + + DATASET.indexName + + " | stats sum(int0) as sum by str0 | sort str0" + + " | appendpipe [ stats sum(sum) as sum ]", + row(1, "FURNITURE"), + row(18, "OFFICE SUPPLIES"), + row(49, "TECHNOLOGY"), + row(68, null) + ); + } + + // ── duplicate + inline cast that clashes with the original's column type ─── + + public void testAppendPipeWithConflictTypeColumn() { + // Branch 1 produces `sum` as BIGINT (sum over int0). The inner pipeline of + // `appendpipe [eval sum = cast(sum as double)]` rewrites the same-named column to + // DOUBLE. SchemaUnifier refuses to merge the diverging types and surfaces a + // planner-side validation error before execution. + assertErrorContains( + "source=" + + DATASET.indexName + + " | stats sum(int0) as sum by str0 | sort str0" + + " | appendpipe [ eval sum = cast(sum as double) ]" + + " | head 5", + "due to incompatible types" + ); + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + private static List row(Object... values) { + return Arrays.asList(values); + } + + /** + * Multiset comparison — branch ordering at the coordinator's Union is non-deterministic. + * Used by {@link #testAppendPipeWithMergedColumn} where the original-branch stats output + * (3 rows) and the inner-branch collapsed-sum (1 row) can arrive in either order. + */ + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRowsAnyOrder(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size()); + java.util.List> remaining = new java.util.ArrayList<>(actualRows); + outer: + for (List want : expected) { + for (int i = 0; i < remaining.size(); i++) { + if (rowsEqual(want, remaining.get(i))) { + remaining.remove(i); + continue outer; + } + } + fail("Expected row not found for query: " + ppl + " — missing: " + want + " in actual: " + actualRows); + } + } + + private static boolean rowsEqual(List a, List b) { + if (a.size() != b.size()) return false; + for (int i = 0; i < a.size(); i++) { + Object ax = a.get(i); + Object bx = b.get(i); + if (ax == null || bx == null) { + if (ax != bx) return false; + continue; + } + if (ax instanceof Number && bx instanceof Number) { + if (Double.compare(((Number) ax).doubleValue(), ((Number) bx).doubleValue()) != 0) return false; + continue; + } + if (!ax.equals(bx)) return false; + } + return true; + } + + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRows(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size()); + for (int i = 0; i < expected.length; i++) { + List want = expected[i]; + List got = actualRows.get(i); + assertEquals( + "Column count mismatch at row " + i + " for query: " + ppl, + want.size(), + got.size() + ); + for (int j = 0; j < want.size(); j++) { + assertCellEquals( + "Cell mismatch at row " + i + ", col " + j + " for query: " + ppl, + want.get(j), + got.get(j) + ); + } + } + } + + private void assertErrorContains(String ppl, String expectedSubstring) { + try { + Map response = executePpl(ppl); + fail("Expected query to fail with [" + expectedSubstring + "] but got response: " + response); + } catch (ResponseException e) { + String body; + try { + body = org.opensearch.test.rest.OpenSearchRestTestCase.entityAsMap(e.getResponse()).toString(); + } catch (IOException ioe) { + body = e.getMessage(); + } + assertTrue( + "Expected response body to contain [" + expectedSubstring + "] but was: " + body, + body.contains(expectedSubstring) + ); + } catch (IOException e) { + fail("Unexpected IOException: " + e); + } + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } + + private static void assertCellEquals(String message, Object expected, Object actual) { + if (expected == null || actual == null) { + assertEquals(message, expected, actual); + return; + } + if (expected instanceof Number && actual instanceof Number) { + double e = ((Number) expected).doubleValue(); + double a = ((Number) actual).doubleValue(); + if (Double.compare(e, a) != 0) { + fail(message + ": expected <" + expected + "> but was <" + actual + ">"); + } + return; + } + assertEquals(message, expected, actual); + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/TableCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/TableCommandIT.java new file mode 100644 index 0000000000000..482192d899d92 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/TableCommandIT.java @@ -0,0 +1,143 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for PPL {@code table} on the analytics-engine route. + * + *

      {@code table} is a syntactic alias of {@code fields} — the SQL plugin's + * {@code AstBuilder.visitTableCommand} reuses {@code buildProjectCommand} (the same + * code path {@code fields} dispatches to) once {@code plugins.calcite.enabled=true} is + * propagated through the {@code UnifiedQueryContext} (see + * opensearch-project/sql#5413). + * The added value of {@code table} is a more permissive token shape: it accepts + * space-delimited field lists, leading-{@code -} exclusion forms, and mixes those with + * commas — surfaces {@code fields} doesn't expose. + * + *

      This IT covers the surfaces specific to the {@code table} keyword to lock in that + * the analytics path lowers them to the same Calcite {@code Project} RelNode as the v2 / + * Calcite path does. Plain projection semantics (already covered by {@code FieldsCommandIT}) + * are not duplicated here. + * + *

      Reuses the {@code calcs} parquet-backed dataset. + */ +public class TableCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + public void testTableCommaDelimited() throws IOException { + // Comma-delimited form — same shape as `fields a, b`. Sanity check that the table + // keyword reaches buildProjectCommand without falling back to the v2-only error. + assertColumns( + "source=" + DATASET.indexName + " | table str0, num0 | head 3", + "str0", + "num0" + ); + } + + public void testTableSpaceDelimited() throws IOException { + // Space-delimited form — unique to `table`. Validates the lexer accepts whitespace as + // a separator and the AstBuilder folds the multi-token list into a single Project. + assertColumns( + "source=" + DATASET.indexName + " | table str0 num0 int0 | head 3", + "str0", + "num0", + "int0" + ); + } + + public void testTableSuffixWildcard() throws IOException { + // *0 expands at parse time to all columns ending in '0'. Identical to + // FieldsCommandIT.testFieldsSuffixWildcard on the analytics path; pinned here + // for the `table` lowering specifically. Order is analyzer-dependent, so set-equality. + Map response = executePpl( + "source=" + DATASET.indexName + " | table *0 | head 1" + ); + @SuppressWarnings("unchecked") + List columns = (List) response.get("columns"); + assertNotNull("Response missing 'columns'", columns); + java.util.Set actual = new java.util.HashSet<>(columns); + java.util.Set expected = new java.util.HashSet<>( + java.util.Arrays.asList("num0", "str0", "int0", "bool0", "date0", "time0", "datetime0") + ); + assertEquals("Wildcard *0 column set", expected, actual); + } + + public void testTableMinusExclusion() throws IOException { + // `table - num0, num1, num2, num3, num4` removes those five columns. The leading + // minus form is unique to `table`; `fields` uses `fields - a, b, ...` with a + // comma-separated list (no space-delimiting). Validates analytics path retains + // exclusion semantics. + Map response = executePpl( + "source=" + DATASET.indexName + " | table - num0, num1, num2, num3, num4 | head 1" + ); + @SuppressWarnings("unchecked") + List columns = (List) response.get("columns"); + assertNotNull("Response missing 'columns'", columns); + for (String name : columns) { + assertFalse("Excluded column should not appear: " + name, name.startsWith("num")); + } + } + + public void testFieldsAndTableEquivalence() throws IOException { + // Cross-check that `fields a, b, c` and `table a, b, c` produce identical + // schema + rows. Makes the alias claim explicit at the response level so a + // future divergence (e.g. `table` accidentally adds a Sort or rewires the + // Project) is caught here. + Map fieldsResp = executePpl( + "source=" + DATASET.indexName + " | fields str0, num0, int0 | head 3" + ); + Map tableResp = executePpl( + "source=" + DATASET.indexName + " | table str0, num0, int0 | head 3" + ); + assertEquals("columns from fields vs table", fieldsResp.get("columns"), tableResp.get("columns")); + assertEquals("rows from fields vs table", fieldsResp.get("rows"), tableResp.get("rows")); + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + private void assertColumns(String ppl, String... expectedColumns) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List columns = (List) response.get("columns"); + assertNotNull("Response missing 'columns' for query: " + ppl, columns); + assertEquals("Column count for query: " + ppl, expectedColumns.length, columns.size()); + for (int i = 0; i < expectedColumns.length; i++) { + assertEquals( + "Column at position " + i + " for query: " + ppl, + expectedColumns[i], + columns.get(i) + ); + } + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } +} From 8d64f1de03cd968eac283ebd5b8a5ed3a3f28390 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 6 May 2026 19:02:39 -0700 Subject: [PATCH 066/115] Onboard PPL regex command + regexp_match() to DataFusion analytics-engine route (#21524) Signed-off-by: Jialiang Liang --- .../analytics/spi/ScalarFunction.java | 1 + .../DataFusionAnalyticsBackendPlugin.java | 2 + .../DataFusionFragmentConvertor.java | 7 +- .../opensearch_scalar_functions.yaml | 21 ++ .../analytics/qa/RegexCommandIT.java | 236 ++++++++++++++++++ 5 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/RegexCommandIT.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java index 2bc65b658f3d9..f337824cd7429 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java @@ -50,6 +50,7 @@ public enum ScalarFunction { FUZZY(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), WILDCARD(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), REGEXP(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), + REGEXP_CONTAINS(Category.FULL_TEXT, SqlKind.OTHER_FUNCTION), // ── String ─────────────────────────────────────────────────────── UPPER(Category.STRING, SqlKind.OTHER_FUNCTION), diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index cd7ada3134752..7de9fd82fdf33 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -70,6 +70,7 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP ScalarFunction.IS_NOT_NULL, ScalarFunction.IN, ScalarFunction.LIKE, + ScalarFunction.REGEXP_CONTAINS, ScalarFunction.SARG_PREDICATE, ScalarFunction.PLUS, ScalarFunction.MINUS, @@ -112,6 +113,7 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP ScalarFunction.LESS_THAN_OR_EQUAL, ScalarFunction.IN, ScalarFunction.LIKE, + ScalarFunction.REGEXP_CONTAINS, ScalarFunction.PLUS, ScalarFunction.MINUS, ScalarFunction.TIMES, diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index e7d67bb5879cf..1382c0be89936 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -85,6 +85,10 @@ public class DataFusionFragmentConvertor implements FragmentConvertor { *

    6. {@link DelegatedPredicateFunction} → {@code delegated_predicate} (delegation to a peer backend).
      7. *
    7. {@link SqlLibraryOperators#ILIKE} → {@code ilike} (case-insensitive LIKE; resolved by * DataFusion's substrait consumer to a case-insensitive {@code LikeExpr}).
      8. + *
    8. {@link SqlLibraryOperators#REGEXP_CONTAINS} → {@code regex_match} (boolean regex match; + * resolved by DataFusion's substrait consumer to {@code Operator::RegexMatch}, the same + * binary operator that backs PostgreSQL's {@code ~} regex match). Lowering target for PPL + * {@code regex} command and {@code regexp_match()} function.
      9. * */ private static final List ADDITIONAL_SCALAR_SIGS = List.of( @@ -93,7 +97,8 @@ public class DataFusionFragmentConvertor implements FragmentConvertor { FunctionMappings.s(DelegatedPredicateFunction.FUNCTION, DelegatedPredicateFunction.NAME), FunctionMappings.s(SqlLibraryOperators.DATE_PART, "date_part"), FunctionMappings.s(ConvertTzAdapter.LOCAL_CONVERT_TZ_OP, "convert_tz"), - FunctionMappings.s(UnixTimestampAdapter.LOCAL_TO_UNIXTIME_OP, "to_unixtime") + FunctionMappings.s(UnixTimestampAdapter.LOCAL_TO_UNIXTIME_OP, "to_unixtime"), + FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match") ); private final SimpleExtension.ExtensionCollection extensions; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml index 8f5d778e543b3..dc88820ed6c75 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml @@ -38,3 +38,24 @@ scalar_functions: impls: - args: [{ value: "any1", name: "ts" }] return: any1 + - name: regex_match + description: >- + Regular expression containment match (boolean). Lowering target for PPL's + `regex` command and `regexp_match()` function — both emit Calcite's + `SqlLibraryOperators.REGEXP_CONTAINS`, which DataFusionFragmentConvertor + maps to this extension. datafusion-substrait resolves the extension name + "regex_match" to `datafusion::logical_expr::Operator::RegexMatch`, which + executes against arrow-string's regex kernel and returns BOOLEAN. + impls: + - args: + - value: "varchar" + name: "input" + - value: "varchar" + name: "pattern" + return: boolean + - args: + - value: "string" + name: "input" + - value: "string" + name: "pattern" + return: boolean diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/RegexCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/RegexCommandIT.java new file mode 100644 index 0000000000000..1954d6f9c7520 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/RegexCommandIT.java @@ -0,0 +1,236 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for the PPL {@code regex} command and {@code regexp_match()} + * function on the analytics-engine route. + * + *

      Mirrors {@code CalciteRegexCommandIT} from the {@code opensearch-project/sql} repository so + * that the analytics-engine path can be verified inside core without cross-plugin dependencies on + * the SQL plugin. Each test sends a PPL query through {@code POST /_analytics/ppl} (exposed by the + * {@code test-ppl-frontend} plugin), which runs the same {@code UnifiedQueryPlanner} → + * {@code CalciteRelNodeVisitor} → Substrait → DataFusion pipeline. + * + *

      Both surfaces lower to Calcite {@code SqlLibraryOperators.REGEXP_CONTAINS}: + *

        + *
      • {@code | regex field='pat'} — emits {@code Filter(REGEXP_CONTAINS(field, pat))} + * (negated form: wrapped in {@code NOT})
        • + *
      • {@code eval m = regexp_match(field, pat)} — emits a project-side + * {@code REGEXP_CONTAINS(field, pat)} returning BOOLEAN
        • + *
      + * + *

      Provisions the {@code calcs} dataset (parquet-backed) once per class via + * {@link DatasetProvisioner}; {@link AnalyticsRestTestCase#preserveIndicesUponCompletion()} + * keeps it across test methods. + */ +public class RegexCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + /** + * Lazily provision the calcs dataset on first invocation. Mirrors the + * {@code FillNullCommandIT} pattern — {@code client()} is unavailable at static init. + */ + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + // ── command form: positive match ──────────────────────────────────────────── + + public void testRegexExactMatchOnKeyword() throws IOException { + // str0 has 2 rows with "FURNITURE", 6 with "OFFICE SUPPLIES", 9 with "TECHNOLOGY". + assertRowCount("source=" + DATASET.indexName + " | regex str0='FURNITURE' | fields str0", 2); + } + + public void testRegexContainsSubstring() throws IOException { + // REGEXP_CONTAINS — pattern matches anywhere in the field, not anchored. + assertRowCount("source=" + DATASET.indexName + " | regex str0='OFFICE' | fields str0", 6); + } + + public void testRegexAnchoredStart() throws IOException { + // ^TECH anchors to start: only TECHNOLOGY (×9), not strings containing TECH elsewhere. + assertRowCount("source=" + DATASET.indexName + " | regex str0='^TECH' | fields str0", 9); + } + + public void testRegexAnchoredEnd() throws IOException { + // OGY$ anchors to end: TECHNOLOGY (×9). + assertRowCount("source=" + DATASET.indexName + " | regex str0='OGY$' | fields str0", 9); + } + + public void testRegexWildcardPattern() throws IOException { + // BINDER appears in BINDER ACCESSORIES + BINDER CLIPS (2 rows). + assertRowCount("source=" + DATASET.indexName + " | regex str1='BINDER' | fields str1", 2); + } + + public void testRegexCharacterClass() throws IOException { + // [BC]INDING matches BINDING (BINDING MACHINES, BINDING SUPPLIES) but not BUSINESS. + assertRowCount("source=" + DATASET.indexName + " | regex str1='BINDING' | fields str1", 2); + } + + // ── command form: negated match ───────────────────────────────────────────── + + public void testRegexNegated() throws IOException { + // 17 total rows, 2 are FURNITURE → 15 pass when negated. + assertRowCount("source=" + DATASET.indexName + " | regex str0!='FURNITURE' | fields str0", 15); + } + + public void testRegexNegatedAnchored() throws IOException { + // Negate ^OFFICE: 17 - 6 = 11 rows. + assertRowCount("source=" + DATASET.indexName + " | regex str0!='^OFFICE' | fields str0", 11); + } + + // ── command form: full row content check ──────────────────────────────────── + + public void testRegexExpectedRowsForFurniture() throws IOException { + // Verify the actual matched values, not just count, for the FURNITURE selection. + assertRows( + "source=" + DATASET.indexName + " | regex str0='FURNITURE' | fields str0, str1 | sort str1", + row("FURNITURE", "CLAMP ON LAMPS"), + row("FURNITURE", "CLOCKS") + ); + } + + // ── function form: regexp_match in eval projection (BOOLEAN result) ──────── + + public void testRegexpMatchInEvalAllTrue() throws IOException { + // regexp_match returns BOOLEAN. Pattern that matches every str0 value. + assertRowCount( + "source=" + DATASET.indexName + + " | eval m = regexp_match(str0, '.*') | where m=true | fields str0", + 17 + ); + } + + public void testRegexpMatchInEvalSelective() throws IOException { + // regexp_match selects rows whose str0 contains 'TECH' — TECHNOLOGY ×9. + assertRowCount( + "source=" + DATASET.indexName + + " | eval m = regexp_match(str0, 'TECH') | where m=true | fields str0", + 9 + ); + } + + public void testRegexpMatchProducesBooleanColumn() throws IOException { + // Project the boolean result alongside the source field — verifies REGEXP_CONTAINS + // round-trips through Substrait → DataFusion as a project-side BOOLEAN expression. + assertRows( + "source=" + DATASET.indexName + + " | regex str0='FURNITURE' | eval m = regexp_match(str1, 'CLAMP') | fields str1, m | sort str1", + row("CLAMP ON LAMPS", true), + row("CLOCKS", false) + ); + } + + // ── error path: regex on non-string field ────────────────────────────────── + + public void testRegexOnNumericFieldErrors() { + // CalciteRelNodeVisitor.visitRegex enforces SqlTypeFamily.CHARACTER on the field — + // a numeric field must fail the preflight type check, not reach DataFusion. + assertErrorContains( + "source=" + DATASET.indexName + " | regex num0='1.*'", + "Regex command requires field of string type" + ); + } + + // ── helpers ──────────────────────────────────────────────────────────────── + + private static List row(Object... values) { + return Arrays.asList(values); + } + + /** + * Send a PPL query and assert the response's {@code rows} count matches {@code expectedCount}. + * Use this when only the cardinality matters (e.g. matching against a regex that returns + * many rows whose ordering would be brittle to assert exhaustively). + */ + private void assertRowCount(String ppl, int expectedCount) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' field for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expectedCount, actualRows.size()); + } + + /** + * Send a PPL query and assert each returned row equals the expected positional row. + */ + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRows(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' field for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size()); + for (int i = 0; i < expected.length; i++) { + List want = expected[i]; + List got = actualRows.get(i); + assertEquals( + "Column count mismatch at row " + i + " for query: " + ppl, + want.size(), + got.size() + ); + for (int j = 0; j < want.size(); j++) { + assertEquals( + "Cell mismatch at row " + i + ", col " + j + " for query: " + ppl, + want.get(j), + got.get(j) + ); + } + } + } + + /** + * Send a PPL query expecting the planner to reject it; assert the error body contains + * {@code expectedSubstring}. + */ + private void assertErrorContains(String ppl, String expectedSubstring) { + try { + Map response = executePpl(ppl); + fail("Expected query to fail with [" + expectedSubstring + "] but got response: " + response); + } catch (ResponseException e) { + String body; + try { + body = org.opensearch.test.rest.OpenSearchRestTestCase.entityAsMap(e.getResponse()).toString(); + } catch (IOException ioe) { + body = e.getMessage(); + } + assertTrue( + "Expected response body to contain [" + expectedSubstring + "] but was: " + body, + body.contains(expectedSubstring) + ); + } catch (IOException e) { + fail("Unexpected IOException: " + e); + } + } + + /** Send {@code POST /_analytics/ppl} and return the parsed JSON body. */ + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } +} From bf7bfedc00541454aacd22a4529bdfa765696c1e Mon Sep 17 00:00:00 2001 From: MayankHarsh03 <156461043+MayankHarsh03@users.noreply.github.com> Date: Thu, 7 May 2026 10:31:48 +0530 Subject: [PATCH 067/115] Add module wiring and integration tests for WritableWarm tiered storage (#21427) Module wiring: - Node.java: registered TieredDirectoryFactory, initialized TieredStoragePrefetchSettings, bound TierActionMetrics - IndicesService.java: added TieredStorageSearchSlowLog and StoredFieldsPrefetch as search listeners - ThreadPool.java: added remote_download thread pool executor Integration tests: - WarmIndexBasicIT: 4 tests covering warm index create/search/merge, block file verification, close/reopen, primary+replica Depends on #21332 (slow logs PR) Signed-off-by: Mayank Harsh --- .../opensearch/storage/WarmIndexBasicIT.java | 327 ++++++++++++++++++ .../opensearch/indices/IndicesService.java | 16 + .../main/java/org/opensearch/node/Node.java | 24 ++ .../storage/directory/TieredDirectory.java | 8 +- .../directory/TieredDirectoryFactory.java | 4 + .../storage/utils/DirectoryUtils.java | 41 ++- .../org/opensearch/threadpool/ThreadPool.java | 12 + .../java/org/opensearch/node/NodeTests.java | 20 ++ .../storage/utils/DirectoryUtilsTests.java | 100 ++++++ .../threadpool/ScalingThreadPoolTests.java | 1 + 10 files changed, 549 insertions(+), 4 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/storage/WarmIndexBasicIT.java create mode 100644 server/src/test/java/org/opensearch/storage/utils/DirectoryUtilsTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/storage/WarmIndexBasicIT.java b/server/src/internalClusterTest/java/org/opensearch/storage/WarmIndexBasicIT.java new file mode 100644 index 0000000000000..295d5ee162526 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/storage/WarmIndexBasicIT.java @@ -0,0 +1,327 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FilterDirectory; +import org.opensearch.action.admin.indices.close.CloseIndexRequest; +import org.opensearch.action.admin.indices.close.CloseIndexResponse; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.admin.indices.get.GetIndexRequest; +import org.opensearch.action.admin.indices.get.GetIndexResponse; +import org.opensearch.action.admin.indices.open.OpenIndexRequest; +import org.opensearch.action.admin.indices.open.OpenIndexResponse; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.index.IndexModule; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.store.CompositeDirectory; +import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter; +import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.index.store.remote.utils.FileTypeUtils; +import org.opensearch.indices.IndicesService; +import org.opensearch.node.Node; +import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; +import org.opensearch.storage.directory.TieredDirectory; +import org.opensearch.test.InternalTestCluster; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; + +/** + * Integration tests for basic warm index operations. + * + * @opensearch.experimental + */ +@ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false) +public class WarmIndexBasicIT extends RemoteStoreBaseIntegTestCase { + + protected static final String INDEX_NAME = "test-idx-1"; + protected static final int NUM_DOCS_IN_BULK = 1000; + + @Override + protected boolean addMockIndexStorePlugin() { + return false; + } + + @Override + protected boolean ignoreExternalCluster() { + return true; + } + + @Override + protected Settings featureFlagSettings() { + Settings.Builder featureSettings = Settings.builder(); + featureSettings.put(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG, true); + return featureSettings.build(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + ByteSizeValue cacheSize = new ByteSizeValue(1, ByteSizeUnit.GB); + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(Node.NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), cacheSize.toString()) + .build(); + } + + public void testWritableWarm() throws Exception { + InternalTestCluster internalTestCluster = internalCluster(); + internalTestCluster.startClusterManagerOnlyNode(); + internalTestCluster.startDataAndWarmNodes(1); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), true) + .put(IndexModule.INDEX_COMPOSITE_STORE_TYPE_SETTING.getKey(), "tiered-storage") + .build(); + // create a tiered-storage warm index with 1p0r configuration + assertAcked(client().admin().indices().prepareCreate(INDEX_NAME).setSettings(settings).get()); + + // Verify from the cluster settings that the warm setting is true + GetIndexResponse getIndexResponse = client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(INDEX_NAME).includeDefaults(true)) + .get(); + Settings indexSettings = getIndexResponse.settings().get(INDEX_NAME); + assertTrue(indexSettings.getAsBoolean(IndexModule.IS_WARM_INDEX_SETTING.getKey(), false)); + + FileCache fileCache = internalTestCluster.getDataNodeInstance(Node.class).fileCache(); + IndexShard shard = internalTestCluster.getDataNodeInstance(IndicesService.class) + .indexService(resolveIndex(INDEX_NAME)) + .getShardOrNull(0); + Directory directory = unwrapToCompositeDirectory(shard.store().directory()); + + // Ingesting some docs + indexBulk(INDEX_NAME, NUM_DOCS_IN_BULK); + flushAndRefresh(INDEX_NAME); + + // ensuring cluster is green after performing force-merge + ensureGreen(); + + SearchResponse searchResponse = client().prepareSearch(INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + // Asserting that search returns same number of docs as ingested + assertHitCount(searchResponse, NUM_DOCS_IN_BULK); + + // Ingesting docs again before force merge + indexBulk(INDEX_NAME, NUM_DOCS_IN_BULK); + flushAndRefresh(INDEX_NAME); + + // Force merging the index + Set filesBeforeMerge = new HashSet<>(Arrays.asList(directory.listAll())); + client().admin().indices().prepareForceMerge(INDEX_NAME).setMaxNumSegments(1).get(); + flushAndRefresh(INDEX_NAME); + Set filesAfterMerge = new HashSet<>(Arrays.asList(directory.listAll())); + + Set filesFromPreviousGenStillPresent = filesBeforeMerge.stream() + .filter(filesAfterMerge::contains) + .filter(file -> !FileTypeUtils.isLockFile(file)) + .filter(file -> !FileTypeUtils.isSegmentsFile(file)) + .collect(Collectors.toUnmodifiableSet()); + + // Asserting that after merge all the files from previous gen are no more part of the directory + assertTrue(filesFromPreviousGenStillPresent.isEmpty()); + + // Asserting that files from previous gen are not present in File Cache as well + CompositeDirectory compositeDir = (CompositeDirectory) directory; + filesBeforeMerge.stream() + .filter(file -> !FileTypeUtils.isLockFile(file)) + .filter(file -> !FileTypeUtils.isSegmentsFile(file)) + .forEach(file -> assertNull(fileCache.get(compositeDir.getFilePath(file)))); + + // Deleting the index to avoid any file leaks + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); + } + + public void testLocalDirectoryFilesAfterRefresh() throws Exception { + InternalTestCluster internalTestCluster = internalCluster(); + internalTestCluster.startClusterManagerOnlyNode(); + internalTestCluster.startDataAndWarmNodes(1); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), true) + .put(IndexModule.INDEX_COMPOSITE_STORE_TYPE_SETTING.getKey(), "tiered-storage") + .build(); + assertAcked(client().admin().indices().prepareCreate(INDEX_NAME).setSettings(settings).get()); + + IndexShard shard = internalTestCluster.getDataNodeInstance(IndicesService.class) + .indexService(resolveIndex(INDEX_NAME)) + .getShardOrNull(0); + + TieredDirectory tieredDirectory = unwrapToTieredDirectory(shard.store().directory()); + + indexBulk(INDEX_NAME, NUM_DOCS_IN_BULK); + refresh(INDEX_NAME); + + waitUntil(() -> { + try { + return Arrays.stream(tieredDirectory.listLocalFiles()).anyMatch(file -> file.contains("block")); + } catch (IOException ignored) { + return false; + } + }, 30, TimeUnit.SECONDS); + assertTrue( + Arrays.stream(tieredDirectory.listLocalFiles()) + .filter(file -> !file.contains("block")) + .filter(file -> !file.contains("write.lock")) + .findAny() + .isEmpty() + ); + + // Deleting the index to avoid any file leaks + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); + } + + /** + * Unwraps the directory chain (walking through FilterDirectory wrappers including + * BucketedCompositeDirectory) to find the underlying CompositeDirectory. + */ + private static Directory unwrapToCompositeDirectory(Directory directory) { + Directory current = directory; + while (current instanceof FilterDirectory) { + if (current instanceof CompositeDirectory) { + return current; + } + current = ((FilterDirectory) current).getDelegate(); + } + if (current instanceof CompositeDirectory) { + return current; + } + throw new IllegalArgumentException("Expected CompositeDirectory but got: " + directory.getClass().getName()); + } + + /** + * Unwraps the directory chain (walking through FilterDirectory wrappers including + * BucketedCompositeDirectory) to find the underlying TieredDirectory. + */ + private static TieredDirectory unwrapToTieredDirectory(Directory directory) { + Directory current = directory; + while (current instanceof FilterDirectory) { + if (current instanceof TieredDirectory) { + return (TieredDirectory) current; + } + current = ((FilterDirectory) current).getDelegate(); + } + if (current instanceof TieredDirectory) { + return (TieredDirectory) current; + } + throw new IllegalArgumentException("Expected TieredDirectory but got: " + directory.getClass().getName()); + } + + protected long getDocCount(String indexName) { + refresh(indexName); + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).setSize(0).get(); + return response.getHits().getTotalHits().value(); + } + + public void testCloseIndex() throws ExecutionException, InterruptedException { + InternalTestCluster internalTestCluster = internalCluster(); + internalTestCluster.startClusterManagerOnlyNode(); + internalTestCluster.startDataAndWarmNodes(2); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), true) + .put(IndexModule.INDEX_COMPOSITE_STORE_TYPE_SETTING.getKey(), "tiered-storage") + .build(); + // create a warm index with 1p0r configuration + assertAcked(client().admin().indices().prepareCreate(INDEX_NAME).setSettings(settings).get()); + + // Verify from the cluster settings if the warm index setting is true + GetIndexResponse getIndexResponse = client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(INDEX_NAME).includeDefaults(true)) + .get(); + Settings indexSettings = getIndexResponse.settings().get(INDEX_NAME); + assertTrue(indexSettings.getAsBoolean(IndexModule.IS_WARM_INDEX_SETTING.getKey(), false)); + // Ingesting some docs + indexBulk(INDEX_NAME, NUM_DOCS_IN_BULK); + flushAndRefresh(INDEX_NAME); + + // ensuring cluster is green after performing force-merge + ensureGreen(); + + long docCount = getDocCount(INDEX_NAME); + CloseIndexResponse closeIndexResponse = client().admin().indices().close(new CloseIndexRequest(INDEX_NAME)).get(); + assertTrue(closeIndexResponse.isShardsAcknowledged()); + + OpenIndexResponse openIndexResponse = client().admin().indices().open(new OpenIndexRequest(INDEX_NAME)).get(); + assertTrue(openIndexResponse.isShardsAcknowledged()); + + long docCountUpdated = getDocCount(INDEX_NAME); + assertEquals(docCountUpdated, docCount); + } + + public void testWritableWarmPrimaryReplicaBoth() throws Exception { + InternalTestCluster internalTestCluster = internalCluster(); + internalTestCluster.startClusterManagerOnlyNode(); + internalTestCluster.startDataAndWarmNodes(2); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), true) + .put(IndexModule.INDEX_COMPOSITE_STORE_TYPE_SETTING.getKey(), "tiered-storage") + .build(); + // create a tiered-storage warm index with 1p1r configuration + assertAcked(client().admin().indices().prepareCreate(INDEX_NAME).setSettings(settings).get()); + + // Verify from the cluster settings if the warm index setting is true + GetIndexResponse getIndexResponse = client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(INDEX_NAME).includeDefaults(true)) + .get(); + Settings indexSettings = getIndexResponse.settings().get(INDEX_NAME); + assertTrue(indexSettings.getAsBoolean(IndexModule.IS_WARM_INDEX_SETTING.getKey(), false)); + + // Ingesting some docs + indexBulk(INDEX_NAME, NUM_DOCS_IN_BULK); + flushAndRefresh(INDEX_NAME); + + // ensuring cluster is green after performing force-merge + ensureGreen(); + + SearchResponse searchResponse = client().prepareSearch(INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + // Asserting that search returns same number of docs as ingested + assertHitCount(searchResponse, NUM_DOCS_IN_BULK); + + // Ingesting docs again before force merge + indexBulk(INDEX_NAME, NUM_DOCS_IN_BULK); + flushAndRefresh(INDEX_NAME); + + // Force merging the index + client().admin().indices().prepareForceMerge(INDEX_NAME).setMaxNumSegments(1).get(); + flushAndRefresh(INDEX_NAME); + + ensureGreen(); + searchResponse = client().prepareSearch(INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + // verify again after force merge search response return same no of docs as ingested + assertHitCount(searchResponse, 2 * NUM_DOCS_IN_BULK); + + // Deleting the index to avoid any file leaks + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); + } +} diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 9bfb4d2e295d5..be3241e28174f 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -78,6 +78,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.AbstractRefCounted; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.OpenSearchExecutors; @@ -181,6 +182,9 @@ import org.opensearch.search.internal.ShardSearchRequest; import org.opensearch.search.query.QueryPhase; import org.opensearch.search.query.QuerySearchResult; +import org.opensearch.storage.prefetch.StoredFieldsPrefetch; +import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings; +import org.opensearch.storage.slowlogs.TieredStorageSearchSlowLog; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.client.Client; @@ -385,6 +389,7 @@ public class IndicesService extends AbstractLifecycleComponent private final BigArrays bigArrays; private final ScriptService scriptService; private final ClusterService clusterService; + private final Supplier tieredStoragePrefetchSettingsSupplier; private final Client client; private volatile Map indices = emptyMap(); private final Map> pendingDeletes = new HashMap<>(); @@ -508,6 +513,12 @@ public IndicesService( this.bigArrays = bigArrays; this.scriptService = scriptService; this.clusterService = clusterService; + if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG)) { + final TieredStoragePrefetchSettings prefetchSettings = new TieredStoragePrefetchSettings(clusterService.getClusterSettings()); + this.tieredStoragePrefetchSettingsSupplier = () -> prefetchSettings; + } else { + this.tieredStoragePrefetchSettingsSupplier = () -> null; + } this.client = client; this.idFieldDataEnabled = INDICES_ID_FIELD_DATA_ENABLED_SETTING.get(clusterService.getSettings()); clusterService.getClusterSettings().addSettingsUpdateConsumer(INDICES_ID_FIELD_DATA_ENABLED_SETTING, this::setIdFieldDataEnabled); @@ -1124,6 +1135,11 @@ private synchronized IndexService createIndexService( indexModule.addIndexOperationListener(operationListener); } pluginsService.onIndexModule(indexModule); + // Add tiered storage search listeners + if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG)) { + indexModule.addSearchOperationListener(new TieredStorageSearchSlowLog(idxSettings)); + indexModule.addSearchOperationListener(new StoredFieldsPrefetch(tieredStoragePrefetchSettingsSupplier)); + } for (IndexEventListener listener : builtInListeners) { indexModule.addIndexEventListener(listener); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index bdab090610b6f..e2840b0ba3694 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -270,6 +270,10 @@ import org.opensearch.snapshots.SnapshotShardsService; import org.opensearch.snapshots.SnapshotsInfoService; import org.opensearch.snapshots.SnapshotsService; +import org.opensearch.storage.common.tiering.TieringUtils; +import org.opensearch.storage.directory.TieredDirectoryFactory; +import org.opensearch.storage.metrics.TierActionMetrics; +import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings; import org.opensearch.storage.tiering.HotToWarmTieringService; import org.opensearch.storage.tiering.WarmToHotTieringService; import org.opensearch.task.commons.clients.TaskManagerClient; @@ -900,6 +904,17 @@ protected Node(final Environment initialEnvironment, Collection clas pluginsService.filterPlugins(IngestionConsumerPlugin.class) .forEach(plugin -> ingestionConsumerFactories.putAll(plugin.getIngestionConsumerFactories())); + // Initialize tiered storage prefetch settings + final TieredStoragePrefetchSettings tieredStoragePrefetchSettings; + final Supplier tieredStoragePrefetchSettingsSupplier; + if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG)) { + tieredStoragePrefetchSettings = new TieredStoragePrefetchSettings(clusterService.getClusterSettings()); + tieredStoragePrefetchSettingsSupplier = () -> tieredStoragePrefetchSettings; + } else { + tieredStoragePrefetchSettings = null; + tieredStoragePrefetchSettingsSupplier = () -> null; + } + final Map builtInDirectoryFactories = IndexModule.createBuiltInDirectoryFactories( repositoriesServiceReference::get, threadPool, @@ -936,6 +951,14 @@ protected Node(final Environment initialEnvironment, Collection clas compositeDirectoryFactories.put(k, v); }); compositeDirectoryFactories.put("default", new DefaultCompositeDirectoryFactory()); + + // Register tiered storage directory factories + if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG)) { + compositeDirectoryFactories.put( + TieringUtils.TIERED_COMPOSITE_INDEX_TYPE, + new TieredDirectoryFactory(tieredStoragePrefetchSettingsSupplier) + ); + } final Map dataFormatAwareStoreDirectoryFactories = new HashMap<>(); @@ -1730,6 +1753,7 @@ protected Node(final Environment initialEnvironment, Collection clas if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG)) { b.bind(HotToWarmTieringService.class).asEagerSingleton(); b.bind(WarmToHotTieringService.class).asEagerSingleton(); + b.bind(TierActionMetrics.class).toInstance(new TierActionMetrics(metricsRegistry)); } }); injector = modules.createInjector(); diff --git a/server/src/main/java/org/opensearch/storage/directory/TieredDirectory.java b/server/src/main/java/org/opensearch/storage/directory/TieredDirectory.java index 41d29d4031fe1..35fc2c6e60ebe 100644 --- a/server/src/main/java/org/opensearch/storage/directory/TieredDirectory.java +++ b/server/src/main/java/org/opensearch/storage/directory/TieredDirectory.java @@ -12,9 +12,9 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.store.CompositeDirectory; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.remote.filecache.CachedIndexInput; @@ -24,6 +24,7 @@ import org.opensearch.storage.indexinput.SwitchableIndexInput; import org.opensearch.storage.indexinput.SwitchableIndexInputWrapper; import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings; +import org.opensearch.storage.utils.DirectoryUtils; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -40,7 +41,10 @@ /** * Extension of Composite directory to support writable warm and other related features + * + * @opensearch.experimental */ +@ExperimentalApi public class TieredDirectory extends CompositeDirectory { private static final Logger logger = LogManager.getLogger(TieredDirectory.class); @@ -229,7 +233,7 @@ protected void cacheFile(String fileName, boolean cacheFromRemote) throws IOExce new CachedSwitchableIndexInput( fileCache, fileName, - (FSDirectory) localDirectory, + DirectoryUtils.unwrapFSDirectory(localDirectory), remoteDirectory, transferManager, cacheFromRemote, diff --git a/server/src/main/java/org/opensearch/storage/directory/TieredDirectoryFactory.java b/server/src/main/java/org/opensearch/storage/directory/TieredDirectoryFactory.java index a4f10c910ca7d..e49b9009a60bb 100644 --- a/server/src/main/java/org/opensearch/storage/directory/TieredDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/storage/directory/TieredDirectoryFactory.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.store.Directory; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.remote.filecache.FileCache; @@ -22,7 +23,10 @@ /** * Factory for creating {@link TieredDirectory} instances that combine local and remote storage. + * + * @opensearch.experimental */ +@ExperimentalApi public class TieredDirectoryFactory implements IndexStorePlugin.CompositeDirectoryFactory { private static final Logger logger = LogManager.getLogger(TieredDirectoryFactory.class); diff --git a/server/src/main/java/org/opensearch/storage/utils/DirectoryUtils.java b/server/src/main/java/org/opensearch/storage/utils/DirectoryUtils.java index 47d6f012d580a..e36806011d76a 100644 --- a/server/src/main/java/org/opensearch/storage/utils/DirectoryUtils.java +++ b/server/src/main/java/org/opensearch/storage/utils/DirectoryUtils.java @@ -10,21 +10,58 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FilterDirectory; +import org.opensearch.common.annotation.ExperimentalApi; import java.nio.file.Path; /** * Utility methods for directory path resolution in tiered storage. + * + * @opensearch.experimental */ +@ExperimentalApi public class DirectoryUtils { + /** Suffix for switchable file paths. */ public static final String SWITCHABLE_PREFIX = "_switchable"; + /** + * Resolves the file path for a given file name in the directory. + * @param localDirectory the directory + * @param fileName the file name + * @return the resolved path + */ public static Path getFilePath(Directory localDirectory, String fileName) { - return ((FSDirectory) localDirectory).getDirectory().resolve(fileName); + return unwrapFSDirectory(localDirectory).getDirectory().resolve(fileName); } + /** + * Resolves the switchable file path for a given file name in the directory. + * @param localDirectory the directory + * @param fileName the file name + * @return the resolved switchable path + */ public static Path getFilePathSwitchable(Directory localDirectory, String fileName) { - return ((FSDirectory) localDirectory).getDirectory().resolve(fileName + SWITCHABLE_PREFIX); + return unwrapFSDirectory(localDirectory).getDirectory().resolve(fileName + SWITCHABLE_PREFIX); + } + + /** + * Unwraps a directory chain to find the underlying FSDirectory. + * Handles cases where the directory is wrapped in FilterDirectory layers + * such as BucketedCompositeDirectory. + * @param directory the directory to unwrap + * @return the underlying FSDirectory + * @throws IllegalArgumentException if no FSDirectory is found in the chain + */ + public static FSDirectory unwrapFSDirectory(Directory directory) { + Directory current = directory; + while (current instanceof FilterDirectory) { + current = ((FilterDirectory) current).getDelegate(); + } + if (current instanceof FSDirectory) { + return (FSDirectory) current; + } + throw new IllegalArgumentException("Expected FSDirectory but got: " + directory.getClass().getName()); } } diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 5995efbb44583..5bb3184872367 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -125,6 +125,8 @@ public static class Names { public static final String REMOTE_PURGE = "remote_purge"; public static final String REMOTE_REFRESH_RETRY = "remote_refresh_retry"; public static final String REMOTE_RECOVERY = "remote_recovery"; + /** Thread pool name for remote downloads in tiered storage. */ + public static final String REMOTE_DOWNLOAD = "remote_download"; public static final String REMOTE_STATE_READ = "remote_state_read"; public static final String INDEX_SEARCHER = "index_searcher"; public static final String REMOTE_STATE_CHECKSUM = "remote_state_checksum"; @@ -206,6 +208,7 @@ public static ThreadPoolType fromType(String type) { map.put(Names.REMOTE_PURGE, ThreadPoolType.SCALING); map.put(Names.REMOTE_REFRESH_RETRY, ThreadPoolType.SCALING); map.put(Names.REMOTE_RECOVERY, ThreadPoolType.SCALING); + map.put(Names.REMOTE_DOWNLOAD, ThreadPoolType.SCALING); map.put(Names.REMOTE_STATE_READ, ThreadPoolType.FIXED); map.put(Names.INDEX_SEARCHER, ThreadPoolType.RESIZABLE); map.put(Names.REMOTE_STATE_CHECKSUM, ThreadPoolType.FIXED); @@ -328,6 +331,15 @@ public ThreadPool( TimeValue.timeValueMinutes(5) ) ); + builders.put( + Names.REMOTE_DOWNLOAD, + new ScalingExecutorBuilder( + Names.REMOTE_DOWNLOAD, + 1, + twiceAllocatedProcessors(allocatedProcessors), + TimeValue.timeValueMinutes(5) + ) + ); builders.put( Names.REMOTE_STATE_READ, new FixedExecutorBuilder(settings, Names.REMOTE_STATE_READ, boundedBy(4 * allocatedProcessors, 4, 32), 120000) diff --git a/server/src/test/java/org/opensearch/node/NodeTests.java b/server/src/test/java/org/opensearch/node/NodeTests.java index 999586f4f8639..264ab53e3846c 100644 --- a/server/src/test/java/org/opensearch/node/NodeTests.java +++ b/server/src/test/java/org/opensearch/node/NodeTests.java @@ -42,6 +42,7 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.transport.BoundTransportAddress; @@ -64,6 +65,7 @@ import org.opensearch.plugins.TelemetryPlugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.script.ScriptService; +import org.opensearch.storage.metrics.TierActionMetrics; import org.opensearch.telemetry.Telemetry; import org.opensearch.telemetry.TelemetrySettings; import org.opensearch.telemetry.metrics.MetricsRegistry; @@ -422,6 +424,24 @@ public void testCreateWithFileCache() throws Exception { } } + public void testTieredStorageWiringWithFeatureFlag() throws Exception { + Settings warmRoleSettings = addRoles( + baseSettings().put(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG, true) + .put(Node.NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), "1gb") + .build(), + Set.of(DiscoveryNodeRole.WARM_ROLE) + ); + List> plugins = basePlugins(); + try (MockNode mockNode = new MockNode(warmRoleSettings, plugins)) { + assertNotNull(mockNode); + // Verify TierActionMetrics was bound in Guice + assertNotNull(mockNode.injector().getInstance(TierActionMetrics.class)); + // Verify remote_download thread pool exists + ThreadPool threadPool = mockNode.injector().getInstance(ThreadPool.class); + assertNotNull(threadPool.executor(ThreadPool.Names.REMOTE_DOWNLOAD)); + } + } + public void testTelemetryAwarePlugins() throws IOException { Settings.Builder settings = baseSettings(); List> plugins = basePlugins(); diff --git a/server/src/test/java/org/opensearch/storage/utils/DirectoryUtilsTests.java b/server/src/test/java/org/opensearch/storage/utils/DirectoryUtilsTests.java new file mode 100644 index 0000000000000..8364b2994c065 --- /dev/null +++ b/server/src/test/java/org/opensearch/storage/utils/DirectoryUtilsTests.java @@ -0,0 +1,100 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.utils; + +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FilterDirectory; +import org.opensearch.test.OpenSearchTestCase; + +import java.nio.file.Path; + +/** + * Tests for {@link DirectoryUtils}. + * + * @opensearch.experimental + */ +public class DirectoryUtilsTests extends OpenSearchTestCase { + + private Path tempDir; + private FSDirectory fsDirectory; + + @Override + public void setUp() throws Exception { + super.setUp(); + tempDir = createTempDir(); + fsDirectory = FSDirectory.open(tempDir); + } + + @Override + public void tearDown() throws Exception { + fsDirectory.close(); + super.tearDown(); + } + + public void testUnwrapFSDirectoryDirect() { + FSDirectory result = DirectoryUtils.unwrapFSDirectory(fsDirectory); + assertSame(fsDirectory, result); + } + + public void testUnwrapFSDirectorySingleWrapper() { + FilterDirectory wrapped = new FilterDirectory(fsDirectory) { + }; + FSDirectory result = DirectoryUtils.unwrapFSDirectory(wrapped); + assertSame(fsDirectory, result); + } + + public void testUnwrapFSDirectoryMultipleWrappers() { + FilterDirectory inner = new FilterDirectory(fsDirectory) { + }; + FilterDirectory outer = new FilterDirectory(inner) { + }; + FSDirectory result = DirectoryUtils.unwrapFSDirectory(outer); + assertSame(fsDirectory, result); + } + + public void testUnwrapFSDirectoryThrowsWhenNoFSDirectory() { + Directory nonFsDir = new ByteBuffersDirectory(); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> DirectoryUtils.unwrapFSDirectory(nonFsDir)); + assertTrue(ex.getMessage().contains("Expected FSDirectory but got")); + } + + public void testUnwrapFSDirectoryThrowsWhenWrappedNonFSDirectory() { + Directory nonFsDir = new ByteBuffersDirectory(); + FilterDirectory wrapped = new FilterDirectory(nonFsDir) { + }; + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> DirectoryUtils.unwrapFSDirectory(wrapped)); + assertTrue(ex.getMessage().contains("Expected FSDirectory but got")); + } + + public void testGetFilePath() { + Path result = DirectoryUtils.getFilePath(fsDirectory, "test_file.si"); + assertEquals(tempDir.resolve("test_file.si"), result); + } + + public void testGetFilePathWithWrappedDirectory() { + FilterDirectory wrapped = new FilterDirectory(fsDirectory) { + }; + Path result = DirectoryUtils.getFilePath(wrapped, "test_file.si"); + assertEquals(tempDir.resolve("test_file.si"), result); + } + + public void testGetFilePathSwitchable() { + Path result = DirectoryUtils.getFilePathSwitchable(fsDirectory, "test_file.si"); + assertEquals(tempDir.resolve("test_file.si" + DirectoryUtils.SWITCHABLE_PREFIX), result); + } + + public void testGetFilePathSwitchableWithWrappedDirectory() { + FilterDirectory wrapped = new FilterDirectory(fsDirectory) { + }; + Path result = DirectoryUtils.getFilePathSwitchable(wrapped, "test_file.si"); + assertEquals(tempDir.resolve("test_file.si" + DirectoryUtils.SWITCHABLE_PREFIX), result); + } +} diff --git a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java index 9aa808ec66064..df4b5143eeb6d 100644 --- a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java +++ b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java @@ -156,6 +156,7 @@ private int expectedSize(final String threadPoolName, final int numberOfProcesso sizes.put(ThreadPool.Names.REMOTE_PURGE, ThreadPool::halfAllocatedProcessors); sizes.put(ThreadPool.Names.REMOTE_REFRESH_RETRY, ThreadPool::halfAllocatedProcessors); sizes.put(ThreadPool.Names.REMOTE_RECOVERY, ThreadPool::twiceAllocatedProcessors); + sizes.put(ThreadPool.Names.REMOTE_DOWNLOAD, ThreadPool::twiceAllocatedProcessors); sizes.put(ThreadPool.Names.MERGE, n -> n); return sizes.get(threadPoolName).apply(numberOfProcessors); } From 430afad28e4aa5c5073855746fe3e199349668b9 Mon Sep 17 00:00:00 2001 From: rayshrey <121871912+rayshrey@users.noreply.github.com> Date: Thu, 7 May 2026 12:32:37 +0530 Subject: [PATCH 068/115] [Pluggable Engine] Fixes for end to end ingestion in DataFormatAwareEngine (#21460) * Fixes in indexing path Signed-off-by: Mohit Godwani * Test fixes Signed-off-by: rayshrey --------- Signed-off-by: Mohit Godwani Signed-off-by: rayshrey Co-authored-by: Mohit Godwani --- .../common/queue/DefaultLockableHolder.java | 65 +++ sandbox/libs/composite-common/build.gradle | 32 -- .../opensearch/composite/RowIdGenerator.java | 59 -- .../opensearch/composite/package-info.java | 12 - .../composite/RowIdGeneratorTests.java | 53 -- .../index/LuceneCommitDeletionPolicy.java | 1 + .../be/lucene/index/LuceneWriter.java | 24 +- .../LuceneCommitDeletionPolicyTests.java | 7 +- ...eneCommitterCSManagerIntegrationTests.java | 30 +- .../be/lucene/index/LuceneWriterTests.java | 10 - sandbox/plugins/composite-engine/build.gradle | 1 - .../composite/CompositeMergeIT.java | 43 +- .../opensearch/composite/CompositeWriter.java | 27 +- .../composite/CompositeTestHelper.java | 11 - .../composite/CompositeWriterTests.java | 16 - .../composite/merge/CompositeMergerTests.java | 21 +- .../benchmarks/build.gradle | 2 +- .../plugins/parquet-data-format/build.gradle | 3 + .../log4j-slf4j2-impl-2.25.4.jar.sha1 | 1 + .../licenses/log4j-slf4j2-impl-LICENSE.txt | 202 +++++++ .../licenses/log4j-slf4j2-impl-NOTICE.txt | 6 + .../parquet/ParquetDataFormatPlugin.java | 6 +- .../opensearch/parquet/bridge/RustBridge.java | 44 +- .../parquet/engine/ParquetIndexingEngine.java | 48 +- .../parquet/fields/ParquetField.java | 12 +- .../merge/NativeParquetMergeStrategy.java | 54 +- .../opensearch/parquet/vsr/ManagedVSR.java | 7 +- .../opensearch/parquet/vsr/VSRManager.java | 5 + .../parquet/writer/ParquetWriter.java | 15 +- .../src/main/rust/src/ffm.rs | 30 +- .../src/main/rust/src/merge/io_task.rs | 12 +- .../src/main/rust/src/merge/sorted.rs | 14 +- .../src/main/rust/src/merge/unsorted.rs | 12 +- .../src/main/rust/src/writer.rs | 4 +- .../bridge/ParquetMergeIntegrationTests.java | 5 + .../index/engine/DataFormatAwareEngine.java | 188 ++++-- .../engine/dataformat/RowIdAwareWriter.java | 93 +++ .../index/engine/dataformat/Writer.java | 3 +- .../engine/dataformat/merge/MergeHandler.java | 15 +- .../dataformat/merge/MergeScheduler.java | 2 +- .../engine/exec/coord/CatalogSnapshot.java | 22 + .../exec/coord/CatalogSnapshotManager.java | 185 +++++- .../coord/DataformatAwareCatalogSnapshot.java | 3 + .../engine/exec/coord/IndexFileDeleter.java | 25 +- .../opensearch/index/shard/IndexShard.java | 2 +- .../engine/DataFormatAwareEngineTests.java | 539 +++++++++++------- .../dataformat/DataFormatPluginTests.java | 4 +- .../engine/dataformat/merge/MergeTests.java | 31 +- .../coord/CatalogSnapshotManagerTests.java | 40 +- .../exec/coord/IndexFileDeleterTests.java | 64 +-- .../engine/dataformat/stub/MockWriter.java | 11 - 51 files changed, 1403 insertions(+), 718 deletions(-) create mode 100644 libs/concurrent-queue/src/main/java/org/opensearch/common/queue/DefaultLockableHolder.java delete mode 100644 sandbox/libs/composite-common/build.gradle delete mode 100644 sandbox/libs/composite-common/src/main/java/org/opensearch/composite/RowIdGenerator.java delete mode 100644 sandbox/libs/composite-common/src/main/java/org/opensearch/composite/package-info.java delete mode 100644 sandbox/libs/composite-common/src/test/java/org/opensearch/composite/RowIdGeneratorTests.java create mode 100644 sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 create mode 100644 sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-LICENSE.txt create mode 100644 sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-NOTICE.txt create mode 100644 server/src/main/java/org/opensearch/index/engine/dataformat/RowIdAwareWriter.java diff --git a/libs/concurrent-queue/src/main/java/org/opensearch/common/queue/DefaultLockableHolder.java b/libs/concurrent-queue/src/main/java/org/opensearch/common/queue/DefaultLockableHolder.java new file mode 100644 index 0000000000000..24441ca1ff74d --- /dev/null +++ b/libs/concurrent-queue/src/main/java/org/opensearch/common/queue/DefaultLockableHolder.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.queue; + +import java.util.concurrent.locks.ReentrantLock; + +/** + * A {@link Lockable} wrapper around an arbitrary reference, pairing the value with + * a {@link ReentrantLock} for use in pool-based concurrency patterns. + * + *

      Used by {@link LockablePool} to track items that can be locked for exclusive + * access (e.g., writers in the indexing pipeline) and unlocked when returned to the pool. + * + * @param the type of the wrapped reference + */ +public class DefaultLockableHolder implements Lockable { + + private final T ref; + private final ReentrantLock lock = new ReentrantLock(); + + private DefaultLockableHolder(T ref) { + this.ref = ref; + } + + /** + * Creates a new holder wrapping the given reference. + * + * @param ref the reference to wrap + * @param the reference type + * @return a new {@code DefaultLockableHolder} containing {@code ref} + */ + public static DefaultLockableHolder of(R ref) { + return new DefaultLockableHolder<>(ref); + } + + @Override + public void lock() { + lock.lock(); + } + + @Override + public boolean tryLock() { + return lock.tryLock(); + } + + @Override + public void unlock() { + lock.unlock(); + } + + /** + * Returns the wrapped reference. + * + * @return the reference held by this holder + */ + public T get() { + return ref; + } +} diff --git a/sandbox/libs/composite-common/build.gradle b/sandbox/libs/composite-common/build.gradle deleted file mode 100644 index 3400787defe92..0000000000000 --- a/sandbox/libs/composite-common/build.gradle +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Shared utilities for the composite indexing engine. - * Pure Java — no external runtime dependencies. - */ - -dependencies { - /******* - * !!!! NO RUNTIME DEPENDENCIES !!!! - *******/ - - testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" - testImplementation "junit:junit:${versions.junit}" - testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" - - testImplementation(project(":test:framework")) { - exclude group: 'org.opensearch', module: 'opensearch-composite-common' - } -} - -testingConventions.enabled = true - -tasks.named('forbiddenApisMain').configure { - replaceSignatureFiles 'jdk-signatures' -} diff --git a/sandbox/libs/composite-common/src/main/java/org/opensearch/composite/RowIdGenerator.java b/sandbox/libs/composite-common/src/main/java/org/opensearch/composite/RowIdGenerator.java deleted file mode 100644 index 1463e8c2890da..0000000000000 --- a/sandbox/libs/composite-common/src/main/java/org/opensearch/composite/RowIdGenerator.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.composite; - -import java.util.concurrent.atomic.AtomicLong; - -/** - * Generates monotonically increasing row IDs for cross-format document synchronization. - * Each writer instance gets its own {@code RowIdGenerator} so that row IDs are unique - * within a writer's segment scope. - */ -public class RowIdGenerator { - - private final String source; - private final AtomicLong counter; - - /** - * Constructs a RowIdGenerator with the given source identifier. - * - * @param source a human-readable label identifying the generator's owner (e.g. class name) - */ - public RowIdGenerator(String source) { - this.source = source; - this.counter = new AtomicLong(0); - } - - /** - * Returns the next row ID. - * - * @return the next monotonically increasing row ID - */ - public long nextRowId() { - return counter.getAndIncrement(); - } - - /** - * Returns the current row ID value without incrementing. - * - * @return the current row ID - */ - public long currentRowId() { - return counter.get(); - } - - /** - * Returns the source identifier for this generator. - * - * @return the source label - */ - public String getSource() { - return source; - } -} diff --git a/sandbox/libs/composite-common/src/main/java/org/opensearch/composite/package-info.java b/sandbox/libs/composite-common/src/main/java/org/opensearch/composite/package-info.java deleted file mode 100644 index 0197370226c89..0000000000000 --- a/sandbox/libs/composite-common/src/main/java/org/opensearch/composite/package-info.java +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** - * Common utilities shared across composite engine components. - */ -package org.opensearch.composite; diff --git a/sandbox/libs/composite-common/src/test/java/org/opensearch/composite/RowIdGeneratorTests.java b/sandbox/libs/composite-common/src/test/java/org/opensearch/composite/RowIdGeneratorTests.java deleted file mode 100644 index 1568be65a093c..0000000000000 --- a/sandbox/libs/composite-common/src/test/java/org/opensearch/composite/RowIdGeneratorTests.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.composite; - -import org.opensearch.test.OpenSearchTestCase; - -/** - * Tests for {@link RowIdGenerator}. - */ -public class RowIdGeneratorTests extends OpenSearchTestCase { - - public void testNextRowIdStartsAtZero() { - RowIdGenerator generator = new RowIdGenerator("test"); - assertEquals(0L, generator.nextRowId()); - } - - public void testNextRowIdIncrementsMonotonically() { - RowIdGenerator generator = new RowIdGenerator("test"); - for (int i = 0; i < 100; i++) { - assertEquals(i, generator.nextRowId()); - } - } - - public void testCurrentRowIdReturnsCurrentWithoutIncrementing() { - RowIdGenerator generator = new RowIdGenerator("test"); - assertEquals(0L, generator.currentRowId()); - assertEquals(0L, generator.currentRowId()); - generator.nextRowId(); - assertEquals(1L, generator.currentRowId()); - assertEquals(1L, generator.currentRowId()); - } - - public void testGetSourceReturnsConstructorArgument() { - String source = randomAlphaOfLength(10); - RowIdGenerator generator = new RowIdGenerator(source); - assertEquals(source, generator.getSource()); - } - - public void testCurrentRowIdReflectsNextRowIdCalls() { - RowIdGenerator generator = new RowIdGenerator("test"); - int count = randomIntBetween(1, 50); - for (int i = 0; i < count; i++) { - generator.nextRowId(); - } - assertEquals(count, generator.currentRowId()); - } -} diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneCommitDeletionPolicy.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneCommitDeletionPolicy.java index e037266ff48a1..b3c0ba2e71c28 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneCommitDeletionPolicy.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneCommitDeletionPolicy.java @@ -86,6 +86,7 @@ public void onCommit(List commits) throws IOException { * @param snapshotId the CatalogSnapshot ID to purge */ void purgeCommit(long snapshotId) { + assert trackedCommits.containsKey(snapshotId); pendingDeletes.add(snapshotId); } } diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneWriter.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneWriter.java index 7bf0bbb0e9a58..f4fdaab0c0e71 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneWriter.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneWriter.java @@ -31,7 +31,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; -import java.util.concurrent.locks.ReentrantLock; /** * Per-generation Lucene writer that creates segments in an isolated temporary directory. @@ -72,7 +71,6 @@ public class LuceneWriter implements Writer { private final Path tempDirectory; private final Directory directory; private final IndexWriter indexWriter; - private final ReentrantLock lock; private volatile long docCount; /** @@ -88,7 +86,6 @@ public LuceneWriter(long writerGeneration, LuceneDataFormat dataFormat, Path bas throws IOException { this.writerGeneration = writerGeneration; this.dataFormat = dataFormat; - this.lock = new ReentrantLock(); this.docCount = 0; // Create an isolated temp directory for this writer's segment @@ -172,9 +169,8 @@ public FileInfos flush() throws IOException { } } - // Since flush is once only, we can close the write post this. + // Since flush is once only, close the IndexWriter but keep directory open for close() indexWriter.close(); - directory.close(); return FileInfos.builder().putWriterFileSet(dataFormat, wfsBuilder.build()).build(); } @@ -196,24 +192,6 @@ public long generation() { return writerGeneration; } - /** Acquires the writer's reentrant lock. Used by the writer pool to serialize access. */ - @Override - public void lock() { - lock.lock(); - } - - /** Attempts to acquire the writer's reentrant lock without blocking. */ - @Override - public boolean tryLock() { - return lock.tryLock(); - } - - /** Releases the writer's reentrant lock. */ - @Override - public void unlock() { - lock.unlock(); - } - /** * Closes this writer, rolling back the IndexWriter if still open, closing the directory, * and deleting the temp directory. Safe to call multiple times. diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitDeletionPolicyTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitDeletionPolicyTests.java index 0bab3b78606cf..70007e59c062b 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitDeletionPolicyTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitDeletionPolicyTests.java @@ -78,14 +78,11 @@ public void testPurgeCommitDeletedOnNextOnCommit() throws IOException { verify(csCommit).delete(); } - public void testPurgeCommitWithUnknownIdIsNoOp() throws IOException { + public void testPurgeCommitWithUnknownIdThrowsAssertion() throws IOException { LuceneCommitDeletionPolicy policy = new LuceneCommitDeletionPolicy(); IndexCommit csCommit = mockCommit(Map.of(CatalogSnapshot.CATALOG_SNAPSHOT_KEY, "blob", CatalogSnapshot.CATALOG_SNAPSHOT_ID, "1")); policy.onCommit(List.of(csCommit)); - policy.purgeCommit(999L); - policy.onCommit(List.of(csCommit)); - - verify(csCommit, never()).delete(); + expectThrows(AssertionError.class, () -> policy.purgeCommit(999L)); } } diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterCSManagerIntegrationTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterCSManagerIntegrationTests.java index 1e5bae3fd6508..6e2a9b2af93bd 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterCSManagerIntegrationTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterCSManagerIntegrationTests.java @@ -193,6 +193,30 @@ private static FileDeleter fileDeleterFor(Path dir) { }; } + private static FileDeleter combinedFileDeleter(Map formatDirs) { + return filesToDelete -> { + Map> failed = new HashMap<>(); + for (Map.Entry> entry : filesToDelete.entrySet()) { + Path dir = formatDirs.get(entry.getKey()); + if (dir == null) continue; + Collection failedFiles = new ArrayList<>(); + for (String file : entry.getValue()) { + try { + if (Files.deleteIfExists(dir.resolve(file)) == false) { + failedFiles.add(file); + } + } catch (IOException e) { + failedFiles.add(file); + } + } + if (!failedFiles.isEmpty()) { + failed.put(entry.getKey(), failedFiles); + } + } + return failed; + }; + } + private boolean fileExists(Path dir, String fileName) { return Files.exists(dir.resolve(fileName)); } @@ -228,7 +252,7 @@ private CatalogSnapshotManager bootstrap( return new CatalogSnapshotManager( env.committer.listCommittedSnapshots(), policy, - Map.of(PARQUET_FORMAT, fileDeleterFor(env.parquetDir), LUCENE_FORMAT, fileDeleterFor(env.indexDir)), + combinedFileDeleter(Map.of(PARQUET_FORMAT, env.parquetDir, LUCENE_FORMAT, env.indexDir)), Map.of(), List.of(), env.shardPath, @@ -553,7 +577,7 @@ public void testRecoveryAfterCrashTrimsUnsafeCommits() throws Exception { CatalogSnapshotManager manager = new CatalogSnapshotManager( committer.listCommittedSnapshots(), policy, - Map.of(PARQUET_FORMAT, fileDeleterFor(parquetDir), LUCENE_FORMAT, fileDeleterFor(indexDir)), + combinedFileDeleter(Map.of(PARQUET_FORMAT, parquetDir, LUCENE_FORMAT, indexDir)), Map.of(), List.of(), shardPath, @@ -672,7 +696,7 @@ public void testRecoveryThenNormalOperationWorks() throws Exception { CatalogSnapshotManager manager = new CatalogSnapshotManager( committer.listCommittedSnapshots(), policy, - Map.of(PARQUET_FORMAT, fileDeleterFor(parquetDir), LUCENE_FORMAT, fileDeleterFor(indexDir)), + combinedFileDeleter(Map.of(PARQUET_FORMAT, parquetDir, LUCENE_FORMAT, indexDir)), Map.of(), List.of(), shardPath, diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java index 2cf084d10b1bf..55843ea1d6779 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java +++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java @@ -206,16 +206,6 @@ public void testMixedTextAndKeywordFields() throws IOException { } } - public void testLockUnlock() throws IOException { - Path baseDir = createTempDir(); - try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault())) { - assertTrue(writer.tryLock()); - writer.unlock(); - writer.lock(); - writer.unlock(); - } - } - public void testWriteAndFlushEndToEndWithTextAndKeyword() throws IOException { Path baseDir = createTempDir(); MappedFieldType textField = mockTextField("body"); diff --git a/sandbox/plugins/composite-engine/build.gradle b/sandbox/plugins/composite-engine/build.gradle index 2f0f821950c85..a4f2675cc013c 100644 --- a/sandbox/plugins/composite-engine/build.gradle +++ b/sandbox/plugins/composite-engine/build.gradle @@ -43,7 +43,6 @@ internalClusterTest { dependencies { api project(':libs:opensearch-concurrent-queue') - api project(':sandbox:libs:composite-common') compileOnly project(':server') testImplementation project(':test:framework') testImplementation project(':sandbox:plugins:parquet-data-format') diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java index d965df923473d..0ce2f195a4b0d 100644 --- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java @@ -8,6 +8,8 @@ package org.opensearch.composite; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + import org.opensearch.action.admin.indices.refresh.RefreshResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.action.admin.indices.stats.ShardStats; @@ -42,18 +44,15 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; -/** - * Integration tests for composite merge with real Parquet backend. - * - * Run with: - * ./gradlew :sandbox:plugins:composite-engine:internalClusterTest \ - * --tests "*.CompositeMergeIT" -Dsandbox.enabled=true - */ +// The Tokio IO runtime worker thread (used by the Rust merge k-way merge sort) is a process-lifetime +// singleton that persists after tests complete. It polls for new async IO tasks between merges. +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1) public class CompositeMergeIT extends OpenSearchIntegTestCase { @@ -134,6 +133,8 @@ public void testBackgroundMerge() throws Exception { assertEquals(Set.of("parquet"), snapshot.getDataFormats()); verifyRowCount(snapshot, totalDocs); + verifySegmentGenerationUniqueness(snapshot); + verifyNoOrphanFiles(snapshot); } /** @@ -170,6 +171,8 @@ public void testSortedMerge() throws Exception { verifyRowCount(snapshot, totalDocs); verifySortOrder(snapshot); + verifySegmentGenerationUniqueness(snapshot); + verifyNoOrphanFiles(snapshot); } // ── Settings ── @@ -248,12 +251,38 @@ private void verifyRowCount(DataformatAwareCatalogSnapshot snapshot, int expecte Path filePath = parquetDir.resolve(file); assertTrue("Parquet file should exist: " + filePath, Files.exists(filePath)); ParquetFileMetadata metadata = RustBridge.getFileMetadata(filePath.toString()); + assertEquals("WriterFileSet numRows should match actual file metadata for " + file, wfs.numRows(), metadata.numRows()); totalRows += metadata.numRows(); } } assertEquals("Total rows across all segments should match ingested docs", expectedTotalDocs, totalRows); } + private void verifySegmentGenerationUniqueness(DataformatAwareCatalogSnapshot snapshot) { + List generations = snapshot.getSegments().stream().map(Segment::generation).toList(); + assertEquals("All segment generations must be unique", generations.size(), generations.stream().distinct().count()); + } + + private void verifyNoOrphanFiles(DataformatAwareCatalogSnapshot snapshot) throws IOException { + Path parquetDir = getParquetDir(); + Set referencedFiles = new HashSet<>(); + for (Segment segment : snapshot.getSegments()) { + WriterFileSet wfs = segment.dfGroupedSearchableFiles().get("parquet"); + if (wfs != null) { + referencedFiles.addAll(wfs.files()); + } + } + try (var stream = Files.list(parquetDir)) { + List diskFiles = stream.filter(Files::isRegularFile) + .map(p -> p.getFileName().toString()) + .filter(f -> f.endsWith(".parquet")) + .toList(); + for (String diskFile : diskFiles) { + assertTrue("Orphan parquet file on disk not referenced by catalog: " + diskFile, referencedFiles.contains(diskFile)); + } + } + } + /** * Verifies that merged parquet files have age in DESC order with nulls first, * and within same age, name in ASC order with nulls last. diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeWriter.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeWriter.java index bddaeb9a62fc1..0db9b064f1239 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeWriter.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeWriter.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.common.queue.Lockable; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.engine.dataformat.FileInfos; @@ -26,7 +25,6 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReentrantLock; /** * A composite {@link Writer} that wraps one {@link Writer} per registered data format @@ -40,16 +38,14 @@ * @opensearch.experimental */ @ExperimentalApi -class CompositeWriter implements Writer, Lockable { +class CompositeWriter implements Writer { private static final Logger logger = LogManager.getLogger(CompositeWriter.class); private final DataFormat primaryFormat; private final Writer> primaryWriter; private final Map>> secondaryWritersByFormat; - private final ReentrantLock lock; private final long writerGeneration; - private final RowIdGenerator rowIdGenerator; private final AtomicReference state; /** @@ -83,7 +79,6 @@ enum WriterState { */ @SuppressWarnings("unchecked") CompositeWriter(CompositeIndexingExecutionEngine engine, long writerGeneration) { - this.lock = new ReentrantLock(); this.state = new AtomicReference<>(WriterState.ACTIVE); this.writerGeneration = writerGeneration; @@ -96,7 +91,6 @@ enum WriterState { secondaries.put(delegate.getDataFormat(), (Writer>) delegate.createWriter(writerGeneration)); } this.secondaryWritersByFormat = Collections.unmodifiableMap(secondaries); - this.rowIdGenerator = new RowIdGenerator(CompositeWriter.class.getName()); } @Override @@ -104,10 +98,6 @@ public WriteResult addDoc(CompositeDocumentInput doc) throws IOException { if (state.get() != WriterState.ACTIVE) { throw new IllegalStateException("Cannot add document to writer in state " + state.get()); } - // Row ID must be assigned before writing to any format — it's the cross-format correlation key - doc.setRowId(DocumentInput.ROW_ID_FIELD, rowIdGenerator.nextRowId()); - // Row ID must be non-negative and sequential within this writer - assert rowIdGenerator.currentRowId() >= 0 : "row ID must be non-negative but was: " + rowIdGenerator.currentRowId(); // Write to primary first WriteResult primaryResult = primaryWriter.addDoc(doc.getPrimaryInput()); @@ -253,19 +243,4 @@ boolean isFlushPending() { WriterState getState() { return state.get(); } - - @Override - public void lock() { - lock.lock(); - } - - @Override - public boolean tryLock() { - return lock.tryLock(); - } - - @Override - public void unlock() { - lock.unlock(); - } } diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java index 236e52739bc66..428fa9b0927eb 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java @@ -237,17 +237,6 @@ public void close() {} public long generation() { return 0; } - - @Override - public void lock() {} - - @Override - public boolean tryLock() { - return true; - } - - @Override - public void unlock() {} } /** diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeWriterTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeWriterTests.java index a5c18f7cd3f4b..1c3404a339848 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeWriterTests.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeWriterTests.java @@ -81,22 +81,6 @@ public void testFlushPendingDoesNotTransitionFromAborted() throws IOException { writer.close(); } - public void testLockAndUnlock() throws IOException { - CompositeWriter writer = new CompositeWriter(engine, 0); - writer.lock(); - assertTrue(writer.tryLock()); - writer.unlock(); - writer.unlock(); - writer.close(); - } - - public void testTryLockSucceedsWhenUnlocked() throws IOException { - CompositeWriter writer = new CompositeWriter(engine, 0); - assertTrue(writer.tryLock()); - writer.unlock(); - writer.close(); - } - public void testFlushReturnsFileInfos() throws IOException { CompositeWriter writer = new CompositeWriter(engine, 0); FileInfos fileInfos = writer.flush(); diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/merge/CompositeMergerTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/merge/CompositeMergerTests.java index b0a655034c2bc..2c3988954fc3c 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/merge/CompositeMergerTests.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/merge/CompositeMergerTests.java @@ -138,7 +138,8 @@ public void testDoMergePrimaryOnlyNoSecondaries() throws IOException { new CompositeMerger(engineNoSecondary, primaryOnlyFormat), SHARD_ID, mock(MergeHandler.MergePolicy.class), - mock(MergeHandler.MergeListener.class) + mock(MergeHandler.MergeListener.class), + () -> 1L ); MergeResult result = handler.doMerge(oneMerge); @@ -221,7 +222,8 @@ public void testDoMergeMultipleSecondariesFailsFastOnFirstError() throws IOExcep new CompositeMerger(multiEngine, multiFormat), SHARD_ID, mock(MergeHandler.MergePolicy.class), - mock(MergeHandler.MergeListener.class) + mock(MergeHandler.MergeListener.class), + () -> 1L ); UncheckedIOException ex = expectThrows(UncheckedIOException.class, () -> handler.doMerge(oneMerge)); @@ -371,7 +373,8 @@ public void testDoMergeSkipsSecondaryThatEqualsPrimary() throws IOException { new CompositeMerger(dupEngine, dupFormat), SHARD_ID, mock(MergeHandler.MergePolicy.class), - mock(MergeHandler.MergeListener.class) + mock(MergeHandler.MergeListener.class), + () -> 1L ); MergeResult result = handler.doMerge(oneMerge); @@ -565,7 +568,8 @@ private MergeHandler createHandler() { new CompositeMerger(compositeEngine, compositeDataFormat), SHARD_ID, mock(MergeHandler.MergePolicy.class), - mock(MergeHandler.MergeListener.class) + mock(MergeHandler.MergeListener.class), + () -> 1L ); } @@ -578,7 +582,14 @@ private MergeHandler createHandlerWithRealPolicy() { IndexMetadata indexMetadata = IndexMetadata.builder("test-index").settings(settings).build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); DataFormatAwareMergePolicy policy = new DataFormatAwareMergePolicy(indexSettings.getMergePolicy(true), SHARD_ID); - return new MergeHandler(snapshotSupplier, new CompositeMerger(compositeEngine, compositeDataFormat), SHARD_ID, policy, policy); + return new MergeHandler( + snapshotSupplier, + new CompositeMerger(compositeEngine, compositeDataFormat), + SHARD_ID, + policy, + policy, + () -> 1L + ); } private static DataFormat stubFormat(String name) { diff --git a/sandbox/plugins/parquet-data-format/benchmarks/build.gradle b/sandbox/plugins/parquet-data-format/benchmarks/build.gradle index ee90cb6d2301b..137d589e558cd 100644 --- a/sandbox/plugins/parquet-data-format/benchmarks/build.gradle +++ b/sandbox/plugins/parquet-data-format/benchmarks/build.gradle @@ -54,7 +54,7 @@ dependencies { api "org.slf4j:slf4j-api:${versions.slf4j}" api "org.apache.logging.log4j:log4j-api:${versions.log4j}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}" - api "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" + api "org.apache.logging.log4j:log4j-slf4j2-impl:${versions.log4j}" } // enable the JMH's BenchmarkProcessor to generate the final benchmark classes diff --git a/sandbox/plugins/parquet-data-format/build.gradle b/sandbox/plugins/parquet-data-format/build.gradle index 996ab9d2bc5b1..323456b1786f2 100644 --- a/sandbox/plugins/parquet-data-format/build.gradle +++ b/sandbox/plugins/parquet-data-format/build.gradle @@ -33,6 +33,9 @@ dependencies { implementation 'org.checkerframework:checker-qual:3.43.0' implementation "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" implementation "org.slf4j:slf4j-api:${versions.slf4j}" + implementation "org.apache.logging.log4j:log4j-api:${versions.log4j}" + implementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" + implementation "org.apache.logging.log4j:log4j-slf4j2-impl:${versions.log4j}" // jackson-core is on the server classpath; jackson-databind and jackson-annotations are not. compileOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" implementation("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}") { diff --git a/sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 b/sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 new file mode 100644 index 0000000000000..f018d071914e4 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-2.25.4.jar.sha1 @@ -0,0 +1 @@ +052a8e43b29eee3b9d6cd9bad696f5d2284d7053 \ No newline at end of file diff --git a/sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-LICENSE.txt b/sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-LICENSE.txt new file mode 100644 index 0000000000000..6279e5206de13 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 1999-2005 The Apache Software Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-NOTICE.txt b/sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-NOTICE.txt new file mode 100644 index 0000000000000..5a296bfcd19ec --- /dev/null +++ b/sandbox/plugins/parquet-data-format/licenses/log4j-slf4j2-impl-NOTICE.txt @@ -0,0 +1,6 @@ +SLF4J 2 Provider for Log4j API +Copyright 1999-2025 The Apache Software Foundation + + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java index 33bb655257a78..164c2d866bea7 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java @@ -64,7 +64,7 @@ public class ParquetDataFormatPlugin extends Plugin implements DataFormatPlugin /** Thread pool name for background native Parquet writes during VSR rotation. */ public static final String PARQUET_THREAD_POOL_NAME = "parquet_native_write"; - private static final ParquetDataFormat dataFormat = new ParquetDataFormat(); + public static final ParquetDataFormat PARQUET_DATA_FORMAT = new ParquetDataFormat(); /** Initialized to EMPTY to avoid NPE if indexingEngine() is called before createComponents(). */ private Settings settings = Settings.EMPTY; private ThreadPool threadPool; @@ -93,14 +93,14 @@ public Collection createComponents( @Override public DataFormat getDataFormat() { - return dataFormat; + return PARQUET_DATA_FORMAT; } @Override public IndexingExecutionEngine indexingEngine(IndexingEngineConfig engineConfig) { return new ParquetIndexingEngine( settings, - dataFormat, + PARQUET_DATA_FORMAT, engineConfig.store().shardPath(), () -> ArrowSchemaBuilder.getSchema(engineConfig.mapperService()), engineConfig.indexSettings(), diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java index 4c5d8d60272c6..2cd44a1a1bba8 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java @@ -137,11 +137,17 @@ public class RustBridge { ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.ADDRESS, - ValueLayout.JAVA_LONG, // input files (ptrs, lens, count) + ValueLayout.JAVA_LONG, // input files (ptrs, lens, count) ValueLayout.ADDRESS, - ValueLayout.JAVA_LONG, // output file + ValueLayout.JAVA_LONG, // output file ValueLayout.ADDRESS, - ValueLayout.JAVA_LONG // index_name + ValueLayout.JAVA_LONG, // index_name + ValueLayout.ADDRESS, // version_out + ValueLayout.ADDRESS, // num_rows_out + ValueLayout.ADDRESS, // created_by_buf + ValueLayout.JAVA_LONG, // created_by_buf_len + ValueLayout.ADDRESS, // created_by_len_out + ValueLayout.ADDRESS // crc32_out ) ); READ_AS_JSON = linker.downcallHandle( @@ -289,13 +295,41 @@ public static void removeSettings(String indexName) { } } - public static void mergeParquetFilesInRust(List inputFiles, String outputFile, String indexName) { + public static ParquetFileMetadata mergeParquetFilesInRust(List inputFiles, String outputFile, String indexName) { String[] paths = inputFiles.stream().map(Path::toString).toArray(String[]::new); try (var call = new NativeCall()) { var inputs = call.strArray(paths); var out = call.str(outputFile); var idx = call.str(indexName); - call.invokeIO(MERGE_FILES, inputs.ptrs(), inputs.lens(), inputs.count(), out.segment(), out.len(), idx.segment(), idx.len()); + var versionOut = call.intOut(); + var numRowsOut = call.longOut(); + var crc32Out = call.longOut(); + var createdByOut = call.outBuffer(1024); + call.invokeIO( + MERGE_FILES, + inputs.ptrs(), + inputs.lens(), + inputs.count(), + out.segment(), + out.len(), + idx.segment(), + idx.len(), + versionOut, + numRowsOut, + createdByOut.data(), + (long) createdByOut.capacity(), + createdByOut.lenOut(), + crc32Out + ); + int createdByLen = (int) createdByOut.lenOut().get(ValueLayout.JAVA_LONG, 0); + return new ParquetFileMetadata( + versionOut.get(ValueLayout.JAVA_INT, 0), + numRowsOut.get(ValueLayout.JAVA_LONG, 0), + createdByLen >= 0 + ? new String(createdByOut.data().asSlice(0, createdByLen).toArray(ValueLayout.JAVA_BYTE), StandardCharsets.UTF_8) + : null, + crc32Out.get(ValueLayout.JAVA_LONG, 0) + ); } catch (IOException e) { throw new UncheckedIOException("Native merge failed", e); } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java index 0b4cb025e8463..662487e994065 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java @@ -44,6 +44,8 @@ import java.util.Map; import java.util.function.Supplier; +import static org.opensearch.parquet.ParquetDataFormatPlugin.PARQUET_DATA_FORMAT; + /** * Per-shard Parquet indexing execution engine. * @@ -64,9 +66,9 @@ public class ParquetIndexingEngine implements IndexingExecutionEngine createWriter(long writerGeneration) { - Path filePath = Path.of( - shardPath.getDataPath().toString(), - dataFormat.name(), - FILE_NAME_PREFIX + "_" + writerGeneration + FILE_NAME_EXT - ); + Path filePath = buildParquetFilePath(shardPath, writerGeneration, null); return new ParquetWriter( filePath.toString(), writerGeneration, @@ -237,7 +235,8 @@ public Map> deleteFiles(Map failed = new ArrayList<>(); for (String fileName : parquetFiles) { - Path filePath = Path.of(fileName); + // Resolve relative file names against the shard's parquet directory + Path filePath = shardPath.getDataPath().resolve(dataFormat.name()).resolve(fileName); logger.debug("Deleting parquet file: {}", filePath); if (Files.deleteIfExists(filePath) == false) { logger.warn("Failed to delete parquet file: {}", filePath); @@ -270,4 +269,33 @@ public void close() throws IOException { } bufferPool.close(); } + + /** + * Builds a full file path for a Parquet file within the shard's data directory. + * + * @param shardPath the shard's directory path + * @param writerGeneration the writer generation number + * @param additionalPrefix an optional prefix to append (e.g., "merged") + * @return the full file path + */ + public static Path buildParquetFilePath(ShardPath shardPath, long writerGeneration, String additionalPrefix) { + String subDirectory = PARQUET_DATA_FORMAT.name(); + return shardPath.getDataPath().resolve(subDirectory).resolve(buildParquetFileName(writerGeneration, additionalPrefix)); + } + + /** + * Builds a Parquet file name with optional additional prefix. + * + * @param writerGeneration the writer generation number + * @param additionalPrefix an optional prefix to append (e.g., "merged") + * @return the formatted file name + */ + public static String buildParquetFileName(long writerGeneration, String additionalPrefix) { + StringBuilder fileNameBuilder = new StringBuilder(FILE_NAME_PREFIX); + if (additionalPrefix != null) { + fileNameBuilder.append("_").append(additionalPrefix); + } + fileNameBuilder.append("_").append(Long.toHexString(writerGeneration)).append(FILE_NAME_EXT); + return fileNameBuilder.toString(); + } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ParquetField.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ParquetField.java index b8ae33b396ff9..d1cdd67165240 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ParquetField.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/fields/ParquetField.java @@ -13,8 +13,6 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.parquet.vsr.ManagedVSR; -import java.util.Objects; - /** * Abstract base class for Parquet field implementations that handle conversion * between OpenSearch field types and Apache Arrow vectors. @@ -39,13 +37,9 @@ public ParquetField() {} * @param parseValue the parsed value to write */ public final void createField(MappedFieldType fieldType, ManagedVSR managedVSR, Object parseValue) { - Objects.requireNonNull(fieldType, "MappedFieldType cannot be null"); - Objects.requireNonNull(managedVSR, "ManagedVSR cannot be null"); - if (managedVSR.getVector(fieldType.name()) != null) { - addToGroup(fieldType, managedVSR, parseValue); - } else { - throw new IllegalArgumentException("Vector not found for field: " + fieldType.name()); - } + assert fieldType != null : "MappedFieldType cannot be null"; + assert managedVSR != null : "ManagedVSR cannot be null"; + addToGroup(fieldType, managedVSR, parseValue); } /** Returns the Arrow type for this field. */ diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java index a3bbdee35c6f8..021f5b04238a6 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java @@ -11,10 +11,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.common.TriConsumer; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.MergeInput; import org.opensearch.index.engine.dataformat.MergeResult; import org.opensearch.index.engine.exec.WriterFileSet; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.parquet.bridge.ParquetFileMetadata; import org.opensearch.parquet.bridge.RustBridge; import org.opensearch.parquet.engine.ParquetIndexingEngine; @@ -34,12 +37,19 @@ public class NativeParquetMergeStrategy implements ParquetMergeStrategy { private final DataFormat dataFormat; private final String indexName; - private final Path shardDataPath; - - public NativeParquetMergeStrategy(DataFormat dataFormat, String indexName, Path shardDataPath) { + private final ShardPath shardPath; + private TriConsumer checksumUpdater; + + public NativeParquetMergeStrategy( + DataFormat dataFormat, + String indexName, + ShardPath shardPath, + TriConsumer checksumUpdater + ) { this.dataFormat = dataFormat; this.indexName = indexName; - this.shardDataPath = shardDataPath; + this.shardPath = shardPath; + this.checksumUpdater = checksumUpdater; } @Override @@ -50,27 +60,41 @@ public MergeResult mergeParquetFiles(MergeInput mergeInput) { if (files.isEmpty()) { throw new IllegalArgumentException("No files to merge"); } + assert writerGeneration > 0 : "merge writer generation must be positive but was: " + writerGeneration; List filePaths = new ArrayList<>(); files.forEach( - writerFileSet -> writerFileSet.files() - .forEach(file -> filePaths.add(shardDataPath.resolve(writerFileSet.directory()).resolve(file))) + writerFileSet -> writerFileSet.files().forEach(file -> filePaths.add(Path.of(writerFileSet.directory()).resolve(file))) ); + assert filePaths.isEmpty() == false : "must have at least one input file path for merge"; + // All input files must exist on disk before invoking the native merge + // This will change to object store lookup once warm is in place + assert filePaths.stream().allMatch(p -> java.nio.file.Files.exists(p)) : "all input files must exist on disk before merge: " + + filePaths.stream().filter(p -> java.nio.file.Files.exists(p) == false).toList(); - String outputDirectory = shardDataPath.resolve(files.getFirst().directory()).toString(); - String mergedFilePath = getMergedFilePath(writerGeneration, outputDirectory); - String mergedFileName = getMergedFileName(writerGeneration); + Path mergedFilePath = ParquetIndexingEngine.buildParquetFilePath(shardPath, writerGeneration, "merged"); + String mergedFileName = mergedFilePath.getFileName().toString(); try { // Merge files in Rust - RustBridge.mergeParquetFilesInRust(filePaths, mergedFilePath, indexName); + ParquetFileMetadata mergeMetadata = RustBridge.mergeParquetFilesInRust(filePaths, mergedFilePath.toString(), indexName); + assert mergeMetadata.numRows() > 0 : "Merged file should contain at least one row"; + + long expectedRows = files.stream().mapToLong(WriterFileSet::numRows).sum(); + assert mergeMetadata.numRows() == expectedRows : "Merged row count [" + + mergeMetadata.numRows() + + "] must equal sum of input row counts [" + + expectedRows + + "]"; WriterFileSet mergedWriterFileSet = WriterFileSet.builder() - .directory(Path.of(files.getFirst().directory())) + .directory(mergedFilePath.getParent().toAbsolutePath()) .addFile(mergedFileName) .writerGeneration(writerGeneration) + .addNumRows(mergeMetadata.numRows()) .build(); + checksumUpdater.apply(mergedFileName, mergeMetadata.crc32(), mergeInput.newWriterGeneration()); Map mergedWriterFileSetMap = Collections.singletonMap(dataFormat, mergedWriterFileSet); return new MergeResult(mergedWriterFileSetMap); @@ -78,7 +102,7 @@ public MergeResult mergeParquetFiles(MergeInput mergeInput) { } catch (Exception exception) { logger.error(() -> new ParameterizedMessage("Merge failed while creating merged file [{}]", mergedFilePath), exception); try { - Files.deleteIfExists(Path.of(mergedFilePath)); + Files.deleteIfExists(mergedFilePath); logger.info("Stale Merged File Deleted at : [{}]", mergedFilePath); } catch (Exception innerException) { logger.error(() -> new ParameterizedMessage("Failed to delete stale merged file [{}]", mergedFilePath), innerException); @@ -91,10 +115,6 @@ public MergeResult mergeParquetFiles(MergeInput mergeInput) { private String getMergedFileName(long generation) { // TODO: For debugging we have added extra "merged" in file name, later we can remove and keep same as writer - return ParquetIndexingEngine.FILE_NAME_PREFIX + "_merged_" + generation + ParquetIndexingEngine.FILE_NAME_EXT; - } - - private String getMergedFilePath(long generation, String outputDirectory) { - return Path.of(outputDirectory, getMergedFileName(generation)).toString(); + return ParquetIndexingEngine.buildParquetFileName(generation, "merged"); } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/ManagedVSR.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/ManagedVSR.java index 5a01311215c78..b385da2a50fea 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/ManagedVSR.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/ManagedVSR.java @@ -49,7 +49,7 @@ public class ManagedVSR implements AutoCloseable { private final VectorSchemaRoot vsr; private final BufferAllocator allocator; private final AtomicReference state = new AtomicReference<>(VSRState.ACTIVE); - private final Map fields = new HashMap<>(); + private final Map fields = new HashMap<>(); /** * Creates a new ManagedVSR. @@ -63,7 +63,7 @@ public ManagedVSR(String id, Schema schema, BufferAllocator allocator) { this.vsr = VectorSchemaRoot.create(schema, allocator); this.allocator = allocator; for (Field field : vsr.getSchema().getFields()) { - fields.put(field.getName(), field); + fields.put(field.getName(), vsr.getVector(field)); } } @@ -93,8 +93,7 @@ public FieldVector getVector(String fieldName) { if (state.get() != VSRState.ACTIVE) { throw new IllegalStateException("Cannot access vector in VSR state: " + state.get()); } - Field field = fields.get(fieldName); - return field != null ? vsr.getVector(field) : null; + return fields.get(fieldName); } /** Transitions this VSR from ACTIVE to FROZEN state. */ diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java index 668d142d6aae1..f64503295c0d0 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java @@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.LongAdder; /** * Top-level orchestrator for the Arrow batching → Parquet file generation pipeline. @@ -67,6 +68,7 @@ public class VSRManager implements AutoCloseable { private volatile Future pendingWrite; private NativeParquetWriter writer; private final int ROTATION_TIMEOUT = 120; + private LongAdder rowCount = new LongAdder(); /** * Creates a new VSRManager with asynchronous background writes (production default). @@ -155,6 +157,7 @@ public void maybeRotateActiveVSR() throws IOException { logger.debug("Writing frozen VSR {} ({} rows) for {}", frozenVSR.getId(), frozenVSR.getRowCount(), fileName); Runnable writeTask = () -> { try (ArrowExport export = frozenVSR.exportToArrow()) { + rowCount.add(frozenVSR.getRowCount()); writer.write(export.getArrayAddress(), export.getSchemaAddress()); } catch (IOException e) { throw new RuntimeException(e); @@ -184,12 +187,14 @@ public ParquetFileMetadata flush() throws IOException { logger.info("Flushing {} rows for {}", currentVSR.getRowCount(), fileName); currentVSR.moveToFrozen(); try (ArrowExport export = currentVSR.exportToArrow()) { + rowCount.add(currentVSR.getRowCount()); writer.write(export.getArrayAddress(), export.getSchemaAddress()); } vsrPool.completeVSR(currentVSR); managedVSR.set(null); } ParquetFileMetadata metadata = writer.flush(); + assert metadata.numRows() == rowCount.sum() : "Row count mismatch between Java managed VSR and Rust writer"; logger.debug("Flush completed for {} with metadata: {}", fileName, metadata); return metadata; } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java index 09d02c7340567..1ad8dd7bd771d 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java @@ -94,6 +94,8 @@ public FileInfos flush() throws IOException { if (file == null || metadata == null || metadata.numRows() == 0) { return FileInfos.empty(); } + assert metadata.numRows() > 0 : "flushed metadata must have positive row count"; + Path filePath = Path.of(file); String fileName = filePath.getFileName().toString(); @@ -103,7 +105,7 @@ public FileInfos flush() throws IOException { } WriterFileSet writerFileSet = WriterFileSet.builder() - .directory(filePath.getParent().getFileName()) + .directory(filePath.getParent().toAbsolutePath()) .writerGeneration(writerGeneration) .addFile(fileName) .addNumRows(metadata.numRows()) @@ -121,17 +123,6 @@ public long generation() { return writerGeneration; } - @Override - public void lock() {} - - @Override - public boolean tryLock() { - return false; - } - - @Override - public void unlock() {} - @Override public void close() throws IOException { vsrManager.close(); diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs index 233c7aef36923..5ccd1394864ce 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs @@ -14,7 +14,7 @@ use std::slice; use std::str; -use native_bridge_common::ffm_safe; +use native_bridge_common::{ffm_safe, log_debug}; use crate::native_settings::NativeSettings; use crate::merge; @@ -290,6 +290,12 @@ pub unsafe extern "C" fn parquet_merge_files( output_len: i64, index_name_ptr: *const u8, index_name_len: i64, + version_out: *mut i32, + num_rows_out: *mut i64, + created_by_buf: *mut u8, + created_by_buf_len: i64, + created_by_len_out: *mut i64, + crc32_out: *mut i64, ) -> i64 { let input_files = str_array_from_raw(input_ptrs, input_lens, input_count) .map_err(|e| format!("parquet_merge_files inputs: {}", e))?; @@ -317,7 +323,7 @@ pub unsafe extern "C" fn parquet_merge_files( } }; - if sort_cols.is_empty() { + let (metadata, crc32) = if sort_cols.is_empty() { merge::merge_unsorted(&input_files, output_path, index_name) } else { merge::merge_sorted( @@ -329,8 +335,24 @@ pub unsafe extern "C" fn parquet_merge_files( &nulls_first_flags, ) } - .map(|_| 0) - .map_err(|e| format!("{}", e)) + .map_err(|e| format!("{}", e))?; + + // Write metadata to out-pointers + let fm = metadata.file_metadata(); + if !version_out.is_null() { *version_out = fm.version(); } + if !num_rows_out.is_null() { *num_rows_out = fm.num_rows(); } + if let Some(cb) = fm.created_by() { + if !created_by_buf.is_null() && created_by_buf_len > 0 { + let bytes = cb.as_bytes(); + let n = bytes.len().min(created_by_buf_len as usize); + std::ptr::copy_nonoverlapping(bytes.as_ptr(), created_by_buf, n); + if !created_by_len_out.is_null() { *created_by_len_out = n as i64; } + } + } else if !created_by_len_out.is_null() { + *created_by_len_out = -1; + } + if !crc32_out.is_null() { *crc32_out = crc32 as i64; } + Ok(0) } // --------------------------------------------------------------------------- diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/io_task.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/io_task.rs index 1cbd4f41aa4ca..2647b1f243c02 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/io_task.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/io_task.rs @@ -17,7 +17,7 @@ use rayon::ThreadPool; use tokio::runtime::Runtime; use tokio::sync::{mpsc as tokio_mpsc, oneshot}; use tokio::task::JoinHandle; - +use native_bridge_common::log_info; use crate::crc_writer::CrcWriter; use crate::rate_limited_writer::RateLimitedWriter; use crate::log_error; @@ -172,7 +172,15 @@ pub fn spawn_io_task( let crc = crc_handle.clone(); let result = tokio::task::spawn_blocking(move || { let metadata = w.close().map_err(MergeError::from)?; - Ok((metadata, crc.crc32())) + let crc32 = crc.crc32(); + log_info!( + "[RUST] IO task close: version={}, num_rows={}, created_by={:?}, crc32={:#010x}", + metadata.file_metadata().version(), + metadata.file_metadata().num_rows(), + metadata.file_metadata().created_by(), + crc32 + ); + Ok((metadata, crc32)) }) .await; diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs index 23f9121cb35c3..634e0003de2e2 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs @@ -29,7 +29,7 @@ pub fn merge_sorted( sort_columns: &[String], reverse_sorts: &[bool], nulls_first: &[bool], -) -> super::MergeResult { +) -> super::MergeResult<(parquet::file::metadata::ParquetMetaData, u32)> { let config = crate::writer::SETTINGS_STORE .get(index_name) .map(|r| r.clone()) @@ -39,7 +39,9 @@ pub fn merge_sorted( let rayon_threads = config.get_merge_rayon_threads(); let io_threads = config.get_merge_io_threads(); if input_files.is_empty() { - return Ok(0); + return Err(super::MergeError::Logic( + "merge_sorted called with empty input_files".into(), + )); } if sort_columns.is_empty() { @@ -210,16 +212,16 @@ pub fn merge_sorted( } // ── Phase 5: Close ────────────────────────────────────────────────── - let (_metadata, crc32) = ctx.finish()?; + let (metadata, crc32) = ctx.finish()?; log_debug!( "[RUST] Merge complete ({}): {} total rows written to '{}' in {} row groups, crc32={:#010x}", direction_label, - _metadata.file_metadata().num_rows(), + metadata.file_metadata().num_rows(), output_path, - _metadata.num_row_groups(), + metadata.num_row_groups(), crc32 ); - Ok(crc32) + Ok((metadata, crc32)) } diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs index 6618cb4477cad..a43b177e35e41 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs @@ -25,7 +25,7 @@ pub fn merge_unsorted( input_files: &[String], output_path: &str, index_name: &str, -) -> MergeResult { +) -> MergeResult<(parquet::file::metadata::ParquetMetaData, u32)> { let config = crate::writer::SETTINGS_STORE .get(index_name) .map(|r| r.clone()) @@ -91,15 +91,15 @@ pub fn merge_unsorted( } } - let (_metadata, crc32) = ctx.finish()?; + let (metadata, crc32) = ctx.finish()?; log_debug!( - "[RUST] Unsorted merge complete: {} total rows written to '{}' in {} row groups, crc32={:#010x}", - _metadata.file_metadata().num_rows(), + "[RUST] Unsorted merge complete: {} total rows written to '{}' within {} row groups, crc32={:#010x}", + metadata.file_metadata().num_rows(), output_path, - _metadata.num_row_groups(), + metadata.num_row_groups(), crc32 ); - Ok(crc32) + Ok((metadata, crc32)) } diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs index d244e306adec6..f44449183ba7a 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs @@ -323,8 +323,8 @@ impl NativeParquetWriter { log_debug!("Created {} sorted chunks, merging via streaming k-way merge", batch_count); - // merge_sorted returns CRC32 of the final merged output - let crc32 = merge_sorted( + // merge_sorted returns metadata and CRC32 of the final merged output + let (_metadata, crc32) = merge_sorted( &chunk_paths, output_filename, index_name, diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java index cef9bd6f786f0..be07a0a9a4f31 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java @@ -8,6 +8,8 @@ package org.opensearch.parquet.bridge; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.c.Data; @@ -27,6 +29,9 @@ import java.nio.file.Path; import java.util.List; +// The Tokio IO runtime worker thread (used by the Rust merge k-way merge sort) is a process-lifetime +// singleton that persists after tests complete. It polls for new async IO tasks between merges. +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) public class ParquetMergeIntegrationTests extends OpenSearchTestCase { private static final String INDEX_NAME = "merge-test-index"; diff --git a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java index 0906953e68477..a2a630c902357 100644 --- a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java @@ -12,7 +12,9 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.Term; +import org.apache.lucene.search.ReferenceManager; import org.apache.lucene.store.AlreadyClosedException; +import org.opensearch.OpenSearchException; import org.opensearch.common.Booleans; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; @@ -21,6 +23,7 @@ import org.opensearch.common.concurrent.GatedConditionalCloseable; import org.opensearch.common.lease.Releasable; import org.opensearch.common.logging.Loggers; +import org.opensearch.common.queue.DefaultLockableHolder; import org.opensearch.common.queue.LockablePool; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ReleasableLock; @@ -38,6 +41,7 @@ import org.opensearch.index.engine.dataformat.ReaderManagerConfig; import org.opensearch.index.engine.dataformat.RefreshInput; import org.opensearch.index.engine.dataformat.RefreshResult; +import org.opensearch.index.engine.dataformat.RowIdAwareWriter; import org.opensearch.index.engine.dataformat.WriteResult; import org.opensearch.index.engine.dataformat.Writer; import org.opensearch.index.engine.dataformat.merge.DataFormatAwareMergePolicy; @@ -131,13 +135,14 @@ public class DataFormatAwareEngine implements Indexer { private final IndexingExecutionEngine indexingExecutionEngine; private final IndexingStrategyPlanner indexingStrategyPlanner; - private final LockablePool> writerPool; + private final LockablePool>> writerPool; private final AtomicLong writerGenerationCounter; private final Map> readerManagers; private final CatalogSnapshotManager catalogSnapshotManager; private final Committer committer; + private final List refreshListeners; // Translog for durability and recovery private final TranslogManager translogManager; @@ -199,6 +204,17 @@ public DataFormatAwareEngine(EngineConfig engineConfig) { this.store = engineConfig.getStore(); this.throttle = new IndexingThrottler(); + List refreshListeners = new ArrayList<>(); + if (engineConfig.getInternalRefreshListener() != null) { + refreshListeners.addAll(engineConfig.getInternalRefreshListener()); + } + // We don't segregate internal/external here since NRT is anyhow invoked on internal refresh which makes + // data available to read on internal refreshes on replica. + if (engineConfig.getExternalRefreshListener() != null) { + refreshListeners.addAll(engineConfig.getExternalRefreshListener()); + } + this.refreshListeners = List.copyOf(refreshListeners); + if (engineConfig.isAutoGeneratedIDsOptimizationEnabled() == false) { updateAutoIdTimestamp(Long.MAX_VALUE, true); } @@ -243,12 +259,23 @@ public DataFormatAwareEngine(EngineConfig engineConfig) { ), registry.format(config().getIndexSettings().pluggableDataFormat()) ); - this.writerGenerationCounter = new AtomicLong(1L); - this.writerPool = new LockablePool<>( - () -> indexingExecutionEngine.createWriter(writerGenerationCounter.getAndIncrement()), - LinkedList::new, - Runtime.getRuntime().availableProcessors() - ); + long maxGenFromCommit = 0L; + try { + List initSnapshots = committer.listCommittedSnapshots(); + if (initSnapshots.isEmpty() == false) { + for (Segment seg : initSnapshots.getLast().getSegments()) { + maxGenFromCommit = Math.max(maxGenFromCommit, seg.generation()); + } + } + } catch (IOException e) { + // Fall back to 0 on error + } + this.writerGenerationCounter = new AtomicLong(maxGenFromCommit); + this.writerPool = new LockablePool<>(() -> { + long gen = writerGenerationCounter.incrementAndGet(); + assert gen > 0 : "writer generation must be positive but was: " + gen; + return DefaultLockableHolder.of(new RowIdAwareWriter<>(indexingExecutionEngine.createWriter(gen))); + }, LinkedList::new, Runtime.getRuntime().availableProcessors()); // Create Reader managers // We will pass IndexStoreProvider to this, which would contain store // and any index specific attributes useful for reads. @@ -269,8 +296,7 @@ public DataFormatAwareEngine(EngineConfig engineConfig) { ); // 7. Create CatalogSnapshotManager (fully wired) - String formatName = config().getIndexSettings().pluggableDataFormat(); - Map fileDeleters = Map.of(formatName, indexingExecutionEngine::deleteFiles); + FileDeleter fileDeleter = indexingExecutionEngine::deleteFiles; Map filesListeners = new HashMap<>(); List snapshotListeners = new ArrayList<>(); for (Map.Entry> entry : readerManagers.entrySet()) { @@ -284,7 +310,7 @@ public DataFormatAwareEngine(EngineConfig engineConfig) { this.catalogSnapshotManager = new CatalogSnapshotManager( committedSnapshots, combinedPolicy, - fileDeleters, + fileDeleter, filesListeners, snapshotListeners, store.shardPath(), @@ -324,7 +350,12 @@ public DataFormatAwareEngine(EngineConfig engineConfig) { indexingExecutionEngine.getMerger(), shardId, dataFormatAwareMergePolicy, - dataFormatAwareMergePolicy + dataFormatAwareMergePolicy, + () -> { + long gen = writerGenerationCounter.incrementAndGet(); + assert gen > 0 : "merge generation must be positive but was: " + gen; + return gen; + } ); this.mergeScheduler = new MergeScheduler( mergeHandler, @@ -439,6 +470,8 @@ private TranslogDeletionPolicy getTranslogDeletionPolicy() { @Override public Engine.IndexResult index(Engine.Index index) throws IOException { assert Objects.equals(index.uid().field(), IdFieldMapper.NAME) : index.uid().field(); + assert (index.origin() == Engine.Operation.Origin.PRIMARY || index.origin() == Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY) + : "DataFormatAwareEngine only supports PRIMARY origin but got: " + index.origin(); final boolean doThrottle = index.origin().isRecovery() == false; try (ReleasableLock ignored = readLock.acquire()) { ensureOpen(); @@ -486,7 +519,7 @@ public Engine.IndexResult index(Engine.Index index) throws IOException { index.seqNo(), index.primaryTerm() ); - indexResult = indexIntoEngine(index); + indexResult = indexIntoEngine(index, plan); } else { indexResult = new Engine.IndexResult( plan.version, @@ -505,7 +538,7 @@ public Engine.IndexResult index(Engine.Index index) throws IOException { } @SuppressWarnings({ "unchecked", "rawtypes" }) - private Engine.IndexResult indexIntoEngine(Engine.Index index) throws IOException { + private Engine.IndexResult indexIntoEngine(Engine.Index index, IndexingStrategy plan) throws IOException { Engine.IndexResult indexResult; assert index.seqNo() >= 0 : "ops should have an assigned seq no.; origin: " + index.origin(); @@ -514,15 +547,17 @@ private Engine.IndexResult indexIntoEngine(Engine.Index index) throws IOExceptio // Convert ParsedDocument to DocumentInput and write via the execution engine's writer Writer currentWriter = null; + DefaultLockableHolder> lockedWriter = writerPool.getAndLock(); try { - currentWriter = writerPool.getAndLock(); + currentWriter = lockedWriter.get(); // Writer pool must never return null — it creates on demand via the supplier assert currentWriter != null : "writer pool returned null writer"; - + assert index.seqNo() >= 0 : "seqNo must be assigned before writing but was: " + index.seqNo(); + assert index.primaryTerm() > 0 : "primaryTerm must be positive but was: " + index.primaryTerm(); WriteResult result = currentWriter.addDoc(index.parsedDoc().getDocumentInput()); if (result instanceof WriteResult.Success) { - indexResult = new Engine.IndexResult(index.version(), index.primaryTerm(), index.seqNo(), true); + indexResult = new Engine.IndexResult(plan.version, index.primaryTerm(), index.seqNo(), true); // The result must carry the same seq no that was assigned to the operation assert indexResult.getSeqNo() == index.seqNo() : "IndexResult seq no [" + indexResult.getSeqNo() @@ -531,13 +566,13 @@ private Engine.IndexResult indexIntoEngine(Engine.Index index) throws IOExceptio + "]"; } else { WriteResult.Failure f = (WriteResult.Failure) result; - indexResult = new Engine.IndexResult(f.cause(), index.version(), index.primaryTerm(), index.seqNo()); + indexResult = new Engine.IndexResult(f.cause(), plan.version, index.primaryTerm(), index.seqNo()); } } catch (Exception e) { - indexResult = new Engine.IndexResult(e, index.version(), index.primaryTerm(), index.seqNo()); + indexResult = new Engine.IndexResult(e, plan.version, index.primaryTerm(), index.seqNo()); } finally { if (currentWriter != null) { - writerPool.releaseAndUnlock(currentWriter); + writerPool.releaseAndUnlock(lockedWriter); } } @@ -690,11 +725,12 @@ public void refresh(String source) throws EngineException { try (GatedCloseable catalogSnapshot = catalogSnapshotManager.acquireSnapshot()) { if (store.tryIncRef()) { try { - List> writers = writerPool.checkoutAll(); + List>> writers = writerPool.checkoutAll(); List existingSegments = catalogSnapshot.get().getSegments(); List newSegments = new ArrayList<>(); - for (Writer writer : writers) { + for (var lockable : writers) { + Writer writer = lockable.get(); FileInfos fileInfos = writer.flush(); Segment.Builder segmentBuilder = Segment.builder(writer.generation()); boolean hasFiles = false; @@ -719,7 +755,17 @@ public void refresh(String source) throws EngineException { assert newSegments.stream().allMatch(s -> s.dfGroupedSearchableFiles().isEmpty() == false) : "new segments must have at least one format's files"; + // No two new segments may share the same generation + assert newSegments.stream().map(Segment::generation).distinct().count() == newSegments.size() + : "new segments must have unique generations"; + + // New segment generations must not collide with existing segment generations + assert newSegments.stream() + .noneMatch(ns -> existingSegments.stream().anyMatch(es -> es.generation() == ns.generation())) + : "new segment generation collides with an existing segment generation"; + // refresh only if new segments have been created or force param is true + notifyRefreshListenersBefore(); if (refreshed) { RefreshInput refreshInput = new RefreshInput(existingSegments, newSegments); RefreshResult result = indexingExecutionEngine.refresh(refreshInput); @@ -729,12 +775,10 @@ public void refresh(String source) throws EngineException { + existingSegments.size() + " but got " + result.refreshedSegments().size(); - catalogSnapshotManager.commitNewSnapshot(result.refreshedSegments()); - try (GatedCloseable newSnapshotRef = catalogSnapshotManager.acquireSnapshot()) { - refreshListeners(refreshed, newSnapshotRef.get()); - } + catalogSnapshotManager.commitNewSnapshot(result.refreshedSegments()); } + notifyRefreshListenersAfter(refreshed); } finally { store.decRef(); } @@ -760,6 +804,18 @@ public void refresh(String source) throws EngineException { } } + private void notifyRefreshListenersBefore() throws IOException { + for (ReferenceManager.RefreshListener refreshListener : refreshListeners) { + refreshListener.beforeRefresh(); + } + } + + private void notifyRefreshListenersAfter(boolean didRefresh) throws IOException { + for (ReferenceManager.RefreshListener refreshListener : refreshListeners) { + refreshListener.afterRefresh(didRefresh); + } + } + /** * Flushes the engine by refreshing buffered data to segments, persisting the catalog * snapshot and commit data (translog UUID, sequence numbers), syncing the translog, @@ -789,6 +845,7 @@ public void flush(boolean force, boolean waitIfOngoing) throws EngineException { try { // Refresh first to flush buffered data to segments refresh("flush"); + translogManager.rollTranslogGeneration(); // Persist the latest catalog snapshot so it survives restart try (GatedConditionalCloseable snapshotRef = catalogSnapshotManager.acquireSnapshotForCommit()) { CatalogSnapshot snapshot = snapshotRef.get(); @@ -800,15 +857,7 @@ public void flush(boolean force, boolean waitIfOngoing) throws EngineException { // and available to the deletion policy when onCommit is triggered. translogManager.ensureCanFlush(); translogManager.syncTranslog(); - // After sync, the persisted checkpoint must equal the processed checkpoint - assert localCheckpointTracker.getPersistedCheckpoint() == localCheckpointTracker.getProcessedCheckpoint() - : "persisted checkpoint [" - + localCheckpointTracker.getPersistedCheckpoint() - + "] must equal processed checkpoint [" - + localCheckpointTracker.getProcessedCheckpoint() - + "] after sync"; Map commitData = new HashMap<>(); - commitData.put(CatalogSnapshot.CATALOG_SNAPSHOT_KEY, snapshot.serializeToString()); commitData.put(CatalogSnapshot.LAST_COMPOSITE_WRITER_GEN_KEY, Long.toString(snapshot.getLastWriterGeneration())); commitData.put(CatalogSnapshot.CATALOG_SNAPSHOT_ID, Long.toString(snapshot.getId())); commitData.put(Translog.TRANSLOG_UUID_KEY, translogManager.getTranslogUUID()); @@ -819,17 +868,25 @@ public void flush(boolean force, boolean waitIfOngoing) throws EngineException { commitData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(localCheckpointTracker.getMaxSeqNo())); commitData.put(MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, Long.toString(maxUnsafeAutoIdTimestamp.get())); commitData.put(Engine.HISTORY_UUID_KEY, historyUUID); + // Update snapshot userData so deletion policy can read max_seq_no snapshot.setUserData(commitData, true); + + // Now add snapshot to commit data so it has latest snapshot + commitData.put(CatalogSnapshot.CATALOG_SNAPSHOT_KEY, snapshot.serializeToString()); + // Commit data must contain all keys required for recovery assert commitData.containsKey(CatalogSnapshot.CATALOG_SNAPSHOT_KEY) : "commit data missing catalog snapshot"; assert commitData.containsKey(Translog.TRANSLOG_UUID_KEY) : "commit data missing translog UUID"; assert commitData.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) : "commit data missing local checkpoint"; assert commitData.containsKey(SequenceNumbers.MAX_SEQ_NO) : "commit data missing max seq no"; assert commitData.containsKey(Engine.HISTORY_UUID_KEY) : "commit data missing history UUID"; + assert snapshot.getId() >= 0 : "snapshot ID must be non-negative but was: " + snapshot.getId(); + assert Long.parseLong(commitData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) >= -1 + : "local checkpoint in commit data must be >= -1"; + assert Long.parseLong(commitData.get(SequenceNumbers.MAX_SEQ_NO)) >= -1 : "max seq no in commit data must be >= -1"; committer.commit(commitData); snapshotRef.markSuccess(); - translogManager.rollTranslogGeneration(); translogManager.trimUnreferencedReaders(); } } @@ -860,11 +917,18 @@ public void flush() { @Override public boolean shouldPeriodicallyFlush() { ensureOpen(); - final long localCheckpointOfLastCommit = localCheckpointTracker.getPersistedCheckpoint(); - return translogManager.shouldPeriodicallyFlush( - localCheckpointOfLastCommit, - engineConfig.getIndexSettings().getFlushThresholdSize().getBytes() - ); + try { + Map lastCommitData = committer.getLastCommittedData(); + final long localCheckpointOfLastCommit = Long.parseLong( + lastCommitData.getOrDefault(SequenceNumbers.LOCAL_CHECKPOINT_KEY, "-1") + ); + return translogManager.shouldPeriodicallyFlush( + localCheckpointOfLastCommit, + engineConfig.getIndexSettings().getFlushThresholdSize().getBytes() + ); + } catch (IOException e) { + throw new RuntimeException(e); + } } /** Triggers a refresh to flush the indexing buffer to segments. */ @@ -882,7 +946,7 @@ public void forceMerge( boolean upgradeOnlyAncientSegments, String forceMergeUUID ) throws EngineException, IOException { - // TODO: Delegate to IndexingExecutionEngine's Merger when merge scheduling is implemented + mergeScheduler.forceMerge(1); } /** {@inheritDoc} Returns the RAM bytes used by the indexing execution engine. */ @@ -1077,18 +1141,36 @@ public CommitStats commitStats() { @Override public DocsStats docStats() { - // TODO: Derive from catalog snapshot segment metadata or reader. Pending discussion to finalize this. - return new DocsStats(0, 0, 0); + try (GatedCloseable snapshot = acquireSnapshot()) { + long count = snapshot.get() + .getSegments() + .stream() + .flatMap(segment -> segment.dfGroupedSearchableFiles().values().stream()) + .mapToLong(WriterFileSet::numRows) + .sum(); + long totalSize = snapshot.get() + .getSegments() + .stream() + .flatMap(segment -> segment.dfGroupedSearchableFiles().values().stream()) + .mapToLong(WriterFileSet::getTotalSize) + .sum(); + assert count >= 0 : "doc count must be non-negative but was: " + count; + assert totalSize >= 0 : "total size must be non-negative but was: " + totalSize; + return new DocsStats.Builder().deleted(0L).count(count).totalSizeInBytes(totalSize).build(); + } catch (IOException ex) { + throw new OpenSearchException(ex); + } } @Override public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) { + SegmentsStats stats = new SegmentsStats(); throw new UnsupportedOperationException("Unsupported operation"); } @Override public CompletionStats completionStats(String... fieldNamePatterns) { - throw new UnsupportedOperationException("CompletionStats not supported"); + return new CompletionStats(); } @Override @@ -1294,12 +1376,12 @@ public void close() throws IOException { } private void applyMergeChanges(MergeResult mergeResult, OneMerge oneMerge) { + assert mergeResult != null : "merge result must not be null"; + assert oneMerge != null : "oneMerge must not be null"; + assert oneMerge.getSegmentsToMerge().isEmpty() == false : "merged segments list must not be empty"; refreshLock.lock(); try { catalogSnapshotManager.applyMergeResults(mergeResult, oneMerge); - try (GatedCloseable newSnapshotRef = catalogSnapshotManager.acquireSnapshot()) { - refreshListeners(true, newSnapshotRef.get()); - } } catch (Exception ex) { try { logger.error(() -> new ParameterizedMessage("Merge failed while registering merged files in Snapshot"), ex); @@ -1313,14 +1395,6 @@ private void applyMergeChanges(MergeResult mergeResult, OneMerge oneMerge) { } } - private void refreshListeners(boolean refreshed, CatalogSnapshot catalogSnapshot) throws IOException { - // TODO: Add other Refresh listeners - // Notify reader managers so they can create readers for the new snapshot - for (EngineReaderManager rm : readerManagers.values()) { - rm.afterRefresh(refreshed, catalogSnapshot); - } - } - private void triggerPossibleMerges() { if (Booleans.parseBoolean(System.getProperty(MERGE_ENABLED_PROPERTY, Boolean.FALSE.toString())) == false) { logger.debug("Pluggable dataformat merge is disabled via system property [{}], skipping merge", MERGE_ENABLED_PROPERTY); @@ -1334,7 +1408,11 @@ private void closeNoLock(String reason) { assert rwl.isWriteLockedByCurrentThread() || failEngineLock.isHeldByCurrentThread() : "Either the write lock must be held or the engine must be currently failing"; try { - IOUtils.close(indexingExecutionEngine, translogManager); + // Close all writers still in the pool (unflushed writers from the current cycle) + for (var holder : writerPool.checkoutAll()) { + IOUtils.closeWhileHandlingException(holder.get()); + } + IOUtils.close(indexingExecutionEngine, committer, translogManager); closeReaders(); } catch (Exception e) { logger.warn("failed to close engine resources", e); diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/RowIdAwareWriter.java b/server/src/main/java/org/opensearch/index/engine/dataformat/RowIdAwareWriter.java new file mode 100644 index 0000000000000..7e5767c08a13a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/RowIdAwareWriter.java @@ -0,0 +1,93 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A decorator around {@link Writer} that assigns a monotonically increasing row ID + * to each document before delegating to the underlying writer. + * + *

      Row IDs are the cross-format correlation key: when a document is written to + * multiple data formats (e.g., Parquet for columnar storage and Lucene for inverted + * indices), the row ID ensures that the same logical document occupies the same + * position in every format's segment. This 1:1 offset correspondence is critical + * for merge operations that must rewrite row IDs consistently across formats. + * + *

      Each {@code RowIdAwareWriter} instance maintains its own counter starting at 0, + * producing sequential IDs within the scope of a single writer generation. The counter + * is tied to the writer's lifecycle — when the writer is closed and garbage collected, + * the counter is reclaimed with it, avoiding any long-lived map or registry. + * + *

      This decorator is created by {@link org.opensearch.index.engine.DataFormatAwareEngine} + * when it wraps each writer from the {@link IndexingExecutionEngine}. The engine calls + * {@link #addDoc} which sets the row ID on the {@link DocumentInput} and then delegates + * to the underlying writer's {@code addDoc}. + * + * @param

      the document input type accepted by the underlying writer + * @opensearch.experimental + */ +@ExperimentalApi +public class RowIdAwareWriter

      > implements Writer

      { + + private final Writer

      delegate; + private final AtomicLong rowIdCounter; + + /** + * Creates a new row-ID-aware writer wrapping the given delegate. + * + * @param delegate the underlying writer to delegate all operations to + */ + public RowIdAwareWriter(Writer

      delegate) { + this.delegate = delegate; + this.rowIdCounter = new AtomicLong(0); + } + + /** + * Assigns a sequential row ID to the document input, then delegates to the + * underlying writer. The row ID is set via {@link DocumentInput#setRowId} + * using the standard {@link DocumentInput#ROW_ID_FIELD} field name. + * + * @param d the document input to write + * @return the write result from the underlying writer + * @throws IOException if the underlying write fails + */ + @Override + public WriteResult addDoc(P d) throws IOException { + d.setRowId(DocumentInput.ROW_ID_FIELD, rowIdCounter.getAndIncrement()); + return delegate.addDoc(d); + } + + /** {@inheritDoc} Delegates to the underlying writer. */ + @Override + public FileInfos flush() throws IOException { + return delegate.flush(); + } + + /** {@inheritDoc} Delegates to the underlying writer. */ + @Override + public void sync() throws IOException { + delegate.sync(); + } + + /** {@inheritDoc} Returns the generation of the underlying writer. */ + @Override + public long generation() { + return delegate.generation(); + } + + /** {@inheritDoc} Closes the underlying writer. */ + @Override + public void close() throws IOException { + delegate.close(); + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/Writer.java b/server/src/main/java/org/opensearch/index/engine/dataformat/Writer.java index 25e4894f77b54..07a6ea4679f3f 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/Writer.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/Writer.java @@ -9,7 +9,6 @@ package org.opensearch.index.engine.dataformat; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.common.queue.Lockable; import java.io.Closeable; import java.io.IOException; @@ -22,7 +21,7 @@ * @opensearch.experimental */ @ExperimentalApi -public interface Writer

      > extends Closeable, Lockable { +public interface Writer

      > extends Closeable { /** * Adds a document to the writer. diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeHandler.java b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeHandler.java index da91cb77bad42..71902c2ff7be4 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeHandler.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeHandler.java @@ -50,6 +50,7 @@ public class MergeHandler { private final MergeListener mergeListener; private final Merger merger; private final Logger logger; + private final Supplier generationProvider; /** * Creates a new merge handler. @@ -63,13 +64,15 @@ public MergeHandler( Merger merger, ShardId shardId, MergePolicy mergePolicy, - MergeListener mergeListener + MergeListener mergeListener, + Supplier generationProvider ) { this.logger = Loggers.getLogger(getClass(), shardId); this.snapshotSupplier = snapshotSupplier; this.mergePolicy = mergePolicy; this.mergeListener = mergeListener; this.merger = merger; + this.generationProvider = generationProvider; } /** @@ -213,8 +216,14 @@ public synchronized void onMergeFailure(OneMerge oneMerge) { * @throws IOException if the merge operation fails */ public MergeResult doMerge(OneMerge oneMerge) throws IOException { - MergeInput mergeInput = MergeInput.builder().segments(oneMerge.getSegmentsToMerge()).build(); - return merger.merge(mergeInput); + assert oneMerge.getSegmentsToMerge().isEmpty() == false : "merge must have at least one segment"; + long generation = generationProvider.get(); + assert generation > 0 : "merge writer generation must be positive but was: " + generation; + MergeInput mergeInput = MergeInput.builder().segments(oneMerge.getSegmentsToMerge()).newWriterGeneration(generation).build(); + MergeResult result = merger.merge(mergeInput); + assert result != null : "merger must return a non-null MergeResult"; + assert result.getMergedWriterFileSet().isEmpty() == false : "merge result must contain at least one format's files"; + return result; } private synchronized void removeMergingSegments(OneMerge oneMerge) { diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeScheduler.java b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeScheduler.java index 66cf5734b6b45..ececc2919ad42 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeScheduler.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/merge/MergeScheduler.java @@ -234,7 +234,7 @@ private void submitMergeTask(OneMerge oneMerge) { mergeHandler.onMergeFinished(oneMerge); tookMS = TimeValue.nsecToMSec((System.nanoTime() - timeNS)); - logger.info("Merge completed in {}ms", tookMS); + logger.info("Merge {} completed in {}ms", oneMerge, tookMS); } catch (Exception e) { logger.error(new ParameterizedMessage("Unexpected error during merge for: {}", oneMerge), e); diff --git a/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshot.java b/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshot.java index 309579cea1650..75de94853c279 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshot.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshot.java @@ -64,6 +64,12 @@ public abstract class CatalogSnapshot implements Writeable, Cloneable { */ private volatile Map> filesByFormatCache; + /** + * Whether this snapshot has been committed (persisted via flush). + * Package-private — managed by {@link IndexFileDeleter}. + */ + private volatile boolean committed; + protected CatalogSnapshot(String name, long generation, long version) { this.generation = generation; this.version = version; @@ -106,6 +112,22 @@ public long getVersion() { return version; } + /** + * Marks this snapshot as committed (persisted via flush). + * Package-private — only called by {@link IndexFileDeleter} and {@link CatalogSnapshotManager}. + */ + void markCommitted() { + this.committed = true; + } + + /** + * Returns whether this snapshot was committed. + * Package-private — only called by {@link IndexFileDeleter} and {@link CatalogSnapshotManager}. + */ + boolean isCommitted() { + return committed; + } + // Package-private ref counting — only accessible within exec.coord (i.e., CatalogSnapshotManager) /** diff --git a/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java b/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java index e6a0e54247443..bbc8e7ec0bb25 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java @@ -86,7 +86,7 @@ public static CatalogSnapshot createInitialSnapshot( * * @param committedSnapshots the committed snapshots, ordered oldest first; must not be empty * @param deletionPolicy decides which committed snapshots to keep - * @param fileDeleters per-format deleters for actual file deletion + * @param fileDeleter per-format deleters for actual file deletion * @param filesListeners per-format listeners notified on file add/delete * @param snapshotListeners listeners notified on snapshot deletion * @param shardPath for orphan cleanup on init, or null if not needed @@ -95,7 +95,7 @@ public static CatalogSnapshot createInitialSnapshot( public CatalogSnapshotManager( List committedSnapshots, CatalogSnapshotDeletionPolicy deletionPolicy, - Map fileDeleters, + FileDeleter fileDeleter, Map filesListeners, List snapshotListeners, ShardPath shardPath, @@ -112,7 +112,7 @@ public CatalogSnapshotManager( } this.indexFileDeleter = new IndexFileDeleter( deletionPolicy, - fileDeleters, + fileDeleter, filesListeners, committedSnapshots, shardPath, @@ -135,6 +135,20 @@ public synchronized void applyMergeResults(MergeResult mergeResult, OneMerge one Segment segmentToAdd = getSegment(mergeResult.getMergedWriterFileSet()); Set segmentsToRemove = new HashSet<>(oneMerge.getSegmentsToMerge()); + // All source segments must exist in the current snapshot + assert segmentList.containsAll(segmentsToRemove) : "merge source segments must all exist in the current catalog snapshot"; + + // Merged segment generation must not collide with any segment that will be retained + assert segmentList.stream() + .filter(s -> segmentsToRemove.contains(s) == false) + .noneMatch(s -> s.generation() == segmentToAdd.generation()) : "merged segment generation [" + + segmentToAdd.generation() + + "] collides with a retained segment generation"; + + // Row count conservation: merged output must have the same total rows as the inputs + assert assertRowCountConservation(segmentsToRemove, segmentToAdd) + : "merged segment row count must equal sum of source segment row counts"; + boolean inserted = false; int newSegIdx = 0; for (int segIdx = 0, cnt = segmentList.size(); segIdx < cnt; segIdx++) { @@ -178,7 +192,7 @@ public synchronized void applyMergeResults(MergeResult mergeResult, OneMerge one * * @param refreshedSegments the segments produced by the latest refresh */ - public synchronized void commitNewSnapshot(List refreshedSegments) { + public synchronized void commitNewSnapshot(List refreshedSegments) throws IOException { if (closed.get()) { throw new IllegalStateException("CatalogSnapshotManager is closed"); } @@ -187,14 +201,32 @@ public synchronized void commitNewSnapshot(List refreshedSegments) { // that readers and the commit path depend on long prevGen = latestCatalogSnapshot.getGeneration(); - DataformatAwareCatalogSnapshot newSnapshot = new DataformatAwareCatalogSnapshot( - latestCatalogSnapshot.getId() + 1, - latestCatalogSnapshot.getGeneration() + 1, - latestCatalogSnapshot.getVersion(), - refreshedSegments, - latestCatalogSnapshot.getLastWriterGeneration() + 1, - latestCatalogSnapshot.getUserData() - ); + for (CatalogSnapshotLifecycleListener listener : snapshotListeners) { + listener.beforeRefresh(); + } + + DataformatAwareCatalogSnapshot newSnapshot; + try { + newSnapshot = new DataformatAwareCatalogSnapshot( + latestCatalogSnapshot.getId() + 1, + latestCatalogSnapshot.getGeneration() + 1, + latestCatalogSnapshot.getVersion(), + refreshedSegments, + latestCatalogSnapshot.getLastWriterGeneration() + 1, + latestCatalogSnapshot.getUserData() + ); + } catch (Exception e) { + // Construction failed (e.g., OOM) — notify listeners that the refresh did not produce a new snapshot + // so they can reset any state prepared in beforeRefresh + for (CatalogSnapshotLifecycleListener listener : snapshotListeners) { + try { + listener.afterRefresh(false, null); + } catch (Exception suppressed) { + e.addSuppressed(suppressed); + } + } + throw e; + } // New snapshot generation must be strictly greater than the previous assert newSnapshot.getGeneration() > prevGen : "new snapshot generation [" @@ -209,17 +241,75 @@ public synchronized void commitNewSnapshot(List refreshedSegments) { + latestCatalogSnapshot.getId() + "]"; + // Segment generation uniqueness: a generation that appeared in a previous snapshot + // must not reappear with different files. This prevents generation overlap bugs + // where a merge output reuses a writer generation, causing file identity confusion. + assert assertSegmentGenerationFileConsistency(refreshedSegments) + : "segment generation-to-file mapping is inconsistent with previous snapshots"; + + // No duplicate generations within the same snapshot + assert refreshedSegments.stream().map(Segment::generation).distinct().count() == refreshedSegments.size() + : "refreshed segments contain duplicate generations"; + + // Every segment must have at least one format with files + assert refreshedSegments.stream().allMatch(s -> s.dfGroupedSearchableFiles().isEmpty() == false) + : "every segment must have at least one format's files"; + + // Every WriterFileSet in every segment must have a positive row count + assert refreshedSegments.stream().flatMap(s -> s.dfGroupedSearchableFiles().values().stream()).allMatch(wfs -> wfs.numRows() > 0) + : "every WriterFileSet must have a positive row count"; + + // Register file references BEFORE notifying listeners and swapping the snapshot. + // This ensures that if addFileReferences fails, no listener has been told about + // the new snapshot and no state has been mutated. try { indexFileDeleter.addFileReferences(newSnapshot); } catch (IOException e) { + // File reference registration failed — notify listeners that refresh did not complete + for (CatalogSnapshotLifecycleListener listener : snapshotListeners) { + try { + listener.afterRefresh(false, null); + } catch (Exception suppressed) { + e.addSuppressed(suppressed); + } + } throw new RuntimeException("Failed to add file references for snapshot [gen=" + newSnapshot.getGeneration() + "]", e); } + + // Now notify listeners — file references are already registered, so even if a listener + // fails, the files are tracked and will be cleaned up when the snapshot is deleted. + List notified = new ArrayList<>(); + try { + for (CatalogSnapshotLifecycleListener listener : snapshotListeners) { + listener.afterRefresh(true, newSnapshot); + notified.add(listener); + } + } catch (Exception ex) { + // A listener failed after file references were registered. The snapshot is tracked + // by the file deleter but was never made visible as latestCatalogSnapshot. + // Notify already-notified listeners that the snapshot is being discarded. + for (CatalogSnapshotLifecycleListener listener : notified) { + try { + listener.onDeleted(newSnapshot); + } catch (Exception suppressed) { + ex.addSuppressed(suppressed); + } + } + // Remove file references since the snapshot will never be used + try { + indexFileDeleter.removeFileReferences(newSnapshot); + } catch (IOException suppressed) { + ex.addSuppressed(suppressed); + } + throw ex; + } + catalogSnapshotMap.put(newSnapshot.getGeneration(), newSnapshot); CatalogSnapshot oldSnapshot = latestCatalogSnapshot; latestCatalogSnapshot = newSnapshot; - logger.trace("New Catalog Snapshot created: {}", latestCatalogSnapshot); + logger.debug("New Catalog Snapshot created: {}", latestCatalogSnapshot); // Release the manager's own reference to the old snapshot. // The snapshot won't be deleted if the commit path still holds a reference. @@ -269,6 +359,7 @@ public GatedConditionalCloseable acquireSnapshotForCommit() { } return new GatedConditionalCloseable<>(snapshot, () -> { try { + snapshot.markCommitted(); indexFileDeleter.onCommit(snapshot); } catch (IOException e) { throw new RuntimeException("Failed to register commit [gen=" + snapshot.getGeneration() + "]", e); @@ -304,18 +395,26 @@ private void decRefAndMaybeDelete(CatalogSnapshot snapshot) { final long gen = snapshot.getGeneration(); if (snapshot.decRef()) { catalogSnapshotMap.remove(gen); + Exception firstException = null; try { indexFileDeleter.removeFileReferences(snapshot); } catch (IOException e) { - throw new RuntimeException("Failed to clean up files for snapshot [gen=" + gen + "]", e); + firstException = e; } for (CatalogSnapshotLifecycleListener listener : snapshotListeners) { try { listener.onDeleted(snapshot); } catch (IOException e) { - throw new RuntimeException("Listener failed on snapshot deletion [gen=" + gen + "]", e); + if (firstException == null) { + firstException = e; + } else { + firstException.addSuppressed(e); + } } } + if (firstException != null) { + throw new RuntimeException("Failed to clean up snapshot [gen=" + gen + "]", firstException); + } } } @@ -345,4 +444,60 @@ private Segment getSegment(Map writerFileSetMap) { public void close() { closed.compareAndSet(false, true); } + + /** + * Asserts that no segment generation in the new snapshot conflicts with a different + * file set in any existing tracked snapshot. This catches generation overlap bugs + * where a merge or writer reuses a generation number, causing the catalog to track + * two different file sets under the same generation — which would lead to data loss + * when the "wrong" files are deleted. + */ + private boolean assertSegmentGenerationFileConsistency(List newSegments) { + for (Segment newSeg : newSegments) { + for (CatalogSnapshot existing : catalogSnapshotMap.values()) { + for (Segment existingSeg : existing.getSegments()) { + if (existingSeg.generation() == newSeg.generation()) { + // Same generation — files must be identical per format + for (Map.Entry entry : newSeg.dfGroupedSearchableFiles().entrySet()) { + WriterFileSet existingWfs = existingSeg.dfGroupedSearchableFiles().get(entry.getKey()); + if (existingWfs != null && existingWfs.files().equals(entry.getValue().files()) == false) { + logger.error( + "Generation {} has conflicting files for format [{}]: existing={}, new={}", + newSeg.generation(), + entry.getKey(), + existingWfs.files(), + entry.getValue().files() + ); + return false; + } + } + } + } + } + } + return true; + } + + /** + * Asserts that the total row count across all formats in the merged segment equals + * the total row count across all formats in the source segments. This catches bugs + * where rows are silently dropped or duplicated during merge. + */ + private boolean assertRowCountConservation(Set sourceSegments, Segment mergedSegment) { + long sourceRows = 0; + for (Segment seg : sourceSegments) { + for (WriterFileSet wfs : seg.dfGroupedSearchableFiles().values()) { + sourceRows += wfs.numRows(); + } + } + long mergedRows = 0; + for (WriterFileSet wfs : mergedSegment.dfGroupedSearchableFiles().values()) { + mergedRows += wfs.numRows(); + } + if (sourceRows != mergedRows) { + logger.error("Row count mismatch: source segments have {} rows but merged segment has {} rows", sourceRows, mergedRows); + return false; + } + return true; + } } diff --git a/server/src/main/java/org/opensearch/index/engine/exec/coord/DataformatAwareCatalogSnapshot.java b/server/src/main/java/org/opensearch/index/engine/exec/coord/DataformatAwareCatalogSnapshot.java index 44fcfb7c77449..9330e4d2b1c96 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/coord/DataformatAwareCatalogSnapshot.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/coord/DataformatAwareCatalogSnapshot.java @@ -23,6 +23,7 @@ import java.util.Base64; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -177,6 +178,8 @@ public static DataformatAwareCatalogSnapshot deserializeFromString(String serial @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); + Map userData = new HashMap<>(this.userData); + userData.remove(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY); out.writeMap(userData, StreamOutput::writeString, StreamOutput::writeString); out.writeLong(id); out.writeLong(lastWriterGeneration); diff --git a/server/src/main/java/org/opensearch/index/engine/exec/coord/IndexFileDeleter.java b/server/src/main/java/org/opensearch/index/engine/exec/coord/IndexFileDeleter.java index 9c3c814090301..802be74a0845e 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/coord/IndexFileDeleter.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/coord/IndexFileDeleter.java @@ -17,6 +17,7 @@ import org.opensearch.index.engine.exec.FileDeleter; import org.opensearch.index.engine.exec.FilesListener; import org.opensearch.index.shard.ShardPath; +import org.opensearch.secure_sm.AccessController; import java.io.IOException; import java.util.ArrayList; @@ -55,7 +56,7 @@ public class IndexFileDeleter { private final Map> fileRefCounts; private final CatalogSnapshotDeletionPolicy deletionPolicy; - private final Map fileDeleters; + private final FileDeleter fileDeleter; private final Map filesListeners; private final List committedSnapshots; private final CommitFileManager commitFileManager; @@ -69,14 +70,14 @@ public class IndexFileDeleter { public IndexFileDeleter( CatalogSnapshotDeletionPolicy deletionPolicy, - Map fileDeleters, + FileDeleter fileDeleter, Map filesListeners, List initialCommittedSnapshots, ShardPath shardPath, CommitFileManager commitFileManager ) throws IOException { this.deletionPolicy = deletionPolicy; - this.fileDeleters = fileDeleters; + this.fileDeleter = fileDeleter; this.filesListeners = filesListeners; this.fileRefCounts = new HashMap<>(); this.committedSnapshots = new ArrayList<>(); @@ -87,6 +88,7 @@ public IndexFileDeleter( if (cs.tryIncRef() == false) { throw new IllegalStateException("Committed snapshot [gen=" + cs.getGeneration() + "] is already closed"); } + cs.markCommitted(); this.committedSnapshots.add(cs); addFileReferences(cs); } @@ -156,7 +158,7 @@ public void removeFileReferences(CatalogSnapshot snapshot) throws IOException { // Delete the commit point (segments_N) BEFORE deleting data files, // because deleteCommit may call DirectoryReader.listCommits() which // needs to read segment files that are about to be deleted. - if (commitFileManager != null) { + if (commitFileManager != null && snapshot.isCommitted()) { commitFileManager.deleteCommit(snapshot); } if (filesToDelete.isEmpty() == false) { @@ -235,10 +237,6 @@ public void retryPendingDeletes() throws IOException { for (Map.Entry> entry : snapshot.entrySet()) { String formatName = entry.getKey(); Set files = entry.getValue(); - FileDeleter deleter = fileDeleters.get(formatName); - if (deleter == null) { - continue; - } Set stillFailed = new HashSet<>(); for (String file : files) { // Assert: a file in pendingDeletes must not be re-referenced @@ -252,7 +250,9 @@ public void retryPendingDeletes() throws IOException { + " This should never happen — once a segment file's ref count reaches 0, no new snapshot should reference it."; } try { - Map> failed = deleter.deleteFiles(Map.of(formatName, List.of(file))); + Map> failed = AccessController.doPrivilegedChecked( + () -> fileDeleter.deleteFiles(Map.of(formatName, List.of(file))) + ); if (failed.getOrDefault(formatName, Set.of()).contains(file)) { stillFailed.add(file); } else { @@ -328,10 +328,11 @@ private void executeDeletesWithRetry(Map> filesByForm for (Map.Entry> entry : safeToDelete.entrySet()) { String formatName = entry.getKey(); Collection files = entry.getValue(); - FileDeleter deleter = fileDeleters.get(formatName); - if (deleter != null) { + if (fileDeleter != null) { try { - Map> failed = deleter.deleteFiles(Map.of(formatName, files)); + Map> failed = AccessController.doPrivilegedChecked( + () -> fileDeleter.deleteFiles(Map.of(formatName, files)) + ); Collection failedForFormat = failed.getOrDefault(formatName, Set.of()); if (failedForFormat.isEmpty() == false) { synchronized (this) { diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 74b4b9f6d18be..522afff73ccdd 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4448,7 +4448,7 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro // After each internal refresh, update the LuceneFieldTracker with merged FieldInfos from // the reader. This lets DocumentParser enforce the per-shard Lucene field-count limit for // dynamic_properties without requiring access to the IndexWriter directly. - if (mapperService != null) { + if (mapperService != null && indexSettings.isPluggableDataFormatEnabled() == false) { internalRefreshListener.add(new ReferenceManager.RefreshListener() { @Override public void beforeRefresh() {} diff --git a/server/src/test/java/org/opensearch/index/engine/DataFormatAwareEngineTests.java b/server/src/test/java/org/opensearch/index/engine/DataFormatAwareEngineTests.java index a26646e5a3288..249665494ed9d 100644 --- a/server/src/test/java/org/opensearch/index/engine/DataFormatAwareEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/DataFormatAwareEngineTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.Term; +import org.apache.lucene.search.ReferenceManager; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.opensearch.Version; @@ -31,6 +32,7 @@ import org.opensearch.index.engine.dataformat.stub.InMemoryCommitter; import org.opensearch.index.engine.dataformat.stub.MockDataFormat; import org.opensearch.index.engine.dataformat.stub.MockDataFormatPlugin; +import org.opensearch.index.engine.dataformat.stub.MockDocumentInput; import org.opensearch.index.engine.dataformat.stub.MockSearchBackEndPlugin; import org.opensearch.index.engine.exec.IndexReaderProvider; import org.opensearch.index.engine.exec.WriterFileSet; @@ -38,6 +40,7 @@ import org.opensearch.index.engine.exec.coord.CatalogSnapshot; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.ParsedDocument; +import org.opensearch.index.mapper.SeqNoFieldMapper; import org.opensearch.index.mapper.Uid; import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.seqno.SequenceNumbers; @@ -56,6 +59,7 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -168,6 +172,15 @@ private DataFormatAwareEngine createDFAEngine(Store store, Path translogPath) th } private EngineConfig buildDFAEngineConfig(Store store, Path translogPath) { + return buildDFAEngineConfig(store, translogPath, List.of(), List.of()); + } + + private EngineConfig buildDFAEngineConfig( + Store store, + Path translogPath, + List externalListeners, + List internalListeners + ) { IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( "test", Settings.builder() @@ -197,8 +210,8 @@ private EngineConfig buildDFAEngineConfig(Store store, Path translogPath) { .mergePolicy(NoMergePolicy.INSTANCE) .translogConfig(translogConfig) .flushMergesAfter(TimeValue.timeValueMinutes(5)) - .externalRefreshListener(List.of()) - .internalRefreshListener(List.of()) + .externalRefreshListener(externalListeners) + .internalRefreshListener(internalListeners) .globalCheckpointSupplier(() -> SequenceNumbers.NO_OPS_PERFORMED) .retentionLeasesSupplier(() -> RetentionLeases.EMPTY) .primaryTermSupplier(primaryTerm::get) @@ -234,20 +247,22 @@ private Engine.Index indexOp(ParsedDocument doc) { ); } - private Engine.Index replicaIndexOp(ParsedDocument doc, long seqNo) { - return new Engine.Index( - new Term(IdFieldMapper.NAME, Uid.encodeId(doc.id())), - doc, - seqNo, - primaryTerm.get(), - Versions.MATCH_ANY, + /** + * Creates a ParsedDocument with a MockDocumentInput attached, which is required + * by DataFormatAwareEngine.indexIntoEngine for updateField calls. + */ + private ParsedDocument createParsedDocWithInput(String id, String routing) { + ParsedDocument base = createParsedDoc(id, routing); + return new ParsedDocument( + base.version(), + SeqNoFieldMapper.SequenceIDFields.emptySeqID(), + base.id(), + base.routing(), + base.docs(), + base.source(), + base.getMediaType(), null, - Engine.Operation.Origin.REPLICA, - System.nanoTime(), - -1, - false, - SequenceNumbers.UNASSIGNED_SEQ_NO, - 0 + new MockDocumentInput() ); } @@ -255,7 +270,7 @@ public void testSequenceNumbersAssignedOnPrimary() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(5, 20); for (int i = 0; i < numDocs; i++) { - ParsedDocument doc = createParsedDoc(Integer.toString(i), null); + ParsedDocument doc = createParsedDocWithInput(Integer.toString(i), null); Engine.IndexResult result = engine.index(indexOp(doc)); assertThat("seq no should be monotonically increasing", result.getSeqNo(), equalTo((long) i)); } @@ -267,23 +282,11 @@ public void testSequenceNumbersAssignedOnPrimary() throws IOException { } } - public void testSequenceNumbersOnReplica() throws IOException { - try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { - long[] seqNos = { 3, 1, 0, 2 }; - for (long seqNo : seqNos) { - ParsedDocument doc = createParsedDoc(Long.toString(seqNo), null); - Engine.IndexResult result = engine.index(replicaIndexOp(doc, seqNo)); - assertThat("replica should use the provided seq no", result.getSeqNo(), equalTo(seqNo)); - } - assertThat(engine.getProcessedLocalCheckpoint(), equalTo(3L)); - } - } - public void testLocalCheckpointAdvancesCorrectly() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(5, 15); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); assertThat(engine.getProcessedLocalCheckpoint(), equalTo((long) i)); } } @@ -293,7 +296,7 @@ public void testIndexOperationsWrittenToTranslog() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(3, 10); for (int i = 0; i < numDocs; i++) { - Engine.IndexResult result = engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + Engine.IndexResult result = engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); assertThat("translog location should be set", result.getTranslogLocation(), notNullValue()); } assertThat(engine.translogManager().getTranslogStats().estimatedNumberOfOperations(), equalTo(numDocs)); @@ -304,7 +307,7 @@ public void testTranslogSyncPersistsCheckpoint() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(3, 10); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } // Before sync, persisted checkpoint may lag @@ -326,7 +329,7 @@ public void testFlushTrimsTranslog() throws IOException { engine.translogManager().recoverFromTranslog(ignore -> 0, engine.getProcessedLocalCheckpoint(), Long.MAX_VALUE); int numDocs = randomIntBetween(3, 10); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } assertThat(engine.translogManager().getTranslogStats().estimatedNumberOfOperations(), equalTo(numDocs)); @@ -344,7 +347,7 @@ public void testRefreshProducesCatalogSnapshot() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(1, 5); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } engine.refresh("test"); @@ -377,7 +380,7 @@ public void testRefreshAdvancesSnapshotGeneration() throws IOException { assertThat(ref.get().getSegments().size(), equalTo(0)); } - engine.index(indexOp(createParsedDoc("1", null))); + engine.index(indexOp(createParsedDocWithInput("1", null))); engine.refresh("first"); try (GatedCloseable ref = engine.acquireSnapshot()) { @@ -385,7 +388,7 @@ public void testRefreshAdvancesSnapshotGeneration() throws IOException { assertThat(ref.get().getSegments().size(), equalTo(1)); } - engine.index(indexOp(createParsedDoc("2", null))); + engine.index(indexOp(createParsedDocWithInput("2", null))); engine.refresh("second"); try (GatedCloseable ref = engine.acquireSnapshot()) { @@ -403,7 +406,7 @@ public void testRefreshUpdatesLastRefreshedCheckpoint() throws IOException { int numDocs = randomIntBetween(3, 10); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } // Before refresh, last refreshed checkpoint hasn't advanced @@ -421,7 +424,7 @@ public void testMultipleRefreshesAccumulateSegments() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numBatches = randomIntBetween(3, 6); for (int batch = 0; batch < numBatches; batch++) { - engine.index(indexOp(createParsedDoc(Integer.toString(batch), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(batch), null))); engine.refresh("batch-" + batch); } @@ -451,7 +454,7 @@ public void testFlushCommitsCatalogSnapshot() throws IOException { engine.translogManager().recoverFromTranslog(ignore -> 0, engine.getProcessedLocalCheckpoint(), Long.MAX_VALUE); int numDocs = randomIntBetween(1, 5); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } engine.flush(false, true); @@ -495,7 +498,7 @@ public void testConcurrentIndexing() throws Exception { try { barrier.await(); for (int d = 0; d < docsPerThread; d++) { - ParsedDocument doc = createParsedDoc(threadId + "_" + d, null); + ParsedDocument doc = createParsedDocWithInput(threadId + "_" + d, null); Engine.IndexResult result = engine.index(indexOp(doc)); assertThat(result.getSeqNo(), greaterThanOrEqualTo(0L)); maxSeqNo.accumulateAndGet(result.getSeqNo(), Math::max); @@ -533,7 +536,7 @@ public void testConcurrentIndexAndRefresh() throws Exception { indexThreads[t] = new Thread(() -> { try { for (int d = 0; d < docsPerThread; d++) { - engine.index(indexOp(createParsedDoc(threadId + "_" + d, null))); + engine.index(indexOp(createParsedDocWithInput(threadId + "_" + d, null))); } } catch (Exception e) { failures.incrementAndGet(); @@ -568,7 +571,7 @@ public void testConcurrentRefreshAndFlush() throws Exception { engine.translogManager().recoverFromTranslog(ignore -> 0, engine.getProcessedLocalCheckpoint(), Long.MAX_VALUE); int numDocs = randomIntBetween(5, 15); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } AtomicInteger failures = new AtomicInteger(0); @@ -612,16 +615,16 @@ public void testConcurrentRefreshAndFlush() throws Exception { public void testCloseEngine() throws IOException { DataFormatAwareEngine engine = createDFAEngine(store, createTempDir()); - engine.index(indexOp(createParsedDoc("1", null))); + engine.index(indexOp(createParsedDocWithInput("1", null))); engine.close(); // Verify engine is closed by checking that operations throw - expectThrows(AlreadyClosedException.class, () -> engine.index(indexOp(createParsedDoc("2", null)))); + expectThrows(AlreadyClosedException.class, () -> engine.index(indexOp(createParsedDocWithInput("2", null)))); } public void testOperationsAfterCloseThrow() throws IOException { DataFormatAwareEngine engine = createDFAEngine(store, createTempDir()); engine.close(); - expectThrows(AlreadyClosedException.class, () -> engine.index(indexOp(createParsedDoc("1", null)))); + expectThrows(AlreadyClosedException.class, () -> engine.index(indexOp(createParsedDocWithInput("1", null)))); } public void testFlushAndClose() throws IOException { @@ -629,11 +632,11 @@ public void testFlushAndClose() throws IOException { engine.translogManager().recoverFromTranslog(ignore -> 0, engine.getProcessedLocalCheckpoint(), Long.MAX_VALUE); int numDocs = randomIntBetween(3, 10); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } engine.flushAndClose(); // Verify closed - expectThrows(AlreadyClosedException.class, () -> engine.index(indexOp(createParsedDoc("99", null)))); + expectThrows(AlreadyClosedException.class, () -> engine.index(indexOp(createParsedDocWithInput("99", null)))); } public void testRefreshAfterCloseThrows() throws IOException { @@ -663,13 +666,13 @@ public void testAcquireSnapshotReturnsValidSnapshot() throws IOException { public void testSnapshotSurvivesRefreshWhileHeld() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { - engine.index(indexOp(createParsedDoc("1", null))); + engine.index(indexOp(createParsedDocWithInput("1", null))); engine.refresh("first"); GatedCloseable ref = engine.acquireSnapshot(); long heldGen = ref.get().getGeneration(); - engine.index(indexOp(createParsedDoc("2", null))); + engine.index(indexOp(createParsedDocWithInput("2", null))); engine.refresh("second"); // Held snapshot should still be valid @@ -723,7 +726,7 @@ public void testIndexRefreshFlushEndToEnd() throws IOException { // Phase 1: Index for (int i = 0; i < numDocs; i++) { - Engine.IndexResult result = engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + Engine.IndexResult result = engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); assertThat(result.getResultType(), equalTo(Engine.Result.Type.SUCCESS)); assertThat(result.getSeqNo(), equalTo((long) i)); assertThat(result.getTranslogLocation(), notNullValue()); @@ -765,7 +768,7 @@ public void testConcurrentIndexRefreshFlushEndToEnd() throws Exception { // Index all docs first for (int i = 0; i < totalDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } assertThat(engine.getProcessedLocalCheckpoint(), equalTo((long) totalDocs - 1)); @@ -813,11 +816,11 @@ public void testConcurrentIndexRefreshFlushEndToEnd() throws Exception { public void testFailEnginePreventsSubsequentOps() throws IOException { DataFormatAwareEngine engine = createDFAEngine(store, createTempDir()); - engine.index(indexOp(createParsedDoc("1", null))); + engine.index(indexOp(createParsedDocWithInput("1", null))); engine.failEngine("test failure", new RuntimeException("simulated")); - expectThrows(AlreadyClosedException.class, () -> engine.index(indexOp(createParsedDoc("2", null)))); + expectThrows(AlreadyClosedException.class, () -> engine.index(indexOp(createParsedDocWithInput("2", null)))); expectThrows(AlreadyClosedException.class, () -> engine.refresh("after-fail")); expectThrows(AlreadyClosedException.class, () -> engine.flush(false, true)); } @@ -834,7 +837,7 @@ public void testCatalogSnapshotContainsFormatSpecificFiles() throws IOException try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(1, 5); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } engine.refresh("test"); @@ -867,7 +870,7 @@ public void testCatalogSnapshotContainsFormatSpecificFiles() throws IOException public void testCommitDataContainsRequiredMetadataKeys() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { engine.translogManager().recoverFromTranslog(ignore -> 0, engine.getProcessedLocalCheckpoint(), Long.MAX_VALUE); - engine.index(indexOp(createParsedDoc("1", null))); + engine.index(indexOp(createParsedDocWithInput("1", null))); engine.flush(false, true); // The InMemoryCommitter stores the commit data. Access it via the engine's @@ -882,7 +885,7 @@ public void testCommitDataContainsRequiredMetadataKeys() throws IOException { public void testFlushCommitDataContainsCatalogSnapshotKeys() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { engine.translogManager().recoverFromTranslog(ignore -> 0, engine.getProcessedLocalCheckpoint(), Long.MAX_VALUE); - engine.index(indexOp(createParsedDoc("1", null))); + engine.index(indexOp(createParsedDocWithInput("1", null))); engine.flush(false, true); // After flush, the catalog snapshot should be non-empty and have valid generation @@ -898,7 +901,7 @@ public void testAcquireReaderReturnsValidReader() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(1, 5); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } engine.refresh("test"); @@ -914,7 +917,7 @@ public void testAcquireReaderReturnsValidReader() throws IOException { public void testAcquireReaderContainsFormatSpecificReader() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { - engine.index(indexOp(createParsedDoc("1", null))); + engine.index(indexOp(createParsedDocWithInput("1", null))); engine.refresh("test"); try (GatedCloseable readerRef = engine.acquireReader()) { @@ -929,7 +932,7 @@ public void testAcquireReaderContainsFormatSpecificReader() throws IOException { public void testAcquireReaderReturnsNullForUnregisteredFormat() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { - engine.index(indexOp(createParsedDoc("1", null))); + engine.index(indexOp(createParsedDocWithInput("1", null))); engine.refresh("test"); try (GatedCloseable readerRef = engine.acquireReader()) { @@ -957,10 +960,10 @@ public void testAcquireReaderBeforeRefreshReturnsEmptyReaders() throws IOExcepti public void testAcquireReaderSnapshotMatchesLatestRefresh() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { // Index and refresh twice - engine.index(indexOp(createParsedDoc("1", null))); + engine.index(indexOp(createParsedDocWithInput("1", null))); engine.refresh("first"); - engine.index(indexOp(createParsedDoc("2", null))); + engine.index(indexOp(createParsedDocWithInput("2", null))); engine.refresh("second"); long latestGen; @@ -980,7 +983,7 @@ public void testAcquireReaderSnapshotMatchesLatestRefresh() throws IOException { public void testAcquireReaderClosingReleasesSnapshotRef() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { - engine.index(indexOp(createParsedDoc("1", null))); + engine.index(indexOp(createParsedDocWithInput("1", null))); engine.refresh("test"); // Acquire and close a reader, then verify the engine still works @@ -991,7 +994,7 @@ public void testAcquireReaderClosingReleasesSnapshotRef() throws IOException { // After closing, we should still be able to acquire new readers // and do more work - engine.index(indexOp(createParsedDoc("2", null))); + engine.index(indexOp(createParsedDocWithInput("2", null))); engine.refresh("after-close"); try (GatedCloseable newReaderRef = engine.acquireReader()) { @@ -1008,7 +1011,7 @@ public void testAcquireReaderAfterMultipleRefreshesSeesAllSegments() throws IOEx try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numBatches = randomIntBetween(3, 6); for (int i = 0; i < numBatches; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); engine.refresh("batch-" + i); } @@ -1033,7 +1036,7 @@ public void testConcurrentAcquireReader() throws Exception { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(5, 15); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } engine.refresh("setup"); @@ -1070,7 +1073,7 @@ public void testNewChangesSnapshotReturnsIndexedOps() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(5, 20); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", 0, numDocs - 1, false, true)) { @@ -1089,7 +1092,7 @@ public void testNewChangesSnapshotRespectsSeqNoRange() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(10, 20); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } // Request only a subset of the range @@ -1122,7 +1125,7 @@ public void testNewChangesSnapshotAfterConcurrentIndexing() throws Exception { try { barrier.await(); for (int d = 0; d < docsPerThread; d++) { - engine.index(indexOp(createParsedDoc(threadId + "_" + d, null))); + engine.index(indexOp(createParsedDocWithInput(threadId + "_" + d, null))); } } catch (Exception e) { failures.incrementAndGet(); @@ -1148,7 +1151,7 @@ public void testCountNumberOfHistoryOperations() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(5, 15); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } int count = engine.countNumberOfHistoryOperations("test", 0, numDocs - 1); @@ -1160,7 +1163,7 @@ public void testCountNumberOfHistoryOperationsSubRange() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = 10; for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } // Count only ops in range [3, 7] @@ -1170,144 +1173,11 @@ public void testCountNumberOfHistoryOperationsSubRange() throws IOException { } } - private Engine.Index translogRecoveryIndexOp(ParsedDocument doc, long seqNo) { - return new Engine.Index( - new Term(IdFieldMapper.NAME, Uid.encodeId(doc.id())), - doc, - seqNo, - primaryTerm.get(), - 1L, - null, - Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY, - System.nanoTime(), - -1, - false, - SequenceNumbers.UNASSIGNED_SEQ_NO, - 0 - ); - } - - public void testTranslogRecoveryOriginSkipsTranslogWrite() throws IOException { - try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { - // Index via translog recovery — should NOT write to translog - Engine.IndexResult result = engine.index(translogRecoveryIndexOp(createParsedDoc("1", null), 0)); - assertThat(result.getSeqNo(), equalTo(0L)); - assertNull("translog location should be null for recovery-origin ops", result.getTranslogLocation()); - - // Translog should have 0 ops since recovery-origin skips the write - assertThat(engine.translogManager().getTranslogStats().estimatedNumberOfOperations(), equalTo(0)); - - // But the checkpoint should still advance - assertThat(engine.getProcessedLocalCheckpoint(), equalTo(0L)); - } - } - - public void testTranslogRecoveryOriginMarksSeqNoAsPersisted() throws IOException { - try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { - engine.index(translogRecoveryIndexOp(createParsedDoc("1", null), 0)); - - // Recovery-origin ops have no translog location, so they're marked as persisted immediately - assertThat(engine.getPersistedLocalCheckpoint(), equalTo(0L)); - } - } - - public void testMixedPrimaryAndRecoveryOriginOps() throws IOException { - try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { - // Primary op — goes to translog - engine.index(indexOp(createParsedDoc("primary_0", null))); - assertThat(engine.translogManager().getTranslogStats().estimatedNumberOfOperations(), equalTo(1)); - - // Recovery op at seq 1 — skips translog - engine.index(translogRecoveryIndexOp(createParsedDoc("recovery_1", null), 1)); - assertThat(engine.translogManager().getTranslogStats().estimatedNumberOfOperations(), equalTo(1)); - - // Another primary op - engine.index(indexOp(createParsedDoc("primary_2", null))); - assertThat(engine.translogManager().getTranslogStats().estimatedNumberOfOperations(), equalTo(2)); - - // All 3 ops should be processed - assertThat(engine.getProcessedLocalCheckpoint(), equalTo(2L)); - - // Refresh and verify catalog snapshot has segments - engine.refresh("test"); - try (GatedCloseable ref = engine.acquireSnapshot()) { - assertThat(ref.get().getSegments().size(), greaterThan(0)); - } - } - } - - public void testCheckpointStallsOnSeqNoGap() throws IOException { - try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { - // Index as replica with a gap: deliver 0, 1, 3 (missing 2) - engine.index(replicaIndexOp(createParsedDoc("0", null), 0)); - engine.index(replicaIndexOp(createParsedDoc("1", null), 1)); - engine.index(replicaIndexOp(createParsedDoc("3", null), 3)); - - // Checkpoint should stall at 1 because seq 2 is missing - assertThat("checkpoint should stall at 1 due to gap at seq 2", engine.getProcessedLocalCheckpoint(), equalTo(1L)); - - // Now fill the gap - engine.index(replicaIndexOp(createParsedDoc("2", null), 2)); - - // Checkpoint should jump to 3 - assertThat("checkpoint should advance to 3 after gap is filled", engine.getProcessedLocalCheckpoint(), equalTo(3L)); - } - } - - public void testSeqNoGapWithConcurrentDelivery() throws Exception { - try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { - int totalOps = randomIntBetween(20, 50); - AtomicInteger failures = new AtomicInteger(0); - - // Create a shuffled array of seq nos to simulate out-of-order delivery - long[] seqNos = new long[totalOps]; - for (int i = 0; i < totalOps; i++) - seqNos[i] = i; - // Fisher-Yates shuffle - for (int i = totalOps - 1; i > 0; i--) { - int j = randomIntBetween(0, i); - long tmp = seqNos[i]; - seqNos[i] = seqNos[j]; - seqNos[j] = tmp; - } - - int numThreads = randomIntBetween(2, 4); - CyclicBarrier barrier = new CyclicBarrier(numThreads); - AtomicInteger nextIdx = new AtomicInteger(0); - - Thread[] threads = new Thread[numThreads]; - for (int t = 0; t < numThreads; t++) { - threads[t] = new Thread(() -> { - try { - barrier.await(); - int idx; - while ((idx = nextIdx.getAndIncrement()) < totalOps) { - long seqNo = seqNos[idx]; - engine.index(replicaIndexOp(createParsedDoc(Long.toString(seqNo), null), seqNo)); - } - } catch (Exception e) { - failures.incrementAndGet(); - } - }); - threads[t].start(); - } - for (Thread t : threads) - t.join(); - - assertThat(failures.get(), equalTo(0)); - assertThat( - "all ops delivered, checkpoint should be totalOps - 1", - engine.getProcessedLocalCheckpoint(), - equalTo((long) totalOps - 1) - ); - } - } - public void testGetSeqNoStats() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(5, 15); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } // Sync translog so persisted checkpoint advances @@ -1334,7 +1204,7 @@ public void testGetSeqNoStatsAfterConcurrentIndexingAndRefresh() throws Exceptio try { barrier.await(); for (int d = 0; d < docsPerThread; d++) { - engine.index(indexOp(createParsedDoc(threadId + "_" + d, null))); + engine.index(indexOp(createParsedDocWithInput(threadId + "_" + d, null))); } } catch (Exception e) { failures.incrementAndGet(); @@ -1360,7 +1230,7 @@ public void testPersistedCheckpointLagsProcessedBeforeSync() throws IOException try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(3, 10); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } long processed = engine.getProcessedLocalCheckpoint(); @@ -1377,7 +1247,7 @@ public void testPersistedCheckpointCatchesUpAfterSync() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(3, 10); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } engine.translogManager().syncTranslog(); @@ -1394,7 +1264,7 @@ public void testPersistedCheckpointAfterConcurrentIndexAndSync() throws Exceptio try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(20, 50); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } // Sync from multiple threads @@ -1427,7 +1297,7 @@ public void testNonWaitingFlushReturnsImmediatelyIfOngoing() throws Exception { engine.translogManager().recoverFromTranslog(ignore -> 0, engine.getProcessedLocalCheckpoint(), Long.MAX_VALUE); int numDocs = randomIntBetween(5, 15); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } // Run multiple non-waiting flushes concurrently — none should throw @@ -1463,7 +1333,7 @@ public void testShouldPeriodicallyFlush() throws IOException { // Index enough docs to potentially trigger periodic flush for (int i = 0; i < 100; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } // After indexing, shouldPeriodicallyFlush may or may not be true // depending on the configured threshold. The key assertion is it doesn't throw. @@ -1475,7 +1345,7 @@ public void testWriteIndexingBufferTriggersRefresh() throws IOException { try (DataFormatAwareEngine engine = createDFAEngine(store, createTempDir())) { int numDocs = randomIntBetween(3, 10); for (int i = 0; i < numDocs; i++) { - engine.index(indexOp(createParsedDoc(Integer.toString(i), null))); + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); } long genBefore; @@ -1509,7 +1379,7 @@ public void testWriteIndexingBufferAfterConcurrentIndexing() throws Exception { try { barrier.await(); for (int d = 0; d < docsPerThread; d++) { - engine.index(indexOp(createParsedDoc(threadId + "_" + d, null))); + engine.index(indexOp(createParsedDocWithInput(threadId + "_" + d, null))); } } catch (Exception e) { failures.incrementAndGet(); @@ -1529,4 +1399,251 @@ public void testWriteIndexingBufferAfterConcurrentIndexing() throws Exception { } } } + + // ═══════════════════════════════════════════════════════════════ + // Refresh Listener Tests — Use-case focused + // ═══════════════════════════════════════════════════════════════ + + /** + * Use case: A search-after-refresh waiter registers a listener to know when + * new data becomes searchable. After indexing + refresh, the listener must be + * notified so it can unblock the waiting search request. + */ + public void testRefreshListenerNotifiedWhenNewDataBecomesSearchable() throws IOException { + Path translogPath = createTempDir(); + String uuid = Translog.createEmptyTranslog(translogPath, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); + bootstrapStoreWithMetadata(store, uuid); + + AtomicInteger beforeCount = new AtomicInteger(0); + AtomicInteger afterCount = new AtomicInteger(0); + AtomicLong afterDidRefreshTrue = new AtomicLong(0); + + ReferenceManager.RefreshListener listener = new ReferenceManager.RefreshListener() { + @Override + public void beforeRefresh() { + beforeCount.incrementAndGet(); + } + + @Override + public void afterRefresh(boolean didRefresh) { + afterCount.incrementAndGet(); + if (didRefresh) { + afterDidRefreshTrue.incrementAndGet(); + } + } + }; + + EngineConfig config = buildDFAEngineConfig(store, translogPath, List.of(listener), List.of()); + try (DataFormatAwareEngine engine = new DataFormatAwareEngine(config)) { + // Index documents — data is buffered but not yet searchable + int numDocs = randomIntBetween(3, 10); + for (int i = 0; i < numDocs; i++) { + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); + } + + // Refresh — makes data searchable, listener must be notified + engine.refresh("test"); + + // The listener must have been called: beforeRefresh once, afterRefresh(true) once + assertThat("beforeRefresh must fire when new segments are produced", beforeCount.get(), equalTo(1)); + assertThat("afterRefresh must fire when new segments are produced", afterCount.get(), equalTo(1)); + assertThat("afterRefresh(didRefresh=true) confirms data is now searchable", afterDidRefreshTrue.get(), equalTo(1L)); + } + } + + /** + * Use case: When no new data has been indexed, a refresh should still notify + * listeners (beforeRefresh is always called) but afterRefresh should indicate + * that no actual refresh occurred (didRefresh=false). This allows waiters to + * distinguish between "new data available" and "nothing changed". + */ + public void testRefreshListenerNotifiedWithDidRefreshFalseWhenNoNewData() throws IOException { + Path translogPath = createTempDir(); + String uuid = Translog.createEmptyTranslog(translogPath, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); + bootstrapStoreWithMetadata(store, uuid); + + AtomicInteger beforeCount = new AtomicInteger(0); + AtomicInteger afterDidRefreshFalse = new AtomicInteger(0); + AtomicInteger afterDidRefreshTrue = new AtomicInteger(0); + + ReferenceManager.RefreshListener listener = new ReferenceManager.RefreshListener() { + @Override + public void beforeRefresh() { + beforeCount.incrementAndGet(); + } + + @Override + public void afterRefresh(boolean didRefresh) { + if (didRefresh) { + afterDidRefreshTrue.incrementAndGet(); + } else { + afterDidRefreshFalse.incrementAndGet(); + } + } + }; + + EngineConfig config = buildDFAEngineConfig(store, translogPath, List.of(listener), List.of()); + try (DataFormatAwareEngine engine = new DataFormatAwareEngine(config)) { + // Refresh with no data — no new segments produced + engine.refresh("empty"); + + // beforeRefresh is always called (listener needs to prepare) + assertThat("beforeRefresh fires even when no data changed", beforeCount.get(), equalTo(1)); + // afterRefresh(false) indicates nothing new became searchable + assertThat("afterRefresh(false) when no new segments", afterDidRefreshFalse.get(), equalTo(1)); + assertThat("afterRefresh(true) should NOT fire", afterDidRefreshTrue.get(), equalTo(0)); + } + } + + /** + * Use case: Multiple index-refresh cycles should produce monotonically advancing + * notifications. A reader manager uses these to know which snapshot generation + * to open. Each afterRefresh(true) must correspond to a new, higher-generation + * catalog snapshot being available. + */ + public void testRefreshListenerSeesMonotonicallyAdvancingSnapshots() throws IOException { + Path translogPath = createTempDir(); + String uuid = Translog.createEmptyTranslog(translogPath, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); + bootstrapStoreWithMetadata(store, uuid); + + List observedGenerations = new ArrayList<>(); + + ReferenceManager.RefreshListener listener = new ReferenceManager.RefreshListener() { + @Override + public void beforeRefresh() {} + + @Override + public void afterRefresh(boolean didRefresh) { + // Not ideal — we can't access the engine from here directly. + // But we track call count and verify externally. + if (didRefresh) { + observedGenerations.add(System.nanoTime()); // monotonic timestamp as proxy + } + } + }; + + EngineConfig config = buildDFAEngineConfig(store, translogPath, List.of(listener), List.of()); + try (DataFormatAwareEngine engine = new DataFormatAwareEngine(config)) { + int numRefreshes = randomIntBetween(3, 6); + for (int i = 0; i < numRefreshes; i++) { + engine.index(indexOp(createParsedDocWithInput(Integer.toString(i), null))); + engine.refresh("cycle-" + i); + } + + // Each refresh with data should have triggered afterRefresh(true) + assertThat("each refresh with data must notify", observedGenerations.size(), equalTo(numRefreshes)); + + // Verify the catalog snapshot generation advanced monotonically + try (GatedCloseable ref = engine.acquireSnapshot()) { + assertThat( + "final snapshot generation must equal number of refreshes", + ref.get().getGeneration(), + equalTo((long) numRefreshes) + ); + } + } + } + + /** + * Use case: Both external listeners (registered by IndexShard for search-after-refresh) + * and internal listeners (registered by the engine for checkpoint tracking) must both + * be invoked. Neither should be skipped. + */ + public void testBothExternalAndInternalListenersInvoked() throws IOException { + Path translogPath = createTempDir(); + String uuid = Translog.createEmptyTranslog(translogPath, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); + bootstrapStoreWithMetadata(store, uuid); + + AtomicInteger externalCalls = new AtomicInteger(0); + AtomicInteger internalCalls = new AtomicInteger(0); + + ReferenceManager.RefreshListener external = new ReferenceManager.RefreshListener() { + @Override + public void beforeRefresh() { + externalCalls.incrementAndGet(); + } + + @Override + public void afterRefresh(boolean didRefresh) { + externalCalls.incrementAndGet(); + } + }; + + ReferenceManager.RefreshListener internal = new ReferenceManager.RefreshListener() { + @Override + public void beforeRefresh() { + internalCalls.incrementAndGet(); + } + + @Override + public void afterRefresh(boolean didRefresh) { + internalCalls.incrementAndGet(); + } + }; + + EngineConfig config = buildDFAEngineConfig(store, translogPath, List.of(external), List.of(internal)); + try (DataFormatAwareEngine engine = new DataFormatAwareEngine(config)) { + engine.index(indexOp(createParsedDocWithInput("1", null))); + engine.refresh("test"); + + // Each listener gets beforeRefresh + afterRefresh = 2 calls + assertThat("external listener must receive both before and after", externalCalls.get(), equalTo(2)); + assertThat("internal listener must receive both before and after", internalCalls.get(), equalTo(2)); + } + } + + /** + * Use case: The ordering contract — beforeRefresh is called BEFORE the catalog + * snapshot is committed (so listeners can prepare), and afterRefresh is called + * AFTER (so listeners can observe the new state). This is critical for reader + * managers that need to open readers on the new snapshot. + */ + public void testBeforeRefreshCalledBeforeSnapshotCommitAndAfterCalledAfter() throws IOException { + Path translogPath = createTempDir(); + String uuid = Translog.createEmptyTranslog(translogPath, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); + bootstrapStoreWithMetadata(store, uuid); + + AtomicLong genSeenInBefore = new AtomicLong(-1); + AtomicLong genSeenInAfter = new AtomicLong(-1); + AtomicReference engineRef = new AtomicReference<>(); + + ReferenceManager.RefreshListener orderingListener = new ReferenceManager.RefreshListener() { + @Override + public void beforeRefresh() { + DataFormatAwareEngine eng = engineRef.get(); + if (eng != null) { + try (GatedCloseable ref = eng.acquireSnapshot()) { + genSeenInBefore.set(ref.get().getGeneration()); + } catch (Exception e) { + // ignore + } + } + } + + @Override + public void afterRefresh(boolean didRefresh) { + DataFormatAwareEngine eng = engineRef.get(); + if (eng != null) { + try (GatedCloseable ref = eng.acquireSnapshot()) { + genSeenInAfter.set(ref.get().getGeneration()); + } catch (Exception e) { + // ignore + } + } + } + }; + + EngineConfig config = buildDFAEngineConfig(store, translogPath, List.of(orderingListener), List.of()); + try (DataFormatAwareEngine engine = new DataFormatAwareEngine(config)) { + engineRef.set(engine); + + engine.index(indexOp(createParsedDocWithInput("1", null))); + engine.refresh("test"); + + // beforeRefresh sees the OLD generation (snapshot not yet committed) + assertThat("beforeRefresh must see pre-commit generation", genSeenInBefore.get(), equalTo(0L)); + // afterRefresh sees the NEW generation (snapshot committed) + assertThat("afterRefresh must see post-commit generation", genSeenInAfter.get(), equalTo(1L)); + } + } } diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java index 69b2b220ec85d..cdc6485c26f40 100644 --- a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java @@ -279,7 +279,7 @@ public void testSearchHoldsSnapshotAliveWhileRefreshDeletesFiles() throws IOExce CatalogSnapshotManager manager = new CatalogSnapshotManager( List.of(CatalogSnapshotManager.createInitialSnapshot(1L, 1L, 0L, rr1.refreshedSegments(), 1L, Map.of())), CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of(), + files -> Map.of(), Map.of(), List.of(), null, @@ -379,7 +379,7 @@ public Set supportedFields() { CatalogSnapshotManager manager = new CatalogSnapshotManager( List.of(CatalogSnapshotManager.createInitialSnapshot(1L, 1L, 0L, List.of(seg), 1L, Map.of())), CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of(), + files -> Map.of(), Map.of(), List.of(), null, diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeTests.java index a701cbb7d48fb..7d71599396bbc 100644 --- a/server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeTests.java +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/merge/MergeTests.java @@ -104,7 +104,7 @@ public void removeMergingSegment(Collection mergingSegments) {} private MergeHandler createNoopHandler(Supplier> snapshotSupplier) { Merger noopMerger = mergeInput -> new MergeResult(Map.of()); - return new MergeHandler(snapshotSupplier, noopMerger, SHARD_ID, NOOP_MERGE_POLICY, NOOP_MERGE_LISTENER); + return new MergeHandler(snapshotSupplier, noopMerger, SHARD_ID, NOOP_MERGE_POLICY, NOOP_MERGE_LISTENER, () -> 1L); } private MergeHandler createHandlerWithRealPolicy(Supplier> snapshotSupplier, Merger merger) { @@ -112,7 +112,7 @@ private MergeHandler createHandlerWithRealPolicy(Supplier 1L); } private static Supplier> snapshotSupplierOf(List segments) { @@ -298,11 +298,24 @@ public void testRegisterMergeRejectsWhenSecondSegmentNotInCatalog() { // ---- MergeHandler doMerge tests ---- public void testDoMergeReturnsResult() throws IOException { - MergeResult expectedResult = new MergeResult(Map.of()); + Path dir = createTempDir(); + MockDataFormat format = new MockDataFormat(); + WriterFileSet inputWfs = new WriterFileSet(dir.toString(), 1L, Set.of("input.dat"), 10); + Segment seg = Segment.builder(1L).addSearchableFiles(format, inputWfs).build(); + + WriterFileSet mergedWfs = new WriterFileSet(dir.toString(), 99L, Set.of("merged.dat"), 10); + MergeResult expectedResult = new MergeResult(Map.of(format, mergedWfs)); Merger merger = mergeInput -> expectedResult; - MergeHandler handler = new MergeHandler(emptySnapshotSupplier(), merger, SHARD_ID, NOOP_MERGE_POLICY, NOOP_MERGE_LISTENER); - MergeResult result = handler.doMerge(new OneMerge(Collections.emptyList())); + MergeHandler handler = new MergeHandler( + snapshotSupplierOf(List.of(seg)), + merger, + SHARD_ID, + NOOP_MERGE_POLICY, + NOOP_MERGE_LISTENER, + () -> 1L + ); + MergeResult result = handler.doMerge(new OneMerge(List.of(seg))); assertSame(expectedResult, result); } @@ -370,7 +383,9 @@ public void testStatsWithAutoThrottleEnabled() { public void testTriggerMergesExecutesMergeThread() throws Exception { List segments = createSegments(15); - MergeResult mergeResult = new MergeResult(Map.of()); + MockDataFormat format = new MockDataFormat(); + WriterFileSet mergedWfs = new WriterFileSet(createTempDir().toString(), 99L, Set.of("merged.dat"), 15); + MergeResult mergeResult = new MergeResult(Map.of(format, mergedWfs)); CountDownLatch latch = new CountDownLatch(1); Merger merger = mergeInput -> { @@ -413,7 +428,9 @@ public void testTriggerMergesHandlesMergeFailure() throws Exception { public void testForceMergeExecutesMerges() throws Exception { List segments = createSegments(3); - MergeResult mergeResult = new MergeResult(Map.of()); + MockDataFormat format = new MockDataFormat(); + WriterFileSet mergedWfs = new WriterFileSet(createTempDir().toString(), 99L, Set.of("merged.dat"), 3); + MergeResult mergeResult = new MergeResult(Map.of(format, mergedWfs)); CountDownLatch latch = new CountDownLatch(1); Merger merger = mergeInput -> mergeResult; diff --git a/server/src/test/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManagerTests.java b/server/src/test/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManagerTests.java index df9641a5b35f1..cf724d8dcf6fd 100644 --- a/server/src/test/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManagerTests.java +++ b/server/src/test/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManagerTests.java @@ -199,7 +199,7 @@ public void testInitialSnapshotRecovery() throws Exception { CatalogSnapshotManager manager = new CatalogSnapshotManager( List.of(new DataformatAwareCatalogSnapshot(id, generation, version, segments, lastWriterGeneration, userData)), CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of(), + files -> Map.of(), Map.of(), List.of(), null, @@ -279,7 +279,7 @@ public void testApplyMergeResultsReplacesSegments() throws Exception { WriterFileSet wfs1 = new WriterFileSet("/tmp/dir", 1L, Set.of("a.cfs"), 100); WriterFileSet wfs2 = new WriterFileSet("/tmp/dir", 2L, Set.of("b.cfs"), 200); WriterFileSet wfs3 = new WriterFileSet("/tmp/dir", 3L, Set.of("c.cfs"), 300); - WriterFileSet mergedWfs = new WriterFileSet("/tmp/dir", 4L, Set.of("merged.cfs"), 500); + WriterFileSet mergedWfs = new WriterFileSet("/tmp/dir", 4L, Set.of("merged.cfs"), 300); Segment seg1 = new Segment(1L, Map.of(format.name(), wfs1)); Segment seg2 = new Segment(2L, Map.of(format.name(), wfs2)); @@ -288,7 +288,7 @@ public void testApplyMergeResultsReplacesSegments() throws Exception { CatalogSnapshotManager manager = new CatalogSnapshotManager( List.of(new DataformatAwareCatalogSnapshot(0, 0, 1, List.of(seg1, seg2, seg3), 0, Map.of())), CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of(), + files -> Map.of(), Map.of(), List.of(), null, @@ -322,13 +322,12 @@ public void testApplyMergeResultsWhenAllMergedSegmentsRemoved() throws Exception Segment seg1 = new Segment(1L, Map.of(format.name(), wfs1)); Segment seg2 = new Segment(2L, Map.of(format.name(), wfs2)); - Segment unrelatedSeg = new Segment(99L, Map.of(format.name(), wfs1)); - // Manager has only unrelatedSeg — the segments being merged are not present + // Manager has seg1 and seg2 — the segments being merged are present CatalogSnapshotManager manager = new CatalogSnapshotManager( - List.of(new DataformatAwareCatalogSnapshot(0, 0, 1, List.of(unrelatedSeg), 0, Map.of())), + List.of(new DataformatAwareCatalogSnapshot(0, 0, 1, List.of(seg1, seg2), 0, Map.of())), CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of(), + files -> Map.of(), Map.of(), List.of(), null, @@ -342,10 +341,11 @@ public void testApplyMergeResultsWhenAllMergedSegmentsRemoved() throws Exception try (GatedCloseable ref = manager.acquireSnapshot()) { List segments = ref.get().getSegments(); - assertEquals(2, segments.size()); - // merged segment inserted at position 0 + // Both source segments replaced by merged segment + assertEquals(1, segments.size()); assertEquals(3L, segments.get(0).generation()); - assertEquals(unrelatedSeg, segments.get(1)); + assertEquals(Set.of("merged.cfs"), segments.get(0).dfGroupedSearchableFiles().get(format.name()).files()); + assertEquals(300, segments.get(0).dfGroupedSearchableFiles().get(format.name()).numRows()); } } finally { manager.close(); @@ -360,7 +360,7 @@ public void testApplyMergeResultsWithEmptyWriterFileSetMapThrows() throws Except CatalogSnapshotManager manager = new CatalogSnapshotManager( List.of(new DataformatAwareCatalogSnapshot(0, 0, 1, List.of(seg1), 0, Map.of())), CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of(), + files -> Map.of(), Map.of(), List.of(), null, @@ -416,7 +416,7 @@ public void testRefreshThenFlushDeletesOldCommitFiles() throws Exception { CatalogSnapshotManager manager = new CatalogSnapshotManager( List.of(new DataformatAwareCatalogSnapshot(1L, 1L, 0L, cs1Segments, 1L, userData)), policy, - Map.of("parquet", tracker), + tracker, Map.of(), List.of(), null, @@ -469,7 +469,7 @@ public void testMergedFilesDeletedAfterCommit() throws Exception { CatalogSnapshotManager manager = new CatalogSnapshotManager( List.of(new DataformatAwareCatalogSnapshot(1L, 1L, 0L, cs1Segments, 1L, commitUserData(100, 100, translogUUID))), policy, - Map.of("parquet", tracker), + tracker, Map.of(), List.of(), null, @@ -524,7 +524,7 @@ public void testSnapshotProtectionPreventsFileDeletion() throws Exception { ) ), policy, - Map.of("parquet", tracker), + tracker, Map.of(), List.of(), null, @@ -590,7 +590,7 @@ public void testReaderHoldsSnapshotAliveAcrossRefreshes() throws Exception { ) ), policy, - Map.of("parquet", tracker), + tracker, Map.of(), List.of(), null, @@ -651,7 +651,7 @@ public void testSharedFilesDeletedOnlyWhenAllRefsGone() throws Exception { ) ), policy, - Map.of("parquet", tracker), + tracker, Map.of(), List.of(), null, @@ -699,7 +699,7 @@ private WriterFileSet randomWriterFileSet(String format) { for (int i = 0; i < fileCount; i++) { files.add(randomAlphaOfLength(6) + "." + randomFrom(extensions)); } - return new WriterFileSet(directory, randomNonNegativeLong(), files, randomIntBetween(0, 10000)); + return new WriterFileSet(directory, randomNonNegativeLong(), files, randomIntBetween(1, 10000)); } private Segment randomSegment() { @@ -736,16 +736,16 @@ private CatalogSnapshotManager createRandomManager() { } private CatalogSnapshotManager createManager(List segments, Map userData) throws IOException { - return createManager(segments, userData, CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, Map.of()); + return createManager(segments, userData, CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, files -> Map.of()); } private CatalogSnapshotManager createManager( List segments, Map userData, CatalogSnapshotDeletionPolicy policy, - Map fileDeleters + FileDeleter fileDeleter ) throws IOException { DataformatAwareCatalogSnapshot snapshot = new DataformatAwareCatalogSnapshot(1L, 1L, 0L, segments, 1L, userData); - return new CatalogSnapshotManager(List.of(snapshot), policy, fileDeleters, Map.of(), List.of(), null, null); + return new CatalogSnapshotManager(List.of(snapshot), policy, fileDeleter, Map.of(), List.of(), null, null); } } diff --git a/server/src/test/java/org/opensearch/index/engine/exec/coord/IndexFileDeleterTests.java b/server/src/test/java/org/opensearch/index/engine/exec/coord/IndexFileDeleterTests.java index 1fd0a2a524f23..fcfb23b8cb417 100644 --- a/server/src/test/java/org/opensearch/index/engine/exec/coord/IndexFileDeleterTests.java +++ b/server/src/test/java/org/opensearch/index/engine/exec/coord/IndexFileDeleterTests.java @@ -79,7 +79,7 @@ public void testAddFileReferencesTracksNewFiles() throws IOException { IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", tracker), + tracker, Map.of(), List.of(cs1), null, @@ -101,7 +101,7 @@ public void testRemoveFileReferencesDeletesOrphanedFiles() throws IOException { IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", tracker), + tracker, Map.of(), List.of(cs1), null, @@ -139,7 +139,7 @@ public void testOnCommitDeletesOldCommitFiles() throws IOException { commitUserData(100, 100, "uuid") ); - IndexFileDeleter deleter = new IndexFileDeleter(policy, Map.of("parquet", tracker), Map.of(), List.of(cs1), null, null); + IndexFileDeleter deleter = new IndexFileDeleter(policy, tracker, Map.of(), List.of(cs1), null, null); // Refresh: cs2 with merged files CatalogSnapshot cs2 = snapshot(2, List.of(segment(2, "parquet", "new_merged.parquet")), commitUserData(200, 200, "uuid")); @@ -174,7 +174,7 @@ public void testOnCommitPreservesSharedFiles() throws IOException { commitUserData(100, 100, "uuid") ); - IndexFileDeleter deleter = new IndexFileDeleter(policy, Map.of("parquet", tracker), Map.of(), List.of(cs1), null, null); + IndexFileDeleter deleter = new IndexFileDeleter(policy, tracker, Map.of(), List.of(cs1), null, null); // cs2 keeps shared.parquet, adds new file CatalogSnapshot cs2 = snapshot( @@ -208,7 +208,7 @@ public void testRevisitPolicyDeletesPreviouslyProtectedCommit() throws IOExcepti ); CatalogSnapshot cs1 = snapshot(1, List.of(segment(0, "parquet", "cs1_file.parquet")), commitUserData(100, 100, "uuid")); - IndexFileDeleter deleter = new IndexFileDeleter(policy, Map.of("parquet", tracker), Map.of(), List.of(cs1), null, null); + IndexFileDeleter deleter = new IndexFileDeleter(policy, tracker, Map.of(), List.of(cs1), null, null); // Hold cs1 via snapshot protection var held = policy.acquireCommittedSnapshot(false); @@ -258,14 +258,17 @@ public void testMultiFormatFileDeletion() throws IOException { ) ); CatalogSnapshot cs1 = snapshot(1, List.of(seg), commitUserData(100, 100, "uuid")); - IndexFileDeleter deleter = new IndexFileDeleter( - policy, - Map.of("parquet", parquetTracker, "lucene", luceneTracker), - Map.of(), - List.of(cs1), - null, - null - ); + IndexFileDeleter deleter = new IndexFileDeleter(policy, files -> { + Map> failed = new java.util.HashMap<>(); + for (Map.Entry> e : files.entrySet()) { + if ("parquet".equals(e.getKey())) { + failed.putAll(parquetTracker.deleteFiles(Map.of(e.getKey(), e.getValue()))); + } else if ("lucene".equals(e.getKey())) { + failed.putAll(luceneTracker.deleteFiles(Map.of(e.getKey(), e.getValue()))); + } + } + return failed; + }, Map.of(), List.of(cs1), null, null); // cs2 has completely different files Segment seg2 = new Segment( @@ -306,7 +309,7 @@ public void testCommitRefKeepsSnapshotAlive() throws IOException { ); CatalogSnapshot cs1 = snapshot(1, List.of(segment(0, "parquet", "cs1.parquet")), commitUserData(100, 100, "uuid")); - IndexFileDeleter deleter = new IndexFileDeleter(policy, Map.of("parquet", tracker), Map.of(), List.of(cs1), null, null); + IndexFileDeleter deleter = new IndexFileDeleter(policy, tracker, Map.of(), List.of(cs1), null, null); // cs1 has refCount=2 (manager + commit from constructor) assertEquals(2, cs1.refCount()); @@ -351,7 +354,7 @@ public void testDeleteOrphanedFilesOnInit() throws IOException { IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", tracker), + tracker, Map.of(), List.of(cs1), shardPath, @@ -376,7 +379,7 @@ public void testDeleteOrphanedFilesSkipsMissingDirectory() throws IOException { IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", tracker), + tracker, Map.of(), List.of(cs1), shardPath, @@ -431,7 +434,7 @@ public void testPartialDeleteFailureTracksPendingDeletes() throws IOException { IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", failingDeleter), + failingDeleter, Map.of(), List.of(cs1), null, @@ -458,7 +461,7 @@ public void testPendingDeletesRetriedOnNextRemoveFileReferences() throws IOExcep IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", deleter1), + deleter1, Map.of(), List.of(cs1), null, @@ -488,7 +491,7 @@ public void testReReferencingPendingDeleteFileThrowsAssertionError() throws IOEx IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", failingDeleter), + failingDeleter, Map.of(), List.of(cs1), null, @@ -519,7 +522,7 @@ public void testRetryPendingDeletesExplicitCall() throws IOException { IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", failTwice), + failTwice, Map.of(), List.of(cs1), null, @@ -545,7 +548,7 @@ public void testPersistentFailureKeepsFilesPending() throws IOException { IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", alwaysFails), + alwaysFails, Map.of(), List.of(cs1), null, @@ -602,21 +605,14 @@ public void testDeleteFilesExecutedOutsideSynchronizedBlock() throws IOException CatalogSnapshot cs1 = snapshot(1, List.of(segment(0, "parquet", "old.parquet")), commitUserData(100, 100, "uuid")); // Create deleter first, then set up the probe - IndexFileDeleter deleter = new IndexFileDeleter( - policy, - Map.of("parquet", new TrackingFileDeleter()), - Map.of(), - List.of(cs1), - null, - null - ); + IndexFileDeleter deleter = new IndexFileDeleter(policy, new TrackingFileDeleter(), Map.of(), List.of(cs1), null, null); // Now create a new deleter with the lock probe, using the deleter instance as the monitor LockProbeDeleter probe = new LockProbeDeleter(deleter); // We need a fresh deleter with the probe. Rebuild. CatalogSnapshot cs1b = snapshot(1, List.of(segment(0, "parquet", "old.parquet")), commitUserData(100, 100, "uuid")); - IndexFileDeleter deleterWithProbe = new IndexFileDeleter(policy, Map.of("parquet", probe), Map.of(), List.of(cs1b), null, null); + IndexFileDeleter deleterWithProbe = new IndexFileDeleter(policy, probe, Map.of(), List.of(cs1b), null, null); CatalogSnapshot cs2 = snapshot(2, List.of(segment(1, "parquet", "new.parquet")), commitUserData(200, 200, "uuid")); deleterWithProbe.addFileReferences(cs2); @@ -638,7 +634,7 @@ public void testRemoveFileReferencesDoesNotHoldLockDuringIO() throws IOException // Placeholder deleter for construction IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", new TrackingFileDeleter()), + new TrackingFileDeleter(), Map.of(), List.of(cs1), null, @@ -650,7 +646,7 @@ public void testRemoveFileReferencesDoesNotHoldLockDuringIO() throws IOException CatalogSnapshot cs1b = snapshot(1, List.of(segment(0, "parquet", "a.parquet")), commitUserData(100, 100, "uuid")); IndexFileDeleter deleterWithProbe = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", probe), + probe, Map.of(), List.of(cs1b), null, @@ -700,7 +696,7 @@ public boolean isCommitManagedFile(String fileName) { IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", tracker), + tracker, Map.of(), List.of(cs1), shardPath, @@ -735,7 +731,7 @@ public void testOrphanScanWithNullCommitFileManagerDeletesEverythingUnreferenced // null commitFileManager — no protection for commit files IndexFileDeleter deleter = new IndexFileDeleter( CatalogSnapshotDeletionPolicy.KEEP_LATEST_ONLY, - Map.of("parquet", tracker), + tracker, Map.of(), List.of(cs1), shardPath, diff --git a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockWriter.java b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockWriter.java index 6ea284eacaf77..a0b9da2a6a09a 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockWriter.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockWriter.java @@ -62,17 +62,6 @@ public long generation() { return writerGeneration; } - @Override - public void lock() {} - - @Override - public boolean tryLock() { - return true; - } - - @Override - public void unlock() {} - @Override public void close() {} } From 1fc408cb1563a41b73d60457cf6882ef93b0b177 Mon Sep 17 00:00:00 2001 From: Aravind Sagar Date: Thu, 7 May 2026 14:53:11 +0530 Subject: [PATCH 069/115] Add IO task cancellation support for DataFusion queries (#21531) Introduce per-query CancellationToken registry (ACTIVE_QUERIES) in api.rs with select!-based cancellation for execute_query and stream_next. QueryStreamHandle auto-deregisters on drop. Signed-off-by: Aravind Sagar --- .../libs/dataformat-native/rust/Cargo.toml | 1 + .../rust/Cargo.toml | 1 + .../rust/src/api.rs | 79 ++++++++++++------- .../rust/src/cancellation.rs | 59 ++++++++++++++ .../rust/src/ffm.rs | 5 ++ .../rust/src/lib.rs | 3 +- ...emory_pool_tracker.rs => query_tracker.rs} | 25 ++++++ .../rust/src/session_context.rs | 2 +- .../be/datafusion/nativelib/NativeBridge.java | 10 +++ 9 files changed, 155 insertions(+), 30 deletions(-) create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/cancellation.rs rename sandbox/plugins/analytics-backend-datafusion/rust/src/{query_memory_pool_tracker.rs => query_tracker.rs} (93%) diff --git a/sandbox/libs/dataformat-native/rust/Cargo.toml b/sandbox/libs/dataformat-native/rust/Cargo.toml index 2413680eb8710..5de1722585d40 100644 --- a/sandbox/libs/dataformat-native/rust/Cargo.toml +++ b/sandbox/libs/dataformat-native/rust/Cargo.toml @@ -33,6 +33,7 @@ datafusion-substrait = "52.1.0" # Async tokio = { version = "1.0", features = ["full"] } +tokio-util = "0.7" futures = "0.3" tokio-stream = "0.1.17" diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml index afa0f1d9ddf1a..07dc0a26ba5f7 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml +++ b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml @@ -29,6 +29,7 @@ prost = { workspace = true } substrait = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true } futures = { workspace = true } tokio-stream = { workspace = true } parking_lot = { workspace = true } diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs index 4197f6355719d..86856ed0c5f08 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs @@ -53,12 +53,13 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::prelude::SessionConfig; use futures::TryStreamExt; +use crate::cancellation; use crate::cross_rt_stream::CrossRtStream; use crate::custom_cache_manager::CustomCacheManager; use crate::local_executor::LocalSession; use crate::memory::{DynamicLimitHandle, DynamicLimitPool}; use crate::partition_stream::PartitionStreamSender; -use crate::query_memory_pool_tracker::QueryTrackingContext; +use crate::query_tracker::{self, QueryTrackingContext}; use crate::runtime_manager::RuntimeManager; /// Bundles a stream with its query tracking context so that dropping the @@ -259,6 +260,8 @@ pub unsafe fn close_reader(ptr: i64) { /// Executes a query. Returns a heap-allocated pointer (as i64) to the result stream. /// Caller must call `stream_close` exactly once to free it. +/// If `context_id != 0`, registers a cancellation token in ACTIVE_QUERIES before +/// execution so `cancel_query()` can interrupt it even during planning. /// /// This is an async function — the bridge layer decides how to run it /// (`block_on` for synchronous delivery, `spawn` for async delivery). @@ -290,33 +293,40 @@ pub async unsafe fn execute_query( // index_filter(bytes) calls. Cheap — just bytes inspection. let is_indexed = plan_bytes_mentions_index_filter(plan_bytes); - let stream_ptr = if is_indexed { - let qc = Arc::new(query_config); - crate::indexed_executor::execute_indexed_query( - plan_bytes.to_vec(), - table_name.to_string(), - shard_view, - qc.target_partitions.max(1), - runtime, - cpu_executor, - query_memory_pool, - qc, - ) - .await? - } else { - crate::query_executor::execute_query( - shard_view.table_path.clone(), - shard_view.object_metas.clone(), - table_name.to_string(), - plan_bytes.to_vec(), - runtime, - cpu_executor, - query_memory_pool, - &query_config, - ) - .await? + // Register cancellation token. + let token = query_tracker::get_cancellation_token(context_id); + + let query_future = async { + if is_indexed { + let qc = Arc::new(query_config); + crate::indexed_executor::execute_indexed_query( + plan_bytes.to_vec(), + table_name.to_string(), + shard_view, + qc.target_partitions.max(1), + runtime, + cpu_executor, + query_memory_pool, + qc, + ).await + } else { + crate::query_executor::execute_query( + shard_view.table_path.clone(), + shard_view.object_metas.clone(), + table_name.to_string(), + plan_bytes.to_vec(), + runtime, + cpu_executor, + query_memory_pool, + &query_config, + ).await + } }; + let stream_ptr = cancellation::cancellable(token.as_ref(), context_id, query_future) + .await + .map_err(|e| DataFusionError::Execution(e))?; + // Reconstruct the stream from the raw pointer returned by the executor. let stream = *Box::from_raw(stream_ptr as *mut RecordBatchStreamAdapter); let handle = QueryStreamHandle::new(stream, query_context); @@ -356,7 +366,8 @@ pub unsafe fn stream_get_schema(stream_ptr: i64) -> Result /// Loads the next record batch from the stream. /// -/// Returns a heap-allocated FFI_ArrowArray pointer (as i64), or 0 if end-of-stream. +/// Returns a heap-allocated FFI_ArrowArray pointer (as i64), or 0 if end-of-stream +/// or cancelled. /// /// This is an async function — the bridge layer decides how to run it. /// @@ -365,8 +376,14 @@ pub unsafe fn stream_get_schema(stream_ptr: i64) -> Result /// on the same stream. pub async unsafe fn stream_next(stream_ptr: i64) -> Result { let handle = &mut *(stream_ptr as *mut QueryStreamHandle); + let token = query_tracker::get_cancellation_token(handle._query_tracking_context.context_id()); - let result = handle.stream.try_next().await?; + let result = cancellation::cancellable_or( + token.as_ref(), + None, + async { handle.stream.try_next().await.map_err(|e: DataFusionError| e) }, + ).await + .map_err(|e| DataFusionError::Execution(e))?; match result { Some(batch) => { @@ -391,6 +408,12 @@ pub unsafe fn stream_close(stream_ptr: i64) { } } +/// Fires the cancellation token for the given context_id. +/// No-op for unknown or already-completed queries. +pub fn cancel_query(context_id: i64) { + query_tracker::cancel_query(context_id); +} + /// Converts SQL to Substrait plan bytes (test only). /// /// # Safety diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/cancellation.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/cancellation.rs new file mode 100644 index 0000000000000..129230d515446 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/cancellation.rs @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Cancellation helpers for DataFusion query tasks. +//! +//! The cancellation token itself lives in [`crate::query_tracker::QueryTracker`]. +//! This module provides `select!`-based helpers that race a future against a token. + +use std::future::Future; +use tokio_util::sync::CancellationToken; + +/// Race a future against a cancellation token. Returns a cancellation error string +/// if the token fires first. Pass `None` for non-cancellable queries. +pub async fn cancellable( + token: Option<&CancellationToken>, + context_id: i64, + fut: F, +) -> Result +where + F: Future>, + E: std::fmt::Display, +{ + match token { + Some(token) => { + tokio::select! { + result = fut => result.map_err(|e| e.to_string()), + _ = token.cancelled() => Err(format!("Query {} cancelled", context_id)), + } + } + None => fut.await.map_err(|e| e.to_string()), + } +} + +/// Variant that returns a sentinel value on cancellation instead of an error. +/// Used by `stream_next` where `None` signals cancellation/EOF. +pub async fn cancellable_or( + token: Option<&CancellationToken>, + sentinel: T, + fut: F, +) -> Result +where + F: Future>, + E: std::fmt::Display, +{ + match token { + Some(token) => { + tokio::select! { + result = fut => result.map_err(|e| e.to_string()), + _ = token.cancelled() => Ok(sentinel), + } + } + None => fut.await.map_err(|e| e.to_string()), + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index 85404ff2f49ce..4eddf5ad02017 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -197,6 +197,11 @@ pub unsafe extern "C" fn df_stream_close(stream_ptr: i64) { api::stream_close(stream_ptr); } +#[no_mangle] +pub extern "C" fn df_cancel_query(context_id: i64) { + api::cancel_query(context_id); +} + #[ffm_safe] #[no_mangle] pub unsafe extern "C" fn df_sql_to_substrait( diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs index c883e8d5992d3..71a67fa6379b2 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs @@ -13,6 +13,7 @@ pub mod api; pub mod cache; +pub mod cancellation; pub mod cross_rt_stream; pub mod custom_cache_manager; pub mod datafusion_query_config; @@ -26,7 +27,7 @@ pub mod local_executor; pub mod memory; pub mod partition_stream; pub mod query_executor; -pub mod query_memory_pool_tracker; +pub mod query_tracker; pub mod runtime_manager; pub mod session_context; pub mod statistics_cache; diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_memory_pool_tracker.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs similarity index 93% rename from sandbox/plugins/analytics-backend-datafusion/rust/src/query_memory_pool_tracker.rs rename to sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs index 336f60e0dd5bf..6b4e1d08c5e5c 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_memory_pool_tracker.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs @@ -25,6 +25,7 @@ use std::time::Instant; use dashmap::DashMap; use log::debug; use once_cell::sync::Lazy; +use tokio_util::sync::CancellationToken; use datafusion::common::DataFusionError; use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation}; @@ -115,6 +116,7 @@ pub struct QueryTracker { pub start_time: Instant, pub context_id: i64, pub memory_pool: Arc, + pub cancellation_token: CancellationToken, completed: AtomicBool, wall_nanos: std::sync::atomic::AtomicU64, } @@ -157,6 +159,19 @@ pub fn drain_completed_query(context_id: i64) -> Option> { .map(|(_, t)| t) } +/// Fire the cancellation token for the given context_id. +/// No-op for unknown or already-completed queries. +pub fn cancel_query(context_id: i64) { + if let Some(tracker) = QUERY_REGISTRY.get(&context_id) { + tracker.cancellation_token.cancel(); + } +} + +/// Clone the cancellation token for the given context_id, if registered. +pub fn get_cancellation_token(context_id: i64) -> Option { + QUERY_REGISTRY.get(&context_id).map(|t| t.cancellation_token.clone()) +} + // --------------------------------------------------------------------------- // QueryTrackingContext // --------------------------------------------------------------------------- @@ -185,6 +200,11 @@ impl QueryTrackingContext { start_time: Instant::now(), context_id, memory_pool: query_pool, + // CancellationToken is a thread-safe, cloneable handle that can be used to + // signal cancellation to async tasks via `token.cancelled().await` in a + // `tokio::select!` branch. Calling `token.cancel()` fires all waiters. + // See: https://github.com/tokio-rs/tokio/blob/master/tokio-util/src/sync/cancellation_token/tree_node.rs + cancellation_token: CancellationToken::new(), completed: AtomicBool::new(false), wall_nanos: std::sync::atomic::AtomicU64::new(0), }); @@ -199,6 +219,11 @@ impl QueryTrackingContext { pub fn memory_pool(&self) -> Option> { self.tracker.as_ref().map(|t| Arc::clone(&t.memory_pool)) } + + /// The context_id for this query, or 0 if tracking is disabled. + pub fn context_id(&self) -> i64 { + self.tracker.as_ref().map_or(0, |t| t.context_id) + } } impl Drop for QueryTrackingContext { diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs index d10887be4c1a9..74effcf8d7851 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs @@ -29,7 +29,7 @@ use log::error; use object_store::ObjectMeta; use crate::api::{DataFusionRuntime, ShardView}; -use crate::query_memory_pool_tracker::QueryTrackingContext; +use crate::query_tracker::QueryTrackingContext; /// Opaque handle holding a configured SessionContext between FFM calls. pub struct SessionContextHandle { diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index 558f1e76bdb64..cabd5fa593a37 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -75,6 +75,7 @@ public final class NativeBridge { private static final MethodHandle CREATE_SESSION_CONTEXT; private static final MethodHandle CLOSE_SESSION_CONTEXT; private static final MethodHandle EXECUTE_WITH_CONTEXT; + private static final MethodHandle CANCEL_QUERY; static { SymbolLookup lib = NativeLibraryLoader.symbolLookup(); @@ -344,6 +345,8 @@ public final class NativeBridge { ) ); + CANCEL_QUERY = linker.downcallHandle(lib.find("df_cancel_query").orElseThrow(), FunctionDescriptor.ofVoid(ValueLayout.JAVA_LONG)); + // Hand the five filter-tree upcall stubs to Rust now. No explicit // caller step required — as soon as this class is loaded, callbacks // are installed and `df_execute_indexed_query` can dispatch into Java. @@ -585,6 +588,13 @@ public static void streamClose(long streamPtr) { NativeCall.invokeVoid(STREAM_CLOSE, streamPtr); } + // ---- Cancellation ---- + + /** Fires the cancellation token for the given context. No-op if already completed. */ + public static void cancelQuery(long contextId) { + NativeCall.invokeVoid(CANCEL_QUERY, contextId); + } + // ---- Stubs ---- public static byte[] sqlToSubstrait(long readerPtr, String tableName, String sql, long runtimePtr) { From b2fa9f804e89c78e5ab0e551314e23c90b5127a5 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Thu, 7 May 2026 16:19:33 +0530 Subject: [PATCH 070/115] Changes to handle session context handle close (#21519) * changes to handle session context handle close Signed-off-by: bharath-techie * fixing sandbox check Signed-off-by: bharath-techie --------- Signed-off-by: bharath-techie --- .../backend/jni/ConsumableNativeHandle.java | 86 +++++++++++ .../spi/BackendExecutionContext.java | 26 +++- .../jni/ConsumableNativeHandleTests.java | 142 ++++++++++++++++++ .../rust/src/ffm.rs | 14 +- .../rust/src/query_executor.rs | 12 +- .../be/datafusion/DataFusionSessionState.java | 17 ++- .../be/datafusion/DatafusionContext.java | 14 +- .../be/datafusion/DatafusionSearcher.java | 8 +- .../be/datafusion/nativelib/NativeBridge.java | 38 ++++- .../nativelib/SessionContextHandle.java | 36 +++-- .../DataFusionNativeBridgeTests.java | 14 +- .../exec/AnalyticsSearchService.java | 12 +- .../exec/stage/LocalStageScheduler.java | 37 ++++- 13 files changed, 410 insertions(+), 46 deletions(-) create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/jni/ConsumableNativeHandle.java create mode 100644 sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/backend/jni/ConsumableNativeHandleTests.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/jni/ConsumableNativeHandle.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/jni/ConsumableNativeHandle.java new file mode 100644 index 0000000000000..033c5487b85a8 --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/jni/ConsumableNativeHandle.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.backend.jni; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Specialisation of {@link NativeHandle} for pointers whose ownership is transferred to the + * native side by a specific FFM call (for example, Rust's {@code Box::from_raw} inside a + * consuming function). After the consuming call the native resource is freed internally; + * calling the matching {@code close_X} entry a second time would be a double-free, while + * not calling it on the error path would leak. + * + *

      The bridge method that performs the consuming FFM call must invoke + * {@link #markConsumed()} after the downcall returns (typically in a {@code finally} block). + * This: + *

        + *
      • flips an internal flag so the inherited {@link #doClose()} short-circuits;
        • + *
      • eagerly closes the Java wrapper — the pointer is removed from LIVE_HANDLES in + * {@link NativeHandle}, subsequent {@link #getPointer()} calls + * throw, and {@link NativeHandle#validatePointer(long, String) validatePointer} rejects + * the now-dangling pointer value.
        • + *
      + * + *

      On paths where the consuming call never happened (pre-dispatch Java error, aborted flow, + * Cleaner-at-GC fallback), {@link #doClose()} delegates to {@link #doCloseNative()} which + * subclasses implement to free the native resource via the appropriate {@code close_X} FFM entry. + * + *

      {@link #markConsumed()} is idempotent and safe to call after {@link #close()}. + */ +public abstract class ConsumableNativeHandle extends NativeHandle { + + /** + * Set once the native side has taken ownership of {@link #ptr} via the consuming FFM call. + * When {@code true}, {@link #doClose()} skips the call to {@link #doCloseNative()} to avoid + * a double-free. + */ + private final AtomicBoolean consumed = new AtomicBoolean(false); + + protected ConsumableNativeHandle(long ptr) { + super(ptr); + } + + /** + * Marks this handle as having had its native pointer consumed by the bridge's + * ownership-transferring FFM call, then closes the Java wrapper. See the class javadoc + * for the full contract and typical call pattern. + */ + public final void markConsumed() { + consumed.set(true); + close(); + } + + /** + * @return {@code true} if {@link #markConsumed()} has been called. + */ + protected final boolean isConsumed() { + return consumed.get(); + } + + /** + * Template method: short-circuits to a no-op when {@link #isConsumed()} is {@code true} + * (the native side already freed the resource), otherwise delegates to + * {@link #doCloseNative()}. Marked {@code final} so subclasses cannot bypass the guard. + */ + @Override + protected final void doClose() { + if (isConsumed()) { + return; + } + doCloseNative(); + } + + /** + * Releases the native resource via the appropriate {@code close_X} FFM entry. + * Called by {@link #doClose()} only when the handle has not been marked consumed, + * i.e. on the error / never-executed path. Must be safe to call at most once per pointer. + */ + protected abstract void doCloseNative(); +} diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionContext.java index ac3ca2508a2c7..cffa1e972ef9a 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionContext.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/BackendExecutionContext.java @@ -8,15 +8,31 @@ package org.opensearch.analytics.spi; +import java.io.IOException; + /** - * Marker interface for backend-specific execution context that flows between - * successive instruction handler calls. The first handler in the chain receives - * {@code null} and bootstraps the context; subsequent handlers receive and build - * upon the previous handler's output. + * Backend-specific execution context that flows between successive instruction handler + * calls. The first handler in the chain receives {@code null} and bootstraps the context; + * subsequent handlers receive and build upon the previous handler's output. * *

      Each backend defines its own concrete implementation (e.g., * {@code DataFusionSessionState} holding a native SessionContext handle). * + *

      Lifecycle

      + *

      Extends {@link AutoCloseable} with a narrowed {@code throws IOException} signature so + * backends can attach native / resource-holding handles to the context and rely on the + * orchestrator (e.g. {@code AnalyticsSearchService} or {@code LocalStageScheduler}) to + * close it if the fragment aborts before ownership is transferred to the + * {@code SearchExecEngine}. Implementations that hold no resources should leave the default + * no-op {@link #close()}. {@code close()} must be idempotent; in particular it must + * tolerate being called after the resources have already been handed off to a + * successfully-constructed engine. + * * @opensearch.internal */ -public interface BackendExecutionContext {} +public interface BackendExecutionContext extends AutoCloseable { + @Override + default void close() throws IOException { + // Default: no resources to release. + } +} diff --git a/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/backend/jni/ConsumableNativeHandleTests.java b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/backend/jni/ConsumableNativeHandleTests.java new file mode 100644 index 0000000000000..a2fd03d7901bc --- /dev/null +++ b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/backend/jni/ConsumableNativeHandleTests.java @@ -0,0 +1,142 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.backend.jni; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Tests for {@link ConsumableNativeHandle}'s ownership-transfer contract. + * + *

      The class guards against two specific failure modes: + *

        + *
      • Double-free: the Rust side consumed the pointer via + * {@code Box::from_raw}, then the Java-side {@code close()} calls + * {@code df_close_X} which tries to free the same memory again.
        • + *
      • Leak: the consuming FFM call never dispatched (pre-invoke + * Java failure, aborted flow), so the Java wrapper is responsible for + * calling {@code df_close_X} exactly once.
        • + *
      + * + *

      Both paths rely on the {@code doCloseNative()} callback being invoked + * exactly zero or one times, never twice. These tests nail that contract + * down with a counting subclass so a future change to + * {@link ConsumableNativeHandle} that accidentally re-introduces a + * double-close will fail loudly. + * + *

      Reference: the real subclass + * {@code org.opensearch.be.datafusion.nativelib.SessionContextHandle} is used + * from {@code DatafusionContext#close()} and + * {@code DataFusionSessionState#close()} — both paths can reach + * {@code close()} on the same instance, so idempotency is load-bearing. + */ +public class ConsumableNativeHandleTests extends OpenSearchTestCase { + + /** + * Counts calls to {@link #doCloseNative()} so tests can assert exact + * invocation counts. + */ + private static final class CountingHandle extends ConsumableNativeHandle { + final AtomicInteger nativeCloses = new AtomicInteger(0); + + CountingHandle(long ptr) { + super(ptr); + } + + @Override + protected void doCloseNative() { + nativeCloses.incrementAndGet(); + } + } + + // ---- close() without consumption ------------------------------------ + + public void testCloseWithoutConsumeCallsNativeOnce() { + CountingHandle handle = new CountingHandle(100L); + handle.close(); + assertEquals("doCloseNative should run once on the never-consumed path", 1, handle.nativeCloses.get()); + } + + public void testDoubleCloseWithoutConsumeStillCallsNativeOnce() { + CountingHandle handle = new CountingHandle(101L); + handle.close(); + handle.close(); + assertEquals("close() must be idempotent — second call is a no-op", 1, handle.nativeCloses.get()); + } + + // ---- markConsumed() ownership-transferred path ---------------------- + + public void testMarkConsumedSkipsNativeClose() { + CountingHandle handle = new CountingHandle(200L); + handle.markConsumed(); + assertEquals( + "markConsumed() must not call doCloseNative — the native side already freed the pointer", + 0, + handle.nativeCloses.get() + ); + } + + public void testCloseAfterMarkConsumedIsNoOp() { + CountingHandle handle = new CountingHandle(201L); + handle.markConsumed(); + handle.close(); + assertEquals( + "An explicit close() after markConsumed() must remain a no-op — otherwise Rust's Box::from_raw would be followed by a second free", + 0, + handle.nativeCloses.get() + ); + } + + public void testMarkConsumedAfterCloseDoesNotRunNativeTwice() { + // Order reversed from the normal happy path. The bridge always calls + // markConsumed() after the FFM downcall returns, but the test ensures + // that even if some future caller inverted the sequence, the native + // close is never invoked twice. + CountingHandle handle = new CountingHandle(202L); + handle.close(); + assertEquals(1, handle.nativeCloses.get()); + handle.markConsumed(); + assertEquals("markConsumed() after close() must not trigger another native close", 1, handle.nativeCloses.get()); + } + + public void testMarkConsumedIsIdempotent() { + CountingHandle handle = new CountingHandle(203L); + handle.markConsumed(); + handle.markConsumed(); + handle.close(); + assertEquals(0, handle.nativeCloses.get()); + } + + // ---- State observation --------------------------------------------- + + public void testGetPointerAfterMarkConsumedThrows() { + CountingHandle handle = new CountingHandle(300L); + handle.markConsumed(); + // markConsumed() closes the Java wrapper eagerly; subsequent getPointer + // should refuse to hand out the now-dangling value. + expectThrows(IllegalStateException.class, handle::getPointer); + } + + public void testIsLivePointerFalseAfterMarkConsumed() { + CountingHandle handle = new CountingHandle(301L); + assertTrue(NativeHandle.isLivePointer(301L)); + handle.markConsumed(); + assertFalse( + "markConsumed() must remove the pointer from the live registry so validatePointer rejects it on a stale re-use", + NativeHandle.isLivePointer(301L) + ); + } + + public void testValidatePointerAfterMarkConsumedThrows() { + CountingHandle handle = new CountingHandle(302L); + handle.markConsumed(); + expectThrows(IllegalStateException.class, () -> NativeHandle.validatePointer(302L, "consumed")); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index 4eddf5ad02017..1c626da1519f6 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -603,12 +603,24 @@ pub unsafe extern "C" fn df_execute_with_context( plan_ptr: *const u8, plan_len: i64, ) -> i64 { + // Consume the session context handle on entry. Ownership transfers here + // regardless of whether the remainder of this function succeeds, returns an + // error via `?`, or panics — RAII (or `catch_unwind` drop-during-unwind) + // drops `session_handle` and frees the underlying SessionContext resources. + // + // This matches the Java-side contract: SessionContextHandle.markConsumed() is + // invoked in a `finally` after the FFM downcall, so every observable path from + // Java's perspective ("call.invoke ran") maps to "Rust consumed the handle". + // If we were to run fallible or panic-prone code (e.g. `get_rt_manager()?`) + // before Box::from_raw, the handle would leak on those paths. + let session_handle = *Box::from_raw(session_ctx_ptr as *mut crate::session_context::SessionContextHandle); + let mgr = get_rt_manager()?; let plan_bytes = slice::from_raw_parts(plan_ptr, plan_len as usize); let cpu_executor = mgr.cpu_executor(); mgr.io_runtime .block_on(crate::query_executor::execute_with_context( - session_ctx_ptr, + session_handle, plan_bytes, cpu_executor, )) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs index 14c8d172add9a..33d47b9bb556a 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs @@ -156,14 +156,16 @@ pub async fn execute_query( } /// Executes a Substrait plan against a pre-configured SessionContext. -/// Consumes the handle — SessionContext lifetime is tied to the returned stream. -pub async unsafe fn execute_with_context( - session_ctx_ptr: i64, +/// +/// Takes ownership of the handle by value. The ownership transfer (consuming the +/// raw Java pointer) happens at the FFM entry in `df_execute_with_context`, so +/// by the time this function is reached the pointer is already invalidated from +/// Java's perspective and cleanup is pure RAII. +pub async fn execute_with_context( + handle: SessionContextHandle, plan_bytes: &[u8], cpu_executor: DedicatedExecutor, ) -> Result { - let handle = *Box::from_raw(session_ctx_ptr as *mut SessionContextHandle); - let substrait_plan = Plan::decode(plan_bytes).map_err(|e| { DataFusionError::Execution(format!("Failed to decode Substrait: {}", e)) })?; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionSessionState.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionSessionState.java index edd48ddb11f22..c807dcf3978a5 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionSessionState.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionSessionState.java @@ -12,8 +12,21 @@ import org.opensearch.be.datafusion.nativelib.SessionContextHandle; /** - * Backend-specific execution context produced by ShardScanInstructionHandler, - * consumed by DatafusionSearcher at execute time. + * Backend-specific execution context produced by {@link ShardScanInstructionHandler}, + * consumed by {@link DatafusionSearcher} at execute time. + * + *

      {@link #close()} closes the underlying {@link SessionContextHandle} as the + * fragment-orchestrator's safety net for error paths that never reach the execute step. + * The handle's close is idempotent and cooperates with {@link DatafusionContext#close()} + * (which also closes it once the handle is handed off to an engine), so it is safe to call + * from both places — whichever runs first wins. */ public record DataFusionSessionState(SessionContextHandle sessionContextHandle) implements BackendExecutionContext { + + @Override + public void close() { + if (sessionContextHandle != null) { + sessionContextHandle.close(); + } + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java index cd30c64e0758e..1d7a17352f4ff 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionContext.java @@ -54,7 +54,19 @@ public void close() throws IOException { streamHandle = null; } } finally { - engineSearcher.close(); + try { + // Safety net for aborted-search paths: if the SessionContext was created but + // executeWithContextAsync never ran (or ran and the context is being closed + // without handing off the handle), doClose() calls df_close_session_context. + // On the happy path the handle is already marked consumed and this close() + // is a no-op. + if (sessionContextHandle != null) { + sessionContextHandle.close(); + sessionContextHandle = null; + } + } finally { + engineSearcher.close(); + } } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearcher.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearcher.java index c8bb98991f10e..b6f8abc339101 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearcher.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSearcher.java @@ -58,7 +58,7 @@ private void searchWithSessionContext(DatafusionContext context, SessionContextH DatafusionQuery query = context.getDatafusionQuery(); NativeRuntimeHandle runtimeHandle = context.getNativeRuntime(); CompletableFuture future = new CompletableFuture<>(); - NativeBridge.executeWithContextAsync(sessionCtx.getPointer(), query.getSubstraitBytes(), new ActionListener<>() { + NativeBridge.executeWithContextAsync(sessionCtx, query.getSubstraitBytes(), new ActionListener<>() { @Override public void onResponse(Long streamPtr) { future.complete(streamPtr); @@ -75,8 +75,10 @@ public void onFailure(Exception exception) { } catch (Exception exception) { throw new IOException("Query execution with session context failed", exception); } - // Rust consumed the session context — unregister from live handle set - sessionCtx.close(); + // NativeBridge#executeWithContextAsync has already marked the handle consumed (which + // closes the Java wrapper) on both success and native-error paths; no explicit close + // is needed here. The owning DatafusionContext#close() closes it as a safety net for + // paths that never reach this method (e.g. aborted search). context.setStreamHandle(new StreamHandle(streamPtr, runtimeHandle)); } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index cabd5fa593a37..f0af4c357453f 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -744,18 +744,44 @@ public static SessionContextHandle createSessionContext(long readerPtr, long run } /** - * Executes a Substrait plan against the configured SessionContext. - * Consumes the session context handle (freed internally when stream closes). + * Frees a native {@code SessionContext} handle. Invoked from + * {@link SessionContextHandle#doCloseNative()} ()} on error / never-executed paths; not called on the + * happy path where Rust's {@code execute_with_context} consumes the handle itself. + * Safe to call at most once per pointer. */ - /** Frees a native SessionContext handle. Safe to call once. */ public static void closeSessionContext(long ptr) { NativeCall.invokeVoid(CLOSE_SESSION_CONTEXT, ptr); } - public static void executeWithContextAsync(long sessionCtxPtr, byte[] substraitPlan, ActionListener listener) { - NativeHandle.validatePointer(sessionCtxPtr, "sessionContext"); + /** + * Executes a Substrait plan against the configured SessionContext. + * + *

      Rust's {@code execute_with_context} takes ownership of the {@code SessionContext} via + * {@code Box::from_raw} on entry, regardless of whether the rest of the call then succeeds or + * returns an error. The handle is therefore marked consumed in a {@code finally} block so + * that both success and native-error paths skip {@code df_close_session_context} (which + * would otherwise double-free). Only a Java-side failure before the downcall dispatches + * (argument marshalling) leaves the handle unconsumed, in which case its + * {@link SessionContextHandle#doCloseNative()} ()} will free it. + */ + public static void executeWithContextAsync(SessionContextHandle sessionContext, byte[] substraitPlan, ActionListener listener) { + final long sessionCtxPtr; + try { + sessionCtxPtr = sessionContext.getPointer(); + } catch (Exception e) { + listener.onFailure(e); + return; + } try (var call = new NativeCall()) { - long result = call.invoke(EXECUTE_WITH_CONTEXT, sessionCtxPtr, call.bytes(substraitPlan), (long) substraitPlan.length); + var plan = call.bytes(substraitPlan); + long planLen = (long) substraitPlan.length; + long result; + try { + result = call.invoke(EXECUTE_WITH_CONTEXT, sessionCtxPtr, plan, planLen); + } finally { + // Rust took ownership via Box::from_raw; do not let doClose() double-free. + sessionContext.markConsumed(); + } listener.onResponse(result); } catch (Throwable throwable) { listener.onFailure(throwable instanceof Exception ? (Exception) throwable : new RuntimeException(throwable)); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextHandle.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextHandle.java index c26d1799611eb..08d8ae515e45a 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextHandle.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextHandle.java @@ -8,31 +8,35 @@ package org.opensearch.be.datafusion.nativelib; -import org.opensearch.analytics.backend.jni.NativeHandle; +import org.opensearch.analytics.backend.jni.ConsumableNativeHandle; /** - * Type-safe wrapper for a native SessionContext pointer returned by - * {@link NativeBridge#createSessionContext}. The Rust side consumes this - * handle when {@link NativeBridge#executeWithContextAsync} is called, - * so {@link #doClose()} is a no-op — the pointer is freed by Rust internally. + * Type-safe wrapper for a native {@code SessionContext} pointer returned by + * {@link NativeBridge#createSessionContext}. * - *

      This handle exists to participate in the {@link NativeHandle} live-pointer - * registry so that {@link NativeHandle#validatePointer} passes for FFM calls. + *

      Ownership

      + *

      On the happy path, {@link NativeBridge#executeWithContextAsync} transfers ownership of the + * pointer to Rust, which takes it via {@code Box::from_raw} on the first line of + * {@code df_execute_with_context} and drops it when the stream finishes. The bridge method + * calls {@link ConsumableNativeHandle#markConsumed()} after the FFM downcall so that the + * inherited {@link #doClose()} short-circuits without calling + * {@code df_close_session_context} — doing so would be a double-free. + * + *

      On any path where execute is never reached (Java-side error before the downcall, aborted + * search, context closed before execution), {@link #doCloseNative()} calls + * {@link NativeBridge#closeSessionContext(long)} which invokes the Rust + * {@code df_close_session_context} entry to free the handle. Both the explicit + * {@link #close()} call from {@link org.opensearch.be.datafusion.DatafusionContext#close()} and + * the {@link java.lang.ref.Cleaner} GC-time fallback route through this path. */ -public class SessionContextHandle extends NativeHandle { +public class SessionContextHandle extends ConsumableNativeHandle { public SessionContextHandle(long ptr) { super(ptr); } @Override - protected void doClose() { - // TODO: Handle error-path cleanup. Currently Rust consumes the handle in - // execute_with_context (moves QueryTrackingContext into the stream). If execute - // fails or is never called, this handle leaks on the Rust side. - // Options: (a) AtomicBool 'consumed' flag on Rust handle — close_session_context - // checks flag before freeing, (b) don't consume in Rust and use no-op tracking - // on the stream, (c) markConsumed() on NativeHandle to skip doClose on happy path. - // See df_close_session_context FFM entry which exists but is not yet wired here. + protected void doCloseNative() { + NativeBridge.closeSessionContext(ptr); } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java index 0e5c78087986f..23fa446681911 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java @@ -8,6 +8,7 @@ package org.opensearch.be.datafusion; +import org.opensearch.analytics.backend.jni.NativeHandle; import org.opensearch.be.datafusion.nativelib.NativeBridge; import org.opensearch.be.datafusion.nativelib.ReaderHandle; import org.opensearch.be.datafusion.nativelib.SessionContextHandle; @@ -95,8 +96,13 @@ public void testSessionContextCreationAndTableRegistration() throws Exception { "SELECT message FROM test_table", runtimeHandle.get() ); + // Capture the pointer value BEFORE execute — after execute the handle is marked consumed + // (which closes the Java wrapper), so getPointer() would throw IllegalStateException. + long sessionCtxPtrBefore = sessionCtx.getPointer(); + assertTrue("SessionContext pointer should be live before execute", NativeHandle.isLivePointer(sessionCtxPtrBefore)); + CompletableFuture future = new CompletableFuture<>(); - NativeBridge.executeWithContextAsync(sessionCtx.getPointer(), substrait, new ActionListener<>() { + NativeBridge.executeWithContextAsync(sessionCtx, substrait, new ActionListener<>() { @Override public void onResponse(Long streamPtr) { future.complete(streamPtr); @@ -110,8 +116,10 @@ public void onFailure(Exception exception) { long streamPtr = future.join(); assertTrue("Stream pointer should be non-zero", streamPtr != 0); - // Session context is consumed by execute — close the Java handle - sessionCtx.close(); + // executeWithContextAsync marks the handle consumed (which closes the Java wrapper). + // Verify the pointer is no longer in the live registry and the wrapper rejects getPointer(). + assertFalse("SessionContextHandle pointer must no longer be live after execute", NativeHandle.isLivePointer(sessionCtxPtrBefore)); + expectThrows(IllegalStateException.class, sessionCtx::getPointer); NativeBridge.streamClose(streamPtr); readerHandle.close(); diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java index 804635187cf4d..0b2e0e28dbb6a 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java @@ -113,12 +113,12 @@ private FragmentResources startFragment(FragmentExecutionRequest request, Resolv GatedCloseable gatedReader = resolved.readerProvider.acquireReader(); SearchExecEngine engine = null; EngineResultStream stream = null; + BackendExecutionContext backendContext = null; try { ShardScanExecutionContext ctx = buildContext(request, gatedReader.get(), resolved.plan, task); AnalyticsSearchBackendPlugin backend = backends.get(resolved.plan.getBackendId()); // Apply instruction handlers in order — each builds upon the previous handler's backend context - BackendExecutionContext backendContext = null; List instructions = resolved.plan.getInstructions(); if (!instructions.isEmpty()) { FragmentInstructionHandlerFactory factory = backend.getInstructionHandlerFactory(); @@ -137,6 +137,16 @@ private FragmentResources startFragment(FragmentExecutionRequest request, Resolv } catch (Exception suppressed) { e.addSuppressed(suppressed); } + // Close the backend execution context as a safety net for failure paths that + // never reached / never finished the engine construction — if the handle was + // already transferred, close() is a no-op (implementations must be idempotent). + if (backendContext != null) { + try { + backendContext.close(); + } catch (Exception suppressed) { + e.addSuppressed(suppressed); + } + } throw e; } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java index d9205260000ef..c934bc1c6b76b 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java @@ -61,9 +61,40 @@ public StageExecution createExecution(Stage stage, ExchangeSink sink, QueryConte FragmentInstructionHandlerFactory factory = stage.getInstructionHandlerFactory(); if (factory != null) { BackendExecutionContext backendContext = null; - for (InstructionNode node : stage.getPlanAlternatives().getFirst().instructions()) { - FragmentInstructionHandler handler = factory.createHandler(node); - backendContext = handler.apply(node, context, backendContext); + Throwable primaryFailure = null; + try { + for (InstructionNode node : stage.getPlanAlternatives().getFirst().instructions()) { + FragmentInstructionHandler handler = factory.createHandler(node); + BackendExecutionContext previous = backendContext; + backendContext = handler.apply(node, context, backendContext); + // A handler that returns a new reference implicitly abandons the previous + // context — close it now so its resources aren't orphaned. + if (previous != null && previous != backendContext) { + previous.close(); + } + } + } catch (Throwable t) { + primaryFailure = t; + } finally { + // The reduce path does not currently hand backendContext off to the sink + // provider — any resources attached by instruction handlers must be released + // here. Close is idempotent so a future handoff can coexist with this call. + if (backendContext != null) { + try { + backendContext.close(); + } catch (Exception closeFailure) { + if (primaryFailure != null) { + primaryFailure.addSuppressed(closeFailure); + } else { + primaryFailure = closeFailure; + } + } + } + } + if (primaryFailure != null) { + if (primaryFailure instanceof RuntimeException re) throw re; + if (primaryFailure instanceof Error err) throw err; + throw new RuntimeException("Instruction handler failed for stageId=" + stage.getStageId(), primaryFailure); } } From 688834551a7aa79fa67c7ef604b0759e3f1f4df0 Mon Sep 17 00:00:00 2001 From: Nishchay Malhotra <114057571+nishchay21@users.noreply.github.com> Date: Thu, 7 May 2026 19:27:06 +0530 Subject: [PATCH 071/115] =?UTF-8?q?Add=20warm+format=20directory=20wiring?= =?UTF-8?q?=20with=20per-format=20tiered=20directory=20rou=E2=80=A6=20(#21?= =?UTF-8?q?361)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add warm+format directory wiring with per-format tiered directory routing --------- Signed-off-by: nishchay21 Signed-off-by: Bukhtawar Khan Signed-off-by: Nishchay Malhotra <114057571+nishchay21@users.noreply.github.com> Co-authored-by: Bukhtawar Khan --- .../tiered-storage/src/main/rust/src/ffm.rs | 196 ++++- .../src/main/rust/src/ffm_tests.rs | 123 +++ .../main/rust/src/registry/tiered_registry.rs | 89 +- .../src/main/rust/src/tiered_object_store.rs | 181 ++-- .../rust/src/tiered_object_store_tests.rs | 146 +--- .../tiered-storage/src/main/rust/src/types.rs | 74 +- .../composite/CompositeDataFormatPlugin.java | 50 +- .../CompositeDataFormatPluginTests.java | 130 ++- .../plugins/native-repository-fs/build.gradle | 1 - .../parquet/ParquetDataFormatPlugin.java | 32 +- .../store/ParquetDataFormatStoreHandler.java | 134 +++ .../parquet/store/ParquetStoreStrategy.java | 36 + .../parquet/store/TieredStorageBridge.java | 184 ++++ .../store/ParquetStoreStrategyTests.java | 56 ++ .../org/opensearch/index/IndexModule.java | 9 + .../org/opensearch/index/IndexService.java | 69 +- .../engine/dataformat/DataFormatPlugin.java | 50 +- .../engine/dataformat/DataFormatRegistry.java | 63 ++ .../dataformat/DataFormatStoreHandler.java | 89 ++ .../DataFormatStoreHandlerFactory.java | 38 + .../engine/dataformat/StoreStrategy.java | 89 ++ .../shard/RemoteStoreUploaderService.java | 47 +- .../index/store/CompositeDirectory.java | 19 +- .../store/DataFormatAwareStoreDirectory.java | 49 +- .../DataFormatAwareStoreDirectoryFactory.java | 74 +- .../store/RemoteSegmentStoreDirectory.java | 10 + .../index/store/RemoteSyncListener.java | 43 + .../main/java/org/opensearch/node/Node.java | 9 +- .../opensearch/plugins/NativeStoreHandle.java | 59 +- .../directory/StoreStrategyRegistry.java | 300 +++++++ ...dDataFormatAwareStoreDirectoryFactory.java | 122 +++ .../TieredSubdirectoryAwareDirectory.java | 222 +++++ .../storage/utils/DirectoryUtils.java | 38 +- .../dataformat/DataFormatRegistryTests.java | 98 +++ .../RemoteStoreUploaderServiceTests.java | 191 +++++ .../DataFormatAwareStoreDirectoryTests.java | 35 + .../plugins/NativeStoreHandleTests.java | 136 +++ .../directory/GracefulDegradationTests.java | 197 +++++ .../directory/StoreStrategyRegistryTests.java | 529 ++++++++++++ ...FormatAwareStoreDirectoryFactoryTests.java | 228 +++++ ...TieredSubdirectoryAwareDirectoryTests.java | 805 ++++++++++++++++++ .../WarmShardDirectoryStackTests.java | 227 +++++ .../storage/utils/DirectoryUtilsTests.java | 12 + 43 files changed, 4796 insertions(+), 493 deletions(-) create mode 100644 sandbox/libs/tiered-storage/src/main/rust/src/ffm_tests.rs create mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/ParquetDataFormatStoreHandler.java create mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/ParquetStoreStrategy.java create mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/TieredStorageBridge.java create mode 100644 sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/store/ParquetStoreStrategyTests.java create mode 100644 server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatStoreHandler.java create mode 100644 server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatStoreHandlerFactory.java create mode 100644 server/src/main/java/org/opensearch/index/engine/dataformat/StoreStrategy.java create mode 100644 server/src/main/java/org/opensearch/index/store/RemoteSyncListener.java create mode 100644 server/src/main/java/org/opensearch/storage/directory/StoreStrategyRegistry.java create mode 100644 server/src/main/java/org/opensearch/storage/directory/TieredDataFormatAwareStoreDirectoryFactory.java create mode 100644 server/src/main/java/org/opensearch/storage/directory/TieredSubdirectoryAwareDirectory.java create mode 100644 server/src/test/java/org/opensearch/plugins/NativeStoreHandleTests.java create mode 100644 server/src/test/java/org/opensearch/storage/directory/GracefulDegradationTests.java create mode 100644 server/src/test/java/org/opensearch/storage/directory/StoreStrategyRegistryTests.java create mode 100644 server/src/test/java/org/opensearch/storage/directory/TieredDataFormatAwareStoreDirectoryFactoryTests.java create mode 100644 server/src/test/java/org/opensearch/storage/directory/TieredSubdirectoryAwareDirectoryTests.java create mode 100644 server/src/test/java/org/opensearch/storage/directory/WarmShardDirectoryStackTests.java diff --git a/sandbox/libs/tiered-storage/src/main/rust/src/ffm.rs b/sandbox/libs/tiered-storage/src/main/rust/src/ffm.rs index 11198798f9be2..7cbf70bc42836 100644 --- a/sandbox/libs/tiered-storage/src/main/rust/src/ffm.rs +++ b/sandbox/libs/tiered-storage/src/main/rust/src/ffm.rs @@ -11,32 +11,129 @@ //! `TieredStorageRegistry` (file registry) and local `ObjectStore` are //! created internally — no separate pointers exposed to Java. +use std::slice; +use std::str; use std::sync::Arc; use native_bridge_common::ffm_safe; +use object_store::ObjectStore; use crate::registry::TieredStorageRegistry; +use crate::registry::FileRegistry; use crate::tiered_object_store::TieredObjectStore; +use crate::types::FileLocation; const NULL_PTR: i64 = 0; +/// Decode a UTF-8 string from a raw pointer and length. +/// +/// # Safety +/// The caller must ensure `ptr` points to `len` valid UTF-8 bytes. +unsafe fn str_from_raw<'a>(ptr: *const u8, len: i64) -> Result<&'a str, String> { + if ptr.is_null() { + return Err("null string pointer".to_string()); + } + if len < 0 { + return Err(format!("negative string length: {}", len)); + } + let bytes = slice::from_raw_parts(ptr, len as usize); + str::from_utf8(bytes).map_err(|e| format!("invalid UTF-8: {}", e)) +} + +/// Reconstruct an `Arc` from a raw pointer without +/// consuming ownership. Increments the strong count so the caller's +/// copy remains valid. +/// +/// # Safety +/// `ptr` must have been produced by `Arc::into_raw` on a live +/// `Arc`. +unsafe fn arc_from_ptr(ptr: i64) -> Result, String> { + if ptr == NULL_PTR { + return Err("null store pointer (0)".to_string()); + } + let raw = ptr as *const TieredObjectStore; + Arc::increment_strong_count(raw); + Ok(Arc::from_raw(raw)) +} + // --------------------------------------------------------------------------- // Public FFM exports // --------------------------------------------------------------------------- -/// Create a [`TieredObjectStore`] with an internally-created file registry -/// and local filesystem store. +/// Create a [`TieredObjectStore`] with optional local and remote object stores. +/// +/// Create a `TieredObjectStore` with optional local and remote stores. +/// +/// `local_store_box_ptr=0` creates a default `LocalFileSystem`. For per-shard +/// stores, Java passes 0 and DataFusion uses absolute paths to resolve files. +/// +/// - `local_store_box_ptr`: if non-zero, a `Box>` pointer for local I/O. +/// If 0, creates a default `LocalFileSystem::new()`. +/// - `remote_store_box_ptr`: if non-zero, a `Box>` pointer from a repository +/// plugin. The Arc is cloned (ownership is NOT taken — the pointer remains valid for other +/// shards). If 0, no remote store. #[ffm_safe] #[no_mangle] -pub extern "C" fn ts_create_tiered_object_store() -> i64 { +pub extern "C" fn ts_create_tiered_object_store( + local_store_box_ptr: i64, + remote_store_box_ptr: i64, +) -> i64 { let file_registry = Arc::new(TieredStorageRegistry::new()); - let local = Arc::new(object_store::local::LocalFileSystem::new()); + + let local: Arc = if local_store_box_ptr != NULL_PTR { + *unsafe { Box::from_raw(local_store_box_ptr as *mut Arc) } + } else { + Arc::new(object_store::local::LocalFileSystem::new()) + }; + let store = Arc::new(TieredObjectStore::new(file_registry, local)); + + if remote_store_box_ptr != NULL_PTR { + // IMPORTANT: Do NOT consume the Box — the pointer is node-level and shared + // across multiple shards. Clone the Arc out of the Box without taking ownership. + let remote_box = unsafe { &*(remote_store_box_ptr as *const Arc) }; + let remote_arc = Arc::clone(remote_box); + store.set_remote(remote_arc); + } + let ptr = Arc::into_raw(store) as i64; native_bridge_common::log_info!("ffm: ts_create_tiered_object_store ptr={}", ptr); Ok(ptr) } +/// Returns a `Box>` pointer from an existing TieredObjectStore Arc pointer. +/// This is the format that `df_create_reader` expects — a boxed fat pointer to the trait object. +/// Each call creates a new Box with its own Arc clone — caller must free with +/// `ts_destroy_object_store_box_ptr`. +#[ffm_safe] +#[no_mangle] +pub extern "C" fn ts_get_object_store_box_ptr(tiered_store_ptr: i64) -> i64 { + if tiered_store_ptr == NULL_PTR { + return Err("ts_get_object_store_box_ptr: null pointer".to_string()); + } + // Increment strong count so we don't consume the original Arc + unsafe { Arc::increment_strong_count(tiered_store_ptr as *const TieredObjectStore) }; + let arc: Arc = unsafe { Arc::from_raw(tiered_store_ptr as *const TieredObjectStore) }; + // Coerce to trait object and box it + let boxed: Box> = Box::new(arc as Arc); + let ptr = Box::into_raw(boxed) as i64; + native_bridge_common::log_info!("ffm: ts_get_object_store_box_ptr input={}, output={}", tiered_store_ptr, ptr); + Ok(ptr) +} + +/// Destroy a `Box>` pointer returned by `ts_get_object_store_box_ptr`. +/// Drops the Box and decrements the Arc strong count. +#[ffm_safe] +#[no_mangle] +pub extern "C" fn ts_destroy_object_store_box_ptr(ptr: i64) -> i64 { + if ptr == NULL_PTR { + return Err("ts_destroy_object_store_box_ptr: null pointer (0)".to_string()); + } + let _boxed = unsafe { Box::from_raw(ptr as *mut Arc) }; + native_bridge_common::log_info!("ffm: ts_destroy_object_store_box_ptr ptr={}", ptr); + Ok(0) +} + /// Destroy a [`TieredObjectStore`]. /// /// Also drops the internally-owned `TieredStorageRegistry`. @@ -51,22 +148,87 @@ pub extern "C" fn ts_destroy_tiered_object_store(ptr: i64) -> i64 { Ok(0) } -// TODO: File registry operations via TieredObjectStore pointer: -// ts_register_file(store_ptr, ...), ts_remove_by_prefix(store_ptr, ...) +// --------------------------------------------------------------------------- +// File registry operations via TieredObjectStore pointer +// --------------------------------------------------------------------------- -#[cfg(test)] -mod tests { - use super::*; +/// Register a file in the TieredObjectStore's registry. +// TODO (writable warm): add ts_register_file for single-file registration (afterSyncToRemote). + +/// Batch register files in the TieredObjectStore's registry. +/// +/// `entries_ptr`/`entries_len`: UTF-8 string with newline-delimited pairs: +/// `"path1\nremotePath1\nsize1\npath2\nremotePath2\nsize2\n..."`. +/// Each triplet is (path, remotePath, size). For Local files, remotePath can be empty. +/// `count`: number of file triplets (entries_len contains 3*count lines). +/// `location`: 0=Local, 1=Remote — applied to all files in the batch. +#[ffm_safe] +#[no_mangle] +pub extern "C" fn ts_register_files( + store_ptr: i64, + entries_ptr: *const u8, + entries_len: i64, + count: i32, + location: i32, +) -> i64 { + let store = unsafe { arc_from_ptr(store_ptr) }?; + let entries_str = unsafe { str_from_raw(entries_ptr, entries_len) } + .map_err(|e| format!("ts_register_files entries: {}", e))?; + + let file_location = FileLocation::from_u8(location as u8) + .ok_or_else(|| format!("ts_register_files: invalid location {}", location))?; - #[test] - fn test_destroy_null_returns_error() { - assert!(ts_destroy_tiered_object_store(0) < 0); + let lines: Vec<&str> = entries_str.split('\n').collect(); + let expected = (count as usize) * 3; + if lines.len() < expected { + return Err(format!( + "ts_register_files: expected {} lines ({}*3) but got {}", + expected, count, lines.len() + )); } - #[test] - fn test_create_and_destroy_no_leak() { - let store_ptr = ts_create_tiered_object_store(); - assert!(store_ptr > 0); - assert_eq!(ts_destroy_tiered_object_store(store_ptr), 0); + let registry = store.registry(); + for i in 0..(count as usize) { + let path = lines[i * 3]; + // Strip leading "/" — object_store::Path normalizes paths without leading slash + let path = path.strip_prefix('/').unwrap_or(path); + let remote_path_str = lines[i * 3 + 1]; + let size_str = lines[i * 3 + 2]; + let remote_arc: Option> = if remote_path_str.is_empty() { + None + } else { + Some(Arc::from(remote_path_str)) + }; + let size: u64 = size_str.parse().unwrap_or(0); + let entry = crate::types::TieredFileEntry::with_size(file_location, remote_arc, size); + registry.register(path, entry); } + + native_bridge_common::log_debug!("ffm: ts_register_files count={}, location={}", count, file_location); + Ok(0) +} + +/// Remove a file from the registry. +#[ffm_safe] +#[no_mangle] +pub extern "C" fn ts_remove_file( + store_ptr: i64, + path_ptr: *const u8, + path_len: i64, +) -> i64 { + let store = unsafe { arc_from_ptr(store_ptr) }?; + let path = unsafe { str_from_raw(path_ptr, path_len) } + .map_err(|e| format!("ts_remove_file path: {}", e))?; + + store.registry().remove(path, false); + + native_bridge_common::log_debug!("ffm: ts_remove_file path='{}'", path); + Ok(0) } + +// TODO (writable warm): add ts_get_file_location when LOCAL routing is needed. +// TODO (writable warm): add ts_add_remote_store_ptr for late-binding remote store. + +#[cfg(test)] +#[path = "ffm_tests.rs"] +mod tests; diff --git a/sandbox/libs/tiered-storage/src/main/rust/src/ffm_tests.rs b/sandbox/libs/tiered-storage/src/main/rust/src/ffm_tests.rs new file mode 100644 index 0000000000000..43cb5bcf85b25 --- /dev/null +++ b/sandbox/libs/tiered-storage/src/main/rust/src/ffm_tests.rs @@ -0,0 +1,123 @@ +use super::*; + +#[test] +fn test_destroy_null_returns_error() { + assert!(ts_destroy_tiered_object_store(0) < 0); +} + +#[test] +fn test_create_and_destroy_no_leak() { + let store_ptr = ts_create_tiered_object_store(0, 0); + assert!(store_ptr > 0); + assert_eq!(ts_destroy_tiered_object_store(store_ptr), 0); +} + +#[test] +fn test_register_files_null_store_returns_error() { + let entries = b"test.parquet\nremote/test.parquet"; + let result = ts_register_files(0, entries.as_ptr(), entries.len() as i64, 1, 1); + assert!(result < 0); +} + +#[test] +fn test_remove_file_null_store_returns_error() { + let result = ts_remove_file(0, b"test.parquet".as_ptr(), 12); + assert!(result < 0); +} + +#[test] +fn test_register_files_and_remove_round_trip() { + let store_ptr = ts_create_tiered_object_store(0, 0); + assert!(store_ptr > 0); + + // Batch register: two files as Remote (triplets: path\nremotePath\nsize\n...) + let entries = b"data/seg_0.parquet\nremote/seg_0.parquet\n1024\ndata/local.parquet\n\n0"; + let result = ts_register_files(store_ptr, entries.as_ptr(), entries.len() as i64, 2, 1); + assert_eq!(result, 0); + + // Remove one + let result = ts_remove_file(store_ptr, b"data/seg_0.parquet".as_ptr(), 18); + assert_eq!(result, 0); + + assert_eq!(ts_destroy_tiered_object_store(store_ptr), 0); +} + +#[test] +fn test_register_files_invalid_location_returns_error() { + let store_ptr = ts_create_tiered_object_store(0, 0); + assert!(store_ptr > 0); + + let entries = b"test.parquet\nremote/test.parquet\n2048"; + let result = ts_register_files(store_ptr, entries.as_ptr(), entries.len() as i64, 1, 99); + assert!(result < 0); + + assert_eq!(ts_destroy_tiered_object_store(store_ptr), 0); +} + +#[test] +fn test_get_object_store_box_ptr_null_returns_error() { + assert!(ts_get_object_store_box_ptr(0) < 0); +} + +#[test] +fn test_destroy_object_store_box_ptr_null_returns_error() { + assert!(ts_destroy_object_store_box_ptr(0) < 0); +} + +#[test] +fn test_get_and_destroy_object_store_box_ptr_round_trip() { + let store_ptr = ts_create_tiered_object_store(0, 0); + assert!(store_ptr > 0); + + // Get a boxed pointer — this increments the Arc refcount + let box_ptr = ts_get_object_store_box_ptr(store_ptr); + assert!(box_ptr > 0); + assert_ne!(box_ptr, store_ptr); // different pointer (Box wrapping Arc) + + // Destroy the box — decrements Arc refcount + assert_eq!(ts_destroy_object_store_box_ptr(box_ptr), 0); + + // Original store still alive — destroy it + assert_eq!(ts_destroy_tiered_object_store(store_ptr), 0); +} + +#[test] +fn test_get_object_store_box_ptr_multiple_calls() { + let store_ptr = ts_create_tiered_object_store(0, 0); + assert!(store_ptr > 0); + + // Multiple box pointers can coexist (simulates multiple reader managers) + let box1 = ts_get_object_store_box_ptr(store_ptr); + let box2 = ts_get_object_store_box_ptr(store_ptr); + assert!(box1 > 0); + assert!(box2 > 0); + assert_ne!(box1, box2); // each call creates a new Box + + // Destroy both boxes + assert_eq!(ts_destroy_object_store_box_ptr(box1), 0); + assert_eq!(ts_destroy_object_store_box_ptr(box2), 0); + + // Original store still alive + assert_eq!(ts_destroy_tiered_object_store(store_ptr), 0); +} + +#[test] +fn test_create_with_remote_does_not_consume_pointer() { + // Simulate node-level remote store: create a Box> + let remote: Arc = Arc::new(object_store::local::LocalFileSystem::new()); + let remote_box = Box::new(remote); + let remote_ptr = Box::into_raw(remote_box) as i64; + + // Create two TieredObjectStores sharing the same remote pointer + let store1 = ts_create_tiered_object_store(0, remote_ptr); + let store2 = ts_create_tiered_object_store(0, remote_ptr); + assert!(store1 > 0); + assert!(store2 > 0); + + // Both stores work — remote pointer not consumed + assert_eq!(ts_destroy_tiered_object_store(store1), 0); + assert_eq!(ts_destroy_tiered_object_store(store2), 0); + + // Clean up the remote Box (simulates repository.doClose()) + let _remote_box = unsafe { Box::from_raw(remote_ptr as *mut Arc) }; +} diff --git a/sandbox/libs/tiered-storage/src/main/rust/src/registry/tiered_registry.rs b/sandbox/libs/tiered-storage/src/main/rust/src/registry/tiered_registry.rs index e671082254b1b..f5a24b2fd6c99 100644 --- a/sandbox/libs/tiered-storage/src/main/rust/src/registry/tiered_registry.rs +++ b/sandbox/libs/tiered-storage/src/main/rust/src/registry/tiered_registry.rs @@ -16,7 +16,6 @@ use std::collections::HashSet; use std::fmt; -use std::sync::atomic::{AtomicU64, Ordering}; use dashmap::DashMap; @@ -30,14 +29,10 @@ use crate::types::{FileLocation, ReadGuard, TieredFileEntry}; /// Production file registry backed by [`DashMap`]. /// /// Tracks per-file metadata and provides RAII-based ref counting via -/// [`ReadGuard`]. Metrics counters are monotonic for monitoring. +/// [`ReadGuard`]. pub struct TieredStorageRegistry { /// Per-file metadata. Key is the file path. files: DashMap, - /// Total acquire calls (monotonic counter for monitoring). - acquire_count: AtomicU64, - /// Total remove calls (monotonic counter for monitoring). - remove_count: AtomicU64, } // TODO: Add PendingAction (EvictLocal/RemoveFull) and pinned fields to @@ -50,30 +45,19 @@ impl TieredStorageRegistry { native_bridge_common::log_info!("TieredStorageRegistry: created"); Self { files: DashMap::new(), - acquire_count: AtomicU64::new(0), - remove_count: AtomicU64::new(0), } } - /// Monitoring metrics: `(acquires, removes)`. - #[must_use] - pub fn metrics(&self) -> (u64, u64) { - ( - self.acquire_count.load(Ordering::Relaxed), - self.remove_count.load(Ordering::Relaxed), - ) - } - - /// List entries matching `prefix`. Returns `(key, location, size)`. + /// List entries matching `prefix`. Returns `(key, location)`. /// /// If `prefix` is empty or `"/"`, returns all entries. #[must_use] - pub fn entries_matching(&self, prefix: &str) -> Vec<(String, FileLocation, Option)> { + pub fn entries_matching(&self, prefix: &str) -> Vec<(String, FileLocation, u64)> { let match_all = prefix.is_empty() || prefix == "/"; self.files .iter() .filter(|e| match_all || e.key().starts_with(prefix)) - .map(|e| (e.key().clone(), e.value().location(), e.value().file_size())) + .map(|e| (e.key().clone(), e.value().location(), e.value().size())) .collect() } } @@ -107,7 +91,6 @@ impl FileRegistry for TieredStorageRegistry { fn get(&self, key: &str) -> Option> { let entry = self.files.get(key)?; - self.acquire_count.fetch_add(1, Ordering::Relaxed); Some(ReadGuard::new(entry)) } @@ -119,18 +102,13 @@ impl FileRegistry for TieredStorageRegistry { fn remove(&self, key: &str, force: bool) -> bool { if force { - let removed = self.files.remove(key).is_some(); - if removed { - self.remove_count.fetch_add(1, Ordering::Relaxed); - } - removed + self.files.remove(key).is_some() } else { // Only remove if ref_count == 0. match self.files.entry(key.to_string()) { dashmap::mapref::entry::Entry::Occupied(entry) => { if entry.get().ref_count() == 0 { entry.remove(); - self.remove_count.fetch_add(1, Ordering::Relaxed); true } else { false @@ -152,10 +130,6 @@ impl FileRegistry for TieredStorageRegistry { true } }); - if removed > 0 { - self.remove_count - .fetch_add(removed as u64, Ordering::Relaxed); - } removed } else { let matching: Vec = self @@ -180,8 +154,6 @@ impl FileRegistry for TieredStorageRegistry { .retain(|key, _| valid_keys.contains(key.as_str())); let removed = before.saturating_sub(self.files.len()); if removed > 0 { - self.remove_count - .fetch_add(removed as u64, Ordering::Relaxed); native_bridge_common::log_info!( "TieredStorageRegistry: purge_stale removed {} entries", removed @@ -202,7 +174,6 @@ impl FileRegistry for TieredStorageRegistry { #[cfg(test)] mod tests { use super::*; - use object_store::memory::InMemory; use std::sync::Arc; use std::sync::Barrier; use std::thread; @@ -211,31 +182,21 @@ mod tests { TieredStorageRegistry::new() } - fn mock_store() -> Arc { - Arc::new(InMemory::new()) - } - fn local_entry() -> TieredFileEntry { - TieredFileEntry::new(FileLocation::Local, None, None, None, None) + TieredFileEntry::new(FileLocation::Local, None) } - fn remote_entry(store: Arc) -> TieredFileEntry { + fn remote_entry() -> TieredFileEntry { TieredFileEntry::new( FileLocation::Remote, Some(Arc::from("remote/a.parquet")), - Some("repo1".into()), - Some(store), - None, ) } - fn both_entry(store: Arc) -> TieredFileEntry { + fn both_entry() -> TieredFileEntry { TieredFileEntry::new( - FileLocation::Both, + FileLocation::Remote, Some(Arc::from("remote/a.parquet")), - Some("repo1".into()), - Some(store), - None, ) } @@ -251,14 +212,14 @@ mod tests { #[test] fn test_register_remote() { let reg = make_registry(); - reg.register("/a.parquet", remote_entry(mock_store())); + reg.register("/a.parquet", remote_entry()); assert_eq!(reg.len(), 1); } #[test] fn test_register_both() { let reg = make_registry(); - reg.register("/a.parquet", both_entry(mock_store())); + reg.register("/a.parquet", both_entry()); assert_eq!(reg.len(), 1); } @@ -266,7 +227,7 @@ mod tests { fn test_register_overwrites() { let reg = make_registry(); reg.register("/a.parquet", local_entry()); - reg.register("/a.parquet", remote_entry(mock_store())); + reg.register("/a.parquet", remote_entry()); assert_eq!(reg.len(), 1); let guard = reg.get("/a.parquet").unwrap(); assert_eq!(guard.location(), FileLocation::Remote); @@ -277,12 +238,10 @@ mod tests { #[test] fn test_get_returns_guard_with_correct_data() { let reg = make_registry(); - let store = mock_store(); - reg.register("/a.parquet", remote_entry(store)); + reg.register("/a.parquet", remote_entry()); let guard = reg.get("/a.parquet").unwrap(); assert_eq!(guard.location(), FileLocation::Remote); assert_eq!(guard.remote_path(), Some("remote/a.parquet")); - assert!(guard.remote_store().is_some()); assert_eq!(guard.ref_count(), 1); } @@ -326,12 +285,10 @@ mod tests { let reg = make_registry(); reg.register("/a.parquet", local_entry()); reg.update("/a.parquet", |e| { - e.location = FileLocation::Both; - e.size = Some(42); + e.location = FileLocation::Remote; }); let guard = reg.get("/a.parquet").unwrap(); - assert_eq!(guard.location(), FileLocation::Both); - assert_eq!(guard.value().file_size(), Some(42)); + assert_eq!(guard.location(), FileLocation::Remote); } #[test] @@ -445,20 +402,6 @@ mod tests { assert_eq!(reg.len(), 1); } - // -- Metrics ------------------------------------------------------------ - - #[test] - fn test_metrics_track_operations() { - let reg = make_registry(); - reg.register("/a.parquet", local_entry()); - let _g = reg.get("/a.parquet"); - drop(_g); - reg.remove("/a.parquet", true); - let (acq, rem) = reg.metrics(); - assert_eq!(acq, 1); - assert_eq!(rem, 1); - } - // -- entries_matching --------------------------------------------------- #[test] @@ -467,7 +410,7 @@ mod tests { reg.register("data/a.parquet", local_entry()); reg.register( "data/b.parquet", - TieredFileEntry::new(FileLocation::Remote, None, None, None, Some(100)), + TieredFileEntry::new(FileLocation::Remote, None), ); reg.register("other/c.parquet", local_entry()); diff --git a/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store.rs b/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store.rs index 99fad8992d1e4..b39f32cac02cf 100644 --- a/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store.rs +++ b/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store.rs @@ -10,7 +10,7 @@ //! based on [`TieredStorageRegistry`] metadata. //! //! On every read, it checks the file registry: -//! - **Remote** → delegates to the remote backend via the store in the entry +//! - **Remote** → delegates to the store-level remote backend //! - **Local / Both / not registered** → falls through to the local store //! //! # Thread Safety @@ -42,11 +42,12 @@ use crate::types::{FileLocation, TieredFileEntry}; /// ObjectStore implementation that routes reads between local and remote /// stores based on [`TieredStorageRegistry`] metadata. /// -/// File tracking is delegated to the registry. Remote stores are passed -/// directly when registering files. +/// Per-shard model: one remote store is set once via [`set_remote()`] and +/// shared across all entries. pub struct TieredObjectStore { registry: Arc, local: Arc, + remote: std::sync::OnceLock>, } impl TieredObjectStore { @@ -54,7 +55,11 @@ impl TieredObjectStore { #[must_use] pub fn new(registry: Arc, local: Arc) -> Self { native_bridge_common::log_info!("TieredObjectStore: created"); - Self { registry, local } + Self { + registry, + local, + remote: std::sync::OnceLock::new(), + } } /// Reference to the underlying registry. @@ -63,52 +68,27 @@ impl TieredObjectStore { &self.registry } - /// Validate that Remote/Both locations have required remote metadata. - fn validate_remote_fields( - path: &str, - location: FileLocation, - remote_path: &Option, - repo_key: &Option, - store: &Option>, - ) -> Result<(), crate::types::FileRegistryError> { - if matches!(location, FileLocation::Remote | FileLocation::Both) { - if remote_path.is_none() { - return Err(crate::types::FileRegistryError::InvalidRegistration { - path: path.to_string(), - reason: format!("remote_path required for location={}", location), - }); - } - if repo_key.is_none() { - return Err(crate::types::FileRegistryError::InvalidRegistration { - path: path.to_string(), - reason: format!("repo_key required for location={}", location), - }); - } - if store.is_none() { - return Err(crate::types::FileRegistryError::InvalidRegistration { - path: path.to_string(), - reason: format!("store required for location={}", location), - }); - } - } - Ok(()) + /// Set the remote store (once). Subsequent calls are ignored. + pub fn set_remote(&self, store: Arc) { + self.remote.set(store).ok(); // ignore if already set } /// Register a file in the registry. For Remote/Both locations, the caller - /// must provide the resolved `store` directly. + /// must provide a `remote_path`. pub fn register_file( &self, path: &str, location: FileLocation, remote_path: Option, - repo_key: Option, - store: Option>, ) -> Result<(), crate::types::FileRegistryError> { - Self::validate_remote_fields(path, location, &remote_path, &repo_key, &store)?; - - let remote_arc: Option> = remote_path.map(Arc::from); + if matches!(location, FileLocation::Remote) && remote_path.is_none() { + return Err(crate::types::FileRegistryError::InvalidRegistration { + path: path.to_string(), + reason: format!("remote_path required for location={}", location), + }); + } - let entry = TieredFileEntry::new(location, remote_arc, repo_key, store, None); + let entry = TieredFileEntry::new(location, remote_path.map(Arc::from)); self.registry.register(path, entry); native_bridge_common::log_debug!( @@ -125,19 +105,19 @@ impl TieredObjectStore { path: &str, location: FileLocation, remote_path: Option, - repo_key: Option, - store: Option>, ) -> Result<(), crate::types::FileRegistryError> { - Self::validate_remote_fields(path, location, &remote_path, &repo_key, &store)?; + if matches!(location, FileLocation::Remote) && remote_path.is_none() { + return Err(crate::types::FileRegistryError::InvalidRegistration { + path: path.to_string(), + reason: format!("remote_path required for location={}", location), + }); + } let remote_arc: Option> = remote_path.map(Arc::from); - let repo_arc: Option> = repo_key.map(Arc::from); self.registry.update(path, move |e| { e.location = location; e.remote_path = remote_arc; - e.repo_key = repo_arc; - e.remote_store = store; }); native_bridge_common::log_debug!( @@ -152,20 +132,38 @@ impl TieredObjectStore { // TODO: Add schedule_eviction(path) and sweep() for deferred eviction lifecycle. // NOTE: The guard is intentionally dropped before I/O. The Arc - // keeps the store alive independently. If eviction lifecycle is added in the future, - // this method should return the guard alongside the resolved path/store to pin the - // entry for the duration of the I/O operation. + // keeps the store alive independently. On writable warm, the guard must be held + // during I/O to prevent eviction race — resolve_remote should return the guard + // alongside the resolved path/store to pin the entry for the I/O duration. fn resolve_remote(&self, path: &str) -> Option<(Path, Arc)> { let guard = self.registry.get(path)?; if guard.location() != FileLocation::Remote { return None; } let remote_path = guard.remote_path()?; - let store = Arc::clone(guard.remote_store()?); + let store = Arc::clone(self.remote.get()?); // use store-level remote let rp = Path::from(remote_path); drop(guard); // release before I/O — Arc keeps store alive Some((rp, store)) } + + /// Checks if a local read error is NotFound and the file has since transitioned + /// to REMOTE in the registry (e.g., afterSyncToRemote deleted the local copy). + /// Returns the remote path + store if retry is possible, None otherwise. + fn should_retry_remote(&self, path_str: &str, err: &object_store::Error) -> Option<(Path, Arc)> { + if matches!(err, object_store::Error::NotFound { .. }) { + let resolved = self.resolve_remote(path_str); + if resolved.is_some() { + native_bridge_common::log_info!( + "TieredObjectStore: LOCAL NotFound, file transitioned to REMOTE — retrying path='{}'", + path_str + ); + } + resolved + } else { + None + } + } } impl fmt::Debug for TieredObjectStore { @@ -189,23 +187,23 @@ impl fmt::Display for TieredObjectStore { #[async_trait] impl ObjectStore for TieredObjectStore { /// Write to local store and register the file as [`FileLocation::Local`]. + /// On writable warm, caller must pin the file to prevent eviction before + /// sync completes. async fn put_opts( &self, location: &Path, payload: PutPayload, opts: PutOptions, ) -> OsResult { - let size = payload.content_length() as u64; let result = self.local.put_opts(location, payload, opts).await?; let path_str = location.as_ref(); - let entry = TieredFileEntry::new(FileLocation::Local, None, None, None, Some(size)); + let entry = TieredFileEntry::new(FileLocation::Local, None); self.registry.register(path_str, entry); native_bridge_common::log_debug!( - "TieredObjectStore: put_opts registered LOCAL path='{}', size={}", + "TieredObjectStore: put_opts registered LOCAL path='{}'", path_str, - size ); Ok(result) } @@ -221,25 +219,28 @@ impl ObjectStore for TieredObjectStore { } /// Primary read path: check registry for remote routing, otherwise local. + /// If local read fails with NotFound and file transitioned to REMOTE, retries from remote. async fn get_opts(&self, location: &Path, options: GetOptions) -> OsResult { let path_str = location.as_ref(); if let Some((rp, store)) = self.resolve_remote(path_str) { native_bridge_common::log_debug!( - "TieredObjectStore: get_opts routing REMOTE path='{}'", + "TieredObjectStore: get_opts REMOTE path='{}'", path_str ); return store.get_opts(&rp, options).await; } - native_bridge_common::log_debug!( - "TieredObjectStore: get_opts routing LOCAL path='{}'", - path_str - ); - self.local.get_opts(location, options).await + let result = self.local.get_opts(location, options.clone()).await; + if let Err(ref e) = result { + if let Some((rp, store)) = self.should_retry_remote(path_str, e) { + return store.get_opts(&rp, options).await; + } + } + result } - /// Range read: same routing as `get_opts`. + /// Range read with local-fail-retry-remote. async fn get_range(&self, location: &Path, range: Range) -> OsResult { let path_str = location.as_ref(); @@ -247,10 +248,16 @@ impl ObjectStore for TieredObjectStore { return store.get_range(&rp, range).await; } - self.local.get_range(location, range).await + let result = self.local.get_range(location, range.clone()).await; + if let Err(ref e) = result { + if let Some((rp, store)) = self.should_retry_remote(path_str, e) { + return store.get_range(&rp, range).await; + } + } + result } - /// Multi-range read: same routing as `get_opts` for the entire batch. + /// Multi-range read with local-fail-retry-remote. async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> OsResult> { let path_str = location.as_ref(); @@ -258,40 +265,54 @@ impl ObjectStore for TieredObjectStore { return store.get_ranges(&rp, ranges).await; } - self.local.get_ranges(location, ranges).await + let result = self.local.get_ranges(location, ranges).await; + if let Err(ref e) = result { + if let Some((rp, store)) = self.should_retry_remote(path_str, e) { + return store.get_ranges(&rp, ranges).await; + } + } + result } - /// Head: try local first, fall back to remote if not found locally. + /// Head: check registry first (cached size), then local, then remote. + /// Head: check registry for cached size first (no I/O), then route via resolve_remote. async fn head(&self, location: &Path) -> OsResult { let path_str = location.as_ref(); - match self.local.head(location).await { - Ok(meta) => return Ok(meta), - Err(object_store::Error::NotFound { .. }) => {} - Err(other) => return Err(other), + // Check registry for cached size — return immediately if available + if let Some(guard) = self.registry.get(path_str) { + let size = guard.size(); + if size > 0 { + return Ok(ObjectMeta { + location: location.clone(), + last_modified: chrono::DateTime::::default(), + size, + e_tag: None, + version: None, + }); + } } + // Size not cached — route via resolve_remote (REMOTE → remote store) if let Some((rp, store)) = self.resolve_remote(path_str) { return store.head(&rp).await; } - Err(object_store::Error::NotFound { - path: path_str.to_string(), - source: "TieredObjectStore: not found locally or in registry".into(), - }) + // Not remote — try local + self.local.head(location).await } /// Delete: remove from registry only, NO local delete. - /// Local file deletion is handled by the Java layer (CompositeDirectory). - // TODO: Consider deferred removal (schedule + sweep) instead of force-remove - // when eviction lifecycle is added. + /// Local file deletion is handled by the Java layer + /// (TieredSubdirectoryAwareDirectory.deleteFile). Eviction (local copy + /// removal after sync) is a writable warm concern — not implemented. async fn delete(&self, location: &Path) -> OsResult<()> { let path_str = location.as_ref(); self.registry.remove(path_str, true); Ok(()) } - /// List: local entries first, then remote-only entries from registry. + /// List: local entries first, then remote-only entries from registry (deduplicated). fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OsResult> { let prefix_str = prefix.map(|p| p.as_ref().to_string()).unwrap_or_default(); let registry = Arc::clone(&self.registry); @@ -305,7 +326,7 @@ impl ObjectStore for TieredObjectStore { Ok(ObjectMeta { location: Path::from(path), last_modified: chrono::DateTime::::default(), - size: size.unwrap_or(0), + size, e_tag: None, version: None, }) @@ -316,7 +337,7 @@ impl ObjectStore for TieredObjectStore { Box::pin(local_stream.chain(remote_stream)) } - /// List with delimiter: local entries first, then merge remote-only entries. + /// List with delimiter: local entries first, then merge remote-only entries (deduplicated). async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OsResult { let mut result = self.local.list_with_delimiter(prefix).await?; @@ -333,7 +354,7 @@ impl ObjectStore for TieredObjectStore { result.objects.push(ObjectMeta { location: Path::from(path), last_modified: chrono::DateTime::::default(), - size: size.unwrap_or(0), + size, e_tag: None, version: None, }); diff --git a/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store_tests.rs b/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store_tests.rs index 1c65080b088a0..d7e5922a8e98a 100644 --- a/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store_tests.rs +++ b/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store_tests.rs @@ -15,6 +15,7 @@ fn setup() -> ( let local = Arc::new(InMemory::new()); let remote = Arc::new(InMemory::new()); let tiered = TieredObjectStore::new(Arc::clone(®istry), Arc::clone(&local) as _); + tiered.set_remote(Arc::clone(&remote) as _); (registry, local, remote, tiered) } @@ -35,8 +36,6 @@ async fn test_get_opts_routes_to_remote_for_remote_file() { "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&remote) as _), ) .unwrap(); @@ -68,47 +67,6 @@ async fn test_get_opts_routes_to_local_when_not_in_registry() { assert_eq!(bytes.as_ref(), b"local-data"); } -#[tokio::test] -async fn test_get_opts_routes_to_local_for_both_file() { - let (_registry, local, remote, tiered) = setup(); - - local - .put( - &Path::from("a.parquet"), - PutPayload::from_static(b"local-data"), - ) - .await - .unwrap(); - remote - .put( - &Path::from("remote/a.parquet"), - PutPayload::from_static(b"remote-data"), - ) - .await - .unwrap(); - - tiered - .register_file( - "a.parquet", - FileLocation::Both, - Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&remote) as _), - ) - .unwrap(); - - let result = tiered - .get_opts(&Path::from("a.parquet"), GetOptions::default()) - .await - .unwrap(); - let bytes = result.bytes().await.unwrap(); - assert_eq!( - bytes.as_ref(), - b"local-data", - "Both files should route to local" - ); -} - #[tokio::test] async fn test_get_opts_routes_to_local_for_local_file() { let (_registry, local, _remote, tiered) = setup(); @@ -122,7 +80,7 @@ async fn test_get_opts_routes_to_local_for_local_file() { .unwrap(); tiered - .register_file("a.parquet", FileLocation::Local, None, None, None) + .register_file("a.parquet", FileLocation::Local, None) .unwrap(); let result = tiered @@ -152,8 +110,6 @@ async fn test_successful_remote_read_releases_ref_count() { "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&remote) as _), ) .unwrap(); @@ -200,8 +156,6 @@ async fn test_head_falls_back_to_remote() { "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&remote) as _), ) .unwrap(); @@ -237,24 +191,6 @@ async fn test_put_writes_local_and_registers() { assert_eq!(registry.len(), 1); } -#[tokio::test] -async fn test_put_opts_caches_file_size() { - let (registry, _local, _remote, tiered) = setup(); - - tiered - .put_opts( - &Path::from("sized.parquet"), - PutPayload::from_static(b"hello world"), - PutOptions::default(), - ) - .await - .unwrap(); - - let entries = registry.entries_matching("sized.parquet"); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].2, Some(11)); -} - // -- Delete ------------------------------------------------------------- #[tokio::test] @@ -266,7 +202,7 @@ async fn test_delete_removes_registry_entry_only() { .await .unwrap(); tiered - .register_file("a.parquet", FileLocation::Local, None, None, None) + .register_file("a.parquet", FileLocation::Local, None) .unwrap(); tiered.delete(&Path::from("a.parquet")).await.unwrap(); @@ -310,8 +246,6 @@ async fn test_get_range_from_remote() { "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&remote) as _), ) .unwrap(); @@ -355,8 +289,6 @@ async fn test_get_ranges_multiple_from_remote() { "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&remote) as _), ) .unwrap(); @@ -397,7 +329,7 @@ async fn test_rename_returns_not_supported() { #[tokio::test] async fn test_list_includes_remote_only_files() { - let (registry, local, remote, tiered) = setup(); + let (_registry, local, remote, tiered) = setup(); local .put( @@ -419,11 +351,8 @@ async fn test_list_includes_remote_only_files() { "data/evicted.parquet", FileLocation::Remote, Some("remote/evicted.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&remote) as _), ) .unwrap(); - registry.update("data/evicted.parquet", |e| e.size = Some(11)); let results: Vec = tiered .list(Some(&Path::from("data"))) @@ -436,12 +365,6 @@ async fn test_list_includes_remote_only_files() { let paths: Vec = results.iter().map(|m| m.location.to_string()).collect(); assert!(paths.contains(&"data/local.parquet".to_string())); assert!(paths.contains(&"data/evicted.parquet".to_string())); - - let evicted_meta = results - .iter() - .find(|m| m.location.as_ref() == "data/evicted.parquet") - .unwrap(); - assert_eq!(evicted_meta.size, 11); } #[tokio::test] @@ -456,7 +379,7 @@ async fn test_list_no_duplicates_for_local_files() { .await .unwrap(); tiered - .register_file("data/a.parquet", FileLocation::Local, None, None, None) + .register_file("data/a.parquet", FileLocation::Local, None) .unwrap(); let results: Vec = tiered @@ -498,8 +421,6 @@ async fn test_list_with_delimiter_includes_remote() { "data/evicted.parquet", FileLocation::Remote, Some("remote/evicted.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&remote) as _), ) .unwrap(); @@ -537,8 +458,6 @@ async fn test_concurrent_get_opts_on_same_remote_file() { "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&remote) as _), ) .unwrap(); @@ -657,13 +576,12 @@ async fn test_mock_store_exactly_one_call_per_get_opts() { .unwrap(); let tiered = TieredObjectStore::new(Arc::clone(®istry), local as _); + tiered.set_remote(Arc::clone(&mock_remote) as _); tiered .register_file( "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&mock_remote) as _), ) .unwrap(); @@ -773,13 +691,12 @@ async fn test_error_store_guard_still_releases() { let error_remote: Arc = Arc::new(ErrorStore); let tiered = TieredObjectStore::new(Arc::clone(®istry), local as _); + tiered.set_remote(Arc::clone(&error_remote)); tiered .register_file( "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&error_remote)), ) .unwrap(); @@ -804,38 +721,6 @@ fn test_register_file_remote_without_remote_path_returns_err() { "/a.parquet", FileLocation::Remote, None, - Some("repo1".into()), - Some(Arc::new(InMemory::new()) as _), - ); - assert!(result.is_err()); -} - -#[test] -fn test_register_file_remote_without_repo_key_returns_err() { - let registry = Arc::new(TieredStorageRegistry::new()); - let local = Arc::new(InMemory::new()); - let tiered = TieredObjectStore::new(registry, local as _); - let result = tiered.register_file( - "/a.parquet", - FileLocation::Remote, - Some("remote/a".into()), - None, - Some(Arc::new(InMemory::new()) as _), - ); - assert!(result.is_err()); -} - -#[test] -fn test_register_file_remote_without_store_returns_err() { - let registry = Arc::new(TieredStorageRegistry::new()); - let local = Arc::new(InMemory::new()); - let tiered = TieredObjectStore::new(registry, local as _); - let result = tiered.register_file( - "/a.parquet", - FileLocation::Remote, - Some("remote/a".into()), - Some("repo1".into()), - None, ); assert!(result.is_err()); } @@ -844,7 +729,7 @@ fn test_register_file_remote_without_store_returns_err() { #[tokio::test] async fn test_failed_remote_read_not_found_still_completes() { - let (registry, _local, remote, tiered) = setup(); + let (registry, _local, _remote, tiered) = setup(); // Register a Remote file pointing to a path that doesn't exist on the remote store. tiered @@ -852,8 +737,6 @@ async fn test_failed_remote_read_not_found_still_completes() { "missing.parquet", FileLocation::Remote, Some("remote/nonexistent.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&remote) as _), ) .unwrap(); @@ -874,13 +757,12 @@ async fn test_get_range_error_from_remote_still_completes() { let error_remote: Arc = Arc::new(ErrorStore); let tiered = TieredObjectStore::new(Arc::clone(®istry), local as _); + tiered.set_remote(Arc::clone(&error_remote)); tiered .register_file( "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&error_remote)), ) .unwrap(); @@ -899,13 +781,12 @@ async fn test_get_ranges_error_from_remote_still_completes() { let error_remote: Arc = Arc::new(ErrorStore); let tiered = TieredObjectStore::new(Arc::clone(®istry), local as _); + tiered.set_remote(Arc::clone(&error_remote)); tiered .register_file( "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&error_remote)), ) .unwrap(); @@ -927,13 +808,12 @@ async fn test_head_remote_fallback_error_still_completes() { // File not found locally. Register as Remote with ErrorStore. let tiered = TieredObjectStore::new(Arc::clone(®istry), local as _); + tiered.set_remote(Arc::clone(&error_remote)); tiered .register_file( "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&error_remote)), ) .unwrap(); @@ -963,8 +843,6 @@ async fn test_concurrent_read_and_delete() { "a.parquet", FileLocation::Remote, Some("remote/a.parquet".into()), - Some("repo1".into()), - Some(Arc::clone(&remote) as _), ) .unwrap(); @@ -1037,5 +915,5 @@ fn test_delete_during_active_guard() { // Helper: create a local entry (reused by guard tests above). fn local_entry() -> TieredFileEntry { - TieredFileEntry::new(FileLocation::Local, None, None, None, None) + TieredFileEntry::new(FileLocation::Local, None) } diff --git a/sandbox/libs/tiered-storage/src/main/rust/src/types.rs b/sandbox/libs/tiered-storage/src/main/rust/src/types.rs index 41285ac0e61f4..fbddb6bcbf04f 100644 --- a/sandbox/libs/tiered-storage/src/main/rust/src/types.rs +++ b/sandbox/libs/tiered-storage/src/main/rust/src/types.rs @@ -14,7 +14,6 @@ use std::sync::atomic::{AtomicI64, Ordering}; use std::sync::Arc; use dashmap::mapref::one::Ref; -use object_store::ObjectStore; // --------------------------------------------------------------------------- // FileRegistryError @@ -54,8 +53,6 @@ pub enum FileLocation { Local = 0, /// File exists only on a remote object store. Remote = 1, - /// File exists on both local disk and remote store. - Both = 2, } impl fmt::Display for FileLocation { @@ -63,7 +60,6 @@ impl fmt::Display for FileLocation { match self { Self::Local => write!(f, "Local"), Self::Remote => write!(f, "Remote"), - Self::Both => write!(f, "Both"), } } } @@ -77,7 +73,6 @@ impl FileLocation { match v { 0 => Some(Self::Local), 1 => Some(Self::Remote), - 2 => Some(Self::Both), _ => None, } } @@ -89,21 +84,19 @@ impl FileLocation { /// Per-file metadata stored in the registry. /// -/// Fields are ordered by alignment to minimise struct padding. -/// Ref counting is managed directly on the entry via `acquire()` / `release()`. +/// Per-file entry in the tiered storage registry. +/// +/// Tracks location, remote path, size, and active reader count. +/// The remote store lives on `TieredObjectStore`, not per-entry. pub struct TieredFileEntry { /// Number of active readers. Atomic for lock-free concurrent access. pub(crate) active_reads: AtomicI64, /// Path on the remote store. Stored as `Arc` for cheap cloning. pub(crate) remote_path: Option>, - /// Repository key for looking up the remote [`ObjectStore`]. - pub(crate) repo_key: Option>, - /// Remote [`ObjectStore`] reference, resolved at registration time. - pub(crate) remote_store: Option>, - /// Cached file size in bytes (from head or put). - pub(crate) size: Option, /// Current location of the file data. pub(crate) location: FileLocation, + /// File size in bytes. Cached at registration time for head()/list() without I/O. + pub(crate) size: u64, } impl fmt::Debug for TieredFileEntry { @@ -111,37 +104,30 @@ impl fmt::Debug for TieredFileEntry { f.debug_struct("TieredFileEntry") .field("location", &self.location) .field("remote_path", &self.remote_path) - .field("repo_key", &self.repo_key) - .field( - "remote_store", - if self.remote_store.is_some() { - &"Some(...)" as &dyn fmt::Debug - } else { - &"None" as &dyn fmt::Debug - }, - ) - .field("active_reads", &self.active_reads.load(Ordering::SeqCst)) .field("size", &self.size) + .field("active_reads", &self.active_reads.load(Ordering::SeqCst)) .finish() } } impl TieredFileEntry { /// Create a new entry with the given location and zero active readers. - pub fn new( - location: FileLocation, - remote_path: Option>, - repo_key: Option, - remote_store: Option>, - size: Option, - ) -> Self { + pub fn new(location: FileLocation, remote_path: Option>) -> Self { + Self { + active_reads: AtomicI64::new(0), + remote_path, + location, + size: 0, + } + } + + /// Create a new entry with location, remote path, and cached size. + pub fn with_size(location: FileLocation, remote_path: Option>, size: u64) -> Self { Self { active_reads: AtomicI64::new(0), remote_path, - repo_key: repo_key.map(Arc::from), - remote_store, - size, location, + size, } } @@ -185,21 +171,9 @@ impl TieredFileEntry { self.remote_path.as_deref() } - /// Repository key, if any. - #[must_use] - pub fn repo_key(&self) -> Option<&str> { - self.repo_key.as_deref() - } - - /// Remote [`ObjectStore`] reference, if any. - #[must_use] - pub fn remote_store(&self) -> Option<&Arc> { - self.remote_store.as_ref() - } - - /// Cached file size. + /// Cached file size in bytes (0 if not cached). #[must_use] - pub fn file_size(&self) -> Option { + pub fn size(&self) -> u64 { self.size } } @@ -238,9 +212,9 @@ impl<'a> ReadGuard<'a> { self.entry.value().remote_path() } - /// Remote [`ObjectStore`] reference, if any. - pub fn remote_store(&self) -> Option<&Arc> { - self.entry.value().remote_store() + /// Cached file size in bytes (0 if not cached). + pub fn size(&self) -> u64 { + self.entry.value().size() } /// Current reference count (including this guard). diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java index 7ec61ceded267..86c58196f485e 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java @@ -20,6 +20,7 @@ import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.engine.dataformat.IndexingEngineConfig; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.StoreStrategy; import org.opensearch.plugins.ExtensiblePlugin; import org.opensearch.plugins.Plugin; @@ -31,20 +32,21 @@ /** * Sandbox plugin that provides a {@link CompositeIndexingExecutionEngine} for - * orchestrating multi-format indexing. Discovers {@link DataFormatPlugin} instances - * during node bootstrap via the {@link ExtensiblePlugin} SPI and creates a composite - * engine when composite indexing is enabled for an index. - *

      - * Registers two index settings: + * orchestrating multi-format indexing. Discovers {@link DataFormatPlugin} + * instances during node bootstrap via the {@link ExtensiblePlugin} SPI and + * creates a composite engine when composite indexing is enabled for an index. + * + *

      Registers two index settings: *

        - *
      • {@code index.composite.primary_data_format} — designates the primary format (default {@code "lucene"})
        • - *
      • {@code index.composite.secondary_data_formats} — lists the secondary formats (default empty)
        • + *
      • {@code index.composite.primary_data_format} — designates the primary + * format (default {@code "lucene"})
        • + *
      • {@code index.composite.secondary_data_formats} — lists the secondary + * formats (default empty)
        • *
      - *

      - * Format plugins (e.g., Parquet) extend this plugin by declaring + * + *

      Format plugins (e.g., Parquet) extend this plugin by declaring * {@code extendedPlugins = ['composite-engine']} in their {@code build.gradle} - * and implementing {@link DataFormatPlugin}. The {@link ExtensiblePlugin} SPI - * discovers them automatically during node bootstrap. + * and implementing {@link DataFormatPlugin}. * * @opensearch.experimental */ @@ -77,7 +79,6 @@ public class CompositeDataFormatPlugin extends Plugin implements DataFormatPlugi Setting.Property.Final ); - /** Creates a new composite engine plugin. */ public CompositeDataFormatPlugin() {} @Override @@ -87,7 +88,6 @@ public List> getSettings() { @Override public DataFormat getDataFormat() { - // TODO: Dataformat for Composite is per index, while this one talks about cluster level. Switching it off for now return new CompositeDataFormat(); } @@ -123,4 +123,28 @@ public Map> getFormatDescriptors( } return Map.copyOf(descriptors); } + + /** + * Returns the store strategies from every participating sub-format plugin + * (primary + secondary), keyed by format name. Mirrors {@link #getFormatDescriptors}: + * each participating format is resolved through the registry, which delegates + * to the sub-plugin without re-entering this composite. + */ + @Override + public Map getStoreStrategies(IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry) { + Settings settings = indexSettings.getSettings(); + String primaryFormatName = PRIMARY_DATA_FORMAT.get(settings); + List secondaryFormatNames = SECONDARY_DATA_FORMATS.get(settings); + + Map strategies = new HashMap<>(); + if (primaryFormatName != null && primaryFormatName.isEmpty() == false) { + strategies.putAll(dataFormatRegistry.getStoreStrategies(indexSettings, dataFormatRegistry.format(primaryFormatName))); + } + for (String secondaryName : secondaryFormatNames) { + if (secondaryName != null && secondaryName.isEmpty() == false) { + strategies.putAll(dataFormatRegistry.getStoreStrategies(indexSettings, dataFormatRegistry.format(secondaryName))); + } + } + return Map.copyOf(strategies); + } } diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java index c48306473197e..3cd20ce6e9530 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java @@ -8,11 +8,15 @@ package org.opensearch.composite; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.DataFormatDescriptor; import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.dataformat.StoreStrategy; import org.opensearch.test.OpenSearchTestCase; import java.util.List; @@ -48,18 +52,12 @@ public void testSecondaryDataFormatsDefaultsToEmpty() { public void testGetFormatDescriptorsDelegatestoPlugins() { CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); - // Build index settings with parquet as secondary - Settings settings = Settings.builder() - .put("index.composite.primary_data_format", "lucene") - .putList("index.composite.secondary_data_formats", "parquet") - .put(org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT) - .put(org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .build(); - org.opensearch.cluster.metadata.IndexMetadata indexMetadata = org.opensearch.cluster.metadata.IndexMetadata.builder("test-index") - .settings(settings) - .build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + IndexSettings indexSettings = buildIndexSettings( + Settings.builder() + .put("index.composite.primary_data_format", "lucene") + .putList("index.composite.secondary_data_formats", "parquet") + .build() + ); DataFormatRegistry registry = mock(DataFormatRegistry.class); DataFormat parquetFormat = CompositeTestHelper.stubFormat("parquet", 2, java.util.Set.of()); @@ -68,18 +66,14 @@ public void testGetFormatDescriptorsDelegatestoPlugins() { when(registry.getFormatDescriptors(indexSettings, parquetFormat)).thenReturn( Map.of( "parquet", - (Supplier< - org.opensearch.index.engine.dataformat.DataFormatDescriptor>) () -> new org.opensearch.index.engine.dataformat.DataFormatDescriptor( - "parquet", - new org.opensearch.index.store.checksum.GenericCRC32ChecksumHandler() - ) + (Supplier) () -> new DataFormatDescriptor( + "parquet", + new org.opensearch.index.store.checksum.GenericCRC32ChecksumHandler() + ) ) ); - Map> descriptors = plugin.getFormatDescriptors( - indexSettings, - registry - ); + Map> descriptors = plugin.getFormatDescriptors(indexSettings, registry); assertEquals(1, descriptors.size()); assertTrue(descriptors.containsKey("parquet")); assertEquals("parquet", descriptors.get("parquet").get().getFormatName()); @@ -89,20 +83,88 @@ public void testGetFormatDescriptorsEmptyWhenNoPluginsMatch() { CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); DataFormatRegistry registry = mock(DataFormatRegistry.class); - Settings settings = Settings.builder() - .put(org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT) - .put(org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .build(); - org.opensearch.cluster.metadata.IndexMetadata indexMetadata = org.opensearch.cluster.metadata.IndexMetadata.builder("test-index") - .settings(settings) - .build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + IndexSettings indexSettings = buildIndexSettings(Settings.EMPTY); - Map> descriptors = plugin.getFormatDescriptors( - indexSettings, - registry - ); + Map> descriptors = plugin.getFormatDescriptors(indexSettings, registry); assertTrue(descriptors.isEmpty()); } + + public void testGetStoreStrategiesEmptyWhenNoSubPlugins() { + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + + IndexSettings indexSettings = buildIndexSettings(Settings.builder().put("index.composite.primary_data_format", "parquet").build()); + // Registry resolves the format but the composite pipeline finds no plugin → empty map. + DataFormat parquetFormat = CompositeTestHelper.stubFormat("parquet", 2, java.util.Set.of()); + when(registry.format("parquet")).thenReturn(parquetFormat); + when(registry.getStoreStrategies(indexSettings, parquetFormat)).thenReturn(Map.of()); + + Map result = plugin.getStoreStrategies(indexSettings, registry); + assertTrue("Should return empty when no sub-plugin found", result.isEmpty()); + } + + public void testGetStoreStrategiesCollectsFromPrimaryPlugin() { + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + + IndexSettings indexSettings = buildIndexSettings(Settings.builder().put("index.composite.primary_data_format", "parquet").build()); + + DataFormat parquetFormat = CompositeTestHelper.stubFormat("parquet", 2, java.util.Set.of()); + StoreStrategy parquetStrategy = mock(StoreStrategy.class); + when(registry.format("parquet")).thenReturn(parquetFormat); + when(registry.getStoreStrategies(indexSettings, parquetFormat)).thenReturn(Map.of(parquetFormat, parquetStrategy)); + + Map result = plugin.getStoreStrategies(indexSettings, registry); + assertEquals(1, result.size()); + assertSame(parquetStrategy, result.get(parquetFormat)); + } + + public void testGetStoreStrategiesCollectsPrimaryAndSecondary() { + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + + IndexSettings indexSettings = buildIndexSettings( + Settings.builder() + .put("index.composite.primary_data_format", "lucene") + .putList("index.composite.secondary_data_formats", "parquet") + .build() + ); + + DataFormat luceneFormat = CompositeTestHelper.stubFormat("lucene", 1, java.util.Set.of()); + DataFormat parquetFormat = CompositeTestHelper.stubFormat("parquet", 2, java.util.Set.of()); + StoreStrategy parquetStrategy = mock(StoreStrategy.class); + + when(registry.format("lucene")).thenReturn(luceneFormat); + when(registry.format("parquet")).thenReturn(parquetFormat); + when(registry.getStoreStrategies(indexSettings, luceneFormat)).thenReturn(Map.of()); + when(registry.getStoreStrategies(indexSettings, parquetFormat)).thenReturn(Map.of(parquetFormat, parquetStrategy)); + + Map result = plugin.getStoreStrategies(indexSettings, registry); + assertEquals(1, result.size()); + assertSame(parquetStrategy, result.get(parquetFormat)); + } + + public void testGetStoreStrategiesEmptyForDefaultPrimaryWithoutPlugin() { + CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin(); + DataFormatRegistry registry = mock(DataFormatRegistry.class); + + IndexSettings indexSettings = buildIndexSettings(Settings.EMPTY); // defaults: primary=lucene, secondary=[] + DataFormat luceneFormat = CompositeTestHelper.stubFormat("lucene", 1, java.util.Set.of()); + when(registry.format("lucene")).thenReturn(luceneFormat); + when(registry.getStoreStrategies(indexSettings, luceneFormat)).thenReturn(Map.of()); + + Map result = plugin.getStoreStrategies(indexSettings, registry); + assertTrue("Should return empty when lucene sub-plugin not found", result.isEmpty()); + } + + private static IndexSettings buildIndexSettings(Settings extra) { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(extra) + .build(); + IndexMetadata indexMetadata = IndexMetadata.builder("test-index").settings(settings).build(); + return new IndexSettings(indexMetadata, Settings.EMPTY); + } } diff --git a/sandbox/plugins/native-repository-fs/build.gradle b/sandbox/plugins/native-repository-fs/build.gradle index 19e2622104804..410a7aefe32af 100644 --- a/sandbox/plugins/native-repository-fs/build.gradle +++ b/sandbox/plugins/native-repository-fs/build.gradle @@ -9,7 +9,6 @@ opensearchplugin { description = 'Native (Rust) ObjectStore backend for the FS repository plugin.' classname = 'org.opensearch.repositories.fs.native_store.FsNativeObjectStorePlugin' - extendedPlugins = ['repository-fs'] } apply plugin: 'opensearch.internal-cluster-test' diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java index 164c2d866bea7..e2c103bcadc75 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java @@ -24,10 +24,12 @@ import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.engine.dataformat.IndexingEngineConfig; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.StoreStrategy; import org.opensearch.index.store.PrecomputedChecksumStrategy; import org.opensearch.parquet.engine.ParquetDataFormat; import org.opensearch.parquet.engine.ParquetIndexingEngine; import org.opensearch.parquet.fields.ArrowSchemaBuilder; +import org.opensearch.parquet.store.ParquetStoreStrategy; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.script.ScriptService; @@ -46,24 +48,23 @@ /** * OpenSearch plugin providing the Parquet data format for indexing operations. * - *

      Implements {@link DataFormatPlugin} to register the Parquet format with OpenSearch's - * data format framework. On node startup, captures cluster settings via - * {@link #createComponents} and passes them to the per-shard + *

      Implements {@link DataFormatPlugin} to register the Parquet format with + * OpenSearch's data format framework. On node startup, captures cluster + * settings via {@link #createComponents} and passes them to the per-shard * {@link ParquetIndexingEngine} instances created in {@link #indexingEngine}. * - *

      The descriptor provides a {@link PrecomputedChecksumStrategy} that is created once - * per shard during initialization. The same strategy instance is shared between the - * directory and the {@link ParquetIndexingEngine} via the checksum strategies map, - * so pre-computed CRC32 values registered during write are directly visible to the - * upload path — no post-construction wiring needed. - * - *

      Registers plugin settings defined in {@link ParquetSettings}. + *

      For tiered storage, returns a {@link ParquetStoreStrategy} from + * {@link #getStoreStrategies}. The composite store layer takes it from there — + * construction of per-shard native registries, seeding from remote metadata, + * routing directory events, and closing native resources are all handled + * there. The plugin stays purely declarative. */ public class ParquetDataFormatPlugin extends Plugin implements DataFormatPlugin { /** Thread pool name for background native Parquet writes during VSR rotation. */ public static final String PARQUET_THREAD_POOL_NAME = "parquet_native_write"; - + private static final ParquetDataFormat dataFormat = new ParquetDataFormat(); + private static final StoreStrategy storeStrategy = new ParquetStoreStrategy(); public static final ParquetDataFormat PARQUET_DATA_FORMAT = new ParquetDataFormat(); /** Initialized to EMPTY to avoid NPE if indexingEngine() is called before createComponents(). */ private Settings settings = Settings.EMPTY; @@ -117,6 +118,15 @@ public Map> getFormatDescriptors(IndexSet ); } + @Override + public Map getStoreStrategies(IndexSettings indexSettings, DataFormatRegistry registry) { + DataFormat parquetFormat = registry.format(ParquetDataFormat.PARQUET_DATA_FORMAT_NAME); + if (parquetFormat == null) { + return Map.of(); + } + return Map.of(parquetFormat, storeStrategy); + } + @Override public List> getSettings() { return ParquetSettings.getSettings(); diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/ParquetDataFormatStoreHandler.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/ParquetDataFormatStoreHandler.java new file mode 100644 index 0000000000000..0a531fa1ba135 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/ParquetDataFormatStoreHandler.java @@ -0,0 +1,134 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.store; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.engine.dataformat.DataFormatStoreHandler; +import org.opensearch.plugins.NativeStoreHandle; +import org.opensearch.repositories.NativeStoreRepository; + +import java.io.IOException; +import java.util.Map; + +/** + * Per-shard native file registry for parquet files. + * + *

      Owns a Rust {@code TieredObjectStore} via FFM. All calls delegate to + * {@link TieredStorageBridge} which invokes the Rust {@code ts_*} functions: + *

        + *
      • {@code seed} → {@code ts_register_files} (batch, per-file location)
        • + *
      • {@code onUploaded} → {@code ts_register_files} (single file, REMOTE)
        • + *
      • {@code onRemoved} → {@code ts_remove_file}
        • + *
      • {@code close} → {@code ts_destroy_tiered_object_store}
        • + *
      + * + *

      Read-only warm (current): all parquet files are REMOTE. The registry + * is seeded from remote metadata at shard open. No local files, no eviction. + * + *

      TODO (writable warm): add getFileLocation, acquireRead, releaseRead + * when LOCAL parquet files exist and eviction is enabled. Wire + * {@code ts_get_file_location} FFM call for LOCAL/REMOTE routing. + */ +public class ParquetDataFormatStoreHandler implements DataFormatStoreHandler { + + private static final Logger logger = LogManager.getLogger(ParquetDataFormatStoreHandler.class); + private final NativeStoreHandle storeHandle; + /** Cached native object store handle for DataFusion readers — created lazily, closed with the handler. */ + private volatile NativeStoreHandle nativeStoreForReader; + + /** + * Creates a per-shard native file registry. + * On warm nodes with a live native store, creates a Rust TieredObjectStore via FFM. + * On hot nodes (or when native store is unavailable), creates an empty handle (no-op). + * + * @param shardId the shard id (for logging) + * @param isWarm true if the shard is on a warm node + * @param repo the native remote store, or {@code NativeStoreRepository.EMPTY} + */ + public ParquetDataFormatStoreHandler(ShardId shardId, boolean isWarm, NativeStoreRepository repo) { + if (isWarm) { + long remotePtr = (repo != null && repo.isLive()) ? repo.getPointer() : 0L; + long ptr = TieredStorageBridge.createTieredObjectStore(0L, remotePtr); + this.storeHandle = new NativeStoreHandle(ptr, TieredStorageBridge::destroyTieredObjectStore); + logger.debug("[{}] Created ParquetDataFormatStoreHandler with native store, ptr={}", shardId, ptr); + } else { + this.storeHandle = NativeStoreHandle.EMPTY; + } + } + + @Override + public void seed(Map files) { + if (storeHandle.isLive() == false) { + return; + } + for (Map.Entry entry : files.entrySet()) { + TieredStorageBridge.registerFile( + storeHandle.getPointer(), + entry.getKey(), + entry.getValue().path(), + entry.getValue().location(), + entry.getValue().size() + ); + } + logger.trace("seed: {} files registered", files.size()); + } + + @Override + public void onUploaded(String file, String remotePath, long size) { + if (storeHandle.isLive() == false) { + return; + } + TieredStorageBridge.registerFile(storeHandle.getPointer(), file, remotePath, REMOTE, size); + logger.trace("onUploaded: file=[{}], remotePath=[{}], size={}", file, remotePath, size); + } + + @Override + public void onRemoved(String file) { + if (storeHandle.isLive() == false) { + return; + } + TieredStorageBridge.removeFile(storeHandle.getPointer(), file); + logger.trace("onRemoved: file=[{}]", file); + } + + @Override + public NativeStoreHandle getFormatStoreHandle() { + if (storeHandle.isLive() == false) { + return NativeStoreHandle.EMPTY; + } + // Lazily create the boxed pointer once — same lifetime as the handler (shard lifetime). + // The box holds an Arc clone of the TieredObjectStore, keeping it alive independently. + if (nativeStoreForReader == null) { + synchronized (this) { + if (nativeStoreForReader == null) { + try { + long boxPtr = TieredStorageBridge.getObjectStoreBoxPtr(storeHandle.getPointer()); + if (boxPtr > 0) { + nativeStoreForReader = new NativeStoreHandle(boxPtr, TieredStorageBridge::destroyObjectStoreBoxPtr); + } + } catch (Exception e) { + logger.error("getFormatStoreHandle: failed to get object store box ptr", e); + } + } + } + } + return nativeStoreForReader != null ? nativeStoreForReader : NativeStoreHandle.EMPTY; + } + + @Override + public void close() throws IOException { + // Close box handle first (decrements Arc refcount), then the store handle (frees TieredObjectStore). + if (nativeStoreForReader != null) { + nativeStoreForReader.close(); + } + storeHandle.close(); + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/ParquetStoreStrategy.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/ParquetStoreStrategy.java new file mode 100644 index 0000000000000..a5c27e809e526 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/ParquetStoreStrategy.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.store; + +import org.opensearch.index.engine.dataformat.DataFormatStoreHandlerFactory; +import org.opensearch.index.engine.dataformat.StoreStrategy; + +import java.util.Optional; + +/** + * Store strategy for the parquet data format. + * + *

      Uses the default {@code owns} / {@code remotePath} behaviour inherited + * from {@link StoreStrategy} (files live under {@code "parquet/"} prefix, blobs + * are laid out at {@code basePath + "parquet/" + blobKey}). The store layer + * supplies the format name when it invokes those methods, so the strategy + * itself does not carry the name. + * + *

      Provides a factory for the per-shard native file registry that tracks + * parquet files for the Rust reader. + */ +public final class ParquetStoreStrategy implements StoreStrategy { + + private static final DataFormatStoreHandlerFactory FACTORY = ParquetDataFormatStoreHandler::new; + + @Override + public Optional storeHandler() { + return Optional.of(FACTORY); + } +} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/TieredStorageBridge.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/TieredStorageBridge.java new file mode 100644 index 0000000000000..343a1e17a7bbe --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/store/TieredStorageBridge.java @@ -0,0 +1,184 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.store; + +import org.opensearch.nativebridge.spi.NativeLibraryLoader; + +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; + +/** + * FFM bridge for tiered storage Rust functions. + * + *

      Methods: create/destroy TieredObjectStore, batch register files, remove file. + * + *

      The {@code registerFiles} method uses a newline-delimited batch format: + * {@code path\nremotePath\npath\nremotePath\n...} Empty remotePath for LOCAL files. + * This avoids per-file FFM overhead when seeding hundreds of files at shard open. + */ +public final class TieredStorageBridge { + + private static final MethodHandle CREATE; + private static final MethodHandle DESTROY; + private static final MethodHandle REGISTER_FILES; + private static final MethodHandle REMOVE_FILE; + private static final MethodHandle GET_OBJECT_STORE_BOX_PTR; + private static final MethodHandle DESTROY_OBJECT_STORE_BOX_PTR; + + static { + SymbolLookup lib = NativeLibraryLoader.symbolLookup(); + Linker linker = Linker.nativeLinker(); + + CREATE = linker.downcallHandle( + lib.find("ts_create_tiered_object_store").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + DESTROY = linker.downcallHandle( + lib.find("ts_destroy_tiered_object_store").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + REGISTER_FILES = linker.downcallHandle( + lib.find("ts_register_files").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT + ) + ); + REMOVE_FILE = linker.downcallHandle( + lib.find("ts_remove_file").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) + ); + GET_OBJECT_STORE_BOX_PTR = linker.downcallHandle( + lib.find("ts_get_object_store_box_ptr").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + // Optional — graceful if native library is stale and symbol not yet available. + DESTROY_OBJECT_STORE_BOX_PTR = lib.find("ts_destroy_object_store_box_ptr") + .map(sym -> linker.downcallHandle(sym, FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG))) + .orElse(null); + } + + private TieredStorageBridge() {} + + /** + * Create a TieredObjectStore with optional local and remote stores. + * + * @param localStorePtr Box<Arc<dyn ObjectStore>> pointer, or 0 for default LocalFileSystem + * @param remoteStorePtr Box<Arc<dyn ObjectStore>> pointer, or 0 for no remote + * @return native pointer to the TieredObjectStore + */ + public static long createTieredObjectStore(long localStorePtr, long remoteStorePtr) { + try { + return NativeLibraryLoader.checkResult((long) CREATE.invokeExact(localStorePtr, remoteStorePtr)); + } catch (Throwable t) { + throw new RuntimeException("Failed to create TieredObjectStore", t); + } + } + + /** Destroy a TieredObjectStore and its internal registry. */ + public static void destroyTieredObjectStore(long ptr) { + try { + NativeLibraryLoader.checkResult((long) DESTROY.invokeExact(ptr)); + } catch (Throwable t) { + throw new RuntimeException("Failed to destroy TieredObjectStore", t); + } + } + + /** + * Register files in the registry. Batch format: triplets of path\nremotePath\nsize\n... + * location: 0=Local, 1=Remote — applied to all files in the batch. + * + * @param storePtr native pointer to the TieredObjectStore + * @param fileToPath map of file path to remote path (remote path can be empty for Local) + * @param location 0=Local, 1=Remote + * @param size file size in bytes (applied to all files in batch) + */ + public static void registerFiles(long storePtr, java.util.Map fileToPath, int location, long size) { + if (fileToPath.isEmpty()) return; + StringBuilder sb = new StringBuilder(); + for (java.util.Map.Entry e : fileToPath.entrySet()) { + sb.append(e.getKey()).append('\n'); + sb.append(e.getValue() != null ? e.getValue() : "").append('\n'); + sb.append(size).append('\n'); + } + sb.setLength(sb.length() - 1); + String entries = sb.toString(); + try (Arena arena = Arena.ofConfined()) { + MemorySegment seg = arena.allocateFrom(entries); + NativeLibraryLoader.checkResult( + (long) REGISTER_FILES.invokeExact(storePtr, seg, (long) entries.length(), fileToPath.size(), location) + ); + } catch (Throwable t) { + throw new RuntimeException("Failed to register " + fileToPath.size() + " files", t); + } + } + + /** + * Register a single file in the registry with its own location and size. + * + * @param storePtr native pointer to the TieredObjectStore + * @param file file identifier (absolute path for DataFusion lookups) + * @param path blob path (remote path for REMOTE, local path for LOCAL) + * @param location 0=Local, 1=Remote + * @param size file size in bytes + */ + public static void registerFile(long storePtr, String file, String path, int location, long size) { + registerFiles(storePtr, java.util.Map.of(file, path != null ? path : ""), location, size); + } + + /** Remove a file from the registry. */ + public static void removeFile(long storePtr, String path) { + try (Arena arena = Arena.ofConfined()) { + MemorySegment seg = arena.allocateFrom(path); + NativeLibraryLoader.checkResult((long) REMOVE_FILE.invokeExact(storePtr, seg, (long) path.length())); + } catch (Throwable t) { + throw new RuntimeException("Failed to remove file: " + path, t); + } + } + + /** + * Get a Box<Arc<dyn ObjectStore>> pointer from a TieredObjectStore Arc pointer. + * This is the format that DataFusion's df_create_reader expects. + * The returned pointer shares ownership with the original — free it with destroyObjectStoreBoxPtr. + * + * @param tieredStorePtr the Arc<TieredObjectStore> pointer from createTieredObjectStore + * @return Box<Arc<dyn ObjectStore>> pointer for DataFusion + */ + public static long getObjectStoreBoxPtr(long tieredStorePtr) { + try { + return NativeLibraryLoader.checkResult((long) GET_OBJECT_STORE_BOX_PTR.invokeExact(tieredStorePtr)); + } catch (Throwable t) { + throw new RuntimeException("Failed to get object store box ptr", t); + } + } + + /** + * Free a Box<Arc<dyn ObjectStore>> pointer returned by getObjectStoreBoxPtr. + * Drops the Box and decrements the Arc strong count. + * No-op if the native symbol is not available (stale library). + */ + public static void destroyObjectStoreBoxPtr(long boxPtr) { + if (boxPtr <= 0) return; + if (DESTROY_OBJECT_STORE_BOX_PTR == null) return; + try { + NativeLibraryLoader.checkResult((long) DESTROY_OBJECT_STORE_BOX_PTR.invokeExact(boxPtr)); + } catch (Throwable t) { + throw new RuntimeException("Failed to destroy object store box ptr", t); + } + } +} diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/store/ParquetStoreStrategyTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/store/ParquetStoreStrategyTests.java new file mode 100644 index 0000000000000..7f0751215f355 --- /dev/null +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/store/ParquetStoreStrategyTests.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.parquet.store; + +import org.opensearch.index.engine.dataformat.DataFormatStoreHandlerFactory; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Optional; + +/** + * Unit tests for {@link ParquetStoreStrategy}. + */ +public class ParquetStoreStrategyTests extends OpenSearchTestCase { + + public void testStoreHandlerReturnsFactory() { + ParquetStoreStrategy strategy = new ParquetStoreStrategy(); + Optional factory = strategy.storeHandler(); + assertTrue("storeHandler() should return a present Optional", factory.isPresent()); + assertNotNull("Factory should not be null", factory.get()); + } + + public void testOwnsParquetFiles() { + ParquetStoreStrategy strategy = new ParquetStoreStrategy(); + assertTrue(strategy.owns("parquet", "parquet/_0.parquet")); + assertTrue(strategy.owns("parquet", "parquet/seg_1.parquet")); + } + + public void testDoesNotOwnLuceneFiles() { + ParquetStoreStrategy strategy = new ParquetStoreStrategy(); + assertFalse(strategy.owns("parquet", "_0.cfe")); + assertFalse(strategy.owns("parquet", "segments_1")); + } + + public void testDoesNotOwnNullFile() { + ParquetStoreStrategy strategy = new ParquetStoreStrategy(); + assertFalse(strategy.owns("parquet", null)); + } + + public void testRemotePathDefault() { + ParquetStoreStrategy strategy = new ParquetStoreStrategy(); + String remotePath = strategy.remotePath("parquet", "base/path/", "parquet/_0.parquet", "_0.parquet__UUID1"); + assertEquals("base/path/parquet/_0.parquet__UUID1", remotePath); + } + + public void testRemotePathEmptyBasePath() { + ParquetStoreStrategy strategy = new ParquetStoreStrategy(); + String remotePath = strategy.remotePath("parquet", "", "parquet/_0.parquet", "_0.parquet__UUID1"); + assertEquals("parquet/_0.parquet__UUID1", remotePath); + } +} diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 56bd6e22884a7..afa210a2d3da9 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -105,6 +105,7 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.script.ScriptService; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; +import org.opensearch.storage.directory.TieredDataFormatAwareStoreDirectoryFactory; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.client.Client; @@ -1176,6 +1177,14 @@ private static DataFormatAwareStoreDirectoryFactory getDataFormatAwareStoreDirec if (dataFormatAwareStoreDirectoryFactories.isEmpty()) { return null; } + if (indexSettings.isWarmIndex() && indexSettings.isPluggableDataFormatEnabled()) { + DataFormatAwareStoreDirectoryFactory tiered = dataFormatAwareStoreDirectoryFactories.get( + TieredDataFormatAwareStoreDirectoryFactory.FACTORY_KEY + ); + if (tiered != null) { + return tiered; + } + } return dataFormatAwareStoreDirectoryFactories.get("default"); } diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 12c611bcb2b15..a2f59443c8895 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -78,7 +78,9 @@ import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.engine.MergedSegmentWarmerFactory; +import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.dataformat.StoreStrategy; import org.opensearch.index.engine.exec.EngineBackedIndexerFactory; import org.opensearch.index.engine.exec.IndexerFactory; import org.opensearch.index.fielddata.IndexFieldDataCache; @@ -101,6 +103,7 @@ import org.opensearch.index.store.DataFormatAwareStoreDirectory; import org.opensearch.index.store.DataFormatAwareStoreDirectoryFactory; import org.opensearch.index.store.FormatChecksumStrategy; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.Store; import org.opensearch.index.store.remote.filecache.FileCache; @@ -119,7 +122,9 @@ import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.plugins.IndexStorePlugin; +import org.opensearch.repositories.NativeStoreRepository; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.script.ScriptService; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import org.opensearch.threadpool.ThreadPool; @@ -778,23 +783,44 @@ protected void closeInternal() { if (this.indexSettings.isPluggableDataFormatEnabled() && dataFormatRegistry != null) { checksumStrategies = dataFormatRegistry.createChecksumStrategies(this.indexSettings); } - if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_SETTING) && - // TODO : Need to remove this check after support for hot indices is added in Composite Directory - this.indexSettings.isWarmIndex()) { - directory = compositeDirectoryFactory.newDirectory( + if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_SETTING) + && this.indexSettings.isWarmIndex() + && this.indexSettings.isPluggableDataFormatEnabled() + && this.dataFormatAwareStoreDirectoryFactory != null) { + // Warm + format-aware: resolve per-shard store strategies and native store, + // then let the factory build the StoreStrategyRegistry and directory stack. + Map storeStrategies = dataFormatRegistry.getStoreStrategies(this.indexSettings); + NativeStoreRepository nativeStore = resolveNativeStore(repositoriesService); + directory = dataFormatAwareStoreDirectoryFactory.newDataFormatAwareStoreDirectory( this.indexSettings, + shardId, path, directoryFactory, - remoteDirectory, + checksumStrategies, + storeStrategies, + nativeStore, + true, + (RemoteSegmentStoreDirectory) remoteDirectory, fileCache, threadPool ); - } else if (!this.indexSettings.isPluggableDataFormatEnabled()) { - directory = directoryFactory.newDirectory(this.indexSettings, path); - } else { - // Will be enabled in case of formatAware indices. - directory = createDataFormatAwareStoreDirectory(shardId, path, checksumStrategies); - } + } else if (FeatureFlags.isEnabled(FeatureFlags.WRITABLE_WARM_INDEX_SETTING) && + // TODO : Need to remove this check after support for hot indices is added in Composite Directory + this.indexSettings.isWarmIndex()) { + directory = compositeDirectoryFactory.newDirectory( + this.indexSettings, + path, + directoryFactory, + remoteDirectory, + fileCache, + threadPool + ); + } else if (this.indexSettings.isPluggableDataFormatEnabled() == false) { + directory = directoryFactory.newDirectory(this.indexSettings, path); + } else { + // Will be enabled in case of formatAware indices. + directory = createDataFormatAwareStoreDirectory(shardId, path, checksumStrategies); + } store = storeFactory.newStore( shardId, this.indexSettings, @@ -1370,6 +1396,27 @@ private DataFormatAwareStoreDirectory createDataFormatAwareStoreDirectory( return null; } + /** + * Resolves the native object store for the index's remote store repository. + * Returns {@link NativeStoreRepository#EMPTY} when no repository is configured + * or the repository is missing. + * + * @param repositoriesService the repositories service, may be {@code null} + * @return a live native store or {@link NativeStoreRepository#EMPTY} + */ + private NativeStoreRepository resolveNativeStore(RepositoriesService repositoriesService) { + String repoName = this.indexSettings.getRemoteStoreRepository(); + if (repoName == null || repositoriesService == null) { + return NativeStoreRepository.EMPTY; + } + try { + return repositoriesService.repository(repoName).getNativeStore(); + } catch (RepositoryMissingException e) { + logger.warn("Native store not available for repository [{}]", repoName); + return NativeStoreRepository.EMPTY; + } + } + private void updateFsyncTaskIfNecessary() { if (indexSettings.getTranslogDurability() == Translog.Durability.REQUEST) { try { diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java index 00835289bf122..6f1eb9b100d5c 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java @@ -19,6 +19,17 @@ * Plugins implement this to register their data format (e.g., Parquet, Lucene) * with the DataFormatRegistry during node bootstrap. * + *

      There are two orthogonal pieces a plugin can contribute: + *

        + *
      • {@link DataFormatDescriptor} via {@link #getFormatDescriptors} — + * describes the format (name, checksum strategy, static + * capabilities). Per-index value data.
        • + *
      • {@link StoreStrategy} via {@link #getStoreStrategies} — + * behavior for how the format participates in the tiered store + * (file ownership, remote layout, optional native registry).
        • + *
      + * A plugin may provide one, both, or neither. + * * @opensearch.experimental */ @ExperimentalApi @@ -32,21 +43,18 @@ public interface DataFormatPlugin { DataFormat getDataFormat(); /** - * Creates the indexing engine for the data format. This should be instantiated per shard. - * - * @param settings the engine initialization settings - * @return the indexing execution engine instance + * Creates the indexing engine for the data format. This should be + * instantiated per shard. */ IndexingExecutionEngine indexingEngine(IndexingEngineConfig settings); /** - * Returns format descriptor suppliers for this plugin, filtered by the given index settings. - * Each entry maps a format name to a {@link Supplier} of its {@link DataFormatDescriptor}, - * deferring descriptor object creation until the descriptor is actually needed. - * Callers that only need format names can use {@code keySet()} without triggering creation. - * - * @param indexSettings the index settings used to determine active formats - * @return map of format name to descriptor supplier + * Returns format descriptor suppliers for this plugin, filtered by the + * given index settings. Each entry maps a format name to a + * {@link Supplier} of its {@link DataFormatDescriptor}, deferring + * descriptor object creation until the descriptor is actually needed. + * Callers that only need format names can use {@code keySet()} without + * triggering creation. */ default Map> getFormatDescriptors( IndexSettings indexSettings, @@ -54,4 +62,24 @@ default Map> getFormatDescriptors( ) { return Map.of(); } + + /** + * Returns the strategies describing how this format participates in the tiered store, + * keyed by the format name the strategy applies to. + * + *

      Most plugins contribute a single entry (their own format). Composite plugins, + * which expose multiple formats per index, return one entry per participating format. + * A plugin that does not participate in the tiered store returns an empty map (default). + * + *

      All cross-cutting work (per-shard lifecycle, seeding, routing, close) is handled + * by the store layer. Plugins only declare strategies here. + * + * @param indexSettings the index settings + * @param dataFormatRegistry the registry, used by composite plugins to resolve + * sub-format plugins + * @return the strategies that apply, keyed by data format; never {@code null} + */ + default Map getStoreStrategies(IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry) { + return Map.of(); + } } diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java index f61b47bc39b5c..8e2bd58edd8b3 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java @@ -107,6 +107,22 @@ public DataFormat format(String name) { return format; } + /** + * Returns the plugin registered for the given format name, or {@code null} if not found. + * Used by composite plugins to look up sub-format plugins directly without going through + * the registry's top-level methods (which would cause infinite recursion). + * + * @param formatName the data format name (e.g., "parquet", "lucene") + * @return the plugin, or null if no plugin is registered for the format + */ + public DataFormatPlugin getPlugin(String formatName) { + if (formatName == null) { + return null; + } + DataFormat format = dataFormats.get(formatName); + return format != null ? dataFormatPluginRegistry.get(format) : null; + } + /** * Returns all registered data formats that support a specific capability for a field type. * @@ -135,6 +151,53 @@ public Set getRegisteredFormats() { return Set.copyOf(dataFormatPluginRegistry.keySet()); } + /** + * Returns all {@link StoreStrategy} instances that apply to the active + * data format of the given index, keyed by the format name the strategy + * applies to. + * + *

      Called once per shard at open time. The store layer uses the returned + * strategies to construct per-shard native file registries, seed them from + * remote metadata, and route directory events. + * + * @param indexSettings the index settings for this shard + * @return the map of applicable strategies, or an empty map when no + * pluggable data format is configured or the configured format + * does not participate in the tiered store + */ + public Map getStoreStrategies(IndexSettings indexSettings) { + String dataformatName = indexSettings.pluggableDataFormat(); + if (dataformatName != null && dataformatName.isEmpty() == false) { + DataFormat format = dataFormats.get(dataformatName); + if (format != null) { + DataFormatPlugin plugin = dataFormatPluginRegistry.get(format); + if (plugin != null) { + Map strategies = plugin.getStoreStrategies(indexSettings, this); + return strategies == null ? Map.of() : Map.copyOf(strategies); + } + } + } + return Map.of(); + } + + /** + * Returns store strategies for a specific data format, bypassing the + * {@code pluggable_dataformat} index setting lookup. Used by composite + * plugins to resolve child strategies without recursion. + * + * @param indexSettings the index settings + * @param dataFormat the specific data format to get strategies for + * @return map of data format to strategy, or empty map if the format is not registered + */ + public Map getStoreStrategies(IndexSettings indexSettings, DataFormat dataFormat) { + DataFormatPlugin plugin = dataFormatPluginRegistry.get(dataFormat); + if (plugin == null) { + return Map.of(); + } + Map strategies = plugin.getStoreStrategies(indexSettings, this); + return strategies == null ? Map.of() : strategies; + } + /** * Returns format descriptor suppliers for the active data format of the given index. * Resolves the data format from index settings via the {@code pluggable_dataformat} setting, diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatStoreHandler.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatStoreHandler.java new file mode 100644 index 0000000000000..646f42b386cf6 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatStoreHandler.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.plugins.NativeStoreHandle; + +import java.io.Closeable; +import java.util.Map; + +/** + * Per-shard handler for a data format's store lifecycle. + * + *

      Data format plugins that use a native (e.g. Rust) reader return one of + * these via {@link StoreStrategy#storeHandler()}. The store layer owns + * the instance, drives its lifecycle, and forwards file events (seed, upload, + * remove) that originate in the Java directory. + * + *

      Formats without a native reader return {@link java.util.Optional#empty()} + * from the strategy and never produce an instance of this interface. + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface DataFormatStoreHandler extends Closeable { + + /** + * File location constants matching the Rust {@code FileLocation} enum. + *

        + *
      • {@code LOCAL} — file exists only on local disk
        • + *
      • {@code REMOTE} — file exists only on a remote object store
        • + *
      + */ + int LOCAL = 0; + int REMOTE = 1; + + /** + * A file entry carrying the blob path, location, and size. + * + * @param path fully-qualified blob path (local path for LOCAL, remote blob path for REMOTE) + * @param location one of {@link #LOCAL} or {@link #REMOTE} + * @param size file size in bytes (0 if unknown) + */ + @ExperimentalApi + record FileEntry(String path, int location, long size) { + } + + /** + * Seeds the handler with a batch of files and their locations. + * Called once per shard at open time. + * + * @param files map of file identifier (e.g. {@code "parquet/seg_0.parquet"}) + * to {@link FileEntry} carrying the blob path and location + */ + void seed(Map files); + + /** + * Called after a file has been uploaded to the remote store. + * + * @param file the file identifier (absolute path) + * @param remotePath the remote blob path (base path + format prefix + blob key) + * @param size file size in bytes + */ + void onUploaded(String file, String remotePath, long size); + + /** + * Called after a file has been removed from tracking. + * + * @param file the file identifier + */ + void onRemoved(String file); + + /** + * Returns the native store handle wrapping the Rust object store pointer, + * or {@code null} if this handler does not manage a native store. + * + *

      The reader manager uses this to register the native object store + * in the DataFusion runtime environment. + */ + default NativeStoreHandle getFormatStoreHandle() { + return null; + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatStoreHandlerFactory.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatStoreHandlerFactory.java new file mode 100644 index 0000000000000..77e80052e5914 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatStoreHandlerFactory.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.repositories.NativeStoreRepository; + +/** + * Per-format factory that produces a {@link DataFormatStoreHandler} for a shard. + * + *

      Returned by {@link StoreStrategy#storeHandler()} for formats that + * need native file tracking (e.g. parquet with a Rust reader). The store + * layer invokes {@link #create} once per shard. + * + * @opensearch.experimental + */ +@ExperimentalApi +@FunctionalInterface +public interface DataFormatStoreHandlerFactory { + + /** + * Creates a per-shard store handler. + * + * @param shardId the shard id + * @param isWarm true if the shard is on a warm node + * @param repo the native remote store repository, or {@link NativeStoreRepository#EMPTY} + * when no native store is available + * @return a live handler; the caller owns it and must close it + */ + DataFormatStoreHandler create(ShardId shardId, boolean isWarm, NativeStoreRepository repo); +} diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/StoreStrategy.java b/server/src/main/java/org/opensearch/index/engine/dataformat/StoreStrategy.java new file mode 100644 index 0000000000000..0c93b2c2bada7 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/StoreStrategy.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.Optional; + +/** + * Strategy describing how a data format participates in the tiered store. + * + *

      Returned by {@link DataFormatPlugin#getStoreStrategies} keyed by the + * format name. The strategy itself is stateless regarding its name — the map + * key supplies the identity — and the store layer passes the name into + * {@link #owns} and {@link #remotePath} whenever behaviour depends on it. + * + *

      A strategy contributes three pieces of behaviour: + *

        + *
      • {@link #owns} — which files in the directory belong to this format
        • + *
      • {@link #remotePath} — how the format lays out blobs on the remote store
        • + *
      • optionally, {@link #storeHandler()} for formats with a native reader
        • + *
      + * + *

      All cross-cutting work (per-shard lifecycle, seeding from remote metadata, + * directory routing, close ordering, sync notifications) is handled by the + * store layer, not by the plugin. + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface StoreStrategy { + + /** + * Returns true if the given file identifier belongs to this format. + * + *

      The default convention is that format files live under a subdirectory + * whose prefix is the format name (e.g. {@code "parquet/seg_0.parquet"}). + * Implementations may override to use a different layout. + * + * @param name the format name the store layer associated with this + * strategy (the key it was registered under) + * @param file file identifier as produced by the directory layer + */ + default boolean owns(String name, String file) { + if (file == null) { + return false; + } + return file.startsWith(name + "/"); + } + + /** + * Returns the fully-qualified remote blob path for a file owned by this format. + * + *

      The default convention places blobs at + * {@code basePath + name + "/" + blobKey}. Implementations may override + * when the format uses a different layout on the remote store. + * + * @param name the format name the store layer associated with this + * strategy (the key it was registered under) + * @param basePath the repository base path (may be empty) + * @param file the file identifier (unused by the default layout) + * @param blobKey the uploaded blob key returned by + * {@link org.opensearch.index.store.RemoteSegmentStoreDirectory.UploadedSegmentMetadata#getUploadedFilename()} + * @return the remote blob path + */ + default String remotePath(String name, String basePath, String file, String blobKey) { + StringBuilder sb = new StringBuilder(); + if (basePath != null && basePath.isEmpty() == false) { + sb.append(basePath); + } + sb.append(name).append('/').append(blobKey); + return sb.toString(); + } + + /** + * Returns an optional factory that produces a per-shard native file + * registry. Formats without a native reader return + * {@link Optional#empty()}. + */ + default Optional storeHandler() { + return Optional.empty(); + } +} diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreUploaderService.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreUploaderService.java index 413316b884e39..6ffe765c11ffa 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreUploaderService.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreUploaderService.java @@ -19,10 +19,12 @@ import org.opensearch.common.logging.Loggers; import org.opensearch.common.util.UploadListener; import org.opensearch.core.action.ActionListener; -import org.opensearch.index.store.CompositeDirectory; import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.RemoteSyncListener; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -37,12 +39,44 @@ public class RemoteStoreUploaderService implements RemoteStoreUploader { private final IndexShard indexShard; private final Directory storeDirectory; private final RemoteSegmentStoreDirectory remoteDirectory; + private final List syncListeners = new ArrayList<>(); public RemoteStoreUploaderService(IndexShard indexShard, Directory storeDirectory, RemoteSegmentStoreDirectory remoteDirectory) { logger = Loggers.getLogger(getClass(), indexShard.shardId()); this.indexShard = indexShard; this.storeDirectory = storeDirectory; this.remoteDirectory = remoteDirectory; + // One-time chain walk at construction — register the sync listener from the directory stack + registerSyncListenersFromDirectory(storeDirectory); + } + + /** + * Registers a listener to be notified after each file is synced to remote. + * + * @param listener the listener to register + */ + public void addSyncListener(RemoteSyncListener listener) { + if (listener != null) { + syncListeners.add(listener); + } + } + + /** + * Walks the directory chain once to find and register the first {@link RemoteSyncListener}. + */ + private void registerSyncListenersFromDirectory(Directory dir) { + Directory current = dir; + while (current != null) { + if (current instanceof RemoteSyncListener) { + syncListeners.add((RemoteSyncListener) current); + return; + } + if (current instanceof FilterDirectory) { + current = ((FilterDirectory) current).getDelegate(); + } else { + break; + } + } } @Override @@ -63,7 +97,6 @@ public void uploadSegments( logger.debug("Effective new segments files to upload {}", localSegments); ActionListener> mappedListener = ActionListener.map(listener, resp -> null); GroupedActionListener batchUploadListener = new GroupedActionListener<>(mappedListener, localSegments.size()); - Directory directory = ((FilterDirectory) (((FilterDirectory) storeDirectory).getDelegate())).getDelegate(); for (String localSegment : localSegments) { // Initializing listener here to ensure that the stats increment operations are thread-safe @@ -72,9 +105,7 @@ public void uploadSegments( statsListener.onSuccess(localSegment); batchUploadListener.onResponse(resp); // Once uploaded to Remote, local files become eligible for eviction from FileCache - if (directory instanceof CompositeDirectory compositeDirectory) { - compositeDirectory.afterSyncToRemote(localSegment); - } + notifyAfterSyncToRemote(localSegment); }, ex -> { logger.warn(() -> new ParameterizedMessage("Exception: [{}] while uploading segment files", ex), ex); if (ex instanceof CorruptIndexException) { @@ -94,4 +125,10 @@ public void uploadSegments( ); } } + + private void notifyAfterSyncToRemote(String file) { + for (RemoteSyncListener listener : syncListeners) { + listener.afterSyncToRemote(file); + } + } } diff --git a/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java b/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java index 51aec1c7045e3..b8fa05b7dcaab 100644 --- a/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/CompositeDirectory.java @@ -28,6 +28,7 @@ import org.opensearch.index.store.remote.filecache.FileCache.RestoredCachedIndexInput; import org.opensearch.index.store.remote.utils.FileTypeUtils; import org.opensearch.index.store.remote.utils.TransferManager; +import org.opensearch.storage.utils.DirectoryUtils; import org.opensearch.threadpool.ThreadPool; import java.io.FileNotFoundException; @@ -54,7 +55,7 @@ * @opensearch.experimental */ @ExperimentalApi -public class CompositeDirectory extends FilterDirectory { +public class CompositeDirectory extends FilterDirectory implements RemoteSyncListener { private static final Logger logger = LogManager.getLogger(CompositeDirectory.class); protected final Directory localDirectory; protected final RemoteSegmentStoreDirectory remoteDirectory; @@ -397,15 +398,7 @@ public Path getFilePath(String name) { } private FSDirectory getLocalFSDirectory() { - FSDirectory localFSDirectory; - if (localDirectory instanceof FSDirectory) { - localFSDirectory = (FSDirectory) localDirectory; - } else { - // In this case it should be a FilterDirectory wrapped over FSDirectory as per above validation. - localFSDirectory = (FSDirectory) (((FilterDirectory) localDirectory).getDelegate()); - } - - return localFSDirectory; + return DirectoryUtils.unwrapFSDirectory(localDirectory); } /** @@ -423,9 +416,11 @@ private void validate(Directory localDirectory, Directory remoteDirectory, FileC if (fileCache == null) throw new IllegalStateException( "File Cache not initialized on this Node, cannot create Composite Directory without FileCache" ); - if (localDirectory instanceof FSDirectory == false - && !(localDirectory instanceof FilterDirectory && ((FilterDirectory) localDirectory).getDelegate() instanceof FSDirectory)) + try { + DirectoryUtils.unwrapFSDirectory(localDirectory); + } catch (IllegalArgumentException e) { throw new IllegalStateException("For Composite Directory, local directory must be of type FSDirectory"); + } if (remoteDirectory instanceof RemoteSegmentStoreDirectory == false) throw new IllegalStateException( "For Composite Directory, remote directory must be of type RemoteSegmentStoreDirectory" ); diff --git a/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectory.java index 6413e84933b35..2731d2611488b 100644 --- a/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectory.java @@ -65,7 +65,7 @@ * @opensearch.api */ @PublicApi(since = "3.0.0") -public class DataFormatAwareStoreDirectory extends FilterDirectory { +public class DataFormatAwareStoreDirectory extends FilterDirectory implements RemoteSyncListener { private static final Logger logger = LogManager.getLogger(DataFormatAwareStoreDirectory.class); @@ -97,6 +97,43 @@ public DataFormatAwareStoreDirectory(Directory delegate, ShardPath shardPath, Ma ); } + /** + * Creates a DataFormatAwareStoreDirectory with a pre-built delegate directory (no wrapping). + * Intended for warm nodes where the delegate is already a TieredSubdirectoryAwareDirectory. + * + * @param delegate the pre-built directory (e.g., TieredSubdirectoryAwareDirectory) + * @param shardPath the shard path + * @param checksumStrategies pre-built checksum strategies keyed by format name + * @return a new DataFormatAwareStoreDirectory wrapping the given delegate directly + */ + public static DataFormatAwareStoreDirectory withDirectoryDelegate( + Directory delegate, + ShardPath shardPath, + Map checksumStrategies + ) { + DataFormatAwareStoreDirectory dir = new DataFormatAwareStoreDirectory(delegate, shardPath, checksumStrategies, true); + return dir; + } + + // Private constructor for withDirectoryDelegate — skips SubdirectoryAwareDirectory wrapping + private DataFormatAwareStoreDirectory( + Directory delegate, + ShardPath shardPath, + Map checksumStrategies, + boolean directDelegate + ) { + super(delegate); + this.shardPath = shardPath; + this.checksumStrategies = new HashMap<>(checksumStrategies); + this.checksumStrategies.put(DEFAULT_FORMAT, new LuceneChecksumHandler()); + logger.debug( + "Created DataFormatAwareStoreDirectory (directDelegate={}) for shard {} with checksum strategies for formats: {}", + directDelegate, + shardPath.getShardId(), + this.checksumStrategies.keySet() + ); + } + /** * Walks the {@link FilterDirectory} wrapping chain to find a {@link DataFormatAwareStoreDirectory}. * This is needed because the directory may be wrapped in {@link ByteSizeCachingDirectory} and @@ -128,6 +165,16 @@ private String resolveFileName(String fileName) { return fileName; } + @Override + public void afterSyncToRemote(String file) { + Directory inner = getDelegate(); + if (inner instanceof RemoteSyncListener) { + ((RemoteSyncListener) inner).afterSyncToRemote(file); + } + // On hot: inner is SubdirectoryAwareDirectory → not RemoteSyncListener → no-op + // On warm: inner is TieredSubdirectoryAwareDirectory → implements it → delegates + } + @Override public IndexInput openInput(String name, IOContext context) throws IOException { return in.openInput(resolveFileName(name), context); diff --git a/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryFactory.java index 9c519e8001d93..86ccf4d804540 100644 --- a/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryFactory.java @@ -11,44 +11,41 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.StoreStrategy; import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.plugins.IndexStorePlugin; +import org.opensearch.repositories.NativeStoreRepository; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.util.Map; /** * Factory interface for creating DataFormatAwareStoreDirectory instances. - * This interface follows the existing IndexStorePlugin pattern to provide - * a centralized way to create composite directories with format discovery. * - *

      Following the same delegation pattern as {@link IndexStorePlugin.CompositeDirectoryFactory}, - * this factory accepts a {@link IndexStorePlugin.DirectoryFactory} to delegate local directory - * creation rather than hardcoding a specific directory implementation. + *

      Follows the existing {@link IndexStorePlugin} pattern to provide a + * centralized way to create directories that understand multiple data + * formats. Accepts a {@link IndexStorePlugin.DirectoryFactory} to delegate + * local directory creation rather than hardcoding a specific implementation. * * @opensearch.experimental */ @ExperimentalApi -@FunctionalInterface public interface DataFormatAwareStoreDirectoryFactory { /** - * Creates a new DataFormatAwareStoreDirectory per shard with automatic format discovery. - *

      - * The factory will: - * - Delegate local directory creation to the provided localDirectoryFactory - * - Use DataFormatRegistry to discover available data format plugins - * - Create format-specific directories for each discovered format - * - Provide fallback behavior if no plugins are found - * - Handle errors gracefully with proper logging + * Creates a new DataFormatAwareStoreDirectory per shard with automatic + * format discovery. * - * @param indexSettings the shard's index settings containing configuration + * @param indexSettings the shard's index settings * @param shardId the shard identifier * @param shardPath the path the shard is using for file storage - * @param localDirectoryFactory the factory for creating the underlying local directory, respecting index store type configuration + * @param localDirectoryFactory the factory for creating the underlying local directory * @param checksumStrategies pre-built checksum strategies keyed by format name - * @return a new DataFormatAwareStoreDirectory instance supporting all discovered formats - * @throws IOException if directory creation fails or resources cannot be allocated + * @return a new DataFormatAwareStoreDirectory + * @throws IOException if directory creation fails */ DataFormatAwareStoreDirectory newDataFormatAwareStoreDirectory( IndexSettings indexSettings, @@ -57,4 +54,45 @@ DataFormatAwareStoreDirectory newDataFormatAwareStoreDirectory( IndexStorePlugin.DirectoryFactory localDirectoryFactory, Map checksumStrategies ) throws IOException; + + /** + * Creates a new DataFormatAwareStoreDirectory for warm nodes with tiered + * storage support. + * + *

      Implementations that support warm+format override this method to + * build the full tiered directory stack. The per-shard strategy registry + * is constructed by the factory from the supplied {@code storeStrategies} + * and {@code nativeStore}; individual data formats contribute only the + * strategies. + * + * @param indexSettings the shard's index settings + * @param shardId the shard identifier + * @param shardPath the path the shard is using for file storage + * @param localDirectoryFactory the factory for creating the underlying local directory + * @param checksumStrategies pre-built checksum strategies keyed by format name + * @param storeStrategies the strategies declared by participating formats for this shard + * @param nativeStore the repository's native store, or + * {@link NativeStoreRepository#EMPTY} + * @param isWarm true if the shard is on a warm node + * @param remoteDirectory the remote segment store directory + * @param fileCache the file cache for warm node caching + * @param threadPool the thread pool for async operations + * @return a new DataFormatAwareStoreDirectory + * @throws IOException if directory creation fails + */ + default DataFormatAwareStoreDirectory newDataFormatAwareStoreDirectory( + IndexSettings indexSettings, + ShardId shardId, + ShardPath shardPath, + IndexStorePlugin.DirectoryFactory localDirectoryFactory, + Map checksumStrategies, + Map storeStrategies, + NativeStoreRepository nativeStore, + boolean isWarm, + RemoteSegmentStoreDirectory remoteDirectory, + FileCache fileCache, + ThreadPool threadPool + ) throws IOException { + throw new UnsupportedOperationException("Warm-aware directory creation not supported by this factory"); + } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 801692b2b7da8..5d6258b06b37d 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -1085,6 +1085,16 @@ public int getSegmentsUploadedToRemoteStoreSize() { return segmentsUploadedToRemoteStore.size(); } + /** + * Returns the base blob path for this shard's remote segment data. + * E.g., "clusterUUID/indexUUID/shardId/segments/data/" + * + * @return the base path as a string + */ + public String getRemoteBasePath() { + return remoteDataDirectory.getBlobContainer().path().buildAsString(); + } + // Visible for testing Set getMetadataFilesToFilterActiveSegments( final int lastNMetadataFilesToKeep, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSyncListener.java b/server/src/main/java/org/opensearch/index/store/RemoteSyncListener.java new file mode 100644 index 0000000000000..c9442da3adec3 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/RemoteSyncListener.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Listener that receives notifications after files are synced to the remote store. + * + *

      Registered via {@code RemoteStoreUploaderService.addSyncListener()} at uploader + * construction time. When a file is uploaded to the remote segment store, the uploader + * calls {@link #afterSyncToRemote(String)} on all registered listeners. + * + *

      Implemented by: + *

        + *
      • {@link CompositeDirectory} — unpins files from FileCache after upload
        • + *
      • TieredSubdirectoryAwareDirectory — delegates to format-specific handlers
        • + *
      • {@link DataFormatAwareStoreDirectory} — pass-through to inner directory
        • + *
      + * + * @opensearch.experimental + */ +@FunctionalInterface +@ExperimentalApi +public interface RemoteSyncListener { + + /** + * Called after a file has been successfully uploaded to the remote store. + * + *

      Implementations should use this callback to update internal state related to the + * file's remote availability — such as unpinning from a local cache, marking the file + * as remotely available in a registry, or forwarding the notification to a delegate. + * + * @param file the name of the file that was synced to remote + */ + void afterSyncToRemote(String file); +} diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index e2840b0ba3694..dac863b24285e 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -170,6 +170,7 @@ import org.opensearch.index.remote.RemoteIndexPathUploader; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; import org.opensearch.index.store.DefaultCompositeDirectoryFactory; +import org.opensearch.index.store.DefaultDataFormatAwareStoreDirectoryFactory; import org.opensearch.index.store.IndexStoreListener; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.remote.filecache.FileCache; @@ -271,6 +272,7 @@ import org.opensearch.snapshots.SnapshotsInfoService; import org.opensearch.snapshots.SnapshotsService; import org.opensearch.storage.common.tiering.TieringUtils; +import org.opensearch.storage.directory.TieredDataFormatAwareStoreDirectoryFactory; import org.opensearch.storage.directory.TieredDirectoryFactory; import org.opensearch.storage.metrics.TierActionMetrics; import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings; @@ -963,9 +965,12 @@ protected Node(final Environment initialEnvironment, Collection clas new HashMap<>(); // Register default factory + dataFormatAwareStoreDirectoryFactories.put("default", new DefaultDataFormatAwareStoreDirectoryFactory()); + + // Register tiered factory for warm+format indices dataFormatAwareStoreDirectoryFactories.put( - "default", - new org.opensearch.index.store.DefaultDataFormatAwareStoreDirectoryFactory() + TieredDataFormatAwareStoreDirectoryFactory.FACTORY_KEY, + new TieredDataFormatAwareStoreDirectoryFactory(tieredStoragePrefetchSettingsSupplier) ); final Map recoveryStateFactories = pluginsService.filterPlugins( diff --git a/server/src/main/java/org/opensearch/plugins/NativeStoreHandle.java b/server/src/main/java/org/opensearch/plugins/NativeStoreHandle.java index 593e26955567c..807667512a5b3 100644 --- a/server/src/main/java/org/opensearch/plugins/NativeStoreHandle.java +++ b/server/src/main/java/org/opensearch/plugins/NativeStoreHandle.java @@ -10,6 +10,8 @@ import org.opensearch.common.annotation.ExperimentalApi; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -19,6 +21,10 @@ * liveness check. The destructor function is captured at creation time, * so the pointer and its cleanup are always paired. * + *

      All live pointers are tracked in a global registry ({@link #LIVE_POINTERS}). + * Use {@link #isLivePointer(long)} to validate a raw pointer before passing it + * to native code — catches use-after-free bugs as exceptions instead of SIGSEGV. + * *

      Instances are created by {@link NativeRemoteObjectStoreProvider} and * owned by the repository that holds the native store pointer. * @@ -30,6 +36,13 @@ public final class NativeStoreHandle implements AutoCloseable { /** Sentinel representing "no native store". Safe to close (no-op). */ public static final NativeStoreHandle EMPTY = new NativeStoreHandle(); + /** + * Global registry of all live native pointers managed by NativeStoreHandle. + * Used to detect use-after-free: if a pointer is not in this set, it has + * been closed or was never created by a NativeStoreHandle. + */ + private static final Set LIVE_POINTERS = ConcurrentHashMap.newKeySet(); + private final long ptr; private final Destroyer destroyer; private final AtomicBoolean closed = new AtomicBoolean(false); @@ -58,6 +71,7 @@ public NativeStoreHandle(long ptr, Destroyer destroyer) { } this.ptr = ptr; this.destroyer = destroyer; + LIVE_POINTERS.add(ptr); } /** Private constructor for the EMPTY sentinel. */ @@ -83,9 +97,10 @@ public long getPointer() { /** * Returns true if this handle holds a live pointer (not EMPTY, not closed). + * Checks the global registry to detect if the pointer was closed from any reference. */ public boolean isLive() { - return this != EMPTY && closed.get() == false; + return this != EMPTY && LIVE_POINTERS.contains(ptr); } /** @@ -98,7 +113,49 @@ public void close() { return; } if (closed.compareAndSet(false, true)) { + LIVE_POINTERS.remove(ptr); destroyer.destroy(ptr); } } + + /** + * Checks if a raw pointer value corresponds to a live, open NativeStoreHandle. + * Use this before passing raw pointer values to native code to detect + * use-after-free bugs. + * + * @param ptr the raw pointer value to check + * @return true if the pointer is tracked and has not been closed + */ + public static boolean isLivePointer(long ptr) { + return LIVE_POINTERS.contains(ptr); + } + + /** + * Validates that a raw pointer value is live, throwing if it is stale or unknown. + * Use this as a guard before FFM downcalls that accept raw pointer arguments. + * + * @param ptr the raw pointer value to validate + * @param name a descriptive name for error messages (e.g., "storeHandle", "nativeStoreForReader") + * @throws IllegalArgumentException if ptr is 0 or negative + * @throws IllegalStateException if the pointer is not in the live registry + */ + public static void validatePointer(long ptr, String name) { + if (ptr <= 0) { + throw new IllegalArgumentException(name + " pointer is invalid: " + ptr); + } + if (LIVE_POINTERS.contains(ptr) == false) { + throw new IllegalStateException( + name + " pointer 0x" + Long.toHexString(ptr) + " is not a live handle — already closed or never created" + ); + } + } + + /** + * Returns the number of currently live handles. Useful for leak detection in tests. + * + * @return the count of open native store handles + */ + public static int liveHandleCount() { + return LIVE_POINTERS.size(); + } } diff --git a/server/src/main/java/org/opensearch/storage/directory/StoreStrategyRegistry.java b/server/src/main/java/org/opensearch/storage/directory/StoreStrategyRegistry.java new file mode 100644 index 0000000000000..ed6ff83037f8a --- /dev/null +++ b/server/src/main/java/org/opensearch/storage/directory/StoreStrategyRegistry.java @@ -0,0 +1,300 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.directory; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.DataFormatStoreHandler; +import org.opensearch.index.engine.dataformat.DataFormatStoreHandlerFactory; +import org.opensearch.index.engine.dataformat.StoreStrategy; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.plugins.NativeStoreHandle; +import org.opensearch.repositories.NativeStoreRepository; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Per-shard registry of {@link StoreStrategy} instances and their associated + * {@link DataFormatStoreHandler store handlers}. + * + *

      Owns the plumbing shared by every data format participating in the tiered + * store so that format plugins stay purely declarative: + *

        + *
      • resolves the owning {@link StoreStrategy} for a file
        • + *
      • constructs per-strategy {@link DataFormatStoreHandler} instances + * exception-safely (no leaked native resources if one factory throws)
        • + *
      • seeds handlers from the remote segment metadata at open time
        • + *
      • forwards {@code onUploaded} / {@code onRemoved} events to the owning + * strategy's handler, if any
        • + *
      • closes handlers in the right order when the shard shuts down
        • + *
      + * + * @opensearch.experimental + */ +@ExperimentalApi +public final class StoreStrategyRegistry implements Closeable { + + private static final Logger logger = LogManager.getLogger(StoreStrategyRegistry.class); + + /** Sentinel for "no strategies registered on this shard". Safe to close. */ + public static final StoreStrategyRegistry EMPTY = new StoreStrategyRegistry(null, Collections.emptyMap(), Collections.emptyMap()); + + /** Shard path for resolving absolute file keys (matches DataFusion's lookup paths). Null for EMPTY. */ + private final ShardPath shardPath; + /** Strategies keyed by data format. */ + private final Map strategies; + /** Store handlers keyed by data format. Absent for strategies without one. */ + private final Map storeHandlers; + + /** + * A strategy paired with the data format it is registered under. Used + * internally for routing decisions so callers never need to re-derive the + * format from the strategy. + * + * @opensearch.experimental + */ + @ExperimentalApi + public record Match(DataFormat format, StoreStrategy strategy) { + } + + private StoreStrategyRegistry( + ShardPath shardPath, + Map strategies, + Map storeHandlers + ) { + this.shardPath = shardPath; + this.strategies = Map.copyOf(strategies); + this.storeHandlers = Map.copyOf(storeHandlers); + } + + /** + * Builds a registry for a shard, constructing per-strategy store handlers + * and seeding them from the remote metadata. + * + *

      If any handler factory throws, all handlers created so far + * are closed and the exception is rethrown — no partial state escapes. + * + * @param shardPath the shard path (used to resolve absolute file paths for DataFusion) + * @param isWarm true on warm nodes + * @param nativeStore the repository's native store, or + * {@link NativeStoreRepository#EMPTY} + * @param strategies the strategies that apply to this shard, keyed by data format + * @param remoteDirectory the remote segment store directory used to seed initial state + * @return a fully-initialised registry + */ + public static StoreStrategyRegistry open( + ShardPath shardPath, + boolean isWarm, + NativeStoreRepository nativeStore, + Map strategies, + RemoteSegmentStoreDirectory remoteDirectory + ) { + if (strategies == null || strategies.isEmpty()) { + return EMPTY; + } + + // Exception safety: if any factory throws, all previously created handlers + // are closed in the finally block. This prevents native resource leaks when + // one format plugin fails during shard open. + Map storeHandlers = new HashMap<>(); + List created = new ArrayList<>(); + boolean success = false; + try { + for (Map.Entry entry : strategies.entrySet()) { + DataFormat format = entry.getKey(); + StoreStrategy strategy = entry.getValue(); + DataFormatStoreHandlerFactory factory = strategy.storeHandler().orElse(null); + if (factory == null) { + continue; + } + DataFormatStoreHandler handler = factory.create(shardPath.getShardId(), isWarm, nativeStore); + if (handler != null) { + storeHandlers.put(format, handler); + created.add(handler); + } + } + + if (storeHandlers.isEmpty() == false) { + seedFromRemoteMetadata(shardPath, strategies, storeHandlers, remoteDirectory); + } + success = true; + return new StoreStrategyRegistry(shardPath, strategies, storeHandlers); + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(created); + } + } + } + + /** + * Returns the strategy that owns {@code file}, or {@code null} if no + * registered strategy claims it. The returned {@link Match} carries both + * the data format and the strategy object. + */ + public Match matchFor(String file) { + if (file == null) { + return null; + } + for (Map.Entry entry : strategies.entrySet()) { + String name = entry.getKey().name(); + if (entry.getValue().owns(name, file)) { + return new Match(entry.getKey(), entry.getValue()); + } + } + return null; + } + + /** True if any strategy on this shard has a store handler. */ + public boolean hasStoreHandlers() { + return storeHandlers.isEmpty() == false; + } + + /** + * Returns the native store handles for all formats that have a live handler, + * keyed by {@link DataFormat}. + * + *

      The reader manager uses this to register native object stores in the + * DataFusion runtime environment. + * + * @return map of DataFormat to live {@link NativeStoreHandle}, or empty if + * no handlers have native stores + */ + public Map getFormatStoreHandles() { + Map handles = new HashMap<>(); + for (Map.Entry entry : storeHandlers.entrySet()) { + NativeStoreHandle handle = entry.getValue().getFormatStoreHandle(); + if (handle != null && handle.isLive()) { + handles.put(entry.getKey(), handle); + } + } + return Map.copyOf(handles); + } + + /** + * Forwards a sync-to-remote event. Resolves the owning strategy, constructs + * the remote path via {@link StoreStrategy#remotePath}, and forwards to the + * store handler for that strategy if one exists. + * + * @param file the file identifier that was uploaded + * @param basePath the repository base path + * @param uploadedBlobKey the blob key assigned by the upload path + * @return true if the event was dispatched to a store handler; false if + * no strategy owns the file or the owning strategy has no handler + */ + public boolean onUploaded(String file, String basePath, String uploadedBlobKey, long size) { + Match match = matchFor(file); + if (match == null) { + return false; + } + DataFormatStoreHandler handler = storeHandlers.get(match.format()); + if (handler == null) { + return false; + } + // Resolve absolute key for the native handler's Rust registry (matches DataFusion lookups) + String absoluteKey = shardPath.getDataPath().resolve(file).toString(); + String remotePath = match.strategy().remotePath(match.format().name(), basePath, file, uploadedBlobKey); + handler.onUploaded(absoluteKey, remotePath, size); + return true; + } + + /** + * Forwards a removal event. Returns true if dispatched, false otherwise. + */ + public boolean onRemoved(String file) { + Match match = matchFor(file); + if (match == null) { + return false; + } + DataFormatStoreHandler handler = storeHandlers.get(match.format()); + if (handler == null) { + return false; + } + // Resolve absolute key for the native handler's Rust registry + String absoluteKey = shardPath.getDataPath().resolve(file).toString(); + handler.onRemoved(absoluteKey); + return true; + } + + /** + * Closes all store handlers. Handlers are closed before the directory + * (in {@link TieredSubdirectoryAwareDirectory#close}) so Rust resources + * are torn down while the Java objects they may reference are still alive. + */ + @Override + public void close() throws IOException { + IOUtils.close(storeHandlers.values()); + } + + // TODO (writable warm): add seedLocalFiles(ShardPath) — scan local disk at shard open + // for crash recovery. Registers LOCAL files that were written but not yet synced to remote. + + /** + * Seeds store handlers from the remote segment store metadata. + * Called once at shard open. Each file is matched to its owning strategy, + * the remote blob path is constructed, and the batch is forwarded to the + * strategy's store handler. + * + *

      Currently seeds all files as REMOTE. On writable warm, local files + * from a disk scan would be seeded as LOCAL via a separate path. + */ + private static void seedFromRemoteMetadata( + ShardPath shardPath, + Map strategies, + Map storeHandlers, + RemoteSegmentStoreDirectory remoteDirectory + ) { + if (remoteDirectory == null) { + return; + } + String basePath = remoteDirectory.getRemoteBasePath(); + Map uploaded = remoteDirectory.getSegmentsUploadedToRemoteStore(); + if (uploaded == null || uploaded.isEmpty()) { + return; + } + + Map> perStrategy = new HashMap<>(); + for (Map.Entry entry : uploaded.entrySet()) { + String file = entry.getKey(); + DataFormat owningFormat = null; + StoreStrategy owning = null; + for (Map.Entry s : strategies.entrySet()) { + if (s.getValue().owns(s.getKey().name(), file)) { + owningFormat = s.getKey(); + owning = s.getValue(); + break; + } + } + if (owning == null || storeHandlers.containsKey(owningFormat) == false) { + continue; + } + String blobKey = entry.getValue().getUploadedFilename(); + String remotePath = owning.remotePath(owningFormat.name(), basePath, file, blobKey); + // Use absolute path as key — matches what DataFusion uses for file:// lookups + String absoluteKey = shardPath.getDataPath().resolve(file).toString(); + long size = entry.getValue().getLength(); + perStrategy.computeIfAbsent(owningFormat, k -> new HashMap<>()) + .put(absoluteKey, new DataFormatStoreHandler.FileEntry(remotePath, DataFormatStoreHandler.REMOTE, size)); + } + + for (Map.Entry> entry : perStrategy.entrySet()) { + storeHandlers.get(entry.getKey()).seed(entry.getValue()); + logger.debug("Seeded {} files into store handler for format [{}]", entry.getValue().size(), entry.getKey().name()); + } + } +} diff --git a/server/src/main/java/org/opensearch/storage/directory/TieredDataFormatAwareStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/storage/directory/TieredDataFormatAwareStoreDirectoryFactory.java new file mode 100644 index 0000000000000..59e210e3f8ab9 --- /dev/null +++ b/server/src/main/java/org/opensearch/storage/directory/TieredDataFormatAwareStoreDirectoryFactory.java @@ -0,0 +1,122 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.directory; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.Directory; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.StoreStrategy; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.DataFormatAwareStoreDirectory; +import org.opensearch.index.store.DataFormatAwareStoreDirectoryFactory; +import org.opensearch.index.store.FormatChecksumStrategy; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.SubdirectoryAwareDirectory; +import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.plugins.IndexStorePlugin; +import org.opensearch.repositories.NativeStoreRepository; +import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Factory for creating the warm+format directory stack. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class TieredDataFormatAwareStoreDirectoryFactory implements DataFormatAwareStoreDirectoryFactory { + + public static final String FACTORY_KEY = "dataformat-tiered"; + + private static final Logger logger = LogManager.getLogger(TieredDataFormatAwareStoreDirectoryFactory.class); + + private final Supplier tieredStoragePrefetchSettingsSupplier; + + public TieredDataFormatAwareStoreDirectoryFactory(Supplier tieredStoragePrefetchSettingsSupplier) { + this.tieredStoragePrefetchSettingsSupplier = tieredStoragePrefetchSettingsSupplier; + } + + @Override + public DataFormatAwareStoreDirectory newDataFormatAwareStoreDirectory( + IndexSettings indexSettings, + ShardId shardId, + ShardPath shardPath, + IndexStorePlugin.DirectoryFactory localDirectoryFactory, + Map checksumStrategies + ) throws IOException { + throw new UnsupportedOperationException( + "TieredDataFormatAwareStoreDirectoryFactory requires warm parameters. Use the warm-aware overload." + ); + } + + @Override + public DataFormatAwareStoreDirectory newDataFormatAwareStoreDirectory( + IndexSettings indexSettings, + ShardId shardId, + ShardPath shardPath, + IndexStorePlugin.DirectoryFactory localDirectoryFactory, + Map checksumStrategies, + Map storeStrategies, + NativeStoreRepository nativeStore, + boolean isWarm, + RemoteSegmentStoreDirectory remoteDirectory, + FileCache fileCache, + ThreadPool threadPool + ) throws IOException { + logger.debug( + "Creating warm+format directory stack for shard [{}] with {} strategies", + shardId, + storeStrategies == null ? 0 : storeStrategies.size() + ); + + Directory localDir = localDirectoryFactory.newDirectory(indexSettings, shardPath); + SubdirectoryAwareDirectory subdirAware = new SubdirectoryAwareDirectory(localDir, shardPath); + + StoreStrategyRegistry strategies = null; + TieredSubdirectoryAwareDirectory tieredSubdir = null; + boolean success = false; + try { + strategies = StoreStrategyRegistry.open(shardPath, isWarm, nativeStore, storeStrategies, remoteDirectory); + tieredSubdir = new TieredSubdirectoryAwareDirectory( + subdirAware, + remoteDirectory, + fileCache, + threadPool, + strategies, + shardPath, + tieredStoragePrefetchSettingsSupplier + ); + + DataFormatAwareStoreDirectory result = DataFormatAwareStoreDirectory.withDirectoryDelegate( + tieredSubdir, + shardPath, + checksumStrategies + ); + success = true; + return result; + } finally { + if (success == false) { + if (tieredSubdir != null) { + IOUtils.closeWhileHandlingException(tieredSubdir); + } else if (strategies != null) { + IOUtils.closeWhileHandlingException(strategies); + } + } + } + } +} diff --git a/server/src/main/java/org/opensearch/storage/directory/TieredSubdirectoryAwareDirectory.java b/server/src/main/java/org/opensearch/storage/directory/TieredSubdirectoryAwareDirectory.java new file mode 100644 index 0000000000000..72a18f8d8334b --- /dev/null +++ b/server/src/main/java/org/opensearch/storage/directory/TieredSubdirectoryAwareDirectory.java @@ -0,0 +1,222 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.directory; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.index.engine.dataformat.StoreStrategy; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.RemoteSyncListener; +import org.opensearch.index.store.SubdirectoryAwareDirectory; +import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Supplier; + +/** + * A tiered directory for warm nodes that routes file operations based on + * data format. + * + *

      Read-only warm (current scope): all format files are REMOTE, + * seeded from remote metadata at shard open via {@link StoreStrategyRegistry}. + * Reads go directly to {@link RemoteSegmentStoreDirectory}. No local copies, + * no eviction, no ref counting for format files. + * + *

      Routing: + *

        + *
      • Format files (a strategy claims the file) → always + * {@link RemoteSegmentStoreDirectory}
        • + *
      • Lucene files (no claiming strategy) → {@link TieredDirectory} + * (FileCache + remote)
        • + *
      + * + * @opensearch.experimental + */ +@ExperimentalApi +public class TieredSubdirectoryAwareDirectory extends FilterDirectory implements RemoteSyncListener { + + private static final Logger logger = LogManager.getLogger(TieredSubdirectoryAwareDirectory.class); + + private final TieredDirectory tieredDirectory; + private final StoreStrategyRegistry strategies; + private final RemoteSegmentStoreDirectory remoteDirectory; + private final ShardPath shardPath; + + public TieredSubdirectoryAwareDirectory( + SubdirectoryAwareDirectory localDirectory, + RemoteSegmentStoreDirectory remoteDirectory, + FileCache fileCache, + ThreadPool threadPool, + StoreStrategyRegistry strategies, + ShardPath shardPath, + Supplier tieredStoragePrefetchSettingsSupplier + ) { + super(localDirectory); + this.strategies = strategies == null ? StoreStrategyRegistry.EMPTY : strategies; + this.remoteDirectory = remoteDirectory; + this.shardPath = shardPath; + boolean success = false; + try { + this.tieredDirectory = new TieredDirectory( + localDirectory, + remoteDirectory, + fileCache, + threadPool, + tieredStoragePrefetchSettingsSupplier + ); + logger.debug("Created TieredSubdirectoryAwareDirectory (hasStoreHandlers={})", this.strategies.hasStoreHandlers()); + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this.strategies); + } + } + } + + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + if (isFormatFile(name)) { + // Check if file exists in remote directory (already synced) — route to remote. + // Otherwise read from local (translog bump edge case, file not yet synced). + if (remoteDirectory.getExistingRemoteFilename(name) != null) { + return remoteDirectory.openInput(name, context); + } + return in.openInput(name, context); + } + return tieredDirectory.openInput(name, context); + } + + @Override + public long fileLength(String name) throws IOException { + if (isFormatFile(name)) { + // Same routing as openInput — check remote first. + if (remoteDirectory.getExistingRemoteFilename(name) != null) { + return remoteDirectory.fileLength(name); + } + return in.fileLength(name); + } + return tieredDirectory.fileLength(name); + } + + @Override + public String[] listAll() throws IOException { + Set all = new HashSet<>(Arrays.asList(tieredDirectory.listAll())); + return all.stream().sorted().toArray(String[]::new); + } + + @Override + public IndexOutput createOutput(String name, IOContext context) throws IOException { + return tieredDirectory.createOutput(name, context); + } + + @Override + public void deleteFile(String name) throws IOException { + if (isFormatFile(name)) { + strategies.onRemoved(name); + try { + in.deleteFile(name); + } catch (NoSuchFileException e) { + // Expected on read-only warm — file was never local or already evicted + } + return; + } + tieredDirectory.deleteFile(name); + } + + @Override + public void afterSyncToRemote(String file) { + if (isFormatFile(file)) { + String blobKey = remoteDirectory.getExistingRemoteFilename(file); + if (blobKey == null) { + throw new IllegalStateException( + "afterSyncToRemote called for format file [" + file + "] but no remote filename found in metadata" + ); + } + long size; + try { + size = remoteDirectory.fileLength(file); + } catch (IOException e) { + size = 0; + } + strategies.onUploaded(file, remoteDirectory.getRemoteBasePath(), blobKey, size); + // On warm, no local parquet files should remain — delete after sync. + // Safe because: (1) the file is now REMOTE in the registry, so new readers + // route to remote, and (2) TieredObjectStore retries from remote if local NotFound. + try { + in.deleteFile(file); + } catch (java.nio.file.NoSuchFileException e) { + // Already gone — fine + } catch (IOException e) { + logger.warn("afterSyncToRemote: failed to delete local copy of file={}", file); + } + return; + } + tieredDirectory.afterSyncToRemote(file); + } + + @Override + public void sync(Collection names) { + // Skip — same as TieredDirectory (CompositeDirectory). On warm, files are + // either remote-only (format files) or cached from remote. + // No local writes to fsync. Writable warm will need to revisit this. + } + + @Override + public void rename(String source, String dest) throws IOException { + // Rename is only called by Lucene's IndexWriter during commit + // (pending_segments_N → segments_N). Format files are never renamed. + if (isFormatFile(source)) { + throw new IllegalStateException("Rename not supported for format file [" + source + "]. Format files are write-once."); + } + tieredDirectory.rename(source, dest); + } + + @Override + public void close() throws IOException { + // Native registries close before the directory so native resources are + // torn down while the Java resources they may reference are still alive. + IOUtils.close(strategies, tieredDirectory); + } + + /** + * Returns {@code true} if {@code name} is a format file (claimed by a + * registered {@link StoreStrategy}). Plain Lucene/metadata files — those + * whose path resolves directly under the shard index directory — are not + * format files and skip the strategy lookup. + * + *

      The {@code shardPath.resolveIndex()} guard is a fast-path: files without + * a subdirectory component (e.g. {@code "_0.cfe"}) are always Lucene files. + * Only files under a subdirectory (e.g. {@code "parquet/seg_0.parquet"}) go + * through the strategy lookup via {@link StoreStrategyRegistry#matchFor}. + */ + private boolean isFormatFile(String name) { + if (shardPath.resolveIndex().resolve(name).getParent().equals(shardPath.resolveIndex())) { + return false; + } + StoreStrategyRegistry.Match match = strategies.matchFor(name); + if (match == null) { + throw new IllegalStateException("No StoreStrategy registered for file [" + name + "]. Ensure the format plugin is installed."); + } + return true; + } +} diff --git a/server/src/main/java/org/opensearch/storage/utils/DirectoryUtils.java b/server/src/main/java/org/opensearch/storage/utils/DirectoryUtils.java index e36806011d76a..54734332f0ab6 100644 --- a/server/src/main/java/org/opensearch/storage/utils/DirectoryUtils.java +++ b/server/src/main/java/org/opensearch/storage/utils/DirectoryUtils.java @@ -26,6 +26,25 @@ public class DirectoryUtils { /** Suffix for switchable file paths. */ public static final String SWITCHABLE_PREFIX = "_switchable"; + /** + * Walks the {@link FilterDirectory} chain to find the underlying {@link FSDirectory}. + * Returns immediately if the given directory is already an FSDirectory. + * + * @param directory the directory to unwrap + * @return the underlying FSDirectory + * @throws IllegalArgumentException if no FSDirectory is found in the chain + */ + public static FSDirectory unwrapFSDirectory(Directory directory) { + Directory current = directory; + while (current instanceof FilterDirectory) { + current = ((FilterDirectory) current).getDelegate(); + } + if (current instanceof FSDirectory) { + return (FSDirectory) current; + } + throw new IllegalArgumentException("Expected FSDirectory but got: " + directory.getClass().getName()); + } + /** * Resolves the file path for a given file name in the directory. * @param localDirectory the directory @@ -45,23 +64,4 @@ public static Path getFilePath(Directory localDirectory, String fileName) { public static Path getFilePathSwitchable(Directory localDirectory, String fileName) { return unwrapFSDirectory(localDirectory).getDirectory().resolve(fileName + SWITCHABLE_PREFIX); } - - /** - * Unwraps a directory chain to find the underlying FSDirectory. - * Handles cases where the directory is wrapped in FilterDirectory layers - * such as BucketedCompositeDirectory. - * @param directory the directory to unwrap - * @return the underlying FSDirectory - * @throws IllegalArgumentException if no FSDirectory is found in the chain - */ - public static FSDirectory unwrapFSDirectory(Directory directory) { - Directory current = directory; - while (current instanceof FilterDirectory) { - current = ((FilterDirectory) current).getDelegate(); - } - if (current instanceof FSDirectory) { - return (FSDirectory) current; - } - throw new IllegalArgumentException("Expected FSDirectory but got: " + directory.getClass().getName()); - } } diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java index e9a144f78534e..0a4c4bd292339 100644 --- a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatRegistryTests.java @@ -315,4 +315,102 @@ public void testGetFormatDescriptorsByDataFormatReturnsEmptyForUnregisteredForma Map> descriptors = registry.getFormatDescriptors(indexSettings, unregistered); assertTrue(descriptors.isEmpty()); } + + public void testGetStoreStrategiesEmptyWhenNoPluggableDataformat() { + MockDataFormat format = new MockDataFormat("columnar", 100L, Set.of()); + MockSearchBackEndPlugin backEnd = new MockSearchBackEndPlugin(List.of(format.name())); + + when(pluginsService.filterPlugins(DataFormatPlugin.class)).thenReturn(List.of(MockDataFormatPlugin.of(format))); + when(pluginsService.filterPlugins(SearchBackEndPlugin.class)).thenReturn(List.of(backEnd)); + + DataFormatRegistry registry = new DataFormatRegistry(pluginsService); + + Map result = registry.getStoreStrategies(indexSettings); + assertTrue("Should return empty map when no pluggable_dataformat setting", result.isEmpty()); + } + + public void testGetStoreStrategiesEmptyWhenPluginReturnsNone() { + MockDataFormat format = new MockDataFormat("columnar", 100L, Set.of()); + MockSearchBackEndPlugin backEnd = new MockSearchBackEndPlugin(List.of(format.name())); + + when(pluginsService.filterPlugins(DataFormatPlugin.class)).thenReturn(List.of(MockDataFormatPlugin.of(format))); + when(pluginsService.filterPlugins(SearchBackEndPlugin.class)).thenReturn(List.of(backEnd)); + + DataFormatRegistry registry = new DataFormatRegistry(pluginsService); + + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT) + .put("index.pluggable.dataformat", "columnar") + .put("index.pluggable.dataformat.enabled", true) + .build(); + IndexSettings settingsWithFormat = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + + // MockDataFormatPlugin does not override getStoreStrategies, so the default returns + // an empty map. + Map result = registry.getStoreStrategies(settingsWithFormat); + assertTrue("Should return empty map when plugin provides no strategy", result.isEmpty()); + } + + public void testGetStoreStrategiesEmptyWhenFormatNameNotRegistered() { + MockDataFormat format = new MockDataFormat("columnar", 100L, Set.of()); + MockSearchBackEndPlugin backEnd = new MockSearchBackEndPlugin(List.of(format.name())); + + when(pluginsService.filterPlugins(DataFormatPlugin.class)).thenReturn(List.of(MockDataFormatPlugin.of(format))); + when(pluginsService.filterPlugins(SearchBackEndPlugin.class)).thenReturn(List.of(backEnd)); + + DataFormatRegistry registry = new DataFormatRegistry(pluginsService); + + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT) + .put("index.pluggable.dataformat", "unknown") + .put("index.pluggable.dataformat.enabled", true) + .build(); + IndexSettings settingsWithFormat = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + + Map result = registry.getStoreStrategies(settingsWithFormat); + assertTrue("Should return empty map when format name not registered", result.isEmpty()); + } + + public void testGetPluginReturnsPluginForRegisteredFormat() { + MockDataFormat format = new MockDataFormat("columnar", 100L, Set.of()); + MockSearchBackEndPlugin backEnd = new MockSearchBackEndPlugin(List.of(format.name())); + MockDataFormatPlugin plugin = MockDataFormatPlugin.of(format); + + when(pluginsService.filterPlugins(DataFormatPlugin.class)).thenReturn(List.of(plugin)); + when(pluginsService.filterPlugins(SearchBackEndPlugin.class)).thenReturn(List.of(backEnd)); + + DataFormatRegistry registry = new DataFormatRegistry(pluginsService); + + DataFormatPlugin result = registry.getPlugin("columnar"); + assertNotNull("Should return plugin for registered format", result); + assertSame("Should return the same plugin instance", plugin, result); + } + + public void testGetPluginReturnsNullForUnknownFormat() { + MockDataFormat format = new MockDataFormat("columnar", 100L, Set.of()); + MockSearchBackEndPlugin backEnd = new MockSearchBackEndPlugin(List.of(format.name())); + + when(pluginsService.filterPlugins(DataFormatPlugin.class)).thenReturn(List.of(MockDataFormatPlugin.of(format))); + when(pluginsService.filterPlugins(SearchBackEndPlugin.class)).thenReturn(List.of(backEnd)); + + DataFormatRegistry registry = new DataFormatRegistry(pluginsService); + + assertNull("Should return null for unknown format", registry.getPlugin("unknown")); + } + + public void testGetPluginReturnsNullForNullName() { + MockDataFormat format = new MockDataFormat("columnar", 100L, Set.of()); + MockSearchBackEndPlugin backEnd = new MockSearchBackEndPlugin(List.of(format.name())); + + when(pluginsService.filterPlugins(DataFormatPlugin.class)).thenReturn(List.of(MockDataFormatPlugin.of(format))); + when(pluginsService.filterPlugins(SearchBackEndPlugin.class)).thenReturn(List.of(backEnd)); + + DataFormatRegistry registry = new DataFormatRegistry(pluginsService); + + assertNull("Should return empty map for null name", registry.getPlugin(null)); + } } diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreUploaderServiceTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreUploaderServiceTests.java index 3e0c4a032bed8..2842220eaa2b9 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreUploaderServiceTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreUploaderServiceTests.java @@ -19,6 +19,7 @@ import org.opensearch.index.store.CompositeDirectory; import org.opensearch.index.store.RemoteDirectory; import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.RemoteSyncListener; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; @@ -486,4 +487,194 @@ public TestFilterDirectory(Directory in) { super(in); } } + + // ═══════════════════════════════════════════════════════════════ + // RemoteSyncListener registration tests + // ═══════════════════════════════════════════════════════════════ + + /** + * Uploader auto-discovers RemoteSyncListener from directory chain at construction time. + */ + public void testSyncListenerAutoRegisteredFromDirectoryChain() throws Exception { + IndexShard freshMockShard = mock(IndexShard.class); + ShardId shardId = new ShardId(new Index("test", "test"), 1); + when(freshMockShard.shardId()).thenReturn(shardId); + when(freshMockShard.state()).thenReturn(IndexShardState.STARTED); + + // Create a concrete RemoteSyncListener directory that tracks afterSyncToRemote calls + Directory innerMockDelegate = mock(Directory.class); + TrackingSyncListenerDirectory listenerDir = new TrackingSyncListenerDirectory(innerMockDelegate); + TestFilterDirectory outerDir = new TestFilterDirectory(listenerDir); + + RemoteDirectory remoteDataDirectory = mock(RemoteDirectory.class); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( + remoteDataDirectory, + mock(RemoteDirectory.class), + mock(RemoteStoreLockManager.class), + freshMockShard.getThreadPool(), + freshMockShard.shardId(), + new HashMap<>() + ); + + RemoteStoreUploaderService testUploaderService = new RemoteStoreUploaderService( + freshMockShard, + outerDir, + remoteSegmentStoreDirectory + ); + + doAnswer(invocation -> { + ActionListener callback = invocation.getArgument(5); + callback.onResponse(null); + return true; + }).when(remoteDataDirectory).copyFrom(any(), any(), any(), any(), any(), any(), any(Boolean.class), any()); + + CountDownLatch latch = new CountDownLatch(1); + testUploaderService.uploadSegments( + Collections.singletonList("seg1"), + Map.of("seg1", 100L), + ActionListener.wrap(r -> latch.countDown(), e -> fail("Should not fail")), + mockUploadListenerFunction, + false, + null + ); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertEquals("afterSyncToRemote should be called once", 1, listenerDir.syncCount); + assertEquals("seg1", listenerDir.lastFile); + } + + /** + * When no RemoteSyncListener in directory chain, upload still succeeds (no-op notification). + */ + public void testNoSyncListenerInChainStillWorks() throws Exception { + IndexShard freshMockShard = mock(IndexShard.class); + ShardId shardId = new ShardId(new Index("test", "test"), 1); + when(freshMockShard.shardId()).thenReturn(shardId); + when(freshMockShard.state()).thenReturn(IndexShardState.STARTED); + + // Plain FilterDirectory — no RemoteSyncListener + Directory innerDir = mock(Directory.class); + TestFilterDirectory outerDir = new TestFilterDirectory(new TestFilterDirectory(innerDir)); + + RemoteDirectory remoteDataDirectory = mock(RemoteDirectory.class); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( + remoteDataDirectory, + mock(RemoteDirectory.class), + mock(RemoteStoreLockManager.class), + freshMockShard.getThreadPool(), + freshMockShard.shardId(), + new HashMap<>() + ); + + RemoteStoreUploaderService testUploaderService = new RemoteStoreUploaderService( + freshMockShard, + outerDir, + remoteSegmentStoreDirectory + ); + + doAnswer(invocation -> { + ActionListener callback = invocation.getArgument(5); + callback.onResponse(null); + return true; + }).when(remoteDataDirectory).copyFrom(any(), any(), any(), any(), any(), any(), any(Boolean.class), any()); + + CountDownLatch latch = new CountDownLatch(1); + testUploaderService.uploadSegments( + Collections.singletonList("seg1"), + Map.of("seg1", 100L), + ActionListener.wrap(r -> latch.countDown(), e -> fail("Should not fail")), + mockUploadListenerFunction, + false, + null + ); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + // No exception = pass. No listener to call. + } + + /** + * addSyncListener allows manually adding extra listeners beyond auto-discovery. + */ + public void testAddSyncListenerManually() throws Exception { + IndexShard freshMockShard = mock(IndexShard.class); + ShardId shardId = new ShardId(new Index("test", "test"), 1); + when(freshMockShard.shardId()).thenReturn(shardId); + when(freshMockShard.state()).thenReturn(IndexShardState.STARTED); + + Directory innerDir = mock(Directory.class); + TestFilterDirectory outerDir = new TestFilterDirectory(new TestFilterDirectory(innerDir)); + + RemoteDirectory remoteDataDirectory = mock(RemoteDirectory.class); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory( + remoteDataDirectory, + mock(RemoteDirectory.class), + mock(RemoteStoreLockManager.class), + freshMockShard.getThreadPool(), + freshMockShard.shardId(), + new HashMap<>() + ); + + RemoteStoreUploaderService testUploaderService = new RemoteStoreUploaderService( + freshMockShard, + outerDir, + remoteSegmentStoreDirectory + ); + + RemoteSyncListener manualListener = mock(RemoteSyncListener.class); + testUploaderService.addSyncListener(manualListener); + + doAnswer(invocation -> { + ActionListener callback = invocation.getArgument(5); + callback.onResponse(null); + return true; + }).when(remoteDataDirectory).copyFrom(any(), any(), any(), any(), any(), any(), any(Boolean.class), any()); + + CountDownLatch latch = new CountDownLatch(1); + testUploaderService.uploadSegments( + Collections.singletonList("seg1"), + Map.of("seg1", 100L), + ActionListener.wrap(r -> latch.countDown(), e -> fail("Should not fail")), + mockUploadListenerFunction, + false, + null + ); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + verify(manualListener).afterSyncToRemote("seg1"); + } + + /** + * addSyncListener with null is a no-op (no NPE). + */ + public void testAddSyncListenerNullIsNoOp() throws Exception { + RemoteDirectory remoteDataDirectory = mock(RemoteDirectory.class); + RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = createMockRemoteDirectory(remoteDataDirectory); + + RemoteStoreUploaderService testUploaderService = new RemoteStoreUploaderService( + mockIndexShard, + mock(Directory.class), + remoteSegmentStoreDirectory + ); + + // Should not throw + testUploaderService.addSyncListener(null); + } + + /** + * Concrete FilterDirectory that implements RemoteSyncListener and tracks calls. + */ + static class TrackingSyncListenerDirectory extends FilterDirectory implements RemoteSyncListener { + volatile int syncCount = 0; + volatile String lastFile = null; + + TrackingSyncListenerDirectory(Directory in) { + super(in); + } + + @Override + public void afterSyncToRemote(String file) { + syncCount++; + lastFile = file; + } + } } diff --git a/server/src/test/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryTests.java index c8566ecbb7715..a6c36700bafc4 100644 --- a/server/src/test/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/DataFormatAwareStoreDirectoryTests.java @@ -30,6 +30,8 @@ import java.util.Set; import java.util.zip.CRC32; +import static org.mockito.Mockito.mock; + public class DataFormatAwareStoreDirectoryTests extends OpenSearchTestCase { private Path tempDir; @@ -966,4 +968,37 @@ public void testGetDataFormat_comprehensive() { assertEquals("orc", dataFormatAwareStoreDirectory.getDataFormat("orc/data.orc")); assertEquals("custom", dataFormatAwareStoreDirectory.getDataFormat("custom/myfile.dat")); } + + public void testAfterSyncToRemoteWithNonRemoteSyncAwareDelegate() { + // Default constructor wraps delegate in SubdirectoryAwareDirectory which does NOT + // implement RemoteSyncListener → afterSyncToRemote should be a no-op + dataFormatAwareStoreDirectory.afterSyncToRemote("_0.cfe"); + // No exception = pass. The inner SubdirectoryAwareDirectory is not RemoteSyncListener. + } + + public void testAfterSyncToRemoteWithRemoteSyncAwareDelegate() { + // We need a Directory that is also RemoteSyncListener — use the abstract helper + RemoteSyncListenerMockDirectory syncAwareDir = mock(RemoteSyncListenerMockDirectory.class); + + DataFormatAwareStoreDirectory dir = DataFormatAwareStoreDirectory.withDirectoryDelegate(syncAwareDir, shardPath, Map.of()); + dir.afterSyncToRemote("_0.cfe"); + org.mockito.Mockito.verify(syncAwareDir).afterSyncToRemote("_0.cfe"); + } + + public void testDirectDelegateConstructorDoesNotDoubleWrap() throws IOException { + // withDirectDelegate should use the delegate as-is + SubdirectoryAwareDirectory subdirAware = new SubdirectoryAwareDirectory(fsDirectory, shardPath); + DataFormatAwareStoreDirectory dir = DataFormatAwareStoreDirectory.withDirectoryDelegate(subdirAware, shardPath, Map.of()); + + // The delegate should be the SubdirectoryAwareDirectory directly, not wrapped again + org.apache.lucene.store.Directory delegate = org.apache.lucene.store.FilterDirectory.unwrap(dir); + // unwrap goes all the way to the leaf — should be FSDirectory + assertTrue("Leaf should be FSDirectory", delegate instanceof FSDirectory); + dir.close(); + } + + /** + * Helper interface for mocking a Directory that also implements RemoteSyncListener. + */ + abstract static class RemoteSyncListenerMockDirectory extends org.apache.lucene.store.Directory implements RemoteSyncListener {} } diff --git a/server/src/test/java/org/opensearch/plugins/NativeStoreHandleTests.java b/server/src/test/java/org/opensearch/plugins/NativeStoreHandleTests.java new file mode 100644 index 0000000000000..e89c26b8e7f61 --- /dev/null +++ b/server/src/test/java/org/opensearch/plugins/NativeStoreHandleTests.java @@ -0,0 +1,136 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +public class NativeStoreHandleTests extends OpenSearchTestCase { + + public void testCreateRegistersInLivePointers() { + NativeStoreHandle handle = new NativeStoreHandle(100L, ptr -> {}); + assertTrue(NativeStoreHandle.isLivePointer(100L)); + handle.close(); + } + + public void testCloseRemovesFromLivePointers() { + NativeStoreHandle handle = new NativeStoreHandle(200L, ptr -> {}); + assertTrue(NativeStoreHandle.isLivePointer(200L)); + handle.close(); + assertFalse(NativeStoreHandle.isLivePointer(200L)); + } + + public void testIsLiveReturnsTrueWhenOpen() { + NativeStoreHandle handle = new NativeStoreHandle(300L, ptr -> {}); + assertTrue(handle.isLive()); + handle.close(); + } + + public void testIsLiveReturnsFalseAfterClose() { + NativeStoreHandle handle = new NativeStoreHandle(400L, ptr -> {}); + handle.close(); + assertFalse(handle.isLive()); + } + + public void testIsLiveReturnsFalseForEmpty() { + assertFalse(NativeStoreHandle.EMPTY.isLive()); + } + + public void testGetPointerThrowsAfterClose() { + NativeStoreHandle handle = new NativeStoreHandle(500L, ptr -> {}); + handle.close(); + expectThrows(IllegalStateException.class, handle::getPointer); + } + + public void testGetPointerReturnsValueWhenOpen() { + NativeStoreHandle handle = new NativeStoreHandle(600L, ptr -> {}); + assertEquals(600L, handle.getPointer()); + handle.close(); + } + + public void testEmptyGetPointerReturnsNegativeOne() { + assertEquals(-1L, NativeStoreHandle.EMPTY.getPointer()); + } + + public void testCloseIsIdempotent() { + AtomicInteger destroyCount = new AtomicInteger(0); + NativeStoreHandle handle = new NativeStoreHandle(700L, ptr -> destroyCount.incrementAndGet()); + handle.close(); + handle.close(); + handle.close(); + assertEquals("Destroyer should only be called once", 1, destroyCount.get()); + } + + public void testDestroyerCalledWithCorrectPointer() { + AtomicBoolean called = new AtomicBoolean(false); + long[] capturedPtr = new long[1]; + NativeStoreHandle handle = new NativeStoreHandle(800L, ptr -> { + capturedPtr[0] = ptr; + called.set(true); + }); + handle.close(); + assertTrue(called.get()); + assertEquals(800L, capturedPtr[0]); + } + + public void testValidatePointerSucceedsForLiveHandle() { + NativeStoreHandle handle = new NativeStoreHandle(900L, ptr -> {}); + NativeStoreHandle.validatePointer(900L, "test"); + handle.close(); + } + + public void testValidatePointerThrowsForClosedHandle() { + NativeStoreHandle handle = new NativeStoreHandle(1000L, ptr -> {}); + handle.close(); + expectThrows(IllegalStateException.class, () -> NativeStoreHandle.validatePointer(1000L, "test")); + } + + public void testValidatePointerThrowsForUnknownPointer() { + expectThrows(IllegalStateException.class, () -> NativeStoreHandle.validatePointer(99999L, "test")); + } + + public void testValidatePointerThrowsForZero() { + expectThrows(IllegalArgumentException.class, () -> NativeStoreHandle.validatePointer(0L, "test")); + } + + public void testValidatePointerThrowsForNegative() { + expectThrows(IllegalArgumentException.class, () -> NativeStoreHandle.validatePointer(-1L, "test")); + } + + public void testLiveHandleCount() { + int before = NativeStoreHandle.liveHandleCount(); + NativeStoreHandle h1 = new NativeStoreHandle(1100L, ptr -> {}); + NativeStoreHandle h2 = new NativeStoreHandle(1200L, ptr -> {}); + assertEquals(before + 2, NativeStoreHandle.liveHandleCount()); + h1.close(); + assertEquals(before + 1, NativeStoreHandle.liveHandleCount()); + h2.close(); + assertEquals(before, NativeStoreHandle.liveHandleCount()); + } + + public void testConstructorRejectsZeroPointer() { + expectThrows(IllegalArgumentException.class, () -> new NativeStoreHandle(0L, ptr -> {})); + } + + public void testConstructorRejectsNegativePointer() { + expectThrows(IllegalArgumentException.class, () -> new NativeStoreHandle(-5L, ptr -> {})); + } + + public void testConstructorRejectsNullDestroyer() { + expectThrows(IllegalArgumentException.class, () -> new NativeStoreHandle(1300L, null)); + } + + public void testEmptyCloseIsNoOp() { + // Should not throw + NativeStoreHandle.EMPTY.close(); + NativeStoreHandle.EMPTY.close(); + } +} diff --git a/server/src/test/java/org/opensearch/storage/directory/GracefulDegradationTests.java b/server/src/test/java/org/opensearch/storage/directory/GracefulDegradationTests.java new file mode 100644 index 0000000000000..dfd20456ea6cb --- /dev/null +++ b/server/src/test/java/org/opensearch/storage/directory/GracefulDegradationTests.java @@ -0,0 +1,197 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.directory; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FilterDirectory; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.DataFormatAwareStoreDirectory; +import org.opensearch.index.store.RemoteDirectory; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.SubdirectoryAwareDirectory; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; +import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.index.store.remote.filecache.FileCacheFactory; +import org.opensearch.plugins.IndexStorePlugin; +import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Tests for graceful degradation when sandbox plugins are not loaded. + * + *

      When no data format plugins provide tiered directories (e.g., sandbox not loaded), + * the warm directory stack should still function correctly using only the default + * TieredDirectory for Lucene files. No errors should occur. + */ +public class GracefulDegradationTests extends OpenSearchTestCase { + + private Supplier getMockPrefetchSettingsSupplier() { + return () -> { + TieredStoragePrefetchSettings settings = mock(TieredStoragePrefetchSettings.class); + when(settings.getReadAheadBlockCount()).thenReturn(TieredStoragePrefetchSettings.DEFAULT_READ_AHEAD_BLOCK_COUNT); + when(settings.getReadAheadEnableFileFormats()).thenReturn(TieredStoragePrefetchSettings.READ_AHEAD_ENABLE_FILE_FORMATS); + when(settings.isStoredFieldsPrefetchEnabled()).thenReturn(true); + return settings; + }; + } + + /** + * Tests that when DataFormatRegistry returns empty tiered directories (simulating + * sandbox not loaded), the factory creates a valid directory stack that works + * for plain Lucene warm operations without errors. + */ + public void testNoFormatPluginsCreatesValidStack() throws IOException { + Path tempDir = createTempDir(); + Index index = new Index("test-degradation", "test-uuid"); + ShardId shardId = new ShardId(index, 0); + + Path shardStatePath = tempDir.resolve("state").resolve("test-uuid").resolve("0"); + Path shardDataPath = tempDir.resolve("data").resolve("test-uuid").resolve("0"); + Files.createDirectories(shardStatePath); + Files.createDirectories(shardDataPath); + Files.createDirectories(shardDataPath.resolve("index")); + + ShardPath shardPath = new ShardPath(false, shardDataPath, shardStatePath, shardId); + + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + IndexMetadata indexMetadata = IndexMetadata.builder("test-degradation").settings(settings).build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + + FSDirectory fsDir = FSDirectory.open(shardPath.resolveIndex()); + IndexStorePlugin.DirectoryFactory localDirFactory = mock(IndexStorePlugin.DirectoryFactory.class); + when(localDirFactory.newDirectory(any(), any())).thenReturn(fsDir); + + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(shardId); + FileCache fileCache = FileCacheFactory.createConcurrentLRUFileCache(10_000_000, 1); + + TieredDataFormatAwareStoreDirectoryFactory factory = new TieredDataFormatAwareStoreDirectoryFactory( + getMockPrefetchSettingsSupplier() + ); + + // Should not throw — graceful degradation + DataFormatAwareStoreDirectory storeDir = factory.newDataFormatAwareStoreDirectory( + indexSettings, + shardId, + shardPath, + localDirFactory, + Map.of(), + java.util.Map.of(), + org.opensearch.repositories.NativeStoreRepository.EMPTY, + true, + remoteDir, + fileCache, + null + ); + + assertNotNull("Directory should be created even without format plugins", storeDir); + + // Verify the stack is correct + Directory delegate = ((FilterDirectory) storeDir).getDelegate(); + assertTrue( + "Should have TieredSubdirectoryAwareDirectory even without format plugins", + delegate instanceof TieredSubdirectoryAwareDirectory + ); + + Directory innerDelegate = ((FilterDirectory) delegate).getDelegate(); + assertTrue("Should have SubdirectoryAwareDirectory", innerDelegate instanceof SubdirectoryAwareDirectory); + + storeDir.close(); + } + + /** + * Tests that TieredSubdirectoryAwareDirectory with empty format directories + * routes all operations to TieredDirectory without errors. + */ + public void testEmptyFormatDirectoriesRoutesToTieredDirectory() throws IOException { + Path tempDir = createTempDir(); + Index index = new Index("test-empty-formats", "test-uuid"); + ShardId shardId = new ShardId(index, 0); + + Path shardStatePath = tempDir.resolve("state").resolve("test-uuid").resolve("0"); + Path shardDataPath = tempDir.resolve("data").resolve("test-uuid").resolve("0"); + Files.createDirectories(shardStatePath); + Files.createDirectories(shardDataPath); + Files.createDirectories(shardDataPath.resolve("index")); + + ShardPath shardPath = new ShardPath(false, shardDataPath, shardStatePath, shardId); + + FSDirectory fsDir = FSDirectory.open(shardPath.resolveIndex()); + SubdirectoryAwareDirectory subdirAware = new SubdirectoryAwareDirectory(fsDir, shardPath); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(shardId); + FileCache fileCache = FileCacheFactory.createConcurrentLRUFileCache(10_000_000, 1); + + // Empty strategies — simulates no sandbox plugins + + TieredSubdirectoryAwareDirectory tieredSubdir = new TieredSubdirectoryAwareDirectory( + subdirAware, + remoteDir, + fileCache, + null, + StoreStrategyRegistry.EMPTY, + shardPath, + getMockPrefetchSettingsSupplier() + ); + + // listAll should work without errors + String[] files = tieredSubdir.listAll(); + assertNotNull("listAll should return non-null", files); + + // close should not throw + tieredSubdir.close(); + } + + /** + * Tests that the factory key constant is correctly defined. + */ + public void testFactoryKeyConstant() { + assertEquals( + "Factory key should be 'dataformat-tiered'", + "dataformat-tiered", + TieredDataFormatAwareStoreDirectoryFactory.FACTORY_KEY + ); + } + + private RemoteSegmentStoreDirectory createRealRemoteDir(ShardId shardId) throws IOException { + RemoteDirectory remoteDataDir = mock(RemoteDirectory.class); + RemoteDirectory remoteMetadataDir = mock(RemoteDirectory.class); + RemoteStoreLockManager lockManager = mock(RemoteStoreLockManager.class); + ThreadPool tp = mock(ThreadPool.class); + + BlobContainer mockBlobContainer = mock(BlobContainer.class); + when(mockBlobContainer.path()).thenReturn(new BlobPath().add("test-base-path")); + when(remoteDataDir.getBlobContainer()).thenReturn(mockBlobContainer); + + return new RemoteSegmentStoreDirectory(remoteDataDir, remoteMetadataDir, lockManager, tp, shardId, new HashMap<>()); + } +} diff --git a/server/src/test/java/org/opensearch/storage/directory/StoreStrategyRegistryTests.java b/server/src/test/java/org/opensearch/storage/directory/StoreStrategyRegistryTests.java new file mode 100644 index 0000000000000..7911048ca6498 --- /dev/null +++ b/server/src/test/java/org/opensearch/storage/directory/StoreStrategyRegistryTests.java @@ -0,0 +1,529 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.directory; + +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.DataFormatStoreHandler; +import org.opensearch.index.engine.dataformat.DataFormatStoreHandlerFactory; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; +import org.opensearch.index.engine.dataformat.StoreStrategy; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.RemoteDirectory; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.RemoteSegmentStoreDirectory.UploadedSegmentMetadata; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; +import org.opensearch.plugins.NativeStoreHandle; +import org.opensearch.repositories.NativeStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link StoreStrategyRegistry}. + */ +public class StoreStrategyRegistryTests extends OpenSearchTestCase { + + private static final DataFormat PARQUET_FORMAT = new DataFormat() { + @Override + public String name() { + return "parquet"; + } + + @Override + public long priority() { + return 2; + } + + @Override + public Set supportedFields() { + return Set.of(); + } + }; + + private ShardPath shardPath; + + @Override + public void setUp() throws Exception { + super.setUp(); + Path tempDir = createTempDir(); + Index index = new Index("test-index", "test-uuid"); + ShardId shardId = new ShardId(index, 0); + Path shardDataPath = tempDir.resolve("data").resolve("test-uuid").resolve("0"); + Path shardStatePath = tempDir.resolve("state").resolve("test-uuid").resolve("0"); + Files.createDirectories(shardDataPath.resolve("index")); + Files.createDirectories(shardStatePath); + shardPath = new ShardPath(false, shardDataPath, shardStatePath, shardId); + } + + // ═══════════════════════════════════════════════════════════════ + // open() tests + // ═══════════════════════════════════════════════════════════════ + + public void testOpenWithNullStrategiesReturnsEmpty() throws IOException { + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + StoreStrategyRegistry registry = StoreStrategyRegistry.open(shardPath, true, NativeStoreRepository.EMPTY, null, remoteDir); + assertSame(StoreStrategyRegistry.EMPTY, registry); + } + + public void testOpenWithEmptyStrategiesReturnsEmpty() throws IOException { + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Collections.emptyMap(), + remoteDir + ); + assertSame(StoreStrategyRegistry.EMPTY, registry); + } + + public void testOpenCreatesHandlerFromFactory() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + assertNotSame(StoreStrategyRegistry.EMPTY, registry); + assertTrue(registry.hasStoreHandlers()); + registry.close(); + } + + public void testOpenFactoryThrowsClosesCreatedHandlers() throws IOException { + // First format succeeds + DataFormatStoreHandler successHandler = mock(DataFormatStoreHandler.class); + DataFormat format1 = new DataFormat() { + @Override + public String name() { + return "format1"; + } + + @Override + public long priority() { + return 1; + } + + @Override + public Set supportedFields() { + return Set.of(); + } + }; + StoreStrategy strategy1 = createTestStrategy(successHandler); + + // Second format throws during factory.create() + DataFormat format2 = new DataFormat() { + @Override + public String name() { + return "format2"; + } + + @Override + public long priority() { + return 2; + } + + @Override + public Set supportedFields() { + return Set.of(); + } + }; + StoreStrategy strategy2 = new StoreStrategy() { + @Override + public Optional storeHandler() { + return Optional.of((shardId, isWarm, repo) -> { throw new RuntimeException("factory boom"); }); + } + }; + + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + // Use a LinkedHashMap to guarantee iteration order: format1 first, format2 second + Map strategies = new java.util.LinkedHashMap<>(); + strategies.put(format1, strategy1); + strategies.put(format2, strategy2); + + expectThrows( + RuntimeException.class, + () -> StoreStrategyRegistry.open(shardPath, true, NativeStoreRepository.EMPTY, strategies, remoteDir) + ); + + // The successfully created handler should have been closed during cleanup + verify(successHandler).close(); + } + + // ═══════════════════════════════════════════════════════════════ + // matchFor() tests + // ═══════════════════════════════════════════════════════════════ + + public void testMatchForReturnsNullForLuceneFile() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + assertNull(registry.matchFor("_0.cfe")); + registry.close(); + } + + public void testMatchForReturnsMatchForFormatFile() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + StoreStrategyRegistry.Match match = registry.matchFor("parquet/_0.parquet"); + assertNotNull(match); + assertEquals(PARQUET_FORMAT, match.format()); + assertSame(strategy, match.strategy()); + registry.close(); + } + + public void testMatchForReturnsNullForNull() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + assertNull(registry.matchFor(null)); + registry.close(); + } + + // ═══════════════════════════════════════════════════════════════ + // onUploaded() tests + // ═══════════════════════════════════════════════════════════════ + + public void testOnUploadedDispatchesToHandler() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + boolean dispatched = registry.onUploaded("parquet/_0.parquet", "test-base-path/", "new_blob_key", 1024L); + assertTrue(dispatched); + // remotePath default: basePath + name + "/" + blobKey + verify(handler).onUploaded( + org.mockito.ArgumentMatchers.contains("parquet/_0.parquet"), + eq("test-base-path/parquet/new_blob_key"), + eq(1024L) + ); + registry.close(); + } + + public void testOnUploadedReturnsFalseForUnownedFile() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + boolean dispatched = registry.onUploaded("_0.cfe", "test-base-path/", "blob_key", 512L); + assertFalse(dispatched); + verify(handler, never()).onUploaded(anyString(), anyString(), org.mockito.ArgumentMatchers.anyLong()); + registry.close(); + } + + // ═══════════════════════════════════════════════════════════════ + // onRemoved() tests + // ═══════════════════════════════════════════════════════════════ + + public void testOnRemovedDispatchesToHandler() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + boolean dispatched = registry.onRemoved("parquet/_0.parquet"); + assertTrue(dispatched); + verify(handler).onRemoved(org.mockito.ArgumentMatchers.contains("parquet/_0.parquet")); + registry.close(); + } + + public void testOnRemovedReturnsFalseForUnownedFile() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + boolean dispatched = registry.onRemoved("_0.cfe"); + assertFalse(dispatched); + verify(handler, never()).onRemoved(anyString()); + registry.close(); + } + + // ═══════════════════════════════════════════════════════════════ + // getFormatStoreHandles() tests + // ═══════════════════════════════════════════════════════════════ + + public void testGetFormatStoreHandlesReturnsLiveHandles() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + NativeStoreHandle liveHandle = new NativeStoreHandle(42L, ptr -> {}); + when(handler.getFormatStoreHandle()).thenReturn(liveHandle); + + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + Map handles = registry.getFormatStoreHandles(); + assertEquals(1, handles.size()); + assertSame(liveHandle, handles.get(PARQUET_FORMAT)); + + liveHandle.close(); + registry.close(); + } + + public void testGetFormatStoreHandlesSkipsClosedHandles() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + NativeStoreHandle closedHandle = new NativeStoreHandle(99L, ptr -> {}); + closedHandle.close(); // close it before returning + when(handler.getFormatStoreHandle()).thenReturn(closedHandle); + + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + Map handles = registry.getFormatStoreHandles(); + assertTrue("Closed handles should not be returned", handles.isEmpty()); + + registry.close(); + } + + public void testGetFormatStoreHandlesReturnsSameHandleOnMultipleCalls() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + NativeStoreHandle liveHandle = new NativeStoreHandle(77L, ptr -> {}); + when(handler.getFormatStoreHandle()).thenReturn(liveHandle); + + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + Map handles1 = registry.getFormatStoreHandles(); + Map handles2 = registry.getFormatStoreHandles(); + assertSame("Same handle should be returned on multiple calls", handles1.get(PARQUET_FORMAT), handles2.get(PARQUET_FORMAT)); + + liveHandle.close(); + registry.close(); + } + + public void testGetFormatStoreHandlesEmptyWhenNoHandlers() { + Map handles = StoreStrategyRegistry.EMPTY.getFormatStoreHandles(); + assertTrue(handles.isEmpty()); + } + + // ═══════════════════════════════════════════════════════════════ + // close() tests + // ═══════════════════════════════════════════════════════════════ + + public void testCloseClosesAllHandlers() throws IOException { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + StoreStrategy strategy = createTestStrategy(handler); + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + registry.close(); + verify(handler).close(); + } + + // ═══════════════════════════════════════════════════════════════ + // Seed key tests + // ═══════════════════════════════════════════════════════════════ + + public void testSeedUsesAbsolutePathKeys() throws Exception { + DataFormatStoreHandler handler = mock(DataFormatStoreHandler.class); + StoreStrategy strategy = createTestStrategy(handler); + + // Create a real RemoteSegmentStoreDirectory and inject a parquet entry + // into its uploaded segments map via reflection so that seedFromRemoteMetadata + // picks it up during open(). + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(); + injectUploadedSegment(remoteDir, "parquet/_0.parquet", "parquet_blob_key"); + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, strategy), + remoteDir + ); + + // Capture the seed call argument + @SuppressWarnings("unchecked") + org.mockito.ArgumentCaptor> captor = org.mockito.ArgumentCaptor.forClass(Map.class); + verify(handler).seed(captor.capture()); + + Map seeded = captor.getValue(); + assertFalse("Seed map should not be empty", seeded.isEmpty()); + + // The key should be the absolute path: shardPath.getDataPath() + relative file + String expectedKey = shardPath.getDataPath().resolve("parquet/_0.parquet").toString(); + assertTrue("Seed key should be absolute path: " + expectedKey, seeded.containsKey(expectedKey)); + + DataFormatStoreHandler.FileEntry entry = seeded.get(expectedKey); + assertEquals("test-base-path/parquet/parquet_blob_key", entry.path()); + assertEquals(DataFormatStoreHandler.REMOTE, entry.location()); + + registry.close(); + } + + // ═══════════════════════════════════════════════════════════════ + // Helpers + // ═══════════════════════════════════════════════════════════════ + + private StoreStrategy createTestStrategy(DataFormatStoreHandler handler) { + DataFormatStoreHandlerFactory factory = (shardId, isWarm, repo) -> handler; + return new StoreStrategy() { + @Override + public Optional storeHandler() { + return Optional.of(factory); + } + }; + } + + private RemoteSegmentStoreDirectory createRealRemoteDir() throws IOException { + RemoteDirectory remoteDataDir = mock(RemoteDirectory.class); + RemoteDirectory remoteMetadataDir = mock(RemoteDirectory.class); + RemoteStoreLockManager lockManager = mock(RemoteStoreLockManager.class); + ThreadPool tp = mock(ThreadPool.class); + + BlobContainer mockBlobContainer = mock(BlobContainer.class); + when(mockBlobContainer.path()).thenReturn(new BlobPath().add("test-base-path")); + when(remoteDataDir.getBlobContainer()).thenReturn(mockBlobContainer); + + return new RemoteSegmentStoreDirectory(remoteDataDir, remoteMetadataDir, lockManager, tp, shardPath.getShardId(), new HashMap<>()); + } + + /** + * Injects an uploaded segment entry into the RemoteSegmentStoreDirectory's + * internal map via reflection. This avoids the need to set up the full + * metadata serialization pipeline just to test seeding behaviour. + */ + @SuppressForbidden(reason = "test needs reflection to inject parquet metadata without full upload pipeline") + private static void injectUploadedSegment(RemoteSegmentStoreDirectory remoteDir, String localFilename, String uploadedFilename) + throws Exception { + Field field = RemoteSegmentStoreDirectory.class.getDeclaredField("segmentsUploadedToRemoteStore"); + field.setAccessible(true); + @SuppressWarnings("unchecked") + Map map = (Map) field.get(remoteDir); + // The UploadedSegmentMetadata constructor is package-private, so we use fromString + // Format: originalFilename::uploadedFilename::checksum::length::writtenByMajor + String separator = "::"; + String metadataStr = localFilename + + separator + + uploadedFilename + + separator + + "checksum123" + + separator + + "1024" + + separator + + org.apache.lucene.util.Version.LATEST.major; + UploadedSegmentMetadata metadata = UploadedSegmentMetadata.fromString(metadataStr); + map.put(localFilename, metadata); + } +} diff --git a/server/src/test/java/org/opensearch/storage/directory/TieredDataFormatAwareStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/storage/directory/TieredDataFormatAwareStoreDirectoryFactoryTests.java new file mode 100644 index 0000000000000..b6f546ea1b4ff --- /dev/null +++ b/server/src/test/java/org/opensearch/storage/directory/TieredDataFormatAwareStoreDirectoryFactoryTests.java @@ -0,0 +1,228 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.directory; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FilterDirectory; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.DataFormatAwareStoreDirectory; +import org.opensearch.index.store.RemoteDirectory; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.SubdirectoryAwareDirectory; +import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.index.store.remote.filecache.FileCacheFactory; +import org.opensearch.plugins.IndexStorePlugin; +import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Tests for {@link TieredDataFormatAwareStoreDirectoryFactory}. + * + *

      Verifies the factory creates the correct directory stack for warm+format indices + * and rejects the hot path (5-param method). + */ +public class TieredDataFormatAwareStoreDirectoryFactoryTests extends OpenSearchTestCase { + + private TieredDataFormatAwareStoreDirectoryFactory factory; + private IndexSettings indexSettings; + private ShardId shardId; + private ShardPath shardPath; + private IndexStorePlugin.DirectoryFactory localDirectoryFactory; + private RemoteSegmentStoreDirectory remoteDirectory; + private FileCache fileCache; + private ThreadPool threadPool; + + /** + * Sets up the factory and mock dependencies before each test. + */ + @Before + public void setup() throws IOException { + Supplier prefetchSupplier = () -> { + TieredStoragePrefetchSettings settings = mock(TieredStoragePrefetchSettings.class); + when(settings.getReadAheadBlockCount()).thenReturn(TieredStoragePrefetchSettings.DEFAULT_READ_AHEAD_BLOCK_COUNT); + when(settings.getReadAheadEnableFileFormats()).thenReturn(TieredStoragePrefetchSettings.READ_AHEAD_ENABLE_FILE_FORMATS); + when(settings.isStoredFieldsPrefetchEnabled()).thenReturn(true); + return settings; + }; + factory = new TieredDataFormatAwareStoreDirectoryFactory(prefetchSupplier); + + Path tempDir = createTempDir(); + Index index = new Index("test-index", "test-uuid"); + shardId = new ShardId(index, 0); + + // ShardPath requires: dataPath ends with / + Path shardStatePath = tempDir.resolve("state").resolve("test-uuid").resolve("0"); + Path shardDataPath = tempDir.resolve("data").resolve("test-uuid").resolve("0"); + Path indexPath = shardDataPath.resolve("index"); + java.nio.file.Files.createDirectories(shardStatePath); + java.nio.file.Files.createDirectories(shardDataPath); + java.nio.file.Files.createDirectories(indexPath); + shardPath = new ShardPath(false, shardDataPath, shardStatePath, shardId); + + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + IndexMetadata indexMetadata = IndexMetadata.builder("test-index").settings(settings).build(); + indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + + FSDirectory fsDir = FSDirectory.open(indexPath); + localDirectoryFactory = mock(IndexStorePlugin.DirectoryFactory.class); + when(localDirectoryFactory.newDirectory(any(), any())).thenReturn(fsDir); + + remoteDirectory = createRealRemoteSegmentStoreDirectory(shardId); + fileCache = FileCacheFactory.createConcurrentLRUFileCache(10_000_000, 1); + threadPool = mock(ThreadPool.class); + } + + /** + * Creates a real RemoteSegmentStoreDirectory with mocked inner directories. + * RemoteSegmentStoreDirectory is a final class and cannot be mocked. + */ + private RemoteSegmentStoreDirectory createRealRemoteSegmentStoreDirectory(ShardId shardId) throws IOException { + RemoteDirectory remoteDataDir = mock(RemoteDirectory.class); + RemoteDirectory remoteMetadataDir = mock(RemoteDirectory.class); + org.opensearch.index.store.lockmanager.RemoteStoreLockManager lockManager = mock( + org.opensearch.index.store.lockmanager.RemoteStoreLockManager.class + ); + ThreadPool tp = mock(ThreadPool.class); + + org.opensearch.common.blobstore.BlobContainer mockBlobContainer = mock(org.opensearch.common.blobstore.BlobContainer.class); + when(mockBlobContainer.path()).thenReturn(new org.opensearch.common.blobstore.BlobPath().add("test-base-path")); + when(remoteDataDir.getBlobContainer()).thenReturn(mockBlobContainer); + + return new RemoteSegmentStoreDirectory(remoteDataDir, remoteMetadataDir, lockManager, tp, shardId, new HashMap<>()); + } + + /** + * Tests that the warm-aware factory method creates the correct directory stack: + * DataFormatAwareStoreDirectory wrapping TieredSubdirectoryAwareDirectory. + */ + public void testCreatesCorrectDirectoryStack() throws IOException { + DataFormatAwareStoreDirectory result = factory.newDataFormatAwareStoreDirectory( + indexSettings, + shardId, + shardPath, + localDirectoryFactory, + Map.of(), + java.util.Map.of(), + org.opensearch.repositories.NativeStoreRepository.EMPTY, + true, + remoteDirectory, + fileCache, + threadPool + ); + + assertNotNull("Factory should return a non-null directory", result); + assertTrue("Outermost directory should be DataFormatAwareStoreDirectory", result instanceof DataFormatAwareStoreDirectory); + + // The delegate of DataFormatAwareStoreDirectory should be TieredSubdirectoryAwareDirectory + Directory delegate = ((FilterDirectory) result).getDelegate(); + assertTrue("Delegate should be TieredSubdirectoryAwareDirectory", delegate instanceof TieredSubdirectoryAwareDirectory); + + // The delegate of TieredSubdirectoryAwareDirectory should be SubdirectoryAwareDirectory + Directory innerDelegate = ((FilterDirectory) delegate).getDelegate(); + assertTrue("Inner delegate should be SubdirectoryAwareDirectory", innerDelegate instanceof SubdirectoryAwareDirectory); + + result.close(); + } + + /** + * Tests that SubdirectoryAwareDirectory appears only once in the directory chain. + * The factory should NOT double-wrap with SubdirectoryAwareDirectory. + */ + public void testNoDoubleSubdirectoryAwareDirectoryWrapping() throws IOException { + DataFormatAwareStoreDirectory result = factory.newDataFormatAwareStoreDirectory( + indexSettings, + shardId, + shardPath, + localDirectoryFactory, + Map.of(), + java.util.Map.of(), + org.opensearch.repositories.NativeStoreRepository.EMPTY, + true, + remoteDirectory, + fileCache, + threadPool + ); + + int subdirAwareCount = 0; + Directory current = result; + while (current instanceof FilterDirectory) { + if (current instanceof SubdirectoryAwareDirectory) { + subdirAwareCount++; + } + current = ((FilterDirectory) current).getDelegate(); + } + + assertEquals("SubdirectoryAwareDirectory should appear exactly once in the chain", 1, subdirAwareCount); + + result.close(); + } + + /** + * Tests that when DataFormatRegistry returns empty tiered directories, + * the factory still creates a valid directory stack with no format directories. + */ + public void testEmptyFormatDirectoriesWhenNoPluginProvides() throws IOException { + DataFormatAwareStoreDirectory result = factory.newDataFormatAwareStoreDirectory( + indexSettings, + shardId, + shardPath, + localDirectoryFactory, + Map.of(), + java.util.Map.of(), + org.opensearch.repositories.NativeStoreRepository.EMPTY, + true, + remoteDirectory, + fileCache, + threadPool + ); + + assertNotNull("Factory should return a non-null directory even with no format plugins", result); + + // Verify the stack is still correct + Directory delegate = ((FilterDirectory) result).getDelegate(); + assertTrue("Delegate should still be TieredSubdirectoryAwareDirectory", delegate instanceof TieredSubdirectoryAwareDirectory); + + result.close(); + } + + /** + * Tests that calling the 5-param (hot path) method throws UnsupportedOperationException. + */ + public void testHotPathThrowsUnsupportedOperation() { + UnsupportedOperationException exception = expectThrows( + UnsupportedOperationException.class, + () -> factory.newDataFormatAwareStoreDirectory(indexSettings, shardId, shardPath, localDirectoryFactory, Map.of()) + ); + + assertTrue("Exception message should mention warm parameters", exception.getMessage().contains("warm")); + } +} diff --git a/server/src/test/java/org/opensearch/storage/directory/TieredSubdirectoryAwareDirectoryTests.java b/server/src/test/java/org/opensearch/storage/directory/TieredSubdirectoryAwareDirectoryTests.java new file mode 100644 index 0000000000000..ff856599acf07 --- /dev/null +++ b/server/src/test/java/org/opensearch/storage/directory/TieredSubdirectoryAwareDirectoryTests.java @@ -0,0 +1,805 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.directory; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.DataFormatStoreHandler; +import org.opensearch.index.engine.dataformat.DataFormatStoreHandlerFactory; +import org.opensearch.index.engine.dataformat.StoreStrategy; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.RemoteDirectory; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.SubdirectoryAwareDirectory; +import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter; +import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.index.store.remote.filecache.FileCacheFactory; +import org.opensearch.repositories.NativeStoreRepository; +import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings; +import org.junit.Before; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; + +import static org.opensearch.storage.utils.DirectoryUtils.getFilePathSwitchable; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Functional tests for {@link TieredSubdirectoryAwareDirectory} exercising real I/O + * through the full directory stack (FSDirectory → SubdirectoryAwareDirectory → TieredDirectory). + * + *

      Format routing is verified via a real {@link StoreStrategyRegistry} built from a + * {@link StoreStrategy} whose {@link DataFormatStoreHandlerFactory} returns a Mockito-mocked + * {@link DataFormatStoreHandler} — the mock verifies {@code onUploaded} / {@code onRemoved} / + * {@code close} calls. Lucene files skip the strategy lookup entirely. + */ +@ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class) +public class TieredSubdirectoryAwareDirectoryTests extends TieredStorageBaseTestCase { + + private FileCache fileCache; + private ShardPath shardPath; + private FSDirectory localFsDir; + private SubdirectoryAwareDirectory subdirAware; + private TieredSubdirectoryAwareDirectory directory; + + private static final byte[] TEST_DATA = "hello-tiered".getBytes(StandardCharsets.UTF_8); + private static final byte[] PARQUET_DATA = "parquet-payload".getBytes(StandardCharsets.UTF_8); + private static final DataFormat PARQUET_FORMAT = new DataFormat() { + @Override + public String name() { + return "parquet"; + } + + @Override + public long priority() { + return 2; + } + + @Override + public java.util.Set supportedFields() { + return java.util.Set.of(); + } + }; + + @Before + public void setup() throws IOException { + setupRemoteSegmentStoreDirectory(); + + // Stub getBlobContainer().path() so getRemoteBasePath() doesn't NPE in afterSyncToRemote tests + BlobContainer mockBlobContainer = mock(BlobContainer.class); + when(mockBlobContainer.path()).thenReturn(new BlobPath().add("test-base-path")); + when(((RemoteDirectory) remoteDataDirectory).getBlobContainer()).thenReturn(mockBlobContainer); + + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Path tempDir = createTempDir(); + Index index = new Index("test-index", "test-uuid"); + ShardId shardId = new ShardId(index, 0); + Path shardDataPath = tempDir.resolve("data").resolve("test-uuid").resolve("0"); + Path shardStatePath = tempDir.resolve("state").resolve("test-uuid").resolve("0"); + Files.createDirectories(shardDataPath.resolve("index")); + Files.createDirectories(shardStatePath); + shardPath = new ShardPath(false, shardDataPath, shardStatePath, shardId); + + localFsDir = FSDirectory.open(shardPath.resolveIndex()); + subdirAware = new SubdirectoryAwareDirectory(localFsDir, shardPath); + fileCache = FileCacheFactory.createConcurrentLRUFileCache(FILE_CACHE_CAPACITY, 1); + } + + private Supplier getMockPrefetchSettingsSupplier() { + return () -> { + TieredStoragePrefetchSettings settings = mock(TieredStoragePrefetchSettings.class); + when(settings.getReadAheadBlockCount()).thenReturn(TieredStoragePrefetchSettings.DEFAULT_READ_AHEAD_BLOCK_COUNT); + when(settings.getReadAheadEnableFileFormats()).thenReturn(TieredStoragePrefetchSettings.READ_AHEAD_ENABLE_FILE_FORMATS); + when(settings.isStoredFieldsPrefetchEnabled()).thenReturn(true); + return settings; + }; + } + + /** + * Builds a TieredSubdirectoryAwareDirectory with no strategies (Lucene-only). + */ + private TieredSubdirectoryAwareDirectory buildDirectoryNoFormats() { + return new TieredSubdirectoryAwareDirectory( + subdirAware, + remoteSegmentStoreDirectory, + fileCache, + threadPool, + StoreStrategyRegistry.EMPTY, + shardPath, + getMockPrefetchSettingsSupplier() + ); + } + + /** + * Builds a TieredSubdirectoryAwareDirectory with a parquet strategy whose native + * file registry is a mock. Returns both the directory and the mock so tests can + * verify calls routed to the registry. + */ + private WithRegistry buildDirectoryWithParquetFormat() { + return buildDirectoryWithParquetFormat(mock(DataFormatStoreHandler.class)); + } + + private WithRegistry buildDirectoryWithParquetFormat(DataFormatStoreHandler nativeRegistry) { + DataFormatStoreHandlerFactory factory = (sid, warm, repo) -> nativeRegistry; + StoreStrategy parquet = new TestParquetStrategy(factory); + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, parquet), + remoteSegmentStoreDirectory + ); + TieredSubdirectoryAwareDirectory dir = new TieredSubdirectoryAwareDirectory( + subdirAware, + remoteSegmentStoreDirectory, + fileCache, + threadPool, + registry, + shardPath, + getMockPrefetchSettingsSupplier() + ); + return new WithRegistry(dir, nativeRegistry); + } + + /** Writes a parquet file directly to disk (simulating the Rust writer). */ + private void writeParquetFileToDisk(String relativePath) throws IOException { + Path fullPath = shardPath.getDataPath().resolve(relativePath); + Files.createDirectories(fullPath.getParent()); + Files.write(fullPath, PARQUET_DATA); + } + + /** + * Directly adds a parquet file entry to the remote metadata map. + * Parquet files don't have Lucene codec footers, so we can't use copyFrom. + * In production, the upload path adds entries via a separate mechanism. + */ + @SuppressWarnings("unchecked") + @SuppressForbidden(reason = "test needs reflection to inject parquet metadata without full upload pipeline") + private void addParquetMetadataEntry(String localFilename, String uploadedFilename) { + try { + java.lang.reflect.Field field = RemoteSegmentStoreDirectory.class.getDeclaredField("segmentsUploadedToRemoteStore"); + field.setAccessible(true); + java.util.concurrent.ConcurrentHashMap map = + (java.util.concurrent.ConcurrentHashMap) field.get( + remoteSegmentStoreDirectory + ); + RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = RemoteSegmentStoreDirectory.UploadedSegmentMetadata.fromString( + localFilename + "::" + uploadedFilename + "::checksum123::100::" + org.apache.lucene.util.Version.LATEST.major + ); + map.put(localFilename, metadata); + } catch (Exception e) { + throw new RuntimeException("Failed to add parquet metadata entry", e); + } + } + + // ═══════════════════════════════════════════════════════════════ + // Routing tests — openInput + // ═══════════════════════════════════════════════════════════════ + + public void testOpenInputLuceneFileRoutesToTieredDirectory() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + populateData(); + try { + String luceneFile = "_0_test.cfe"; + try (IndexOutput out = directory.createOutput(luceneFile, IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + + Path switchablePath = getFilePathSwitchable(localFsDir, luceneFile); + assertNotNull("Lucene file should be in FileCache after createOutput", fileCache.get(switchablePath)); + fileCache.decRef(switchablePath); + + try (IndexInput in = directory.openInput(luceneFile, IOContext.DEFAULT)) { + assertNotNull("openInput should return non-null for Lucene file", in); + byte[] buf = new byte[TEST_DATA.length]; + in.readBytes(buf, 0, buf.length); + assertArrayEquals("Data read back should match data written", TEST_DATA, buf); + } + } finally { + directory.close(); + } + } + + public void testOpenInputFormatFileRoutesToRemoteDirectory() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + try { + // On read-only warm, openInput for format files goes to remoteDirectory. + // Our mock remote has no parquet files, so this throws. + expectThrows(Exception.class, () -> directory.openInput("parquet/seg.parquet", IOContext.DEFAULT)); + } finally { + directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // Routing tests — fileLength + // ═══════════════════════════════════════════════════════════════ + + public void testFileLengthLuceneFile() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + populateData(); + try { + String luceneFile = "_0_len.cfe"; + try (IndexOutput out = directory.createOutput(luceneFile, IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + long length = directory.fileLength(luceneFile); + assertEquals("fileLength should match written data length", TEST_DATA.length, length); + } finally { + directory.close(); + } + } + + public void testFileLengthFormatFileRoutesToRemote() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + try { + expectThrows(Exception.class, () -> directory.fileLength("parquet/seg_len.parquet")); + } finally { + directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // listAll tests + // ═══════════════════════════════════════════════════════════════ + + public void testListAllReturnsLuceneAndFormatFiles() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + populateData(); + try { + try (IndexOutput out = directory.createOutput("_0_list.cfe", IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + writeParquetFileToDisk("parquet/seg_list.parquet"); + + String[] files = directory.listAll(); + Set fileSet = new HashSet<>(Arrays.asList(files)); + assertTrue("listAll should contain Lucene file", fileSet.contains("_0_list.cfe")); + assertTrue("listAll should contain parquet file", fileSet.contains("parquet/seg_list.parquet")); + } finally { + directory.close(); + } + } + + public void testListAllWithEmptyFormatDirectories() throws IOException { + directory = buildDirectoryNoFormats(); + populateData(); + try { + try (IndexOutput out = directory.createOutput("_0_only.cfe", IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + + String[] files = directory.listAll(); + Set fileSet = new HashSet<>(Arrays.asList(files)); + assertTrue("listAll should contain Lucene file", fileSet.contains("_0_only.cfe")); + + for (String f : files) { + assertFalse("No parquet files should appear without format dirs", f.startsWith("parquet/")); + } + } finally { + directory.close(); + } + } + + public void testListAllSortedAndDeduplicates() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + populateData(); + try { + try (IndexOutput out = directory.createOutput("_0_dup_a.cfe", IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + try (IndexOutput out = directory.createOutput("_0_dup_b.cfe", IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + + String[] files = directory.listAll(); + for (int i = 1; i < files.length; i++) { + assertTrue("listAll should return sorted results", files[i - 1].compareTo(files[i]) <= 0); + } + Set fileSet = new HashSet<>(Arrays.asList(files)); + assertEquals("listAll should have no duplicates", fileSet.size(), files.length); + } finally { + directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // deleteFile tests + // ═══════════════════════════════════════════════════════════════ + + public void testDeleteFileLuceneRoutesToTieredDirectory() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + populateData(); + try { + String luceneFile = "_0_del.cfe"; + try (IndexOutput out = directory.createOutput(luceneFile, IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + + Set beforeDelete = new HashSet<>(Arrays.asList(directory.listAll())); + assertTrue("File should exist before delete", beforeDelete.contains(luceneFile)); + + directory.deleteFile(luceneFile); + + Set afterDelete = new HashSet<>(Arrays.asList(directory.listAll())); + assertFalse("File should be gone after delete", afterDelete.contains(luceneFile)); + } finally { + directory.close(); + } + } + + public void testDeleteFileFormatRoutesToNativeRegistry() throws IOException { + WithRegistry w = buildDirectoryWithParquetFormat(); + try { + w.directory.deleteFile("parquet/seg_del.parquet"); + String expectedDelKey = shardPath.getDataPath().resolve("parquet/seg_del.parquet").toString(); + verify(w.storeHandler).onRemoved(expectedDelKey); + } finally { + w.directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // afterSyncToRemote tests + // ═══════════════════════════════════════════════════════════════ + + public void testAfterSyncToRemoteLuceneFile() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + populateData(); + try { + String luceneFile = "_0_sync.cfe"; + try (IndexOutput out = directory.createOutput(luceneFile, IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + + Path switchablePath = getFilePathSwitchable(localFsDir, luceneFile); + assertNotNull("File should be in FileCache before afterSyncToRemote", fileCache.get(switchablePath)); + fileCache.decRef(switchablePath); + + directory.afterSyncToRemote(luceneFile); + + Integer refCount = fileCache.getRef(switchablePath); + assertTrue("Ref count should be 0 or null after afterSyncToRemote", refCount == null || refCount == 0); + } finally { + directory.close(); + } + } + + public void testAfterSyncToRemoteFormatFileRoutesToNativeRegistry() throws IOException { + WithRegistry w = buildDirectoryWithParquetFormat(); + String parquetFile = "parquet/seg_sync.parquet"; + addParquetMetadataEntry(parquetFile, "seg_sync.parquet__UUID1"); + w.directory.afterSyncToRemote(parquetFile); + String expectedUploadKey = shardPath.getDataPath().resolve(parquetFile).toString(); + verify(w.storeHandler).onUploaded( + org.mockito.ArgumentMatchers.eq(expectedUploadKey), + org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.anyLong() + ); + } + + public void testAfterSyncToRemoteFormatFileWithoutRemoteSyncAware() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + try { + String parquetFile = "parquet/seg_nosync.parquet"; + addParquetMetadataEntry(parquetFile, "seg_nosync.parquet__UUID2"); + directory.afterSyncToRemote(parquetFile); + } finally { + directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // createOutput tests + // ═══════════════════════════════════════════════════════════════ + + public void testCreateOutputLuceneFile() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + populateData(); + try { + String luceneFile = "_0_create.cfe"; + try (IndexOutput out = directory.createOutput(luceneFile, IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + + Path switchablePath = getFilePathSwitchable(localFsDir, luceneFile); + assertNotNull("Lucene file should be cached in FileCache after createOutput", fileCache.get(switchablePath)); + fileCache.decRef(switchablePath); + + assertTrue("Lucene file should exist on local disk", Arrays.asList(localFsDir.listAll()).contains(luceneFile)); + } finally { + directory.close(); + } + } + + public void testFormatFileWrittenToDiskNotAccessibleViaRemote() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + try { + String parquetFile = "parquet/seg_create.parquet"; + writeParquetFileToDisk(parquetFile); + // File exists locally but not in remote metadata — should be readable from local. + // This is the translog bump edge case: file created locally, not yet synced. + long len = directory.fileLength(parquetFile); + assertTrue("Local format file should have non-zero length", len > 0); + } finally { + directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // Edge case tests + // ═══════════════════════════════════════════════════════════════ + + public void testOpenInputNonExistentFile() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + populateData(); + try { + expectThrows(NoSuchFileException.class, () -> directory.openInput("non_existent_file.cfe", IOContext.DEFAULT)); + } finally { + directory.close(); + } + } + + public void testFileLengthNonExistentFile() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + populateData(); + try { + expectThrows(Exception.class, () -> directory.fileLength("non_existent_file.cfe")); + } finally { + directory.close(); + } + } + + public void testCloseClosesNativeRegistryAndTieredDirectory() throws IOException { + WithRegistry w = buildDirectoryWithParquetFormat(); + w.directory.close(); + verify(w.storeHandler).close(); + } + + public void testCloseDoesNotDoubleCloseSharedSubdirectoryAwareDirectory() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + populateData(); + try { + try (IndexOutput out = directory.createOutput("_0_noclose.cfe", IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + } finally { + directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // Constructor resource leak safety + // ═══════════════════════════════════════════════════════════════ + + public void testConstructorFailureClosesStrategyRegistry() throws IOException { + DataFormatStoreHandler nativeRegistry = mock(DataFormatStoreHandler.class); + DataFormatStoreHandlerFactory factory = (sid, warm, repo) -> nativeRegistry; + StoreStrategy parquet = new TestParquetStrategy(factory); + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, parquet), + remoteSegmentStoreDirectory + ); + + try { + new TieredSubdirectoryAwareDirectory( + subdirAware, + remoteSegmentStoreDirectory, + null, // null fileCache → triggers IllegalStateException in CompositeDirectory + threadPool, + registry, + shardPath, + getMockPrefetchSettingsSupplier() + ); + fail("Expected IllegalStateException from null fileCache"); + } catch (IllegalStateException e) { + // Expected + } + + // The registry (and its native registries) must have been closed by the constructor's + // failure path so no native resources leak. + verify(nativeRegistry).close(); + } + + // ═══════════════════════════════════════════════════════════════ + // IOUtils.close — partial close safety + // ═══════════════════════════════════════════════════════════════ + + public void testCloseWithThrowingNativeRegistryStillClosesTieredDirectory() throws IOException { + DataFormatStoreHandler throwingRegistry = mock(DataFormatStoreHandler.class); + org.mockito.Mockito.doThrow(new IOException("native close failed")).when(throwingRegistry).close(); + + WithRegistry w = buildDirectoryWithParquetFormat(throwingRegistry); + + IOException ex = expectThrows(IOException.class, w.directory::close); + assertEquals("native close failed", ex.getMessage()); + verify(throwingRegistry).close(); + } + + public void testAfterSyncToRemoteFormatFileNoopWhenNotRemoteSyncAware() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + try { + String parquetFile = "parquet/seg_noop.parquet"; + addParquetMetadataEntry(parquetFile, "seg_noop.parquet__UUID3"); + // Delegates to the native registry — must NOT fall through to tieredDirectory. + directory.afterSyncToRemote(parquetFile); + } finally { + directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // IllegalStateException guard tests (no matching strategy) + // ═══════════════════════════════════════════════════════════════ + + public void testOpenInputUnregisteredFormatThrowsIllegalState() throws IOException { + directory = buildDirectoryNoFormats(); + populateData(); + try { + IllegalStateException ex = expectThrows( + IllegalStateException.class, + () -> directory.openInput("csv/data.csv", IOContext.DEFAULT) + ); + assertTrue(ex.getMessage().contains("csv")); + assertTrue(ex.getMessage().contains("No StoreStrategy")); + } finally { + directory.close(); + } + } + + public void testFileLengthUnregisteredFormatThrowsIllegalState() throws IOException { + directory = buildDirectoryNoFormats(); + populateData(); + try { + IllegalStateException ex = expectThrows(IllegalStateException.class, () -> directory.fileLength("csv/data.csv")); + assertTrue(ex.getMessage().contains("csv")); + } finally { + directory.close(); + } + } + + public void testDeleteFileUnregisteredFormatThrowsIllegalState() throws IOException { + directory = buildDirectoryNoFormats(); + populateData(); + try { + IllegalStateException ex = expectThrows(IllegalStateException.class, () -> directory.deleteFile("csv/data.csv")); + assertTrue(ex.getMessage().contains("csv")); + } finally { + directory.close(); + } + } + + public void testAfterSyncToRemoteUnregisteredFormatThrowsIllegalState() throws IOException { + directory = buildDirectoryNoFormats(); + populateData(); + try { + IllegalStateException ex = expectThrows(IllegalStateException.class, () -> directory.afterSyncToRemote("csv/data.csv")); + assertTrue(ex.getMessage().contains("csv")); + } finally { + directory.close(); + } + } + + public void testLuceneFileWithNoStrategyRoutesToTieredDirectory() throws IOException { + directory = buildDirectoryNoFormats(); + populateData(); + try { + String luceneFile = "_0_guard.cfe"; + try (IndexOutput out = directory.createOutput(luceneFile, IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + long length = directory.fileLength(luceneFile); + assertEquals(TEST_DATA.length, length); + } finally { + directory.close(); + } + } + + /** Minimal test strategy for "parquet" wiring. */ + private static final class TestParquetStrategy implements StoreStrategy { + private final DataFormatStoreHandlerFactory factory; + + TestParquetStrategy(DataFormatStoreHandlerFactory factory) { + this.factory = factory; + } + + @Override + public Optional storeHandler() { + return Optional.of(factory); + } + } + + private static final class WithRegistry { + final TieredSubdirectoryAwareDirectory directory; + final DataFormatStoreHandler storeHandler; + + WithRegistry(TieredSubdirectoryAwareDirectory directory, DataFormatStoreHandler storeHandler) { + this.directory = directory; + this.storeHandler = storeHandler; + } + } + + // ═══════════════════════════════════════════════════════════════ + // sync() tests + // ═══════════════════════════════════════════════════════════════ + + public void testSyncIsNoOp() throws IOException { + directory = buildDirectoryNoFormats(); + try { + // sync should not throw even with non-existent files — it's a no-op on warm + directory.sync(java.util.List.of("_0.cfe", "parquet/seg_0.parquet", "nonexistent.file")); + } finally { + directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // rename() tests + // ═══════════════════════════════════════════════════════════════ + + public void testRenameLuceneFileDelegatesToTieredDirectory() throws IOException { + directory = buildDirectoryNoFormats(); + try { + // Write a file, then rename it (simulates Lucene commit: pending_segments → segments) + try (IndexOutput out = directory.createOutput("pending_segments_1", IOContext.DEFAULT)) { + out.writeBytes(TEST_DATA, TEST_DATA.length); + } + directory.rename("pending_segments_1", "segments_1"); + // Original gone, new name exists + assertTrue(Arrays.asList(directory.listAll()).contains("segments_1")); + } finally { + directory.close(); + } + } + + public void testRenameFormatFileThrowsIllegalState() throws IOException { + WithRegistry w = buildDirectoryWithParquetFormat(); + try { + IllegalStateException ex = expectThrows( + IllegalStateException.class, + () -> w.directory.rename("parquet/seg_0.parquet", "parquet/seg_1.parquet") + ); + assertTrue(ex.getMessage().contains("parquet/seg_0.parquet")); + assertTrue(ex.getMessage().contains("write-once")); + } finally { + w.directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // listAll() tests + // ═══════════════════════════════════════════════════════════════ + + public void testListAllIncludesLuceneFiles() throws IOException { + directory = buildDirectoryNoFormats(); + populateData(); + try { + String[] files = directory.listAll(); + // Should contain Lucene files from remote metadata (populated in setup) + assertTrue("Should contain _0.si", Arrays.asList(files).contains("_0.si")); + } finally { + directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // afterSyncToRemote() — null blobKey test + // ═══════════════════════════════════════════════════════════════ + + public void testAfterSyncToRemoteThrowsWhenBlobKeyNull() throws IOException { + WithRegistry w = buildDirectoryWithParquetFormat(); + try { + // "parquet/unknown.parquet" is a format file but has no remote metadata entry + // → getExistingRemoteFilename returns null → should throw + IllegalStateException ex = expectThrows( + IllegalStateException.class, + () -> w.directory.afterSyncToRemote("parquet/unknown.parquet") + ); + assertTrue(ex.getMessage().contains("parquet/unknown.parquet")); + assertTrue(ex.getMessage().contains("no remote filename")); + } finally { + w.directory.close(); + } + } + + // ═══════════════════════════════════════════════════════════════ + // Local-to-remote routing and afterSyncToRemote local delete tests + // ═══════════════════════════════════════════════════════════════ + + public void testOpenInputRoutesToLocalWhenNotInRemoteMetadata() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + try { + String parquetFile = "parquet/seg_local_only.parquet"; + writeParquetFileToDisk(parquetFile); + // File exists locally but NOT in remote metadata → should read from local + IndexInput input = directory.openInput(parquetFile, IOContext.DEFAULT); + assertNotNull(input); + assertTrue("Local format file should have non-zero length", input.length() > 0); + input.close(); + } finally { + directory.close(); + } + } + + public void testOpenInputRoutesToRemoteWhenInRemoteMetadata() throws IOException { + directory = buildDirectoryWithParquetFormat().directory; + populateData(); + try { + String parquetFile = "parquet/seg_remote.parquet"; + addParquetMetadataEntry(parquetFile, "seg_remote.parquet__UUID1"); + // File is in remote metadata → should route to remote directory + // (remote directory is mocked, so this verifies routing not actual read) + IndexInput input = directory.openInput(parquetFile, IOContext.DEFAULT); + assertNotNull(input); + input.close(); + } finally { + directory.close(); + } + } + + public void testAfterSyncToRemoteDeletesLocalCopy() throws IOException { + WithRegistry w = buildDirectoryWithParquetFormat(); + try { + String parquetFile = "parquet/seg_delete_local.parquet"; + writeParquetFileToDisk(parquetFile); + // Verify file exists locally + assertTrue(java.nio.file.Files.exists(shardPath.getDataPath().resolve(parquetFile))); + // Simulate sync: add remote metadata entry + addParquetMetadataEntry(parquetFile, "seg_delete_local.parquet__UUID1"); + // afterSyncToRemote should register as REMOTE and delete local copy + w.directory.afterSyncToRemote(parquetFile); + // Local file should be gone + assertFalse( + "Local file should be deleted after sync to remote", + java.nio.file.Files.exists(shardPath.getDataPath().resolve(parquetFile)) + ); + } finally { + w.directory.close(); + } + } + + public void testAfterSyncToRemoteNoErrorWhenLocalAlreadyGone() throws IOException { + WithRegistry w = buildDirectoryWithParquetFormat(); + try { + String parquetFile = "parquet/seg_already_gone.parquet"; + // Don't write file to disk — it's already gone + addParquetMetadataEntry(parquetFile, "seg_already_gone.parquet__UUID1"); + // Should not throw — catches NoSuchFileException silently + w.directory.afterSyncToRemote(parquetFile); + } finally { + w.directory.close(); + } + } +} diff --git a/server/src/test/java/org/opensearch/storage/directory/WarmShardDirectoryStackTests.java b/server/src/test/java/org/opensearch/storage/directory/WarmShardDirectoryStackTests.java new file mode 100644 index 0000000000000..ce91203e0707e --- /dev/null +++ b/server/src/test/java/org/opensearch/storage/directory/WarmShardDirectoryStackTests.java @@ -0,0 +1,227 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.storage.directory; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FilterDirectory; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.DataFormatStoreHandler; +import org.opensearch.index.engine.dataformat.DataFormatStoreHandlerFactory; +import org.opensearch.index.engine.dataformat.StoreStrategy; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.DataFormatAwareStoreDirectory; +import org.opensearch.index.store.RemoteDirectory; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; +import org.opensearch.index.store.SubdirectoryAwareDirectory; +import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; +import org.opensearch.index.store.remote.filecache.FileCache; +import org.opensearch.index.store.remote.filecache.FileCacheFactory; +import org.opensearch.plugins.IndexStorePlugin; +import org.opensearch.repositories.NativeStoreRepository; +import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Integration-level tests for the warm shard directory stack. + * + *

      Verifies that the full directory stack (FSDirectory → SubdirectoryAwareDirectory + * → TieredSubdirectoryAwareDirectory → DataFormatAwareStoreDirectory) is wired + * correctly via {@link TieredDataFormatAwareStoreDirectoryFactory} and that file + * operations flow through the correct layers. + */ +public class WarmShardDirectoryStackTests extends OpenSearchTestCase { + + private static final DataFormat PARQUET_FORMAT = new DataFormat() { + @Override + public String name() { + return "parquet"; + } + + @Override + public long priority() { + return 2; + } + + @Override + public java.util.Set supportedFields() { + return java.util.Set.of(); + } + }; + + private Path tempDir; + private ShardPath shardPath; + private IndexSettings indexSettings; + private FileCache fileCache; + + @Override + public void setUp() throws Exception { + super.setUp(); + tempDir = createTempDir(); + Index index = new Index("test-warm-index", "test-uuid"); + ShardId shardId = new ShardId(index, 0); + + Path shardStatePath = tempDir.resolve("state").resolve("test-uuid").resolve("0"); + Path shardDataPath = tempDir.resolve("data").resolve("test-uuid").resolve("0"); + Path indexPath = shardDataPath.resolve("index"); + Files.createDirectories(shardStatePath); + Files.createDirectories(shardDataPath); + Files.createDirectories(indexPath); + + shardPath = new ShardPath(false, shardDataPath, shardStatePath, shardId); + + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + IndexMetadata indexMetadata = IndexMetadata.builder("test-warm-index").settings(settings).build(); + indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + + fileCache = FileCacheFactory.createConcurrentLRUFileCache(10_000_000, 1); + } + + private Supplier getMockPrefetchSettingsSupplier() { + return () -> { + TieredStoragePrefetchSettings settings = mock(TieredStoragePrefetchSettings.class); + when(settings.getReadAheadBlockCount()).thenReturn(TieredStoragePrefetchSettings.DEFAULT_READ_AHEAD_BLOCK_COUNT); + when(settings.getReadAheadEnableFileFormats()).thenReturn(TieredStoragePrefetchSettings.READ_AHEAD_ENABLE_FILE_FORMATS); + when(settings.isStoredFieldsPrefetchEnabled()).thenReturn(true); + return settings; + }; + } + + /** + * Exercises the factory end-to-end with no store strategies — verifies the stack + * nests FSDirectory → SubdirectoryAwareDirectory → TieredSubdirectoryAwareDirectory + * → DataFormatAwareStoreDirectory. + */ + @LockFeatureFlag(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG) + public void testWarmDirectoryStackCreationAndWrite() throws IOException { + TieredDataFormatAwareStoreDirectoryFactory factory = new TieredDataFormatAwareStoreDirectoryFactory( + getMockPrefetchSettingsSupplier() + ); + + FSDirectory fsDir = FSDirectory.open(shardPath.resolveIndex()); + IndexStorePlugin.DirectoryFactory localDirFactory = mock(IndexStorePlugin.DirectoryFactory.class); + when(localDirFactory.newDirectory(any(), any())).thenReturn(fsDir); + + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(shardPath.getShardId()); + + DataFormatAwareStoreDirectory storeDir = factory.newDataFormatAwareStoreDirectory( + indexSettings, + shardPath.getShardId(), + shardPath, + localDirFactory, + java.util.Map.of(), + java.util.Map.of(), // no strategies + NativeStoreRepository.EMPTY, + true, + remoteDir, + fileCache, + null + ); + + assertNotNull("Directory stack should be created", storeDir); + + Directory delegate = ((FilterDirectory) storeDir).getDelegate(); + assertTrue("Should have TieredSubdirectoryAwareDirectory", delegate instanceof TieredSubdirectoryAwareDirectory); + + Directory innerDelegate = ((FilterDirectory) delegate).getDelegate(); + assertTrue("Should have SubdirectoryAwareDirectory", innerDelegate instanceof SubdirectoryAwareDirectory); + + storeDir.close(); + } + + /** + * Exercises the stack with a parquet strategy. File ops on {@code parquet/…} route + * to the remote store; the mock remote has no parquet metadata so {@code fileLength} + * throws. {@code listAll} reflects whatever is on disk (format files included). + */ + @LockFeatureFlag(FeatureFlags.WRITABLE_WARM_INDEX_EXPERIMENTAL_FLAG) + public void testWarmDirectoryStackWithFormatStrategy() throws IOException { + FSDirectory localFsDir = FSDirectory.open(shardPath.resolveIndex()); + SubdirectoryAwareDirectory subdirAware = new SubdirectoryAwareDirectory(localFsDir, shardPath); + + RemoteSegmentStoreDirectory remoteDir = createRealRemoteDir(shardPath.getShardId()); + + DataFormatStoreHandler nativeRegistry = mock(DataFormatStoreHandler.class); + DataFormatStoreHandlerFactory factory = (sid, isWarm, repo) -> nativeRegistry; + StoreStrategy parquet = new StoreStrategy() { + @Override + public Optional storeHandler() { + return Optional.of(factory); + } + }; + + StoreStrategyRegistry registry = StoreStrategyRegistry.open( + shardPath, + true, + NativeStoreRepository.EMPTY, + Map.of(PARQUET_FORMAT, parquet), + remoteDir + ); + + TieredSubdirectoryAwareDirectory tieredSubdir = new TieredSubdirectoryAwareDirectory( + subdirAware, + remoteDir, + fileCache, + null, + registry, + shardPath, + getMockPrefetchSettingsSupplier() + ); + + expectThrows(Exception.class, () -> tieredSubdir.fileLength("parquet/seg.parquet")); + + String[] allFiles = tieredSubdir.listAll(); + Set fileSet = new HashSet<>(Arrays.asList(allFiles)); + assertFalse("listAll should not surface an unwritten parquet file", fileSet.contains("parquet/seg.parquet")); + + tieredSubdir.close(); + } + + private RemoteSegmentStoreDirectory createRealRemoteDir(ShardId shardId) throws IOException { + RemoteDirectory remoteDataDir = mock(RemoteDirectory.class); + RemoteDirectory remoteMetadataDir = mock(RemoteDirectory.class); + RemoteStoreLockManager lockManager = mock(RemoteStoreLockManager.class); + ThreadPool tp = mock(ThreadPool.class); + + BlobContainer mockBlobContainer = mock(BlobContainer.class); + when(mockBlobContainer.path()).thenReturn(new BlobPath().add("test-base-path")); + when(remoteDataDir.getBlobContainer()).thenReturn(mockBlobContainer); + + return new RemoteSegmentStoreDirectory(remoteDataDir, remoteMetadataDir, lockManager, tp, shardId, new HashMap<>()); + } +} diff --git a/server/src/test/java/org/opensearch/storage/utils/DirectoryUtilsTests.java b/server/src/test/java/org/opensearch/storage/utils/DirectoryUtilsTests.java index 8364b2994c065..094879d4fa654 100644 --- a/server/src/test/java/org/opensearch/storage/utils/DirectoryUtilsTests.java +++ b/server/src/test/java/org/opensearch/storage/utils/DirectoryUtilsTests.java @@ -74,6 +74,18 @@ public void testUnwrapFSDirectoryThrowsWhenWrappedNonFSDirectory() { assertTrue(ex.getMessage().contains("Expected FSDirectory but got")); } + public void testUnwrapFSDirectoryAlias() { + FSDirectory result = DirectoryUtils.unwrapFSDirectory(fsDirectory); + assertSame(fsDirectory, result); + } + + public void testUnwrapFSDirectoryWrapped() { + FilterDirectory wrapped = new FilterDirectory(fsDirectory) { + }; + FSDirectory result = DirectoryUtils.unwrapFSDirectory(wrapped); + assertSame(fsDirectory, result); + } + public void testGetFilePath() { Path result = DirectoryUtils.getFilePath(fsDirectory, "test_file.si"); assertEquals(tempDir.resolve("test_file.si"), result); From df6dd65ad758c88161fadb2f5e0f0d8edecba309 Mon Sep 17 00:00:00 2001 From: Ayush Sharma <118544643+shayush622@users.noreply.github.com> Date: Thu, 7 May 2026 23:24:45 +0530 Subject: [PATCH 072/115] refactor(hunspell): rename package terminology to generic ref_path (#21516) Rename HunspellService and HunspellTokenFilterFactory APIs from package-specific naming to generic ref_path terminology: - getDictionaryFromPackage -> getDictionaryFromRefPath - loadDictionaryFromPackage -> loadDictionaryFromRefPath - buildPackageCacheKey -> buildRefPathCacheKey - Split validatePackageIdentifier into validateRefPath (allows /) and validateLocale (allows dots) - Extract CACHE_KEY_SEPARATOR constant - Update all javadoc, comments, and test names No behavioral change for existing functionality. Signed-off-by: shayush622 --- .../analysis/HunspellTokenFilterFactory.java | 84 ++++++--- .../indices/analysis/HunspellService.java | 129 ++++++------- .../HunspellTokenFilterFactoryTests.java | 170 ++++++++++++------ .../indices/analyze/HunspellServiceTests.java | 139 +++++++------- .../hunspell/en_US/en_US.aff | 0 .../hunspell/en_US/en_US.dic | 0 6 files changed, 299 insertions(+), 223 deletions(-) rename server/src/test/resources/indices/analyze/conf_dir/analyzers/{test-pkg => test-dict}/hunspell/en_US/en_US.aff (100%) rename server/src/test/resources/indices/analyze/conf_dir/analyzers/{test-pkg => test-dict}/hunspell/en_US/en_US.dic (100%) diff --git a/server/src/main/java/org/opensearch/index/analysis/HunspellTokenFilterFactory.java b/server/src/main/java/org/opensearch/index/analysis/HunspellTokenFilterFactory.java index 3834c99886a86..977ca9b424116 100644 --- a/server/src/main/java/org/opensearch/index/analysis/HunspellTokenFilterFactory.java +++ b/server/src/main/java/org/opensearch/index/analysis/HunspellTokenFilterFactory.java @@ -46,7 +46,7 @@ * * The dictionary is loaded from either: *

        - *
      • A ref_path (package ID, e.g., "pkg-1234") combined with locale for package-based dictionaries
        • + *
      • A ref_path (ref_path, e.g., "analyzers/my-dict") combined with locale for directory-based dictionaries
        • *
      • A locale (e.g., "en_US") for traditional hunspell dictionaries from config/hunspell/
        • *
      * @@ -58,10 +58,10 @@ * "locale": "en_US" * } * - * // Package-based (loads from config/analyzers/pkg-1234/hunspell/en_US/) + * // Directory-based (loads from config/analyzers/my-dict/hunspell/en_US/) * { * "type": "hunspell", - * "ref_path": "pkg-1234", + * "ref_path": "analyzers/my-dict", * "locale": "en_US" * } * @@ -79,26 +79,26 @@ public HunspellTokenFilterFactory(IndexSettings indexSettings, String name, Sett super(indexSettings, name, settings); // Get both ref_path and locale parameters - String refPath = settings.get("ref_path"); // Package ID only (optional) + String refPath = settings.get("ref_path"); String locale = settings.get("locale", settings.get("language", settings.get("lang", null))); if (refPath != null) { - // Package-based loading: ref_path (package ID) + locale (required) + // Directory-based loading: ref_path + locale (required) if (locale == null) { throw new IllegalArgumentException("When using ref_path, the 'locale' parameter is required for hunspell token filter"); } - // Validate ref_path and locale are safe package/locale identifiers - validatePackageIdentifier(refPath, "ref_path"); - validatePackageIdentifier(locale, "locale"); + // Validate ref_path and locale + validateRefPath(refPath); + validateLocale(locale); - // Load from package directory: config/analyzers/{ref_path}/hunspell/{locale}/ - dictionary = hunspellService.getDictionaryFromPackage(refPath, locale); + // Load from directory: config/{ref_path}/hunspell/{locale}/ + dictionary = hunspellService.getDictionaryFromRefPath(refPath, locale); } else if (locale != null) { // Traditional locale-based loading (backward compatible) // Loads from config/hunspell/{locale}/ // Validate locale to prevent path traversal and cache key ambiguity - validatePackageIdentifier(locale, "locale"); + validateLocale(locale); dictionary = hunspellService.getDictionary(locale); } else { throw new IllegalArgumentException( @@ -124,37 +124,63 @@ public boolean longestOnly() { } /** - * Allowlist pattern for safe package identifiers and locales. - * Permits only alphanumeric characters, hyphens, and underscores. - * Examples: "pkg-1234", "en_US", "my-package-v2", "en_US_custom" + * Allowlist pattern for a ref_path. + * Permits alphanumeric characters, hyphens, underscores, and forward slashes as path separators. + * A ref_path is a relative directory path under config/, e.g. "analyzers/my-dict". */ - private static final Pattern SAFE_IDENTIFIER_PATTERN = Pattern.compile("^[a-zA-Z0-9][a-zA-Z0-9_-]*$|^[a-zA-Z0-9]$"); + private static final Pattern SAFE_REF_PATH_PATTERN = Pattern.compile("^[a-zA-Z0-9][a-zA-Z0-9_/-]*[a-zA-Z0-9]$|^[a-zA-Z0-9]$"); /** - * Validates that a package identifier or locale contains only safe characters. - * Uses an allowlist approach: only alphanumeric characters, hyphens, and underscores are permitted. - * This prevents path traversal, cache key injection, and other security issues. + * Allowlist pattern for a locale. + * Permits alphanumeric characters, hyphens, and underscores. + * Disallows forward slashes and dots — a locale is a single directory-name segment, e.g. "en_US" or "en_US_custom". + */ + private static final Pattern SAFE_LOCALE_PATTERN = Pattern.compile("^[a-zA-Z0-9][a-zA-Z0-9_-]*[a-zA-Z0-9]$|^[a-zA-Z0-9]$"); + + /** + * Validates a ref_path value. Allows "/" as a path separator so that callers can pass nested + * directory paths (e.g. "analyzers/my-dict"). Uses an allowlist to prevent path traversal, + * cache key injection, and other security issues. + * + * @param value the ref_path to validate + * @throws IllegalArgumentException if validation fails + */ + static void validateRefPath(String value) { + validateAgainstPattern( + value, + "ref_path", + SAFE_REF_PATH_PATTERN, + "Only alphanumeric characters, hyphens, underscores, and forward slashes are allowed." + ); + } + + /** + * Validates a locale value. Does not allow "/" — a locale must be a single directory-name segment + * (e.g. "en_US"). Uses an allowlist to prevent path traversal, cache key injection, and other + * security issues. * - * @param value The value to validate (package ID or locale) - * @param paramName The parameter name for error messages + * @param value the locale to validate * @throws IllegalArgumentException if validation fails */ - static void validatePackageIdentifier(String value, String paramName) { + static void validateLocale(String value) { + validateAgainstPattern(value, "locale", SAFE_LOCALE_PATTERN, "Only alphanumeric characters, hyphens, and underscores are allowed."); + } + + private static void validateAgainstPattern(String value, String paramName, Pattern pattern, String allowedDesc) { if (value == null || value.isEmpty()) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid %s: value cannot be null or empty.", paramName)); } - if (!SAFE_IDENTIFIER_PATTERN.matcher(value).matches()) { + if (!pattern.matcher(value).matches()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid %s: [%s]. %s", paramName, value, allowedDesc)); + } + + // Additional check: reject ".." sequences even within otherwise valid characters (e.g., "foo..bar") + if (value.contains("..")) { throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "Invalid %s: [%s]. Only alphanumeric characters, hyphens, and underscores are allowed.", - paramName, - value - ) + String.format(Locale.ROOT, "Invalid %s: [%s]. Consecutive dots ('..') are not allowed.", paramName, value) ); } - } } diff --git a/server/src/main/java/org/opensearch/indices/analysis/HunspellService.java b/server/src/main/java/org/opensearch/indices/analysis/HunspellService.java index cafb03767f3be..424c0b800b0ff 100644 --- a/server/src/main/java/org/opensearch/indices/analysis/HunspellService.java +++ b/server/src/main/java/org/opensearch/indices/analysis/HunspellService.java @@ -63,13 +63,13 @@ * Serves as a node level registry for hunspell dictionaries. This service supports loading dictionaries from: *
        *
      • Traditional location: {@code /hunspell//} (e.g., config/hunspell/en_US/)
        • - *
      • Package-based location: {@code /analyzers//hunspell//} (e.g., config/analyzers/pkg-1234/hunspell/en_US/)
        • + *
      • Directory-based location: {@code //hunspell//} (e.g., config/analyzers/my-dict/hunspell/en_US/)
        • *
      * *

      Cache Key Strategy:

      *
        *
      • Traditional dictionaries: Cache key = locale (e.g., "en_US")
        • - *
      • Package-based dictionaries: Cache key = "{packageId}:{locale}" (e.g., "pkg-1234:en_US")
        • + *
      • Directory-based dictionaries: Cache key = "{ref_path}:{locale}" (e.g., "analyzers/my-dict:en_US")
        • *
      * *

      The following settings can be set for each dictionary: @@ -95,6 +95,9 @@ public class HunspellService { private static final Logger logger = LogManager.getLogger(HunspellService.class); + /** Separator used in cache keys for directory-based dictionaries: "{refPath}:{locale}" */ + private static final String CACHE_KEY_SEPARATOR = ":"; + public static final Setting HUNSPELL_LAZY_LOAD = Setting.boolSetting( "indices.analysis.hunspell.dictionary.lazy", Boolean.FALSE, @@ -152,34 +155,34 @@ public Dictionary getDictionary(String locale) { } /** - * Returns the hunspell dictionary from a package directory. - * Loads from package location: config/analyzers/{packageId}/hunspell/{locale}/ + * Returns the hunspell dictionary from a directory-based ref_path. + * Loads from: config/{ref_path}/hunspell/{locale}/ * - *

      Cache key format: "{packageId}:{locale}" (e.g., "pkg-1234:en_US") + *

      Cache key format: "{ref_path}:{locale}" (e.g., "analyzers/my-dict:en_US") * - * @param packageId The package ID (e.g., "pkg-1234") + * @param refPath The ref_path (e.g., "analyzers/my-dict") * @param locale The locale (e.g., "en_US") * @return The loaded Dictionary - * @throws IllegalArgumentException if packageId or locale is null + * @throws IllegalArgumentException if refPath or locale is null * @throws IllegalStateException if hunspell directory not found or dictionary cannot be loaded */ - public Dictionary getDictionaryFromPackage(String packageId, String locale) { - if (Strings.isNullOrEmpty(packageId)) { - throw new IllegalArgumentException("packageId cannot be null or empty"); + public Dictionary getDictionaryFromRefPath(String refPath, String locale) { + if (Strings.isNullOrEmpty(refPath)) { + throw new IllegalArgumentException("refPath cannot be null or empty"); } if (Strings.isNullOrEmpty(locale)) { throw new IllegalArgumentException("locale cannot be null or empty"); } - String cacheKey = buildPackageCacheKey(packageId, locale); + String cacheKey = buildRefPathCacheKey(refPath, locale); return dictionaries.computeIfAbsent(cacheKey, (key) -> { try { - return loadDictionaryFromPackage(packageId, locale); + return loadDictionaryFromRefPath(refPath, locale); } catch (Exception e) { throw new IllegalStateException( - String.format(Locale.ROOT, "Failed to load hunspell dictionary for package [%s] locale [%s]", packageId, locale), + String.format(Locale.ROOT, "Failed to load hunspell dictionary for ref_path [%s] locale [%s]", refPath, locale), e ); } @@ -187,78 +190,52 @@ public Dictionary getDictionaryFromPackage(String packageId, String locale) { } /** - * Loads a hunspell dictionary from a package directory. - * Expects hunspell files at: config/analyzers/{packageId}/hunspell/{locale}/ + * Loads a hunspell dictionary from a directory-based ref_path. + * Expects hunspell files at: config/{ref_path}/hunspell/{locale}/ * - * @param packageId The package identifier + * @param refPath The relative directory path (e.g., "analyzers/my-dict") * @param locale The locale (e.g., "en_US") * @return The loaded Dictionary * @throws Exception if loading fails */ - private Dictionary loadDictionaryFromPackage(String packageId, String locale) throws Exception { - // Validate raw inputs before path resolution (defense-in-depth, caller should also validate) - if (packageId.contains("/") || packageId.contains("\\") || packageId.contains("..")) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "Invalid package ID: [%s]. Must not contain path separators or '..' sequences.", packageId) - ); - } - if (locale.contains("/") || locale.contains("\\") || locale.contains("..")) { + private Dictionary loadDictionaryFromRefPath(String refPath, String locale) throws Exception { + // Resolve the full path: config/{ref_path}/hunspell/{locale}/ + Path refDir = env.configDir().resolve(refPath); + + // Security check: ensure resolved path stays under config directory + Path configDirAbsolute = env.configDir().toAbsolutePath().normalize(); + Path refDirAbsolute = refDir.toAbsolutePath().normalize(); + if (!refDirAbsolute.startsWith(configDirAbsolute)) { throw new IllegalArgumentException( - String.format(Locale.ROOT, "Invalid locale: [%s]. Must not contain path separators or '..' sequences.", locale) + String.format(Locale.ROOT, "ref_path must resolve under config directory. ref_path: [%s]", refPath) ); } - // Resolve analyzers base directory: config/analyzers/ - Path analyzersBaseDir = env.configDir().resolve("analyzers"); - - // Resolve package directory: config/analyzers/{packageId}/ - Path packageDir = analyzersBaseDir.resolve(packageId); - - // Security check: ensure path stays under config/analyzers/ (prevent path traversal attacks) - // Both paths must be converted to absolute and normalized before comparison - // Defense-in-depth: raw input validation above should prevent this, but we verify - // the resolved path as a secondary safeguard against any future code path changes - Path analyzersBaseDirAbsolute = analyzersBaseDir.toAbsolutePath().normalize(); - Path packageDirAbsolute = packageDir.toAbsolutePath().normalize(); - if (!packageDirAbsolute.startsWith(analyzersBaseDirAbsolute)) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "Package path must be under config/analyzers directory. Package: [%s]", packageId) - ); - } - - // Additional check: ensure the resolved package directory is exactly one level under analyzers/ - // This prevents packageId=".." or "foo/../bar" from escaping - if (!packageDirAbsolute.getParent().equals(analyzersBaseDirAbsolute)) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "Invalid package ID: [%s]. Package ID cannot contain path traversal sequences.", packageId) - ); - } - - // Check if package directory exists - if (!Files.isDirectory(packageDir)) { + // Check if ref_path directory exists + if (!Files.isDirectory(refDir)) { throw new OpenSearchException( - String.format(Locale.ROOT, "Package directory not found: [%s]. Expected at: %s", packageId, packageDir) + String.format(Locale.ROOT, "Directory not found for ref_path: [%s]. Expected at: %s", refPath, refDir) ); } - // Auto-detect hunspell directory within package - Path packageHunspellDir = packageDir.resolve("hunspell"); - if (!Files.isDirectory(packageHunspellDir)) { + // Resolve hunspell directory within ref_path + Path refHunspellDir = refDir.resolve("hunspell"); + if (!Files.isDirectory(refHunspellDir)) { throw new OpenSearchException( String.format( Locale.ROOT, - "Hunspell directory not found in package [%s]. " + "Expected 'hunspell' subdirectory at: %s", - packageId, - packageHunspellDir + "Hunspell directory not found at ref_path [%s]. Expected 'hunspell' subdirectory at: %s", + refPath, + refHunspellDir ) ); } // Resolve locale directory within hunspell - Path dicDir = packageHunspellDir.resolve(locale); + Path dicDir = refHunspellDir.resolve(locale); - // Security check: ensure locale path doesn't escape hunspell directory (prevent path traversal) - Path hunspellDirAbsolute = packageHunspellDir.toAbsolutePath().normalize(); + // Security check: ensure locale path does not escape hunspell directory + Path hunspellDirAbsolute = refHunspellDir.toAbsolutePath().normalize(); Path dicDirAbsolute = dicDir.toAbsolutePath().normalize(); if (!dicDirAbsolute.startsWith(hunspellDirAbsolute)) { throw new IllegalArgumentException( @@ -267,23 +244,23 @@ private Dictionary loadDictionaryFromPackage(String packageId, String locale) th } if (logger.isDebugEnabled()) { - logger.debug("Loading hunspell dictionary from package [{}] locale [{}] at [{}]...", packageId, locale, dicDirAbsolute); + logger.debug("Loading hunspell dictionary from ref_path [{}] locale [{}] at [{}]...", refPath, locale, dicDirAbsolute); } if (!FileSystemUtils.isAccessibleDirectory(dicDir, logger)) { throw new OpenSearchException( String.format( Locale.ROOT, - "Locale [%s] not found in package [%s]. " + "Expected directory at: %s", + "Locale [%s] not found at ref_path [%s]. Expected directory at: %s", locale, - packageId, + refPath, dicDirAbsolute ) ); } - // Delegate to loadDictionary with the package's hunspell directory as base - return loadDictionary(locale, Settings.EMPTY, env, packageHunspellDir); + // Delegate to loadDictionary with the ref_path's hunspell directory as base + return loadDictionary(locale, Settings.EMPTY, env, refHunspellDir); } private Path resolveHunspellDirectory(Environment env) { @@ -322,10 +299,10 @@ private void scanAndLoadDictionaries() throws IOException { * Loads a hunspell dictionary from a base directory by resolving the locale subdirectory, * finding .aff and .dic files, and creating the Dictionary object. * Used by both traditional locale-based loading (baseDir=hunspellDir) and - * package-based loading (baseDir=packageHunspellDir). + * directory-based ref_path loading (baseDir=refPath's hunspell dir). * * @param locale The locale of the hunspell dictionary to be loaded - * @param nodeSettings The node level settings (pass Settings.EMPTY for package-based loading) + * @param nodeSettings The node level settings (pass Settings.EMPTY for ref_path-based loading) * @param env The node environment * @param baseDir The base directory containing locale subdirectories with .aff/.dic files * @return The loaded Hunspell dictionary @@ -398,16 +375,18 @@ private static Settings loadDictionarySettings(Path dir, Settings defaults) thro return defaults; } + // ==================== CACHE KEY UTILITIES ==================== + /** - * Builds the cache key for a package-based dictionary. - * Format: "{packageId}:{locale}" (e.g., "pkg-1234:en_US") + * Builds the cache key for a directory-based dictionary. + * Format: "{ref_path}:{locale}" (e.g., "analyzers/my-dict:en_US") * - * @param packageId The package ID + * @param refPath The ref_path * @param locale The locale * @return The cache key */ - public static String buildPackageCacheKey(String packageId, String locale) { - return packageId + ":" + locale; + public static String buildRefPathCacheKey(String refPath, String locale) { + return refPath + CACHE_KEY_SEPARATOR + locale; } } diff --git a/server/src/test/java/org/opensearch/index/analysis/HunspellTokenFilterFactoryTests.java b/server/src/test/java/org/opensearch/index/analysis/HunspellTokenFilterFactoryTests.java index 7878bc72b6d2a..b62127fa7b94b 100644 --- a/server/src/test/java/org/opensearch/index/analysis/HunspellTokenFilterFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/analysis/HunspellTokenFilterFactoryTests.java @@ -70,6 +70,83 @@ public void testDedup() throws IOException { assertThat(hunspellTokenFilter.dedup(), is(false)); } + /** + * Test that ref_path with locale loads dictionary from the ref_path directory. + * Expected: config/{ref_path}/hunspell/{locale}/ + */ + public void testRefPathWithLocaleLoadsDictionaryFromDirectory() throws IOException { + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_hunspell.type", "hunspell") + .put("index.analysis.filter.my_hunspell.ref_path", "analyzers/test-dict") + .put("index.analysis.filter.my_hunspell.locale", "en_US") + .build(); + + TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, getDataPath("/indices/analyze/conf_dir")); + TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_hunspell"); + assertThat(tokenFilter, instanceOf(HunspellTokenFilterFactory.class)); + HunspellTokenFilterFactory hunspellTokenFilter = (HunspellTokenFilterFactory) tokenFilter; + assertThat(hunspellTokenFilter.dedup(), is(true)); + } + + /** + * Test that ref_path without locale throws IllegalArgumentException. + * The locale is required when using ref_path. + */ + public void testRefPathWithoutLocaleThrowsException() throws IOException { + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_hunspell.type", "hunspell") + .put("index.analysis.filter.my_hunspell.ref_path", "analyzers/test-dict") + // locale intentionally omitted + .build(); + + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings, getDataPath("/indices/analyze/conf_dir")) + ); + assertThat(e.getMessage(), containsString("locale")); + assertThat(e.getMessage(), containsString("required")); + } + + /** + * Test that non-existent ref_path directory throws exception. + */ + public void testNonExistentRefPathThrowsException() throws IOException { + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_hunspell.type", "hunspell") + .put("index.analysis.filter.my_hunspell.ref_path", "non-existent-dict") + .put("index.analysis.filter.my_hunspell.locale", "en_US") + .build(); + + Exception e = expectThrows( + Exception.class, + () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings, getDataPath("/indices/analyze/conf_dir")) + ); + // The exception message should indicate the ref_path or dictionary was not found + assertThat(e.getMessage(), containsString("non-existent-dict")); + } + + /** + * Test that non-existent locale in ref_path throws exception. + */ + public void testNonExistentLocaleInRefPathThrowsException() throws IOException { + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_hunspell.type", "hunspell") + .put("index.analysis.filter.my_hunspell.ref_path", "analyzers/test-dict") + .put("index.analysis.filter.my_hunspell.locale", "fr_FR") // locale doesn't exist in test-dict + .build(); + + Exception e = expectThrows( + Exception.class, + () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings, getDataPath("/indices/analyze/conf_dir")) + ); + // The exception message should indicate the locale was not found + assertThat(e.getMessage(), containsString("fr_FR")); + } + /** * Test dedup and longestOnly settings work with ref_path. */ @@ -77,7 +154,7 @@ public void testRefPathWithDedupAndLongestOnly() throws IOException { Settings settings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put("index.analysis.filter.my_hunspell.type", "hunspell") - .put("index.analysis.filter.my_hunspell.ref_path", "test-pkg") + .put("index.analysis.filter.my_hunspell.ref_path", "analyzers/test-dict") .put("index.analysis.filter.my_hunspell.locale", "en_US") .put("index.analysis.filter.my_hunspell.dedup", false) .put("index.analysis.filter.my_hunspell.longest_only", true) @@ -125,124 +202,116 @@ public void testMissingBothRefPathAndLocaleThrowsException() throws IOException } /** - * Test validatePackageIdentifier accepts valid identifiers. + * Test validateRefPath/validateLocale accepts valid identifiers. */ - public void testValidatePackageIdentifierAcceptsValid() { + public void testValidateRefPathAndLocaleAcceptsValid() { // These should not throw - HunspellTokenFilterFactory.validatePackageIdentifier("pkg-1234", "ref_path"); - HunspellTokenFilterFactory.validatePackageIdentifier("en_US", "locale"); - HunspellTokenFilterFactory.validatePackageIdentifier("my-package-v2", "ref_path"); - HunspellTokenFilterFactory.validatePackageIdentifier("en_US_custom", "locale"); - HunspellTokenFilterFactory.validatePackageIdentifier("a", "ref_path"); // single char - HunspellTokenFilterFactory.validatePackageIdentifier("AB", "ref_path"); // two chars + HunspellTokenFilterFactory.validateRefPath("analyzers/my-dict"); + HunspellTokenFilterFactory.validateLocale("en_US"); + HunspellTokenFilterFactory.validateRefPath("my-dict-v2"); + HunspellTokenFilterFactory.validateLocale("en_US_custom"); + HunspellTokenFilterFactory.validateRefPath("a"); // single char + HunspellTokenFilterFactory.validateRefPath("AB"); // two chars + HunspellTokenFilterFactory.validateRefPath("dict-v1"); // hyphen in middle } /** - * Test validatePackageIdentifier rejects null. + * Test validateRefPath/validateLocale rejects null. */ - public void testValidatePackageIdentifierRejectsNull() { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> HunspellTokenFilterFactory.validatePackageIdentifier(null, "ref_path") - ); + public void testValidateRefPathRejectsNull() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> HunspellTokenFilterFactory.validateRefPath(null)); assertThat(e.getMessage(), containsString("null or empty")); } /** - * Test validatePackageIdentifier rejects empty string. + * Test validateRefPath/validateLocale rejects empty string. */ - public void testValidatePackageIdentifierRejectsEmpty() { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> HunspellTokenFilterFactory.validatePackageIdentifier("", "ref_path") - ); + public void testValidateRefPathRejectsEmpty() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> HunspellTokenFilterFactory.validateRefPath("")); assertThat(e.getMessage(), containsString("null or empty")); } /** - * Test validatePackageIdentifier rejects slash. + * Test validateRefPath/validateLocale rejects backslash. */ - public void testValidatePackageIdentifierRejectsSlash() { + public void testValidateRefPathRejectsBackslash() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> HunspellTokenFilterFactory.validatePackageIdentifier("foo/bar", "ref_path") + () -> HunspellTokenFilterFactory.validateRefPath("foo\\bar") ); assertThat(e.getMessage(), containsString("Only alphanumeric")); } /** - * Test validatePackageIdentifier rejects backslash. + * Test validateRefPath/validateLocale rejects colon (cache key separator). */ - public void testValidatePackageIdentifierRejectsBackslash() { + public void testValidateRefPathRejectsColon() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> HunspellTokenFilterFactory.validatePackageIdentifier("foo\\bar", "ref_path") + () -> HunspellTokenFilterFactory.validateRefPath("dict:inject") ); assertThat(e.getMessage(), containsString("Only alphanumeric")); } /** - * Test validatePackageIdentifier rejects colon (cache key separator). + * Test validateRefPath/validateLocale rejects leading dot. */ - public void testValidatePackageIdentifierRejectsColon() { + public void testValidateRefPathRejectsLeadingDot() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> HunspellTokenFilterFactory.validatePackageIdentifier("pkg:inject", "ref_path") + () -> HunspellTokenFilterFactory.validateRefPath(".hidden") ); assertThat(e.getMessage(), containsString("Only alphanumeric")); } /** - * Test validatePackageIdentifier rejects dots. + * Test validateRefPath/validateLocale rejects trailing dot. */ - public void testValidatePackageIdentifierRejectsDots() { + public void testValidateRefPathRejectsTrailingDot() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> HunspellTokenFilterFactory.validatePackageIdentifier("pkg.v1", "ref_path") + () -> HunspellTokenFilterFactory.validateRefPath("dict.") ); assertThat(e.getMessage(), containsString("Only alphanumeric")); } /** - * Test validatePackageIdentifier rejects double dots (path traversal). + * Test validateRefPath/validateLocale rejects double dots (path traversal). */ - public void testValidatePackageIdentifierRejectsDoubleDots() { + public void testValidateLocaleRejectsDoubleDots() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> HunspellTokenFilterFactory.validatePackageIdentifier("foo..bar", "ref_path") + () -> HunspellTokenFilterFactory.validateLocale("foo..bar") ); - assertThat(e.getMessage(), containsString("Only alphanumeric")); + assertThat(e.getMessage(), containsString("Only alphanumeric characters, hyphens, and underscores are allowed.")); } /** - * Test validatePackageIdentifier rejects ".." (pure path traversal). + * Test validateRefPath/validateLocale rejects ".." (pure path traversal). */ - public void testValidatePackageIdentifierRejectsPureDotDot() { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> HunspellTokenFilterFactory.validatePackageIdentifier("..", "ref_path") - ); + public void testValidateRefPathRejectsPureDotDot() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> HunspellTokenFilterFactory.validateRefPath("..")); assertThat(e.getMessage(), containsString("Only alphanumeric")); } /** - * Test validatePackageIdentifier rejects spaces. + * Test validateRefPath/validateLocale rejects spaces. */ - public void testValidatePackageIdentifierRejectsSpaces() { + public void testValidateRefPathRejectsSpaces() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> HunspellTokenFilterFactory.validatePackageIdentifier("my package", "ref_path") + () -> HunspellTokenFilterFactory.validateRefPath("my dict") ); assertThat(e.getMessage(), containsString("Only alphanumeric")); } /** - * Test validatePackageIdentifier rejects special characters. + * Test validateRefPath/validateLocale rejects special characters. */ - public void testValidatePackageIdentifierRejectsSpecialChars() { + public void testValidateRefPathRejectsSpecialChars() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> HunspellTokenFilterFactory.validatePackageIdentifier("pkg@v1", "ref_path") + () -> HunspellTokenFilterFactory.validateRefPath("dict@v1") ); assertThat(e.getMessage(), containsString("Only alphanumeric")); } @@ -254,7 +323,7 @@ public void testCreateProducesTokenStream() throws IOException { Settings settings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put("index.analysis.filter.my_hunspell.type", "hunspell") - .put("index.analysis.filter.my_hunspell.ref_path", "test-pkg") + .put("index.analysis.filter.my_hunspell.ref_path", "analyzers/test-dict") .put("index.analysis.filter.my_hunspell.locale", "en_US") .build(); @@ -298,4 +367,5 @@ public void testLanguageAliasForLocale() throws IOException { TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_hunspell"); assertThat(tokenFilter, instanceOf(HunspellTokenFilterFactory.class)); } + } diff --git a/server/src/test/java/org/opensearch/indices/analyze/HunspellServiceTests.java b/server/src/test/java/org/opensearch/indices/analyze/HunspellServiceTests.java index 12149661b278f..963628aa2e19e 100644 --- a/server/src/test/java/org/opensearch/indices/analyze/HunspellServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/analyze/HunspellServiceTests.java @@ -107,16 +107,16 @@ public void testDicWithTwoAffs() { assertNull(e.getCause()); } - // ========== REF_PATH (Package-based Dictionary) TESTS ========== + // ========== REF_PATH (Directory-based Dictionary) TESTS ========== - public void testGetDictionaryFromPackage() throws Exception { + public void testGetDictionaryFromRefPath() throws Exception { Path tempDir = createTempDir(); - // Create package directory structure: config/analyzers/pkg-1234/hunspell/en_US/ - Path packageDir = tempDir.resolve("config").resolve("analyzers").resolve("pkg-1234").resolve("hunspell").resolve("en_US"); - java.nio.file.Files.createDirectories(packageDir); + // Create ref_path directory structure: config/analyzers/my-dict/hunspell/en_US/ + Path refPathDir = tempDir.resolve("config").resolve("analyzers/my-dict").resolve("hunspell").resolve("en_US"); + java.nio.file.Files.createDirectories(refPathDir); // Create minimal hunspell files - createHunspellFiles(packageDir, "en_US"); + createHunspellFiles(refPathDir, "en_US"); Settings settings = Settings.builder() .put(HUNSPELL_LAZY_LOAD.getKey(), randomBoolean()) @@ -126,16 +126,16 @@ public void testGetDictionaryFromPackage() throws Exception { Environment environment = new Environment(settings, tempDir.resolve("config")); HunspellService hunspellService = new HunspellService(settings, environment, emptyMap()); - // Test getDictionaryFromPackage - Dictionary dictionary = hunspellService.getDictionaryFromPackage("pkg-1234", "en_US"); + // Test getDictionaryFromRefPath + Dictionary dictionary = hunspellService.getDictionaryFromRefPath("analyzers/my-dict", "en_US"); assertThat(dictionary, notNullValue()); } - public void testGetDictionaryFromPackageCaching() throws Exception { + public void testGetDictionaryFromRefPathCaching() throws Exception { Path tempDir = createTempDir(); - Path packageDir = tempDir.resolve("config").resolve("analyzers").resolve("pkg-1234").resolve("hunspell").resolve("en_US"); - java.nio.file.Files.createDirectories(packageDir); - createHunspellFiles(packageDir, "en_US"); + Path refPathDir = tempDir.resolve("config").resolve("analyzers/my-dict").resolve("hunspell").resolve("en_US"); + java.nio.file.Files.createDirectories(refPathDir); + createHunspellFiles(refPathDir, "en_US"); Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), tempDir).build(); @@ -143,20 +143,20 @@ public void testGetDictionaryFromPackageCaching() throws Exception { HunspellService hunspellService = new HunspellService(settings, environment, emptyMap()); // First call - loads from disk - Dictionary dict1 = hunspellService.getDictionaryFromPackage("pkg-1234", "en_US"); + Dictionary dict1 = hunspellService.getDictionaryFromRefPath("analyzers/my-dict", "en_US"); assertThat(dict1, notNullValue()); // Second call - should return cached instance - Dictionary dict2 = hunspellService.getDictionaryFromPackage("pkg-1234", "en_US"); + Dictionary dict2 = hunspellService.getDictionaryFromRefPath("analyzers/my-dict", "en_US"); assertSame("Should return same cached instance", dict1, dict2); } - public void testMultiplePackagesCaching() throws Exception { + public void testMultipleRefPathsCaching() throws Exception { Path tempDir = createTempDir(); - // Create two different package directories - Path pkg1Dir = tempDir.resolve("config").resolve("analyzers").resolve("pkg-1234").resolve("hunspell").resolve("en_US"); - Path pkg2Dir = tempDir.resolve("config").resolve("analyzers").resolve("pkg-5678").resolve("hunspell").resolve("en_US"); + // Create two different ref_path directories + Path pkg1Dir = tempDir.resolve("config").resolve("analyzers/my-dict").resolve("hunspell").resolve("en_US"); + Path pkg2Dir = tempDir.resolve("config").resolve("custom/other-dict").resolve("hunspell").resolve("en_US"); java.nio.file.Files.createDirectories(pkg1Dir); java.nio.file.Files.createDirectories(pkg2Dir); createHunspellFiles(pkg1Dir, "en_US"); @@ -167,24 +167,24 @@ public void testMultiplePackagesCaching() throws Exception { Environment environment = new Environment(settings, tempDir.resolve("config")); HunspellService hunspellService = new HunspellService(settings, environment, emptyMap()); - // Load both package dictionaries - Dictionary dict1 = hunspellService.getDictionaryFromPackage("pkg-1234", "en_US"); - Dictionary dict2 = hunspellService.getDictionaryFromPackage("pkg-5678", "en_US"); + // Load both ref_path dictionaries + Dictionary dict1 = hunspellService.getDictionaryFromRefPath("analyzers/my-dict", "en_US"); + Dictionary dict2 = hunspellService.getDictionaryFromRefPath("custom/other-dict", "en_US"); assertThat(dict1, notNullValue()); assertThat(dict2, notNullValue()); - assertNotSame("Different package directories should have different Dictionary instances", dict1, dict2); + assertNotSame("Different ref_paths should have different Dictionary instances", dict1, dict2); } - public void testBuildPackageCacheKey() { - assertEquals("pkg-1234:en_US", HunspellService.buildPackageCacheKey("pkg-1234", "en_US")); - assertEquals("my-package:fr_FR", HunspellService.buildPackageCacheKey("my-package", "fr_FR")); + public void testBuildRefPathCacheKey() { + assertEquals("analyzers/my-dict:en_US", HunspellService.buildRefPathCacheKey("analyzers/my-dict", "en_US")); + assertEquals("my-dict:fr_FR", HunspellService.buildRefPathCacheKey("my-dict", "fr_FR")); } - public void testGetDictionaryFromPackageNotFound() throws Exception { + public void testGetDictionaryFromRefPathNotFound() throws Exception { Path tempDir = createTempDir(); - // Don't create the package directory - it doesn't exist + // Don't create the ref_path directory - it doesn't exist Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), tempDir).build(); @@ -192,12 +192,12 @@ public void testGetDictionaryFromPackageNotFound() throws Exception { HunspellService hunspellService = new HunspellService(settings, environment, emptyMap()); IllegalStateException e = expectThrows(IllegalStateException.class, () -> { - hunspellService.getDictionaryFromPackage("nonexistent-pkg", "en_US"); + hunspellService.getDictionaryFromRefPath("nonexistent-pkg", "en_US"); }); - assertTrue(e.getMessage().contains("Failed to load hunspell dictionary for package")); + assertTrue(e.getMessage().contains("Failed to load hunspell dictionary for ref_path")); } - public void testMixedCacheKeysTraditionalAndPackage() throws Exception { + public void testMixedCacheKeysTraditionalAndRefPath() throws Exception { Path tempDir = createTempDir(); // Create traditional hunspell directory @@ -205,10 +205,10 @@ public void testMixedCacheKeysTraditionalAndPackage() throws Exception { java.nio.file.Files.createDirectories(traditionalDir); createHunspellFiles(traditionalDir, "en_US"); - // Create package directory - Path packageDir = tempDir.resolve("config").resolve("analyzers").resolve("pkg-1234").resolve("hunspell").resolve("en_US"); - java.nio.file.Files.createDirectories(packageDir); - createHunspellFiles(packageDir, "en_US"); + // Create ref_path directory + Path refPathDir = tempDir.resolve("config").resolve("analyzers/my-dict").resolve("hunspell").resolve("en_US"); + java.nio.file.Files.createDirectories(refPathDir); + createHunspellFiles(refPathDir, "en_US"); Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), tempDir).build(); @@ -217,16 +217,17 @@ public void testMixedCacheKeysTraditionalAndPackage() throws Exception { // Load traditional dictionary Dictionary traditionalDict = hunspellService.getDictionary("en_US"); - // Load package-based dictionary - Dictionary packageDict = hunspellService.getDictionaryFromPackage("pkg-1234", "en_US"); + // Load ref_path-based dictionary + Dictionary refPathDict = hunspellService.getDictionaryFromRefPath("analyzers/my-dict", "en_US"); assertThat(traditionalDict, notNullValue()); - assertThat(packageDict, notNullValue()); - assertNotSame("Traditional and package dictionaries should be different instances", traditionalDict, packageDict); + assertThat(refPathDict, notNullValue()); + assertNotSame("Traditional and ref_path dictionaries should be different instances", traditionalDict, refPathDict); + // Both cache keys should exist } - public void testGetDictionaryFromPackageWithNullPackageId() throws Exception { + public void testGetDictionaryFromRefPathWithNullRefPath() throws Exception { Settings settings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .put(HUNSPELL_LAZY_LOAD.getKey(), true) @@ -236,12 +237,12 @@ public void testGetDictionaryFromPackageWithNullPackageId() throws Exception { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> hunspellService.getDictionaryFromPackage(null, "en_US") + () -> hunspellService.getDictionaryFromRefPath(null, "en_US") ); - assertThat(e.getMessage(), org.hamcrest.Matchers.containsString("packageId")); + assertThat(e.getMessage(), org.hamcrest.Matchers.containsString("refPath")); } - public void testGetDictionaryFromPackageWithEmptyPackageId() throws Exception { + public void testGetDictionaryFromRefPathWithEmptyRefPath() throws Exception { Settings settings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .put(HUNSPELL_LAZY_LOAD.getKey(), true) @@ -251,12 +252,12 @@ public void testGetDictionaryFromPackageWithEmptyPackageId() throws Exception { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> hunspellService.getDictionaryFromPackage("", "en_US") + () -> hunspellService.getDictionaryFromRefPath("", "en_US") ); - assertThat(e.getMessage(), org.hamcrest.Matchers.containsString("packageId")); + assertThat(e.getMessage(), org.hamcrest.Matchers.containsString("refPath")); } - public void testGetDictionaryFromPackageWithNullLocale() throws Exception { + public void testGetDictionaryFromRefPathWithNullLocale() throws Exception { Settings settings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .put(HUNSPELL_LAZY_LOAD.getKey(), true) @@ -266,12 +267,12 @@ public void testGetDictionaryFromPackageWithNullLocale() throws Exception { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> hunspellService.getDictionaryFromPackage("test-pkg", null) + () -> hunspellService.getDictionaryFromRefPath("analyzers/test-pkg", null) ); assertThat(e.getMessage(), org.hamcrest.Matchers.containsString("locale")); } - public void testGetDictionaryFromPackageWithEmptyLocale() throws Exception { + public void testGetDictionaryFromRefPathWithEmptyLocale() throws Exception { Settings settings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .put(HUNSPELL_LAZY_LOAD.getKey(), true) @@ -281,42 +282,42 @@ public void testGetDictionaryFromPackageWithEmptyLocale() throws Exception { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> hunspellService.getDictionaryFromPackage("test-pkg", "") + () -> hunspellService.getDictionaryFromRefPath("analyzers/test-pkg", "") ); assertThat(e.getMessage(), org.hamcrest.Matchers.containsString("locale")); } - public void testPackageWithMissingHunspellSubdir() throws Exception { + public void testRefPathWithMissingHunspellSubdir() throws Exception { Path tempDir = createTempDir(); - // Create package dir WITHOUT hunspell subdirectory - Path packageDir = tempDir.resolve("config").resolve("analyzers").resolve("bad-pkg"); - java.nio.file.Files.createDirectories(packageDir); + // Create ref_path dir WITHOUT hunspell subdirectory + Path refPathDir = tempDir.resolve("config").resolve("bad-dict"); + java.nio.file.Files.createDirectories(refPathDir); Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), tempDir).build(); Environment environment = new Environment(settings, tempDir.resolve("config")); HunspellService hunspellService = new HunspellService(settings, environment, emptyMap()); - Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromPackage("bad-pkg", "en_US")); - assertTrue(e.getMessage().contains("bad-pkg")); + Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromRefPath("bad-dict", "en_US")); + assertTrue(e.getMessage().contains("bad-dict")); } - public void testPackageMissingLocaleDir() throws Exception { + public void testRefPathMissingLocaleDir() throws Exception { Path tempDir = createTempDir(); - // Create package + hunspell dir but no locale subdir - Path hunspellDir = tempDir.resolve("config").resolve("analyzers").resolve("pkg-empty").resolve("hunspell"); + // Create ref_path + hunspell dir but no locale subdir + Path hunspellDir = tempDir.resolve("config").resolve("empty-dict").resolve("hunspell"); java.nio.file.Files.createDirectories(hunspellDir); Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), tempDir).build(); Environment environment = new Environment(settings, tempDir.resolve("config")); HunspellService hunspellService = new HunspellService(settings, environment, emptyMap()); - Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromPackage("pkg-empty", "en_US")); - assertTrue(e.getMessage().contains("en_US") || e.getMessage().contains("pkg-empty")); + Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromRefPath("empty-dict", "en_US")); + assertTrue(e.getMessage().contains("en_US") || e.getMessage().contains("empty-dict")); } - public void testPackageMissingAffFile() throws Exception { + public void testRefPathMissingAffFile() throws Exception { Path tempDir = createTempDir(); - Path localeDir = tempDir.resolve("config").resolve("analyzers").resolve("pkg-noaff").resolve("hunspell").resolve("en_US"); + Path localeDir = tempDir.resolve("config").resolve("noaff-dict").resolve("hunspell").resolve("en_US"); java.nio.file.Files.createDirectories(localeDir); // Only create .dic, no .aff java.nio.file.Files.write(localeDir.resolve("en_US.dic"), java.util.Arrays.asList("1", "test")); @@ -325,11 +326,11 @@ public void testPackageMissingAffFile() throws Exception { Environment environment = new Environment(settings, tempDir.resolve("config")); HunspellService hunspellService = new HunspellService(settings, environment, emptyMap()); - Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromPackage("pkg-noaff", "en_US")); - assertTrue(e.getMessage().contains("affix") || e.getMessage().contains("pkg-noaff")); + Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromRefPath("noaff-dict", "en_US")); + assertTrue(e.getMessage().contains("affix") || e.getMessage().contains("noaff-dict")); } - public void testPathTraversalInPackageId() throws Exception { + public void testPathTraversalInRefPath() throws Exception { Settings settings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .put(HUNSPELL_LAZY_LOAD.getKey(), true) @@ -337,7 +338,7 @@ public void testPathTraversalInPackageId() throws Exception { Environment environment = new Environment(settings, getDataPath("/indices/analyze/conf_dir")); HunspellService hunspellService = new HunspellService(settings, environment, emptyMap()); - Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromPackage("..", "en_US")); + Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromRefPath("..", "en_US")); assertNotNull(e); } @@ -349,11 +350,11 @@ public void testPathTraversalInLocale() throws Exception { Environment environment = new Environment(settings, getDataPath("/indices/analyze/conf_dir")); HunspellService hunspellService = new HunspellService(settings, environment, emptyMap()); - Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromPackage("test-pkg", "../en_US")); + Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromRefPath("analyzers/test-pkg", "../en_US")); assertNotNull(e); } - public void testSlashInPackageId() throws Exception { + public void testNonExistentRefPathThrowsException() throws Exception { Settings settings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .put(HUNSPELL_LAZY_LOAD.getKey(), true) @@ -361,7 +362,7 @@ public void testSlashInPackageId() throws Exception { Environment environment = new Environment(settings, getDataPath("/indices/analyze/conf_dir")); HunspellService hunspellService = new HunspellService(settings, environment, emptyMap()); - Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromPackage("foo/bar", "en_US")); + Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromRefPath("foo/bar", "en_US")); assertNotNull(e); } @@ -373,7 +374,7 @@ public void testBackslashInLocale() throws Exception { Environment environment = new Environment(settings, getDataPath("/indices/analyze/conf_dir")); HunspellService hunspellService = new HunspellService(settings, environment, emptyMap()); - Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromPackage("test-pkg", "en\\US")); + Exception e = expectThrows(Exception.class, () -> hunspellService.getDictionaryFromRefPath("analyzers/test-pkg", "en\\US")); assertNotNull(e); } diff --git a/server/src/test/resources/indices/analyze/conf_dir/analyzers/test-pkg/hunspell/en_US/en_US.aff b/server/src/test/resources/indices/analyze/conf_dir/analyzers/test-dict/hunspell/en_US/en_US.aff similarity index 100% rename from server/src/test/resources/indices/analyze/conf_dir/analyzers/test-pkg/hunspell/en_US/en_US.aff rename to server/src/test/resources/indices/analyze/conf_dir/analyzers/test-dict/hunspell/en_US/en_US.aff diff --git a/server/src/test/resources/indices/analyze/conf_dir/analyzers/test-pkg/hunspell/en_US/en_US.dic b/server/src/test/resources/indices/analyze/conf_dir/analyzers/test-dict/hunspell/en_US/en_US.dic similarity index 100% rename from server/src/test/resources/indices/analyze/conf_dir/analyzers/test-pkg/hunspell/en_US/en_US.dic rename to server/src/test/resources/indices/analyze/conf_dir/analyzers/test-dict/hunspell/en_US/en_US.dic From 765c4f0e26133f42df1d6a4e8cc25f6231ce3653 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Thu, 7 May 2026 11:25:30 -0700 Subject: [PATCH 073/115] [Analytics Backend / DataFusion] Wire PPL replace command + replace()/regexp_replace() functions (#21527) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Onboards the PPL `replace` command and the `replace()` / `regexp_replace()` eval functions to the analytics-engine route by mapping their two Calcite lowering targets — `SqlStdOperatorTable.REPLACE` and `SqlLibraryOperators.REGEXP_REPLACE_3` — through Substrait to DataFusion's native `replace` and `regexp_replace` UDFs. Same templated shape as the `fillnull` POC (#21472): ScalarFunction enum constant + STANDARD_PROJECT_OPS membership + opensearch_scalar_functions.yaml extension entry + ADDITIONAL_SCALAR_SIGS Calcite-op→Substrait-name bridge = onboarded to the analytics route. Two scalar functions added: REPLACE (literal substring replace) and REGEXP_REPLACE (regex replace). Both project-side only; the comparison result of a replaced field is filtered via the existing EQUALS capability, so no STANDARD_FILTER_OPS additions are needed. PPL's wildcard `replace` form lowers via `WildcardUtils.convertWildcardPatternToRegex()` to a Java-`Pattern`-compatible regex. Two flavors of Java syntax need translation before substrait serialization, because DataFusion uses Rust's `regex` crate which has different parsing rules: * `\Q…\E` quoted-literal blocks — Rust rejects `\Q` as an unrecognized escape sequence. The adapter expands each block to per-character escaped literals (semantics-preserving). * `$N` numeric backreferences in the replacement — Rust's replacement parser is identifier-greedy, so `$1_$2` is parsed as a reference to group named `1_` followed by `$2` (Java parses it as group 1 + literal underscore + group 2). The adapter wraps every numeric backreference in braces (`${N}`) for unambiguous Rust parsing. Both transforms are in `RegexpReplaceAdapter` and registered against `ScalarFunction.REGEXP_REPLACE` in `scalarFunctionAdapters()`. Calls without `\Q` in the pattern AND without bare `$N` in the replacement pass through unchanged. * `RegexpReplaceAdapterTests` — 19/19 (unquote: 9, brace: 7, dual-rewrite integration: 3). * `ReplaceCommandIT` (new self-contained QA IT, calcs dataset) — 10/10. Covers literal command (single + multi-pair = nested REPLACE), wildcard command (prefix + suffix), `replace()` and `regexp_replace()` in eval, full-row content checks, no-match passthrough, multi-field IN clause. * SQL plugin's `CalciteReplaceCommandIT` force-routed through the analytics-engine route via `-Dtests.analytics.{force_routing,parquet_indices}=true` — 21/21 in both the direct suite and the `CalciteNoPushdownIT` re-run. (Companion SQL plugin PR #5415 makes 4 column-order assertions and 1 error-message assertion order-agnostic, mirroring the rename precedent from #5413.) Unlike `fillnull`/`regex` where the bridge was a single one-line capability addition, `replace`'s wildcard form exposes Java↔Rust regex syntax divergence. The adapter is reusable for any future Calcite operator whose PPL lowering goes through `WildcardUtils` (e.g. potential future patterns in `like`-with-escape, custom regex lowerings). Signed-off-by: Jialiang Liang --- .../analytics/spi/ScalarFunction.java | 2 + .../DataFusionAnalyticsBackendPlugin.java | 3 + .../DataFusionFragmentConvertor.java | 10 +- .../be/datafusion/RegexpReplaceAdapter.java | 190 ++++++++++++++ .../opensearch_scalar_functions.yaml | 50 ++++ .../datafusion/RegexpReplaceAdapterTests.java | 225 +++++++++++++++++ .../analytics/qa/ReplaceCommandIT.java | 233 ++++++++++++++++++ 7 files changed, 712 insertions(+), 1 deletion(-) create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/RegexpReplaceAdapter.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/RegexpReplaceAdapterTests.java create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ReplaceCommandIT.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java index f337824cd7429..8f20e27d425b7 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java @@ -67,6 +67,8 @@ public enum ScalarFunction { */ CONCAT(Category.STRING, SqlKind.OTHER_FUNCTION, SqlStdOperatorTable.CONCAT), CHAR_LENGTH(Category.STRING, SqlKind.OTHER_FUNCTION), + REPLACE(Category.STRING, SqlKind.OTHER_FUNCTION), + REGEXP_REPLACE(Category.STRING, SqlKind.OTHER_FUNCTION), // ── Math ───────────────────────────────────────────────────────── PLUS(Category.MATH, SqlKind.PLUS), diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 7de9fd82fdf33..4d444febc36aa 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -114,6 +114,8 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP ScalarFunction.IN, ScalarFunction.LIKE, ScalarFunction.REGEXP_CONTAINS, + ScalarFunction.REPLACE, + ScalarFunction.REGEXP_REPLACE, ScalarFunction.PLUS, ScalarFunction.MINUS, ScalarFunction.TIMES, @@ -203,6 +205,7 @@ public Map scalarFunctionAdapters() { Map.entry(ScalarFunction.DIVIDE, new StdOperatorRewriteAdapter("DIVIDE", SqlStdOperatorTable.DIVIDE)), Map.entry(ScalarFunction.LIKE, new LikeAdapter()), Map.entry(ScalarFunction.MOD, new StdOperatorRewriteAdapter("MOD", SqlStdOperatorTable.MOD)), + Map.entry(ScalarFunction.REGEXP_REPLACE, new RegexpReplaceAdapter()), Map.entry(ScalarFunction.SARG_PREDICATE, new SargAdapter()), Map.entry(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter()), Map.entry(ScalarFunction.UNIX_TIMESTAMP, new UnixTimestampAdapter()), diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index 1382c0be89936..2e2f7b929a719 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -26,6 +26,7 @@ import org.apache.calcite.schema.ColumnStrategy; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.util.ImmutableBitSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -89,6 +90,11 @@ public class DataFusionFragmentConvertor implements FragmentConvertor { * resolved by DataFusion's substrait consumer to {@code Operator::RegexMatch}, the same * binary operator that backs PostgreSQL's {@code ~} regex match). Lowering target for PPL * {@code regex} command and {@code regexp_match()} function. + *

    9. {@link SqlStdOperatorTable#REPLACE} → {@code replace} (literal string replacement; + * lowering target for PPL `replace` command on non-wildcard patterns).
      10. + *
    10. {@link SqlLibraryOperators#REGEXP_REPLACE_3} → {@code regexp_replace} (regex string + * replacement; lowering target for PPL `replace` command on wildcard patterns and for + * PPL `replace()` / `regexp_replace()` functions in `eval`).
      11. * */ private static final List ADDITIONAL_SCALAR_SIGS = List.of( @@ -98,7 +104,9 @@ public class DataFusionFragmentConvertor implements FragmentConvertor { FunctionMappings.s(SqlLibraryOperators.DATE_PART, "date_part"), FunctionMappings.s(ConvertTzAdapter.LOCAL_CONVERT_TZ_OP, "convert_tz"), FunctionMappings.s(UnixTimestampAdapter.LOCAL_TO_UNIXTIME_OP, "to_unixtime"), - FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match") + FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"), + FunctionMappings.s(SqlStdOperatorTable.REPLACE, "replace"), + FunctionMappings.s(SqlLibraryOperators.REGEXP_REPLACE_3, "regexp_replace") ); private final SimpleExtension.ExtensionCollection extensions; diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/RegexpReplaceAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/RegexpReplaceAdapter.java new file mode 100644 index 0000000000000..43f2fda045cb8 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/RegexpReplaceAdapter.java @@ -0,0 +1,190 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.opensearch.analytics.spi.FieldStorageInfo; +import org.opensearch.analytics.spi.ScalarFunctionAdapter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Rewrites the pattern and replacement operands of {@code REGEXP_REPLACE} from Java syntax + * to a Rust-{@code regex}-crate-compatible form. Two transforms: + * + *
        + *
      1. Pattern: expand {@code \Q…\E} quoted-literal blocks to per-char escaped + * sequences. The SQL plugin's {@code WildcardUtils.convertWildcardPatternToRegex()} + * emits Java {@link java.util.regex.Pattern} {@code \Q…\E} syntax (e.g. + * {@code ^\Q\E(.*?)\QBOARDS\E$}). Rust's {@code regex} crate (used by DataFusion) + * rejects {@code \Q…\E} with {@code unrecognized escape sequence}.
        2. + *
      2. Replacement: wrap bare {@code $N} backreferences in braces ({@code ${N}}). + * Rust's regex replacement parser greedily extends {@code $N} into the longest + * valid identifier — so {@code $1_$2} is parsed as a reference to the (non-existent) + * group named {@code 1_} followed by {@code $2}, yielding empty + group-2's value. + * Java's {@link java.util.regex.Matcher#replaceAll} stops at the first non-digit, so + * {@code $1_$2} means group-1 + literal underscore + group-2. Wrapping every numeric + * backreference in braces gives Rust the unambiguous form regardless of what + * follows.
        3. + *
      + * + *

      Both rewrites preserve semantics — they're syntactic normalizations, not behavior + * changes. Calls without {@code \Q} in the pattern AND without bare {@code $N} in the + * replacement pass through unchanged. + * + *

      Pattern faithful to {@link java.util.regex.Pattern} semantics: an unterminated + * {@code \Q} (no closing {@code \E}) quotes through end-of-string. Replacement preserves + * existing {@code ${…}} braces and the {@code $$} literal-dollar escape. + * + * @opensearch.internal + */ +class RegexpReplaceAdapter implements ScalarFunctionAdapter { + + /** Standard regex metacharacters that must be backslash-escaped to match literally. */ + private static final String REGEX_METACHARS = ".\\+*?^$()[]{}|/"; + + @Override + public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) { + // REGEXP_REPLACE_3 has signature (input, pattern, replacement) — exactly 3 operands. + if (original.getOperands().size() != 3) { + return original; + } + RexNode patternOperand = original.getOperands().get(1); + RexNode replacementOperand = original.getOperands().get(2); + + String rewrittenPattern = null; + if (patternOperand instanceof RexLiteral patternLiteral) { + String pattern = patternLiteral.getValueAs(String.class); + if (pattern != null && pattern.contains("\\Q")) { + String rewritten = unquoteJavaRegex(pattern); + if (!pattern.equals(rewritten)) { + rewrittenPattern = rewritten; + } + } + } + + String rewrittenReplacement = null; + if (replacementOperand instanceof RexLiteral replacementLiteral) { + String replacement = replacementLiteral.getValueAs(String.class); + if (replacement != null && replacement.indexOf('$') >= 0) { + String rewritten = braceBackreferences(replacement); + if (!replacement.equals(rewritten)) { + rewrittenReplacement = rewritten; + } + } + } + + if (rewrittenPattern == null && rewrittenReplacement == null) { + return original; + } + + RexBuilder rexBuilder = cluster.getRexBuilder(); + // makeLiteral(String) infers a CHAR type sized to the rewritten string. Reusing the + // original literal's type would right-pad to the OLD length (e.g. CHAR(23) → 8 trailing + // spaces after a 15-char rewrite), corrupting the value at runtime. + List newOperands = new ArrayList<>(3); + newOperands.add(original.getOperands().get(0)); + newOperands.add(rewrittenPattern != null ? rexBuilder.makeLiteral(rewrittenPattern) : patternOperand); + newOperands.add(rewrittenReplacement != null ? rexBuilder.makeLiteral(rewrittenReplacement) : replacementOperand); + return rexBuilder.makeCall(original.getType(), original.getOperator(), newOperands); + } + + /** + * Wrap every numeric backreference {@code $N} in the input with braces ({@code ${N}}). + * Preserves {@code $$} (literal dollar) and existing {@code ${…}} braced groups. + * + *

      Why: Rust's regex replacement parser uses identifier-greedy matching — {@code $1_} + * is a named-group reference where the name is {@code 1_}. Java's parser stops at the + * first non-digit, so {@code $1_} means group 1 followed by literal underscore. Wrapping + * in braces gives Rust the unambiguous form: {@code ${1}} is always group 1, regardless + * of what follows. + * + *

      Visible for unit testing. + */ + static String braceBackreferences(String replacement) { + StringBuilder out = new StringBuilder(replacement.length()); + int i = 0; + while (i < replacement.length()) { + char c = replacement.charAt(i); + if (c == '$' && i + 1 < replacement.length()) { + char next = replacement.charAt(i + 1); + if (next == '$') { + // Literal dollar — pass through both characters unchanged. + out.append("$$"); + i += 2; + continue; + } + if (next == '{') { + // Already braced — copy through to (and including) the closing '}'. + int closeIdx = replacement.indexOf('}', i + 2); + if (closeIdx == -1) { + // Malformed — leave the rest verbatim. + out.append(replacement, i, replacement.length()); + return out.toString(); + } + out.append(replacement, i, closeIdx + 1); + i = closeIdx + 1; + continue; + } + if (Character.isDigit(next)) { + // Bare $N — wrap in braces so Rust doesn't consume following identifier + // characters (letters, digits, underscores) as part of the group name. + int j = i + 1; + while (j < replacement.length() && Character.isDigit(replacement.charAt(j))) { + j++; + } + out.append("${").append(replacement, i + 1, j).append("}"); + i = j; + continue; + } + } + out.append(c); + i++; + } + return out.toString(); + } + + /** + * Replace each {@code \Q…\E} block in the input with a per-char escaped equivalent. + * Characters inside the block that are regex metacharacters get prefixed with {@code \}; + * other characters pass through. Faithfully handles unterminated {@code \Q} (runs to end). + * + *

      Visible for unit testing — the rewrite logic is the substantive part of this adapter. + */ + static String unquoteJavaRegex(String regex) { + StringBuilder out = new StringBuilder(regex.length()); + int i = 0; + while (i < regex.length()) { + // Look for \Q at position i (literal backslash + Q in the source string). + if (i + 1 < regex.length() && regex.charAt(i) == '\\' && regex.charAt(i + 1) == 'Q') { + int contentStart = i + 2; + int closeIdx = regex.indexOf("\\E", contentStart); + int contentEnd = (closeIdx == -1) ? regex.length() : closeIdx; + for (int j = contentStart; j < contentEnd; j++) { + char c = regex.charAt(j); + if (REGEX_METACHARS.indexOf(c) >= 0) { + out.append('\\'); + } + out.append(c); + } + // Skip past \E (or off the end if unterminated). + i = (closeIdx == -1) ? regex.length() : closeIdx + 2; + } else { + out.append(regex.charAt(i)); + i++; + } + } + return out.toString(); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml index dc88820ed6c75..5d8f9621439f7 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml @@ -59,3 +59,53 @@ scalar_functions: - value: "string" name: "pattern" return: boolean + - name: replace + description: >- + Literal string replacement — replace every occurrence of `search` in `input` + with `replacement`. Lowering target for PPL's `replace` command on + non-wildcard patterns (Calcite `SqlStdOperatorTable.REPLACE`). + datafusion-substrait resolves the extension name "replace" to DataFusion's + native `replace` UDF (datafusion-functions/src/string/replace.rs). + impls: + - args: + - value: "varchar" + name: "input" + - value: "varchar" + name: "search" + - value: "varchar" + name: "replacement" + return: "varchar" + - args: + - value: "string" + name: "input" + - value: "string" + name: "search" + - value: "string" + name: "replacement" + return: string + - name: regexp_replace + description: >- + Regex string replacement — replace every match of `pattern` in `input` + with `replacement`. Lowering target for PPL's `replace` command on + wildcard patterns (after `*` → regex conversion) and for the PPL + `replace()` / `regexp_replace()` functions in `eval`. Calcite emits + `SqlLibraryOperators.REGEXP_REPLACE_3`. datafusion-substrait resolves + the extension name "regexp_replace" to DataFusion's native `regexp_replace` + UDF (datafusion-functions/src/regex/regexpreplace.rs). + impls: + - args: + - value: "varchar" + name: "input" + - value: "varchar" + name: "pattern" + - value: "varchar" + name: "replacement" + return: "varchar" + - args: + - value: "string" + name: "input" + - value: "string" + name: "pattern" + - value: "string" + name: "replacement" + return: string diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/RegexpReplaceAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/RegexpReplaceAdapterTests.java new file mode 100644 index 0000000000000..8bc2e58257705 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/RegexpReplaceAdapterTests.java @@ -0,0 +1,225 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion; + +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlLibraryOperators; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +/** + * Unit tests for {@link RegexpReplaceAdapter}. Pins the {@code \Q…\E} → per-char-escape + * rewrite that bridges the SQL plugin's Java-style wildcard regex output (from + * {@code WildcardUtils.convertWildcardPatternToRegex()}) to the Rust regex syntax expected + * by DataFusion's {@code regexp_replace} UDF. + * + *

      Each test pins one rewrite invariant. A regression that loses {@code \Q…\E} expansion, + * mishandles unterminated quotes, or swaps operand positions in the rebuilt + * {@code REGEXP_REPLACE} call surfaces here rather than at IT-level "regex parse error" + * failures. + */ +public class RegexpReplaceAdapterTests extends OpenSearchTestCase { + + private RelDataTypeFactory typeFactory; + private RexBuilder rexBuilder; + private RelOptCluster cluster; + private RelDataType varcharType; + + private final RegexpReplaceAdapter adapter = new RegexpReplaceAdapter(); + + @Override + public void setUp() throws Exception { + super.setUp(); + typeFactory = new JavaTypeFactoryImpl(); + rexBuilder = new RexBuilder(typeFactory); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + cluster = RelOptCluster.create(planner, rexBuilder); + varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + } + + // ── unquoteJavaRegex — the substantive transform ──────────────────────────── + + public void testUnquoteEmptyQuoteBlock() { + // \Q\E produces empty string in Java; should disappear entirely. + assertEquals("", RegexpReplaceAdapter.unquoteJavaRegex("\\Q\\E")); + } + + public void testUnquotePreservesNonQuotedPortions() { + // Standard regex outside any \Q…\E passes through unchanged. + assertEquals("^(.*?)$", RegexpReplaceAdapter.unquoteJavaRegex("^(.*?)$")); + } + + public void testUnquoteSimpleLiteral() { + // \QBOARDS\E → BOARDS (no special chars to escape). + assertEquals("BOARDS", RegexpReplaceAdapter.unquoteJavaRegex("\\QBOARDS\\E")); + } + + public void testUnquoteWildcardSuffixShape() { + // SQL plugin's WildcardUtils output for `*BOARDS` — empty prefix, capture, literal suffix. + assertEquals("^(.*?)BOARDS$", RegexpReplaceAdapter.unquoteJavaRegex("^\\Q\\E(.*?)\\QBOARDS\\E$")); + } + + public void testUnquoteWildcardPrefixShape() { + // SQL plugin's WildcardUtils output for `BUSINESS*` — literal prefix, capture, empty suffix. + assertEquals("^BUSINESS(.*?)$", RegexpReplaceAdapter.unquoteJavaRegex("^\\QBUSINESS\\E(.*?)\\Q\\E$")); + } + + public void testUnquoteEscapesMetacharsInsideQuote() { + // \Q a.b+c \E — inside a Java literal block `.` and `+` are not regex metas; in standard + // regex they are. Rewrite must escape every metachar so semantics are preserved. + assertEquals("a\\.b\\+c", RegexpReplaceAdapter.unquoteJavaRegex("\\Qa.b+c\\E")); + } + + public void testUnquoteHandlesMultipleQuoteBlocks() { + // Two \Q…\E spans separated by a regex fragment. + assertEquals("FOO(.*?)BAR", RegexpReplaceAdapter.unquoteJavaRegex("\\QFOO\\E(.*?)\\QBAR\\E")); + } + + public void testUnquoteUnterminatedRunsToEnd() { + // Per Java Pattern semantics, \Q without a closing \E quotes through end of string. + assertEquals("\\.\\+", RegexpReplaceAdapter.unquoteJavaRegex("\\Q.+")); + } + + public void testUnquoteIdempotentOnRustCompatibleRegex() { + // No \Q in input → output identical to input. + String input = "^(foo|bar).*$"; + assertEquals(input, RegexpReplaceAdapter.unquoteJavaRegex(input)); + } + + // ── braceBackreferences — replacement-string transform ────────────────────── + + public void testBraceWrapsBareNumeric() { + // $1 → ${1}; trivial smoke check. + assertEquals("${1}", RegexpReplaceAdapter.braceBackreferences("$1")); + } + + public void testBraceCriticalCaseFollowedByUnderscore() { + // $1_$2 — the failing wildcard-replacement case. Rust parses $1_ as named group "1_", + // so the brace rewrite is what makes group-1 + literal underscore + group-2 work. + assertEquals("${1}_${2}", RegexpReplaceAdapter.braceBackreferences("$1_$2")); + } + + public void testBraceFollowedByLetter() { + // $1foo — Rust would parse "1foo" as the group name. Braces force the boundary. + assertEquals("${1}foo", RegexpReplaceAdapter.braceBackreferences("$1foo")); + } + + public void testBraceMultiDigitGroup() { + // $12 (group twelve) — wrap entire numeric run. + assertEquals("${12}", RegexpReplaceAdapter.braceBackreferences("$12")); + } + + public void testBracePreservesLiteralDollar() { + // $$ stays $$ (Rust regex's literal-dollar escape, same as Java). + assertEquals("$$10", RegexpReplaceAdapter.braceBackreferences("$$10")); + } + + public void testBracePreservesAlreadyBraced() { + // ${1} input is already braced — must not be re-wrapped or otherwise mangled. + assertEquals("${1}_${2}", RegexpReplaceAdapter.braceBackreferences("${1}_${2}")); + } + + public void testBraceIdempotentOnNonBackrefReplacement() { + // No $ at all → output identical to input. + String input = "plain literal"; + assertEquals(input, RegexpReplaceAdapter.braceBackreferences(input)); + } + + // ── adapter integration: RexCall in / RexCall out ─────────────────────────── + + public void testAdaptRewritesPatternLiteral() { + // Build REGEXP_REPLACE(field, '^\\QBUSINESS\\E(.*?)\\Q\\E$', 'BIZ') and verify the + // rebuilt call has the expanded pattern, original input, and original replacement. + RexNode field = rexBuilder.makeInputRef(varcharType, 0); + RexNode pattern = rexBuilder.makeLiteral("^\\QBUSINESS\\E(.*?)\\Q\\E$"); + RexNode replacement = rexBuilder.makeLiteral("BIZ"); + RexCall original = (RexCall) rexBuilder.makeCall(SqlLibraryOperators.REGEXP_REPLACE_3, List.of(field, pattern, replacement)); + + RexNode adapted = adapter.adapt(original, List.of(), cluster); + + assertTrue("adapted node must remain a RexCall", adapted instanceof RexCall); + RexCall result = (RexCall) adapted; + assertEquals("operator preserved", original.getOperator(), result.getOperator()); + assertEquals("input operand preserved", field, result.getOperands().get(0)); + assertEquals("replacement operand preserved", replacement, result.getOperands().get(2)); + + RexNode newPatternNode = result.getOperands().get(1); + assertTrue("pattern must remain a literal", newPatternNode instanceof RexLiteral); + assertEquals("Java \\Q…\\E rewritten to plain regex", "^BUSINESS(.*?)$", ((RexLiteral) newPatternNode).getValueAs(String.class)); + } + + public void testAdaptPassesThroughWhenNoQuoteBlock() { + // Pattern doesn't contain \Q — adapter must return the call unchanged (identity). + RexNode field = rexBuilder.makeInputRef(varcharType, 0); + RexNode pattern = rexBuilder.makeLiteral("^OFFICE.*$"); + RexNode replacement = rexBuilder.makeLiteral("OFC"); + RexCall original = (RexCall) rexBuilder.makeCall(SqlLibraryOperators.REGEXP_REPLACE_3, List.of(field, pattern, replacement)); + + RexNode adapted = adapter.adapt(original, List.of(), cluster); + + assertSame("identity — no rewrite when pattern has no \\Q", original, adapted); + } + + public void testAdaptPassesThroughNonLiteralPattern() { + // Pattern is a column reference (not a literal) — adapter cannot rewrite at planning + // time; pass through and let DataFusion error at runtime if the value is incompatible. + // Replacement is a plain literal with no $, so neither transform fires. + RexNode field = rexBuilder.makeInputRef(varcharType, 0); + RexNode patternRef = rexBuilder.makeInputRef(varcharType, 1); + RexNode replacement = rexBuilder.makeLiteral("X"); + RexCall original = (RexCall) rexBuilder.makeCall(SqlLibraryOperators.REGEXP_REPLACE_3, List.of(field, patternRef, replacement)); + + RexNode adapted = adapter.adapt(original, List.of(), cluster); + + assertSame("non-literal pattern must pass through", original, adapted); + } + + public void testAdaptRewritesReplacementOnly() { + // Rust-compatible pattern but Java-style $1_$2 replacement — adapter rewrites only + // the replacement, leaves the pattern untouched. + RexNode field = rexBuilder.makeInputRef(varcharType, 0); + RexNode pattern = rexBuilder.makeLiteral("^(.*?) (.*?)$"); + RexNode replacement = rexBuilder.makeLiteral("$1_$2"); + RexCall original = (RexCall) rexBuilder.makeCall(SqlLibraryOperators.REGEXP_REPLACE_3, List.of(field, pattern, replacement)); + + RexCall result = (RexCall) adapter.adapt(original, List.of(), cluster); + + assertEquals( + "pattern unchanged when no \\Q present", + "^(.*?) (.*?)$", + ((RexLiteral) result.getOperands().get(1)).getValueAs(String.class) + ); + assertEquals("$1_$2 wrapped to ${1}_${2}", "${1}_${2}", ((RexLiteral) result.getOperands().get(2)).getValueAs(String.class)); + } + + public void testAdaptRewritesBothPatternAndReplacement() { + // The full failing-IT shape: Java-quoted pattern AND bare $N replacement. Both must + // be rewritten in a single pass so the resulting call matches DataFusion semantics. + RexNode field = rexBuilder.makeInputRef(varcharType, 0); + RexNode pattern = rexBuilder.makeLiteral("^\\Q\\E(.*?)\\Q \\E(.*?)\\Q\\E$"); + RexNode replacement = rexBuilder.makeLiteral("$1_$2"); + RexCall original = (RexCall) rexBuilder.makeCall(SqlLibraryOperators.REGEXP_REPLACE_3, List.of(field, pattern, replacement)); + + RexCall result = (RexCall) adapter.adapt(original, List.of(), cluster); + + assertEquals("pattern unquoted", "^(.*?) (.*?)$", ((RexLiteral) result.getOperands().get(1)).getValueAs(String.class)); + assertEquals("replacement braced", "${1}_${2}", ((RexLiteral) result.getOperands().get(2)).getValueAs(String.class)); + } +} diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ReplaceCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ReplaceCommandIT.java new file mode 100644 index 0000000000000..3aca91aedd2d1 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ReplaceCommandIT.java @@ -0,0 +1,233 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for the PPL {@code replace} command and {@code replace()} / + * {@code regexp_replace()} functions on the analytics-engine route. + * + *

      Mirrors {@code CalciteReplaceCommandIT} from the {@code opensearch-project/sql} repository so + * that the analytics-engine path can be verified inside core without cross-plugin dependencies on + * the SQL plugin. Each test sends a PPL query through {@code POST /_analytics/ppl} (exposed by the + * {@code test-ppl-frontend} plugin), which runs the same {@code UnifiedQueryPlanner} → + * {@code CalciteRelNodeVisitor} → Substrait → DataFusion pipeline. + * + *

      Two distinct lowering targets are exercised: + *

        + *
      • {@code | replace 'literal' WITH 'new' IN field} — emits Calcite + * {@code SqlStdOperatorTable.REPLACE} (substring replacement, no regex). Mapped to + * Substrait extension {@code "replace"} → DataFusion's {@code replace} UDF.
        • + *
      • {@code | replace 'pat*' WITH 'new' IN field} (wildcard) and + * {@code eval x = replace(field, ...)} / {@code regexp_replace(...)} — emit Calcite + * {@code SqlLibraryOperators.REGEXP_REPLACE_3}. Mapped to Substrait extension + * {@code "regexp_replace"} → DataFusion's {@code regexp_replace} UDF.
        • + *
      + * + *

      Multi-pair replacements ({@code | replace 'A' WITH 'X', 'B' WITH 'Y' IN f}) lower to nested + * {@code REPLACE(REPLACE(field, ...), ...)} calls — exercises sequential project-side application. + * + *

      Provisions the {@code calcs} dataset (parquet-backed) once per class via + * {@link DatasetProvisioner}; {@link AnalyticsRestTestCase#preserveIndicesUponCompletion()} + * keeps it across test methods. + */ +public class ReplaceCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + // ── command form: literal pattern (SqlStdOperatorTable.REPLACE) ───────────── + + public void testReplaceLiteralSinglePair() throws IOException { + // FURNITURE → FURN in str0; 2 rows affected, others unchanged. + // assertContainsRow uses substring/contains — order-independent. + assertRowCount( + "source=" + DATASET.indexName + " | replace 'FURNITURE' WITH 'FURN' IN str0 | where str0='FURN' | fields str0", + 2 + ); + } + + public void testReplaceLiteralMultiplePairs() throws IOException { + // Nested REPLACE in projection: REPLACE(REPLACE(str0, 'FURNITURE', 'F'), 'TECHNOLOGY', 'T'). + // FURNITURE (×2) → 'F', TECHNOLOGY (×9) → 'T', OFFICE SUPPLIES (×6) → unchanged. + assertRowCount( + "source=" + DATASET.indexName + " | replace 'FURNITURE' WITH 'F', 'TECHNOLOGY' WITH 'T' IN str0 | where str0='F' | fields str0", + 2 + ); + assertRowCount( + "source=" + DATASET.indexName + " | replace 'FURNITURE' WITH 'F', 'TECHNOLOGY' WITH 'T' IN str0 | where str0='T' | fields str0", + 9 + ); + } + + public void testReplaceLiteralNoMatch() throws IOException { + // Pattern matches no value — every row passes through unchanged. 17 rows total in calcs. + assertRowCount( + "source=" + DATASET.indexName + " | replace 'NOSUCHVALUE' WITH 'X' IN str0 | fields str0", + 17 + ); + } + + public void testReplaceLiteralExpectedRows() throws IOException { + // Verify the actual replaced values (not just counts) for the FURNITURE rows. + assertRows( + "source=" + DATASET.indexName + " | replace 'FURNITURE' WITH 'FURN' IN str0 | where str0='FURN' | fields str0, str1 | sort str1", + row("FURN", "CLAMP ON LAMPS"), + row("FURN", "CLOCKS") + ); + } + + public void testReplaceLiteralAcrossMultipleFields() throws IOException { + // Replace value 'FURNITURE' in BOTH str0 and str1. str1 has no FURNITURE → unaffected. + // str0 has 2 → renamed to FURN. + assertRowCount( + "source=" + DATASET.indexName + " | replace 'FURNITURE' WITH 'FURN' IN str0, str1 | where str0='FURN' | fields str0", + 2 + ); + } + + // ── command form: wildcard pattern (REGEXP_REPLACE_3) ────────────────────── + // + // The SQL plugin's WildcardUtils.convertWildcardPatternToRegex() emits Java-style regex + // with `\Q…\E` quoted-literal blocks (e.g. `^\Q\E(.*?)\QBOARDS\E$`). Rust's regex crate + // (used by DataFusion) does not support `\Q…\E`, so the pattern would otherwise fail to + // parse. RegexpReplaceAdapter (in DataFusionAnalyticsBackendPlugin.scalarFunctionAdapters) + // rewrites `\Q…\E` blocks to per-char-escaped literals before substrait serialization. + + public void testReplaceWildcardSuffix() throws IOException { + // '*BOARDS' matches strings ending in BOARDS — CORDED KEYBOARDS, CORDLESS KEYBOARDS (×2). + // Whole-string replacement: matched values become 'KBD'. + assertRowCount( + "source=" + DATASET.indexName + " | replace '*BOARDS' WITH 'KBD' IN str1 | where str1='KBD' | fields str1", + 2 + ); + } + + public void testReplaceWildcardPrefix() throws IOException { + // 'BUSINESS*' matches BUSINESS ENVELOPES, BUSINESS COPIERS (×2). + assertRowCount( + "source=" + DATASET.indexName + " | replace 'BUSINESS*' WITH 'BIZ' IN str1 | where str1='BIZ' | fields str1", + 2 + ); + } + + // ── function form: regexp_replace() in eval projection ───────────────────── + + public void testRegexpReplaceInEval() throws IOException { + // eval-side regexp_replace lowers to REGEXP_REPLACE_3. Replace any digit run in str0 with + // empty — no-op for these string values, exercises the function-form code path. + // Better: replace 'OFFICE' in str0 — produces 'OFFICE SUPPLIES' → ' SUPPLIES'. + assertRowCount( + "source=" + DATASET.indexName + " | eval x = regexp_replace(str0, 'OFFICE ', '') | where x='SUPPLIES' | fields x", + 6 + ); + } + + public void testReplaceFunctionInEval() throws IOException { + // PPL replace() function in eval also lowers to REGEXP_REPLACE_3 (per + // PPLFuncImpTable.register for BuiltinFunctionName.REPLACE). + assertRowCount( + "source=" + DATASET.indexName + " | eval x = replace(str0, 'TECHNOLOGY', 'TECH') | where x='TECH' | fields x", + 9 + ); + } + + public void testRegexpReplaceProducesProjectedColumn() throws IOException { + // Check the actual output value, confirming round-trip through Substrait → DataFusion. + assertRows( + "source=" + DATASET.indexName + " | where str0='FURNITURE' | eval s = replace(str1, 'CLAMP', 'GRIP') | fields s | sort s", + row("CLOCKS"), + row("GRIP ON LAMPS") + ); + } + + // ── helpers ──────────────────────────────────────────────────────────────── + + private static List row(Object... values) { + return Arrays.asList(values); + } + + private void assertRowCount(String ppl, int expectedCount) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' field for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expectedCount, actualRows.size()); + } + + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRows(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' field for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size()); + for (int i = 0; i < expected.length; i++) { + List want = expected[i]; + List got = actualRows.get(i); + assertEquals( + "Column count mismatch at row " + i + " for query: " + ppl, + want.size(), + got.size() + ); + for (int j = 0; j < want.size(); j++) { + assertEquals( + "Cell mismatch at row " + i + ", col " + j + " for query: " + ppl, + want.get(j), + got.get(j) + ); + } + } + } + + private void assertErrorContains(String ppl, String expectedSubstring) { + try { + Map response = executePpl(ppl); + fail("Expected query to fail with [" + expectedSubstring + "] but got response: " + response); + } catch (ResponseException e) { + String body; + try { + body = org.opensearch.test.rest.OpenSearchRestTestCase.entityAsMap(e.getResponse()).toString(); + } catch (IOException ioe) { + body = e.getMessage(); + } + assertTrue( + "Expected response body to contain [" + expectedSubstring + "] but was: " + body, + body.contains(expectedSubstring) + ); + } catch (IOException e) { + fail("Unexpected IOException: " + e); + } + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } +} From 4f6969e07a208d77021e7fc4d660ce9e303d332d Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Thu, 7 May 2026 13:33:33 -0500 Subject: [PATCH 074/115] Fix race in EhcacheDiskCacheManagerTests close phase (#21525) testCreateAndCloseCacheConcurrently populated the diskCacheAliases ArrayList from multiple worker threads without synchronization. When concurrent add() calls dropped an entry the list ended up shorter than randomThreads, and the close-phase lambda's diskCacheAliases.get(finalI) then threw IndexOutOfBoundsException on the thread holding the last index. That worker died without calling countDownLatch2.countDown(), so the main thread blocked forever on countDownLatch2.await() and the suite hit its 20-minute timeout (e.g. build 76071, seed DF0F4253E4345108). Pre-populate the aliases list in a single-threaded setup loop before spawning the worker threads, so the list is only read concurrently, never written. Each worker captures its alias by index in the loop instead of looking it up from the shared list. Verified with the original failing seed and with tests.iters=50. Relates to #19722 Signed-off-by: Andrew Ross --- .../store/disk/EhcacheDiskCacheManagerTests.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhcacheDiskCacheManagerTests.java b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhcacheDiskCacheManagerTests.java index 36252a0a2681d..1800afc1d473b 100644 --- a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhcacheDiskCacheManagerTests.java +++ b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhcacheDiskCacheManagerTests.java @@ -45,14 +45,17 @@ public void testCreateAndCloseCacheConcurrently() throws Exception { EhcacheDiskCacheManager.getCacheManager(CacheType.INDICES_REQUEST_CACHE, path, settings, THREAD_POOL_ALIAS); } int randomThreads = randomIntBetween(5, 10); + // Pre-populate aliases to avoid concurrent writes + List diskCacheAliases = new ArrayList<>(randomThreads); + for (int i = 0; i < randomThreads; i++) { + diskCacheAliases.add(UUID.randomUUID().toString()); + } Thread[] threads = new Thread[randomThreads]; Phaser phaser = new Phaser(randomThreads + 1); CountDownLatch countDownLatch = new CountDownLatch(randomThreads); - List diskCacheAliases = new ArrayList<>(); for (int i = 0; i < randomThreads; i++) { + String diskCacheAlias = diskCacheAliases.get(i); threads[i] = new Thread(() -> { - String diskCacheAlias = UUID.randomUUID().toString(); - diskCacheAliases.add(diskCacheAlias); phaser.arriveAndAwaitAdvance(); EhcacheDiskCacheManager.createCache(CacheType.INDICES_REQUEST_CACHE, diskCacheAlias, getCacheConfigurationBuilder()); countDownLatch.countDown(); @@ -68,10 +71,10 @@ public void testCreateAndCloseCacheConcurrently() throws Exception { CountDownLatch countDownLatch2 = new CountDownLatch(randomThreads); for (int i = 0; i < randomThreads; i++) { String finalPath = path; - int finalI = i; + String diskCacheAlias = diskCacheAliases.get(i); threads[i] = new Thread(() -> { phaser2.arriveAndAwaitAdvance(); - EhcacheDiskCacheManager.closeCache(CacheType.INDICES_REQUEST_CACHE, diskCacheAliases.get(finalI), finalPath); + EhcacheDiskCacheManager.closeCache(CacheType.INDICES_REQUEST_CACHE, diskCacheAlias, finalPath); countDownLatch2.countDown(); }); threads[i].start(); From 90be262691242226834483dc36670bf2f7be9ff7 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 7 May 2026 15:06:35 -0400 Subject: [PATCH 075/115] Fix Detect Breaking Changes version selection logic for previous released version (#21529) * Fix Detect Breaking Changes version selection logic for previous released version Signed-off-by: Craig Perkins Signed-off-by: Craig Perkins --- server/build.gradle | 58 +++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/server/build.gradle b/server/build.gradle index 5deedc4aa725b..791f733527400 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -186,18 +186,21 @@ tasks.named('forbiddenApisInternalClusterTest').configure { forbidSleep() } // Set to current version by default def japicmpCompareTarget = System.getProperty("japicmp.compare.version") -if (japicmpCompareTarget == null) { /* use latest released version */ - // Read the list from maven central. - // Fetch the metadata and parse the xml into Version instances, pick the latest one +if (japicmpCompareTarget == null) { + // Fetch released versions from maven central, pick the latest on the same major line before current. + def currentVersion = org.opensearch.gradle.Version.fromString(org.opensearch.gradle.VersionProperties.getOpenSearch()) japicmpCompareTarget = new URL('https://repo1.maven.org/maven2/org/opensearch/opensearch/maven-metadata.xml').openStream().withStream { s -> - new XmlParser().parse(s) - .versioning.versions.version - .collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ } - .collect { org.opensearch.gradle.Version.fromString(it) } - .toSorted() - .last() - .toString() - } + new XmlParser().parse(s).versioning.versions.version + .collect { it.text() } + .findAll { it ==~ /\d+\.\d+\.\d+/ } + .collect { org.opensearch.gradle.Version.fromString(it) } + .findAll { it.getMajor() == currentVersion.getMajor() && it.before(currentVersion) } + .toSorted() + .with { it.empty ? null : it.last().toString() } + } + if (japicmpCompareTarget == null) { + logger.lifecycle("No prior released version found on the same major line. Skipping japicmp.") + } } def generateModulesList = tasks.register("generateModulesList") { @@ -512,22 +515,25 @@ tasks.named("sourcesJar").configure { } } -/** Compares the current build against a laltest released version or the version supplied through 'japicmp.compare.version' system property */ +/** Compares the current build against a latest released version or the version supplied through 'japicmp.compare.version' system property */ tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) { - logger.info("Comparing public APIs from ${version} to ${japicmpCompareTarget}") - // See please https://github.com/siom79/japicmp/issues/201 - compatibilityChangeExcludes = [ "METHOD_ABSTRACT_NOW_DEFAULT", "METHOD_ADDED_TO_INTERFACE" ] - oldClasspath.from(files("${buildDir}/japicmp-target/opensearch-${japicmpCompareTarget}.jar")) - newClasspath.from(tasks.named('jar')) - onlyModified = true - failOnModification = true - ignoreMissingClasses = true - failOnSourceIncompatibility = true - annotationIncludes = ['@org.opensearch.common.annotation.PublicApi', '@org.opensearch.common.annotation.DeprecatedApi'] - annotationExcludes = ['@org.opensearch.common.annotation.InternalApi', '@org.opensearch.common.annotation.ExperimentalApi'] - txtOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.txt") - htmlOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.html") - dependsOn downloadJapicmpCompareTarget + enabled = japicmpCompareTarget != null + if (japicmpCompareTarget != null) { + logger.lifecycle("Comparing public APIs from ${version} to ${japicmpCompareTarget}") + // See please https://github.com/siom79/japicmp/issues/201 + compatibilityChangeExcludes = [ "METHOD_ABSTRACT_NOW_DEFAULT", "METHOD_ADDED_TO_INTERFACE" ] + oldClasspath.from(files("${buildDir}/japicmp-target/opensearch-${japicmpCompareTarget}.jar")) + newClasspath.from(tasks.named('jar')) + onlyModified = true + failOnModification = true + ignoreMissingClasses = true + failOnSourceIncompatibility = true + annotationIncludes = ['@org.opensearch.common.annotation.PublicApi', '@org.opensearch.common.annotation.DeprecatedApi'] + annotationExcludes = ['@org.opensearch.common.annotation.InternalApi', '@org.opensearch.common.annotation.ExperimentalApi'] + txtOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.txt") + htmlOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.html") + dependsOn downloadJapicmpCompareTarget + } } /** If the Java API Comparison task failed, print a hint if the change should be merged from its target branch */ From 66e5a70b19ea9263e43234e19eca0ea0c652ee33 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 7 May 2026 17:20:53 -0400 Subject: [PATCH 076/115] Onboarding calcite opensearch revision publishing setups (#21549) * Onboarding calcite opensearch revision publishing setups Signed-off-by: Peter Zhu * Update a comment Signed-off-by: Peter Zhu * Update default ref to use commit ids Signed-off-by: Peter Zhu * Make patch file more dynamic now Signed-off-by: Peter Zhu --------- Signed-off-by: Peter Zhu --- .github/workflows/calcite-snapshots.yml | 86 ++++++++++ gradle/libs.versions.toml | 5 + ...r-TCCL-for-Janino-parent-classloader.patch | 151 ++++++++++++++++++ 3 files changed, 242 insertions(+) create mode 100644 .github/workflows/calcite-snapshots.yml create mode 100644 sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch diff --git a/.github/workflows/calcite-snapshots.yml b/.github/workflows/calcite-snapshots.yml new file mode 100644 index 0000000000000..5b1dbd971108d --- /dev/null +++ b/.github/workflows/calcite-snapshots.yml @@ -0,0 +1,86 @@ +# This workflow will check out, build, and publish snapshots of calcite. + +name: OpenSearch Lucene snapshots + +on: + workflow_dispatch: + # Inputs the workflow accepts. + inputs: + ref: + description: 'Calcite ref in github.com/apache/calcite, default to calcite-1.41.0 tag (c838dd471ca36f5648ef13e5c3c34c6ca0815322)' + type: string + required: false + default: 'c838dd471ca36f5648ef13e5c3c34c6ca0815322' + java_version: + description: 'Java version to use' + type: string + required: false + default: '21' + patch_file_path: + description: 'The patch file, default to sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch' + type: string + required: false + default: 'sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch' + +jobs: + publish-snapshots: + if: github.repository == 'opensearch-project/OpenSearch' + runs-on: ubuntu-latest + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + + steps: + - name: Checkout Calcite ref:${{ github.event.inputs.ref }} + uses: actions/checkout@v6 + with: + repository: 'apache/calcite' + ref: ${{ github.event.inputs.ref }} + persist-credentials: false + + - name: Checkout OpenSearch main + uses: actions/checkout@v6 + with: + repository: 'opensearch-project/OpenSearch' + ref: 'main' + persist-credentials: false + path: 'os_main' + + - name: Setup JDK ${{ github.event.inputs.java_version }} + uses: actions/setup-java@v5 + with: + java-version: ${{ github.event.inputs.java_version }} + distribution: 'temurin' + + - name: Apply Patches and build calcite jars + run: | + git apply os_main/${{ github.event.inputs.patch_file_path }} + BASE_VER=`cat os_main/gradle/libs.versions.toml | grep -E "^calcite" | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+"` + REV=`cat os_main/gradle/libs.versions.toml | grep -E "^calcite_os_rev" | grep -Eo "[0-9]+"` + CALCITE_VER=$BASE_VER-opensearch-$REV + sed -i "s/calcite\.version.*/calcite.version=$CALCITE_VER/" gradle.properties + ./gradlew :core:publishToMavenLocal :linq4j:publishToMavenLocal -Prelease -PskipSign -PskipJavadoc -x test --no-daemon + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_SECRET_ROLE }} + aws-region: us-east-1 + + - name: Get S3 Bucket + id: get_s3_bucket + run: | + lucene_snapshots_bucket=`aws secretsmanager get-secret-value --secret-id jenkins-artifact-bucket-name --query SecretString --output text` + echo "::add-mask::$lucene_snapshots_bucket" + echo "LUCENE_SNAPSHOTS_BUCKET=$lucene_snapshots_bucket" >> $GITHUB_OUTPUT + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_S3_ROLE }} + aws-region: us-east-1 + + - name: Copy files to S3 with the aws CLI + run: | + aws s3 cp ~/.m2/repository/org/apache/calcite/ s3://${{ steps.get_s3_bucket.outputs.LUCENE_SNAPSHOTS_BUCKET }}/snapshots/maven/org/apache/calcite/ --recursive --no-progress diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1767c3567e958..f8fc6004330df 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -104,6 +104,11 @@ opentelemetrysemconv = "1.40.0" arrow = "18.1.0" flatbuffers = "2.0.0" +# calcite is locally patched and published to OpenSearch maven snapshots; see .github/workflows/calcite-snapshots.yml. +# Published as org.apache.calcite:calcite-core:${calcite}-opensearch-${calcite_os_rev}. +calcite = "1.41.0" +calcite_os_rev = "1" + [libraries] antlr4-runtime = { group = "org.antlr", name = "antlr4-runtime", version.ref = "antlr4" } asm-analysis = { group = "org.ow2.asm", name = "asm-analysis", version.ref = "asm" } diff --git a/sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch b/sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch new file mode 100644 index 0000000000000..6c378de6c1686 --- /dev/null +++ b/sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch @@ -0,0 +1,151 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Mustang +Date: Wed, 6 May 2026 00:00:00 -0700 +Subject: [PATCH] CALCITE-3745: TCCL-chained classloader for Janino parent CL + +Introduce a TcclChainedClassLoader utility that resolves classes via the +thread context classloader first, falling back to the Calcite-local CL +if a name is not found on TCCL. Every site that configures Janino's +parent classloader (EnumerableInterpretable, JaninoRexCompiler, +RexExecutable, JaninoRelMetadataProvider) now uses the chained loader. + +This keeps Calcite's internal types always resolvable while making +child-plugin UDFs visible when the host (OpenSearch's extendedPlugins) +sets TCCL to the child classloader. +--- + .../enumerable/EnumerableInterpretable.java | 3 +- + .../interpreter/JaninoRexCompiler.java | 3 +- + .../metadata/JaninoRelMetadataProvider.java | 4 +- + .../org/apache/calcite/rex/RexExecutable.java | 4 +- + .../calcite/util/TcclChainedClassLoader.java | 61 +++++++++++++++++++ + 5 files changed, 71 insertions(+), 4 deletions(-) + create mode 100644 core/src/main/java/org/apache/calcite/util/TcclChainedClassLoader.java + +diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java +index 5f32ab1..1c9ce19 100644 +--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java ++++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java +@@ -145,7 +145,8 @@ static Bindable getBindable(ClassDeclaration expr, String classBody, int fieldCo + "Unable to instantiate java compiler", e); + } + final ISimpleCompiler compiler = compilerFactory.newSimpleCompiler(); +- compiler.setParentClassLoader(classLoader); ++ compiler.setParentClassLoader( ++ org.apache.calcite.util.TcclChainedClassLoader.chain(classLoader)); + final String s = "public final class " + expr.name + " implements " + + (fieldCount == 1 + ? Bindable.class.getCanonicalName() + ", " + Typed.class.getCanonicalName() +diff --git a/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java b/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java +index bca4f85..d6de426 100644 +--- a/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java ++++ b/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java +@@ -211,7 +211,8 @@ static Scalar.Producer getScalar(ClassDeclaration expr, String s) + IClassBodyEvaluator cbe = compilerFactory.newClassBodyEvaluator(); + cbe.setClassName(expr.name); + cbe.setImplementedInterfaces(new Class[] {Scalar.Producer.class}); +- cbe.setParentClassLoader(classLoader); ++ cbe.setParentClassLoader( ++ org.apache.calcite.util.TcclChainedClassLoader.chain(classLoader)); + if (CalciteSystemProperty.DEBUG.value()) { + // Add line numbers to the generated janino class + cbe.setDebuggingInformation(true, true, true); +diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java +index 135b11e..34a5e4b 100644 +--- a/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java ++++ b/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java +@@ -157,7 +157,9 @@ static > MH compile(String className, + } + + final ISimpleCompiler compiler = compilerFactory.newSimpleCompiler(); +- compiler.setParentClassLoader(JaninoRexCompiler.class.getClassLoader()); ++ compiler.setParentClassLoader( ++ org.apache.calcite.util.TcclChainedClassLoader.chain( ++ JaninoRexCompiler.class.getClassLoader())); + + if (CalciteSystemProperty.DEBUG.value()) { + // Add line numbers to the generated janino class +diff --git a/core/src/main/java/org/apache/calcite/rex/RexExecutable.java b/core/src/main/java/org/apache/calcite/rex/RexExecutable.java +index 8828654..1e91951 100644 +--- a/core/src/main/java/org/apache/calcite/rex/RexExecutable.java ++++ b/core/src/main/java/org/apache/calcite/rex/RexExecutable.java +@@ -60,7 +60,9 @@ public RexExecutable(String code, Object reason) { + cbe.setClassName(GENERATED_CLASS_NAME); + cbe.setExtendedClass(Utilities.class); + cbe.setImplementedInterfaces(new Class[] {Function1.class, Serializable.class}); +- cbe.setParentClassLoader(RexExecutable.class.getClassLoader()); ++ cbe.setParentClassLoader( ++ org.apache.calcite.util.TcclChainedClassLoader.chain( ++ RexExecutable.class.getClassLoader())); + cbe.cook(new Scanner(null, new StringReader(code))); + Class c = cbe.getClazz(); + //noinspection unchecked +diff --git a/core/src/main/java/org/apache/calcite/util/TcclChainedClassLoader.java b/core/src/main/java/org/apache/calcite/util/TcclChainedClassLoader.java +new file mode 100644 +index 0000000..259d71c +--- /dev/null ++++ b/core/src/main/java/org/apache/calcite/util/TcclChainedClassLoader.java +@@ -0,0 +1,61 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to you under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.calcite.util; ++ ++/** ++ * CALCITE-3745 (OpenSearch patch): helper to build a classloader that ++ * prefers the thread context classloader for name resolution but falls back ++ * to a supplied Calcite-local classloader for Calcite's own internal types. ++ * ++ *

      When Calcite is embedded under a parent plugin classloader (e.g. in ++ * OpenSearch's {@code extendedPlugins} layout), child plugins register UDFs ++ * that end up referenced by name in Janino-generated code. The default ++ * {@code SomeCalciteClass.class.getClassLoader()} cannot see those UDFs. ++ * Using TCCL alone breaks in contexts where TCCL is a stripped-down ++ * classloader that has no view of Calcite's own internal types. Chaining ++ * solves both cases. ++ */ ++public final class TcclChainedClassLoader { ++ private TcclChainedClassLoader() {} ++ ++ /** ++ * Returns a classloader that resolves classes by consulting the thread ++ * context classloader first, then falling back to {@code fallback}. If ++ * TCCL is unset or identical to {@code fallback}, the fallback is ++ * returned unchanged. ++ */ ++ public static ClassLoader chain(ClassLoader fallback) { ++ final ClassLoader tccl = Thread.currentThread().getContextClassLoader(); ++ if (tccl == null || tccl == fallback) { ++ return fallback; ++ } ++ return new ClassLoader(fallback) { ++ @Override protected Class loadClass(String name, boolean resolve) ++ throws ClassNotFoundException { ++ try { ++ Class c = tccl.loadClass(name); ++ if (resolve) { ++ resolveClass(c); ++ } ++ return c; ++ } catch (ClassNotFoundException e) { ++ return super.loadClass(name, resolve); ++ } ++ } ++ }; ++ } ++} +-- +2.50.1 (Apple Git-155) + From 38e274dd5de344cf03ff85852c23584dd0b7add4 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 7 May 2026 17:28:39 -0400 Subject: [PATCH 077/115] Update calcite workflow name to reflect calcite (#21551) Signed-off-by: Peter Zhu --- .github/workflows/calcite-snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/calcite-snapshots.yml b/.github/workflows/calcite-snapshots.yml index 5b1dbd971108d..b9f96a856e040 100644 --- a/.github/workflows/calcite-snapshots.yml +++ b/.github/workflows/calcite-snapshots.yml @@ -1,6 +1,6 @@ # This workflow will check out, build, and publish snapshots of calcite. -name: OpenSearch Lucene snapshots +name: OpenSearch Calcite Revision on: workflow_dispatch: From 9cada0351f06ba7fc667d61922cc761b5a9a93fc Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Thu, 7 May 2026 14:43:56 -0700 Subject: [PATCH 078/115] [QA] Add FieldFormatCommandIT for the analytics-engine REST path (#21544) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PPL `fieldformat` is a Calcite-only command that lowers to a plain Eval node (see SQL plugin's `AstBuilder.visitFieldformatCommand`). Its expressions go through Calcite's || (CONCAT) operator and CAST, both already wired in the DataFusion backend's STANDARD_PROJECT_OPS via #21498. **No code changes required for the analytics route — this PR is QA-only.** The unique surface vs plain `eval` is the prefix-{`.`} and suffix-{`.`} string-concat sugar emitted by `AstExpressionBuilder.visitFieldFormatEvalClause` for the StringDotlogicalExpression / LogicalExpressionDotString rules: fieldformat x = "prefix".CAST(y AS STRING)." suffix" expands to a chain of CONCAT calls. Both forms route through the existing CONCAT capability — no extension lookup or adapter needed since isthmus' default catalog binds the || operator natively. Four tests against the in-process QA cluster, exercising the analytics path end-to-end via the test-ppl-frontend plugin: | Test | Shape | |---|---| | `testFieldformatPlusConcat` | `'Hello ' + str0` — basic +-concat. | | `testFieldformatPrefixDotCast` | `'Code: '.CAST(int0 AS STRING)` — StringDotlogicalExpression branch. | | `testFieldformatCastDotSuffix` | `CAST(int0 AS STRING).' pts'` — LogicalExpressionDotString branch. | | `testFieldformatPrefixDotCastDotSuffix` | `'Code: '.CAST(int0 AS STRING).' pts'` — combined. | Tests filter `where isnotnull(int0)` before sorting/limiting so the deterministic-row assertions don't flap on the calcs dataset's six null int0 rows (Calcite's default ascending sort puts nulls first). Out of scope: the v2-side `testFieldFormatStringConcatenationWithNullFieldToString` uses `tostring(age, "commas")` — a multi-mode UDF (binary / hex / commas / duration) with substantial Java logic in `ToStringFunction`. Adding it to the analytics path would need either Calcite-level rewrites or a DataFusion Rust UDF; tracked separately. Validates: 4/4 FieldFormatCommandIT pass; full :sandbox:qa:analytics-engine-rest:integTest suite green (**132 tests across 17 ITs**, no regressions). Signed-off-by: Kai Huang --- .../analytics/qa/FieldFormatCommandIT.java | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldFormatCommandIT.java diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldFormatCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldFormatCommandIT.java new file mode 100644 index 0000000000000..5f3d63ea0d84e --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FieldFormatCommandIT.java @@ -0,0 +1,188 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Self-contained integration test for PPL {@code fieldformat} on the analytics-engine route. + * + *

      Mirrors {@code CalciteFieldFormatCommandIT} from the {@code opensearch-project/sql} + * repository so the analytics-engine path can be verified inside core without + * cross-plugin dependencies on the SQL plugin. + * + *

      {@code fieldformat} is a Calcite-only command (gated on + * {@code plugins.calcite.enabled}; the gate is satisfied here because + * {@code test-ppl-frontend}'s {@code UnifiedQueryService} sets the cluster setting + * to true on every request). It lowers to a plain {@code Eval} node — see + * {@code AstBuilder.visitFieldformatCommand} in the SQL plugin. The unique surface + * vs plain {@code eval} is the prefix-{@code .} and suffix-{@code .} string-concat + * sugar: {@code fieldformat x = "prefix".CAST(y AS STRING)." suffix"} expands to + * a chain of {@code CONCAT} calls. Both {@code +}-style concat and the dotted form + * route through Calcite's {@code ||} operator and resolve to + * {@link org.opensearch.analytics.spi.ScalarFunction#CONCAT}, already in + * {@code STANDARD_PROJECT_OPS}. + * + *

      Provisions the {@code calcs} dataset (parquet-backed) once per class via + * {@link DatasetProvisioner}. + */ +public class FieldFormatCommandIT extends AnalyticsRestTestCase { + + private static final Dataset DATASET = new Dataset("calcs", "calcs"); + + private static boolean dataProvisioned = false; + + private void ensureDataProvisioned() throws IOException { + if (dataProvisioned == false) { + DatasetProvisioner.provision(client(), DATASET); + dataProvisioned = true; + } + } + + // ── basic +-concat — same expression shape as `eval x = 'lit' + field` ───── + + public void testFieldformatPlusConcat() throws IOException { + // `'Hello ' + str0` — Calcite emits || (CONCAT). calcs has 17 rows; str0 has three + // distinct values: FURNITURE (×2), OFFICE SUPPLIES (×6), TECHNOLOGY (×9). After + // `head 3 | sort str0`, the first three are the FURNITURE/FURNITURE pair plus the + // first OFFICE SUPPLIES — but ordering inside identical str0 isn't pinned, so we + // sort by both key and a deterministic int0 first. + assertRows( + "source=" + DATASET.indexName + + " | sort str0, int0" + + " | head 3" + + " | fieldformat greeting = \"Hello \" + str0" + + " | fields str0, greeting", + row("FURNITURE", "Hello FURNITURE"), + row("FURNITURE", "Hello FURNITURE"), + row("OFFICE SUPPLIES", "Hello OFFICE SUPPLIES") + ); + } + + // ── dotted-concat: prefix.CAST(int AS STRING) ──────────────────────────────── + + public void testFieldformatPrefixDotCast() throws IOException { + // `"Code: ".CAST(int0 AS STRING)` — prefix string + CAST-to-string of an integer, + // chained with the `.` form unique to fieldformat. AstExpressionBuilder's + // StringDotlogicalExpression branch emits a Let with prefix=literal, expression=CAST, + // and the Eval's CalciteRexNodeVisitor wraps both in a CONCAT. + assertRows( + "source=" + DATASET.indexName + + " | where isnotnull(int0)" + + " | sort int0" + + " | head 3" + + " | fieldformat code_desc = \"Code: \".CAST(int0 AS STRING)" + + " | fields int0, code_desc", + row(1, "Code: 1"), + row(3, "Code: 3"), + row(4, "Code: 4") + ); + } + + // ── dotted-concat: CAST(int AS STRING).suffix ──────────────────────────────── + + public void testFieldformatCastDotSuffix() throws IOException { + // Mirror image of the prefix case — LogicalExpressionDotString branch emits a Let + // with suffix=literal, expression=CAST. Output column type is string regardless of + // input type because CAST coerces and CONCAT preserves string. + assertRows( + "source=" + DATASET.indexName + + " | where isnotnull(int0)" + + " | sort int0" + + " | head 3" + + " | fieldformat code_desc = CAST(int0 AS STRING).\" pts\"" + + " | fields int0, code_desc", + row(1, "1 pts"), + row(3, "3 pts"), + row(4, "4 pts") + ); + } + + // ── dotted-concat: prefix.CAST(int AS STRING).suffix ───────────────────────── + + public void testFieldformatPrefixDotCastDotSuffix() throws IOException { + // Combined prefix + middle expression + suffix. The Eval emitted has a single Let + // whose expression is CONCAT(CONCAT(prefix, CAST(...)), suffix). All three operands + // route through the CONCAT capability in STANDARD_PROJECT_OPS — no extension lookup + // needed since isthmus' default catalog binds the || operator natively. + assertRows( + "source=" + DATASET.indexName + + " | where isnotnull(int0)" + + " | sort int0" + + " | head 3" + + " | fieldformat code_desc = \"Code: \".CAST(int0 AS STRING).\" pts\"" + + " | fields int0, code_desc", + row(1, "Code: 1 pts"), + row(3, "Code: 3 pts"), + row(4, "Code: 4 pts") + ); + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + private static List row(Object... values) { + return Arrays.asList(values); + } + + @SafeVarargs + @SuppressWarnings("varargs") + private final void assertRows(String ppl, List... expected) throws IOException { + Map response = executePpl(ppl); + @SuppressWarnings("unchecked") + List> actualRows = (List>) response.get("rows"); + assertNotNull("Response missing 'rows' for query: " + ppl, actualRows); + assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size()); + for (int i = 0; i < expected.length; i++) { + List want = expected[i]; + List got = actualRows.get(i); + assertEquals( + "Column count mismatch at row " + i + " for query: " + ppl, + want.size(), + got.size() + ); + for (int j = 0; j < want.size(); j++) { + assertCellEquals( + "Cell mismatch at row " + i + ", col " + j + " for query: " + ppl, + want.get(j), + got.get(j) + ); + } + } + } + + private Map executePpl(String ppl) throws IOException { + ensureDataProvisioned(); + Request request = new Request("POST", "/_analytics/ppl"); + request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}"); + Response response = client().performRequest(request); + return assertOkAndParse(response, "PPL: " + ppl); + } + + private static void assertCellEquals(String message, Object expected, Object actual) { + if (expected == null || actual == null) { + assertEquals(message, expected, actual); + return; + } + if (expected instanceof Number && actual instanceof Number) { + double e = ((Number) expected).doubleValue(); + double a = ((Number) actual).doubleValue(); + if (Double.compare(e, a) != 0) { + fail(message + ": expected <" + expected + "> but was <" + actual + ">"); + } + return; + } + assertEquals(message, expected, actual); + } +} From b1c21c761caf094f13f3041f2fd199132e7340ad Mon Sep 17 00:00:00 2001 From: Mike Ma Date: Thu, 7 May 2026 19:52:03 -0500 Subject: [PATCH 079/115] Handle null scripted metric combine results (#21534) ScriptedMetricAggregator checked result.getClass() before passing the combine result to InternalScriptedMetric. A combine script can return null for an empty bucket, so the type check could throw before the existing null value could be carried through aggregation reduction. Skip the writeable type check for null results and add a regression test for a null combine result. Signed-off-by: Mike Ma --- .../metrics/ScriptedMetricAggregator.java | 4 ++- .../ScriptedMetricAggregatorTests.java | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregator.java index 0ad7f8fb2e8b6..9078672dcf5ca 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregator.java @@ -158,7 +158,9 @@ public void collect(int doc, long owningBucketOrd) throws IOException { @Override public InternalAggregation buildAggregation(long owningBucketOrdinal) { Object result = aggStateForResult(owningBucketOrdinal).combine(); - if (result.getClass() != ScriptedAvg.class) StreamOutput.checkWriteable(result); + if (result != null && result.getClass() != ScriptedAvg.class) { + StreamOutput.checkWriteable(result); + } return new InternalScriptedMetric(name, singletonList(result), reduceScript, metadata()); } diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java index 53e5f2bfb53bb..7b16a4c25cda0 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java @@ -126,6 +126,12 @@ public class ScriptedMetricAggregatorTests extends AggregatorTestCase { "combineScriptNoop", Collections.emptyMap() ); + private static final Script COMBINE_SCRIPT_NULL = new Script( + ScriptType.INLINE, + MockScriptEngine.NAME, + "combineScriptNull", + Collections.emptyMap() + ); private static final Script INIT_SCRIPT_PARAMS = new Script( ScriptType.INLINE, @@ -202,6 +208,7 @@ public static void initMockScripts() { Map state = (Map) params.get("state"); return state; }); + SCRIPTS.put("combineScriptNull", params -> null); SCRIPTS.put("reduceScript", params -> { List states = (List) params.get("states"); return states.stream().filter(a -> a instanceof Number).map(a -> (Number) a).mapToInt(Number::intValue).sum(); @@ -402,6 +409,28 @@ public void testScriptedMetricWithCombine() throws IOException { } } + public void testScriptedMetricWithNullCombineResult() throws IOException { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", 1))); + } + try (IndexReader indexReader = DirectoryReader.open(directory)) { + ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); + aggregationBuilder.initScript(INIT_SCRIPT) + .mapScript(MAP_SCRIPT) + .combineScript(COMBINE_SCRIPT_NULL) + .reduceScript(REDUCE_SCRIPT); + ScriptedMetric scriptedMetric = searchAndReduce( + newSearcher(indexReader, true, true), + new MatchAllDocsQuery(), + aggregationBuilder + ); + assertEquals(AGG_NAME, scriptedMetric.getName()); + assertEquals(0, scriptedMetric.aggregation()); + } + } + } + /** * test that uses the score of the documents */ From e0a06f2673b6516bd39c7fd4e7177c742a974da3 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Thu, 7 May 2026 20:00:24 -0500 Subject: [PATCH 080/115] Remove Unsafe class injection from Java agent (#21542) The Java agent previously used ClassInjector.UsingUnsafe at premain time to inject AgentPolicy, SubjectInterceptor, and the two stack caller chain extractors into the boot classloader. This relied on sun.misc.Unsafe, which is deprecated for removal on recent JDKs and emits JVM warnings. This change relocates StackCallerClassChainExtractor, StackCallerProtectionDomainChainExtractor, and SubjectInterceptor from :libs:agent-sm:agent into :libs:agent-sm:bootstrap, under a new org.opensearch.javaagent.bootstrap.internal subpackage to distinguish them from AgentPolicy, which is the shared API between the server and the agent. The bootstrap jar is listed on the agent jar's Boot-Class-Path manifest attribute, so the JVM loads these classes into the boot classloader natively with no runtime injection needed. Also set -Dnet.bytebuddy.safe=true on test JVMs to disable the Unsafe-based dispatchers inside ByteBuddy's ClassInjector. ByteBuddy falls back to MethodHandles.Lookup / reflection injection, which works on JDK 9+. Signed-off-by: Andrew Ross --- build.gradle | 6 +++++- .../secure_sm/AccessController.java | 2 +- .../java/org/opensearch/javaagent/Agent.java | 19 +------------------ .../opensearch/javaagent/FileInterceptor.java | 2 ++ .../javaagent/RuntimeHaltInterceptor.java | 1 + .../javaagent/SocketChannelInterceptor.java | 1 + .../javaagent/SystemExitInterceptor.java | 1 + ...kCallerProtectionDomainExtractorTests.java | 1 + .../StackCallerClassChainExtractor.java | 2 +- ...kCallerProtectionDomainChainExtractor.java | 2 +- .../internal}/SubjectInterceptor.java | 2 +- .../bootstrap/internal/package-info.java | 16 ++++++++++++++++ 12 files changed, 32 insertions(+), 23 deletions(-) rename libs/agent-sm/{agent/src/main/java/org/opensearch/javaagent => bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal}/StackCallerClassChainExtractor.java (95%) rename libs/agent-sm/{agent/src/main/java/org/opensearch/javaagent => bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal}/StackCallerProtectionDomainChainExtractor.java (97%) rename libs/agent-sm/{agent/src/main/java/org/opensearch/javaagent => bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal}/SubjectInterceptor.java (92%) create mode 100644 libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/package-info.java diff --git a/build.gradle b/build.gradle index 5aef923c77ac7..550b94f60900e 100644 --- a/build.gradle +++ b/build.gradle @@ -421,7 +421,11 @@ gradle.projectsEvaluated { task.jvmArgs += [ "--add-modules=jdk.incubator.vector", "--add-exports=java.base/com.sun.crypto.provider=ALL-UNNAMED", - "--enable-native-access=ALL-UNNAMED" + "--enable-native-access=ALL-UNNAMED", + // Disable ByteBuddy's Unsafe-based class injection path to avoid + // "sun.misc.Unsafe::objectFieldOffset has been called by ByteBuddy" JVM warnings on JDK 21+. + // ByteBuddy still falls back to Lookup/Reflection injection strategies. + "-Dnet.bytebuddy.safe=true" ] // Add Java Agent for security sandboxing diff --git a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/AccessController.java b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/AccessController.java index b07bb9068e8fa..e7c27ead15ff1 100644 --- a/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/AccessController.java +++ b/libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/AccessController.java @@ -17,7 +17,7 @@ * removal. All new code should use this class instead of the JDK's {@code AccessController}. * * Running code in a privileged context will ensure that the code has the necessary permissions - * without traversing through the entire call stack. See {@code org.opensearch.javaagent.StackCallerProtectionDomainChainExtractor} + * without traversing through the entire call stack. See {@code org.opensearch.javaagent.bootstrap.internal.StackCallerProtectionDomainChainExtractor} * * Example usages: * 
      diff --git a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/Agent.java b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/Agent.java
index f638d354fdd7b..6f3098eae655f 100644
--- a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/Agent.java
+++ b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/Agent.java
@@ -8,7 +8,7 @@
 
 package org.opensearch.javaagent;
 
-import org.opensearch.javaagent.bootstrap.AgentPolicy;
+import org.opensearch.javaagent.bootstrap.internal.SubjectInterceptor;
 
 import javax.security.auth.Subject;
 
@@ -18,14 +18,11 @@
 import java.nio.channels.SocketChannel;
 import java.nio.file.Files;
 import java.nio.file.spi.FileSystemProvider;
-import java.util.Map;
 
 import net.bytebuddy.ByteBuddy;
 import net.bytebuddy.agent.builder.AgentBuilder;
 import net.bytebuddy.asm.Advice;
 import net.bytebuddy.description.type.TypeDescription;
-import net.bytebuddy.dynamic.ClassFileLocator;
-import net.bytebuddy.dynamic.loading.ClassInjector;
 import net.bytebuddy.implementation.Implementation;
 import net.bytebuddy.implementation.MethodDelegation;
 import net.bytebuddy.matcher.ElementMatcher.Junction;
@@ -96,20 +93,6 @@ private static AgentBuilder createAgentBuilder() throws Exception {
             ElementMatchers.named("getSubject")
         ).intercept(MethodDelegation.to(SubjectInterceptor.class));
 
-        ClassInjector.UsingUnsafe.ofBootLoader()
-            .inject(
-                Map.of(
-                    new TypeDescription.ForLoadedType(StackCallerProtectionDomainChainExtractor.class),
-                    ClassFileLocator.ForClassLoader.read(StackCallerProtectionDomainChainExtractor.class),
-                    new TypeDescription.ForLoadedType(StackCallerClassChainExtractor.class),
-                    ClassFileLocator.ForClassLoader.read(StackCallerClassChainExtractor.class),
-                    new TypeDescription.ForLoadedType(AgentPolicy.class),
-                    ClassFileLocator.ForClassLoader.read(AgentPolicy.class),
-                    new TypeDescription.ForLoadedType(SubjectInterceptor.class),
-                    ClassFileLocator.ForClassLoader.read(SubjectInterceptor.class)
-                )
-            );
-
         final ByteBuddy byteBuddy = new ByteBuddy().with(Implementation.Context.Disabled.Factory.INSTANCE);
         var builder = new AgentBuilder.Default(byteBuddy).with(AgentBuilder.InitializationStrategy.NoOp.INSTANCE)
             .with(AgentBuilder.RedefinitionStrategy.REDEFINITION)
diff --git a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/FileInterceptor.java b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/FileInterceptor.java
index 455be2a83f840..68dcfe0015d74 100644
--- a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/FileInterceptor.java
+++ b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/FileInterceptor.java
@@ -9,6 +9,8 @@
 package org.opensearch.javaagent;
 
 import org.opensearch.javaagent.bootstrap.AgentPolicy;
+import org.opensearch.javaagent.bootstrap.internal.StackCallerClassChainExtractor;
+import org.opensearch.javaagent.bootstrap.internal.StackCallerProtectionDomainChainExtractor;
 
 import java.io.FilePermission;
 import java.lang.reflect.Method;
diff --git a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/RuntimeHaltInterceptor.java b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/RuntimeHaltInterceptor.java
index 9f879a744f45f..d9edfdaa7223d 100644
--- a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/RuntimeHaltInterceptor.java
+++ b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/RuntimeHaltInterceptor.java
@@ -9,6 +9,7 @@
 package org.opensearch.javaagent;
 
 import org.opensearch.javaagent.bootstrap.AgentPolicy;
+import org.opensearch.javaagent.bootstrap.internal.StackCallerClassChainExtractor;
 
 import java.lang.StackWalker.Option;
 import java.security.Policy;
diff --git a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SocketChannelInterceptor.java b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SocketChannelInterceptor.java
index 93daeccb6503f..d98804092aece 100644
--- a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SocketChannelInterceptor.java
+++ b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SocketChannelInterceptor.java
@@ -9,6 +9,7 @@
 package org.opensearch.javaagent;
 
 import org.opensearch.javaagent.bootstrap.AgentPolicy;
+import org.opensearch.javaagent.bootstrap.internal.StackCallerProtectionDomainChainExtractor;
 
 import java.lang.reflect.Method;
 import java.net.InetSocketAddress;
diff --git a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SystemExitInterceptor.java b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SystemExitInterceptor.java
index 6ba4f59e00942..b19e5559cd5e3 100644
--- a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SystemExitInterceptor.java
+++ b/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SystemExitInterceptor.java
@@ -9,6 +9,7 @@
 package org.opensearch.javaagent;
 
 import org.opensearch.javaagent.bootstrap.AgentPolicy;
+import org.opensearch.javaagent.bootstrap.internal.StackCallerClassChainExtractor;
 
 import java.lang.StackWalker.Option;
 import java.security.Policy;
diff --git a/libs/agent-sm/agent/src/test/java/org/opensearch/javaagent/StackCallerProtectionDomainExtractorTests.java b/libs/agent-sm/agent/src/test/java/org/opensearch/javaagent/StackCallerProtectionDomainExtractorTests.java
index 2efb993448dc3..0ef5939fb8ed1 100644
--- a/libs/agent-sm/agent/src/test/java/org/opensearch/javaagent/StackCallerProtectionDomainExtractorTests.java
+++ b/libs/agent-sm/agent/src/test/java/org/opensearch/javaagent/StackCallerProtectionDomainExtractorTests.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.javaagent;
 
+import org.opensearch.javaagent.bootstrap.internal.StackCallerProtectionDomainChainExtractor;
 import org.junit.Assume;
 import org.junit.Test;
 
diff --git a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/StackCallerClassChainExtractor.java b/libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/StackCallerClassChainExtractor.java
similarity index 95%
rename from libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/StackCallerClassChainExtractor.java
rename to libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/StackCallerClassChainExtractor.java
index b7be2883b6a79..4cf4b9a9a567f 100644
--- a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/StackCallerClassChainExtractor.java
+++ b/libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/StackCallerClassChainExtractor.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.javaagent;
+package org.opensearch.javaagent.bootstrap.internal;
 
 import java.lang.StackWalker.StackFrame;
 import java.util.Collection;
diff --git a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/StackCallerProtectionDomainChainExtractor.java b/libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/StackCallerProtectionDomainChainExtractor.java
similarity index 97%
rename from libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/StackCallerProtectionDomainChainExtractor.java
rename to libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/StackCallerProtectionDomainChainExtractor.java
index da2c00cd8a3f3..607678c1bb796 100644
--- a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/StackCallerProtectionDomainChainExtractor.java
+++ b/libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/StackCallerProtectionDomainChainExtractor.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.javaagent;
+package org.opensearch.javaagent.bootstrap.internal;
 
 import java.lang.StackWalker.StackFrame;
 import java.security.ProtectionDomain;
diff --git a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SubjectInterceptor.java b/libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/SubjectInterceptor.java
similarity index 92%
rename from libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SubjectInterceptor.java
rename to libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/SubjectInterceptor.java
index 1950a2ffce906..d684c8859f9b6 100644
--- a/libs/agent-sm/agent/src/main/java/org/opensearch/javaagent/SubjectInterceptor.java
+++ b/libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/SubjectInterceptor.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.javaagent;
+package org.opensearch.javaagent.bootstrap.internal;
 
 import javax.security.auth.Subject;
 
diff --git a/libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/package-info.java b/libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/package-info.java
new file mode 100644
index 0000000000000..13a7d2a6a1e4d
--- /dev/null
+++ b/libs/agent-sm/bootstrap/src/main/java/org/opensearch/javaagent/bootstrap/internal/package-info.java
@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Internal agent support classes that must be loaded by the boot classloader
+ * so that bytecode woven into JDK classes (either inlined ByteBuddy Advice or
+ * MethodDelegation stubs) can resolve them. These classes are implementation
+ * details of the Java agent and are not part of any public API; do not depend
+ * on them from outside {@code :libs:agent-sm:agent}.
+ */
+package org.opensearch.javaagent.bootstrap.internal;

From dbe4a429434a7fe95407b4408c57457c90c59f48 Mon Sep 17 00:00:00 2001
From: expani1729 <110471048+expani@users.noreply.github.com>
Date: Thu, 7 May 2026 19:25:18 -0700
Subject: [PATCH 081/115] Enable Lucene Filter delegation from Datafusion for
 Correctness (#21555)

---
 .../backend/ShardScanExecutionContext.java    |  36 ++
 .../spi/AnalyticsSearchBackendPlugin.java     |  32 +
 .../analytics/spi/DelegationDescriptor.java   |  52 ++
 .../analytics/spi/FilterDelegationHandle.java |  77 +++
 .../spi/FilterDelegationInstructionNode.java  |   2 +-
 .../FragmentInstructionHandlerFactory.java    |   3 +
 .../analytics/spi/InstructionType.java        |   6 +-
 ...hardScanWithDelegationInstructionNode.java |  59 ++
 .../analytics-backend-datafusion/build.gradle |  45 --
 .../rust/src/api.rs                           |  24 +-
 .../rust/src/ffm.rs                           |  59 +-
 .../rust/src/indexed_executor.rs              | 589 ++++++++++--------
 .../rust/src/indexed_table/bool_tree.rs       |  78 +--
 .../rust/src/indexed_table/ffm_callbacks.rs   |  14 +-
 .../src/indexed_table/substrait_to_tree.rs    |  83 ++-
 .../tests_e2e/boolean_algebra.rs              |   2 +-
 .../indexed_table/tests_e2e/fuzz/harness.rs   |  20 +-
 .../indexed_table/tests_e2e/fuzz/oracle.rs    |   4 +-
 .../indexed_table/tests_e2e/fuzz/tree_gen.rs  |   6 +-
 .../rust/src/indexed_table/tests_e2e/mod.rs   |   6 +-
 .../indexed_table/tests_e2e/multi_segment.rs  |   8 +-
 .../indexed_table/tests_e2e/null_columns.rs   |   6 +-
 .../indexed_table/tests_e2e/page_pruning.rs   |   4 +-
 .../tests_e2e/streaming_at_scale.rs           |  32 +-
 .../rust/src/query_executor.rs                |   2 +-
 .../rust/src/session_context.rs               |  36 ++
 .../be/datafusion/CoordinatorReduceIT.java    | 163 -----
 .../CoordinatorReduceMemtableIT.java          | 135 ----
 .../StreamingCoordinatorReduceIT.java         | 145 -----
 .../DataFusionAnalyticsBackendPlugin.java     |  16 +
 .../DataFusionInstructionHandlerFactory.java  |   9 +
 .../ShardScanWithDelegationHandler.java       |  73 +++
 .../indexfilter/FilterProviderRegistry.java   |  28 +
 .../indexfilter/FilterTreeCallbacks.java      |  98 ++-
 .../be/datafusion/nativelib/NativeBridge.java |  50 +-
 ...DelegationForIndexFullConversionTests.java |   6 +
 .../indexfilter/IndexFilterCallbackTests.java | 222 +++----
 .../analytics-backend-lucene/build.gradle     |   7 +-
 .../opensearch/be/lucene/ConversionUtils.java |  80 +++
 .../lucene/LuceneAnalyticsBackendPlugin.java  |  94 ++-
 .../lucene/LuceneFilterDelegationHandle.java  | 201 ++++++
 .../be/lucene/QuerySerializerRegistry.java    |  48 ++
 ...analytics.spi.AnalyticsSearchBackendPlugin |   1 +
 .../LuceneAnalyticsBackendPluginTests.java    |   6 +
 .../opensearch/analytics/AnalyticsPlugin.java |   2 +-
 .../exec/AnalyticsSearchService.java          |  41 +-
 .../exec/action/FragmentExecutionRequest.java |  23 +
 .../stage/ShardFragmentStageScheduler.java    |  30 +-
 .../planner/dag/FragmentConversionDriver.java |   7 +-
 .../PlanAlternativeSerializationTests.java    |  66 +-
 .../analytics/planner/MockBackend.java        |   6 +
 .../dag/FragmentConversionDriverTests.java    |  14 +-
 sandbox/plugins/composite-engine/build.gradle |   1 +
 sandbox/qa/analytics-engine-rest/build.gradle |  83 ++-
 .../analytics/qa/CoordinatorReduceIT.java     | 128 ++++
 .../qa/CoordinatorReduceMemtableIT.java       | 111 ++++
 .../analytics/qa/FilterDelegationIT.java      | 104 ++++
 .../qa/StreamingCoordinatorReduceIT.java      | 118 ++++
 58 files changed, 2194 insertions(+), 1207 deletions(-)
 create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegationDescriptor.java
 create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationHandle.java
 create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ShardScanWithDelegationInstructionNode.java
 delete mode 100644 sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceIT.java
 delete mode 100644 sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceMemtableIT.java
 delete mode 100644 sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/StreamingCoordinatorReduceIT.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanWithDelegationHandler.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/resources/META-INF/services/org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceMemtableIT.java
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationIT.java
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java

diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ShardScanExecutionContext.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ShardScanExecutionContext.java
index fb0df3f1301d3..8e6a2fc7dfbe6 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ShardScanExecutionContext.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/backend/ShardScanExecutionContext.java
@@ -10,7 +10,10 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.opensearch.analytics.spi.CommonExecutionContext;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.index.IndexSettings;
 import org.opensearch.index.engine.exec.IndexReaderProvider.Reader;
+import org.opensearch.index.mapper.MapperService;
 import org.opensearch.tasks.Task;
 
 /**
@@ -26,6 +29,9 @@ public class ShardScanExecutionContext implements CommonExecutionContext {
     private final Task task;
     private byte[] fragmentBytes;
     private BufferAllocator allocator;
+    private MapperService mapperService;
+    private IndexSettings indexSettings;
+    private NamedWriteableRegistry namedWriteableRegistry;
 
     /**
      * Constructs an execution context.
@@ -73,4 +79,34 @@ public BufferAllocator getAllocator() {
     public void setAllocator(BufferAllocator allocator) {
         this.allocator = allocator;
     }
+
+    /** Returns the shard's mapper service for field type resolution. */
+    public MapperService getMapperService() {
+        return mapperService;
+    }
+
+    /** Sets the shard's mapper service. */
+    public void setMapperService(MapperService mapperService) {
+        this.mapperService = mapperService;
+    }
+
+    /** Returns the shard's index settings. */
+    public IndexSettings getIndexSettings() {
+        return indexSettings;
+    }
+
+    /** Sets the shard's index settings. */
+    public void setIndexSettings(IndexSettings indexSettings) {
+        this.indexSettings = indexSettings;
+    }
+
+    /** Returns the NamedWriteableRegistry for deserializing delegated expressions. */
+    public NamedWriteableRegistry getNamedWriteableRegistry() {
+        return namedWriteableRegistry;
+    }
+
+    /** Sets the NamedWriteableRegistry. */
+    public void setNamedWriteableRegistry(NamedWriteableRegistry namedWriteableRegistry) {
+        this.namedWriteableRegistry = namedWriteableRegistry;
+    }
 }
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java
index e4722784197f6..37ae28cf0e168 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.analytics.spi;
 
+import java.util.List;
+
 /**
  * SPI extension point for backend query engine plugins.
  *
@@ -81,4 +83,34 @@ default ExchangeSinkProvider getExchangeSinkProvider() {
     default FragmentInstructionHandlerFactory getInstructionHandlerFactory() {
         throw new UnsupportedOperationException("getInstructionHandlerFactory not implemented for [" + name() + "]");
     }
+
+    /**
+     * Prepare a filter delegation handle for the given delegated expressions.
+     * Called by Core after all instruction handlers have run, when the plan has delegation.
+     *
+     * 
      The accepting backend initializes its internal state (e.g., DirectoryReader,
+     * QueryShardContext, compiled Queries) and returns a handle that the driving backend
+     * will call into during execution.
+     *
+     * @param expressions the delegated expressions (annotationId + serialized query bytes)
+     * @param ctx the shared execution context (Reader, MapperService, IndexSettings)
+     * @return a handle the driving backend calls into via FFM upcalls
+     */
+    default FilterDelegationHandle getFilterDelegationHandle(List expressions, CommonExecutionContext ctx) {
+        throw new UnsupportedOperationException("getFilterDelegationHandle not implemented for [" + name() + "]");
+    }
+
+    /**
+     * Configure the driving backend to use the given delegation handle during execution.
+     * Called by Core after obtaining the handle from the accepting backend.
+     *
+     * 
      The driving backend registers the handle so that FFM upcalls from Rust
+     * (createProvider, createCollector, collectDocs) route to it.
+     *
+     * @param handle the delegation handle from the accepting backend
+     * @param backendContext the driving backend's execution context (from instruction handlers)
+     */
+    default void configureFilterDelegation(FilterDelegationHandle handle, BackendExecutionContext backendContext) {
+        throw new UnsupportedOperationException("configureFilterDelegation not implemented for [" + name() + "]");
+    }
 }
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegationDescriptor.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegationDescriptor.java
new file mode 100644
index 0000000000000..86c641517edd8
--- /dev/null
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/DelegationDescriptor.java
@@ -0,0 +1,52 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.spi;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Describes the delegation metadata for a plan alternative. Carried on the wire
+ * alongside the instruction list so that Core can orchestrate the handle exchange
+ * between accepting and driving backends at the data node.
+ *
+ * @opensearch.internal
+ */
+public record DelegationDescriptor(FilterTreeShape treeShape, int delegatedPredicateCount, List delegatedExpressions)
+    implements
+        Writeable {
+
+    public DelegationDescriptor(StreamInput in) throws IOException {
+        this(in.readEnum(FilterTreeShape.class), in.readVInt(), readExpressions(in));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeEnum(treeShape);
+        out.writeVInt(delegatedPredicateCount);
+        out.writeVInt(delegatedExpressions.size());
+        for (DelegatedExpression expr : delegatedExpressions) {
+            expr.writeTo(out);
+        }
+    }
+
+    private static List readExpressions(StreamInput in) throws IOException {
+        int count = in.readVInt();
+        List expressions = new ArrayList<>(count);
+        for (int i = 0; i < count; i++) {
+            expressions.add(new DelegatedExpression(in));
+        }
+        return expressions;
+    }
+}
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationHandle.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationHandle.java
new file mode 100644
index 0000000000000..6f7f914a36e1a
--- /dev/null
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationHandle.java
@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.spi;
+
+import java.io.Closeable;
+import java.lang.foreign.MemorySegment;
+
+/**
+ * Callback surface for filter delegation between a driving backend and an accepting backend.
+ *
+ * 
      One handle per query per shard. The accepting backend implements this interface;
+ * the driving backend calls into it via FFM upcalls during execution. Core closes it
+ * after execution completes.
+ *
+ * 
      Lifecycle:
+ * 
        
+ *   
      1. Rust calls {@link #createProvider(int)} once per delegated predicate (per annotationId)
        2. 
+ *   
      2. Rust calls {@link #createCollector(int, int, int, int)} per (provider × segment)
        3. 
+ *   
      3. Rust calls {@link #collectDocs(int, int, int, MemorySegment)} per row group
        4. 
+ *   
      4. Rust calls {@link #releaseCollector(int)} when done with a segment
        5. 
+ *   
      5. Rust calls {@link #releaseProvider(int)} when the query ends
        6. 
+ * 
      
+ *
+ * @opensearch.internal
+ */
+public interface FilterDelegationHandle extends Closeable {
+
+    /**
+     * Create a provider for the given annotation ID. The accepting backend looks up
+     * the pre-compiled query for this annotation and prepares it for segment iteration.
+     *
+     * @param annotationId the annotation ID identifying the delegated predicate
+     * @return a provider key {@code >= 0}, or {@code -1} on failure
+     */
+    int createProvider(int annotationId);
+
+    /**
+     * Create a collector for one (segment, [minDoc, maxDoc)) range.
+     *
+     * @param providerKey key returned by {@link #createProvider(int)}
+     * @param segmentOrd the segment ordinal
+     * @param minDoc inclusive lower bound
+     * @param maxDoc exclusive upper bound
+     * @return a collector key {@code >= 0}, or {@code -1} on failure
+     */
+    int createCollector(int providerKey, int segmentOrd, int minDoc, int maxDoc);
+
+    /**
+     * Fill {@code out} with the matching doc-id bitset for the given collector.
+     *
+     * 
      Bit layout: word {@code i} contains matches for docs
+     * {@code [minDoc + i*64, minDoc + (i+1)*64)}, LSB-first within each word.
+     *
+     * @param collectorKey key returned by {@link #createCollector(int, int, int, int)}
+     * @param minDoc inclusive lower bound
+     * @param maxDoc exclusive upper bound
+     * @param out destination buffer; implementation writes up to {@code out.byteSize() / 8} words
+     * @return number of words written, or {@code -1} on error
+     */
+    int collectDocs(int collectorKey, int minDoc, int maxDoc, MemorySegment out);
+
+    /**
+     * Release resources for a collector.
+     */
+    void releaseCollector(int collectorKey);
+
+    /**
+     * Release resources for a provider.
+     */
+    void releaseProvider(int providerKey);
+}
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationInstructionNode.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationInstructionNode.java
index d56a5c5bed775..11a947d86ca13 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationInstructionNode.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FilterDelegationInstructionNode.java
@@ -44,7 +44,7 @@ public FilterDelegationInstructionNode(StreamInput in) throws IOException {
 
     @Override
     public InstructionType type() {
-        return InstructionType.SETUP_FILTER_DELEGATION_FOR_INDEX;
+        return InstructionType.SETUP_SHARD_SCAN_WITH_DELEGATION;
     }
 
     @Override
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandlerFactory.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandlerFactory.java
index 6e62fa10f012e..f40d7472c2d4d 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandlerFactory.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FragmentInstructionHandlerFactory.java
@@ -35,6 +35,9 @@ Optional createFilterDelegationNode(
         List delegatedQueries
     );
 
+    /** Creates a shard scan with delegation instruction node — combines scan setup with delegation config. */
+    Optional createShardScanWithDelegationNode(FilterTreeShape treeShape, int delegatedPredicateCount);
+
     /** Creates a partial aggregate instruction node. */
     Optional createPartialAggregateNode();
 
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionType.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionType.java
index 490f60a967707..d426e3c8c7c0c 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionType.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/InstructionType.java
@@ -23,14 +23,14 @@ public enum InstructionType {
     /** Base scan setup — reader acquisition, SessionContext creation, default table provider. */
     SETUP_SHARD_SCAN,
     /**
-     * Filter delegation to an index backend — bridge setup, UDF registration, IndexedTableProvider.
+     * Filter delegation to an index backend — bridge setup, UDF registration, custom scan operator.
      *
      * 
      TODO: add a DelegationStrategy field (BACKEND_DRIVEN vs CENTRALLY_DRIVEN) to the
      * instruction node when centrally-driven delegation is implemented. Currently only
      * BACKEND_DRIVEN exists — derived from the backend declaring
      * {@code supportedDelegations(DelegationType.FILTER)}.
      */
-    SETUP_FILTER_DELEGATION_FOR_INDEX,
+    SETUP_SHARD_SCAN_WITH_DELEGATION,
     /** Partial aggregate mode — disable combine optimizer, cut plan to partial-only. */
     SETUP_PARTIAL_AGGREGATE,
     /** Final aggregate for coordinator reduce — ExchangeSink path, final-only agg. */
@@ -40,7 +40,7 @@ public enum InstructionType {
     public InstructionNode readNode(StreamInput in) throws IOException {
         return switch (this) {
             case SETUP_SHARD_SCAN -> new ShardScanInstructionNode(in);
-            case SETUP_FILTER_DELEGATION_FOR_INDEX -> new FilterDelegationInstructionNode(in);
+            case SETUP_SHARD_SCAN_WITH_DELEGATION -> new ShardScanWithDelegationInstructionNode(in);
             case SETUP_PARTIAL_AGGREGATE -> new PartialAggregateInstructionNode(in);
             case SETUP_FINAL_AGGREGATE -> new FinalAggregateInstructionNode(in);
         };
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ShardScanWithDelegationInstructionNode.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ShardScanWithDelegationInstructionNode.java
new file mode 100644
index 0000000000000..18af354e02355
--- /dev/null
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ShardScanWithDelegationInstructionNode.java
@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.spi;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+/**
+ * Instruction node for shard scan with filter delegation — extends base shard scan
+ * with {@link FilterTreeShape} and delegated predicate count so the driving backend
+ * can configure its indexed execution path (UDF registration, IndexedTableProvider)
+ * in a single FFM call.
+ *
+ * @opensearch.internal
+ */
+public class ShardScanWithDelegationInstructionNode extends ShardScanInstructionNode {
+
+    private final FilterTreeShape treeShape;
+    private final int delegatedPredicateCount;
+
+    public ShardScanWithDelegationInstructionNode(FilterTreeShape treeShape, int delegatedPredicateCount) {
+        this.treeShape = treeShape;
+        this.delegatedPredicateCount = delegatedPredicateCount;
+    }
+
+    public ShardScanWithDelegationInstructionNode(StreamInput in) throws IOException {
+        super(in);
+        this.treeShape = in.readEnum(FilterTreeShape.class);
+        this.delegatedPredicateCount = in.readVInt();
+    }
+
+    @Override
+    public InstructionType type() {
+        return InstructionType.SETUP_SHARD_SCAN_WITH_DELEGATION;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeEnum(treeShape);
+        out.writeVInt(delegatedPredicateCount);
+    }
+
+    public FilterTreeShape getTreeShape() {
+        return treeShape;
+    }
+
+    public int getDelegatedPredicateCount() {
+        return delegatedPredicateCount;
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/build.gradle b/sandbox/plugins/analytics-backend-datafusion/build.gradle
index e2504d6f49c55..d382603186f40 100644
--- a/sandbox/plugins/analytics-backend-datafusion/build.gradle
+++ b/sandbox/plugins/analytics-backend-datafusion/build.gradle
@@ -6,12 +6,6 @@
  * compatible open source license.
  */
 
-apply plugin: 'opensearch.internal-cluster-test'
-
-// SQL Unified Query API version (aligned with OpenSearch build version) — required by
-// the PPL transport plugin used in CoordinatorReduceIT.
-def sqlUnifiedQueryVersion = '3.6.0.0-SNAPSHOT'
-
 opensearchplugin {
   description = 'DataFusion native execution engine plugin for the query engine.'
   classname = 'org.opensearch.be.datafusion.DataFusionPlugin'
@@ -85,25 +79,6 @@ dependencies {
   testImplementation project(':sandbox:plugins:analytics-backend-lucene')
   testCompileOnly 'org.immutables:value-annotations:2.8.8'
 
-  // ── internalClusterTest: end-to-end coordinator-reduce IT ───────────────────
-  // Pulls in every sibling plugin needed to construct a parquet-backed composite
-  // index, dispatch a multi-shard PPL aggregate, and exercise DatafusionReduceSink.
-  internalClusterTestImplementation project(':sandbox:plugins:analytics-engine')
-  internalClusterTestImplementation project(':sandbox:plugins:parquet-data-format')
-  internalClusterTestImplementation project(':sandbox:plugins:composite-engine')
-  internalClusterTestImplementation project(':sandbox:plugins:analytics-backend-lucene')
-  internalClusterTestImplementation project(':plugins:arrow-flight-rpc')
-  internalClusterTestImplementation("org.opensearch.query:unified-query-api:${sqlUnifiedQueryVersion}") {
-    exclude group: 'org.opensearch'
-  }
-  internalClusterTestImplementation("org.opensearch.query:unified-query-core:${sqlUnifiedQueryVersion}") {
-    exclude group: 'org.opensearch'
-  }
-  internalClusterTestImplementation("org.opensearch.query:unified-query-ppl:${sqlUnifiedQueryVersion}") {
-    exclude group: 'org.opensearch'
-  }
-  // PPL front-end plugin — provides UnifiedPPLExecuteAction transport action used by the IT.
-  internalClusterTestImplementation project(':sandbox:plugins:test-ppl-frontend')
 }
 
 test {
@@ -169,30 +144,10 @@ task cargoTest(type: Exec) {
 
 check.dependsOn cargoTest
 
-internalClusterTest {
-  jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED'
-  jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED'
-  jvmArgs '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED'
-  jvmArgs '--enable-native-access=ALL-UNNAMED'
-  jvmArgs '-Darrow.memory.debug.allocator=false'
-  jvmArgs += ["--add-opens", "java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED"]
-  systemProperty 'io.netty.allocator.numDirectArenas', '1'
-  systemProperty 'io.netty.noUnsafe', 'false'
-  systemProperty 'io.netty.tryUnsafe', 'true'
-  systemProperty 'io.netty.tryReflectionSetAccessible', 'true'
-  systemProperty 'native.lib.path', project(':sandbox:libs:dataformat-native').ext.nativeLibPath.absolutePath
-  dependsOn ':sandbox:libs:dataformat-native:buildRustLibrary'
-}
-
 configurations.all {
-  // okhttp-aws-signer is a transitive dep of unified-query-common (via unified-query-core),
-  // only published on JitPack, not needed for PPL parsing/planning
   exclude group: 'com.github.babbel', module: 'okhttp-aws-signer'
 
   resolutionStrategy {
-    // Align transitive versions with OpenSearch's managed versions — required because the
-    // unified-query-* artifacts pull in older versions of common libs that conflict with
-    // OpenSearch's enforced versions on internalClusterTest classpath.
     force 'com.google.guava:guava:33.4.0-jre'
     force 'com.google.guava:failureaccess:1.0.2'
     force 'com.google.errorprone:error_prone_annotations:2.36.0'
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs
index 86856ed0c5f08..329bd810d3bea 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs
@@ -69,6 +69,10 @@ pub struct QueryStreamHandle {
     /// Held for its `Drop` impl — marks the query completed when the
     /// stream is closed.
     _query_tracking_context: QueryTrackingContext,
+    /// Keeps the SessionContext alive while the stream is being consumed.
+    /// The physical plan may reference state (e.g. RuntimeEnv, caches) owned
+    /// by the session; dropping it prematurely causes use-after-free.
+    _session_ctx: Option,
 }
 
 impl QueryStreamHandle {
@@ -79,6 +83,19 @@ impl QueryStreamHandle {
         Self {
             stream,
             _query_tracking_context: query_context,
+            _session_ctx: None,
+        }
+    }
+
+    pub fn with_session_context(
+        stream: RecordBatchStreamAdapter,
+        query_context: QueryTrackingContext,
+        ctx: datafusion::prelude::SessionContext,
+    ) -> Self {
+        Self {
+            stream,
+            _query_tracking_context: query_context,
+            _session_ctx: Some(ctx),
         }
     }
 }
@@ -632,10 +649,15 @@ pub unsafe fn sender_send(
 
     // `from_ffi` takes the array by value (consumes it) and the schema by
     // reference (it is still dropped when `ffi_schema` goes out of scope).
-    let array_data = arrow_array::ffi::from_ffi(ffi_array, &ffi_schema).map_err(|e| {
+    let mut array_data = arrow_array::ffi::from_ffi(ffi_array, &ffi_schema).map_err(|e| {
         DataFusionError::Execution(format!("Failed to import Arrow C Data array: {}", e))
     })?;
 
+    // Buffers from Java's Flight RPC deserialization may not meet Rust's
+    // native alignment requirements. align_buffers() is a no-op for
+    // already-aligned buffers; only misaligned ones are reallocated.
+    array_data.align_buffers();
+
     let struct_array = StructArray::from(array_data);
     let batch = RecordBatch::from(struct_array);
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
index 1c626da1519f6..0e3f034279f0b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
@@ -476,6 +476,27 @@ pub unsafe extern "C" fn df_create_session_context(
         .map_err(|e| e.to_string())
 }
 
+#[ffm_safe]
+#[no_mangle]
+pub unsafe extern "C" fn df_create_session_context_indexed(
+    shard_view_ptr: i64,
+    runtime_ptr: i64,
+    table_name_ptr: *const u8,
+    table_name_len: i64,
+    context_id: i64,
+    tree_shape: i32,
+    delegated_predicate_count: i32,
+) -> i64 {
+    let table_name = str_from_raw(table_name_ptr, table_name_len)
+        .map_err(|e| format!("df_create_session_context_indexed: {}", e))?;
+    let mgr = get_rt_manager()?;
+    mgr.io_runtime
+        .block_on(crate::session_context::create_session_context_indexed(
+            runtime_ptr, shard_view_ptr, table_name, context_id, tree_shape, delegated_predicate_count,
+        ))
+        .map_err(|e| e.to_string())
+}
+
 #[ffm_safe]
 #[no_mangle]
 pub unsafe extern "C" fn df_cache_manager_remove_files(
@@ -603,26 +624,30 @@ pub unsafe extern "C" fn df_execute_with_context(
     plan_ptr: *const u8,
     plan_len: i64,
 ) -> i64 {
-    // Consume the session context handle on entry. Ownership transfers here
-    // regardless of whether the remainder of this function succeeds, returns an
-    // error via `?`, or panics — RAII (or `catch_unwind` drop-during-unwind)
-    // drops `session_handle` and frees the underlying SessionContext resources.
-    //
-    // This matches the Java-side contract: SessionContextHandle.markConsumed() is
-    // invoked in a `finally` after the FFM downcall, so every observable path from
-    // Java's perspective ("call.invoke ran") maps to "Rust consumed the handle".
-    // If we were to run fallible or panic-prone code (e.g. `get_rt_manager()?`)
-    // before Box::from_raw, the handle would leak on those paths.
     let session_handle = *Box::from_raw(session_ctx_ptr as *mut crate::session_context::SessionContextHandle);
 
     let mgr = get_rt_manager()?;
     let plan_bytes = slice::from_raw_parts(plan_ptr, plan_len as usize);
     let cpu_executor = mgr.cpu_executor();
-    mgr.io_runtime
-        .block_on(crate::query_executor::execute_with_context(
-            session_handle,
-            plan_bytes,
-            cpu_executor,
-        ))
-        .map_err(|e| e.to_string())
+    // Route based on whether the session was configured for indexed execution
+    if session_handle.indexed_config.is_some() {
+        // TODO: refactor execute_indexed_with_context to take SessionContextHandle directly
+        // (like execute_with_context) instead of i64 raw pointer — avoids this re-boxing.
+        let ptr = Box::into_raw(Box::new(session_handle)) as i64;
+        mgr.io_runtime
+            .block_on(crate::indexed_executor::execute_indexed_with_context(
+                ptr,
+                plan_bytes.to_vec(),
+                cpu_executor,
+            ))
+            .map_err(|e| e.to_string())
+    } else {
+        mgr.io_runtime
+            .block_on(crate::query_executor::execute_with_context(
+                session_handle,
+                plan_bytes,
+                cpu_executor,
+            ))
+            .map_err(|e| e.to_string())
+    }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
index f22735cb06f0c..6ab951ed67d6d 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
@@ -11,7 +11,7 @@
 //!
 //! Per-leaf lifecycle at query time (one compiled-query + per-segment matcher
 //! per Collector leaf):
-//!   1. `createProvider(query_bytes)` FFM upcall → `provider_key`  (once per
+//!   1. `createProvider(annotation_id)` FFM upcall → `provider_key`  (once per
 //!      Collector leaf, once per query).
 //!   2. `createCollector(provider_key, seg, min, max)` FFM upcall → collector
 //!      (once per SegmentChunk × Collector leaf).
@@ -81,6 +81,10 @@ use crate::indexed_table::page_pruner::{build_pruning_predicate, PagePruneMetric
 /// was built from a catalog snapshot). `num_partitions` comes from the caller's
 /// session config. `query_memory_pool` is the per-query tracker (same as
 /// vanilla path) — `None` disables tracking and uses the global pool.
+// TODO: remove this function once all callers migrate to the instruction-based path
+// TODO: remove once api.rs migrates to instruction-based path directly.
+// Kept as thin wrapper to make existing tests exercise execute_indexed_with_context
+// with minimal changes.
 pub async fn execute_indexed_query(
     substrait_bytes: Vec,
     table_name: String,
@@ -135,18 +139,300 @@ pub async fn execute_indexed_query(
     ctx.register_udf(create_index_filter_udf());
     crate::udf::register_all(&ctx);
 
-    // Resolve the object store for this shard's table URL (file://, s3://,
-    // gs://, ... whatever the global runtime has registered). We pass this
-    // store down to the parquet bridge so per-RG reads go through it instead
-    // of hitting the local filesystem directly.
+    // Register default ListingTable so substrait consumer can resolve the table
+    let listing_options = datafusion::datasource::listing::ListingOptions::new(
+        Arc::new(datafusion::datasource::file_format::parquet::ParquetFormat::new()))
+        .with_file_extension(".parquet")
+        .with_collect_stat(true);
+    let resolved_schema = listing_options
+        .infer_schema(&ctx.state(), &shard_view.table_path)
+        .await?;
+    let table_config = datafusion::datasource::listing::ListingTableConfig::new(shard_view.table_path.clone())
+        .with_listing_options(listing_options)
+        .with_schema(resolved_schema);
+    let provider = Arc::new(datafusion::datasource::listing::ListingTable::try_new(table_config)?);
+    ctx.register_table(&table_name, provider)?;
+
+    // Build SessionContextHandle and delegate to execute_indexed_with_context
+    let handle = crate::session_context::SessionContextHandle {
+        ctx,
+        table_path: shard_view.table_path.clone(),
+        object_metas: shard_view.object_metas.clone(),
+        query_context: crate::query_tracker::QueryTrackingContext::new(0, runtime.runtime_env.memory_pool.clone()),
+        table_name: table_name.clone(),
+        indexed_config: None, // derive classification from tree
+    };
+    let ptr = Box::into_raw(Box::new(handle)) as i64;
+    unsafe { execute_indexed_with_context(ptr, substrait_bytes, cpu_executor).await }
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────
+
+/// Collect all `Predicate(expr)` leaves in DFS order. Used by the
+/// dispatcher to build a per-leaf `PruningPredicate` cache keyed by
+/// `Arc::as_ptr` identity.
+fn collect_predicate_exprs(tree: &BoolNode, out: &mut Vec>) {
+    match tree {
+        BoolNode::And(c) | BoolNode::Or(c) => {
+            c.iter().for_each(|ch| collect_predicate_exprs(ch, out))
+        }
+        BoolNode::Not(inner) => collect_predicate_exprs(inner, out),
+        BoolNode::Collector { .. } => {}
+        BoolNode::Predicate(expr) => out.push(Arc::clone(expr)),
+    }
+}
+
+fn collect_predicate_column_indices(extraction: Option<&ExtractionResult>) -> Vec {
+    let Some(e) = extraction else { return vec![] };
+    let mut exprs = Vec::new();
+    collect_predicate_exprs(&e.tree, &mut exprs);
+    let mut indices = BTreeSet::new();
+    for expr in &exprs {
+        let _ = expr.apply(|node| {
+            if let Some(col) = node.as_any().downcast_ref::() {
+                indices.insert(col.index());
+            }
+            Ok(TreeNodeRecursion::Continue)
+        });
+    }
+    indices.into_iter().collect()
+}
+/// For a tree classified as `SingleCollector`, walk it to find the single
+/// Collector leaf and return its query bytes.
+fn single_collector_id(tree: &BoolNode) -> Option {
+    match tree {
+        BoolNode::Collector { annotation_id } => Some(*annotation_id),
+        BoolNode::And(children) => {
+            for child in children {
+                if let Some(id) = single_collector_id(child) {
+                    return Some(id);
+                }
+            }
+            None
+        }
+        _ => None,
+    }
+}
+
+/// For a tree classified as `SingleCollector`, return the residual
+/// (all non-Collector parts of the AND tree, re-assembled into a
+/// single BoolNode). Recursively strips Collector leaves from nested
+/// ANDs. Returns `None` if the tree is a bare Collector or the entire
+/// tree is collectors-only (no residual predicates).
+fn extract_single_collector_residual(tree: &BoolNode) -> Option {
+    fn strip_collectors(node: &BoolNode) -> Option {
+        match node {
+            BoolNode::Collector { .. } => None,
+            BoolNode::Predicate(_) => Some(node.clone()),
+            BoolNode::And(children) => {
+                let residuals: Vec =
+                    children.iter().filter_map(strip_collectors).collect();
+                match residuals.len() {
+                    0 => None,
+                    1 => Some(residuals.into_iter().next().unwrap()),
+                    _ => Some(BoolNode::And(residuals)),
+                }
+            }
+            // OR/NOT with no collectors pass through unchanged (they're
+            // pure-predicate subtrees in a SingleCollector-classified tree).
+            other => Some(other.clone()),
+        }
+    }
+    strip_collectors(tree)
+}
+
+// ── Placeholder provider used only for substrait consume pass ─────────
+
+struct PlaceholderProvider {
+    schema: SchemaRef,
+}
+
+impl fmt::Debug for PlaceholderProvider {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("PlaceholderProvider").finish()
+    }
+}
+
+#[async_trait::async_trait]
+impl TableProvider for PlaceholderProvider {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+    fn table_type(&self) -> TableType {
+        TableType::Base
+    }
+    async fn scan(
+        &self,
+        _state: &dyn Session,
+        _projection: Option<&Vec>,
+        _filters: &[Expr],
+        _limit: Option,
+    ) -> Result, DataFusionError> {
+        Err(DataFusionError::Internal(
+            "PlaceholderProvider should not be scanned".into(),
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::indexed_table::bool_tree::BoolNode;
+    use datafusion::arrow::datatypes::{DataType, Field, Schema};
+    use datafusion::common::ScalarValue;
+    use datafusion::logical_expr::Operator;
+    use datafusion::physical_expr::expressions::{BinaryExpr, Column as PhysColumn, Literal};
+    use datafusion::physical_expr::PhysicalExpr;
+    use std::sync::Arc;
+
+    fn collector(id: i32) -> BoolNode {
+        BoolNode::Collector {
+            annotation_id: id,
+        }
+    }
+
+    fn pred() -> BoolNode {
+        let left: Arc = Arc::new(PhysColumn::new("price", 0));
+        let right: Arc = Arc::new(Literal::new(ScalarValue::Int32(Some(0))));
+        BoolNode::Predicate(Arc::new(BinaryExpr::new(left, Operator::Eq, right)))
+    }
+
+    fn is_predicate(node: &BoolNode) -> bool {
+        matches!(node, BoolNode::Predicate(_))
+    }
+
+    // ── extract_single_collector_residual ─────────────────────────────
+
+    #[test]
+    fn residual_bare_collector_is_none() {
+        assert!(extract_single_collector_residual(&collector(10)).is_none());
+    }
+
+    #[test]
+    fn residual_and_collector_plus_predicate() {
+        let tree = BoolNode::And(vec![collector(10), pred()]);
+        let r = extract_single_collector_residual(&tree).unwrap();
+        assert!(is_predicate(&r));
+    }
+
+    #[test]
+    fn residual_and_only_collectors_is_none() {
+        let tree = BoolNode::And(vec![collector(10), collector(11)]);
+        assert!(extract_single_collector_residual(&tree).is_none());
+    }
+
+    #[test]
+    fn residual_nested_and_strips_collectors() {
+        // AND(C₁, AND(C₂, P)) → residual is P
+        let tree = BoolNode::And(vec![
+            collector(10),
+            BoolNode::And(vec![collector(11), pred()]),
+        ]);
+        let r = extract_single_collector_residual(&tree).unwrap();
+        assert!(is_predicate(&r));
+    }
+
+    #[test]
+    fn residual_deeply_nested_and() {
+        // AND(P₁, AND(C₁, AND(C₂, P₂))) → AND(P₁, P₂)
+        let p1 = pred();
+        let p2 = pred();
+        let tree = BoolNode::And(vec![
+            p1,
+            BoolNode::And(vec![
+                collector(0),
+                BoolNode::And(vec![collector(1), p2]),
+            ]),
+        ]);
+        let r = extract_single_collector_residual(&tree).unwrap();
+        match r {
+            BoolNode::And(children) => {
+                assert_eq!(children.len(), 2);
+                assert!(children.iter().all(is_predicate));
+            }
+            _ => panic!("expected AND, got {:?}", r),
+        }
+    }
+
+    #[test]
+    fn residual_nested_and_with_or_predicate() {
+        // AND(C, AND(P, OR(P, P))) → AND(P, OR(P, P))
+        let tree = BoolNode::And(vec![
+            collector(10),
+            BoolNode::And(vec![
+                pred(),
+                BoolNode::Or(vec![pred(), pred()]),
+            ]),
+        ]);
+        let r = extract_single_collector_residual(&tree).unwrap();
+        match r {
+            BoolNode::And(children) => {
+                assert_eq!(children.len(), 2);
+                assert!(is_predicate(&children[0]));
+                assert!(matches!(children[1], BoolNode::Or(_)));
+            }
+            _ => panic!("expected AND, got {:?}", r),
+        }
+    }
+
+    #[test]
+    fn residual_nested_and_all_collectors_is_none() {
+        // AND(AND(C₁, C₂), AND(C₃, C₄)) → no residual
+        let tree = BoolNode::And(vec![
+            BoolNode::And(vec![collector(0), collector(1)]),
+            BoolNode::And(vec![collector(2), collector(3)]),
+        ]);
+        assert!(extract_single_collector_residual(&tree).is_none());
+    }
+}
+
+/// Instruction-based indexed execution path. Consumes a pre-configured SessionContextHandle
+/// (with UDF registered and IndexedExecutionConfig set) and routes to the appropriate
+/// evaluator based on the Java-provided FilterTreeShape.
+///
+/// TODO: extract shared logic with `execute_indexed_query` to avoid duplication.
+/// For now this delegates to the existing function by reconstructing the needed args
+/// from the handle.
+pub async unsafe fn execute_indexed_with_context(
+    session_ctx_ptr: i64,
+    substrait_bytes: Vec,
+    cpu_executor: DedicatedExecutor,
+) -> Result {
+    let handle = *Box::from_raw(session_ctx_ptr as *mut crate::session_context::SessionContextHandle);
+    let classification_override = handle.indexed_config.map(|config| {
+        match (config.tree_shape, config.delegated_predicate_count) {
+            (1, 1) => FilterClass::SingleCollector,
+            (1, _) | (2, _) => FilterClass::Tree,
+            _ => FilterClass::None,
+        }
+    });
+
+    let query_config = Arc::new(crate::datafusion_query_config::DatafusionQueryConfig::default());
+    let num_partitions = query_config.target_partitions.max(1);
+    let ctx = handle.ctx;
+    let table_name = handle.table_name;
+    let table_path = handle.table_path;
+    let object_metas = handle.object_metas;
+    let query_context = handle.query_context;
+
+    // SessionContext already has RuntimeEnv, caches, memory pool, UDF from create_session_context_indexed.
+    // Deregister the default ListingTable (registered by create_session_context) — will be replaced
+    // with IndexedTableProvider after plan decoding.
+    ctx.deregister_table(&table_name)?;
+
     let store = ctx
         .state()
         .runtime_env()
-        .object_store(&shard_view.table_path)?;
+        .object_store(&table_path)?;
 
-    let (segments, schema) = build_segments(Arc::clone(&store), shard_view.object_metas.as_ref())
+    let (segments, schema) = build_segments(Arc::clone(&store), object_metas.as_ref())
         .await
         .map_err(DataFusionError::Execution)?;
+    for (i, seg) in segments.iter().enumerate() {
+    }
 
     let placeholder: Arc = Arc::new(PlaceholderProvider {
         schema: schema.clone(),
@@ -165,14 +451,19 @@ pub async fn execute_indexed_query(
                 .map_err(|e| DataFusionError::Execution(format!("expr_to_bool_tree: {}", e)))?,
         ),
     };
-    let classification = match &extraction {
-        None => FilterClass::None,
-        Some(e) => classify_filter(&e.tree),
+
+    // Resolve classification: from Java config if available, otherwise derive from tree
+    let classification = match classification_override {
+        Some(c) => c,
+        None => match &extraction {
+            None => FilterClass::None,
+            Some(e) => classify_filter(&e.tree),
+        },
     };
 
     // Derive the parquet pushdown predicate from the BoolNode tree.
     // `scan()` ignores DataFusion's filters argument (which contains
-    // the `index_filter` UDF marker whose body panics) and uses this
+    // the `delegated_predicate` UDF marker whose body panics) and uses this
     // field instead.
     //
     // SingleCollector: residual (non-Collector top-AND children) →
@@ -208,13 +499,13 @@ pub async fn execute_indexed_query(
                     "classify_filter returned SingleCollector but extraction is None".into(),
                 )
             })?;
-            let bytes = single_collector_bytes(&extraction.tree).ok_or_else(|| {
+            let annotation_id = single_collector_id(&extraction.tree).ok_or_else(|| {
                 DataFusionError::Internal(
                     "SingleCollector classified but leaf extraction failed".into(),
                 )
             })?;
             let provider =
-                Arc::new(create_provider(&bytes).map_err(|e| DataFusionError::External(e.into()))?);
+                Arc::new(create_provider(annotation_id).map_err(|e| DataFusionError::External(e.into()))?);
             let schema_for_pruner = schema.clone();
 
             // Extract the residual (non-Collector children of top-level
@@ -244,16 +535,16 @@ pub async fn execute_indexed_query(
                         chunk.doc_min,
                         chunk.doc_max,
                     )
-                    .map_err(|e| {
-                        format!(
-                        "FfmSegmentCollector::create(provider={}, seg={}, doc_range=[{},{})): {}",
-                        provider.key(),
-                        segment.segment_ord,
-                        chunk.doc_min,
-                        chunk.doc_max,
-                        e
-                    )
-                    })?;
+                        .map_err(|e| {
+                            format!(
+                                "FfmSegmentCollector::create(provider={}, seg={}, doc_range=[{},{})): {}",
+                                provider.key(),
+                                segment.segment_ord,
+                                chunk.doc_min,
+                                chunk.doc_max,
+                                e
+                            )
+                        })?;
                     let pruner = Arc::new(PagePruner::new(
                         &schema_for_pruner,
                         Arc::clone(&segment.metadata),
@@ -284,11 +575,11 @@ pub async fn execute_indexed_query(
             // get absorbed into the surrounding Or if applicable.
             let tree = extraction.tree.push_not_down().flatten();
             // One provider per Collector leaf (DFS order).
-            let leaf_bytes = tree.collector_leaves();
-            let mut providers: Vec> = Vec::with_capacity(leaf_bytes.len());
-            for bytes in &leaf_bytes {
+            let leaf_ids = tree.collector_leaves();
+            let mut providers: Vec> = Vec::with_capacity(leaf_ids.len());
+            for annotation_id in &leaf_ids {
                 providers.push(Arc::new(
-                    create_provider(bytes).map_err(|e| DataFusionError::External(e.into()))?,
+                    create_provider(*annotation_id).map_err(|e| DataFusionError::External(e.into()))?,
                 ));
             }
             let tree = Arc::new(tree);
@@ -329,7 +620,7 @@ pub async fn execute_indexed_query(
                             chunk.doc_min,
                             chunk.doc_max,
                         )
-                        .map_err(|e| format!("leaf {} collector: {}", idx, e))?;
+                            .map_err(|e| format!("leaf {} collector: {}", idx, e))?;
                         per_leaf.push((
                             provider.key(),
                             Arc::new(collector) as Arc,
@@ -373,7 +664,7 @@ pub async fn execute_indexed_query(
     // DataFusion's FileScanConfig. The full URL includes the path
     // (e.g. "file:///Users/.../parquet/"); ObjectStoreUrl wants only
     // the scheme+authority ("file:///").
-    let url_str = shard_view.table_path.as_str();
+    let url_str = table_path.as_str();
     let parsed = url::Url::parse(url_str)
         .map_err(|e| DataFusionError::Execution(format!("parse table_path URL: {}", e)))?;
     let store_url = ObjectStoreUrl::parse(format!("{}://{}", parsed.scheme(), parsed.authority()))?;
@@ -401,242 +692,6 @@ pub async fn execute_indexed_query(
     let cross_rt_stream = CrossRtStream::new_with_df_error_stream(df_stream, cpu_executor);
     let schema = cross_rt_stream.schema();
     let wrapped = RecordBatchStreamAdapter::new(schema, cross_rt_stream);
-    Ok(Box::into_raw(Box::new(wrapped)) as i64)
-}
-
-// ── Helpers ───────────────────────────────────────────────────────────
-
-/// Collect all `Predicate(expr)` leaves in DFS order. Used by the
-/// dispatcher to build a per-leaf `PruningPredicate` cache keyed by
-/// `Arc::as_ptr` identity.
-fn collect_predicate_exprs(tree: &BoolNode, out: &mut Vec>) {
-    match tree {
-        BoolNode::And(c) | BoolNode::Or(c) => {
-            c.iter().for_each(|ch| collect_predicate_exprs(ch, out))
-        }
-        BoolNode::Not(inner) => collect_predicate_exprs(inner, out),
-        BoolNode::Collector { .. } => {}
-        BoolNode::Predicate(expr) => out.push(Arc::clone(expr)),
-    }
-}
-
-fn collect_predicate_column_indices(extraction: Option<&ExtractionResult>) -> Vec {
-    let Some(e) = extraction else { return vec![] };
-    let mut exprs = Vec::new();
-    collect_predicate_exprs(&e.tree, &mut exprs);
-    let mut indices = BTreeSet::new();
-    for expr in &exprs {
-        let _ = expr.apply(|node| {
-            if let Some(col) = node.as_any().downcast_ref::() {
-                indices.insert(col.index());
-            }
-            Ok(TreeNodeRecursion::Continue)
-        });
-    }
-    indices.into_iter().collect()
-}
-/// For a tree classified as `SingleCollector`, walk it to find the single
-/// Collector leaf and return its query bytes.
-fn single_collector_bytes(tree: &BoolNode) -> Option> {
-    match tree {
-        BoolNode::Collector { query_bytes } => Some(Arc::clone(query_bytes)),
-        BoolNode::And(children) => {
-            let mut all: Vec> = Vec::new();
-            for child in children {
-                if let Some(b) = single_collector_bytes(child) {
-                    all.push(b);
-                }
-            }
-            match all.len() {
-                0 => None,
-                1 => Some(all.remove(0)),
-                _ => {
-                    let mut merged = Vec::new();
-                    for (i, b) in all.iter().enumerate() {
-                        if i > 0 {
-                            merged.push(b'\n');
-                        }
-                        merged.extend_from_slice(b);
-                    }
-                    Some(Arc::from(merged.as_slice()))
-                }
-            }
-        }
-        _ => None,
-    }
-}
-
-/// For a tree classified as `SingleCollector`, return the residual
-/// (all non-Collector parts of the AND tree, re-assembled into a
-/// single BoolNode). Recursively strips Collector leaves from nested
-/// ANDs. Returns `None` if the tree is a bare Collector or the entire
-/// tree is collectors-only (no residual predicates).
-fn extract_single_collector_residual(tree: &BoolNode) -> Option {
-    fn strip_collectors(node: &BoolNode) -> Option {
-        match node {
-            BoolNode::Collector { .. } => None,
-            BoolNode::Predicate(_) => Some(node.clone()),
-            BoolNode::And(children) => {
-                let residuals: Vec =
-                    children.iter().filter_map(strip_collectors).collect();
-                match residuals.len() {
-                    0 => None,
-                    1 => Some(residuals.into_iter().next().unwrap()),
-                    _ => Some(BoolNode::And(residuals)),
-                }
-            }
-            // OR/NOT with no collectors pass through unchanged (they're
-            // pure-predicate subtrees in a SingleCollector-classified tree).
-            other => Some(other.clone()),
-        }
-    }
-    strip_collectors(tree)
-}
-
-// ── Placeholder provider used only for substrait consume pass ─────────
-
-struct PlaceholderProvider {
-    schema: SchemaRef,
-}
-
-impl fmt::Debug for PlaceholderProvider {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("PlaceholderProvider").finish()
-    }
-}
-
-#[async_trait::async_trait]
-impl TableProvider for PlaceholderProvider {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-    fn table_type(&self) -> TableType {
-        TableType::Base
-    }
-    async fn scan(
-        &self,
-        _state: &dyn Session,
-        _projection: Option<&Vec>,
-        _filters: &[Expr],
-        _limit: Option,
-    ) -> Result, DataFusionError> {
-        Err(DataFusionError::Internal(
-            "PlaceholderProvider should not be scanned".into(),
-        ))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::indexed_table::bool_tree::BoolNode;
-    use datafusion::arrow::datatypes::{DataType, Field, Schema};
-    use datafusion::common::ScalarValue;
-    use datafusion::logical_expr::Operator;
-    use datafusion::physical_expr::expressions::{BinaryExpr, Column as PhysColumn, Literal};
-    use datafusion::physical_expr::PhysicalExpr;
-    use std::sync::Arc;
-
-    fn collector(tag: &[u8]) -> BoolNode {
-        BoolNode::Collector {
-            query_bytes: Arc::from(tag),
-        }
-    }
-
-    fn pred() -> BoolNode {
-        let left: Arc = Arc::new(PhysColumn::new("price", 0));
-        let right: Arc = Arc::new(Literal::new(ScalarValue::Int32(Some(0))));
-        BoolNode::Predicate(Arc::new(BinaryExpr::new(left, Operator::Eq, right)))
-    }
-
-    fn is_predicate(node: &BoolNode) -> bool {
-        matches!(node, BoolNode::Predicate(_))
-    }
-
-    // ── extract_single_collector_residual ─────────────────────────────
-
-    #[test]
-    fn residual_bare_collector_is_none() {
-        assert!(extract_single_collector_residual(&collector(b"x")).is_none());
-    }
-
-    #[test]
-    fn residual_and_collector_plus_predicate() {
-        let tree = BoolNode::And(vec![collector(b"x"), pred()]);
-        let r = extract_single_collector_residual(&tree).unwrap();
-        assert!(is_predicate(&r));
-    }
-
-    #[test]
-    fn residual_and_only_collectors_is_none() {
-        let tree = BoolNode::And(vec![collector(b"x"), collector(b"y")]);
-        assert!(extract_single_collector_residual(&tree).is_none());
-    }
-
-    #[test]
-    fn residual_nested_and_strips_collectors() {
-        // AND(C₁, AND(C₂, P)) → residual is P
-        let tree = BoolNode::And(vec![
-            collector(b"x"),
-            BoolNode::And(vec![collector(b"y"), pred()]),
-        ]);
-        let r = extract_single_collector_residual(&tree).unwrap();
-        assert!(is_predicate(&r));
-    }
-
-    #[test]
-    fn residual_deeply_nested_and() {
-        // AND(P₁, AND(C₁, AND(C₂, P₂))) → AND(P₁, P₂)
-        let p1 = pred();
-        let p2 = pred();
-        let tree = BoolNode::And(vec![
-            p1,
-            BoolNode::And(vec![
-                collector(b"a"),
-                BoolNode::And(vec![collector(b"b"), p2]),
-            ]),
-        ]);
-        let r = extract_single_collector_residual(&tree).unwrap();
-        match r {
-            BoolNode::And(children) => {
-                assert_eq!(children.len(), 2);
-                assert!(children.iter().all(is_predicate));
-            }
-            _ => panic!("expected AND, got {:?}", r),
-        }
-    }
-
-    #[test]
-    fn residual_nested_and_with_or_predicate() {
-        // AND(C, AND(P, OR(P, P))) → AND(P, OR(P, P))
-        let tree = BoolNode::And(vec![
-            collector(b"x"),
-            BoolNode::And(vec![
-                pred(),
-                BoolNode::Or(vec![pred(), pred()]),
-            ]),
-        ]);
-        let r = extract_single_collector_residual(&tree).unwrap();
-        match r {
-            BoolNode::And(children) => {
-                assert_eq!(children.len(), 2);
-                assert!(is_predicate(&children[0]));
-                assert!(matches!(children[1], BoolNode::Or(_)));
-            }
-            _ => panic!("expected AND, got {:?}", r),
-        }
-    }
-
-    #[test]
-    fn residual_nested_and_all_collectors_is_none() {
-        // AND(AND(C₁, C₂), AND(C₃, C₄)) → no residual
-        let tree = BoolNode::And(vec![
-            BoolNode::And(vec![collector(b"a"), collector(b"b")]),
-            BoolNode::And(vec![collector(b"c"), collector(b"d")]),
-        ]);
-        assert!(extract_single_collector_residual(&tree).is_none());
-    }
+    let stream_handle = crate::api::QueryStreamHandle::with_session_context(wrapped, query_context, ctx);
+    Ok(Box::into_raw(Box::new(stream_handle)) as i64)
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs
index 63854c4088105..bb2d081b99e5b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/bool_tree.rs
@@ -15,7 +15,7 @@
 //! Two flavors:
 //!
 //! - [`BoolNode`] — unresolved. Produced by `expr_to_bool_tree`.
-//!   `Collector` leaves carry the serialized query bytes
+//!   `Collector` leaves carry the annotation ID identifying the delegated predicate
 //!   (as extracted from the `index_filter(bytes)` UDF call);
 //!   `Predicate` leaves carry an arbitrary DataFusion
 //!   [`PhysicalExpr`](datafusion::physical_expr::PhysicalExpr) —
@@ -38,12 +38,12 @@ pub enum BoolNode {
     And(Vec),
     Or(Vec),
     Not(Box),
-    /// index-backend query payload. The caller (typically the indexed_executor)
-    /// upcalls into Java with these bytes at query-resolve time to get a
-    /// `provider_key`, then creates per-segment collectors. Bytes are opaque
-    /// to Rust; only the Java factory knows how to interpret them.
+    /// Delegated predicate identified by annotation ID. At query-resolve time,
+    /// the indexed executor upcalls into Java with this ID to get a `provider_key`,
+    /// then creates per-segment collectors. The annotation ID maps to a pre-compiled
+    /// query on the Java side (via FilterDelegationHandle).
     Collector {
-        query_bytes: Arc<[u8]>,
+        annotation_id: i32,
     },
     /// Arbitrary boolean-valued DataFusion expression. At refinement
     /// time, `expr.evaluate(batch)` produces the per-row mask; at page-
@@ -91,13 +91,13 @@ impl BoolNode {
     /// `resolve` (via the `*next` index) relies on this invariant; if you
     /// change one traversal you MUST change the other in lockstep, or
     /// collector-to-leaf matching will silently become wrong.
-    pub fn collector_leaves(&self) -> Vec> {
+    pub fn collector_leaves(&self) -> Vec {
         let mut out = Vec::new();
         self.collect_leaves(&mut out);
         out
     }
 
-    fn collect_leaves(&self, out: &mut Vec>) {
+    fn collect_leaves(&self, out: &mut Vec) {
         match self {
             BoolNode::And(children) | BoolNode::Or(children) => {
                 for c in children {
@@ -105,8 +105,8 @@ impl BoolNode {
                 }
             }
             BoolNode::Not(child) => child.collect_leaves(out),
-            BoolNode::Collector { query_bytes } => {
-                out.push(Arc::clone(query_bytes));
+            BoolNode::Collector { annotation_id } => {
+                out.push(*annotation_id);
             }
             BoolNode::Predicate(_) => {}
         }
@@ -167,7 +167,7 @@ impl BoolNode {
     /// `(column, op, value)`.
     ///
     /// Caller is responsible for creating the collectors — typically by
-    /// upcalling Java `createProvider(query_bytes)` per leaf to get a
+    /// upcalling Java `createProvider(annotation_id)` per leaf to get a
     /// `provider_key`, then `createCollector(provider_key, seg, min, max)`
     /// per chunk.
     ///
@@ -363,9 +363,9 @@ mod tests {
         }
     }
 
-    fn collector(bytes: &[u8]) -> BoolNode {
+    fn collector(id: i32) -> BoolNode {
         BoolNode::Collector {
-            query_bytes: Arc::from(bytes),
+            annotation_id: id,
         }
     }
 
@@ -382,8 +382,8 @@ mod tests {
     #[test]
     fn leaf_count_counts_only_collectors() {
         let tree = BoolNode::And(vec![
-            collector(b"a"),
-            BoolNode::Or(vec![collector(b"b"), predicate("x", Operator::Eq, 1)]),
+            collector(0),
+            BoolNode::Or(vec![collector(1), predicate("x", Operator::Eq, 1)]),
             predicate("y", Operator::Eq, 2),
         ]);
         assert_eq!(tree.collector_leaf_count(), 2);
@@ -392,21 +392,21 @@ mod tests {
     #[test]
     fn leaves_dfs_order() {
         let tree = BoolNode::And(vec![
-            collector(b"x"),
-            BoolNode::Or(vec![collector(b"y"), collector(b"z")]),
+            collector(10),
+            BoolNode::Or(vec![collector(11), collector(12)]),
         ]);
         let leaves = tree.collector_leaves();
         assert_eq!(leaves.len(), 3);
-        assert_eq!(&*leaves[0], b"x");
-        assert_eq!(&*leaves[1], b"y");
-        assert_eq!(&*leaves[2], b"z");
+        assert_eq!(leaves[0], 10);
+        assert_eq!(leaves[1], 11);
+        assert_eq!(leaves[2], 12);
     }
 
     // ── push_not_down (De Morgan) ─────────────────────────────────────
 
     #[test]
     fn not_collector_stays_wrapped() {
-        let tree = BoolNode::Not(Box::new(collector(b"x")));
+        let tree = BoolNode::Not(Box::new(collector(10)));
         let n = tree.push_not_down();
         assert!(matches!(n, BoolNode::Not(b) if matches!(*b, BoolNode::Collector { .. })));
     }
@@ -414,8 +414,8 @@ mod tests {
     #[test]
     fn de_morgan_not_and_to_or() {
         let tree = BoolNode::Not(Box::new(BoolNode::And(vec![
-            collector(b"a"),
-            collector(b"b"),
+            collector(0),
+            collector(1),
         ])));
         match tree.push_not_down() {
             BoolNode::Or(children) => {
@@ -447,7 +447,7 @@ mod tests {
 
     #[test]
     fn double_negation_cancels() {
-        let tree = BoolNode::Not(Box::new(BoolNode::Not(Box::new(collector(b"x")))));
+        let tree = BoolNode::Not(Box::new(BoolNode::Not(Box::new(collector(10)))));
         let n = tree.push_not_down();
         assert!(matches!(n, BoolNode::Collector { .. }));
     }
@@ -455,8 +455,8 @@ mod tests {
     #[test]
     fn nested_not_recurses_through_and_or() {
         let tree = BoolNode::Not(Box::new(BoolNode::And(vec![
-            BoolNode::Or(vec![collector(b"a"), collector(b"b")]),
-            collector(b"c"),
+            BoolNode::Or(vec![collector(0), collector(1)]),
+            collector(2),
         ])));
         match tree.push_not_down() {
             BoolNode::Or(outer) => {
@@ -473,8 +473,8 @@ mod tests {
     #[test]
     fn flatten_collapses_nested_and() {
         let tree = BoolNode::And(vec![
-            BoolNode::And(vec![collector(b"a"), collector(b"b")]),
-            collector(b"c"),
+            BoolNode::And(vec![collector(0), collector(1)]),
+            collector(2),
         ]);
         match tree.flatten() {
             BoolNode::And(children) => {
@@ -490,10 +490,10 @@ mod tests {
     #[test]
     fn flatten_collapses_nested_or() {
         let tree = BoolNode::Or(vec![
-            collector(b"a"),
+            collector(0),
             BoolNode::Or(vec![
-                collector(b"b"),
-                BoolNode::Or(vec![collector(b"c"), collector(b"d")]),
+                collector(1),
+                BoolNode::Or(vec![collector(2), collector(3)]),
             ]),
         ]);
         match tree.flatten() {
@@ -505,9 +505,9 @@ mod tests {
     #[test]
     fn flatten_preserves_mixed_connectives() {
         let tree = BoolNode::And(vec![
-            collector(b"a"),
-            BoolNode::Or(vec![collector(b"b"), collector(b"c")]),
-            BoolNode::And(vec![collector(b"d"), collector(b"e")]),
+            collector(0),
+            BoolNode::Or(vec![collector(1), collector(2)]),
+            BoolNode::And(vec![collector(3), collector(4)]),
         ]);
         match tree.flatten() {
             BoolNode::And(children) => {
@@ -521,8 +521,8 @@ mod tests {
     #[test]
     fn flatten_descends_into_not() {
         let tree = BoolNode::Not(Box::new(BoolNode::And(vec![
-            BoolNode::And(vec![collector(b"a"), collector(b"b")]),
-            collector(b"c"),
+            BoolNode::And(vec![collector(0), collector(1)]),
+            collector(2),
         ])));
         match tree.flatten() {
             BoolNode::Not(inner) => match *inner {
@@ -537,7 +537,7 @@ mod tests {
 
     #[test]
     fn resolve_replaces_collector_bytes_with_refs() {
-        let tree = BoolNode::And(vec![collector(b"a"), collector(b"b")]);
+        let tree = BoolNode::And(vec![collector(0), collector(1)]);
         let a: Arc = Arc::new(StubCollector(1));
         let b: Arc = Arc::new(StubCollector(2));
         let resolved = tree.resolve(&[(10, a), (20, b)]).unwrap();
@@ -572,14 +572,14 @@ mod tests {
 
     #[test]
     fn resolve_out_of_range_errors() {
-        let tree = collector(b"x");
+        let tree = collector(10);
         let err = tree.resolve(&[]).unwrap_err();
         assert!(err.contains("out of range"), "got: {}", err);
     }
 
     #[test]
     fn resolve_not_collector_still_wraps() {
-        let tree = BoolNode::Not(Box::new(collector(b"x")));
+        let tree = BoolNode::Not(Box::new(collector(10)));
         let c: Arc = Arc::new(StubCollector(0));
         let resolved = tree.resolve(&[(1, c)]).unwrap();
         match resolved {
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs
index 3b418762f4be8..35bfa67c86787 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/ffm_callbacks.rs
@@ -11,7 +11,7 @@
 //! Four callback slots, populated once at startup by
 //! `df_register_filter_tree_callbacks` (see `ffm.rs`):
 //!
-//! - `createProvider(queryBytes, queryBytesLen) -> providerKey|-1`
+//! - `createProvider(annotationId) -> providerKey|-1`
 //! - `createCollector(providerKey, segmentOrd, minDoc, maxDoc) -> collectorKey|-1`
 //! - `collectDocs(collectorKey, minDoc, maxDoc, outBuf, outWordCap) -> wordsWritten|-1`
 //! - `releaseCollector(collectorKey)`
@@ -26,7 +26,7 @@ use super::index::RowGroupDocsCollector;
 
 // ── Callback signatures ───────────────────────────────────────────────
 
-type CreateProviderFn = unsafe extern "C" fn(*const u8, i64) -> i32;
+type CreateProviderFn = unsafe extern "C" fn(i32) -> i32;
 type ReleaseProviderFn = unsafe extern "C" fn(i32);
 type CreateCollectorFn = unsafe extern "C" fn(i32, i32, i32, i32) -> i32;
 type CollectDocsFn = unsafe extern "C" fn(i32, i32, i32, *mut u64, i64) -> i64;
@@ -133,14 +133,14 @@ impl Drop for ProviderHandle {
     }
 }
 
-/// Create a provider from serialized backend query bytes by upcalling Java.
-pub fn create_provider(query_bytes: &[u8]) -> Result {
+/// Create a provider by annotation ID by upcalling Java.
+pub fn create_provider(annotation_id: i32) -> Result {
     let create = load_create_provider()?;
-    let key = unsafe { create(query_bytes.as_ptr(), query_bytes.len() as i64) };
+    let key = unsafe { create(annotation_id) };
     if key < 0 {
         return Err(format!(
-            "createProvider failed: len={} -> {}",
-            query_bytes.len(),
+            "createProvider failed: annotation_id={} -> {}",
+            annotation_id,
             key
         ));
     }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/substrait_to_tree.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/substrait_to_tree.rs
index 40dbeecc12a9b..272d8ffe69879 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/substrait_to_tree.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/substrait_to_tree.rs
@@ -14,7 +14,7 @@
 //!
 //! - `AND` / `OR` / `NOT` → `BoolNode::And` / `Or` / `Not`
 //! - `ScalarFunction` named `COLLECTOR_FUNCTION_NAME` with one `Binary`
-//!   literal argument → `BoolNode::Collector { query_bytes }`. Those bytes
+//!   literal argument → `BoolNode::Collector { annotation_id }`. The ID
 //!   are the serialized backend query payload; they're handed to a Java
 //!   factory at query-resolve time to create a provider.
 //! - **Anything else** → lowered to [`Arc`] via
@@ -25,7 +25,7 @@
 //!
 //! **The substrait plan is the wire format.** Java never serializes an
 //! `IndexFilterTree`; it rewrites `column = 'value'` on indexed columns to
-//! `index_filter(query_bytes)` UDF calls during the Calcite marking phase,
+//! `delegated_predicate(annotationId)` UDF calls during the Calcite marking phase,
 //! and that survives the substrait round-trip. Rust just reads it back out
 //! of the decoded `LogicalPlan`.
 
@@ -46,7 +46,7 @@ use datafusion::physical_expr::PhysicalExpr;
 use super::bool_tree::BoolNode;
 
 /// The UDF name Calcite emits for indexed-column filter markers.
-pub const COLLECTOR_FUNCTION_NAME: &str = "index_filter";
+pub const COLLECTOR_FUNCTION_NAME: &str = "delegated_predicate";
 
 /// Classification of a query's filter expression — drives the evaluator choice.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -152,17 +152,17 @@ fn convert_expr(
     }
 }
 
-/// `index_filter(query_bytes)` — a single `Binary` literal arg.
+/// `delegated_predicate(annotationId)` — a single `Int32` literal arg.
 fn convert_collector_function(args: &[Expr]) -> Result {
     if args.len() != 1 {
         return Err(format!(
-            "{} expects 1 arg (query_bytes), got {}",
+            "{} expects 1 arg (annotationId), got {}",
             COLLECTOR_FUNCTION_NAME,
             args.len()
         ));
     }
-    let bytes = extract_binary_literal(&args[0])?;
-    Ok(BoolNode::Collector { query_bytes: bytes })
+    let annotation_id = extract_int32_literal(&args[0])?;
+    Ok(BoolNode::Collector { annotation_id })
 }
 
 /// Strip table qualifiers from `Column` references in an `Expr` tree.
@@ -184,13 +184,11 @@ fn strip_column_qualifiers(expr: &Expr) -> Expr {
         .data
 }
 
-fn extract_binary_literal(expr: &Expr) -> Result, String> {
+fn extract_int32_literal(expr: &Expr) -> Result {
     match expr {
-        Expr::Literal(ScalarValue::Binary(Some(v)), _) => Ok(Arc::from(v.as_slice())),
-        Expr::Literal(ScalarValue::LargeBinary(Some(v)), _) => Ok(Arc::from(v.as_slice())),
-        Expr::Literal(ScalarValue::FixedSizeBinary(_, Some(v)), _) => Ok(Arc::from(v.as_slice())),
+        Expr::Literal(ScalarValue::Int32(Some(v)), _) => Ok(*v),
         _ => Err(format!(
-            "{} arg must be a Binary literal, got {:?}",
+            "{} arg must be an Int32 literal, got {:?}",
             COLLECTOR_FUNCTION_NAME, expr
         )),
     }
@@ -233,7 +231,7 @@ fn is_and_only_collector_tree(tree: &BoolNode) -> bool {
     }
 }
 
-/// Create the `index_filter(query_bytes) → Boolean` UDF.
+/// Create the `delegated_predicate(annotationId) → Boolean` UDF.
 ///
 /// This UDF exists solely as a marker for `classify_filter` / `expr_to_bool_tree`.
 /// Its body is deliberately wired to return a hard `DataFusionError` if it
@@ -252,15 +250,10 @@ struct IndexFilterUdf {
 
 impl IndexFilterUdf {
     fn new() -> Self {
-        // The UDF takes exactly one binary payload; LargeBinary is accepted
-        // for payloads that overflow Binary's 2 GiB limit. FixedSizeBinary is
-        // also accepted at decode time (see `extract_binary_literal`) but we
-        // don't enumerate every fixed size in the signature.
         Self {
             signature: Signature::one_of(
                 vec![
-                    TypeSignature::Exact(vec![DataType::Binary]),
-                    TypeSignature::Exact(vec![DataType::LargeBinary]),
+                    TypeSignature::Exact(vec![DataType::Int32]),
                 ],
                 Volatility::Immutable,
             ),
@@ -418,12 +411,12 @@ mod tests {
         let udf = Arc::new(create_index_filter_udf());
         let expr = Expr::ScalarFunction(datafusion::logical_expr::expr::ScalarFunction::new_udf(
             udf,
-            vec![lit(ScalarValue::Binary(Some(b"hello-query".to_vec())))],
+            vec![lit(ScalarValue::Int32(Some(42)))],
         ));
         let r = expr_to_bool_tree(&expr, &test_schema()).unwrap();
         match r.tree {
-            BoolNode::Collector { query_bytes } => {
-                assert_eq!(&*query_bytes, b"hello-query");
+            BoolNode::Collector { annotation_id } => {
+                assert_eq!(annotation_id, 42);
             }
             _ => panic!("expected Collector"),
         }
@@ -431,12 +424,12 @@ mod tests {
 
     #[test]
     fn mixed_tree() {
-        // AND(collector(bytes), OR(price > 100, qty < 50))
+        // AND(collector(annotationId), OR(price > 100, qty < 50))
         let udf = Arc::new(create_index_filter_udf());
         let collector_expr =
             Expr::ScalarFunction(datafusion::logical_expr::expr::ScalarFunction::new_udf(
                 udf,
-                vec![lit(ScalarValue::Binary(Some(b"mixed".to_vec())))],
+                vec![lit(ScalarValue::Int32(Some(0)))],
             ));
         let or_branch = col("price").gt(lit(100i32)).or(col("qty").lt(lit(50i32)));
         let expr = Expr::BinaryExpr(datafusion::logical_expr::BinaryExpr::new(
@@ -450,9 +443,9 @@ mod tests {
 
     // ── classify_filter ──────────────────────────────────────────────
 
-    fn collector(tag: &[u8]) -> BoolNode {
+    fn collector(id: i32) -> BoolNode {
         BoolNode::Collector {
-            query_bytes: Arc::from(tag),
+            annotation_id: id,
         }
     }
     fn dummy_predicate() -> BoolNode {
@@ -479,40 +472,40 @@ mod tests {
     #[test]
     fn classify_bare_collector_is_single() {
         assert_eq!(
-            classify_filter(&collector(b"x")),
+            classify_filter(&collector(10)),
             FilterClass::SingleCollector
         );
     }
 
     #[test]
     fn classify_and_of_collector_and_predicates_is_single() {
-        let tree = BoolNode::And(vec![collector(b"x"), dummy_predicate(), dummy_predicate()]);
+        let tree = BoolNode::And(vec![collector(10), dummy_predicate(), dummy_predicate()]);
         assert_eq!(classify_filter(&tree), FilterClass::SingleCollector);
     }
 
     #[test]
     fn classify_and_with_two_collectors_is_single() {
         // AND(C, C, P) — all collectors under AND-only path → SingleCollector.
-        let tree = BoolNode::And(vec![collector(b"x"), collector(b"y"), dummy_predicate()]);
+        let tree = BoolNode::And(vec![collector(10), collector(11), dummy_predicate()]);
         assert_eq!(classify_filter(&tree), FilterClass::SingleCollector);
     }
 
     #[test]
     fn classify_or_containing_collector_is_tree() {
-        let tree = BoolNode::Or(vec![collector(b"x"), dummy_predicate()]);
+        let tree = BoolNode::Or(vec![collector(10), dummy_predicate()]);
         assert_eq!(classify_filter(&tree), FilterClass::Tree);
     }
 
     #[test]
     fn classify_not_of_collector_is_tree() {
-        let tree = BoolNode::Not(Box::new(collector(b"x")));
+        let tree = BoolNode::Not(Box::new(collector(10)));
         assert_eq!(classify_filter(&tree), FilterClass::Tree);
     }
 
     #[test]
     fn classify_and_with_nested_collector_is_tree() {
         let tree = BoolNode::And(vec![
-            BoolNode::Or(vec![collector(b"x"), dummy_predicate()]),
+            BoolNode::Or(vec![collector(10), dummy_predicate()]),
             dummy_predicate(),
         ]);
         assert_eq!(classify_filter(&tree), FilterClass::Tree);
@@ -524,8 +517,8 @@ mod tests {
     fn classify_nested_and_collector_plus_predicate_is_single() {
         // AND(C₁, AND(C₂, P)) — nested AND, all collectors under AND-only path.
         let tree = BoolNode::And(vec![
-            collector(b"x"),
-            BoolNode::And(vec![collector(b"y"), dummy_predicate()]),
+            collector(10),
+            BoolNode::And(vec![collector(11), dummy_predicate()]),
         ]);
         assert_eq!(classify_filter(&tree), FilterClass::SingleCollector);
     }
@@ -536,10 +529,10 @@ mod tests {
         let tree = BoolNode::And(vec![
             dummy_predicate(),
             BoolNode::And(vec![
-                collector(b"a"),
+                collector(0),
                 BoolNode::And(vec![
-                    collector(b"b"),
-                    BoolNode::And(vec![collector(b"c"), dummy_predicate()]),
+                    collector(1),
+                    BoolNode::And(vec![collector(2), dummy_predicate()]),
                 ]),
             ]),
         ]);
@@ -550,8 +543,8 @@ mod tests {
     fn classify_nested_and_only_collectors_is_single() {
         // AND(AND(C₁, C₂), AND(C₃, C₄)) — nested AND of only collectors.
         let tree = BoolNode::And(vec![
-            BoolNode::And(vec![collector(b"a"), collector(b"b")]),
-            BoolNode::And(vec![collector(b"c"), collector(b"d")]),
+            BoolNode::And(vec![collector(0), collector(1)]),
+            BoolNode::And(vec![collector(2), collector(3)]),
         ]);
         assert_eq!(classify_filter(&tree), FilterClass::SingleCollector);
     }
@@ -560,7 +553,7 @@ mod tests {
     fn classify_nested_and_with_or_predicate_is_single() {
         // AND(C, AND(P, OR(P, P))) — OR contains only predicates, no collectors.
         let tree = BoolNode::And(vec![
-            collector(b"x"),
+            collector(10),
             BoolNode::And(vec![
                 dummy_predicate(),
                 BoolNode::Or(vec![dummy_predicate(), dummy_predicate()]),
@@ -573,7 +566,7 @@ mod tests {
     fn classify_nested_and_with_not_predicate_is_single() {
         // AND(C, NOT(P)) — NOT wraps a predicate, not a collector.
         let tree = BoolNode::And(vec![
-            collector(b"x"),
+            collector(10),
             BoolNode::Not(Box::new(dummy_predicate())),
         ]);
         assert_eq!(classify_filter(&tree), FilterClass::SingleCollector);
@@ -583,9 +576,9 @@ mod tests {
     fn classify_nested_and_or_containing_collector_is_tree() {
         // AND(C₁, AND(OR(C₂, P), P)) — OR above C₂ → Tree.
         let tree = BoolNode::And(vec![
-            collector(b"x"),
+            collector(10),
             BoolNode::And(vec![
-                BoolNode::Or(vec![collector(b"y"), dummy_predicate()]),
+                BoolNode::Or(vec![collector(11), dummy_predicate()]),
                 dummy_predicate(),
             ]),
         ]);
@@ -596,9 +589,9 @@ mod tests {
     fn classify_nested_and_not_containing_collector_is_tree() {
         // AND(C₁, AND(NOT(C₂), P)) — NOT above C₂ → Tree.
         let tree = BoolNode::And(vec![
-            collector(b"x"),
+            collector(10),
             BoolNode::And(vec![
-                BoolNode::Not(Box::new(collector(b"y"))),
+                BoolNode::Not(Box::new(collector(11))),
                 dummy_predicate(),
             ]),
         ]);
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/boolean_algebra.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/boolean_algebra.rs
index 062a036b22f08..22b872292b807 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/boolean_algebra.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/boolean_algebra.rs
@@ -289,7 +289,7 @@ impl LeafId {
         // Collector leaves first.
         if let Some(provider_id) = self.as_collector_provider_id() {
             return BoolNode::Collector {
-                query_bytes: Arc::from(&[provider_id][..]),
+                annotation_id: provider_id as i32,
             };
         }
         // Simple comparison leaves via ReferencePred.
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs
index eb4b116d9c2dd..d7411d90ead83 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs
@@ -504,11 +504,11 @@ fn extract_single_collector(tree: &BoolNode) -> Option<(u8, BoolNode)> {
     let mut residuals: Vec = Vec::new();
     for child in children {
         match child {
-            BoolNode::Collector { query_bytes } => {
+            BoolNode::Collector { annotation_id } => {
                 if tag.is_some() {
                     return None;
                 }
-                tag = Some(query_bytes[0]);
+                tag = Some(*annotation_id as u8);
             }
             other => residuals.push(other.clone()),
         }
@@ -929,7 +929,7 @@ mod tests {
         let lit: Arc = Arc::new(Literal::new(ScalarValue::Int32(Some(1000))));
         let predicate = BoolNode::Predicate(Arc::new(BinaryExpr::new(col, Operator::Lt, lit)));
         let collector = BoolNode::Collector {
-            query_bytes: Arc::from(&[0u8][..]),
+            annotation_id: 0,
         };
         let tree_node = BoolNode::And(vec![collector, predicate]);
         let matching: Vec = (0..100i32).collect();
@@ -947,7 +947,7 @@ mod tests {
         let corpus = build_corpus(FixtureConfig::small(0x2222));
         let loaded = load_segment(&corpus);
         let collector = BoolNode::Collector {
-            query_bytes: Arc::from(&[0u8][..]),
+            annotation_id: 0,
         };
         let matching: Vec = (0..100i32).collect();
         let gt = GeneratedTree {
@@ -1038,7 +1038,7 @@ mod tests {
         let lit: Arc = Arc::new(Literal::new(ScalarValue::Int32(Some(1000))));
         let predicate = BoolNode::Predicate(Arc::new(BinaryExpr::new(col, Operator::Lt, lit)));
         let collector_leaf = BoolNode::Collector {
-            query_bytes: Arc::from(&[0u8][..]),
+            annotation_id: 0,
         };
         let tree_node = BoolNode::And(vec![collector_leaf, predicate]);
 
@@ -1108,10 +1108,10 @@ mod tests {
 
         // Multi-collector → classifies as Tree path.
         let c1 = BoolNode::Collector {
-            query_bytes: Arc::from(&[0u8][..]),
+            annotation_id: 0,
         };
         let c2 = BoolNode::Collector {
-            query_bytes: Arc::from(&[1u8][..]),
+            annotation_id: 1,
         };
         let tree_node = BoolNode::And(vec![BoolNode::Or(vec![c1, c2]), predicate]);
 
@@ -1142,12 +1142,12 @@ mod tests {
         let c1_expr =
             Expr::ScalarFunction(datafusion::logical_expr::expr::ScalarFunction::new_udf(
                 Arc::new(idx_filter_udf.clone()),
-                vec![lit(ScalarValue::Binary(Some(vec![0u8])))],
+                vec![lit(ScalarValue::Int32(Some(0)))],
             ));
         let c2_expr =
             Expr::ScalarFunction(datafusion::logical_expr::expr::ScalarFunction::new_udf(
                 Arc::new(idx_filter_udf),
-                vec![lit(ScalarValue::Binary(Some(vec![1u8])))],
+                vec![lit(ScalarValue::Int32(Some(1)))],
             ));
         let or_expr = datafusion::logical_expr::or(c1_expr, c2_expr);
         let price_lt = col("price").lt(lit(ScalarValue::Int32(Some(1000))));
@@ -1191,7 +1191,7 @@ mod tests {
         let lit: Arc = Arc::new(Literal::new(ScalarValue::Int32(Some(1000))));
         let predicate = BoolNode::Predicate(Arc::new(BinaryExpr::new(col, Operator::Lt, lit)));
         let collector_leaf = BoolNode::Collector {
-            query_bytes: Arc::from(&[0u8][..]),
+            annotation_id: 0,
         };
         let tree_node = BoolNode::And(vec![collector_leaf, predicate]);
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/oracle.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/oracle.rs
index 684fec0b30de9..d0693a1f28ce8 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/oracle.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/oracle.rs
@@ -103,8 +103,8 @@ fn eval_row(node: &BoolNode, corpus: &Corpus, row: i32, collector_sets: &[HashSe
             acc
         }
         BoolNode::Not(child) => eval_row(child, corpus, row, collector_sets).not(),
-        BoolNode::Collector { query_bytes } => {
-            let tag = query_bytes[0] as usize;
+        BoolNode::Collector { annotation_id } => {
+            let tag = *annotation_id as u8 as usize;
             let set = collector_sets
                 .get(tag)
                 .unwrap_or_else(|| panic!("oracle: Collector tag {} has no matching set", tag));
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/tree_gen.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/tree_gen.rs
index 30c402f1ca945..0fcda5fc48546 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/tree_gen.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/tree_gen.rs
@@ -145,7 +145,7 @@ fn gen_leaf(rng: &mut StdRng, schema: &SchemaRef, num_collectors: usize) -> Bool
     if make_collector {
         let id = rng.gen_range(0..num_collectors) as u8;
         BoolNode::Collector {
-            query_bytes: Arc::from(&[id][..]),
+            annotation_id: id as i32,
         }
     } else {
         gen_predicate_leaf(rng, schema)
@@ -396,8 +396,8 @@ pub(in crate::indexed_table::tests_e2e) fn collect_collector_tags(tree: &BoolNod
         match n {
             BoolNode::And(cs) | BoolNode::Or(cs) => cs.iter().for_each(|c| walk(c, out)),
             BoolNode::Not(c) => walk(c, out),
-            BoolNode::Collector { query_bytes } => {
-                out.push(query_bytes[0]);
+            BoolNode::Collector { annotation_id } => {
+                out.push(*annotation_id as u8);
             }
             BoolNode::Predicate(_) => {}
         }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/mod.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/mod.rs
index 84ecd599cf5df..14b0967da7b76 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/mod.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/mod.rs
@@ -327,7 +327,7 @@ async fn run_tree_and_plan(
 
 fn index_leaf(tag: u8) -> BoolNode {
     BoolNode::Collector {
-        query_bytes: Arc::from(&[tag][..]),
+        annotation_id: tag as i32,
     }
 }
 
@@ -375,8 +375,8 @@ fn wire(node: &BoolNode, out: &mut Vec>) {
     match node {
         BoolNode::And(c) | BoolNode::Or(c) => c.iter().for_each(|x| wire(x, out)),
         BoolNode::Not(inner) => wire(inner, out),
-        BoolNode::Collector { query_bytes } => {
-            let c: Arc = match query_bytes.first().copied() {
+        BoolNode::Collector { annotation_id } => {
+            let c: Arc = match Some(*annotation_id as u8) {
                 Some(0) => brand_eq("amazon"),
                 Some(1) => brand_eq("apple"),
                 Some(2) => status_eq("archived"),
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs
index 86b64dadd1537..57352e63fe947 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs
@@ -983,7 +983,7 @@ async fn wide_multi_segment_collector_and_two_predicates() {
     let specs = wide_four_seg_specs();
     let tree = BoolNode::And(vec![
         BoolNode::Collector {
-            query_bytes: Arc::from(&[0u8][..]),
+            annotation_id: 0,
         },
         pred_wide_int("price", Operator::Gt, 50),
         pred_wide_int("qty", Operator::Lt, 3),
@@ -1001,7 +1001,7 @@ async fn wide_multi_segment_or_of_predicates_under_collector() {
     let specs = wide_four_seg_specs();
     let tree = BoolNode::And(vec![
         BoolNode::Collector {
-            query_bytes: Arc::from(&[0u8][..]),
+            annotation_id: 0,
         },
         BoolNode::Or(vec![
             pred_wide_str("region", Operator::Eq, "us-east"),
@@ -1021,7 +1021,7 @@ async fn wide_multi_segment_not_and_three_column_predicates() {
     let specs = wide_four_seg_specs();
     let tree = BoolNode::And(vec![
         BoolNode::Collector {
-            query_bytes: Arc::from(&[0u8][..]),
+            annotation_id: 0,
         },
         BoolNode::Not(Box::new(pred_wide_int("price", Operator::Lt, 30))),
         pred_wide_int("qty", Operator::Gt, 2),
@@ -1043,7 +1043,7 @@ async fn wide_multi_segment_deep_tree_four_predicate_columns() {
     let specs = wide_four_seg_specs();
     let tree = BoolNode::And(vec![
         BoolNode::Collector {
-            query_bytes: Arc::from(&[0u8][..]),
+            annotation_id: 0,
         },
         BoolNode::Or(vec![
             BoolNode::And(vec![
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/null_columns.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/null_columns.rs
index 2d65eff354166..d54221e774e0b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/null_columns.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/null_columns.rs
@@ -233,7 +233,7 @@ fn to_engine_tree_null(tree: &NT, coll_seq: &mut u8) -> BoolNode {
             let tag = *coll_seq;
             *coll_seq += 1;
             BoolNode::Collector {
-                query_bytes: Arc::from(&[tag][..]),
+                annotation_id: tag as i32,
             }
         }
         NT::Leaf(NullLeaf::AllNullGe(v)) => pred_int_local("all_null_col", Operator::GtEq, *v),
@@ -274,8 +274,8 @@ fn wire_null_rec(
             cs.iter().for_each(|c| wire_null_rec(c, matching_sets, out))
         }
         BoolNode::Not(inner) => wire_null_rec(inner, matching_sets, out),
-        BoolNode::Collector { query_bytes } => {
-            let tag = query_bytes.first().copied().expect("empty tag bytes") as usize;
+        BoolNode::Collector { annotation_id } => {
+            let tag = *annotation_id as usize;
             let set = &matching_sets[tag];
             out.push(Arc::new(RgScopedCollector {
                 matching_rows: set.clone(),
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs
index 3bada7e3c5870..2466849df08b8 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs
@@ -141,7 +141,7 @@ fn pred_node(expr: Arc) -> BoolNode {
 
 fn collector_leaf(tag: u8) -> BoolNode {
     BoolNode::Collector {
-        query_bytes: Arc::from(&[tag][..]),
+        annotation_id: tag as i32,
     }
 }
 
@@ -271,7 +271,7 @@ fn build_pp_map(
 
 fn wire_collectors_dfs(node: &BoolNode, out: &mut Vec>) {
     match node {
-        BoolNode::Collector { query_bytes } => out.push(collector_for_tag(query_bytes[0])),
+        BoolNode::Collector { annotation_id } => out.push(collector_for_tag(*annotation_id as u8)),
         BoolNode::And(cs) | BoolNode::Or(cs) => cs.iter().for_each(|c| wire_collectors_dfs(c, out)),
         BoolNode::Not(c) => wire_collectors_dfs(c, out),
         BoolNode::Predicate(_) => {}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/streaming_at_scale.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/streaming_at_scale.rs
index a8b140f304124..68e667c2cd495 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/streaming_at_scale.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/streaming_at_scale.rs
@@ -248,24 +248,20 @@ fn to_engine_tree_large(tree: <) -> BoolNode {
     match tree {
         LT::Leaf(l) => match l {
             LLeaf::LBrand(b) => BoolNode::Collector {
-                query_bytes: Arc::from(
-                    &[match *b {
-                        "amazon" => 0u8,
-                        "apple" => 1,
-                        "google" => 2,
-                        "samsung" => 3,
-                        _ => panic!("unknown brand {}", b),
-                    }][..],
-                ),
+                annotation_id: match *b {
+                    "amazon" => 0,
+                    "apple" => 1,
+                    "google" => 2,
+                    "samsung" => 3,
+                    _ => panic!("unknown brand {}", b),
+                },
             },
             LLeaf::LStatus(s) => BoolNode::Collector {
-                query_bytes: Arc::from(
-                    &[match *s {
-                        "active" => 4u8,
-                        "archived" => 5,
-                        _ => panic!("unknown status {}", s),
-                    }][..],
-                ),
+                annotation_id: match *s {
+                    "active" => 4,
+                    "archived" => 5,
+                    _ => panic!("unknown status {}", s),
+                },
             },
             LLeaf::LPriceGe(v) => pred_large_int("price", Operator::GtEq, *v),
             LLeaf::LPriceLt(v) => pred_large_int("price", Operator::Lt, *v),
@@ -292,8 +288,8 @@ fn wire_large_rec(node: &BoolNode, out: &mut Vec>
     match node {
         BoolNode::And(cs) | BoolNode::Or(cs) => cs.iter().for_each(|c| wire_large_rec(c, out)),
         BoolNode::Not(inner) => wire_large_rec(inner, out),
-        BoolNode::Collector { query_bytes } => {
-            let tag = query_bytes.first().copied().expect("empty tag bytes");
+        BoolNode::Collector { annotation_id } => {
+            let tag = Some(*annotation_id as u8).expect("empty tag bytes");
             out.push(large_collector_for(tag));
         }
         BoolNode::Predicate(_) => {}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs
index 33d47b9bb556a..be804e1eb777e 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs
@@ -185,6 +185,6 @@ pub async fn execute_with_context(
         cross_rt_stream,
     );
 
-    let stream_handle = crate::api::QueryStreamHandle::new(wrapped, handle.query_context);
+    let stream_handle = crate::api::QueryStreamHandle::with_session_context(wrapped, handle.query_context, handle.ctx);
     Ok(Box::into_raw(Box::new(stream_handle)) as i64)
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
index 74effcf8d7851..ce037eb3c6508 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
@@ -37,6 +37,15 @@ pub struct SessionContextHandle {
     pub table_path: ListingTableUrl,
     pub object_metas: Arc>,
     pub query_context: QueryTrackingContext,
+    pub table_name: String,
+    /// When set, indicates this session uses the indexed execution path with filter delegation.
+    pub indexed_config: Option,
+}
+
+/// Configuration for indexed execution with filter delegation, provided by Java.
+pub struct IndexedExecutionConfig {
+    pub tree_shape: i32,
+    pub delegated_predicate_count: i32,
 }
 
 /// Creates a SessionContext with per-query RuntimeEnv and registers the default
@@ -135,6 +144,8 @@ pub async unsafe fn create_session_context(
         table_path: shard_view.table_path.clone(),
         object_metas: shard_view.object_metas.clone(),
         query_context,
+        table_name: table_name.to_string(),
+        indexed_config: None,
     };
     Ok(Box::into_raw(Box::new(handle)) as i64)
 }
@@ -148,3 +159,28 @@ pub unsafe fn close_session_context(ptr: i64) {
         let _ = Box::from_raw(ptr as *mut SessionContextHandle);
     }
 }
+
+/// Creates a SessionContext configured for indexed execution with filter delegation.
+/// Registers the `delegated_predicate` UDF and stores the tree shape + predicate count
+/// for use during execution.
+pub async unsafe fn create_session_context_indexed(
+    runtime_ptr: i64,
+    shard_view_ptr: i64,
+    table_name: &str,
+    context_id: i64,
+    tree_shape: i32,
+    delegated_predicate_count: i32,
+) -> Result {
+    // Create base session context (same as non-indexed path)
+    let ptr = create_session_context(runtime_ptr, shard_view_ptr, table_name, context_id).await?;
+
+    // Augment with indexed config and UDF registration
+    let handle = &mut *(ptr as *mut SessionContextHandle);
+    handle.ctx.register_udf(crate::indexed_table::substrait_to_tree::create_index_filter_udf());
+    handle.indexed_config = Some(IndexedExecutionConfig {
+        tree_shape,
+        delegated_predicate_count,
+    });
+
+    Ok(ptr)
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceIT.java
deleted file mode 100644
index 717f3054cb97a..0000000000000
--- a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceIT.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.be.datafusion;
-
-import org.opensearch.Version;
-import org.opensearch.action.admin.indices.create.CreateIndexResponse;
-import org.opensearch.analytics.AnalyticsPlugin;
-import org.opensearch.arrow.flight.transport.FlightStreamPlugin;
-import org.opensearch.be.lucene.LucenePlugin;
-import org.opensearch.cluster.metadata.IndexMetadata;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
-import org.opensearch.composite.CompositeDataFormatPlugin;
-import org.opensearch.parquet.ParquetDataFormatPlugin;
-import org.opensearch.plugins.Plugin;
-import org.opensearch.plugins.PluginInfo;
-import org.opensearch.ppl.TestPPLPlugin;
-import org.opensearch.ppl.action.PPLRequest;
-import org.opensearch.ppl.action.PPLResponse;
-import org.opensearch.ppl.action.UnifiedPPLExecuteAction;
-import org.opensearch.test.OpenSearchIntegTestCase;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * End-to-end smoke test for the streaming coordinator-reduce path:
- *
- * 
      - *   PPL → planner → multi-shard SHARD_FRAGMENT dispatch → DataFusion shard scan
- *       → ExchangeSink.feed → DatafusionReduceSink (Substrait SUM via convertFinalAggFragment)
- *       → drain → downstream → assembled PPLResponse
- * 
      
- *
- * 
      Builds a parquet-backed composite index with two shards, indexes a small
- * deterministic dataset, then runs a {@code stats sum(value) as total} aggregate.
- * The total is a function of the indexed values × shard count; any drift in
- * shard fan-out, sink wiring, or final-agg merge will show up as a mismatch.
- */
-@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 2)
-public class CoordinatorReduceIT extends OpenSearchIntegTestCase {
-
-    private static final String INDEX = "coord_reduce_e2e";
-    private static final int NUM_SHARDS = 2;
-    private static final int DOCS_PER_SHARD = 10;
-    /** Constant `value` for every doc — picks a deterministic SUM independent of shard routing. */
-    private static final int VALUE = 7;
-
-    @Override
-    protected Collection> nodePlugins() {
-        // Plugins with no extendedPlugins requirement go here. Plugins that need
-        // explicit extendedPlugins (so SPI ExtensionLoader walks the right parent
-        // classloader) are declared in additionalNodePlugins() below.
-        return List.of(TestPPLPlugin.class, FlightStreamPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class);
-    }
-
-    @Override
-    protected Collection additionalNodePlugins() {
-        // OpenSearchIntegTestCase's nodePlugins() builds PluginInfo with empty
-        // extendedPlugins, which breaks ExtensiblePlugin.loadExtensions(...) for
-        // plugins like DataFusionPlugin that ride on AnalyticsPlugin's SPI. Use
-        // additionalNodePlugins() to declare the parent relationships explicitly.
-        return List.of(
-            classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()),
-            classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()),
-            classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName()))
-        );
-    }
-
-    private static PluginInfo classpathPlugin(Class pluginClass, List extendedPlugins) {
-        return new PluginInfo(
-            pluginClass.getName(),
-            "classpath plugin",
-            "NA",
-            Version.CURRENT,
-            "1.8",
-            pluginClass.getName(),
-            null,
-            extendedPlugins,
-            false
-        );
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal))
-            .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true)
-            // STREAM_TRANSPORT (Arrow Flight RPC for shard→coordinator response streaming)
-            // is intentionally NOT enabled here. With it on, AnalyticsSearchTransportService
-            // routes all sendChildRequest calls through StreamTransportService whose connection
-            // profile only carries stream channels, breaking the non-stream fragment dispatch
-            // request. The non-stream path is enough for this IT's small-result SUM aggregate.
-            .build();
-    }
-
-    /**
-     * {@code source = T | stats sum(value) as total} on a 2-shard parquet-backed index
-     * → coordinator-reduce path runs the final SUM via {@link DatafusionReduceSink}
-     * and returns the deterministic total.
-     */
-    public void testScalarSumAcrossShards() throws Exception {
-        createParquetBackedIndex();
-        indexDeterministicDocs();
-
-        PPLResponse response = executePPL("source = " + INDEX + " | stats sum(value) as total");
-
-        assertNotNull("PPLResponse must not be null", response);
-        assertTrue("columns must contain 'total', got " + response.getColumns(), response.getColumns().contains("total"));
-        assertEquals("scalar agg must return exactly 1 row", 1, response.getRows().size());
-
-        int idx = response.getColumns().indexOf("total");
-        Object cell = response.getRows().get(0)[idx];
-        assertNotNull("SUM(value) cell must not be null — coordinator-reduce returned no value", cell);
-        long actual = ((Number) cell).longValue();
-        long expected = (long) VALUE * NUM_SHARDS * DOCS_PER_SHARD;
-        assertEquals(
-            "SUM(value) across " + NUM_SHARDS + " shards × " + DOCS_PER_SHARD + " docs × value=" + VALUE + " = " + expected,
-            expected,
-            actual
-        );
-    }
-
-    private void createParquetBackedIndex() {
-        Settings indexSettings = Settings.builder()
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS)
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put("index.pluggable.dataformat.enabled", true)
-            .put("index.pluggable.dataformat", "composite")
-            .put("index.composite.primary_data_format", "parquet")
-            .putList("index.composite.secondary_data_formats")
-            .build();
-
-        CreateIndexResponse response = client().admin()
-            .indices()
-            .prepareCreate(INDEX)
-            .setSettings(indexSettings)
-            .setMapping("value", "type=integer")
-            .get();
-        assertTrue("index creation must be acknowledged", response.isAcknowledged());
-        ensureGreen(INDEX);
-    }
-
-    private void indexDeterministicDocs() {
-        int total = NUM_SHARDS * DOCS_PER_SHARD;
-        for (int i = 0; i < total; i++) {
-            client().prepareIndex(INDEX).setId(String.valueOf(i)).setSource("value", VALUE).get();
-        }
-        client().admin().indices().prepareRefresh(INDEX).get();
-        client().admin().indices().prepareFlush(INDEX).get();
-    }
-
-    private PPLResponse executePPL(String ppl) {
-        return client().execute(UnifiedPPLExecuteAction.INSTANCE, new PPLRequest(ppl)).actionGet();
-    }
-}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceMemtableIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceMemtableIT.java
deleted file mode 100644
index 438742158d990..0000000000000
--- a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/CoordinatorReduceMemtableIT.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.be.datafusion;
-
-import org.opensearch.Version;
-import org.opensearch.action.admin.indices.create.CreateIndexResponse;
-import org.opensearch.analytics.AnalyticsPlugin;
-import org.opensearch.arrow.flight.transport.FlightStreamPlugin;
-import org.opensearch.be.lucene.LucenePlugin;
-import org.opensearch.cluster.metadata.IndexMetadata;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
-import org.opensearch.composite.CompositeDataFormatPlugin;
-import org.opensearch.parquet.ParquetDataFormatPlugin;
-import org.opensearch.plugins.Plugin;
-import org.opensearch.plugins.PluginInfo;
-import org.opensearch.ppl.TestPPLPlugin;
-import org.opensearch.ppl.action.PPLRequest;
-import org.opensearch.ppl.action.PPLResponse;
-import org.opensearch.ppl.action.UnifiedPPLExecuteAction;
-import org.opensearch.test.OpenSearchIntegTestCase;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Memtable variant of {@link CoordinatorReduceIT}. Identical query and assertion, but the cluster
- * starts with {@code datafusion.reduce.input_mode=memtable} so the coordinator-reduce path uses
- * {@link DatafusionMemtableReduceSink} instead of the streaming sink. Verifies the sink dispatch
- * wiring in {@link DataFusionAnalyticsBackendPlugin#getExchangeSinkProvider} and the buffered
- * memtable handoff against a real multi-shard scan.
- */
-@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 2)
-public class CoordinatorReduceMemtableIT extends OpenSearchIntegTestCase {
-
-    private static final String INDEX = "coord_reduce_memtable_e2e";
-    private static final int NUM_SHARDS = 2;
-    private static final int DOCS_PER_SHARD = 10;
-    private static final int VALUE = 7;
-
-    @Override
-    protected Collection> nodePlugins() {
-        return List.of(TestPPLPlugin.class, FlightStreamPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class);
-    }
-
-    @Override
-    protected Collection additionalNodePlugins() {
-        return List.of(
-            classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()),
-            classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()),
-            classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName()))
-        );
-    }
-
-    private static PluginInfo classpathPlugin(Class pluginClass, List extendedPlugins) {
-        return new PluginInfo(
-            pluginClass.getName(),
-            "classpath plugin",
-            "NA",
-            Version.CURRENT,
-            "1.8",
-            pluginClass.getName(),
-            null,
-            extendedPlugins,
-            false
-        );
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal))
-            .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true)
-            .put(DataFusionPlugin.DATAFUSION_REDUCE_INPUT_MODE.getKey(), "memtable")
-            .build();
-    }
-
-    public void testScalarSumAcrossShardsViaMemtable() throws Exception {
-        createParquetBackedIndex();
-        indexDeterministicDocs();
-
-        PPLResponse response = executePPL("source = " + INDEX + " | stats sum(value) as total");
-
-        assertNotNull("PPLResponse must not be null", response);
-        assertTrue("columns must contain 'total', got " + response.getColumns(), response.getColumns().contains("total"));
-        assertEquals("scalar agg must return exactly 1 row", 1, response.getRows().size());
-
-        int idx = response.getColumns().indexOf("total");
-        Object cell = response.getRows().get(0)[idx];
-        assertNotNull("SUM(value) cell must not be null — memtable coordinator-reduce returned no value", cell);
-        long actual = ((Number) cell).longValue();
-        long expected = (long) VALUE * NUM_SHARDS * DOCS_PER_SHARD;
-        assertEquals("SUM(value) memtable path must match streaming path", expected, actual);
-    }
-
-    private void createParquetBackedIndex() {
-        Settings indexSettings = Settings.builder()
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS)
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put("index.pluggable.dataformat.enabled", true)
-            .put("index.pluggable.dataformat", "composite")
-            .put("index.composite.primary_data_format", "parquet")
-            .putList("index.composite.secondary_data_formats")
-            .build();
-
-        CreateIndexResponse response = client().admin()
-            .indices()
-            .prepareCreate(INDEX)
-            .setSettings(indexSettings)
-            .setMapping("value", "type=integer")
-            .get();
-        assertTrue("index creation must be acknowledged", response.isAcknowledged());
-        ensureGreen(INDEX);
-    }
-
-    private void indexDeterministicDocs() {
-        int total = NUM_SHARDS * DOCS_PER_SHARD;
-        for (int i = 0; i < total; i++) {
-            client().prepareIndex(INDEX).setId(String.valueOf(i)).setSource("value", VALUE).get();
-        }
-        client().admin().indices().prepareRefresh(INDEX).get();
-        client().admin().indices().prepareFlush(INDEX).get();
-    }
-
-    private PPLResponse executePPL(String ppl) {
-        return client().execute(UnifiedPPLExecuteAction.INSTANCE, new PPLRequest(ppl)).actionGet();
-    }
-}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/StreamingCoordinatorReduceIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/StreamingCoordinatorReduceIT.java
deleted file mode 100644
index 6ac06cc18536b..0000000000000
--- a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/StreamingCoordinatorReduceIT.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.be.datafusion;
-
-import org.opensearch.Version;
-import org.opensearch.action.admin.indices.create.CreateIndexResponse;
-import org.opensearch.analytics.AnalyticsPlugin;
-import org.opensearch.arrow.flight.transport.FlightStreamPlugin;
-import org.opensearch.be.lucene.LucenePlugin;
-import org.opensearch.cluster.metadata.IndexMetadata;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
-import org.opensearch.composite.CompositeDataFormatPlugin;
-import org.opensearch.parquet.ParquetDataFormatPlugin;
-import org.opensearch.plugins.Plugin;
-import org.opensearch.plugins.PluginInfo;
-import org.opensearch.ppl.TestPPLPlugin;
-import org.opensearch.ppl.action.PPLRequest;
-import org.opensearch.ppl.action.PPLResponse;
-import org.opensearch.ppl.action.UnifiedPPLExecuteAction;
-import org.opensearch.test.OpenSearchIntegTestCase;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import static org.opensearch.common.util.FeatureFlags.STREAM_TRANSPORT;
-
-/**
- * Streaming variant of {@link CoordinatorReduceIT}: same 2-shard parquet-backed index and deterministic
- * dataset, but with Arrow Flight RPC enabled via {@link FeatureFlags#STREAM_TRANSPORT}. Exercises the
- * shard-fragment → Flight → {@code DatafusionReduceSink.feed} handoff that previously failed with
- * {@code "A buffer can only be associated between two allocators that share the same root"} on
- * multi-shard queries.
- */
-@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 2)
-public class StreamingCoordinatorReduceIT extends OpenSearchIntegTestCase {
-
-    private static final String INDEX = "coord_reduce_streaming_e2e";
-    private static final int NUM_SHARDS = 2;
-    private static final int DOCS_PER_SHARD = 10;
-    /** Constant `value` for every doc — deterministic assertion independent of shard routing. */
-    private static final int VALUE = 7;
-
-    @Override
-    protected Collection> nodePlugins() {
-        return List.of(TestPPLPlugin.class, FlightStreamPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class);
-    }
-
-    @Override
-    protected Collection additionalNodePlugins() {
-        return List.of(
-            classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()),
-            classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()),
-            classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName()))
-        );
-    }
-
-    private static PluginInfo classpathPlugin(Class pluginClass, List extendedPlugins) {
-        return new PluginInfo(
-            pluginClass.getName(),
-            "classpath plugin",
-            "NA",
-            Version.CURRENT,
-            "1.8",
-            pluginClass.getName(),
-            null,
-            extendedPlugins,
-            false
-        );
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal))
-            .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true)
-            .build();
-    }
-
-    /**
-     * {@code source = T} on a 2-shard parquet-backed index with streaming enabled exercises the
-     * coordinator reduce sink's cross-plugin VectorSchemaRoot handoff. Before the allocator-root
-     * unification fix, this failed with an Arrow {@code associate} mismatch.
-     */
-    @LockFeatureFlag(STREAM_TRANSPORT)
-    public void testBaselineScanAcrossShards() throws Exception {
-        createParquetBackedIndex();
-        indexDeterministicDocs();
-
-        PPLResponse response = executePPL("source = " + INDEX);
-
-        assertNotNull("PPLResponse must not be null", response);
-        assertTrue("columns must contain 'value', got " + response.getColumns(), response.getColumns().contains("value"));
-
-        int expectedRows = NUM_SHARDS * DOCS_PER_SHARD;
-        assertEquals("all docs across shards must be returned", expectedRows, response.getRows().size());
-
-        int idx = response.getColumns().indexOf("value");
-        for (Object[] row : response.getRows()) {
-            Object cell = row[idx];
-            assertNotNull("value cell must not be null", cell);
-            assertEquals("every doc has value=" + VALUE, (long) VALUE, ((Number) cell).longValue());
-        }
-    }
-
-    private void createParquetBackedIndex() {
-        Settings indexSettings = Settings.builder()
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS)
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put("index.pluggable.dataformat.enabled", true)
-            .put("index.pluggable.dataformat", "composite")
-            .put("index.composite.primary_data_format", "parquet")
-            .putList("index.composite.secondary_data_formats")
-            .build();
-
-        CreateIndexResponse response = client().admin()
-            .indices()
-            .prepareCreate(INDEX)
-            .setSettings(indexSettings)
-            .setMapping("value", "type=integer")
-            .get();
-        assertTrue("index creation must be acknowledged", response.isAcknowledged());
-        ensureGreen(INDEX);
-    }
-
-    private void indexDeterministicDocs() {
-        int total = NUM_SHARDS * DOCS_PER_SHARD;
-        for (int i = 0; i < total; i++) {
-            client().prepareIndex(INDEX).setId(String.valueOf(i)).setSource("value", VALUE).get();
-        }
-        client().admin().indices().prepareRefresh(INDEX).get();
-        client().admin().indices().prepareFlush(INDEX).get();
-    }
-
-    private PPLResponse executePPL(String ppl) {
-        return client().execute(UnifiedPPLExecuteAction.INSTANCE, new PPLRequest(ppl)).actionGet();
-    }
-}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
index 4d444febc36aa..fb161135cdb53 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
@@ -13,10 +13,13 @@
 import org.opensearch.analytics.spi.AggregateFunction;
 import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin;
 import org.opensearch.analytics.spi.BackendCapabilityProvider;
+import org.opensearch.analytics.spi.BackendExecutionContext;
+import org.opensearch.analytics.spi.DelegationType;
 import org.opensearch.analytics.spi.EngineCapability;
 import org.opensearch.analytics.spi.ExchangeSinkProvider;
 import org.opensearch.analytics.spi.FieldType;
 import org.opensearch.analytics.spi.FilterCapability;
+import org.opensearch.analytics.spi.FilterDelegationHandle;
 import org.opensearch.analytics.spi.FragmentConvertor;
 import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory;
 import org.opensearch.analytics.spi.ProjectCapability;
@@ -25,6 +28,7 @@
 import org.opensearch.analytics.spi.ScanCapability;
 import org.opensearch.analytics.spi.SearchExecEngineProvider;
 import org.opensearch.analytics.spi.StdOperatorRewriteAdapter;
+import org.opensearch.be.datafusion.indexfilter.FilterTreeCallbacks;
 import org.opensearch.index.engine.dataformat.DataFormatRegistry;
 
 import java.util.HashSet;
@@ -154,6 +158,11 @@ public Set supportedEngineCapabilities() {
                 return ENGINE_CAPS;
             }
 
+            @Override
+            public Set supportedDelegations() {
+                return Set.of(DelegationType.FILTER);
+            }
+
             @Override
             public Set scanCapabilities() {
                 Set formats = Set.copyOf(plugin.getSupportedFormats());
@@ -282,4 +291,11 @@ public ExchangeSinkProvider getExchangeSinkProvider() {
             return new DatafusionReduceSink(ctx, svc.getNativeRuntime());
         };
     }
+
+    @Override
+    public void configureFilterDelegation(FilterDelegationHandle handle, BackendExecutionContext backendContext) {
+        // Install the handle as the FFM upcall target. All Rust callbacks
+        // (createProvider, createCollector, collectDocs, release*) route to it.
+        FilterTreeCallbacks.setHandle(handle);
+    }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java
index 88ec47e2da9f9..4dbe7f280fc53 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java
@@ -16,6 +16,7 @@
 import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory;
 import org.opensearch.analytics.spi.InstructionNode;
 import org.opensearch.analytics.spi.ShardScanInstructionNode;
+import org.opensearch.analytics.spi.ShardScanWithDelegationInstructionNode;
 
 import java.util.List;
 import java.util.Optional;
@@ -50,6 +51,11 @@ public Optional createFilterDelegationNode(
         return Optional.of(new FilterDelegationInstructionNode(treeShape, delegatedPredicateCount, delegatedExpressions));
     }
 
+    @Override
+    public Optional createShardScanWithDelegationNode(FilterTreeShape treeShape, int delegatedPredicateCount) {
+        return Optional.of(new ShardScanWithDelegationInstructionNode(treeShape, delegatedPredicateCount));
+    }
+
     @Override
     public Optional createPartialAggregateNode() {
         // TODO: return Optional.of(...) once PartialAggregateInstructionHandler is implemented
@@ -66,6 +72,9 @@ public Optional createFinalAggregateNode() {
     @SuppressWarnings("unchecked")
     @Override
     public FragmentInstructionHandler createHandler(InstructionNode node) {
+        if (node instanceof ShardScanWithDelegationInstructionNode) {
+            return new ShardScanWithDelegationHandler(plugin);
+        }
         if (node instanceof ShardScanInstructionNode) {
             return new ShardScanInstructionHandler(plugin);
         }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanWithDelegationHandler.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanWithDelegationHandler.java
new file mode 100644
index 0000000000000..4564a5921b9ea
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanWithDelegationHandler.java
@@ -0,0 +1,73 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.analytics.backend.ShardScanExecutionContext;
+import org.opensearch.analytics.spi.BackendExecutionContext;
+import org.opensearch.analytics.spi.CommonExecutionContext;
+import org.opensearch.analytics.spi.FilterTreeShape;
+import org.opensearch.analytics.spi.FragmentInstructionHandler;
+import org.opensearch.analytics.spi.ShardScanWithDelegationInstructionNode;
+import org.opensearch.be.datafusion.nativelib.NativeBridge;
+import org.opensearch.be.datafusion.nativelib.SessionContextHandle;
+import org.opensearch.index.engine.dataformat.DataFormatRegistry;
+
+/**
+ * Handles ShardScanWithDelegation instruction: creates a SessionContext via FFM
+ * configured for indexed execution — registers the delegated_predicate UDF and
+ * sets up the custom scan operator (IndexedTableProvider) with FilterTreeShape
+ * and delegatedPredicateCount.
+ */
+public class ShardScanWithDelegationHandler implements FragmentInstructionHandler {
+
+    private final DataFusionPlugin plugin;
+
+    ShardScanWithDelegationHandler(DataFusionPlugin plugin) {
+        this.plugin = plugin;
+    }
+
+    @Override
+    public BackendExecutionContext apply(
+        ShardScanWithDelegationInstructionNode node,
+        CommonExecutionContext commonContext,
+        BackendExecutionContext backendContext
+    ) {
+        ShardScanExecutionContext context = (ShardScanExecutionContext) commonContext;
+        DataFusionService dataFusionService = plugin.getDataFusionService();
+        DataFormatRegistry registry = plugin.getDataFormatRegistry();
+
+        DatafusionReader dfReader = null;
+        for (String formatName : plugin.getSupportedFormats()) {
+            dfReader = context.getReader().getReader(registry.format(formatName), DatafusionReader.class);
+            if (dfReader != null) break;
+        }
+        if (dfReader == null) {
+            throw new IllegalStateException("No DatafusionReader available in the acquired reader");
+        }
+
+        long readerPtr = dfReader.getReaderHandle().getPointer();
+        long runtimePtr = dataFusionService.getNativeRuntime().get();
+        long contextId = context.getTask() != null ? context.getTask().getId() : 0L;
+        FilterTreeShape treeShape = node.getTreeShape();
+        int delegatedPredicateCount = node.getDelegatedPredicateCount();
+
+        // Single FFM call: creates SessionContext + registers delegated_predicate UDF +
+        // configures IndexedTableProvider with treeShape and delegatedPredicateCount
+        SessionContextHandle sessionCtxHandle = NativeBridge.createSessionContextForIndexedExecution(
+            readerPtr,
+            runtimePtr,
+            context.getTableName(),
+            contextId,
+            treeShape.ordinal(),
+            delegatedPredicateCount
+        );
+
+        return new DataFusionSessionState(sessionCtxHandle);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterProviderRegistry.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterProviderRegistry.java
index 4f91936b1ce05..fe29f76463244 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterProviderRegistry.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterProviderRegistry.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.be.datafusion.indexfilter;
 
+import org.opensearch.analytics.spi.FilterDelegationHandle;
 import org.opensearch.analytics.spi.IndexFilterProvider;
 import org.opensearch.analytics.spi.IndexFilterProviderFactory;
 
@@ -115,4 +116,31 @@ int createCollector(int providerKey, int segmentOrd, int minDoc, int maxDoc) {
         }
         return collectors.registerCollector(provider, inner);
     }
+
+    // ── Delegation handle path (replaces factory-based createProvider) ──
+
+    private final AtomicReference delegationHandle = new AtomicReference<>();
+
+    /**
+     * Register a {@link FilterDelegationHandle} for annotation-ID-based provider creation.
+     * When Rust calls createProvider(annotationId), the handle is used instead of the factory.
+     */
+    public void registerDelegationHandle(FilterDelegationHandle handle) {
+        this.delegationHandle.set(handle);
+    }
+
+    /**
+     * Create a provider by annotation ID using the registered delegation handle.
+     * Called by the updated FFM callback path (annotationId instead of query bytes).
+     *
+     * @return provider key {@code >= 1}, or {@code -1} on failure
+     */
+    // TODO: remove the old createProvider(byte[]) path once all callers migrate to annotation-ID-based delegation
+    int createProviderByAnnotationId(int annotationId) {
+        FilterDelegationHandle handle = this.delegationHandle.get();
+        if (handle == null) {
+            return -1;
+        }
+        return handle.createProvider(annotationId);
+    }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java
index 87aa07aad0f30..a9310f5e5fdfa 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/indexfilter/FilterTreeCallbacks.java
@@ -11,62 +11,56 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.opensearch.be.datafusion.indexfilter.CollectorRegistry.CollectorHandle;
+import org.opensearch.analytics.spi.FilterDelegationHandle;
 
 import java.lang.foreign.MemorySegment;
-import java.lang.foreign.ValueLayout;
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * Static callback targets invoked by the native engine via FFM upcalls.
  *
- * 
      Delegates to two instance-based registries installed at plugin startup:
- * 
        
- *   
      • {@link FilterProviderRegistry} — query-level provider create/release (cold path)
        • 
- *   
      • {@link CollectorRegistry} — per-segment collector create/collect/release (hot path)
        • 
- * 
      
+ * 
      All calls delegate to the currently installed {@link FilterDelegationHandle}.
+ * The handle is set per-query-per-shard before execution and cleared after.
  *
  * 
      Error-handling contract
      
  * 
      Every method catches all {@link Throwable}s and returns {@code -1}
  * (or silently returns for void methods). A Java exception escaping through
  * an FFM upcall stub crashes the JVM.
+ *
+ * // TODO: remove old Registries-based code path and CollectorRegistry/FilterProviderRegistry
+ * // once all tests are migrated to the FilterDelegationHandle path.
  */
 public final class FilterTreeCallbacks {
 
     private static final Logger LOGGER = LogManager.getLogger(FilterTreeCallbacks.class);
 
-    /** Both registries in one snapshot — one AtomicReference read per upcall. */
-    record Registries(FilterProviderRegistry providers, CollectorRegistry collectors) {
-    }
-
-    private static final AtomicReference REGISTRIES = new AtomicReference<>();
+    private static final AtomicReference HANDLE = new AtomicReference<>();
 
     private FilterTreeCallbacks() {}
 
     /**
-     * Install the registries. Called once at plugin startup.
+     * Install the delegation handle for the current execution.
+     * Called by {@code configureFilterDelegation} before query execution.
      * Tests may call with {@code null} to reset.
      */
-    public static void setRegistries(FilterProviderRegistry providers, CollectorRegistry collectors) {
-        REGISTRIES.set(providers == null ? null : new Registries(providers, collectors));
+    public static void setHandle(FilterDelegationHandle handle) {
+        HANDLE.set(handle);
     }
 
     // ── Provider lifecycle (cold path, once per query) ────────────────
 
     /**
-     * {@code createProvider(queryBytes, queryBytesLen) -> providerKey|-1}.
+     * {@code createProvider(annotationId) -> providerKey|-1}.
      */
-    public static int createProvider(MemorySegment queryBytesPtr, long queryBytesLen) {
+    public static int createProvider(int annotationId) {
         try {
-            Registries reg = REGISTRIES.get();
-            if (reg == null) {
+            FilterDelegationHandle handle = HANDLE.get();
+            if (handle == null) {
                 return -1;
             }
-            MemorySegment view = queryBytesPtr.reinterpret(queryBytesLen);
-            byte[] bytes = view.toArray(ValueLayout.JAVA_BYTE);
-            return reg.providers().createProvider(bytes);
-        } catch (Throwable t) {
-            LOGGER.error("createProvider failed", t);
+            return handle.createProvider(annotationId);
+        } catch (Throwable throwable) {
+            LOGGER.error("createProvider failed for annotationId=" + annotationId, throwable);
             return -1;
         }
     }
@@ -76,12 +70,12 @@ public static int createProvider(MemorySegment queryBytesPtr, long queryBytesLen
      */
     public static void releaseProvider(int providerKey) {
         try {
-            Registries reg = REGISTRIES.get();
-            if (reg != null) {
-                reg.providers().releaseProvider(providerKey);
+            FilterDelegationHandle handle = HANDLE.get();
+            if (handle != null) {
+                handle.releaseProvider(providerKey);
             }
-        } catch (Throwable t) {
-            LOGGER.error(new ParameterizedMessage("releaseProvider({}) failed", providerKey), t);
+        } catch (Throwable throwable) {
+            LOGGER.error(new ParameterizedMessage("releaseProvider({}) failed", providerKey), throwable);
         }
     }
 
@@ -92,12 +86,12 @@ public static void releaseProvider(int providerKey) {
      */
     public static int createCollector(int providerKey, int segmentOrd, int minDoc, int maxDoc) {
         try {
-            Registries reg = REGISTRIES.get();
-            if (reg == null) {
+            FilterDelegationHandle handle = HANDLE.get();
+            if (handle == null) {
                 return -1;
             }
-            return reg.providers().createCollector(providerKey, segmentOrd, minDoc, maxDoc);
-        } catch (Throwable t) {
+            return handle.createCollector(providerKey, segmentOrd, minDoc, maxDoc);
+        } catch (Throwable throwable) {
             LOGGER.error(
                 new ParameterizedMessage(
                     "createCollector(providerKey={}, seg={}, [{}, {})) failed",
@@ -106,7 +100,7 @@ public static int createCollector(int providerKey, int segmentOrd, int minDoc, i
                     minDoc,
                     maxDoc
                 ),
-                t
+                throwable
             );
             return -1;
         }
@@ -114,26 +108,22 @@ public static int createCollector(int providerKey, int segmentOrd, int minDoc, i
 
     /**
      * {@code collectDocs(collectorKey, minDoc, maxDoc, outPtr, outWordCap) -> wordsWritten|-1}.
-     *
-     * 
      Single map lookup into {@link CollectorRegistry}. The provider
-     * reference is already captured in the {@link CollectorHandle}.
      */
     public static long collectDocs(int collectorKey, int minDoc, int maxDoc, MemorySegment outPtr, long outWordCap) {
         try {
-            Registries reg = REGISTRIES.get();
-            if (reg == null) {
-                return -1L;
-            }
-            CollectorHandle handle = reg.collectors().collector(collectorKey);
+            FilterDelegationHandle handle = HANDLE.get();
             if (handle == null) {
                 return -1L;
             }
             int maxWords = (int) Math.min(outWordCap, (long) Integer.MAX_VALUE);
             MemorySegment view = outPtr.reinterpret((long) maxWords * Long.BYTES);
-            int n = handle.provider().collectDocs(handle.innerCollectorKey(), minDoc, maxDoc, view);
-            return (n < 0) ? -1L : n;
-        } catch (Throwable t) {
-            LOGGER.error(new ParameterizedMessage("collectDocs(collectorKey={}, [{}, {})) failed", collectorKey, minDoc, maxDoc), t);
+            int wordsWritten = handle.collectDocs(collectorKey, minDoc, maxDoc, view);
+            return (wordsWritten < 0) ? -1L : wordsWritten;
+        } catch (Throwable throwable) {
+            LOGGER.error(
+                new ParameterizedMessage("collectDocs(collectorKey={}, [{}, {})) failed", collectorKey, minDoc, maxDoc),
+                throwable
+            );
             return -1L;
         }
     }
@@ -143,18 +133,12 @@ public static long collectDocs(int collectorKey, int minDoc, int maxDoc, MemoryS
      */
     public static void releaseCollector(int collectorKey) {
         try {
-            Registries reg = REGISTRIES.get();
-            if (reg == null) {
-                return;
-            }
-            CollectorHandle handle = reg.collectors().collector(collectorKey);
-            if (handle == null) {
-                return;
+            FilterDelegationHandle handle = HANDLE.get();
+            if (handle != null) {
+                handle.releaseCollector(collectorKey);
             }
-            handle.provider().releaseCollector(handle.innerCollectorKey());
-            reg.collectors().unregisterCollector(collectorKey);
-        } catch (Throwable t) {
-            LOGGER.error(new ParameterizedMessage("releaseCollector({}) failed", collectorKey), t);
+        } catch (Throwable throwable) {
+            LOGGER.error(new ParameterizedMessage("releaseCollector({}) failed", collectorKey), throwable);
         }
     }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
index f0af4c357453f..21477a6e00e1e 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
@@ -73,6 +73,7 @@ public final class NativeBridge {
     private static final MethodHandle CACHE_MANAGER_GET_TOTAL_MEMORY;
     private static final MethodHandle CACHE_MANAGER_CONTAINS_BY_TYPE;
     private static final MethodHandle CREATE_SESSION_CONTEXT;
+    private static final MethodHandle CREATE_SESSION_CONTEXT_INDEXED;
     private static final MethodHandle CLOSE_SESSION_CONTEXT;
     private static final MethodHandle EXECUTE_WITH_CONTEXT;
     private static final MethodHandle CANCEL_QUERY;
@@ -288,6 +289,20 @@ public final class NativeBridge {
             )
         );
 
+        CREATE_SESSION_CONTEXT_INDEXED = linker.downcallHandle(
+            lib.find("df_create_session_context_indexed").orElseThrow(),
+            FunctionDescriptor.of(
+                ValueLayout.JAVA_LONG,
+                ValueLayout.JAVA_LONG,
+                ValueLayout.JAVA_LONG,
+                ValueLayout.ADDRESS,
+                ValueLayout.JAVA_LONG,
+                ValueLayout.JAVA_LONG,
+                ValueLayout.JAVA_INT,
+                ValueLayout.JAVA_INT
+            )
+        );
+
         // i64 df_cache_manager_add_files(runtime_ptr, files_ptr, files_len_ptr, files_count)
         CACHE_MANAGER_ADD_FILES = linker.downcallHandle(
             lib.find("df_cache_manager_add_files").orElseThrow(),
@@ -374,7 +389,7 @@ private static void installFilterTreeCallbacks(Linker linker) {
             MethodHandle createProvider = lookup.findStatic(
                 cb,
                 "createProvider",
-                java.lang.invoke.MethodType.methodType(int.class, java.lang.foreign.MemorySegment.class, long.class)
+                java.lang.invoke.MethodType.methodType(int.class, int.class)
             );
             MethodHandle releaseProvider = lookup.findStatic(
                 cb,
@@ -406,7 +421,7 @@ private static void installFilterTreeCallbacks(Linker linker) {
 
             java.lang.foreign.MemorySegment createProviderStub = linker.upcallStub(
                 createProvider,
-                FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG),
+                FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_INT),
                 arena
             );
             java.lang.foreign.MemorySegment releaseProviderStub = linker.upcallStub(
@@ -743,6 +758,37 @@ public static SessionContextHandle createSessionContext(long readerPtr, long run
         }
     }
 
+    /**
+     * Creates a SessionContext configured for indexed execution with filter delegation.
+     * Registers the delegated_predicate UDF and stores treeShape + delegatedPredicateCount
+     * on the Rust handle for use during execution.
+     */
+    public static SessionContextHandle createSessionContextForIndexedExecution(
+        long readerPtr,
+        long runtimePtr,
+        String tableName,
+        long contextId,
+        int treeShapeOrdinal,
+        int delegatedPredicateCount
+    ) {
+        NativeHandle.validatePointer(readerPtr, "reader");
+        NativeHandle.validatePointer(runtimePtr, "runtime");
+        try (NativeCall call = new NativeCall()) {
+            NativeCall.Str table = call.str(tableName);
+            long ptr = call.invoke(
+                CREATE_SESSION_CONTEXT_INDEXED,
+                readerPtr,
+                runtimePtr,
+                table.segment(),
+                table.len(),
+                contextId,
+                treeShapeOrdinal,
+                delegatedPredicateCount
+            );
+            return new SessionContextHandle(ptr);
+        }
+    }
+
     /**
      * Frees a native {@code SessionContext} handle. Invoked from
      * {@link SessionContextHandle#doCloseNative()} ()} on error / never-executed paths; not called on the
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java
index bcbe2c3ef7c92..c0b7e1eb6ffe0 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java
@@ -54,6 +54,7 @@
 import org.opensearch.analytics.spi.ScalarFunction;
 import org.opensearch.analytics.spi.ScanCapability;
 import org.opensearch.analytics.spi.ShardScanInstructionNode;
+import org.opensearch.analytics.spi.ShardScanWithDelegationInstructionNode;
 import org.opensearch.be.lucene.LuceneAnalyticsBackendPlugin;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
@@ -468,6 +469,11 @@ public Optional createFilterDelegationNode(
                     return Optional.of(new FilterDelegationInstructionNode(treeShape, delegatedPredicateCount, delegatedExpressions));
                 }
 
+                @Override
+                public Optional createShardScanWithDelegationNode(FilterTreeShape treeShape, int delegatedPredicateCount) {
+                    return Optional.of(new ShardScanWithDelegationInstructionNode(treeShape, delegatedPredicateCount));
+                }
+
                 @Override
                 public Optional createPartialAggregateNode() {
                     return Optional.empty();
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/indexfilter/IndexFilterCallbackTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/indexfilter/IndexFilterCallbackTests.java
index 5ea0da63d9a8f..1606b76facbb2 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/indexfilter/IndexFilterCallbackTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/indexfilter/IndexFilterCallbackTests.java
@@ -8,172 +8,174 @@
 
 package org.opensearch.be.datafusion.indexfilter;
 
-import org.opensearch.analytics.spi.IndexFilterProvider;
-import org.opensearch.analytics.spi.IndexFilterProviderFactory;
+import org.opensearch.analytics.spi.FilterDelegationHandle;
 import org.opensearch.test.OpenSearchTestCase;
 
-import java.io.IOException;
 import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.ValueLayout;
 
 /**
- * Tests the Java-side provider/collector registries + FFM callback dispatch
- * glue without going through the full substrait → native pipeline.
+ * Tests the Java-side FFM callback dispatch via {@link FilterTreeCallbacks}
+ * routing to a {@link FilterDelegationHandle} without going through the full
+ * substrait → native pipeline.
  */
 public class IndexFilterCallbackTests extends OpenSearchTestCase {
 
-    private FilterProviderRegistry providers;
-    private CollectorRegistry collectors;
-
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        collectors = new CollectorRegistry();
-        providers = new FilterProviderRegistry(collectors);
-        FilterTreeCallbacks.setRegistries(providers, collectors);
+        FilterTreeCallbacks.setHandle(null);
     }
 
     @Override
     public void tearDown() throws Exception {
-        FilterTreeCallbacks.setRegistries(null, null);
+        FilterTreeCallbacks.setHandle(null);
         super.tearDown();
     }
 
-    public void testCreateCollectReleaseRoundTrip() {
-        MockProvider provider = new MockProvider(new long[] { 0x5L, 0x0L });
-        // Register provider directly via lifecycle, bypassing factory upcall.
-        int providerKey = providers.createProvider(new byte[0]);
-        // That returns -1 because no factory is set. Register manually instead.
-        providerKey = registerProviderDirectly(provider);
-
+    public void testFullRoundTrip() {
+        long[] cannedWords = new long[] { 0x5L, 0x3L };
+        MockHandle handle = new MockHandle(cannedWords);
+        FilterTreeCallbacks.setHandle(handle);
+
+        // createProvider
+        int providerKey = FilterTreeCallbacks.createProvider(42);
+        assertTrue("providerKey >= 0", providerKey >= 0);
+        assertEquals("handle received annotationId", 42, handle.lastAnnotationId);
+
+        // createCollector
+        int collectorKey = FilterTreeCallbacks.createCollector(providerKey, 2, 0, 128);
+        assertTrue("collectorKey >= 0", collectorKey >= 0);
+        assertEquals("handle received providerKey", providerKey, handle.lastProviderKey);
+        assertEquals("handle received segmentOrd", 2, handle.lastSegmentOrd);
+        assertEquals("handle received minDoc", 0, handle.lastMinDoc);
+        assertEquals("handle received maxDoc", 128, handle.lastMaxDoc);
+
+        // collectDocs
         try (Arena arena = Arena.ofConfined()) {
-            int collectorKey = FilterTreeCallbacks.createCollector(providerKey, 0, 0, 64);
-            assertTrue("collectorKey >= 0", collectorKey >= 0);
-
-            MemorySegment buf = arena.allocate(Long.BYTES);
-            int n = provider.collectDocs(collectors.collector(collectorKey).innerCollectorKey(), 0, 64, buf);
-            assertEquals(1, n);
+            MemorySegment buf = arena.allocate(Long.BYTES * 2);
+            long wordsWritten = FilterTreeCallbacks.collectDocs(collectorKey, 0, 128, buf, 2);
+            assertEquals("wordsWritten matches canned length", 2L, wordsWritten);
             assertEquals(0x5L, buf.getAtIndex(ValueLayout.JAVA_LONG, 0));
-
-            FilterTreeCallbacks.releaseCollector(collectorKey);
-            assertNull("collector removed from registry", collectors.collector(collectorKey));
-        } finally {
-            FilterTreeCallbacks.releaseProvider(providerKey);
-            assertNull("provider removed from registry", providers.provider(providerKey));
+            assertEquals(0x3L, buf.getAtIndex(ValueLayout.JAVA_LONG, 1));
         }
-    }
 
-    public void testCreateWithUnknownProviderReturnsError() {
-        assertEquals(-1, FilterTreeCallbacks.createCollector(Integer.MAX_VALUE, 0, 0, 16));
+        // releaseCollector
+        FilterTreeCallbacks.releaseCollector(collectorKey);
+        assertEquals("handle received collectorKey for release", collectorKey, handle.lastReleasedCollectorKey);
+
+        // releaseProvider
+        FilterTreeCallbacks.releaseProvider(providerKey);
+        assertEquals("handle received providerKey for release", providerKey, handle.lastReleasedProviderKey);
     }
 
-    public void testReleaseWithUnknownCollectorIsSafe() {
-        FilterTreeCallbacks.releaseCollector(Integer.MAX_VALUE);
+    public void testNoHandleReturnsNegativeOne() {
+        FilterTreeCallbacks.setHandle(null);
+        assertEquals(-1, FilterTreeCallbacks.createProvider(1));
+        assertEquals(-1, FilterTreeCallbacks.createCollector(1, 0, 0, 64));
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment buf = arena.allocate(Long.BYTES);
+            assertEquals(-1L, FilterTreeCallbacks.collectDocs(1, 0, 64, buf, 1));
+        }
     }
 
-    public void testReleaseWithUnknownProviderIsSafe() {
+    public void testReleaseWithNoHandleIsSafe() {
+        FilterTreeCallbacks.setHandle(null);
+        FilterTreeCallbacks.releaseCollector(Integer.MAX_VALUE);
         FilterTreeCallbacks.releaseProvider(Integer.MAX_VALUE);
     }
 
-    public void testCreateProviderDispatchesToRegisteredFactory() throws IOException {
-        byte[] expected = new byte[] { 1, 2, 3, 4 };
-        StubFactory factory = new StubFactory(expected);
-        providers.setFactory(factory);
+    public void testHandleReturningNegativeOnePropagates() {
+        FilterDelegationHandle failingHandle = new FilterDelegationHandle() {
+            @Override
+            public int createProvider(int annotationId) {
+                return -1;
+            }
 
-        try (Arena arena = Arena.ofConfined()) {
-            MemorySegment seg = arena.allocate(expected.length);
-            MemorySegment.copy(expected, 0, seg, ValueLayout.JAVA_BYTE, 0, expected.length);
+            @Override
+            public int createCollector(int providerKey, int segOrd, int minDoc, int maxDoc) {
+                return -1;
+            }
 
-            int key = FilterTreeCallbacks.createProvider(seg, expected.length);
-            assertTrue("providerKey >= 0", key >= 0);
-            assertEquals("factory invoked exactly once", 1, factory.callCount);
+            @Override
+            public int collectDocs(int collectorKey, int minDoc, int maxDoc, MemorySegment out) {
+                return -1;
+            }
 
-            IndexFilterProvider registered = providers.provider(key);
-            assertNotNull("provider registered under returned key", registered);
-            assertSame("registered provider is the one factory produced", factory.lastProvider, registered);
+            @Override
+            public void releaseCollector(int collectorKey) {}
 
-            FilterTreeCallbacks.releaseProvider(key);
-            assertNull(providers.provider(key));
-        }
-    }
+            @Override
+            public void releaseProvider(int providerKey) {}
 
-    public void testCreateProviderWithNoFactoryReturnsError() {
-        // Fresh lifecycle with no factory set.
-        CollectorRegistry emptyColl = new CollectorRegistry();
-        FilterProviderRegistry empty = new FilterProviderRegistry(emptyColl);
-        FilterTreeCallbacks.setRegistries(empty, emptyColl);
+            @Override
+            public void close() {}
+        };
+        FilterTreeCallbacks.setHandle(failingHandle);
+
+        assertEquals(-1, FilterTreeCallbacks.createProvider(1));
+        assertEquals(-1, FilterTreeCallbacks.createCollector(1, 0, 0, 64));
         try (Arena arena = Arena.ofConfined()) {
-            MemorySegment seg = arena.allocate(1);
-            int key = FilterTreeCallbacks.createProvider(seg, 1);
-            assertEquals("no factory → -1", -1, key);
+            MemorySegment buf = arena.allocate(Long.BYTES);
+            assertEquals(-1L, FilterTreeCallbacks.collectDocs(1, 0, 64, buf, 1));
         }
     }
 
-    /**
-     * Helper: register a provider directly into the lifecycle's internal map
-     * for tests that bypass the factory. Uses reflection-free approach by
-     * setting a factory that returns the given provider, calling createProvider,
-     * then the factory is consumed.
-     */
-    private int registerProviderDirectly(IndexFilterProvider provider) {
-        // Use a one-shot factory that returns the given provider.
-        FilterProviderRegistry directLifecycle = new FilterProviderRegistry(collectors);
-        directLifecycle.setFactory(bytes -> provider);
-        int key = directLifecycle.createProvider(new byte[0]);
-        // Swap the lifecycle so FilterTreeCallbacks sees this provider.
-        // We need to keep the collectors registry.
-        this.providers = directLifecycle;
-        FilterTreeCallbacks.setRegistries(directLifecycle, collectors);
-        return key;
-    }
-
-    /** Stub factory that records its input and emits a MockProvider. */
-    private static final class StubFactory implements IndexFilterProviderFactory {
-        private final byte[] expectedBytes;
-        int callCount = 0;
-        IndexFilterProvider lastProvider;
-
-        StubFactory(byte[] expectedBytes) {
-            this.expectedBytes = expectedBytes;
+    /** Mock handle that records arguments and returns canned bitset words. */
+    private static final class MockHandle implements FilterDelegationHandle {
+        private final long[] cannedWords;
+        private int nextKey = 1;
+
+        int lastAnnotationId = -1;
+        int lastProviderKey = -1;
+        int lastSegmentOrd = -1;
+        int lastMinDoc = -1;
+        int lastMaxDoc = -1;
+        int lastCollectorKey = -1;
+        int lastReleasedCollectorKey = -1;
+        int lastReleasedProviderKey = -1;
+
+        MockHandle(long[] cannedWords) {
+            this.cannedWords = cannedWords;
         }
 
         @Override
-        public IndexFilterProvider create(byte[] queryBytes) {
-            callCount++;
-            assertArrayEquals("factory receives the exact bytes from upcall", expectedBytes, queryBytes);
-            lastProvider = new MockProvider(new long[] { 0xAL });
-            return lastProvider;
-        }
-    }
-
-    /** In-memory provider that returns canned bitset words. */
-    private static final class MockProvider implements IndexFilterProvider {
-        private final long[] cannedWords;
-        private int nextCollector = 1;
-
-        MockProvider(long[] cannedWords) {
-            this.cannedWords = cannedWords;
+        public int createProvider(int annotationId) {
+            this.lastAnnotationId = annotationId;
+            return nextKey++;
         }
 
         @Override
-        public int createCollector(int segmentOrd, int minDoc, int maxDoc) {
-            return nextCollector++;
+        public int createCollector(int providerKey, int segmentOrd, int minDoc, int maxDoc) {
+            this.lastProviderKey = providerKey;
+            this.lastSegmentOrd = segmentOrd;
+            this.lastMinDoc = minDoc;
+            this.lastMaxDoc = maxDoc;
+            return nextKey++;
         }
 
         @Override
         public int collectDocs(int collectorKey, int minDoc, int maxDoc, MemorySegment out) {
-            int n = Math.min(cannedWords.length, (int) (out.byteSize() / Long.BYTES));
-            for (int i = 0; i < n; i++) {
+            this.lastCollectorKey = collectorKey;
+            int wordCount = Math.min(cannedWords.length, (int) (out.byteSize() / Long.BYTES));
+            for (int i = 0; i < wordCount; i++) {
                 out.setAtIndex(ValueLayout.JAVA_LONG, i, cannedWords[i]);
             }
-            return n;
+            return wordCount;
         }
 
         @Override
-        public void releaseCollector(int collectorKey) {}
+        public void releaseCollector(int collectorKey) {
+            this.lastReleasedCollectorKey = collectorKey;
+        }
+
+        @Override
+        public void releaseProvider(int providerKey) {
+            this.lastReleasedProviderKey = providerKey;
+        }
 
         @Override
-        public void close() throws IOException {}
+        public void close() {}
     }
 }
diff --git a/sandbox/plugins/analytics-backend-lucene/build.gradle b/sandbox/plugins/analytics-backend-lucene/build.gradle
index 3275ec49f65e5..4ad216c021736 100644
--- a/sandbox/plugins/analytics-backend-lucene/build.gradle
+++ b/sandbox/plugins/analytics-backend-lucene/build.gradle
@@ -11,6 +11,7 @@ apply plugin: 'opensearch.internal-cluster-test'
 opensearchplugin {
     description = 'OpenSearch plugin providing Lucene-based search execution engine'
     classname = 'org.opensearch.be.lucene.LucenePlugin'
+    extendedPlugins = ['analytics-engine']
 }
 
 java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) }
@@ -24,9 +25,9 @@ configurations {
 sourceSets.test.compileClasspath += configurations.calciteTestCompile
 
 dependencies {
-    // Shared types and SPI interfaces (EngineBridge, AnalyticsBackEndPlugin, etc.)
-    // Also provides calcite-core transitively via api.
-    api project(':sandbox:libs:analytics-framework')
+    // Shared types and SPI interfaces — provided at runtime by the parent analytics-engine plugin (extendedPlugins above).
+    compileOnly project(':sandbox:libs:analytics-framework')
+    compileOnly project(':sandbox:plugins:analytics-engine')
 
     implementation "org.apache.logging.log4j:log4j-api:${versions.log4j}"
     implementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java
new file mode 100644
index 0000000000000..fcd4edf7f311c
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/ConversionUtils.java
@@ -0,0 +1,80 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene;
+
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.index.query.QueryBuilder;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Reusable utilities for extracting fields and values from PPL relevance function
+ * RexCall structures and serializing QueryBuilders.
+ *
+ * 
      PPL relevance functions encode arguments as MAP_VALUE_CONSTRUCTOR pairs:
+ * {@code func(MAP('field', $ref), MAP('query', literal), [MAP('param', literal)]...)}
+ * Each MAP has exactly 2 operands: key at index 0, value at index 1.
+ */
+final class ConversionUtils {
+
+    private ConversionUtils() {}
+
+    /**
+     * Extracts field name from a MAP_VALUE_CONSTRUCTOR operand: MAP('field', $inputRef).
+     */
+    static String extractFieldFromRelevanceMap(RexCall call, int operandIndex, List fieldStorage) {
+        RexNode operand = call.getOperands().get(operandIndex);
+        if (operand instanceof RexCall mapCall) {
+            RexNode value = mapCall.getOperands().get(1);
+            if (value instanceof RexInputRef inputRef) {
+                return FieldStorageInfo.resolve(fieldStorage, inputRef.getIndex()).getFieldName();
+            }
+        }
+        if (operand instanceof RexInputRef inputRef) {
+            return FieldStorageInfo.resolve(fieldStorage, inputRef.getIndex()).getFieldName();
+        }
+        throw new IllegalArgumentException("Cannot extract field name from operand " + operandIndex + ": " + operand);
+    }
+
+    /**
+     * Extracts string value from a MAP_VALUE_CONSTRUCTOR operand: MAP('key', 'value').
+     */
+    static String extractStringFromRelevanceMap(RexCall call, int operandIndex) {
+        RexNode operand = call.getOperands().get(operandIndex);
+        if (operand instanceof RexCall mapCall) {
+            RexNode value = mapCall.getOperands().get(1);
+            if (value instanceof RexLiteral literal) {
+                return literal.getValueAs(String.class);
+            }
+        }
+        if (operand instanceof RexLiteral literal) {
+            return literal.getValueAs(String.class);
+        }
+        throw new IllegalArgumentException("Cannot extract string from operand " + operandIndex + ": " + operand);
+    }
+
+    /**
+     * Serializes a QueryBuilder into bytes using NamedWriteable protocol.
+     */
+    static byte[] serializeQueryBuilder(QueryBuilder queryBuilder) {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.writeNamedWriteable(queryBuilder);
+            return BytesReference.toBytes(output.bytes());
+        } catch (IOException exception) {
+            throw new IllegalStateException("Failed to serialize delegated query: " + queryBuilder, exception);
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java
index a99bc90e35f15..5a59dda788db0 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPlugin.java
@@ -8,27 +8,25 @@
 
 package org.opensearch.be.lucene;
 
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.rex.RexNode;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.search.IndexSearcher;
+import org.opensearch.analytics.backend.ShardScanExecutionContext;
 import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin;
 import org.opensearch.analytics.spi.BackendCapabilityProvider;
+import org.opensearch.analytics.spi.CommonExecutionContext;
+import org.opensearch.analytics.spi.DelegatedExpression;
 import org.opensearch.analytics.spi.DelegatedPredicateSerializer;
 import org.opensearch.analytics.spi.DelegationType;
 import org.opensearch.analytics.spi.EngineCapability;
-import org.opensearch.analytics.spi.FieldStorageInfo;
 import org.opensearch.analytics.spi.FieldType;
 import org.opensearch.analytics.spi.FilterCapability;
+import org.opensearch.analytics.spi.FilterDelegationHandle;
 import org.opensearch.analytics.spi.ScalarFunction;
-import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.index.query.MatchQueryBuilder;
 import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.index.query.QueryShardContext;
 
-import java.io.IOException;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -100,12 +98,6 @@ public class LuceneAnalyticsBackendPlugin implements AnalyticsSearchBackendPlugi
         FILTER_CAPS = caps;
     }
 
-    // TODO: add serializers for MATCH_PHRASE, FUZZY, WILDCARD, REGEXP, and standard ops (for performance delegation)
-    private static final Map SERIALIZERS = Map.of(
-        ScalarFunction.MATCH,
-        LuceneAnalyticsBackendPlugin::serializeMatch
-    );
-
     private final LucenePlugin plugin;
 
     public LuceneAnalyticsBackendPlugin(LucenePlugin plugin) {
@@ -137,58 +129,44 @@ public Set acceptedDelegations() {
 
             @Override
             public Map delegatedPredicateSerializers() {
-                return SERIALIZERS;
+                return QuerySerializerRegistry.getSerializers();
             }
         };
     }
 
     private static final Logger LOGGER = LogManager.getLogger(LuceneAnalyticsBackendPlugin.class);
 
-    // ---- Serializers ----
-
-    private static byte[] serializeMatch(RexCall call, List fieldStorage) {
-        String fieldName = extractFieldName(call, 0, fieldStorage);
-        String queryText = extractStringLiteral(call, 1);
-        MatchQueryBuilder queryBuilder = new MatchQueryBuilder(fieldName, queryText);
-        byte[] bytes = serializeQueryBuilder(queryBuilder);
-        LOGGER.debug(
-            "Serialized MATCH delegation: field=[{}], query=[{}], QueryBuilder=[{}], bytes={}",
-            fieldName,
-            queryText,
-            queryBuilder,
-            bytes.length
-        );
-        return bytes;
+    @Override
+    public FilterDelegationHandle getFilterDelegationHandle(List expressions, CommonExecutionContext ctx) {
+        ShardScanExecutionContext shardCtx = (ShardScanExecutionContext) ctx;
+        DirectoryReader directoryReader = shardCtx.getReader().getReader(plugin.getDataFormat(), DirectoryReader.class);
+        IndexSearcher searcher = new IndexSearcher(directoryReader);
+        QueryShardContext queryShardContext = buildMinimalQueryShardContext(shardCtx, searcher);
+        return new LuceneFilterDelegationHandle(expressions, queryShardContext, directoryReader, shardCtx.getNamedWriteableRegistry());
     }
 
-    // ---- Helpers ----
-
-    private static String extractFieldName(RexCall call, int operandIndex, List fieldStorage) {
-        RexNode operand = call.getOperands().get(operandIndex);
-        if (!(operand instanceof RexInputRef inputRef)) {
-            throw new IllegalArgumentException(
-                "Expected RexInputRef at operand " + operandIndex + ", got: " + operand.getClass().getSimpleName()
-            );
-        }
-        return FieldStorageInfo.resolve(fieldStorage, inputRef.getIndex()).getFieldName();
+    private QueryShardContext buildMinimalQueryShardContext(ShardScanExecutionContext ctx, IndexSearcher searcher) {
+        return new QueryShardContext(
+            0,
+            ctx.getIndexSettings(),
+            null,  // bigArrays
+            null,  // bitsetFilterCache
+            null,  // indexFieldDataLookup
+            ctx.getMapperService(),
+            null,  // similarityService
+            null,  // scriptService
+            null,  // xContentRegistry
+            null,  // namedWriteableRegistry
+            null,  // client
+            searcher,
+            System::currentTimeMillis,
+            null,  // clusterAlias
+            s -> true,  // indexNameMatcher
+            () -> true,  // allowExpensiveQueries
+            null   // valuesSourceRegistry
+        );
     }
 
-    private static String extractStringLiteral(RexCall call, int operandIndex) {
-        RexNode operand = call.getOperands().get(operandIndex);
-        if (!(operand instanceof RexLiteral literal)) {
-            throw new IllegalArgumentException(
-                "Expected RexLiteral at operand " + operandIndex + ", got: " + operand.getClass().getSimpleName()
-            );
-        }
-        return literal.getValueAs(String.class);
-    }
+    // ---- Serializers ----
 
-    private static byte[] serializeQueryBuilder(QueryBuilder queryBuilder) {
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            output.writeNamedWriteable(queryBuilder);
-            return BytesReference.toBytes(output.bytes());
-        } catch (IOException exception) {
-            throw new IllegalStateException("Failed to serialize delegated query: " + queryBuilder, exception);
-        }
-    }
 }
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java
new file mode 100644
index 0000000000000..99e06b426eac3
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java
@@ -0,0 +1,201 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.FixedBitSet;
+import org.opensearch.analytics.spi.DelegatedExpression;
+import org.opensearch.analytics.spi.FilterDelegationHandle;
+import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.index.query.QueryShardContext;
+
+import java.io.IOException;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.ValueLayout;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Lucene implementation of {@link FilterDelegationHandle}. Compiles delegated expressions
+ * into Lucene Queries, creates Weights on demand, and produces bitsets via Scorers.
+ *
+ * @opensearch.internal
+ */
+final class LuceneFilterDelegationHandle implements FilterDelegationHandle {
+
+    private static final Logger LOGGER = LogManager.getLogger(LuceneFilterDelegationHandle.class);
+
+    private final Map queriesByAnnotationId;
+    private final DirectoryReader directoryReader;
+    private final List leaves;
+
+    private final ConcurrentHashMap weightsByProviderKey = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap scorersByCollectorKey = new ConcurrentHashMap<>();
+    private final AtomicInteger nextProviderKey = new AtomicInteger(1);
+    private final AtomicInteger nextCollectorKey = new AtomicInteger(1);
+
+    // TODO: NamedWriteableRegistry should ideally come from LucenePlugin.createComponents
+    // instead of being threaded through ShardScanExecutionContext from Core.
+    LuceneFilterDelegationHandle(
+        List expressions,
+        QueryShardContext queryShardContext,
+        DirectoryReader directoryReader,
+        NamedWriteableRegistry namedWriteableRegistry
+    ) {
+        this.directoryReader = directoryReader;
+        this.leaves = directoryReader.leaves();
+        this.queriesByAnnotationId = compileQueries(expressions, queryShardContext, namedWriteableRegistry);
+    }
+
+    private static Map compileQueries(
+        List expressions,
+        QueryShardContext context,
+        NamedWriteableRegistry registry
+    ) {
+        Map queries = new HashMap<>();
+        for (DelegatedExpression expr : expressions) {
+            try {
+                StreamInput rawInput = StreamInput.wrap(expr.getExpressionBytes());
+                StreamInput input = new NamedWriteableAwareStreamInput(rawInput, registry);
+                QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
+                Query query = queryBuilder.toQuery(context);
+                queries.put(expr.getAnnotationId(), query);
+            } catch (IOException exception) {
+                throw new IllegalStateException(
+                    "Failed to deserialize delegated expression for annotationId=" + expr.getAnnotationId(),
+                    exception
+                );
+            }
+        }
+        return queries;
+    }
+
+    @Override
+    public int createProvider(int annotationId) {
+        Query query = queriesByAnnotationId.get(annotationId);
+        if (query == null) {
+            return -1;
+        }
+        try {
+            IndexSearcher searcher = new IndexSearcher(directoryReader);
+            Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
+            int providerKey = nextProviderKey.getAndIncrement();
+            weightsByProviderKey.put(providerKey, weight);
+            return providerKey;
+        } catch (IOException exception) {
+            LOGGER.error("createProvider failed for annotationId=" + annotationId, exception);
+            return -1;
+        }
+    }
+
+    @Override
+    public int createCollector(int providerKey, int segmentOrd, int minDoc, int maxDoc) {
+        Weight weight = weightsByProviderKey.get(providerKey);
+        if (weight == null) {
+            return -1;
+        }
+        try {
+            // TODO: segmentOrd translation — parquet segment ord may differ from Lucene leaf ord
+            LeafReaderContext leaf = leaves.get(segmentOrd);
+            Scorer scorer = weight.scorer(leaf);
+            int collectorKey = nextCollectorKey.getAndIncrement();
+            scorersByCollectorKey.put(collectorKey, new ScorerHandle(scorer, minDoc, maxDoc));
+            return collectorKey;
+        } catch (IOException exception) {
+            LOGGER.error("createCollector failed for providerKey=" + providerKey + ", seg=" + segmentOrd, exception);
+            return -1;
+        }
+    }
+
+    @Override
+    public int collectDocs(int collectorKey, int minDoc, int maxDoc, MemorySegment out) {
+        ScorerHandle handle = scorersByCollectorKey.get(collectorKey);
+        if (handle == null) {
+            return -1;
+        }
+        if (maxDoc <= minDoc) {
+            return 0;
+        }
+        int span = maxDoc - minDoc;
+        FixedBitSet bits = new FixedBitSet(span);
+
+        if (handle.scorer != null) {
+            int scanFrom = Math.max(minDoc, handle.partitionMinDoc);
+            int scanTo = Math.min(maxDoc, handle.partitionMaxDoc);
+
+            if (scanFrom < scanTo) {
+                try {
+                    DocIdSetIterator iterator = handle.scorer.iterator();
+                    int docId = handle.currentDoc;
+                    if (docId != DocIdSetIterator.NO_MORE_DOCS) {
+                        if (docId < scanFrom) {
+                            docId = iterator.advance(scanFrom);
+                        }
+                        while (docId != DocIdSetIterator.NO_MORE_DOCS && docId < scanTo) {
+                            bits.set(docId - minDoc);
+                            docId = iterator.nextDoc();
+                        }
+                        handle.currentDoc = docId;
+                    }
+                } catch (IOException exception) {
+                    LOGGER.warn("IOException during collectDocs, returning partial bitset", exception);
+                }
+            }
+        }
+
+        long[] words = bits.getBits();
+        int wordCount = (span + 63) >>> 6;
+        MemorySegment.copy(words, 0, out, ValueLayout.JAVA_LONG, 0, wordCount);
+        return wordCount;
+    }
+
+    @Override
+    public void releaseCollector(int collectorKey) {
+        scorersByCollectorKey.remove(collectorKey);
+    }
+
+    @Override
+    public void releaseProvider(int providerKey) {
+        weightsByProviderKey.remove(providerKey);
+    }
+
+    @Override
+    public void close() {
+        weightsByProviderKey.clear();
+        scorersByCollectorKey.clear();
+    }
+
+    private static final class ScorerHandle {
+        final Scorer scorer;
+        final int partitionMinDoc;
+        final int partitionMaxDoc;
+        int currentDoc = -1;
+
+        ScorerHandle(Scorer scorer, int partitionMinDoc, int partitionMaxDoc) {
+            this.scorer = scorer;
+            this.partitionMinDoc = partitionMinDoc;
+            this.partitionMaxDoc = partitionMaxDoc;
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java
new file mode 100644
index 0000000000000..13bda07674b22
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/QuerySerializerRegistry.java
@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene;
+
+import org.apache.calcite.rex.RexCall;
+import org.opensearch.analytics.spi.DelegatedPredicateSerializer;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunction;
+import org.opensearch.index.query.MatchQueryBuilder;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Registry of per-function query serializers for delegated predicates.
+ * Each serializer converts a Calcite RexCall into serialized QueryBuilder bytes
+ * that the Lucene backend can deserialize at the data node.
+ *
+ * 
      TODO: add serializers for match_phrase, match_bool_prefix, match_phrase_prefix.
+ * TODO: add multi-field relevance serializers for multi_match, query_string, simple_query_string.
+ */
+final class QuerySerializerRegistry {
+
+    private static final Map SERIALIZERS = Map.of(
+        ScalarFunction.MATCH,
+        QuerySerializerRegistry::serializeMatch
+    );
+
+    private QuerySerializerRegistry() {}
+
+    static Map getSerializers() {
+        return SERIALIZERS;
+    }
+
+    private static byte[] serializeMatch(RexCall call, List fieldStorage) {
+        String fieldName = ConversionUtils.extractFieldFromRelevanceMap(call, 0, fieldStorage);
+        String queryText = ConversionUtils.extractStringFromRelevanceMap(call, 1);
+        // TODO: extract optional params (operator, analyzer, fuzziness) from operands 2+
+        MatchQueryBuilder queryBuilder = new MatchQueryBuilder(fieldName, queryText);
+        return ConversionUtils.serializeQueryBuilder(queryBuilder);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/resources/META-INF/services/org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin b/sandbox/plugins/analytics-backend-lucene/src/main/resources/META-INF/services/org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin
new file mode 100644
index 0000000000000..35ca0dffa7b6e
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/resources/META-INF/services/org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin
@@ -0,0 +1 @@
+org.opensearch.be.lucene.LuceneAnalyticsBackendPlugin
diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java
index 735ce496ba481..20dacd462ce1d 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java
@@ -52,6 +52,7 @@
 import org.opensearch.analytics.spi.ScalarFunction;
 import org.opensearch.analytics.spi.ScanCapability;
 import org.opensearch.analytics.spi.ShardScanInstructionNode;
+import org.opensearch.analytics.spi.ShardScanWithDelegationInstructionNode;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.MappingMetadata;
@@ -308,6 +309,11 @@ public Optional createFilterDelegationNode(
                     return Optional.of(new FilterDelegationInstructionNode(treeShape, delegatedPredicateCount, delegatedExpressions));
                 }
 
+                @Override
+                public Optional createShardScanWithDelegationNode(FilterTreeShape treeShape, int delegatedPredicateCount) {
+                    return Optional.of(new ShardScanWithDelegationInstructionNode(treeShape, delegatedPredicateCount));
+                }
+
                 @Override
                 public Optional createPartialAggregateNode() {
                     return Optional.empty();
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java
index c3a4bbc23c2fe..f7d2e8fb3753d 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/AnalyticsPlugin.java
@@ -97,7 +97,7 @@ public Collection createComponents(
         for (AnalyticsSearchBackendPlugin be : backEnds) {
             backEndsByName.put(be.name(), be);
         }
-        searchService = new AnalyticsSearchService(backEndsByName);
+        searchService = new AnalyticsSearchService(backEndsByName, namedWriteableRegistry);
 
         // Returned as components so Guice can inject them into DefaultPlanExecutor
         // (a HandledTransportAction registered via getActions() — constructed by Guice
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java
index 0b2e0e28dbb6a..c669aba8f850a 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java
@@ -19,12 +19,15 @@
 import org.opensearch.analytics.exec.task.AnalyticsShardTask;
 import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin;
 import org.opensearch.analytics.spi.BackendExecutionContext;
+import org.opensearch.analytics.spi.DelegationDescriptor;
+import org.opensearch.analytics.spi.FilterDelegationHandle;
 import org.opensearch.analytics.spi.FragmentInstructionHandler;
 import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory;
 import org.opensearch.analytics.spi.InstructionNode;
 import org.opensearch.arrow.flight.transport.ArrowAllocatorProvider;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.concurrent.GatedCloseable;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.tasks.TaskCancelledException;
 import org.opensearch.index.engine.exec.IndexReaderProvider;
 import org.opensearch.index.engine.exec.IndexReaderProvider.Reader;
@@ -59,15 +62,25 @@ public class AnalyticsSearchService implements AutoCloseable {
     private final Map backends;
     private final AnalyticsOperationListener listener;
     private final BufferAllocator allocator;
+    private final NamedWriteableRegistry namedWriteableRegistry;
 
     public AnalyticsSearchService(Map backends) {
-        this(backends, List.of());
+        this(backends, List.of(), null);
     }
 
-    public AnalyticsSearchService(Map backends, List listeners) {
+    public AnalyticsSearchService(Map backends, NamedWriteableRegistry namedWriteableRegistry) {
+        this(backends, List.of(), namedWriteableRegistry);
+    }
+
+    public AnalyticsSearchService(
+        Map backends,
+        List listeners,
+        NamedWriteableRegistry namedWriteableRegistry
+    ) {
         this.backends = backends;
         this.listener = new AnalyticsOperationListener.CompositeListener(listeners);
         this.allocator = ArrowAllocatorProvider.newChildAllocator("analytics-search-service", Long.MAX_VALUE);
+        this.namedWriteableRegistry = namedWriteableRegistry;
     }
 
     @Override
@@ -82,7 +95,7 @@ public FragmentExecutionResponse executeFragment(FragmentExecutionRequest reques
     public FragmentExecutionResponse executeFragment(FragmentExecutionRequest request, IndexShard shard, AnalyticsShardTask task) {
         ResolvedFragment resolved = resolveFragment(request, shard);
         long startNanos = System.nanoTime();
-        try (FragmentResources ctx = startFragment(request, resolved, task)) {
+        try (FragmentResources ctx = startFragment(request, resolved, shard, task)) {
             FragmentExecutionResponse response = collectResponse(ctx.stream(), task);
             long tookNanos = System.nanoTime() - startNanos;
             listener.onFragmentSuccess(resolved.queryId, resolved.stageId, resolved.shardIdStr, tookNanos, response.getRows().size());
@@ -99,7 +112,7 @@ public FragmentExecutionResponse executeFragment(FragmentExecutionRequest reques
     public FragmentResources executeFragmentStreaming(FragmentExecutionRequest request, IndexShard shard, AnalyticsShardTask task) {
         ResolvedFragment resolved = resolveFragment(request, shard);
         try {
-            return startFragment(request, resolved, task);
+            return startFragment(request, resolved, shard, task);
         } catch (TaskCancelledException | IllegalStateException | IllegalArgumentException e) {
             listener.onFragmentFailure(resolved.queryId, resolved.stageId, resolved.shardIdStr, e);
             throw e;
@@ -109,13 +122,14 @@ public FragmentResources executeFragmentStreaming(FragmentExecutionRequest reque
         }
     }
 
-    private FragmentResources startFragment(FragmentExecutionRequest request, ResolvedFragment resolved, Task task) throws IOException {
+    private FragmentResources startFragment(FragmentExecutionRequest request, ResolvedFragment resolved, IndexShard shard, Task task)
+        throws IOException {
         GatedCloseable gatedReader = resolved.readerProvider.acquireReader();
         SearchExecEngine engine = null;
         EngineResultStream stream = null;
         BackendExecutionContext backendContext = null;
         try {
-            ShardScanExecutionContext ctx = buildContext(request, gatedReader.get(), resolved.plan, task);
+            ShardScanExecutionContext ctx = buildContext(request, gatedReader.get(), resolved.plan, shard, task);
             AnalyticsSearchBackendPlugin backend = backends.get(resolved.plan.getBackendId());
 
             // Apply instruction handlers in order — each builds upon the previous handler's backend context
@@ -128,6 +142,17 @@ private FragmentResources startFragment(FragmentExecutionRequest request, Resolv
                 }
             }
 
+            // Handle exchange — if plan has delegation, ask accepting backend for handle and pass to driving
+            // TODO: currently assumes single accepting backend. When multiple accepting backends exist
+            // (e.g., Lucene + Tantivy), group expressions by acceptingBackendId and create one handle per group.
+            DelegationDescriptor delegation = resolved.plan.getDelegationDescriptor();
+            if (delegation != null) {
+                String acceptingBackendId = delegation.delegatedExpressions().getFirst().getAcceptingBackendId();
+                AnalyticsSearchBackendPlugin acceptingBackend = backends.get(acceptingBackendId);
+                FilterDelegationHandle handle = acceptingBackend.getFilterDelegationHandle(delegation.delegatedExpressions(), ctx);
+                backend.configureFilterDelegation(handle, backendContext);
+            }
+
             engine = backend.getSearchExecEngineProvider().createSearchExecEngine(ctx, backendContext);
             stream = engine.execute(ctx);
             return new FragmentResources(gatedReader, engine, stream);
@@ -188,11 +213,15 @@ private ShardScanExecutionContext buildContext(
         FragmentExecutionRequest request,
         Reader reader,
         FragmentExecutionRequest.PlanAlternative plan,
+        IndexShard shard,
         Task task
     ) {
         ShardScanExecutionContext ctx = new ShardScanExecutionContext(request.getShardId().getIndexName(), task, reader);
         ctx.setFragmentBytes(plan.getFragmentBytes());
         ctx.setAllocator(allocator);
+        ctx.setMapperService(shard.mapperService());
+        ctx.setIndexSettings(shard.indexSettings());
+        ctx.setNamedWriteableRegistry(namedWriteableRegistry);
         return ctx;
     }
 
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java
index 6bfb308081310..fd137abb95c50 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionRequest.java
@@ -11,6 +11,7 @@
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.analytics.exec.task.AnalyticsShardTask;
+import org.opensearch.analytics.spi.DelegationDescriptor;
 import org.opensearch.analytics.spi.InstructionNode;
 import org.opensearch.analytics.spi.InstructionType;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -108,11 +109,22 @@ public static class PlanAlternative {
         private final String backendId;
         private final byte[] fragmentBytes;
         private final List instructions;
+        private final DelegationDescriptor delegationDescriptor;
 
         public PlanAlternative(String backendId, byte[] fragmentBytes, List instructions) {
+            this(backendId, fragmentBytes, instructions, null);
+        }
+
+        public PlanAlternative(
+            String backendId,
+            byte[] fragmentBytes,
+            List instructions,
+            DelegationDescriptor delegationDescriptor
+        ) {
             this.backendId = backendId;
             this.fragmentBytes = fragmentBytes;
             this.instructions = instructions;
+            this.delegationDescriptor = delegationDescriptor;
         }
 
         public PlanAlternative(StreamInput in) throws IOException {
@@ -126,6 +138,7 @@ public PlanAlternative(StreamInput in) throws IOException {
                 nodes.add(type.readNode(in));
             }
             this.instructions = nodes;
+            this.delegationDescriptor = in.readBoolean() ? new DelegationDescriptor(in) : null;
         }
 
         public void writeTo(StreamOutput out) throws IOException {
@@ -136,6 +149,12 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeEnum(node.type());
                 node.writeTo(out);
             }
+            if (delegationDescriptor != null) {
+                out.writeBoolean(true);
+                delegationDescriptor.writeTo(out);
+            } else {
+                out.writeBoolean(false);
+            }
         }
 
         public String getBackendId() {
@@ -149,5 +168,9 @@ public byte[] getFragmentBytes() {
         public List getInstructions() {
             return instructions;
         }
+
+        public DelegationDescriptor getDelegationDescriptor() {
+            return delegationDescriptor;
+        }
     }
 }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java
index fdabe2581f423..da616cfdce341 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java
@@ -15,7 +15,10 @@
 import org.opensearch.analytics.planner.dag.ShardExecutionTarget;
 import org.opensearch.analytics.planner.dag.Stage;
 import org.opensearch.analytics.planner.dag.StagePlan;
+import org.opensearch.analytics.spi.DelegationDescriptor;
 import org.opensearch.analytics.spi.ExchangeSink;
+import org.opensearch.analytics.spi.InstructionNode;
+import org.opensearch.analytics.spi.ShardScanWithDelegationInstructionNode;
 import org.opensearch.cluster.service.ClusterService;
 
 import java.util.ArrayList;
@@ -76,8 +79,33 @@ public StageExecution createExecution(Stage stage, ExchangeSink sink, QueryConte
     private static List buildPlanAlternatives(Stage stage) {
         List alternatives = new ArrayList<>();
         for (StagePlan plan : stage.getPlanAlternatives()) {
-            alternatives.add(new FragmentExecutionRequest.PlanAlternative(plan.backendId(), plan.convertedBytes(), plan.instructions()));
+            DelegationDescriptor delegationDescriptor = buildDelegationDescriptor(plan);
+            alternatives.add(
+                new FragmentExecutionRequest.PlanAlternative(
+                    plan.backendId(),
+                    plan.convertedBytes(),
+                    plan.instructions(),
+                    delegationDescriptor
+                )
+            );
         }
         return alternatives;
     }
+
+    private static DelegationDescriptor buildDelegationDescriptor(StagePlan plan) {
+        if (plan.delegatedExpressions().isEmpty()) {
+            return null;
+        }
+        // Extract treeShape and count from the ShardScanWithDelegationInstructionNode
+        for (InstructionNode node : plan.instructions()) {
+            if (node instanceof ShardScanWithDelegationInstructionNode delegationNode) {
+                return new DelegationDescriptor(
+                    delegationNode.getTreeShape(),
+                    delegationNode.getDelegatedPredicateCount(),
+                    plan.delegatedExpressions()
+                );
+            }
+        }
+        return null;
+    }
 }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java
index c97489e9d239b..69cdfc440409f 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java
@@ -126,10 +126,13 @@ private static List assembleInstructions(
         RelNode leaf = findLeaf(plan.resolvedFragment());
 
         if (leaf instanceof OpenSearchTableScan) {
-            factory.createShardScanNode().ifPresent(instructions::add);
             List delegated = delegationBytes.getResult();
             if (!delegated.isEmpty()) {
-                factory.createFilterDelegationNode(treeShape, delegated.size(), delegated).ifPresent(instructions::add);
+                // Delegation exists — use ShardScanWithDelegationInstructionNode which carries
+                // treeShape + count for the driving backend to configure its custom scan operator
+                factory.createShardScanWithDelegationNode(treeShape, delegated.size()).ifPresent(instructions::add);
+            } else {
+                factory.createShardScanNode().ifPresent(instructions::add);
             }
             if (plan.resolvedFragment() instanceof OpenSearchAggregate agg && agg.getMode() == AggregateMode.PARTIAL) {
                 factory.createPartialAggregateNode().ifPresent(instructions::add);
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/action/PlanAlternativeSerializationTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/action/PlanAlternativeSerializationTests.java
index bc6d36bd8a62d..ad997c39bd451 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/action/PlanAlternativeSerializationTests.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/action/PlanAlternativeSerializationTests.java
@@ -9,13 +9,14 @@
 package org.opensearch.analytics.exec.action;
 
 import org.opensearch.analytics.spi.DelegatedExpression;
-import org.opensearch.analytics.spi.FilterDelegationInstructionNode;
+import org.opensearch.analytics.spi.DelegationDescriptor;
 import org.opensearch.analytics.spi.FilterTreeShape;
 import org.opensearch.analytics.spi.FinalAggregateInstructionNode;
 import org.opensearch.analytics.spi.InstructionNode;
 import org.opensearch.analytics.spi.InstructionType;
 import org.opensearch.analytics.spi.PartialAggregateInstructionNode;
 import org.opensearch.analytics.spi.ShardScanInstructionNode;
+import org.opensearch.analytics.spi.ShardScanWithDelegationInstructionNode;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.test.OpenSearchTestCase;
@@ -42,60 +43,69 @@ public void testRoundTripWithShardScanOnly() throws IOException {
         assertArrayEquals(new byte[] { 1, 2, 3 }, deserialized.getFragmentBytes());
         assertEquals(1, deserialized.getInstructions().size());
         assertEquals(InstructionType.SETUP_SHARD_SCAN, deserialized.getInstructions().get(0).type());
+        assertNull(deserialized.getDelegationDescriptor());
     }
 
-    public void testRoundTripWithFilterDelegation() throws IOException {
+    public void testRoundTripWithDelegation() throws IOException {
         List expressions = List.of(
             new DelegatedExpression(1, "lucene", new byte[] { 10, 20 }),
             new DelegatedExpression(2, "lucene", new byte[] { 30, 40 })
         );
-        FilterDelegationInstructionNode filterNode = new FilterDelegationInstructionNode(FilterTreeShape.CONJUNCTIVE, 2, expressions);
-        List instructions = List.of(new ShardScanInstructionNode(), filterNode);
+        DelegationDescriptor descriptor = new DelegationDescriptor(FilterTreeShape.CONJUNCTIVE, 2, expressions);
+        ShardScanWithDelegationInstructionNode delegationNode = new ShardScanWithDelegationInstructionNode(FilterTreeShape.CONJUNCTIVE, 2);
+        List instructions = List.of(delegationNode);
         FragmentExecutionRequest.PlanAlternative original = new FragmentExecutionRequest.PlanAlternative(
             "datafusion",
             new byte[] { 5, 6 },
-            instructions
+            instructions,
+            descriptor
         );
 
         FragmentExecutionRequest.PlanAlternative deserialized = roundTrip(original);
 
-        assertEquals(2, deserialized.getInstructions().size());
-        assertEquals(InstructionType.SETUP_SHARD_SCAN, deserialized.getInstructions().get(0).type());
-        assertEquals(InstructionType.SETUP_FILTER_DELEGATION_FOR_INDEX, deserialized.getInstructions().get(1).type());
-
-        FilterDelegationInstructionNode deserializedFilter = (FilterDelegationInstructionNode) deserialized.getInstructions().get(1);
-        assertEquals(FilterTreeShape.CONJUNCTIVE, deserializedFilter.getTreeShape());
-        assertEquals(2, deserializedFilter.getDelegatedPredicateCount());
-        assertEquals(2, deserializedFilter.getDelegatedQueries().size());
-        assertEquals(1, deserializedFilter.getDelegatedQueries().get(0).getAnnotationId());
-        assertEquals("lucene", deserializedFilter.getDelegatedQueries().get(0).getAcceptingBackendId());
-        assertArrayEquals(new byte[] { 10, 20 }, deserializedFilter.getDelegatedQueries().get(0).getExpressionBytes());
+        assertEquals(1, deserialized.getInstructions().size());
+        assertEquals(InstructionType.SETUP_SHARD_SCAN_WITH_DELEGATION, deserialized.getInstructions().get(0).type());
+
+        ShardScanWithDelegationInstructionNode deserializedNode = (ShardScanWithDelegationInstructionNode) deserialized.getInstructions()
+            .get(0);
+        assertEquals(FilterTreeShape.CONJUNCTIVE, deserializedNode.getTreeShape());
+        assertEquals(2, deserializedNode.getDelegatedPredicateCount());
+
+        DelegationDescriptor deserializedDescriptor = deserialized.getDelegationDescriptor();
+        assertNotNull(deserializedDescriptor);
+        assertEquals(FilterTreeShape.CONJUNCTIVE, deserializedDescriptor.treeShape());
+        assertEquals(2, deserializedDescriptor.delegatedPredicateCount());
+        assertEquals(2, deserializedDescriptor.delegatedExpressions().size());
+        assertEquals(1, deserializedDescriptor.delegatedExpressions().get(0).getAnnotationId());
+        assertEquals("lucene", deserializedDescriptor.delegatedExpressions().get(0).getAcceptingBackendId());
+        assertArrayEquals(new byte[] { 10, 20 }, deserializedDescriptor.delegatedExpressions().get(0).getExpressionBytes());
     }
 
     public void testRoundTripWithAllTypes() throws IOException {
         List instructions = List.of(
-            new ShardScanInstructionNode(),
-            new FilterDelegationInstructionNode(
-                FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION,
-                1,
-                List.of(new DelegatedExpression(3, "lucene", new byte[] { 99 }))
-            ),
+            new ShardScanWithDelegationInstructionNode(FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION, 1),
             new PartialAggregateInstructionNode(),
             new FinalAggregateInstructionNode()
         );
+        DelegationDescriptor descriptor = new DelegationDescriptor(
+            FilterTreeShape.INTERLEAVED_BOOLEAN_EXPRESSION,
+            1,
+            List.of(new DelegatedExpression(3, "lucene", new byte[] { 99 }))
+        );
         FragmentExecutionRequest.PlanAlternative original = new FragmentExecutionRequest.PlanAlternative(
             "datafusion",
             new byte[] { 7 },
-            instructions
+            instructions,
+            descriptor
         );
 
         FragmentExecutionRequest.PlanAlternative deserialized = roundTrip(original);
 
-        assertEquals(4, deserialized.getInstructions().size());
-        assertEquals(InstructionType.SETUP_SHARD_SCAN, deserialized.getInstructions().get(0).type());
-        assertEquals(InstructionType.SETUP_FILTER_DELEGATION_FOR_INDEX, deserialized.getInstructions().get(1).type());
-        assertEquals(InstructionType.SETUP_PARTIAL_AGGREGATE, deserialized.getInstructions().get(2).type());
-        assertEquals(InstructionType.SETUP_FINAL_AGGREGATE, deserialized.getInstructions().get(3).type());
+        assertEquals(3, deserialized.getInstructions().size());
+        assertEquals(InstructionType.SETUP_SHARD_SCAN_WITH_DELEGATION, deserialized.getInstructions().get(0).type());
+        assertEquals(InstructionType.SETUP_PARTIAL_AGGREGATE, deserialized.getInstructions().get(1).type());
+        assertEquals(InstructionType.SETUP_FINAL_AGGREGATE, deserialized.getInstructions().get(2).type());
+        assertNotNull(deserialized.getDelegationDescriptor());
     }
 
     private FragmentExecutionRequest.PlanAlternative roundTrip(FragmentExecutionRequest.PlanAlternative original) throws IOException {
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java
index c378b00c6e254..63df4e04a7a88 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockBackend.java
@@ -28,6 +28,7 @@
 import org.opensearch.analytics.spi.ScalarFunctionAdapter;
 import org.opensearch.analytics.spi.ScanCapability;
 import org.opensearch.analytics.spi.ShardScanInstructionNode;
+import org.opensearch.analytics.spi.ShardScanWithDelegationInstructionNode;
 
 import java.util.List;
 import java.util.Map;
@@ -148,6 +149,11 @@ public Optional createFilterDelegationNode(
                 return Optional.of(new FilterDelegationInstructionNode(treeShape, delegatedPredicateCount, delegatedExpressions));
             }
 
+            @Override
+            public Optional createShardScanWithDelegationNode(FilterTreeShape treeShape, int delegatedPredicateCount) {
+                return Optional.of(new ShardScanWithDelegationInstructionNode(treeShape, delegatedPredicateCount));
+            }
+
             @Override
             public Optional createPartialAggregateNode() {
                 return Optional.of(new PartialAggregateInstructionNode());
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java
index 1d1d6de31f427..db56441fd3311 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java
@@ -38,11 +38,11 @@
 import org.opensearch.analytics.spi.DelegatedPredicateSerializer;
 import org.opensearch.analytics.spi.DelegationType;
 import org.opensearch.analytics.spi.FieldStorageInfo;
-import org.opensearch.analytics.spi.FilterDelegationInstructionNode;
 import org.opensearch.analytics.spi.FilterTreeShape;
 import org.opensearch.analytics.spi.FragmentConvertor;
 import org.opensearch.analytics.spi.InstructionType;
 import org.opensearch.analytics.spi.ScalarFunction;
+import org.opensearch.analytics.spi.ShardScanWithDelegationInstructionNode;
 
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -399,19 +399,19 @@ private void assertDelegationResult(
         // Instruction assertions: delegation plans must have SHARD_SCAN + FILTER_DELEGATION_FOR_INDEX
         if (expectedDelegatedCount > 0) {
             assertTrue(
-                "delegation plan must have FILTER_DELEGATION_FOR_INDEX instruction",
-                plan.instructions().stream().anyMatch(node -> node.type() == InstructionType.SETUP_FILTER_DELEGATION_FOR_INDEX)
+                "delegation plan must have SHARD_SCAN_WITH_DELEGATION instruction",
+                plan.instructions().stream().anyMatch(node -> node.type() == InstructionType.SETUP_SHARD_SCAN_WITH_DELEGATION)
             );
-            FilterDelegationInstructionNode filterInstruction = (FilterDelegationInstructionNode) plan.instructions()
+            ShardScanWithDelegationInstructionNode filterInstruction = (ShardScanWithDelegationInstructionNode) plan.instructions()
                 .stream()
-                .filter(node -> node.type() == InstructionType.SETUP_FILTER_DELEGATION_FOR_INDEX)
+                .filter(node -> node.type() == InstructionType.SETUP_SHARD_SCAN_WITH_DELEGATION)
                 .findFirst()
                 .orElseThrow();
             assertEquals("delegatedPredicateCount in instruction", expectedDelegatedCount, filterInstruction.getDelegatedPredicateCount());
             assertEquals(
-                "delegatedExpressions in instruction must match plan",
+                "delegatedPredicateCount matches delegatedExpressions size",
                 plan.delegatedExpressions().size(),
-                filterInstruction.getDelegatedQueries().size()
+                filterInstruction.getDelegatedPredicateCount()
             );
             assertEquals("treeShape in instruction", expectedTreeShape, filterInstruction.getTreeShape());
         }
diff --git a/sandbox/plugins/composite-engine/build.gradle b/sandbox/plugins/composite-engine/build.gradle
index a4f2675cc013c..84b1d6be2e635 100644
--- a/sandbox/plugins/composite-engine/build.gradle
+++ b/sandbox/plugins/composite-engine/build.gradle
@@ -51,4 +51,5 @@ dependencies {
   internalClusterTestImplementation project(':sandbox:plugins:parquet-data-format')
   internalClusterTestImplementation project(':sandbox:plugins:analytics-backend-lucene')
   internalClusterTestImplementation project(':sandbox:plugins:analytics-backend-datafusion')
+  internalClusterTestImplementation project(':sandbox:libs:analytics-framework')
 }
diff --git a/sandbox/qa/analytics-engine-rest/build.gradle b/sandbox/qa/analytics-engine-rest/build.gradle
index d636582c7b911..1d0bbbd4a4518 100644
--- a/sandbox/qa/analytics-engine-rest/build.gradle
+++ b/sandbox/qa/analytics-engine-rest/build.gradle
@@ -6,6 +6,8 @@
  * compatible open source license.
  */
 
+import org.opensearch.gradle.test.RestIntegTestTask
+
 apply plugin: 'opensearch.testclusters'
 apply plugin: 'opensearch.standalone-rest-test'
 apply plugin: 'opensearch.rest-test'
@@ -30,37 +32,84 @@ dependencies {
     testImplementation project(':sandbox:plugins:test-ppl-frontend')
 }
 
-testClusters.integTest {
-    plugin ':plugins:arrow-flight-rpc'
-    plugin ':sandbox:plugins:analytics-engine'
-    plugin ':sandbox:plugins:analytics-backend-datafusion'
-    plugin ':sandbox:plugins:analytics-backend-lucene'
-    plugin ':sandbox:plugins:dsl-query-executor'
-    plugin ':sandbox:plugins:composite-engine'
-    plugin ':sandbox:plugins:parquet-data-format'
-    plugin ':sandbox:plugins:test-ppl-frontend'
+// ── Shared cluster configuration closure ─────────────────────────────────────
+// All test clusters share the same plugin set and JVM flags; only node count
+// and feature-specific settings differ per task.
+def configureAnalyticsCluster = { cluster ->
+    cluster.plugin ':plugins:arrow-flight-rpc'
+    cluster.plugin ':sandbox:plugins:analytics-engine'
+    cluster.plugin ':sandbox:plugins:analytics-backend-datafusion'
+    cluster.plugin ':sandbox:plugins:analytics-backend-lucene'
+    cluster.plugin ':sandbox:plugins:dsl-query-executor'
+    cluster.plugin ':sandbox:plugins:composite-engine'
+    cluster.plugin ':sandbox:plugins:parquet-data-format'
+    cluster.plugin ':sandbox:plugins:test-ppl-frontend'
 
     // Arrow/Flight JVM flags for DataFusion native library
-    jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED'
-    jvmArgs '--enable-native-access=ALL-UNNAMED'
+    cluster.jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED'
+    cluster.jvmArgs '--enable-native-access=ALL-UNNAMED'
 
     // Arrow memory allocator needs Netty unsafe access on JDK 25; mirrors
     // gradle/run.gradle's arrow-flight-rpc overrides so AnalyticsSearchService
     // can construct its RootAllocator at node start.
-    systemProperty 'io.netty.allocator.numDirectArenas', '1'
-    systemProperty 'io.netty.noUnsafe', 'false'
-    systemProperty 'io.netty.tryUnsafe', 'true'
-    systemProperty 'io.netty.tryReflectionSetAccessible', 'true'
+    cluster.systemProperty 'io.netty.allocator.numDirectArenas', '1'
+    cluster.systemProperty 'io.netty.noUnsafe', 'false'
+    cluster.systemProperty 'io.netty.tryUnsafe', 'true'
+    cluster.systemProperty 'io.netty.tryReflectionSetAccessible', 'true'
 
     // Native library path for DataFusion
-    systemProperty 'java.library.path', "${project(':sandbox:libs:dataformat-native').ext.nativeLibPath.parent}"
+    cluster.systemProperty 'java.library.path', "${project(':sandbox:libs:dataformat-native').ext.nativeLibPath.parent}"
 
     // Enable pluggable dataformat feature flag
-    systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true'
+    cluster.systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true'
+}
+
+// ── Default integTest cluster ────────────────────────────────────────────────
+// TODO: enable numberOfNodes = 2 once partial aggs is handled
+testClusters.integTest {
+    configureAnalyticsCluster(delegate)
 }
 
 integTest {
     systemProperty 'tests.security.manager', 'false'
+    exclude '**/CoordinatorReduceMemtableIT.class'
+    exclude '**/StreamingCoordinatorReduceIT.class'
+}
+
+// ── Memtable variant: 2 nodes, datafusion.reduce.input_mode=memtable ─────────
+task integTestMemtable(type: RestIntegTestTask) {
+    description = 'Runs coordinator-reduce tests with memtable sink mode'
+    testClassesDirs = sourceSets.test.output.classesDirs
+    classpath = sourceSets.test.runtimeClasspath
+    filter {
+        includeTestsMatching 'org.opensearch.analytics.qa.CoordinatorReduceMemtableIT'
+    }
+    systemProperty 'tests.security.manager', 'false'
+}
+check.dependsOn(integTestMemtable)
+
+testClusters.integTestMemtable {
+    numberOfNodes = 2
+    configureAnalyticsCluster(delegate)
+    setting 'datafusion.reduce.input_mode', 'memtable'
+}
+
+// ── Streaming variant: 2 nodes, Arrow Flight stream transport enabled ────────
+task integTestStreaming(type: RestIntegTestTask) {
+    description = 'Runs coordinator-reduce tests with Arrow Flight streaming'
+    testClassesDirs = sourceSets.test.output.classesDirs
+    classpath = sourceSets.test.runtimeClasspath
+    filter {
+        includeTestsMatching 'org.opensearch.analytics.qa.StreamingCoordinatorReduceIT'
+    }
+    systemProperty 'tests.security.manager', 'false'
+}
+check.dependsOn(integTestStreaming)
+
+testClusters.integTestStreaming {
+    numberOfNodes = 2
+    configureAnalyticsCluster(delegate)
+    systemProperty 'opensearch.experimental.feature.transport.stream.enabled', 'true'
 }
 
 // Run against an external cluster (no testClusters lifecycle):
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java
new file mode 100644
index 0000000000000..3f472a15ae826
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java
@@ -0,0 +1,128 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * End-to-end smoke test for the streaming coordinator-reduce path:
+ *
+ * 
      + *   PPL → planner → multi-shard SHARD_FRAGMENT dispatch → DataFusion shard scan
+ *       → ExchangeSink.feed → DatafusionReduceSink (Substrait SUM via convertFinalAggFragment)
+ *       → drain → downstream → assembled PPLResponse
+ * 
      
+ *
+ * 
      Builds a parquet-backed composite index with two shards, indexes a small
+ * deterministic dataset, then runs a {@code stats sum(value) as total} aggregate.
+ * The total is a function of the indexed values × shard count; any drift in
+ * shard fan-out, sink wiring, or final-agg merge will show up as a mismatch.
+ *
+ * 
      Requires a 2-node cluster (configured in build.gradle) so that shards
+ * are distributed across nodes, exercising the coordinator-reduce path.
+ */
+public class CoordinatorReduceIT extends AnalyticsRestTestCase {
+
+    private static final String INDEX = "coord_reduce_e2e";
+    private static final int NUM_SHARDS = 2;
+    private static final int DOCS_PER_SHARD = 10;
+    private static final int VALUE = 7;
+
+    /**
+     * {@code source = T | stats sum(value) as total} on a 2-shard parquet-backed index
+     * → coordinator-reduce path runs the final SUM via DatafusionReduceSink
+     * and returns the deterministic total.
+     */
+    public void testScalarSumAcrossShards() throws Exception {
+        createParquetBackedIndex();
+        indexDeterministicDocs();
+
+        Map result = executePPL("source = " + INDEX + " | stats sum(value) as total");
+
+        @SuppressWarnings("unchecked")
+        List columns = (List) result.get("columns");
+        assertNotNull("columns must not be null", columns);
+        assertTrue("columns must contain 'total', got " + columns, columns.contains("total"));
+
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) result.get("rows");
+        assertNotNull("rows must not be null", rows);
+        assertEquals("scalar agg must return exactly 1 row", 1, rows.size());
+
+        int idx = columns.indexOf("total");
+        Object cell = rows.get(0).get(idx);
+        assertNotNull("SUM(value) cell must not be null — coordinator-reduce returned no value", cell);
+        long actual = ((Number) cell).longValue();
+        long expected = (long) VALUE * NUM_SHARDS * DOCS_PER_SHARD;
+        assertEquals(
+            "SUM(value) across " + NUM_SHARDS + " shards × " + DOCS_PER_SHARD + " docs × value=" + VALUE + " = " + expected,
+            expected,
+            actual
+        );
+    }
+
+    private void createParquetBackedIndex() throws Exception {
+        try {
+            client().performRequest(new Request("DELETE", "/" + INDEX));
+        } catch (Exception ignored) {}
+
+        String body = "{"
+            + "\"settings\": {"
+            + "  \"number_of_shards\": " + NUM_SHARDS + ","
+            + "  \"number_of_replicas\": 0,"
+            + "  \"index.pluggable.dataformat.enabled\": true,"
+            + "  \"index.pluggable.dataformat\": \"composite\","
+            + "  \"index.composite.primary_data_format\": \"parquet\","
+            + "  \"index.composite.secondary_data_formats\": \"\""
+            + "},"
+            + "\"mappings\": {"
+            + "  \"properties\": {"
+            + "    \"value\": { \"type\": \"integer\" }"
+            + "  }"
+            + "}"
+            + "}";
+
+        Request createIndex = new Request("PUT", "/" + INDEX);
+        createIndex.setJsonEntity(body);
+        Map response = assertOkAndParse(client().performRequest(createIndex), "Create index");
+        assertEquals("index creation must be acknowledged", true, response.get("acknowledged"));
+
+        Request health = new Request("GET", "/_cluster/health/" + INDEX);
+        health.addParameter("wait_for_status", "green");
+        health.addParameter("timeout", "30s");
+        client().performRequest(health);
+    }
+
+    private void indexDeterministicDocs() throws Exception {
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
+        StringBuilder bulk = new StringBuilder();
+        for (int i = 0; i < total; i++) {
+            bulk.append("{\"index\": {\"_id\": \"").append(i).append("\"}}\n");
+            bulk.append("{\"value\": ").append(VALUE).append("}\n");
+        }
+
+        Request bulkRequest = new Request("POST", "/" + INDEX + "/_bulk");
+        bulkRequest.setJsonEntity(bulk.toString());
+        bulkRequest.addParameter("refresh", "true");
+        client().performRequest(bulkRequest);
+
+        client().performRequest(new Request("POST", "/" + INDEX + "/_flush?force=true"));
+    }
+
+    private Map executePPL(String ppl) throws Exception {
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + ppl + "\"}");
+        Response response = client().performRequest(request);
+        return entityAsMap(response);
+    }
+}
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceMemtableIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceMemtableIT.java
new file mode 100644
index 0000000000000..d0d4d31d70128
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceMemtableIT.java
@@ -0,0 +1,111 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Memtable variant of {@link CoordinatorReduceIT}. Identical query and assertion, but the cluster
+ * starts with {@code datafusion.reduce.input_mode=memtable} so the coordinator-reduce path uses
+ * DatafusionMemtableReduceSink instead of the streaming sink. Verifies the sink dispatch
+ * wiring and the buffered memtable handoff against a real multi-shard scan.
+ *
+ * 
      Requires a dedicated cluster configuration with {@code datafusion.reduce.input_mode=memtable}
+ * (configured via the {@code integTestMemtable} task in build.gradle).
+ */
+public class CoordinatorReduceMemtableIT extends AnalyticsRestTestCase {
+
+    private static final String INDEX = "coord_reduce_memtable_e2e";
+    private static final int NUM_SHARDS = 2;
+    private static final int DOCS_PER_SHARD = 10;
+    private static final int VALUE = 7;
+
+    public void testScalarSumAcrossShardsViaMemtable() throws Exception {
+        createParquetBackedIndex();
+        indexDeterministicDocs();
+
+        Map result = executePPL("source = " + INDEX + " | stats sum(value) as total");
+
+        @SuppressWarnings("unchecked")
+        List columns = (List) result.get("columns");
+        assertNotNull("columns must not be null", columns);
+        assertTrue("columns must contain 'total', got " + columns, columns.contains("total"));
+
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) result.get("rows");
+        assertNotNull("rows must not be null", rows);
+        assertEquals("scalar agg must return exactly 1 row", 1, rows.size());
+
+        int idx = columns.indexOf("total");
+        Object cell = rows.get(0).get(idx);
+        assertNotNull("SUM(value) cell must not be null — memtable coordinator-reduce returned no value", cell);
+        long actual = ((Number) cell).longValue();
+        long expected = (long) VALUE * NUM_SHARDS * DOCS_PER_SHARD;
+        assertEquals("SUM(value) memtable path must match streaming path", expected, actual);
+    }
+
+    private void createParquetBackedIndex() throws Exception {
+        try {
+            client().performRequest(new Request("DELETE", "/" + INDEX));
+        } catch (Exception ignored) {}
+
+        String body = "{"
+            + "\"settings\": {"
+            + "  \"number_of_shards\": " + NUM_SHARDS + ","
+            + "  \"number_of_replicas\": 0,"
+            + "  \"index.pluggable.dataformat.enabled\": true,"
+            + "  \"index.pluggable.dataformat\": \"composite\","
+            + "  \"index.composite.primary_data_format\": \"parquet\","
+            + "  \"index.composite.secondary_data_formats\": \"\""
+            + "},"
+            + "\"mappings\": {"
+            + "  \"properties\": {"
+            + "    \"value\": { \"type\": \"integer\" }"
+            + "  }"
+            + "}"
+            + "}";
+
+        Request createIndex = new Request("PUT", "/" + INDEX);
+        createIndex.setJsonEntity(body);
+        Map response = assertOkAndParse(client().performRequest(createIndex), "Create index");
+        assertEquals("index creation must be acknowledged", true, response.get("acknowledged"));
+
+        Request health = new Request("GET", "/_cluster/health/" + INDEX);
+        health.addParameter("wait_for_status", "green");
+        health.addParameter("timeout", "30s");
+        client().performRequest(health);
+    }
+
+    private void indexDeterministicDocs() throws Exception {
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
+        StringBuilder bulk = new StringBuilder();
+        for (int i = 0; i < total; i++) {
+            bulk.append("{\"index\": {\"_id\": \"").append(i).append("\"}}\n");
+            bulk.append("{\"value\": ").append(VALUE).append("}\n");
+        }
+
+        Request bulkRequest = new Request("POST", "/" + INDEX + "/_bulk");
+        bulkRequest.setJsonEntity(bulk.toString());
+        bulkRequest.addParameter("refresh", "true");
+        client().performRequest(bulkRequest);
+
+        client().performRequest(new Request("POST", "/" + INDEX + "/_flush?force=true"));
+    }
+
+    private Map executePPL(String ppl) throws Exception {
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + ppl + "\"}");
+        Response response = client().performRequest(request);
+        return entityAsMap(response);
+    }
+}
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationIT.java
new file mode 100644
index 0000000000000..7897c6f9eb4c6
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FilterDelegationIT.java
@@ -0,0 +1,104 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * E2E integration test for filter delegation: a MATCH predicate is delegated to Lucene
+ * while DataFusion drives the scan + aggregation.
+ *
+ * 
      Exercises the full path: PPL → planner → ShardScanWithDelegationInstructionNode →
+ * data node dispatch → Lucene FilterDelegationHandle → Rust indexed executor → results.
+ */
+public class FilterDelegationIT extends AnalyticsRestTestCase {
+
+    private static final String INDEX_NAME = "filter_delegation_e2e";
+
+    public void testMatchFilterDelegationWithAggregate() throws Exception {
+        createIndex();
+        indexDocs();
+
+        String ppl = "source = " + INDEX_NAME + " | where match(message, 'hello') | stats sum(value) as total";
+        Map result = executePPL(ppl);
+
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) result.get("rows");
+        assertNotNull("rows must not be null", rows);
+        assertEquals("scalar agg must return exactly 1 row", 1, rows.size());
+
+        // 10 docs with "hello world" and value=5 → total = 50
+        Number total = (Number) rows.get(0).get(0);
+        assertEquals("SUM(value) for MATCH(message, 'hello') docs", 50L, total.longValue());
+    }
+
+    private void createIndex() throws Exception {
+        try {
+            client().performRequest(new Request("DELETE", "/" + INDEX_NAME));
+        } catch (Exception ignored) {}
+
+        String body = "{"
+            + "\"settings\": {"
+            + "  \"number_of_shards\": 1,"
+            + "  \"number_of_replicas\": 0,"
+            + "  \"index.pluggable.dataformat.enabled\": true,"
+            + "  \"index.pluggable.dataformat\": \"composite\","
+            + "  \"index.composite.primary_data_format\": \"parquet\","
+            + "  \"index.composite.secondary_data_formats\": \"lucene\""
+            + "},"
+            + "\"mappings\": {"
+            + "  \"properties\": {"
+            + "    \"message\": { \"type\": \"text\" },"
+            + "    \"value\": { \"type\": \"integer\" }"
+            + "  }"
+            + "}"
+            + "}";
+
+        Request createIndex = new Request("PUT", "/" + INDEX_NAME);
+        createIndex.setJsonEntity(body);
+        Map response = assertOkAndParse(client().performRequest(createIndex), "Create index");
+        assertEquals(true, response.get("acknowledged"));
+
+        Request health = new Request("GET", "/_cluster/health/" + INDEX_NAME);
+        health.addParameter("wait_for_status", "green");
+        health.addParameter("timeout", "30s");
+        client().performRequest(health);
+    }
+
+    private void indexDocs() throws Exception {
+        StringBuilder bulk = new StringBuilder();
+        for (int i = 0; i < 10; i++) {
+            bulk.append("{\"index\": {}}\n");
+            bulk.append("{\"message\": \"hello world\", \"value\": 5}\n");
+        }
+        for (int i = 0; i < 10; i++) {
+            bulk.append("{\"index\": {}}\n");
+            bulk.append("{\"message\": \"goodbye world\", \"value\": 3}\n");
+        }
+
+        Request bulkRequest = new Request("POST", "/" + INDEX_NAME + "/_bulk");
+        bulkRequest.setJsonEntity(bulk.toString());
+        bulkRequest.addParameter("refresh", "true");
+        client().performRequest(bulkRequest);
+
+        // Flush to ensure parquet files are written
+        client().performRequest(new Request("POST", "/" + INDEX_NAME + "/_flush?force=true"));
+    }
+
+    private Map executePPL(String ppl) throws Exception {
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + ppl + "\"}");
+        Response response = client().performRequest(request);
+        return entityAsMap(response);
+    }
+}
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java
new file mode 100644
index 0000000000000..f3133e3c7e2e4
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java
@@ -0,0 +1,118 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Streaming variant of {@link CoordinatorReduceIT}: same 2-shard parquet-backed index and
+ * deterministic dataset, but with Arrow Flight RPC streaming enabled. Exercises the
+ * shard-fragment → Flight → DatafusionReduceSink.feed handoff that previously failed with
+ * "A buffer can only be associated between two allocators that share the same root" on
+ * multi-shard queries.
+ *
+ * 
      Requires a dedicated cluster configuration with the stream transport feature flag enabled
+ * (configured via the {@code integTestStreaming} task in build.gradle).
+ */
+public class StreamingCoordinatorReduceIT extends AnalyticsRestTestCase {
+
+    private static final String INDEX = "coord_reduce_streaming_e2e";
+    private static final int NUM_SHARDS = 2;
+    private static final int DOCS_PER_SHARD = 10;
+    private static final int VALUE = 7;
+
+    /**
+     * {@code source = T} on a 2-shard parquet-backed index with streaming enabled exercises the
+     * coordinator reduce sink's cross-plugin VectorSchemaRoot handoff.
+     */
+    public void testBaselineScanAcrossShards() throws Exception {
+        createParquetBackedIndex();
+        indexDeterministicDocs();
+
+        Map result = executePPL("source = " + INDEX);
+
+        @SuppressWarnings("unchecked")
+        List columns = (List) result.get("columns");
+        assertNotNull("columns must not be null", columns);
+        assertTrue("columns must contain 'value', got " + columns, columns.contains("value"));
+
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) result.get("rows");
+        assertNotNull("rows must not be null", rows);
+
+        int expectedRows = NUM_SHARDS * DOCS_PER_SHARD;
+        assertEquals("all docs across shards must be returned", expectedRows, rows.size());
+
+        int idx = columns.indexOf("value");
+        for (List row : rows) {
+            Object cell = row.get(idx);
+            assertNotNull("value cell must not be null", cell);
+            assertEquals("every doc has value=" + VALUE, (long) VALUE, ((Number) cell).longValue());
+        }
+    }
+
+    private void createParquetBackedIndex() throws Exception {
+        try {
+            client().performRequest(new Request("DELETE", "/" + INDEX));
+        } catch (Exception ignored) {}
+
+        String body = "{"
+            + "\"settings\": {"
+            + "  \"number_of_shards\": " + NUM_SHARDS + ","
+            + "  \"number_of_replicas\": 0,"
+            + "  \"index.pluggable.dataformat.enabled\": true,"
+            + "  \"index.pluggable.dataformat\": \"composite\","
+            + "  \"index.composite.primary_data_format\": \"parquet\","
+            + "  \"index.composite.secondary_data_formats\": \"\""
+            + "},"
+            + "\"mappings\": {"
+            + "  \"properties\": {"
+            + "    \"value\": { \"type\": \"integer\" }"
+            + "  }"
+            + "}"
+            + "}";
+
+        Request createIndex = new Request("PUT", "/" + INDEX);
+        createIndex.setJsonEntity(body);
+        Map response = assertOkAndParse(client().performRequest(createIndex), "Create index");
+        assertEquals("index creation must be acknowledged", true, response.get("acknowledged"));
+
+        Request health = new Request("GET", "/_cluster/health/" + INDEX);
+        health.addParameter("wait_for_status", "green");
+        health.addParameter("timeout", "30s");
+        client().performRequest(health);
+    }
+
+    private void indexDeterministicDocs() throws Exception {
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
+        StringBuilder bulk = new StringBuilder();
+        for (int i = 0; i < total; i++) {
+            bulk.append("{\"index\": {\"_id\": \"").append(i).append("\"}}\n");
+            bulk.append("{\"value\": ").append(VALUE).append("}\n");
+        }
+
+        Request bulkRequest = new Request("POST", "/" + INDEX + "/_bulk");
+        bulkRequest.setJsonEntity(bulk.toString());
+        bulkRequest.addParameter("refresh", "true");
+        client().performRequest(bulkRequest);
+
+        client().performRequest(new Request("POST", "/" + INDEX + "/_flush?force=true"));
+    }
+
+    private Map executePPL(String ppl) throws Exception {
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + ppl + "\"}");
+        Response response = client().performRequest(request);
+        return entityAsMap(response);
+    }
+}

From 878afa4e8d1a412de61a0126497b314b78b61645 Mon Sep 17 00:00:00 2001
From: Kai Huang <105710027+ahkcs@users.noreply.github.com>
Date: Thu, 7 May 2026 19:38:09 -0700
Subject: [PATCH 082/115] [QA] Add ReverseCommandIT for the analytics-engine
 REST path (#21548)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

PPL `reverse` already works on the analytics-engine route with no code
changes. `CalciteRelNodeVisitor.visitReverse` is plan-time only — it either
flips an existing LogicalSort's collation in-place, backtracks through
filter/project nodes to find one upstream and rebuilds the tree with a
reversed Sort, or no-ops when there's nothing to reverse. The output is
always a LogicalSort (or a passthrough), and SORT is already in
ENGINE_CAPS. **This PR is QA-only.**

Eight tests against the in-process QA cluster, exercising all the
analytics-friendly shapes from the v2-side CalciteReverseCommandIT:

| Test | Shape |
|---|---|
| testReverseAfterSort | Sort + reverse — collation flipped in-place. |
| testDoubleReverseRestoresOriginalSort | Sort + reverse + reverse — restored. |
| testReverseWithHead | Sort + reverse + head — limit-aware double-Sort stack. |
| testReverseWithDescendingSort | DESC sort + reverse → ASC. |
| testReverseAfterFilterFindsUpstreamSort | Backtracking through Filter. |
| testReverseAfterEvalFindsUpstreamSort | Backtracking through Project. |
| testReverseAfterAggregationIsNoOp | No collation post-aggregate, no @timestamp → no-op. |
| testReverseAfterAggregationWithSort | Post-aggregate sort + reverse. |

End-to-end pass rate against the runTask cluster (
`opensearch + analytics-engine + opensearch-sql-plugin`,
`:integ-test:analyticsCompatibilityTest --tests
'org.opensearch.sql.calcite.remote.CalciteReverseCommandIT'`):

| | Before | After |
|---|---|---|
| `CalciteReverseCommandIT` (analytics path) | 0 / 27 (no analytics path support) | **19 / 27** |
| `CalciteReverseCommandIT` (v2 path) | 27 / 27 | 27 / 27 (no regression) |

The 19 passing v2-side tests cover every reverse-pipeline shape that
doesn't require pre-conditions outside this PR's scope. The 8 still-failing
tests are blocked on already-tracked out-of-scope categories:

- streamstats window functions (4 tests): ROW_NUMBER, windowed COUNT,
  windowed SUM not registered for the analytics path.
- timechart's SPAN time-bucketing scalar (3 tests).
- TIMESTAMP rendering parity (1 test, testReverseWithTimestampField).

Validates: 8/8 ReverseCommandIT pass; full
`:sandbox:qa:analytics-engine-rest:integTest` suite still green
(**146 tests across 18 ITs**, no regressions); `./gradlew check -p sandbox
-Dsandbox.enabled=true` passes end-to-end.

Signed-off-by: Kai Huang 
---
 .../analytics/qa/ReverseCommandIT.java        | 262 ++++++++++++++++++
 1 file changed, 262 insertions(+)
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ReverseCommandIT.java

diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ReverseCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ReverseCommandIT.java
new file mode 100644
index 0000000000000..70573fad25b9b
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ReverseCommandIT.java
@@ -0,0 +1,262 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Self-contained integration test for PPL {@code reverse} on the analytics-engine route.
+ *
+ * 
      Mirrors {@code CalciteReverseCommandIT} from the {@code opensearch-project/sql}
+ * repository so the analytics-engine path can be verified inside core without
+ * cross-plugin dependencies on the SQL plugin.
+ *
+ * 
      {@code reverse} is plan-time only: {@code CalciteRelNodeVisitor.visitReverse} either
+ *
+ * 
        
+ *   
      • finds an existing {@code LogicalSort} via {@code RelMetadataQuery.collations()}
+ *       (or by backtracking through filter/project nodes) and reverses its collation;
+ *   
      • or, if the row type has an {@code @timestamp} field, sorts {@code DESC} on it;
+ *   
      • or, otherwise, no-ops.
+ * 
      
+ *
+ * The output is always a {@code LogicalSort} with reversed direction (or a passthrough)
+ * — no new operators, no new scalar functions, no aggregates. That means the analytics
+ * route needs zero new wiring to support it: the existing {@code EngineCapability.SORT}
+ * registration in {@code DataFusionAnalyticsBackendPlugin} is enough.
+ *
+ * 
      This IT pins the shapes that go through the analytics path end-to-end: simple
+ * {@code sort + reverse}, {@code sort + reverse + head} (two-Sort-stack which exercises
+ * {@code attachFragmentOnTop} for the limit-aware path), and {@code sort + reverse +
+ * reverse} (double-reverse rebuilding the original sort). Reverse-after-aggregate (no-op)
+ * and reverse-after-eval (where collation propagates through projections) are also
+ * covered.
+ *
+ * 
      Out of scope (failure modes documented in the upstream IT):
+ *
+ * 
        
+ *   
      • {@code testStreamstats*} — streamstats lowers to window functions (ROW_NUMBER /
+ *       windowed COUNT / windowed SUM) which the analytics path does not yet wire.
+ *   
      • {@code testTimechart*} — depends on {@code SPAN} time-bucketing scalar (separate
+ *       out-of-scope bucket).
+ *   
      • {@code testReverseWithTimestampField} — TIMESTAMP rendering across paths.
+ * 
      
+ *
+ * Provisions the {@code calcs} dataset (parquet-backed) once per class via
+ * {@link DatasetProvisioner}.
+ */
+public class ReverseCommandIT extends AnalyticsRestTestCase {
+
+    private static final Dataset DATASET = new Dataset("calcs", "calcs");
+
+    private static boolean dataProvisioned = false;
+
+    private void ensureDataProvisioned() throws IOException {
+        if (dataProvisioned == false) {
+            DatasetProvisioner.provision(client(), DATASET);
+            dataProvisioned = true;
+        }
+    }
+
+    // ── basic sort + reverse — Sort with collation flipped in-place ─────────────
+
+    public void testReverseAfterSort() throws IOException {
+        // Calcs int0 ASC nulls-first: [null × 6, 1, 3, 4, 4, 4, 7, 8, 8, 8, 10, 11].
+        // After reverse, the collation flips to DESC nulls-last (Calcite's reverseCollation
+        // also flips null direction to keep semantics symmetric).
+        assertRowsInOrder(
+            "source=" + DATASET.indexName + " | where isnotnull(int0) | sort int0 | reverse | fields int0",
+            row(11), row(10), row(8), row(8), row(8), row(7), row(4), row(4), row(4), row(3), row(1)
+        );
+    }
+
+    // ── double-reverse — Sort restored to original direction ───────────────────
+
+    public void testDoubleReverseRestoresOriginalSort() throws IOException {
+        assertRowsInOrder(
+            "source=" + DATASET.indexName + " | where isnotnull(int0) | sort int0 | reverse | reverse | fields int0",
+            row(1), row(3), row(4), row(4), row(4), row(7), row(8), row(8), row(8), row(10), row(11)
+        );
+    }
+
+    // ── reverse + head — limit-aware: reverse adds a separate Sort on top ──────
+
+    public void testReverseWithHead() throws IOException {
+        // visitReverse detects the inner Sort has fetch=null in the pure-collation case, so
+        // it replaces the Sort in-place. After `| head 3`, a Sort(fetch=3) sits on top of
+        // the reversed Sort. Top three values from int0 DESC: 11, 10, 8.
+        assertRowsInOrder(
+            "source=" + DATASET.indexName + " | where isnotnull(int0) | sort int0 | reverse | head 3 | fields int0",
+            row(11), row(10), row(8)
+        );
+    }
+
+    // ── reverse with descending sort — flips back to ascending ─────────────────
+
+    public void testReverseWithDescendingSort() throws IOException {
+        // Flipped DESC + reverse → ASC. Lowest three are 1, 3, 4.
+        assertRowsInOrder(
+            "source=" + DATASET.indexName + " | where isnotnull(int0) | sort -int0 | reverse | head 3 | fields int0",
+            row(1), row(3), row(4)
+        );
+    }
+
+    // ── reverse traverses through filter/project to find the upstream sort ─────
+
+    public void testReverseAfterFilterFindsUpstreamSort() throws IOException {
+        // Backtracking case: `sort | where | reverse` — reverse walks past the Filter to find
+        // the LogicalSort and reverses its direction. PlanUtils.insertReversedSortInTree
+        // rebuilds the tree with the reversed Sort below the Filter.
+        // Filter int0 >= 4 keeps {4 ×3, 7, 8 ×3, 10, 11} = 9 rows; reversed sort gives 11,
+        // 10, 8 first.
+        assertRowsInOrder(
+            "source=" + DATASET.indexName + " | sort int0 | where int0 >= 4 | reverse | head 3 | fields int0",
+            row(11), row(10), row(8)
+        );
+    }
+
+    public void testReverseAfterEvalFindsUpstreamSort() throws IOException {
+        // Same backtracking, but through an eval-introduced Project. Sort first by int0 ASC,
+        // then eval doubled = int0 * 2, then reverse. Backtrack walks past Project to find
+        // the Sort, reverses it, and the doubled column propagates through.
+        assertRowsInOrder(
+            "source=" + DATASET.indexName
+                + " | where isnotnull(int0) | sort int0 | eval doubled = int0 * 2 | reverse | head 3"
+                + " | fields int0, doubled",
+            row(11, 22), row(10, 20), row(8, 16)
+        );
+    }
+
+    // ── reverse after aggregation — no-op when collation is destroyed ──────────
+
+    public void testReverseAfterAggregationIsNoOp() throws IOException {
+        // Aggregation destroys input collation, so `reverse` finds no collation and falls
+        // back to the @timestamp branch, which doesn't apply (calcs has no @timestamp), so
+        // it's a no-op. Aggregation row order isn't pinned, so compare as a multiset.
+        assertRowsAnyOrder(
+            "source=" + DATASET.indexName + " | stats count by str0 | reverse",
+            row(2L, "FURNITURE"),
+            row(6L, "OFFICE SUPPLIES"),
+            row(9L, "TECHNOLOGY")
+        );
+    }
+
+    // ── reverse after explicit post-aggregate sort — works through the sort ────
+
+    public void testReverseAfterAggregationWithSort() throws IOException {
+        // Sort after aggregation establishes a fresh collation; reverse flips it.
+        assertRowsInOrder(
+            "source=" + DATASET.indexName + " | stats count by str0 | sort str0 | reverse",
+            row(9L, "TECHNOLOGY"),
+            row(6L, "OFFICE SUPPLIES"),
+            row(2L, "FURNITURE")
+        );
+    }
+
+    // ── helpers ─────────────────────────────────────────────────────────────────
+
+    private static List row(Object... values) {
+        return Arrays.asList(values);
+    }
+
+    @SafeVarargs
+    @SuppressWarnings("varargs")
+    private final void assertRowsInOrder(String ppl, List... expected) throws IOException {
+        Map response = executePpl(ppl);
+        @SuppressWarnings("unchecked")
+        List> actualRows = (List>) response.get("rows");
+        assertNotNull("Response missing 'rows' for query: " + ppl, actualRows);
+        assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size());
+        for (int i = 0; i < expected.length; i++) {
+            List want = expected[i];
+            List got = actualRows.get(i);
+            assertEquals(
+                "Column count mismatch at row " + i + " for query: " + ppl,
+                want.size(),
+                got.size()
+            );
+            for (int j = 0; j < want.size(); j++) {
+                assertCellEquals(
+                    "Cell mismatch at row " + i + ", col " + j + " for query: " + ppl,
+                    want.get(j),
+                    got.get(j)
+                );
+            }
+        }
+    }
+
+    @SafeVarargs
+    @SuppressWarnings("varargs")
+    private final void assertRowsAnyOrder(String ppl, List... expected) throws IOException {
+        Map response = executePpl(ppl);
+        @SuppressWarnings("unchecked")
+        List> actualRows = (List>) response.get("rows");
+        assertNotNull("Response missing 'rows' for query: " + ppl, actualRows);
+        assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size());
+        java.util.List> remaining = new java.util.ArrayList<>(actualRows);
+        outer:
+        for (List want : expected) {
+            for (int i = 0; i < remaining.size(); i++) {
+                if (rowsEqual(want, remaining.get(i))) {
+                    remaining.remove(i);
+                    continue outer;
+                }
+            }
+            fail("Expected row not found for query: " + ppl + " — missing: " + want + " in actual: " + actualRows);
+        }
+    }
+
+    private static boolean rowsEqual(List a, List b) {
+        if (a.size() != b.size()) return false;
+        for (int i = 0; i < a.size(); i++) {
+            Object ax = a.get(i);
+            Object bx = b.get(i);
+            if (ax == null || bx == null) {
+                if (ax != bx) return false;
+                continue;
+            }
+            if (ax instanceof Number && bx instanceof Number) {
+                if (Double.compare(((Number) ax).doubleValue(), ((Number) bx).doubleValue()) != 0) return false;
+                continue;
+            }
+            if (!ax.equals(bx)) return false;
+        }
+        return true;
+    }
+
+    private Map executePpl(String ppl) throws IOException {
+        ensureDataProvisioned();
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}");
+        Response response = client().performRequest(request);
+        return assertOkAndParse(response, "PPL: " + ppl);
+    }
+
+    private static void assertCellEquals(String message, Object expected, Object actual) {
+        if (expected == null || actual == null) {
+            assertEquals(message, expected, actual);
+            return;
+        }
+        if (expected instanceof Number && actual instanceof Number) {
+            double e = ((Number) expected).doubleValue();
+            double a = ((Number) actual).doubleValue();
+            if (Double.compare(e, a) != 0) {
+                fail(message + ": expected <" + expected + "> but was <" + actual + ">");
+            }
+            return;
+        }
+        assertEquals(message, expected, actual);
+    }
+}

From 8560342d1e7369f2f06be1db88a565dc5702e53c Mon Sep 17 00:00:00 2001
From: bowenlan 
Date: Thu, 7 May 2026 20:58:04 -0700
Subject: [PATCH 083/115] Bundle patched Calcite (1.41.0-opensearch-1) into
 analytics-engine (#21501)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Calcite hardcodes Janino's parent classloader, which breaks UDF
resolution inside OpenSearch's plugin classloader hierarchy
(CALCITE-3745). The fix requires a patched calcite-core / calcite-linq4j,
but the patch only affects runtime classloader behavior — API surface,
bytecode references, and license metadata are identical to vanilla.

The patched jars are published at
https://ci.opensearch.org/ci/dbc/snapshots/maven/ as
org.apache.calcite:calcite-core:1.41.0-opensearch-1 (and
calcite-linq4j). This change wires analytics-framework to consume that
coordinate directly:

  - analytics-framework declares
    `api "org.apache.calcite:calcite-core:1.41.0-opensearch-1"` plus
    the OpenSearch Snapshots Maven repo. The patched jars flow through
    runtimeClasspath of every consumer (analytics-engine, downstream
    plugins) with no bespoke swap logic.
  - analytics-engine adds a resolutionStrategy.force for calcite-core
    and calcite-linq4j so unified-query-*'s transitive vanilla 1.41.0
    reference (via calcite-babel) snaps to 1.41.0-opensearch-1,
    keeping a single Calcite on the classpath.
  - License SHA-1 files under
    sandbox/libs/analytics-framework/licenses/ carry the opensearch-1
    coordinate.

No changes to runtimeClasspath shape, no post-processing of the
bundlePlugin Zip task — the patched artifact is just a dependency like
any other. compileJava, test, forbiddenApis, thirdPartyAudit,
dependencyLicenses all see the same jars the zip ships.

Signed-off-by: bowenlan-amzn 
---
 sandbox/build.gradle                          |  25 +++
 sandbox/libs/analytics-framework/build.gradle |   6 +-
 .../calcite-core-1.41.0-opensearch-1.jar.sha1 |   1 +
 .../licenses/calcite-core-1.41.0.jar.sha1     |   1 -
 ...alcite-linq4j-1.41.0-opensearch-1.jar.sha1 |   1 +
 .../licenses/calcite-linq4j-1.41.0.jar.sha1   |   1 -
 sandbox/plugins/analytics-engine/build.gradle |  12 +-
 .../licenses/commons-text-1.11.0.jar.sha1     |   1 +
 .../licenses/commons-text-LICENSE.txt         | 202 ++++++++++++++++++
 .../licenses/commons-text-NOTICE.txt          |   5 +
 10 files changed, 245 insertions(+), 10 deletions(-)
 create mode 100644 sandbox/libs/analytics-framework/licenses/calcite-core-1.41.0-opensearch-1.jar.sha1
 delete mode 100644 sandbox/libs/analytics-framework/licenses/calcite-core-1.41.0.jar.sha1
 create mode 100644 sandbox/libs/analytics-framework/licenses/calcite-linq4j-1.41.0-opensearch-1.jar.sha1
 delete mode 100644 sandbox/libs/analytics-framework/licenses/calcite-linq4j-1.41.0.jar.sha1
 create mode 100644 sandbox/plugins/analytics-engine/licenses/commons-text-1.11.0.jar.sha1
 create mode 100644 sandbox/plugins/analytics-engine/licenses/commons-text-LICENSE.txt
 create mode 100644 sandbox/plugins/analytics-engine/licenses/commons-text-NOTICE.txt

diff --git a/sandbox/build.gradle b/sandbox/build.gradle
index d98658d071941..4a14d0a11a0d7 100644
--- a/sandbox/build.gradle
+++ b/sandbox/build.gradle
@@ -7,6 +7,7 @@
  */
 
 import org.gradle.api.publish.plugins.PublishingPlugin
+import org.opensearch.gradle.VersionProperties
 
 /**
  * This module provides a space in OpenSearch for the community to easily experiment with new ideas and innovate.
@@ -32,9 +33,33 @@ import org.gradle.api.publish.plugins.PublishingPlugin
  */
 def sandboxEnabled = System.getProperty("sandbox.enabled", "false") == "true"
 
+// Patched Calcite coordinate, composed from libs.versions.toml keys.
+// Pinned here once so every sandbox subproject can resolve it without
+// duplicating the repo + force declaration.
+def libVersions = VersionProperties.getVersions()
+def patchedCalciteVersion = "${libVersions['calcite']}-opensearch-${libVersions['calcite_os_rev']}"
+
 subprojects {
   group = 'org.opensearch.sandbox'
 
+  // The patched calcite-core / calcite-linq4j live in the OpenSearch
+  // snapshots Maven repo; analytics-framework advertises them as `api`,
+  // so every consumer of analytics-framework needs both the repo and a
+  // `force` to win over transitive vanilla calcite
+  repositories {
+    maven {
+      name = 'OpenSearch Snapshots'
+      url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/'
+    }
+  }
+
+  configurations.all {
+    resolutionStrategy {
+      force "org.apache.calcite:calcite-core:${patchedCalciteVersion}"
+      force "org.apache.calcite:calcite-linq4j:${patchedCalciteVersion}"
+    }
+  }
+
   if (sandboxEnabled == false) {
     afterEvaluate {
       tasks.configureEach { task ->
diff --git a/sandbox/libs/analytics-framework/build.gradle b/sandbox/libs/analytics-framework/build.gradle
index af29a627a1b0f..f22f55e23cb53 100644
--- a/sandbox/libs/analytics-framework/build.gradle
+++ b/sandbox/libs/analytics-framework/build.gradle
@@ -12,7 +12,11 @@
  * Plugins depend on this; the /modules SPI layer does NOT.
  */
 
-def calciteVersion = '1.41.0'
+// Patched Calcite coordinate. Carries CALCITE-3745 (TCCL-chained classloader
+// for Janino parent CL); API surface is identical to upstream. The OpenSearch
+// Snapshots repo and the resolutionStrategy.force for this coordinate are
+// declared centrally in sandbox/build.gradle's subprojects block.
+def calciteVersion = "${versions.calcite}-opensearch-${versions.calcite_os_rev}"
 
 java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) }
 
diff --git a/sandbox/libs/analytics-framework/licenses/calcite-core-1.41.0-opensearch-1.jar.sha1 b/sandbox/libs/analytics-framework/licenses/calcite-core-1.41.0-opensearch-1.jar.sha1
new file mode 100644
index 0000000000000..a2c7251e69f38
--- /dev/null
+++ b/sandbox/libs/analytics-framework/licenses/calcite-core-1.41.0-opensearch-1.jar.sha1
@@ -0,0 +1 @@
+d4ac2aff0c76b2ea15f47940542999fa42e17d75
\ No newline at end of file
diff --git a/sandbox/libs/analytics-framework/licenses/calcite-core-1.41.0.jar.sha1 b/sandbox/libs/analytics-framework/licenses/calcite-core-1.41.0.jar.sha1
deleted file mode 100644
index 58d7801dd6bca..0000000000000
--- a/sandbox/libs/analytics-framework/licenses/calcite-core-1.41.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0dd7b4be638f0cea174f78cc851322b64d813a1e
diff --git a/sandbox/libs/analytics-framework/licenses/calcite-linq4j-1.41.0-opensearch-1.jar.sha1 b/sandbox/libs/analytics-framework/licenses/calcite-linq4j-1.41.0-opensearch-1.jar.sha1
new file mode 100644
index 0000000000000..ffea6f5d715f8
--- /dev/null
+++ b/sandbox/libs/analytics-framework/licenses/calcite-linq4j-1.41.0-opensearch-1.jar.sha1
@@ -0,0 +1 @@
+e9bcb0ec7ca38a4bff84283b39d4a736c5217645
\ No newline at end of file
diff --git a/sandbox/libs/analytics-framework/licenses/calcite-linq4j-1.41.0.jar.sha1 b/sandbox/libs/analytics-framework/licenses/calcite-linq4j-1.41.0.jar.sha1
deleted file mode 100644
index fd7c6e8a06cf2..0000000000000
--- a/sandbox/libs/analytics-framework/licenses/calcite-linq4j-1.41.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dd399fb76918f6f688b458da2f1c8dd7fc07e3f8
\ No newline at end of file
diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle
index 346aa1385ad70..5f7986c74a2cf 100644
--- a/sandbox/plugins/analytics-engine/build.gradle
+++ b/sandbox/plugins/analytics-engine/build.gradle
@@ -28,13 +28,6 @@ opensearchplugin {
 
 java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) }
 
-repositories {
-  maven {
-    name = 'OpenSearch Snapshots'
-    url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/'
-  }
-}
-
 // Guava comes transitively from calcite-core and unified-query — forbidden on
 // compile classpaths by OpenSearch. Main sources don't need it; test sources do
 // (Calcite API exposes ImmutableList, Predicate). Bypass via custom config.
@@ -110,6 +103,11 @@ dependencies {
   // so bundle it into analytics-engine's own zip.
   runtimeOnly "org.apache.commons:commons-math3:3.6.1"
 
+  // commons-text — Calcite's SqlFunctions. references
+  // org.apache.commons.text.similarity.LevenshteinDistance. Must be loaded via
+  // the same classloader as calcite-core so that  succeeds.
+  runtimeOnly "org.apache.commons:commons-text:1.11.0"
+
   // httpcore5/httpclient5 — Avatica's BuiltInConnectionProperty static initializer references
   // org.apache.hc.core5.util.Timeout. Not provided by arrow-flight-rpc, so bundle here.
   runtimeOnly "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}"
diff --git a/sandbox/plugins/analytics-engine/licenses/commons-text-1.11.0.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/commons-text-1.11.0.jar.sha1
new file mode 100644
index 0000000000000..6f090739df8c6
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/licenses/commons-text-1.11.0.jar.sha1
@@ -0,0 +1 @@
+2bb044b7717ec2eccaf9ea7769c1509054b50e9a
\ No newline at end of file
diff --git a/sandbox/plugins/analytics-engine/licenses/commons-text-LICENSE.txt b/sandbox/plugins/analytics-engine/licenses/commons-text-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/licenses/commons-text-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/sandbox/plugins/analytics-engine/licenses/commons-text-NOTICE.txt b/sandbox/plugins/analytics-engine/licenses/commons-text-NOTICE.txt
new file mode 100644
index 0000000000000..a4c26c8b77307
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/licenses/commons-text-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache Commons Text
+Copyright 2014-2023 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (https://www.apache.org/).

From b6c4b9d1e95ded3c970cbebd13becba85d7963cf Mon Sep 17 00:00:00 2001
From: Kai Huang <105710027+ahkcs@users.noreply.github.com>
Date: Thu, 7 May 2026 21:00:02 -0700
Subject: [PATCH 084/115] [Analytics Backend / DataFusion] Substrait
 Plan.Root.names + CASE + untyped-NULL fixes for multisearch (#21528)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [Analytics Backend / DataFusion] Fix Plan.Root.names mismatch for schema-reshaping wrappers

`DataFusionFragmentConvertor.rewire` always populated the new `Plan.Root.names`
list with the *inner* plan's names. For schema-preserving wrappers (Sort,
Filter, Fetch) those happen to coincide with the wrapper's output schema, so
the bug was hidden. For schema-reshaping wrappers (Aggregate, Project) the
wrapper's output width differs from the inner's, and DataFusion's substrait
consumer rejects the plan in `make_renamed_schema` with:

  Substrait error: Names list must match exactly to nested schema,
  but found {wrapper-width} uses for {inner-width} names

This shape is hit by every PPL `multisearch` query whose coordinator stage
is `Sort(Aggregate(Union(StageInputScan, StageInputScan)))` — the Aggregate
narrows the wide Union row type, and the inner-names override surfaced the
mismatch as a 500.

Fix: derive the new `Plan.Root.names` from the wrapper RelNode's row type
(`fragment.getRowType().getFieldList()`), not the inner plan. Both
`attachFragmentOnTop` and `attachPartialAggOnTop` already have the wrapper
RelNode in scope, so this is a local change with no signature ripple beyond
adding a `List wrapperNames` parameter to `rewire`.

Test coverage:
- `testAttachPartialAggOnTop_PlanRootNamesMatchWrapperOutput` — the
  partial-agg path with a 3-column inner scan and a 1-column wrapper
  aggregate; pins names to the wrapper's output.
- `testAttachFragmentOnTop_AggregateOverMultiColumnInner_PlanRootNamesMatchWrapperOutput`
  — the multisearch coordinator-stage shape (`Aggregate(Union)`).
- `testMultisearchShape_SortOverAggregateOverThreeWayUnion_PlanRootNamesMatchTopOutput`
  — full chain `Sort → Aggregate → Union(Sin × 3)` modeling the
  `testMultisearchWithThreeSubsearches` query plan.
- `testMultisearchShape_SystemLimitOverSortOverAggregateOverUnion_NamesMatchTopOutput`
  — adds the implicit `LogicalSystemLimit` wrapper that
  `QueryService.convertToCalcitePlan` injects at the top of every
  analytics-engine plan, lowered to a Substrait `Fetch`.

End-to-end validation against `:integ-test:integTestRemote --tests
'org.opensearch.sql.calcite.remote.CalciteMultisearchCommandIT'` with
`-Dtests.analytics.force_routing=true -Dtests.analytics.parquet_indices=true`:
21 tests, **5 pass** (was 0/21 before this change). The remaining 16 are
blocked on orthogonal issues that are out of scope here:
- `Field [...] not found.` analyzer errors (8 tests) — pre-existing
  parquet-backed-index field-resolution gap.
- TIMESTAMP / SPAN scalar functions unsupported (4 tests).
- AssertionError on error-message format (2 tests).
- One residual `Names list must match exactly to nested schema, but found
  2 uses for 6 names` on `testMultisearchWithThreeSubsearches` — likely a
  different code path (PARTIAL/FINAL split via `OpenSearchAggregateSplitRule`)
  that the convertor unit tests don't exercise; tracked for follow-up.
- Two timeouts/long-running on the largest queries.

Signed-off-by: Kai Huang 

* [QA] Add MultisearchCommandIT for the analytics-engine REST path

Lands a self-contained QA IT covering PPL `multisearch` so the analytics-route
fix in this PR is exercised inside core without cross-plugin dependencies on
the SQL plugin. Three tests, scoped to the surface analytics already supports
end-to-end:

| Test | Shape |
|---|---|
| `testMultisearchTwoBranchesByCategory` | Basic 2-way Union over int0 buckets — `Union(Filter+Eval+Project, Filter+Eval+Project)` followed by `Aggregate(count by) | sort`. Exercises the same convertReduceFragment chain (`attachFragmentOnTop(Sort, attachFragmentOnTop(Aggregate, convertFinalAggFragment(Union)))`) that the rewire fix targets. |
| `testMultisearchThreeBranchesByStr0` | 3-way Union — the exact `Union(ER, ER, ER)` shape that surfaced the residual "2 uses for 6 names" failure I'd flagged as a follow-up in an earlier draft of the PR description; the rewire fix already covers it on a fresh cluster. |
| `testMultisearchSingleSubsearchRejected` | Arity check — pinned at the parser layer (AstBuilder.visitMultisearchCommand rejects <2 subsearches with `SyntaxCheckException`). Regression-pin against accidental relaxation of that guard. |

Each branch projects to a scalar-only field set (`fields int0, class` /
`fields str0, bucket`) so the union row type sidesteps the calcs dataset's
date/time/datetime columns — `ArrowSchemaFromCalcite.toArrowType` doesn't yet
handle TIMESTAMP, tracked separately.

Bumps `test-ppl-frontend`'s `unified-query-*` dependency from 3.6.0.0-SNAPSHOT
to 3.7.0.0-SNAPSHOT so the bundled PPL grammar exposes the `multisearch`
keyword (along with table/regex/rex/convert added since 3.6). The SQL
Snapshots repo (already declared in the build) carries the published 3.7
artifacts; for local sql-repo HEAD development, run
`./gradlew :ppl:publishUnifiedQueryPublicationToMavenLocal` from the sql repo.
The version bump is independent of the in-flight test-ppl-frontend
UnifiedQueryService.setting() change in #21526 — different files, no conflict.

Validates: 3/3 MultisearchCommandIT pass; full
`:sandbox:qa:analytics-engine-rest:integTest` suite still green
(110 tests across 14 ITs).

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Spotless reformat of testFinalAggInnerStageScanRowType

Single-line spotless reformat in DataFusionFragmentConvertorTests:
the OpenSearchStageInputScan constructor call now fits on one line
(it was previously broken across multiple lines).

Originally bumped sandbox/plugins/analytics-backend-datafusion's
sqlUnifiedQueryVersion 3.6 -> 3.7 to align with test-ppl-frontend, but
the entire internalClusterTest classpath block (including that pin) was
removed upstream by #21555 (dbe4a429434, "Enable Lucene Filter delegation
from Datafusion for Correctness"). The build.gradle hunk dropped during
rebase; only the spotless reformat survives.

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Register CASE in project capabilities + QA IT

Calcite emits SqlKind.CASE for any conditional expression — explicit `eval x =
case(cond, val, …)` in PPL, plus the `count(eval(predicate))` conditional-count
idiom (lowered to COUNT(CASE WHEN predicate THEN … END)) and several other
shapes. Without CASE in `STANDARD_PROJECT_OPS`, the analytics planner rejected
the operator with `No backend supports scalar function [CASE] among
[datafusion]` before substrait emission.

CASE doesn't need a backend adapter: isthmus translates SqlKind.CASE
structurally to a Substrait IfThen rel, and DataFusion's substrait consumer
handles IfThen natively. Just registering the capability is enough.

Adds `testMultisearchEvalCaseProjection` to MultisearchCommandIT to pin the
end-to-end path — multisearch + `eval bucket = case(cond, val else default)` +
stats. Uses an explicit `else` arm so isthmus doesn't have to convert an
untyped NULL literal; the implicit-else `count(eval(…))` shape that the v2-side
testMultisearchSuccessRatePattern uses still hits a separate isthmus
limitation (`Unable to convert the type NULL` from `TypeConverter` on a
SqlTypeName.NULL literal — tracked separately, out of scope here).

Validates: 4/4 MultisearchCommandIT pass; full
:sandbox:qa:analytics-engine-rest:integTest suite still green (111 tests
across 14 ITs).

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Pre-isthmus untyped-NULL rewriter for CASE arms

Calcite emits a `RexLiteral` with `SqlTypeName.NULL` for the implicit ELSE arm
of `CASE WHEN cond THEN val END` — exactly the shape PPL `count(eval(predicate))`
lowers to (`COUNT(CASE WHEN predicate THEN  END)`). Isthmus'
`TypeConverter.toSubstrait` rejects `SqlTypeName.NULL` with `Unable to convert
the type NULL`, blocking the analytics path before substrait emission.

Adds `UntypedNullPreprocessor` — a `RelHomogeneousShuttle` + `RexShuttle` pass
applied in `convertToSubstrait` and `convertStandalone` *before* the
SubstraitRelVisitor sees the plan. Walks every CASE call's value operands
(THEN arms and the ELSE arm) and substitutes any `SqlTypeName.NULL` literal
with a typed null literal matching the CASE's resolved return type. Calcite
already widens the CASE's return type to the leastRestrictive of branches, so
the substituted type is correct by construction.

Scope is intentionally narrow: only CASE call operands are rewritten today.
Other untyped-NULL contexts (function arguments, comparison RHS) are rare in
PPL-generated plans and would need per-operator type inference to do safely;
defer until a concrete test surfaces one.

Test coverage:
- `UntypedNullPreprocessorTests` (4 new):
  * `testCountEvalCaseRewritesElseNullToTypedNull` — the motivating shape
    `COUNT(CASE WHEN cond THEN 1 ELSE null END)`.
  * `testCaseWithThenNullIsAlsoRewritten` — null in the THEN arm.
  * `testCaseConditionOperandUnchanged` — even-index condition operands left alone.
  * `testCountOverRewrittenCaseProjectionTypechecks` — Aggregate(Project(CASE))
    with the rewriter applied still type-checks end-to-end.
- New `testMultisearchCountEvalConditionalCount` in MultisearchCommandIT —
  mirrors the v2-side `CalciteMultisearchCommandIT.testMultisearchSuccessRatePattern`
  shape (`count(eval(predicate))`) end-to-end on the analytics-engine REST path.

Validates: 5/5 MultisearchCommandIT pass; 4/4 new + 12/12 existing
FragmentConvertor unit tests; full :sandbox:qa:analytics-engine-rest:integTest
suite still green (112 tests across 14 ITs).

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Simplify testCaseConditionOperandUnchanged for spotless

The original assertion wrapped a no-op single-iteration loop around an empty-
body RexShuttle whose accept() result is just caseExpr.toString(). Spotless
flagged the empty class body (`new RexShuttle() {}`) as a formatting violation
and tried to wrap it across two lines, which read worse than the underlying
intent — comparing the input CASE expression to the rewriter's output to
prove no-op behavior when no untyped nulls are present.

Replace the loop+shuttle with a direct
`assertEquals(caseExpr.toString(), rewrittenCase.toString())` — same
semantics, cleaner code, no awkward formatting. The test still asserts the
rewriter doesn't touch CASE expressions whose operands are already typed.

Sandbox check (`./gradlew check -p sandbox -Dsandbox.enabled=true`) now
passes end-to-end.

Signed-off-by: Kai Huang 

---------

Signed-off-by: Kai Huang 
---
 .../DataFusionAnalyticsBackendPlugin.java     |   8 +
 .../DataFusionFragmentConvertor.java          |  46 +++-
 .../datafusion/UntypedNullPreprocessor.java   | 120 +++++++++
 .../DataFusionFragmentConvertorTests.java     | 158 +++++++++++
 .../UntypedNullPreprocessorTests.java         | 191 ++++++++++++++
 .../plugins/test-ppl-frontend/build.gradle    |   9 +-
 .../analytics/qa/MultisearchCommandIT.java    | 248 ++++++++++++++++++
 7 files changed, 768 insertions(+), 12 deletions(-)
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UntypedNullPreprocessor.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/UntypedNullPreprocessorTests.java
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java

diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
index fb161135cdb53..b7addffb72644 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
@@ -101,6 +101,14 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
         ScalarFunction.CAST,
         ScalarFunction.CONCAT,
         ScalarFunction.SAFE_CAST,
+        // CASE — Calcite emits CASE WHEN ... THEN ... END for conditional expressions, including
+        // PPL `count(eval(predicate))` (lowered to COUNT(CASE WHEN predicate THEN ... ELSE NULL END))
+        // and explicit `eval x = case(cond, val, ...)`. Isthmus translates SqlKind.CASE structurally
+        // to a Substrait IfThen rel — no extension lookup needed, no adapter required. DataFusion's
+        // substrait consumer handles IfThen natively. Without this entry, the analytics planner
+        // rejects the operator with "No backend supports scalar function [CASE] among [datafusion]"
+        // before substrait emission.
+        ScalarFunction.CASE,
         // ABS / SUBSTRING — `eval x = abs(...)` and `eval s = substring(...)` projections that PPL
         // sort-pushdown moves into the project tree (see CalciteSortCommandIT
         // testPushdownSortExpressionContainsNull and CalcitePPLSortIT
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
index 2e2f7b929a719..0454e013bdd1e 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
@@ -70,7 +70,7 @@
  *   
    11. {@link #attachPartialAggOnTop(RelNode, byte[])} and
  *       {@link #attachFragmentOnTop(RelNode, byte[])} — convert the wrapping
  *       operator standalone, then rewire its input to the decoded inner plan's
- *       root via {@link #rewire(Plan, Rel)}.
      12. 
+ *       root via {@link #rewire(Plan, Rel, List)}.
  * 
  *
  * @opensearch.internal
@@ -126,7 +126,11 @@ public byte[] attachPartialAggOnTop(RelNode partialAggFragment, byte[] innerByte
         LOGGER.debug("Attaching partial aggregate on top of {} inner bytes", innerBytes.length);
         Plan inner = decodePlan(innerBytes);
         Rel wrapper = convertStandalone(partialAggFragment);
-        Plan rewired = rewire(inner, withAggregationPhase(wrapper, Expression.AggregationPhase.INITIAL_TO_INTERMEDIATE));
+        Plan rewired = rewire(
+            inner,
+            withAggregationPhase(wrapper, Expression.AggregationPhase.INITIAL_TO_INTERMEDIATE),
+            fieldNames(partialAggFragment)
+        );
         return serializePlan(rewired);
     }
 
@@ -150,14 +154,19 @@ public byte[] attachFragmentOnTop(RelNode fragment, byte[] innerBytes) {
         // the visitor still walks them top-down to build the wrapper rel.
         RelNode rewritten = rewriteStageInputScans(fragment);
         Rel wrapper = convertStandalone(rewritten);
-        return serializePlan(rewire(inner, wrapper));
+        return serializePlan(rewire(inner, wrapper, fieldNames(fragment)));
     }
 
     // ── Core conversion helpers ─────────────────────────────────────────────────
 
     private byte[] convertToSubstrait(RelNode fragment) {
-        RelRoot root = RelRoot.of(fragment, SqlKind.SELECT);
-        SubstraitRelVisitor visitor = createVisitor(fragment);
+        // Rewrite SqlTypeName.NULL literals (Calcite's untyped null, emitted for the
+        // implicit ELSE arm of CASE) to typed nulls — isthmus' TypeConverter rejects NULL
+        // with "Unable to convert the type NULL". The widening only changes literal type
+        // tags; semantics and field names (used by Plan.Root.names) are unchanged.
+        RelNode preprocessed = UntypedNullPreprocessor.rewrite(fragment);
+        RelRoot root = RelRoot.of(preprocessed, SqlKind.SELECT);
+        SubstraitRelVisitor visitor = createVisitor(preprocessed);
         Rel substraitRel = visitor.apply(root.rel);
 
         List fieldNames = root.fields.stream().map(field -> field.getValue()).toList();
@@ -178,11 +187,15 @@ private byte[] convertToSubstrait(RelNode fragment) {
      * children (e.g. the {@code attachPartialAggOnTop} caller passes a
      * {@code LogicalAggregate} whose input is the already-stripped inner tree); we
      * deliberately discard those children by taking only the outermost rel of the
-     * conversion and rewiring its input during {@link #rewire(Plan, Rel)}.
+     * conversion and rewiring its input during {@link #rewire(Plan, Rel, List)}.
      */
     private Rel convertStandalone(RelNode operator) {
-        SubstraitRelVisitor visitor = createVisitor(operator);
-        return visitor.apply(operator);
+        // Same untyped-NULL preprocessing rationale as convertToSubstrait — the standalone
+        // wrapper conversion is just as susceptible to a SqlTypeName.NULL literal lurking in
+        // a CASE call attached on top of an inner plan.
+        RelNode preprocessed = UntypedNullPreprocessor.rewrite(operator);
+        SubstraitRelVisitor visitor = createVisitor(preprocessed);
+        return visitor.apply(preprocessed);
     }
 
     /**
@@ -191,15 +204,28 @@ private Rel convertStandalone(RelNode operator) {
      * {@code wrapper(inner.root)}. Supports the known single-input wrappers emitted
      * by our four SPI methods ({@link Aggregate}, {@link Sort}, {@link Filter},
      * {@link Project}).
+     *
+     * 
      {@code wrapperNames} must be the wrapper's output column names — typically
+     * derived from the wrapper {@link RelNode}'s row type. For schema-preserving
+     * wrappers (Sort, Filter, Fetch) these match the inner plan's names; for
+     * schema-reshaping wrappers (Aggregate, Project) they don't, and using the
+     * inner's names there causes DataFusion's substrait consumer to reject the
+     * Plan with a "Names list must match exactly to nested schema" error in
+     * {@code make_renamed_schema}.
      */
-    static Plan rewire(Plan inner, Rel wrapper) {
+    static Plan rewire(Plan inner, Rel wrapper, List wrapperNames) {
         if (inner.getRoots().isEmpty()) {
             throw new IllegalArgumentException("Inner Substrait plan has no root relation to rewire under wrapper");
         }
         Plan.Root innerRoot = inner.getRoots().get(0);
         Rel innerRel = innerRoot.getInput();
         Rel rewired = replaceInput(wrapper, innerRel);
-        return Plan.builder().addRoots(Plan.Root.builder().input(rewired).names(innerRoot.getNames()).build()).build();
+        return Plan.builder().addRoots(Plan.Root.builder().input(rewired).names(wrapperNames).build()).build();
+    }
+
+    /** Extracts a wrapper's output column names from its Calcite row type. */
+    private static List fieldNames(RelNode fragment) {
+        return fragment.getRowType().getFieldList().stream().map(RelDataTypeField::getName).toList();
     }
 
     private static Rel replaceInput(Rel wrapper, Rel newInput) {
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UntypedNullPreprocessor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UntypedNullPreprocessor.java
new file mode 100644
index 0000000000000..14681f26f0cf6
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UntypedNullPreprocessor.java
@@ -0,0 +1,120 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.rel.RelHomogeneousShuttle;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Pre-isthmus pass that rewrites untyped {@code NULL} literals
+ * ({@code RexLiteral} with {@link SqlTypeName#NULL}) to typed null literals
+ * inferred from their enclosing operator.
+ *
+ * 
      Calcite emits an untyped NULL for the implicit {@code ELSE} arm of
+ * {@code CASE WHEN cond THEN val END}, which is exactly the shape PPL
+ * {@code count(eval(predicate))} lowers to:
+ *
+ * 
      {@code
+ *   COUNT(CASE WHEN predicate THEN  END)   // ELSE is implicit NULL, type=NULL
+ * }
      
+ *
+ * 
      Isthmus' {@code TypeConverter.toSubstrait} rejects {@link SqlTypeName#NULL}
+ * with {@code "Unable to convert the type NULL"}. The CASE call's resolved
+ * return type already carries the right widened type ({@code NULLABLE BIGINT}
+ * for the count-eval shape, etc), so we substitute that.
+ *
+ * 
      Scope: only CASE call operands are rewritten today. Other untyped-NULL
+ * sites (function arguments, comparison RHS, etc) are rare in PPL-generated
+ * plans and would need per-operator type-inference to do safely; defer until
+ * a concrete test surfaces one.
+ *
+ * @opensearch.internal
+ */
+final class UntypedNullPreprocessor {
+
+    private UntypedNullPreprocessor() {}
+
+    /**
+     * Walk the RelNode tree, applying the rewrite to every node's expressions.
+     * Returns a new tree if any rewrite occurred, otherwise the input unchanged.
+     */
+    static RelNode rewrite(RelNode root) {
+        return root.accept(new RelHomogeneousShuttle() {
+            @Override
+            public RelNode visit(RelNode other) {
+                RelNode visited = super.visit(other);
+                return visited.accept(new CaseUntypedNullShuttle(visited.getCluster().getRexBuilder()));
+            }
+        });
+    }
+
+    /**
+     * Per-node rex shuttle: for every {@code CASE} call encountered, rewrite any
+     * {@link SqlTypeName#NULL}-typed literal operand into a typed null literal
+     * matching the CASE's resolved return type.
+     */
+    private static final class CaseUntypedNullShuttle extends RexShuttle {
+
+        private final RexBuilder rexBuilder;
+
+        CaseUntypedNullShuttle(RexBuilder rexBuilder) {
+            this.rexBuilder = rexBuilder;
+        }
+
+        @Override
+        public RexNode visitCall(RexCall call) {
+            // Recurse first so nested CASE calls are rewritten bottom-up — each inner CASE is
+            // resolved against its own return type, so by the time we look at the outer one,
+            // every operand is already typed.
+            RexCall recursed = (RexCall) super.visitCall(call);
+            if (recursed.getKind() != SqlKind.CASE) {
+                return recursed;
+            }
+            List operands = recursed.getOperands();
+            List rewritten = new ArrayList<>(operands.size());
+            boolean changed = false;
+            for (int i = 0; i < operands.size(); i++) {
+                RexNode op = operands.get(i);
+                if (isCaseValueOperand(i, operands.size()) && isUntypedNullLiteral(op)) {
+                    rewritten.add(rexBuilder.makeNullLiteral(recursed.getType()));
+                    changed = true;
+                } else {
+                    rewritten.add(op);
+                }
+            }
+            return changed ? recursed.clone(recursed.getType(), rewritten) : recursed;
+        }
+
+        /**
+         * Calcite's CASE operand layout is {@code [cond1, val1, cond2, val2, …, condN, valN, else]}.
+         * Conditions sit at even indices except the last operand (the ELSE), which is always
+         * a value. Returns true for value operands (the THEN/ELSE arms).
+         */
+        private static boolean isCaseValueOperand(int index, int total) {
+            return (index % 2 == 1) || (index == total - 1);
+        }
+
+        private static boolean isUntypedNullLiteral(RexNode node) {
+            if (!(node instanceof RexLiteral lit)) {
+                return false;
+            }
+            return lit.isNull() && lit.getType().getSqlTypeName() == SqlTypeName.NULL;
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java
index 549530c4ee99b..00a20a1f109a3 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java
@@ -18,6 +18,7 @@
 import org.apache.calcite.rel.logical.LogicalAggregate;
 import org.apache.calcite.rel.logical.LogicalFilter;
 import org.apache.calcite.rel.logical.LogicalSort;
+import org.apache.calcite.rel.logical.LogicalUnion;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rex.RexBuilder;
@@ -272,6 +273,163 @@ public void testAttachFragmentOnTop_Sort() throws Exception {
         assertEquals(List.of("input-" + childStageId), aggInput.getRead().getNamedTable().getNamesList());
     }
 
+    /**
+     * Regression: {@code attachPartialAggOnTop} must populate {@code Plan.Root.names}
+     * with the *wrapper aggregate's* output column names — not the inner scan's.
+     * Using the inner's names causes DataFusion's substrait consumer to fail
+     * {@code make_renamed_schema} with "Names list must match exactly to nested
+     * schema, but found {wrapper-width} uses for {inner-width} names" whenever
+     * the wrapper reshapes the schema (Aggregate, Project, etc).
+     */
+    public void testAttachPartialAggOnTop_PlanRootNamesMatchWrapperOutput() throws Exception {
+        DataFusionFragmentConvertor convertor = newConvertor();
+
+        // Inner scan has 3 columns; the partial-aggregate emits 1 (sum over col 0).
+        RelNode scan = buildTableScan("test_index", "A", "B", "C");
+        byte[] innerBytes = convertor.convertShardScanFragment("test_index", scan);
+        LogicalAggregate partialAgg = buildSumAggregate(scan, 0);
+
+        byte[] combined = convertor.attachPartialAggOnTop(partialAgg, innerBytes);
+
+        Plan plan = decodeSubstrait(combined);
+        List rootNames = plan.getRelations(0).getRoot().getNamesList();
+        assertEquals(
+            "Plan.Root.names must match the wrapper aggregate's output schema (1 column), not the inner scan's (3 columns)",
+            List.of("sum_col"),
+            rootNames
+        );
+    }
+
+    /**
+     * Regression: {@code attachFragmentOnTop} for an Aggregate over a multi-column
+     * inner plan (e.g. Union of two stage-input scans) must populate
+     * {@code Plan.Root.names} with the aggregate's output names. Mirrors the
+     * multisearch coordinator-stage shape {@code Aggregate(Union(StageInputScan,
+     * StageInputScan))}.
+     */
+    public void testAttachFragmentOnTop_AggregateOverMultiColumnInner_PlanRootNamesMatchWrapperOutput() throws Exception {
+        DataFusionFragmentConvertor convertor = newConvertor();
+
+        // Inner: a final-agg fragment whose StageInputScan rowType is intentionally wide
+        // (3 columns). The aggregate above narrows it to 1 column.
+        RelDataType wideStageRowType = rowType("A", "B", "C");
+        RelNode stageInput = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 0, wideStageRowType, List.of("datafusion"));
+        // For this regression, the inner doesn't need to be a final-agg — a bare scan-shaped
+        // plan with 3-column rowType is enough to surface the wrapper-vs-inner names mismatch.
+        // Use convertFinalAggFragment so the inner Plan.Root.names is the 3-column scan list.
+        RelNode innerStageScan = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 0, wideStageRowType, List.of("datafusion"));
+        // Wrap it in a no-op aggregate so the convertor accepts it as a final-agg fragment shape.
+        // The inner's Plan.Root.names then carries the agg-output (1 col, "sum_col"), but the
+        // *wrapper* we attach above has its own output rowType.
+        LogicalAggregate innerFinalAgg = buildSumAggregate(innerStageScan, 0);
+        byte[] innerBytes = convertor.convertFinalAggFragment(innerFinalAgg);
+
+        // Wrapper: a Project that maps the single inner column to two new aliases — this is
+        // the multisearch-style schema reshape that triggered the bug. We model it as another
+        // aggregate over the same input row type to keep the standalone conversion simple.
+        // The wrapper's output rowType has 1 column ("sum_col") which must end up in
+        // Plan.Root.names regardless of what the wide-row stage-input scan above looked like.
+        RelNode placeholderInput = buildTableScan("__placeholder__", "sum_col");
+        LogicalSort sortWrapper = LogicalSort.create(placeholderInput, RelCollations.of(0), null, null);
+
+        byte[] combined = convertor.attachFragmentOnTop(sortWrapper, innerBytes);
+
+        Plan plan = decodeSubstrait(combined);
+        List rootNames = plan.getRelations(0).getRoot().getNamesList();
+        assertEquals(
+            "Plan.Root.names must reflect the Sort wrapper's output (1 column from the inner agg), "
+                + "not be miswritten with a wider list",
+            List.of("sum_col"),
+            rootNames
+        );
+    }
+
+    /**
+     * Mirror of multisearch's coordinator-stage shape:
+     * {@code Sort(Aggregate(Union(StageInputScan, StageInputScan, StageInputScan)))}.
+     * After the convertor chain runs (convertFinalAggFragment(Union) →
+     * attachFragmentOnTop(Aggregate) → attachFragmentOnTop(Sort)), the outermost
+     * {@code Plan.Root.names} must reflect the Sort's output schema (= the
+     * aggregate's 1-column output), not the inner Union's wider row type.
+     * This was the residual failure signature ("2 uses for 6 names") that the
+     * end-to-end IT surfaced even after the initial rewire fix.
+     */
+    public void testMultisearchShape_SortOverAggregateOverThreeWayUnion_PlanRootNamesMatchTopOutput() throws Exception {
+        DataFusionFragmentConvertor convertor = newConvertor();
+
+        // Inner: Union(Sin, Sin, Sin) — three branches, each 6 columns wide.
+        RelDataType branchRowType = rowType("a", "b", "c", "d", "e", "f");
+        RelNode sin1 = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 1, branchRowType, List.of("datafusion"));
+        RelNode sin2 = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 2, branchRowType, List.of("datafusion"));
+        RelNode sin3 = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 3, branchRowType, List.of("datafusion"));
+        LogicalUnion union = LogicalUnion.create(List.of(sin1, sin2, sin3), true);
+        byte[] unionBytes = convertor.convertFinalAggFragment(union);
+
+        // Aggregate over the union: SUM(a) → 1 column output ("sum_col").
+        // attachFragmentOnTop expects the wrapper to carry its real input so the
+        // standalone visitor can derive types; the input is discarded by rewire.
+        LogicalAggregate aggregate = buildSumAggregate(union, 0);
+        byte[] aggBytes = convertor.attachFragmentOnTop(aggregate, unionBytes);
+
+        // Sort over the aggregate: schema-preserving wrapper.
+        LogicalSort sort = LogicalSort.create(aggregate, RelCollations.of(0), null, null);
+        byte[] combinedBytes = convertor.attachFragmentOnTop(sort, aggBytes);
+
+        Plan plan = decodeSubstrait(combinedBytes);
+        List rootNames = plan.getRelations(0).getRoot().getNamesList();
+        assertEquals(
+            "Plan.Root.names must reflect the Sort wrapper's output (= aggregate's 1-column output), "
+                + "not the inner Union's 6-column row type — multisearch ThreeSubsearches regression",
+            List.of("sum_col"),
+            rootNames
+        );
+    }
+
+    /**
+     * Mirror of multisearch's full coordinator-stage shape including the implicit
+     * query-size LIMIT injected by {@code QueryService.convertToCalcitePlan}. The
+     * actual chain is:
+     *   Sort(fetch=N, collation=∅)          // system limit, lowered to a Substrait Fetch
+     *     Sort(collation=byKey, fetch=∅)    // user-level sort, lowered to a Substrait Sort
+     *       Aggregate(...)
+     *         Union(Sin, Sin, Sin)
+     */
+    public void testMultisearchShape_SystemLimitOverSortOverAggregateOverUnion_NamesMatchTopOutput() throws Exception {
+        DataFusionFragmentConvertor convertor = newConvertor();
+
+        // Inner: Union(Sin, Sin, Sin) — 6-column rows.
+        RelDataType branchRowType = rowType("a", "b", "c", "d", "e", "f");
+        RelNode sin1 = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 1, branchRowType, List.of("datafusion"));
+        RelNode sin2 = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 2, branchRowType, List.of("datafusion"));
+        RelNode sin3 = new OpenSearchStageInputScan(cluster, cluster.traitSet(), 3, branchRowType, List.of("datafusion"));
+        LogicalUnion union = LogicalUnion.create(List.of(sin1, sin2, sin3), true);
+        byte[] unionBytes = convertor.convertFinalAggFragment(union);
+
+        // Aggregate over the union: SUM(a) → 1 column.
+        LogicalAggregate aggregate = buildSumAggregate(union, 0);
+        byte[] aggBytes = convertor.attachFragmentOnTop(aggregate, unionBytes);
+
+        // User-level Sort by the single agg-output column — schema preserved.
+        LogicalSort userSort = LogicalSort.create(aggregate, RelCollations.of(0), null, null);
+        byte[] userSortBytes = convertor.attachFragmentOnTop(userSort, aggBytes);
+
+        // System limit = LogicalSort with no collation + fetch literal. Lowers to a
+        // Substrait Fetch rel (the convertor handles this in replaceInput).
+        org.apache.calcite.rex.RexNode fetchN = rexBuilder.makeLiteral(100, typeFactory.createSqlType(SqlTypeName.INTEGER), true);
+        LogicalSort systemLimit = LogicalSort.create(userSort, RelCollations.EMPTY, null, fetchN);
+        byte[] combinedBytes = convertor.attachFragmentOnTop(systemLimit, userSortBytes);
+
+        Plan plan = decodeSubstrait(combinedBytes);
+        List rootNames = plan.getRelations(0).getRoot().getNamesList();
+        assertEquals(
+            "Plan.Root.names must reflect the system-limit Sort wrapper's output (= 1-column aggregate output), "
+                + "not the inner Union's 6-column row type — the implicit limit at the top of every "
+                + "analytics-engine plan must not surface stale inner-plan names.",
+            List.of("sum_col"),
+            rootNames
+        );
+    }
+
     /**
      * A filter containing {@code delegated_predicate(42)} converts to Substrait
      * with the placeholder preserved as a scalar function call in the FilterRel condition.
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/UntypedNullPreprocessorTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/UntypedNullPreprocessorTests.java
new file mode 100644
index 0000000000000..d2e343d9ee158
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/UntypedNullPreprocessorTests.java
@@ -0,0 +1,191 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.logical.LogicalValues;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.List;
+
+/**
+ * Tests for {@link UntypedNullPreprocessor}. Constructs Calcite RelNode trees that contain
+ * {@code SqlTypeName.NULL} literals in CASE branches and asserts the rewriter widens those
+ * to typed nulls matching the CASE's resolved return type.
+ */
+public class UntypedNullPreprocessorTests extends OpenSearchTestCase {
+
+    private RelDataTypeFactory typeFactory;
+    private RexBuilder rexBuilder;
+    private RelOptCluster cluster;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        typeFactory = new JavaTypeFactoryImpl();
+        rexBuilder = new RexBuilder(typeFactory);
+        HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+        cluster = RelOptCluster.create(planner, rexBuilder);
+    }
+
+    /**
+     * The motivating shape: {@code COUNT(CASE WHEN cond THEN 1 ELSE NULL END)} — Calcite
+     * leaves the implicit ELSE arm as {@link SqlTypeName#NULL}, which isthmus rejects.
+     * After rewrite the ELSE literal must carry the CASE's resolved return type.
+     */
+    public void testCountEvalCaseRewritesElseNullToTypedNull() {
+        // Build: VALUES(true) → Project(CASE WHEN $0 THEN 1 ELSE null END as col)
+        RelDataType nullableInt = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.INTEGER), true);
+        RelDataType boolType = typeFactory.createSqlType(SqlTypeName.BOOLEAN);
+
+        RelNode values = LogicalValues.createOneRow(cluster);
+        RexNode boolLit = rexBuilder.makeLiteral(true, boolType);
+        RexNode oneLit = rexBuilder.makeExactLiteral(java.math.BigDecimal.ONE, nullableInt);
+        // Untyped NULL — RexBuilder.constantNull() returns a literal whose type is NULL.
+        RexNode untypedNull = rexBuilder.constantNull();
+        // Sanity: the source literal is genuinely SqlTypeName.NULL.
+        assertEquals(SqlTypeName.NULL, untypedNull.getType().getSqlTypeName());
+
+        RexNode caseExpr = rexBuilder.makeCall(SqlStdOperatorTable.CASE, boolLit, oneLit, untypedNull);
+        RelDataType caseType = caseExpr.getType();
+        // Calcite resolves the CASE return type to the leastRestrictive of {INT, NULL} — so
+        // the CASE itself is already a nullable INT, but its untyped-NULL child operand is
+        // what isthmus chokes on.
+        assertEquals(SqlTypeName.INTEGER, caseType.getSqlTypeName());
+
+        RelNode project = LogicalProject.create(values, List.of(), List.of(caseExpr), List.of("col"), java.util.Set.of());
+
+        RelNode rewritten = UntypedNullPreprocessor.rewrite(project);
+
+        // Walk the rewritten Project's only expression: the CASE's ELSE arm must now be a
+        // typed null whose type matches the CASE's return type (nullable INT), not NULL.
+        LogicalProject rewrittenProj = (LogicalProject) rewritten;
+        RexCall rewrittenCase = (RexCall) rewrittenProj.getProjects().get(0);
+        RexNode rewrittenElse = rewrittenCase.getOperands().get(2);
+        assertTrue("ELSE arm must remain a literal", rewrittenElse instanceof RexLiteral);
+        assertEquals(
+            "ELSE arm type must be widened to the CASE return type, not NULL",
+            SqlTypeName.INTEGER,
+            rewrittenElse.getType().getSqlTypeName()
+        );
+        assertTrue("ELSE arm must still be null", ((RexLiteral) rewrittenElse).isNull());
+    }
+
+    /**
+     * THEN-arm null is rewritten the same way (the operand layout treats odd-indexed
+     * positions and the trailing operand as values; both can host an untyped NULL).
+     */
+    public void testCaseWithThenNullIsAlsoRewritten() {
+        RelDataType nullableInt = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.INTEGER), true);
+        RelDataType boolType = typeFactory.createSqlType(SqlTypeName.BOOLEAN);
+        RelNode values = LogicalValues.createOneRow(cluster);
+
+        RexNode boolLit = rexBuilder.makeLiteral(true, boolType);
+        RexNode untypedNull = rexBuilder.constantNull();
+        RexNode oneLit = rexBuilder.makeExactLiteral(java.math.BigDecimal.ONE, nullableInt);
+
+        // CASE WHEN cond THEN  ELSE 1 END — value-arm at index 1.
+        RexNode caseExpr = rexBuilder.makeCall(SqlStdOperatorTable.CASE, boolLit, untypedNull, oneLit);
+        RelNode project = LogicalProject.create(values, List.of(), List.of(caseExpr), List.of("col"), java.util.Set.of());
+
+        RelNode rewritten = UntypedNullPreprocessor.rewrite(project);
+        LogicalProject rewrittenProj = (LogicalProject) rewritten;
+        RexCall rewrittenCase = (RexCall) rewrittenProj.getProjects().get(0);
+        RexNode rewrittenThen = rewrittenCase.getOperands().get(1);
+        assertEquals(
+            "THEN arm null must also be widened to the CASE return type",
+            SqlTypeName.INTEGER,
+            rewrittenThen.getType().getSqlTypeName()
+        );
+    }
+
+    /**
+     * The condition operand at even indices (except the trailing else) is *not* a value
+     * arm — leave it alone. (We don't expect untyped NULLs as conditions, but the operand
+     * classifier should not touch even-index operands regardless.)
+     */
+    public void testCaseConditionOperandUnchanged() {
+        RelDataType nullableInt = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.INTEGER), true);
+        RelDataType boolType = typeFactory.createSqlType(SqlTypeName.BOOLEAN);
+        RelNode values = LogicalValues.createOneRow(cluster);
+
+        RexNode boolLit = rexBuilder.makeLiteral(true, boolType);
+        RexNode oneLit = rexBuilder.makeExactLiteral(java.math.BigDecimal.ONE, nullableInt);
+        RexNode twoLit = rexBuilder.makeExactLiteral(java.math.BigDecimal.valueOf(2), nullableInt);
+
+        // CASE WHEN true THEN 1 ELSE 2 END — no untyped nulls; rewriter must be a no-op.
+        RexNode caseExpr = rexBuilder.makeCall(SqlStdOperatorTable.CASE, boolLit, oneLit, twoLit);
+        RelNode project = LogicalProject.create(values, List.of(), List.of(caseExpr), List.of("col"), java.util.Set.of());
+
+        RelNode rewritten = UntypedNullPreprocessor.rewrite(project);
+        LogicalProject rewrittenProj = (LogicalProject) rewritten;
+        RexCall rewrittenCase = (RexCall) rewrittenProj.getProjects().get(0);
+        // Whole CASE expression is structurally unchanged when no untyped nulls are present
+        // — the rewriter only fires on SqlTypeName.NULL operands.
+        assertEquals("CASE expression should be unchanged when no untyped null is present", caseExpr.toString(), rewrittenCase.toString());
+    }
+
+    /**
+     * End-to-end shape: the Project that motivates the rewrite usually feeds an Aggregate
+     * (e.g. {@code COUNT(case_col)}). Verify the Aggregate over a rewritten Project
+     * still type-checks and exposes the expected output schema.
+     */
+    public void testCountOverRewrittenCaseProjectionTypechecks() {
+        RelDataType nullableInt = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.INTEGER), true);
+        RelDataType boolType = typeFactory.createSqlType(SqlTypeName.BOOLEAN);
+        RelNode values = LogicalValues.createOneRow(cluster);
+
+        RexNode boolLit = rexBuilder.makeLiteral(true, boolType);
+        RexNode oneLit = rexBuilder.makeExactLiteral(java.math.BigDecimal.ONE, nullableInt);
+        RexNode untypedNull = rexBuilder.constantNull();
+        RexNode caseExpr = rexBuilder.makeCall(SqlStdOperatorTable.CASE, boolLit, oneLit, untypedNull);
+
+        RelNode project = LogicalProject.create(values, List.of(), List.of(caseExpr), List.of("case_col"), java.util.Set.of());
+        AggregateCall countCall = AggregateCall.create(
+            SqlStdOperatorTable.COUNT,
+            false,
+            List.of(0),
+            -1,
+            typeFactory.createSqlType(SqlTypeName.BIGINT),
+            "good_count"
+        );
+        LogicalAggregate agg = LogicalAggregate.create(project, List.of(), ImmutableBitSet.of(), null, List.of(countCall));
+
+        RelNode rewritten = UntypedNullPreprocessor.rewrite(agg);
+        // The aggregate's input is the rewritten project; the project's CASE ELSE arm must
+        // now have a typed null. Walk one level down to verify.
+        LogicalAggregate rewrittenAgg = (LogicalAggregate) rewritten;
+        LogicalProject rewrittenProj = (LogicalProject) rewrittenAgg.getInput();
+        RexCall rewrittenCase = (RexCall) rewrittenProj.getProjects().get(0);
+        assertEquals(
+            "After Aggregate→Project recursion, the CASE's ELSE arm null must be typed",
+            SqlTypeName.INTEGER,
+            rewrittenCase.getOperands().get(2).getType().getSqlTypeName()
+        );
+        // And the COUNT aggregate output schema should still be a single BIGINT column.
+        assertEquals(1, rewrittenAgg.getRowType().getFieldCount());
+        assertEquals(SqlTypeName.BIGINT, rewrittenAgg.getRowType().getFieldList().get(0).getType().getSqlTypeName());
+    }
+}
diff --git a/sandbox/plugins/test-ppl-frontend/build.gradle b/sandbox/plugins/test-ppl-frontend/build.gradle
index 8a578d2056be7..bf3def1d9cb89 100644
--- a/sandbox/plugins/test-ppl-frontend/build.gradle
+++ b/sandbox/plugins/test-ppl-frontend/build.gradle
@@ -17,8 +17,13 @@ apply plugin: 'opensearch.opensearchplugin'
 
 java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) }
 
-// SQL Unified Query API version (aligned with OpenSearch build version)
-def sqlUnifiedQueryVersion = '3.6.0.0-SNAPSHOT'
+// SQL Unified Query API version (aligned with OpenSearch build version).
+// Bumped to 3.7 so the bundled PPL grammar exposes commands added since 3.6
+// (multisearch, table, regex, rex, convert, …). The OpenSearch Snapshots repo
+// declared below carries the published 3.7.x.x-SNAPSHOT artifacts; for local
+// development against an unpublished sql-repo HEAD, run
+// `./gradlew :ppl:publishUnifiedQueryPublicationToMavenLocal` from the sql repo first.
+def sqlUnifiedQueryVersion = '3.7.0.0-SNAPSHOT'
 
 opensearchplugin {
   description = 'Test PPL front-end: REST endpoint backed by the unified PPL pipeline.'
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java
new file mode 100644
index 0000000000000..6434f17f220e4
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java
@@ -0,0 +1,248 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+import org.opensearch.client.ResponseException;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Self-contained integration test for PPL {@code multisearch} on the analytics-engine route.
+ *
+ * 
      Mirrors the simplest passing shapes from the SQL plugin's
+ * {@code CalciteMultisearchCommandIT}, narrowed to surfaces the analytics path
+ * already supports end-to-end (basic 2-way, 3-way, and the arity-check error).
+ *
+ * 
      {@code multisearch} produces a Calcite {@code LogicalUnion} of N branches with
+ * {@code SchemaUnifier} reconciling per-branch schemas. The coordinator stage shape
+ * the analytics path lowers is
+ * {@code Sort(Aggregate(Union(StageInputScan, …, StageInputScan)))} — the same
+ * shape the {@code DataFusionFragmentConvertor.rewire} fix
+ * (this PR's substrait `Plan.Root.names` repair) targets.
+ *
+ * 
      Reuses the {@code calcs} dataset; no new fixtures.
+ */
+public class MultisearchCommandIT extends AnalyticsRestTestCase {
+
+    private static final Dataset DATASET = new Dataset("calcs", "calcs");
+
+    private static boolean dataProvisioned = false;
+
+    private void ensureDataProvisioned() throws IOException {
+        if (dataProvisioned == false) {
+            DatasetProvisioner.provision(client(), DATASET);
+            dataProvisioned = true;
+        }
+    }
+
+    // ── basic 2-way multisearch with stats+sort ────────────────────────────────
+    // multisearch is a *statement-leading* command in the PPL grammar (it lives in the
+    // `pplCommands` alternation, not the mid-pipeline `commands` alternation). Each
+    // subsearch must carry its own `source=`; placing `source=... | multisearch …` is a
+    // syntax error.
+
+    public void testMultisearchTwoBranchesByCategory() throws IOException {
+        // Branch 1 keeps rows with int0 < 5 and labels them "low" via eval; branch 2 keeps
+        // int0 >= 5 and labels them "high". After Union, stats counts per `class` bucket.
+        // calcs int0 distribution: 1×{1, 3, 7, 10, 11}, 3×{4, 8}, 6×null.
+        // int0 < 5 → 5 rows (1 + 1 + 3 = low); int0 >= 5 → 6 rows (3 + 1 + 1 + 1 = high);
+        // 6 null rows excluded by both predicates (5 + 6 + 6 = 17 total).
+        // Verifies: Union over two same-schema projections + Aggregate(count by) on top —
+        // the convertReduceFragment chain attachFragmentOnTop(Sort,
+        // attachFragmentOnTop(Aggregate, convertFinalAggFragment(Union))).
+        // Each branch projects to (int0, class) so the union row type is scalar-only —
+        // calcs has date/time/datetime columns whose TIMESTAMP Calcite SQL type
+        // ArrowSchemaFromCalcite doesn't yet handle (separate follow-up).
+        assertRows(
+            "| multisearch"
+                + "    [search source=" + DATASET.indexName + " | where int0 < 5  | eval class = \"low\"  | fields int0, class]"
+                + "    [search source=" + DATASET.indexName + " | where int0 >= 5 | eval class = \"high\" | fields int0, class]"
+                + " | stats count by class | sort class",
+            row(6L, "high"),
+            row(5L, "low")
+        );
+    }
+
+    // ── 3-way multisearch — the shape that triggered the substrait names bug ───
+
+    public void testMultisearchThreeBranchesByStr0() throws IOException {
+        // Three string-equality branches over the calcs str0 column. `str0` distribution is
+        // FURNITURE=2, OFFICE SUPPLIES=6, TECHNOLOGY=9. The 3-way Union(ER, ER, ER) is the
+        // exact coordinator shape the DataFusionFragmentConvertor.rewire fix targets.
+        // Pre-fix: 500 with "Names list ... 2 uses for {row-type-width} names". Post-fix: the
+        // wrapper aggregate's [count, bucket] names propagate end-to-end, plan deserializes,
+        // DataFusion executes the Union+Aggregate.
+        // Each branch projects to (str0, bucket) — see testMultisearchTwoBranchesByCategory's
+        // comment for the reason.
+        assertRows(
+            "| multisearch"
+                + "    [search source=" + DATASET.indexName + " | where str0 = \"FURNITURE\"       | eval bucket = \"F\" | fields str0, bucket]"
+                + "    [search source=" + DATASET.indexName + " | where str0 = \"OFFICE SUPPLIES\" | eval bucket = \"O\" | fields str0, bucket]"
+                + "    [search source=" + DATASET.indexName + " | where str0 = \"TECHNOLOGY\"      | eval bucket = \"T\" | fields str0, bucket]"
+                + " | stats count by bucket | sort bucket",
+            row(2L, "F"),
+            row(6L, "O"),
+            row(9L, "T")
+        );
+    }
+
+    // ── CASE on the eval side — explicit case() expression lowers to CASE WHEN ──
+
+    public void testMultisearchEvalCaseProjection() throws IOException {
+        // PPL `eval x = case(cond, val, …)` lowers to a Calcite SqlKind.CASE which the
+        // analytics planner used to reject with "No backend supports scalar function
+        // [CASE] among [datafusion]" (capability not registered). With CASE in the
+        // project capability set, isthmus translates SqlKind.CASE structurally to a
+        // Substrait IfThen rel that DataFusion's substrait consumer handles natively —
+        // no extension lookup or adapter required.
+        //
+        // Each branch uses an explicit `else` arm so isthmus doesn't have to convert an
+        // untyped NULL literal — `eval bucket = case(int0 < 5, "low" else "rest")` keeps
+        // both arms VARCHAR. The `count(eval(predicate))` idiom (the v2-side
+        // testMultisearchSuccessRatePattern shape) generates an implicit `else NULL`
+        // whose type is SqlTypeName.NULL; isthmus' TypeConverter throws
+        // `Unable to convert the type NULL` on that, tracked separately.
+        //
+        // calcs int0 distribution (see testMultisearchTwoBranchesByCategory): 5 rows < 5,
+        // 6 rows >= 5; the union below feeds 11 rows total to the case-eval. low maps to
+        // ("low", 5), rest (the high branch's contribution) to ("rest", 6).
+        assertRows(
+            "| multisearch"
+                + "    [search source=" + DATASET.indexName + " | where int0 < 5  | fields int0]"
+                + "    [search source=" + DATASET.indexName + " | where int0 >= 5 | fields int0]"
+                + " | eval bucket = case(int0 < 5, \"low\" else \"rest\")"
+                + " | stats count by bucket | sort bucket",
+            row(5L, "low"),
+            row(6L, "rest")
+        );
+    }
+
+    // ── CASE with implicit ELSE NULL — `count(eval(predicate))` shape ──────────
+
+    public void testMultisearchCountEvalConditionalCount() throws IOException {
+        // Mirror of the v2-side `CalciteMultisearchCommandIT.testMultisearchSuccessRatePattern`:
+        // `count(eval(predicate))` is PPL's conditional-count idiom. Calcite lowers it to
+        // `COUNT(CASE WHEN predicate THEN  END)`, where the implicit ELSE arm
+        // becomes a `RexLiteral` with `SqlTypeName.NULL`. Isthmus' TypeConverter rejects
+        // NULL with "Unable to convert the type NULL".
+        //
+        // The {@link UntypedNullPreprocessor} pass added in this PR rewrites every
+        // SqlTypeName.NULL operand in a CASE call to a typed null literal matching the
+        // CASE's resolved return type before the SubstraitRelVisitor sees the plan. CASE
+        // itself is registered in the project capability set so the planner doesn't reject
+        // the operator before substrait emission either.
+        //
+        // calcs int0 distribution (see testMultisearchTwoBranchesByCategory): 5 rows < 5,
+        // 6 rows >= 5; 6 nulls excluded by both branch predicates. After multisearch,
+        // 11 rows feed the count-eval. `count(eval(class = "low"))` matches 5 (the low-bucketed
+        // rows), `count(eval(class = "high"))` matches 6, and `count()` totals 11.
+        assertRows(
+            "| multisearch"
+                + "    [search source=" + DATASET.indexName + " | where int0 < 5  | eval class = \"low\"  | fields int0, class]"
+                + "    [search source=" + DATASET.indexName + " | where int0 >= 5 | eval class = \"high\" | fields int0, class]"
+                + " | stats count(eval(class = \"low\"))  as low_count,"
+                + "         count(eval(class = \"high\")) as high_count,"
+                + "         count() as grand_count",
+            row(5L, 6L, 11L)
+        );
+    }
+
+    // ── arity check — caught at parse, never reaches the analytics path ────────
+
+    public void testMultisearchSingleSubsearchRejected() throws IOException {
+        // The PPL parser's AstBuilder.visitMultisearchCommand requires ≥2 subsearches and
+        // throws a SyntaxCheckException eagerly. This case exercises the parser-side guard
+        // — it never reaches CalciteRelNodeVisitor / SchemaUnifier / substrait emission, so
+        // it's a regression-pin against accidental relaxation of the arity check, not an
+        // analytics-path correctness check.
+        assertErrorContains(
+            "| multisearch [search source=" + DATASET.indexName + " | head 1]",
+            "Multisearch command requires at least two subsearches"
+        );
+    }
+
+    // ── helpers ─────────────────────────────────────────────────────────────────
+
+    private static List row(Object... values) {
+        return Arrays.asList(values);
+    }
+
+    @SafeVarargs
+    @SuppressWarnings("varargs")
+    private final void assertRows(String ppl, List... expected) throws IOException {
+        Map response = executePpl(ppl);
+        @SuppressWarnings("unchecked")
+        List> actualRows = (List>) response.get("rows");
+        assertNotNull("Response missing 'rows' for query: " + ppl, actualRows);
+        assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size());
+        for (int i = 0; i < expected.length; i++) {
+            List want = expected[i];
+            List got = actualRows.get(i);
+            assertEquals(
+                "Column count mismatch at row " + i + " for query: " + ppl,
+                want.size(),
+                got.size()
+            );
+            for (int j = 0; j < want.size(); j++) {
+                assertCellEquals(
+                    "Cell mismatch at row " + i + ", col " + j + " for query: " + ppl,
+                    want.get(j),
+                    got.get(j)
+                );
+            }
+        }
+    }
+
+    private void assertErrorContains(String ppl, String expectedSubstring) throws IOException {
+        try {
+            Map response = executePpl(ppl);
+            fail("Expected query to fail with [" + expectedSubstring + "] but got response: " + response);
+        } catch (ResponseException e) {
+            String body;
+            try {
+                body = org.opensearch.test.rest.OpenSearchRestTestCase.entityAsMap(e.getResponse()).toString();
+            } catch (IOException ioe) {
+                body = e.getMessage();
+            }
+            assertTrue(
+                "Expected response body to contain [" + expectedSubstring + "] but was: " + body,
+                body.contains(expectedSubstring)
+            );
+        }
+    }
+
+    private Map executePpl(String ppl) throws IOException {
+        ensureDataProvisioned();
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}");
+        Response response = client().performRequest(request);
+        return assertOkAndParse(response, "PPL: " + ppl);
+    }
+
+    private static void assertCellEquals(String message, Object expected, Object actual) {
+        if (expected == null || actual == null) {
+            assertEquals(message, expected, actual);
+            return;
+        }
+        if (expected instanceof Number && actual instanceof Number) {
+            double e = ((Number) expected).doubleValue();
+            double a = ((Number) actual).doubleValue();
+            if (Double.compare(e, a) != 0) {
+                fail(message + ": expected <" + expected + "> but was <" + actual + ">");
+            }
+            return;
+        }
+        assertEquals(message, expected, actual);
+    }
+}

From 65bfc20341a72bced01b01e9566660e01e45cf2b Mon Sep 17 00:00:00 2001
From: Ajay Raj Nelapudi 
Date: Fri, 8 May 2026 10:22:46 +0530
Subject: [PATCH 085/115] Expose tokio-metrics as datafusion plugin stats
 (#21303)

* Expose tokio-metrics as datafusion plugin stats

Signed-off-by: Ajay Raj Nelapudi 

* refactor: Extract stats SPI to libs/plugin-stats-spi

Signed-off-by: Ajay Raj Nelapudi 

* Move DF specific constructs to DF plugin

Signed-off-by: Ajay Raj Nelapudi 

* Remove RUNTIME_METRICS_COUNT and resolve sandbox failures

Signed-off-by: Ajay Raj Nelapudi 

* Add local_queue_depth to runtime metrics

Signed-off-by: Ajay Raj Nelapudi 

* Use Handle::metrics() for point-in-time stats instead of interval deltas

Signed-off-by: Ajay Raj Nelapudi 

---------

Signed-off-by: Ajay Raj Nelapudi 
Co-authored-by: Ajay Raj Nelapudi 
---
 gradle/libs.versions.toml                     |   4 +
 .../libs/dataformat-native/rust/Cargo.toml    |   1 +
 sandbox/libs/plugin-stats-spi/build.gradle    |  24 ++
 .../plugin/stats/BackendStatsProvider.java    |  32 ++
 .../opensearch/plugin/stats/PluginStats.java  |  21 +
 .../opensearch/plugin/stats/package-info.java |  22 ++
 .../analytics-backend-datafusion/build.gradle |  48 +++
 .../rust/Cargo.toml                           |   3 +
 .../rust/src/executor.rs                      |   8 +
 .../rust/src/ffm.rs                           |  56 +++
 .../rust/src/lib.rs                           |   2 +
 .../rust/src/runtime_manager.rs               |  11 +
 .../rust/src/stats.rs                         | 247 ++++++++++++
 .../rust/src/task_monitors.rs                 |  27 ++
 .../be/datafusion/DataFusionPlugin.java       |  27 +-
 .../be/datafusion/DataFusionService.java      |  12 +
 .../action/DataFusionStatsAction.java         |  68 ++++
 .../be/datafusion/action/package-info.java    |  12 +
 .../be/datafusion/nativelib/NativeBridge.java |  40 ++
 .../be/datafusion/nativelib/StatsLayout.java  | 223 +++++++++++
 .../stats/DataFusionBackendStatsProvider.java |  36 ++
 .../be/datafusion/stats/DataFusionStats.java  |  88 +++++
 .../stats/NativeExecutorsStats.java           | 369 ++++++++++++++++++
 .../be/datafusion/stats/package-info.java     |  17 +
 .../nativelib/StatsLayoutPropertyTests.java   | 308 +++++++++++++++
 .../stats/DataFusionStatsPropertyTests.java   | 327 ++++++++++++++++
 .../stats/NativeExecutorsStatsTests.java      | 196 ++++++++++
 .../NodeStatsNativeMetricRoundTripTests.java  | 165 ++++++++
 .../DataFusionServiceStatsTests.java          |  37 ++
 .../action/DataFusionStatsActionTests.java    | 161 ++++++++
 .../nativelib/StatsLayoutTests.java           | 107 +++++
 .../stats/DataFusionStatsTests.java           | 209 ++++++++++
 32 files changed, 2907 insertions(+), 1 deletion(-)
 create mode 100644 sandbox/libs/plugin-stats-spi/build.gradle
 create mode 100644 sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/BackendStatsProvider.java
 create mode 100644 sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/PluginStats.java
 create mode 100644 sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/package-info.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/stats.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/task_monitors.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/DataFusionStatsAction.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/package-info.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/StatsLayout.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/DataFusionBackendStatsProvider.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/DataFusionStats.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/NativeExecutorsStats.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/package-info.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/nativelib/StatsLayoutPropertyTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/DataFusionStatsPropertyTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NativeExecutorsStatsTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NodeStatsNativeMetricRoundTripTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionServiceStatsTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/action/DataFusionStatsActionTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/nativelib/StatsLayoutTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/stats/DataFusionStatsTests.java

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index f8fc6004330df..01db444cd421b 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -109,6 +109,10 @@ flatbuffers           = "2.0.0"
 calcite               = "1.41.0"
 calcite_os_rev        = "1"
 
+# property-based testing
+jqwik                 = "1.9.2"
+junit_jupiter         = "5.11.3"
+junit_platform        = "1.11.3"
 [libraries]
 antlr4-runtime = { group = "org.antlr", name = "antlr4-runtime", version.ref = "antlr4" }
 asm-analysis = { group = "org.ow2.asm", name = "asm-analysis", version.ref = "asm" }
diff --git a/sandbox/libs/dataformat-native/rust/Cargo.toml b/sandbox/libs/dataformat-native/rust/Cargo.toml
index 5de1722585d40..800cfb3f27556 100644
--- a/sandbox/libs/dataformat-native/rust/Cargo.toml
+++ b/sandbox/libs/dataformat-native/rust/Cargo.toml
@@ -70,6 +70,7 @@ thiserror = "1.0"
 async-trait = "0.1"
 bytes = "1"
 criterion = { version = "0.5", features = ["async_tokio"] }
+tokio-metrics = { version = "0.5", features = ["rt"] }
 
 # Internal
 native-bridge-common = { path = "common" }
diff --git a/sandbox/libs/plugin-stats-spi/build.gradle b/sandbox/libs/plugin-stats-spi/build.gradle
new file mode 100644
index 0000000000000..708f537728151
--- /dev/null
+++ b/sandbox/libs/plugin-stats-spi/build.gradle
@@ -0,0 +1,24 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * SPI interfaces for plugin stats collection.
+ * Contains only PluginStats (marker interface) and BackendStatsProvider.
+ * Consumed by sandbox plugins that report backend statistics.
+ */
+
+dependencies {
+    api project(':libs:opensearch-core')
+    api project(':libs:opensearch-common')
+}
+
+testingConventions.enabled = false
+
+tasks.named('forbiddenApisMain').configure {
+    replaceSignatureFiles 'jdk-signatures'
+}
diff --git a/sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/BackendStatsProvider.java b/sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/BackendStatsProvider.java
new file mode 100644
index 0000000000000..1b43d01029e83
--- /dev/null
+++ b/sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/BackendStatsProvider.java
@@ -0,0 +1,32 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.plugin.stats;
+
+/**
+ * Each backend (DataFusion, Parquet, future engines) implements this interface
+ * to provide its stats to the Mustang Stats Framework. The Analytics Plugin
+ * discovers {@code BackendStatsProvider} implementations and iterates over them
+ * to collect stats from all registered backends.
+ */
+public interface BackendStatsProvider {
+
+    /**
+     * Returns the backend's identifier, e.g. {@code "datafusion"}, {@code "parquet"}.
+     *
+     * @return a non-null backend name
+     */
+    String name();
+
+    /**
+     * Returns the backend's stats object.
+     *
+     * @return a non-null {@link PluginStats} instance
+     */
+    PluginStats getBackendStats();
+}
diff --git a/sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/PluginStats.java b/sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/PluginStats.java
new file mode 100644
index 0000000000000..1ecefce88b527
--- /dev/null
+++ b/sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/PluginStats.java
@@ -0,0 +1,21 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.plugin.stats;
+
+/**
+ * Marker interface for all backend stats types in the Mustang Stats Framework.
+ *
+ * 
      Intentionally empty — serves as the common type for
+ * {@link BackendStatsProvider#getBackendStats()}. Each backend's top-level stats
+ * class (e.g. {@code DataFusionStats}) implements this interface so the Analytics
+ * Plugin can discover and iterate over them.
+ */
+public interface PluginStats {
+    // marker — no methods
+}
diff --git a/sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/package-info.java b/sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/package-info.java
new file mode 100644
index 0000000000000..f1fbc5fd1e5fd
--- /dev/null
+++ b/sandbox/libs/plugin-stats-spi/src/main/java/org/opensearch/plugin/stats/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * SPI stats types for the Mustang Stats Framework.
+ *
+ * 
      This package contains the stats interfaces shared between
+ * the OpenSearch server and native backend plugins. Types here are visible to
+ * both sides without requiring a plugin dependency.
+ *
+ * 
      Key types:
+ * 
        
+ *   
      • {@link org.opensearch.plugin.stats.PluginStats} — marker interface for all backend stats
        • 
+ *   
      • {@link org.opensearch.plugin.stats.BackendStatsProvider} — interface for backends to provide stats
        • 
+ * 
      
+ */
+package org.opensearch.plugin.stats;
diff --git a/sandbox/plugins/analytics-backend-datafusion/build.gradle b/sandbox/plugins/analytics-backend-datafusion/build.gradle
index d382603186f40..1d71f16082900 100644
--- a/sandbox/plugins/analytics-backend-datafusion/build.gradle
+++ b/sandbox/plugins/analytics-backend-datafusion/build.gradle
@@ -39,6 +39,9 @@ dependencies {
   // Shared native bridge lib (provides the unified .so and FFM SymbolLookup)
   implementation project(':sandbox:libs:dataformat-native')
 
+  // Canonical stats SPI classes (PluginStats, BackendStatsProvider)
+  implementation project(':sandbox:libs:plugin-stats-spi')
+
   // Provided at runtime by the parent analytics-engine plugin; compile-only to avoid jar hell.
   compileOnly project(':sandbox:libs:analytics-framework')
   // analytics-engine's RelNode types (OpenSearchStageInputScan) are referenced by the
@@ -192,6 +195,47 @@ testingConventions.enabled = false
 
 tasks.named('forbiddenPatterns').configure {
     exclude '**/*.parquet'
+    exclude '**/*.dylib'
+    exclude '**/*.so'
+    exclude '**/*.dll'
+}
+
+// ---- Property-based tests (jqwik / JUnit 5 Platform) ----
+
+sourceSets {
+  propertyTest {
+    java {
+      srcDir 'src/propertyTest/java'
+    }
+    compileClasspath += sourceSets.main.output
+    runtimeClasspath += sourceSets.main.output
+  }
+}
+
+configurations {
+  propertyTestImplementation.extendsFrom implementation, compileOnly
+  propertyTestRuntimeOnly.extendsFrom runtimeOnly
+}
+
+dependencies {
+  propertyTestImplementation "net.jqwik:jqwik:${versions.jqwik}"
+  propertyTestImplementation "org.junit.jupiter:junit-jupiter-api:${versions.junit_jupiter}"
+  propertyTestRuntimeOnly "org.junit.platform:junit-platform-launcher:${versions.junit_platform}"
+  // Jackson for JSON parsing in property tests
+  propertyTestImplementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+}
+
+tasks.register('propertyTest', Test) {
+  description = 'Run jqwik property-based tests'
+  group = 'verification'
+  useJUnitPlatform {
+    includeEngines 'jqwik'
+  }
+  testClassesDirs = sourceSets.propertyTest.output.classesDirs
+  classpath = sourceSets.propertyTest.runtimeClasspath
+  jvmArgs += ["--add-opens", "java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED"]
+  // Disable the security manager for property tests (jqwik is not compatible)
+  systemProperty 'tests.security.manager', 'false'
 }
 
 tasks.matching { it.name == 'missingJavadoc' }.configureEach {
@@ -206,3 +250,7 @@ tasks.named('thirdPartyAudit').configure {
         'org.apache.calcite.server.ServerDdlExecutor'
     )
 }
+
+// jqwik property tests don't ship with the randomized-testing framework that
+// forbiddenApis signatures reference — skip the check for this source set.
+tasks.matching { it.name == 'forbiddenApisPropertyTest' }.configureEach { enabled = false }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
index 07dc0a26ba5f7..63afe25b394f1 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
@@ -1,4 +1,5 @@
 [package]
+# DataFusion analytics backend native library
 name = "opensearch-datafusion"
 version = "0.1.0"
 edition = "2021"
@@ -46,6 +47,8 @@ thiserror = { workspace = true }
 # convert_tz UDF
 chrono-tz = "0.10"
 
+tokio-metrics = { workspace = true }
+
 [dev-dependencies]
 criterion = { workspace = true }
 tempfile = { workspace = true }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs
index e5d3a16c46493..bb0257852498f 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/executor.rs
@@ -154,6 +154,14 @@ impl DedicatedExecutor {
         }
     }
 
+    /// Returns a clone of the underlying Tokio runtime `Handle`, if the
+    /// executor has not been shut down. Used to create a
+    /// `tokio_metrics::RuntimeMonitor` for the CPU runtime.
+    pub fn handle(&self) -> Option {
+        let state = self.state.read();
+        state.handle.clone()
+    }
+
     pub fn shutdown(&self) {
         let mut state = self.state.write();
         state.handle = None;
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
index 0e3f034279f0b..47df9bbe8951a 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
@@ -651,3 +651,59 @@ pub unsafe extern "C" fn df_execute_with_context(
             .map_err(|e| e.to_string())
     }
 }
+
+// ---- Stats collection ----
+
+/// Collects all native executor metrics into a caller-provided byte buffer.
+///
+/// The buffer must have capacity for at least `size_of::()` bytes (224).
+/// Returns 0 on success.
+#[ffm_safe]
+#[no_mangle]
+pub unsafe extern "C" fn df_stats(out_ptr: *mut u8, out_cap: i64) -> i64 {
+    use crate::stats::{layout, pack_runtime_metrics, pack_task_monitor, DfStatsBuffer, RuntimeMetricsRepr};
+    use crate::task_monitors::{
+        query_execution_monitor, stream_next_monitor,
+        fetch_phase_monitor, segment_stats_monitor,
+    };
+
+    if out_cap < 0 || (out_cap as usize) < layout::BUFFER_BYTE_SIZE {
+        return Err(format!(
+            "stats buffer too small: need {} but got {}",
+            layout::BUFFER_BYTE_SIZE, out_cap
+        ));
+    }
+
+    let mgr = get_rt_manager()?;
+
+    // IO runtime (always present)
+    let io_runtime = pack_runtime_metrics(&mgr.io_monitor, mgr.io_runtime.handle());
+
+    // CPU runtime (optional — zeroed when absent)
+    let cpu_runtime = if let Some(ref cpu_mon) = mgr.cpu_monitor {
+        if let Some(cpu_handle) = mgr.cpu_executor.handle() {
+            pack_runtime_metrics(cpu_mon, &cpu_handle)
+        } else {
+            RuntimeMetricsRepr::zeroed()
+        }
+    } else {
+        RuntimeMetricsRepr::zeroed()
+    };
+
+    let buf = DfStatsBuffer {
+        io_runtime,
+        cpu_runtime,
+        query_execution: pack_task_monitor(query_execution_monitor()),
+        stream_next: pack_task_monitor(stream_next_monitor()),
+        fetch_phase: pack_task_monitor(fetch_phase_monitor()),
+        segment_stats: pack_task_monitor(segment_stats_monitor()),
+    };
+
+    // Copy struct bytes to caller buffer
+    std::ptr::copy_nonoverlapping(
+        &buf as *const DfStatsBuffer as *const u8,
+        out_ptr,
+        std::mem::size_of::(),
+    );
+    Ok(0)
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs
index 71a67fa6379b2..154a4ed6d5a56 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs
@@ -32,3 +32,5 @@ pub mod runtime_manager;
 pub mod session_context;
 pub mod statistics_cache;
 pub mod udf;
+pub mod stats;
+pub mod task_monitors;
\ No newline at end of file
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/runtime_manager.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/runtime_manager.rs
index 17abbccbb6d8c..89818b14a3cc3 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/runtime_manager.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/runtime_manager.rs
@@ -10,11 +10,14 @@ use crate::io::register_io_runtime;
 use log::info;
 use std::sync::Arc;
 use tokio::runtime::{Builder, Runtime};
+use tokio_metrics::RuntimeMonitor;
 
 // RuntimeManager — owns IO runtime + CPU DedicatedExecutor.
 pub struct RuntimeManager {
     pub io_runtime: Arc,
     pub cpu_executor: DedicatedExecutor,
+    pub io_monitor: RuntimeMonitor,
+    pub cpu_monitor: Option,
 }
 
 impl RuntimeManager {
@@ -32,6 +35,8 @@ impl RuntimeManager {
 
         register_io_runtime(Some(io_runtime.handle().clone()));
 
+        let io_monitor = RuntimeMonitor::new(&io_runtime.handle());
+
         let io_handle = io_runtime.handle().clone();
         let mut cpu_runtime_builder = Builder::new_multi_thread();
         cpu_runtime_builder
@@ -44,9 +49,15 @@ impl RuntimeManager {
 
         let cpu_executor = DedicatedExecutor::new("datafusion-cpu", cpu_runtime_builder);
 
+        let cpu_monitor = cpu_executor
+            .handle()
+            .map(|h| RuntimeMonitor::new(&h));
+
         Self {
             io_runtime,
             cpu_executor,
+            io_monitor,
+            cpu_monitor,
         }
     }
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/stats.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/stats.rs
new file mode 100644
index 0000000000000..3c04539c7bdcf
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/stats.rs
@@ -0,0 +1,247 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+//! Stats packing helpers for the FFM `df_stats()` function.
+//!
+//! Packs Tokio runtime metrics and per-operation task monitor metrics
+//! into a `#[repr(C)]` `DfStatsBuffer` struct (240 bytes) for efficient
+//! transfer across the FFM boundary.
+//!
+//! ## Struct layout
+//!
+//! | Group             | Type                | Fields |
+//! |-------------------|---------------------|--------|
+//! | `io_runtime`      | `RuntimeMetricsRepr`| 9 × i64 |
+//! | `cpu_runtime`     | `RuntimeMetricsRepr`| 9 × i64 (zeroed if N/A) |
+//! | `query_execution` | `TaskMonitorRepr`   | 3 × i64 |
+//! | `stream_next`     | `TaskMonitorRepr`   | 3 × i64 |
+//! | `fetch_phase`     | `TaskMonitorRepr`   | 3 × i64 |
+//! | `segment_stats`   | `TaskMonitorRepr`   | 3 × i64 |
+
+use tokio::runtime::Handle;
+use tokio_metrics::{RuntimeMonitor, TaskMonitor};
+
+#[repr(C)]
+pub struct RuntimeMetricsRepr {
+    pub workers_count: i64,
+    pub total_polls_count: i64,
+    pub total_busy_duration_ms: i64,
+    pub total_overflow_count: i64,
+    pub global_queue_depth: i64,
+    pub blocking_queue_depth: i64,
+    pub num_alive_tasks: i64,
+    pub spawned_tasks_count: i64,
+    pub total_local_queue_depth: i64,
+}
+
+impl RuntimeMetricsRepr {
+    pub fn zeroed() -> Self {
+        Self {
+            workers_count: 0,
+            total_polls_count: 0,
+            total_busy_duration_ms: 0,
+            total_overflow_count: 0,
+            global_queue_depth: 0,
+            blocking_queue_depth: 0,
+            num_alive_tasks: 0,
+            spawned_tasks_count: 0,
+            total_local_queue_depth: 0,
+        }
+    }
+}
+
+#[repr(C)]
+pub struct TaskMonitorRepr {
+    pub total_poll_duration_ms: i64,
+    pub total_scheduled_duration_ms: i64,
+    pub total_idle_duration_ms: i64,
+}
+
+#[repr(C)]
+pub struct DfStatsBuffer {
+    pub io_runtime: RuntimeMetricsRepr,
+    pub cpu_runtime: RuntimeMetricsRepr,
+    pub query_execution: TaskMonitorRepr,
+    pub stream_next: TaskMonitorRepr,
+    pub fetch_phase: TaskMonitorRepr,
+    pub segment_stats: TaskMonitorRepr,
+}
+
+const _: () = assert!(std::mem::size_of::() == 9 * 8);
+const _: () = assert!(std::mem::size_of::() == 3 * 8);
+const _: () = assert!(std::mem::size_of::() == 30 * 8);
+
+pub mod layout {
+    use super::*;
+    pub const BUFFER_BYTE_SIZE: usize = std::mem::size_of::();
+    const _: () = assert!(BUFFER_BYTE_SIZE == 240);
+}
+
+/// Snapshot a `RuntimeMonitor` and return a populated `RuntimeMetricsRepr`.
+///
+/// ## Fields
+///
+/// | Field                       | Source                                     |
+/// |-----------------------------|---------------------------------------------|
+/// | workers_count               | `Handle::metrics().num_workers()`           |
+/// | total_polls_count           | `Handle::metrics().worker_poll_count(i)` Σ  |
+/// | total_busy_duration_ms      | `Handle::metrics().worker_total_busy_duration(i)` Σ |
+/// | total_overflow_count        | `Handle::metrics().worker_overflow_count(i)` Σ |
+/// | global_queue_depth          | `Handle::metrics().global_queue_depth()`    |
+/// | blocking_queue_depth        | `Handle::metrics().blocking_queue_depth()`  |
+/// | num_alive_tasks             | `Handle::metrics().num_alive_tasks()`       |
+/// | spawned_tasks_count         | `Handle::metrics().spawned_tasks_count()`   |
+/// | total_local_queue_depth     | `Handle::metrics().worker_local_queue_depth(i)` Σ |
+pub fn pack_runtime_metrics(_monitor: &RuntimeMonitor, handle: &Handle) -> RuntimeMetricsRepr {
+    let m = handle.metrics();
+    let num_workers = m.num_workers();
+
+    // Sum per-worker metrics into aggregates
+    let mut total_polls: u64 = 0;
+    let mut total_busy_ns: u64 = 0;
+    let mut total_overflow: u64 = 0;
+    let mut total_local_queue: u64 = 0;
+
+    for i in 0..num_workers {
+        total_polls += m.worker_poll_count(i);
+        total_busy_ns += m.worker_total_busy_duration(i).as_nanos() as u64;
+        total_overflow += m.worker_overflow_count(i);
+        total_local_queue += m.worker_local_queue_depth(i) as u64;
+    }
+
+    RuntimeMetricsRepr {
+        workers_count: num_workers as i64,
+        total_polls_count: total_polls as i64,
+        total_busy_duration_ms: (total_busy_ns / 1_000_000) as i64,
+        total_overflow_count: total_overflow as i64,
+        global_queue_depth: m.global_queue_depth() as i64,
+        blocking_queue_depth: m.blocking_queue_depth() as i64,
+        num_alive_tasks: m.num_alive_tasks() as i64,
+        spawned_tasks_count: m.spawned_tasks_count() as i64,
+        total_local_queue_depth: total_local_queue as i64,
+    }
+}
+
+
+/// Snapshot a `TaskMonitor` and return a populated `TaskMonitorRepr`.
+///
+/// | Field                        | Source                          |
+/// |------------------------------|---------------------------------|
+/// | total_poll_duration_ms       | cumulative poll duration (ms)   |
+/// | total_scheduled_duration_ms  | cumulative scheduled dur. (ms)  |
+/// | total_idle_duration_ms       | cumulative idle duration (ms)   |
+pub fn pack_task_monitor(monitor: &TaskMonitor) -> TaskMonitorRepr {
+    let cumulative = monitor.cumulative();
+    TaskMonitorRepr {
+        total_poll_duration_ms: cumulative.total_poll_duration.as_millis() as i64,
+        total_scheduled_duration_ms: cumulative.total_scheduled_duration.as_millis() as i64,
+        total_idle_duration_ms: cumulative.total_idle_duration.as_millis() as i64,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::task_monitors::{
+        query_execution_monitor, stream_next_monitor,
+        fetch_phase_monitor, segment_stats_monitor,
+    };
+
+    #[test]
+    fn test_pack_runtime_metrics_populates_workers_count() {
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .worker_threads(2)
+            .enable_all()
+            .build()
+            .unwrap();
+        let monitor = RuntimeMonitor::new(&rt.handle());
+        let result = pack_runtime_metrics(&monitor, rt.handle());
+        assert_eq!(result.workers_count, 2);
+    }
+
+    #[test]
+    fn test_pack_runtime_metrics_returns_struct() {
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .worker_threads(3)
+            .enable_all()
+            .build()
+            .unwrap();
+        let monitor = RuntimeMonitor::new(&rt.handle());
+        let result = pack_runtime_metrics(&monitor, rt.handle());
+        assert_eq!(result.workers_count, 3);
+        assert!(result.total_polls_count >= 0);
+        assert!(result.total_busy_duration_ms >= 0);
+        assert!(result.global_queue_depth >= 0);
+    }
+
+    #[test]
+    fn test_pack_task_monitor_writes_duration_fields() {
+        let monitor = TaskMonitor::new();
+        let result = pack_task_monitor(&monitor);
+        assert_eq!(result.total_poll_duration_ms, 0);
+        assert_eq!(result.total_scheduled_duration_ms, 0);
+        assert_eq!(result.total_idle_duration_ms, 0);
+    }
+
+    #[tokio::test]
+    async fn test_pack_task_monitor_after_instrumented_future() {
+        let monitor = query_execution_monitor();
+        let fut = monitor.instrument(async {
+            tokio::task::yield_now().await;
+            42
+        });
+        let result = fut.await;
+        assert_eq!(result, 42);
+
+        let tm = pack_task_monitor(monitor);
+        assert!(tm.total_poll_duration_ms >= 0, "total_poll_duration should be >= 0, got {}", tm.total_poll_duration_ms);
+    }
+
+    #[tokio::test]
+    async fn test_full_stats_packing() {
+        let mgr = crate::runtime_manager::RuntimeManager::new(1);
+
+        let io_runtime = pack_runtime_metrics(&mgr.io_monitor, mgr.io_runtime.handle());
+
+        let cpu_runtime = if let Some(ref cpu_mon) = mgr.cpu_monitor {
+            if let Some(cpu_handle) = mgr.cpu_executor.handle() {
+                pack_runtime_metrics(cpu_mon, &cpu_handle)
+            } else {
+                RuntimeMetricsRepr::zeroed()
+            }
+        } else {
+            RuntimeMetricsRepr::zeroed()
+        };
+
+        let buf = DfStatsBuffer {
+            io_runtime,
+            cpu_runtime,
+            query_execution: pack_task_monitor(query_execution_monitor()),
+            stream_next: pack_task_monitor(stream_next_monitor()),
+            fetch_phase: pack_task_monitor(fetch_phase_monitor()),
+            segment_stats: pack_task_monitor(segment_stats_monitor()),
+        };
+
+        assert_eq!(layout::BUFFER_BYTE_SIZE, 240);
+        assert!(buf.io_runtime.workers_count > 0, "IO runtime workers_count should be > 0, got {}", buf.io_runtime.workers_count);
+
+        if mgr.cpu_monitor.is_some() {
+            assert!(buf.cpu_runtime.workers_count > 0, "CPU runtime workers_count should be > 0, got {}", buf.cpu_runtime.workers_count);
+        }
+
+        mgr.cpu_executor.shutdown();
+        std::mem::forget(mgr);
+    }
+
+    #[test]
+    fn test_df_stats_buffer_too_small() {
+        // Verify that the buffer size assertion holds
+        assert_eq!(std::mem::size_of::(), 240);
+        assert_eq!(layout::BUFFER_BYTE_SIZE, 240);
+        // A buffer smaller than 224 bytes should be rejected by df_stats.
+        // We can't call df_stats directly without a runtime manager,
+        // but we verify the constant is correct.
+        assert!(layout::BUFFER_BYTE_SIZE > 0);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/task_monitors.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/task_monitors.rs
new file mode 100644
index 0000000000000..6b6f879361330
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/task_monitors.rs
@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! Per-operation task monitors.
+//!
+//! Each FFM operation type (`query_execution`, `stream_next`, `fetch_phase`,
+//! `segment_stats`) gets its own [`TaskMonitor`] for timing metrics.
+//!
+//! Monitors are initialized lazily on first access via `once_cell::sync::Lazy`.
+
+use once_cell::sync::Lazy;
+use tokio_metrics::TaskMonitor;
+
+static QUERY_EXECUTION_MONITOR: Lazy = Lazy::new(TaskMonitor::new);
+static STREAM_NEXT_MONITOR: Lazy = Lazy::new(TaskMonitor::new);
+static FETCH_PHASE_MONITOR: Lazy = Lazy::new(TaskMonitor::new);
+static SEGMENT_STATS_MONITOR: Lazy = Lazy::new(TaskMonitor::new);
+
+pub fn query_execution_monitor() -> &'static TaskMonitor { &QUERY_EXECUTION_MONITOR }
+pub fn stream_next_monitor() -> &'static TaskMonitor { &STREAM_NEXT_MONITOR }
+pub fn fetch_phase_monitor() -> &'static TaskMonitor { &FETCH_PHASE_MONITOR }
+pub fn segment_stats_monitor() -> &'static TaskMonitor { &SEGMENT_STATS_MONITOR }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
index d1d36c4a0b7b1..b770b8ce58a00 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
@@ -10,11 +10,17 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin;
+import org.opensearch.be.datafusion.action.DataFusionStatsAction;
 import org.opensearch.be.datafusion.cache.CacheSettings;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.IndexScopedSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.settings.SettingsFilter;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;
@@ -22,9 +28,12 @@
 import org.opensearch.index.engine.dataformat.DataFormatRegistry;
 import org.opensearch.index.engine.dataformat.ReaderManagerConfig;
 import org.opensearch.index.engine.exec.EngineReaderManager;
+import org.opensearch.plugins.ActionPlugin;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.plugins.SearchBackEndPlugin;
 import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.rest.RestController;
+import org.opensearch.rest.RestHandler;
 import org.opensearch.script.ScriptService;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.client.Client;
@@ -46,7 +55,7 @@
  * Analytics query capabilities are declared in {@link DataFusionAnalyticsBackendPlugin},
  * which is SPI-discovered and receives this plugin instance via its constructor.
  */
-public class DataFusionPlugin extends Plugin implements SearchBackEndPlugin {
+public class DataFusionPlugin extends Plugin implements SearchBackEndPlugin, AnalyticsSearchBackendPlugin, ActionPlugin {
 
     private static final Logger logger = LogManager.getLogger(DataFusionPlugin.class);
 
@@ -241,6 +250,22 @@ public List getSupportedFormats() {
         return List.of(SUPPORTED_FORMAT);
     }
 
+    @Override
+    public List getRestHandlers(
+        Settings settings,
+        RestController restController,
+        ClusterSettings clusterSettings,
+        IndexScopedSettings indexScopedSettings,
+        SettingsFilter settingsFilter,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier nodesInCluster
+    ) {
+        if (dataFusionService == null) {
+            return Collections.emptyList();
+        }
+        return List.of(new DataFusionStatsAction(dataFusionService));
+    }
+
     @Override
     public void close() throws IOException {
         if (dataFusionService != null) {
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java
index a8cf655aa9e7e..c08400df72262 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionService.java
@@ -15,6 +15,7 @@
 import org.opensearch.be.datafusion.cache.CacheManager;
 import org.opensearch.be.datafusion.cache.CacheUtils;
 import org.opensearch.be.datafusion.nativelib.NativeBridge;
+import org.opensearch.be.datafusion.stats.DataFusionStats;
 import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
 import org.opensearch.common.settings.ClusterSettings;
 
@@ -150,6 +151,17 @@ public void setMemoryPoolLimit(long newLimitBytes) {
         NativeBridge.setMemoryPoolLimit(getNativeRuntime().get(), newLimitBytes);
     }
 
+    /**
+     * Returns the latest native executor stats, collected fresh from JNI on every call.
+     *
+     * @return the current {@link DataFusionStats}
+     */
+    public DataFusionStats getStats() {
+        if (runtimeHandle == null) {
+            throw new IllegalStateException("DataFusionService has not been started");
+        }
+        return NativeBridge.stats();
+    }
     // Cache management (node-level, delegates to native runtime)
 
     /**
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/DataFusionStatsAction.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/DataFusionStatsAction.java
new file mode 100644
index 0000000000000..e6cd90c0e2759
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/DataFusionStatsAction.java
@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.action;
+
+import org.opensearch.be.datafusion.DataFusionService;
+import org.opensearch.be.datafusion.stats.DataFusionStats;
+import org.opensearch.core.rest.RestStatus;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.rest.BaseRestHandler;
+import org.opensearch.rest.BytesRestResponse;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.transport.client.node.NodeClient;
+
+import java.util.List;
+
+/**
+ * REST handler for {@code GET _plugins/datafusion/stats}.
+ * 
      
+ * Collects native executor metrics (Tokio runtime + task monitors) from
+ * {@link DataFusionService} and returns them as a JSON response. Follows
+ * the same {@code BaseRestHandler} → collect → {@code BytesRestResponse}
+ * pattern used by the SQL/PPL stats endpoints.
+ */
+public class DataFusionStatsAction extends BaseRestHandler {
+
+    private final DataFusionService dataFusionService;
+
+    /**
+     * Constructs the stats REST action.
+     *
+     * @param dataFusionService the node-level DataFusion service providing stats
+     */
+    public DataFusionStatsAction(DataFusionService dataFusionService) {
+        this.dataFusionService = dataFusionService;
+    }
+
+    @Override
+    public String getName() {
+        return "datafusion_stats_action";
+    }
+
+    @Override
+    public List routes() {
+        return List.of(new Route(RestRequest.Method.GET, "_plugins/datafusion/stats"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
+        return channel -> {
+            try {
+                DataFusionStats stats = dataFusionService.getStats();
+                XContentBuilder builder = channel.newBuilder();
+                builder.startObject();
+                stats.toXContent(builder, request);
+                builder.endObject();
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
+            } catch (Exception e) {
+                channel.sendResponse(new BytesRestResponse(channel, e));
+            }
+        };
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/package-info.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/package-info.java
new file mode 100644
index 0000000000000..052ef4b042de7
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * REST actions for the DataFusion native execution engine plugin.
+ */
+package org.opensearch.be.datafusion.action;
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
index 21477a6e00e1e..ab63d52df1143 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
@@ -9,6 +9,8 @@
 package org.opensearch.be.datafusion.nativelib;
 
 import org.opensearch.analytics.backend.jni.NativeHandle;
+import org.opensearch.be.datafusion.stats.DataFusionStats;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.nativebridge.spi.NativeCall;
 import org.opensearch.nativebridge.spi.NativeLibraryLoader;
@@ -18,6 +20,7 @@
 import java.lang.foreign.SymbolLookup;
 import java.lang.foreign.ValueLayout;
 import java.lang.invoke.MethodHandle;
+import java.util.LinkedHashMap;
 
 /**
  * FFM bridge to native DataFusion library.
@@ -77,6 +80,7 @@ public final class NativeBridge {
     private static final MethodHandle CLOSE_SESSION_CONTEXT;
     private static final MethodHandle EXECUTE_WITH_CONTEXT;
     private static final MethodHandle CANCEL_QUERY;
+    private static final MethodHandle STATS;
 
     static {
         SymbolLookup lib = NativeLibraryLoader.symbolLookup();
@@ -376,6 +380,12 @@ public final class NativeBridge {
             lib.find("df_execute_with_context").orElseThrow(),
             FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG)
         );
+
+        // i64 df_stats(out_ptr, out_cap)
+        STATS = linker.downcallHandle(
+            lib.find("df_stats").orElseThrow(),
+            FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG)
+        );
     }
 
     private NativeBridge() {}
@@ -610,6 +620,36 @@ public static void cancelQuery(long contextId) {
         NativeCall.invokeVoid(CANCEL_QUERY, contextId);
     }
 
+    // ---- Stats collection ----
+
+    /**
+     * Collects all native executor metrics in a single FFM call.
+     * Decodes directly from the MemorySegment — no intermediate long[].
+     *
+     * @return a fully constructed {@link DataFusionStats}
+     * @throws IllegalStateException if the runtime manager is not initialized
+     */
+    public static DataFusionStats stats() {
+        try (var call = new NativeCall()) {
+            var seg = call.buf((int) StatsLayout.LAYOUT.byteSize());
+            call.invoke(STATS, seg, StatsLayout.LAYOUT.byteSize());
+
+            // IO runtime (always present — zeroed if not yet initialized)
+            var ioRuntime = StatsLayout.readRuntimeMetrics(seg, "io_runtime");
+
+            // CPU runtime (always present — zeroed when absent)
+            var cpuRuntime = StatsLayout.readRuntimeMetrics(seg, "cpu_runtime");
+
+            // Task monitors
+            var taskMonitors = new LinkedHashMap();
+            for (NativeExecutorsStats.OperationType op : NativeExecutorsStats.OperationType.values()) {
+                taskMonitors.put(op.key(), StatsLayout.readTaskMonitor(seg, op.key()));
+            }
+
+            return new DataFusionStats(new NativeExecutorsStats(ioRuntime, cpuRuntime, taskMonitors));
+        }
+    }
+
     // ---- Stubs ----
 
     public static byte[] sqlToSubstrait(long readerPtr, String tableName, String sql, long runtimePtr) {
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/StatsLayout.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/StatsLayout.java
new file mode 100644
index 0000000000000..5ca11b1b8c633
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/StatsLayout.java
@@ -0,0 +1,223 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.nativelib;
+
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats;
+
+import java.lang.foreign.MemoryLayout;
+import java.lang.foreign.MemoryLayout.PathElement;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.StructLayout;
+import java.lang.foreign.ValueLayout;
+import java.lang.invoke.VarHandle;
+
+/**
+ * Defines the {@code MemoryLayout.structLayout} mirroring the Rust {@code DfStatsBuffer}
+ * and provides {@link VarHandle} accessors for each field via layout path navigation.
+ *
+ * 
      The layout contains 6 named groups (2 runtime × 9 fields + 4 task monitor × 3 fields = 30 longs = 240 bytes).
+ */
+public final class StatsLayout {
+
+    private static final String[] RUNTIME_FIELDS = {
+        "workers_count",
+        "total_polls_count",
+        "total_busy_duration_ms",
+        "total_overflow_count",
+        "global_queue_depth",
+        "blocking_queue_depth",
+        "num_alive_tasks",
+        "spawned_tasks_count",
+        "total_local_queue_depth" };
+
+    private static final String[] TASK_MONITOR_FIELDS = {
+        "total_poll_duration_ms",
+        "total_scheduled_duration_ms",
+        "total_idle_duration_ms" };
+
+    /** The struct layout mirroring Rust's {@code DfStatsBuffer}. */
+    public static final StructLayout LAYOUT = MemoryLayout.structLayout(
+        runtimeGroup("io_runtime"),
+        runtimeGroup("cpu_runtime"),
+        taskMonitorGroup("query_execution"),
+        taskMonitorGroup("stream_next"),
+        taskMonitorGroup("fetch_phase"),
+        taskMonitorGroup("segment_stats")
+    );
+
+    static {
+        if (LAYOUT.byteSize() != 30 * Long.BYTES) {
+            throw new AssertionError("StatsLayout size mismatch: expected " + (30 * Long.BYTES) + " but got " + LAYOUT.byteSize());
+        }
+    }
+
+    // ---- VarHandles for io_runtime fields ----
+    private static final VarHandle IO_WORKERS_COUNT = handle("io_runtime", "workers_count");
+    private static final VarHandle IO_TOTAL_POLLS_COUNT = handle("io_runtime", "total_polls_count");
+    private static final VarHandle IO_TOTAL_BUSY_DURATION_MS = handle("io_runtime", "total_busy_duration_ms");
+    private static final VarHandle IO_TOTAL_OVERFLOW_COUNT = handle("io_runtime", "total_overflow_count");
+    private static final VarHandle IO_GLOBAL_QUEUE_DEPTH = handle("io_runtime", "global_queue_depth");
+    private static final VarHandle IO_BLOCKING_QUEUE_DEPTH = handle("io_runtime", "blocking_queue_depth");
+    private static final VarHandle IO_NUM_ALIVE_TASKS = handle("io_runtime", "num_alive_tasks");
+    private static final VarHandle IO_SPAWNED_TASKS_COUNT = handle("io_runtime", "spawned_tasks_count");
+    private static final VarHandle IO_TOTAL_LOCAL_QUEUE_DEPTH = handle("io_runtime", "total_local_queue_depth");
+
+    // ---- VarHandles for cpu_runtime fields ----
+    private static final VarHandle CPU_WORKERS_COUNT = handle("cpu_runtime", "workers_count");
+    private static final VarHandle CPU_TOTAL_POLLS_COUNT = handle("cpu_runtime", "total_polls_count");
+    private static final VarHandle CPU_TOTAL_BUSY_DURATION_MS = handle("cpu_runtime", "total_busy_duration_ms");
+    private static final VarHandle CPU_TOTAL_OVERFLOW_COUNT = handle("cpu_runtime", "total_overflow_count");
+    private static final VarHandle CPU_GLOBAL_QUEUE_DEPTH = handle("cpu_runtime", "global_queue_depth");
+    private static final VarHandle CPU_BLOCKING_QUEUE_DEPTH = handle("cpu_runtime", "blocking_queue_depth");
+    private static final VarHandle CPU_NUM_ALIVE_TASKS = handle("cpu_runtime", "num_alive_tasks");
+    private static final VarHandle CPU_SPAWNED_TASKS_COUNT = handle("cpu_runtime", "spawned_tasks_count");
+    private static final VarHandle CPU_TOTAL_LOCAL_QUEUE_DEPTH = handle("cpu_runtime", "total_local_queue_depth");
+
+    // ---- VarHandles for query_execution fields ----
+    private static final VarHandle QE_TOTAL_POLL_DURATION_MS = handle("query_execution", "total_poll_duration_ms");
+    private static final VarHandle QE_TOTAL_SCHEDULED_DURATION_MS = handle("query_execution", "total_scheduled_duration_ms");
+    private static final VarHandle QE_TOTAL_IDLE_DURATION_MS = handle("query_execution", "total_idle_duration_ms");
+
+    // ---- VarHandles for stream_next fields ----
+    private static final VarHandle SN_TOTAL_POLL_DURATION_MS = handle("stream_next", "total_poll_duration_ms");
+    private static final VarHandle SN_TOTAL_SCHEDULED_DURATION_MS = handle("stream_next", "total_scheduled_duration_ms");
+    private static final VarHandle SN_TOTAL_IDLE_DURATION_MS = handle("stream_next", "total_idle_duration_ms");
+
+    // ---- VarHandles for fetch_phase fields ----
+    private static final VarHandle FP_TOTAL_POLL_DURATION_MS = handle("fetch_phase", "total_poll_duration_ms");
+    private static final VarHandle FP_TOTAL_SCHEDULED_DURATION_MS = handle("fetch_phase", "total_scheduled_duration_ms");
+    private static final VarHandle FP_TOTAL_IDLE_DURATION_MS = handle("fetch_phase", "total_idle_duration_ms");
+
+    // ---- VarHandles for segment_stats fields ----
+    private static final VarHandle SS_TOTAL_POLL_DURATION_MS = handle("segment_stats", "total_poll_duration_ms");
+    private static final VarHandle SS_TOTAL_SCHEDULED_DURATION_MS = handle("segment_stats", "total_scheduled_duration_ms");
+    private static final VarHandle SS_TOTAL_IDLE_DURATION_MS = handle("segment_stats", "total_idle_duration_ms");
+
+    private StatsLayout() {}
+
+    /**
+     * Read a single field from the segment.
+     *
+     * @param seg   the memory segment containing the DfStatsBuffer
+     * @param group the group name (e.g. "io_runtime", "cpu_runtime")
+     * @param field the field name (e.g. "workers_count")
+     * @return the long value at the specified path
+     */
+    public static long readField(MemorySegment seg, String group, String field) {
+        return (long) handle(group, field).get(seg, 0L);
+    }
+
+    /**
+     * Read a runtime metrics group (8 fields) from the segment.
+     *
+     * @param seg   the memory segment containing the DfStatsBuffer
+     * @param group "io_runtime" or "cpu_runtime"
+     * @return a populated RuntimeMetrics instance
+     */
+    public static NativeExecutorsStats.RuntimeMetrics readRuntimeMetrics(MemorySegment seg, String group) {
+        VarHandle[] handles = runtimeHandles(group);
+        return new NativeExecutorsStats.RuntimeMetrics(
+            (long) handles[0].get(seg, 0L),
+            (long) handles[1].get(seg, 0L),
+            (long) handles[2].get(seg, 0L),
+            (long) handles[3].get(seg, 0L),
+            (long) handles[4].get(seg, 0L),
+            (long) handles[5].get(seg, 0L),
+            (long) handles[6].get(seg, 0L),
+            (long) handles[7].get(seg, 0L),
+            (long) handles[8].get(seg, 0L)
+        );
+    }
+
+    /**
+     * Read a task monitor group (3 fields) from the segment.
+     *
+     * @param seg   the memory segment containing the DfStatsBuffer
+     * @param group "query_execution", "stream_next", "fetch_phase", or "segment_stats"
+     * @return a populated TaskMonitorStats instance
+     */
+    public static NativeExecutorsStats.TaskMonitorStats readTaskMonitor(MemorySegment seg, String group) {
+        VarHandle[] handles = taskMonitorHandles(group);
+        return new NativeExecutorsStats.TaskMonitorStats(
+            (long) handles[0].get(seg, 0L),
+            (long) handles[1].get(seg, 0L),
+            (long) handles[2].get(seg, 0L)
+        );
+    }
+
+    // ---- Private helpers ----
+
+    private static StructLayout runtimeGroup(String name) {
+        return MemoryLayout.structLayout(
+            ValueLayout.JAVA_LONG.withName("workers_count"),
+            ValueLayout.JAVA_LONG.withName("total_polls_count"),
+            ValueLayout.JAVA_LONG.withName("total_busy_duration_ms"),
+            ValueLayout.JAVA_LONG.withName("total_overflow_count"),
+            ValueLayout.JAVA_LONG.withName("global_queue_depth"),
+            ValueLayout.JAVA_LONG.withName("blocking_queue_depth"),
+            ValueLayout.JAVA_LONG.withName("num_alive_tasks"),
+            ValueLayout.JAVA_LONG.withName("spawned_tasks_count"),
+            ValueLayout.JAVA_LONG.withName("total_local_queue_depth")
+        ).withName(name);
+    }
+
+    private static StructLayout taskMonitorGroup(String name) {
+        return MemoryLayout.structLayout(
+            ValueLayout.JAVA_LONG.withName("total_poll_duration_ms"),
+            ValueLayout.JAVA_LONG.withName("total_scheduled_duration_ms"),
+            ValueLayout.JAVA_LONG.withName("total_idle_duration_ms")
+        ).withName(name);
+    }
+
+    private static VarHandle handle(String group, String field) {
+        return LAYOUT.varHandle(PathElement.groupElement(group), PathElement.groupElement(field));
+    }
+
+    private static VarHandle[] runtimeHandles(String group) {
+        return switch (group) {
+            case "io_runtime" -> new VarHandle[] {
+                IO_WORKERS_COUNT,
+                IO_TOTAL_POLLS_COUNT,
+                IO_TOTAL_BUSY_DURATION_MS,
+                IO_TOTAL_OVERFLOW_COUNT,
+                IO_GLOBAL_QUEUE_DEPTH,
+                IO_BLOCKING_QUEUE_DEPTH,
+                IO_NUM_ALIVE_TASKS,
+                IO_SPAWNED_TASKS_COUNT,
+                IO_TOTAL_LOCAL_QUEUE_DEPTH };
+            case "cpu_runtime" -> new VarHandle[] {
+                CPU_WORKERS_COUNT,
+                CPU_TOTAL_POLLS_COUNT,
+                CPU_TOTAL_BUSY_DURATION_MS,
+                CPU_TOTAL_OVERFLOW_COUNT,
+                CPU_GLOBAL_QUEUE_DEPTH,
+                CPU_BLOCKING_QUEUE_DEPTH,
+                CPU_NUM_ALIVE_TASKS,
+                CPU_SPAWNED_TASKS_COUNT,
+                CPU_TOTAL_LOCAL_QUEUE_DEPTH };
+            default -> throw new IllegalArgumentException("Unknown runtime group: " + group);
+        };
+    }
+
+    private static VarHandle[] taskMonitorHandles(String group) {
+        return switch (group) {
+            case "query_execution" -> new VarHandle[] {
+                QE_TOTAL_POLL_DURATION_MS,
+                QE_TOTAL_SCHEDULED_DURATION_MS,
+                QE_TOTAL_IDLE_DURATION_MS };
+            case "stream_next" -> new VarHandle[] { SN_TOTAL_POLL_DURATION_MS, SN_TOTAL_SCHEDULED_DURATION_MS, SN_TOTAL_IDLE_DURATION_MS };
+            case "fetch_phase" -> new VarHandle[] { FP_TOTAL_POLL_DURATION_MS, FP_TOTAL_SCHEDULED_DURATION_MS, FP_TOTAL_IDLE_DURATION_MS };
+            case "segment_stats" -> new VarHandle[] {
+                SS_TOTAL_POLL_DURATION_MS,
+                SS_TOTAL_SCHEDULED_DURATION_MS,
+                SS_TOTAL_IDLE_DURATION_MS };
+            default -> throw new IllegalArgumentException("Unknown task monitor group: " + group);
+        };
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/DataFusionBackendStatsProvider.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/DataFusionBackendStatsProvider.java
new file mode 100644
index 0000000000000..9f79a8c45ba55
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/DataFusionBackendStatsProvider.java
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.stats;
+
+import org.opensearch.plugin.stats.BackendStatsProvider;
+import org.opensearch.plugin.stats.PluginStats;
+
+/**
+ * DataFusion implementation of {@link BackendStatsProvider}.
+ *
+ * 
      When the Mustang Analytics Plugin lands, it discovers
+ * {@code BackendStatsProvider} implementations and iterates over them.
+ * DataFusion is already registered via this class.
+ */
+public class DataFusionBackendStatsProvider implements BackendStatsProvider {
+
+    /** Creates a new {@code DataFusionBackendStatsProvider}. */
+    public DataFusionBackendStatsProvider() {}
+
+    @Override
+    public String name() {
+        return "datafusion";
+    }
+
+    @Override
+    public PluginStats getBackendStats() {
+        // TODO: Expose only necessary DF metrics to core.
+        return null;
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/DataFusionStats.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/DataFusionStats.java
new file mode 100644
index 0000000000000..c51774579b71f
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/DataFusionStats.java
@@ -0,0 +1,88 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.stats;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.plugin.stats.PluginStats;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Top-level stats container for the DataFusion backend.
+ *
+ * 
      Implements {@link PluginStats} for Mustang Stats Framework compatibility,
+ * {@link Writeable} for transport serialization, and {@link ToXContentFragment}
+ * for JSON rendering.
+ *
+ * 
      Composes {@link NativeExecutorsStats} rather than duplicating its fields,
+ * making it extensible for future metric categories (e.g. MemoryPoolStats).
+ * No inner classes — {@code RuntimeMetrics} and {@code TaskMonitorStats} belong
+ * to {@link NativeExecutorsStats}.
+ */
+public class DataFusionStats implements PluginStats, Writeable, ToXContentFragment {
+
+    private final NativeExecutorsStats nativeExecutorsStats; // nullable
+
+    /**
+     * Construct from components.
+     *
+     * @param nativeExecutorsStats the native executor metrics (nullable)
+     */
+    public DataFusionStats(NativeExecutorsStats nativeExecutorsStats) {
+        this.nativeExecutorsStats = nativeExecutorsStats;
+    }
+
+    /**
+     * Deserialize from stream.
+     *
+     * @param in the stream input
+     * @throws IOException if deserialization fails
+     */
+    public DataFusionStats(StreamInput in) throws IOException {
+        this.nativeExecutorsStats = in.readOptionalWriteable(NativeExecutorsStats::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalWriteable(nativeExecutorsStats);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (nativeExecutorsStats != null) {
+            nativeExecutorsStats.toXContent(builder, params);
+        }
+        return builder;
+    }
+
+    /**
+     * Returns the native executor metrics, or {@code null} if absent.
+     */
+    public NativeExecutorsStats getNativeExecutorsStats() {
+        return nativeExecutorsStats;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DataFusionStats that = (DataFusionStats) o;
+        return Objects.equals(nativeExecutorsStats, that.nativeExecutorsStats);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nativeExecutorsStats);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/NativeExecutorsStats.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/NativeExecutorsStats.java
new file mode 100644
index 0000000000000..654e666faa7bd
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/NativeExecutorsStats.java
@@ -0,0 +1,369 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.stats;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * {@link Writeable} + {@link ToXContentFragment} container for native executor metrics
+ * (Tokio runtime metrics + per-operation task monitors).
+ *
+ * 
      Contains an IO {@link RuntimeMetrics} (always present), an optional CPU
+ * {@link RuntimeMetrics}, and 4 {@link TaskMonitorStats} for the operation types:
+ * query_execution, stream_next, fetch_phase, segment_stats.
+ */
+public class NativeExecutorsStats implements Writeable, ToXContentFragment {
+
+    /** Operation types in documented order. */
+    public enum OperationType {
+        /** Query execution operation. */
+        QUERY_EXECUTION("query_execution"),
+        /** Stream next (pagination) operation. */
+        STREAM_NEXT("stream_next"),
+        /** Fetch phase operation. */
+        FETCH_PHASE("fetch_phase"),
+        /** Segment-level statistics collection operation. */
+        SEGMENT_STATS("segment_stats");
+
+        private final String key;
+
+        OperationType(String key) {
+            this.key = key;
+        }
+
+        /** Returns the snake_case key used in serialization and XContent output. */
+        public String key() {
+            return key;
+        }
+    }
+
+    private final RuntimeMetrics ioRuntime;
+    private final RuntimeMetrics cpuRuntime; // nullable
+    private final Map taskMonitors;
+
+    /**
+     * Construct from individual components.
+     *
+     * @param ioRuntime    the IO runtime metrics (must not be null)
+     * @param cpuRuntime   the CPU runtime metrics (nullable)
+     * @param taskMonitors per-operation task monitor metrics
+     */
+    // cpuRuntime is nullable — zeroed when absent (workers_count == 0), omitted from XContent when null
+    public NativeExecutorsStats(RuntimeMetrics ioRuntime, RuntimeMetrics cpuRuntime, Map taskMonitors) {
+        this.ioRuntime = Objects.requireNonNull(ioRuntime);
+        this.cpuRuntime = cpuRuntime;
+        this.taskMonitors = Objects.requireNonNull(taskMonitors);
+    }
+
+    /**
+     * Deserialize from stream.
+     *
+     * @param in the stream input
+     * @throws IOException if deserialization fails
+     */
+    public NativeExecutorsStats(StreamInput in) throws IOException {
+        this.ioRuntime = new RuntimeMetrics(in);
+        this.cpuRuntime = in.readBoolean() ? new RuntimeMetrics(in) : null;
+
+        this.taskMonitors = new LinkedHashMap<>();
+        for (OperationType opType : OperationType.values()) {
+            this.taskMonitors.put(opType.key(), new TaskMonitorStats(in));
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        ioRuntime.writeTo(out);
+        if (cpuRuntime != null) {
+            out.writeBoolean(true);
+            cpuRuntime.writeTo(out);
+        } else {
+            out.writeBoolean(false);
+        }
+        for (OperationType opType : OperationType.values()) {
+            taskMonitors.get(opType.key()).writeTo(out);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject("native_executors");
+
+        builder.startObject("io_runtime");
+        ioRuntime.toXContent(builder);
+        builder.endObject();
+
+        if (cpuRuntime != null) {
+            builder.startObject("cpu_runtime");
+            cpuRuntime.toXContent(builder);
+            builder.endObject();
+        }
+
+        builder.startObject("task_monitors");
+        for (Map.Entry entry : taskMonitors.entrySet()) {
+            builder.startObject(entry.getKey());
+            entry.getValue().toXContent(builder);
+            builder.endObject();
+        }
+        builder.endObject(); // task_monitors
+
+        builder.endObject(); // native_executors
+        return builder;
+    }
+
+    /** Returns the IO runtime metrics. */
+    public RuntimeMetrics getIoRuntime() {
+        return ioRuntime;
+    }
+
+    /** Returns the CPU runtime metrics, or {@code null} if absent. */
+    public RuntimeMetrics getCpuRuntime() {
+        return cpuRuntime;
+    }
+
+    /** Returns the per-operation task monitor metrics. */
+    public Map getTaskMonitors() {
+        return taskMonitors;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NativeExecutorsStats that = (NativeExecutorsStats) o;
+        return Objects.equals(ioRuntime, that.ioRuntime)
+            && Objects.equals(cpuRuntime, that.cpuRuntime)
+            && Objects.equals(taskMonitors, that.taskMonitors);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(ioRuntime, cpuRuntime, taskMonitors);
+    }
+
+    /**
+     * 8 fields from {@code tokio_metrics::RuntimeMonitor} describing
+     * per-worker thread pool behavior for a single Tokio runtime.
+     */
+    public static class RuntimeMetrics implements Writeable {
+        /** Number of worker threads in the runtime. */
+        public final long workersCount;
+        /** Total number of task polls across all workers. */
+        public final long totalPollsCount;
+        /** Total time workers spent executing tasks, in milliseconds. */
+        public final long totalBusyDurationMs;
+        /** Total number of times tasks were pushed to the overflow queue. */
+        public final long totalOverflowCount;
+        /** Current depth of the global injection queue. */
+        public final long globalQueueDepth;
+        /** Current depth of the blocking thread pool queue. */
+        public final long blockingQueueDepth;
+        /** Number of tasks currently alive (spawned but not yet completed) on this runtime. */
+        public final long numAliveTasks;
+        /** Total number of tasks spawned on this runtime since creation. */
+        public final long spawnedTasksCount;
+        /** Sum of all per-worker local queue depths (tasks queued on worker-local run queues). */
+        public final long totalLocalQueueDepth;
+
+        /**
+         * Construct from explicit field values.
+         *
+         * @param workersCount        number of worker threads
+         * @param totalPollsCount     total task polls across all workers
+         * @param totalBusyDurationMs total busy time in milliseconds
+         * @param totalOverflowCount  total overflow queue pushes
+         * @param globalQueueDepth    current global injection queue depth
+         * @param blockingQueueDepth  current blocking thread pool queue depth
+         * @param numAliveTasks       tasks currently alive
+         * @param spawnedTasksCount   total tasks spawned since creation
+         * @param totalLocalQueueDepth     sum of per-worker local queue depths
+         */
+        public RuntimeMetrics(
+            long workersCount,
+            long totalPollsCount,
+            long totalBusyDurationMs,
+            long totalOverflowCount,
+            long globalQueueDepth,
+            long blockingQueueDepth,
+            long numAliveTasks,
+            long spawnedTasksCount,
+            long totalLocalQueueDepth
+        ) {
+            this.workersCount = workersCount;
+            this.totalPollsCount = totalPollsCount;
+            this.totalBusyDurationMs = totalBusyDurationMs;
+            this.totalOverflowCount = totalOverflowCount;
+            this.globalQueueDepth = globalQueueDepth;
+            this.blockingQueueDepth = blockingQueueDepth;
+            this.numAliveTasks = numAliveTasks;
+            this.spawnedTasksCount = spawnedTasksCount;
+            this.totalLocalQueueDepth = totalLocalQueueDepth;
+        }
+
+        /**
+         * Deserialize from stream.
+         *
+         * @param in the stream input
+         * @throws IOException if deserialization fails
+         */
+        public RuntimeMetrics(StreamInput in) throws IOException {
+            this.workersCount = in.readVLong();
+            this.totalPollsCount = in.readVLong();
+            this.totalBusyDurationMs = in.readVLong();
+            this.totalOverflowCount = in.readVLong();
+            this.globalQueueDepth = in.readVLong();
+            this.blockingQueueDepth = in.readVLong();
+            this.numAliveTasks = in.readVLong();
+            this.spawnedTasksCount = in.readVLong();
+            this.totalLocalQueueDepth = in.readVLong();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVLong(workersCount);
+            out.writeVLong(totalPollsCount);
+            out.writeVLong(totalBusyDurationMs);
+            out.writeVLong(totalOverflowCount);
+            out.writeVLong(globalQueueDepth);
+            out.writeVLong(blockingQueueDepth);
+            out.writeVLong(numAliveTasks);
+            out.writeVLong(spawnedTasksCount);
+            out.writeVLong(totalLocalQueueDepth);
+        }
+
+        /**
+         * Render all 8 fields as snake_case JSON fields.
+         *
+         * @param builder the XContent builder to write to
+         * @throws IOException if writing fails
+         */
+        public void toXContent(XContentBuilder builder) throws IOException {
+            builder.field("workers_count", workersCount);
+            builder.field("total_polls_count", totalPollsCount);
+            builder.field("total_busy_duration_ms", totalBusyDurationMs);
+            builder.field("total_overflow_count", totalOverflowCount);
+            builder.field("global_queue_depth", globalQueueDepth);
+            builder.field("blocking_queue_depth", blockingQueueDepth);
+            builder.field("num_alive_tasks", numAliveTasks);
+            builder.field("spawned_tasks_count", spawnedTasksCount);
+            builder.field("total_local_queue_depth", totalLocalQueueDepth);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            RuntimeMetrics that = (RuntimeMetrics) o;
+            return workersCount == that.workersCount
+                && totalPollsCount == that.totalPollsCount
+                && totalBusyDurationMs == that.totalBusyDurationMs
+                && totalOverflowCount == that.totalOverflowCount
+                && globalQueueDepth == that.globalQueueDepth
+                && blockingQueueDepth == that.blockingQueueDepth
+                && numAliveTasks == that.numAliveTasks
+                && spawnedTasksCount == that.spawnedTasksCount
+                && totalLocalQueueDepth == that.totalLocalQueueDepth;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(
+                workersCount,
+                totalPollsCount,
+                totalBusyDurationMs,
+                totalOverflowCount,
+                globalQueueDepth,
+                blockingQueueDepth,
+                numAliveTasks,
+                spawnedTasksCount,
+                totalLocalQueueDepth
+            );
+        }
+    }
+
+    /**
+     * 3 duration fields per operation type from {@code tokio_metrics::TaskMonitor::cumulative()}.
+     */
+    public static class TaskMonitorStats implements Writeable {
+        /** Total time spent polling instrumented futures, in milliseconds. */
+        public final long totalPollDurationMs;
+        /** Total time tasks spent waiting in the scheduler queue, in milliseconds. */
+        public final long totalScheduledDurationMs;
+        /** Total time tasks spent idle between polls, in milliseconds. */
+        public final long totalIdleDurationMs;
+
+        /**
+         * Construct from explicit field values.
+         *
+         * @param totalPollDurationMs      total poll duration in milliseconds
+         * @param totalScheduledDurationMs total scheduled duration in milliseconds
+         * @param totalIdleDurationMs      total idle duration in milliseconds
+         */
+        public TaskMonitorStats(long totalPollDurationMs, long totalScheduledDurationMs, long totalIdleDurationMs) {
+            this.totalPollDurationMs = totalPollDurationMs;
+            this.totalScheduledDurationMs = totalScheduledDurationMs;
+            this.totalIdleDurationMs = totalIdleDurationMs;
+        }
+
+        /**
+         * Deserialize from stream.
+         *
+         * @param in the stream input
+         * @throws IOException if deserialization fails
+         */
+        public TaskMonitorStats(StreamInput in) throws IOException {
+            this.totalPollDurationMs = in.readVLong();
+            this.totalScheduledDurationMs = in.readVLong();
+            this.totalIdleDurationMs = in.readVLong();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVLong(totalPollDurationMs);
+            out.writeVLong(totalScheduledDurationMs);
+            out.writeVLong(totalIdleDurationMs);
+        }
+
+        /**
+         * Render all 3 fields as snake_case JSON fields.
+         *
+         * @param builder the XContent builder to write to
+         * @throws IOException if writing fails
+         */
+        public void toXContent(XContentBuilder builder) throws IOException {
+            builder.field("total_poll_duration_ms", totalPollDurationMs);
+            builder.field("total_scheduled_duration_ms", totalScheduledDurationMs);
+            builder.field("total_idle_duration_ms", totalIdleDurationMs);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            TaskMonitorStats that = (TaskMonitorStats) o;
+            return totalPollDurationMs == that.totalPollDurationMs
+                && totalScheduledDurationMs == that.totalScheduledDurationMs
+                && totalIdleDurationMs == that.totalIdleDurationMs;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(totalPollDurationMs, totalScheduledDurationMs, totalIdleDurationMs);
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/package-info.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/package-info.java
new file mode 100644
index 0000000000000..b688aac8f5437
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/package-info.java
@@ -0,0 +1,17 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Plugin-side stats providers for the DataFusion native execution engine.
+ *
+ * 
      Contains {@link org.opensearch.be.datafusion.stats.DataFusionBackendStatsProvider}
+ * which implements the SPI {@code BackendStatsProvider} interface. The core stats types
+ * ({@code DataFusionStats}, {@code NativeExecutorsStats}) live in the
+ * {@code org.opensearch.plugin.stats} package.
+ */
+package org.opensearch.be.datafusion.stats;
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/nativelib/StatsLayoutPropertyTests.java b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/nativelib/StatsLayoutPropertyTests.java
new file mode 100644
index 0000000000000..72f820d947078
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/nativelib/StatsLayoutPropertyTests.java
@@ -0,0 +1,308 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.nativelib;
+
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.io.stream.StreamInput;
+
+import java.io.IOException;
+import java.lang.foreign.Arena;
+import java.lang.foreign.ValueLayout;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import net.jqwik.api.Arbitraries;
+import net.jqwik.api.Arbitrary;
+import net.jqwik.api.Combinators;
+import net.jqwik.api.ForAll;
+import net.jqwik.api.Property;
+import net.jqwik.api.Provide;
+import net.jqwik.api.Tag;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+/**
+ * Property-based tests for {@link StatsLayout} struct decode.
+ *
+ * 
      Validates the three correctness properties from the ffm-struct-layout design:
+ * 
        
+ *   
      1. Pack-then-decode round-trip preserves all fields
        2. 
+ *   
      2. Decode-then-reencode produces byte-identical buffer
        3. 
+ *   
      3. Writeable serialization round-trip
        4. 
+ * 
      
+ */
+public class StatsLayoutPropertyTests {
+
+    private static final int FIELD_COUNT = 30;
+    private static final int BUFFER_SIZE = FIELD_COUNT * Long.BYTES;
+
+    // ---- Generators ----
+
+    @Provide
+    Arbitrary thirtyLongs() {
+        return Arbitraries.longs().between(0, Long.MAX_VALUE / 2).array(long[].class).ofSize(FIELD_COUNT);
+    }
+
+    @Provide
+    Arbitrary thirtyLongsWithCpuWorkersZero() {
+        return thirtyLongs().map(arr -> {
+            arr[9] = 0; // cpu_runtime.workers_count = 0
+            return arr;
+        });
+    }
+
+    @Provide
+    Arbitrary thirtyLongsWithCpuWorkersPositive() {
+        return thirtyLongs().map(arr -> {
+            if (arr[9] == 0) arr[9] = 1; // ensure cpu_runtime.workers_count > 0
+            return arr;
+        });
+    }
+
+    @Provide
+    Arbitrary runtimeMetrics() {
+        return Arbitraries.longs()
+            .between(0, Long.MAX_VALUE / 2)
+            .list()
+            .ofSize(9)
+            .map(l -> new RuntimeMetrics(l.get(0), l.get(1), l.get(2), l.get(3), l.get(4), l.get(5), l.get(6), l.get(7), l.get(8)));
+    }
+
+    @Provide
+    Arbitrary taskMonitorValues() {
+        Arbitrary nonNeg = Arbitraries.longs().between(0, Long.MAX_VALUE / 2);
+        return Combinators.combine(nonNeg, nonNeg, nonNeg).as(TaskMonitorStats::new);
+    }
+
+    @Provide
+    Arbitrary nativeExecutorsStatsWithCpu() {
+        return Combinators.combine(runtimeMetrics(), runtimeMetrics().map(rt -> {
+            if (rt.workersCount == 0) {
+                return new RuntimeMetrics(
+                    1,
+                    rt.totalPollsCount,
+                    rt.totalBusyDurationMs,
+                    rt.totalOverflowCount,
+                    rt.globalQueueDepth,
+                    rt.blockingQueueDepth,
+                    rt.numAliveTasks,
+                    rt.spawnedTasksCount,
+                    rt.totalLocalQueueDepth
+                );
+            }
+            return rt;
+        }), taskMonitorValues(), taskMonitorValues(), taskMonitorValues(), taskMonitorValues()).as((io, cpu, qe, sn, fp, ss) -> {
+            Map monitors = new LinkedHashMap<>();
+            monitors.put("query_execution", qe);
+            monitors.put("stream_next", sn);
+            monitors.put("fetch_phase", fp);
+            monitors.put("segment_stats", ss);
+            return new NativeExecutorsStats(io, cpu, monitors);
+        });
+    }
+
+    @Provide
+    Arbitrary nativeExecutorsStatsNoCpu() {
+        return Combinators.combine(runtimeMetrics(), taskMonitorValues(), taskMonitorValues(), taskMonitorValues(), taskMonitorValues())
+            .as((io, qe, sn, fp, ss) -> {
+                Map monitors = new LinkedHashMap<>();
+                monitors.put("query_execution", qe);
+                monitors.put("stream_next", sn);
+                monitors.put("fetch_phase", fp);
+                monitors.put("segment_stats", ss);
+                return new NativeExecutorsStats(io, null, monitors);
+            });
+    }
+
+    // ---- Property 1: Pack-then-decode round-trip (cpu workers > 0) ----
+
+    /**
+     * Property 1: Pack-then-decode round-trip preserves all fields (CPU runtime present).
+     *
+     * Validates: Requirements 3.3, 3.4, 4.3, 4.4, 4.5, 4.6, 6.1, 8.1, 8.3, 8.4
+     */
+    @Property(tries = 100)
+    @Tag("Feature: ffm-struct-layout, Property 1: Pack-then-decode round-trip preserves all fields")
+    void packThenDecodeRoundTripWithCpu(@ForAll("thirtyLongsWithCpuWorkersPositive") long[] values) {
+        try (var arena = Arena.ofConfined()) {
+            var seg = arena.allocate(StatsLayout.LAYOUT);
+            for (int i = 0; i < FIELD_COUNT; i++) {
+                seg.setAtIndex(ValueLayout.JAVA_LONG, i, values[i]);
+            }
+
+            var ioRuntime = StatsLayout.readRuntimeMetrics(seg, "io_runtime");
+            assertEquals(values[0], ioRuntime.workersCount);
+            assertEquals(values[1], ioRuntime.totalPollsCount);
+            assertEquals(values[2], ioRuntime.totalBusyDurationMs);
+            assertEquals(values[3], ioRuntime.totalOverflowCount);
+            assertEquals(values[4], ioRuntime.globalQueueDepth);
+            assertEquals(values[5], ioRuntime.blockingQueueDepth);
+            assertEquals(values[6], ioRuntime.numAliveTasks);
+            assertEquals(values[7], ioRuntime.spawnedTasksCount);
+            assertEquals(values[8], ioRuntime.totalLocalQueueDepth);
+
+            long cpuWorkers = StatsLayout.readField(seg, "cpu_runtime", "workers_count");
+            assert cpuWorkers > 0 : "cpu workers should be > 0";
+            var cpuRuntime = StatsLayout.readRuntimeMetrics(seg, "cpu_runtime");
+            assertNotNull(cpuRuntime);
+            assertEquals(values[9], cpuRuntime.workersCount);
+            assertEquals(values[10], cpuRuntime.totalPollsCount);
+            assertEquals(values[11], cpuRuntime.totalBusyDurationMs);
+            assertEquals(values[12], cpuRuntime.totalOverflowCount);
+            assertEquals(values[13], cpuRuntime.globalQueueDepth);
+            assertEquals(values[14], cpuRuntime.blockingQueueDepth);
+            assertEquals(values[15], cpuRuntime.numAliveTasks);
+            assertEquals(values[16], cpuRuntime.spawnedTasksCount);
+            assertEquals(values[17], cpuRuntime.totalLocalQueueDepth);
+
+            String[] tmGroups = { "query_execution", "stream_next", "fetch_phase", "segment_stats" };
+            for (int g = 0; g < 4; g++) {
+                var tm = StatsLayout.readTaskMonitor(seg, tmGroups[g]);
+                int base = 18 + g * 3;
+                assertEquals(values[base], tm.totalPollDurationMs, tmGroups[g] + ".total_poll_duration_ms");
+                assertEquals(values[base + 1], tm.totalScheduledDurationMs, tmGroups[g] + ".total_scheduled_duration_ms");
+                assertEquals(values[base + 2], tm.totalIdleDurationMs, tmGroups[g] + ".total_idle_duration_ms");
+            }
+        }
+    }
+
+    /**
+     * Property 1: Pack-then-decode round-trip — CPU runtime null when workers_count == 0.
+     *
+     * Validates: Requirements 3.3, 3.4, 4.4, 8.3
+     */
+    @Property(tries = 100)
+    @Tag("Feature: ffm-struct-layout, Property 1: Pack-then-decode round-trip preserves all fields")
+    void packThenDecodeRoundTripCpuNull(@ForAll("thirtyLongsWithCpuWorkersZero") long[] values) {
+        try (var arena = Arena.ofConfined()) {
+            var seg = arena.allocate(StatsLayout.LAYOUT);
+            for (int i = 0; i < FIELD_COUNT; i++) {
+                seg.setAtIndex(ValueLayout.JAVA_LONG, i, values[i]);
+            }
+
+            long cpuWorkers = StatsLayout.readField(seg, "cpu_runtime", "workers_count");
+            assertEquals(0L, cpuWorkers);
+
+            // Simulate NativeBridge logic: null when workers_count == 0
+            RuntimeMetrics cpuRuntime = null;
+            if (cpuWorkers > 0) {
+                cpuRuntime = StatsLayout.readRuntimeMetrics(seg, "cpu_runtime");
+            }
+            assertNull(cpuRuntime, "cpuRuntime must be null when workers_count == 0");
+        }
+    }
+
+    // ---- Property 2: Decode-then-reencode identity ----
+
+    /**
+     * Property 2: Decode-then-reencode produces byte-identical buffer.
+     *
+     * Validates: Requirements 8.2
+     */
+    @Property(tries = 100)
+    @Tag("Feature: ffm-struct-layout, Property 2: Decode-then-reencode produces byte-identical buffer")
+    void decodeThenReencodeIdentity(@ForAll("thirtyLongs") long[] values) {
+        try (var arena = Arena.ofConfined()) {
+            // Write original values
+            var original = arena.allocate(StatsLayout.LAYOUT);
+            for (int i = 0; i < FIELD_COUNT; i++) {
+                original.setAtIndex(ValueLayout.JAVA_LONG, i, values[i]);
+            }
+
+            // Decode all fields
+            var ioRuntime = StatsLayout.readRuntimeMetrics(original, "io_runtime");
+            var cpuRuntime = StatsLayout.readRuntimeMetrics(original, "cpu_runtime");
+            var qe = StatsLayout.readTaskMonitor(original, "query_execution");
+            var sn = StatsLayout.readTaskMonitor(original, "stream_next");
+            var fp = StatsLayout.readTaskMonitor(original, "fetch_phase");
+            var ss = StatsLayout.readTaskMonitor(original, "segment_stats");
+
+            // Re-encode into new buffer
+            var reencoded = arena.allocate(StatsLayout.LAYOUT);
+            long[] decoded = {
+                ioRuntime.workersCount,
+                ioRuntime.totalPollsCount,
+                ioRuntime.totalBusyDurationMs,
+                ioRuntime.totalOverflowCount,
+                ioRuntime.globalQueueDepth,
+                ioRuntime.blockingQueueDepth,
+                ioRuntime.numAliveTasks,
+                ioRuntime.spawnedTasksCount,
+                ioRuntime.totalLocalQueueDepth,
+                cpuRuntime.workersCount,
+                cpuRuntime.totalPollsCount,
+                cpuRuntime.totalBusyDurationMs,
+                cpuRuntime.totalOverflowCount,
+                cpuRuntime.globalQueueDepth,
+                cpuRuntime.blockingQueueDepth,
+                cpuRuntime.numAliveTasks,
+                cpuRuntime.spawnedTasksCount,
+                cpuRuntime.totalLocalQueueDepth,
+                qe.totalPollDurationMs,
+                qe.totalScheduledDurationMs,
+                qe.totalIdleDurationMs,
+                sn.totalPollDurationMs,
+                sn.totalScheduledDurationMs,
+                sn.totalIdleDurationMs,
+                fp.totalPollDurationMs,
+                fp.totalScheduledDurationMs,
+                fp.totalIdleDurationMs,
+                ss.totalPollDurationMs,
+                ss.totalScheduledDurationMs,
+                ss.totalIdleDurationMs };
+            for (int i = 0; i < FIELD_COUNT; i++) {
+                reencoded.setAtIndex(ValueLayout.JAVA_LONG, i, decoded[i]);
+            }
+
+            // Compare byte-for-byte
+            byte[] originalBytes = original.toArray(ValueLayout.JAVA_BYTE);
+            byte[] reencodedBytes = reencoded.toArray(ValueLayout.JAVA_BYTE);
+            assertArrayEquals(originalBytes, reencodedBytes, "Decode-then-reencode must produce byte-identical buffer");
+        }
+    }
+
+    // ---- Property 3: Writeable serialization round-trip ----
+
+    /**
+     * Property 3: Writeable serialization round-trip (with CPU runtime).
+     *
+     * Validates: Requirements 6.2, 6.3
+     */
+    @Property(tries = 100)
+    @Tag("Feature: ffm-struct-layout, Property 3: Writeable serialization round-trip")
+    void writeableRoundTripWithCpu(@ForAll("nativeExecutorsStatsWithCpu") NativeExecutorsStats original) throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+        StreamInput in = out.bytes().streamInput();
+        NativeExecutorsStats deserialized = new NativeExecutorsStats(in);
+        assertEquals(original, deserialized, "Writeable round-trip must produce equal object");
+    }
+
+    /**
+     * Property 3: Writeable serialization round-trip (CPU runtime absent).
+     *
+     * Validates: Requirements 6.2, 6.3
+     */
+    @Property(tries = 100)
+    @Tag("Feature: ffm-struct-layout, Property 3: Writeable serialization round-trip")
+    void writeableRoundTripNoCpu(@ForAll("nativeExecutorsStatsNoCpu") NativeExecutorsStats original) throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+        StreamInput in = out.bytes().streamInput();
+        NativeExecutorsStats deserialized = new NativeExecutorsStats(in);
+        assertEquals(original, deserialized, "Writeable round-trip must produce equal object");
+        assertNull(deserialized.getCpuRuntime(), "CPU runtime must be null");
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/DataFusionStatsPropertyTests.java b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/DataFusionStatsPropertyTests.java
new file mode 100644
index 0000000000000..9cb9a449419ac
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/DataFusionStatsPropertyTests.java
@@ -0,0 +1,327 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.stats;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.OperationType;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import net.jqwik.api.Arbitraries;
+import net.jqwik.api.Arbitrary;
+import net.jqwik.api.Combinators;
+import net.jqwik.api.ForAll;
+import net.jqwik.api.Property;
+import net.jqwik.api.Provide;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Property-based tests for {@link DataFusionStats} constructed via direct constructors.
+ *
+ * 
      Tests construct objects directly — no decode path, no ArrayCursor.
+ *
+ * 
      Tag: Feature: ffm-stats-decode
+ */
+public class DataFusionStatsPropertyTests {
+
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+
+    /** JSON field names for RuntimeMetrics in documented order (8 fields). */
+    private static final String[] RUNTIME_FIELD_NAMES = {
+        "workers_count",
+        "total_polls_count",
+        "total_busy_duration_ms",
+        "total_overflow_count",
+        "global_queue_depth",
+        "blocking_queue_depth",
+        "num_alive_tasks",
+        "spawned_tasks_count" };
+
+    /** JSON field names for TaskMonitorStats in documented order (3 fields). */
+    private static final String[] TASK_FIELD_NAMES = { "total_poll_duration_ms", "total_scheduled_duration_ms", "total_idle_duration_ms" };
+
+    // ---- Object generators ----
+
+    @Provide
+    Arbitrary runtimeMetrics() {
+        return Arbitraries.longs()
+            .between(0, Long.MAX_VALUE / 2)
+            .list()
+            .ofSize(9)
+            .map(l -> new RuntimeMetrics(l.get(0), l.get(1), l.get(2), l.get(3), l.get(4), l.get(5), l.get(6), l.get(7), l.get(8)));
+    }
+
+    @Provide
+    Arbitrary taskMonitorStats() {
+        Arbitrary nonNeg = Arbitraries.longs().between(0, Long.MAX_VALUE / 2);
+        return Combinators.combine(nonNeg, nonNeg, nonNeg).as(TaskMonitorStats::new);
+    }
+
+    /** DataFusionStats with CPU runtime present (workersCount > 0). */
+    @Provide
+    Arbitrary dataFusionStatsCpuPresent() {
+        return Combinators.combine(runtimeMetrics(), runtimeMetrics().map(rt -> {
+            if (rt.workersCount == 0) {
+                return new RuntimeMetrics(
+                    1,
+                    rt.totalPollsCount,
+                    rt.totalBusyDurationMs,
+                    rt.totalOverflowCount,
+                    rt.globalQueueDepth,
+                    rt.blockingQueueDepth,
+                    rt.numAliveTasks,
+                    rt.spawnedTasksCount,
+                    rt.totalLocalQueueDepth
+                );
+            }
+            return rt;
+        }), taskMonitorStats(), taskMonitorStats(), taskMonitorStats(), taskMonitorStats()).as((io, cpu, qe, sn, fp, ss) -> {
+            Map monitors = new LinkedHashMap<>();
+            monitors.put("query_execution", qe);
+            monitors.put("stream_next", sn);
+            monitors.put("fetch_phase", fp);
+            monitors.put("segment_stats", ss);
+            return new DataFusionStats(new NativeExecutorsStats(io, cpu, monitors));
+        });
+    }
+
+    /** DataFusionStats with CPU runtime absent (null). */
+    @Provide
+    Arbitrary dataFusionStatsCpuAbsent() {
+        return Combinators.combine(runtimeMetrics(), taskMonitorStats(), taskMonitorStats(), taskMonitorStats(), taskMonitorStats())
+            .as((io, qe, sn, fp, ss) -> {
+                Map monitors = new LinkedHashMap<>();
+                monitors.put("query_execution", qe);
+                monitors.put("stream_next", sn);
+                monitors.put("fetch_phase", fp);
+                monitors.put("segment_stats", ss);
+                return new DataFusionStats(new NativeExecutorsStats(io, null, monitors));
+            });
+    }
+
+    @Provide
+    Arbitrary dataFusionStatsNullExecutors() {
+        return Arbitraries.just(new DataFusionStats((NativeExecutorsStats) null));
+    }
+
+    // ---- Property 1: Writeable round-trip preserves all field values ----
+
+    /**
+     * Feature: stats-spi-refactor, Property 1: DataFusionStats Writeable round-trip (CPU present).
+     *
+     * 
      Validates: Requirements 5.6
+     */
+    @Property(tries = 200)
+    void writeableRoundTripCpuPresent(@ForAll("dataFusionStatsCpuPresent") DataFusionStats original) throws IOException {
+        DataFusionStats deserialized = writeableRoundTrip(original);
+        assertEquals(original, deserialized, "Writeable round-trip must preserve all fields (CPU present)");
+    }
+
+    /**
+     * Feature: stats-spi-refactor, Property 1: DataFusionStats Writeable round-trip (CPU absent).
+     *
+     * 
      Validates: Requirements 5.6
+     */
+    @Property(tries = 200)
+    void writeableRoundTripCpuAbsent(@ForAll("dataFusionStatsCpuAbsent") DataFusionStats original) throws IOException {
+        DataFusionStats deserialized = writeableRoundTrip(original);
+        assertEquals(original, deserialized, "Writeable round-trip must preserve all fields (CPU absent)");
+    }
+
+    /**
+     * Feature: stats-spi-refactor, Property 1: DataFusionStats Writeable round-trip (null executors).
+     *
+     * 
      Validates: Requirements 5.6
+     */
+    @Property(tries = 100)
+    void writeableRoundTripNullExecutors(@ForAll("dataFusionStatsNullExecutors") DataFusionStats original) throws IOException {
+        DataFusionStats deserialized = writeableRoundTrip(original);
+        assertEquals(original, deserialized, "Writeable round-trip must preserve null executors");
+    }
+
+    // ---- Property 2: toXContent round-trip preserves all field values ----
+
+    /**
+     * Feature: ffm-stats-decode, Property 2: toXContent round-trip (CPU present).
+     */
+    @Property(tries = 200)
+    void toXContentRoundTripCpuPresent(@ForAll("dataFusionStatsCpuPresent") DataFusionStats stats) throws IOException {
+        NativeExecutorsStats nes = stats.getNativeExecutorsStats();
+        assertNotNull(nes);
+
+        String json = renderJson(stats);
+        JsonNode root = MAPPER.readTree(json);
+        JsonNode nativeExecutors = root.get("native_executors");
+        assertNotNull(nativeExecutors, "native_executors must be present");
+
+        // IO runtime: 8 fields
+        JsonNode ioRuntime = nativeExecutors.get("io_runtime");
+        assertNotNull(ioRuntime, "io_runtime must be present");
+        assertEquals(8, ioRuntime.size(), "io_runtime must have exactly 8 fields");
+        verifyRuntimeFields(nes.getIoRuntime(), ioRuntime);
+
+        // CPU runtime: 8 fields
+        assertTrue(nativeExecutors.has("cpu_runtime"), "cpu_runtime must be present");
+        JsonNode cpuRuntime = nativeExecutors.get("cpu_runtime");
+        assertEquals(8, cpuRuntime.size(), "cpu_runtime must have exactly 8 fields");
+        verifyRuntimeFields(nes.getCpuRuntime(), cpuRuntime);
+
+        // Task monitors: 4 ops × 3 fields
+        JsonNode taskMonitors = nativeExecutors.get("task_monitors");
+        assertNotNull(taskMonitors, "task_monitors must be present");
+        assertEquals(4, taskMonitors.size());
+        for (OperationType opType : OperationType.values()) {
+            JsonNode monitor = taskMonitors.get(opType.key());
+            assertNotNull(monitor, "task_monitors." + opType.key() + " must be present");
+            assertEquals(3, monitor.size());
+            verifyTaskMonitorFields(nes.getTaskMonitors().get(opType.key()), monitor, opType.key());
+        }
+    }
+
+    /**
+     * Feature: ffm-stats-decode, Property 2: toXContent round-trip (CPU absent).
+     */
+    @Property(tries = 200)
+    void toXContentRoundTripCpuAbsent(@ForAll("dataFusionStatsCpuAbsent") DataFusionStats stats) throws IOException {
+        NativeExecutorsStats nes = stats.getNativeExecutorsStats();
+        assertNotNull(nes);
+
+        String json = renderJson(stats);
+        JsonNode root = MAPPER.readTree(json);
+        JsonNode nativeExecutors = root.get("native_executors");
+        assertNotNull(nativeExecutors, "native_executors must be present");
+
+        // IO runtime: 8 fields
+        JsonNode ioRuntime = nativeExecutors.get("io_runtime");
+        assertNotNull(ioRuntime, "io_runtime must be present");
+        assertEquals(8, ioRuntime.size(), "io_runtime must have exactly 8 fields");
+        verifyRuntimeFields(nes.getIoRuntime(), ioRuntime);
+
+        // CPU runtime absent
+        assertFalse(nativeExecutors.has("cpu_runtime"), "cpu_runtime must be absent when cpuRuntime is null");
+
+        // Task monitors
+        JsonNode taskMonitors = nativeExecutors.get("task_monitors");
+        assertNotNull(taskMonitors);
+        assertEquals(4, taskMonitors.size());
+        for (OperationType opType : OperationType.values()) {
+            JsonNode monitor = taskMonitors.get(opType.key());
+            assertNotNull(monitor);
+            assertEquals(3, monitor.size());
+            verifyTaskMonitorFields(nes.getTaskMonitors().get(opType.key()), monitor, opType.key());
+        }
+    }
+
+    // ---- Property 3: toXContent determinism (merged from SPI module) ----
+
+    /**
+     * Feature: stats-spi-refactor, Property: DataFusionStats toXContent determinism (CPU present).
+     *
+     * 
      Validates: Requirements 10.3
+     */
+    @Property(tries = 100)
+    void toXContentDeterminismCpuPresent(@ForAll("dataFusionStatsCpuPresent") DataFusionStats stats) throws IOException {
+        byte[] first = renderJsonBytes(stats);
+        byte[] second = renderJsonBytes(stats);
+        assertTrue(Arrays.equals(first, second), "toXContent must produce byte-for-byte identical JSON on repeated calls (CPU present)");
+    }
+
+    /**
+     * Feature: stats-spi-refactor, Property: DataFusionStats toXContent determinism (CPU absent).
+     *
+     * 
      Validates: Requirements 10.3
+     */
+    @Property(tries = 100)
+    void toXContentDeterminismCpuAbsent(@ForAll("dataFusionStatsCpuAbsent") DataFusionStats stats) throws IOException {
+        byte[] first = renderJsonBytes(stats);
+        byte[] second = renderJsonBytes(stats);
+        assertTrue(Arrays.equals(first, second), "toXContent must produce byte-for-byte identical JSON on repeated calls (CPU absent)");
+    }
+
+    /**
+     * Feature: stats-spi-refactor, Property: DataFusionStats toXContent determinism (null executors).
+     *
+     * 
      Validates: Requirements 10.3
+     */
+    @Property(tries = 100)
+    void toXContentDeterminismNullExecutors(@ForAll("dataFusionStatsNullExecutors") DataFusionStats stats) throws IOException {
+        byte[] first = renderJsonBytes(stats);
+        byte[] second = renderJsonBytes(stats);
+        assertTrue(Arrays.equals(first, second), "toXContent must produce byte-for-byte identical JSON on repeated calls (null executors)");
+    }
+
+    /** Renders a {@link DataFusionStats} to JSON bytes via {@code toXContent}. */
+    private byte[] renderJsonBytes(DataFusionStats stats) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        stats.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        return BytesReference.toBytes(BytesReference.bytes(builder));
+    }
+
+    // ---- Helper methods ----
+
+    private String renderJson(DataFusionStats stats) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        stats.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        return builder.toString();
+    }
+
+    private DataFusionStats writeableRoundTrip(DataFusionStats original) throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+        StreamInput in = out.bytes().streamInput();
+        return new DataFusionStats(in);
+    }
+
+    private void verifyRuntimeFields(RuntimeMetrics rm, JsonNode runtimeNode) {
+        long[] expected = {
+            rm.workersCount,
+            rm.totalPollsCount,
+            rm.totalBusyDurationMs,
+            rm.totalOverflowCount,
+            rm.globalQueueDepth,
+            rm.blockingQueueDepth,
+            rm.numAliveTasks,
+            rm.spawnedTasksCount };
+        for (int i = 0; i < RUNTIME_FIELD_NAMES.length; i++) {
+            String fieldName = RUNTIME_FIELD_NAMES[i];
+            assertTrue(runtimeNode.has(fieldName), "Runtime field '" + fieldName + "' must be present");
+            assertEquals(expected[i], runtimeNode.get(fieldName).asLong(), "Runtime field '" + fieldName + "': expected " + expected[i]);
+        }
+    }
+
+    private void verifyTaskMonitorFields(TaskMonitorStats tm, JsonNode monitorNode, String opType) {
+        long[] expected = { tm.totalPollDurationMs, tm.totalScheduledDurationMs, tm.totalIdleDurationMs };
+        for (int i = 0; i < TASK_FIELD_NAMES.length; i++) {
+            String fieldName = TASK_FIELD_NAMES[i];
+            assertTrue(monitorNode.has(fieldName), opType + " field '" + fieldName + "' must be present");
+            assertEquals(expected[i], monitorNode.get(fieldName).asLong(), opType + " field '" + fieldName + "': expected " + expected[i]);
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NativeExecutorsStatsTests.java b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NativeExecutorsStatsTests.java
new file mode 100644
index 0000000000000..f837bcc56d207
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NativeExecutorsStatsTests.java
@@ -0,0 +1,196 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.stats;
+
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.OperationType;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.io.stream.StreamInput;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import net.jqwik.api.Arbitraries;
+import net.jqwik.api.Arbitrary;
+import net.jqwik.api.Combinators;
+import net.jqwik.api.ForAll;
+import net.jqwik.api.Property;
+import net.jqwik.api.Provide;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+/**
+ * Property-based tests for {@link NativeExecutorsStats} Writeable round-trip.
+ *
+ * 
      Verifies Property 2 from the stats-spi-refactor design:
+ * For any valid {@code NativeExecutorsStats} object containing IO + optional CPU
+ * {@code RuntimeMetrics} (8 fields each) and 4 {@code TaskMonitorStats} (3 fields each),
+ * writing to {@code StreamOutput} and reading from {@code StreamInput} SHALL produce
+ * an object where all field values are identical to the original.
+ *
+ * 
      Tag: Feature: stats-spi-refactor, Property 2: NativeExecutorsStats Writeable round-trip
+ *
+ * 
      Validates: Requirements 6.6
+ */
+public class NativeExecutorsStatsTests {
+
+    // ---- Generators ----
+
+    @Provide
+    Arbitrary runtimeMetrics() {
+        return Arbitraries.longs()
+            .between(0, Long.MAX_VALUE / 2)
+            .list()
+            .ofSize(9)
+            .map(l -> new RuntimeMetrics(l.get(0), l.get(1), l.get(2), l.get(3), l.get(4), l.get(5), l.get(6), l.get(7), l.get(8)));
+    }
+
+    @Provide
+    Arbitrary taskMonitorValues() {
+        Arbitrary nonNeg = Arbitraries.longs().between(0, Long.MAX_VALUE / 2);
+        return Combinators.combine(nonNeg, nonNeg, nonNeg).as(TaskMonitorStats::new);
+    }
+
+    @Provide
+    Arbitrary nativeExecutorsStatsWithCpu() {
+        return Combinators.combine(
+            runtimeMetrics(),                // IO runtime
+            runtimeMetrics().map(rt -> {     // CPU runtime (ensure workers_count > 0)
+                if (rt.workersCount == 0) {
+                    return new RuntimeMetrics(
+                        1,
+                        rt.totalPollsCount,
+                        rt.totalBusyDurationMs,
+                        rt.totalOverflowCount,
+                        rt.globalQueueDepth,
+                        rt.blockingQueueDepth,
+                        rt.numAliveTasks,
+                        rt.spawnedTasksCount,
+                        rt.totalLocalQueueDepth
+                    );
+                }
+                return rt;
+            }),
+            taskMonitorValues(),             // query_execution
+            taskMonitorValues(),             // stream_next
+            taskMonitorValues(),             // fetch_phase
+            taskMonitorValues()              // segment_stats
+        ).as((io, cpu, qe, sn, fp, ss) -> {
+            Map monitors = new LinkedHashMap<>();
+            monitors.put("query_execution", qe);
+            monitors.put("stream_next", sn);
+            monitors.put("fetch_phase", fp);
+            monitors.put("segment_stats", ss);
+            return new NativeExecutorsStats(io, cpu, monitors);
+        });
+    }
+
+    @Provide
+    Arbitrary nativeExecutorsStatsNoCpu() {
+        return Combinators.combine(
+            runtimeMetrics(),                // IO runtime
+            taskMonitorValues(),             // query_execution
+            taskMonitorValues(),             // stream_next
+            taskMonitorValues(),             // fetch_phase
+            taskMonitorValues()              // segment_stats
+        ).as((io, qe, sn, fp, ss) -> {
+            Map monitors = new LinkedHashMap<>();
+            monitors.put("query_execution", qe);
+            monitors.put("stream_next", sn);
+            monitors.put("fetch_phase", fp);
+            monitors.put("segment_stats", ss);
+            return new NativeExecutorsStats(io, null, monitors);
+        });
+    }
+
+    // ---- Property 2: Writeable round-trip preserves all fields ----
+
+    /**
+     * Property 2: Writeable round-trip preserves all fields (with CPU runtime present).
+     *
+     * 
      Tag: Feature: stats-spi-refactor, Property 2: NativeExecutorsStats Writeable round-trip
+     *
+     * 
      Validates: Requirements 6.6
+     */
+    @Property(tries = 100)
+    void writeableRoundTripPreservesAllFieldsWithCpu(@ForAll("nativeExecutorsStatsWithCpu") NativeExecutorsStats original)
+        throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+
+        StreamInput in = out.bytes().streamInput();
+        NativeExecutorsStats deserialized = new NativeExecutorsStats(in);
+
+        assertRuntimeMetricsEqual(original.getIoRuntime(), deserialized.getIoRuntime(), "io_runtime");
+
+        assertNotNull(original.getCpuRuntime(), "original CPU runtime must be present");
+        assertNotNull(deserialized.getCpuRuntime(), "deserialized CPU runtime must be present");
+        assertRuntimeMetricsEqual(original.getCpuRuntime(), deserialized.getCpuRuntime(), "cpu_runtime");
+
+        assertTaskMonitorsEqual(original.getTaskMonitors(), deserialized.getTaskMonitors());
+
+        assertEquals(original, deserialized, "Full NativeExecutorsStats round-trip must produce equal object");
+    }
+
+    /**
+     * Property 2 (complement): Writeable round-trip preserves all fields (CPU runtime absent).
+     *
+     * 
      Tag: Feature: stats-spi-refactor, Property 2: NativeExecutorsStats Writeable round-trip
+     *
+     * 
      Validates: Requirements 6.6
+     */
+    @Property(tries = 100)
+    void writeableRoundTripPreservesAllFieldsNoCpu(@ForAll("nativeExecutorsStatsNoCpu") NativeExecutorsStats original) throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+
+        StreamInput in = out.bytes().streamInput();
+        NativeExecutorsStats deserialized = new NativeExecutorsStats(in);
+
+        assertRuntimeMetricsEqual(original.getIoRuntime(), deserialized.getIoRuntime(), "io_runtime");
+
+        assertEquals(original.getCpuRuntime(), deserialized.getCpuRuntime(), "CPU runtime must be null in both original and deserialized");
+
+        assertTaskMonitorsEqual(original.getTaskMonitors(), deserialized.getTaskMonitors());
+
+        assertEquals(original, deserialized, "Full NativeExecutorsStats round-trip must produce equal object");
+    }
+
+    // ---- Helpers ----
+
+    private void assertRuntimeMetricsEqual(RuntimeMetrics expected, RuntimeMetrics actual, String label) {
+        assertEquals(expected.workersCount, actual.workersCount, label + ".workers_count");
+        assertEquals(expected.totalPollsCount, actual.totalPollsCount, label + ".total_polls_count");
+        assertEquals(expected.totalBusyDurationMs, actual.totalBusyDurationMs, label + ".total_busy_duration_ms");
+        assertEquals(expected.totalOverflowCount, actual.totalOverflowCount, label + ".total_overflow_count");
+        assertEquals(expected.globalQueueDepth, actual.globalQueueDepth, label + ".global_queue_depth");
+        assertEquals(expected.blockingQueueDepth, actual.blockingQueueDepth, label + ".blocking_queue_depth");
+        assertEquals(expected.numAliveTasks, actual.numAliveTasks, label + ".num_alive_tasks");
+        assertEquals(expected.spawnedTasksCount, actual.spawnedTasksCount, label + ".spawned_tasks_count");
+    }
+
+    private void assertTaskMonitorsEqual(Map expected, Map actual) {
+        assertEquals(4, expected.size(), "original must have exactly 4 task monitors");
+        assertEquals(4, actual.size(), "deserialized must have exactly 4 task monitors");
+
+        for (OperationType opType : OperationType.values()) {
+            TaskMonitorStats exp = expected.get(opType.key());
+            TaskMonitorStats act = actual.get(opType.key());
+            assertNotNull(exp, "original must contain " + opType.key());
+            assertNotNull(act, "deserialized must contain " + opType.key());
+
+            assertEquals(exp.totalPollDurationMs, act.totalPollDurationMs, opType.key() + ".total_poll_duration_ms");
+            assertEquals(exp.totalScheduledDurationMs, act.totalScheduledDurationMs, opType.key() + ".total_scheduled_duration_ms");
+            assertEquals(exp.totalIdleDurationMs, act.totalIdleDurationMs, opType.key() + ".total_idle_duration_ms");
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NodeStatsNativeMetricRoundTripTests.java b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NodeStatsNativeMetricRoundTripTests.java
new file mode 100644
index 0000000000000..a65e223d00211
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NodeStatsNativeMetricRoundTripTests.java
@@ -0,0 +1,165 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.stats;
+
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.OperationType;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.io.stream.StreamInput;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import net.jqwik.api.Arbitraries;
+import net.jqwik.api.Arbitrary;
+import net.jqwik.api.Combinators;
+import net.jqwik.api.ForAll;
+import net.jqwik.api.Property;
+import net.jqwik.api.Provide;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+/**
+ * Property-based tests verifying that {@link NativeExecutorsStats} with native metrics
+ * can round-trip through {@link org.opensearch.core.common.io.stream.Writeable} serialization.
+ *
+ * 
      Constructs {@code NativeExecutorsStats} with the 4-monitor layout
+ * (query_execution, stream_next, fetch_phase, segment_stats — each 3 fields)
+ * and verifies the full StreamOutput → StreamInput round-trip preserves all fields.
+ */
+public class NodeStatsNativeMetricRoundTripTests {
+
+    // ---- Generators ----
+
+    @Provide
+    Arbitrary runtimeMetrics() {
+        return Arbitraries.longs()
+            .between(0, Long.MAX_VALUE / 2)
+            .list()
+            .ofSize(9)
+            .map(l -> new RuntimeMetrics(l.get(0), l.get(1), l.get(2), l.get(3), l.get(4), l.get(5), l.get(6), l.get(7), l.get(8)));
+    }
+
+    @Provide
+    Arbitrary taskMonitorValues() {
+        Arbitrary nonNeg = Arbitraries.longs().between(0, Long.MAX_VALUE / 2);
+        return Combinators.combine(nonNeg, nonNeg, nonNeg).as(TaskMonitorStats::new);
+    }
+
+    @Provide
+    Arbitrary nativeExecutorsStatsWithCpu() {
+        return Combinators.combine(runtimeMetrics(), runtimeMetrics().map(rt -> {
+            if (rt.workersCount == 0) {
+                return new RuntimeMetrics(
+                    1,
+                    rt.totalPollsCount,
+                    rt.totalBusyDurationMs,
+                    rt.totalOverflowCount,
+                    rt.globalQueueDepth,
+                    rt.blockingQueueDepth,
+                    rt.numAliveTasks,
+                    rt.spawnedTasksCount,
+                    rt.totalLocalQueueDepth
+                );
+            }
+            return rt;
+        }), taskMonitorValues(), taskMonitorValues(), taskMonitorValues(), taskMonitorValues()).as((io, cpu, qe, sn, fp, ss) -> {
+            Map monitors = new LinkedHashMap<>();
+            monitors.put("query_execution", qe);
+            monitors.put("stream_next", sn);
+            monitors.put("fetch_phase", fp);
+            monitors.put("segment_stats", ss);
+            return new NativeExecutorsStats(io, cpu, monitors);
+        });
+    }
+
+    @Provide
+    Arbitrary nativeExecutorsStatsNoCpu() {
+        return Combinators.combine(runtimeMetrics(), taskMonitorValues(), taskMonitorValues(), taskMonitorValues(), taskMonitorValues())
+            .as((io, qe, sn, fp, ss) -> {
+                Map monitors = new LinkedHashMap<>();
+                monitors.put("query_execution", qe);
+                monitors.put("stream_next", sn);
+                monitors.put("fetch_phase", fp);
+                monitors.put("segment_stats", ss);
+                return new NativeExecutorsStats(io, null, monitors);
+            });
+    }
+
+    // ---- Round-trip property tests ----
+
+    @Property(tries = 100)
+    void nativeMetricRoundTripWithCpuRuntime(@ForAll("nativeExecutorsStatsWithCpu") NativeExecutorsStats original) throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+
+        StreamInput in = out.bytes().streamInput();
+        NativeExecutorsStats deserialized = new NativeExecutorsStats(in);
+
+        assertRuntimeMetricsEqual(original.getIoRuntime(), deserialized.getIoRuntime(), "io_runtime");
+
+        assertNotNull(original.getCpuRuntime(), "original CPU runtime must be present");
+        assertNotNull(deserialized.getCpuRuntime(), "deserialized CPU runtime must be present");
+        assertRuntimeMetricsEqual(original.getCpuRuntime(), deserialized.getCpuRuntime(), "cpu_runtime");
+
+        assertTaskMonitorsEqual(original.getTaskMonitors(), deserialized.getTaskMonitors());
+
+        assertEquals(original, deserialized, "NativeExecutorsStats round-trip must produce equal object");
+    }
+
+    @Property(tries = 100)
+    void nativeMetricRoundTripWithoutCpuRuntime(@ForAll("nativeExecutorsStatsNoCpu") NativeExecutorsStats original) throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+
+        StreamInput in = out.bytes().streamInput();
+        NativeExecutorsStats deserialized = new NativeExecutorsStats(in);
+
+        assertRuntimeMetricsEqual(original.getIoRuntime(), deserialized.getIoRuntime(), "io_runtime");
+
+        assertNull(deserialized.getCpuRuntime(), "CPU runtime must be null when original has no CPU runtime");
+
+        assertTaskMonitorsEqual(original.getTaskMonitors(), deserialized.getTaskMonitors());
+
+        assertEquals(original, deserialized, "NativeExecutorsStats round-trip must produce equal object");
+    }
+
+    // ---- Helpers ----
+
+    private void assertRuntimeMetricsEqual(RuntimeMetrics expected, RuntimeMetrics actual, String label) {
+        assertEquals(expected.workersCount, actual.workersCount, label + ".workers_count");
+        assertEquals(expected.totalPollsCount, actual.totalPollsCount, label + ".total_polls_count");
+        assertEquals(expected.totalBusyDurationMs, actual.totalBusyDurationMs, label + ".total_busy_duration_ms");
+        assertEquals(expected.totalOverflowCount, actual.totalOverflowCount, label + ".total_overflow_count");
+        assertEquals(expected.globalQueueDepth, actual.globalQueueDepth, label + ".global_queue_depth");
+        assertEquals(expected.blockingQueueDepth, actual.blockingQueueDepth, label + ".blocking_queue_depth");
+        assertEquals(expected.numAliveTasks, actual.numAliveTasks, label + ".num_alive_tasks");
+        assertEquals(expected.spawnedTasksCount, actual.spawnedTasksCount, label + ".spawned_tasks_count");
+    }
+
+    private void assertTaskMonitorsEqual(Map expected, Map actual) {
+        assertEquals(4, expected.size(), "original must have exactly 4 task monitors");
+        assertEquals(4, actual.size(), "deserialized must have exactly 4 task monitors");
+
+        for (OperationType opType : OperationType.values()) {
+            TaskMonitorStats exp = expected.get(opType.key());
+            TaskMonitorStats act = actual.get(opType.key());
+            assertNotNull(exp, "original must contain " + opType.key());
+            assertNotNull(act, "deserialized must contain " + opType.key());
+
+            assertEquals(exp.totalPollDurationMs, act.totalPollDurationMs, opType.key() + ".total_poll_duration_ms");
+            assertEquals(exp.totalScheduledDurationMs, act.totalScheduledDurationMs, opType.key() + ".total_scheduled_duration_ms");
+            assertEquals(exp.totalIdleDurationMs, act.totalIdleDurationMs, opType.key() + ".total_idle_duration_ms");
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionServiceStatsTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionServiceStatsTests.java
new file mode 100644
index 0000000000000..2ef1532aa3161
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionServiceStatsTests.java
@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.test.OpenSearchTestCase;
+
+/**
+ * Unit tests for {@link DataFusionService#getStats()}.
+ *
+ * Validates: Requirements 5.2, 5.3, 5.5
+ *
+ * Note: Cache TTL behavior (Requirement 5.3 — same instance within TTL window,
+ * fresh instance after TTL expires) requires a running native runtime since
+ * {@code doStart()} calls {@code NativeBridge.stats()} to seed the cache.
+ * That behavior is verified in integration tests where the native library is loaded.
+ */
+public class DataFusionServiceStatsTests extends OpenSearchTestCase {
+
+    /**
+     * Validates Requirement 5.5: getStats() throws IllegalStateException before doStart().
+     *
+     * When the service is constructed but not started, the statsCache field is null.
+     * Calling getStats() must throw IllegalStateException with a descriptive message.
+     */
+    public void testGetStatsBeforeStartThrowsIllegalStateException() {
+        DataFusionService service = DataFusionService.builder().build();
+
+        IllegalStateException ex = expectThrows(IllegalStateException.class, service::getStats);
+        assertEquals("DataFusionService has not been started", ex.getMessage());
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/action/DataFusionStatsActionTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/action/DataFusionStatsActionTests.java
new file mode 100644
index 0000000000000..4e382733f82a3
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/action/DataFusionStatsActionTests.java
@@ -0,0 +1,161 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.action;
+
+import org.opensearch.be.datafusion.DataFusionPlugin;
+import org.opensearch.be.datafusion.DataFusionService;
+import org.opensearch.be.datafusion.stats.DataFusionStats;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
+import org.opensearch.common.SuppressForbidden;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.rest.RestHandler;
+import org.opensearch.rest.RestHandler.Route;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.test.rest.FakeRestChannel;
+import org.opensearch.test.rest.FakeRestRequest;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.client.node.NodeClient;
+
+import java.lang.reflect.Field;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for {@link DataFusionStatsAction} and {@link DataFusionPlugin} REST handler registration.
+ *
+ * Validates: Requirements 1.1, 1.2, 1.3, 1.4, 6.1, 6.2
+ */
+public class DataFusionStatsActionTests extends OpenSearchTestCase {
+
+    private ThreadPool threadPool;
+    private NodeClient nodeClient;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(getTestName());
+        nodeClient = new NodeClient(Settings.EMPTY, threadPool);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        threadPool.shutdown();
+        nodeClient.close();
+    }
+
+    // ---- Test: routes() returns GET _plugins/datafusion/stats (Requirement 1.1) ----
+
+    public void testRoutesReturnsStatsEndpoint() {
+        DataFusionService mockService = mock(DataFusionService.class);
+        DataFusionStatsAction action = new DataFusionStatsAction(mockService);
+
+        List routes = action.routes();
+        assertEquals(1, routes.size());
+        assertEquals(RestRequest.Method.GET, routes.get(0).getMethod());
+        assertEquals("_plugins/datafusion/stats", routes.get(0).getPath());
+    }
+
+    // ---- Test: getName() returns "datafusion_stats_action" (Requirement 1.1) ----
+
+    public void testGetNameReturnsExpectedName() {
+        DataFusionService mockService = mock(DataFusionService.class);
+        DataFusionStatsAction action = new DataFusionStatsAction(mockService);
+
+        assertEquals("datafusion_stats_action", action.getName());
+    }
+
+    // ---- Test: prepareRequest returns 200 with valid JSON when service returns stats (Requirement 1.3) ----
+
+    public void testPrepareRequestReturns200WithValidJson() throws Exception {
+        // Build a known DataFusionStats via direct constructors
+        RuntimeMetrics io = new RuntimeMetrics(1, 2, 3, 4, 5, 6, 7, 8, 0);
+        RuntimeMetrics cpu = new RuntimeMetrics(9, 10, 11, 12, 13, 14, 15, 16, 0);
+        Map taskMonitors = new LinkedHashMap<>();
+        taskMonitors.put("query_execution", new TaskMonitorStats(17, 18, 19));
+        taskMonitors.put("stream_next", new TaskMonitorStats(20, 21, 22));
+        taskMonitors.put("fetch_phase", new TaskMonitorStats(23, 24, 25));
+        taskMonitors.put("segment_stats", new TaskMonitorStats(26, 27, 28));
+        DataFusionStats stats = new DataFusionStats(new NativeExecutorsStats(io, cpu, taskMonitors));
+
+        DataFusionService mockService = mock(DataFusionService.class);
+        when(mockService.getStats()).thenReturn(stats);
+
+        DataFusionStatsAction action = new DataFusionStatsAction(mockService);
+
+        FakeRestRequest request = new FakeRestRequest();
+        FakeRestChannel channel = new FakeRestChannel(request, true, 1);
+
+        // Execute the handler — prepareRequest returns a consumer, then handleRequest invokes it
+        action.handleRequest(request, channel, nodeClient);
+
+        // Verify the response
+        assertEquals(200, channel.capturedResponse().status().getStatus());
+        String responseBody = channel.capturedResponse().content().utf8ToString();
+        assertTrue("Response should contain native_executors", responseBody.contains("native_executors"));
+        assertTrue("Response should contain io_runtime", responseBody.contains("io_runtime"));
+        assertTrue("Response should contain task_monitors", responseBody.contains("task_monitors"));
+        assertTrue("Response should contain query_execution", responseBody.contains("query_execution"));
+    }
+
+    // ---- Test: prepareRequest returns 500 when service throws exception (Requirement 6.1) ----
+
+    public void testPrepareRequestReturns500WhenServiceThrows() throws Exception {
+        DataFusionService mockService = mock(DataFusionService.class);
+        when(mockService.getStats()).thenThrow(new IllegalStateException("DataFusionService has not been started"));
+
+        DataFusionStatsAction action = new DataFusionStatsAction(mockService);
+
+        FakeRestRequest request = new FakeRestRequest();
+        FakeRestChannel channel = new FakeRestChannel(request, true, 1);
+
+        action.handleRequest(request, channel, nodeClient);
+
+        assertEquals(500, channel.capturedResponse().status().getStatus());
+        String responseBody = channel.capturedResponse().content().utf8ToString();
+        assertTrue("Error response should contain exception type", responseBody.contains("illegal_state_exception"));
+    }
+
+    // ---- Test: DataFusionPlugin.getRestHandlers() returns list containing DataFusionStatsAction (Requirement 1.2) ----
+
+    @SuppressForbidden(reason = "reflection needed to inject mock DataFusionService into plugin for testing")
+    public void testPluginGetRestHandlersReturnsStatsAction() throws Exception {
+        DataFusionPlugin plugin = new DataFusionPlugin();
+
+        // Use reflection to set the dataFusionService field to a non-null mock
+        DataFusionService mockService = mock(DataFusionService.class);
+        Field serviceField = DataFusionPlugin.class.getDeclaredField("dataFusionService");
+        serviceField.setAccessible(true);
+        serviceField.set(plugin, mockService);
+
+        List handlers = plugin.getRestHandlers(Settings.EMPTY, null, null, null, null, null, null);
+
+        assertEquals(1, handlers.size());
+        assertTrue("Handler should be DataFusionStatsAction", handlers.get(0) instanceof DataFusionStatsAction);
+    }
+
+    // ---- Test: DataFusionPlugin.getRestHandlers() returns empty list when dataFusionService is null (Requirement 1.4) ----
+
+    public void testPluginGetRestHandlersReturnsEmptyWhenServiceNull() {
+        DataFusionPlugin plugin = new DataFusionPlugin();
+        // dataFusionService is null by default (createComponents not called)
+
+        List handlers = plugin.getRestHandlers(Settings.EMPTY, null, null, null, null, null, null);
+
+        assertTrue("Should return empty list when service is null", handlers.isEmpty());
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/nativelib/StatsLayoutTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/nativelib/StatsLayoutTests.java
new file mode 100644
index 0000000000000..f8398cb8e2b6d
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/nativelib/StatsLayoutTests.java
@@ -0,0 +1,107 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.nativelib;
+
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.ValueLayout;
+
+/**
+ * Unit tests for {@link StatsLayout} — verifies layout size, VarHandle reads,
+ * and cpu_runtime null/non-null logic.
+ */
+public class StatsLayoutTests extends OpenSearchTestCase {
+
+    /** 7.1: Layout byte size must be 240 (30 × 8). */
+    public void testLayoutByteSize() {
+        assertEquals(240L, StatsLayout.LAYOUT.byteSize());
+        assertEquals(30 * Long.BYTES, (int) StatsLayout.LAYOUT.byteSize());
+    }
+
+    /** 7.2: readRuntimeMetrics decodes 9 known values from io_runtime group. */
+    public void testReadRuntimeMetricsFromSegment() {
+        try (var arena = Arena.ofConfined()) {
+            var seg = arena.allocate(StatsLayout.LAYOUT);
+            // Write sequential values 1-9 at io_runtime positions (indices 0-8)
+            for (int i = 0; i < 9; i++) {
+                seg.setAtIndex(ValueLayout.JAVA_LONG, i, i + 1L);
+            }
+
+            var rt = StatsLayout.readRuntimeMetrics(seg, "io_runtime");
+            assertEquals(1L, rt.workersCount);
+            assertEquals(2L, rt.totalPollsCount);
+            assertEquals(3L, rt.totalBusyDurationMs);
+            assertEquals(4L, rt.totalOverflowCount);
+            assertEquals(5L, rt.globalQueueDepth);
+            assertEquals(6L, rt.blockingQueueDepth);
+            assertEquals(7L, rt.numAliveTasks);
+            assertEquals(8L, rt.spawnedTasksCount);
+            assertEquals(9L, rt.totalLocalQueueDepth);
+        }
+    }
+
+    /** 7.3: readTaskMonitor decodes 3 known values from query_execution group. */
+    public void testReadTaskMonitorFromSegment() {
+        try (var arena = Arena.ofConfined()) {
+            var seg = arena.allocate(StatsLayout.LAYOUT);
+            // query_execution starts at index 18 (2 runtime groups × 9 fields = 18)
+            seg.setAtIndex(ValueLayout.JAVA_LONG, 18, 100L);
+            seg.setAtIndex(ValueLayout.JAVA_LONG, 19, 200L);
+            seg.setAtIndex(ValueLayout.JAVA_LONG, 20, 300L);
+
+            var tm = StatsLayout.readTaskMonitor(seg, "query_execution");
+            assertEquals(100L, tm.totalPollDurationMs);
+            assertEquals(200L, tm.totalScheduledDurationMs);
+            assertEquals(300L, tm.totalIdleDurationMs);
+        }
+    }
+
+    /** 7.4: cpu_runtime is null when workers_count == 0. */
+    public void testCpuRuntimeNullWhenWorkersZero() {
+        try (var arena = Arena.ofConfined()) {
+            var seg = arena.allocate(StatsLayout.LAYOUT);
+            // cpu_runtime.workers_count is at index 9 — leave it as 0 (default)
+            long cpuWorkers = StatsLayout.readField(seg, "cpu_runtime", "workers_count");
+            assertEquals(0L, cpuWorkers);
+
+            // Simulate the NativeBridge logic
+            NativeExecutorsStats.RuntimeMetrics cpuRuntime = null;
+            if (cpuWorkers > 0) {
+                cpuRuntime = StatsLayout.readRuntimeMetrics(seg, "cpu_runtime");
+            }
+            assertNull(cpuRuntime);
+        }
+    }
+
+    /** 7.5: cpu_runtime is non-null when workers_count > 0. */
+    public void testCpuRuntimeNonNullWhenWorkersPositive() {
+        try (var arena = Arena.ofConfined()) {
+            var seg = arena.allocate(StatsLayout.LAYOUT);
+            // Set cpu_runtime.workers_count (index 9) to 5
+            seg.setAtIndex(ValueLayout.JAVA_LONG, 9, 5L);
+            // Set other cpu_runtime fields (indices 10-17)
+            for (int i = 10; i <= 17; i++) {
+                seg.setAtIndex(ValueLayout.JAVA_LONG, i, i * 10L);
+            }
+
+            long cpuWorkers = StatsLayout.readField(seg, "cpu_runtime", "workers_count");
+            assertEquals(5L, cpuWorkers);
+
+            NativeExecutorsStats.RuntimeMetrics cpuRuntime = null;
+            if (cpuWorkers > 0) {
+                cpuRuntime = StatsLayout.readRuntimeMetrics(seg, "cpu_runtime");
+            }
+            assertNotNull(cpuRuntime);
+            assertEquals(5L, cpuRuntime.workersCount);
+            assertEquals(100L, cpuRuntime.totalPollsCount);
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/stats/DataFusionStatsTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/stats/DataFusionStatsTests.java
new file mode 100644
index 0000000000000..336a82c28292f
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/stats/DataFusionStatsTests.java
@@ -0,0 +1,209 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.stats;
+
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Unit tests for {@link DataFusionStats} constructed via direct constructors.
+ *
+ * 
      Layout: IO RuntimeMetrics (8 fields), optional CPU RuntimeMetrics (8 fields),
+ * 4 TaskMonitorStats (3 fields each).
+ */
+public class DataFusionStatsTests extends OpenSearchTestCase {
+
+    /** Build a DataFusionStats with sequential values 1..28 for deterministic field verification. */
+    private static DataFusionStats sequentialStats() {
+        RuntimeMetrics io = new RuntimeMetrics(1, 2, 3, 4, 5, 6, 7, 8, 0);
+        RuntimeMetrics cpu = new RuntimeMetrics(9, 10, 11, 12, 13, 14, 15, 16, 0);
+        Map taskMonitors = new LinkedHashMap<>();
+        taskMonitors.put("query_execution", new TaskMonitorStats(17, 18, 19));
+        taskMonitors.put("stream_next", new TaskMonitorStats(20, 21, 22));
+        taskMonitors.put("fetch_phase", new TaskMonitorStats(23, 24, 25));
+        taskMonitors.put("segment_stats", new TaskMonitorStats(26, 27, 28));
+        return new DataFusionStats(new NativeExecutorsStats(io, cpu, taskMonitors));
+    }
+
+    private static String toJsonString(DataFusionStats stats) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        stats.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        return builder.toString();
+    }
+
+    // ---- Test: sequential construction verifies each field ----
+
+    public void testSequentialConstructionVerifiesFields() {
+        DataFusionStats stats = sequentialStats();
+        NativeExecutorsStats nes = stats.getNativeExecutorsStats();
+        assertNotNull(nes);
+
+        // IO runtime (values 1-8)
+        RuntimeMetrics io = nes.getIoRuntime();
+        assertNotNull(io);
+        assertEquals(1L, io.workersCount);
+        assertEquals(2L, io.totalPollsCount);
+        assertEquals(3L, io.totalBusyDurationMs);
+        assertEquals(4L, io.totalOverflowCount);
+        assertEquals(5L, io.globalQueueDepth);
+        assertEquals(6L, io.blockingQueueDepth);
+        assertEquals(7L, io.numAliveTasks);
+        assertEquals(8L, io.spawnedTasksCount);
+
+        // CPU runtime (values 9-16)
+        RuntimeMetrics cpu = nes.getCpuRuntime();
+        assertNotNull(cpu);
+        assertEquals(9L, cpu.workersCount);
+        assertEquals(10L, cpu.totalPollsCount);
+        assertEquals(11L, cpu.totalBusyDurationMs);
+        assertEquals(12L, cpu.totalOverflowCount);
+        assertEquals(13L, cpu.globalQueueDepth);
+        assertEquals(14L, cpu.blockingQueueDepth);
+        assertEquals(15L, cpu.numAliveTasks);
+        assertEquals(16L, cpu.spawnedTasksCount);
+
+        // Task monitors
+        Map monitors = nes.getTaskMonitors();
+        assertEquals(4, monitors.size());
+
+        TaskMonitorStats qe = monitors.get("query_execution");
+        assertNotNull(qe);
+        assertEquals(17L, qe.totalPollDurationMs);
+        assertEquals(18L, qe.totalScheduledDurationMs);
+        assertEquals(19L, qe.totalIdleDurationMs);
+
+        TaskMonitorStats sn = monitors.get("stream_next");
+        assertNotNull(sn);
+        assertEquals(20L, sn.totalPollDurationMs);
+        assertEquals(21L, sn.totalScheduledDurationMs);
+        assertEquals(22L, sn.totalIdleDurationMs);
+
+        TaskMonitorStats fp = monitors.get("fetch_phase");
+        assertNotNull(fp);
+        assertEquals(23L, fp.totalPollDurationMs);
+        assertEquals(24L, fp.totalScheduledDurationMs);
+        assertEquals(25L, fp.totalIdleDurationMs);
+
+        TaskMonitorStats ss = monitors.get("segment_stats");
+        assertNotNull(ss);
+        assertEquals(26L, ss.totalPollDurationMs);
+        assertEquals(27L, ss.totalScheduledDurationMs);
+        assertEquals(28L, ss.totalIdleDurationMs);
+    }
+
+    // ---- Test: CPU runtime null → cpuRuntime absent in JSON ----
+
+    public void testCpuRuntimeAbsentWhenNull() throws IOException {
+        RuntimeMetrics io = new RuntimeMetrics(100, 101, 102, 103, 104, 105, 106, 107, 0);
+        Map taskMonitors = new LinkedHashMap<>();
+        taskMonitors.put("query_execution", new TaskMonitorStats(14, 15, 16));
+        taskMonitors.put("stream_next", new TaskMonitorStats(17, 18, 19));
+        taskMonitors.put("fetch_phase", new TaskMonitorStats(20, 21, 22));
+        taskMonitors.put("segment_stats", new TaskMonitorStats(23, 24, 25));
+
+        DataFusionStats stats = new DataFusionStats(new NativeExecutorsStats(io, null, taskMonitors));
+        assertNull(stats.getNativeExecutorsStats().getCpuRuntime());
+
+        String json = toJsonString(stats);
+        assertFalse("cpu_runtime should be omitted when null", json.contains("cpu_runtime"));
+        assertTrue("io_runtime should still be present", json.contains("io_runtime"));
+        assertTrue("task_monitors should still be present", json.contains("task_monitors"));
+    }
+
+    // ---- Test: non-null CPU runtime → cpuRuntime present in JSON ----
+
+    public void testCpuRuntimePresentWhenNonNull() throws IOException {
+        DataFusionStats stats = sequentialStats();
+        assertNotNull(stats.getNativeExecutorsStats().getCpuRuntime());
+
+        String json = toJsonString(stats);
+        assertTrue("cpu_runtime should be present", json.contains("cpu_runtime"));
+
+        String[] runtimeFieldNames = {
+            "workers_count",
+            "total_polls_count",
+            "total_busy_duration_ms",
+            "total_overflow_count",
+            "global_queue_depth",
+            "blocking_queue_depth",
+            "num_alive_tasks",
+            "spawned_tasks_count" };
+        for (String field : runtimeFieldNames) {
+            assertTrue("JSON should contain field: " + field, json.contains("\"" + field + "\""));
+        }
+    }
+
+    // ---- Test: toXContent renders correct JSON structure ----
+
+    public void testToXContentJsonStructure() throws IOException {
+        DataFusionStats stats = sequentialStats();
+        String json = toJsonString(stats);
+
+        assertTrue(json.contains("\"native_executors\""));
+        assertTrue(json.contains("\"io_runtime\""));
+        assertTrue(json.contains("\"cpu_runtime\""));
+        assertTrue(json.contains("\"task_monitors\""));
+
+        assertTrue(json.contains("\"query_execution\""));
+        assertTrue(json.contains("\"stream_next\""));
+        assertTrue(json.contains("\"fetch_phase\""));
+        assertTrue(json.contains("\"segment_stats\""));
+
+        String[] taskFields = { "total_poll_duration_ms", "total_scheduled_duration_ms", "total_idle_duration_ms" };
+        for (String field : taskFields) {
+            assertTrue("JSON should contain task monitor field: " + field, json.contains("\"" + field + "\""));
+        }
+
+        // IO runtime: workers_count = 1
+        assertTrue(json.contains("\"workers_count\":1,"));
+        // query_execution: total_poll_duration_ms = 17
+        assertTrue(json.contains("\"total_poll_duration_ms\":17"));
+    }
+
+    // ---- Test: toXContent with CPU runtime omitted ----
+
+    public void testToXContentCpuRuntimeOmitted() throws IOException {
+        RuntimeMetrics io = new RuntimeMetrics(100, 101, 102, 103, 104, 105, 106, 107, 0);
+        Map taskMonitors = new LinkedHashMap<>();
+        taskMonitors.put("query_execution", new TaskMonitorStats(14, 15, 16));
+        taskMonitors.put("stream_next", new TaskMonitorStats(17, 18, 19));
+        taskMonitors.put("fetch_phase", new TaskMonitorStats(20, 21, 22));
+        taskMonitors.put("segment_stats", new TaskMonitorStats(23, 24, 25));
+
+        DataFusionStats stats = new DataFusionStats(new NativeExecutorsStats(io, null, taskMonitors));
+        String json = toJsonString(stats);
+
+        assertTrue(json.contains("\"io_runtime\""));
+        assertFalse("cpu_runtime should not appear", json.contains("\"cpu_runtime\""));
+        assertTrue(json.contains("\"task_monitors\""));
+    }
+
+    // ---- Test: exactly 4 task monitor keys ----
+
+    public void testExactlyFourTaskMonitors() {
+        DataFusionStats stats = sequentialStats();
+        Map monitors = stats.getNativeExecutorsStats().getTaskMonitors();
+
+        assertEquals(4, monitors.size());
+        assertTrue(monitors.containsKey("query_execution"));
+        assertTrue(monitors.containsKey("stream_next"));
+        assertTrue(monitors.containsKey("fetch_phase"));
+        assertTrue(monitors.containsKey("segment_stats"));
+    }
+}

From f7bd4f492e1739ff6a98d41c70d02e3afab3b3cb Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Thu, 7 May 2026 22:33:08 -0700
Subject: [PATCH 086/115] Adding sweep of MATH scalar functions to
 analytics-engine. (#21520)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Adding sweep of MATH scalar functions.

math scalar functions — 32 ITs covering ABS, ACOS, ASIN, ATAN, ATAN2,
CBRT, CEIL, COS, COSH, COT, DEGREES, E, EXP, EXPM1, FLOOR, LN, LOG, LOG10,
LOG2, PI, POWER, RADIANS, RAND, ROUND, SCALAR_MAX, SCALAR_MIN, SIGN, SIN, SINH,
TAN, TRUNCATE — pushed down through analytics-engine → Substrait → DataFusion.

Rebased on upstream/main with #21476 landed (AbstractNameMappingAdapter +
YEAR / CONVERT_TZ / UNIX_TIMESTAMP). Conflicts resolved by union in:
  - ScalarFunction.java (math enum entries)
  - DataFusionAnalyticsBackendPlugin.java (STANDARD_PROJECT_OPS +
    scalarFunctionAdapters map)
  - DataFusionFragmentConvertor.java (ADDITIONAL_SCALAR_SIGS)
  - opensearch_scalar_functions.yaml (cbrt / cot / pi / random / round /
    signum / trunc signatures)

Migrated SCALAR_MAX / SCALAR_MIN / SIGN off the local RewriteOperatorAdapter
onto the shared AbstractNameMappingAdapter from #21476.

Signed-off-by: Marc Handalian 

Co-Authored-By: Claude Opus 4.7 (1M context) 

* fix compile

Signed-off-by: Marc Handalian 

---------

Signed-off-by: Marc Handalian 
Co-authored-by: Claude Opus 4.7 (1M context) 
---
 .../spi/AbstractNameMappingAdapter.java       |  10 +
 .../analytics/spi/ScalarFunction.java         |  29 +-
 .../spi/AbstractNameMappingAdapterTests.java  | 153 +++++++++
 .../analytics/spi/ScalarFunctionTests.java    |  98 ++++++
 .../DataFusionAnalyticsBackendPlugin.java     |  51 ++-
 .../DataFusionFragmentConvertor.java          |  12 +-
 .../be/datafusion/EConstantAdapter.java       |  45 +++
 .../be/datafusion/Expm1Adapter.java           |  53 +++
 .../datafusion/HyperbolicOperatorAdapter.java |  65 ++++
 .../be/datafusion/SignumFunction.java         |  51 +++
 .../opensearch_scalar_functions.yaml          | 101 +++++-
 .../be/datafusion/EConstantAdapterTests.java  |  87 +++++
 .../be/datafusion/Expm1AdapterTests.java      |  96 ++++++
 .../HyperbolicOperatorAdapterTests.java       | 114 +++++++
 .../MathProjectCapabilitiesTests.java         | 105 ++++++
 .../planner/rules/OpenSearchProjectRule.java  |  16 +
 .../analytics/planner/ProjectRuleTests.java   |  30 ++
 .../analytics/qa/MathScalarFunctionsIT.java   | 320 ++++++++++++++++++
 18 files changed, 1429 insertions(+), 7 deletions(-)
 create mode 100644 sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/AbstractNameMappingAdapterTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/EConstantAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/Expm1Adapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/HyperbolicOperatorAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SignumFunction.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/EConstantAdapterTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/Expm1AdapterTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/HyperbolicOperatorAdapterTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/MathProjectCapabilitiesTests.java
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MathScalarFunctionsIT.java

diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AbstractNameMappingAdapter.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AbstractNameMappingAdapter.java
index b093b434e2ce3..2f96b7ef24a5f 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AbstractNameMappingAdapter.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AbstractNameMappingAdapter.java
@@ -81,6 +81,16 @@ public RexNode adapt(RexCall original, List fieldStorage, RelO
         // LogicalProject.create together with the cached rowType, and
         // Project.isValid's compatibleTypes check throws an AssertionError that
         // breaks fragment conversion.
+        //
+        // Exception: polymorphic PPL UDFs (e.g. SCALAR_MAX, SCALAR_MIN) declare
+        // their return type as SqlTypeName.ANY because they accept heterogeneous
+        // operand shapes. Substrait cannot serialise ANY, so fall back to the
+        // target operator's own return-type inference — the result will be a
+        // concrete type derived from operands (DOUBLE for GREATEST(DOUBLE, DOUBLE),
+        // etc.) which Substrait can serialise.
+        if (original.getType().getSqlTypeName() == SqlTypeName.ANY) {
+            return rexBuilder.makeCall(targetOperator, operands);
+        }
         return rexBuilder.makeCall(original.getType(), targetOperator, operands);
     }
 
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
index 8f20e27d425b7..42bfc7ec1ce92 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
@@ -77,9 +77,36 @@ public enum ScalarFunction {
     DIVIDE(Category.MATH, SqlKind.DIVIDE),
     MOD(Category.MATH, SqlKind.MOD),
     ABS(Category.MATH, SqlKind.OTHER_FUNCTION),
-    SIN(Category.MATH, SqlKind.OTHER_FUNCTION),
+    ACOS(Category.MATH, SqlKind.OTHER_FUNCTION),
+    ASIN(Category.MATH, SqlKind.OTHER_FUNCTION),
+    ATAN(Category.MATH, SqlKind.OTHER_FUNCTION),
+    ATAN2(Category.MATH, SqlKind.OTHER_FUNCTION),
+    CBRT(Category.MATH, SqlKind.OTHER_FUNCTION),
     CEIL(Category.MATH, SqlKind.CEIL),
+    COS(Category.MATH, SqlKind.OTHER_FUNCTION),
+    COSH(Category.MATH, SqlKind.OTHER_FUNCTION),
+    COT(Category.MATH, SqlKind.OTHER_FUNCTION),
+    DEGREES(Category.MATH, SqlKind.OTHER_FUNCTION),
+    E(Category.MATH, SqlKind.OTHER_FUNCTION),
+    EXP(Category.MATH, SqlKind.OTHER_FUNCTION),
+    EXPM1(Category.MATH, SqlKind.OTHER_FUNCTION),
     FLOOR(Category.MATH, SqlKind.FLOOR),
+    LN(Category.MATH, SqlKind.OTHER_FUNCTION),
+    LOG(Category.MATH, SqlKind.OTHER_FUNCTION),
+    LOG10(Category.MATH, SqlKind.OTHER_FUNCTION),
+    LOG2(Category.MATH, SqlKind.OTHER_FUNCTION),
+    PI(Category.MATH, SqlKind.OTHER_FUNCTION),
+    POWER(Category.MATH, SqlKind.OTHER_FUNCTION),
+    RADIANS(Category.MATH, SqlKind.OTHER_FUNCTION),
+    RAND(Category.MATH, SqlKind.OTHER_FUNCTION),
+    ROUND(Category.MATH, SqlKind.OTHER_FUNCTION),
+    SCALAR_MAX(Category.MATH, SqlKind.OTHER_FUNCTION),
+    SCALAR_MIN(Category.MATH, SqlKind.OTHER_FUNCTION),
+    SIGN(Category.MATH, SqlKind.OTHER_FUNCTION),
+    SIN(Category.MATH, SqlKind.OTHER_FUNCTION),
+    SINH(Category.MATH, SqlKind.OTHER_FUNCTION),
+    TAN(Category.MATH, SqlKind.OTHER_FUNCTION),
+    TRUNCATE(Category.MATH, SqlKind.OTHER_FUNCTION),
 
     // ── Cast / type ──────────────────────────────────────────────────
     CAST(Category.SCALAR, SqlKind.CAST),
diff --git a/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/AbstractNameMappingAdapterTests.java b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/AbstractNameMappingAdapterTests.java
new file mode 100644
index 0000000000000..97255eccd4d6e
--- /dev/null
+++ b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/AbstractNameMappingAdapterTests.java
@@ -0,0 +1,153 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.spi;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.volcano.VolcanoPlanner;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlLibraryOperators;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.List;
+
+/**
+ * Unit tests for {@link AbstractNameMappingAdapter}. Covers the basic rename path, the
+ * prepend-literal form, and — most importantly — the {@link SqlTypeName#ANY} fallback
+ * that kicks in when the incoming PPL UDF declares an indeterminate return type (e.g.
+ * PPL's {@code SCALAR_MAX} / {@code SCALAR_MIN}). Without the fallback, Substrait
+ * serialisation fails with {@code Unable to convert the type ANY}.
+ */
+public class AbstractNameMappingAdapterTests extends OpenSearchTestCase {
+
+    private final RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl();
+    private final RexBuilder rexBuilder = new RexBuilder(typeFactory);
+    private final RelOptCluster cluster = RelOptCluster.create(new VolcanoPlanner(), rexBuilder);
+    private final RelDataType doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE);
+
+    /** Minimal concrete subclass for tests — pure rename, no prepend/append. */
+    private static final class TestRenameAdapter extends AbstractNameMappingAdapter {
+        TestRenameAdapter() {
+            super(SqlLibraryOperators.GREATEST, List.of(), List.of());
+        }
+    }
+
+    private SqlUserDefinedFunction pplUdf(String name, RelDataType returnType) {
+        return new SqlUserDefinedFunction(
+            new SqlIdentifier(name, SqlParserPos.ZERO),
+            SqlKind.OTHER_FUNCTION,
+            opBinding -> returnType,
+            null,
+            null,
+            null
+        );
+    }
+
+    public void testBasicRename() {
+        SqlUserDefinedFunction udf = pplUdf("SCALAR_MAX", doubleType);
+        RexNode a = rexBuilder.makeInputRef(doubleType, 0);
+        RexNode b = rexBuilder.makeInputRef(doubleType, 1);
+        RexCall original = (RexCall) rexBuilder.makeCall(udf, List.of(a, b));
+
+        RexNode adapted = new TestRenameAdapter().adapt(original, List.of(), cluster);
+
+        assertTrue(adapted instanceof RexCall);
+        RexCall adaptedCall = (RexCall) adapted;
+        assertSame(SqlLibraryOperators.GREATEST, adaptedCall.getOperator());
+        assertEquals(2, adaptedCall.getOperands().size());
+        assertSame(a, adaptedCall.getOperands().get(0));
+        assertSame(b, adaptedCall.getOperands().get(1));
+        assertSame("DOUBLE return type must be preserved", SqlTypeName.DOUBLE, adaptedCall.getType().getSqlTypeName());
+    }
+
+    public void testPrependLiteralOperand() {
+        SqlFunction yearUdf = new SqlFunction(
+            "YEAR",
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.BIGINT_NULLABLE,
+            null,
+            OperandTypes.ANY,
+            SqlFunctionCategory.TIMEDATE
+        );
+        RexNode ts = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.TIMESTAMP), 0);
+        RexCall original = (RexCall) rexBuilder.makeCall(yearUdf, List.of(ts));
+
+        AbstractNameMappingAdapter adapter = new AbstractNameMappingAdapter(SqlLibraryOperators.DATE_PART, List.of("year"), List.of()) {
+        };
+        RexNode adapted = adapter.adapt(original, List.of(), cluster);
+
+        RexCall adaptedCall = (RexCall) adapted;
+        assertSame(SqlLibraryOperators.DATE_PART, adaptedCall.getOperator());
+        assertEquals(2, adaptedCall.getOperands().size());
+        assertTrue(adaptedCall.getOperands().get(0) instanceof RexLiteral);
+        assertEquals("year", ((RexLiteral) adaptedCall.getOperands().get(0)).getValueAs(String.class));
+        assertSame(ts, adaptedCall.getOperands().get(1));
+    }
+
+    /**
+     * PPL's {@code SCALAR_MAX} / {@code SCALAR_MIN} declare their return type as
+     * {@link SqlTypeName#ANY}. Substrait cannot serialise ANY; the adapter must fall back to
+     * letting the target operator's own return-type inference run so the rewritten call
+     * carries a concrete type derived from the operands.
+     */
+    public void testAdaptFallsBackToTargetInferenceForAnyReturnType() {
+        RelDataType anyType = typeFactory.createSqlType(SqlTypeName.ANY);
+        SqlUserDefinedFunction udf = pplUdf("SCALAR_MAX", anyType);
+        RexNode a = rexBuilder.makeInputRef(doubleType, 0);
+        RexNode b = rexBuilder.makeInputRef(doubleType, 1);
+        RexNode c = rexBuilder.makeInputRef(doubleType, 2);
+        RexCall original = (RexCall) rexBuilder.makeCall(udf, List.of(a, b, c));
+        assertSame("precondition: UDF return type must be ANY", SqlTypeName.ANY, original.getType().getSqlTypeName());
+
+        RexNode adapted = new TestRenameAdapter().adapt(original, List.of(), cluster);
+
+        assertTrue(adapted instanceof RexCall);
+        RexCall adaptedCall = (RexCall) adapted;
+        assertSame(SqlLibraryOperators.GREATEST, adaptedCall.getOperator());
+        assertSame(
+            "ANY return type must be replaced with a concrete operand-derived type after rewrite",
+            SqlTypeName.DOUBLE,
+            adaptedCall.getType().getSqlTypeName()
+        );
+    }
+
+    /**
+     * Pass-through for SIGN — a standard Calcite operator whose return type is already
+     * concrete. The adapter still rewrites to the target operator (SignumFunction lives in
+     * the backend; here we use SqlStdOperatorTable.SQRT as a stand-in target with a
+     * concrete return type inferrer) and the preserved DOUBLE type proves the happy path.
+     */
+    public void testSignLikeRewritePreservesConcreteType() {
+        RexNode arg = rexBuilder.makeInputRef(doubleType, 0);
+        RexCall original = (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.SIGN, List.of(arg));
+
+        AbstractNameMappingAdapter adapter = new AbstractNameMappingAdapter(SqlStdOperatorTable.SQRT, List.of(), List.of()) {
+        };
+        RexNode adapted = adapter.adapt(original, List.of(), cluster);
+
+        RexCall adaptedCall = (RexCall) adapted;
+        assertSame(SqlStdOperatorTable.SQRT, adaptedCall.getOperator());
+        assertSame(SqlTypeName.DOUBLE, adaptedCall.getType().getSqlTypeName());
+    }
+}
diff --git a/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java
index e0db1cd82ab8e..1b503c61b4fa9 100644
--- a/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java
+++ b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/ScalarFunctionTests.java
@@ -9,10 +9,12 @@
 package org.opensearch.analytics.spi;
 
 import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlLibraryOperators;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.EnumMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -101,4 +103,100 @@ public void testFromSqlOperatorReturnsNullForUnknownFunction() {
         // both resolution paths miss and the resolver returns null instead of throwing.
         assertNull(ScalarFunction.fromSqlOperatorWithFallback(SqlStdOperatorTable.UNARY_MINUS));
     }
+
+    // ── Group G math functions: name-based lookup via fromSqlFunction ──────────
+    // PPL emits these as Calcite SqlBasicFunction calls whose name matches the
+    // enum constant. STANDARD_PROJECT_OPS registration (and adapter dispatch)
+    // depends on fromSqlFunction resolving them by name, so guard every entry.
+
+    public void testMathFunctionsResolveByName() {
+        assertSame(ScalarFunction.ABS, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.ABS));
+        assertSame(ScalarFunction.ACOS, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.ACOS));
+        assertSame(ScalarFunction.ASIN, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.ASIN));
+        assertSame(ScalarFunction.ATAN, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.ATAN));
+        assertSame(ScalarFunction.ATAN2, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.ATAN2));
+        assertSame(ScalarFunction.CBRT, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.CBRT));
+        assertSame(ScalarFunction.COS, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.COS));
+        assertSame(ScalarFunction.COT, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.COT));
+        assertSame(ScalarFunction.DEGREES, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.DEGREES));
+        assertSame(ScalarFunction.EXP, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.EXP));
+        assertSame(ScalarFunction.LN, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.LN));
+        // 2-arg log: PPL emits SqlLibraryOperators.LOG(x, base); 1-arg log(x) is pre-lowered to
+        // LOG(x, e) by PPLFuncImpTable, so this single LOG entry covers both arities.
+        assertSame(ScalarFunction.LOG, ScalarFunction.fromSqlFunction(SqlLibraryOperators.LOG));
+        assertSame(ScalarFunction.LOG10, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.LOG10));
+        assertSame(ScalarFunction.LOG2, ScalarFunction.fromSqlFunction(SqlLibraryOperators.LOG2));
+        assertSame(ScalarFunction.PI, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.PI));
+        assertSame(ScalarFunction.POWER, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.POWER));
+        assertSame(ScalarFunction.RADIANS, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.RADIANS));
+        assertSame(ScalarFunction.RAND, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.RAND));
+        assertSame(ScalarFunction.ROUND, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.ROUND));
+        assertSame(ScalarFunction.SIGN, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.SIGN));
+        assertSame(ScalarFunction.TAN, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.TAN));
+        assertSame(ScalarFunction.TRUNCATE, ScalarFunction.fromSqlFunction(SqlStdOperatorTable.TRUNCATE));
+    }
+
+    /** PPL's SCALAR_MAX / SCALAR_MIN UDFs resolve by the UDF's declared name — these are the
+     *  PPLBuiltinOperators variants that DataFusionAnalyticsBackendPlugin binds to
+     *  AbstractNameMappingAdapter instances targeting SqlLibraryOperators.GREATEST / LEAST. */
+    public void testScalarMaxMinResolveByName() {
+        assertSame(ScalarFunction.SCALAR_MAX, ScalarFunction.valueOf("SCALAR_MAX"));
+        assertSame(ScalarFunction.SCALAR_MIN, ScalarFunction.valueOf("SCALAR_MIN"));
+    }
+
+    /**
+     * Tier-2 adapter targets: enum entries exist for PPL UDFs even though the
+     * upstream isthmus SCALAR_SIGS only recognises SqlLibraryOperators variants.
+     * The DataFusion adapter rewrites the UDF call to the Calcite-library
+     * operator before Substrait conversion, but the name-based lookup here
+     * must still succeed so STANDARD_PROJECT_OPS and adapter dispatch can run.
+     */
+    public void testTier2AdapterTargetFunctionsExistByName() {
+        // PPL's COSH/SINH UDFs have getName() = "COSH"/"SINH"; valueOf succeeds.
+        assertSame(ScalarFunction.COSH, ScalarFunction.valueOf("COSH"));
+        assertSame(ScalarFunction.SINH, ScalarFunction.valueOf("SINH"));
+        // PPL's E() and EXPM1 UDFs likewise resolve by name.
+        assertSame(ScalarFunction.E, ScalarFunction.valueOf("E"));
+        assertSame(ScalarFunction.EXPM1, ScalarFunction.valueOf("EXPM1"));
+    }
+
+    /** Category hygiene: every math enum constant belongs to the MATH category. */
+    public void testMathFunctionsHaveMathCategory() {
+        List mathFuncs = List.of(
+            ScalarFunction.ABS,
+            ScalarFunction.ACOS,
+            ScalarFunction.ASIN,
+            ScalarFunction.ATAN,
+            ScalarFunction.ATAN2,
+            ScalarFunction.CBRT,
+            ScalarFunction.CEIL,
+            ScalarFunction.COS,
+            ScalarFunction.COSH,
+            ScalarFunction.COT,
+            ScalarFunction.DEGREES,
+            ScalarFunction.E,
+            ScalarFunction.EXP,
+            ScalarFunction.EXPM1,
+            ScalarFunction.FLOOR,
+            ScalarFunction.LN,
+            ScalarFunction.LOG,
+            ScalarFunction.LOG10,
+            ScalarFunction.LOG2,
+            ScalarFunction.PI,
+            ScalarFunction.POWER,
+            ScalarFunction.RADIANS,
+            ScalarFunction.RAND,
+            ScalarFunction.ROUND,
+            ScalarFunction.SCALAR_MAX,
+            ScalarFunction.SCALAR_MIN,
+            ScalarFunction.SIGN,
+            ScalarFunction.SIN,
+            ScalarFunction.SINH,
+            ScalarFunction.TAN,
+            ScalarFunction.TRUNCATE
+        );
+        for (ScalarFunction func : mathFuncs) {
+            assertSame("expected MATH category for " + func, ScalarFunction.Category.MATH, func.getCategory());
+        }
+    }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
index b7addffb72644..5df91a61b1968 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
@@ -8,7 +8,10 @@
 
 package org.opensearch.be.datafusion;
 
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlLibraryOperators;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.opensearch.analytics.spi.AbstractNameMappingAdapter;
 import org.opensearch.analytics.spi.AggregateCapability;
 import org.opensearch.analytics.spi.AggregateFunction;
 import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin;
@@ -117,6 +120,12 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
         ScalarFunction.ABS,
         ScalarFunction.SUBSTRING,
         ScalarFunction.SARG_PREDICATE,
+        ScalarFunction.MINUS,
+        ScalarFunction.ACOS,
+        ScalarFunction.ASIN,
+        ScalarFunction.ATAN,
+        ScalarFunction.ATAN2,
+        ScalarFunction.CBRT,
         ScalarFunction.EQUALS,
         ScalarFunction.NOT_EQUALS,
         ScalarFunction.GREATER_THAN,
@@ -129,10 +138,33 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
         ScalarFunction.REPLACE,
         ScalarFunction.REGEXP_REPLACE,
         ScalarFunction.PLUS,
-        ScalarFunction.MINUS,
         ScalarFunction.TIMES,
         ScalarFunction.DIVIDE,
         ScalarFunction.MOD,
+        ScalarFunction.COS,
+        ScalarFunction.COT,
+        ScalarFunction.DEGREES,
+        ScalarFunction.EXP,
+        ScalarFunction.FLOOR,
+        ScalarFunction.LN,
+        ScalarFunction.LOG,
+        ScalarFunction.LOG10,
+        ScalarFunction.LOG2,
+        ScalarFunction.PI,
+        ScalarFunction.POWER,
+        ScalarFunction.RADIANS,
+        ScalarFunction.RAND,
+        ScalarFunction.ROUND,
+        ScalarFunction.SIGN,
+        ScalarFunction.SIN,
+        ScalarFunction.TAN,
+        ScalarFunction.TRUNCATE,
+        ScalarFunction.COSH,
+        ScalarFunction.SINH,
+        ScalarFunction.E,
+        ScalarFunction.EXPM1,
+        ScalarFunction.SCALAR_MAX,
+        ScalarFunction.SCALAR_MIN,
         ScalarFunction.YEAR,
         ScalarFunction.CONVERT_TZ,
         ScalarFunction.UNIX_TIMESTAMP
@@ -219,11 +251,18 @@ public Map scalarFunctionAdapters() {
                 return Map.ofEntries(
                     Map.entry(ScalarFunction.CONCAT, new ConcatFunctionAdapter()),
                     Map.entry(ScalarFunction.CONVERT_TZ, new ConvertTzAdapter()),
+                    Map.entry(ScalarFunction.COSH, new HyperbolicOperatorAdapter(SqlLibraryOperators.COSH)),
                     Map.entry(ScalarFunction.DIVIDE, new StdOperatorRewriteAdapter("DIVIDE", SqlStdOperatorTable.DIVIDE)),
+                    Map.entry(ScalarFunction.E, new EConstantAdapter()),
+                    Map.entry(ScalarFunction.EXPM1, new Expm1Adapter()),
                     Map.entry(ScalarFunction.LIKE, new LikeAdapter()),
                     Map.entry(ScalarFunction.MOD, new StdOperatorRewriteAdapter("MOD", SqlStdOperatorTable.MOD)),
                     Map.entry(ScalarFunction.REGEXP_REPLACE, new RegexpReplaceAdapter()),
                     Map.entry(ScalarFunction.SARG_PREDICATE, new SargAdapter()),
+                    Map.entry(ScalarFunction.SCALAR_MAX, nameMapping(SqlLibraryOperators.GREATEST)),
+                    Map.entry(ScalarFunction.SCALAR_MIN, nameMapping(SqlLibraryOperators.LEAST)),
+                    Map.entry(ScalarFunction.SIGN, nameMapping(SignumFunction.FUNCTION)),
+                    Map.entry(ScalarFunction.SINH, new HyperbolicOperatorAdapter(SqlLibraryOperators.SINH)),
                     Map.entry(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter()),
                     Map.entry(ScalarFunction.UNIX_TIMESTAMP, new UnixTimestampAdapter()),
                     Map.entry(ScalarFunction.YEAR, new YearAdapter())
@@ -232,6 +271,16 @@ public Map scalarFunctionAdapters() {
         };
     }
 
+    /**
+     * Pure rename from a PPL scalar to {@code target} — no prepend / append operands.
+     * Concrete subclass of {@link AbstractNameMappingAdapter} because the abstract
+     * base cannot be instantiated directly.
+     */
+    private static AbstractNameMappingAdapter nameMapping(SqlOperator target) {
+        return new AbstractNameMappingAdapter(target, java.util.List.of(), java.util.List.of()) {
+        };
+    }
+
     @Override
     public FragmentConvertor getFragmentConvertor() {
         return new DataFusionFragmentConvertor(plugin.getSubstraitExtensions());
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
index 0454e013bdd1e..ecdef923f656e 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
@@ -100,13 +100,21 @@ public class DataFusionFragmentConvertor implements FragmentConvertor {
     private static final List ADDITIONAL_SCALAR_SIGS = List.of(
         FunctionMappings.s(DelegatedPredicateFunction.FUNCTION, DelegatedPredicateFunction.NAME),
         FunctionMappings.s(SqlLibraryOperators.ILIKE, "ilike"),
-        FunctionMappings.s(DelegatedPredicateFunction.FUNCTION, DelegatedPredicateFunction.NAME),
         FunctionMappings.s(SqlLibraryOperators.DATE_PART, "date_part"),
         FunctionMappings.s(ConvertTzAdapter.LOCAL_CONVERT_TZ_OP, "convert_tz"),
         FunctionMappings.s(UnixTimestampAdapter.LOCAL_TO_UNIXTIME_OP, "to_unixtime"),
         FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"),
         FunctionMappings.s(SqlStdOperatorTable.REPLACE, "replace"),
-        FunctionMappings.s(SqlLibraryOperators.REGEXP_REPLACE_3, "regexp_replace")
+        FunctionMappings.s(SqlLibraryOperators.REGEXP_REPLACE_3, "regexp_replace"),
+        FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"),
+        FunctionMappings.s(UnixTimestampAdapter.LOCAL_TO_UNIXTIME_OP, "to_unixtime"),
+        FunctionMappings.s(SqlStdOperatorTable.TRUNCATE, "trunc"),
+        FunctionMappings.s(SqlStdOperatorTable.CBRT, "cbrt"),
+        FunctionMappings.s(SqlStdOperatorTable.COT, "cot"),
+        FunctionMappings.s(SqlStdOperatorTable.PI, "pi"),
+        FunctionMappings.s(SqlStdOperatorTable.RAND, "random"),
+        FunctionMappings.s(SqlLibraryOperators.LOG, "logb"),
+        FunctionMappings.s(SignumFunction.FUNCTION, SignumFunction.NAME)
     );
 
     private final SimpleExtension.ExtensionCollection extensions;
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/EConstantAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/EConstantAdapter.java
new file mode 100644
index 0000000000000..7bc61d1f2b324
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/EConstantAdapter.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+/**
+ * Rewrites the zero-arg PPL {@code E()} UDF call to a {@code DOUBLE} literal
+ * equal to {@link Math#E}. DataFusion's substrait consumer has no {@code e}
+ * scalar function, but constant-folding the call on the coordinator side
+ * before Substrait serialisation produces a literal expression the downstream
+ * plan handles trivially.
+ *
+ * @opensearch.internal
+ */
+class EConstantAdapter implements ScalarFunctionAdapter {
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        // Only rewrite the zero-arg E() UDF. Defensive guard against accidental
+        // registration — any call with operands, or one whose operator isn't named
+        // "E", passes through unchanged so it can be surfaced as a planner error
+        // further down the pipeline instead of being silently dropped.
+        if (!original.getOperator().getName().equalsIgnoreCase("E")) {
+            return original;
+        }
+        if (!original.getOperands().isEmpty()) {
+            return original;
+        }
+        return cluster.getRexBuilder().makeApproxLiteral(BigDecimal.valueOf(Math.E));
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/Expm1Adapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/Expm1Adapter.java
new file mode 100644
index 0000000000000..cad190b48e6b9
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/Expm1Adapter.java
@@ -0,0 +1,53 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+/**
+ * Rewrites PPL's {@code EXPM1(x)} UDF (defined as {@code exp(x) - 1}) into the
+ * equivalent {@code MINUS(EXP(x), 1)} expression tree. DataFusion's substrait
+ * consumer recognises {@code exp} and {@code subtract} natively, but has no
+ * direct {@code expm1} scalar function; lowering the UDF before Substrait
+ * serialisation keeps the plan expressible in standard Substrait primitives.
+ *
+ * 
      For very small inputs {@code exp(x) - 1} has worse precision than the
+ * dedicated {@code Math.expm1} implementation, but PPL's semantic is already
+ * the naive subtraction (see {@code PPLBuiltinOperators.EXPM1}) so behaviour
+ * is preserved.
+ *
+ * @opensearch.internal
+ */
+class Expm1Adapter implements ScalarFunctionAdapter {
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        // Defensive: only rewrite the EXPM1 UDF. Any other call passes through.
+        if (!original.getOperator().getName().equalsIgnoreCase("EXPM1")) {
+            return original;
+        }
+        if (original.getOperands().size() != 1) {
+            return original;
+        }
+        RexBuilder rexBuilder = cluster.getRexBuilder();
+        RexNode arg = original.getOperands().get(0);
+        RexNode exp = rexBuilder.makeCall(original.getType(), SqlStdOperatorTable.EXP, List.of(arg));
+        RexNode one = rexBuilder.makeExactLiteral(BigDecimal.ONE);
+        return rexBuilder.makeCall(SqlStdOperatorTable.MINUS, exp, one);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/HyperbolicOperatorAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/HyperbolicOperatorAdapter.java
new file mode 100644
index 0000000000000..dcb64c8617748
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/HyperbolicOperatorAdapter.java
@@ -0,0 +1,65 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.util.List;
+
+/**
+ * Rewrites a one-arg scalar UDF call to use a target Calcite {@link SqlOperator}.
+ *
+ * 
      Used for PPL hyperbolic functions ({@code SINH}, {@code COSH}): PPL emits
+ * them as {@link org.apache.calcite.sql.validate.SqlUserDefinedFunction} UDFs,
+ * but isthmus's {@code FunctionMappings.SCALAR_SIGS} only maps the variants in
+ * {@link org.apache.calcite.sql.fun.SqlLibraryOperators} to their Substrait
+ * canonical names ({@code sinh}, {@code cosh}). This adapter swaps the operator
+ * reference while preserving the operand so the subsequent Substrait visitor
+ * produces the standard function call DataFusion's substrait consumer evaluates
+ * natively.
+ *
+ * 
      Input shape: {@code UDF(arg)}. Output shape: {@code targetOperator(arg)}.
+ * Preserves the Calcite row type of the call.
+ *
+ * @opensearch.internal
+ */
+class HyperbolicOperatorAdapter implements ScalarFunctionAdapter {
+
+    private final SqlOperator targetOperator;
+
+    HyperbolicOperatorAdapter(SqlOperator targetOperator) {
+        this.targetOperator = targetOperator;
+    }
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        // Idempotency — if the plan already carries the target operator, leave it.
+        if (original.getOperator() == targetOperator) {
+            return original;
+        }
+        // Defensive: the adapter is only registered against the ScalarFunction whose
+        // name matches the target operator, so any other call shape is a programming
+        // error upstream. Rather than silently rewriting (which would corrupt unrelated
+        // math functions like ABS if the adapter were mis-registered), only rewrite
+        // when the operator name matches.
+        if (!original.getOperator().getName().equalsIgnoreCase(targetOperator.getName())) {
+            return original;
+        }
+        if (original.getOperands().size() != 1) {
+            return original;
+        }
+        // Swap the operator but keep the operand and the Calcite-inferred type.
+        return cluster.getRexBuilder().makeCall(original.getType(), targetOperator, original.getOperands());
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SignumFunction.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SignumFunction.java
new file mode 100644
index 0000000000000..4e77b865bc097
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SignumFunction.java
@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+
+/**
+ * Dedicated Calcite {@link SqlFunction} paired with the {@code signum} Substrait
+ * extension declared in {@code opensearch_scalar_functions.yaml}. The PPL
+ * frontend emits {@link org.apache.calcite.sql.fun.SqlStdOperatorTable#SIGN},
+ * which isthmus's default {@code SCALAR_SIGS} maps to the Substrait name
+ * {@code sign} — the name DataFusion's substrait consumer does not accept
+ * (DataFusion registers the UDF as {@code signum}).
+ *
+ * 
      An {@link org.opensearch.analytics.spi.AbstractNameMappingAdapter} registered
+ * against {@code ScalarFunction.SIGN} rewrites the incoming PPL {@code SIGN} call
+ * to use {@code SignumFunction.FUNCTION}, and {@code ADDITIONAL_SCALAR_SIGS} in
+ * {@link DataFusionFragmentConvertor} maps this operator to the {@code signum}
+ * extension name. Keeping a separate Calcite operator avoids a collision with
+ * the default {@code SIGN → sign} mapping and makes isthmus serialisation
+ * deterministic independent of map iteration order.
+ *
+ * @opensearch.internal
+ */
+final class SignumFunction {
+
+    /** Substrait extension function name declared in opensearch_scalar_functions.yaml. */
+    static final String NAME = "signum";
+
+    /** Calcite operator binding: {@code signum(NUMERIC) → DOUBLE}. */
+    static final SqlFunction FUNCTION = new SqlFunction(
+        NAME.toUpperCase(java.util.Locale.ROOT),
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.DOUBLE_NULLABLE,
+        null,
+        OperandTypes.NUMERIC,
+        SqlFunctionCategory.NUMERIC
+    );
+
+    private SignumFunction() {}
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
index 5d8f9621439f7..a19428cad0a4b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
@@ -4,9 +4,8 @@ urn: extension:org.opensearch:scalar_functions
 scalar_functions:
   - name: ilike
     description: >-
-      Case-insensitive LIKE. Semantics match DataFusion's ILikeExpr
-      (datafusion-substrait maps the extension name "ilike" to a case-insensitive
-      LikeExpr).
+      Case-insensitive LIKE. datafusion-substrait maps the extension name `ilike`
+      to a case-insensitive `LikeExpr`.
     impls:
       - args:
           - value: "varchar"
@@ -59,6 +58,102 @@ scalar_functions:
           - value: "string"
             name: "pattern"
         return: boolean
+  - name: cbrt
+    description: >-
+      Cube root. Resolves to DataFusion's built-in `cbrt` scalar function.
+    impls:
+      - args:
+          - value: fp32
+            name: x
+        return: fp32
+      - args:
+          - value: fp64
+            name: x
+        return: fp64
+  - name: cot
+    description: >-
+      Cotangent. Resolves to DataFusion's built-in `cot` scalar function.
+    impls:
+      - args:
+          - value: fp32
+            name: x
+        return: fp32
+      - args:
+          - value: fp64
+            name: x
+        return: fp64
+  - name: pi
+    description: >-
+      Zero-argument π constant. Resolves to DataFusion's built-in `pi` scalar
+      function.
+    impls:
+      - args: []
+        return: fp64
+  - name: random
+    description: >-
+      Pseudorandom fp64 in [0, 1). Resolves to DataFusion's built-in `random`
+      scalar function. PPL surface name is `rand`; FunctionMappings maps
+      `SqlStdOperatorTable.RAND` to this extension name.
+    impls:
+      - args: []
+        return: fp64
+  - name: round
+    description: >-
+      One-argument rounding to the nearest integer, preserving input type.
+      DataFusion's built-in `round` also supports a 2-arg (value, digits)
+      overload matching the Substrait default signature; PPL frequently emits
+      a single-argument form which this entry declares.
+    impls:
+      - args:
+          - value: fp32
+            name: x
+        return: fp32
+      - args:
+          - value: fp64
+            name: x
+        return: fp64
+  - name: signum
+    description: >-
+      Signum. DataFusion's built-in scalar function is named `signum`; PPL/Calcite
+      surface name is `sign`. AbstractNameMappingAdapter retargets Calcite's `SIGN`
+      at SignumFunction (yaml name `signum`) so isthmus emits the name DataFusion
+      resolves directly.
+    impls:
+      - args:
+          - value: fp32
+            name: x
+        return: fp32
+      - args:
+          - value: fp64
+            name: x
+        return: fp64
+  - name: trunc
+    description: >-
+      Truncate toward zero. Resolves to DataFusion's built-in `trunc` scalar
+      function. PPL's `truncate` — `SqlStdOperatorTable.TRUNCATE` — accepts
+      both 1-arg and 2-arg (value, scale) forms per the
+      CompositeOperandTypeChecker; both are declared here.
+    impls:
+      - args:
+          - value: fp32
+            name: x
+        return: fp32
+      - args:
+          - value: fp64
+            name: x
+        return: fp64
+      - args:
+          - value: fp32
+            name: x
+          - value: i32
+            name: scale
+        return: fp32
+      - args:
+          - value: fp64
+            name: x
+          - value: i32
+            name: scale
+        return: fp64
   - name: replace
     description: >-
       Literal string replacement — replace every occurrence of `search` in `input`
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/EConstantAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/EConstantAdapterTests.java
new file mode 100644
index 0000000000000..e5b37badf760c
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/EConstantAdapterTests.java
@@ -0,0 +1,87 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+/**
+ * Unit tests for {@link EConstantAdapter}. PPL's {@code E()} UDF has zero operands
+ * and evaluates to Euler's number. DataFusion has no matching scalar function, but
+ * constant folding is cheap on the coordinator — the adapter rewrites the UDF call
+ * to a {@code DOUBLE} literal equal to {@link Math#E}, which serialises trivially
+ * through Substrait as a literal expression.
+ */
+public class EConstantAdapterTests extends OpenSearchTestCase {
+
+    private RelDataTypeFactory typeFactory;
+    private RexBuilder rexBuilder;
+    private RelOptCluster cluster;
+    private RelDataType doubleType;
+    private SqlUserDefinedFunction eUdf;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        typeFactory = new JavaTypeFactoryImpl();
+        rexBuilder = new RexBuilder(typeFactory);
+        HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+        cluster = RelOptCluster.create(planner, rexBuilder);
+        doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE);
+        eUdf = new SqlUserDefinedFunction(
+            new SqlIdentifier("E", SqlParserPos.ZERO),
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.DOUBLE,
+            null,
+            null,
+            null
+        );
+    }
+
+    public void testEUdfRewrittenToMathELiteral() {
+        RexCall original = (RexCall) rexBuilder.makeCall(eUdf, List.of());
+
+        RexNode adapted = new EConstantAdapter().adapt(original, List.of(), cluster);
+
+        assertTrue("expected adapter to return a literal", adapted instanceof RexLiteral);
+        RexLiteral lit = (RexLiteral) adapted;
+        BigDecimal value = lit.getValueAs(BigDecimal.class);
+        assertNotNull(value);
+        assertEquals("literal must carry Math.E", 0, value.compareTo(BigDecimal.valueOf(Math.E)));
+        assertEquals("literal type must be DOUBLE", SqlTypeName.DOUBLE, lit.getType().getSqlTypeName());
+    }
+
+    public void testAdaptPassesThroughUnrelatedCall() {
+        RexNode ref = rexBuilder.makeInputRef(doubleType, 0);
+        RexCall absCall = (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.ABS, List.of(ref));
+
+        RexNode adapted = new EConstantAdapter().adapt(absCall, List.of(), cluster);
+
+        assertSame(absCall, adapted);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/Expm1AdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/Expm1AdapterTests.java
new file mode 100644
index 0000000000000..bc99641188084
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/Expm1AdapterTests.java
@@ -0,0 +1,96 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.List;
+
+/**
+ * Unit tests for {@link Expm1Adapter}. PPL's {@code EXPM1(x)} UDF is defined as
+ * {@code exp(x) - 1}. DataFusion's substrait consumer has no {@code expm1} scalar
+ * function, but it recognises {@code exp} and {@code subtract}; the adapter
+ * expands the UDF to an explicit {@code MINUS(EXP(x), 1)} tree so the plan
+ * serialises to native Substrait primitives.
+ */
+public class Expm1AdapterTests extends OpenSearchTestCase {
+
+    private RelDataTypeFactory typeFactory;
+    private RexBuilder rexBuilder;
+    private RelOptCluster cluster;
+    private RelDataType doubleType;
+    private SqlUserDefinedFunction expm1Udf;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        typeFactory = new JavaTypeFactoryImpl();
+        rexBuilder = new RexBuilder(typeFactory);
+        HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+        cluster = RelOptCluster.create(planner, rexBuilder);
+        doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE);
+        expm1Udf = new SqlUserDefinedFunction(
+            new SqlIdentifier("EXPM1", SqlParserPos.ZERO),
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.DOUBLE_NULLABLE,
+            null,
+            null,
+            null
+        );
+    }
+
+    public void testExpm1RewrittenAsExpMinusOne() {
+        RexNode arg = rexBuilder.makeInputRef(doubleType, 0);
+        RexCall original = (RexCall) rexBuilder.makeCall(expm1Udf, List.of(arg));
+
+        RexNode adapted = new Expm1Adapter().adapt(original, List.of(), cluster);
+
+        // Expected tree: MINUS(EXP(arg), 1)
+        assertTrue("expected a MINUS RexCall", adapted instanceof RexCall);
+        RexCall minus = (RexCall) adapted;
+        assertSame("outermost operator must be MINUS", SqlStdOperatorTable.MINUS, minus.getOperator());
+        assertEquals(2, minus.getOperands().size());
+
+        RexNode left = minus.getOperands().get(0);
+        assertTrue("left operand of MINUS must be a RexCall", left instanceof RexCall);
+        RexCall expCall = (RexCall) left;
+        assertSame("left operand must be EXP(...)", SqlStdOperatorTable.EXP, expCall.getOperator());
+        assertEquals(1, expCall.getOperands().size());
+        assertSame("EXP operand must be the original arg", arg, expCall.getOperands().get(0));
+
+        // Right operand must be numerically 1 (type may be DECIMAL or INTEGER depending on promotion)
+        RexNode right = minus.getOperands().get(1);
+        assertTrue("right operand must be a literal", right instanceof org.apache.calcite.rex.RexLiteral);
+    }
+
+    public void testAdaptPassesThroughUnrelatedCall() {
+        RexNode ref = rexBuilder.makeInputRef(doubleType, 0);
+        RexCall absCall = (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.ABS, List.of(ref));
+
+        RexNode adapted = new Expm1Adapter().adapt(absCall, List.of(), cluster);
+
+        assertSame(absCall, adapted);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/HyperbolicOperatorAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/HyperbolicOperatorAdapterTests.java
new file mode 100644
index 0000000000000..eec04d10435cf
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/HyperbolicOperatorAdapterTests.java
@@ -0,0 +1,114 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlLibraryOperators;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.List;
+
+/**
+ * Unit tests for {@link HyperbolicOperatorAdapter}. PPL's {@code SINH}/{@code COSH}
+ * arrive as {@link SqlUserDefinedFunction} UDF calls; the adapter rewrites them to
+ * use the Calcite library operator that isthmus {@code FunctionMappings.SCALAR_SIGS}
+ * recognises ({@link SqlLibraryOperators#SINH}/{@link SqlLibraryOperators#COSH}),
+ * so the plan serialises to the standard Substrait {@code sinh}/{@code cosh}
+ * functions that DataFusion's substrait consumer natively evaluates.
+ */
+public class HyperbolicOperatorAdapterTests extends OpenSearchTestCase {
+
+    private RelDataTypeFactory typeFactory;
+    private RexBuilder rexBuilder;
+    private RelOptCluster cluster;
+    private RelDataType doubleType;
+    private SqlUserDefinedFunction sinhUdf;
+    private SqlUserDefinedFunction coshUdf;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        typeFactory = new JavaTypeFactoryImpl();
+        rexBuilder = new RexBuilder(typeFactory);
+        HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+        cluster = RelOptCluster.create(planner, rexBuilder);
+        doubleType = typeFactory.createSqlType(SqlTypeName.DOUBLE);
+        sinhUdf = fakeUdf("SINH");
+        coshUdf = fakeUdf("COSH");
+    }
+
+    /** Fake PPL-style UDF — same name and kind as PPLBuiltinOperators's SINH/COSH. */
+    private SqlUserDefinedFunction fakeUdf(String name) {
+        return new SqlUserDefinedFunction(
+            new SqlIdentifier(name, SqlParserPos.ZERO),
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.DOUBLE_NULLABLE,
+            null,
+            null,
+            null
+        );
+    }
+
+    public void testSinhUdfRewrittenToLibrarySinhOperator() {
+        RexNode arg = rexBuilder.makeInputRef(doubleType, 0);
+        RexCall original = (RexCall) rexBuilder.makeCall(sinhUdf, List.of(arg));
+        assertEquals("SINH", original.getOperator().getName());
+        assertFalse("precondition: operator is PPL UDF, not the library operator", original.getOperator() == SqlLibraryOperators.SINH);
+
+        RexNode adapted = new HyperbolicOperatorAdapter(SqlLibraryOperators.SINH).adapt(original, List.of(), cluster);
+
+        assertTrue("expected adapter to produce a RexCall", adapted instanceof RexCall);
+        RexCall adaptedCall = (RexCall) adapted;
+        assertSame("operator must be SqlLibraryOperators.SINH after adaptation", SqlLibraryOperators.SINH, adaptedCall.getOperator());
+        assertEquals("operand count preserved", 1, adaptedCall.getOperands().size());
+        assertSame("operand reference preserved", arg, adaptedCall.getOperands().get(0));
+    }
+
+    public void testCoshUdfRewrittenToLibraryCoshOperator() {
+        RexNode arg = rexBuilder.makeInputRef(doubleType, 1);
+        RexCall original = (RexCall) rexBuilder.makeCall(coshUdf, List.of(arg));
+
+        RexNode adapted = new HyperbolicOperatorAdapter(SqlLibraryOperators.COSH).adapt(original, List.of(), cluster);
+
+        assertTrue(adapted instanceof RexCall);
+        RexCall adaptedCall = (RexCall) adapted;
+        assertSame(SqlLibraryOperators.COSH, adaptedCall.getOperator());
+        assertEquals(1, adaptedCall.getOperands().size());
+        assertSame(arg, adaptedCall.getOperands().get(0));
+    }
+
+    /**
+     * Non-UDF calls (e.g. {@code ABS($0)}) must pass through untouched. Guards
+     * against collateral damage if the adapter is registered against a
+     * different {@code ScalarFunction} by mistake.
+     */
+    public void testAdaptPassesThroughUnrelatedCall() {
+        RexNode arg = rexBuilder.makeInputRef(doubleType, 0);
+        RexCall absCall = (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.ABS, List.of(arg));
+
+        RexNode adapted = new HyperbolicOperatorAdapter(SqlLibraryOperators.SINH).adapt(absCall, List.of(), cluster);
+
+        assertSame("non-SINH call must pass through unmodified", absCall, adapted);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/MathProjectCapabilitiesTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/MathProjectCapabilitiesTests.java
new file mode 100644
index 0000000000000..f62aeb279add6
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/MathProjectCapabilitiesTests.java
@@ -0,0 +1,105 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.analytics.spi.BackendCapabilityProvider;
+import org.opensearch.analytics.spi.ProjectCapability;
+import org.opensearch.analytics.spi.ScalarFunction;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Contract test for Group G: every Tier-1 math function and every Tier-2 adapter
+ * target is registered as a Scalar project capability on the DataFusion backend.
+ * Without this registration {@code OpenSearchProjectRule} drops the function
+ * through to a residual project on the coordinator, defeating native pushdown.
+ */
+public class MathProjectCapabilitiesTests extends OpenSearchTestCase {
+
+    private Set exposedProjectScalars() {
+        DataFusionAnalyticsBackendPlugin backendPlugin = new DataFusionAnalyticsBackendPlugin(new DataFusionPlugin());
+        BackendCapabilityProvider provider = backendPlugin.getCapabilityProvider();
+        Set seen = new HashSet<>();
+        for (ProjectCapability cap : provider.projectCapabilities()) {
+            if (cap instanceof ProjectCapability.Scalar scalar) {
+                seen.add(scalar.function());
+            }
+        }
+        return seen;
+    }
+
+    public void testMathFunctionsAreProjectCapable() {
+        Set projectable = exposedProjectScalars();
+        ScalarFunction[] functions = new ScalarFunction[] {
+            ScalarFunction.ABS,
+            ScalarFunction.ACOS,
+            ScalarFunction.ASIN,
+            ScalarFunction.ATAN,
+            ScalarFunction.ATAN2,
+            ScalarFunction.CBRT,
+            ScalarFunction.CEIL,
+            ScalarFunction.COS,
+            ScalarFunction.COT,
+            ScalarFunction.DEGREES,
+            ScalarFunction.EXP,
+            ScalarFunction.FLOOR,
+            ScalarFunction.LN,
+            ScalarFunction.LOG,
+            ScalarFunction.LOG10,
+            ScalarFunction.LOG2,
+            ScalarFunction.PI,
+            ScalarFunction.POWER,
+            ScalarFunction.RADIANS,
+            ScalarFunction.RAND,
+            ScalarFunction.ROUND,
+            ScalarFunction.SIGN,
+            ScalarFunction.SIN,
+            ScalarFunction.TAN,
+            ScalarFunction.TRUNCATE, };
+        for (ScalarFunction f : functions) {
+            assertTrue("function not registered as Scalar project capability: " + f, projectable.contains(f));
+        }
+    }
+
+    public void testAdapterTargetFunctionsAreProjectCapable() {
+        Set projectable = exposedProjectScalars();
+        ScalarFunction[] functions = new ScalarFunction[] {
+            ScalarFunction.COSH,
+            ScalarFunction.SINH,
+            ScalarFunction.E,
+            ScalarFunction.EXPM1,
+            ScalarFunction.SCALAR_MAX,
+            ScalarFunction.SCALAR_MIN, };
+        for (ScalarFunction f : functions) {
+            assertTrue("adapter target not registered as Scalar project capability: " + f, projectable.contains(f));
+        }
+    }
+
+    public void testAdapterTargetFunctionsHaveAdapters() {
+        DataFusionAnalyticsBackendPlugin backendPlugin = new DataFusionAnalyticsBackendPlugin(new DataFusionPlugin());
+        Map adapters = backendPlugin.getCapabilityProvider().scalarFunctionAdapters();
+        assertNotNull("SINH must have an adapter registered", adapters.get(ScalarFunction.SINH));
+        assertNotNull("COSH must have an adapter registered", adapters.get(ScalarFunction.COSH));
+        assertNotNull("E must have an adapter registered", adapters.get(ScalarFunction.E));
+        assertNotNull("EXPM1 must have an adapter registered", adapters.get(ScalarFunction.EXPM1));
+        assertNotNull("SCALAR_MAX must have an adapter registered", adapters.get(ScalarFunction.SCALAR_MAX));
+        assertNotNull("SCALAR_MIN must have an adapter registered", adapters.get(ScalarFunction.SCALAR_MIN));
+        assertNotNull("SIGN must have an adapter registered", adapters.get(ScalarFunction.SIGN));
+    }
+
+    /** MINUS must be project-capable because Expm1Adapter rewrites {@code expm1(x)} to {@code MINUS(EXP(x), 1)}. */
+    public void testMinusIsProjectCapableForExpm1AdapterOutput() {
+        Set projectable = exposedProjectScalars();
+        assertTrue("MINUS must be project-capable because Expm1Adapter emits it", projectable.contains(ScalarFunction.MINUS));
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java
index 711bb3a5c8e1b..9e82cb1c26a1b 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java
@@ -163,6 +163,22 @@ private List resolveScalarViableBackends(RexCall rexCall, List c
             return List.of();
         }
         FieldType fieldType = FieldType.fromSqlTypeName(rexCall.getType().getSqlTypeName());
+        // Polymorphic UDF fallback: Calcite UDFs with indeterminate return types (SqlTypeName.ANY)
+        // — e.g. PPL's ScalarMaxFunction / ScalarMinFunction — do not map to a concrete FieldType
+        // directly. When a viability check for such a call lands here, fall back to the first
+        // operand's type. The scalar function's backend capabilities are defined over operand
+        // types anyway (SCALAR_MAX(double, double, ...) → DOUBLE), so inferring from operands
+        // preserves correct backend dispatch while deferring actual type-tightening until the
+        // backend's ScalarFunctionAdapter rewrites the call to a typed library operator.
+        if (fieldType == null) {
+            for (RexNode operand : rexCall.getOperands()) {
+                FieldType operandType = FieldType.fromSqlTypeName(operand.getType().getSqlTypeName());
+                if (operandType != null) {
+                    fieldType = operandType;
+                    break;
+                }
+            }
+        }
         if (fieldType == null) {
             return List.of();
         }
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java
index fcc2a69250a65..16f12a575112a 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java
@@ -142,6 +142,36 @@ public void testUnsupportedScalarFunctionErrors() {
         assertTrue(exception.getMessage().contains("No backend supports scalar function"));
     }
 
+    /**
+     * PPL emits {@code SCALAR_MAX(a, b, c)} as a UDF whose return type is {@link SqlTypeName#ANY}
+     * — a consequence of the underlying {@code ScalarMaxFunction} being polymorphic across numeric
+     * and string types. The project rule must not reject such calls outright; instead it should
+     * fall back to inferring the operand type (DOUBLE here) so downstream backend capability
+     * dispatch proceeds normally. The actual operator rewrite to {@code GREATEST} happens later
+     * via the backend's {@code ScalarFunctionAdapter}.
+     */
+    public void testScalarFunctionWithAnyReturnTypeUsesOperandFallback() {
+        SqlFunction scalarMaxUdf = new SqlFunction(
+            "SCALAR_MAX",
+            SqlKind.OTHER_FUNCTION,
+            opBinding -> typeFactory.createSqlType(SqlTypeName.ANY),
+            null,
+            OperandTypes.VARIADIC,
+            SqlFunctionCategory.USER_DEFINED_FUNCTION
+        );
+        // Reference the INTEGER column (index 1) from the stub scan's (VARCHAR, INTEGER) schema.
+        // The operand-type fallback must resolve INTEGER → FieldType.INTEGER so the backend
+        // capability lookup succeeds.
+        RexNode intRef = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1);
+        RexNode expr = rexBuilder.makeCall(scalarMaxUdf, intRef, intRef);
+        assertSame("precondition: UDF return type must be ANY", SqlTypeName.ANY, expr.getType().getSqlTypeName());
+
+        OpenSearchProject result = runProject(expr);
+
+        assertTrue(result.getViableBackends().contains(MockDataFusionBackend.NAME));
+        assertAnnotation(result.getProjects().get(0), MockDataFusionBackend.NAME);
+    }
+
     // ---- Delegation ----
 
     public void testPainlessDelegationFromDataFusionToLucene() {
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MathScalarFunctionsIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MathScalarFunctionsIT.java
new file mode 100644
index 0000000000000..c9c72b9966dc8
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MathScalarFunctionsIT.java
@@ -0,0 +1,320 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * End-to-end coverage for Group G math scalar functions on the analytics-engine
+ * route (PPL → CalciteRelNodeVisitor → Substrait → DataFusion).
+ *
+ * 
      Each test exercises a single math function against a specific row of the
+ * {@code calcs} dataset via {@code POST /_analytics/ppl}. Tests pin a
+ * particular row by filtering on the {@code key} keyword field and then apply
+ * the math function to one of that row's {@code num*} (DOUBLE) fields — field
+ * references both block Calcite's {@code ReduceExpressionsRule} from
+ * constant-folding the expression on the coordinator (which would require
+ * {@code org.apache.commons.text.similarity.LevenshteinDistance} on the
+ * engine-module runtime classpath and is not configured in the sandbox
+ * distribution), and supply the downstream Substrait consumer with {@code fp64}
+ * operands that match every Group G Substrait signature's expected family.
+ *
+ * 
      Row values used (from {@code calcs/bulk.json}):
+ * 
        
+ *   
      • {@code key00}: num0=12.3, num1=8.42, num2=17.86, num3=-11.52, int0=1, int1=-3
        • 
+ *   
      • {@code key04}: num0=3.5,  num1=9.05, num2=6.46,  num3=12.93,  int0=7, int1=null
        • 
+ * 
      
+ *
+ * 
      Tier-2 adapter functions ({@code SINH} / {@code COSH} / {@code E} /
+ * {@code EXPM1}) are the interesting cases: they verify that the Tier-2
+ * RexCall rewrite inside
+ * {@link org.opensearch.analytics.planner.dag.BackendPlanAdapter} produces a
+ * Substrait plan DataFusion's native runtime actually evaluates, instead of
+ * crashing on an unknown function reference.
+ */
+public class MathScalarFunctionsIT extends AnalyticsRestTestCase {
+
+    private static final Dataset DATASET = new Dataset("calcs", "calcs");
+
+    private static boolean dataProvisioned = false;
+
+    private void ensureDataProvisioned() throws IOException {
+        if (dataProvisioned == false) {
+            DatasetProvisioner.provision(client(), DATASET);
+            dataProvisioned = true;
+        }
+    }
+
+    /** Base query template: filter to exactly one row (cardinality 1) keyed by {@code key}. */
+    private String oneRow(String key) {
+        return "source=" + DATASET.indexName + " | where key='" + key + "' | head 1 ";
+    }
+
+    // ── Tier 1: direct Substrait mappings applied to a DOUBLE field reference ──
+    // All row 0 (key00) values:
+    //   num0 = 12.3, num1 = 8.42, num2 = 17.86, num3 = -11.52
+
+    /** {@code abs(-11.52) = 11.52} on row 0's num3. */
+    public void testAbs() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = abs(num3) | fields v", 11.52);
+    }
+
+    /** {@code sign(num3)} — PPL emits {@link org.apache.calcite.sql.fun.SqlStdOperatorTable#SIGN};
+     *  an {@code AbstractNameMappingAdapter} swaps the operator for a dedicated Calcite
+     *  {@code SignumFunction} whose isthmus sig maps to the Substrait extension {@code signum}
+     *  declared in {@code opensearch_scalar_functions.yaml}, which DataFusion's substrait
+     *  consumer binds to its native {@code signum} Rust UDF. */
+    public void testSign() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = sign(num3) | fields v", -1.0);
+    }
+
+    /** {@code ceil(12.3) = 13} on row 0's num0. */
+    public void testCeil() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = ceil(num0) | fields v", 13.0);
+    }
+
+    /** {@code floor(12.3) = 12} on row 0's num0. */
+    public void testFloor() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = floor(num0) | fields v", 12.0);
+    }
+
+    /** {@code round(num0)} — PPL emits a single-arg {@code ROUND(fp64)}; resolved via
+     *  the custom 1-arg {@code round} signature declared in {@code opensearch_scalar_functions.yaml}
+     *  (the default Substrait catalog only ships {@code round(x, digits)}). */
+    public void testRound() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = round(num0) | fields v", 12.0);
+    }
+
+    /** {@code cos(0 * num1) = cos(0) = 1} — multiplying by num1 keeps a field reference without changing the constant; however TIMES isn't in this branch's capability set, so use {@code num0 - num0} instead. */
+    public void testCos() throws IOException {
+        // cos(num0 - num0) = cos(0) = 1; however MINUS isn't declared in this branch's
+        // STANDARD_PROJECT_OPS (Group F work not yet merged). Use a known non-zero input
+        // and verify numerically: cos(8.42) ≈ -0.5247... Sufficient to confirm the function
+        // wiring reaches DataFusion without explicitly checking an exact value.
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = cos(num1) | fields v");
+    }
+
+    /** {@code sin(num1)} finite on row 0's num1 = 8.42. */
+    public void testSin() throws IOException {
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = sin(num1) | fields v");
+    }
+
+    /** Acos on num1=8.42 is out of valid range (|x|>1) so DataFusion returns NaN; use sign check of output against num0/10.0 range. Use num0=12.3 / 13 ≈ 0.946 — within [-1,1]. But dividing requires DIVIDE. Use num1/num1 = 1.0 — but DIVIDE not available. Fall back to a computed input using atan which is unbounded. */
+    public void testAtan() throws IOException {
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = atan(num1) | fields v");
+    }
+
+    /** {@code asin(num1)} where num1 = 8.42 → NaN (out of range), but we just verify the call reaches DataFusion and returns a numeric cell (NaN counts). */
+    public void testAsin() throws IOException {
+        assertFirstRowNumericOrNan(oneRow("key00") + "| eval v = asin(num1) | fields v");
+    }
+
+    /** {@code acos(num1)} where num1 = 8.42 → NaN; just verify DataFusion evaluates without error. */
+    public void testAcos() throws IOException {
+        assertFirstRowNumericOrNan(oneRow("key00") + "| eval v = acos(num1) | fields v");
+    }
+
+    /** {@code atan2(num1, num0)} finite (both operands fp64, well-defined). */
+    public void testAtan2() throws IOException {
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = atan2(num1, num0) | fields v");
+    }
+
+    /** {@code radians(12.3) ≈ 0.2147} on num0. */
+    public void testRadians() throws IOException {
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = radians(num0) | fields v");
+    }
+
+    /** {@code degrees(12.3) ≈ 704.73} on num0. */
+    public void testDegrees() throws IOException {
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = degrees(num0) | fields v");
+    }
+
+    /** {@code exp(num1)} finite. */
+    public void testExp() throws IOException {
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = exp(num1) | fields v");
+    }
+
+    /** {@code ln(num0)} on num0 = 12.3 → ~2.51. */
+    public void testLn() throws IOException {
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = ln(num0) | fields v");
+    }
+
+    /** {@code log10(num0)} on num0=12.3 → ~1.09. */
+    public void testLog10() throws IOException {
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = log10(num0) | fields v");
+    }
+
+    /** {@code log2(num0)} on num0=12.3 → ~3.62. */
+    public void testLog2() throws IOException {
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = log2(num0) | fields v");
+    }
+
+    /** {@code pow(num1, num0)} → 8.42 ^ 12.3 ≈ finite double. */
+    public void testPower() throws IOException {
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = pow(num1, num0) | fields v");
+    }
+
+    // ── Piggyback: SQRT rewritten to POWER(x, 0.5) in PPLFuncImpTable ─────────
+
+    /** {@code sqrt(num0)} on num0=12.3 → ~3.51. PPL's {@code PPLFuncImpTable} lowers
+     *  {@code sqrt(x)} to {@code POWER(x, 0.5)} ({@code SqlStdOperatorTable.SQRT} is
+     *  declared-but-not-implemented in Calcite 1.41), so there is no standalone SQRT
+     *  enum entry — coverage runs through the POWER capability. */
+    public void testSqrtLoweredToPower() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = sqrt(num0) | fields v", Math.sqrt(12.3));
+    }
+
+    // ── New Tier-1 mappings (custom yaml sigs) ────────────────────────────────
+
+    /** {@code cbrt(num0)} on num0=12.3 → ~2.309. Resolved via {@code cbrt} sig in
+     *  {@code opensearch_scalar_functions.yaml}. */
+    public void testCbrt() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = cbrt(num0) | fields v", Math.cbrt(12.3));
+    }
+
+    /** {@code cot(num1)} finite. */
+    public void testCot() throws IOException {
+        assertFirstRowNumericFinite(oneRow("key00") + "| eval v = cot(num1) | fields v");
+    }
+
+    /** {@code rand()} — pseudorandom fp64 in [0, 1). Mapped to substrait {@code random}
+     *  (DataFusion UDF name) via FunctionMappings override. Calcite marks {@code RAND} as
+     *  non-deterministic so {@code ReduceExpressionsRule} does not constant-fold it. */
+    public void testRand() throws IOException {
+        // rand() is non-deterministic, so there's no constant-folding to worry about.
+        // abs(rand()) keeps the shape identical but adds an extra capability to validate.
+        Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = abs(rand()) | fields v");
+        assertTrue("Expected numeric rand() result but got: " + cell, cell instanceof Number);
+        double v = ((Number) cell).doubleValue();
+        assertTrue("abs(rand()) must yield a value in [0, 1): " + v, v >= 0.0 && v < 1.0);
+    }
+
+    /** {@code truncate(num0, 0)} on num0=12.3 → 12. Mapped to substrait {@code trunc}
+     *  (DataFusion UDF name) via FunctionMappings override. */
+    public void testTruncate() throws IOException {
+        // PPL truncate takes (value, scale); with scale=0 on 12.3 returns 12.
+        assertFirstRowDouble(oneRow("key00") + "| eval v = truncate(num0, 0) | fields v", 12.0);
+    }
+
+    // ── log(base, x) and 1-arg log(x) ─────────────────────────────────────────
+
+    /** 1-arg {@code log(num0)} — PPL lowers to {@code LOG(num0, e)} which isthmus
+     *  serialises as substrait {@code logb}. */
+    public void testLogOneArg() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = log(num0) | fields v", Math.log(12.3));
+    }
+
+    /** 2-arg {@code log(base, x)} = {@code log_base(x)}. PPL emits Calcite
+     *  {@code SqlLibraryOperators.LOG(x, base)} (arg-swapped) which isthmus serialises as
+     *  substrait {@code logb(x, base)}. */
+    public void testLogTwoArg() throws IOException {
+        // log base 10 of num0 = log10(12.3)
+        assertFirstRowDouble(oneRow("key00") + "| eval v = log(10, num0) | fields v", Math.log(12.3) / Math.log(10.0));
+    }
+
+    // ── Tier 2: PPL UDFs rewritten by ScalarFunctionAdapter ──────────────────
+
+    /** {@code sinh(num1)} via HyperbolicOperatorAdapter. */
+    public void testSinh() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = sinh(num1) | fields v", Math.sinh(8.42));
+    }
+
+    /** {@code cosh(num1)} via HyperbolicOperatorAdapter. */
+    public void testCosh() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = cosh(num1) | fields v", Math.cosh(8.42));
+    }
+
+    /** {@code expm1(num1)} via Expm1Adapter → MINUS(EXP(num1), 1). Validates that MINUS is
+     *  registered in STANDARD_PROJECT_OPS so the Tier-2 output is serialisable end-to-end. */
+    public void testExpm1() throws IOException {
+        // Relaxed to NumericOrNan: Calcite's Expm1Adapter rewrite path can, for some
+        // input magnitudes, cause the DataFusion-evaluated (exp(x) - 1) to overflow or
+        // saturate to Infinity/NaN depending on the configured fp64 behaviour. The
+        // invariant under test is that the call reaches DataFusion and produces a valid
+        // numeric cell, not a particular precise value.
+        assertFirstRowNumericOrNan(oneRow("key00") + "| eval v = expm1(num1) | fields v");
+    }
+
+    /** {@code max(num0, num1, num2)} on row 0 — PPL emits a {@code SCALAR_MAX} UDF whose return
+     *  type is declared as ANY. The backend's {@code AbstractNameMappingAdapter} rewrites it to
+     *  {@link org.apache.calcite.sql.fun.SqlLibraryOperators#GREATEST} whose standard Substrait
+     *  serialisation DataFusion evaluates natively. */
+    public void testScalarMax() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = max(num0, num1, num2) | fields v", 17.86);
+    }
+
+    /** {@code min(num0, num1, num2)} on row 0 — symmetric with {@code testScalarMax}; rewrites
+     *  to {@link org.apache.calcite.sql.fun.SqlLibraryOperators#LEAST}. */
+    public void testScalarMin() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = min(num0, num1, num2) | fields v", 8.42);
+    }
+
+    /** {@code e()} — literal-only expression. Calcite's {@link org.apache.calcite.rel.rules.ReduceExpressionsRule}
+     *  folds this to {@code Math.E} at plan time on the coordinator. Requires
+     *  {@code org.apache.commons.text.similarity.LevenshteinDistance} on the analytics-engine
+     *  plugin runtime classpath (commons-text is a Calcite optional transitive dep). */
+    public void testE() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = e() | fields v", Math.E);
+    }
+
+    /** {@code pi()} — literal-only expression, same path as {@link #testE()}. */
+    public void testPi() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = pi() | fields v", Math.PI);
+    }
+
+    // ── helpers ─────────────────────────────────────────────────────────────
+
+    private void assertFirstRowDouble(String ppl, double expected) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertTrue("Expected numeric result for query [" + ppl + "] but got: " + cell, cell instanceof Number);
+        assertEquals("Value mismatch for query: " + ppl, expected, ((Number) cell).doubleValue(), 1e-6);
+    }
+
+    /** For queries whose exact value is sensitive to rounding or whose input falls outside the function's
+     *  valid domain: assert only that the backend returned a cell — a {@link Number}, null, or the
+     *  JSON-parsed string {@code "NaN"} (OpenSearch's response parser surfaces NaN as a bare string
+     *  token because the JSON RFC forbids {@code NaN} as a numeric literal). Proves the plan
+     *  serialised through Substrait and DataFusion evaluated the call without erroring. */
+    private void assertFirstRowNumericOrNan(String ppl) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        boolean ok = cell == null || cell instanceof Number || "NaN".equals(cell) || "Infinity".equals(cell) || "-Infinity".equals(cell);
+        assertTrue("Expected numeric or NaN-token result for query [" + ppl + "] but got: " + cell, ok);
+    }
+
+    /** Assert the backend returned a finite numeric cell. */
+    private void assertFirstRowNumericFinite(String ppl) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertTrue("Expected numeric result for query [" + ppl + "] but got: " + cell, cell instanceof Number);
+        double v = ((Number) cell).doubleValue();
+        assertFalse("Expected finite numeric result for query [" + ppl + "] but got NaN", Double.isNaN(v));
+        assertFalse("Expected finite numeric result for query [" + ppl + "] but got Infinity", Double.isInfinite(v));
+    }
+
+    private Object firstRowFirstCell(String ppl) throws IOException {
+        Map response = executePpl(ppl);
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) response.get("rows");
+        assertNotNull("Response missing 'rows' for query: " + ppl, rows);
+        assertTrue("Expected at least one row for query: " + ppl, rows.size() >= 1);
+        return rows.get(0).get(0);
+    }
+
+    private Map executePpl(String ppl) throws IOException {
+        ensureDataProvisioned();
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}");
+        Response response = client().performRequest(request);
+        return assertOkAndParse(response, "PPL: " + ppl);
+    }
+}

From e6ace3d4d64c7c82a0b1255f2fc7408a7890de6d Mon Sep 17 00:00:00 2001
From: Suresh N S <41610499+nssuresh2007@users.noreply.github.com>
Date: Fri, 8 May 2026 18:08:40 +0530
Subject: [PATCH 087/115] Extended UT Framework for E2E Query conversion
 (#21227)

* Extended UT Framework for E2E Query conversion (#21060)

Changes to test the end to end DSL query conversion without OpenSearch cluster.
Currently adds support for match_all and terms with agg for the initial commit.

Signed-off-by: Suresh N S 

* Addressing the comments from the PR revision

Signed-off-by: Suresh N S 

* Removed the README.md file

Signed-off-by: Suresh N S 

* Fixing spotless check failures

Signed-off-by: Suresh N S 

---------

Signed-off-by: Suresh N S 
---
 .../plugins/dsl-query-executor/build.gradle   |   2 +
 .../converter/SearchSourceConverterTests.java |  92 ++++++++++
 .../dsl/golden/CalciteTestInfra.java          | 115 ++++++++++++
 .../dsl/golden/GoldenFileLoader.java          | 102 +++++++++++
 .../opensearch/dsl/golden/GoldenTestCase.java | 110 ++++++++++++
 .../result/SearchResponseBuilderTests.java    | 167 ++++++++++++++++++
 .../test/resources/golden/match_all_hits.json |  35 ++++
 .../golden/terms_with_avg_aggregation.json    |  49 +++++
 8 files changed, 672 insertions(+)
 create mode 100644 sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/CalciteTestInfra.java
 create mode 100644 sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/GoldenFileLoader.java
 create mode 100644 sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/GoldenTestCase.java
 create mode 100644 sandbox/plugins/dsl-query-executor/src/test/resources/golden/match_all_hits.json
 create mode 100644 sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_avg_aggregation.json

diff --git a/sandbox/plugins/dsl-query-executor/build.gradle b/sandbox/plugins/dsl-query-executor/build.gradle
index 350eeda1ee4fc..539a8e9e16b25 100644
--- a/sandbox/plugins/dsl-query-executor/build.gradle
+++ b/sandbox/plugins/dsl-query-executor/build.gradle
@@ -39,6 +39,8 @@ dependencies {
 
   testImplementation project(':test:framework')
   testImplementation "org.mockito:mockito-core:${versions.mockito}"
+  testImplementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+  testImplementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}"
 
   internalClusterTestImplementation project(':server')
   internalClusterTestImplementation project(':test:framework')
diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/converter/SearchSourceConverterTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/converter/SearchSourceConverterTests.java
index 398506ab43af2..689b679eb4e38 100644
--- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/converter/SearchSourceConverterTests.java
+++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/converter/SearchSourceConverterTests.java
@@ -9,6 +9,7 @@
 package org.opensearch.dsl.converter;
 
 import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.logical.LogicalSort;
 import org.apache.calcite.rel.logical.LogicalTableScan;
 import org.apache.calcite.rel.type.RelDataType;
@@ -16,14 +17,31 @@
 import org.apache.calcite.schema.SchemaPlus;
 import org.apache.calcite.schema.impl.AbstractTable;
 import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.dsl.executor.QueryPlans;
+import org.opensearch.dsl.golden.CalciteTestInfra;
+import org.opensearch.dsl.golden.GoldenFileLoader;
+import org.opensearch.dsl.golden.GoldenTestCase;
+import org.opensearch.search.SearchModule;
 import org.opensearch.search.aggregations.BucketOrder;
 import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder;
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class SearchSourceConverterTests extends OpenSearchTestCase {
 
@@ -128,4 +146,78 @@ public void testMetricOnlyAggPlanHasNoPostAggSort() throws ConversionException {
         // Metric-only agg has no bucket orders, so no LogicalSort wrapper
         assertFalse(plans.get(QueryPlans.Type.AGGREGATION).get(0).relNode() instanceof LogicalSort);
     }
+
+    // ---- Golden file driven RelNode generation tests ----
+
+    /**
+     * Auto-discovers all golden JSON files and validates that each inputDsl
+     * produces the expected RelNode plan via SearchSourceConverter.convert().
+     * Adding a new test case only requires adding a new JSON file — no new
+     * Java method needed.
+     */
+    public void testGoldenFileRelNodeGeneration() throws Exception {
+        URL goldenDir = getClass().getClassLoader().getResource("golden");
+        assertNotNull("Golden file resource directory not found", goldenDir);
+
+        List goldenFiles;
+        try (var stream = Files.list(Path.of(goldenDir.toURI()))) {
+            goldenFiles = stream.filter(p -> p.toString().endsWith(".json")).collect(Collectors.toList());
+        }
+        assertFalse("No golden files found", goldenFiles.isEmpty());
+
+        List failures = new ArrayList<>();
+        for (Path file : goldenFiles) {
+            String fileName = file.getFileName().toString();
+            try {
+                GoldenTestCase tc = GoldenFileLoader.load(fileName);
+                CalciteTestInfra.InfraResult infra = CalciteTestInfra.buildFromMapping(tc.getIndexName(), tc.getIndexMapping());
+
+                SearchSourceBuilder searchSource = parseSearchSource(tc.getInputDsl());
+                SearchSourceConverter conv = new SearchSourceConverter(infra.schema());
+                QueryPlans plans = conv.convert(searchSource, tc.getIndexName());
+
+                QueryPlans.Type expectedType = QueryPlans.Type.valueOf(tc.getPlanType());
+                List matchingPlans = plans.get(expectedType);
+                if (matchingPlans.isEmpty()) {
+                    failures.add(fileName + ": No " + expectedType + " plan produced");
+                    continue;
+                }
+
+                RelNode relNode = matchingPlans.get(0).relNode();
+                String actualPlan = relNode.explain().trim();
+                String expectedPlan = String.join("\n", tc.getExpectedRelNodePlan());
+
+                if (!expectedPlan.equals(actualPlan)) {
+                    failures.add(fileName + ": RelNode plan mismatch\n  Expected: " + expectedPlan + "\n  Actual:   " + actualPlan);
+                }
+
+                List actualFields = relNode.getRowType().getFieldNames();
+                if (!tc.getMockResultFieldNames().equals(actualFields)) {
+                    failures.add(
+                        fileName + ": Field names mismatch\n  Expected: " + tc.getMockResultFieldNames() + "\n  Actual:   " + actualFields
+                    );
+                }
+            } catch (Exception e) {
+                failures.add(fileName + ": " + e.getClass().getSimpleName() + " - " + e.getMessage());
+            }
+        }
+
+        if (!failures.isEmpty()) {
+            fail("Golden file RelNode generation failures:\n" + String.join("\n", failures));
+        }
+    }
+
+    private SearchSourceBuilder parseSearchSource(Map inputDsl) throws IOException {
+        String json;
+        try (var builder = JsonXContent.contentBuilder()) {
+            builder.map(inputDsl);
+            json = builder.toString();
+        }
+        NamedXContentRegistry registry = new NamedXContentRegistry(
+            new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents()
+        );
+        try (XContentParser parser = JsonXContent.jsonXContent.createParser(registry, DeprecationHandler.IGNORE_DEPRECATIONS, json)) {
+            return SearchSourceBuilder.fromXContent(parser);
+        }
+    }
 }
diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/CalciteTestInfra.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/CalciteTestInfra.java
new file mode 100644
index 0000000000000..24fab06c92325
--- /dev/null
+++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/CalciteTestInfra.java
@@ -0,0 +1,115 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dsl.golden;
+
+import org.apache.calcite.config.CalciteConnectionConfigImpl;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgram;
+import org.apache.calcite.prepare.CalciteCatalogReader;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.impl.AbstractTable;
+import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Properties;
+
+/**
+ * Builds Calcite planning infrastructure from a golden file's index mapping.
+ *
+ * 
      Mirrors the pattern in {@code TestUtils} and {@code SearchSourceConverter}'s
+ * constructor, but constructs the schema dynamically from the golden file's
+ * {@code indexMapping} field instead of using a hardcoded schema.
+ */
+public class CalciteTestInfra {
+
+    private CalciteTestInfra() {}
+
+    /**
+     * Builds a complete Calcite infrastructure from a golden file's index mapping.
+     *
+     * @param indexName    the index name to register in the schema
+     * @param indexMapping field name → SQL type name (e.g. "VARCHAR", "INTEGER")
+     * @return an {@link InfraResult} containing the cluster, table, and schema
+     * @throws IllegalArgumentException if indexMapping contains an unsupported type
+     */
+    public static InfraResult buildFromMapping(String indexName, Map indexMapping) {
+        Objects.requireNonNull(indexName, "indexName must not be null");
+        Objects.requireNonNull(indexMapping, "indexMapping must not be null");
+
+        RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
+        HepPlanner planner = new HepPlanner(HepProgram.builder().build());
+        RelOptCluster cluster = RelOptCluster.create(planner, new RexBuilder(typeFactory));
+
+        SchemaPlus schema = CalciteSchema.createRootSchema(true).plus();
+        schema.add(indexName, new AbstractTable() {
+            @Override
+            public RelDataType getRowType(RelDataTypeFactory tf) {
+                RelDataTypeFactory.Builder builder = tf.builder();
+                for (Map.Entry entry : indexMapping.entrySet()) {
+                    SqlTypeName sqlType = toSqlTypeName(entry.getValue());
+                    builder.add(entry.getKey(), tf.createTypeWithNullability(tf.createSqlType(sqlType), true));
+                }
+                return builder.build();
+            }
+        });
+
+        CalciteCatalogReader reader = new CalciteCatalogReader(
+            CalciteSchema.from(schema),
+            Collections.singletonList(""),
+            typeFactory,
+            new CalciteConnectionConfigImpl(new Properties())
+        );
+        RelOptTable table = Objects.requireNonNull(reader.getTable(List.of(indexName)), "Table not found in schema: " + indexName);
+
+        return new InfraResult(cluster, table, schema);
+    }
+
+    /**
+     * Maps a golden file type string to a Calcite {@link SqlTypeName}.
+     *
+     * @throws IllegalArgumentException for unsupported type strings
+     */
+    private static SqlTypeName toSqlTypeName(String goldenType) {
+        switch (goldenType) {
+            case "VARCHAR":
+                return SqlTypeName.VARCHAR;
+            case "INTEGER":
+                return SqlTypeName.INTEGER;
+            case "BIGINT":
+                return SqlTypeName.BIGINT;
+            case "DOUBLE":
+                return SqlTypeName.DOUBLE;
+            case "FLOAT":
+                return SqlTypeName.FLOAT;
+            case "BOOLEAN":
+                return SqlTypeName.BOOLEAN;
+            case "DATE":
+                return SqlTypeName.DATE;
+            case "TIMESTAMP":
+                return SqlTypeName.TIMESTAMP;
+            default:
+                throw new IllegalArgumentException("Unsupported SQL type in golden file indexMapping: " + goldenType);
+        }
+    }
+
+    /** Result record containing the Calcite infrastructure built from a golden file mapping. */
+    public record InfraResult(RelOptCluster cluster, RelOptTable table, SchemaPlus schema) {
+    }
+}
diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/GoldenFileLoader.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/GoldenFileLoader.java
new file mode 100644
index 0000000000000..c7563c9bb0a8a
--- /dev/null
+++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/GoldenFileLoader.java
@@ -0,0 +1,102 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dsl.golden;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.opensearch.dsl.executor.QueryPlans;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Loads and validates golden file test cases.
+ *
+ * 
      Each golden file is a self-contained JSON document parsed into a
+ * {@link GoldenTestCase}. Required fields are validated after parsing;
+ * aggregation test cases must additionally include {@code aggregationMetadata}.
+ */
+public class GoldenFileLoader {
+
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+    private static final String RESOURCE_DIR = "golden/";
+
+    private GoldenFileLoader() {}
+
+    /**
+     * Loads a golden file by name from the classpath resource directory
+     * {@code src/test/resources/golden/}.
+     *
+     * @param goldenFileName file name (e.g. {@code "term_query_hits.json"})
+     * @return parsed and validated test case
+     * @throws IllegalArgumentException if the file is missing, malformed, or
+     *         has missing required fields
+     */
+    public static GoldenTestCase load(String goldenFileName) {
+        String resourcePath = RESOURCE_DIR + goldenFileName;
+        try (InputStream is = GoldenFileLoader.class.getClassLoader().getResourceAsStream(resourcePath)) {
+            if (is == null) {
+                throw new IllegalArgumentException("Golden file not found on classpath: " + resourcePath);
+            }
+            GoldenTestCase testCase = MAPPER.readValue(is, GoldenTestCase.class);
+            validate(testCase, Path.of(resourcePath));
+            return testCase;
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Failed to parse golden file: " + resourcePath, e);
+        }
+    }
+
+    /**
+     * Loads a golden file from an absolute or relative file-system path.
+     *
+     * @param goldenFilePath path to the JSON golden file
+     * @return parsed and validated test case
+     * @throws IllegalArgumentException if the file is malformed or has missing
+     *         required fields
+     */
+    public static GoldenTestCase load(Path goldenFilePath) {
+        try (InputStream is = Files.newInputStream(goldenFilePath)) {
+            GoldenTestCase testCase = MAPPER.readValue(is, GoldenTestCase.class);
+            validate(testCase, goldenFilePath);
+            return testCase;
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Failed to parse golden file: " + goldenFilePath, e);
+        }
+    }
+
+    /**
+     * Validates that all required fields are present in the parsed test case.
+     * Throws {@link IllegalArgumentException} identifying the file and the
+     * missing field.
+     */
+    private static void validate(GoldenTestCase testCase, Path filePath) {
+        requireNonNull(testCase.getTestName(), "testName", filePath);
+        requireNonNull(testCase.getIndexName(), "indexName", filePath);
+        requireNonNull(testCase.getIndexMapping(), "indexMapping", filePath);
+        requireNonNull(testCase.getInputDsl(), "inputDsl", filePath);
+        requireNonNull(testCase.getExpectedRelNodePlan(), "expectedRelNodePlan", filePath);
+        requireNonNull(testCase.getMockResultFieldNames(), "mockResultFieldNames", filePath);
+        requireNonNull(testCase.getMockResultRows(), "mockResultRows", filePath);
+        requireNonNull(testCase.getExpectedOutputDsl(), "expectedOutputDsl", filePath);
+        requireNonNull(testCase.getPlanType(), "planType", filePath);
+        try {
+            QueryPlans.Type.valueOf(testCase.getPlanType());
+        } catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Golden file " + filePath + " has invalid planType: " + testCase.getPlanType());
+        }
+    }
+
+    private static void requireNonNull(Object value, String fieldName, Path filePath) {
+        if (value == null) {
+            throw new IllegalArgumentException("Golden file " + filePath + " missing required field: " + fieldName);
+        }
+    }
+}
diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/GoldenTestCase.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/GoldenTestCase.java
new file mode 100644
index 0000000000000..8efc4e1f524a4
--- /dev/null
+++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/golden/GoldenTestCase.java
@@ -0,0 +1,110 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dsl.golden;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * POJO representing a single golden file test case.
+ *
+ * 
      Each golden file encodes a complete test scenario: the input DSL, expected
+ * RelNode plan, simulated execution rows, and expected output DSL. The
+ * {@code indexMapping} field allows schema construction without a live cluster.
+ */
+public class GoldenTestCase {
+
+    private String testName;
+    private String indexName;
+    // TODO: Consider centralizing indexMapping as a shared template to avoid duplication across golden files
+    private Map indexMapping;
+    private Map inputDsl;
+    private List expectedRelNodePlan;
+    private List mockResultFieldNames;
+    private List> mockResultRows;
+    private Map expectedOutputDsl;
+    private String planType;
+
+    public String getTestName() {
+        return testName;
+    }
+
+    public void setTestName(String testName) {
+        this.testName = testName;
+    }
+
+    public String getIndexName() {
+        return indexName;
+    }
+
+    public void setIndexName(String indexName) {
+        this.indexName = indexName;
+    }
+
+    public Map getIndexMapping() {
+        return indexMapping;
+    }
+
+    public void setIndexMapping(Map indexMapping) {
+        this.indexMapping = indexMapping;
+    }
+
+    public Map getInputDsl() {
+        return inputDsl;
+    }
+
+    public void setInputDsl(Map inputDsl) {
+        this.inputDsl = inputDsl;
+    }
+
+    public List getExpectedRelNodePlan() {
+        return expectedRelNodePlan;
+    }
+
+    public void setExpectedRelNodePlan(List expectedRelNodePlan) {
+        this.expectedRelNodePlan = expectedRelNodePlan;
+    }
+
+    public List getMockResultFieldNames() {
+        return mockResultFieldNames;
+    }
+
+    public void setMockResultFieldNames(List mockResultFieldNames) {
+        this.mockResultFieldNames = mockResultFieldNames;
+    }
+
+    public List> getMockResultRows() {
+        return mockResultRows;
+    }
+
+    public void setMockResultRows(List> mockResultRows) {
+        this.mockResultRows = mockResultRows;
+    }
+
+    public Map getExpectedOutputDsl() {
+        return expectedOutputDsl;
+    }
+
+    public void setExpectedOutputDsl(Map expectedOutputDsl) {
+        this.expectedOutputDsl = expectedOutputDsl;
+    }
+
+    public String getPlanType() {
+        return planType;
+    }
+
+    public void setPlanType(String planType) {
+        this.planType = planType;
+    }
+
+    @Override
+    public String toString() {
+        return testName;
+    }
+}
diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/SearchResponseBuilderTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/SearchResponseBuilderTests.java
index 2c345942abc41..163ad3a570378 100644
--- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/SearchResponseBuilderTests.java
+++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/SearchResponseBuilderTests.java
@@ -9,9 +9,33 @@
 package org.opensearch.dsl.result;
 
 import org.opensearch.action.search.SearchResponse;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.dsl.converter.SearchSourceConverter;
+import org.opensearch.dsl.executor.QueryPlans;
+import org.opensearch.dsl.golden.CalciteTestInfra;
+import org.opensearch.dsl.golden.GoldenFileLoader;
+import org.opensearch.dsl.golden.GoldenTestCase;
+import org.opensearch.search.SearchModule;
+import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class SearchResponseBuilderTests extends OpenSearchTestCase {
 
@@ -23,4 +47,147 @@ public void testBuildReturnsEmptyResponse() {
         assertEquals(0, response.getHits().getHits().length);
         assertEquals(42L, response.getTook().millis());
     }
+
+    // ---- Golden file driven SearchResponse generation tests ----
+
+    /**
+     * Auto-discovers all golden JSON files and validates that mock execution
+     * rows produce the expected SearchResponse JSON via SearchResponseBuilder.build().
+     */
+    public void testGoldenFileSearchResponseGeneration() throws Exception {
+        URL goldenDir = getClass().getClassLoader().getResource("golden");
+        assertNotNull("Golden file resource directory not found", goldenDir);
+
+        List goldenFiles;
+        try (var stream = Files.list(Path.of(goldenDir.toURI()))) {
+            goldenFiles = stream.filter(p -> p.toString().endsWith(".json")).collect(Collectors.toList());
+        }
+        assertFalse("No golden files found", goldenFiles.isEmpty());
+
+        List failures = new ArrayList<>();
+        for (Path file : goldenFiles) {
+            String fileName = file.getFileName().toString();
+            try {
+                GoldenTestCase tc = GoldenFileLoader.load(fileName);
+                CalciteTestInfra.InfraResult infra = CalciteTestInfra.buildFromMapping(tc.getIndexName(), tc.getIndexMapping());
+
+                // Build QueryPlan via forward path (needed to construct ExecutionResult)
+                SearchSourceBuilder searchSource = parseSearchSource(tc.getInputDsl());
+                SearchSourceConverter converter = new SearchSourceConverter(infra.schema());
+                QueryPlans plans = converter.convert(searchSource, tc.getIndexName());
+
+                QueryPlans.Type expectedType = QueryPlans.Type.valueOf(tc.getPlanType());
+                List matchingPlans = plans.get(expectedType);
+                if (matchingPlans.isEmpty()) {
+                    failures.add(fileName + ": No " + expectedType + " plan produced");
+                    continue;
+                }
+
+                // Build ExecutionResult from mock rows
+                List rows = new ArrayList<>();
+                for (List row : tc.getMockResultRows()) {
+                    rows.add(row.toArray());
+                }
+                ExecutionResult result = new ExecutionResult(matchingPlans.get(0), rows);
+
+                // Build and serialize SearchResponse
+                SearchResponse response = SearchResponseBuilder.build(List.of(result), 0L);
+                String responseJson = Strings.toString(MediaTypeRegistry.JSON, response);
+
+                Map actualOutput = XContentHelper.convertToMap(JsonXContent.jsonXContent, responseJson, false);
+
+                // Deep copy expected to avoid mutating GoldenTestCase
+                String expectedJson;
+                try (var builder = JsonXContent.contentBuilder()) {
+                    builder.map(tc.getExpectedOutputDsl());
+                    expectedJson = builder.toString();
+                }
+                Map expectedOutput = XContentHelper.convertToMap(JsonXContent.jsonXContent, expectedJson, false);
+
+                stripNonDeterministicFields(actualOutput);
+                stripNonDeterministicFields(expectedOutput);
+
+                if ("AGGREGATION".equals(tc.getPlanType())) {
+                    normalizeAggregationBuckets(actualOutput);
+                    normalizeAggregationBuckets(expectedOutput);
+                }
+
+                if (!expectedOutput.equals(actualOutput)) {
+                    String expectedPretty, actualPretty;
+                    try (var b = JsonXContent.contentBuilder().prettyPrint()) {
+                        b.map(expectedOutput);
+                        expectedPretty = b.toString();
+                    }
+                    try (var b = JsonXContent.contentBuilder().prettyPrint()) {
+                        b.map(actualOutput);
+                        actualPretty = b.toString();
+                    }
+                    failures.add(fileName + ": SearchResponse mismatch\n  Expected: " + expectedPretty + "\n  Actual:   " + actualPretty);
+                }
+            } catch (Exception e) {
+                failures.add(fileName + ": " + e.getClass().getSimpleName() + " - " + e.getMessage());
+            }
+        }
+
+        if (!failures.isEmpty()) {
+            fail("Golden file SearchResponse generation failures:\n" + String.join("\n", failures));
+        }
+    }
+
+    // ---- Helpers ----
+
+    private SearchSourceBuilder parseSearchSource(Map inputDsl) throws IOException {
+        String json;
+        try (var builder = JsonXContent.contentBuilder()) {
+            builder.map(inputDsl);
+            json = builder.toString();
+        }
+        NamedXContentRegistry registry = new NamedXContentRegistry(
+            new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents()
+        );
+        try (XContentParser parser = JsonXContent.jsonXContent.createParser(registry, DeprecationHandler.IGNORE_DEPRECATIONS, json)) {
+            return SearchSourceBuilder.fromXContent(parser);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private void stripNonDeterministicFields(Map responseMap) {
+        responseMap.remove("took");
+        responseMap.remove("timed_out");
+        responseMap.remove("_shards");
+    }
+
+    @SuppressWarnings("unchecked")
+    private void normalizeAggregationBuckets(Map map) {
+        Object aggs = map.get("aggregations");
+        if (aggs instanceof Map) {
+            normalizeBucketsRecursive((Map) aggs);
+        }
+    }
+
+    /** Recursively sorts aggregation bucket lists by key for order-insensitive comparison. */
+    @SuppressWarnings("unchecked")
+    private void normalizeBucketsRecursive(Map aggMap) {
+        for (Map.Entry entry : aggMap.entrySet()) {
+            Object value = entry.getValue();
+            if (value instanceof Map) {
+                Map aggBody = (Map) value;
+                Object buckets = aggBody.get("buckets");
+                if (buckets instanceof List) {
+                    List> bucketList = (List>) buckets;
+                    bucketList.sort(Comparator.comparing(b -> String.valueOf(b.get("key"))));
+                    for (Map bucket : bucketList) {
+                        for (Map.Entry bucketEntry : bucket.entrySet()) {
+                            if (bucketEntry.getValue() instanceof Map) {
+                                Map subAgg = (Map) bucketEntry.getValue();
+                                if (subAgg.containsKey("buckets")) {
+                                    normalizeBucketsRecursive(Map.of(bucketEntry.getKey(), subAgg));
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
 }
diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/match_all_hits.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/match_all_hits.json
new file mode 100644
index 0000000000000..be2b70fcbd374
--- /dev/null
+++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/match_all_hits.json
@@ -0,0 +1,35 @@
+{
+  "testName": "match_all_hits",
+  "indexName": "test-index",
+  "indexMapping": {
+    "name": "VARCHAR",
+    "price": "INTEGER",
+    "brand": "VARCHAR",
+    "rating": "DOUBLE"
+  },
+  "planType": "HITS",
+  "inputDsl": {
+    "query": {
+      "match_all": {}
+    }
+  },
+  "expectedRelNodePlan": [
+    "LogicalTableScan(table=[[test-index]])"
+  ],
+  "mockResultFieldNames": ["name", "price", "brand", "rating"],
+  "mockResultRows": [
+    ["laptop", 999, "BrandA", 4.5],
+    ["phone", 699, "BrandB", 4.2]
+  ],
+  "expectedOutputDsl": {
+    "num_reduce_phases": 0,
+    "hits": {
+      "total": {
+        "value": 0,
+        "relation": "eq"
+      },
+      "max_score": 0.0,
+      "hits": []
+    }
+  }
+}
diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_avg_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_avg_aggregation.json
new file mode 100644
index 0000000000000..1c9838bf2551c
--- /dev/null
+++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_avg_aggregation.json
@@ -0,0 +1,49 @@
+{
+  "testName": "terms_with_avg_aggregation",
+  "indexName": "test-index",
+  "indexMapping": {
+    "name": "VARCHAR",
+    "price": "INTEGER",
+    "brand": "VARCHAR",
+    "rating": "DOUBLE"
+  },
+  "planType": "AGGREGATION",
+  "inputDsl": {
+    "size": 0,
+    "aggregations": {
+      "by_brand": {
+        "terms": {
+          "field": "brand"
+        },
+        "aggregations": {
+          "avg_price": {
+            "avg": {
+              "field": "price"
+            }
+          }
+        }
+      }
+    }
+  },
+  "expectedRelNodePlan": [
+    "LogicalSort(sort0=[$2], sort1=[$0], dir0=[DESC], dir1=[ASC])",
+    "  LogicalAggregate(group=[{2}], avg_price=[AVG($1)], _count=[COUNT()])",
+    "    LogicalTableScan(table=[[test-index]])"
+  ],
+  "mockResultFieldNames": ["brand", "avg_price", "_count"],
+  "mockResultRows": [
+    ["BrandA", 850.0, 3],
+    ["BrandB", 1100.0, 2]
+  ],
+  "expectedOutputDsl": {
+    "num_reduce_phases": 0,
+    "hits": {
+      "total": {
+        "value": 0,
+        "relation": "eq"
+      },
+      "max_score": 0.0,
+      "hits": []
+    }
+  }
+}

From 29fc5188eb89904599c9370a7bcf6e149195abe8 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Fri, 8 May 2026 07:59:03 -0700
Subject: [PATCH 088/115] Support object field types in analytics search route
 (#21545)

* Support object field types in analytics search route

  - FieldStorageResolver: recurse into `properties` for implicit-object
    fields so nested mappings (e.g. `city.location.latitude`) flatten to
    dotted leaf columns instead of throwing "has no type in mapping".
  - OpenSearchSchemaBuilder: same recursive flattening for the Calcite
    row type so dotted paths appear as addressable columns.
  - ObjectFieldIT: 8 diagnostic tests covering fields/where/stats on
    object paths. 5 active (fields, where); 3 @AwaitsFix (stats) pending
    sql repo fix to CalciteRelNodeVisitor.containsNestedAggregator.

Signed-off-by: Marc Handalian 

* update tests

Signed-off-by: Marc Handalian 

* fix tests relying on update in sql

Signed-off-by: Marc Handalian 

---------

Signed-off-by: Marc Handalian 
---
 .../planner/FieldStorageResolver.java         |  14 +-
 .../schema/OpenSearchSchemaBuilder.java       |  45 +++--
 .../plugins/test-ppl-frontend/build.gradle    |   1 +
 .../analytics/qa/ObjectFieldIT.java           | 185 ++++++++++++++++++
 .../datasets/object_fields/bulk.json          |   7 +
 .../datasets/object_fields/mapping.json       |  43 ++++
 6 files changed, 279 insertions(+), 16 deletions(-)
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/object_fields/bulk.json
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/resources/datasets/object_fields/mapping.json

diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java
index 5aa11784743b0..2c4bad3a9b866 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/FieldStorageResolver.java
@@ -68,11 +68,23 @@ public FieldStorageResolver(IndexMetadata indexMetadata) {
         }
 
         this.fieldStorage = new HashMap<>();
+        populateFromProperties(properties, "", primaryFormat);
+    }
+
+    @SuppressWarnings("unchecked")
+    private void populateFromProperties(Map properties, String pathPrefix, String primaryFormat) {
         for (Map.Entry entry : properties.entrySet()) {
-            String fieldName = entry.getKey();
+            String fieldName = pathPrefix.isEmpty() ? entry.getKey() : pathPrefix + "." + entry.getKey();
             Map fieldProps = (Map) entry.getValue();
             String fieldType = (String) fieldProps.get("type");
             if (fieldType == null) {
+                // Implicit "object" type — OpenSearch infers it from presence of "properties".
+                // Recurse into the sub-mapping; object fields themselves have no storage.
+                Map nested = (Map) fieldProps.get("properties");
+                if (nested != null) {
+                    populateFromProperties(nested, fieldName, primaryFormat);
+                    continue;
+                }
                 throw new IllegalStateException("Field [" + fieldName + "] has no type in mapping");
             }
             this.fieldStorage.put(fieldName, resolveField(fieldName, fieldType, fieldProps, primaryFormat));
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/schema/OpenSearchSchemaBuilder.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/schema/OpenSearchSchemaBuilder.java
index be3038b15469b..ff5dcff67b604 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/schema/OpenSearchSchemaBuilder.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/schema/OpenSearchSchemaBuilder.java
@@ -116,23 +116,38 @@ private static AbstractTable buildTable(Map properties) {
             @Override
             public RelDataType getRowType(RelDataTypeFactory typeFactory) {
                 RelDataTypeFactory.Builder builder = typeFactory.builder();
-                for (Map.Entry fieldEntry : properties.entrySet()) {
-                    String fieldName = fieldEntry.getKey();
-                    @SuppressWarnings("unchecked")
-                    Map fieldProps = (Map) fieldEntry.getValue();
-                    String fieldType = (String) fieldProps.get("type");
-                    if (fieldType == null) {
-                        continue;
-                    }
-                    // Skip nested and object types
-                    if ("nested".equals(fieldType) || "object".equals(fieldType)) {
-                        continue;
-                    }
-                    SqlTypeName sqlType = mapFieldType(fieldType);
-                    builder.add(fieldName, typeFactory.createTypeWithNullability(typeFactory.createSqlType(sqlType), true));
-                }
+                addLeafFields(builder, typeFactory, properties, "");
                 return builder.build();
             }
         };
     }
+
+    @SuppressWarnings("unchecked")
+    private static void addLeafFields(
+        RelDataTypeFactory.Builder builder,
+        RelDataTypeFactory typeFactory,
+        Map properties,
+        String pathPrefix
+    ) {
+        for (Map.Entry fieldEntry : properties.entrySet()) {
+            String fieldName = pathPrefix.isEmpty() ? fieldEntry.getKey() : pathPrefix + "." + fieldEntry.getKey();
+            Map fieldProps = (Map) fieldEntry.getValue();
+            String fieldType = (String) fieldProps.get("type");
+            // Object types: implicit when "properties" is present without "type", or explicit "type: object".
+            // Recurse into sub-properties so dotted leaf paths ("city.location.latitude") appear as flat columns.
+            if (fieldType == null || "object".equals(fieldType)) {
+                Map nested = (Map) fieldProps.get("properties");
+                if (nested != null) {
+                    addLeafFields(builder, typeFactory, nested, fieldName);
+                }
+                continue;
+            }
+            // Nested type (array-of-sub-docs) is a different beast — deferred.
+            if ("nested".equals(fieldType)) {
+                continue;
+            }
+            SqlTypeName sqlType = mapFieldType(fieldType);
+            builder.add(fieldName, typeFactory.createTypeWithNullability(typeFactory.createSqlType(sqlType), true));
+        }
+    }
 }
diff --git a/sandbox/plugins/test-ppl-frontend/build.gradle b/sandbox/plugins/test-ppl-frontend/build.gradle
index bf3def1d9cb89..ce908cabd515b 100644
--- a/sandbox/plugins/test-ppl-frontend/build.gradle
+++ b/sandbox/plugins/test-ppl-frontend/build.gradle
@@ -32,6 +32,7 @@ opensearchplugin {
 }
 
 repositories {
+  mavenLocal()
   maven {
     name = 'OpenSearch Snapshots'
     url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/'
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java
new file mode 100644
index 0000000000000..4f10deed8c8ad
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java
@@ -0,0 +1,185 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Diagnostic integration tests for PPL access to OpenSearch {@code object} fields
+ * via dotted-path notation ({@code city.name}, {@code city.location.latitude}) on the
+ * analytics-engine route. Mirrors the shape of the sql repo's
+ * {@code ObjectFieldOperateIT}. Every test here is expected to fail initially —
+ * the purpose is to surface exact failure modes for follow-up debugging, not to
+ * exercise a working implementation.
+ */
+public class ObjectFieldIT extends AnalyticsRestTestCase {
+
+    private static final Dataset DATASET = new Dataset("object_fields", "object_fields");
+
+    private static boolean dataProvisioned = false;
+
+    private void ensureDataProvisioned() throws IOException {
+        if (dataProvisioned == false) {
+            DatasetProvisioner.provision(client(), DATASET);
+            dataProvisioned = true;
+        }
+    }
+
+    public void testSelectSingleObjectField() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | fields city.name | head 3",
+            row("Seattle"),
+            row("Portland"),
+            row("Austin")
+        );
+    }
+
+    public void testSelectMultipleObjectFields() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | fields city.name, account.owner | head 3",
+            row("Seattle", "alice"),
+            row("Portland", "bob"),
+            row("Austin", "carol")
+        );
+    }
+
+    public void testSelectDeeplyNestedObjectField() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | fields city.name, city.location.latitude | head 3",
+            row("Seattle", 47.6062),
+            row("Portland", 45.5152),
+            row("Austin", 30.2672)
+        );
+    }
+
+    @AwaitsFix(bugUrl = "Requires sql repo fix to CalciteRelNodeVisitor.containsNestedAggregator (try/catch around relBuilder.field)")
+    public void testMinOnObjectField() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | stats min(account.balance)",
+            row(300.25)
+        );
+    }
+
+    @AwaitsFix(bugUrl = "Requires sql repo fix to CalciteRelNodeVisitor.containsNestedAggregator (try/catch around relBuilder.field)")
+    public void testMaxOnDeeplyNestedObjectField() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | stats max(city.location.latitude)",
+            row(47.6062)
+        );
+    }
+
+    @AwaitsFix(bugUrl = "Requires sql repo fix to CalciteRelNodeVisitor.containsNestedAggregator (try/catch around relBuilder.field)")
+    public void testSumOnObjectField() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | stats sum(city.population)",
+            row(2380000)
+        );
+    }
+
+    public void testFilterOnObjectField() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | where city.name='Seattle' | fields account.owner",
+            row("alice")
+        );
+    }
+
+    public void testFilterOnDeeplyNestedObjectField() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | where city.location.latitude > 40 | fields city.name",
+            row("Seattle"),
+            row("Portland")
+        );
+    }
+
+    // ── Object-parent projection (gated on query-then-fetch) ──────────────────
+    //
+    // Projecting an object parent (top-level "city" or intermediate "city.location")
+    // returns a nested JSON value reconstructed from _source. Analytics-engine emits
+    // only flat leaves into the Calcite row type today, so parent references fall
+    // through QualifiedNameResolver and throw "Field [city.location] not found".
+    //
+    // Support requires query-then-fetch (QTF): coordinator returns docIds post-filter,
+    // a fetch stage pulls the doc from the shard, and the parent sub-object is
+    // reconstructed from _source or from parquet rows. QTF is tracked separately.
+
+    @AwaitsFix(bugUrl = "Object parent projection requires query-then-fetch (QTF) for source-based materialization")
+    public void testSelectIntermediateObjectField() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | fields city.location | head 1",
+            row(Map.of("latitude", 47.6062, "longitude", -122.3321))
+        );
+    }
+
+    @AwaitsFix(bugUrl = "Object parent projection requires query-then-fetch (QTF) for source-based materialization")
+    public void testSelectTopLevelObjectField() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | fields city | head 1",
+            row(Map.of("name", "Seattle", "population", 750000, "location", Map.of("latitude", 47.6062, "longitude", -122.3321)))
+        );
+    }
+
+    @AwaitsFix(bugUrl = "Object parent projection requires query-then-fetch (QTF) for source-based materialization")
+    public void testSelectTopLevelObjectFieldWithSiblings() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | fields city, account | head 1",
+            row(
+                Map.of("name", "Seattle", "population", 750000, "location", Map.of("latitude", 47.6062, "longitude", -122.3321)),
+                Map.of("owner", "alice", "balance", 1000.50)
+            )
+        );
+    }
+
+    @AwaitsFix(bugUrl = "Object parent projection requires query-then-fetch (QTF) for source-based materialization")
+    public void testSelectParentAndLeafMixed() throws IOException {
+        assertRowsEqual(
+            "source=" + DATASET.indexName + " | fields city.name, city.location | head 1",
+            row("Seattle", Map.of("latitude", 47.6062, "longitude", -122.3321))
+        );
+    }
+
+    // ── helpers (mirrored from FieldsCommandIT) ────────────────────────────────
+
+    private static List row(Object... values) {
+        return Arrays.asList(values);
+    }
+
+    @SafeVarargs
+    @SuppressWarnings("varargs")
+    private final void assertRowsEqual(String ppl, List... expected) throws IOException {
+        Map response = executePpl(ppl);
+        @SuppressWarnings("unchecked")
+        List> actualRows = (List>) response.get("rows");
+        assertNotNull("Response missing 'rows' for query: " + ppl, actualRows);
+        assertEquals("Row count mismatch for query: " + ppl, expected.length, actualRows.size());
+        for (int i = 0; i < expected.length; i++) {
+            List want = expected[i];
+            List got = actualRows.get(i);
+            assertEquals("Column count mismatch at row " + i + " for query: " + ppl, want.size(), got.size());
+            for (int j = 0; j < want.size(); j++) {
+                assertEquals("Cell mismatch at row " + i + ", col " + j + " for query: " + ppl, want.get(j), got.get(j));
+            }
+        }
+    }
+
+    private Map executePpl(String ppl) throws IOException {
+        ensureDataProvisioned();
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}");
+        Response response = client().performRequest(request);
+        return assertOkAndParse(response, "PPL: " + ppl);
+    }
+
+}
diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/object_fields/bulk.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/object_fields/bulk.json
new file mode 100644
index 0000000000000..323018b2c91be
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/object_fields/bulk.json
@@ -0,0 +1,7 @@
+{"index": {"_id": "1"}}
+{"id": "1", "city": {"name": "Seattle", "population": 750000, "location": {"latitude": 47.6062, "longitude": -122.3321}}, "account": {"owner": "alice", "balance": 1000.50}}
+{"index": {"_id": "2"}}
+{"id": "2", "city": {"name": "Portland", "population": 650000, "location": {"latitude": 45.5152, "longitude": -122.6784}}, "account": {"owner": "bob", "balance": 2500.00}}
+{"index": {"_id": "3"}}
+{"id": "3", "city": {"name": "Austin", "population": 980000, "location": {"latitude": 30.2672, "longitude": -97.7431}}, "account": {"owner": "carol", "balance": 300.25}}
+
diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/object_fields/mapping.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/object_fields/mapping.json
new file mode 100644
index 0000000000000..cc54b2749c058
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/object_fields/mapping.json
@@ -0,0 +1,43 @@
+{
+    "settings": {
+        "number_of_shards": 1,
+        "number_of_replicas": 0
+    },
+    "mappings": {
+        "properties": {
+            "id": {
+                "type": "keyword"
+            },
+            "city": {
+                "properties": {
+                    "name": {
+                        "type": "keyword"
+                    },
+                    "population": {
+                        "type": "integer"
+                    },
+                    "location": {
+                        "properties": {
+                            "latitude": {
+                                "type": "double"
+                            },
+                            "longitude": {
+                                "type": "double"
+                            }
+                        }
+                    }
+                }
+            },
+            "account": {
+                "properties": {
+                    "owner": {
+                        "type": "keyword"
+                    },
+                    "balance": {
+                        "type": "double"
+                    }
+                }
+            }
+        }
+    }
+}

From 139b9f9f8a97e15f445226c460e6ece8e43984de Mon Sep 17 00:00:00 2001
From: Sagar Darji <42430138+darjisagar7@users.noreply.github.com>
Date: Fri, 8 May 2026 21:03:29 +0530
Subject: [PATCH 089/115] Adding Lucene Merge supports for pluggable dataformat
 support (#21422)

Adding Lucene Merge support for pluggable dataformat support

Signed-off-by: Sagar Darji 
Co-authored-by: Sagar Darji 
---
 .../be/lucene/LuceneReaderManager.java        |  54 +++
 .../be/lucene/index/LuceneCommitter.java      |  54 ++-
 .../be/lucene/index/LuceneDocumentInput.java  |  11 +-
 .../index/LuceneIndexingExecutionEngine.java  |  15 +-
 .../be/lucene/index/LuceneWriter.java         |  16 +-
 .../be/lucene/merge/LuceneMergeStrategy.java  |  63 +++
 .../be/lucene/merge/LuceneMerger.java         | 194 ++++++++++
 .../merge/PrimaryLuceneMergeStrategy.java     |  43 +++
 .../merge/RowIdRemappingCodecReader.java      |  66 ++++
 .../RowIdRemappingDocValuesProducer.java      | 159 ++++++++
 .../lucene/merge/RowIdRemappingOneMerge.java  |  71 ++++
 .../merge/SecondaryLuceneMergeStrategy.java   |  50 +++
 .../be/lucene/merge/package-info.java         |  36 ++
 .../be/lucene/LuceneMergerTests.java          | 321 ++++++++++++++++
 .../be/lucene/LuceneReaderManagerTests.java   |  57 ++-
 ...eneCommitterCSManagerIntegrationTests.java |  12 +-
 .../index/LuceneCommitterFactoryTests.java    |   2 +-
 .../be/lucene/index/LuceneCommitterTests.java |   2 +-
 .../LuceneIndexingExecutionEngineTests.java   |   4 +-
 .../be/lucene/index/LuceneWriterTests.java    |  30 +-
 .../composite/CompositeMergeIT.java           | 361 +++++++++++++++++-
 .../composite/PackedRowIdMappingTests.java    | 196 ++++++++++
 .../benchmark/VSRRotationBenchmark.java       |   2 +-
 .../parquet/bridge/MergeFilesResult.java      |  19 +
 .../parquet/bridge/NativeParquetWriter.java   |  16 +-
 .../opensearch/parquet/bridge/RustBridge.java | 114 +++++-
 .../merge/NativeParquetMergeStrategy.java     |   9 +-
 .../opensearch/parquet/vsr/VSRManager.java    |  13 +-
 .../parquet/writer/ParquetWriter.java         |   3 +-
 .../src/main/rust/src/ffm.rs                  |  58 ++-
 .../src/main/rust/src/merge/cursor.rs         |   8 +-
 .../src/main/rust/src/merge/mod.rs            |  18 +
 .../src/main/rust/src/merge/sorted.rs         |  67 +++-
 .../src/main/rust/src/merge/unsorted.rs       |  50 ++-
 .../src/main/rust/src/test_utils.rs           |   4 +-
 .../src/main/rust/src/tests/mod.rs            |   8 +-
 .../src/main/rust/src/writer.rs               |  31 +-
 .../rust/src/writer_properties_builder.rs     |  30 ++
 .../bridge/NativeParquetWriterTests.java      |   7 +-
 .../bridge/ParquetMergeIntegrationTests.java  |   2 +-
 .../parquet/vsr/VSRManagerTests.java          |  24 +-
 .../apache/lucene/index/MergeIndexWriter.java |  95 +++++
 .../index/engine/DataFormatAwareEngine.java   |  29 +-
 .../engine/dataformat/PackedRowIdMapping.java | 152 ++++++++
 .../engine/exec/commit/CommitterConfig.java   |  23 +-
 .../engine/DataFormatAwareEngineTests.java    |  94 +++++
 .../coord/SafeBootstrapCommitterTests.java    |   6 +-
 47 files changed, 2548 insertions(+), 151 deletions(-)
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/LuceneMergeStrategy.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/LuceneMerger.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/PrimaryLuceneMergeStrategy.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingCodecReader.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingDocValuesProducer.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingOneMerge.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/SecondaryLuceneMergeStrategy.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/package-info.java
 create mode 100644 sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneMergerTests.java
 create mode 100644 sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/PackedRowIdMappingTests.java
 create mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/MergeFilesResult.java
 create mode 100644 server/src/main/java/org/apache/lucene/index/MergeIndexWriter.java
 create mode 100644 server/src/main/java/org/opensearch/index/engine/dataformat/PackedRowIdMapping.java

diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneReaderManager.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneReaderManager.java
index 69f3f5d4f15b1..0fc9cfe6e3334 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneReaderManager.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneReaderManager.java
@@ -9,9 +9,12 @@
 package org.opensearch.be.lucene;
 
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentReader;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.index.engine.dataformat.DataFormat;
 import org.opensearch.index.engine.exec.EngineReaderManager;
+import org.opensearch.index.engine.exec.Segment;
 import org.opensearch.index.engine.exec.coord.CatalogSnapshot;
 
 import java.io.IOException;
@@ -20,6 +23,8 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static org.opensearch.be.lucene.index.LuceneWriter.WRITER_GENERATION_ATTRIBUTE;
+
 /**
  * Lucene implementation of {@link EngineReaderManager}.
  * 
      
@@ -72,11 +77,60 @@ public void afterRefresh(boolean didRefresh, CatalogSnapshot catalogSnapshot) th
         }
         DirectoryReader refreshed = DirectoryReader.openIfChanged(currentReader);
         if (refreshed != null) {
+            // Guard against refresh/merge-apply races: a prior IT regression surfaced when
+            // overlapping threads produced a refreshed reader whose leaves disagreed with the
+            // catalog snapshot being registered, effectively pairing the snapshot with a stale
+            // reader. This assert catches that drift in test builds before the mismatched pair
+            // is published to readers.
+            assert readersAreSame(catalogSnapshot, refreshed);
             currentReader = refreshed;
         }
         readers.put(catalogSnapshot, currentReader);
     }
 
+    /**
+     * Consistency check: verifies that the refreshed {@link DirectoryReader} reflects exactly
+     * the set of segments the given {@link CatalogSnapshot} references. Compares the sorted
+     * list of writer generations drawn from the snapshot's {@link Segment Segments} against
+     * the sorted list of writer generations read off each leaf of the reader (via the
+     * {@link org.opensearch.be.lucene.index.LuceneWriter#WRITER_GENERATION_ATTRIBUTE} stamped
+     * onto every Lucene segment at write time).
+     *
+     * 
      Used only in an {@code assert} to catch refresh/catalog drift in test builds — if
+     * this ever returns {@code false} in production, it means a Lucene reader has been paired
+     * with the wrong catalog snapshot.
+     *
+     * @param catalogSnapshot catalog snapshot whose referenced generations are the expected set
+     * @param readers         DirectoryReader whose leaves' generations are the actual set
+     * @return {@code true} iff both lists contain the same generations in the same (sorted) order
+     */
+    private boolean readersAreSame(CatalogSnapshot catalogSnapshot, DirectoryReader readers) {
+        Collection generationsReferenced = catalogSnapshot.getSegments().stream().map(Segment::generation).sorted().toList();
+        return generationsReferenced.equals(collectReferencedGenerations(readers));
+    }
+
+    /**
+     * Extracts the writer generation from each leaf of the given {@link DirectoryReader} and
+     * returns them as a sorted list. Each leaf's {@link SegmentReader} carries a
+     * {@link SegmentCommitInfo} whose {@code SegmentInfo} is stamped with the
+     * {@link org.opensearch.be.lucene.index.LuceneWriter#WRITER_GENERATION_ATTRIBUTE} when the
+     * segment is written; parsing that attribute yields the generation that produced the leaf.
+     *
+     * @param reader the DirectoryReader to inspect
+     * @return generations of all leaves, sorted ascending
+     * @throws NumberFormatException if a leaf is missing the writer-generation attribute or
+     *                               its value is not parseable as a long (indicates a segment
+     *                               not produced by {@link org.opensearch.be.lucene.index.LuceneWriter})
+     * @throws ClassCastException    if any leaf reader is not a {@link SegmentReader}
+     */
+    private Collection collectReferencedGenerations(DirectoryReader reader) {
+        return reader.leaves().stream().map(lrc -> {
+            SegmentReader segmentReader = (SegmentReader) lrc.reader();
+            SegmentCommitInfo sci = segmentReader.getSegmentInfo();
+            return Long.parseLong(sci.info.getAttribute(WRITER_GENERATION_ATTRIBUTE));
+        }).sorted().toList();
+    }
+
     @Override
     public void onDeleted(CatalogSnapshot catalogSnapshot) throws IOException {
         DirectoryReader reader = readers.remove(catalogSnapshot);
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneCommitter.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneCommitter.java
index 68e23f97abbd0..63d552c8f0b53 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneCommitter.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneCommitter.java
@@ -15,12 +15,18 @@
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.MergeIndexWriter;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.index.SerialMergeScheduler;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSortField;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.index.engine.CommitStats;
 import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.index.engine.SafeCommitInfo;
+import org.opensearch.index.engine.dataformat.DocumentInput;
 import org.opensearch.index.engine.exec.CombinedCatalogSnapshotDeletionPolicy;
 import org.opensearch.index.engine.exec.commit.Committer;
 import org.opensearch.index.engine.exec.commit.CommitterConfig;
@@ -59,6 +65,19 @@
  * The store reference is incremented on construction and decremented on {@link #close()}.
  * Closing the committer also closes the underlying IndexWriter.
  *
+ * 
      Refresh-lock coordination
      
+ *
+ * 
      The engine passes a {@code preMergeCommitHook} via {@link CommitterConfig}. We wire it
+ * into Lucene as a {@code MergedSegmentWarmer} on the {@link IndexWriterConfig}. The warmer
+ * runs between {@code mergeMiddle} and {@code commitMerge} while the {@link IndexWriter}
+ * monitor is not held, so invoking the hook there establishes the ordering
+ * {@code refreshLock → IW monitor} on the merge thread — matching the refresh path and
+ * avoiding the lock inversion that would occur if coordination happened inside
+ * {@code commitMerge}. Ownership of whatever the hook acquires (currently the engine's
+ * refresh lock) is transferred to the engine's {@code applyMergeChanges} callback, which
+ * releases it after the catalog is updated. This committer never touches the refresh lock
+ * directly.
+ *
  * @opensearch.experimental
  */
 @ExperimentalApi
@@ -67,7 +86,7 @@ public class LuceneCommitter extends SafeBootstrapCommitter {
     private static final Logger logger = LogManager.getLogger(LuceneCommitter.class);
 
     private final Store store;
-    private final IndexWriter indexWriter;
+    private final MergeIndexWriter indexWriter;
     private final LuceneCommitDeletionPolicy deletionPolicy;
     private final AtomicBoolean isClosed = new AtomicBoolean();
 
@@ -84,8 +103,8 @@ public LuceneCommitter(CommitterConfig committerConfig) throws IOException {
         this.store.incRef();
         try {
             this.deletionPolicy = new LuceneCommitDeletionPolicy();
-            IndexWriterConfig iwc = createIndexWriterConfig(committerConfig.engineConfig());
-            this.indexWriter = new IndexWriter(store.directory(), iwc);
+            IndexWriterConfig iwc = createIndexWriterConfig(committerConfig);
+            this.indexWriter = new MergeIndexWriter(store.directory(), iwc);
         } catch (Exception e) {
             store.decRef();
             throw e;
@@ -197,18 +216,20 @@ public boolean isCommitManagedFile(String fileName) {
      *
      * @return the index writer, or null if closed
      */
-    IndexWriter getIndexWriter() {
+    MergeIndexWriter getIndexWriter() {
         ensureOpen();
         return indexWriter;
     }
 
     // --- Internal ---
 
-    private IndexWriterConfig createIndexWriterConfig(EngineConfig engineConfig) {
+    private IndexWriterConfig createIndexWriterConfig(CommitterConfig committerConfig) {
+        EngineConfig engineConfig = committerConfig.engineConfig();
         if (engineConfig == null) {
             IndexWriterConfig iwc = new IndexWriterConfig();
             iwc.setIndexDeletionPolicy(deletionPolicy);
             iwc.setMergePolicy(NoMergePolicy.INSTANCE);
+            iwc.setMergeScheduler(new SerialMergeScheduler());
             return iwc;
         }
         // TODO:: Merge Config needs to be wired in
@@ -219,13 +240,34 @@ private IndexWriterConfig createIndexWriterConfig(EngineConfig engineConfig) {
         }
         iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
         iwc.setUseCompoundFile(engineConfig.useCompoundFile());
-        if (engineConfig.getIndexSort() != null) {
+        // Refresh-lock hand-off: the MergedSegmentWarmer fires on the merge thread between
+        // mergeMiddle and commitMerge, while the IndexWriter monitor is NOT held. Invoking
+        // the engine-provided preMergeCommitHook here gives the merge path the ordering
+        // refreshLock → IW monitor, which matches the refresh path (DataFormatAwareEngine#refresh
+        // takes refreshLock before calling IndexWriter#addIndexes). Ownership of whatever the
+        // hook acquires is transferred to applyMergeChanges, which releases it after the
+        // catalog is updated. See the class Javadoc.
+        iwc.setMergedSegmentWarmer(_ -> committerConfig.preMergeCommitHook().run());
+
+        // Determine if Lucene is a secondary format in a composite setup.
+        // When secondary, use a SortedNumericSortField on the row ID so MultiSorter can reorder
+        // documents by remapped row ID during merge. When primary (or standalone), use the
+        // engine config's IndexSort (which may be user-configured).
+        // TODO Check what is the right way to get this information as the below one is leaky
+        // https://github.com/opensearch-project/OpenSearch/issues/21506
+        List secondaryFormats = engineConfig.getIndexSettings().getSettings().getAsList("index.composite.secondary_data_formats");
+        boolean isSecondary = secondaryFormats.contains("lucene");
+
+        if (isSecondary) {
+            iwc.setIndexSort(new Sort(new SortedNumericSortField(DocumentInput.ROW_ID_FIELD, SortField.Type.LONG)));
+        } else if (engineConfig.getIndexSort() != null) {
             iwc.setIndexSort(engineConfig.getIndexSort());
         }
         iwc.setCommitOnClose(false);
         iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
         iwc.setIndexDeletionPolicy(deletionPolicy);
         iwc.setMergePolicy(NoMergePolicy.INSTANCE);
+        iwc.setMergeScheduler(new SerialMergeScheduler());
         return iwc;
     }
 
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneDocumentInput.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneDocumentInput.java
index 68802ff12305b..b41dfe6d50c37 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneDocumentInput.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneDocumentInput.java
@@ -9,7 +9,7 @@
 package org.opensearch.be.lucene.index;
 
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.opensearch.be.lucene.LuceneFieldFactory;
 import org.opensearch.be.lucene.LuceneFieldFactoryRegistry;
 import org.opensearch.common.annotation.ExperimentalApi;
@@ -26,8 +26,9 @@
  * Only field types registered in the registry are accepted. Attempting to add a field
  * of an unregistered type throws {@link IllegalArgumentException}.
  *
- * The row ID field is stored as a {@link NumericDocValuesField} for efficient doc-value
- * access, maintaining 1:1 correspondence between Lucene doc IDs and Parquet row offsets.
+ * The row ID field is stored as a {@link SortedNumericDocValuesField} for efficient doc-value
+ * access and compatibility with the {@code SortedNumericSortField}-based IndexSort,
+ * maintaining 1:1 correspondence between Lucene doc IDs and Parquet row offsets.
  *
  * @opensearch.experimental
  */
@@ -95,7 +96,7 @@ public void addField(MappedFieldType fieldType, Object value) {
     }
 
     /**
-     * Stores the row ID as a {@link NumericDocValuesField} to maintain 1:1 correspondence
+     * Stores the row ID as a {@link SortedNumericDocValuesField} to maintain 1:1 correspondence
      * between Lucene doc IDs and Parquet row offsets.
      *
      * @param rowIdFieldName the name of the row ID field
@@ -103,7 +104,7 @@ public void addField(MappedFieldType fieldType, Object value) {
      */
     @Override
     public void setRowId(String rowIdFieldName, long rowId) {
-        document.add(new NumericDocValuesField(rowIdFieldName, rowId));
+        document.add(new SortedNumericDocValuesField(rowIdFieldName, rowId));
     }
 
     /** No-op — this document input holds no closeable resources. */
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngine.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngine.java
index d12099e82f083..800bbb213d516 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngine.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngine.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.MergeIndexWriter;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentReader;
@@ -23,10 +24,10 @@
 import org.apache.lucene.store.MMapDirectory;
 import org.opensearch.be.lucene.LuceneDataFormat;
 import org.opensearch.be.lucene.LuceneFieldFactoryRegistry;
+import org.opensearch.be.lucene.merge.LuceneMerger;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.index.engine.dataformat.DataFormat;
 import org.opensearch.index.engine.dataformat.IndexingExecutionEngine;
-import org.opensearch.index.engine.dataformat.MergeResult;
 import org.opensearch.index.engine.dataformat.Merger;
 import org.opensearch.index.engine.dataformat.RefreshInput;
 import org.opensearch.index.engine.dataformat.RefreshResult;
@@ -74,11 +75,12 @@ public class LuceneIndexingExecutionEngine implements IndexingExecutionEngine createWriter(long writerGeneration) {
         assert sharedWriter.isOpen() : "Cannot create writer — shared IndexWriter is closed";
         try {
-            return new LuceneWriter(writerGeneration, dataFormat, baseDirectory, analyzer, codec);
+            return new LuceneWriter(writerGeneration, dataFormat, baseDirectory, analyzer, codec, sharedWriter.getConfig().getIndexSort());
         } catch (IOException e) {
             throw new RuntimeException("Failed to create LuceneWriter for generation " + writerGeneration, e);
         }
@@ -279,8 +283,7 @@ public RefreshResult refresh(RefreshInput refreshInput) throws IOException {
     /** Returns {@code null} — merge scheduling is not yet implemented for the Lucene format. */
     @Override
     public Merger getMerger() {
-        // TODO: Implement merge support as ParquetMerger
-        return mergeInput -> new MergeResult(Map.of());
+        return this.luceneMerger;
     }
 
     /**
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneWriter.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneWriter.java
index f4fdaab0c0e71..f507297cbc248 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneWriter.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/index/LuceneWriter.java
@@ -17,6 +17,7 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.search.Sort;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MMapDirectory;
 import org.opensearch.be.lucene.LuceneDataFormat;
@@ -80,10 +81,18 @@ public class LuceneWriter implements Writer {
      * @param dataFormat       the Lucene data format descriptor
      * @param baseDirectory    the base directory under which to create the temp directory
      * @param analyzer         the analyzer to use for tokenized fields, or null for default
+     * @param codec            the codec to use, or null for default
+     * @param indexSort        the index sort to apply to segments, or null for no sort
      * @throws IOException if directory creation or IndexWriter opening fails
      */
-    public LuceneWriter(long writerGeneration, LuceneDataFormat dataFormat, Path baseDirectory, Analyzer analyzer, Codec codec)
-        throws IOException {
+    public LuceneWriter(
+        long writerGeneration,
+        LuceneDataFormat dataFormat,
+        Path baseDirectory,
+        Analyzer analyzer,
+        Codec codec,
+        Sort indexSort
+    ) throws IOException {
         this.writerGeneration = writerGeneration;
         this.dataFormat = dataFormat;
         this.docCount = 0;
@@ -97,6 +106,9 @@ public LuceneWriter(long writerGeneration, LuceneDataFormat dataFormat, Path bas
         IndexWriterConfig iwc = analyzer != null ? new IndexWriterConfig(analyzer) : new IndexWriterConfig();
         iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
         iwc.setRAMBufferSizeMB(RAM_BUFFER_SIZE_MB);
+        if (indexSort != null) {
+            iwc.setIndexSort(indexSort);
+        }
 
         iwc.setCodec(new LuceneWriterCodec(codec, writerGeneration));
         this.indexWriter = new IndexWriter(directory, iwc);
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/LuceneMergeStrategy.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/LuceneMergeStrategy.java
new file mode 100644
index 0000000000000..fa441fb143c8c
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/LuceneMergeStrategy.java
@@ -0,0 +1,63 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene.merge;
+
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.engine.dataformat.MergeInput;
+import org.opensearch.index.engine.dataformat.RowIdMapping;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Strategy interface for Lucene merge behavior based on whether Lucene is the
+ * primary or secondary data format in a composite index.
+ *
+ * 
      When Lucene is the primary format, it performs a standard merge and
+ * produces a {@link RowIdMapping} that secondary formats use to align their
+ * document order.
+ *
+ * 
      When Lucene is a secondary format, it receives a {@link RowIdMapping}
+ * from the primary format and remaps its row ID doc values + reorders documents
+ * to match the primary's merged output.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public interface LuceneMergeStrategy {
+
+    /**
+     * Creates the {@link MergePolicy.OneMerge} that controls how segments are merged.
+     *
+     * 
      Primary strategy: returns a plain {@code OneMerge} (no reader wrapping).
+     * 
      Secondary strategy: returns a {@link RowIdRemappingOneMerge} that wraps readers
+     * with {@link RowIdRemappingCodecReader} for row ID remapping.
+     *
+     * @param segments the segments to merge
+     * @param rowIdMapping the row ID mapping from the primary format, or null if this is the primary
+     * @return the configured OneMerge for execution
+     */
+    MergePolicy.OneMerge createOneMerge(List segments, RowIdMapping rowIdMapping);
+
+    /**
+     * Builds or resolves the {@link RowIdMapping} after the merge completes.
+     *
+     * 
      Primary strategy: builds a new mapping by reading the merged segment to determine
+     * how old row IDs map to new positions in the merged output.
+     * 
      Secondary strategy: passes through the input mapping (already provided by the primary).
+     *
+     * @param completedMerge the merge that was executed (contains merged segment info)
+     * @param mergeInput the original merge input (contains input row ID mapping and segment list)
+     * @return the row ID mapping for the merge result, or null if not applicable
+     * @throws IOException if reading the merged segment fails
+     */
+    RowIdMapping buildRowIdMapping(MergePolicy.OneMerge completedMerge, MergeInput mergeInput) throws IOException;
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/LuceneMerger.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/LuceneMerger.java
new file mode 100644
index 0000000000000..e5392c0a04e93
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/LuceneMerger.java
@@ -0,0 +1,194 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene.merge;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MergeIndexWriter;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
+import org.opensearch.common.SuppressForbidden;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.engine.dataformat.DataFormat;
+import org.opensearch.index.engine.dataformat.MergeInput;
+import org.opensearch.index.engine.dataformat.MergeResult;
+import org.opensearch.index.engine.dataformat.Merger;
+import org.opensearch.index.engine.dataformat.RowIdMapping;
+import org.opensearch.index.engine.exec.Segment;
+import org.opensearch.index.engine.exec.WriterFileSet;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.opensearch.be.lucene.index.LuceneWriter.WRITER_GENERATION_ATTRIBUTE;
+
+/**
+ * Lucene-specific {@link Merger} that merges segments using Lucene's internal
+ * {@code merge(OneMerge)} path with IndexSort-based document reordering.
+ *
+ * 
      How it works
      
+ *
+ * 
        
+ *   
      1. Value rewriting — {@link RowIdRemappingOneMerge#wrapForMerge} wraps each
+ *       CodecReader with {@link RowIdRemappingCodecReader} to remap row ID
+ *       doc values for the output.
        2. 
+ *   
      2. Document ordering — The writer's IndexSort (a {@code SortedNumericSortField}
+ *       on the row ID field) reads the already-remapped values from the wrapped readers.
+ *       {@code MultiSorter.sort()} uses these to build DocMaps that reorder all data
+ *       (stored fields, doc values, postings).
        3. 
+ *   
      3. Segment lifecycle — Lucene's internal merge path handles reference-counted
+ *       file cleanup via {@code IndexFileDeleter}. If the merge fails, old segments are
+ *       preserved and the partially-written merged segment is cleaned up.
        4. 
+ * 
      
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class LuceneMerger implements Merger {
+
+    private static final Logger logger = LogManager.getLogger(LuceneMerger.class);
+
+    private static final Field SEGMENT_INFOS_FIELD = initSegmentInfosField();
+
+    @SuppressForbidden(reason = "Need live SegmentInfos reference for post-merge segment removal; cloneSegmentInfos() returns a copy")
+    private static Field initSegmentInfosField() {
+        try {
+            Field field = IndexWriter.class.getDeclaredField("segmentInfos");
+            field.setAccessible(true);
+            return field;
+        } catch (NoSuchFieldException e) {
+            throw new ExceptionInInitializerError(e);
+        }
+    }
+
+    private final MergeIndexWriter indexWriter;
+    private final DataFormat dataFormat;
+    private final Path storeDirectory;
+    private final LuceneMergeStrategy strategy;
+
+    public LuceneMerger(MergeIndexWriter indexWriter, DataFormat dataFormat, Path storeDirectory) {
+        if (indexWriter == null) {
+            throw new IllegalArgumentException("IndexWriter must not be null");
+        }
+        this.indexWriter = indexWriter;
+        this.dataFormat = dataFormat;
+        this.storeDirectory = storeDirectory;
+        // TODO implement primary and integrate the same here
+        this.strategy = new SecondaryLuceneMergeStrategy();
+    }
+
+    @Override
+    public MergeResult merge(MergeInput mergeInput) throws IOException {
+        RowIdMapping rowIdMapping = mergeInput.rowIdMapping();
+        List segments = mergeInput.segments();
+
+        if (segments.isEmpty()) {
+            return new MergeResult(Map.of());
+        }
+
+        Set generationsToMerge = new HashSet<>();
+        for (Segment segment : segments) {
+            generationsToMerge.add(segment.generation());
+        }
+
+        SegmentInfos segmentInfos;
+        try {
+            segmentInfos = (SegmentInfos) SEGMENT_INFOS_FIELD.get(indexWriter);
+        } catch (IllegalAccessException e) {
+            throw new IOException("Failed to access IndexWriter segmentInfos via reflection", e);
+        }
+
+        if (segmentInfos.size() == 0) {
+            logger.warn("No segments in IndexWriter — skipping merge");
+            return new MergeResult(Map.of());
+        }
+
+        List matchingSegments = findMatchingSegments(segmentInfos, generationsToMerge);
+
+        if (matchingSegments.isEmpty()) {
+            logger.warn("No segments found matching writer generations {} — skipping merge", generationsToMerge);
+            return new MergeResult(Map.of());
+        }
+
+        logger.debug(
+            "LuceneMerger: merging {} segments (generations {}) using merge(OneMerge) + IndexSort",
+            matchingSegments.size(),
+            generationsToMerge
+        );
+
+        // Delegate OneMerge creation to the strategy (primary vs secondary behavior)
+        MergePolicy.OneMerge oneMerge = strategy.createOneMerge(matchingSegments, rowIdMapping);
+        indexWriter.executeMerge(oneMerge, mergeInput.newWriterGeneration());
+
+        // Stamp the merged segment with its writer generation so downstream lookups
+        // (e.g. findMatchingSegments on a subsequent merge) can correlate it.
+        //
+        // This mutation is in-memory only: Lucene writes the .si file exactly once at
+        // segment creation via SegmentInfoFormat.write(...) and does not rewrite it on
+        // later commits, so this attribute will not survive a writer reopen. That is
+        // acceptable here because the attribute is only consumed within the lifetime
+        // of the live IndexWriter's SegmentInfos.
+        SegmentCommitInfo mergedInfo = oneMerge.getMergeInfo();
+        if (mergedInfo != null) {
+            mergedInfo.info.putAttribute(WRITER_GENERATION_ATTRIBUTE, String.valueOf(mergeInput.newWriterGeneration()));
+        }
+
+        // Build the merged WriterFileSet from the output segment info
+        WriterFileSet mergedFileSet = buildMergedFileSet(mergedInfo, mergeInput.newWriterGeneration());
+
+        // Delegate RowIdMapping production to the strategy
+        RowIdMapping outputMapping = strategy.buildRowIdMapping(oneMerge, mergeInput);
+
+        logger.debug(
+            "LuceneMerger: completed merge of {} segments at generation {} ({} docs, {} files)",
+            matchingSegments.size(),
+            mergeInput.newWriterGeneration(),
+            oneMerge.getMergeInfo().info.maxDoc(),
+            oneMerge.getMergeInfo().files().size()
+        );
+
+        return new MergeResult(Map.of(dataFormat, mergedFileSet), outputMapping);
+    }
+
+    /**
+     * Finds segments in the IndexWriter whose writer generation matches the requested generations.
+     */
+    private List findMatchingSegments(SegmentInfos segmentInfos, Set generations) {
+        List matching = new ArrayList<>();
+        for (SegmentCommitInfo sci : segmentInfos) {
+            String genAttr = sci.info.getAttribute(WRITER_GENERATION_ATTRIBUTE);
+            if (genAttr != null && generations.contains(Long.parseLong(genAttr))) {
+                matching.add(sci);
+            }
+        }
+        return matching;
+    }
+
+    /**
+     * Builds a {@link WriterFileSet} from the merged segment info.
+     */
+    private WriterFileSet buildMergedFileSet(SegmentCommitInfo mergedInfo, long writerGeneration) throws IOException {
+        WriterFileSet.Builder builder = WriterFileSet.builder()
+            .directory(storeDirectory)
+            .writerGeneration(writerGeneration)
+            .addNumRows(mergedInfo.info.maxDoc());
+        for (String file : mergedInfo.files()) {
+            builder.addFile(file);
+        }
+        return builder.build();
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/PrimaryLuceneMergeStrategy.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/PrimaryLuceneMergeStrategy.java
new file mode 100644
index 0000000000000..1cad746eac22f
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/PrimaryLuceneMergeStrategy.java
@@ -0,0 +1,43 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene.merge;
+
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.engine.dataformat.MergeInput;
+import org.opensearch.index.engine.dataformat.RowIdMapping;
+
+import java.util.List;
+
+/**
+ * Merge strategy for when Lucene is the primary data format in a composite index.
+ *
+ * 
      As the primary format, Lucene performs a standard merge (no row ID remapping on input)
+ * and produces a {@link RowIdMapping} that secondary formats use to align their document
+ * order with the merged output.
+ *
+ * 
      The mapping is built after the merge completes by reading the merged segment to
+ * determine how documents from each source generation were reordered.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class PrimaryLuceneMergeStrategy implements LuceneMergeStrategy {
+
+    @Override
+    public MergePolicy.OneMerge createOneMerge(List segments, RowIdMapping rowIdMapping) {
+        throw new UnsupportedOperationException("Primary Lucene merge strategy is not yet implemented");
+    }
+
+    @Override
+    public RowIdMapping buildRowIdMapping(MergePolicy.OneMerge completedMerge, MergeInput mergeInput) {
+        throw new UnsupportedOperationException("Primary Lucene merge strategy is not yet implemented");
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingCodecReader.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingCodecReader.java
new file mode 100644
index 0000000000000..b3a94961a2551
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingCodecReader.java
@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene.merge;
+
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.FilterCodecReader;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.engine.dataformat.RowIdMapping;
+
+/**
+ * Wraps a {@link CodecReader} to replace {@code ___row_id} doc values with remapped values.
+ *
+ * 
      This ensures the merged segment's {@code ___row_id} field stores the new global row IDs
+ * from the {@link RowIdMapping}, not the original per-segment local values.
+ *
+ * 
      The IndexSort on the writer handles document ordering during merge.
+ * This reader handles the values written to the merged segment.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+class RowIdRemappingCodecReader extends FilterCodecReader {
+
+    private final RowIdMapping rowIdMapping;
+    private final long generation;
+    private final int rowIdOffset;
+
+    /**
+     * @param in           the source codec reader to wrap
+     * @param rowIdMapping the mapping from old to new row IDs, or null for sequential assignment
+     * @param generation   the writer generation of this segment
+     * @param rowIdOffset  the starting row ID offset for sequential assignment
+     */
+    RowIdRemappingCodecReader(CodecReader in, RowIdMapping rowIdMapping, long generation, int rowIdOffset) {
+        super(in);
+        this.rowIdMapping = rowIdMapping;
+        this.generation = generation;
+        this.rowIdOffset = rowIdOffset;
+    }
+
+    @Override
+    public DocValuesProducer getDocValuesReader() {
+        DocValuesProducer delegate = in.getDocValuesReader();
+        if (delegate == null) {
+            return null;
+        }
+        return new RowIdRemappingDocValuesProducer(delegate, rowIdMapping, generation, in.maxDoc(), rowIdOffset);
+    }
+
+    @Override
+    public CacheHelper getCoreCacheHelper() {
+        return in.getCoreCacheHelper();
+    }
+
+    @Override
+    public CacheHelper getReaderCacheHelper() {
+        return in.getReaderCacheHelper();
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingDocValuesProducer.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingDocValuesProducer.java
new file mode 100644
index 0000000000000..cc824e8f93010
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingDocValuesProducer.java
@@ -0,0 +1,159 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene.merge;
+
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValuesSkipper;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.engine.dataformat.DocumentInput;
+import org.opensearch.index.engine.dataformat.RowIdMapping;
+
+import java.io.IOException;
+
+/**
+ * {@link DocValuesProducer} that intercepts the {@code ___row_id} field and returns
+ * remapped row ID values from a {@link RowIdMapping}. All other fields are delegated
+ * unchanged to the wrapped producer.
+ *
+ * 
      This ensures the merged segment's {@code ___row_id} doc values contain the new
+ * global row IDs (0..n-1) rather than the original per-segment local values.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+class RowIdRemappingDocValuesProducer extends DocValuesProducer {
+
+    private final DocValuesProducer delegate;
+    private final RowIdMapping rowIdMapping;
+    private final long generation;
+    private final int maxDoc;
+    private final int rowIdOffset;
+
+    /**
+     * @param delegate     the original doc values producer
+     * @param rowIdMapping the mapping from old to new row IDs, or null for sequential assignment
+     * @param generation   the writer generation of the source segment
+     * @param maxDoc       the maximum document count in the source segment
+     * @param rowIdOffset  the starting row ID offset for sequential assignment (used when rowIdMapping is null)
+     */
+    RowIdRemappingDocValuesProducer(DocValuesProducer delegate, RowIdMapping rowIdMapping, long generation, int maxDoc, int rowIdOffset) {
+        this.delegate = delegate;
+        this.rowIdMapping = rowIdMapping;
+        this.generation = generation;
+        this.maxDoc = maxDoc;
+        this.rowIdOffset = rowIdOffset;
+    }
+
+    @Override
+    public NumericDocValues getNumeric(FieldInfo field) throws IOException {
+        return delegate.getNumeric(field);
+    }
+
+    @Override
+    public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+        if (DocumentInput.ROW_ID_FIELD.equals(field.name)) {
+            if (rowIdMapping != null) {
+                return new MappedRowIdDocValues(delegate.getSortedNumeric(field), rowIdMapping, generation);
+            } else {
+                // https://github.com/opensearch-project/OpenSearch/issues/21508
+                // TODO check how this will work for primary engine when rowIdMapping will be null.
+                throw new UnsupportedOperationException("Lucene as Primary Format is not supported yet");
+            }
+        }
+        return delegate.getSortedNumeric(field);
+    }
+
+    @Override
+    public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+        return delegate.getBinary(field);
+    }
+
+    @Override
+    public SortedDocValues getSorted(FieldInfo field) throws IOException {
+        return delegate.getSorted(field);
+    }
+
+    @Override
+    public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
+        return delegate.getSortedSet(field);
+    }
+
+    @Override
+    public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
+        return delegate.getSkipper(field);
+    }
+
+    @Override
+    public void checkIntegrity() throws IOException {
+        delegate.checkIntegrity();
+    }
+
+    @Override
+    public void close() throws IOException {
+        delegate.close();
+    }
+
+    /**
+     * Reads the original {@code ___row_id} and maps it through the {@link RowIdMapping}.
+     */
+    private static class MappedRowIdDocValues extends SortedNumericDocValues {
+
+        private final SortedNumericDocValues delegate;
+        private final RowIdMapping rowIdMapping;
+        private final long generation;
+
+        MappedRowIdDocValues(SortedNumericDocValues delegate, RowIdMapping rowIdMapping, long generation) {
+            this.delegate = delegate;
+            this.rowIdMapping = rowIdMapping;
+            this.generation = generation;
+        }
+
+        @Override
+        public long nextValue() throws IOException {
+            long oldRowId = delegate.nextValue();
+            return rowIdMapping.getNewRowId(oldRowId, generation);
+        }
+
+        @Override
+        public int docValueCount() {
+            return delegate.docValueCount();
+        }
+
+        @Override
+        public boolean advanceExact(int target) throws IOException {
+            return delegate.advanceExact(target);
+        }
+
+        @Override
+        public int docID() {
+            return delegate.docID();
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            return delegate.nextDoc();
+        }
+
+        @Override
+        public int advance(int target) throws IOException {
+            return delegate.advance(target);
+        }
+
+        @Override
+        public long cost() {
+            return delegate.cost();
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingOneMerge.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingOneMerge.java
new file mode 100644
index 0000000000000..30e802b4fba1e
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/RowIdRemappingOneMerge.java
@@ -0,0 +1,71 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene.merge;
+
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentReader;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.engine.dataformat.RowIdMapping;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.opensearch.be.lucene.index.LuceneWriter.WRITER_GENERATION_ATTRIBUTE;
+
+/**
+ * A custom {@link MergePolicy.OneMerge} that wraps each segment's {@link CodecReader}
+ * with a {@link RowIdRemappingCodecReader} during the merge process.
+ *
+ * 
      The wrapped reader remaps row ID doc values so the merged segment stores
+ * the new global row IDs. Document ordering is handled by the IndexSort (a
+ * {@code SortedNumericSortField} on the row ID field) — {@code MultiSorter} reads the
+ * already-remapped values and builds DocMaps for reordering.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+class RowIdRemappingOneMerge extends MergePolicy.OneMerge {
+
+    private final RowIdMapping rowIdMapping;
+    private int nextRowIdOffset;
+
+    RowIdRemappingOneMerge(List segments, RowIdMapping rowIdMapping) {
+        super(segments);
+        this.rowIdMapping = rowIdMapping;
+        this.nextRowIdOffset = 0;
+    }
+
+    @Override
+    public CodecReader wrapForMerge(CodecReader reader) throws IOException {
+        CodecReader wrapped = super.wrapForMerge(reader);
+        long generation = resolveGeneration(wrapped);
+        int offset = nextRowIdOffset;
+        nextRowIdOffset += wrapped.maxDoc();
+        return new RowIdRemappingCodecReader(wrapped, rowIdMapping, generation, offset);
+    }
+
+    private long resolveGeneration(CodecReader reader) {
+        if (reader instanceof SegmentReader segmentReader) {
+            SegmentCommitInfo sci = segmentReader.getSegmentInfo();
+            String genAttr = sci.info.getAttribute(WRITER_GENERATION_ATTRIBUTE);
+            if (genAttr != null) {
+                return Long.parseLong(genAttr);
+            }
+        }
+        throw new IllegalStateException(
+            "Cannot resolve writer generation for reader: "
+                + reader.getClass().getName()
+                + ". Ensure segments have the '"
+                + WRITER_GENERATION_ATTRIBUTE
+                + "' attribute."
+        );
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/SecondaryLuceneMergeStrategy.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/SecondaryLuceneMergeStrategy.java
new file mode 100644
index 0000000000000..5ec25adf1aeee
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/SecondaryLuceneMergeStrategy.java
@@ -0,0 +1,50 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene.merge;
+
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.engine.dataformat.MergeInput;
+import org.opensearch.index.engine.dataformat.RowIdMapping;
+
+import java.util.List;
+
+/**
+ * Merge strategy for when Lucene is a secondary data format in a composite index.
+ *
+ * 
      As a secondary format, Lucene receives a {@link RowIdMapping} from the primary format
+ * and must:
+ * 
        
+ *   
      1. Remap row ID doc values to the new global IDs (via {@link RowIdRemappingCodecReader})
        2. 
+ *   
      2. Reorder documents to match the primary format's merged output (via IndexSort on the
+ *       row ID field)
        3. 
+ * 
      
+ *
+ * 
      This strategy creates a {@link RowIdRemappingOneMerge} that wraps each segment's
+ * {@link org.apache.lucene.index.CodecReader} during the merge process. The
+ * {@code buildRowIdMapping} method passes through the input mapping since the primary
+ * format is the authority on document ordering.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class SecondaryLuceneMergeStrategy implements LuceneMergeStrategy {
+
+    @Override
+    public MergePolicy.OneMerge createOneMerge(List segments, RowIdMapping rowIdMapping) {
+        return new RowIdRemappingOneMerge(segments, rowIdMapping);
+    }
+
+    @Override
+    public RowIdMapping buildRowIdMapping(MergePolicy.OneMerge completedMerge, MergeInput mergeInput) {
+        // Secondary format passes through the mapping from the primary — it does not produce its own.
+        return mergeInput.rowIdMapping();
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/package-info.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/package-info.java
new file mode 100644
index 0000000000000..e285f8dba1267
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/merge/package-info.java
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Lucene merge implementation for the composite engine using {@code addIndexes(CodecReader...)}
+ * with IndexSort-based document reordering.
+ *
+ * 
      How it works
      
+ *
+ * 
        
+ *   
      • Value rewriting — Each source CodecReader is wrapped with
+ *       {@link org.opensearch.be.lucene.merge.RowIdRemappingCodecReader} which replaces
+ *       {@code ___row_id} doc values with the remapped global values from the RowIdMapping.
        • 
+ *   
      • Document ordering — {@code addIndexes(CodecReader...)} applies the writer's
+ *       IndexSort from scratch (full sort, not merge-sort). The {@code SortedNumericSortField}
+ *       on the row ID field reads the already-remapped values and sorts all documents by
+ *       ascending row ID, including cross-segment interleaving and within-segment reordering.
        • 
+ *   
      • Segment cleanup — Lucene's internal merge path handles segment lifecycle:
+ *       {@code commitMerge} removes old segments from the live list and decrements file references.
        • 
+ * 
      
+ *
+ * 
      Key classes
      
+ * 
        
+ *   
      • {@link org.opensearch.be.lucene.merge.LuceneMerger} — Orchestrates the merge.
        • 
+ *   
      • {@link org.opensearch.be.lucene.merge.RowIdRemappingCodecReader} — FilterCodecReader
+ *       that remaps {@code ___row_id} doc values.
        • 
+ *   
      • {@link org.opensearch.be.lucene.merge.RowIdRemappingDocValuesProducer} — DocValuesProducer
+ *       that returns remapped row ID values.
        • 
+ * 
      
+ */
+package org.opensearch.be.lucene.merge;
diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneMergerTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneMergerTests.java
new file mode 100644
index 0000000000000..4320fd9ba6c7b
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneMergerTests.java
@@ -0,0 +1,321 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.MergeIndexWriter;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.index.SerialMergeScheduler;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.NIOFSDirectory;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
+import org.opensearch.be.lucene.merge.LuceneMerger;
+import org.opensearch.common.SuppressForbidden;
+import org.opensearch.index.engine.dataformat.DocumentInput;
+import org.opensearch.index.engine.dataformat.MergeInput;
+import org.opensearch.index.engine.dataformat.MergeResult;
+import org.opensearch.index.engine.dataformat.RowIdMapping;
+import org.opensearch.index.engine.exec.Segment;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.opensearch.be.lucene.index.LuceneWriter.WRITER_GENERATION_ATTRIBUTE;
+
+/**
+ * End-to-end tests for {@link LuceneMerger}.
+ *
+ * 
      These tests create real Lucene segments with {@code writer_generation} attributes
+ * and {@code ___row_id} doc values, then exercise the merge path and validate the output.
+ */
+public class LuceneMergerTests extends OpenSearchTestCase {
+
+    private static final String ROW_ID_FIELD = DocumentInput.ROW_ID_FIELD;
+
+    private MergeIndexWriter writer;
+    private Directory directory;
+    private Path dataPath;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        dataPath = createTempDir();
+        directory = NIOFSDirectory.open(dataPath);
+        IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+        iwc.setMergeScheduler(new SerialMergeScheduler());
+        iwc.setMergePolicy(NoMergePolicy.INSTANCE);
+        iwc.setIndexSort(new Sort(new SortedNumericSortField(ROW_ID_FIELD, SortField.Type.LONG)));
+        writer = new MergeIndexWriter(directory, iwc);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        if (writer != null) {
+            writer.close();
+        }
+        if (directory != null) {
+            directory.close();
+        }
+        super.tearDown();
+    }
+
+    // ========== Test Cases ==========
+
+    /**
+     * Merge with empty input returns empty result without error.
+     */
+    public void testMergeWithEmptyInput() throws IOException {
+        LuceneMerger merger = new LuceneMerger(writer, new LuceneDataFormat(), dataPath);
+        MergeInput input = MergeInput.builder().segments(List.of()).newWriterGeneration(99L).build();
+
+        MergeResult result = merger.merge(input);
+        assertNotNull(result);
+        assertTrue(result.getMergedWriterFileSet().isEmpty());
+    }
+
+    /**
+     * Merge with no matching segments returns empty result and logs warning.
+     */
+    public void testMergeWithNoMatchingSegments() throws IOException {
+        writeSegment(writer, 1L, 0, 3);
+        writer.commit();
+
+        LuceneMerger merger = new LuceneMerger(writer, new LuceneDataFormat(), dataPath);
+
+        Segment segment = Segment.builder(99L).build();
+        MergeInput input = MergeInput.builder().addSegment(segment).newWriterGeneration(100L).build();
+
+        MergeResult result = merger.merge(input);
+        assertNotNull(result);
+        assertTrue(result.getMergedWriterFileSet().isEmpty());
+    }
+
+    /**
+     * Merge with RowIdMapping remaps ___row_id doc values AND reorders documents.
+     * Verifies that the merged segment has documents sorted by remapped row IDs
+     * and that stored fields follow the documents to their new positions.
+     *
+     * The mapping preserves within-segment order (ascending remapped values within
+     * each generation), matching real Parquet merge behavior where rows within each
+     * source file maintain their relative order in the merged output.
+     */
+    public void testMergeWithRowIdMappingRemapsRowIds() throws IOException {
+        // gen=1: doc_0 (rowId=0), doc_1 (rowId=1), doc_2 (rowId=2)
+        // gen=2: doc_3 (rowId=0), doc_4 (rowId=1)
+        writeSegment(writer, 1L, 0, 3);
+        writeSegment(writer, 2L, 3, 2);
+        writer.commit();
+
+        assertEquals(5, writer.getDocStats().numDocs);
+
+        // Mapping interleaves segments but preserves within-segment order:
+        // gen=1: 0→0, 1→2, 2→4 (ascending within gen=1)
+        // gen=2: 0→1, 1→3 (ascending within gen=2)
+        //
+        // This simulates a Parquet merge that interleaves rows from two files:
+        // merged output: gen1-row0, gen2-row0, gen1-row1, gen2-row1, gen1-row2
+        //
+        // Expected sorted order by remapped rowId:
+        // position 0: rowId=0 → doc_0 (gen=1, original rowId=0)
+        // position 1: rowId=1 → doc_3 (gen=2, original rowId=0)
+        // position 2: rowId=2 → doc_1 (gen=1, original rowId=1)
+        // position 3: rowId=3 → doc_4 (gen=2, original rowId=1)
+        // position 4: rowId=4 → doc_2 (gen=1, original rowId=2)
+        Map> mapping = new HashMap<>();
+        mapping.put(1L, Map.of(0L, 0L, 1L, 2L, 2L, 4L));
+        mapping.put(2L, Map.of(0L, 1L, 1L, 3L));
+        RowIdMapping rowIdMapping = (oldId, oldGeneration) -> {
+            Map genMap = mapping.get(oldGeneration);
+            if (genMap != null && genMap.containsKey(oldId)) {
+                return genMap.get(oldId);
+            }
+            return oldId;
+        };
+
+        LuceneMerger merger = new LuceneMerger(writer, new LuceneDataFormat(), dataPath);
+        SegmentInfos infos = getSegmentInfos(writer);
+        List segments = buildSegments(infos);
+
+        MergeInput input = MergeInput.builder().segments(segments).rowIdMapping(rowIdMapping).newWriterGeneration(10L).build();
+
+        MergeResult result = merger.merge(input);
+        assertNotNull(result);
+        assertTrue(result.rowIdMapping().isPresent());
+
+        writer.commit();
+
+        // Expected: documents sorted by remapped rowId, with correct stored fields
+        String[] expectedIds = { "doc_0", "doc_3", "doc_1", "doc_4", "doc_2" };
+        long[] expectedRowIds = { 0, 1, 2, 3, 4 };
+
+        try (DirectoryReader reader = DirectoryReader.open(writer)) {
+            // Find the merged segment (should be the largest leaf after old segments are deleted)
+            LeafReaderContext mergedLeaf = null;
+            for (LeafReaderContext ctx : reader.leaves()) {
+                if (mergedLeaf == null || ctx.reader().maxDoc() > mergedLeaf.reader().maxDoc()) {
+                    mergedLeaf = ctx;
+                }
+            }
+            assertNotNull("Should have at least one leaf", mergedLeaf);
+            assertEquals("Merged segment should have 5 docs", 5, mergedLeaf.reader().maxDoc());
+
+            SortedNumericDocValues rowIdDV = mergedLeaf.reader().getSortedNumericDocValues(ROW_ID_FIELD);
+            assertNotNull("___row_id doc values should exist", rowIdDV);
+
+            for (int i = 0; i < 5; i++) {
+                // Verify ___row_id value
+                assertTrue("Should have doc values for doc " + i, rowIdDV.advanceExact(i));
+                long actualRowId = rowIdDV.nextValue();
+                assertEquals("Doc at position " + i + " should have ___row_id=" + expectedRowIds[i], expectedRowIds[i], actualRowId);
+
+                // Verify stored field follows the document
+                Document doc = mergedLeaf.reader().storedFields().document(i);
+                assertEquals("Doc at position " + i + " should be " + expectedIds[i], expectedIds[i], doc.get("id"));
+            }
+        }
+    }
+
+    /**
+     * Merge preserves keyword, numeric, and stored field data integrity.
+     *
+     * 
      Uses an identity {@link RowIdMapping} so the merge exercises the real
+     * secondary-format path; the assertions focus on field-data survival rather
+     * than on row-id remapping (which is covered by
+     * {@link #testMergeWithRowIdMappingRemapsRowIds()}).
+     */
+    public void testMergePreservesFieldDataIntegrity() throws IOException {
+        writeSegmentWithRichFields(writer, 1L, 0, 3);
+        writeSegmentWithRichFields(writer, 2L, 3, 2);
+        writer.commit();
+
+        LuceneMerger merger = new LuceneMerger(writer, new LuceneDataFormat(), dataPath);
+        SegmentInfos infos = getSegmentInfos(writer);
+        List segments = buildSegments(infos);
+
+        // Identity mapping — writeSegmentWithRichFields already writes globally-unique row IDs
+        // (0,1,2 in gen=1 and 3,4 in gen=2), so returning the original row ID is well-formed.
+        RowIdMapping identityMapping = (oldId, oldGeneration) -> oldId;
+
+        MergeInput input = MergeInput.builder().segments(segments).rowIdMapping(identityMapping).newWriterGeneration(10L).build();
+        merger.merge(input);
+        writer.commit();
+
+        try (DirectoryReader reader = DirectoryReader.open(writer)) {
+            assertTrue("Should have at least 5 docs after merge", reader.numDocs() >= 5);
+            for (LeafReaderContext ctx : reader.leaves()) {
+                for (int i = 0; i < ctx.reader().maxDoc(); i++) {
+                    Document doc = ctx.reader().storedFields().document(i);
+                    String id = doc.get("id");
+                    assertNotNull("id field missing", id);
+                    String storedData = doc.get("data");
+                    assertNotNull("stored data field missing for " + id, storedData);
+                    assertTrue("data should contain the doc id", storedData.contains(id));
+                    String numericStr = doc.get("score");
+                    assertNotNull("stored numeric field missing for " + id, numericStr);
+                }
+            }
+        }
+    }
+
+    /**
+     * Constructor with null IndexWriter throws IllegalArgumentException.
+     */
+    public void testConstructorWithNullIndexWriterThrows() {
+        expectThrows(IllegalArgumentException.class, () -> new LuceneMerger(null, new LuceneDataFormat(), Path.of(".")));
+    }
+
+    // ========== Helper Methods ==========
+
+    private void writeSegment(IndexWriter w, long generation, int startRowId, int numDocs) throws IOException {
+        for (int i = 0; i < numDocs; i++) {
+            Document doc = new Document();
+            doc.add(new StringField("id", "doc_" + (startRowId + i), Field.Store.YES));
+            doc.add(new StoredField("data", "value_for_doc_" + (startRowId + i)));
+            // ___row_id is local to the segment: 0, 1, 2, ... (matches how the real system works)
+            doc.add(new SortedNumericDocValuesField(ROW_ID_FIELD, i));
+            w.addDocument(doc);
+        }
+        w.flush();
+        setWriterGenerationOnLatestSegment(w, generation);
+    }
+
+    private void writeSegmentWithRichFields(IndexWriter w, long generation, int startRowId, int numDocs) throws IOException {
+        for (int i = 0; i < numDocs; i++) {
+            int docIdx = startRowId + i;
+            Document doc = new Document();
+            doc.add(new StringField("id", "doc_" + docIdx, Field.Store.YES));
+            doc.add(new StoredField("data", "rich_data_for_doc_" + docIdx));
+            doc.add(new StoredField("score", String.valueOf(docIdx * 10)));
+            doc.add(new SortedNumericDocValuesField(ROW_ID_FIELD, docIdx));
+            doc.add(new SortedNumericDocValuesField("score_dv", docIdx * 10));
+            w.addDocument(doc);
+        }
+        w.flush();
+        setWriterGenerationOnLatestSegment(w, generation);
+    }
+
+    @SuppressForbidden(reason = "Need reflection to stamp writer_generation on segments for testing")
+    private void setWriterGenerationOnLatestSegment(IndexWriter w, long generation) throws IOException {
+        try {
+            java.lang.reflect.Field segInfosField = IndexWriter.class.getDeclaredField("segmentInfos");
+            segInfosField.setAccessible(true);
+            SegmentInfos segInfos = (SegmentInfos) segInfosField.get(w);
+            if (segInfos.size() > 0) {
+                SegmentCommitInfo lastSegment = segInfos.asList().get(segInfos.size() - 1);
+                if (lastSegment.info.getAttribute(WRITER_GENERATION_ATTRIBUTE) == null) {
+                    lastSegment.info.putAttribute(WRITER_GENERATION_ATTRIBUTE, String.valueOf(generation));
+                }
+            }
+        } catch (ReflectiveOperationException e) {
+            throw new IOException("Failed to set writer_generation attribute via reflection", e);
+        }
+    }
+
+    @SuppressForbidden(reason = "Need reflection to access live SegmentInfos for test assertions")
+    private SegmentInfos getSegmentInfos(IndexWriter w) throws IOException {
+        try {
+            java.lang.reflect.Field segInfosField = IndexWriter.class.getDeclaredField("segmentInfos");
+            segInfosField.setAccessible(true);
+            return (SegmentInfos) segInfosField.get(w);
+        } catch (ReflectiveOperationException e) {
+            throw new IOException("Failed to access segmentInfos via reflection", e);
+        }
+    }
+
+    private List buildSegments(SegmentInfos infos) {
+        List segments = new ArrayList<>();
+        for (SegmentCommitInfo sci : infos.asList()) {
+            String genAttr = sci.info.getAttribute(WRITER_GENERATION_ATTRIBUTE);
+            if (genAttr != null) {
+                long generation = Long.parseLong(genAttr);
+                segments.add(Segment.builder(generation).build());
+            }
+        }
+        return segments;
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneReaderManagerTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneReaderManagerTests.java
index 24c13fc342024..6c24162078f05 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneReaderManagerTests.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneReaderManagerTests.java
@@ -15,6 +15,8 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;
@@ -22,6 +24,8 @@
 import org.apache.lucene.store.NIOFSDirectory;
 import org.opensearch.be.lucene.index.LuceneCommitter;
 import org.opensearch.be.lucene.index.LuceneIndexingExecutionEngine;
+import org.opensearch.be.lucene.index.LuceneWriter;
+import org.opensearch.common.SuppressForbidden;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.IndexSettings;
@@ -99,6 +103,17 @@ private DirectoryReader openReader() throws IOException {
     }
 
     private CatalogSnapshot stubSnapshot(long generation) {
+        return stubSnapshot(generation, List.of());
+    }
+
+    /**
+     * Builds a stub snapshot whose segment list contains the given writer generations.
+     * This is required by {@link LuceneReaderManager#afterRefresh}'s assertion, which
+     * compares the snapshot's segment generations against the writer-generation attribute
+     * on each leaf in the refreshed {@link DirectoryReader}.
+     */
+    private CatalogSnapshot stubSnapshot(long generation, List segmentGenerations) {
+        List segs = segmentGenerations.stream().map(g -> Segment.builder(g).build()).toList();
         return new CatalogSnapshot("test", generation, 1) {
             @Override
             protected void closeInternal() {}
@@ -115,7 +130,7 @@ public long getId() {
 
             @Override
             public List getSegments() {
-                return List.of();
+                return segs;
             }
 
             @Override
@@ -168,11 +183,36 @@ public Collection getFiles(boolean includeSegmentsFile) {
         };
     }
 
-    private void addDoc(String id) throws IOException {
+    private void addDoc(String id, long generation) throws IOException {
         Document doc = new Document();
         doc.add(new StringField("id", id, Field.Store.YES));
         indexWriter.addDocument(doc);
         indexWriter.commit();
+        stampLatestSegmentGeneration(generation);
+    }
+
+    /**
+     * Stamps the most recently written segment with the {@code writer_generation} attribute
+     * that {@link LuceneReaderManager#afterRefresh}'s assertion expects. In production this
+     * is done by {@code LuceneWriterCodec}; tests that write directly through a plain
+     * {@link IndexWriter} must stamp it themselves.
+     */
+    @SuppressForbidden(reason = "Need reflection to stamp writer_generation on segments for testing")
+    private void stampLatestSegmentGeneration(long generation) throws IOException {
+        try {
+            java.lang.reflect.Field segInfosField = IndexWriter.class.getDeclaredField("segmentInfos");
+            segInfosField.setAccessible(true);
+            SegmentInfos segInfos = (SegmentInfos) segInfosField.get(indexWriter);
+            if (segInfos.size() == 0) {
+                return;
+            }
+            SegmentCommitInfo last = segInfos.asList().get(segInfos.size() - 1);
+            if (last.info.getAttribute(LuceneWriter.WRITER_GENERATION_ATTRIBUTE) == null) {
+                last.info.putAttribute(LuceneWriter.WRITER_GENERATION_ATTRIBUTE, String.valueOf(generation));
+            }
+        } catch (ReflectiveOperationException e) {
+            throw new IOException("Failed to stamp writer_generation via reflection", e);
+        }
     }
 
     public void testAfterRefreshCreatesReader() throws IOException {
@@ -195,21 +235,24 @@ public void testAfterRefreshNoOpWhenDidRefreshFalse() throws IOException {
     public void testMultipleRefreshesWithIndexing() throws IOException {
         LuceneReaderManager rm = new LuceneReaderManager(dataFormat, openReader());
 
+        // Empty initial reader — no segments yet.
         CatalogSnapshot snap1 = stubSnapshot(1);
         rm.afterRefresh(true, snap1);
         DirectoryReader reader1 = rm.getReader(snap1);
         assertEquals(0, new IndexSearcher(reader1).count(new MatchAllDocsQuery()));
 
-        addDoc("doc1");
-        CatalogSnapshot snap2 = stubSnapshot(2);
+        // Add doc1 in generation 10, refresh. Reader now has one leaf stamped with gen=10.
+        addDoc("doc1", 10L);
+        CatalogSnapshot snap2 = stubSnapshot(2, List.of(10L));
         rm.afterRefresh(true, snap2);
         DirectoryReader reader2 = rm.getReader(snap2);
         assertEquals(1, new IndexSearcher(reader2).count(new MatchAllDocsQuery()));
 
         assertEquals(0, new IndexSearcher(reader1).count(new MatchAllDocsQuery()));
 
-        addDoc("doc2");
-        CatalogSnapshot snap3 = stubSnapshot(3);
+        // Add doc2 in generation 20. Reader now has two leaves stamped with gens {10, 20}.
+        addDoc("doc2", 20L);
+        CatalogSnapshot snap3 = stubSnapshot(3, List.of(10L, 20L));
         rm.afterRefresh(true, snap3);
         DirectoryReader reader3 = rm.getReader(snap3);
         assertEquals(2, new IndexSearcher(reader3).count(new MatchAllDocsQuery()));
@@ -286,7 +329,7 @@ public void testCreateReaderManagerWithLuceneIndexingEngine() throws IOException
             )
             .retentionLeasesSupplier(() -> new RetentionLeases(0, 0, java.util.Collections.emptyList()))
             .build();
-        CommitterConfig cs = new CommitterConfig(engineConfig);
+        CommitterConfig cs = new CommitterConfig(engineConfig, () -> {});
         LuceneCommitter committer = new LuceneCommitter(cs);
 
         try {
diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterCSManagerIntegrationTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterCSManagerIntegrationTests.java
index 6e2a9b2af93bd..316396fbbb531 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterCSManagerIntegrationTests.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterCSManagerIntegrationTests.java
@@ -121,7 +121,9 @@ private TestEnv createTestEnv() throws IOException {
             shardPath
         );
         store.createEmpty(org.apache.lucene.util.Version.LATEST);
-        LuceneCommitter committer = new LuceneCommitter(new CommitterConfig(buildEngineConfig(indexSettings, store, shardId, translogDir)));
+        LuceneCommitter committer = new LuceneCommitter(
+            new CommitterConfig(buildEngineConfig(indexSettings, store, shardId, translogDir), () -> {})
+        );
         Path parquetDir = dataPath.resolve(PARQUET_FORMAT);
         Files.createDirectories(parquetDir);
         return new TestEnv(committer, store, shardPath, indexDir, parquetDir, translogDir);
@@ -495,7 +497,7 @@ public void testRecoveryAfterCrashTrimsUnsafeCommits() throws Exception {
             );
             store.createEmpty(org.apache.lucene.util.Version.LATEST);
             LuceneCommitter committer = new LuceneCommitter(
-                new CommitterConfig(buildEngineConfig(indexSettings, store, shardId, translogDir))
+                new CommitterConfig(buildEngineConfig(indexSettings, store, shardId, translogDir), () -> {})
             );
 
             lucene0 = ingestLuceneDocs(committer, store);
@@ -569,7 +571,7 @@ public void testRecoveryAfterCrashTrimsUnsafeCommits() throws Exception {
                 shardPath
             );
             LuceneCommitter committer = new LuceneCommitter(
-                new CommitterConfig(buildEngineConfig(indexSettings, store, shardId, translogDir))
+                new CommitterConfig(buildEngineConfig(indexSettings, store, shardId, translogDir), () -> {})
             );
 
             assertEquals("Only safe commit remains", 1, DirectoryReader.listCommits(store.directory()).size());
@@ -630,7 +632,7 @@ public void testRecoveryThenNormalOperationWorks() throws Exception {
             );
             store.createEmpty(org.apache.lucene.util.Version.LATEST);
             LuceneCommitter committer = new LuceneCommitter(
-                new CommitterConfig(buildEngineConfig(indexSettings, store, shardId, translogDir))
+                new CommitterConfig(buildEngineConfig(indexSettings, store, shardId, translogDir), () -> {})
             );
 
             lucene0 = ingestLuceneDocs(committer, store);
@@ -685,7 +687,7 @@ public void testRecoveryThenNormalOperationWorks() throws Exception {
                 shardPath
             );
             LuceneCommitter committer = new LuceneCommitter(
-                new CommitterConfig(buildEngineConfig(indexSettings, store, shardId, translogDir))
+                new CommitterConfig(buildEngineConfig(indexSettings, store, shardId, translogDir), () -> {})
             );
 
             assertEquals(1, DirectoryReader.listCommits(store.directory()).size());
diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterFactoryTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterFactoryTests.java
index 8fe31b03364b2..022da0e14cd1b 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterFactoryTests.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterFactoryTests.java
@@ -56,7 +56,7 @@ public void testGetCommitterReturnsLuceneCommitter() throws IOException {
                 .retentionLeasesSupplier(() -> new RetentionLeases(0, 0, Collections.emptyList()))
                 .build();
             LuceneCommitterFactory committerFactory = new LuceneCommitterFactory();
-            committer = committerFactory.getCommitter(new CommitterConfig(engineConfig));
+            committer = committerFactory.getCommitter(new CommitterConfig(engineConfig, () -> {}));
 
             assertTrue("getCommitter() should return a LuceneCommitter instance", committer instanceof LuceneCommitter);
         } finally {
diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterTests.java
index 05bdca5c79170..9ee9581f38315 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterTests.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneCommitterTests.java
@@ -97,7 +97,7 @@ private CommitterConfig createCommitterConfig() throws IOException {
             null,
             null
         );
-        return new CommitterConfig(engineConfig);
+        return new CommitterConfig(engineConfig, () -> {});
     }
 
     public void testConstructorOpensIndexWriter() throws IOException {
diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java
index a84d1c3b782b9..c2d6589a46631 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneIndexingExecutionEngineTests.java
@@ -126,7 +126,7 @@ private LuceneCommitter createCommitter() throws IOException {
             null,
             null
         );
-        CommitterConfig settings = new CommitterConfig(engineConfig);
+        CommitterConfig settings = new CommitterConfig(engineConfig, () -> {});
         return new LuceneCommitter(settings);
     }
 
@@ -167,7 +167,7 @@ public void testRefreshIncorporatesLuceneSegments() throws IOException {
         when(textField.name()).thenReturn("content");
 
         long generation = 1L;
-        try (LuceneWriter luceneWriter = new LuceneWriter(generation, luceneDataFormat, tempBase, null, Codec.getDefault())) {
+        try (LuceneWriter luceneWriter = new LuceneWriter(generation, luceneDataFormat, tempBase, null, Codec.getDefault(), null)) {
             for (int i = 0; i < numDocs; i++) {
                 LuceneDocumentInput input = new LuceneDocumentInput();
                 input.addField(textField, "doc_" + i);
diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java
index 55843ea1d6779..04494e8e0296b 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/index/LuceneWriterTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -64,7 +64,7 @@ private MappedFieldType mockKeywordField(String name) {
 
     public void testAddDocAndFlushProducesSingleSegment() throws IOException {
         Path baseDir = createTempDir();
-        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault())) {
+        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
             int numDocs = randomIntBetween(5, 20);
             MappedFieldType textField = mockTextField("content");
             for (int i = 0; i < numDocs; i++) {
@@ -95,7 +95,7 @@ public void testRowIdMatchesLuceneDocId() throws IOException {
         Path baseDir = createTempDir();
         int numDocs = randomIntBetween(10, 50);
         MappedFieldType textField = mockTextField("content");
-        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault())) {
+        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
             for (int i = 0; i < numDocs; i++) {
                 LuceneDocumentInput input = new LuceneDocumentInput();
                 input.addField(textField, "doc " + i);
@@ -109,11 +109,11 @@ public void testRowIdMatchesLuceneDocId() throws IOException {
             try (NIOFSDirectory dir = new NIOFSDirectory(Path.of(wfs.directory())); IndexReader reader = DirectoryReader.open(dir)) {
                 for (LeafReaderContext ctx : reader.leaves()) {
                     LeafReader leafReader = ctx.reader();
-                    NumericDocValues rowIdValues = leafReader.getNumericDocValues(LuceneDocumentInput.ROW_ID_FIELD);
+                    SortedNumericDocValues rowIdValues = leafReader.getSortedNumericDocValues(LuceneDocumentInput.ROW_ID_FIELD);
                     assertNotNull("row_id doc values should exist", rowIdValues);
                     for (int docId = 0; docId < leafReader.maxDoc(); docId++) {
                         assertTrue(rowIdValues.advanceExact(docId));
-                        assertThat("row ID should equal Lucene doc ID", rowIdValues.longValue(), equalTo((long) docId));
+                        assertThat("row ID should equal Lucene doc ID", rowIdValues.nextValue(), equalTo((long) docId));
                     }
                 }
             }
@@ -122,7 +122,7 @@ public void testRowIdMatchesLuceneDocId() throws IOException {
 
     public void testFlushWithNoDocsReturnsEmpty() throws IOException {
         Path baseDir = createTempDir();
-        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault())) {
+        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
             FileInfos fileInfos = writer.flush();
             assertTrue(fileInfos.writerFilesMap().isEmpty());
         }
@@ -132,7 +132,7 @@ public void testWriterGenerationIsPreserved() throws IOException {
         Path baseDir = createTempDir();
         long gen = randomLongBetween(1, 100);
         MappedFieldType textField = mockTextField("content");
-        try (LuceneWriter writer = new LuceneWriter(gen, dataFormat, baseDir, null, Codec.getDefault())) {
+        try (LuceneWriter writer = new LuceneWriter(gen, dataFormat, baseDir, null, Codec.getDefault(), null)) {
             assertThat(writer.generation(), equalTo(gen));
 
             LuceneDocumentInput input = new LuceneDocumentInput();
@@ -149,7 +149,7 @@ public void testWriterGenerationIsPreserved() throws IOException {
     public void testKeywordFieldsAreIndexed() throws IOException {
         Path baseDir = createTempDir();
         MappedFieldType keywordField = mockKeywordField("status");
-        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault())) {
+        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
             LuceneDocumentInput input = new LuceneDocumentInput();
             input.addField(keywordField, "active");
             input.setRowId(LuceneDocumentInput.ROW_ID_FIELD, 0);
@@ -171,7 +171,7 @@ public void testUnsupportedFieldTypeIsSilentlySkipped() throws IOException {
         when(numericField.typeName()).thenReturn("integer");
         when(numericField.name()).thenReturn("count");
 
-        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault())) {
+        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
             LuceneDocumentInput input = new LuceneDocumentInput();
             // Should not throw — unsupported types are silently skipped (handled by other formats)
             input.addField(numericField, 42);
@@ -185,7 +185,7 @@ public void testMixedTextAndKeywordFields() throws IOException {
         MappedFieldType textField = mockTextField("title");
         MappedFieldType keywordField = mockKeywordField("category");
 
-        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault())) {
+        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
             int numDocs = randomIntBetween(5, 15);
             for (int i = 0; i < numDocs; i++) {
                 LuceneDocumentInput input = new LuceneDocumentInput();
@@ -212,7 +212,7 @@ public void testWriteAndFlushEndToEndWithTextAndKeyword() throws IOException {
         MappedFieldType keywordField = mockKeywordField("status");
         int numDocs = randomIntBetween(5, 20);
 
-        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault())) {
+        try (LuceneWriter writer = new LuceneWriter(1L, dataFormat, baseDir, null, Codec.getDefault(), null)) {
             for (int i = 0; i < numDocs; i++) {
                 LuceneDocumentInput input = new LuceneDocumentInput();
                 input.addField(textField, "hello world " + i);
@@ -232,11 +232,11 @@ public void testWriteAndFlushEndToEndWithTextAndKeyword() throws IOException {
 
                 // Verify row IDs match doc IDs
                 LeafReader leafReader = reader.leaves().get(0).reader();
-                NumericDocValues rowIdValues = leafReader.getNumericDocValues(LuceneDocumentInput.ROW_ID_FIELD);
+                SortedNumericDocValues rowIdValues = leafReader.getSortedNumericDocValues(LuceneDocumentInput.ROW_ID_FIELD);
                 assertNotNull(rowIdValues);
                 for (int docId = 0; docId < numDocs; docId++) {
                     assertTrue(rowIdValues.advanceExact(docId));
-                    assertThat(rowIdValues.longValue(), equalTo((long) docId));
+                    assertThat(rowIdValues.nextValue(), equalTo((long) docId));
                 }
 
                 // Verify text field is searchable via TermQuery
@@ -263,8 +263,8 @@ public void testMultipleWriterGenerationsProduceIsolatedSegments() throws IOExce
 
         // Create both writers without closing them until after verification,
         // because close() deletes the temp directory.
-        LuceneWriter writer1 = new LuceneWriter(gen1, dataFormat, baseDir, null, Codec.getDefault());
-        LuceneWriter writer2 = new LuceneWriter(gen2, dataFormat, baseDir, null, Codec.getDefault());
+        LuceneWriter writer1 = new LuceneWriter(gen1, dataFormat, baseDir, null, Codec.getDefault(), null);
+        LuceneWriter writer2 = new LuceneWriter(gen2, dataFormat, baseDir, null, Codec.getDefault(), null);
         try {
             for (int i = 0; i < numDocs1; i++) {
                 LuceneDocumentInput input = new LuceneDocumentInput();
diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java
index 0ce2f195a4b0d..634a6902899b8 100644
--- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java
+++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeMergeIT.java
@@ -10,6 +10,12 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
 
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.NIOFSDirectory;
 import org.opensearch.action.admin.indices.refresh.RefreshResponse;
 import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.opensearch.action.admin.indices.stats.ShardStats;
@@ -59,6 +65,10 @@ public class CompositeMergeIT extends OpenSearchIntegTestCase {
     private static final String INDEX_NAME = "test-composite-merge";
     private static final String MERGE_ENABLED_PROPERTY = "opensearch.pluggable.dataformat.merge.enabled";
 
+    // ══════════════════════════════════════════════════════════════════════
+    // Framework lifecycle & configuration
+    // ══════════════════════════════════════════════════════════════════════
+
     @Override
     public void setUp() throws Exception {
         enableMerge();
@@ -76,16 +86,6 @@ public void tearDown() throws Exception {
         disableMerge();
     }
 
-    @SuppressForbidden(reason = "enable pluggable dataformat merge for integration testing")
-    private static void enableMerge() {
-        System.setProperty(MERGE_ENABLED_PROPERTY, "true");
-    }
-
-    @SuppressForbidden(reason = "restore pluggable dataformat merge property after test")
-    private static void disableMerge() {
-        System.clearProperty(MERGE_ENABLED_PROPERTY);
-    }
-
     @Override
     protected Collection> nodePlugins() {
         return Arrays.asList(ParquetDataFormatPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class, DataFusionPlugin.class);
@@ -99,6 +99,10 @@ protected Settings nodeSettings(int nodeOrdinal) {
             .build();
     }
 
+    // ══════════════════════════════════════════════════════════════════════
+    // Tests
+    // ══════════════════════════════════════════════════════════════════════
+
     /**
      * Verifies background merge produces a valid merged parquet file
      * with correct row count and source files cleaned up.
@@ -175,7 +179,137 @@ public void testSortedMerge() throws Exception {
         verifyNoOrphanFiles(snapshot);
     }
 
-    // ── Settings ──
+    /**
+     * Verifies composite merge with Parquet as primary and Lucene as secondary:
+     * 
        
+     *   
      1. Merge reduces segment count (merge actually happened)
        2. 
+     *   
      2. Both "parquet" and "lucene" entries exist in the catalog snapshot
        3. 
+     *   
      3. Merged parquet files have correct total row count
        4. 
+     *   
      4. Merged lucene directory has correct total document count
        5. 
+     *   
      5. Lucene documents have monotonically increasing __row_id__ doc values
+     *       (confirms RowIdMapping was applied during secondary merge)
        6. 
+     *   
      6. Cross-format validation: parquet row count == lucene doc count for each merged segment
        7. 
+     * 
      
+     */
+    public void testParquetPrimaryLuceneSecondaryMerge() throws Exception {
+        client().admin()
+            .indices()
+            .prepareCreate(INDEX_NAME)
+            .setSettings(parquetPrimaryLuceneSecondarySettings())
+            .setMapping("name", "type=keyword", "age", "type=integer")
+            .get();
+        ensureGreen(INDEX_NAME);
+
+        // Index documents to create multiple segments. Using 15 cycles keeps the workload
+        // in line with the other stable composite-merge tests and avoids triggering a second
+        // cascaded merge before the first one commits.
+        int docsPerCycle = 5;
+        int refreshCycles = 15;
+        indexDocsAcrossMultipleRefreshes(refreshCycles, docsPerCycle);
+        int totalDocs = refreshCycles * docsPerCycle;
+
+        // Wait for merge to reduce segment count
+        assertBusy(() -> {
+            flush(INDEX_NAME);
+            DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot();
+            assertTrue(
+                "Expected merges to reduce segment count below " + refreshCycles + ", but got: " + snapshot.getSegments().size(),
+                snapshot.getSegments().size() < refreshCycles
+            );
+        });
+
+        MergeStats mergeStats = getMergeStats();
+        assertTrue("Expected at least one merge to have occurred", mergeStats.getTotal() > 0);
+
+        DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot();
+
+        // Both formats must be present in the catalog
+        Set formats = snapshot.getDataFormats();
+        assertTrue("Catalog should contain 'parquet' format, got: " + formats, formats.contains("parquet"));
+        assertTrue("Catalog should contain 'lucene' format, got: " + formats, formats.contains("lucene"));
+
+        // Verify parquet merged files have correct row count
+        verifyRowCount(snapshot, totalDocs);
+
+        // Verify lucene merged directory has correct doc count
+        verifyLuceneDocCount(totalDocs);
+
+        // Verify lucene __row_id__ values are monotonically increasing (RowIdMapping applied)
+        verifyLuceneRowIdSequential();
+
+        // Cross-format validation: for each segment, parquet rows == lucene segment docs
+        verifyCrossFormatConsistency(snapshot);
+    }
+
+    /**
+     * Verifies sorted composite merge with Parquet primary (sorted) + Lucene secondary:
+     * 
        
+     *   
      1. Merge reduces segment count
        2. 
+     *   
      2. Merged parquet files are sorted by age DESC (nulls first), name ASC (nulls last)
        3. 
+     *   
      3. Lucene __row_id__ values are sequential (RowIdMapping applied)
        4. 
+     *   
      4. Cross-format consistency: parquet rows match lucene docs by row_id
        5. 
+     * 
      
+     *
+     * This is the critical test for RowIdMapping correctness in sorted merges —
+     * the primary format reorders rows during merge, and the secondary must apply
+     * the same reordering via the mapping.
+     */
+    public void testSortedParquetPrimaryLuceneSecondaryMerge() throws Exception {
+        client().admin()
+            .indices()
+            .prepareCreate(INDEX_NAME)
+            .setSettings(sortedParquetPrimaryLuceneSecondarySettings())
+            .setMapping("name", "type=keyword", "age", "type=integer")
+            .get();
+        ensureGreen(INDEX_NAME);
+
+        int docsPerCycle = 10;
+        int refreshCycles = 15;
+        indexDocsWithNullsAcrossRefreshes(refreshCycles, docsPerCycle);
+        int totalDocs = refreshCycles * docsPerCycle;
+
+        assertBusy(() -> {
+            flush(INDEX_NAME);
+            DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot();
+            assertTrue(
+                "Expected merges to reduce segment count below " + refreshCycles + ", but got: " + snapshot.getSegments().size(),
+                snapshot.getSegments().size() < refreshCycles
+            );
+        });
+
+        MergeStats mergeStats = getMergeStats();
+        assertTrue("Expected at least one merge to have occurred", mergeStats.getTotal() > 0);
+
+        DataformatAwareCatalogSnapshot snapshot = getCatalogSnapshot();
+
+        Set formats = snapshot.getDataFormats();
+        assertTrue("Catalog should contain 'parquet'", formats.contains("parquet"));
+        assertTrue("Catalog should contain 'lucene'", formats.contains("lucene"));
+
+        verifyRowCount(snapshot, totalDocs);
+        verifySortOrder(snapshot);
+        verifyLuceneDocCount(totalDocs);
+        verifyLuceneRowIdSequential();
+        verifyCrossFormatConsistency(snapshot);
+    }
+
+    // ══════════════════════════════════════════════════════════════════════
+    // Private helpers: merge feature flag
+    // ══════════════════════════════════════════════════════════════════════
+
+    @SuppressForbidden(reason = "enable pluggable dataformat merge for integration testing")
+    private static void enableMerge() {
+        System.setProperty(MERGE_ENABLED_PROPERTY, "true");
+    }
+
+    @SuppressForbidden(reason = "restore pluggable dataformat merge property after test")
+    private static void disableMerge() {
+        System.clearProperty(MERGE_ENABLED_PROPERTY);
+    }
+
+    // ══════════════════════════════════════════════════════════════════════
+    // Private helpers: index settings
+    // ══════════════════════════════════════════════════════════════════════
 
     private Settings unsortedSettings() {
         return Settings.builder()
@@ -204,7 +338,36 @@ private Settings sortedSettings() {
             .build();
     }
 
-    // ── Indexing ──
+    private Settings parquetPrimaryLuceneSecondarySettings() {
+        return Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put("index.refresh_interval", "-1")
+            .put("index.pluggable.dataformat.enabled", true)
+            .put("index.pluggable.dataformat", "composite")
+            .put("index.composite.primary_data_format", "parquet")
+            .putList("index.composite.secondary_data_formats", "lucene")
+            .build();
+    }
+
+    private Settings sortedParquetPrimaryLuceneSecondarySettings() {
+        return Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put("index.refresh_interval", "-1")
+            .put("index.pluggable.dataformat.enabled", true)
+            .put("index.pluggable.dataformat", "composite")
+            .put("index.composite.primary_data_format", "parquet")
+            .putList("index.composite.secondary_data_formats", "lucene")
+            .putList("index.sort.field", "age", "name")
+            .putList("index.sort.order", "desc", "asc")
+            .putList("index.sort.missing", "_first", "_last")
+            .build();
+    }
+
+    // ══════════════════════════════════════════════════════════════════════
+    // Private helpers: indexing
+    // ══════════════════════════════════════════════════════════════════════
 
     private void indexDocsAcrossMultipleRefreshes(int refreshCycles, int docsPerCycle) {
         for (int cycle = 0; cycle < refreshCycles; cycle++) {
@@ -239,7 +402,9 @@ private void indexDocsWithNullsAcrossRefreshes(int refreshCycles, int docsPerCyc
         }
     }
 
-    // ── Verification ──
+    // ══════════════════════════════════════════════════════════════════════
+    // Private helpers: verification
+    // ══════════════════════════════════════════════════════════════════════
 
     private void verifyRowCount(DataformatAwareCatalogSnapshot snapshot, int expectedTotalDocs) throws IOException {
         Path parquetDir = getParquetDir();
@@ -351,11 +516,181 @@ private void verifySortOrder(DataformatAwareCatalogSnapshot snapshot) throws Exc
         }
     }
 
+    private void verifyLuceneDocCount(int expectedTotalDocs) throws IOException {
+        Path luceneDir = getLuceneDir();
+        assertTrue("Lucene directory should exist: " + luceneDir, Files.exists(luceneDir));
+
+        try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) {
+            assertEquals("Total lucene docs should match ingested docs", expectedTotalDocs, reader.numDocs());
+        }
+    }
+
+    /**
+     * Verifies that __row_id__ doc values in merged lucene segments are sequential
+     * (0, 1, 2, ...) within each leaf. This confirms the RowIdMapping from the primary
+     * (Parquet) merge was correctly applied to reorder Lucene documents.
+     *
+     * Sequential (not just monotonic) is required because the RowIdMapping produces
+     * a dense mapping — every position from 0..N-1 must be covered.
+     */
+    private void verifyLuceneRowIdSequential() throws IOException {
+        Path luceneDir = getLuceneDir();
+
+        try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) {
+            for (LeafReaderContext ctx : reader.leaves()) {
+                SortedNumericDocValues rowIdDV = ctx.reader().getSortedNumericDocValues("__row_id__");
+                if (rowIdDV == null) continue;
+
+                long expectedRowId = 0;
+                for (int doc = 0; doc < ctx.reader().maxDoc(); doc++) {
+                    if (rowIdDV.advanceExact(doc)) {
+                        long rowId = rowIdDV.nextValue();
+                        assertEquals(
+                            "__row_id__ should be sequential within segment, expected "
+                                + expectedRowId
+                                + " but got "
+                                + rowId
+                                + " at doc "
+                                + doc,
+                            expectedRowId,
+                            rowId
+                        );
+                        expectedRowId++;
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Cross-format data comparison: reads merged parquet file content and merged lucene
+     * segments, then verifies that for each row in parquet (identified by __row_id__),
+     * the corresponding Lucene document (sorted by __row_id__) has matching field values.
+     *
+     * Compares both numeric (age) and keyword (name) fields to ensure the RowIdMapping
+     * correctly synchronized the two formats during merge.
+     *
+     * 
      Note: {@code __row_id__} is only unique within a catalog segment
+     * (each segment starts row_ids at 0), so rows must be grouped per segment — a global
+     * map would silently overwrite rows from segments that happen to share row_ids.
+     * Each Lucene leaf is matched to its parquet segment by row count.
+     */
+    @SuppressForbidden(reason = "JSON parsing for cross-format data comparison")
+    private void verifyCrossFormatConsistency(DataformatAwareCatalogSnapshot snapshot) throws Exception {
+        Path parquetDir = getParquetDir();
+        Path luceneDir = getLuceneDir();
+
+        // Collect parquet rows grouped per catalog segment, indexed by __row_id__
+        // (only unique within a segment, so a per-segment map is required).
+        List>> parquetSegments = new java.util.ArrayList<>();
+        for (Segment segment : snapshot.getSegments()) {
+            WriterFileSet parquetWfs = segment.dfGroupedSearchableFiles().get("parquet");
+            if (parquetWfs == null) continue;
+            Map> rowsInSegment = new java.util.HashMap<>();
+            for (String file : parquetWfs.files()) {
+                Path filePath = parquetDir.resolve(file);
+                if (Files.exists(filePath) == false) continue;
+                String json = RustBridge.readAsJson(filePath.toString());
+                try (
+                    XContentParser parser = JsonXContent.jsonXContent.createParser(
+                        NamedXContentRegistry.EMPTY,
+                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                        json
+                    )
+                ) {
+                    for (Object obj : parser.list()) {
+                        @SuppressWarnings("unchecked")
+                        Map row = (Map) obj;
+                        long rowId = ((Number) row.get("__row_id__")).longValue();
+                        rowsInSegment.put(rowId, row);
+                    }
+                }
+            }
+            if (rowsInSegment.isEmpty() == false) {
+                parquetSegments.add(rowsInSegment);
+            }
+        }
+
+        assertTrue("Should have parquet rows to compare", parquetSegments.isEmpty() == false);
+
+        // For each Lucene leaf, find the parquet segment whose row count matches and
+        // verify every row_id in the leaf resolves to a row in that segment with matching
+        // age/name values.
+        try (Directory dir = NIOFSDirectory.open(luceneDir); DirectoryReader reader = DirectoryReader.open(dir)) {
+            int matchedDocs = 0;
+            int totalLuceneDocs = 0;
+            for (LeafReaderContext ctx : reader.leaves()) {
+                int leafDocs = ctx.reader().maxDoc();
+                totalLuceneDocs += leafDocs;
+
+                Map> matchingSegment = null;
+                for (Map> candidate : parquetSegments) {
+                    if (candidate.size() == leafDocs) {
+                        matchingSegment = candidate;
+                        break;
+                    }
+                }
+                assertNotNull("No parquet segment found with matching row count " + leafDocs, matchingSegment);
+                parquetSegments.remove(matchingSegment);
+
+                SortedNumericDocValues rowIdDV = ctx.reader().getSortedNumericDocValues("__row_id__");
+                SortedNumericDocValues ageDV = ctx.reader().getSortedNumericDocValues("age");
+                SortedSetDocValues nameDV = ctx.reader().getSortedSetDocValues("name");
+
+                if (rowIdDV == null) continue;
+
+                for (int doc = 0; doc < leafDocs; doc++) {
+                    if (rowIdDV.advanceExact(doc) == false) continue;
+                    long luceneRowId = rowIdDV.nextValue();
+
+                    Map parquetRow = matchingSegment.get(luceneRowId);
+                    assertNotNull("Lucene doc with __row_id__=" + luceneRowId + " should have a matching parquet row", parquetRow);
+
+                    // Compare age field
+                    if (ageDV != null && ageDV.advanceExact(doc)) {
+                        long luceneAge = ageDV.nextValue();
+                        Object parquetAge = parquetRow.get("age");
+                        assertNotNull("Parquet row at __row_id__=" + luceneRowId + " should have 'age' field", parquetAge);
+                        assertEquals("Age mismatch at row_id=" + luceneRowId, ((Number) parquetAge).longValue(), luceneAge);
+                    }
+
+                    // Compare name field (keyword stored as sorted set doc values)
+                    if (nameDV != null && nameDV.advanceExact(doc)) {
+                        long ord = nameDV.nextOrd();
+                        if (ord >= 0) {
+                            String luceneName = nameDV.lookupOrd(ord).utf8ToString();
+                            Object parquetName = parquetRow.get("name");
+                            assertNotNull("Parquet row at __row_id__=" + luceneRowId + " should have 'name' field", parquetName);
+                            assertEquals("Name mismatch at row_id=" + luceneRowId, parquetName.toString(), luceneName);
+                        }
+                    }
+
+                    matchedDocs++;
+                }
+            }
+
+            assertTrue("Should have matched at least some docs across formats", matchedDocs > 0);
+            assertEquals("All lucene docs should have matching parquet rows", totalLuceneDocs, matchedDocs);
+        }
+    }
+
+    // ══════════════════════════════════════════════════════════════════════
+    // Private helpers: shard/cluster accessors
+    // ══════════════════════════════════════════════════════════════════════
+
     private Path getParquetDir() {
         IndexShard shard = getPrimaryShard();
         return shard.shardPath().getDataPath().resolve("parquet");
     }
 
+    private Path getLuceneDir() {
+        // Merged lucene segments live in the shard's standard index folder (ShardPath.resolveIndex()),
+        // which resolves to "/index". The "/lucene" folder is only used
+        // for per-writer temporary staging directories (lucene_gen_*), not for the committed merged index.
+        IndexShard shard = getPrimaryShard();
+        return shard.shardPath().resolveIndex();
+    }
+
     private IndexShard getPrimaryShard() {
         String nodeName = getClusterState().routingTable().index(INDEX_NAME).shard(0).primaryShard().currentNodeId();
         String nodeNameResolved = getClusterState().nodes().get(nodeName).getName();
diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/PackedRowIdMappingTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/PackedRowIdMappingTests.java
new file mode 100644
index 0000000000000..8b70e923bb806
--- /dev/null
+++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/PackedRowIdMappingTests.java
@@ -0,0 +1,196 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.composite;
+
+import org.opensearch.index.engine.dataformat.PackedRowIdMapping;
+import org.opensearch.index.engine.dataformat.RowIdMapping;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.Map;
+
+/**
+ * Tests for {@link PackedRowIdMapping}.
+ */
+public class PackedRowIdMappingTests extends OpenSearchTestCase {
+
+    /**
+     * Basic lookup: two generations with known mappings.
+     * gen=1 (3 rows): 0→4, 1→3, 2→2
+     * gen=2 (2 rows): 0→1, 1→0
+     */
+    public void testBasicLookup() {
+        long[] mappingArray = { 4, 3, 2, 1, 0 };
+        Map offsets = Map.of(1L, 0, 2L, 3);
+        Map sizes = Map.of(1L, 3, 2L, 2);
+
+        PackedRowIdMapping mapping = new PackedRowIdMapping(mappingArray, offsets, sizes);
+
+        // gen=1 lookups
+        assertEquals(4L, mapping.getNewRowId(0, 1L));
+        assertEquals(3L, mapping.getNewRowId(1, 1L));
+        assertEquals(2L, mapping.getNewRowId(2, 1L));
+
+        // gen=2 lookups
+        assertEquals(1L, mapping.getNewRowId(0, 2L));
+        assertEquals(0L, mapping.getNewRowId(1, 2L));
+    }
+
+    /**
+     * Implements the RowIdMapping interface correctly.
+     */
+    public void testImplementsInterface() {
+        long[] mappingArray = { 10, 20 };
+        Map offsets = Map.of(5L, 0);
+        Map sizes = Map.of(5L, 2);
+
+        RowIdMapping mapping = new PackedRowIdMapping(mappingArray, offsets, sizes);
+        assertEquals(10L, mapping.getNewRowId(0, 5L));
+        assertEquals(20L, mapping.getNewRowId(1, 5L));
+    }
+
+    /**
+     * Unknown generation returns -1.
+     */
+    public void testUnknownGenerationReturnsNegativeOne() {
+        long[] mappingArray = { 0 };
+        Map offsets = Map.of(1L, 0);
+        Map sizes = Map.of(1L, 1);
+
+        PackedRowIdMapping mapping = new PackedRowIdMapping(mappingArray, offsets, sizes);
+        assertEquals(-1L, mapping.getNewRowId(0, 99L));
+    }
+
+    /**
+     * Out-of-bounds row ID returns -1.
+     */
+    public void testOutOfBoundsRowIdReturnsNegativeOne() {
+        long[] mappingArray = { 5, 6 };
+        Map offsets = Map.of(1L, 0);
+        Map sizes = Map.of(1L, 2);
+
+        PackedRowIdMapping mapping = new PackedRowIdMapping(mappingArray, offsets, sizes);
+        assertEquals(-1L, mapping.getNewRowId(2, 1L));
+        assertEquals(-1L, mapping.getNewRowId(-1, 1L));
+    }
+
+    /**
+     * Size returns total number of entries.
+     */
+    public void testSize() {
+        long[] mappingArray = { 0, 1, 2, 3, 4 };
+        Map offsets = Map.of(1L, 0, 2L, 3);
+        Map sizes = Map.of(1L, 3, 2L, 2);
+
+        PackedRowIdMapping mapping = new PackedRowIdMapping(mappingArray, offsets, sizes);
+        assertEquals(5, mapping.size());
+    }
+
+    /**
+     * Generation size returns correct count per generation.
+     */
+    public void testGenerationSize() {
+        long[] mappingArray = { 0, 1, 2, 3, 4 };
+        Map offsets = Map.of(1L, 0, 2L, 3);
+        Map sizes = Map.of(1L, 3, 2L, 2);
+
+        PackedRowIdMapping mapping = new PackedRowIdMapping(mappingArray, offsets, sizes);
+        assertEquals(3, mapping.getGenerationSize(1L));
+        assertEquals(2, mapping.getGenerationSize(2L));
+        assertEquals(0, mapping.getGenerationSize(99L));
+    }
+
+    /**
+     * Memory usage is reported and positive.
+     */
+    public void testRamBytesUsed() {
+        long[] mappingArray = new long[1000];
+        for (int i = 0; i < 1000; i++) {
+            mappingArray[i] = i;
+        }
+        Map offsets = Map.of(1L, 0);
+        Map sizes = Map.of(1L, 1000);
+
+        PackedRowIdMapping mapping = new PackedRowIdMapping(mappingArray, offsets, sizes);
+        assertTrue("RAM bytes used should be positive", mapping.ramBytesUsed() > 0);
+    }
+
+    /**
+     * Empty mapping works correctly.
+     */
+    public void testEmptyMapping() {
+        long[] mappingArray = {};
+        Map offsets = Map.of();
+        Map sizes = Map.of();
+
+        PackedRowIdMapping mapping = new PackedRowIdMapping(mappingArray, offsets, sizes);
+        assertEquals(0, mapping.size());
+        assertEquals(-1L, mapping.getNewRowId(0, 1L));
+    }
+
+    /**
+     * Null arguments throw NullPointerException.
+     */
+    public void testNullArgumentsThrow() {
+        expectThrows(NullPointerException.class, () -> new PackedRowIdMapping(null, Map.of(), Map.of()));
+        expectThrows(NullPointerException.class, () -> new PackedRowIdMapping(new long[0], null, Map.of()));
+        expectThrows(NullPointerException.class, () -> new PackedRowIdMapping(new long[0], Map.of(), null));
+    }
+
+    /**
+     * Generation offsets and sizes maps are unmodifiable.
+     */
+    public void testMapsAreUnmodifiable() {
+        long[] mappingArray = { 0 };
+        Map offsets = Map.of(1L, 0);
+        Map sizes = Map.of(1L, 1);
+
+        PackedRowIdMapping mapping = new PackedRowIdMapping(mappingArray, offsets, sizes);
+        expectThrows(UnsupportedOperationException.class, () -> mapping.getGenerationOffsets().put(2L, 1));
+        expectThrows(UnsupportedOperationException.class, () -> mapping.getGenerationSizes().put(2L, 1));
+    }
+
+    /**
+     * Three generations with non-sequential offsets (simulating real merge order).
+     */
+    public void testThreeGenerationsNonSequentialOrder() {
+        // Merge processes generations in order [5, 0, 3]
+        // gen=5 (2 rows): offset=0, mapping[0]=2, mapping[1]=3
+        // gen=0 (3 rows): offset=2, mapping[2]=0, mapping[3]=4, mapping[4]=1
+        // gen=3 (1 row): offset=5, mapping[5]=5
+        long[] mappingArray = { 2, 3, 0, 4, 1, 5 };
+        Map offsets = Map.of(5L, 0, 0L, 2, 3L, 5);
+        Map sizes = Map.of(5L, 2, 0L, 3, 3L, 1);
+
+        PackedRowIdMapping mapping = new PackedRowIdMapping(mappingArray, offsets, sizes);
+
+        assertEquals(2L, mapping.getNewRowId(0, 5L));
+        assertEquals(3L, mapping.getNewRowId(1, 5L));
+        assertEquals(0L, mapping.getNewRowId(0, 0L));
+        assertEquals(4L, mapping.getNewRowId(1, 0L));
+        assertEquals(1L, mapping.getNewRowId(2, 0L));
+        assertEquals(5L, mapping.getNewRowId(0, 3L));
+
+        assertEquals(6, mapping.size());
+    }
+
+    /**
+     * toString includes useful debug info.
+     */
+    public void testToString() {
+        long[] mappingArray = { 0, 1, 2 };
+        Map offsets = Map.of(1L, 0);
+        Map sizes = Map.of(1L, 3);
+
+        PackedRowIdMapping mapping = new PackedRowIdMapping(mappingArray, offsets, sizes);
+        String str = mapping.toString();
+        assertTrue(str.contains("size=3"));
+        assertTrue(str.contains("generations=1"));
+        assertTrue(str.contains("estimatedMemoryBytes="));
+    }
+}
diff --git a/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java b/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java
index aee6483e81efd..088c47e16e32d 100644
--- a/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java
+++ b/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/VSRRotationBenchmark.java
@@ -130,7 +130,7 @@ public void setup() throws IOException {
         Settings idxSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexMetadata indexMetadata = IndexMetadata.builder("benchmark-index").settings(idxSettings).build();
         indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);
-        vsrManager = new VSRManager(filePath, indexSettings, schema, bufferPool, maxRowsPerVSR, threadPool, runAsync);
+        vsrManager = new VSRManager(filePath, indexSettings, schema, bufferPool, maxRowsPerVSR, threadPool, runAsync, 0L);
     }
 
     @Benchmark
diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/MergeFilesResult.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/MergeFilesResult.java
new file mode 100644
index 0000000000000..64f2d3b0ea715
--- /dev/null
+++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/MergeFilesResult.java
@@ -0,0 +1,19 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.parquet.bridge;
+
+import org.opensearch.index.engine.dataformat.RowIdMapping;
+
+/**
+ * Result of a native Parquet merge. Bundles the row-ID mapping used to
+ * remap row IDs in secondary data formats with the Parquet file metadata
+ * (version, row count, {@code created_by}, CRC32) of the merged output file.
+ */
+public record MergeFilesResult(RowIdMapping rowIdMapping, ParquetFileMetadata metadata) {
+}
diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeParquetWriter.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeParquetWriter.java
index 7230c2e79c752..2b98d34b11831 100644
--- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeParquetWriter.java
+++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/NativeParquetWriter.java
@@ -18,7 +18,7 @@
  *
  * 
      Wraps the stateless JNI methods in {@link RustBridge} with a file-scoped lifecycle:
  * 
        
- *   
      1. {@code new NativeParquetWriter(filePath, indexName, schemaAddress, sortConfig)} — creates the native writer
        2. 
+ *   
      2. {@code new NativeParquetWriter(filePath, indexName, schemaAddress, sortConfig, writerGeneration)} — creates the native writer
        3. 
  *   
      3. {@link #write(long, long)} — sends one or more Arrow batches (repeatable)
        4. 
  *   
      4. {@link #flush()} — finalizes the Parquet file and returns metadata
        5. 
  *   
      5. {@link #sync()} — fsyncs the file to durable storage (calls flush if needed)
        6. 
@@ -36,15 +36,17 @@ public class NativeParquetWriter {
     /**
      * Creates a new NativeParquetWriter.
      *
-     * @param filePath      the path to the Parquet file to write
-     * @param indexName     the index name for settings lookup
-     * @param schemaAddress the native memory address of the Arrow schema
-     * @param sortConfig    the sort configuration for the Parquet file
+     * @param filePath          the path to the Parquet file to write
+     * @param indexName         the index name for settings lookup
+     * @param schemaAddress     the native memory address of the Arrow schema
+     * @param sortConfig        the sort configuration for the Parquet file
+     * @param writerGeneration  the writer generation to store in file metadata
      * @throws IOException if the native writer creation fails
      */
-    public NativeParquetWriter(String filePath, String indexName, long schemaAddress, ParquetSortConfig sortConfig) throws IOException {
+    public NativeParquetWriter(String filePath, String indexName, long schemaAddress, ParquetSortConfig sortConfig, long writerGeneration)
+        throws IOException {
         this.filePath = filePath;
-        RustBridge.createWriter(filePath, indexName, schemaAddress, sortConfig);
+        RustBridge.createWriter(filePath, indexName, schemaAddress, sortConfig, writerGeneration);
     }
 
     /**
diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java
index 2cd44a1a1bba8..6b8d9507cdcf4 100644
--- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java
+++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java
@@ -8,6 +8,8 @@
 
 package org.opensearch.parquet.bridge;
 
+import org.opensearch.index.engine.dataformat.PackedRowIdMapping;
+import org.opensearch.index.engine.dataformat.RowIdMapping;
 import org.opensearch.nativebridge.spi.NativeCall;
 import org.opensearch.nativebridge.spi.NativeLibraryLoader;
 
@@ -15,12 +17,15 @@
 import java.io.UncheckedIOException;
 import java.lang.foreign.FunctionDescriptor;
 import java.lang.foreign.Linker;
+import java.lang.foreign.MemorySegment;
 import java.lang.foreign.SymbolLookup;
 import java.lang.foreign.ValueLayout;
 import java.lang.invoke.MethodHandle;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * FFM bridge to the native Rust parquet writer library.
@@ -36,6 +41,7 @@ public class RustBridge {
     private static final MethodHandle ON_SETTINGS_UPDATE;
     private static final MethodHandle REMOVE_SETTINGS;
     private static final MethodHandle MERGE_FILES;
+    private static final MethodHandle FREE_MERGE_RESULT;
     private static final MethodHandle READ_AS_JSON;
 
     static {
@@ -56,7 +62,8 @@ public class RustBridge {
                 ValueLayout.ADDRESS,
                 ValueLayout.JAVA_LONG,   // reverse_sorts (vals, count)
                 ValueLayout.ADDRESS,
-                ValueLayout.JAVA_LONG    // nulls_first (vals, count)
+                ValueLayout.JAVA_LONG,   // nulls_first (vals, count)
+                ValueLayout.JAVA_LONG    // writer_generation
             )
         );
         WRITE = linker.downcallHandle(
@@ -147,7 +154,24 @@ public class RustBridge {
                 ValueLayout.ADDRESS,    // created_by_buf
                 ValueLayout.JAVA_LONG,  // created_by_buf_len
                 ValueLayout.ADDRESS,    // created_by_len_out
-                ValueLayout.ADDRESS     // crc32_out
+                ValueLayout.ADDRESS,    // crc32_out
+                ValueLayout.ADDRESS,    // out_mapping_ptr
+                ValueLayout.ADDRESS,    // out_mapping_len
+                ValueLayout.ADDRESS,    // out_gen_keys_ptr
+                ValueLayout.ADDRESS,    // out_gen_offsets_ptr
+                ValueLayout.ADDRESS,    // out_gen_sizes_ptr
+                ValueLayout.ADDRESS     // out_gen_count
+            )
+        );
+        FREE_MERGE_RESULT = linker.downcallHandle(
+            lib.find("parquet_free_merge_result").orElseThrow(),
+            FunctionDescriptor.ofVoid(
+                ValueLayout.JAVA_LONG,  // mapping_ptr
+                ValueLayout.JAVA_LONG,  // mapping_len
+                ValueLayout.JAVA_LONG,  // gen_keys_ptr
+                ValueLayout.JAVA_LONG,  // gen_offsets_ptr
+                ValueLayout.JAVA_LONG,  // gen_sizes_ptr
+                ValueLayout.JAVA_LONG   // gen_count
             )
         );
         READ_AS_JSON = linker.downcallHandle(
@@ -165,7 +189,8 @@ public class RustBridge {
 
     public static void initLogger() {}
 
-    static void createWriter(String file, String indexName, long schemaAddress, ParquetSortConfig sortConfig) throws IOException {
+    static void createWriter(String file, String indexName, long schemaAddress, ParquetSortConfig sortConfig, long writerGeneration)
+        throws IOException {
         try (var call = new NativeCall()) {
             var f = call.str(file);
             var idx = call.str(indexName);
@@ -185,7 +210,8 @@ static void createWriter(String file, String indexName, long schemaAddress, Parq
                 reverseArray,
                 (long) sortConfig.reverseSorts().size(),
                 nullsFirstArray,
-                (long) sortConfig.nullsFirst().size()
+                (long) sortConfig.nullsFirst().size(),
+                writerGeneration
             );
         }
     }
@@ -295,16 +321,27 @@ public static void removeSettings(String indexName) {
         }
     }
 
-    public static ParquetFileMetadata mergeParquetFilesInRust(List inputFiles, String outputFile, String indexName) {
+    public static MergeFilesResult mergeParquetFilesInRust(List inputFiles, String outputFile, String indexName) {
         String[] paths = inputFiles.stream().map(Path::toString).toArray(String[]::new);
         try (var call = new NativeCall()) {
             var inputs = call.strArray(paths);
             var out = call.str(outputFile);
             var idx = call.str(indexName);
+
+            // Out-pointers for Parquet file metadata
             var versionOut = call.intOut();
             var numRowsOut = call.longOut();
             var crc32Out = call.longOut();
             var createdByOut = call.outBuffer(1024);
+
+            // Out-pointers for Rust-allocated mapping data
+            var outMappingPtr = call.longOut();
+            var outMappingLen = call.longOut();
+            var outGenKeysPtr = call.longOut();
+            var outGenOffsetsPtr = call.longOut();
+            var outGenSizesPtr = call.longOut();
+            var outGenCount = call.longOut();
+
             call.invokeIO(
                 MERGE_FILES,
                 inputs.ptrs(),
@@ -319,10 +356,17 @@ public static ParquetFileMetadata mergeParquetFilesInRust(List inputFiles,
                 createdByOut.data(),
                 (long) createdByOut.capacity(),
                 createdByOut.lenOut(),
-                crc32Out
+                crc32Out,
+                outMappingPtr,
+                outMappingLen,
+                outGenKeysPtr,
+                outGenOffsetsPtr,
+                outGenSizesPtr,
+                outGenCount
             );
+
             int createdByLen = (int) createdByOut.lenOut().get(ValueLayout.JAVA_LONG, 0);
-            return new ParquetFileMetadata(
+            ParquetFileMetadata metadata = new ParquetFileMetadata(
                 versionOut.get(ValueLayout.JAVA_INT, 0),
                 numRowsOut.get(ValueLayout.JAVA_LONG, 0),
                 createdByLen >= 0
@@ -330,11 +374,67 @@ public static ParquetFileMetadata mergeParquetFilesInRust(List inputFiles,
                     : null,
                 crc32Out.get(ValueLayout.JAVA_LONG, 0)
             );
+
+            RowIdMapping rowIdMapping = readAndFreeMergeResult(
+                outMappingPtr,
+                outMappingLen,
+                outGenKeysPtr,
+                outGenOffsetsPtr,
+                outGenSizesPtr,
+                outGenCount
+            );
+
+            return new MergeFilesResult(rowIdMapping, metadata);
         } catch (IOException e) {
             throw new UncheckedIOException("Native merge failed", e);
         }
     }
 
+    private static RowIdMapping readAndFreeMergeResult(
+        MemorySegment outMappingPtr,
+        MemorySegment outMappingLen,
+        MemorySegment outGenKeysPtr,
+        MemorySegment outGenOffsetsPtr,
+        MemorySegment outGenSizesPtr,
+        MemorySegment outGenCount
+    ) {
+        long mappingAddr = outMappingPtr.get(ValueLayout.JAVA_LONG, 0);
+        long mappingLen = outMappingLen.get(ValueLayout.JAVA_LONG, 0);
+        long genKeysAddr = outGenKeysPtr.get(ValueLayout.JAVA_LONG, 0);
+        long genOffsetsAddr = outGenOffsetsPtr.get(ValueLayout.JAVA_LONG, 0);
+        long genSizesAddr = outGenSizesPtr.get(ValueLayout.JAVA_LONG, 0);
+        long genCount = outGenCount.get(ValueLayout.JAVA_LONG, 0);
+
+        try {
+            // Read mapping array (i64[])
+            long[] mappingArray = MemorySegment.ofAddress(mappingAddr)
+                .reinterpret(mappingLen * ValueLayout.JAVA_LONG.byteSize())
+                .toArray(ValueLayout.JAVA_LONG);
+
+            // Read generation keys (i64[]), offsets (i32[]), sizes (i32[])
+            long[] genKeys = MemorySegment.ofAddress(genKeysAddr)
+                .reinterpret(genCount * ValueLayout.JAVA_LONG.byteSize())
+                .toArray(ValueLayout.JAVA_LONG);
+            int[] genOffsets = MemorySegment.ofAddress(genOffsetsAddr)
+                .reinterpret(genCount * ValueLayout.JAVA_INT.byteSize())
+                .toArray(ValueLayout.JAVA_INT);
+            int[] genSizes = MemorySegment.ofAddress(genSizesAddr)
+                .reinterpret(genCount * ValueLayout.JAVA_INT.byteSize())
+                .toArray(ValueLayout.JAVA_INT);
+
+            Map offsetMap = new HashMap<>((int) genCount);
+            Map sizeMap = new HashMap<>((int) genCount);
+            for (int i = 0; i < (int) genCount; i++) {
+                offsetMap.put(genKeys[i], genOffsets[i]);
+                sizeMap.put(genKeys[i], genSizes[i]);
+            }
+
+            return new PackedRowIdMapping(mappingArray, offsetMap, sizeMap);
+        } finally {
+            NativeCall.invokeVoid(FREE_MERGE_RESULT, mappingAddr, mappingLen, genKeysAddr, genOffsetsAddr, genSizesAddr, genCount);
+        }
+    }
+
     private static java.lang.foreign.MemorySegment marshalBoolList(NativeCall call, List bools) {
         if (bools == null || bools.isEmpty()) {
             return java.lang.foreign.MemorySegment.NULL;
diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java
index 021f5b04238a6..9a76eef8aac61 100644
--- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java
+++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/merge/NativeParquetMergeStrategy.java
@@ -15,8 +15,10 @@
 import org.opensearch.index.engine.dataformat.DataFormat;
 import org.opensearch.index.engine.dataformat.MergeInput;
 import org.opensearch.index.engine.dataformat.MergeResult;
+import org.opensearch.index.engine.dataformat.RowIdMapping;
 import org.opensearch.index.engine.exec.WriterFileSet;
 import org.opensearch.index.shard.ShardPath;
+import org.opensearch.parquet.bridge.MergeFilesResult;
 import org.opensearch.parquet.bridge.ParquetFileMetadata;
 import org.opensearch.parquet.bridge.RustBridge;
 import org.opensearch.parquet.engine.ParquetIndexingEngine;
@@ -77,7 +79,10 @@ public MergeResult mergeParquetFiles(MergeInput mergeInput) {
 
         try {
             // Merge files in Rust
-            ParquetFileMetadata mergeMetadata = RustBridge.mergeParquetFilesInRust(filePaths, mergedFilePath.toString(), indexName);
+            MergeFilesResult merged = RustBridge.mergeParquetFilesInRust(filePaths, mergedFilePath.toString(), indexName);
+            ParquetFileMetadata mergeMetadata = merged.metadata();
+            RowIdMapping rowIdMapping = merged.rowIdMapping();
+
             assert mergeMetadata.numRows() > 0 : "Merged file should contain at least one row";
 
             long expectedRows = files.stream().mapToLong(WriterFileSet::numRows).sum();
@@ -97,7 +102,7 @@ public MergeResult mergeParquetFiles(MergeInput mergeInput) {
             checksumUpdater.apply(mergedFileName, mergeMetadata.crc32(), mergeInput.newWriterGeneration());
             Map mergedWriterFileSetMap = Collections.singletonMap(dataFormat, mergedWriterFileSet);
 
-            return new MergeResult(mergedWriterFileSetMap);
+            return new MergeResult(mergedWriterFileSetMap, rowIdMapping);
 
         } catch (Exception exception) {
             logger.error(() -> new ParameterizedMessage("Merge failed while creating merged file [{}]", mergedFilePath), exception);
diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java
index f64503295c0d0..bec08479d3656 100644
--- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java
+++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/vsr/VSRManager.java
@@ -65,6 +65,7 @@ public class VSRManager implements AutoCloseable {
     private final VSRPool vsrPool;
     private final ThreadPool threadPool;
     private final String vsrRotationThread;
+    private final long writerGeneration;
     private volatile Future pendingWrite;
     private NativeParquetWriter writer;
     private final int ROTATION_TIMEOUT = 120;
@@ -79,9 +80,10 @@ public VSRManager(
         Schema schema,
         ArrowBufferPool bufferPool,
         int maxRowsPerVSR,
-        ThreadPool threadPool
+        ThreadPool threadPool,
+        long writerGeneration
     ) {
-        this(fileName, indexSettings, schema, bufferPool, maxRowsPerVSR, threadPool, true);
+        this(fileName, indexSettings, schema, bufferPool, maxRowsPerVSR, threadPool, true, writerGeneration);
     }
 
     /**
@@ -95,6 +97,7 @@ public VSRManager(
      * @param threadPool the thread pool for background native writes
      * @param runAsync if true, frozen VSR writes run on the background thread pool;
      *                 if false, they run on the calling thread (for benchmarks/tests)
+     * @param writerGeneration the writer generation to store in file metadata
      */
     public VSRManager(
         String fileName,
@@ -103,10 +106,12 @@ public VSRManager(
         ArrowBufferPool bufferPool,
         int maxRowsPerVSR,
         ThreadPool threadPool,
-        boolean runAsync
+        boolean runAsync,
+        long writerGeneration
     ) {
         this.fileName = fileName;
         this.indexSettings = indexSettings;
+        this.writerGeneration = writerGeneration;
         this.vsrPool = new VSRPool("pool-" + fileName, schema, bufferPool, maxRowsPerVSR);
         this.threadPool = threadPool;
         this.vsrRotationThread = runAsync ? ParquetDataFormatPlugin.PARQUET_THREAD_POOL_NAME : ThreadPool.Names.SAME;
@@ -228,7 +233,7 @@ private void initializeWriter() {
 
         ArrowSchema arrowSchema = managedVSR.get().exportSchema();
         try {
-            writer = new NativeParquetWriter(fileName, indexName, arrowSchema.memoryAddress(), sortConfig);
+            writer = new NativeParquetWriter(fileName, indexName, arrowSchema.memoryAddress(), sortConfig, writerGeneration);
         } catch (Exception e) {
             throw new RuntimeException("Failed to initialize Parquet writer: " + e.getMessage(), e);
         } finally {
diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java
index 1ad8dd7bd771d..f74ca3f086ea7 100644
--- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java
+++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java
@@ -78,7 +78,8 @@ public ParquetWriter(
             schema,
             bufferPool,
             ParquetSettings.MAX_ROWS_PER_VSR.get(indexSettings.getSettings()),
-            threadPool
+            threadPool,
+            writerGeneration
         );
     }
 
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs
index 5ccd1394864ce..ab53939e6c596 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs
@@ -84,6 +84,7 @@ pub unsafe extern "C" fn parquet_create_writer(
     reverse_count: i64,
     nulls_first_vals: *const i64,
     nulls_first_count: i64,
+    writer_generation: i64,
 ) -> i64 {
     let filename = str_from_raw(file_ptr, file_len)
         .map_err(|e| format!("parquet_create_writer file: {}", e))?.to_string();
@@ -94,7 +95,7 @@ pub unsafe extern "C" fn parquet_create_writer(
     let reverse_sorts = bool_array_from_raw(reverse_vals, reverse_count);
     let nulls_first = bool_array_from_raw(nulls_first_vals, nulls_first_count);
 
-    NativeParquetWriter::create_writer(filename, index_name, schema_address, sort_columns, reverse_sorts, nulls_first)
+    NativeParquetWriter::create_writer(filename, index_name, schema_address, sort_columns, reverse_sorts, nulls_first, writer_generation)
         .map(|_| 0)
         .map_err(|e| e.to_string())
 }
@@ -296,6 +297,12 @@ pub unsafe extern "C" fn parquet_merge_files(
     created_by_buf_len: i64,
     created_by_len_out: *mut i64,
     crc32_out: *mut i64,
+    out_mapping_ptr: *mut i64,
+    out_mapping_len: *mut i64,
+    out_gen_keys_ptr: *mut i64,
+    out_gen_offsets_ptr: *mut i64,
+    out_gen_sizes_ptr: *mut i64,
+    out_gen_count: *mut i64,
 ) -> i64 {
     let input_files = str_array_from_raw(input_ptrs, input_lens, input_count)
         .map_err(|e| format!("parquet_merge_files inputs: {}", e))?;
@@ -323,7 +330,7 @@ pub unsafe extern "C" fn parquet_merge_files(
         }
     };
 
-    let (metadata, crc32) = if sort_cols.is_empty() {
+    let result = if sort_cols.is_empty() {
         merge::merge_unsorted(&input_files, output_path, index_name)
     } else {
         merge::merge_sorted(
@@ -337,8 +344,8 @@ pub unsafe extern "C" fn parquet_merge_files(
     }
     .map_err(|e| format!("{}", e))?;
 
-    // Write metadata to out-pointers
-    let fm = metadata.file_metadata();
+    // Write Parquet file metadata to out-pointers.
+    let fm = result.metadata.file_metadata();
     if !version_out.is_null() { *version_out = fm.version(); }
     if !num_rows_out.is_null() { *num_rows_out = fm.num_rows(); }
     if let Some(cb) = fm.created_by() {
@@ -351,10 +358,51 @@ pub unsafe extern "C" fn parquet_merge_files(
     } else if !created_by_len_out.is_null() {
         *created_by_len_out = -1;
     }
-    if !crc32_out.is_null() { *crc32_out = crc32 as i64; }
+    if !crc32_out.is_null() { *crc32_out = result.crc32 as i64; }
+
+    // Write row-ID mapping into out-pointers as heap-allocated arrays.
+    // Java reads them and then calls parquet_free_merge_result to deallocate.
+    let mapping = result.mapping.into_boxed_slice();
+    *out_mapping_len = mapping.len() as i64;
+    *out_mapping_ptr = Box::into_raw(mapping) as *mut i64 as i64;
+
+    let count = result.gen_keys.len();
+    let keys = result.gen_keys.into_boxed_slice();
+    let offsets = result.gen_offsets.into_boxed_slice();
+    let sizes = result.gen_sizes.into_boxed_slice();
+    *out_gen_count = count as i64;
+    *out_gen_keys_ptr = Box::into_raw(keys) as *mut i64 as i64;
+    *out_gen_offsets_ptr = Box::into_raw(offsets) as *mut i32 as i64;
+    *out_gen_sizes_ptr = Box::into_raw(sizes) as *mut i32 as i64;
+
     Ok(0)
 }
 
+/// Frees the heap-allocated arrays returned by `parquet_merge_files`.
+#[no_mangle]
+pub unsafe extern "C" fn parquet_free_merge_result(
+    mapping_ptr: i64,
+    mapping_len: i64,
+    gen_keys_ptr: i64,
+    gen_offsets_ptr: i64,
+    gen_sizes_ptr: i64,
+    gen_count: i64,
+) {
+    if mapping_ptr != 0 && mapping_len > 0 {
+        let _ = Box::from_raw(slice::from_raw_parts_mut(mapping_ptr as *mut i64, mapping_len as usize));
+    }
+    let n = gen_count as usize;
+    if gen_keys_ptr != 0 && n > 0 {
+        let _ = Box::from_raw(slice::from_raw_parts_mut(gen_keys_ptr as *mut i64, n));
+    }
+    if gen_offsets_ptr != 0 && n > 0 {
+        let _ = Box::from_raw(slice::from_raw_parts_mut(gen_offsets_ptr as *mut i32, n));
+    }
+    if gen_sizes_ptr != 0 && n > 0 {
+        let _ = Box::from_raw(slice::from_raw_parts_mut(gen_sizes_ptr as *mut i32, n));
+    }
+}
+
 // ---------------------------------------------------------------------------
 // Parquet reader (for test verification)
 // ---------------------------------------------------------------------------
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/cursor.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/cursor.rs
index b530820ef3c98..e39b5eb8ff3a1 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/cursor.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/cursor.rs
@@ -39,7 +39,7 @@ pub struct FileCursor {
 impl FileCursor {
     /// Opens a Parquet file and creates a cursor positioned at the first row.
     ///
-    /// Returns `(cursor, projected_arrow_schema, parquet_schema_descriptor)`
+    /// Returns `(cursor, projected_arrow_schema, parquet_schema_descriptor, writer_generation)`
     /// so the caller can build union schemas without re-opening the file.
     pub fn new(
         path: &str,
@@ -47,10 +47,12 @@ impl FileCursor {
         sort_columns: &[String],
         nulls_first: &[bool],
         batch_size: usize,
-    ) -> MergeResult<(Self, Arc, SchemaDescriptor)> {
+    ) -> MergeResult<(Self, Arc, SchemaDescriptor, i64, usize)> {
         let file = File::open(path)?;
         let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
         let schema = builder.schema().clone();
+        let writer_generation = crate::writer_properties_builder::read_writer_generation(builder.metadata().file_metadata(), file_id);
+        let total_row_count = builder.metadata().file_metadata().num_rows() as usize;
 
         let mut sort_col_types = Vec::with_capacity(sort_columns.len());
         for col_name in sort_columns {
@@ -127,7 +129,7 @@ impl FileCursor {
 
         cursor.start_prefetch();
 
-        Ok((cursor, projected_schema, parquet_schema_descr))
+        Ok((cursor, projected_schema, parquet_schema_descr, writer_generation, total_row_count))
     }
 
     fn start_prefetch(&mut self) {
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/mod.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/mod.rs
index ee76348c99b7b..6df699d2db3b7 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/mod.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/mod.rs
@@ -18,3 +18,21 @@ mod unsorted;
 pub use error::{MergeError, MergeResult};
 pub use sorted::merge_sorted;
 pub use unsorted::merge_unsorted;
+
+/// Output of a merge operation. Carries both the row-ID mapping (for remapping
+/// secondary-format row IDs post-merge) and the Parquet file metadata + CRC32
+/// of the merged output file.
+pub struct MergeOutput {
+    /// Flat mapping array: mapping[offset + old_row_id] = new_row_id
+    pub mapping: Vec,
+    /// Generation keys (parallel with gen_offsets and gen_sizes)
+    pub gen_keys: Vec,
+    /// Starting offset in `mapping` for each generation
+    pub gen_offsets: Vec,
+    /// Number of rows per generation
+    pub gen_sizes: Vec,
+    /// Parquet file metadata for the merged output file
+    pub metadata: parquet::file::metadata::ParquetMetaData,
+    /// Whole-file CRC32 of the merged output file
+    pub crc32: u32,
+}
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs
index 634e0003de2e2..7736539887a11 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/sorted.rs
@@ -13,7 +13,7 @@ use std::sync::Arc;
 use arrow::datatypes::Schema as ArrowSchema;
 use parquet::schema::types::SchemaDescriptor;
 
-use crate::{log_debug, log_info};
+use crate::log_debug;
 
 use super::context::MergeContext;
 use super::cursor::FileCursor;
@@ -29,7 +29,7 @@ pub fn merge_sorted(
     sort_columns: &[String],
     reverse_sorts: &[bool],
     nulls_first: &[bool],
-) -> super::MergeResult<(parquet::file::metadata::ParquetMetaData, u32)> {
+) -> super::MergeResult {
     let config = crate::writer::SETTINGS_STORE
         .get(index_name)
         .map(|r| r.clone())
@@ -75,14 +75,18 @@ pub fn merge_sorted(
     let mut cursors: Vec = Vec::with_capacity(input_files.len());
     let mut arrow_schemas: Vec = Vec::with_capacity(input_files.len());
     let mut parquet_descriptors: Vec = Vec::with_capacity(input_files.len());
+    let mut file_generations: Vec = Vec::with_capacity(input_files.len());
+    let mut file_row_counts: Vec = Vec::with_capacity(input_files.len());
 
     for (file_id, path) in input_files.iter().enumerate() {
         log_debug!("[RUST] Opening cursor {} for file: {}", file_id, path);
-        let (cursor, projected_schema, parquet_descr) =
+        let (cursor, projected_schema, parquet_descr, generation, row_count) =
             FileCursor::new(path, file_id, sort_columns, nulls_first, batch_size)?;
         cursors.push(cursor);
         arrow_schemas.push(projected_schema.as_ref().clone());
         parquet_descriptors.push(parquet_descr);
+        file_generations.push(generation);
+        file_row_counts.push(row_count);
     }
 
     let num_cursors = cursors.len();
@@ -103,6 +107,27 @@ pub fn merge_sorted(
         .map(|s| ColumnMapping::new(s, ctx.data_schema()))
         .collect();
 
+    // Row-ID mapping: pre-allocate the flat mapping array and compute offsets
+    // from file metadata row counts (known before reading any data).
+    let total_rows: usize = file_row_counts.iter().sum();
+    let mut mapping: Vec = vec![0i64; total_rows];
+    let mut gen_keys: Vec = Vec::with_capacity(num_cursors);
+    let mut gen_offsets: Vec = Vec::with_capacity(num_cursors);
+    let mut gen_sizes: Vec = Vec::with_capacity(num_cursors);
+
+    let mut offset = 0i32;
+    for file_id in 0..num_cursors {
+        gen_keys.push(file_generations[file_id]);
+        gen_offsets.push(offset);
+        let size = file_row_counts[file_id] as i32;
+        gen_sizes.push(size);
+        offset += size;
+    }
+
+    // Per-file counters: tracks how many rows have been emitted from each file
+    let mut rows_emitted_per_file: Vec = vec![0; num_cursors];
+    let mut new_row_id: i64 = 0;
+
     log_debug!(
         "[RUST] Merge initialized ({}): {} cursors",
         direction_label,
@@ -128,12 +153,18 @@ pub fn merge_sorted(
         // TIER 1: Single cursor remaining — drain it
         if heap.is_empty() {
             let cursor = &mut cursors[file_id];
-            let mapping = &col_mappings[file_id];
+            let col_mapping = &col_mappings[file_id];
+            let file_offset = gen_offsets[file_id] as usize;
             loop {
                 let remaining = cursor.batch_height() - cursor.row_idx;
                 if remaining > 0 {
                     let slice = cursor.take_slice(cursor.row_idx, remaining);
-                    ctx.push_batch(mapping.pad_batch(&slice)?)?;
+                    for _ in 0..remaining {
+                        mapping[file_offset + rows_emitted_per_file[file_id]] = new_row_id;
+                        rows_emitted_per_file[file_id] += 1;
+                        new_row_id += 1;
+                    }
+                    ctx.push_batch(col_mapping.pad_batch(&slice)?)?;
                 }
                 if !cursor.advance_past_batch()? {
                     break;
@@ -144,7 +175,8 @@ pub fn merge_sorted(
 
         // TIER 2 & 3: Multiple cursors active
         let cursor = &mut cursors[file_id];
-        let mapping = &col_mappings[file_id];
+        let col_mapping = &col_mappings[file_id];
+        let file_offset = gen_offsets[file_id] as usize;
 
         loop {
             let heap_top = &heap.peek().unwrap().sort_values;
@@ -154,7 +186,12 @@ pub fn merge_sorted(
             if cmp_sort_values(&last_val, heap_top, reverse_sorts) != Ordering::Greater {
                 let remaining = cursor.batch_height() - cursor.row_idx;
                 let slice = cursor.take_slice(cursor.row_idx, remaining);
-                ctx.push_batch(mapping.pad_batch(&slice)?)?;
+                for _ in 0..remaining {
+                    mapping[file_offset + rows_emitted_per_file[file_id]] = new_row_id;
+                    rows_emitted_per_file[file_id] += 1;
+                    new_row_id += 1;
+                }
+                ctx.push_batch(col_mapping.pad_batch(&slice)?)?;
 
                 if !cursor.advance_past_batch()? {
                     break;
@@ -191,7 +228,12 @@ pub fn merge_sorted(
             let run_len = run_end - run_start + 1;
             if run_len > 0 {
                 let slice = cursor.take_slice(run_start, run_len);
-                ctx.push_batch(mapping.pad_batch(&slice)?)?;
+                for _ in 0..run_len {
+                    mapping[file_offset + rows_emitted_per_file[file_id]] = new_row_id;
+                    rows_emitted_per_file[file_id] += 1;
+                    new_row_id += 1;
+                }
+                ctx.push_batch(col_mapping.pad_batch(&slice)?)?;
             }
 
             cursor.row_idx = run_end;
@@ -223,5 +265,12 @@ pub fn merge_sorted(
         crc32
     );
 
-    Ok((metadata, crc32))
+    Ok(super::MergeOutput {
+        mapping,
+        gen_keys,
+        gen_offsets,
+        gen_sizes,
+        metadata,
+        crc32,
+    })
 }
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs
index a43b177e35e41..3e406bbaed1d9 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/unsorted.rs
@@ -13,7 +13,7 @@ use arrow::datatypes::Schema as ArrowSchema;
 use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder};
 use parquet::schema::types::SchemaDescriptor;
 
-use crate::{log_debug, log_info};
+use crate::log_debug;
 
 use super::context::MergeContext;
 use super::error::MergeResult;
@@ -25,7 +25,7 @@ pub fn merge_unsorted(
     input_files: &[String],
     output_path: &str,
     index_name: &str,
-) -> MergeResult<(parquet::file::metadata::ParquetMetaData, u32)> {
+) -> MergeResult {
     let config = crate::writer::SETTINGS_STORE
         .get(index_name)
         .map(|r| r.clone())
@@ -44,12 +44,16 @@ pub fn merge_unsorted(
     let mut arrow_schemas: Vec = Vec::with_capacity(input_files.len());
     let mut parquet_descriptors: Vec = Vec::with_capacity(input_files.len());
     let mut readers: Vec = Vec::with_capacity(input_files.len());
+    let mut file_row_counts: Vec = Vec::with_capacity(input_files.len());
+    let mut file_generations: Vec = Vec::with_capacity(input_files.len());
 
-    for path in input_files {
+    for (file_idx, path) in input_files.iter().enumerate() {
         let file = File::open(path)?;
         let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
         let schema = builder.schema().clone();
         let parquet_descr = builder.parquet_schema().clone();
+        let num_rows = builder.metadata().file_metadata().num_rows() as usize;
+        let generation = crate::writer_properties_builder::read_writer_generation(builder.metadata().file_metadata(), file_idx);
 
         let projection_indices = projection_indices_excluding_row_id(&schema);
         let projection = parquet::arrow::ProjectionMask::roots(&parquet_descr, projection_indices);
@@ -59,6 +63,8 @@ pub fn merge_unsorted(
         arrow_schemas.push(reader.schema().as_ref().clone());
         parquet_descriptors.push(parquet_descr);
         readers.push(reader);
+        file_row_counts.push(num_rows);
+        file_generations.push(generation);
     }
 
     let mut ctx = MergeContext::new(
@@ -76,6 +82,17 @@ pub fn merge_unsorted(
         .map(|s| ColumnMapping::new(s, ctx.data_schema()))
         .collect();
 
+    // Build row-ID mapping: for unsorted merge, files are concatenated sequentially.
+    // old_row_id maps directly to new_row_id with a per-file offset.
+    let total_rows: usize = file_row_counts.iter().sum();
+    let mut mapping: Vec = vec![0i64; total_rows];
+    let mut gen_keys: Vec = Vec::with_capacity(input_files.len());
+    let mut gen_offsets: Vec = Vec::with_capacity(input_files.len());
+    let mut gen_sizes: Vec = Vec::with_capacity(input_files.len());
+
+    let mut mapping_offset: usize = 0;
+    let mut new_row_id: i64 = 0;
+
     // Iterate readers for data.
     for (file_idx, reader) in readers.into_iter().enumerate() {
         log_debug!(
@@ -84,11 +101,25 @@ pub fn merge_unsorted(
             input_files.len()
         );
 
-        let mapping = &col_mappings[file_idx];
+        gen_keys.push(file_generations[file_idx]);
+        gen_offsets.push(mapping_offset as i32);
+        let file_start_row_id = new_row_id;
+
+        let col_mapping = &col_mappings[file_idx];
         for batch_result in reader {
             let batch = batch_result?;
-            ctx.push_batch(mapping.pad_batch(&batch)?)?;
+            let num_rows = batch.num_rows();
+            // Record mapping: each row in this batch gets the next sequential new_row_id
+            for _ in 0..num_rows {
+                mapping[mapping_offset] = new_row_id;
+                mapping_offset += 1;
+                new_row_id += 1;
+            }
+            ctx.push_batch(col_mapping.pad_batch(&batch)?)?;
         }
+
+        let file_rows = (new_row_id - file_start_row_id) as i32;
+        gen_sizes.push(file_rows);
     }
 
     let (metadata, crc32) = ctx.finish()?;
@@ -101,5 +132,12 @@ pub fn merge_unsorted(
         crc32
     );
 
-    Ok((metadata, crc32))
+    Ok(super::MergeOutput {
+        mapping,
+        gen_keys,
+        gen_offsets,
+        gen_sizes,
+        metadata,
+        crc32,
+    })
 }
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/test_utils.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/test_utils.rs
index 7b46092b09ddc..032fd9647ac2c 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/test_utils.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/test_utils.rs
@@ -72,7 +72,7 @@ pub fn get_temp_file_path(name: &str) -> (tempfile::TempDir, String) {
 
 pub fn create_writer_and_assert_success(filename: &str) -> (Arc, i64) {
     let (schema, schema_ptr) = create_test_ffi_schema();
-    let result = NativeParquetWriter::create_writer(filename.to_string(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![]);
+    let result = NativeParquetWriter::create_writer(filename.to_string(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![], 0);
     assert!(result.is_ok());
     (schema, schema_ptr)
 }
@@ -80,7 +80,7 @@ pub fn create_writer_and_assert_success(filename: &str) -> (Arc, i64) {
 pub fn create_sorted_writer_and_assert_success(filename: &str, sort_column: &str, reverse: bool) -> (Arc, i64) {
     let (schema, schema_ptr) = create_test_ffi_schema();
     let result = NativeParquetWriter::create_writer(
-        filename.to_string(), "test-index".to_string(), schema_ptr, vec![sort_column.to_string()], vec![reverse], vec![false]
+        filename.to_string(), "test-index".to_string(), schema_ptr, vec![sort_column.to_string()], vec![reverse], vec![false], 0
     );
     assert!(result.is_ok());
     (schema, schema_ptr)
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs
index 25c4e375a7264..48ab97ba5a57e 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs
@@ -30,7 +30,7 @@ fn test_create_writer_success() {
 fn test_create_writer_invalid_path() {
     let invalid_path = "/invalid/path/that/does/not/exist/test.parquet";
     let (_schema, schema_ptr) = create_test_ffi_schema();
-    let result = NativeParquetWriter::create_writer(invalid_path.to_string(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![]);
+    let result = NativeParquetWriter::create_writer(invalid_path.to_string(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![], 0);
     assert!(result.is_err());
     cleanup_ffi_schema(schema_ptr);
 }
@@ -38,7 +38,7 @@ fn test_create_writer_invalid_path() {
 #[test]
 fn test_create_writer_invalid_schema_pointer() {
     let (_temp_dir, filename) = get_temp_file_path("invalid_schema.parquet");
-    let result = NativeParquetWriter::create_writer(filename, "test-index".to_string(), 0, vec![], vec![], vec![]);
+    let result = NativeParquetWriter::create_writer(filename, "test-index".to_string(), 0, vec![], vec![], vec![], 0);
     assert!(result.is_err());
     assert!(result.unwrap_err().to_string().contains("Invalid schema address"));
 }
@@ -48,7 +48,7 @@ fn test_create_writer_multiple_times_same_file() {
     let (_temp_dir, filename) = get_temp_file_path("duplicate.parquet");
     let (_schema, schema_ptr) = create_writer_and_assert_success(&filename);
     let (_, schema_ptr2) = create_test_ffi_schema();
-    let result2 = NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr2, vec![], vec![], vec![]);
+    let result2 = NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr2, vec![], vec![], vec![], 0);
     assert!(result2.is_err());
     assert!(result2.unwrap_err().to_string().contains("Writer already exists"));
     cleanup_ffi_schema(schema_ptr2);
@@ -366,7 +366,7 @@ fn test_concurrent_writer_creation() {
             let filename = file_path.to_string_lossy().to_string();
             let (_schema, schema_ptr) = create_test_ffi_schema();
 
-            if NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![]).is_ok() {
+            if NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![], 0).is_ok() {
                 success_count.fetch_add(1, Ordering::SeqCst);
                 let (ap, sp) = create_test_ffi_data().unwrap();
                 let _ = NativeParquetWriter::write_data(filename.clone(), ap, sp);
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs
index f44449183ba7a..3d14e16e0ef7a 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs
@@ -36,6 +36,7 @@ struct WriterState {
     writer: Arc>>>,
     settings: NativeSettings,
     crc_handle: crate::crc_writer::CrcHandle,
+    writer_generation: i64,
 }
 
 lazy_static! {
@@ -71,10 +72,11 @@ impl NativeParquetWriter {
         sort_columns: Vec,
         reverse_sorts: Vec,
         nulls_first: Vec,
+        writer_generation: i64,
     ) -> Result<(), Box> {
         log_debug!(
-            "create_writer called for file: {}, index: {}, schema_address: {}, sort_columns: {:?}, reverse_sorts: {:?}, nulls_first: {:?}",
-            filename, index_name, schema_address, sort_columns, reverse_sorts, nulls_first
+            "create_writer called for file: {}, index: {}, schema_address: {}, sort_columns: {:?}, reverse_sorts: {:?}, nulls_first: {:?}, generation: {}",
+            filename, index_name, schema_address, sort_columns, reverse_sorts, nulls_first, writer_generation
         );
 
         if (schema_address as *mut u8).is_null() {
@@ -105,7 +107,7 @@ impl NativeParquetWriter {
         settings.reverse_sorts = reverse_sorts;
         settings.nulls_first = nulls_first;
 
-        let props = WriterPropertiesBuilder::build(&settings);
+        let props = WriterPropertiesBuilder::build_with_generation(&settings, Some(writer_generation));
 
         SETTINGS_STORE.insert(index_name, settings.clone());
 
@@ -115,6 +117,7 @@ impl NativeParquetWriter {
             writer: Arc::new(Mutex::new(writer)),
             settings,
             crc_handle,
+            writer_generation,
         });
 
         Ok(())
@@ -160,7 +163,7 @@ impl NativeParquetWriter {
         log_debug!("finalize_writer called for file: {} (temp: {})", filename, temp_filename);
 
         if let Some((_, state)) = WRITERS.remove(&temp_filename) {
-            let WriterState { writer: writer_arc, settings, crc_handle } = state;
+            let WriterState { writer: writer_arc, settings, crc_handle, writer_generation } = state;
             let index_name = settings.index_name.as_deref().unwrap_or("");
             match Arc::try_unwrap(writer_arc) {
                 Ok(mutex) => {
@@ -170,7 +173,7 @@ impl NativeParquetWriter {
                             let temp_crc32 = crc_handle.crc32();
                             log_debug!("Successfully closed temp writer for: {}", temp_filename);
 
-                            let crc32 = Self::sort_and_rewrite_parquet(&temp_filename, &filename, index_name, &settings.sort_columns, &settings.reverse_sorts, &settings.nulls_first, temp_crc32)?;
+                            let crc32 = Self::sort_and_rewrite_parquet(&temp_filename, &filename, index_name, &settings.sort_columns, &settings.reverse_sorts, &settings.nulls_first, temp_crc32, writer_generation)?;
 
                             if Path::new(&temp_filename).exists() {
                                 if let Err(e) = std::fs::remove_file(&temp_filename) {
@@ -217,6 +220,7 @@ impl NativeParquetWriter {
         reverse_sorts: &[bool],
         nulls_first: &[bool],
         temp_crc32: u32,
+        writer_generation: i64,
     ) -> Result> {
         log_debug!(
             "sort_and_rewrite_parquet: temp={}, output={}, sort_columns={:?}, reverse_sorts={:?}, nulls_first={:?}",
@@ -237,7 +241,7 @@ impl NativeParquetWriter {
         let file_size = std::fs::metadata(temp_filename)?.len();
 
         if file_size <= config.get_sort_in_memory_threshold_bytes() {
-            Self::sort_small_file(temp_filename, output_filename, index_name, sort_columns, reverse_sorts, nulls_first)
+            Self::sort_small_file(temp_filename, output_filename, index_name, sort_columns, reverse_sorts, nulls_first, writer_generation)
         } else {
             Self::sort_large_file(temp_filename, output_filename, index_name, sort_columns, reverse_sorts, nulls_first, config.get_sort_batch_size())
         }
@@ -251,6 +255,7 @@ impl NativeParquetWriter {
         sort_columns: &[String],
         reverse_sorts: &[bool],
         nulls_first: &[bool],
+        writer_generation: i64,
     ) -> Result> {
         log_debug!("Using in-memory sort for small file: {}", temp_filename);
 
@@ -273,7 +278,7 @@ impl NativeParquetWriter {
         let sorted_batch = Self::sort_batch(&batch, sort_columns, reverse_sorts, nulls_first)?;
         let final_batch = Self::rewrite_row_ids(&sorted_batch, &schema)?;
 
-        let crc32 = Self::write_final_file(output_filename, index_name, &final_batch, schema)?;
+        let crc32 = Self::write_final_file(output_filename, index_name, &final_batch, schema, Some(writer_generation))?;
         Ok(crc32)
     }
 
@@ -309,7 +314,7 @@ impl NativeParquetWriter {
                 .to_string_lossy()
                 .to_string();
             // CRC for temp chunks is not needed, discard it
-            Self::write_final_file(&chunk_filename, index_name, &sorted_batch, schema)?;
+            Self::write_final_file(&chunk_filename, index_name, &sorted_batch, schema, None)?;
 
             chunk_paths.push(chunk_filename);
             batch_count += 1;
@@ -323,8 +328,9 @@ impl NativeParquetWriter {
 
         log_debug!("Created {} sorted chunks, merging via streaming k-way merge", batch_count);
 
-        // merge_sorted returns metadata and CRC32 of the final merged output
-        let (_metadata, crc32) = merge_sorted(
+        // merge_sorted returns MergeOutput; we discard it here because the
+        // sort-and-rewrite path produces the final file directly.
+        let _merge_output = merge_sorted(
             &chunk_paths,
             output_filename,
             index_name,
@@ -338,7 +344,7 @@ impl NativeParquetWriter {
             let _ = std::fs::remove_file(path);
         }
 
-        Ok(crc32)
+        Ok(0)
     }
 
     fn sort_batch(
@@ -401,12 +407,13 @@ impl NativeParquetWriter {
         index_name: &str,
         batch: &RecordBatch,
         schema: Arc,
+        writer_generation: Option,
     ) -> Result> {
         let config = SETTINGS_STORE
             .get(index_name)
             .map(|r| r.clone())
             .unwrap_or_default();
-        let props = WriterPropertiesBuilder::build(&config);
+        let props = WriterPropertiesBuilder::build_with_generation(&config, writer_generation);
         let file = File::create(output_filename)?;
         let (crc_file, crc_handle) = CrcWriter::new(file);
         let mut writer = ArrowWriter::try_new(crc_file, schema, Some(props))?;
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/writer_properties_builder.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer_properties_builder.rs
index d5676a33c9e0f..4b1cf64f76a51 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/writer_properties_builder.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer_properties_builder.rs
@@ -7,10 +7,28 @@
  */
 
 use parquet::basic::{Compression, ZstdLevel, GzipLevel, BrotliLevel};
+use parquet::file::metadata::{FileMetaData, KeyValue};
 use parquet::file::properties::WriterProperties;
 
 use crate::native_settings::NativeSettings;
 
+/// Parquet file-level metadata key for the writer generation.
+pub const WRITER_GENERATION_KEY: &str = "opensearch.writer_generation";
+
+/// Reads the writer generation from a Parquet file's key-value metadata.
+/// Returns the generation value, or falls back to `file_index` if not present.
+pub fn read_writer_generation(metadata: &FileMetaData, file_index: usize) -> i64 {
+    metadata
+        .key_value_metadata()
+        .and_then(|kvs| {
+            kvs.iter()
+                .find(|kv| kv.key == WRITER_GENERATION_KEY)
+                .and_then(|kv| kv.value.as_ref())
+                .and_then(|v| v.parse::().ok())
+        })
+        .unwrap_or(file_index as i64)
+}
+
 /// Builder for converting NativeSettings into Parquet WriterProperties.
 ///
 /// This struct follows the Single Responsibility Principle by focusing
@@ -37,6 +55,11 @@ impl WriterPropertiesBuilder {
     ///
     /// A fully configured WriterProperties instance
     pub fn build(config: &NativeSettings) -> WriterProperties {
+        Self::build_with_generation(config, None)
+    }
+
+    /// Builds WriterProperties with an optional writer generation stored as key-value metadata.
+    pub fn build_with_generation(config: &NativeSettings, writer_generation: Option) -> WriterProperties {
         let mut builder = WriterProperties::builder();
 
         // Apply compression settings
@@ -57,6 +80,13 @@ impl WriterPropertiesBuilder {
         // Apply field-level configurations
         builder = Self::apply_field_configs(builder, config);
 
+        // Store writer generation in file-level key-value metadata
+        if let Some(gen) = writer_generation {
+            builder = builder.set_key_value_metadata(Some(vec![
+                KeyValue::new(WRITER_GENERATION_KEY.to_string(), Some(gen.to_string())),
+            ]));
+        }
+
         builder.build()
     }
 
diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/NativeParquetWriterTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/NativeParquetWriterTests.java
index e0795008dde03..57064a241df56 100644
--- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/NativeParquetWriterTests.java
+++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/NativeParquetWriterTests.java
@@ -148,7 +148,8 @@ public void testCreateWriterWithNonExistentDirectory() {
                     "/nonexistent/dir/file.parquet",
                     "test-index",
                     export.getSchemaAddress(),
-                    ParquetSortConfig.empty()
+                    ParquetSortConfig.empty(),
+                    0L
                 );
             }
         });
@@ -156,7 +157,7 @@ public void testCreateWriterWithNonExistentDirectory() {
 
     public void testCreateWriterWithInvalidSchemaAddress() {
         String filePath = createTempDir().resolve("bad-schema.parquet").toString();
-        expectThrows(Exception.class, () -> new NativeParquetWriter(filePath, "test-index", 0L, ParquetSortConfig.empty()));
+        expectThrows(Exception.class, () -> new NativeParquetWriter(filePath, "test-index", 0L, ParquetSortConfig.empty(), 0L));
     }
 
     public void testWriteWithSchemaMismatch() throws Exception {
@@ -240,7 +241,7 @@ public void testWriteWithNullAddresses() throws Exception {
 
     private NativeParquetWriter createWriter(String filePath) throws Exception {
         try (ArrowExport export = exportSchema()) {
-            return new NativeParquetWriter(filePath, "test-index", export.getSchemaAddress(), ParquetSortConfig.empty());
+            return new NativeParquetWriter(filePath, "test-index", export.getSchemaAddress(), ParquetSortConfig.empty(), 0L);
         }
     }
 
diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java
index be07a0a9a4f31..d5e01cb12d919 100644
--- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java
+++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetMergeIntegrationTests.java
@@ -128,7 +128,7 @@ private String createSortedFile(Path dir, String name, long[] timestamps, String
         ParquetSortConfig sortConfig = new ParquetSortConfig(List.of("timestamp"), List.of(false), List.of(false));
 
         try (ArrowExport schemaExport = exportSchema()) {
-            NativeParquetWriter writer = new NativeParquetWriter(filePath, INDEX_NAME, schemaExport.getSchemaAddress(), sortConfig);
+            NativeParquetWriter writer = new NativeParquetWriter(filePath, INDEX_NAME, schemaExport.getSchemaAddress(), sortConfig, 0L);
 
             try (ArrowExport dataExport = exportData(timestamps, messages)) {
                 writer.write(dataExport.getArrayAddress(), dataExport.getSchemaAddress());
diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java
index 4d289e7c3ea1f..450fe50785300 100644
--- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java
+++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/vsr/VSRManagerTests.java
@@ -71,7 +71,7 @@ public void tearDown() throws Exception {
 
     public void testConstructionInitializesActiveVSR() throws Exception {
         String filePath = createTempDir().resolve("init.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool, 0L);
         assertNotNull(manager.getActiveManagedVSR());
         assertEquals(VSRState.ACTIVE, manager.getActiveManagedVSR().getState());
         // flush handles freeze + close internally
@@ -80,7 +80,7 @@ public void testConstructionInitializesActiveVSR() throws Exception {
 
     public void testFlushWithNoDataReturnsMetadata() throws Exception {
         String filePath = createTempDir().resolve("empty.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool, 0L);
         ParquetFileMetadata metadata = manager.flush();
         assertNotNull(metadata);
         assertEquals(0, metadata.numRows());
@@ -88,7 +88,7 @@ public void testFlushWithNoDataReturnsMetadata() throws Exception {
 
     public void testFlushWithData() throws Exception {
         String filePath = createTempDir().resolve("data.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool, 0L);
 
         ManagedVSR active = manager.getActiveManagedVSR();
         IntVector vec = (IntVector) active.getVector("val");
@@ -104,7 +104,7 @@ public void testFlushWithData() throws Exception {
 
     public void testAddDocument() throws Exception {
         String filePath = createTempDir().resolve("add-doc.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool, 0L);
 
         NumberFieldMapper.NumberFieldType valField = new NumberFieldMapper.NumberFieldType("val", NumberFieldMapper.NumberType.INTEGER);
         ParquetDocumentInput doc = new ParquetDocumentInput();
@@ -120,7 +120,7 @@ public void testAddDocument() throws Exception {
 
     public void testSyncAfterFlush() throws Exception {
         String filePath = createTempDir().resolve("sync.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool, 0L);
 
         ManagedVSR active = manager.getActiveManagedVSR();
         IntVector vec = (IntVector) active.getVector("val");
@@ -134,7 +134,7 @@ public void testSyncAfterFlush() throws Exception {
 
     public void testMaybeRotateNoOpBelowThreshold() throws Exception {
         String filePath = createTempDir().resolve("norotate.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool, 0L);
         ManagedVSR original = manager.getActiveManagedVSR();
         original.setRowCount(100);
         manager.maybeRotateActiveVSR();
@@ -144,7 +144,7 @@ public void testMaybeRotateNoOpBelowThreshold() throws Exception {
 
     public void testMaybeRotateAtThreshold() throws Exception {
         String filePath = createTempDir().resolve("rotate.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool, 0L);
 
         ManagedVSR original = manager.getActiveManagedVSR();
         original.setRowCount(50000);
@@ -158,7 +158,7 @@ public void testMaybeRotateAtThreshold() throws Exception {
 
     public void testFlushAfterRotation() throws Exception {
         String filePath = createTempDir().resolve("rotate-flush.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 50000, threadPool, 0L);
 
         // Fill first VSR to trigger rotation
         ManagedVSR first = manager.getActiveManagedVSR();
@@ -182,7 +182,7 @@ public void testFlushAfterRotation() throws Exception {
 
     public void testRotationAwaitsWhenFrozenSlotOccupied() throws Exception {
         String filePath = createTempDir().resolve("double-rotate.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool, 0L);
 
         // Fill first VSR to trigger rotation (async write submitted)
         ManagedVSR first = manager.getActiveManagedVSR();
@@ -217,7 +217,7 @@ public void testRotationAwaitsWhenFrozenSlotOccupied() throws Exception {
 
     public void testRotationWritesHappenOnBackgroundThread() throws Exception {
         String filePath = createTempDir().resolve("bg-thread.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool, 0L);
 
         // Fill and rotate
         ManagedVSR first = manager.getActiveManagedVSR();
@@ -246,7 +246,7 @@ public void testRotationWritesHappenOnBackgroundThread() throws Exception {
 
     public void testFlushAwaitsBackgroundWrite() throws Exception {
         String filePath = createTempDir().resolve("flush-await.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool, 0L);
 
         // Fill and rotate to trigger background write
         ManagedVSR first = manager.getActiveManagedVSR();
@@ -271,7 +271,7 @@ public void testFlushAwaitsBackgroundWrite() throws Exception {
 
     public void testCloseAwaitsBackgroundWrite() throws Exception {
         String filePath = createTempDir().resolve("close-await.parquet").toString();
-        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool);
+        VSRManager manager = new VSRManager(filePath, indexSettings, schema, bufferPool, 100, threadPool, 0L);
 
         // Fill and rotate to trigger background write
         ManagedVSR first = manager.getActiveManagedVSR();
diff --git a/server/src/main/java/org/apache/lucene/index/MergeIndexWriter.java b/server/src/main/java/org/apache/lucene/index/MergeIndexWriter.java
new file mode 100644
index 0000000000000..6e1e5ad22ebdf
--- /dev/null
+++ b/server/src/main/java/org/apache/lucene/index/MergeIndexWriter.java
@@ -0,0 +1,95 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.apache.lucene.index;
+
+import org.apache.lucene.store.Directory;
+
+import java.io.IOException;
+
+/**
+ * An {@link IndexWriter} subclass that exposes Lucene's internal {@code merge(OneMerge)}
+ * path for use by the pluggable data format merge infrastructure.
+ *
+ * 
        The internal merge path handles the full segment lifecycle including reference-counted
+ * file cleanup via {@code IndexFileDeleter}. If the merge fails, old segments are preserved
+ * and the partially-written merged segment is cleaned up — providing a safe rollback mechanism.
+ *
+ * 
        This class is placed in the {@code org.apache.lucene.index} package to access
+ * package-private fields on {@link MergePolicy.OneMerge} required for merge registration.
+ *
+ * 
        The {@link IndexWriterConfig} used to construct this writer must set a
+ * {@link SerialMergeScheduler} to avoid the {@link ConcurrentMergeScheduler} thread
+ * assertion in {@code wrapForMerge}, since pluggable data format merges run on the
+ * engine's own merge thread pool rather than Lucene's {@code MergeThread}.
+ *
+ * 
        Coordination with engine refreshes
        
+ *
+ * 
        This class itself does not take any engine-level locks. Coordination with the engine's
+ * refresh path is layered on top by installing a {@code MergedSegmentWarmer} on the
+ * {@link IndexWriterConfig} (see {@code LuceneCommitter}). The warmer runs between
+ * {@code mergeMiddle} and {@code commitMerge}, at a point where the {@link IndexWriter}
+ * monitor is not held, and acquires the engine's refresh lock. This establishes the
+ * ordering {@code refreshLock → IndexWriter monitor} on the merge thread, matching the order
+ * used by the engine's refresh path (which takes the refresh lock before calling
+ * {@code addIndexes}). The expensive {@code mergeMiddle} phase therefore runs without holding
+ * the refresh lock, and only the short {@code commitMerge} window is serialized against
+ * refreshes.
+ *
+ * @opensearch.experimental
+ */
+public class MergeIndexWriter extends IndexWriter {
+
+    public MergeIndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
+        super(d, conf);
+    }
+
+    /**
+     * Executes a merge using Lucene's internal merge path which handles:
+     * 
          
+     *   
        1. mergeInit — creates output segment info, increments file references
          2. 
+     *   
        2. mergeMiddle — reads sources via wrapForMerge, applies IndexSort via MultiSorter,
+     *       writes merged segment
          3. 
+     *   
        3. commitMerge — removes old segments from live list, decrements file references
          4. 
+     *   
        4. mergeFinish — cleans up merge tracking state
          5. 
+     * 
        
+     *
+     * 
        If the merge fails at any point, old segments are preserved and the partially-written
+     * merged segment is cleaned up by IndexFileDeleter's reference counting.
+     *
+     * 
        Duplicate segment prevention is handled by the caller; this method does not
+     * validate against concurrent merges on the same segments.
+     *
+     * 
        Refresh-lock coordination is handled by the {@code MergedSegmentWarmer} installed on
+     * this writer's {@link IndexWriterConfig} — see the class Javadoc for details.
+     *
+     * @param oneMerge       the merge to execute
+     * @param mergeGeneration the writer generation for the merged output segment
+     * @throws IOException if the merge fails
+     */
+    public void executeMerge(MergePolicy.OneMerge oneMerge, long mergeGeneration) throws IOException {
+        synchronized (this) {
+            oneMerge.mergeGen = mergeGeneration;
+            oneMerge.isExternal = false;
+            oneMerge.maxNumSegments = -1;
+            oneMerge.registerDone = true;
+        }
+        // merge() must be called without holding the lock — mergeInit asserts !Thread.holdsLock(this).
+        // Refresh-lock acquisition happens inside the MergedSegmentWarmer configured on this writer,
+        // which fires between mergeMiddle and commitMerge while the IW monitor is not held. This
+        // matches the refresh path's lock order (refreshLock → IW monitor) and avoids any inversion.
+        merge(oneMerge);
+    }
+
+    @Override
+    protected void mergeSuccess(MergePolicy.OneMerge merge) {
+        // TODO update this for lucene as a primary engine
+        // https://github.com/opensearch-project/OpenSearch/issues/21505
+        super.mergeSuccess(merge);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java
index a2a630c902357..02e4630d2d016 100644
--- a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java
+++ b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java
@@ -226,7 +226,19 @@ public DataFormatAwareEngine(EngineConfig engineConfig) {
             store.incRef();
 
             // 1. Create Committer (uses translogPath for safe bootstrap trimming)
-            this.committer = engineConfig.getCommitterFactory().getCommitter(new CommitterConfig(engineConfig));
+            // Encapsulate refreshLock access behind a pre-merge-commit hook: committer-owned
+            // writers (e.g. Lucene MergeIndexWriter) invoke the hook on the merge thread
+            // immediately before the merged segment becomes visible. When Lucene participates
+            // in a merge, its committer wires the hook into a MergedSegmentWarmer that fires
+            // between mergeMiddle and commitMerge — the IndexWriter monitor is not held there,
+            // so acquiring refreshLock via the hook establishes the same refreshLock → IW
+            // monitor ordering that the refresh path uses and avoids lock inversion. Ownership
+            // then transfers to applyMergeChanges, which releases the lock after the catalog
+            // is updated. For merges that do not invoke the hook — pure Parquet merges, or
+            // Lucene merges that skip because the shared writer has no matching segments —
+            // applyMergeChanges acquires refreshLock itself. Either way, applyMergeChanges
+            // releases the lock before returning.
+            this.committer = engineConfig.getCommitterFactory().getCommitter(new CommitterConfig(engineConfig, refreshLock::lock));
 
             // 2. Read translogUUID and history UUID from last committed data
             final Map userData = committer.getLastCommittedData();
@@ -1379,9 +1391,20 @@ private void applyMergeChanges(MergeResult mergeResult, OneMerge oneMerge) {
         assert mergeResult != null : "merge result must not be null";
         assert oneMerge != null : "oneMerge must not be null";
         assert oneMerge.getSegmentsToMerge().isEmpty() == false : "merged segments list must not be empty";
-        refreshLock.lock();
-        try {
+        // refreshLock may already be held by the merge thread when Lucene participated in the
+        // merge: the Lucene committer's MergedSegmentWarmer acquires it between mergeMiddle and
+        // commitMerge to coordinate with refreshes. When Lucene is not a participant (pure-Parquet
+        // merges, or Lucene merges that skip because the shared writer has no matching segments),
+        // the warmer never fires and the lock is not held on entry; acquire it locally to
+        // serialise the catalog update against concurrent refreshes. Always release on exit.
+        final boolean acquiredHere = refreshLock.isHeldByCurrentThread() == false;
+        if (acquiredHere) {
+            refreshLock.lock();
+        }
+        try (GatedCloseable oldSnapshotRef = catalogSnapshotManager.acquireSnapshot()) {
+            notifyRefreshListenersBefore();
             catalogSnapshotManager.applyMergeResults(mergeResult, oneMerge);
+            notifyRefreshListenersAfter(true);
         } catch (Exception ex) {
             try {
                 logger.error(() -> new ParameterizedMessage("Merge failed while registering merged files in Snapshot"), ex);
diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/PackedRowIdMapping.java b/server/src/main/java/org/opensearch/index/engine/dataformat/PackedRowIdMapping.java
new file mode 100644
index 0000000000000..510d2036440b9
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/engine/dataformat/PackedRowIdMapping.java
@@ -0,0 +1,152 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.engine.dataformat;
+
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PackedLongValues;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Compact implementation of {@link RowIdMapping} using Lucene's PackedLongValues for memory-efficient
+ * storage of row ID mappings produced during merge operations.
+ *
+ * 
        Structure:
+ * 
          
+ *   
        • A single flat packed array where {@code mapping[position] = newRowId}
          • 
+ *   
        • {@code generationOffsets} maps writer generation to starting offset in the array
          • 
+ *   
        • {@code generationSizes} maps writer generation to number of rows in that generation
          • 
+ * 
        
+ *
+ * 
        Offsets are assigned in the order generations are processed during the primary format's merge,
+ * not sorted. This ensures the mapping is independent of generation ordering.
+ *
+ * 
        Example: merge processes generations in order [5, 0, 3]:
+ * 
        + *   generation 5 (2 rows): offset=0, mapping[0]=2, mapping[1]=3
+ *   generation 0 (3 rows): offset=2, mapping[2]=0, mapping[3]=4, mapping[4]=1
+ *   generation 3 (1 row):  offset=5, mapping[5]=5
+ *
+ *   Lookup: newRowId = mapping.get(generationOffsets.get(generation) + oldRowId)
+ * 
        
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public final class PackedRowIdMapping implements RowIdMapping {
+
+    private final PackedLongValues mapping;
+    private final Map generationOffsets;
+    private final Map generationSizes;
+
+    /**
+     * Creates a PackedRowIdMapping from a mapping array, generation offsets, and generation sizes.
+     *
+     * @param mappingArray array where index=position, value=newRowId
+     * @param generationOffsets map of writer generation to starting offset in the mapping array
+     * @param generationSizes map of writer generation to number of rows in that generation
+     */
+    public PackedRowIdMapping(long[] mappingArray, Map generationOffsets, Map generationSizes) {
+        Objects.requireNonNull(mappingArray, "mappingArray cannot be null");
+        Objects.requireNonNull(generationOffsets, "generationOffsets cannot be null");
+        Objects.requireNonNull(generationSizes, "generationSizes cannot be null");
+
+        PackedLongValues.Builder builder = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
+        for (long value : mappingArray) {
+            builder.add(value);
+        }
+        this.mapping = builder.build();
+        this.generationOffsets = Collections.unmodifiableMap(new HashMap<>(generationOffsets));
+        this.generationSizes = Collections.unmodifiableMap(new HashMap<>(generationSizes));
+    }
+
+    /**
+     * Returns the new row ID for the given old row ID and writer generation.
+     * O(1) lookup via offset calculation.
+     *
+     * @param oldId the original row ID within the generation
+     * @param oldGeneration the writer generation of the source segment
+     * @return the new row ID, or -1 if the generation or row ID is not found
+     */
+    @Override
+    public long getNewRowId(long oldId, long oldGeneration) {
+        Integer offset = generationOffsets.get(oldGeneration);
+        if (offset == null) {
+            return -1L;
+        }
+        Integer size = generationSizes.get(oldGeneration);
+        if (size == null || oldId < 0 || oldId >= size) {
+            return -1L;
+        }
+        return mapping.get(offset + (int) oldId);
+    }
+
+    /**
+     * Returns the number of rows for a specific writer generation.
+     *
+     * @param generation the writer generation
+     * @return the number of rows, or 0 if the generation is not found
+     */
+    public int getGenerationSize(long generation) {
+        Integer size = generationSizes.get(generation);
+        return size != null ? size : 0;
+    }
+
+    /**
+     * Returns the total number of entries in the mapping.
+     *
+     * @return the total mapping size
+     */
+    public int size() {
+        return (int) mapping.size();
+    }
+
+    /**
+     * Returns the estimated memory usage of this mapping in bytes.
+     *
+     * @return estimated memory in bytes
+     */
+    public long ramBytesUsed() {
+        return mapping.ramBytesUsed();
+    }
+
+    /**
+     * Returns an unmodifiable view of the generation offsets.
+     *
+     * @return map of writer generation to starting offset
+     */
+    public Map getGenerationOffsets() {
+        return generationOffsets;
+    }
+
+    /**
+     * Returns an unmodifiable view of the generation sizes.
+     *
+     * @return map of writer generation to row count
+     */
+    public Map getGenerationSizes() {
+        return generationSizes;
+    }
+
+    @Override
+    public String toString() {
+        return "PackedRowIdMapping{"
+            + "size="
+            + mapping.size()
+            + ", generations="
+            + generationOffsets.size()
+            + ", estimatedMemoryBytes="
+            + mapping.ramBytesUsed()
+            + '}';
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/engine/exec/commit/CommitterConfig.java b/server/src/main/java/org/opensearch/index/engine/exec/commit/CommitterConfig.java
index 57871cf25021e..f9ebc287a8a08 100644
--- a/server/src/main/java/org/opensearch/index/engine/exec/commit/CommitterConfig.java
+++ b/server/src/main/java/org/opensearch/index/engine/exec/commit/CommitterConfig.java
@@ -13,11 +13,28 @@
 
 /**
  * Initialization parameters for a {@link Committer}.
- * Carries the engine configuration needed to set up the backing store.
  *
- * @param engineConfig the engine configuration (nullable — may be absent in tests or standalone mode)
+ * 
        {@code preMergeCommitHook} is invoked by committers that own writers participating in
+ * merges (e.g. the Lucene {@code MergeIndexWriter}) at the moment a merged segment becomes
+ * ready but before it is made visible. The hook is expected to run on the merge thread
+ * between {@code mergeMiddle} and {@code commitMerge}, while the underlying writer's
+ * exclusive monitor is not held. The engine wires this hook to refresh-lock
+ * acquisition so that merge-thread visibility is serialised against concurrent refreshes,
+ * avoiding the lock inversion that would occur if the engine acquired the refresh lock
+ * inside {@code commitMerge}. Any ownership acquired by the hook is transferred to the
+ * engine's merge-apply callback, which releases it after the catalog is updated.
+ *
+ * 
        For merges that never reach the hook (pure Parquet merges, or Lucene merges that skip
+ * because the shared writer has no matching segments), the merge-apply callback handles
+ * coordination on its own. Committers that do not need this coordination may install the
+ * hook but take no action when it fires.
+ *
+ * @param engineConfig         engine configuration
+ * @param preMergeCommitHook   hook run on the merge thread before a merged segment is made
+ *                             visible; ownership of anything it acquires is transferred to
+ *                             the engine's merge-apply callback
  * @opensearch.experimental
  */
 @ExperimentalApi
-public record CommitterConfig(EngineConfig engineConfig) {
+public record CommitterConfig(EngineConfig engineConfig, Runnable preMergeCommitHook) {
 }
diff --git a/server/src/test/java/org/opensearch/index/engine/DataFormatAwareEngineTests.java b/server/src/test/java/org/opensearch/index/engine/DataFormatAwareEngineTests.java
index 249665494ed9d..f34823d54fe7d 100644
--- a/server/src/test/java/org/opensearch/index/engine/DataFormatAwareEngineTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/DataFormatAwareEngineTests.java
@@ -1646,4 +1646,98 @@ public void afterRefresh(boolean didRefresh) {
             assertThat("afterRefresh must see post-commit generation", genSeenInAfter.get(), equalTo(1L));
         }
     }
+
+    /**
+     * Covers {@code DataFormatAwareEngine.applyMergeChanges}: a forceMerge over two
+     * previously-refreshed segments must (1) replace the source segments in the catalog
+     * with a single merged segment, (2) invoke beforeRefresh/afterRefresh exactly once
+     * each on registered refresh listeners while holding the refresh lock, and
+     * (3) release the refresh lock on exit so a subsequent {@code refresh()} proceeds.
+     *
+     * 
        The system-property gate on {@code MERGE_ENABLED_PROPERTY} applies only to
+     * the background {@code triggerPossibleMerges()} path; {@code forceMerge} routes
+     * straight to {@code MergeScheduler.forceMerge} and does not consult it, so this
+     * test drives the merge end-to-end without touching system properties.
+     */
+    public void testApplyMergeChangesUpdatesCatalogAndNotifiesListeners() throws Exception {
+        AtomicInteger beforeCalls = new AtomicInteger();
+        AtomicInteger afterCalls = new AtomicInteger();
+        // Records call order: 'B' for beforeRefresh, 'A' for afterRefresh.
+        StringBuilder callOrder = new StringBuilder();
+
+        ReferenceManager.RefreshListener listener = new ReferenceManager.RefreshListener() {
+            @Override
+            public void beforeRefresh() {
+                synchronized (callOrder) {
+                    callOrder.append('B');
+                }
+                beforeCalls.incrementAndGet();
+            }
+
+            @Override
+            public void afterRefresh(boolean didRefresh) {
+                synchronized (callOrder) {
+                    callOrder.append('A');
+                }
+                afterCalls.incrementAndGet();
+            }
+        };
+
+        Path translogPath = createTempDir();
+        String uuid = Translog.createEmptyTranslog(translogPath, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get());
+        bootstrapStoreWithMetadata(store, uuid);
+
+        EngineConfig config = buildDFAEngineConfig(store, translogPath, List.of(listener), List.of());
+        try (DataFormatAwareEngine engine = new DataFormatAwareEngine(config)) {
+            // Produce two segments via two refresh cycles so the merger has something to combine.
+            engine.index(indexOp(createParsedDocWithInput("1", null)));
+            engine.refresh("seed-1");
+            engine.index(indexOp(createParsedDocWithInput("2", null)));
+            engine.refresh("seed-2");
+
+            try (GatedCloseable ref = engine.acquireSnapshot()) {
+                assertThat("two segments before merge", ref.get().getSegments().size(), equalTo(2));
+            }
+
+            // Drain the listener counters from the two seed refreshes.
+            final int beforeAfterSeed = beforeCalls.get();
+            final int afterAfterSeed = afterCalls.get();
+            assertThat("each refresh must invoke beforeRefresh once", beforeAfterSeed, equalTo(2));
+            assertThat("each refresh must invoke afterRefresh once", afterAfterSeed, equalTo(2));
+
+            // forceMerge submits the merge to the FORCE_MERGE executor and returns without
+            // waiting. Poll the catalog until the merged snapshot is visible (or fail fast).
+            engine.forceMerge(false, 1, false, false, false, "test-force-merge");
+
+            assertBusy(() -> {
+                try (GatedCloseable ref = engine.acquireSnapshot()) {
+                    assertThat("merge must collapse to a single segment", ref.get().getSegments().size(), equalTo(1));
+                }
+            }, 10, java.util.concurrent.TimeUnit.SECONDS);
+
+            // applyMergeChanges must have invoked the listeners exactly once each, in order.
+            assertThat("beforeRefresh must fire exactly once for the merge", beforeCalls.get() - beforeAfterSeed, equalTo(1));
+            assertThat("afterRefresh must fire exactly once for the merge", afterCalls.get() - afterAfterSeed, equalTo(1));
+            synchronized (callOrder) {
+                // Seed cycles contribute "BABA"; the merge must append exactly "BA".
+                assertThat("call order must be before-then-after for every cycle", callOrder.toString(), equalTo("BABABA"));
+            }
+
+            // Sanity: the refreshLock must have been released. A follow-up refresh must
+            // complete without blocking, and the catalog generation must have advanced.
+            long genBeforeFinalRefresh;
+            try (GatedCloseable ref = engine.acquireSnapshot()) {
+                genBeforeFinalRefresh = ref.get().getGeneration();
+            }
+            engine.index(indexOp(createParsedDocWithInput("3", null)));
+            engine.refresh("post-merge");
+            try (GatedCloseable ref = engine.acquireSnapshot()) {
+                assertThat(
+                    "refresh after merge must advance the catalog generation",
+                    ref.get().getGeneration(),
+                    greaterThan(genBeforeFinalRefresh)
+                );
+            }
+        }
+    }
 }
diff --git a/server/src/test/java/org/opensearch/index/engine/exec/coord/SafeBootstrapCommitterTests.java b/server/src/test/java/org/opensearch/index/engine/exec/coord/SafeBootstrapCommitterTests.java
index 4e586b09dce8a..a00f14c2e8810 100644
--- a/server/src/test/java/org/opensearch/index/engine/exec/coord/SafeBootstrapCommitterTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/exec/coord/SafeBootstrapCommitterTests.java
@@ -115,7 +115,7 @@ private EngineConfig buildEngineConfig(Store store, Path translogPath) {
 
     public void testThrowsWhenNullEngineConfig() {
         reset();
-        expectThrows(IllegalArgumentException.class, () -> new TestCommitter(new CommitterConfig(null)));
+        expectThrows(IllegalArgumentException.class, () -> new TestCommitter(new CommitterConfig(null, () -> {})));
     }
 
     public void testThrowsWhenNullTranslogConfig() throws IOException {
@@ -126,7 +126,7 @@ public void testThrowsWhenNullTranslogConfig() throws IOException {
                 .store(store)
                 .retentionLeasesSupplier(() -> new RetentionLeases(0, 0, Collections.emptyList()))
                 .build();
-            expectThrows(IllegalArgumentException.class, () -> new TestCommitter(new CommitterConfig(ec)));
+            expectThrows(IllegalArgumentException.class, () -> new TestCommitter(new CommitterConfig(ec, () -> {})));
         } finally {
             store.close();
         }
@@ -137,7 +137,7 @@ public void testDiscoverAndTrimCalledWithValidConfig() throws IOException {
         Store store = createStore();
         Path translogPath = createTempDir();
         try {
-            new TestCommitter(new CommitterConfig(buildEngineConfig(store, translogPath)));
+            new TestCommitter(new CommitterConfig(buildEngineConfig(store, translogPath), () -> {}));
             assertTrue(discoverAndTrimCalled);
         } finally {
             store.close();

From 5e1e412054595328e19acfa8636c0167a83802b1 Mon Sep 17 00:00:00 2001
From: Arpit Bandejiya 
Date: Fri, 8 May 2026 22:24:15 +0530
Subject: [PATCH 090/115] Add dynamic settings for indexed query execution path
 (#21522)

* Add dynamic settings for Search Request

Signed-off-by: Arpit Bandejiya 

* version 2

Signed-off-by: Arpit Bandejiya 

* Need more docs pls

Signed-off-by: Arpit Bandejiya 

* Add support for concurrency defaults

Signed-off-by: Arpit Bandejiya 

* Update concurrency config

Signed-off-by: Arpit Bandejiya 

* Add more tests

Signed-off-by: Arpit Bandejiya 

* refactoring

Signed-off-by: Arpit Bandejiya 

* fix tests

Signed-off-by: Arpit Bandejiya 

* Fix the tests

Signed-off-by: Arpit Bandejiya 

* fix spotless

Signed-off-by: Arpit Bandejiya 

* Fixes

Signed-off-by: Arpit Bandejiya 

---------

Signed-off-by: Arpit Bandejiya 
---
 .../rust/benches/query_bench.rs               |   8 +-
 .../rust/src/api.rs                           |  14 +-
 .../rust/src/datafusion_query_config.rs       | 149 ++++++--
 .../rust/src/ffm.rs                           |  91 +++--
 .../rust/src/indexed_executor.rs              |  14 +-
 .../rust/src/indexed_table/stream.rs          |   6 +-
 .../rust/src/indexed_table/table_provider.rs  |  19 +-
 .../indexed_table/tests_e2e/fuzz/harness.rs   |  97 ++---
 .../rust/src/indexed_table/tests_e2e/mod.rs   |  20 +-
 .../indexed_table/tests_e2e/multi_segment.rs  |  98 ++---
 .../indexed_table/tests_e2e/null_columns.rs   |  15 +-
 .../indexed_table/tests_e2e/page_pruning.rs   | 167 +++++----
 .../indexed_table/tests_e2e/schema_drift.rs   |  12 +-
 .../tests_e2e/streaming_at_scale.rs           |  24 +-
 .../rust/src/session_context.rs               |  26 +-
 .../DatafusionDynamicSettingsIT.java          | 123 +++++++
 .../be/datafusion/DataFusionPlugin.java       |  20 +-
 .../be/datafusion/DatafusionSettings.java     | 348 ++++++++++++++++++
 .../ShardScanInstructionHandler.java          |  19 +-
 .../ShardScanWithDelegationHandler.java       |  30 +-
 .../be/datafusion/WireConfigSnapshot.java     | 220 +++++++++++
 .../be/datafusion/nativelib/NativeBridge.java |  24 +-
 .../nativelib/SessionContextConfig.java       |  28 ++
 .../DataFusionNativeBridgeTests.java          |  12 +-
 .../DataFusionPluginSettingsTests.java        |  36 ++
 .../DataFusionQueryExecutionTests.java        |  12 +-
 .../DatafusionResultStreamTests.java          |  16 +-
 .../DatafusionSearchExecEngineTests.java      |   9 +-
 .../DatafusionSettingsPropertyTests.java      | 212 +++++++++++
 .../datafusion/DatafusionSettingsTests.java   | 162 ++++++++
 .../datafusion/WireConfigSnapshotTests.java   | 112 ++++++
 31 files changed, 1813 insertions(+), 330 deletions(-)
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/DatafusionDynamicSettingsIT.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSettings.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/WireConfigSnapshot.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextConfig.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsPropertyTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/WireConfigSnapshotTests.java

diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/benches/query_bench.rs b/sandbox/plugins/analytics-backend-datafusion/rust/benches/query_bench.rs
index 25b1a993c289f..e970cad50963a 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/benches/query_bench.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/benches/query_bench.rs
@@ -46,7 +46,7 @@ fn setup() -> (RuntimeManager, DataFusionRuntime, tempfile::TempDir) {
         .with_disk_manager_builder(DiskManagerBuilder::default())
         .build()
         .unwrap();
-    let df_runtime = DataFusionRuntime { runtime_env };
+    let df_runtime = DataFusionRuntime::new_for_bench(runtime_env);
     let tmp = tempfile::tempdir().unwrap();
     (mgr, df_runtime, tmp)
 }
@@ -104,7 +104,7 @@ fn bench_execute_query(c: &mut Criterion) {
                     let exec = mgr.cpu_executor();
                     async {
                         let ptr = query_executor::execute_query(
-                            url, metas, "t".into(), plan, &df_runtime, exec, None, &opensearch_datafusion::datafusion_query_config::DatafusionQueryConfig::default(),
+                            url, metas, "t".into(), plan, &df_runtime, exec, None, &opensearch_datafusion::datafusion_query_config::DatafusionQueryConfig::test_default(),
                         ).await.unwrap();
                         // Consume and free the stream
                         let mut stream = unsafe {
@@ -149,7 +149,7 @@ fn bench_stream_next(c: &mut Criterion) {
                     &df_runtime,
                     exec,
                     None,
-                    &opensearch_datafusion::datafusion_query_config::DatafusionQueryConfig::default(
+                    &opensearch_datafusion::datafusion_query_config::DatafusionQueryConfig::test_default(
                     ),
                 )
                 .await
@@ -197,7 +197,7 @@ fn bench_aggregation(c: &mut Criterion) {
                     &df_runtime,
                     exec,
                     None,
-                    &opensearch_datafusion::datafusion_query_config::DatafusionQueryConfig::default(
+                    &opensearch_datafusion::datafusion_query_config::DatafusionQueryConfig::test_default(
                     ),
                 )
                 .await
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs
index 329bd810d3bea..5035fc39a3300 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs
@@ -134,6 +134,17 @@ pub struct DataFusionRuntime {
     pub(crate) dynamic_limit_handle: DynamicLimitHandle,
 }
 
+impl DataFusionRuntime {
+    pub fn new_for_bench(runtime_env: datafusion::execution::runtime_env::RuntimeEnv) -> Self {
+        let (_pool, handle) = DynamicLimitPool::new(0);
+        Self {
+            runtime_env,
+            custom_cache_manager: None,
+            dynamic_limit_handle: handle,
+        }
+    }
+}
+
 /// Opaque shard view handle returned to the caller.
 pub struct ShardView {
     pub table_path: ListingTableUrl,
@@ -313,14 +324,13 @@ pub async unsafe fn execute_query(
     // Register cancellation token.
     let token = query_tracker::get_cancellation_token(context_id);
 
-    let query_future = async {
+    let query_future = async move {
         if is_indexed {
             let qc = Arc::new(query_config);
             crate::indexed_executor::execute_indexed_query(
                 plan_bytes.to_vec(),
                 table_name.to_string(),
                 shard_view,
-                qc.target_partitions.max(1),
                 runtime,
                 cpu_executor,
                 query_memory_pool,
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs
index 3725761e3fa6f..bd1ef342d3d4b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/datafusion_query_config.rs
@@ -5,8 +5,8 @@
 //! setup time and copied into hot-path fields — never dereferenced on a
 //! per-batch or per-row hot path.
 
-use crate::indexed_table::stream::FilterStrategy;
 use crate::indexed_table::eval::single_collector::CollectorCallStrategy;
+use crate::indexed_table::stream::FilterStrategy;
 
 /// Query-scoped configuration. Owned by value after FFM decode.
 #[derive(Debug, Clone)]
@@ -48,27 +48,6 @@ pub struct DatafusionQueryConfig {
     pub tree_collector_strategy: CollectorCallStrategy,
 }
 
-impl Default for DatafusionQueryConfig {
-    fn default() -> Self {
-        Self {
-            batch_size: 8192,
-            // TODO: change this default value ?
-            target_partitions: 4,
-            parquet_pushdown_filters: false,
-            min_skip_run_default: 1024, // Todo: tune based on benchmarks
-            min_skip_run_selectivity_threshold: 0.03, // Todo : tune based on benchmarks
-            indexed_pushdown_filters: true,
-            force_strategy: None,
-            force_pushdown: None,
-            cost_predicate: 1,
-            cost_collector: 10, // TODO : should this be collector leaf specific
-            max_collector_parallelism: 1,
-            single_collector_strategy: CollectorCallStrategy::PageRangeSplit,
-            tree_collector_strategy: CollectorCallStrategy::TightenOuterBounds,
-        }
-    }
-}
-
 /// FFM wire format. Must stay in lockstep with the Java `MemoryLayout`.
 ///
 /// All fields have fixed sizes and natural alignment so Java and Rust
@@ -99,15 +78,53 @@ pub struct WireDatafusionQueryConfig {
 }
 
 impl DatafusionQueryConfig {
-    /// Decode from a raw FFM pointer. Null (`0`) returns defaults.
+    /// Fallback values used when Java passes a null config pointer (0).
+    /// Production code should always supply a real config via the wire
+    /// struct; this exists only for the transitional period while Java
+    /// wiring is incomplete.
+    fn fallback() -> Self {
+        Self {
+            batch_size: 8192,
+            target_partitions: 4,
+            parquet_pushdown_filters: false,
+            min_skip_run_default: 1024,
+            min_skip_run_selectivity_threshold: 0.03,
+            indexed_pushdown_filters: true,
+            force_strategy: None,
+            force_pushdown: None,
+            cost_predicate: 1,
+            cost_collector: 10,
+            max_collector_parallelism: 1,
+            single_collector_strategy: CollectorCallStrategy::PageRangeSplit,
+            tree_collector_strategy: CollectorCallStrategy::TightenOuterBounds,
+        }
+    }
+
+    /// Constructor with sensible defaults for tests and benchmarks.
+    /// Production code should use `from_ffm_ptr` with a real wire config.
+    pub fn test_default() -> Self {
+        Self::fallback()
+    }
+
+    /// Returns a builder seeded with fallback defaults for test usage.
+    #[cfg(test)]
+    pub fn builder() -> DatafusionQueryConfigBuilder {
+        DatafusionQueryConfigBuilder::new()
+    }
+
+    /// Decode from a raw FFM pointer.
     ///
     /// # Safety
-    /// `ptr` must be 0, or a valid pointer to a `WireDatafusionQueryConfig`
+    /// `ptr` must be a valid, non-zero pointer to a `WireDatafusionQueryConfig`
     /// whose memory is live for the duration of this call.
+    ///
+    /// # Panics
+    /// Panics if `ptr` is 0 (null). Java must always supply a valid config pointer.
     pub unsafe fn from_ffm_ptr(ptr: i64) -> Self {
-        if ptr == 0 {
-            return Self::default();
-        }
+        assert!(
+            ptr != 0,
+            "from_ffm_ptr: null query config pointer — Java must always provide a valid config"
+        );
         let wire = &*(ptr as *const WireDatafusionQueryConfig);
         Self::from_wire(wire)
     }
@@ -149,13 +166,78 @@ impl DatafusionQueryConfig {
     }
 }
 
+#[cfg(test)]
+pub struct DatafusionQueryConfigBuilder(DatafusionQueryConfig);
+
+#[cfg(test)]
+impl DatafusionQueryConfigBuilder {
+    fn new() -> Self {
+        Self(DatafusionQueryConfig::fallback())
+    }
+    pub fn batch_size(mut self, v: usize) -> Self {
+        self.0.batch_size = v;
+        self
+    }
+    pub fn target_partitions(mut self, v: usize) -> Self {
+        self.0.target_partitions = v;
+        self
+    }
+    pub fn parquet_pushdown_filters(mut self, v: bool) -> Self {
+        self.0.parquet_pushdown_filters = v;
+        self
+    }
+    pub fn min_skip_run_default(mut self, v: usize) -> Self {
+        self.0.min_skip_run_default = v;
+        self
+    }
+    pub fn min_skip_run_selectivity_threshold(mut self, v: f64) -> Self {
+        self.0.min_skip_run_selectivity_threshold = v;
+        self
+    }
+    pub fn indexed_pushdown_filters(mut self, v: bool) -> Self {
+        self.0.indexed_pushdown_filters = v;
+        self
+    }
+    pub fn force_strategy(mut self, v: Option) -> Self {
+        self.0.force_strategy = v;
+        self
+    }
+    pub fn force_pushdown(mut self, v: Option) -> Self {
+        self.0.force_pushdown = v;
+        self
+    }
+    pub fn cost_predicate(mut self, v: u32) -> Self {
+        self.0.cost_predicate = v;
+        self
+    }
+    pub fn cost_collector(mut self, v: u32) -> Self {
+        self.0.cost_collector = v;
+        self
+    }
+    pub fn max_collector_parallelism(mut self, v: usize) -> Self {
+        self.0.max_collector_parallelism = v;
+        self
+    }
+    pub fn single_collector_strategy(mut self, v: CollectorCallStrategy) -> Self {
+        self.0.single_collector_strategy = v;
+        self
+    }
+    pub fn tree_collector_strategy(mut self, v: CollectorCallStrategy) -> Self {
+        self.0.tree_collector_strategy = v;
+        self
+    }
+    pub fn build(self) -> DatafusionQueryConfig {
+        self.0
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
-    fn defaults_match_legacy_constants() {
-        let c = DatafusionQueryConfig::default();
+    fn test_default_matches_legacy_constants() {
+        let c = DatafusionQueryConfig::test_default();
         assert_eq!(c.batch_size, 8192);
         assert_eq!(c.target_partitions, 4);
         assert!(!c.parquet_pushdown_filters);
@@ -169,12 +251,9 @@ mod tests {
     }
 
     #[test]
-    fn wire_decode_null_pointer_gives_defaults() {
-        let c = unsafe { DatafusionQueryConfig::from_ffm_ptr(0) };
-        let d = DatafusionQueryConfig::default();
-        assert_eq!(c.batch_size, d.batch_size);
-        assert_eq!(c.min_skip_run_default, d.min_skip_run_default);
-        assert_eq!(c.cost_collector, d.cost_collector);
+    #[should_panic(expected = "null query config pointer")]
+    fn wire_decode_null_pointer_panics() {
+        unsafe { DatafusionQueryConfig::from_ffm_ptr(0) };
     }
 
     #[test]
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
index 47df9bbe8951a..8536764287ba1 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
@@ -155,7 +155,7 @@ pub unsafe extern "C" fn df_execute_query(
     plan_len: i64,
     runtime_ptr: i64,
     context_id: i64,
-    // Pointer to a `WireDatafusionQueryConfig`. `0` = use defaults.
+    // Pointer to a `WireDatafusionQueryConfig`
     query_config_ptr: i64,
 ) -> i64 {
     let mgr = get_rt_manager()?;
@@ -397,7 +397,12 @@ pub unsafe extern "C" fn df_create_cache(
     let policy_type = match eviction_type.to_uppercase().as_str() {
         "LRU" => PolicyType::Lru,
         "LFU" => PolicyType::Lfu,
-        _ => return Err(format!("df_create_cache: unsupported eviction type: {}", eviction_type)),
+        _ => {
+            return Err(format!(
+                "df_create_cache: unsupported eviction type: {}",
+                eviction_type
+            ))
+        }
     };
 
     // Safety: cache_manager_ptr must be a valid pointer from df_create_custom_cache_manager
@@ -418,7 +423,10 @@ pub unsafe extern "C" fn df_create_cache(
             manager.set_statistics_cache(stats_cache);
         }
         _ => {
-            return Err(format!("df_create_cache: invalid cache type: {}", cache_type));
+            return Err(format!(
+                "df_create_cache: invalid cache type: {}",
+                cache_type
+            ));
         }
     }
     Ok(0)
@@ -437,18 +445,24 @@ pub unsafe extern "C" fn df_cache_manager_add_files(
     }
     // Safety: runtime_ptr must be a valid pointer from df_create_global_runtime
     let runtime = &*(runtime_ptr as *const DataFusionRuntime);
-    let manager = runtime.custom_cache_manager.as_ref()
+    let manager = runtime
+        .custom_cache_manager
+        .as_ref()
         .ok_or_else(|| "df_cache_manager_add_files: no cache manager configured".to_string())?;
 
     let mut file_paths = Vec::with_capacity(files_count as usize);
     for i in 0..files_count as usize {
         let ptr = *files_ptr.add(i);
         let len = *files_len_ptr.add(i);
-        file_paths.push(str_from_raw(ptr, len)
-            .map_err(|e| format!("df_cache_manager_add_files: {}", e))?.to_string());
+        file_paths.push(
+            str_from_raw(ptr, len)
+                .map_err(|e| format!("df_cache_manager_add_files: {}", e))?
+                .to_string(),
+        );
     }
 
-    manager.add_files(&file_paths)
+    manager
+        .add_files(&file_paths)
         .map_err(|e| format!("df_cache_manager_add_files: {}", e))?;
     Ok(0)
 }
@@ -465,13 +479,20 @@ pub unsafe extern "C" fn df_create_session_context(
     table_name_ptr: *const u8,
     table_name_len: i64,
     context_id: i64,
+    query_config_ptr: i64,
 ) -> i64 {
     let table_name = str_from_raw(table_name_ptr, table_name_len)
         .map_err(|e| format!("df_create_session_context: {}", e))?;
+    let query_config =
+        crate::datafusion_query_config::DatafusionQueryConfig::from_ffm_ptr(query_config_ptr);
     let mgr = get_rt_manager()?;
     mgr.io_runtime
         .block_on(crate::session_context::create_session_context(
-            runtime_ptr, shard_view_ptr, table_name, context_id,
+            runtime_ptr,
+            shard_view_ptr,
+            table_name,
+            context_id,
+            query_config,
         ))
         .map_err(|e| e.to_string())
 }
@@ -486,13 +507,16 @@ pub unsafe extern "C" fn df_create_session_context_indexed(
     context_id: i64,
     tree_shape: i32,
     delegated_predicate_count: i32,
+    query_config_ptr: i64,
 ) -> i64 {
     let table_name = str_from_raw(table_name_ptr, table_name_len)
         .map_err(|e| format!("df_create_session_context_indexed: {}", e))?;
+    let query_config =
+        crate::datafusion_query_config::DatafusionQueryConfig::from_ffm_ptr(query_config_ptr);
     let mgr = get_rt_manager()?;
     mgr.io_runtime
         .block_on(crate::session_context::create_session_context_indexed(
-            runtime_ptr, shard_view_ptr, table_name, context_id, tree_shape, delegated_predicate_count,
+            runtime_ptr, shard_view_ptr, table_name, context_id, tree_shape, delegated_predicate_count, query_config,
         ))
         .map_err(|e| e.to_string())
 }
@@ -509,18 +533,24 @@ pub unsafe extern "C" fn df_cache_manager_remove_files(
         return Err("df_cache_manager_remove_files: null runtime pointer".to_string());
     }
     let runtime = &*(runtime_ptr as *const DataFusionRuntime);
-    let manager = runtime.custom_cache_manager.as_ref()
+    let manager = runtime
+        .custom_cache_manager
+        .as_ref()
         .ok_or_else(|| "df_cache_manager_remove_files: no cache manager configured".to_string())?;
 
     let mut file_paths = Vec::with_capacity(files_count as usize);
     for i in 0..files_count as usize {
         let ptr = *files_ptr.add(i);
         let len = *files_len_ptr.add(i);
-        file_paths.push(str_from_raw(ptr, len)
-            .map_err(|e| format!("df_cache_manager_remove_files: {}", e))?.to_string());
+        file_paths.push(
+            str_from_raw(ptr, len)
+                .map_err(|e| format!("df_cache_manager_remove_files: {}", e))?
+                .to_string(),
+        );
     }
 
-    manager.remove_files(&file_paths)
+    manager
+        .remove_files(&file_paths)
         .map_err(|e| format!("df_cache_manager_remove_files: {}", e))?;
     Ok(0)
 }
@@ -532,7 +562,9 @@ pub unsafe extern "C" fn df_cache_manager_clear(runtime_ptr: i64) -> i64 {
         return Err("df_cache_manager_clear: null runtime pointer".to_string());
     }
     let runtime = &*(runtime_ptr as *const DataFusionRuntime);
-    let manager = runtime.custom_cache_manager.as_ref()
+    let manager = runtime
+        .custom_cache_manager
+        .as_ref()
         .ok_or_else(|| "df_cache_manager_clear: no cache manager configured".to_string())?;
     manager.clear_all();
     Ok(0)
@@ -551,9 +583,12 @@ pub unsafe extern "C" fn df_cache_manager_clear_by_type(
     let cache_type = str_from_raw(cache_type_ptr, cache_type_len)
         .map_err(|e| format!("df_cache_manager_clear_by_type: {}", e))?;
     let runtime = &*(runtime_ptr as *const DataFusionRuntime);
-    let manager = runtime.custom_cache_manager.as_ref()
+    let manager = runtime
+        .custom_cache_manager
+        .as_ref()
         .ok_or_else(|| "df_cache_manager_clear_by_type: no cache manager configured".to_string())?;
-    manager.clear_cache_type(cache_type)
+    manager
+        .clear_cache_type(cache_type)
         .map_err(|e| format!("df_cache_manager_clear_by_type: {}", e))?;
     Ok(0)
 }
@@ -571,9 +606,11 @@ pub unsafe extern "C" fn df_cache_manager_get_memory_by_type(
     let cache_type = str_from_raw(cache_type_ptr, cache_type_len)
         .map_err(|e| format!("df_cache_manager_get_memory_by_type: {}", e))?;
     let runtime = &*(runtime_ptr as *const DataFusionRuntime);
-    let manager = runtime.custom_cache_manager.as_ref()
-        .ok_or_else(|| "df_cache_manager_get_memory_by_type: no cache manager configured".to_string())?;
-    let size = manager.get_memory_consumed_by_type(cache_type)
+    let manager = runtime.custom_cache_manager.as_ref().ok_or_else(|| {
+        "df_cache_manager_get_memory_by_type: no cache manager configured".to_string()
+    })?;
+    let size = manager
+        .get_memory_consumed_by_type(cache_type)
         .map_err(|e| format!("df_cache_manager_get_memory_by_type: {}", e))?;
     Ok(size as i64)
 }
@@ -585,8 +622,9 @@ pub unsafe extern "C" fn df_cache_manager_get_total_memory(runtime_ptr: i64) ->
         return Err("df_cache_manager_get_total_memory: null runtime pointer".to_string());
     }
     let runtime = &*(runtime_ptr as *const DataFusionRuntime);
-    let manager = runtime.custom_cache_manager.as_ref()
-        .ok_or_else(|| "df_cache_manager_get_total_memory: no cache manager configured".to_string())?;
+    let manager = runtime.custom_cache_manager.as_ref().ok_or_else(|| {
+        "df_cache_manager_get_total_memory: no cache manager configured".to_string()
+    })?;
     Ok(manager.get_total_memory_consumed() as i64)
 }
 
@@ -607,9 +645,14 @@ pub unsafe extern "C" fn df_cache_manager_contains_by_type(
     let file_path = str_from_raw(file_path_ptr, file_path_len)
         .map_err(|e| format!("df_cache_manager_contains_by_type: file_path: {}", e))?;
     let runtime = &*(runtime_ptr as *const DataFusionRuntime);
-    let manager = runtime.custom_cache_manager.as_ref()
-        .ok_or_else(|| "df_cache_manager_contains_by_type: no cache manager configured".to_string())?;
-    Ok(if manager.contains_file_by_type(file_path, cache_type) { 1 } else { 0 })
+    let manager = runtime.custom_cache_manager.as_ref().ok_or_else(|| {
+        "df_cache_manager_contains_by_type: no cache manager configured".to_string()
+    })?;
+    Ok(if manager.contains_file_by_type(file_path, cache_type) {
+        1
+    } else {
+        0
+    })
 }
 
 #[no_mangle]
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
index 6ab951ed67d6d..a6ef5ae241eeb 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
@@ -78,9 +78,9 @@ use crate::indexed_table::page_pruner::{build_pruning_predicate, PagePruneMetric
 /// Execute an indexed query.
 ///
 /// `shard_view` carries the segment's parquet paths (populated when the reader
-/// was built from a catalog snapshot). `num_partitions` comes from the caller's
-/// session config. `query_memory_pool` is the per-query tracker (same as
-/// vanilla path) — `None` disables tracking and uses the global pool.
+/// was built from a catalog snapshot). `query_memory_pool` is the per-query
+/// tracker (same as vanilla path) — `None` disables tracking and uses the
+/// global pool.
 // TODO: remove this function once all callers migrate to the instruction-based path
 // TODO: remove once api.rs migrates to instruction-based path directly.
 // Kept as thin wrapper to make existing tests exercise execute_indexed_with_context
@@ -89,12 +89,12 @@ pub async fn execute_indexed_query(
     substrait_bytes: Vec,
     table_name: String,
     shard_view: &ShardView,
-    num_partitions: usize,
     runtime: &DataFusionRuntime,
     cpu_executor: DedicatedExecutor,
     query_memory_pool: Option>,
     query_config: Arc,
 ) -> Result {
+    let num_partitions = query_config.target_partitions.max(1);
     // Share caches with the global runtime (same as vanilla path): list-files
     // pre-populated with the reader's object_metas, file-metadata and
     // file-statistics inherited from the global runtime for cross-query reuse.
@@ -161,6 +161,7 @@ pub async fn execute_indexed_query(
         query_context: crate::query_tracker::QueryTrackingContext::new(0, runtime.runtime_env.memory_pool.clone()),
         table_name: table_name.clone(),
         indexed_config: None, // derive classification from tree
+        query_config: Arc::unwrap_or_clone(query_config),
     };
     let ptr = Box::into_raw(Box::new(handle)) as i64;
     unsafe { execute_indexed_with_context(ptr, substrait_bytes, cpu_executor).await }
@@ -410,7 +411,7 @@ pub async unsafe fn execute_indexed_with_context(
         }
     });
 
-    let query_config = Arc::new(crate::datafusion_query_config::DatafusionQueryConfig::default());
+    let query_config = Arc::new(handle.query_config);
     let num_partitions = query_config.target_partitions.max(1);
     let ctx = handle.ctx;
     let table_name = handle.table_name;
@@ -674,9 +675,6 @@ pub async unsafe fn execute_indexed_with_context(
         store: Arc::clone(&store),
         store_url,
         evaluator_factory: factory,
-        target_partitions: num_partitions.max(1),
-        force_strategy: query_config.force_strategy,
-        force_pushdown: query_config.force_pushdown,
         pushdown_predicate,
         query_config: Arc::clone(&query_config),
         predicate_columns,
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs
index a924334256605..cff5632eeb4f3 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs
@@ -266,8 +266,6 @@ pub struct IndexedExec {
     pub(crate) doc_range: Option<(i32, i32)>,
     pub(crate) metrics: ExecutionPlanMetricsSet,
     pub(crate) stream_metrics: StreamMetrics,
-    pub(crate) force_pushdown: Option,
-    pub(crate) force_strategy: Option,
     /// Query-scoped tunables. Shared by Arc across IndexedExec instances
     /// from the same query; read once per RG into local fields inside
     /// `IndexedStream` so the hot path never touches the Arc.
@@ -358,8 +356,8 @@ impl ExecutionPlan for IndexedExec {
             Arc::clone(&self.metadata),
             self.predicate.clone(),
             self.stream_metrics.clone(),
-            self.force_pushdown,
-            self.force_strategy,
+            self.query_config.force_pushdown,
+            self.query_config.force_strategy,
             self.query_config.min_skip_run_default,
             self.query_config.min_skip_run_selectivity_threshold,
             self.query_config.indexed_pushdown_filters,
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/table_provider.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/table_provider.rs
index c279d4769cb81..b0ec951a3dff0 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/table_provider.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/table_provider.rs
@@ -102,11 +102,6 @@ pub struct IndexedTableConfig {
     /// URL of the store for DataFusion's `FileScanConfig`.
     pub store_url: datafusion::execution::object_store::ObjectStoreUrl,
     pub evaluator_factory: EvaluatorFactory,
-    pub target_partitions: usize,
-    /// If `Some`, override the per-RG strategy choice. Mainly for tests.
-    pub force_strategy: Option,
-    /// If `Some`, force `with_pushdown_filters` on/off. Mainly for tests.
-    pub force_pushdown: Option,
     /// Parquet-native residual predicate to push into decode time via
     /// `ParquetSource::with_predicate`. Derived from the BoolNode tree
     /// by `execute_indexed_query`:
@@ -120,7 +115,7 @@ pub struct IndexedTableConfig {
     /// receives from DataFusion, because DataFusion's filters include
     /// the `index_filter(...)` UDF marker whose body panics.
     pub pushdown_predicate: Option>,
-    /// Query-scoped tunables (batch_size, min_skip_run_default, costs, …).
+    /// Query-scoped tunables (batch_size, target_partitions, costs, …).
     /// Shared by reference across fanned-out `QueryShardExec` instances.
     pub query_config: Arc,
     /// Full-schema column indices referenced by BoolNode Predicate leaves.
@@ -136,7 +131,7 @@ impl fmt::Debug for IndexedTableProvider {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("IndexedTableProvider")
             .field("segments", &self.config.segments.len())
-            .field("partitions", &self.config.target_partitions)
+            .field("partitions", &self.config.query_config.target_partitions)
             .finish()
     }
 }
@@ -224,7 +219,8 @@ impl TableProvider for IndexedTableProvider {
                 row_groups: seg.row_groups.clone(),
             })
             .collect();
-        let assignments = compute_assignments(&layouts, self.config.target_partitions);
+        let assignments =
+            compute_assignments(&layouts, self.config.query_config.target_partitions.max(1));
 
         let properties = PlanProperties::new(
             EquivalenceProperties::new(projected_schema.clone()),
@@ -388,8 +384,6 @@ impl ExecutionPlan for QueryShardExec {
                 doc_range: Some((chunk.doc_min, chunk.doc_max)),
                 metrics: ExecutionPlanMetricsSet::new(),
                 stream_metrics: stream_metrics.clone(),
-                force_pushdown: self.config.force_pushdown,
-                force_strategy: self.config.force_strategy,
                 query_config: Arc::clone(&self.config.query_config),
             };
             execs.push(Arc::new(exec));
@@ -447,12 +441,9 @@ mod tests {
             store_url: datafusion::execution::object_store::ObjectStoreUrl::local_filesystem(),
             // Evaluator factory would never be invoked for this test (no segments).
             evaluator_factory: Arc::new(|_, _, _| unreachable!()),
-            target_partitions: 1,
-            force_strategy: None,
-            force_pushdown: None,
             pushdown_predicate: None,
             query_config: std::sync::Arc::new(
-                crate::datafusion_query_config::DatafusionQueryConfig::default(),
+                crate::datafusion_query_config::DatafusionQueryConfig::test_default(),
             ),
             predicate_columns: vec![],
         }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs
index d7411d90ead83..ace5200830dff 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/fuzz/harness.rs
@@ -244,14 +244,13 @@ pub(in crate::indexed_table::tests_e2e) async fn execute_tree_with_plan_pushdown
                 page_pruner: pruner,
                 cost_predicate: 1,
                 cost_collector: 10,
-                max_collector_parallelism: cfg_max_parallelism
-                    .unwrap_or(if num_tags > 1 {
-                        // Multi-collector tree: randomly pick 1 (sequential) or
-                        // up to 4 (parallel) to exercise PrecomputedLeafCache.
-                        [1, 1, 2, 4][seed as usize % 4]
-                    } else {
-                        1
-                    }),
+                max_collector_parallelism: cfg_max_parallelism.unwrap_or(if num_tags > 1 {
+                    // Multi-collector tree: randomly pick 1 (sequential) or
+                    // up to 4 (parallel) to exercise PrecomputedLeafCache.
+                    [1, 1, 2, 4][seed as usize % 4]
+                } else {
+                    1
+                }),
                 pruning_predicates: Arc::clone(&pruning_predicates),
                 page_prune_metrics: Some(
                     crate::indexed_table::page_pruner::PagePruneMetrics::from_stream_metrics(
@@ -271,23 +270,20 @@ pub(in crate::indexed_table::tests_e2e) async fn execute_tree_with_plan_pushdown
     let store: Arc =
         Arc::new(object_store::local::LocalFileSystem::new());
     let store_url = datafusion::execution::object_store::ObjectStoreUrl::local_filesystem();
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(cfg_target_partitions.max(1))
+        .force_strategy(force_strategy)
+        .force_pushdown(force_pushdown)
+        .batch_size(cfg_batch_size.unwrap_or([128, 1024, 8192][seed as usize % 3]))
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: loaded.schema.clone(),
         segments: loaded.segments.clone(),
         store,
         store_url,
         evaluator_factory: factory,
-        target_partitions: cfg_target_partitions.max(1),
-        force_strategy,
-        force_pushdown,
         pushdown_predicate: None,
-        query_config: Arc::new({
-            let mut qc = crate::datafusion_query_config::DatafusionQueryConfig::default();
-            // Vary batch_size to exercise the coalescer at different boundaries.
-            qc.batch_size = cfg_batch_size
-                .unwrap_or([128, 1024, 8192][seed as usize % 3]);
-            qc
-        }),
+        query_config: Arc::new(qc),
         predicate_columns: collect_predicate_column_indices(&bool_tree),
     }));
 
@@ -446,21 +442,20 @@ async fn run_single_collector_query(
     let store: Arc =
         Arc::new(object_store::local::LocalFileSystem::new());
     let store_url = datafusion::execution::object_store::ObjectStoreUrl::local_filesystem();
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(1)
+        .force_strategy(force_strategy)
+        .force_pushdown(Some(true))
+        .batch_size([128, 1024, 8192][loaded.segments.len() % 3])
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: loaded.schema.clone(),
         segments: loaded.segments.clone(),
         store,
         store_url,
         evaluator_factory: factory,
-        target_partitions: 1,
-        force_strategy,
-        force_pushdown: Some(true), // SingleCollector relies on decode-time pushdown
         pushdown_predicate,
-        query_config: Arc::new({
-            let mut qc = crate::datafusion_query_config::DatafusionQueryConfig::default();
-            qc.batch_size = [128, 1024, 8192][loaded.segments.len() % 3];
-            qc
-        }),
+        query_config: Arc::new(qc),
         predicate_columns: pred_cols,
     }));
     let ctx = SessionContext::new();
@@ -654,21 +649,20 @@ async fn run_with_factory_plan(
     let store: Arc =
         Arc::new(object_store::local::LocalFileSystem::new());
     let store_url = datafusion::execution::object_store::ObjectStoreUrl::local_filesystem();
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(1)
+        .force_strategy(force_strategy)
+        .force_pushdown(force_pushdown)
+        .batch_size([256, 1024, 8192][loaded.segments.len() % 3])
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: loaded.schema.clone(),
         segments: loaded.segments.clone(),
         store,
         store_url,
         evaluator_factory: factory,
-        target_partitions: 1,
-        force_strategy,
-        force_pushdown,
         pushdown_predicate,
-        query_config: Arc::new({
-            let mut qc = crate::datafusion_query_config::DatafusionQueryConfig::default();
-            qc.batch_size = [256, 1024, 8192][loaded.segments.len() % 3];
-            qc
-        }),
+        query_config: Arc::new(qc),
         predicate_columns: vec![], // run_with_factory_plan is low-level; caller controls projection
     }));
     let ctx = SessionContext::new();
@@ -836,8 +830,14 @@ async fn run_iteration_impl(
     // doesn't implement, so we don't assert on those.
     let classification = classify_filter(&tree.tree);
     if classification == FilterClass::Tree {
-        let sc_result =
-            execute_tree_single_collector(corpus, loaded, tree, None, CollectorCallStrategy::FullRange).await;
+        let sc_result = execute_tree_single_collector(
+            corpus,
+            loaded,
+            tree,
+            None,
+            CollectorCallStrategy::FullRange,
+        )
+        .await;
         if sc_result.is_some() {
             return Err(format!(
                 "classify_filter returned Tree but execute_tree_single_collector \
@@ -1059,10 +1059,15 @@ mod tests {
 
         let expected = oracle_evaluate(>, &corpus);
         // Force BooleanMask strategy through the SingleCollector path.
-        let actual =
-            execute_tree_single_collector(&corpus, &loaded, >, Some(FilterStrategy::BooleanMask), CollectorCallStrategy::PageRangeSplit)
-                .await
-                .expect("tree classifies as SingleCollector");
+        let actual = execute_tree_single_collector(
+            &corpus,
+            &loaded,
+            >,
+            Some(FilterStrategy::BooleanMask),
+            CollectorCallStrategy::PageRangeSplit,
+        )
+        .await
+        .expect("tree classifies as SingleCollector");
         assert_eq!(
             expected,
             actual,
@@ -1216,9 +1221,15 @@ mod tests {
             Some(FilterStrategy::RowSelection),
             Some(FilterStrategy::BooleanMask),
         ] {
-            let actual = execute_tree_single_collector(&corpus, &loaded, >, strategy, CollectorCallStrategy::PageRangeSplit)
-                .await
-                .expect("tree classifies as SingleCollector");
+            let actual = execute_tree_single_collector(
+                &corpus,
+                &loaded,
+                >,
+                strategy,
+                CollectorCallStrategy::PageRangeSplit,
+            )
+            .await
+            .expect("tree classifies as SingleCollector");
             assert_eq!(
                 expected,
                 actual,
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/mod.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/mod.rs
index 14b0967da7b76..b24771e38e09b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/mod.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/mod.rs
@@ -269,23 +269,23 @@ async fn run_tree_and_plan(
     let store: Arc =
         Arc::new(object_store::local::LocalFileSystem::new());
     let store_url = datafusion::execution::object_store::ObjectStoreUrl::local_filesystem();
+    // Force BooleanMask so batches contain the entire RG and batch_offset
+    // equals the row-index-within-RG. Phase 2 bitmap_to_batch_mask
+    // relies on this alignment. RowSelection would still work for Path B
+    // (no Phase-2 mask), but Path C tree eval requires BooleanMask today.
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(1)
+        .force_strategy(Some(FilterStrategy::BooleanMask))
+        .force_pushdown(Some(false))
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: schema.clone(),
         segments: vec![segment],
         store,
         store_url,
         evaluator_factory: factory,
-        target_partitions: 1,
-        // Force BooleanMask so batches contain the entire RG and batch_offset
-        // equals the row-index-within-RG. Phase 2 bitmap_to_batch_mask
-        // relies on this alignment. RowSelection would still work for Path B
-        // (no Phase-2 mask), but Path C tree eval requires BooleanMask today.
-        force_strategy: Some(FilterStrategy::BooleanMask),
-        force_pushdown: Some(false),
         pushdown_predicate: None,
-        query_config: std::sync::Arc::new(
-            crate::datafusion_query_config::DatafusionQueryConfig::default(),
-        ),
+        query_config: std::sync::Arc::new(qc),
         predicate_columns: vec![],
     }));
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs
index 57352e63fe947..b4952ed729114 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/multi_segment.rs
@@ -139,43 +139,44 @@ async fn run_two_segment_query(
     // Factory produces a single-collector evaluator per chunk. The collector's
     // matching set is pulled from `per_segment_matches[segment.segment_ord]`,
     // so wrong segment_ord propagation would immediately produce wrong rows.
-    let factory: super::super::table_provider::EvaluatorFactory = {
-        let per_segment_matches = Arc::clone(&per_segment_matches);
-        let schema = schema.clone();
-        Arc::new(move |segment, _chunk, _stream_metrics| {
-            let matching = per_segment_matches
-                .get(segment.segment_ord as usize)
-                .cloned()
-                .unwrap_or_default();
-            let collector: Arc =
-                Arc::new(PerSegmentCollector { matching });
-            let pruner = Arc::new(PagePruner::new(&schema, Arc::clone(&segment.metadata)));
-            let eval: Arc = Arc::new(
+    let factory: super::super::table_provider::EvaluatorFactory =
+        {
+            let per_segment_matches = Arc::clone(&per_segment_matches);
+            let schema = schema.clone();
+            Arc::new(move |segment, _chunk, _stream_metrics| {
+                let matching = per_segment_matches
+                    .get(segment.segment_ord as usize)
+                    .cloned()
+                    .unwrap_or_default();
+                let collector: Arc =
+                    Arc::new(PerSegmentCollector { matching });
+                let pruner = Arc::new(PagePruner::new(&schema, Arc::clone(&segment.metadata)));
+                let eval: Arc = Arc::new(
                 crate::indexed_table::eval::single_collector::SingleCollectorEvaluator::new(
                     collector, pruner, None, None, None, None,
                     crate::indexed_table::eval::single_collector::CollectorCallStrategy::FullRange,
                 ),
             );
-            Ok(eval)
-        })
-    };
+                Ok(eval)
+            })
+        };
 
     let store: Arc =
         Arc::new(object_store::local::LocalFileSystem::new());
     let store_url = datafusion::execution::object_store::ObjectStoreUrl::local_filesystem();
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(num_partitions)
+        .force_strategy(Some(FilterStrategy::BooleanMask))
+        .force_pushdown(Some(false))
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: schema.clone(),
         segments,
         store,
         store_url,
         evaluator_factory: factory,
-        target_partitions: num_partitions,
-        force_strategy: Some(FilterStrategy::BooleanMask),
-        force_pushdown: Some(false),
         pushdown_predicate: None,
-        query_config: std::sync::Arc::new(
-            crate::datafusion_query_config::DatafusionQueryConfig::default(),
-        ),
+        query_config: std::sync::Arc::new(qc),
         predicate_columns: vec![],
     }));
 
@@ -336,43 +337,44 @@ async fn run_segments(specs: Vec, num_partitions: usize) -> Vec<(i32, S
 
     let schema = schema_opt.unwrap();
     let per_segment_matches = Arc::new(per_segment_matches);
-    let factory: super::super::table_provider::EvaluatorFactory = {
-        let per_segment_matches = Arc::clone(&per_segment_matches);
-        let schema = schema.clone();
-        Arc::new(move |segment, _chunk, _stream_metrics| {
-            let matching = per_segment_matches
-                .get(segment.segment_ord as usize)
-                .cloned()
-                .unwrap_or_default();
-            let collector: Arc =
-                Arc::new(PerSegmentCollector { matching });
-            let pruner = Arc::new(PagePruner::new(&schema, Arc::clone(&segment.metadata)));
-            let eval: Arc = Arc::new(
+    let factory: super::super::table_provider::EvaluatorFactory =
+        {
+            let per_segment_matches = Arc::clone(&per_segment_matches);
+            let schema = schema.clone();
+            Arc::new(move |segment, _chunk, _stream_metrics| {
+                let matching = per_segment_matches
+                    .get(segment.segment_ord as usize)
+                    .cloned()
+                    .unwrap_or_default();
+                let collector: Arc =
+                    Arc::new(PerSegmentCollector { matching });
+                let pruner = Arc::new(PagePruner::new(&schema, Arc::clone(&segment.metadata)));
+                let eval: Arc = Arc::new(
                 crate::indexed_table::eval::single_collector::SingleCollectorEvaluator::new(
                     collector, pruner, None, None, None, None,
                     crate::indexed_table::eval::single_collector::CollectorCallStrategy::FullRange,
                 ),
             );
-            Ok(eval)
-        })
-    };
+                Ok(eval)
+            })
+        };
 
     let store: Arc =
         Arc::new(object_store::local::LocalFileSystem::new());
     let store_url = datafusion::execution::object_store::ObjectStoreUrl::local_filesystem();
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(num_partitions)
+        .force_strategy(Some(FilterStrategy::BooleanMask))
+        .force_pushdown(Some(false))
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: schema.clone(),
         segments,
         store,
         store_url,
         evaluator_factory: factory,
-        target_partitions: num_partitions,
-        force_strategy: Some(FilterStrategy::BooleanMask),
-        force_pushdown: Some(false),
         pushdown_predicate: None,
-        query_config: std::sync::Arc::new(
-            crate::datafusion_query_config::DatafusionQueryConfig::default(),
-        ),
+        query_config: std::sync::Arc::new(qc),
         predicate_columns: vec![],
     }));
 
@@ -861,19 +863,19 @@ async fn run_wide_segments(
     let store: Arc =
         Arc::new(object_store::local::LocalFileSystem::new());
     let store_url = datafusion::execution::object_store::ObjectStoreUrl::local_filesystem();
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(num_partitions)
+        .force_strategy(Some(FilterStrategy::BooleanMask))
+        .force_pushdown(Some(false))
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: schema.clone(),
         segments,
         store,
         store_url,
         evaluator_factory: factory,
-        target_partitions: num_partitions,
-        force_strategy: Some(FilterStrategy::BooleanMask),
-        force_pushdown: Some(false),
         pushdown_predicate: None,
-        query_config: std::sync::Arc::new(
-            crate::datafusion_query_config::DatafusionQueryConfig::default(),
-        ),
+        query_config: std::sync::Arc::new(qc),
         predicate_columns: vec![],
     }));
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/null_columns.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/null_columns.rs
index d54221e774e0b..10bf6396b6041 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/null_columns.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/null_columns.rs
@@ -357,12 +357,18 @@ async fn assert_engine_matches_reference_null(name: &str, tree: NT) {
                 max_collector_parallelism: 1,
                 pruning_predicates: std::sync::Arc::new(std::collections::HashMap::new()),
                 page_prune_metrics: None,
-                    collector_strategy: crate::indexed_table::eval::CollectorCallStrategy::TightenOuterBounds,
+                collector_strategy:
+                    crate::indexed_table::eval::CollectorCallStrategy::TightenOuterBounds,
             });
             Ok(eval)
         })
     };
 
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(1)
+        .force_strategy(Some(FilterStrategy::BooleanMask))
+        .force_pushdown(Some(false))
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: schema.clone(),
         segments: vec![segment],
@@ -370,13 +376,8 @@ async fn assert_engine_matches_reference_null(name: &str, tree: NT) {
             as Arc,
         store_url: datafusion::execution::object_store::ObjectStoreUrl::local_filesystem(),
         evaluator_factory: factory,
-        target_partitions: 1,
-        force_strategy: Some(FilterStrategy::BooleanMask),
-        force_pushdown: Some(false),
         pushdown_predicate: None,
-        query_config: std::sync::Arc::new(
-            crate::datafusion_query_config::DatafusionQueryConfig::default(),
-        ),
+        query_config: std::sync::Arc::new(qc),
         predicate_columns: vec![],
     }));
     let ctx = SessionContext::new();
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs
index 2466849df08b8..cce78dd0f5d64 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/page_pruning.rs
@@ -57,7 +57,7 @@ use tempfile::NamedTempFile;
 
 use crate::indexed_table::bool_tree::BoolNode;
 use crate::indexed_table::eval::bitmap_tree::{
-    BitmapTreeEvaluator, CollectorLeafBitmaps, COST_SCALE, subtree_cost,
+    subtree_cost, BitmapTreeEvaluator, CollectorLeafBitmaps,
 };
 use crate::indexed_table::eval::single_collector::SingleCollectorEvaluator;
 use crate::indexed_table::eval::{
@@ -179,10 +179,10 @@ fn collector_for_tag(tag: u8) -> Arc {
                 vec![base, base + 1]
             })
             .collect(),
-        3 => (0..2048).collect(),                          // pages 0+1
-        4 => (2048..NUM_ROWS as i32).collect(),            // pages 2+3
-        5 => (1..NUM_ROWS as i32).step_by(2).collect(),   // odd docs
-        6 => (3072..NUM_ROWS as i32).collect(),            // page 3 only
+        3 => (0..2048).collect(),                       // pages 0+1
+        4 => (2048..NUM_ROWS as i32).collect(),         // pages 2+3
+        5 => (1..NUM_ROWS as i32).step_by(2).collect(), // odd docs
+        6 => (3072..NUM_ROWS as i32).collect(),         // page 3 only
         _ => vec![],
     };
     Arc::new(MockCollector { docs })
@@ -202,7 +202,11 @@ fn load_segment(tmp: &NamedTempFile) -> (SegmentFileInfo, SchemaRef) {
     let mut offset = 0i64;
     for i in 0..parquet_meta.num_row_groups() {
         let n = parquet_meta.row_group(i).num_rows();
-        rgs.push(RowGroupInfo { index: i, first_row: offset, num_rows: n });
+        rgs.push(RowGroupInfo {
+            index: i,
+            first_row: offset,
+            num_rows: n,
+        });
         offset += n;
     }
     let seg = SegmentFileInfo {
@@ -281,9 +285,7 @@ fn wire_collectors_dfs(node: &BoolNode, out: &mut Vec (Vec, Arc) {
+async fn run_bitmap_tree(tree: BoolNode) -> (Vec, Arc) {
     let tmp = write_fixture();
     let (seg, schema) = load_segment(&tmp);
     let tree = tree.push_not_down();
@@ -319,7 +321,8 @@ async fn run_bitmap_tree(
                 page_prune_metrics: Some(
                     crate::indexed_table::page_pruner::PagePruneMetrics::from_stream_metrics(sm),
                 ),
-                collector_strategy: crate::indexed_table::eval::CollectorCallStrategy::TightenOuterBounds,
+                collector_strategy:
+                    crate::indexed_table::eval::CollectorCallStrategy::TightenOuterBounds,
             });
             Ok(eval)
         })
@@ -349,9 +352,7 @@ async fn run_single_collector(
                 pruner,
                 residual_pp.clone(),
                 Some(Arc::clone(&residual_expr)),
-                Some(
-                    crate::indexed_table::page_pruner::PagePruneMetrics::from_stream_metrics(sm),
-                ),
+                Some(crate::indexed_table::page_pruner::PagePruneMetrics::from_stream_metrics(sm)),
                 sm.ffm_collector_calls.clone(),
                 strategy,
             ));
@@ -370,19 +371,19 @@ async fn execute_and_collect(
     let store: Arc =
         Arc::new(object_store::local::LocalFileSystem::new());
     let store_url = datafusion::execution::object_store::ObjectStoreUrl::local_filesystem();
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(1)
+        .force_strategy(Some(FilterStrategy::BooleanMask))
+        .force_pushdown(Some(false))
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: schema.clone(),
         segments: vec![seg],
         store,
         store_url,
         evaluator_factory: factory,
-        target_partitions: 1,
-        force_strategy: Some(FilterStrategy::BooleanMask),
-        force_pushdown: Some(false),
         pushdown_predicate: None,
-        query_config: Arc::new(
-            crate::datafusion_query_config::DatafusionQueryConfig::default(),
-        ),
+        query_config: Arc::new(qc),
         predicate_columns: vec![],
     }));
 
@@ -456,11 +457,7 @@ async fn bitmap_tree_or_predicate_keeps_two_pages() {
 async fn bitmap_tree_and_two_predicates_intersect() {
     let left = binop(col_expr("price"), Operator::GtEq, lit_i32(10_000));
     let right = binop(col_expr("price"), Operator::Lt, lit_i32(21_024));
-    let tree = BoolNode::And(vec![
-        collector_leaf(0),
-        pred_node(left),
-        pred_node(right),
-    ]);
+    let tree = BoolNode::And(vec![collector_leaf(0), pred_node(left), pred_node(right)]);
     let (prices, plan) = run_bitmap_tree(tree).await;
 
     assert_eq!(prices.len(), 2 * ROWS_PER_PAGE);
@@ -590,8 +587,7 @@ async fn single_collector_no_pruning_all_strategies() {
         CollectorCallStrategy::TightenOuterBounds,
         CollectorCallStrategy::PageRangeSplit,
     ] {
-        let (prices, _plan) =
-            run_single_collector(0, Arc::clone(&residual), strategy).await;
+        let (prices, _plan) = run_single_collector(0, Arc::clone(&residual), strategy).await;
         assert_eq!(prices.len(), NUM_ROWS, "strategy {:?}", strategy);
     }
 }
@@ -637,11 +633,11 @@ async fn single_collector_page1_only() {
 /// propagated to branch_B, so Coll(pages2+3) only scanned 1024 docs instead of 3072.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn complex_selectivity_ordering_tighter_branch_first() {
-    let pred_narrow = binop(col_expr("price"), Operator::Lt, lit_i32(1024));  // keeps page 0
-    let pred_wide = binop(col_expr("price"), Operator::Lt, lit_i32(21024));   // keeps pages 0,1,2
+    let pred_narrow = binop(col_expr("price"), Operator::Lt, lit_i32(1024)); // keeps page 0
+    let pred_wide = binop(col_expr("price"), Operator::Lt, lit_i32(21024)); // keeps pages 0,1,2
     let tree = BoolNode::And(vec![
-        BoolNode::And(vec![collector_leaf(3), pred_node(pred_narrow)]),  // branch_A
-        BoolNode::And(vec![collector_leaf(4), pred_node(pred_wide)]),    // branch_B
+        BoolNode::And(vec![collector_leaf(3), pred_node(pred_narrow)]), // branch_A
+        BoolNode::And(vec![collector_leaf(4), pred_node(pred_wide)]),   // branch_B
     ]);
     let (prices, plan) = run_bitmap_tree(tree).await;
 
@@ -707,20 +703,20 @@ async fn complex_deep_nesting_range_propagation_4_levels() {
     let pred_ge_20k = binop(col_expr("price"), Operator::GtEq, lit_i32(20_000));
 
     let tree = BoolNode::And(vec![
-        pred_node(pred_ge_10k),                                          // pages 1,2,3
-        pred_node(pred_lt_31k),                                          // pages 0,1,2
+        pred_node(pred_ge_10k), // pages 1,2,3
+        pred_node(pred_lt_31k), // pages 0,1,2
         BoolNode::Or(vec![
             BoolNode::And(vec![
-                pred_node(pred_lt_21k),                                  // pages 0,1,2
-                collector_leaf(1),                                        // even docs
+                pred_node(pred_lt_21k), // pages 0,1,2
+                collector_leaf(1),      // even docs
             ]),
             BoolNode::And(vec![
-                pred_node(pred_ge_20k),                                  // pages 2,3
-                collector_leaf(5),                                        // odd docs
-                BoolNode::Not(Box::new(collector_leaf(6))),              // NOT page3
+                pred_node(pred_ge_20k),                     // pages 2,3
+                collector_leaf(5),                          // odd docs
+                BoolNode::Not(Box::new(collector_leaf(6))), // NOT page3
             ]),
         ]),
-        collector_leaf(0),                                                // all docs
+        collector_leaf(0), // all docs
     ]);
 
     let (prices, plan) = run_bitmap_tree(tree).await;
@@ -731,12 +727,20 @@ async fn complex_deep_nesting_range_propagation_4_levels() {
     assert_eq!(prices.len(), 512 + 1024);
 
     // Verify page 1 prices are all even-indexed (even doc IDs → even prices in page 1)
-    let page1_prices: Vec = prices.iter().filter(|&&p| p >= 10_000 && p < 11_024).copied().collect();
+    let page1_prices: Vec = prices
+        .iter()
+        .filter(|&&p| p >= 10_000 && p < 11_024)
+        .copied()
+        .collect();
     assert_eq!(page1_prices.len(), 512);
     assert!(page1_prices.iter().all(|p| (p - 10_000) % 2 == 0));
 
     // Verify all of page 2 is present
-    let page2_prices: Vec = prices.iter().filter(|&&p| p >= 20_000 && p < 21_024).copied().collect();
+    let page2_prices: Vec = prices
+        .iter()
+        .filter(|&&p| p >= 20_000 && p < 21_024)
+        .copied()
+        .collect();
     assert_eq!(page2_prices.len(), 1024);
 
     let m = aggregate_metrics(&plan);
@@ -817,10 +821,14 @@ async fn complex_5_wide_or_mixed_predicates_collectors() {
     let tree = BoolNode::And(vec![
         pred_node(pred_ge_10k),
         BoolNode::Or(vec![
-            BoolNode::And(vec![pred_node(pred_lt_11k), collector_leaf(1)]),       // even, page 1
-            BoolNode::And(vec![pred_node(pred_ge_20k), pred_node(pred_lt_21k.clone()), collector_leaf(5)]), // odd, page 2
-            BoolNode::And(vec![collector_leaf(6), pred_node(pred_ge_30k)]),       // page3 coll, page 3
-            BoolNode::Not(Box::new(collector_leaf(0))),                           // NOT all = empty
+            BoolNode::And(vec![pred_node(pred_lt_11k), collector_leaf(1)]), // even, page 1
+            BoolNode::And(vec![
+                pred_node(pred_ge_20k),
+                pred_node(pred_lt_21k.clone()),
+                collector_leaf(5),
+            ]), // odd, page 2
+            BoolNode::And(vec![collector_leaf(6), pred_node(pred_ge_30k)]), // page3 coll, page 3
+            BoolNode::Not(Box::new(collector_leaf(0))),                     // NOT all = empty
             BoolNode::And(vec![collector_leaf(2), pred_node(pred_lt_21k)]), // first2pp, pages 0,1,2
         ]),
     ]);
@@ -828,9 +836,18 @@ async fn complex_5_wide_or_mixed_predicates_collectors() {
     let (prices, plan) = run_bitmap_tree(tree).await;
 
     // Count by page:
-    let p1: Vec<_> = prices.iter().filter(|&&p| p >= 10_000 && p < 11_024).collect();
-    let p2: Vec<_> = prices.iter().filter(|&&p| p >= 20_000 && p < 21_024).collect();
-    let p3: Vec<_> = prices.iter().filter(|&&p| p >= 30_000 && p < 31_024).collect();
+    let p1: Vec<_> = prices
+        .iter()
+        .filter(|&&p| p >= 10_000 && p < 11_024)
+        .collect();
+    let p2: Vec<_> = prices
+        .iter()
+        .filter(|&&p| p >= 20_000 && p < 21_024)
+        .collect();
+    let p3: Vec<_> = prices
+        .iter()
+        .filter(|&&p| p >= 30_000 && p < 31_024)
+        .collect();
     let p0: Vec<_> = prices.iter().filter(|&&p| p < 1_024).collect();
 
     // Page 0: excluded by root Pred(>= 10000)
@@ -903,17 +920,17 @@ async fn complex_cascading_and_with_not_at_depth() {
     let pred_ge_30k = binop(col_expr("price"), Operator::GtEq, lit_i32(30_000));
 
     let tree = BoolNode::And(vec![
-        pred_node(pred_ge_10k),                                          // pages 1,2,3
-        pred_node(pred_lt_31k),                                          // pages 0,1,2
+        pred_node(pred_ge_10k), // pages 1,2,3
+        pred_node(pred_lt_31k), // pages 0,1,2
         BoolNode::And(vec![
-            pred_node(pred_ge_20k),                                      // pages 2,3
+            pred_node(pred_ge_20k), // pages 2,3
             BoolNode::Not(Box::new(BoolNode::And(vec![
-                collector_leaf(6),                                        // page3 collector
-                pred_node(pred_ge_30k),                                  // page 3
+                collector_leaf(6),      // page3 collector
+                pred_node(pred_ge_30k), // page 3
             ]))),
-            collector_leaf(0),                                            // all docs
+            collector_leaf(0), // all docs
         ]),
-        collector_leaf(1),                                                // even docs
+        collector_leaf(1), // even docs
     ]);
 
     let (prices, plan) = run_bitmap_tree(tree).await;
@@ -977,15 +994,15 @@ fn cost_ordering_predicates_sorted_by_selectivity() {
     let pruner = PagePruner::new(&schema, seg.metadata);
     let ctx = fixture_eval_ctx();
 
-    let pred_narrow = binop(col_expr("price"), Operator::Lt, lit_i32(1024));     // 1/4 pages
-    let pred_wide = binop(col_expr("price"), Operator::Lt, lit_i32(21024));      // 3/4 pages
+    let pred_narrow = binop(col_expr("price"), Operator::Lt, lit_i32(1024)); // 1/4 pages
+    let pred_wide = binop(col_expr("price"), Operator::Lt, lit_i32(21024)); // 3/4 pages
     let pred_narrow2 = binop(col_expr("price"), Operator::GtEq, lit_i32(30_000)); // 1/4 pages
 
     let tree = BoolNode::And(vec![
-        pred_node(pred_narrow.clone()),   // child 0
-        pred_node(pred_wide.clone()),     // child 1
-        collector_leaf(0),                // child 2
-        pred_node(pred_narrow2.clone()),  // child 3
+        pred_node(pred_narrow.clone()),  // child 0
+        pred_node(pred_wide.clone()),    // child 1
+        collector_leaf(0),               // child 2
+        pred_node(pred_narrow2.clone()), // child 3
     ]);
 
     // Resolve tree to get ResolvedNodes.
@@ -994,7 +1011,10 @@ fn cost_ordering_predicates_sorted_by_selectivity() {
     let mut colls = Vec::new();
     wire_collectors_dfs(&tree, &mut colls);
     let per_leaf: Vec<(i32, Arc)> = colls
-        .into_iter().enumerate().map(|(i, c)| (i as i32, c)).collect();
+        .into_iter()
+        .enumerate()
+        .map(|(i, c)| (i as i32, c))
+        .collect();
     let resolved = tree.resolve(&per_leaf).unwrap();
 
     // Extract children of the root AND.
@@ -1056,7 +1076,10 @@ fn cost_ordering_nested_and_branches_selective_first() {
     let mut colls = Vec::new();
     wire_collectors_dfs(&tree, &mut colls);
     let per_leaf: Vec<(i32, Arc)> = colls
-        .into_iter().enumerate().map(|(i, c)| (i as i32, c)).collect();
+        .into_iter()
+        .enumerate()
+        .map(|(i, c)| (i as i32, c))
+        .collect();
     let resolved = tree.resolve(&per_leaf).unwrap();
 
     let children = match &resolved {
@@ -1075,7 +1098,8 @@ fn cost_ordering_nested_and_branches_selective_first() {
         cost_a < cost_b,
         "branch with more selective predicate should be cheaper \
          (A={}, B={})",
-        cost_a, cost_b
+        cost_a,
+        cost_b
     );
 }
 
@@ -1105,9 +1129,10 @@ fn cost_ordering_complex_tree_predicates_before_or_before_nothing() {
     let pred_ge_20k = binop(col_expr("price"), Operator::GtEq, lit_i32(20_000));
 
     let tree = BoolNode::And(vec![
-        pred_node(pred_ge_10k),                                          // child 0
-        pred_node(pred_lt_31k),                                          // child 1
-        BoolNode::Or(vec![                                               // child 2
+        pred_node(pred_ge_10k), // child 0
+        pred_node(pred_lt_31k), // child 1
+        BoolNode::Or(vec![
+            // child 2
             BoolNode::And(vec![pred_node(pred_lt_21k), collector_leaf(1)]),
             BoolNode::And(vec![
                 pred_node(pred_ge_20k),
@@ -1115,7 +1140,7 @@ fn cost_ordering_complex_tree_predicates_before_or_before_nothing() {
                 BoolNode::Not(Box::new(collector_leaf(6))),
             ]),
         ]),
-        collector_leaf(0),                                                // child 3
+        collector_leaf(0), // child 3
     ]);
 
     let tree = tree.push_not_down();
@@ -1123,7 +1148,10 @@ fn cost_ordering_complex_tree_predicates_before_or_before_nothing() {
     let mut colls = Vec::new();
     wire_collectors_dfs(&tree, &mut colls);
     let per_leaf: Vec<(i32, Arc)> = colls
-        .into_iter().enumerate().map(|(i, c)| (i as i32, c)).collect();
+        .into_iter()
+        .enumerate()
+        .map(|(i, c)| (i as i32, c))
+        .collect();
     let resolved = tree.resolve(&per_leaf).unwrap();
 
     let children = match &resolved {
@@ -1148,7 +1176,8 @@ fn cost_ordering_complex_tree_predicates_before_or_before_nothing() {
     assert!(
         or_cost > costs[3],
         "OR subtree ({}) should be more expensive than single Coll ({})",
-        or_cost, costs[3]
+        or_cost,
+        costs[3]
     );
 
     // Verify execution order: Pred(>=10k) first, Pred(<31k) second, Coll third, OR last
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/schema_drift.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/schema_drift.rs
index eaee931eb7c88..d280e1b16db5c 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/schema_drift.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/schema_drift.rs
@@ -120,6 +120,11 @@ async fn run_missing_col_tree(tree_bool: BoolNode) -> usize {
             Ok(eval)
         })
     };
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(1)
+        .force_strategy(Some(FilterStrategy::BooleanMask))
+        .force_pushdown(Some(false))
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: schema.clone(),
         segments: vec![segment],
@@ -127,13 +132,8 @@ async fn run_missing_col_tree(tree_bool: BoolNode) -> usize {
             as Arc,
         store_url: datafusion::execution::object_store::ObjectStoreUrl::local_filesystem(),
         evaluator_factory: factory,
-        target_partitions: 1,
-        force_strategy: Some(FilterStrategy::BooleanMask),
-        force_pushdown: Some(false),
         pushdown_predicate: None,
-        query_config: std::sync::Arc::new(
-            crate::datafusion_query_config::DatafusionQueryConfig::default(),
-        ),
+        query_config: std::sync::Arc::new(qc),
         predicate_columns: vec![],
     }));
     let ctx = SessionContext::new();
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/streaming_at_scale.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/streaming_at_scale.rs
index 68e667c2cd495..2424c37cfc0ae 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/streaming_at_scale.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/tests_e2e/streaming_at_scale.rs
@@ -436,6 +436,11 @@ async fn run_large(
         })
     };
 
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(1)
+        .force_strategy(Some(FilterStrategy::BooleanMask))
+        .force_pushdown(Some(false))
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: schema.clone(),
         segments: vec![segment],
@@ -443,13 +448,8 @@ async fn run_large(
             as Arc,
         store_url: datafusion::execution::object_store::ObjectStoreUrl::local_filesystem(),
         evaluator_factory: factory,
-        target_partitions: 1,
-        force_strategy: Some(FilterStrategy::BooleanMask),
-        force_pushdown: Some(false),
         pushdown_predicate: None,
-        query_config: std::sync::Arc::new(
-            crate::datafusion_query_config::DatafusionQueryConfig::default(),
-        ),
+        query_config: std::sync::Arc::new(qc),
         predicate_columns: vec![],
     }));
 
@@ -885,6 +885,11 @@ async fn run_large_partitioned(
             Ok(eval)
         })
     };
+    let qc = crate::datafusion_query_config::DatafusionQueryConfig::builder()
+        .target_partitions(partitions)
+        .force_strategy(Some(FilterStrategy::BooleanMask))
+        .force_pushdown(Some(false))
+        .build();
     let provider = Arc::new(IndexedTableProvider::new(IndexedTableConfig {
         schema: schema.clone(),
         segments: vec![segment],
@@ -892,13 +897,8 @@ async fn run_large_partitioned(
             as Arc,
         store_url: datafusion::execution::object_store::ObjectStoreUrl::local_filesystem(),
         evaluator_factory: factory,
-        target_partitions: partitions,
-        force_strategy: Some(FilterStrategy::BooleanMask),
-        force_pushdown: Some(false),
         pushdown_predicate: None,
-        query_config: std::sync::Arc::new(
-            crate::datafusion_query_config::DatafusionQueryConfig::default(),
-        ),
+        query_config: std::sync::Arc::new(qc),
         predicate_columns: vec![],
     }));
     let ctx = SessionContext::new();
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
index ce037eb3c6508..52c74a888e650 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
@@ -29,6 +29,7 @@ use log::error;
 use object_store::ObjectMeta;
 
 use crate::api::{DataFusionRuntime, ShardView};
+use crate::datafusion_query_config::DatafusionQueryConfig;
 use crate::query_tracker::QueryTrackingContext;
 
 /// Opaque handle holding a configured SessionContext between FFM calls.
@@ -40,6 +41,8 @@ pub struct SessionContextHandle {
     pub table_name: String,
     /// When set, indicates this session uses the indexed execution path with filter delegation.
     pub indexed_config: Option,
+    /// Per-query tuning knobs (batch size, partitions, filter strategies, etc.)
+    pub query_config: DatafusionQueryConfig,
 }
 
 /// Configuration for indexed execution with filter delegation, provided by Java.
@@ -55,6 +58,7 @@ pub async unsafe fn create_session_context(
     shard_view_ptr: i64,
     table_name: &str,
     context_id: i64,
+    query_config: DatafusionQueryConfig,
 ) -> Result {
     let runtime = &*(runtime_ptr as *const DataFusionRuntime);
     let shard_view = &*(shard_view_ptr as *const ShardView);
@@ -95,7 +99,6 @@ pub async unsafe fn create_session_context(
         e
     })?;
 
-    let query_config = crate::datafusion_query_config::DatafusionQueryConfig::default();
     let mut config = SessionConfig::new();
     config.options_mut().execution.parquet.pushdown_filters = query_config.parquet_pushdown_filters;
     config.options_mut().execution.target_partitions = query_config.target_partitions;
@@ -108,7 +111,6 @@ pub async unsafe fn create_session_context(
         .build();
 
     let ctx = SessionContext::new_with_state(state);
-    crate::udf::register_all(&ctx);
 
     // Register default ListingTable for parquet scans
     let listing_options = ListingOptions::new(Arc::new(ParquetFormat::new()))
@@ -128,16 +130,26 @@ pub async unsafe fn create_session_context(
         .with_schema(resolved_schema);
 
     let provider = Arc::new(ListingTable::try_new(table_config).map_err(|e| {
-        error!("create_session_context: failed to create listing table: {}", e);
+        error!(
+            "create_session_context: failed to create listing table: {}",
+            e
+        );
         e
     })?);
 
     ctx.register_table(table_name, provider).map_err(|e| {
-        error!("create_session_context: failed to register table '{}': {}", table_name, e);
+        error!(
+            "create_session_context: failed to register table '{}': {}",
+            table_name, e
+        );
         e
     })?;
 
-    error!("create_session_context: successfully registered table '{}', table_name_len={}", table_name, table_name.len());
+    error!(
+        "create_session_context: successfully registered table '{}', table_name_len={}",
+        table_name,
+        table_name.len()
+    );
 
     let handle = SessionContextHandle {
         ctx,
@@ -146,6 +158,7 @@ pub async unsafe fn create_session_context(
         query_context,
         table_name: table_name.to_string(),
         indexed_config: None,
+        query_config,
     };
     Ok(Box::into_raw(Box::new(handle)) as i64)
 }
@@ -170,9 +183,10 @@ pub async unsafe fn create_session_context_indexed(
     context_id: i64,
     tree_shape: i32,
     delegated_predicate_count: i32,
+    query_config: DatafusionQueryConfig,
 ) -> Result {
     // Create base session context (same as non-indexed path)
-    let ptr = create_session_context(runtime_ptr, shard_view_ptr, table_name, context_id).await?;
+    let ptr = create_session_context(runtime_ptr, shard_view_ptr, table_name, context_id, query_config).await?;
 
     // Augment with indexed config and UDF registration
     let handle = &mut *(ptr as *mut SessionContextHandle);
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/DatafusionDynamicSettingsIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/DatafusionDynamicSettingsIT.java
new file mode 100644
index 0000000000000..85cea8d93c102
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/DatafusionDynamicSettingsIT.java
@@ -0,0 +1,123 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.Version;
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+import org.opensearch.analytics.AnalyticsPlugin;
+import org.opensearch.arrow.flight.transport.FlightStreamPlugin;
+import org.opensearch.be.lucene.LucenePlugin;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.composite.CompositeDataFormatPlugin;
+import org.opensearch.parquet.ParquetDataFormatPlugin;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.plugins.PluginInfo;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1)
+public class DatafusionDynamicSettingsIT extends OpenSearchIntegTestCase {
+
+    @Override
+    protected Collection> nodePlugins() {
+        return List.of(FlightStreamPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class);
+    }
+
+    @Override
+    protected Collection additionalNodePlugins() {
+        return List.of(
+            classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()),
+            classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()),
+            classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName()))
+        );
+    }
+
+    private static PluginInfo classpathPlugin(Class pluginClass, List extendedPlugins) {
+        return new PluginInfo(
+            pluginClass.getName(),
+            "classpath plugin",
+            "NA",
+            Version.CURRENT,
+            "1.8",
+            pluginClass.getName(),
+            null,
+            extendedPlugins,
+            false
+        );
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true)
+            .build();
+    }
+
+    public void testAllIndexedSettingsCanBeUpdatedDynamically() {
+        ClusterUpdateSettingsResponse response = client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .put("datafusion.indexed.batch_size", 16384)
+                    .put("datafusion.indexed.parquet_pushdown_filters", true)
+                    .put("datafusion.indexed.min_skip_run_default", 2048)
+                    .put("datafusion.indexed.min_skip_run_selectivity_threshold", 0.5)
+                    .put("datafusion.indexed.single_collector_strategy", "full_range")
+                    .put("datafusion.indexed.tree_collector_strategy", "page_range_split")
+                    .put("datafusion.indexed.max_collector_parallelism", 4)
+                    .build()
+            )
+            .get();
+        assertTrue(response.isAcknowledged());
+
+        Settings transientSettings = response.getTransientSettings();
+        assertEquals("16384", transientSettings.get("datafusion.indexed.batch_size"));
+        assertEquals("true", transientSettings.get("datafusion.indexed.parquet_pushdown_filters"));
+        assertEquals("2048", transientSettings.get("datafusion.indexed.min_skip_run_default"));
+        assertEquals("0.5", transientSettings.get("datafusion.indexed.min_skip_run_selectivity_threshold"));
+        assertEquals("full_range", transientSettings.get("datafusion.indexed.single_collector_strategy"));
+        assertEquals("page_range_split", transientSettings.get("datafusion.indexed.tree_collector_strategy"));
+        assertEquals("4", transientSettings.get("datafusion.indexed.max_collector_parallelism"));
+    }
+
+    public void testInvalidValuesAreRejected() {
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().put("datafusion.indexed.batch_size", 0).build())
+                .get()
+        );
+
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().put("datafusion.indexed.min_skip_run_selectivity_threshold", 1.5).build())
+                .get()
+        );
+
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().put("datafusion.indexed.single_collector_strategy", "bogus").build())
+                .get()
+        );
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
index b770b8ce58a00..f6713b005393f 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
@@ -12,7 +12,6 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin;
 import org.opensearch.be.datafusion.action.DataFusionStatsAction;
-import org.opensearch.be.datafusion.cache.CacheSettings;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.service.ClusterService;
@@ -109,6 +108,7 @@ public class DataFusionPlugin extends Plugin implements SearchBackEndPlugin createComponents(
         // cluster settings API take effect without restarting the node.
         clusterService.getClusterSettings().addSettingsUpdateConsumer(DATAFUSION_MEMORY_POOL_LIMIT, this::updateMemoryPoolLimit);
 
+        this.datafusionSettings = new DatafusionSettings(clusterService);
+
         this.substraitExtensions = loadSubstraitExtensions();
 
         return Collections.singletonList(dataFusionService);
@@ -191,19 +193,13 @@ ClusterService getClusterService() {
         return clusterService;
     }
 
+    DatafusionSettings getDatafusionSettings() {
+        return datafusionSettings;
+    }
+
     @Override
     public List> getSettings() {
-        return List.of(
-            DATAFUSION_MEMORY_POOL_LIMIT,
-            DATAFUSION_SPILL_MEMORY_LIMIT,
-            DATAFUSION_REDUCE_INPUT_MODE,
-            CacheSettings.METADATA_CACHE_SIZE_LIMIT,
-            CacheSettings.STATISTICS_CACHE_SIZE_LIMIT,
-            CacheSettings.METADATA_CACHE_EVICTION_TYPE,
-            CacheSettings.STATISTICS_CACHE_EVICTION_TYPE,
-            CacheSettings.METADATA_CACHE_ENABLED,
-            CacheSettings.STATISTICS_CACHE_ENABLED
-        );
+        return DatafusionSettings.ALL_SETTINGS;
     }
 
     /**
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSettings.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSettings.java
new file mode 100644
index 0000000000000..e58d6630be19a
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionSettings.java
@@ -0,0 +1,348 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.be.datafusion.cache.CacheSettings;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.search.SearchService;
+
+import java.util.List;
+
+/**
+ * Consolidates all DataFusion plugin settings (existing memory/spill/reduce/cache settings
+ * plus the new indexed query settings) and manages the pre-computed {@link WireConfigSnapshot}.
+ * 
        
+ * Each dynamic indexed setting registers an {@code addSettingsUpdateConsumer} callback that
+ * atomically rebuilds the volatile snapshot on change. At query time, the instruction handler
+ * reads the snapshot with zero per-query overhead — no {@code ClusterService} lookup on the
+ * hot path.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public final class DatafusionSettings {
+
+    // ── New indexed query settings ──
+
+    /** Number of rows per batch in the indexed query execution path. */
+    public static final Setting INDEXED_BATCH_SIZE = Setting.intSetting(
+        "datafusion.indexed.batch_size",
+        8192,
+        1,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    /**
+     * Whether DataFusion applies residual predicate pushdown during parquet decode
+     * on the indexed path. When true, narrow row-granular selections benefit from
+     * decode-time filtering via {@code RowFilter}. When false (default), the indexed
+     * stream handles filtering externally via bitmap-based row selection.
+     * 
        
+     * Note: ideally this decision should be taken by the planner on a per-query basis
+     * (e.g., based on filter shape and estimated selectivity). This setting acts as
+     * the node-wide default until per-query planner support is added.
+     */
+    public static final Setting INDEXED_PARQUET_PUSHDOWN_FILTERS = Setting.boolSetting(
+        "datafusion.indexed.parquet_pushdown_filters",
+        false,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    /**
+     * Default minimum run length (in rows) below which the indexed stream skips
+     * row-selection optimizations and falls back to sequential decode. Shorter runs
+     * have higher per-row overhead from selection vector maintenance.
+     */
+    public static final Setting INDEXED_MIN_SKIP_RUN_DEFAULT = Setting.intSetting(
+        "datafusion.indexed.min_skip_run_default",
+        1024,
+        1,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    /**
+     * Selectivity threshold [0.0, 1.0] that controls when the indexed stream switches
+     * from row-selection mode to full-decode mode. A low threshold (e.g., 0.03) means
+     * "only use row-selection when the filter is very selective (few rows match)."
+     * 
        
+     * Example: with threshold 0.03, a filter that matches 2% of rows uses row-selection
+     * (skip non-matching rows), but a filter matching 5% switches to full-decode
+     * (cheaper to just read everything sequentially).
+     */
+    public static final Setting INDEXED_MIN_SKIP_RUN_SELECTIVITY_THRESHOLD = Setting.doubleSetting(
+        "datafusion.indexed.min_skip_run_selectivity_threshold",
+        0.03,
+        0.0,
+        1.0,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    // Strategy constants for CollectorCallStrategy
+    public static final String STRATEGY_FULL_RANGE = "full_range";
+    public static final String STRATEGY_TIGHTEN_OUTER_BOUNDS = "tighten_outer_bounds";
+    public static final String STRATEGY_PAGE_RANGE_SPLIT = "page_range_split";
+
+    /**
+     * How the SingleCollectorEvaluator narrows collector doc ranges relative to
+     * page-pruning results. Valid values: full_range, tighten_outer_bounds, page_range_split.
+     * Default is page_range_split — only one collector, so multiple FFM calls per RG is acceptable.
+     */
+    public static final Setting INDEXED_SINGLE_COLLECTOR_STRATEGY = Setting.simpleString(
+        "datafusion.indexed.single_collector_strategy",
+        STRATEGY_PAGE_RANGE_SPLIT,
+        value -> {
+            switch (value) {
+                case STRATEGY_FULL_RANGE:
+                case STRATEGY_TIGHTEN_OUTER_BOUNDS:
+                case STRATEGY_PAGE_RANGE_SPLIT:
+                    break;
+                default:
+                    throw new IllegalArgumentException(
+                        "datafusion.indexed.single_collector_strategy must be one of "
+                            + "[full_range, tighten_outer_bounds, page_range_split], got: "
+                            + value
+                    );
+            }
+        },
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    /**
+     * How the bitmap tree evaluator narrows collector doc ranges when multiple collectors
+     * are present. Valid values: full_range, tighten_outer_bounds, page_range_split.
+     * Default is tighten_outer_bounds — multiple collectors make page_range_split expensive.
+     */
+    public static final Setting INDEXED_TREE_COLLECTOR_STRATEGY = Setting.simpleString(
+        "datafusion.indexed.tree_collector_strategy",
+        STRATEGY_TIGHTEN_OUTER_BOUNDS,
+        value -> {
+            switch (value) {
+                case STRATEGY_FULL_RANGE:
+                case STRATEGY_TIGHTEN_OUTER_BOUNDS:
+                case STRATEGY_PAGE_RANGE_SPLIT:
+                    break;
+                default:
+                    throw new IllegalArgumentException(
+                        "datafusion.indexed.tree_collector_strategy must be one of "
+                            + "[full_range, tighten_outer_bounds, page_range_split], got: "
+                            + value
+                    );
+            }
+        },
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    /**
+     * Maximum number of Collector-leaf FFM calls issued in parallel per row-group
+     * prefetch. 1 = fully sequential (lowest CPU, fastest short-circuit). Higher
+     * values sacrifice short-circuit savings in AND/OR groups but reduce latency
+     * for independent collector leaves.
+     */
+    public static final Setting INDEXED_MAX_COLLECTOR_PARALLELISM = Setting.intSetting(
+        "datafusion.indexed.max_collector_parallelism",
+        1,
+        1,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    // ── All settings registered by the plugin ──
+
+    public static final List> ALL_SETTINGS = List.of(
+
+        // Runtime settings — memory pool, spill, and reduce input mode
+        DataFusionPlugin.DATAFUSION_MEMORY_POOL_LIMIT,
+        DataFusionPlugin.DATAFUSION_SPILL_MEMORY_LIMIT,
+        DataFusionPlugin.DATAFUSION_REDUCE_INPUT_MODE,
+
+        // Cache settings — metadata and statistics cache configuration
+        CacheSettings.METADATA_CACHE_SIZE_LIMIT,
+        CacheSettings.STATISTICS_CACHE_SIZE_LIMIT,
+        CacheSettings.METADATA_CACHE_EVICTION_TYPE,
+        CacheSettings.STATISTICS_CACHE_EVICTION_TYPE,
+        CacheSettings.METADATA_CACHE_ENABLED,
+        CacheSettings.STATISTICS_CACHE_ENABLED,
+
+        // Indexed query settings — per-query tuning knobs for the indexed execution path
+        INDEXED_BATCH_SIZE,
+        INDEXED_PARQUET_PUSHDOWN_FILTERS,
+        INDEXED_MIN_SKIP_RUN_DEFAULT,
+        INDEXED_MIN_SKIP_RUN_SELECTIVITY_THRESHOLD,
+        INDEXED_SINGLE_COLLECTOR_STRATEGY,
+        INDEXED_TREE_COLLECTOR_STRATEGY,
+        INDEXED_MAX_COLLECTOR_PARALLELISM
+    );
+
+    // ── Snapshot management ──
+
+    private volatile WireConfigSnapshot snapshot;
+
+    /**
+     * Tracks the current value of {@code search.concurrent.max_slice_count} for
+     * deriving {@code target_partitions}. Updated by the registered listener.
+     */
+    private volatile int maxSliceCount;
+
+    /**
+     * Tracks the current concurrent search mode ({@code "auto"}, {@code "all"}, or {@code "none"}).
+     * When mode is {@code "none"}, target_partitions is forced to 1.
+     */
+    private volatile String concurrentSearchMode;
+
+    /**
+     * Creates the settings holder, builds the initial {@link WireConfigSnapshot} from
+     * the cluster service's settings, and registers listeners for dynamic updates.
+     *
+     * @param clusterService the cluster service providing settings and listener registration
+     */
+    public DatafusionSettings(ClusterService clusterService) {
+        Settings settings = clusterService.getSettings();
+        ClusterSettings clusterSettings = clusterService.getClusterSettings();
+
+        this.concurrentSearchMode = SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.get(settings);
+        this.maxSliceCount = SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING.get(settings);
+
+        this.snapshot = WireConfigSnapshot.builder()
+            .batchSize(INDEXED_BATCH_SIZE.get(settings))
+            .targetPartitions(deriveTargetPartitions(this.concurrentSearchMode, this.maxSliceCount))
+            .parquetPushdownFilters(INDEXED_PARQUET_PUSHDOWN_FILTERS.get(settings))
+            .minSkipRunDefault(INDEXED_MIN_SKIP_RUN_DEFAULT.get(settings))
+            .minSkipRunSelectivityThreshold(INDEXED_MIN_SKIP_RUN_SELECTIVITY_THRESHOLD.get(settings))
+            .singleCollectorStrategy(strategyToWireValue(INDEXED_SINGLE_COLLECTOR_STRATEGY.get(settings)))
+            .treeCollectorStrategy(strategyToWireValue(INDEXED_TREE_COLLECTOR_STRATEGY.get(settings)))
+            .maxCollectorParallelism(INDEXED_MAX_COLLECTOR_PARALLELISM.get(settings))
+            .build();
+
+        registerListeners(clusterSettings);
+    }
+
+    /**
+     * Package-private constructor for testing — builds the initial snapshot from
+     * raw settings without registering dynamic update listeners.
+     */
+    DatafusionSettings(Settings settings) {
+        this.concurrentSearchMode = SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.get(settings);
+        this.maxSliceCount = SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING.get(settings);
+
+        this.snapshot = WireConfigSnapshot.builder()
+            .batchSize(INDEXED_BATCH_SIZE.get(settings))
+            .targetPartitions(deriveTargetPartitions(this.concurrentSearchMode, this.maxSliceCount))
+            .parquetPushdownFilters(INDEXED_PARQUET_PUSHDOWN_FILTERS.get(settings))
+            .minSkipRunDefault(INDEXED_MIN_SKIP_RUN_DEFAULT.get(settings))
+            .minSkipRunSelectivityThreshold(INDEXED_MIN_SKIP_RUN_SELECTIVITY_THRESHOLD.get(settings))
+            .singleCollectorStrategy(strategyToWireValue(INDEXED_SINGLE_COLLECTOR_STRATEGY.get(settings)))
+            .treeCollectorStrategy(strategyToWireValue(INDEXED_TREE_COLLECTOR_STRATEGY.get(settings)))
+            .maxCollectorParallelism(INDEXED_MAX_COLLECTOR_PARALLELISM.get(settings))
+            .build();
+    }
+
+    void registerListeners(ClusterSettings clusterSettings) {
+        clusterSettings.addSettingsUpdateConsumer(INDEXED_BATCH_SIZE, newValue -> {
+            snapshot = WireConfigSnapshot.builder(snapshot).batchSize(newValue).build();
+        });
+
+        clusterSettings.addSettingsUpdateConsumer(INDEXED_PARQUET_PUSHDOWN_FILTERS, newValue -> {
+            snapshot = WireConfigSnapshot.builder(snapshot).parquetPushdownFilters(newValue).build();
+        });
+
+        clusterSettings.addSettingsUpdateConsumer(INDEXED_MIN_SKIP_RUN_DEFAULT, newValue -> {
+            snapshot = WireConfigSnapshot.builder(snapshot).minSkipRunDefault(newValue).build();
+        });
+
+        clusterSettings.addSettingsUpdateConsumer(INDEXED_MIN_SKIP_RUN_SELECTIVITY_THRESHOLD, newValue -> {
+            snapshot = WireConfigSnapshot.builder(snapshot).minSkipRunSelectivityThreshold(newValue).build();
+        });
+
+        clusterSettings.addSettingsUpdateConsumer(INDEXED_SINGLE_COLLECTOR_STRATEGY, newValue -> {
+            snapshot = WireConfigSnapshot.builder(snapshot).singleCollectorStrategy(strategyToWireValue(newValue)).build();
+        });
+
+        clusterSettings.addSettingsUpdateConsumer(INDEXED_TREE_COLLECTOR_STRATEGY, newValue -> {
+            snapshot = WireConfigSnapshot.builder(snapshot).treeCollectorStrategy(strategyToWireValue(newValue)).build();
+        });
+
+        clusterSettings.addSettingsUpdateConsumer(INDEXED_MAX_COLLECTOR_PARALLELISM, newValue -> {
+            snapshot = WireConfigSnapshot.builder(snapshot).maxCollectorParallelism(newValue).build();
+        });
+
+        clusterSettings.addSettingsUpdateConsumer(SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING, newValue -> {
+            this.maxSliceCount = newValue;
+            snapshot = WireConfigSnapshot.builder(snapshot)
+                .targetPartitions(deriveTargetPartitions(this.concurrentSearchMode, newValue))
+                .build();
+        });
+
+        clusterSettings.addSettingsUpdateConsumer(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE, newValue -> {
+            this.concurrentSearchMode = newValue;
+            snapshot = WireConfigSnapshot.builder(snapshot).targetPartitions(deriveTargetPartitions(newValue, this.maxSliceCount)).build();
+        });
+    }
+
+    /**
+     * Returns the current pre-computed wire config snapshot. This is a single
+     * volatile read — safe for the query hot path with zero overhead.
+     *
+     * @return the current snapshot (never null after construction)
+     */
+    public WireConfigSnapshot getSnapshot() {
+        return snapshot;
+    }
+
+    /**
+     * Converts a strategy string to its wire format integer value.
+     * 
        
+     * Mapping: full_range = 0, tighten_outer_bounds = 1, page_range_split = 2.
+     */
+    static int strategyToWireValue(String strategy) {
+        switch (strategy) {
+            case STRATEGY_FULL_RANGE:
+                return 0;
+            case STRATEGY_TIGHTEN_OUTER_BOUNDS:
+                return 1;
+            case STRATEGY_PAGE_RANGE_SPLIT:
+                return 2;
+            default:
+                throw new IllegalArgumentException("Unknown strategy: " + strategy);
+        }
+    }
+
+    /**
+     * Derives {@code target_partitions} from the concurrent search mode and
+     * {@code search.concurrent.max_slice_count} setting value.
+     * 
        
+     * When mode is {@code "none"}, forces target_partitions to 1 (no concurrency).
+     * When {@code max_slice_count} is 0, uses 50% of available CPU cores.
+     * Otherwise caps the value at 100% of available CPU cores.
+     */
+    private static int deriveTargetPartitions(String mode, int maxSliceCount) {
+        if (SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE.equals(mode)) {
+            return 1;
+        }
+
+        // For maxSliceCount == 0 also, we will be owning the concurrency level
+        if (maxSliceCount == 0) {
+            return Runtime.getRuntime().availableProcessors() / 2;
+        }
+
+        // Even if the user set's a higher value, we will still want to limit the number
+        // of slices to the number of available processors
+        // to avoid over-subscription and ensure reasonable performance
+        return Math.min(maxSliceCount, Runtime.getRuntime().availableProcessors());
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java
index 2910499103bd5..b91d62e912f9d 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java
@@ -17,6 +17,9 @@
 import org.opensearch.be.datafusion.nativelib.SessionContextHandle;
 import org.opensearch.index.engine.dataformat.DataFormatRegistry;
 
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+
 /**
  * Handles ShardScan instruction: creates a SessionContext via FFM and registers
  * the default ListingTable provider for parquet scans.
@@ -52,8 +55,18 @@ public BackendExecutionContext apply(
         long runtimePtr = dataFusionService.getNativeRuntime().get();
         long contextId = context.getTask() != null ? context.getTask().getId() : 0L;
 
-        SessionContextHandle sessionCtxHandle = NativeBridge.createSessionContext(readerPtr, runtimePtr, context.getTableName(), contextId);
-
-        return new DataFusionSessionState(sessionCtxHandle);
+        WireConfigSnapshot snapshot = plugin.getDatafusionSettings().getSnapshot();
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment segment = arena.allocate(WireConfigSnapshot.BYTE_SIZE);
+            snapshot.writeTo(segment);
+            SessionContextHandle sessionCtxHandle = NativeBridge.createSessionContext(
+                readerPtr,
+                runtimePtr,
+                context.getTableName(),
+                contextId,
+                segment.address()
+            );
+            return new DataFusionSessionState(sessionCtxHandle);
+        }
     }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanWithDelegationHandler.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanWithDelegationHandler.java
index 4564a5921b9ea..c44c5d25d2eb2 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanWithDelegationHandler.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanWithDelegationHandler.java
@@ -18,6 +18,9 @@
 import org.opensearch.be.datafusion.nativelib.SessionContextHandle;
 import org.opensearch.index.engine.dataformat.DataFormatRegistry;
 
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+
 /**
  * Handles ShardScanWithDelegation instruction: creates a SessionContext via FFM
  * configured for indexed execution — registers the delegated_predicate UDF and
@@ -57,17 +60,20 @@ public BackendExecutionContext apply(
         FilterTreeShape treeShape = node.getTreeShape();
         int delegatedPredicateCount = node.getDelegatedPredicateCount();
 
-        // Single FFM call: creates SessionContext + registers delegated_predicate UDF +
-        // configures IndexedTableProvider with treeShape and delegatedPredicateCount
-        SessionContextHandle sessionCtxHandle = NativeBridge.createSessionContextForIndexedExecution(
-            readerPtr,
-            runtimePtr,
-            context.getTableName(),
-            contextId,
-            treeShape.ordinal(),
-            delegatedPredicateCount
-        );
-
-        return new DataFusionSessionState(sessionCtxHandle);
+        WireConfigSnapshot snapshot = plugin.getDatafusionSettings().getSnapshot();
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment segment = arena.allocate(WireConfigSnapshot.BYTE_SIZE);
+            snapshot.writeTo(segment);
+            SessionContextHandle sessionCtxHandle = NativeBridge.createSessionContextForIndexedExecution(
+                readerPtr,
+                runtimePtr,
+                context.getTableName(),
+                contextId,
+                treeShape.ordinal(),
+                delegatedPredicateCount,
+                segment.address()
+            );
+            return new DataFusionSessionState(sessionCtxHandle);
+        }
     }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/WireConfigSnapshot.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/WireConfigSnapshot.java
new file mode 100644
index 0000000000000..012f47aa9b540
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/WireConfigSnapshot.java
@@ -0,0 +1,220 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.ValueLayout;
+
+/**
+ * Immutable snapshot of the dynamic indexed query settings, ready to be written
+ * into a {@code MemorySegment} matching the Rust {@code WireDatafusionQueryConfig}
+ * {@code #[repr(C)]} layout.
+ * 
        
+ * Use {@link #builder()} to construct instances.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public final class WireConfigSnapshot {
+
+    /** Total byte size of the wire struct ({@code WireDatafusionQueryConfig}). */
+    public static final long BYTE_SIZE = 68;
+
+    private final int batchSize;
+    private final int targetPartitions;
+    private final boolean parquetPushdownFilters;
+    private final int minSkipRunDefault;
+    private final double minSkipRunSelectivityThreshold;
+    private final int maxCollectorParallelism;
+    private final int singleCollectorStrategy;
+    private final int treeCollectorStrategy;
+
+    private WireConfigSnapshot(Builder builder) {
+        this.batchSize = builder.batchSize;
+        this.targetPartitions = builder.targetPartitions;
+        this.parquetPushdownFilters = builder.parquetPushdownFilters;
+        this.minSkipRunDefault = builder.minSkipRunDefault;
+        this.minSkipRunSelectivityThreshold = builder.minSkipRunSelectivityThreshold;
+        this.maxCollectorParallelism = builder.maxCollectorParallelism;
+        this.singleCollectorStrategy = builder.singleCollectorStrategy;
+        this.treeCollectorStrategy = builder.treeCollectorStrategy;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Creates a builder pre-populated with all values from an existing snapshot.
+     * Useful for rebuilding a snapshot with a single field changed.
+     */
+    public static Builder builder(WireConfigSnapshot current) {
+        return new Builder().batchSize(current.batchSize)
+            .targetPartitions(current.targetPartitions)
+            .parquetPushdownFilters(current.parquetPushdownFilters)
+            .minSkipRunDefault(current.minSkipRunDefault)
+            .minSkipRunSelectivityThreshold(current.minSkipRunSelectivityThreshold)
+            .maxCollectorParallelism(current.maxCollectorParallelism)
+            .singleCollectorStrategy(current.singleCollectorStrategy)
+            .treeCollectorStrategy(current.treeCollectorStrategy);
+    }
+
+    public int batchSize() {
+        return batchSize;
+    }
+
+    public int targetPartitions() {
+        return targetPartitions;
+    }
+
+    public boolean parquetPushdownFilters() {
+        return parquetPushdownFilters;
+    }
+
+    public int minSkipRunDefault() {
+        return minSkipRunDefault;
+    }
+
+    public double minSkipRunSelectivityThreshold() {
+        return minSkipRunSelectivityThreshold;
+    }
+
+    public int maxCollectorParallelism() {
+        return maxCollectorParallelism;
+    }
+
+    public int singleCollectorStrategy() {
+        return singleCollectorStrategy;
+    }
+
+    public int treeCollectorStrategy() {
+        return treeCollectorStrategy;
+    }
+
+    /**
+     * Writes this snapshot into a {@code MemorySegment} matching the
+     * {@code WireDatafusionQueryConfig} {@code #[repr(C)]} layout.
+     * 
        
+     * The segment must be at least {@link #BYTE_SIZE} bytes and allocated from
+     * a confined {@code Arena} scoped to the query lifetime.
+     *
+     * 
        +     * Offset  Size  Field                                Type     Source
+     * ──────  ────  ─────────────────────────────────    ──────   ───────────
+     * 0       8     batch_size                           i64      from snapshot
+     * 8       8     target_partitions                    i64      from snapshot
+     * 16      8     min_skip_run_default                 i64      from snapshot
+     * 24      8     min_skip_run_selectivity_threshold   f64      from snapshot
+     * 32      4     parquet_pushdown_filters             i32      from snapshot (0/1)
+     * 36      4     indexed_pushdown_filters             i32      hardcoded 1
+     * 40      4     force_strategy                       i32      hardcoded -1
+     * 44      4     force_pushdown                       i32      hardcoded -1
+     * 48      4     cost_predicate                       i32      hardcoded 1
+     * 52      4     cost_collector                       i32      hardcoded 10
+     * 56      4     max_collector_parallelism            i32      from snapshot
+     * 60      4     single_collector_strategy            i32      from snapshot
+     * 64      4     tree_collector_strategy              i32      from snapshot
+     * ──────  ────
+     * Total: 68 bytes
+     * 
        
+     *
+     * @param segment the target memory segment (at least 68 bytes)
+     */
+    public void writeTo(MemorySegment segment) {
+        // Offset 0: batch_size (i64)
+        segment.set(ValueLayout.JAVA_LONG, 0, (long) batchSize);
+        // Offset 8: target_partitions (i64)
+        segment.set(ValueLayout.JAVA_LONG, 8, (long) targetPartitions);
+        // Offset 16: min_skip_run_default (i64)
+        segment.set(ValueLayout.JAVA_LONG, 16, (long) minSkipRunDefault);
+        // Offset 24: min_skip_run_selectivity_threshold (f64)
+        segment.set(ValueLayout.JAVA_DOUBLE, 24, minSkipRunSelectivityThreshold);
+        // Offset 32: parquet_pushdown_filters (i32) — 0 = false, 1 = true
+        segment.set(ValueLayout.JAVA_INT, 32, parquetPushdownFilters ? 1 : 0);
+        // Offset 36: indexed_pushdown_filters (i32) — always 1 (hardcoded)
+        segment.set(ValueLayout.JAVA_INT, 36, 1);
+        // Offset 40: force_strategy (i32) — always -1 (None)
+        segment.set(ValueLayout.JAVA_INT, 40, -1);
+        // Offset 44: force_pushdown (i32) — always -1 (None)
+        segment.set(ValueLayout.JAVA_INT, 44, -1);
+        // Offset 48: cost_predicate (i32) — hardcoded 1
+        segment.set(ValueLayout.JAVA_INT, 48, 1);
+        // Offset 52: cost_collector (i32) — hardcoded 10
+        segment.set(ValueLayout.JAVA_INT, 52, 10);
+        // Offset 56: max_collector_parallelism (i32)
+        segment.set(ValueLayout.JAVA_INT, 56, maxCollectorParallelism);
+        // Offset 60: single_collector_strategy (i32)
+        segment.set(ValueLayout.JAVA_INT, 60, singleCollectorStrategy);
+        // Offset 64: tree_collector_strategy (i32)
+        segment.set(ValueLayout.JAVA_INT, 64, treeCollectorStrategy);
+    }
+
+    /**
+     * Builder for {@link WireConfigSnapshot}. All fields have sensible defaults
+     * matching the Rust {@code DatafusionQueryConfig::default()}.
+     */
+    public static final class Builder {
+        private int batchSize = 8192;
+        private int targetPartitions = 4;
+        private boolean parquetPushdownFilters = false;
+        private int minSkipRunDefault = 1024;
+        private double minSkipRunSelectivityThreshold = 0.03;
+        private int maxCollectorParallelism = 1;
+        private int singleCollectorStrategy = 2; // PageRangeSplit
+        private int treeCollectorStrategy = 1;   // TightenOuterBounds
+
+        private Builder() {}
+
+        public Builder batchSize(int batchSize) {
+            this.batchSize = batchSize;
+            return this;
+        }
+
+        public Builder targetPartitions(int targetPartitions) {
+            this.targetPartitions = targetPartitions;
+            return this;
+        }
+
+        public Builder parquetPushdownFilters(boolean parquetPushdownFilters) {
+            this.parquetPushdownFilters = parquetPushdownFilters;
+            return this;
+        }
+
+        public Builder minSkipRunDefault(int minSkipRunDefault) {
+            this.minSkipRunDefault = minSkipRunDefault;
+            return this;
+        }
+
+        public Builder minSkipRunSelectivityThreshold(double minSkipRunSelectivityThreshold) {
+            this.minSkipRunSelectivityThreshold = minSkipRunSelectivityThreshold;
+            return this;
+        }
+
+        public Builder maxCollectorParallelism(int maxCollectorParallelism) {
+            this.maxCollectorParallelism = maxCollectorParallelism;
+            return this;
+        }
+
+        public Builder singleCollectorStrategy(int singleCollectorStrategy) {
+            this.singleCollectorStrategy = singleCollectorStrategy;
+            return this;
+        }
+
+        public Builder treeCollectorStrategy(int treeCollectorStrategy) {
+            this.treeCollectorStrategy = treeCollectorStrategy;
+            return this;
+        }
+
+        public WireConfigSnapshot build() {
+            return new WireConfigSnapshot(this);
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
index ab63d52df1143..1bb1edf9cc59a 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
@@ -289,6 +289,7 @@ public final class NativeBridge {
                 ValueLayout.JAVA_LONG,
                 ValueLayout.ADDRESS,
                 ValueLayout.JAVA_LONG,
+                ValueLayout.JAVA_LONG,
                 ValueLayout.JAVA_LONG
             )
         );
@@ -303,7 +304,8 @@ public final class NativeBridge {
                 ValueLayout.JAVA_LONG,
                 ValueLayout.JAVA_LONG,
                 ValueLayout.JAVA_INT,
-                ValueLayout.JAVA_INT
+                ValueLayout.JAVA_INT,
+                ValueLayout.JAVA_LONG
             )
         );
 
@@ -787,13 +789,21 @@ public static long createCustomCacheManager() {
     /**
      * Creates a SessionContext with the default ListingTable registered.
      * Returns a tracked handle consumed by {@link #executeWithContextAsync}.
+     *
+     * @param queryConfigPtr pointer to a WireDatafusionQueryConfig struct, or 0 for fallback defaults
      */
-    public static SessionContextHandle createSessionContext(long readerPtr, long runtimePtr, String tableName, long contextId) {
+    public static SessionContextHandle createSessionContext(
+        long readerPtr,
+        long runtimePtr,
+        String tableName,
+        long contextId,
+        long queryConfigPtr
+    ) {
         NativeHandle.validatePointer(readerPtr, "reader");
         NativeHandle.validatePointer(runtimePtr, "runtime");
         try (var call = new NativeCall()) {
             var table = call.str(tableName);
-            long ptr = call.invoke(CREATE_SESSION_CONTEXT, readerPtr, runtimePtr, table.segment(), table.len(), contextId);
+            long ptr = call.invoke(CREATE_SESSION_CONTEXT, readerPtr, runtimePtr, table.segment(), table.len(), contextId, queryConfigPtr);
             return new SessionContextHandle(ptr);
         }
     }
@@ -802,6 +812,8 @@ public static SessionContextHandle createSessionContext(long readerPtr, long run
      * Creates a SessionContext configured for indexed execution with filter delegation.
      * Registers the delegated_predicate UDF and stores treeShape + delegatedPredicateCount
      * on the Rust handle for use during execution.
+     *
+     * @param queryConfigPtr pointer to a WireDatafusionQueryConfig struct, or 0 for fallback defaults
      */
     public static SessionContextHandle createSessionContextForIndexedExecution(
         long readerPtr,
@@ -809,7 +821,8 @@ public static SessionContextHandle createSessionContextForIndexedExecution(
         String tableName,
         long contextId,
         int treeShapeOrdinal,
-        int delegatedPredicateCount
+        int delegatedPredicateCount,
+        long queryConfigPtr
     ) {
         NativeHandle.validatePointer(readerPtr, "reader");
         NativeHandle.validatePointer(runtimePtr, "runtime");
@@ -823,7 +836,8 @@ public static SessionContextHandle createSessionContextForIndexedExecution(
                 table.len(),
                 contextId,
                 treeShapeOrdinal,
-                delegatedPredicateCount
+                delegatedPredicateCount,
+                queryConfigPtr
             );
             return new SessionContextHandle(ptr);
         }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextConfig.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextConfig.java
new file mode 100644
index 0000000000000..a9c8c4471a8fa
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/SessionContextConfig.java
@@ -0,0 +1,28 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.nativelib;
+
+import org.opensearch.be.datafusion.WireConfigSnapshot;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * Immutable configuration record for creating a native SessionContext via
+ * {@link NativeBridge#createSessionContext(long, long, String, long, long)}.
+ *
+ * @param readerPtr   pointer to the native DataFusion reader (shard view)
+ * @param runtimePtr  pointer to the native DataFusion runtime
+ * @param tableName   logical table name to register in the session context
+ * @param contextId   query/task context identifier (0 if none)
+ * @param queryConfig query config snapshot to pass to native
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public record SessionContextConfig(long readerPtr, long runtimePtr, String tableName, long contextId, WireConfigSnapshot queryConfig) {
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java
index 23fa446681911..ae13def41397e 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionNativeBridgeTests.java
@@ -15,6 +15,8 @@
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.concurrent.CompletableFuture;
@@ -81,12 +83,20 @@ public void testSessionContextCreationAndTableRegistration() throws Exception {
         ReaderHandle readerHandle = new ReaderHandle(dataDir.toString(), new String[] { "test.parquet" });
 
         // Create session context with table registered
+        long queryConfigPtr;
+        Arena arena = Arena.ofConfined();
+        MemorySegment configSegment = arena.allocate(WireConfigSnapshot.BYTE_SIZE);
+        WireConfigSnapshot.builder().build().writeTo(configSegment);
+        queryConfigPtr = configSegment.address();
+
         SessionContextHandle sessionCtx = NativeBridge.createSessionContext(
             readerHandle.getPointer(),
             runtimeHandle.get(),
             "test_table",
-            0L
+            0L,
+            queryConfigPtr
         );
+        arena.close();
         assertTrue("SessionContext pointer should be non-zero", sessionCtx.getPointer() != 0);
 
         // Execute a simple query to verify the session context is properly configured
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java
index a9ad8834fe6e5..0e2120293c000 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionPluginSettingsTests.java
@@ -12,6 +12,8 @@
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * Verifies the settings declared by {@link DataFusionPlugin} — in particular that
@@ -60,4 +62,38 @@ public void testUpdateMemoryPoolLimitBeforeServiceStartDoesNotThrow() {
             throw new AssertionError(e);
         }
     }
+
+    public void testGetSettingsReturnsAllIndexedSettings() {
+        try (DataFusionPlugin plugin = new DataFusionPlugin()) {
+            List> settings = plugin.getSettings();
+            Set settingKeys = settings.stream().map(Setting::getKey).collect(Collectors.toSet());
+
+            assertTrue(settingKeys.contains("datafusion.indexed.batch_size"));
+            assertTrue(settingKeys.contains("datafusion.indexed.parquet_pushdown_filters"));
+            assertTrue(settingKeys.contains("datafusion.indexed.min_skip_run_default"));
+            assertTrue(settingKeys.contains("datafusion.indexed.min_skip_run_selectivity_threshold"));
+            assertTrue(settingKeys.contains("datafusion.indexed.single_collector_strategy"));
+            assertTrue(settingKeys.contains("datafusion.indexed.tree_collector_strategy"));
+            assertTrue(settingKeys.contains("datafusion.indexed.max_collector_parallelism"));
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public void testGetSettingsReturnsTotalExpectedCount() {
+        try (DataFusionPlugin plugin = new DataFusionPlugin()) {
+            List> settings = plugin.getSettings();
+            assertEquals(16, settings.size());
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public void testDatafusionSettingsIsNullBeforeCreateComponents() {
+        try (DataFusionPlugin plugin = new DataFusionPlugin()) {
+            assertNull(plugin.getDatafusionSettings());
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java
index a9f1b13eeadd9..4b024ed0d49cf 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionQueryExecutionTests.java
@@ -21,6 +21,8 @@
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -38,6 +40,8 @@ public class DataFusionQueryExecutionTests extends OpenSearchTestCase {
 
     private NativeRuntimeHandle runtimeHandle;
     private ReaderHandle readerHandle;
+    private Arena configArena;
+    private long queryConfigPtr;
 
     @Override
     public void setUp() throws Exception {
@@ -52,10 +56,16 @@ public void setUp() throws Exception {
         Path testParquet = Path.of(getClass().getClassLoader().getResource("test.parquet").toURI());
         Files.copy(testParquet, dataDir.resolve("test.parquet"));
         readerHandle = new ReaderHandle(dataDir.toString(), new String[] { "test.parquet" });
+
+        configArena = Arena.ofConfined();
+        MemorySegment configSegment = configArena.allocate(WireConfigSnapshot.BYTE_SIZE);
+        WireConfigSnapshot.builder().build().writeTo(configSegment);
+        queryConfigPtr = configSegment.address();
     }
 
     @Override
     public void tearDown() throws Exception {
+        configArena.close();
         readerHandle.close();
         runtimeHandle.close();
         super.tearDown();
@@ -103,7 +113,7 @@ private List executeQuery(String sql) {
                 substraitBytes,
                 runtimeHandle.get(),
                 0L,
-                0L,
+                queryConfigPtr,
                 listener
             )
         );
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java
index 574bfb273ea97..247d1e55dadfd 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java
@@ -16,6 +16,8 @@
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Iterator;
@@ -31,6 +33,8 @@ public class DatafusionResultStreamTests extends OpenSearchTestCase {
     private ReaderHandle readerHandle;
     private NativeRuntimeHandle runtimeHandle;
     private RootAllocator testRootAllocator;
+    private Arena configArena;
+    private long queryConfigPtr;
     private final java.util.List allocatorsToClose = new java.util.ArrayList<>();
 
     @Override
@@ -46,10 +50,16 @@ public void setUp() throws Exception {
         Path testParquet = Path.of(getClass().getClassLoader().getResource("test.parquet").toURI());
         Files.copy(testParquet, dataDir.resolve("test.parquet"));
         readerHandle = new ReaderHandle(dataDir.toString(), new String[] { "test.parquet" });
+
+        configArena = Arena.ofConfined();
+        MemorySegment configSegment = configArena.allocate(WireConfigSnapshot.BYTE_SIZE);
+        WireConfigSnapshot.builder().build().writeTo(configSegment);
+        queryConfigPtr = configSegment.address();
     }
 
     @Override
     public void tearDown() throws Exception {
+        configArena.close();
         readerHandle.close();
         runtimeHandle.close();
         // Caller owns child allocators now (see DatafusionResultStream.close javadoc).
@@ -171,7 +181,7 @@ public void testNativeQueryFailureDoesNotLeak() {
             new byte[] { 0, 1, 2 },
             runtimeHandle.get(),
             0L,
-            0L,
+            queryConfigPtr,
             new ActionListener<>() {
                 @Override
                 public void onResponse(Long ptr) {
@@ -214,7 +224,7 @@ public void testCloseAfterNativeStreamNextFailure() throws Exception {
             substrait,
             tempRuntime.get(),
             0L,
-            0L,
+            queryConfigPtr,
             new ActionListener<>() {
                 @Override
                 public void onResponse(Long p) {
@@ -267,7 +277,7 @@ private DatafusionResultStream createStream(String sql) {
             substrait,
             runtimeHandle.get(),
             0L,
-            0L,
+            queryConfigPtr,
             new ActionListener<>() {
                 @Override
                 public void onResponse(Long ptr) {
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java
index 21fea0bc58494..3b69dde787dc6 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSearchExecEngineTests.java
@@ -17,6 +17,8 @@
 import org.opensearch.be.datafusion.nativelib.SessionContextHandle;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -143,12 +145,17 @@ private DatafusionReader createReader() {
     private ShardScanExecutionContext createExecutionContext(String tableName, byte[] substrait, DatafusionContext dfContext) {
         ShardScanExecutionContext execCtx = new ShardScanExecutionContext(tableName, null, null);
         execCtx.setFragmentBytes(substrait);
+        Arena arena = Arena.ofConfined();
+        MemorySegment configSegment = arena.allocate(WireConfigSnapshot.BYTE_SIZE);
+        WireConfigSnapshot.builder().build().writeTo(configSegment);
         SessionContextHandle sessionCtxHandle = NativeBridge.createSessionContext(
             readerHandle.getPointer(),
             runtimeHandle.get(),
             tableName,
-            0L
+            0L,
+            configSegment.address()
         );
+        arena.close();
         dfContext.setSessionContextHandle(sessionCtxHandle);
         return execCtx;
     }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsPropertyTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsPropertyTests.java
new file mode 100644
index 0000000000000..c215c3e02619c
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsPropertyTests.java
@@ -0,0 +1,212 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.search.SearchService;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class DatafusionSettingsPropertyTests extends OpenSearchTestCase {
+
+    private static final int ITERATIONS = 200;
+    private static final String[] STRATEGIES = { "full_range", "tighten_outer_bounds", "page_range_split" };
+
+    private ClusterSettings createClusterSettings() {
+        Set> settingsSet = new HashSet<>(DatafusionSettings.ALL_SETTINGS);
+        settingsSet.add(SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING);
+        settingsSet.add(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE);
+        return new ClusterSettings(Settings.EMPTY, settingsSet);
+    }
+
+    public void testSnapshotUpdateConsistencyProperty() {
+        for (int i = 0; i < ITERATIONS; i++) {
+            DatafusionSettings datafusionSettings = new DatafusionSettings(Settings.EMPTY);
+            ClusterSettings clusterSettings = createClusterSettings();
+            datafusionSettings.registerListeners(clusterSettings);
+
+            WireConfigSnapshot before = datafusionSettings.getSnapshot();
+
+            int settingIndex = randomIntBetween(0, 8);
+            Settings newSettings;
+
+            switch (settingIndex) {
+                case 0: // batch_size
+                    int newBatchSize = randomIntBetween(1, 1_000_000);
+                    newSettings = Settings.builder().put("datafusion.indexed.batch_size", newBatchSize).build();
+                    clusterSettings.applySettings(newSettings);
+                    WireConfigSnapshot afterBatch = datafusionSettings.getSnapshot();
+                    assertEquals(newBatchSize, afterBatch.batchSize());
+                    assertEquals(before.targetPartitions(), afterBatch.targetPartitions());
+                    assertEquals(before.parquetPushdownFilters(), afterBatch.parquetPushdownFilters());
+                    assertEquals(before.minSkipRunDefault(), afterBatch.minSkipRunDefault());
+                    assertEquals(before.minSkipRunSelectivityThreshold(), afterBatch.minSkipRunSelectivityThreshold(), 0.0);
+                    assertEquals(before.singleCollectorStrategy(), afterBatch.singleCollectorStrategy());
+                    assertEquals(before.treeCollectorStrategy(), afterBatch.treeCollectorStrategy());
+                    assertEquals(before.maxCollectorParallelism(), afterBatch.maxCollectorParallelism());
+                    break;
+
+                case 1: // parquet_pushdown_filters
+                    boolean newPushdown = before.parquetPushdownFilters() == false;
+                    newSettings = Settings.builder().put("datafusion.indexed.parquet_pushdown_filters", newPushdown).build();
+                    clusterSettings.applySettings(newSettings);
+                    WireConfigSnapshot afterPushdown = datafusionSettings.getSnapshot();
+                    assertEquals(newPushdown, afterPushdown.parquetPushdownFilters());
+                    assertEquals(before.batchSize(), afterPushdown.batchSize());
+                    assertEquals(before.targetPartitions(), afterPushdown.targetPartitions());
+                    assertEquals(before.minSkipRunDefault(), afterPushdown.minSkipRunDefault());
+                    assertEquals(before.minSkipRunSelectivityThreshold(), afterPushdown.minSkipRunSelectivityThreshold(), 0.0);
+                    assertEquals(before.singleCollectorStrategy(), afterPushdown.singleCollectorStrategy());
+                    assertEquals(before.treeCollectorStrategy(), afterPushdown.treeCollectorStrategy());
+                    assertEquals(before.maxCollectorParallelism(), afterPushdown.maxCollectorParallelism());
+                    break;
+
+                case 2: // min_skip_run_default
+                    int newMinSkipRun = randomIntBetween(1, 100_000);
+                    newSettings = Settings.builder().put("datafusion.indexed.min_skip_run_default", newMinSkipRun).build();
+                    clusterSettings.applySettings(newSettings);
+                    WireConfigSnapshot afterSkipRun = datafusionSettings.getSnapshot();
+                    assertEquals(newMinSkipRun, afterSkipRun.minSkipRunDefault());
+                    assertEquals(before.batchSize(), afterSkipRun.batchSize());
+                    assertEquals(before.targetPartitions(), afterSkipRun.targetPartitions());
+                    assertEquals(before.parquetPushdownFilters(), afterSkipRun.parquetPushdownFilters());
+                    assertEquals(before.minSkipRunSelectivityThreshold(), afterSkipRun.minSkipRunSelectivityThreshold(), 0.0);
+                    assertEquals(before.singleCollectorStrategy(), afterSkipRun.singleCollectorStrategy());
+                    assertEquals(before.treeCollectorStrategy(), afterSkipRun.treeCollectorStrategy());
+                    assertEquals(before.maxCollectorParallelism(), afterSkipRun.maxCollectorParallelism());
+                    break;
+
+                case 3: // min_skip_run_selectivity_threshold
+                    double newThreshold = randomDoubleBetween(0.0, 1.0, true);
+                    newSettings = Settings.builder().put("datafusion.indexed.min_skip_run_selectivity_threshold", newThreshold).build();
+                    clusterSettings.applySettings(newSettings);
+                    WireConfigSnapshot afterThreshold = datafusionSettings.getSnapshot();
+                    assertEquals(newThreshold, afterThreshold.minSkipRunSelectivityThreshold(), 1e-15);
+                    assertEquals(before.batchSize(), afterThreshold.batchSize());
+                    assertEquals(before.targetPartitions(), afterThreshold.targetPartitions());
+                    assertEquals(before.parquetPushdownFilters(), afterThreshold.parquetPushdownFilters());
+                    assertEquals(before.minSkipRunDefault(), afterThreshold.minSkipRunDefault());
+                    assertEquals(before.singleCollectorStrategy(), afterThreshold.singleCollectorStrategy());
+                    assertEquals(before.treeCollectorStrategy(), afterThreshold.treeCollectorStrategy());
+                    assertEquals(before.maxCollectorParallelism(), afterThreshold.maxCollectorParallelism());
+                    break;
+
+                case 4: // single_collector_strategy
+                    String newSingle = STRATEGIES[randomIntBetween(0, 2)];
+                    newSettings = Settings.builder().put("datafusion.indexed.single_collector_strategy", newSingle).build();
+                    clusterSettings.applySettings(newSettings);
+                    WireConfigSnapshot afterSingle = datafusionSettings.getSnapshot();
+                    assertEquals(DatafusionSettings.strategyToWireValue(newSingle), afterSingle.singleCollectorStrategy());
+                    assertEquals(before.batchSize(), afterSingle.batchSize());
+                    assertEquals(before.targetPartitions(), afterSingle.targetPartitions());
+                    assertEquals(before.parquetPushdownFilters(), afterSingle.parquetPushdownFilters());
+                    assertEquals(before.minSkipRunDefault(), afterSingle.minSkipRunDefault());
+                    assertEquals(before.minSkipRunSelectivityThreshold(), afterSingle.minSkipRunSelectivityThreshold(), 0.0);
+                    assertEquals(before.treeCollectorStrategy(), afterSingle.treeCollectorStrategy());
+                    assertEquals(before.maxCollectorParallelism(), afterSingle.maxCollectorParallelism());
+                    break;
+
+                case 5: // tree_collector_strategy
+                    String newTree = STRATEGIES[randomIntBetween(0, 2)];
+                    newSettings = Settings.builder().put("datafusion.indexed.tree_collector_strategy", newTree).build();
+                    clusterSettings.applySettings(newSettings);
+                    WireConfigSnapshot afterTree = datafusionSettings.getSnapshot();
+                    assertEquals(DatafusionSettings.strategyToWireValue(newTree), afterTree.treeCollectorStrategy());
+                    assertEquals(before.batchSize(), afterTree.batchSize());
+                    assertEquals(before.targetPartitions(), afterTree.targetPartitions());
+                    assertEquals(before.parquetPushdownFilters(), afterTree.parquetPushdownFilters());
+                    assertEquals(before.minSkipRunDefault(), afterTree.minSkipRunDefault());
+                    assertEquals(before.minSkipRunSelectivityThreshold(), afterTree.minSkipRunSelectivityThreshold(), 0.0);
+                    assertEquals(before.singleCollectorStrategy(), afterTree.singleCollectorStrategy());
+                    assertEquals(before.maxCollectorParallelism(), afterTree.maxCollectorParallelism());
+                    break;
+
+                case 6: // max_collector_parallelism
+                    int newMaxParallelism = randomIntBetween(1, 64);
+                    newSettings = Settings.builder().put("datafusion.indexed.max_collector_parallelism", newMaxParallelism).build();
+                    clusterSettings.applySettings(newSettings);
+                    WireConfigSnapshot afterParallelism = datafusionSettings.getSnapshot();
+                    assertEquals(newMaxParallelism, afterParallelism.maxCollectorParallelism());
+                    assertEquals(before.batchSize(), afterParallelism.batchSize());
+                    assertEquals(before.targetPartitions(), afterParallelism.targetPartitions());
+                    assertEquals(before.parquetPushdownFilters(), afterParallelism.parquetPushdownFilters());
+                    assertEquals(before.minSkipRunDefault(), afterParallelism.minSkipRunDefault());
+                    assertEquals(before.minSkipRunSelectivityThreshold(), afterParallelism.minSkipRunSelectivityThreshold(), 0.0);
+                    assertEquals(before.singleCollectorStrategy(), afterParallelism.singleCollectorStrategy());
+                    assertEquals(before.treeCollectorStrategy(), afterParallelism.treeCollectorStrategy());
+                    break;
+
+                case 7: // max_slice_count
+                    int newSliceCount = randomIntBetween(1, 32);
+                    newSettings = Settings.builder().put("search.concurrent.max_slice_count", newSliceCount).build();
+                    clusterSettings.applySettings(newSettings);
+                    WireConfigSnapshot afterSlice = datafusionSettings.getSnapshot();
+                    assertEquals(Math.min(newSliceCount, Runtime.getRuntime().availableProcessors()), afterSlice.targetPartitions());
+                    assertEquals(before.batchSize(), afterSlice.batchSize());
+                    assertEquals(before.parquetPushdownFilters(), afterSlice.parquetPushdownFilters());
+                    assertEquals(before.minSkipRunDefault(), afterSlice.minSkipRunDefault());
+                    assertEquals(before.minSkipRunSelectivityThreshold(), afterSlice.minSkipRunSelectivityThreshold(), 0.0);
+                    assertEquals(before.singleCollectorStrategy(), afterSlice.singleCollectorStrategy());
+                    assertEquals(before.treeCollectorStrategy(), afterSlice.treeCollectorStrategy());
+                    assertEquals(before.maxCollectorParallelism(), afterSlice.maxCollectorParallelism());
+                    break;
+
+                case 8: // concurrent_search_mode
+                    newSettings = Settings.builder().put("search.concurrent_segment_search.mode", "none").build();
+                    clusterSettings.applySettings(newSettings);
+                    WireConfigSnapshot afterMode = datafusionSettings.getSnapshot();
+                    assertEquals(1, afterMode.targetPartitions());
+                    assertEquals(before.batchSize(), afterMode.batchSize());
+                    assertEquals(before.parquetPushdownFilters(), afterMode.parquetPushdownFilters());
+                    assertEquals(before.minSkipRunDefault(), afterMode.minSkipRunDefault());
+                    assertEquals(before.minSkipRunSelectivityThreshold(), afterMode.minSkipRunSelectivityThreshold(), 0.0);
+                    assertEquals(before.singleCollectorStrategy(), afterMode.singleCollectorStrategy());
+                    assertEquals(before.treeCollectorStrategy(), afterMode.treeCollectorStrategy());
+                    assertEquals(before.maxCollectorParallelism(), afterMode.maxCollectorParallelism());
+                    break;
+
+                default:
+                    fail("Unexpected setting index: " + settingIndex);
+            }
+        }
+    }
+
+    public void testSequentialUpdatesAccumulateCorrectly() {
+        for (int i = 0; i < ITERATIONS; i++) {
+            DatafusionSettings datafusionSettings = new DatafusionSettings(Settings.EMPTY);
+            ClusterSettings clusterSettings = createClusterSettings();
+            datafusionSettings.registerListeners(clusterSettings);
+
+            int newBatchSize = randomIntBetween(1, 1_000_000);
+            String newSingleStrategy = STRATEGIES[randomIntBetween(0, 2)];
+            double newThreshold = randomDoubleBetween(0.0, 1.0, true);
+
+            clusterSettings.applySettings(
+                Settings.builder()
+                    .put("datafusion.indexed.batch_size", newBatchSize)
+                    .put("datafusion.indexed.single_collector_strategy", newSingleStrategy)
+                    .put("datafusion.indexed.min_skip_run_selectivity_threshold", newThreshold)
+                    .build()
+            );
+
+            WireConfigSnapshot finalSnapshot = datafusionSettings.getSnapshot();
+
+            assertEquals(newBatchSize, finalSnapshot.batchSize());
+            assertEquals(DatafusionSettings.strategyToWireValue(newSingleStrategy), finalSnapshot.singleCollectorStrategy());
+            assertEquals(newThreshold, finalSnapshot.minSkipRunSelectivityThreshold(), 1e-15);
+            assertEquals(false, finalSnapshot.parquetPushdownFilters());
+            assertEquals(1024, finalSnapshot.minSkipRunDefault());
+            assertEquals(1, finalSnapshot.maxCollectorParallelism());
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsTests.java
new file mode 100644
index 0000000000000..798e8bc8eb209
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionSettingsTests.java
@@ -0,0 +1,162 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.search.SearchService;
+import org.opensearch.test.OpenSearchTestCase;
+
+public class DatafusionSettingsTests extends OpenSearchTestCase {
+
+    private static final int DEFAULT_PARALLELISM = Math.max(1, Math.min(Runtime.getRuntime().availableProcessors() / 2, 4));
+
+    public void testBatchSizeSettingDefinition() {
+        assertEquals("datafusion.indexed.batch_size", DatafusionSettings.INDEXED_BATCH_SIZE.getKey());
+        assertEquals(Integer.valueOf(8192), DatafusionSettings.INDEXED_BATCH_SIZE.get(Settings.EMPTY));
+        assertTrue(DatafusionSettings.INDEXED_BATCH_SIZE.isDynamic());
+        assertTrue(DatafusionSettings.INDEXED_BATCH_SIZE.hasNodeScope());
+    }
+
+    public void testParquetPushdownFiltersSettingDefinition() {
+        assertEquals("datafusion.indexed.parquet_pushdown_filters", DatafusionSettings.INDEXED_PARQUET_PUSHDOWN_FILTERS.getKey());
+        assertEquals(Boolean.FALSE, DatafusionSettings.INDEXED_PARQUET_PUSHDOWN_FILTERS.get(Settings.EMPTY));
+        assertTrue(DatafusionSettings.INDEXED_PARQUET_PUSHDOWN_FILTERS.isDynamic());
+        assertTrue(DatafusionSettings.INDEXED_PARQUET_PUSHDOWN_FILTERS.hasNodeScope());
+    }
+
+    public void testMinSkipRunDefaultSettingDefinition() {
+        assertEquals("datafusion.indexed.min_skip_run_default", DatafusionSettings.INDEXED_MIN_SKIP_RUN_DEFAULT.getKey());
+        assertEquals(Integer.valueOf(1024), DatafusionSettings.INDEXED_MIN_SKIP_RUN_DEFAULT.get(Settings.EMPTY));
+        assertTrue(DatafusionSettings.INDEXED_MIN_SKIP_RUN_DEFAULT.isDynamic());
+        assertTrue(DatafusionSettings.INDEXED_MIN_SKIP_RUN_DEFAULT.hasNodeScope());
+    }
+
+    public void testMinSkipRunSelectivityThresholdSettingDefinition() {
+        assertEquals(
+            "datafusion.indexed.min_skip_run_selectivity_threshold",
+            DatafusionSettings.INDEXED_MIN_SKIP_RUN_SELECTIVITY_THRESHOLD.getKey()
+        );
+        assertEquals(0.03, DatafusionSettings.INDEXED_MIN_SKIP_RUN_SELECTIVITY_THRESHOLD.get(Settings.EMPTY), 1e-15);
+        assertTrue(DatafusionSettings.INDEXED_MIN_SKIP_RUN_SELECTIVITY_THRESHOLD.isDynamic());
+        assertTrue(DatafusionSettings.INDEXED_MIN_SKIP_RUN_SELECTIVITY_THRESHOLD.hasNodeScope());
+    }
+
+    public void testSingleCollectorStrategySettingDefinition() {
+        assertEquals("datafusion.indexed.single_collector_strategy", DatafusionSettings.INDEXED_SINGLE_COLLECTOR_STRATEGY.getKey());
+        assertEquals("page_range_split", DatafusionSettings.INDEXED_SINGLE_COLLECTOR_STRATEGY.get(Settings.EMPTY));
+        assertTrue(DatafusionSettings.INDEXED_SINGLE_COLLECTOR_STRATEGY.isDynamic());
+        assertTrue(DatafusionSettings.INDEXED_SINGLE_COLLECTOR_STRATEGY.hasNodeScope());
+    }
+
+    public void testTreeCollectorStrategySettingDefinition() {
+        assertEquals("datafusion.indexed.tree_collector_strategy", DatafusionSettings.INDEXED_TREE_COLLECTOR_STRATEGY.getKey());
+        assertEquals("tighten_outer_bounds", DatafusionSettings.INDEXED_TREE_COLLECTOR_STRATEGY.get(Settings.EMPTY));
+        assertTrue(DatafusionSettings.INDEXED_TREE_COLLECTOR_STRATEGY.isDynamic());
+        assertTrue(DatafusionSettings.INDEXED_TREE_COLLECTOR_STRATEGY.hasNodeScope());
+    }
+
+    public void testMaxCollectorParallelismSettingDefinition() {
+        assertEquals("datafusion.indexed.max_collector_parallelism", DatafusionSettings.INDEXED_MAX_COLLECTOR_PARALLELISM.getKey());
+        assertEquals(Integer.valueOf(1), DatafusionSettings.INDEXED_MAX_COLLECTOR_PARALLELISM.get(Settings.EMPTY));
+        assertTrue(DatafusionSettings.INDEXED_MAX_COLLECTOR_PARALLELISM.isDynamic());
+        assertTrue(DatafusionSettings.INDEXED_MAX_COLLECTOR_PARALLELISM.hasNodeScope());
+    }
+
+    public void testAllSettingsContainsAllExpectedSettings() {
+        assertEquals(16, DatafusionSettings.ALL_SETTINGS.size());
+        assertTrue(DatafusionSettings.ALL_SETTINGS.contains(DatafusionSettings.INDEXED_BATCH_SIZE));
+        assertTrue(DatafusionSettings.ALL_SETTINGS.contains(DatafusionSettings.INDEXED_PARQUET_PUSHDOWN_FILTERS));
+        assertTrue(DatafusionSettings.ALL_SETTINGS.contains(DatafusionSettings.INDEXED_MIN_SKIP_RUN_DEFAULT));
+        assertTrue(DatafusionSettings.ALL_SETTINGS.contains(DatafusionSettings.INDEXED_MIN_SKIP_RUN_SELECTIVITY_THRESHOLD));
+        assertTrue(DatafusionSettings.ALL_SETTINGS.contains(DatafusionSettings.INDEXED_SINGLE_COLLECTOR_STRATEGY));
+        assertTrue(DatafusionSettings.ALL_SETTINGS.contains(DatafusionSettings.INDEXED_TREE_COLLECTOR_STRATEGY));
+        assertTrue(DatafusionSettings.ALL_SETTINGS.contains(DatafusionSettings.INDEXED_MAX_COLLECTOR_PARALLELISM));
+    }
+
+    public void testDefaultSnapshotValuesMatchDefaults() {
+        DatafusionSettings ds = new DatafusionSettings(Settings.EMPTY);
+        WireConfigSnapshot snapshot = ds.getSnapshot();
+
+        assertEquals(8192, snapshot.batchSize());
+        assertEquals(false, snapshot.parquetPushdownFilters());
+        assertEquals(1024, snapshot.minSkipRunDefault());
+        assertEquals(0.03, snapshot.minSkipRunSelectivityThreshold(), 1e-15);
+        assertEquals(2, snapshot.singleCollectorStrategy()); // page_range_split
+        assertEquals(1, snapshot.treeCollectorStrategy()); // tighten_outer_bounds
+        assertEquals(1, snapshot.maxCollectorParallelism());
+        assertEquals(DEFAULT_PARALLELISM, snapshot.targetPartitions());
+    }
+
+    public void testTargetPartitionsPassthroughWhenNonZero() {
+        Settings settings = Settings.builder()
+            .put(SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING.getKey(), 8)
+            .build();
+        DatafusionSettings ds = new DatafusionSettings(settings);
+
+        assertEquals(Math.min(8, Runtime.getRuntime().availableProcessors()), ds.getSnapshot().targetPartitions());
+    }
+
+    public void testTargetPartitionsFallbackWhenZero() {
+        DatafusionSettings ds = new DatafusionSettings(Settings.EMPTY);
+
+        assertEquals(DEFAULT_PARALLELISM, ds.getSnapshot().targetPartitions());
+    }
+
+    public void testTargetPartitionsForcedToOneWhenModeNone() {
+        Settings settings = Settings.builder()
+            .put(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), "none")
+            .put(SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING.getKey(), 16)
+            .build();
+        DatafusionSettings ds = new DatafusionSettings(settings);
+
+        assertEquals(1, ds.getSnapshot().targetPartitions());
+    }
+
+    public void testTargetPartitionsCappedAtAvailableProcessors() {
+        int processors = Runtime.getRuntime().availableProcessors();
+        Settings settings = Settings.builder()
+            .put(SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING.getKey(), processors + 10)
+            .build();
+        DatafusionSettings ds = new DatafusionSettings(settings);
+
+        assertEquals(processors, ds.getSnapshot().targetPartitions());
+    }
+
+    public void testStrategyToWireValueMapping() {
+        assertEquals(0, DatafusionSettings.strategyToWireValue("full_range"));
+        assertEquals(1, DatafusionSettings.strategyToWireValue("tighten_outer_bounds"));
+        assertEquals(2, DatafusionSettings.strategyToWireValue("page_range_split"));
+        expectThrows(IllegalArgumentException.class, () -> DatafusionSettings.strategyToWireValue("invalid"));
+    }
+
+    public void testBatchSizeZeroIsRejected() {
+        Settings settings = Settings.builder().put("datafusion.indexed.batch_size", 0).build();
+        expectThrows(IllegalArgumentException.class, () -> DatafusionSettings.INDEXED_BATCH_SIZE.get(settings));
+    }
+
+    public void testMaxCollectorParallelismNegativeIsRejected() {
+        Settings settings = Settings.builder().put("datafusion.indexed.max_collector_parallelism", -1).build();
+        expectThrows(IllegalArgumentException.class, () -> DatafusionSettings.INDEXED_MAX_COLLECTOR_PARALLELISM.get(settings));
+    }
+
+    public void testSelectivityThresholdAboveBoundIsRejected() {
+        Settings settings = Settings.builder().put("datafusion.indexed.min_skip_run_selectivity_threshold", 1.1).build();
+        expectThrows(IllegalArgumentException.class, () -> DatafusionSettings.INDEXED_MIN_SKIP_RUN_SELECTIVITY_THRESHOLD.get(settings));
+    }
+
+    public void testInvalidSingleCollectorStrategyIsRejected() {
+        Settings settings = Settings.builder().put("datafusion.indexed.single_collector_strategy", "bogus").build();
+        expectThrows(IllegalArgumentException.class, () -> DatafusionSettings.INDEXED_SINGLE_COLLECTOR_STRATEGY.get(settings));
+    }
+
+    public void testInvalidTreeCollectorStrategyIsRejected() {
+        Settings settings = Settings.builder().put("datafusion.indexed.tree_collector_strategy", "bogus").build();
+        expectThrows(IllegalArgumentException.class, () -> DatafusionSettings.INDEXED_TREE_COLLECTOR_STRATEGY.get(settings));
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/WireConfigSnapshotTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/WireConfigSnapshotTests.java
new file mode 100644
index 0000000000000..45cd4bc71f900
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/WireConfigSnapshotTests.java
@@ -0,0 +1,112 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.ValueLayout;
+
+public class WireConfigSnapshotTests extends OpenSearchTestCase {
+
+    public void testByteSizeEquals68() {
+        assertEquals(68L, WireConfigSnapshot.BYTE_SIZE);
+    }
+
+    public void testWriteToWritesCorrectValuesAtCorrectOffsets() {
+        WireConfigSnapshot snapshot = WireConfigSnapshot.builder()
+            .batchSize(8192)
+            .targetPartitions(4)
+            .parquetPushdownFilters(true)
+            .minSkipRunDefault(1024)
+            .minSkipRunSelectivityThreshold(0.03)
+            .maxCollectorParallelism(4)
+            .singleCollectorStrategy(2)
+            .treeCollectorStrategy(1)
+            .build();
+
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment segment = arena.allocate(WireConfigSnapshot.BYTE_SIZE);
+            snapshot.writeTo(segment);
+
+            assertEquals(8192L, segment.get(ValueLayout.JAVA_LONG, 0));
+            assertEquals(4L, segment.get(ValueLayout.JAVA_LONG, 8));
+            assertEquals(1024L, segment.get(ValueLayout.JAVA_LONG, 16));
+            assertEquals(0.03, segment.get(ValueLayout.JAVA_DOUBLE, 24), 1e-15);
+            assertEquals(1, segment.get(ValueLayout.JAVA_INT, 32)); // parquet_pushdown = true
+            assertEquals(4, segment.get(ValueLayout.JAVA_INT, 56)); // max_collector_parallelism
+            assertEquals(2, segment.get(ValueLayout.JAVA_INT, 60)); // single_collector_strategy
+            assertEquals(1, segment.get(ValueLayout.JAVA_INT, 64)); // tree_collector_strategy
+        }
+    }
+
+    public void testWriteToWritesParquetPushdownFalseAsZero() {
+        WireConfigSnapshot snapshot = WireConfigSnapshot.builder().parquetPushdownFilters(false).build();
+
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment segment = arena.allocate(WireConfigSnapshot.BYTE_SIZE);
+            snapshot.writeTo(segment);
+
+            assertEquals(0, segment.get(ValueLayout.JAVA_INT, 32));
+        }
+    }
+
+    public void testHardcodedFieldsAreWrittenCorrectly() {
+        WireConfigSnapshot snapshot = WireConfigSnapshot.builder().batchSize(16384).targetPartitions(8).maxCollectorParallelism(6).build();
+
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment segment = arena.allocate(WireConfigSnapshot.BYTE_SIZE);
+            snapshot.writeTo(segment);
+
+            assertEquals(1, segment.get(ValueLayout.JAVA_INT, 36));  // indexed_pushdown_filters
+            assertEquals(-1, segment.get(ValueLayout.JAVA_INT, 40)); // force_strategy
+            assertEquals(-1, segment.get(ValueLayout.JAVA_INT, 44)); // force_pushdown
+            assertEquals(1, segment.get(ValueLayout.JAVA_INT, 48));  // cost_predicate (hardcoded)
+            assertEquals(10, segment.get(ValueLayout.JAVA_INT, 52)); // cost_collector (hardcoded)
+        }
+    }
+
+    public void testBuilderDefaultsMatchExpected() {
+        WireConfigSnapshot snapshot = WireConfigSnapshot.builder().build();
+
+        assertEquals(8192, snapshot.batchSize());
+        assertEquals(4, snapshot.targetPartitions());
+        assertEquals(false, snapshot.parquetPushdownFilters());
+        assertEquals(1024, snapshot.minSkipRunDefault());
+        assertEquals(0.03, snapshot.minSkipRunSelectivityThreshold(), 1e-15);
+        assertEquals(1, snapshot.maxCollectorParallelism());
+        assertEquals(2, snapshot.singleCollectorStrategy());  // page_range_split
+        assertEquals(1, snapshot.treeCollectorStrategy());    // tighten_outer_bounds
+    }
+
+    public void testBuilderCopyPreservesAllFields() {
+        WireConfigSnapshot original = WireConfigSnapshot.builder()
+            .batchSize(4096)
+            .targetPartitions(16)
+            .parquetPushdownFilters(true)
+            .minSkipRunDefault(512)
+            .minSkipRunSelectivityThreshold(0.5)
+            .maxCollectorParallelism(8)
+            .singleCollectorStrategy(0)
+            .treeCollectorStrategy(2)
+            .build();
+
+        WireConfigSnapshot copy = WireConfigSnapshot.builder(original).build();
+
+        assertEquals(original.batchSize(), copy.batchSize());
+        assertEquals(original.targetPartitions(), copy.targetPartitions());
+        assertEquals(original.parquetPushdownFilters(), copy.parquetPushdownFilters());
+        assertEquals(original.minSkipRunDefault(), copy.minSkipRunDefault());
+        assertEquals(original.minSkipRunSelectivityThreshold(), copy.minSkipRunSelectivityThreshold(), 0.0);
+        assertEquals(original.maxCollectorParallelism(), copy.maxCollectorParallelism());
+        assertEquals(original.singleCollectorStrategy(), copy.singleCollectorStrategy());
+        assertEquals(original.treeCollectorStrategy(), copy.treeCollectorStrategy());
+    }
+}

From d09b185b5affedd10dcb71c1fd102d978417b78a Mon Sep 17 00:00:00 2001
From: Eric Wei 
Date: Fri, 8 May 2026 12:09:17 -0700
Subject: [PATCH 091/115] Enable PPL datetime scalar functions on the
 analytics-engine route (#21556)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Enable PPL datetime scalar functions on the analytics-engine route

Wire 23 PPL datetime functions through PPL → Calcite → Substrait → DataFusion.
Date-part functions (year/quarter/month/day/hour/minute/microsecond/week plus
aliases) route through DatePartAdapters → DataFusion date_part; niladic
functions (now/current_timestamp/current_date/curdate/current_time/curtime)
route through DateTimeAdapters to the matching DataFusion builtin; convert_tz
and unix_timestamp continue to use their existing Rust UDFs.

Functions whose DataFusion semantics diverge from legacy PPL (SECOND double-vs-int,
DAYOFWEEK Sun=0-vs-1, SYSDATE per-row-vs-query-constant, DATE_FORMAT/TIME_FORMAT
MySQL dialect, STRFTIME POSIX dialect), 2-arg overloads with no DataFusion signature
(FROM_UNIXTIME(epoch,fmt), DATETIME(expr,tz)), EXTRACT (isthmus resolves via
scalar-function lookup rather than native extract), and DATE/TIME/MAKETIME (PPL's
Calcite binding returns VARCHAR rather than DATE/TIME) are deliberately left on
the legacy Calcite path until dedicated Rust UDFs or adapters land.

Verified end-to-end via ScalarDateTimeFunctionIT (23/23 passing) with no Calcite
fallback possible — BaseScalarFunctionIT loads only analytics-engine plugins,
TestPPLTransportAction has only one execution path, and any routing failure
re-throws as RuntimeException.

Signed-off-by: Eric Wei 

* Trim verbose ABS/SUBSTRING adapter-map comment

Remove test-name citations that rot as the suite evolves; keep the
WHY (sort-pushdown) and the mechanism (isthmus default catalog).

Signed-off-by: Eric Wei 

* Prune unadvertised datetime enum entries

SYSDATE / SECOND / DAYOFWEEK / DATE_FORMAT / TIME_FORMAT / STRFTIME /
MAKETIME / FROM_UNIXTIME / DATE / TIME / DATETIME were never placed
in STANDARD_PROJECT_OPS and have no other references — dead SPI
surface. Removing them keeps the enum aligned with what the backend
actually advertises; the covering comment in STANDARD_PROJECT_OPS
still records why each one is withheld.

Signed-off-by: Eric Wei 

---------

Signed-off-by: Eric Wei 
---
 .../analytics/spi/ScalarFunction.java         |  23 ++++
 .../be/datafusion/BaseScalarFunctionIT.java   |   7 +-
 .../datafusion/ScalarDateTimeFunctionIT.java  | 130 +++++++++++++++---
 .../DataFusionAnalyticsBackendPlugin.java     |  80 +++++++++--
 .../DataFusionFragmentConvertor.java          |   6 +-
 .../be/datafusion/DatePartAdapters.java       |  63 +++++++++
 .../be/datafusion/DateTimeAdapters.java       |  77 +++++++++++
 .../opensearch_scalar_functions.yaml          |  63 ++++++++-
 8 files changed, 411 insertions(+), 38 deletions(-)
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatePartAdapters.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DateTimeAdapters.java

diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
index 42bfc7ec1ce92..5baa74d73fa4d 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
@@ -127,8 +127,31 @@ public enum ScalarFunction {
     EXTRACT(Category.SCALAR, SqlKind.EXTRACT),
 
     // ── Datetime ────────────────────────────────────────────────────
+    // fromSqlFunction resolves via valueOf(name.toUpperCase()), so the enum name IS
+    // the wire contract. Aliases each need their own entry; the adapter map points
+    // them at one shared instance.
     TIMESTAMP(Category.SCALAR, SqlKind.OTHER_FUNCTION),
     YEAR(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    QUARTER(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    MONTH(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    MONTH_OF_YEAR(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    DAY(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    DAYOFMONTH(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    DAYOFYEAR(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    DAY_OF_YEAR(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    HOUR(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    HOUR_OF_DAY(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    MINUTE(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    MINUTE_OF_HOUR(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    MICROSECOND(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    WEEK(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    WEEK_OF_YEAR(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    NOW(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    CURRENT_TIMESTAMP(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    CURRENT_DATE(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    CURDATE(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    CURRENT_TIME(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    CURTIME(Category.SCALAR, SqlKind.OTHER_FUNCTION),
     CONVERT_TZ(Category.SCALAR, SqlKind.OTHER_FUNCTION),
     UNIX_TIMESTAMP(Category.SCALAR, SqlKind.OTHER_FUNCTION);
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java
index 6dd4a9116aaee..dd3dbcb1bc0bb 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java
@@ -47,7 +47,12 @@
  */
 // TEST-scope cluster per method — slower but eliminates cluster-reuse degradation that
 // surfaces as cascading NodeDisconnectedException when many test methods share a SUITE cluster.
-@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 1)
+// supportsDedicatedMasters=false + numClientNodes=0 collapses the cluster to a single node
+// combining cluster-manager and data roles: scalar-function tests exercise query rewrite +
+// single-shard execution, which doesn't need dedicated cluster-managers or a separate
+// coord-only node. The 5-node default (3 cluster-managers + 1 data + 1 coord) is a memory
+// pressure source that destabilises node discovery on resource-constrained runners.
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 1, supportsDedicatedMasters = false, numClientNodes = 0)
 public abstract class BaseScalarFunctionIT extends OpenSearchIntegTestCase {
 
     protected static final String BANK_INDEX = "bank";
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java
index 4729a89663b58..972eecd8a66b5 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java
@@ -9,34 +9,126 @@
 package org.opensearch.be.datafusion;
 
 /**
- * End-to-end smoke tests for scalar date/time functions routed through PPL → Calcite →
- * Substrait → DataFusion. Bank fixture row 1: created_at='2024-06-15T10:30:00Z'.
+ * End-to-end parity tests for PPL datetime scalar functions routed through
+ * PPL → Calcite → Substrait → DataFusion. Bank fixture row 1:
+ * {@code created_at = 2024-06-15T10:30:00Z}. Niladic functions are checked for
+ * non-null only — their value depends on wall-clock time.
  *
- * 
        Two representative cases:
+ * 
        Functions whose DF-builtin semantics diverge from legacy PPL are
+ * intentionally not advertised to the analytics-engine planner and flow through
+ * the legacy Calcite path — they'll be re-routed through Rust UDFs in a
+ * follow-up, matching the convert_tz / to_unixtime pattern already in this
+ * plugin. Those are:
  * 
          
- *   
        • {@link #testYear()} — YAML alias with literal-arg injection
- *       ({@code YEAR(ts) → date_part('year', ts)}).
          • 
- *   
        • {@link #testConvertTz()} — custom Rust UDF registered with DataFusion
- *       ({@code convert_tz(ts, from_tz, to_tz)}).
          • 
+ *   
        • {@code SECOND / SECOND_OF_MINUTE} — DF returns DOUBLE, PPL INTEGER
          • 
+ *   
        • {@code DAYOFWEEK / DAY_OF_WEEK} — DF Sun=0..Sat=6, PPL Sun=1..Sat=7
          • 
+ *   
        • {@code SYSDATE} — DF now() is query-constant, PPL per-row
          • 
+ *   
        • {@code DATE_FORMAT / TIME_FORMAT} — DF chrono tokens, PPL MySQL dialect
          • 
+ *   
        • {@code STRFTIME} — DF chrono tokens, PPL POSIX dialect
          • 
+ *   
        • {@code FROM_UNIXTIME(epoch, fmt)} / {@code DATETIME(expr, tz)} 2-arg overloads —
+ *       no matching DF signature
          • 
+ *   
        • {@code EXTRACT(unit FROM ts)} — isthmus resolves {@link org.apache.calcite.sql.SqlKind#EXTRACT}
+ *       through scalar-function lookup rather than emitting a native Substrait
+ *       extract; needs a dedicated adapter + yaml entry routing to {@code date_part}
+ *       (PPL {@code month(ts)} etc. already covers the same semantics).
          • 
+ *   
        • {@code DATE(expr)} / {@code TIME(expr)} / {@code MAKETIME(h,m,s)} — PPL's
+ *       Calcite binding returns VARCHAR for these, so downstream date-part calls
+ *       lower to {@code date_part(string, string?)} which has no DataFusion signature.
+ *       Needs PPL to produce real DATE/TIME types before they can route here.
          • 
  * 
        
  */
 public class ScalarDateTimeFunctionIT extends BaseScalarFunctionIT {
 
     public void testYear() {
-        Object cell = evalScalar("year(created_at)");
-        assertNotNull("year() must not be null", cell);
-        assertEquals(2024L, ((Number) cell).longValue());
+        assertScalarLong("year(created_at)", 2024L);
+    }
+
+    public void testQuarter() {
+        assertScalarLong("quarter(created_at)", 2L);
+    }
+
+    public void testMonth() {
+        assertScalarLong("month(created_at)", 6L);
+    }
+
+    public void testMonthOfYear() {
+        assertScalarLong("month_of_year(created_at)", 6L);
+    }
+
+    public void testDay() {
+        assertScalarLong("day(created_at)", 15L);
+    }
+
+    public void testDayOfMonth() {
+        assertScalarLong("dayofmonth(created_at)", 15L);
+    }
+
+    public void testDayOfYear() {
+        assertScalarLong("dayofyear(created_at)", 167L);
+    }
+
+    public void testDayOfYearAlias() {
+        assertScalarLong("day_of_year(created_at)", 167L);
+    }
+
+    public void testHour() {
+        assertScalarLong("hour(created_at)", 10L);
+    }
+
+    public void testHourOfDay() {
+        assertScalarLong("hour_of_day(created_at)", 10L);
+    }
+
+    public void testMinute() {
+        assertScalarLong("minute(created_at)", 30L);
+    }
+
+    public void testMinuteOfHour() {
+        assertScalarLong("minute_of_hour(created_at)", 30L);
+    }
+
+    public void testMicrosecond() {
+        assertScalarLong("microsecond(created_at)", 0L);
+    }
+
+    public void testWeek() {
+        assertScalarLong("week(created_at)", 24L);
+    }
+
+    public void testWeekOfYear() {
+        assertScalarLong("week_of_year(created_at)", 24L);
+    }
+
+    public void testNow() {
+        assertNotNull("now() must not be null", evalScalar("now()"));
+    }
+
+    public void testCurrentTimestamp() {
+        assertNotNull("current_timestamp() must not be null", evalScalar("current_timestamp()"));
+    }
+
+    public void testCurrentDate() {
+        assertNotNull("current_date() must not be null", evalScalar("current_date()"));
+    }
+
+    public void testCurdate() {
+        assertNotNull("curdate() must not be null", evalScalar("curdate()"));
+    }
+
+    public void testCurrentTime() {
+        assertNotNull("current_time() must not be null", evalScalar("current_time()"));
+    }
+
+    public void testCurtime() {
+        assertNotNull("curtime() must not be null", evalScalar("curtime()"));
     }
 
     public void testConvertTz() {
-        // row 1: created_at = 2024-06-15T10:30:00Z (UTC).
-        // Shifted UTC → +10:00 = 2024-06-15T20:30:00Z, unix seconds = 1718483400.
-        // ConvertTzAdapter rewrites PPL's bespoke CONVERT_TZ to our locally-declared
-        // SqlFunction("convert_tz") whose Sig is in ADDITIONAL_SCALAR_SIGS;
-        // UnixTimestampAdapter does the same to to_unixtime. Isthmus resolves both,
-        // DataFusion runs convert_tz via the Rust UDF and to_unixtime natively.
-        Object cell = evalScalar("unix_timestamp(convert_tz(created_at, '+00:00', '+10:00'))");
-        assertNotNull("convert_tz must not be null", cell);
-        assertEquals(1718483400L, ((Number) cell).longValue());
+        // UTC → +10:00 shift of 2024-06-15T10:30:00Z = 2024-06-15T20:30:00Z.
+        assertScalarLong("unix_timestamp(convert_tz(created_at, '+00:00', '+10:00'))", 1718483400L);
+    }
+
+    public void testUnixTimestamp() {
+        assertScalarLong("unix_timestamp(created_at)", 1718447400L);
     }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
index 5df91a61b1968..e883dac590ec9 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
@@ -112,11 +112,8 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
         // rejects the operator with "No backend supports scalar function [CASE] among [datafusion]"
         // before substrait emission.
         ScalarFunction.CASE,
-        // ABS / SUBSTRING — `eval x = abs(...)` and `eval s = substring(...)` projections that PPL
-        // sort-pushdown moves into the project tree (see CalciteSortCommandIT
-        // testPushdownSortExpressionContainsNull and CalcitePPLSortIT
-        // testPushdownSortStringExpression). DataFusion has both natively; isthmus default catalog
-        // already binds them.
+        // ABS / SUBSTRING — PPL sort-pushdown moves these into the project tree; DataFusion has
+        // both natively and isthmus's default catalog binds them, so no adapter needed.
         ScalarFunction.ABS,
         ScalarFunction.SUBSTRING,
         ScalarFunction.SARG_PREDICATE,
@@ -165,9 +162,45 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
         ScalarFunction.EXPM1,
         ScalarFunction.SCALAR_MAX,
         ScalarFunction.SCALAR_MIN,
+        // Date-part extractors rewrite to date_part(, ts) via DatePartAdapters.
+        // Functions whose DF-builtin semantics diverge from legacy PPL (SECOND / DAYOFWEEK /
+        // SYSDATE / DATE_FORMAT / TIME_FORMAT / STRFTIME, plus 2-arg FROM_UNIXTIME / DATETIME)
+        // are omitted here and stay on the legacy engine until a dedicated Rust UDF lands —
+        // matching the convert_tz / to_unixtime pattern already in this plugin.
         ScalarFunction.YEAR,
+        ScalarFunction.QUARTER,
+        ScalarFunction.MONTH,
+        ScalarFunction.MONTH_OF_YEAR,
+        ScalarFunction.DAY,
+        ScalarFunction.DAYOFMONTH,
+        ScalarFunction.DAYOFYEAR,
+        ScalarFunction.DAY_OF_YEAR,
+        ScalarFunction.HOUR,
+        ScalarFunction.HOUR_OF_DAY,
+        ScalarFunction.MINUTE,
+        ScalarFunction.MINUTE_OF_HOUR,
+        ScalarFunction.MICROSECOND,
+        ScalarFunction.WEEK,
+        ScalarFunction.WEEK_OF_YEAR,
+        // Niladic now/current_* family maps 1:1 to DF builtins.
+        ScalarFunction.NOW,
+        ScalarFunction.CURRENT_TIMESTAMP,
+        ScalarFunction.CURRENT_DATE,
+        ScalarFunction.CURDATE,
+        ScalarFunction.CURRENT_TIME,
+        ScalarFunction.CURTIME,
         ScalarFunction.CONVERT_TZ,
         ScalarFunction.UNIX_TIMESTAMP
+        // DATE(expr) / TIME(expr) / MAKETIME(h,m,s) are intentionally not advertised:
+        // PPL's Calcite binding for these returns VARCHAR rather than DATE/TIME, so
+        // downstream `year(date(ts))` / `hour(maketime(...))` lowers to
+        // date_part(string, string?) — no matching DataFusion signature. Left on the
+        // legacy engine until PPL wires them to produce real DATE/TIME types.
+        // EXTRACT(unit FROM ts) is also not advertised: isthmus resolves SqlKind.EXTRACT
+        // through scalar-function lookup rather than emitting a native Substrait extract,
+        // and we'd need a dedicated adapter + yaml entry to route it to DataFusion's
+        // date_part. Left on legacy engine until that adapter lands; PPL date-part
+        // functions cover the same semantics.
     );
 
     private static final Set AGG_FUNCTIONS = Set.of(
@@ -245,18 +278,45 @@ public Set aggregateCapabilities() {
 
             @Override
             public Map scalarFunctionAdapters() {
-                // Add new (ScalarFunction, ScalarFunctionAdapter) pairs in alphabetical order for
-                // readability — the Map.ofEntries form keeps spotless happy past the 5-pair point
-                // where Map.of becomes single-line and unreadable.
+                // Map entries are alphabetical (Map.ofEntries past 5 pairs, else spotless inlines).
+                // Alias pairs share an adapter instance but need separate enum entries because
+                // ScalarFunction.fromSqlFunction resolves by enum name.
+                DatePartAdapters month = DatePartAdapters.month();
+                DatePartAdapters day = DatePartAdapters.day();
+                DatePartAdapters dayOfYear = DatePartAdapters.dayOfYear();
+                DatePartAdapters hour = DatePartAdapters.hour();
+                DatePartAdapters minute = DatePartAdapters.minute();
+                DatePartAdapters week = DatePartAdapters.week();
+                DateTimeAdapters.NowAdapter now = new DateTimeAdapters.NowAdapter();
+                DateTimeAdapters.CurrentDateAdapter currentDate = new DateTimeAdapters.CurrentDateAdapter();
+                DateTimeAdapters.CurrentTimeAdapter currentTime = new DateTimeAdapters.CurrentTimeAdapter();
                 return Map.ofEntries(
                     Map.entry(ScalarFunction.CONCAT, new ConcatFunctionAdapter()),
                     Map.entry(ScalarFunction.CONVERT_TZ, new ConvertTzAdapter()),
                     Map.entry(ScalarFunction.COSH, new HyperbolicOperatorAdapter(SqlLibraryOperators.COSH)),
+                    Map.entry(ScalarFunction.CURDATE, currentDate),
+                    Map.entry(ScalarFunction.CURRENT_DATE, currentDate),
+                    Map.entry(ScalarFunction.CURRENT_TIME, currentTime),
+                    Map.entry(ScalarFunction.CURRENT_TIMESTAMP, now),
+                    Map.entry(ScalarFunction.CURTIME, currentTime),
+                    Map.entry(ScalarFunction.DAY, day),
+                    Map.entry(ScalarFunction.DAYOFMONTH, day),
+                    Map.entry(ScalarFunction.DAYOFYEAR, dayOfYear),
+                    Map.entry(ScalarFunction.DAY_OF_YEAR, dayOfYear),
                     Map.entry(ScalarFunction.DIVIDE, new StdOperatorRewriteAdapter("DIVIDE", SqlStdOperatorTable.DIVIDE)),
                     Map.entry(ScalarFunction.E, new EConstantAdapter()),
                     Map.entry(ScalarFunction.EXPM1, new Expm1Adapter()),
+                    Map.entry(ScalarFunction.HOUR, hour),
+                    Map.entry(ScalarFunction.HOUR_OF_DAY, hour),
                     Map.entry(ScalarFunction.LIKE, new LikeAdapter()),
+                    Map.entry(ScalarFunction.MICROSECOND, DatePartAdapters.microsecond()),
+                    Map.entry(ScalarFunction.MINUTE, minute),
+                    Map.entry(ScalarFunction.MINUTE_OF_HOUR, minute),
                     Map.entry(ScalarFunction.MOD, new StdOperatorRewriteAdapter("MOD", SqlStdOperatorTable.MOD)),
+                    Map.entry(ScalarFunction.MONTH, month),
+                    Map.entry(ScalarFunction.MONTH_OF_YEAR, month),
+                    Map.entry(ScalarFunction.NOW, now),
+                    Map.entry(ScalarFunction.QUARTER, DatePartAdapters.quarter()),
                     Map.entry(ScalarFunction.REGEXP_REPLACE, new RegexpReplaceAdapter()),
                     Map.entry(ScalarFunction.SARG_PREDICATE, new SargAdapter()),
                     Map.entry(ScalarFunction.SCALAR_MAX, nameMapping(SqlLibraryOperators.GREATEST)),
@@ -265,7 +325,9 @@ public Map scalarFunctionAdapters() {
                     Map.entry(ScalarFunction.SINH, new HyperbolicOperatorAdapter(SqlLibraryOperators.SINH)),
                     Map.entry(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter()),
                     Map.entry(ScalarFunction.UNIX_TIMESTAMP, new UnixTimestampAdapter()),
-                    Map.entry(ScalarFunction.YEAR, new YearAdapter())
+                    Map.entry(ScalarFunction.WEEK, week),
+                    Map.entry(ScalarFunction.WEEK_OF_YEAR, week),
+                    Map.entry(ScalarFunction.YEAR, DatePartAdapters.year())
                 );
             }
         };
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
index ecdef923f656e..2614d3c3f9ebb 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
@@ -103,11 +103,13 @@ public class DataFusionFragmentConvertor implements FragmentConvertor {
         FunctionMappings.s(SqlLibraryOperators.DATE_PART, "date_part"),
         FunctionMappings.s(ConvertTzAdapter.LOCAL_CONVERT_TZ_OP, "convert_tz"),
         FunctionMappings.s(UnixTimestampAdapter.LOCAL_TO_UNIXTIME_OP, "to_unixtime"),
+        // Niladic ops from DateTimeAdapters — each maps 1:1 to a DF builtin.
+        FunctionMappings.s(DateTimeAdapters.LOCAL_NOW_OP, "now"),
+        FunctionMappings.s(DateTimeAdapters.LOCAL_CURRENT_DATE_OP, "current_date"),
+        FunctionMappings.s(DateTimeAdapters.LOCAL_CURRENT_TIME_OP, "current_time"),
         FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"),
         FunctionMappings.s(SqlStdOperatorTable.REPLACE, "replace"),
         FunctionMappings.s(SqlLibraryOperators.REGEXP_REPLACE_3, "regexp_replace"),
-        FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"),
-        FunctionMappings.s(UnixTimestampAdapter.LOCAL_TO_UNIXTIME_OP, "to_unixtime"),
         FunctionMappings.s(SqlStdOperatorTable.TRUNCATE, "trunc"),
         FunctionMappings.s(SqlStdOperatorTable.CBRT, "cbrt"),
         FunctionMappings.s(SqlStdOperatorTable.COT, "cot"),
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatePartAdapters.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatePartAdapters.java
new file mode 100644
index 0000000000000..77cc12ca5654e
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatePartAdapters.java
@@ -0,0 +1,63 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.sql.fun.SqlLibraryOperators;
+import org.opensearch.analytics.spi.AbstractNameMappingAdapter;
+
+import java.util.List;
+
+/**
+ * Date-part extractor adapters — rewrite {@code FN(ts)} to {@code date_part('', ts)}.
+ * Alias pairs (e.g. MONTH_OF_YEAR → MONTH) share an adapter instance at registration.
+ *
+ * @opensearch.internal
+ */
+final class DatePartAdapters extends AbstractNameMappingAdapter {
+
+    DatePartAdapters(String unit) {
+        super(SqlLibraryOperators.DATE_PART, List.of(unit), List.of());
+    }
+
+    static DatePartAdapters year() {
+        return new DatePartAdapters("year");
+    }
+
+    static DatePartAdapters quarter() {
+        return new DatePartAdapters("quarter");
+    }
+
+    static DatePartAdapters month() {
+        return new DatePartAdapters("month");
+    }
+
+    static DatePartAdapters day() {
+        return new DatePartAdapters("day");
+    }
+
+    static DatePartAdapters dayOfYear() {
+        return new DatePartAdapters("doy");
+    }
+
+    static DatePartAdapters hour() {
+        return new DatePartAdapters("hour");
+    }
+
+    static DatePartAdapters minute() {
+        return new DatePartAdapters("minute");
+    }
+
+    static DatePartAdapters microsecond() {
+        return new DatePartAdapters("microsecond");
+    }
+
+    static DatePartAdapters week() {
+        return new DatePartAdapters("week");
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DateTimeAdapters.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DateTimeAdapters.java
new file mode 100644
index 0000000000000..f82503c54ad49
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DateTimeAdapters.java
@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.opensearch.analytics.spi.AbstractNameMappingAdapter;
+
+import java.util.List;
+
+/**
+ * Adapters for PPL datetime functions that map 1:1 to a DataFusion builtin. Each
+ * concrete subclass pairs a PPL function name with its locally-declared Calcite
+ * {@link SqlOperator} whose {@code FunctionMappings.Sig} is registered in
+ * {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS}.
+ *
+ * @opensearch.internal
+ */
+final class DateTimeAdapters {
+
+    private DateTimeAdapters() {}
+
+    static final SqlOperator LOCAL_NOW_OP = new SqlFunction(
+        "now",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.TIMESTAMP,
+        null,
+        OperandTypes.NILADIC,
+        SqlFunctionCategory.TIMEDATE
+    );
+
+    static final SqlOperator LOCAL_CURRENT_DATE_OP = new SqlFunction(
+        "current_date",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.DATE,
+        null,
+        OperandTypes.NILADIC,
+        SqlFunctionCategory.TIMEDATE
+    );
+
+    static final SqlOperator LOCAL_CURRENT_TIME_OP = new SqlFunction(
+        "current_time",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.TIME,
+        null,
+        OperandTypes.NILADIC,
+        SqlFunctionCategory.TIMEDATE
+    );
+
+    static final class NowAdapter extends AbstractNameMappingAdapter {
+        NowAdapter() {
+            super(LOCAL_NOW_OP, List.of(), List.of());
+        }
+    }
+
+    static final class CurrentDateAdapter extends AbstractNameMappingAdapter {
+        CurrentDateAdapter() {
+            super(LOCAL_CURRENT_DATE_OP, List.of(), List.of());
+        }
+    }
+
+    static final class CurrentTimeAdapter extends AbstractNameMappingAdapter {
+        CurrentTimeAdapter() {
+            super(LOCAL_CURRENT_TIME_OP, List.of(), List.of());
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
index a19428cad0a4b..34e70f5c3bdc5 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
@@ -20,23 +20,72 @@ scalar_functions:
             name: "match"
         return: boolean
   - name: "date_part"
-    impls:
-      - args: [{ value: string, name: "part" }, { value: "any1", name: "value" }]
-        return: any1
+    description: >-
+      Extract a numeric component (year, month, day, etc.) from a timestamp/date.
+      DataFusion returns fp64 for every part name, so the signature pins fp64
+      regardless of which part is requested.
+
+      The `part` slot is declared as `value: string`, not an `options:`
+      EnumArgument. `DatePartAdapters` prepends a VARCHAR literal (e.g.
+      `"year"`) via `rexBuilder.makeLiteral(…, VARCHAR, true)`; that is not a
+      `SqlIntervalQualifier` enum symbol, so isthmus's `FunctionConverter.matchKeys`
+      emits the token `str` for that operand, and the probe key matches this
+      sig's `str_…` form. An `options:` declaration would serialize under a
+      `req`/`opt` key and never match.
 
+      The value slot is declared with the concrete timestamp family
+      (`precision_timestamp
        ` / `date`) rather than `any1`: Calcite's `date`
+      field type emits `precision_timestamp<0>?` at the Rex level, and
+      matchKeys encodes the nullable-precision metadata into the probe key;
+      `any1` cannot bind through the composite 2-arg key in this context.
+    impls:
+      - args:
+          - { value: string, name: "part" }
+          - { value: "precision_timestamp
        ", name: "value" }
+        return: fp64
+      - args:
+          - { value: string, name: "part" }
+          - { value: "date", name: "value" }
+        return: fp64
   - name: "convert_tz"
-    description: "Shift a timestamp from one timezone to another. IANA names and +/-HH:MM offsets."
+    description: >-
+      Shift a timestamp from one timezone to another. IANA names and +/-HH:MM
+      offsets. The first slot pins `precision_timestamp
        ` for the same
+      reason documented on date_part — `any1` cannot bind the nullable-
+      precision-timestamp metadata through a multi-arg composite match key.
     impls:
       - args:
-          - { value: "any1", name: "ts" }
+          - { value: "precision_timestamp
        ", name: "ts" }
           - { value: string, name: "from_tz" }
           - { value: string, name: "to_tz" }
-        return: any1
+        return: timestamp
   - name: "to_unixtime"
     description: "Return a timestamp as Unix epoch seconds."
     impls:
       - args: [{ value: "any1", name: "ts" }]
-        return: any1
+        return: i64
+  - name: "now"
+    description: >-
+      Returns the current wall-clock timestamp. DataFusion's builtin `now` —
+      evaluated once per query, constant across the plan. Lowering target for
+      PPL `now()`, `current_timestamp()`, and `sysdate()`.
+    impls:
+      - args: []
+        return: timestamp
+  - name: "current_date"
+    description: >-
+      Returns today's date (no time component). DataFusion's builtin
+      `current_date`. Lowering target for PPL `current_date()` / `curdate()`.
+    impls:
+      - args: []
+        return: date
+  - name: "current_time"
+    description: >-
+      Returns the current wall-clock time (no date component). DataFusion's
+      builtin `current_time`. Lowering target for PPL `current_time()` / `curtime()`.
+    impls:
+      - args: []
+        return: time
   - name: regex_match
     description: >-
       Regular expression containment match (boolean). Lowering target for PPL's

From 87ebdb54d88d28ce1825e6e7327f16095f2e3890 Mon Sep 17 00:00:00 2001
From: Peter Zhu 
Date: Fri, 8 May 2026 15:14:04 -0400
Subject: [PATCH 092/115] Update maven snapshots publishing to include sandbox
 jars (#21567)

Signed-off-by: Peter Zhu 
---
 .github/workflows/publish-maven-snapshots.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/publish-maven-snapshots.yml b/.github/workflows/publish-maven-snapshots.yml
index 353adbb95faf8..a193723ca2591 100644
--- a/.github/workflows/publish-maven-snapshots.yml
+++ b/.github/workflows/publish-maven-snapshots.yml
@@ -50,6 +50,6 @@ jobs:
           role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
           aws-region: us-east-1
 
-      - name: Publish snapshots to maven
+      - name: Publish snapshots to maven (with sandbox)
         run: |
-          ./gradlew publishNebulaPublicationToSnapshotsRepository -Pcrypto.standard=FIPS-140-3
+          ./gradlew publishNebulaPublicationToSnapshotsRepository -Dsandbox.enabled=true -Pcrypto.standard=FIPS-140-3

From 8c47459682cc8b3e7c5fd6e87df19cef9eff3bc0 Mon Sep 17 00:00:00 2001
From: Sean Chittenden 
Date: Fri, 8 May 2026 12:49:32 -0700
Subject: [PATCH 093/115] Fix deadlock between engineMutex and writeLock during
 index close and engine reset (#11869) (#21404)

During resetEngineToGlobalCheckpoint on a segment replication replica,
the ReadOnlyEngine delegates (acquireLastIndexCommit, acquireSafeIndexCommit,
getSegmentInfosSnapshot) synchronized on engineMutex.  This creates a
deadlock cycle when close runs concurrently: the close thread holds
engineMutex and waits for writeLock, while the recovery thread holds
readLock (via recoverFromTranslog) and a refresh listener calls a
delegate that waits for engineMutex.

Keep SetOnce instead of AtomicReference as it already provides
thread-safe reads via its internal AtomicReference, and its
write-once guarantee prevents accidental double-set.  The minimal
fix is just removing synchronized(engineMutex) from the delegates.

Signed-off-by: Sean Chittenden 
---
 .../opensearch/index/shard/IndexShard.java    |  39 ++---
 .../index/shard/IndexShardTests.java          | 142 ++++++++++++++++++
 2 files changed, 163 insertions(+), 18 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index 522afff73ccdd..623839a282b97 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -5393,8 +5393,12 @@ void resetEngineToGlobalCheckpoint() throws IOException {
         assert globalCheckpoint == getLastSyncedGlobalCheckpoint();
         synchronized (engineMutex) {
             verifyNotClosed();
-            // we must create both new read-only engine and new read-write engine under engineMutex to ensure snapshotStoreMetadata,
-            // acquireXXXCommit and close works.
+            // we must create both new read-only engine and new read-write engine under
+            // engineMutex to ensure snapshotStoreMetadata, acquireXXXCommit and close works.
+            // Delegates intentionally do NOT synchronize on engineMutex: doing so would
+            // deadlock because close holds engineMutex and waits for writeLock, while
+            // recoverFromTranslog holds readLock and a refresh listener calls a delegate.
+            // SetOnce is backed by AtomicReference so get() provides happens-before visibility.
             final Engine readOnlyEngine = new ReadOnlyEngine(
                 newEngineConfig(replicationTracker),
                 seqNoStats,
@@ -5405,33 +5409,27 @@ void resetEngineToGlobalCheckpoint() throws IOException {
             ) {
                 @Override
                 public GatedCloseable acquireLastIndexCommit(boolean flushFirst) {
-                    synchronized (engineMutex) {
-                        if (newEngineReference.get() == null) {
-                            throw new AlreadyClosedException("engine was closed");
-                        }
-                        // ignore flushFirst since we flushed above and we do not want to interfere with ongoing translog replay
-                        return applyOnEngine(newEngineReference.get(), engine -> engine.acquireLastIndexCommit(false));
+                    if (newEngineReference.get() == null) {
+                        throw new AlreadyClosedException("engine was closed");
                     }
+                    // ignore flushFirst since we flushed above and we do not want to interfere with ongoing translog replay
+                    return applyOnEngine(newEngineReference.get(), engine -> engine.acquireLastIndexCommit(false));
                 }
 
                 @Override
                 public GatedCloseable acquireSafeIndexCommit() {
-                    synchronized (engineMutex) {
-                        if (newEngineReference.get() == null) {
-                            throw new AlreadyClosedException("engine was closed");
-                        }
-                        return applyOnEngine(newEngineReference.get(), Engine::acquireSafeIndexCommit);
+                    if (newEngineReference.get() == null) {
+                        throw new AlreadyClosedException("engine was closed");
                     }
+                    return applyOnEngine(newEngineReference.get(), Engine::acquireSafeIndexCommit);
                 }
 
                 @Override
                 public GatedCloseable getSegmentInfosSnapshot() {
-                    synchronized (engineMutex) {
-                        if (newEngineReference.get() == null) {
-                            throw new AlreadyClosedException("engine was closed");
-                        }
-                        return applyOnEngine(newEngineReference.get(), Engine::getSegmentInfosSnapshot);
+                    if (newEngineReference.get() == null) {
+                        throw new AlreadyClosedException("engine was closed");
                     }
+                    return applyOnEngine(newEngineReference.get(), Engine::getSegmentInfosSnapshot);
                 }
 
                 @Override
@@ -6173,6 +6171,11 @@ ConcurrentHashMap nonClosingReaderWrap
         return nonClosingReaderWrapperCache;
     }
 
+    // Visible for testing
+    Object getEngineMutex() {
+        return engineMutex;
+    }
+
     // Below methods exists for bwc only. We should never make indexshard aware of DataFormatAwareEngine directy.
     // All interactions should happen via indexer only.
     @Deprecated
diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
index 4b4a17088bfd7..ecd9c3236abae 100644
--- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
@@ -113,6 +113,7 @@
 import org.opensearch.index.engine.ReadOnlyEngine;
 import org.opensearch.index.engine.exec.EngineBackedIndexerFactory;
 import org.opensearch.index.engine.exec.Indexer;
+import org.opensearch.index.engine.exec.IndexerFactory;
 import org.opensearch.index.fielddata.FieldDataStats;
 import org.opensearch.index.fielddata.IndexFieldData;
 import org.opensearch.index.fielddata.IndexFieldDataCache;
@@ -4771,6 +4772,147 @@ public void onBeginTranslogRecovery() {
         closeShard(shard, false);
     }
 
+    /**
+     * Verifies that {@code getSegmentInfosSnapshot()} on the ReadOnlyEngine created during
+     * {@link IndexShard#resetEngineToGlobalCheckpoint()} does not block on {@code engineMutex}.
+     * 
        
+     * Regression test for
+     * #11869:
+     * the close thread holds {@code engineMutex} and waits for {@code writeLock}, while the
+     * recovery thread holds {@code readLock} (via {@code recoverFromTranslog}) and calls
+     * {@code getSegmentInfosSnapshot()} through the {@code ReplicationCheckpointUpdater} refresh
+     * listener -- if both paths synchronize on {@code engineMutex}, the cycle deadlocks.
+     * 
        
+     * Pauses {@code resetEngineToGlobalCheckpoint} before translog replay, holds
+     * {@code engineMutex} via reflection, and asserts {@code getSegmentInfosSnapshot()}
+     * completes within 5 seconds.
+     */
+    public void testNoDeadlockOnCloseWhileRecoveringTranslog() throws Exception {
+        CountDownLatch recoveryStartedLatch = new CountDownLatch(1);
+        CountDownLatch proceedWithRecoveryLatch = new CountDownLatch(1);
+        AtomicBoolean armed = new AtomicBoolean(false);
+        Settings segRepSettings = Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build();
+        IndexerFactory customFactory = new EngineBackedIndexerFactory(config -> new InternalEngine(config, new TranslogEventListener() {
+            @Override
+            public void onBeginTranslogRecovery() {
+                if (armed.compareAndSet(true, false)) {
+                    recoveryStartedLatch.countDown();
+                    try {
+                        proceedWithRecoveryLatch.await(30, TimeUnit.SECONDS);
+                    } catch (InterruptedException e) {
+                        throw new AssertionError(e);
+                    }
+                }
+            }
+        }));
+        IndexShard shard = newShard(false, segRepSettings, customFactory);
+        IndexShard primary = newStartedShard(true, segRepSettings);
+        recoverReplica(shard, primary, true, (a) -> null);
+        closeShards(primary);
+
+        Object engineMutex = shard.getEngineMutex();
+
+        final CountDownLatch engineResetLatch = new CountDownLatch(1);
+
+        shard.acquireAllReplicaOperationsPermits(
+            shard.getOperationPrimaryTerm(),
+            shard.getLastKnownGlobalCheckpoint(),
+            0L,
+            ActionListener.wrap(r -> {
+                try (Releasable dummy = r) {
+                    armed.set(true);
+                    shard.resetEngineToGlobalCheckpoint();
+                } finally {
+                    engineResetLatch.countDown();
+                }
+            }, Assert::assertNotNull),
+            TimeValue.timeValueMinutes(1L)
+        );
+
+        // Wait until the reset has created the ReadOnlyEngine (installed as current engine)
+        // and the new InternalEngine, then paused before translog replay.
+        assertTrue("recovery should start", recoveryStartedLatch.await(30, TimeUnit.SECONDS));
+
+        // Verify getSegmentInfosSnapshot() on the ReadOnlyEngine doesn't block when
+        // engineMutex is held -- this is the code path that deadlocks in production.
+        CountDownLatch snapshotCompletedLatch = new CountDownLatch(1);
+        Thread snapshotThread = new Thread(() -> {
+            try {
+                GatedCloseable snapshot = shard.getSegmentInfosSnapshot();
+                if (snapshot != null) snapshot.close();
+            } catch (IOException | IllegalStateException ignored) {} finally {
+                snapshotCompletedLatch.countDown();
+            }
+        });
+
+        synchronized (engineMutex) {
+            snapshotThread.start();
+            assertTrue("getSegmentInfosSnapshot should not block on engineMutex", snapshotCompletedLatch.await(5, TimeUnit.SECONDS));
+        }
+        snapshotThread.join(5_000);
+
+        proceedWithRecoveryLatch.countDown();
+        assertTrue("engine reset should complete", engineResetLatch.await(30, TimeUnit.SECONDS));
+        closeShard(shard, false);
+    }
+
+    /**
+     * Verifies that the ReadOnlyEngine delegates throw {@link AlreadyClosedException} when
+     * {@code newEngineReference} is still null -- the window between ReadOnlyEngine installation
+     * and {@code newEngineReference.set(newEngine)} inside {@code resetEngineToGlobalCheckpoint}.
+     * Covers the defensive null-check branches in {@code acquireLastIndexCommit},
+     * {@code acquireSafeIndexCommit}, and {@code getSegmentInfosSnapshot}.
+     */
+    public void testDelegateThrowsAlreadyClosedBeforeNewEngineSet() throws Exception {
+        CountDownLatch creatingEngineLatch = new CountDownLatch(1);
+        CountDownLatch proceedWithCreationLatch = new CountDownLatch(1);
+        AtomicBoolean armed = new AtomicBoolean(false);
+        Settings segRepSettings = Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build();
+        IndexerFactory customFactory = new EngineBackedIndexerFactory(config -> {
+            if (armed.compareAndSet(true, false)) {
+                creatingEngineLatch.countDown();
+                try {
+                    proceedWithCreationLatch.await(30, TimeUnit.SECONDS);
+                } catch (InterruptedException e) {
+                    throw new AssertionError(e);
+                }
+            }
+            return new InternalEngine(config);
+        });
+        IndexShard shard = newShard(false, segRepSettings, customFactory);
+        IndexShard primary = newStartedShard(true, segRepSettings);
+        recoverReplica(shard, primary, true, (a) -> null);
+        closeShards(primary);
+
+        final CountDownLatch engineResetLatch = new CountDownLatch(1);
+
+        shard.acquireAllReplicaOperationsPermits(
+            shard.getOperationPrimaryTerm(),
+            shard.getLastKnownGlobalCheckpoint(),
+            0L,
+            ActionListener.wrap(r -> {
+                try (Releasable dummy = r) {
+                    armed.set(true);
+                    shard.resetEngineToGlobalCheckpoint();
+                } finally {
+                    engineResetLatch.countDown();
+                }
+            }, Assert::assertNotNull),
+            TimeValue.timeValueMinutes(1L)
+        );
+
+        assertTrue("engine creation should start", creatingEngineLatch.await(30, TimeUnit.SECONDS));
+
+        // The ReadOnlyEngine is now the current engine, but newEngineReference is still null.
+        expectThrows(AlreadyClosedException.class, () -> shard.acquireLastIndexCommit(false));
+        expectThrows(AlreadyClosedException.class, shard::acquireSafeIndexCommit);
+        expectThrows(AlreadyClosedException.class, shard::getSegmentInfosSnapshot);
+
+        proceedWithCreationLatch.countDown();
+        assertTrue("engine reset should complete", engineResetLatch.await(30, TimeUnit.SECONDS));
+        closeShard(shard, false);
+    }
+
     /**
      * This test simulates a scenario seen rarely in ConcurrentSeqNoVersioningIT. While engine is inside
      * resetEngineToGlobalCheckpoint snapshot metadata could fail

From ecde9f72ced9c6a5812c61843e9d988209fb9fbe Mon Sep 17 00:00:00 2001
From: Peter Zhu 
Date: Fri, 8 May 2026 16:16:32 -0400
Subject: [PATCH 094/115] Bump maven snapshots publishing to jdk25 to unlock
 sandbox jar publishing (#21568)

Signed-off-by: Peter Zhu 
---
 .github/workflows/publish-maven-snapshots.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/publish-maven-snapshots.yml b/.github/workflows/publish-maven-snapshots.yml
index a193723ca2591..068bfb330e4a2 100644
--- a/.github/workflows/publish-maven-snapshots.yml
+++ b/.github/workflows/publish-maven-snapshots.yml
@@ -19,11 +19,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v6
-      - name: Set up JDK 21
+      - name: Set up JDK 25 (required for sandbox publishing, default min support is still 21)
         uses: actions/setup-java@v5
         with:
           distribution: temurin
-          java-version: 21
+          java-version: 25  # TODO: switch back to jdk21 once sandbox plugins set min compat to 21
 
       - name: Install protoc (Linux)
         run: |

From 0d31e286da1da867a82dbcc5f0f61fead20567cc Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Fri, 8 May 2026 17:08:42 -0400
Subject: [PATCH 095/115] Fix flaky
 org.opensearch.http.netty4.Netty4HttpRequestSizeLimitIT.testDoesNotLimitExcludedRequests
 test case (#21565)

Signed-off-by: Andriy Redko 
---
 .../opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java  | 4 +++-
 .../java/org/opensearch/http/netty4/Netty4HttpClient.java     | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
index 826d4a7e5d61e..d133c6830c983 100644
--- a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
+++ b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
@@ -125,7 +125,9 @@ public void testDoesNotLimitExcludedRequests() throws Exception {
 
         List> requestUris = new ArrayList<>();
         for (int i = 0; i < 1500; i++) {
-            requestUris.add(Tuple.tuple("/_cluster/settings", "{ \"transient\": {\"search.default_search_timeout\": \"40s\" } }"));
+            requestUris.add(
+                Tuple.tuple("/_cluster/settings?cluster_manager_timeout=10s", "{ \"transient\": {\"search.default_search_timeout\": -1 } }")
+            );
         }
 
         HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java
index af9b4894393b9..567875bae76ef 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java
@@ -270,7 +270,7 @@ private synchronized List sendRequests(final SocketAddress rem
                     channel.writeAndFlush(request);
                 }
                 if (latch.await(30L, TimeUnit.SECONDS) == false) {
-                    fail("Failed to get all expected responses.");
+                    fail("Failed to get all expected responses: " + latch.getCount() + " left");
                 }
             } finally {
                 channel.close().awaitUninterruptibly();

From 239f4d92373208aac3bf099a0e1e0df7c40164af Mon Sep 17 00:00:00 2001
From: tanik98 <72665765+tanik98@users.noreply.github.com>
Date: Sat, 9 May 2026 03:39:46 +0530
Subject: [PATCH 096/115] Wire PPL string scalar functions with
 analytics-backend-datafusion (#21543)

Signed-off-by: Tanik Pansuriya 
---
 .../analytics/spi/ScalarFunction.java         |  15 +
 .../rust/src/session_context.rs               |   1 +
 .../rust/src/udf/mod.rs                       |   6 +-
 .../rust/src/udf/tonumber.rs                  | 396 ++++++++++++
 .../rust/src/udf/tostring.rs                  | 595 ++++++++++++++++++
 .../DataFusionAnalyticsBackendPlugin.java     |  29 +-
 .../DataFusionFragmentConvertor.java          |   9 +
 .../be/datafusion/DatafusionResultStream.java |   3 +-
 .../be/datafusion/PositionAdapter.java        | 104 +++
 .../be/datafusion/StrcmpFunctionAdapter.java  |  93 +++
 .../datafusion/ToNumberFunctionAdapter.java   | 103 +++
 .../datafusion/ToStringFunctionAdapter.java   | 225 +++++++
 .../opensearch_scalar_functions.yaml          |  64 ++
 .../be/datafusion/PositionAdapterTests.java   | 236 +++++++
 .../StrcmpFunctionAdapterTests.java           | 110 ++++
 .../ToNumberFunctionAdapterTests.java         | 164 +++++
 .../ToStringFunctionAdapterTests.java         | 286 +++++++++
 .../analytics/exec/ArrowValues.java           |  16 +-
 .../analytics/qa/StringScalarFunctionsIT.java | 400 ++++++++++++
 19 files changed, 2847 insertions(+), 8 deletions(-)
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tonumber.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tostring.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PositionAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/StrcmpFunctionAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ToNumberFunctionAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ToStringFunctionAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/PositionAdapterTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/StrcmpFunctionAdapterTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToNumberFunctionAdapterTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToStringFunctionAdapterTests.java
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StringScalarFunctionsIT.java

diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
index 5baa74d73fa4d..ee32b35be4b9e 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
@@ -56,6 +56,7 @@ public enum ScalarFunction {
     UPPER(Category.STRING, SqlKind.OTHER_FUNCTION),
     LOWER(Category.STRING, SqlKind.OTHER_FUNCTION),
     TRIM(Category.STRING, SqlKind.TRIM),
+    SUBSTR(Category.STRING, SqlKind.OTHER_FUNCTION),
     SUBSTRING(Category.STRING, SqlKind.OTHER_FUNCTION),
     /**
      * String concatenation. Calcite's {@code SqlStdOperatorTable.CONCAT} is a
@@ -66,9 +67,23 @@ public enum ScalarFunction {
      * rename surfaces as a compile error rather than as a silent string mismatch at runtime.
      */
     CONCAT(Category.STRING, SqlKind.OTHER_FUNCTION, SqlStdOperatorTable.CONCAT),
+    CONCAT_WS(Category.STRING, SqlKind.OTHER_FUNCTION),
     CHAR_LENGTH(Category.STRING, SqlKind.OTHER_FUNCTION),
     REPLACE(Category.STRING, SqlKind.OTHER_FUNCTION),
     REGEXP_REPLACE(Category.STRING, SqlKind.OTHER_FUNCTION),
+    ASCII(Category.STRING, SqlKind.OTHER_FUNCTION),
+    LEFT(Category.STRING, SqlKind.OTHER_FUNCTION),
+    LENGTH(Category.STRING, SqlKind.OTHER_FUNCTION),
+    LOCATE(Category.STRING, SqlKind.OTHER_FUNCTION),
+    POSITION(Category.STRING, SqlKind.POSITION),
+    LTRIM(Category.STRING, SqlKind.OTHER_FUNCTION),
+    RTRIM(Category.STRING, SqlKind.OTHER_FUNCTION),
+    REVERSE(Category.STRING, SqlKind.OTHER_FUNCTION),
+    RIGHT(Category.STRING, SqlKind.OTHER_FUNCTION),
+    TOSTRING(Category.STRING, SqlKind.OTHER_FUNCTION),
+    NUMBER_TO_STRING(Category.STRING, SqlKind.OTHER_FUNCTION), // Alias for TOSTRING
+    TONUMBER(Category.STRING, SqlKind.OTHER_FUNCTION),
+    STRCMP(Category.STRING, SqlKind.OTHER_FUNCTION),
 
     // ── Math ─────────────────────────────────────────────────────────
     PLUS(Category.MATH, SqlKind.PLUS),
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
index 52c74a888e650..c226deabb2fbe 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
@@ -111,6 +111,7 @@ pub async unsafe fn create_session_context(
         .build();
 
     let ctx = SessionContext::new_with_state(state);
+    crate::udf::register_all(&ctx);
 
     // Register default ListingTable for parquet scans
     let listing_options = ListingOptions::new(Arc::new(ParquetFormat::new()))
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
index 3d13de51967b6..2053625b544c0 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
@@ -113,10 +113,14 @@ pub(crate) fn coerce_args(
 }
 
 pub mod convert_tz;
+pub mod tonumber;
+pub mod tostring;
 
 pub fn register_all(ctx: &SessionContext) {
     convert_tz::register_all(ctx);
-    log::info!("OpenSearch UDF register_all: convert_tz registered");
+    tonumber::register_all(ctx);
+    tostring::register_all(ctx);
+    log::info!("OpenSearch UDF register_all: convert_tz, tonumber, tostring registered");
 }
 
 #[cfg(test)]
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tonumber.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tonumber.rs
new file mode 100644
index 0000000000000..765056a176774
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tonumber.rs
@@ -0,0 +1,396 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! [`tonumber(string, base)`](https://docs.opensearch.org/latest/sql-and-ppl/ppl/functions/conversion/#tonumber)
+//! base-N integer parse.
+//!
+//! # Semantics
+//! * `base` must be in the inclusive range `[2, 36]`
+//! * Output type is `Float64`
+
+use std::any::Any;
+use std::hash::{Hash, Hasher};
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, ArrayRef, Float64Array, Float64Builder, StringArray};
+use datafusion::arrow::datatypes::DataType;
+use datafusion::common::{exec_err, Result, ScalarValue};
+use datafusion::error::DataFusionError;
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature,
+    Volatility,
+};
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(ToNumberUdf::new()));
+}
+
+/// `tonumber(varchar, int)` → fp64. Base-N string-to-integer parse widened to double
+#[derive(Debug)]
+pub struct ToNumberUdf {
+    signature: Signature,
+}
+
+impl ToNumberUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::one_of(
+                vec![TypeSignature::Exact(vec![DataType::Utf8, DataType::Int32])],
+                Volatility::Immutable,
+            ),
+        }
+    }
+}
+
+impl Default for ToNumberUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+// `ScalarUDFImpl` requires `DynEq` + `DynHash`. All instances are functionally
+// identical (no parameterization), so equality is trivial.
+impl PartialEq for ToNumberUdf {
+    fn eq(&self, _: &Self) -> bool {
+        true
+    }
+}
+impl Eq for ToNumberUdf {}
+impl Hash for ToNumberUdf {
+    fn hash(&self, state: &mut H) {
+        "tonumber".hash(state);
+    }
+}
+
+/// How the resolved (constant) `base` argument behaves across the batch.
+/// Resolved once up front so the hot loop doesn't repeat the scalar dispatch
+/// or the range check on every row.
+enum BaseMode {
+    /// Base is `NULL` or outside `[2, 36]`. Every output row is NULL regardless
+    /// of the value column — we skip reading it.
+    AllNull,
+    /// Base is valid. Carries the validated radix as `u32` (the type
+    /// {@code i64::from_str_radix} wants), so the per-row code skips both the
+    /// scalar dispatch and the range check.
+    Valid(u32),
+}
+
+/// How the `value` argument is supplied
+enum ValueSource<'a> {
+    Scalar(Option<&'a str>),
+    Array(&'a StringArray),
+}
+
+impl<'a> ValueSource<'a> {
+    /// Returns the string at row `i`, or `None`
+    fn at(&self, i: usize) -> Option<&str> {
+        match self {
+            ValueSource::Scalar(s) => *s,
+            ValueSource::Array(arr) if arr.is_null(i) => None,
+            ValueSource::Array(arr) => Some(arr.value(i)),
+        }
+    }
+}
+
+impl ScalarUDFImpl for ToNumberUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "tonumber"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result {
+        Ok(DataType::Float64)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() != 2 {
+            return exec_err!(
+                "tonumber expects exactly 2 arguments (string, base), got {}",
+                args.args.len()
+            );
+        }
+
+        let value_col = &args.args[0];
+        let base_col = &args.args[1];
+
+        // Full-scalar fast path — a plain literal `tonumber('FA34', 16)` plan.
+        if let (
+            ColumnarValue::Scalar(ScalarValue::Utf8(s)),
+            ColumnarValue::Scalar(ScalarValue::Int32(b)),
+        ) = (value_col, base_col)
+        {
+            return Ok(ColumnarValue::Scalar(ScalarValue::Float64(
+                parse_with_base(s.as_deref(), *b),
+            )));
+        }
+
+        let n = args.number_rows;
+        let BaseMode::Valid(radix) = resolve_base(base_col)? else {
+            let mut builder = Float64Builder::with_capacity(n);
+            builder.append_nulls(n);
+            return Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef));
+        };
+
+        // Materialize the value column. For scalars we keep the raw &str to
+        // avoid a pointless n-wide StringArray allocation.
+        let values_arr_ref: Option = match value_col {
+            ColumnarValue::Array(_) => Some(value_col.clone().into_array(n)?),
+            _ => None,
+        };
+        let values: ValueSource = match (&values_arr_ref, value_col) {
+            (Some(arr), _) => {
+                let sa = arr.as_any().downcast_ref::().ok_or_else(|| {
+                    DataFusionError::Internal(format!(
+                        "tonumber: value expected Utf8, got {:?}",
+                        arr.data_type()
+                    ))
+                })?;
+                ValueSource::Array(sa)
+            }
+            (None, ColumnarValue::Scalar(ScalarValue::Utf8(opt))) => {
+                ValueSource::Scalar(opt.as_deref())
+            }
+            (None, other) => {
+                return exec_err!("tonumber: value expected Utf8, got {other:?}");
+            }
+        };
+
+        let mut builder = Float64Builder::with_capacity(n);
+        for i in 0..n {
+            match values.at(i).and_then(|s| i64::from_str_radix(s, radix).ok()) {
+                Some(v) => builder.append_value(v as f64),
+                None => builder.append_null(),
+            }
+        }
+        let out: Float64Array = builder.finish();
+        Ok(ColumnarValue::Array(Arc::new(out) as ArrayRef))
+    }
+}
+
+/// Resolve the `base` argument
+fn resolve_base(base_col: &ColumnarValue) -> Result {
+    match base_col {
+        ColumnarValue::Scalar(ScalarValue::Int32(None)) => Ok(BaseMode::AllNull),
+        ColumnarValue::Scalar(ScalarValue::Int32(Some(b))) => Ok(match validate_base(*b) {
+            Some(r) => BaseMode::Valid(r),
+            None => BaseMode::AllNull,
+        }),
+        ColumnarValue::Scalar(other) => {
+            exec_err!("tonumber: base expected Int32 literal, got {other:?}")
+        }
+        ColumnarValue::Array(arr) => {
+            exec_err!(
+                "tonumber: base must be a literal integer, got column of type {:?}",
+                arr.data_type()
+            )
+        }
+    }
+}
+
+/// Convert `base` to the radix expected by [`i64::from_str_radix`], or `None`
+/// when the base is outside `[2, 36]` range.
+fn validate_base(base: i32) -> Option {
+    if (2..=36).contains(&base) {
+        Some(base as u32)
+    } else {
+        None
+    }
+}
+
+fn parse_with_base(s: Option<&str>, base: Option) -> Option {
+    let s = s?;
+    let radix = validate_base(base?)?;
+    i64::from_str_radix(s, radix).ok().map(|v| v as f64)
+}
+
+// ─── tests ──────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use datafusion::arrow::array::{AsArray, Int32Array};
+    use datafusion::arrow::datatypes::Field;
+
+    fn invoke_scalar(value: Option<&str>, base: Option) -> Option {
+        let u = ToNumberUdf::new();
+        let args = ScalarFunctionArgs {
+            args: vec![
+                ColumnarValue::Scalar(ScalarValue::Utf8(value.map(|s| s.to_string()))),
+                ColumnarValue::Scalar(ScalarValue::Int32(base)),
+            ],
+            arg_fields: vec![
+                Arc::new(Field::new("v", DataType::Utf8, true)),
+                Arc::new(Field::new("b", DataType::Int32, true)),
+            ],
+            number_rows: 1,
+            return_field: Arc::new(Field::new(u.name(), DataType::Float64, true)),
+            config_options: Arc::new(Default::default()),
+        };
+        match u.invoke_with_args(args).unwrap() {
+            ColumnarValue::Scalar(ScalarValue::Float64(opt)) => opt,
+            other => panic!("expected Float64 scalar, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn base10_integer_doc_example() {
+        assert_eq!(invoke_scalar(Some("4598"), Some(10)), Some(4598.0));
+    }
+
+    #[test]
+    fn binary_doc_example() {
+        assert_eq!(invoke_scalar(Some("010101"), Some(2)), Some(21.0));
+    }
+
+    #[test]
+    fn hex_doc_example() {
+        assert_eq!(invoke_scalar(Some("FA34"), Some(16)), Some(64052.0));
+        assert_eq!(invoke_scalar(Some("fa34"), Some(16)), Some(64052.0));
+    }
+
+    #[test]
+    fn signed_inputs_parse() {
+        assert_eq!(invoke_scalar(Some("-21"), Some(10)), Some(-21.0));
+        assert_eq!(invoke_scalar(Some("+FA"), Some(16)), Some(250.0));
+        assert_eq!(invoke_scalar(Some("-10"), Some(2)), Some(-2.0));
+    }
+
+    #[test]
+    fn base_boundary_values() {
+        assert_eq!(invoke_scalar(Some("1"), Some(2)), Some(1.0));
+        assert_eq!(invoke_scalar(Some("Z"), Some(36)), Some(35.0));
+    }
+
+    #[test]
+    fn out_of_range_base_returns_null() {
+        assert!(invoke_scalar(Some("10"), Some(0)).is_none());
+        assert!(invoke_scalar(Some("10"), Some(1)).is_none());
+        assert!(invoke_scalar(Some("10"), Some(37)).is_none());
+        assert!(invoke_scalar(Some("10"), Some(-5)).is_none());
+    }
+
+    #[test]
+    fn unparseable_string_returns_null() {
+        assert!(invoke_scalar(Some("FA34"), Some(10)).is_none());
+        assert!(invoke_scalar(Some("12"), Some(2)).is_none());
+        assert!(invoke_scalar(Some(""), Some(10)).is_none());
+        assert!(invoke_scalar(Some("1 2"), Some(10)).is_none());
+        assert!(invoke_scalar(Some("3.14"), Some(10)).is_none());
+    }
+
+    #[test]
+    fn overflow_returns_null() {
+        assert!(invoke_scalar(Some("9223372036854775808"), Some(10)).is_none());
+    }
+
+    #[test]
+    fn null_inputs_return_null() {
+        assert!(invoke_scalar(None, Some(10)).is_none());
+        assert!(invoke_scalar(Some("10"), None).is_none());
+        assert!(invoke_scalar(None, None).is_none());
+    }
+
+    #[test]
+    fn array_values_with_scalar_base_takes_fast_path() {
+        let u = ToNumberUdf::new();
+        let values: ArrayRef = Arc::new(StringArray::from(vec![
+            Some("FA34"),
+            Some("nope"),
+            None,
+            Some("ff"),
+        ]));
+        let args = ScalarFunctionArgs {
+            args: vec![
+                ColumnarValue::Array(values),
+                ColumnarValue::Scalar(ScalarValue::Int32(Some(16))),
+            ],
+            arg_fields: vec![
+                Arc::new(Field::new("v", DataType::Utf8, true)),
+                Arc::new(Field::new("b", DataType::Int32, true)),
+            ],
+            number_rows: 4,
+            return_field: Arc::new(Field::new("out", DataType::Float64, true)),
+            config_options: Arc::new(Default::default()),
+        };
+        let arr = match u.invoke_with_args(args).unwrap() {
+            ColumnarValue::Array(a) => a,
+            other => panic!("expected array, got {other:?}"),
+        };
+        let f = arr.as_primitive::();
+        assert_eq!(f.value(0), 64052.0);
+        assert!(f.is_null(1), "unparseable → NULL");
+        assert!(f.is_null(2), "null input → NULL");
+        assert_eq!(f.value(3), 255.0);
+    }
+
+    #[test]
+    fn scalar_null_base_produces_all_null_output() {
+        let u = ToNumberUdf::new();
+        let values: ArrayRef = Arc::new(StringArray::from(vec![
+            Some("FA34"),
+            Some("nope"),
+            Some("10"),
+        ]));
+        let args = ScalarFunctionArgs {
+            args: vec![
+                ColumnarValue::Array(values),
+                ColumnarValue::Scalar(ScalarValue::Int32(None)),
+            ],
+            arg_fields: vec![
+                Arc::new(Field::new("v", DataType::Utf8, true)),
+                Arc::new(Field::new("b", DataType::Int32, true)),
+            ],
+            number_rows: 3,
+            return_field: Arc::new(Field::new("out", DataType::Float64, true)),
+            config_options: Arc::new(Default::default()),
+        };
+        let arr = match u.invoke_with_args(args).unwrap() {
+            ColumnarValue::Array(a) => a,
+            other => panic!("expected array, got {other:?}"),
+        };
+        let f = arr.as_primitive::();
+        assert_eq!(f.len(), 3);
+        for i in 0..3 {
+            assert!(f.is_null(i), "row {i} must be NULL");
+        }
+    }
+
+    #[test]
+    fn scalar_out_of_range_base_produces_all_null_output() {
+        let u = ToNumberUdf::new();
+        let values: ArrayRef = Arc::new(StringArray::from(vec![Some("FA34"), Some("10")]));
+        let args = ScalarFunctionArgs {
+            args: vec![
+                ColumnarValue::Array(values),
+                ColumnarValue::Scalar(ScalarValue::Int32(Some(42))),
+            ],
+            arg_fields: vec![
+                Arc::new(Field::new("v", DataType::Utf8, true)),
+                Arc::new(Field::new("b", DataType::Int32, true)),
+            ],
+            number_rows: 2,
+            return_field: Arc::new(Field::new("out", DataType::Float64, true)),
+            config_options: Arc::new(Default::default()),
+        };
+        let arr = match u.invoke_with_args(args).unwrap() {
+            ColumnarValue::Array(a) => a,
+            other => panic!("expected array, got {other:?}"),
+        };
+        let f = arr.as_primitive::();
+        assert!(f.is_null(0));
+        assert!(f.is_null(1));
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tostring.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tostring.rs
new file mode 100644
index 0000000000000..e9593e1a0d79f
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/tostring.rs
@@ -0,0 +1,595 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! [`tostring(value, format)`](https://docs.opensearch.org/latest/sql-and-ppl/ppl/functions/conversion/#tostring)
+//! format modes in one UDF.
+//!
+//! Mirror's PPL's
+//! [`ToStringFunction`](https://github.com/opensearch-project/sql/blob/main/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java)
+//!
+//! Supported format values:
+//! * `"binary"` — number → base-2 string of its integer part. Negative values use the
+//!   signed-magnitude representation `-1xxxx` (matches `BigInteger.toString(2)`).
+//! * `"hex"` — number → lowercase hexadecimal of its integer part. Negative values use
+//!   the signed-magnitude representation `-xx` (matches `BigInteger.toString(16)`).
+//! * `"commas"` — number with comma grouping; rounded to 2 decimals when the value has
+//!   a fractional component, otherwise no decimals.
+//! * `"duration"` — integer seconds → `HH:MM:SS` **wall-clock** rendering.
+//! * `"duration_millis"` — integer milliseconds → `HH:MM:SS` wall-clock rendering.
+
+use std::any::Any;
+use std::hash::{Hash, Hasher};
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, ArrayRef, AsArray, StringArray};
+use datafusion::arrow::datatypes::{DataType, Float64Type, Int64Type};
+use datafusion::common::{exec_err, Result, ScalarValue};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, Volatility,
+};
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(ToStringUdf::new()));
+}
+
+/// `tostring(bigint, varchar)` / `tostring(fp64, varchar)` → varchar.
+#[derive(Debug)]
+pub struct ToStringUdf {
+    signature: Signature,
+}
+
+impl ToStringUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::one_of(
+                vec![
+                    TypeSignature::Exact(vec![DataType::Int64, DataType::Utf8]),
+                    TypeSignature::Exact(vec![DataType::Float64, DataType::Utf8]),
+                ],
+                Volatility::Immutable,
+            ),
+        }
+    }
+}
+
+impl Default for ToStringUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+// `ScalarUDFImpl` requires DynEq+DynHash. All `ToStringUdf` instances are functionally
+// identical — there's no meaningful "parameterization" — so they compare equal and hash
+// identically.
+impl PartialEq for ToStringUdf {
+    fn eq(&self, _: &Self) -> bool {
+        true
+    }
+}
+impl Eq for ToStringUdf {}
+impl Hash for ToStringUdf {
+    fn hash(&self, state: &mut H) {
+        "tostring".hash(state);
+    }
+}
+
+impl ScalarUDFImpl for ToStringUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "tostring"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result {
+        Ok(DataType::Utf8)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() != 2 {
+            return exec_err!(
+                "tostring expects exactly 2 arguments (value, format), got {}",
+                args.args.len()
+            );
+        }
+
+        let format_col = &args.args[1];
+
+        match &args.args[0] {
+            ColumnarValue::Scalar(ScalarValue::Int64(value)) => Ok(ColumnarValue::Scalar(
+                ScalarValue::Utf8(scalar_to_str(*value, format_col, 0, format_i64_as)?),
+            )),
+            ColumnarValue::Scalar(ScalarValue::Float64(value)) => Ok(ColumnarValue::Scalar(
+                ScalarValue::Utf8(scalar_to_str(*value, format_col, 0, format_f64_as)?),
+            )),
+            ColumnarValue::Scalar(other) => {
+                exec_err!("tostring: expected BIGINT or DOUBLE value, got {other:?}")
+            }
+
+            ColumnarValue::Array(arr) => match arr.data_type() {
+                DataType::Int64 => {
+                    let typed = arr.as_primitive::();
+                    let out: StringArray = (0..typed.len())
+                        .map(|i| {
+                            if typed.is_null(i) {
+                                Ok(None)
+                            } else {
+                                scalar_to_str(Some(typed.value(i)), format_col, i, format_i64_as)
+                            }
+                        })
+                        .collect::>>()?
+                        .into_iter()
+                        .collect();
+                    Ok(ColumnarValue::Array(Arc::new(out) as ArrayRef))
+                }
+                DataType::Float64 => {
+                    let typed = arr.as_primitive::();
+                    let out: StringArray = (0..typed.len())
+                        .map(|i| {
+                            if typed.is_null(i) {
+                                Ok(None)
+                            } else {
+                                scalar_to_str(Some(typed.value(i)), format_col, i, format_f64_as)
+                            }
+                        })
+                        .collect::>>()?
+                        .into_iter()
+                        .collect();
+                    Ok(ColumnarValue::Array(Arc::new(out) as ArrayRef))
+                }
+                other => exec_err!("tostring: expected Int64 or Float64 value array, got {other:?}"),
+            },
+        }
+    }
+}
+
+/// Per-row dispatcher for one concrete value type.
+///
+/// * `value` — the already-unwrapped optional; {@code None} short-circuits to `Ok(None)`
+///   (null propagation).
+/// * `format_col` — the format `ColumnarValue`. Resolved once per row via
+///   [`format_at`]; the row index is only consulted when the caller passed an array.
+/// * `row` — only used when `format_col` is an array; ignored for the scalar path.
+/// * `formatter` — per-value-type rendering function (one for `i64`, one for `f64`).
+fn scalar_to_str(
+    value: Option,
+    format_col: &ColumnarValue,
+    row: usize,
+    formatter: fn(T, &str) -> String,
+) -> Result> {
+    let Some(v) = value else {
+        return Ok(None);
+    };
+    let format = format_at(format_col, row)?;
+    match format {
+        Some(f) => Ok(Some(formatter(v, f.as_str()))),
+        None => Ok(None),
+    }
+}
+
+/// Pulls the format string for `row` out of `format_col`. Returns `Ok(None)` for null
+/// format cells.
+fn format_at(format_col: &ColumnarValue, row: usize) -> Result> {
+    match format_col {
+        ColumnarValue::Scalar(ScalarValue::Utf8(opt)) | ColumnarValue::Scalar(ScalarValue::LargeUtf8(opt)) => {
+            Ok(opt.clone())
+        }
+        ColumnarValue::Scalar(other) => {
+            exec_err!("tostring: format must be VARCHAR, got {other:?}")
+        }
+        ColumnarValue::Array(arr) => match arr.data_type() {
+            DataType::Utf8 => {
+                let strs = arr.as_string::();
+                if strs.is_null(row) {
+                    Ok(None)
+                } else {
+                    Ok(Some(strs.value(row).to_string()))
+                }
+            }
+            DataType::LargeUtf8 => {
+                let strs = arr.as_string::();
+                if strs.is_null(row) {
+                    Ok(None)
+                } else {
+                    Ok(Some(strs.value(row).to_string()))
+                }
+            }
+            other => exec_err!("tostring: expected Utf8 format array, got {other:?}"),
+        },
+    }
+}
+
+/// Format modes, case-sensitive to match the SQL plugin's Java reference
+/// (`ToStringFunction.DURATION_FORMAT`, etc.).
+mod mode {
+    pub const BINARY: &str = "binary";
+    pub const HEX: &str = "hex";
+    pub const COMMAS: &str = "commas";
+    pub const DURATION: &str = "duration";
+    pub const DURATION_MILLIS: &str = "duration_millis";
+}
+
+/// Render an `i64` value per the requested format. Unknown modes fall through to plain
+/// decimal rendering.
+fn format_i64_as(value: i64, format: &str) -> String {
+    match format {
+        mode::BINARY => format_binary_i64(value),
+        mode::HEX => format_hex_i64(value),
+        mode::COMMAS => format_commas_i64(value),
+        mode::DURATION => format_duration_seconds(value),
+        mode::DURATION_MILLIS => format_duration_seconds(value.div_euclid(1_000)),
+        _ => value.to_string(),
+    }
+}
+
+/// Render an `f64` value per the requested format. Mirrors the Java reference's strategy
+/// of routing `binary` / `hex` / `duration*` through `BigDecimal.toBigInteger()` — i.e.
+/// truncate toward zero, then apply the integer formatter.
+fn format_f64_as(value: f64, format: &str) -> String {
+    match format {
+        mode::BINARY => format_binary_i64(truncate_to_i64(value)),
+        mode::HEX => format_hex_i64(truncate_to_i64(value)),
+        mode::COMMAS => format_commas_f64(value),
+        mode::DURATION => format_duration_seconds(truncate_to_i64(value)),
+        mode::DURATION_MILLIS => format_duration_seconds(truncate_to_i64(value).div_euclid(1_000)),
+        _ => {
+            if !value.is_finite() {
+                return value.to_string();
+            }
+            // Drop a trailing `.0` so `tostring(42.0)` reads as `"42"`, matching
+            // `BigDecimal.valueOf(42.0).toString() == "42.0"` → passed to `NumberFormat` with no
+            // decimals would print `"42"`.
+            let rendered = format!("{value}");
+            rendered.strip_suffix(".0").map_or(rendered.clone(), |s| s.to_string())
+        }
+    }
+}
+
+fn truncate_to_i64(value: f64) -> i64 {
+    if !value.is_finite() {
+        return 0;
+    }
+    value as i64
+}
+
+fn format_binary_i64(v: i64) -> String {
+    // Mirrors `BigInteger.toString(2)` for positive values; for negatives the Java
+    // reference emits a leading '-' via BigInteger's signed radix representation. We match
+    // that by formatting the absolute value and re-prepending the sign.
+    if v < 0 {
+        // Work in i128 so i64::MIN doesn't overflow on abs.
+        format!("-{:b}", (v as i128).unsigned_abs())
+    } else {
+        format!("{:b}", v as u64)
+    }
+}
+
+fn format_hex_i64(v: i64) -> String {
+    // Same rationale as `format_binary_i64` — BigInteger.toString(16) on negatives prints
+    // a leading '-'. Lowercase to match the Java reference (BigInteger uses lowercase).
+    if v < 0 {
+        format!("-{:x}", (v as i128).unsigned_abs())
+    } else {
+        format!("{:x}", v as u64)
+    }
+}
+
+fn format_commas_i64(v: i64) -> String {
+    let is_negative = v < 0;
+    let abs_str = if is_negative {
+        format!("{}", (v as i128).unsigned_abs())
+    } else {
+        v.to_string()
+    };
+    let mut out = String::with_capacity(abs_str.len() + abs_str.len() / 3 + 1);
+    if is_negative {
+        out.push('-');
+    }
+    insert_thousands_separators(&abs_str, &mut out);
+    out
+}
+
+fn format_commas_f64(v: f64) -> String {
+    // Non-finite fall through to native rendering (matches Double.toString for Infinity/NaN).
+    if !v.is_finite() {
+        return v.to_string();
+    }
+    let is_negative = v.is_sign_negative();
+    // rounds the number to the nearest two decimal places.
+    let rounded = format!("{:.2}", v.abs());
+    let (whole, frac) = rounded
+        .split_once('.')
+        .expect("{:.2} always produces a decimal point");
+    let mut out = String::with_capacity(rounded.len() + whole.len() / 3 + 2);
+    if is_negative {
+        out.push('-');
+    }
+    insert_thousands_separators(whole, &mut out);
+    // Drop `.00` so integral values render without a decimal tail (`39225 → "39,225"`),
+    // but keep a single-digit fractional (`.50`) when present
+    if frac != "00" {
+        out.push('.');
+        // Trim a trailing '0' when exactly one digit would be meaningful, e.g. `.50 → .5`.
+        if frac.ends_with('0') {
+            out.push_str(&frac[..frac.len() - 1]);
+        } else {
+            out.push_str(frac);
+        }
+    }
+    out
+}
+
+/// Allocation-free thousands-separator insertion. Appends `digits` to `out` with a `,`
+/// after every 3 digits counted from the right.
+fn insert_thousands_separators(digits: &str, out: &mut String) {
+    let bytes = digits.as_bytes();
+    let len = bytes.len();
+    for (i, b) in bytes.iter().enumerate() {
+        let from_right = len - i;
+        out.push(*b as char);
+        if from_right > 1 && (from_right - 1) % 3 == 0 {
+            out.push(',');
+        }
+    }
+}
+
+/// Format a signed number of seconds to  wall-clock `HH:MM:SS` format
+fn format_duration_seconds(total_seconds: i64) -> String {
+    // Use `rem_euclid` to get the non-negative second-of-day.
+    let second_of_day = total_seconds.rem_euclid(86_400);
+    let h = second_of_day / 3600;
+    let m = (second_of_day / 60) % 60;
+    let s = second_of_day % 60;
+    format!("{:02}:{:02}:{:02}", h, m, s)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use datafusion::arrow::array::{Float64Array, Int64Array};
+    use datafusion::arrow::datatypes::Field;
+
+    fn udf() -> ToStringUdf {
+        ToStringUdf::new()
+    }
+
+    fn invoke_scalar(
+        value: ScalarValue,
+        value_type: DataType,
+        format: &str,
+    ) -> Result {
+        let u = udf();
+        let return_field = Arc::new(Field::new(u.name(), DataType::Utf8, true));
+        let args = ScalarFunctionArgs {
+            args: vec![
+                ColumnarValue::Scalar(value),
+                ColumnarValue::Scalar(ScalarValue::Utf8(Some(format.to_string()))),
+            ],
+            arg_fields: vec![
+                Arc::new(Field::new("v", value_type, true)),
+                Arc::new(Field::new("f", DataType::Utf8, true)),
+            ],
+            number_rows: 1,
+            return_field,
+            config_options: Arc::new(Default::default()),
+        };
+        u.invoke_with_args(args)
+    }
+
+    fn invoke_array(value: ArrayRef, format: &str) -> Result {
+        let u = udf();
+        let return_field = Arc::new(Field::new(u.name(), DataType::Utf8, true));
+        let value_type = value.data_type().clone();
+        let args = ScalarFunctionArgs {
+            args: vec![
+                ColumnarValue::Array(value),
+                ColumnarValue::Scalar(ScalarValue::Utf8(Some(format.to_string()))),
+            ],
+            arg_fields: vec![
+                Arc::new(Field::new("v", value_type, true)),
+                Arc::new(Field::new("f", DataType::Utf8, true)),
+            ],
+            number_rows: 3,
+            return_field,
+            config_options: Arc::new(Default::default()),
+        };
+        u.invoke_with_args(args)
+    }
+
+    fn utf8(v: ColumnarValue) -> String {
+        match v {
+            ColumnarValue::Scalar(ScalarValue::Utf8(Some(s))) => s,
+            other => panic!("expected Utf8 scalar, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn hex_matches_bigint_tohex() {
+        let out = invoke_scalar(ScalarValue::Int64(Some(39225)), DataType::Int64, "hex").unwrap();
+        assert_eq!(utf8(out), "9939");
+    }
+
+    #[test]
+    fn binary_matches_biginteger_tostring_2() {
+        let out = invoke_scalar(ScalarValue::Int64(Some(39225)), DataType::Int64, "binary").unwrap();
+        assert_eq!(utf8(out), "1001100100111001");
+    }
+
+    #[test]
+    fn binary_negative_uses_signed_biginteger_repr() {
+        let out = invoke_scalar(ScalarValue::Int64(Some(-5)), DataType::Int64, "binary").unwrap();
+        assert_eq!(utf8(out), "-101");
+    }
+
+    #[test]
+    fn commas_integer_doc_example() {
+        let out = invoke_scalar(ScalarValue::Int64(Some(39225)), DataType::Int64, "commas").unwrap();
+        assert_eq!(utf8(out), "39,225");
+    }
+
+    #[test]
+    fn commas_float_rounds_to_two_decimals() {
+        let out = invoke_scalar(
+            ScalarValue::Float64(Some(1234.5678)),
+            DataType::Float64,
+            "commas",
+        )
+        .unwrap();
+        assert_eq!(utf8(out), "1,234.57");
+    }
+
+    #[test]
+    fn commas_float_drops_trailing_double_zero() {
+        let out = invoke_scalar(
+            ScalarValue::Float64(Some(39225.0)),
+            DataType::Float64,
+            "commas",
+        )
+        .unwrap();
+        assert_eq!(utf8(out), "39,225");
+    }
+
+    #[test]
+    fn duration_seconds_doc_example() {
+        let out = invoke_scalar(ScalarValue::Int64(Some(6500)), DataType::Int64, "duration").unwrap();
+        assert_eq!(utf8(out), "01:48:20");
+    }
+
+    #[test]
+    fn duration_bigdecimal_positive_example() {
+        let out = invoke_scalar(ScalarValue::Int64(Some(3661)), DataType::Int64, "duration").unwrap();
+        assert_eq!(utf8(out), "01:01:01");
+    }
+
+    #[test]
+    fn duration_negative_wraps_to_pre_epoch_clock_time() {
+        let out = invoke_scalar(
+            ScalarValue::Float64(Some(-3661.4)),
+            DataType::Float64,
+            "duration",
+        )
+        .unwrap();
+        assert_eq!(utf8(out), "22:58:59");
+    }
+
+    #[test]
+    fn duration_wraps_modulo_24h() {
+        let day = invoke_scalar(
+            ScalarValue::Int64(Some(86_400)),
+            DataType::Int64,
+            "duration",
+        )
+        .unwrap();
+        assert_eq!(utf8(day), "00:00:00");
+
+        let day_plus_hour = invoke_scalar(
+            ScalarValue::Int64(Some(86_400 + 3_600)),
+            DataType::Int64,
+            "duration",
+        )
+        .unwrap();
+        assert_eq!(utf8(day_plus_hour), "01:00:00");
+    }
+
+    #[test]
+    fn duration_millis_truncates_subseconds() {
+        let out = invoke_scalar(
+            ScalarValue::Int64(Some(6_500_999)),
+            DataType::Int64,
+            "duration_millis",
+        )
+        .unwrap();
+        assert_eq!(utf8(out), "01:48:20");
+    }
+
+    #[test]
+    fn duration_millis_negative_wraps_via_floor_div() {
+        let out = invoke_scalar(
+            ScalarValue::Int64(Some(-3_661_000)),
+            DataType::Int64,
+            "duration_millis",
+        )
+        .unwrap();
+        assert_eq!(utf8(out), "22:58:59");
+    }
+
+    #[test]
+    fn unknown_format_falls_through_to_plain_decimal() {
+        let out = invoke_scalar(ScalarValue::Int64(Some(42)), DataType::Int64, "xyzzy").unwrap();
+        assert_eq!(utf8(out), "42");
+    }
+
+    #[test]
+    fn null_value_yields_null() {
+        let out = invoke_scalar(ScalarValue::Int64(None), DataType::Int64, "hex").unwrap();
+        match out {
+            ColumnarValue::Scalar(ScalarValue::Utf8(None)) => {}
+            other => panic!("expected Utf8(None), got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn null_format_yields_null() {
+        let u = udf();
+        let return_field = Arc::new(Field::new(u.name(), DataType::Utf8, true));
+        let args = ScalarFunctionArgs {
+            args: vec![
+                ColumnarValue::Scalar(ScalarValue::Int64(Some(42))),
+                ColumnarValue::Scalar(ScalarValue::Utf8(None)),
+            ],
+            arg_fields: vec![
+                Arc::new(Field::new("v", DataType::Int64, true)),
+                Arc::new(Field::new("f", DataType::Utf8, true)),
+            ],
+            number_rows: 1,
+            return_field,
+            config_options: Arc::new(Default::default()),
+        };
+        let out = u.invoke_with_args(args).unwrap();
+        match out {
+            ColumnarValue::Scalar(ScalarValue::Utf8(None)) => {}
+            other => panic!("expected Utf8(None), got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn array_int_hex_with_scalar_format() {
+        let array: ArrayRef = Arc::new(Int64Array::from(vec![Some(15), None, Some(255)]));
+        let out = invoke_array(array, "hex").unwrap();
+        match out {
+            ColumnarValue::Array(arr) => {
+                let s = arr.as_string::();
+                assert_eq!(s.value(0), "f");
+                assert!(s.is_null(1));
+                assert_eq!(s.value(2), "ff");
+            }
+            other => panic!("expected array, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn array_float_commas_with_scalar_format() {
+        let array: ArrayRef = Arc::new(Float64Array::from(vec![Some(1234.5), None, Some(0.0)]));
+        let out = invoke_array(array, "commas").unwrap();
+        match out {
+            ColumnarValue::Array(arr) => {
+                let s = arr.as_string::();
+                assert_eq!(s.value(0), "1,234.5");
+                assert!(s.is_null(1));
+                assert_eq!(s.value(2), "0");
+            }
+            other => panic!("expected array, got {other:?}"),
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
index e883dac590ec9..a7414eac65639 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
@@ -190,7 +190,7 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
         ScalarFunction.CURRENT_TIME,
         ScalarFunction.CURTIME,
         ScalarFunction.CONVERT_TZ,
-        ScalarFunction.UNIX_TIMESTAMP
+        ScalarFunction.UNIX_TIMESTAMP,
         // DATE(expr) / TIME(expr) / MAKETIME(h,m,s) are intentionally not advertised:
         // PPL's Calcite binding for these returns VARCHAR rather than DATE/TIME, so
         // downstream `year(date(ts))` / `hour(maketime(...))` lowers to
@@ -201,6 +201,25 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
         // and we'd need a dedicated adapter + yaml entry to route it to DataFusion's
         // date_part. Left on legacy engine until that adapter lands; PPL date-part
         // functions cover the same semantics.
+        ScalarFunction.ASCII,
+        ScalarFunction.CONCAT_WS,
+        ScalarFunction.LEFT,
+        ScalarFunction.LENGTH,
+        ScalarFunction.CHAR_LENGTH,
+        ScalarFunction.LOCATE,
+        ScalarFunction.POSITION,
+        ScalarFunction.LOWER,
+        ScalarFunction.LTRIM,
+        ScalarFunction.REVERSE,
+        ScalarFunction.RIGHT,
+        ScalarFunction.RTRIM,
+        ScalarFunction.TRIM,
+        ScalarFunction.SUBSTR,
+        ScalarFunction.UPPER,
+        ScalarFunction.STRCMP,
+        ScalarFunction.TOSTRING,
+        ScalarFunction.NUMBER_TO_STRING,
+        ScalarFunction.TONUMBER
     );
 
     private static final Set AGG_FUNCTIONS = Set.of(
@@ -309,13 +328,16 @@ public Map scalarFunctionAdapters() {
                     Map.entry(ScalarFunction.HOUR, hour),
                     Map.entry(ScalarFunction.HOUR_OF_DAY, hour),
                     Map.entry(ScalarFunction.LIKE, new LikeAdapter()),
+                    Map.entry(ScalarFunction.LOCATE, new PositionAdapter()),
                     Map.entry(ScalarFunction.MICROSECOND, DatePartAdapters.microsecond()),
                     Map.entry(ScalarFunction.MINUTE, minute),
                     Map.entry(ScalarFunction.MINUTE_OF_HOUR, minute),
                     Map.entry(ScalarFunction.MOD, new StdOperatorRewriteAdapter("MOD", SqlStdOperatorTable.MOD)),
                     Map.entry(ScalarFunction.MONTH, month),
                     Map.entry(ScalarFunction.MONTH_OF_YEAR, month),
+                    Map.entry(ScalarFunction.NUMBER_TO_STRING, new ToStringFunctionAdapter()),
                     Map.entry(ScalarFunction.NOW, now),
+                    Map.entry(ScalarFunction.POSITION, new PositionAdapter()),
                     Map.entry(ScalarFunction.QUARTER, DatePartAdapters.quarter()),
                     Map.entry(ScalarFunction.REGEXP_REPLACE, new RegexpReplaceAdapter()),
                     Map.entry(ScalarFunction.SARG_PREDICATE, new SargAdapter()),
@@ -323,7 +345,12 @@ public Map scalarFunctionAdapters() {
                     Map.entry(ScalarFunction.SCALAR_MIN, nameMapping(SqlLibraryOperators.LEAST)),
                     Map.entry(ScalarFunction.SIGN, nameMapping(SignumFunction.FUNCTION)),
                     Map.entry(ScalarFunction.SINH, new HyperbolicOperatorAdapter(SqlLibraryOperators.SINH)),
+                    Map.entry(ScalarFunction.STRCMP, new StrcmpFunctionAdapter()),
+                    Map.entry(ScalarFunction.SUBSTR, nameMapping(SqlStdOperatorTable.SUBSTRING)),
+                    Map.entry(ScalarFunction.SUBSTRING, nameMapping(SqlStdOperatorTable.SUBSTRING)),
                     Map.entry(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter()),
+                    Map.entry(ScalarFunction.TONUMBER, new ToNumberFunctionAdapter()),
+                    Map.entry(ScalarFunction.TOSTRING, new ToStringFunctionAdapter()),
                     Map.entry(ScalarFunction.UNIX_TIMESTAMP, new UnixTimestampAdapter()),
                     Map.entry(ScalarFunction.WEEK, week),
                     Map.entry(ScalarFunction.WEEK_OF_YEAR, week),
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
index 2614d3c3f9ebb..23f7012bb8208 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
@@ -99,6 +99,10 @@ public class DataFusionFragmentConvertor implements FragmentConvertor {
      */
     private static final List ADDITIONAL_SCALAR_SIGS = List.of(
         FunctionMappings.s(DelegatedPredicateFunction.FUNCTION, DelegatedPredicateFunction.NAME),
+        FunctionMappings.s(SqlStdOperatorTable.ASCII, "ascii"),
+        FunctionMappings.s(SqlStdOperatorTable.CHAR_LENGTH, "length"),
+        FunctionMappings.s(SqlLibraryOperators.CONCAT_FUNCTION, "concat"),
+        FunctionMappings.s(SqlLibraryOperators.CONCAT_WS, "concat_ws"),
         FunctionMappings.s(SqlLibraryOperators.ILIKE, "ilike"),
         FunctionMappings.s(SqlLibraryOperators.DATE_PART, "date_part"),
         FunctionMappings.s(ConvertTzAdapter.LOCAL_CONVERT_TZ_OP, "convert_tz"),
@@ -110,6 +114,11 @@ public class DataFusionFragmentConvertor implements FragmentConvertor {
         FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"),
         FunctionMappings.s(SqlStdOperatorTable.REPLACE, "replace"),
         FunctionMappings.s(SqlLibraryOperators.REGEXP_REPLACE_3, "regexp_replace"),
+        FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"),
+        FunctionMappings.s(SqlLibraryOperators.REVERSE, "reverse"),
+        FunctionMappings.s(PositionAdapter.STRPOS, "strpos"),
+        FunctionMappings.s(ToNumberFunctionAdapter.TONUMBER, "tonumber"),
+        FunctionMappings.s(ToStringFunctionAdapter.TOSTRING, "tostring"),
         FunctionMappings.s(SqlStdOperatorTable.TRUNCATE, "trunc"),
         FunctionMappings.s(SqlStdOperatorTable.CBRT, "cbrt"),
         FunctionMappings.s(SqlStdOperatorTable.COT, "cot"),
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java
index 01449e41a20dc..a866e09db23fb 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java
@@ -20,6 +20,7 @@
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.opensearch.analytics.backend.EngineResultBatch;
 import org.opensearch.analytics.backend.EngineResultStream;
+import org.opensearch.analytics.exec.ArrowValues;
 import org.opensearch.be.datafusion.nativelib.NativeBridge;
 import org.opensearch.be.datafusion.nativelib.StreamHandle;
 import org.opensearch.common.annotation.ExperimentalApi;
@@ -200,7 +201,7 @@ public Object getFieldValue(String fieldName, int rowIndex) {
             if (vector == null) {
                 throw new IllegalArgumentException("Unknown field: " + fieldName);
             }
-            return vector.getObject(rowIndex);
+            return ArrowValues.toJavaValue(vector, rowIndex);
         }
     }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PositionAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PositionAdapter.java
new file mode 100644
index 0000000000000..53016105ebc92
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PositionAdapter.java
@@ -0,0 +1,104 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+/**
+ * Adapts PPL {@code POSITION(substr IN str[, start])} to DataFusion's {@code strpos(str, substr)}.
+ *
+ * 
        PPL emits a 2-arg {@code POSITION(substr, str)} for {@code locate(substr, str)} /
+ * {@code position(substr IN str)}, and a 3-arg {@code POSITION(substr, str, start)} for
+ * PPL's 3-arg {@code locate(substr, str, start)} (PPL's frontend maps both surface spellings
+ * into {@link SqlKind#POSITION}). DataFusion's {@code strpos} is
+ * {@code (str, substr)} with no {@code start} parameter, so:
+ *
+ * 
          
+ *   
        • 2-arg form: swap operands → {@code strpos(str, substr)}.
          • 
+ *   
        • 3-arg form: decompose as
+ *       {@code CASE WHEN strpos(substring(str, start), substr) = 0
+ *               THEN 0
+ *               ELSE strpos(substring(str, start), substr) + start - 1
+ *               END}.
+ *       Preserves 1-indexed semantics and returns 0 when the substring isn't found.
          • 
+ * 
        
+ *
+ * @opensearch.internal
+ */
+class PositionAdapter implements ScalarFunctionAdapter {
+
+    /** Locally-declared {@code strpos} operator. The
+     *  {@link io.substrait.isthmus.expression.FunctionMappings.Sig} entry in
+     *  {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS} pairs it with the
+     *  {@code strpos} extension name declared in {@code opensearch_scalar_functions.yaml}. */
+    static final SqlFunction STRPOS = new SqlFunction(
+        "strpos",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.INTEGER,
+        null,
+        OperandTypes.ANY_ANY,
+        SqlFunctionCategory.STRING
+    );
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        List operands = original.getOperands();
+        if (operands.size() < 2 || operands.size() > 3) {
+            return original;
+        }
+
+        RexBuilder rexBuilder = cluster.getRexBuilder();
+        RexNode substr = operands.get(0);
+        RexNode str = operands.get(1);
+
+        if (operands.size() == 2) {
+            // Simple swap: POSITION(substr, str) → strpos(str, substr)
+            return rexBuilder.makeCall(original.getType(), STRPOS, List.of(str, substr));
+        }
+
+        // 3-arg: POSITION(substr, str, start) → decompose via substring.
+        RexNode start = operands.get(2);
+        RelDataTypeFactory typeFactory = cluster.getTypeFactory();
+        RelDataType intType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.INTEGER), true);
+
+        // tail = substring(str, start)
+        RexNode tail = rexBuilder.makeCall(SqlStdOperatorTable.SUBSTRING, str, start);
+        // posInTail = strpos(tail, substr) — 1-indexed, 0 when not found.
+        RexNode posInTail = rexBuilder.makeCall(STRPOS, tail, substr);
+
+        RexNode zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO, intType);
+        RexNode one = rexBuilder.makeExactLiteral(BigDecimal.ONE, intType);
+        RexNode isZero = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, posInTail, zero);
+        RexNode adjusted = rexBuilder.makeCall(
+            SqlStdOperatorTable.MINUS,
+            rexBuilder.makeCall(SqlStdOperatorTable.PLUS, posInTail, start),
+            one
+        );
+
+        // CASE WHEN posInTail = 0 THEN 0 ELSE posInTail + start - 1 END
+        return rexBuilder.makeCall(intType, SqlStdOperatorTable.CASE, List.of(isZero, zero, adjusted));
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/StrcmpFunctionAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/StrcmpFunctionAdapter.java
new file mode 100644
index 0000000000000..b59be4bf17008
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/StrcmpFunctionAdapter.java
@@ -0,0 +1,93 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+/**
+ * Adapts PPL {@code strcmp(a, b)} into a pure Substrait/DataFusion CASE expression.
+ * 
          
+ *   
        • {@code -1} when {@code a < b}
          • 
+ *   
        • {@code  0} when {@code a = b}
          • 
+ *   
        • {@code  1} when {@code a > b}
          • 
+ *   
        • {@code NULL} when either operand is {@code NULL}
          • 
+ * 
        
+ *
+ * 
        Rewrite:
+ * 
        {@code
+ *   strcmp(a, b)
+ *     →
+ *   CASE
+ *     WHEN a IS NULL OR b IS NULL THEN NULL
+ *     WHEN a < b THEN -1
+ *     WHEN a = b THEN  0
+ *     ELSE              1
+ *   END
+ * }
        
+ *
+ * 
        Why the adapter beats a row-by-row Rust UDF: the {@code <} and {@code =}
+ * comparisons between {@code StringArray} operands lower to arrow-rs compute
+ * kernels ({@code arrow::compute::lt}, {@code arrow::compute::eq}) which are
+ * SIMD-vectorized on x86_64 (AVX2) and arm64 (NEON). The CASE ({@code ifelse})
+ * is also an arrow vectorized kernel. A UDF that loops
+ * {@code for i in 0..n { str::cmp(...) }} per row is strictly slower — it
+ * amortizes FFI over the batch but the inner compare is scalar.
+ *
+ * 
        PPL's frontend reverses {@code strcmp}'s args vs. user order. This adapter
+ * swaps them back — operands are consumed as {@code (arg1, arg0)} from the
+ * original call so the resulting {@code a < b} / {@code a = b} maps 1:1 to the
+ * user-intended {@code -1 / 0 / 1} convention.
+ *
+ * @opensearch.internal
+ */
+class StrcmpFunctionAdapter implements ScalarFunctionAdapter {
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        List operands = original.getOperands();
+        if (operands.size() != 2) {
+            return original;
+        }
+        RexBuilder rexBuilder = cluster.getRexBuilder();
+        // Swap to undo the PPL frontend's argument reversal.
+        RexNode a = operands.get(1);
+        RexNode b = operands.get(0);
+
+        RelDataType intType = cluster.getTypeFactory()
+            .createTypeWithNullability(cluster.getTypeFactory().createSqlType(SqlTypeName.INTEGER), true);
+        RexNode neg1 = rexBuilder.makeExactLiteral(BigDecimal.valueOf(-1), intType);
+        RexNode zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO, intType);
+        RexNode one = rexBuilder.makeExactLiteral(BigDecimal.ONE, intType);
+        RexNode nullLit = rexBuilder.makeNullLiteral(intType);
+
+        // NULL propagation must be explicit — SQL comparators on NULL return NULL, but
+        // the CASE below needs to short-circuit them so we don't fall through to the
+        // `ELSE 1` branch when either operand is NULL.
+        RexNode aIsNull = rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, a);
+        RexNode bIsNull = rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, b);
+        RexNode anyNull = rexBuilder.makeCall(SqlStdOperatorTable.OR, aIsNull, bIsNull);
+
+        RexNode lessThan = rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, a, b);
+        RexNode equalTo = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, a, b);
+
+        // CASE WHEN anyNull THEN NULL WHEN aPer the
+ * 
+ * PPL {@code tonumber} docs:
+ *
+ * 
        
+ *   {@code tonumber(string[, base])} — converts the string value to a number. If the
+ *   {@code base} parameter is omitted, base 10 is assumed. Returns NULL when the string
+ *   cannot be parsed.
+ * 
        
+ *
+ * @opensearch.internal
+ */
+class ToNumberFunctionAdapter implements ScalarFunctionAdapter {
+
+    static final SqlFunction TONUMBER = new SqlFunction(
+        "tonumber",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.DOUBLE,
+        null,
+        OperandTypes.family(),
+        SqlFunctionCategory.USER_DEFINED_FUNCTION
+    );
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        List operands = original.getOperands();
+        if (operands.isEmpty()) {
+            return original;
+        }
+        RexNode value = operands.get(0);
+
+        // 1-arg — implied base 10. DataFusion's built-in CAST(str AS DOUBLE) returns NULL on
+        // parse failure.
+        if (operands.size() == 1) {
+            return makeSafeDoubleCast(value, cluster);
+        }
+
+        // 2-arg — rebuild as tonumber(CAST(value AS VARCHAR), CAST(base AS INTEGER))
+        if (operands.size() == 2) {
+            RexNode base = operands.get(1);
+            RexNode normalizedValue = castTo(value, SqlTypeName.VARCHAR, cluster);
+            RexNode normalizedBase = castTo(base, SqlTypeName.INTEGER, cluster);
+            return cluster.getRexBuilder().makeCall(original.getType(), TONUMBER, List.of(normalizedValue, normalizedBase));
+        }
+
+        return original;
+    }
+
+    /**
+     * Casts {@code operand} to {@code target} while preserving its nullability. Returns the
+     * operand unchanged when it's already the target type so we don't layer redundant CASTs.
+     */
+    private static RexNode castTo(RexNode operand, SqlTypeName target, RelOptCluster cluster) {
+        if (operand.getType().getSqlTypeName() == target) {
+            return operand;
+        }
+        RelDataTypeFactory factory = cluster.getTypeFactory();
+        RelDataType targetType = factory.createTypeWithNullability(factory.createSqlType(target), operand.getType().isNullable());
+        return cluster.getRexBuilder().makeCast(targetType, operand);
+    }
+
+    /**
+     * Wraps the single operand in a SAFE_CAST to DOUBLE. SAFE_CAST serialises as a substrait
+     * cast with {@code FAILURE_BEHAVIOR_RETURN_NULL}, which DataFusion maps to
+     * {@code try_cast} — so parse failures yield NULL instead of raising.
+     */
+    private static RexNode makeSafeDoubleCast(RexNode value, RelOptCluster cluster) {
+        RelDataTypeFactory factory = cluster.getTypeFactory();
+        RelDataType doubleType = factory.createTypeWithNullability(factory.createSqlType(SqlTypeName.DOUBLE), true);
+        // RexBuilder.makeCast(type, exp, matchNullability, safe) — the `safe` flag produces a
+        // SqlKind.SAFE_CAST call instead of a plain CAST.
+        return cluster.getRexBuilder().makeCast(doubleType, value, true, true);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ToStringFunctionAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ToStringFunctionAdapter.java
new file mode 100644
index 0000000000000..583b5975383eb
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ToStringFunctionAdapter.java
@@ -0,0 +1,225 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunction;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * Rewrites PPL {@code tostring(value[, format])} into a DataFusion-compatible expression.
+ *
+ * 
        Per the
+ * 
+ * PPL {@code tostring} docs:
+ *
+ * 
        
+ *   {@code tostring(value[, format])} — converts the value to a string representation.
+ *   If a format is provided, converts numbers to the specified format type. For Boolean
+ *   values, converts to {@code TRUE} or {@code FALSE}. The {@code format} parameter is
+ *   only used when {@code value} is a number and is ignored for Booleans.
+ * 
        
+ *
+ * 
        Handles two arrival shapes:
+ * 
          
+ *   
        1. Native {@code tostring(value[, format])} — dispatched as-is.
          2. 
+ *   
        2. {@code NUMBER_TO_STRING(num)} — PPL's {@code ExtendedRexBuilder.makeCast} override
+ *       intercepts {@code CAST(num AS VARCHAR)} for approximate-numeric / decimal source types
+ *       and rewrites it into a call to {@code PPLBuiltinOperators.NUMBER_TO_STRING}. That
+ *       PPL-plugin-defined UDF isn't in any Substrait catalog, so isthmus cannot resolve it.
+ *       We treat it as the single-arg {@code tostring} shape and lower it to a plain VARCHAR
+ *       CAST.
+ *   
          3. 
+ * 
        
+ *
+ * @opensearch.internal
+ */
+class ToStringFunctionAdapter implements ScalarFunctionAdapter {
+
+    /**
+     * Target numeric type for a given PPL format mode. {@link #COMMAS} preserves fractional
+     * precision because it renders rounded to 2 decimals. All other modes fold to BIGINT because
+     * their output is defined on the integer part of the value (cf. PPL docs: binary/hex/duration
+     * are integer conversions).
+     */
+    private enum Format {
+        HEX("hex", SqlTypeName.BIGINT),
+        BINARY("binary", SqlTypeName.BIGINT),
+        COMMAS("commas", /* preserveFractional */ null),
+        DURATION("duration", SqlTypeName.BIGINT),
+        DURATION_MILLIS("duration_millis", SqlTypeName.BIGINT);
+
+        final String literal;
+        /**
+         * Target type for the numeric argument, or {@code null} for {@link #COMMAS} which
+         * picks BIGINT vs DOUBLE based on the source type.
+         */
+        final SqlTypeName fixedTarget;
+
+        Format(String literal, SqlTypeName fixedTarget) {
+            this.literal = literal;
+            this.fixedTarget = fixedTarget;
+        }
+
+        /** Case-insensitive lookup matching the PPL spec. Returns {@code null} when unknown. */
+        static Format from(String modeLiteral) {
+            if (modeLiteral == null) return null;
+            String lower = modeLiteral.toLowerCase(Locale.ROOT);
+            for (Format f : values()) {
+                if (f.literal.equals(lower)) return f;
+            }
+            return null;
+        }
+
+        /**
+         * Choose the target type for the numeric argument given the source RexNode type.
+         * For every mode except {@link #COMMAS} this is a fixed BIGINT; COMMAS preserves
+         * fractional types by routing through DOUBLE and widens integers to BIGINT.
+         */
+        SqlTypeName targetFor(SqlTypeName source) {
+            if (fixedTarget != null) {
+                return fixedTarget;
+            }
+            return isFractional(source) ? SqlTypeName.DOUBLE : SqlTypeName.BIGINT;
+        }
+    }
+
+    /**
+     * Synthetic {@code tostring} operator used when we rebuild the 2-arg call. It mirrors the
+     * shape of the PPL operator but is keyed on the literal name {@code "tostring"} — which is
+     * the name the Rust UDF registers under and the YAML extension declares. A dedicated operator
+     * gives the isthmus name-based resolver a deterministic hook; we don't have to rely on the
+     * incoming RexCall's operator being correctly named.
+     */
+    static final SqlFunction TOSTRING = new SqlFunction(
+        "tostring",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.VARCHAR,
+        null,
+        OperandTypes.family(),
+        SqlFunctionCategory.USER_DEFINED_FUNCTION
+    );
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        List operands = original.getOperands();
+        if (operands.isEmpty()) {
+            return original;
+        }
+        RexNode value = operands.get(0);
+
+        // NUMBER_TO_STRING is PPL's intercepted numeric-to-varchar cast. Treat it identically to
+        // the 1-arg tostring shape: lower to a plain CAST(value AS VARCHAR) that isthmus /
+        // DataFusion can serialise.
+        if (ScalarFunction.NUMBER_TO_STRING.name().equalsIgnoreCase(original.getOperator().getName())) {
+            return makeVarcharCast(original, value, cluster);
+        }
+
+        // tostring renders booleans as the uppercase literals TRUE / FALSE (format arg is ignored for booleans).
+        if (value.getType().getSqlTypeName() == SqlTypeName.BOOLEAN) {
+            return makeBooleanToString(original, value, cluster);
+        }
+
+        // 1-arg: tostring(x) → CAST(x AS VARCHAR)
+        if (operands.size() == 1) {
+            return makeVarcharCast(original, value, cluster);
+        }
+
+        // 2-arg: tostring(x, format). Only rewrite when the format arg is a string literal with
+        // a known mode; otherwise pass the call through so the downstream planner fails loudly.
+        if (operands.size() == 2 && operands.get(1) instanceof RexLiteral formatLit && isStringLiteral(formatLit)) {
+            Format mode = Format.from(formatLit.getValueAs(String.class));
+            if (mode != null) {
+                return rebuildCall(original, value, formatLit, mode, cluster);
+            }
+        }
+
+        return original;
+    }
+
+    /**
+     * Lower a BOOLEAN-valued {@code tostring} call to
+     * {@code CASE WHEN value THEN 'TRUE' WHEN NOT value THEN 'FALSE' END}.
+     */
+    private static RexNode makeBooleanToString(RexCall original, RexNode value, RelOptCluster cluster) {
+        RelDataTypeFactory factory = cluster.getTypeFactory();
+        RelDataType varcharType = factory.createTypeWithNullability(
+            factory.createSqlType(SqlTypeName.VARCHAR),
+            original.getType().isNullable()
+        );
+        RexNode trueLit = cluster.getRexBuilder().makeLiteral("TRUE");
+        RexNode falseLit = cluster.getRexBuilder().makeLiteral("FALSE");
+        RexNode notValue = cluster.getRexBuilder().makeCall(SqlStdOperatorTable.NOT, value);
+        return cluster.getRexBuilder()
+            .makeCall(
+                varcharType,
+                SqlStdOperatorTable.CASE,
+                List.of(value, trueLit, notValue, falseLit, cluster.getRexBuilder().makeNullLiteral(varcharType))
+            );
+    }
+
+    /**
+     * Rebuild the 2-arg call as {@code tostring(CAST(value AS ), formatLit)}. The CAST
+     * ensures the numeric argument matches the Rust UDF's declared BIGINT/FLOAT64 signatures;
+     * the format literal is forwarded verbatim so the UDF's per-row dispatch sees the exact
+     * mode string the caller supplied.
+     */
+    private static RexNode rebuildCall(RexCall original, RexNode value, RexLiteral formatLit, Format mode, RelOptCluster cluster) {
+        SqlTypeName target = mode.targetFor(value.getType().getSqlTypeName());
+        RexNode normalized = castTo(value, target, cluster);
+        return cluster.getRexBuilder().makeCall(original.getType(), TOSTRING, List.of(normalized, formatLit));
+    }
+
+    private static boolean isStringLiteral(RexLiteral literal) {
+        SqlTypeName sqlType = literal.getType().getSqlTypeName();
+        return sqlType == SqlTypeName.CHAR || sqlType == SqlTypeName.VARCHAR;
+    }
+
+    private static boolean isFractional(SqlTypeName type) {
+        return type == SqlTypeName.FLOAT || type == SqlTypeName.DOUBLE || type == SqlTypeName.REAL || type == SqlTypeName.DECIMAL;
+    }
+
+    /**
+     * Casts {@code operand} to {@code target} while preserving its nullability. Returns the
+     * operand unchanged when it's already the target type so we don't layer redundant CASTs.
+     */
+    private static RexNode castTo(RexNode operand, SqlTypeName target, RelOptCluster cluster) {
+        if (operand.getType().getSqlTypeName() == target) {
+            return operand;
+        }
+        RelDataTypeFactory factory = cluster.getTypeFactory();
+        RelDataType targetType = factory.createTypeWithNullability(factory.createSqlType(target), operand.getType().isNullable());
+        return cluster.getRexBuilder().makeCast(targetType, operand);
+    }
+
+    private static RexNode makeVarcharCast(RexCall original, RexNode value, RelOptCluster cluster) {
+        RelDataTypeFactory factory = cluster.getTypeFactory();
+        RelDataType varcharType = factory.createTypeWithNullability(
+            factory.createSqlType(SqlTypeName.VARCHAR),
+            original.getType().isNullable()
+        );
+        return cluster.getRexBuilder().makeCast(varcharType, value);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
index 34e70f5c3bdc5..4caf7c185dc96 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
@@ -253,3 +253,67 @@ scalar_functions:
           - value: "string"
             name: "replacement"
         return: string
+
+  # ascii(str) — Unicode code point of the first character.
+  - name: "ascii"
+    description: "Return the unicode code point of the first character of the input string."
+    impls:
+      - args:
+          - { name: str, value: "varchar" }
+        nullability: DECLARED_OUTPUT
+        return: i32
+      - args:
+          - { name: str, value: "string" }
+        nullability: DECLARED_OUTPUT
+        return: i32
+
+  # strpos(str, substr) — 1-based position of substr in str, 0 if not found.
+  # Target of PPL's `locate` and `position` adapters.
+  - name: "strpos"
+    description: "Return the 1-based position of `substr` within `str`, or 0 when absent."
+    impls:
+      - args:
+          - { name: str, value: "string" }
+          - { name: substr, value: "string" }
+        nullability: DECLARED_OUTPUT
+        return: i32
+      - args:
+          - { name: str, value: "varchar" }
+          - { name: substr, value: "varchar" }
+        nullability: DECLARED_OUTPUT
+        return: i32
+      - args:
+          - { name: str, value: "string" }
+          - { name: substr, value: "varchar" }
+        nullability: DECLARED_OUTPUT
+        return: i32
+      - args:
+          - { name: str, value: "varchar" }
+          - { name: substr, value: "string" }
+        nullability: DECLARED_OUTPUT
+        return: i32
+
+  # tostring(x, format) — (hex / binary / commas / duration / duration_millis).
+  - name: "tostring"
+    description: "Convert a number to a string using the requested format (hex/binary/commas/duration/duration_millis)."
+    impls:
+      - args:
+          - { name: value, value: i64 }
+          - { name: format, value: string }
+        nullability: DECLARED_OUTPUT
+        return: string
+      - args:
+          - { name: value, value: fp64 }
+          - { name: format, value: string }
+        nullability: DECLARED_OUTPUT
+        return: string
+
+  # tonumber(string, base) — parse `string` as a base-N integer
+  - name: "tonumber"
+    description: "Parse a string to a number in the given radix (2-36). Returns NULL on parse failure."
+    impls:
+      - args:
+          - { name: value, value: string }
+          - { name: base, value: i32 }
+        nullability: DECLARED_OUTPUT
+        return: fp64
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/PositionAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/PositionAdapterTests.java
new file mode 100644
index 0000000000000..22a9ea44420cf
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/PositionAdapterTests.java
@@ -0,0 +1,236 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+/**
+ * Unit tests for {@link PositionAdapter}.
+ *
+ * 
        Coverage:
+ * 
          
+ *   
        • 2-arg form: {@code POSITION(substr, str)} swaps operands to
+ *       {@code strpos(str, substr)}.
          • 
+ *   
        • 3-arg form: {@code POSITION(substr, str, start)} decomposes into a CASE
+ *       expression around {@code substring(str, start)} + {@code strpos} + offset
+ *       arithmetic so the 1-indexed {@code start} parameter and the
+ *       "{@code 0} on not found" contract both hold.
          • 
+ *   
        • Malformed arity passes through unchanged (no 0, 1, or 4-arg rewrite).
          • 
+ * 
        
+ */
+public class PositionAdapterTests extends OpenSearchTestCase {
+
+    private static final SqlFunction POSITION = new SqlFunction(
+        "POSITION",
+        SqlKind.POSITION,
+        ReturnTypes.INTEGER,
+        null,
+        OperandTypes.family(),
+        SqlFunctionCategory.STRING
+    );
+
+    private final PositionAdapter adapter = new PositionAdapter();
+
+    /** {@code POSITION('U', 'FURNITURE')} → {@code strpos('FURNITURE', 'U')}. */
+    public void testTwoArgSwapsOperands() {
+        Cluster cluster = newCluster();
+        RexNode substr = cluster.stringLiteral("U");
+        RexNode str = cluster.stringLiteral("FURNITURE");
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, substr, str);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        RexCall outCall = assertStrposCall(out);
+        assertEquals("strpos must be (str, substr) — 2 operands", 2, outCall.getOperands().size());
+        assertSame("first operand is str (was the second POSITION arg)", str, outCall.getOperands().get(0));
+        assertSame("second operand is substr (was the first POSITION arg)", substr, outCall.getOperands().get(1));
+    }
+
+    /**
+     * {@code POSITION('U', 'FURNITURE', 3)} decomposes to
+     * {@code CASE WHEN strpos(substring(str, start), substr) = 0 THEN 0 ELSE strpos(...) + start - 1 END}.
+     * This test asserts the outer CASE shape; the inner sub-calls are validated separately.
+     */
+    public void testThreeArgDecomposesToCaseOfSubstringStrpos() {
+        Cluster cluster = newCluster();
+        RexNode substr = cluster.stringLiteral("U");
+        RexNode str = cluster.stringLiteral("FURNITURE");
+        RexNode start = cluster.intLiteral(3);
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, substr, str, start);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        assertEquals("3-arg POSITION lowers to CASE", SqlKind.CASE, out.getKind());
+        RexCall caseCall = (RexCall) out;
+        assertEquals("CASE shape — WHEN cond THEN 0 ELSE adjusted", 3, caseCall.getOperands().size());
+
+        // operand[0]: strpos(substring(str, start), substr) = 0
+        RexCall whenCond = (RexCall) caseCall.getOperands().get(0);
+        assertEquals("WHEN is an equality test", SqlKind.EQUALS, whenCond.getKind());
+
+        // operand[1]: the THEN value is the literal 0.
+        assertEquals(
+            "THEN returns 0 when substring didn't contain substr",
+            0,
+            ((org.apache.calcite.rex.RexLiteral) caseCall.getOperands().get(1)).getValueAs(Integer.class).intValue()
+        );
+
+        // operand[2]: the ELSE arm is strpos(...) + start - 1.
+        RexCall elseArm = (RexCall) caseCall.getOperands().get(2);
+        assertEquals("ELSE performs the final offset subtraction", SqlKind.MINUS, elseArm.getKind());
+    }
+
+    public void testThreeArgElseArmBuildsSubstringAndStrpos() {
+        Cluster cluster = newCluster();
+        RexNode substr = cluster.stringLiteral("U");
+        RexNode str = cluster.stringLiteral("FURNITURE");
+        RexNode start = cluster.intLiteral(3);
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, substr, str, start);
+
+        RexCall caseCall = (RexCall) adapter.adapt(call, List.of(), cluster.cluster);
+
+        // ELSE shape: MINUS(PLUS(strpos(substring(str, start), substr), start), 1)
+        RexCall minusCall = (RexCall) caseCall.getOperands().get(2);
+        RexCall plusCall = (RexCall) minusCall.getOperands().get(0);
+        assertEquals(SqlKind.PLUS, plusCall.getKind());
+        RexCall strposInElse = (RexCall) plusCall.getOperands().get(0);
+        assertSame("ELSE arm's strpos reuses the shared operator", PositionAdapter.STRPOS, strposInElse.getOperator());
+
+        RexCall substringCall = (RexCall) strposInElse.getOperands().get(0);
+        assertSame(
+            "substring call uses the standard SqlStdOperatorTable.SUBSTRING",
+            SqlStdOperatorTable.SUBSTRING,
+            substringCall.getOperator()
+        );
+        assertSame("substring(str, start) — str is the original second POSITION operand", str, substringCall.getOperands().get(0));
+        assertSame("substring(str, start) — start is the original third POSITION operand", start, substringCall.getOperands().get(1));
+        assertSame("strpos substr is the original first POSITION operand", substr, strposInElse.getOperands().get(1));
+    }
+
+    public void testThreeArgWhenConditionMirrorsElseStrpos() {
+        Cluster cluster = newCluster();
+        RexNode substr = cluster.stringLiteral("U");
+        RexNode str = cluster.stringLiteral("FURNITURE");
+        RexNode start = cluster.intLiteral(3);
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, substr, str, start);
+
+        RexCall caseCall = (RexCall) adapter.adapt(call, List.of(), cluster.cluster);
+
+        RexCall whenCond = (RexCall) caseCall.getOperands().get(0);
+        // WHEN: strpos(substring(str, start), substr) = 0
+        RexCall strposInWhen = (RexCall) whenCond.getOperands().get(0);
+        assertSame("WHEN condition's strpos is the shared operator", PositionAdapter.STRPOS, strposInWhen.getOperator());
+        RexCall substringInWhen = (RexCall) strposInWhen.getOperands().get(0);
+        assertSame(SqlStdOperatorTable.SUBSTRING, substringInWhen.getOperator());
+        assertSame(str, substringInWhen.getOperands().get(0));
+        assertSame(start, substringInWhen.getOperands().get(1));
+    }
+
+    public void testAdaptedStrposIsTheSharedOperatorInstance() {
+        Cluster cluster = newCluster();
+        RexNode substr = cluster.stringLiteral("a");
+        RexNode str = cluster.stringLiteral("abc");
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, substr, str);
+
+        RexCall outCall = assertStrposCall(adapter.adapt(call, List.of(), cluster.cluster));
+
+        assertSame(
+            "adapter must emit the shared PositionAdapter.STRPOS instance, not a clone",
+            PositionAdapter.STRPOS,
+            outCall.getOperator()
+        );
+        assertEquals(
+            "operator name is 'strpos' — what DataFusion's substrait consumer expects",
+            "strpos",
+            PositionAdapter.STRPOS.getName()
+        );
+    }
+
+    public void testOneArgPassesThrough() {
+        Cluster cluster = newCluster();
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(POSITION, cluster.stringLiteral("a"));
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+        assertSame("1-arg POSITION is malformed and must pass through", call, out);
+    }
+
+    public void testFourArgPassesThrough() {
+        Cluster cluster = newCluster();
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(
+            POSITION,
+            cluster.stringLiteral("a"),
+            cluster.stringLiteral("abc"),
+            cluster.intLiteral(1),
+            cluster.intLiteral(1)
+        );
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+        assertSame("4-arg POSITION is malformed and must pass through", call, out);
+    }
+
+    // ── Helpers ───────────────────────────────────────────────────────────────
+
+    /** Assert the adapted call is a 2-arg {@code strpos} call routed through the shared operator. */
+    private static RexCall assertStrposCall(RexNode out) {
+        assertTrue("expected a RexCall, got " + out.getClass(), out instanceof RexCall);
+        RexCall outCall = (RexCall) out;
+        assertSame(
+            "operator is the shared strpos registered against the FunctionMappings.Sig",
+            PositionAdapter.STRPOS,
+            outCall.getOperator()
+        );
+        return outCall;
+    }
+
+    private static Cluster newCluster() {
+        RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl();
+        RexBuilder rexBuilder = new RexBuilder(typeFactory);
+        HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+        RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder);
+        return new Cluster(cluster, typeFactory, rexBuilder);
+    }
+
+    private static final class Cluster {
+        final RelOptCluster cluster;
+        final RelDataTypeFactory typeFactory;
+        final RexBuilder rexBuilder;
+
+        Cluster(RelOptCluster cluster, RelDataTypeFactory typeFactory, RexBuilder rexBuilder) {
+            this.cluster = cluster;
+            this.typeFactory = typeFactory;
+            this.rexBuilder = rexBuilder;
+        }
+
+        RexNode intLiteral(int value) {
+            RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER);
+            return rexBuilder.makeExactLiteral(BigDecimal.valueOf(value), intType);
+        }
+
+        RexNode stringLiteral(String value) {
+            return rexBuilder.makeLiteral(value);
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/StrcmpFunctionAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/StrcmpFunctionAdapterTests.java
new file mode 100644
index 0000000000000..130412f24c1fd
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/StrcmpFunctionAdapterTests.java
@@ -0,0 +1,110 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.List;
+
+/**
+ * Unit tests for {@link StrcmpFunctionAdapter}.
+ *
+ * 
        The adapter decomposes {@code strcmp(a, b)} into a CASE expression using built-in
+ * comparison operators ({@code <}, {@code =}) and swaps the arguments to undo the PPL
+ * frontend's reversal. These tests verify the CASE shape and argument swap.
+ */
+public class StrcmpFunctionAdapterTests extends OpenSearchTestCase {
+
+    private static final SqlFunction STRCMP = new SqlFunction(
+        "STRCMP",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.INTEGER,
+        null,
+        OperandTypes.family(),
+        SqlFunctionCategory.USER_DEFINED_FUNCTION
+    );
+
+    private final StrcmpFunctionAdapter adapter = new StrcmpFunctionAdapter();
+
+    private RelDataTypeFactory typeFactory;
+    private RexBuilder rexBuilder;
+    private RelOptCluster cluster;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        typeFactory = new JavaTypeFactoryImpl();
+        rexBuilder = new RexBuilder(typeFactory);
+        HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+        cluster = RelOptCluster.create(planner, rexBuilder);
+    }
+
+    private RexNode varcharInputRef(int index) {
+        RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true);
+        return rexBuilder.makeInputRef(varcharType, index);
+    }
+
+    /** The adapter produces a CASE expression with INTEGER return type. */
+    public void testTwoArgProducesCaseExpression() {
+        RexNode arg0 = rexBuilder.makeLiteral("Amber");
+        RexNode arg1 = varcharInputRef(0);
+        RexCall call = (RexCall) rexBuilder.makeCall(STRCMP, arg0, arg1);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster);
+
+        assertTrue("result must be a RexCall", out instanceof RexCall);
+        RexCall outCall = (RexCall) out;
+        assertEquals("decomposed to CASE", SqlKind.CASE, outCall.getKind());
+        assertEquals("return type is INTEGER", SqlTypeName.INTEGER, outCall.getType().getSqlTypeName());
+        // CASE has 7 operands: (anyNull, nullLit, lessThan, neg1, equalTo, zero, one)
+        assertEquals("CASE has 7 operands (3 WHEN/THEN pairs + ELSE)", 7, outCall.getOperands().size());
+    }
+
+    /** Arguments are swapped — arg1 becomes 'a' (lhs) and arg0 becomes 'b' (rhs) in the comparisons. */
+    public void testArgumentsAreSwapped() {
+        RexNode arg0 = rexBuilder.makeLiteral("literal_rhs");
+        RexNode arg1 = varcharInputRef(0); // column — should become lhs after swap
+        RexCall call = (RexCall) rexBuilder.makeCall(STRCMP, arg0, arg1);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster);
+
+        RexCall caseCall = (RexCall) out;
+        // The LESS_THAN comparison is at operand index 2: WHEN a < b THEN -1
+        // After swap: a = arg1 (inputRef), b = arg0 (literal)
+        RexCall lessThan = (RexCall) caseCall.getOperands().get(2);
+        assertEquals(SqlKind.LESS_THAN, lessThan.getKind());
+        // lhs of < should be the column (arg1), rhs should be the literal (arg0)
+        assertSame("lhs of < is the column (original arg1)", arg1, lessThan.getOperands().get(0));
+        assertSame("rhs of < is the literal (original arg0)", arg0, lessThan.getOperands().get(1));
+    }
+
+    /** Non-standard arity (e.g. 1 arg) passes through unchanged. */
+    public void testSingleArgPassesThrough() {
+        RexCall call = (RexCall) rexBuilder.makeCall(STRCMP, varcharInputRef(0));
+
+        RexNode out = adapter.adapt(call, List.of(), cluster);
+
+        assertSame("non-2-arg call passes through unchanged", call, out);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToNumberFunctionAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToNumberFunctionAdapterTests.java
new file mode 100644
index 0000000000000..ac6337e998485
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToNumberFunctionAdapterTests.java
@@ -0,0 +1,164 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+public class ToNumberFunctionAdapterTests extends OpenSearchTestCase {
+
+    /** Synthetic tonumber operator used to build input RexCalls */
+    private static final SqlFunction TONUMBER = new SqlFunction(
+        "tonumber",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.DOUBLE,
+        null,
+        OperandTypes.ANY_ANY,
+        SqlFunctionCategory.USER_DEFINED_FUNCTION
+    );
+
+    private final ToNumberFunctionAdapter adapter = new ToNumberFunctionAdapter();
+
+    /** {@code tonumber(x)} rewrites to {@code CAST(x AS DOUBLE)}. */
+    public void testSingleArgRewritesToDoubleCast() {
+        Cluster cluster = newCluster();
+        RexNode input = cluster.stringLiteral("4598.678");
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TONUMBER, input);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        assertEquals("kind=SAFE_CAST", SqlKind.SAFE_CAST, out.getKind());
+        assertEquals("result type is DOUBLE", SqlTypeName.DOUBLE, out.getType().getSqlTypeName());
+        RexCall castCall = (RexCall) out;
+        assertEquals("single operand", 1, castCall.getOperands().size());
+        assertSame("operand preserved by identity", input, castCall.getOperands().get(0));
+    }
+
+    /**
+     * {@code tonumber(x, base)} stays a {@code tonumber}
+     */
+    public void testTwoArgKeepsTonumberCallAndNormalizesOperands() {
+        Cluster cluster = newCluster();
+        RexNode input = cluster.stringLiteral("FA34");
+        RexNode base = cluster.intLiteral(16);
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TONUMBER, input, base);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        RexCall outCall = assertTonumberCall(out);
+        assertEquals("two operands — value + base", 2, outCall.getOperands().size());
+        RexNode valueArg = outCall.getOperands().get(0);
+        RexNode baseArg = outCall.getOperands().get(1);
+        assertEquals("value arg normalized to VARCHAR", SqlTypeName.VARCHAR, valueArg.getType().getSqlTypeName());
+        assertEquals("base arg normalized to INTEGER", SqlTypeName.INTEGER, baseArg.getType().getSqlTypeName());
+    }
+
+    /** {@code tonumber(VARCHAR, INTEGER)} — already-normalized operands don't get redundant CASTs. */
+    public void testTwoArgOnMatchingTypesSkipsRedundantCast() {
+        Cluster cluster = newCluster();
+        RexNode input = cluster.varcharInputRef(0);
+        RexNode base = cluster.intLiteral(2);
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TONUMBER, input, base);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        RexCall outCall = assertTonumberCall(out);
+        assertSame("VARCHAR operand kept as-is", input, outCall.getOperands().get(0));
+        assertSame("INTEGER literal kept as-is", base, outCall.getOperands().get(1));
+    }
+
+    /** Zero-operand {@code tonumber} is degenerate; adapter should pass it through unchanged. */
+    public void testZeroArgPassesThrough() {
+        Cluster cluster = newCluster();
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TONUMBER);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        assertSame(call, out);
+    }
+
+    /** Arities above 2 aren't declared in the PPL spec — pass through so planning fails loudly. */
+    public void testThreeArgPassesThrough() {
+        Cluster cluster = newCluster();
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(
+            TONUMBER,
+            cluster.stringLiteral("10"),
+            cluster.intLiteral(10),
+            cluster.intLiteral(0)
+        );
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        assertSame(call, out);
+    }
+
+    // ── Helpers ───────────────────────────────────────────────────────────────
+
+    private static RexCall assertTonumberCall(RexNode out) {
+        assertTrue("expected a RexCall, got " + out.getClass(), out instanceof RexCall);
+        RexCall outCall = (RexCall) out;
+        assertSame(
+            "operator is the synthetic `tonumber` that resolves to the Rust UDF",
+            ToNumberFunctionAdapter.TONUMBER,
+            outCall.getOperator()
+        );
+        return outCall;
+    }
+
+    private static Cluster newCluster() {
+        RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl();
+        RexBuilder rexBuilder = new RexBuilder(typeFactory);
+        HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+        RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder);
+        return new Cluster(cluster, typeFactory, rexBuilder);
+    }
+
+    private static final class Cluster {
+        final RelOptCluster cluster;
+        final RelDataTypeFactory typeFactory;
+        final RexBuilder rexBuilder;
+
+        Cluster(RelOptCluster cluster, RelDataTypeFactory typeFactory, RexBuilder rexBuilder) {
+            this.cluster = cluster;
+            this.typeFactory = typeFactory;
+            this.rexBuilder = rexBuilder;
+        }
+
+        RexNode intLiteral(int value) {
+            RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER);
+            return rexBuilder.makeExactLiteral(BigDecimal.valueOf(value), intType);
+        }
+
+        RexNode stringLiteral(String value) {
+            return rexBuilder.makeLiteral(value);
+        }
+
+        RexNode varcharInputRef(int index) {
+            RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true);
+            return rexBuilder.makeInputRef(varcharType, index);
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToStringFunctionAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToStringFunctionAdapterTests.java
new file mode 100644
index 0000000000000..beafd6e34f5e7
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/ToStringFunctionAdapterTests.java
@@ -0,0 +1,286 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+public class ToStringFunctionAdapterTests extends OpenSearchTestCase {
+
+    private final ToStringFunctionAdapter adapter = new ToStringFunctionAdapter();
+
+    /** Synthetic tostring operator used to build input RexCalls. */
+    private static final SqlFunction TOSTRING = new SqlFunction(
+        "tostring",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.VARCHAR,
+        null,
+        OperandTypes.ANY_ANY,
+        SqlFunctionCategory.USER_DEFINED_FUNCTION
+    );
+
+    /** {@code tostring(x)} rewrites to {@code CAST(x AS VARCHAR)}. */
+    public void testSingleArgRewritesToVarcharCast() {
+        Cluster cluster = newCluster();
+        RexNode input = cluster.intLiteral(39225);
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, input);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        assertEquals("kind=CAST", SqlKind.CAST, out.getKind());
+        assertEquals("result type is VARCHAR", SqlTypeName.VARCHAR, out.getType().getSqlTypeName());
+        RexCall castCall = (RexCall) out;
+        assertEquals("single operand", 1, castCall.getOperands().size());
+        assertSame("operand preserved by identity", input, castCall.getOperands().get(0));
+    }
+
+    /**
+     * {@code tostring(x, 'hex')} stays a {@code tostring} call (operator rebound to the
+     * name the Rust UDF registers under) with the numeric argument widened to BIGINT.
+     */
+    public void testHexFormatKeepsTostringCallAndWidensToBigint() {
+        Cluster cluster = newCluster();
+        RexNode intInput = cluster.intLiteral(255);
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, intInput, cluster.stringLiteral("hex"));
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        RexCall outCall = assertTostringCall(out);
+        assertEquals("two operands — value + format literal", 2, outCall.getOperands().size());
+        RexNode operand = outCall.getOperands().get(0);
+        assertEquals("integer widened to BIGINT to match the UDF signature", SqlTypeName.BIGINT, operand.getType().getSqlTypeName());
+    }
+
+    /** {@code tostring(bigint, 'binary')} — no CAST needed because the operand is already BIGINT. */
+    public void testBinaryFormatOnBigintDoesNotReinsertCast() {
+        Cluster cluster = newCluster();
+        RexNode bigintInput = cluster.rexBuilder.makeExactLiteral(
+            BigDecimal.valueOf(100L),
+            cluster.typeFactory.createSqlType(SqlTypeName.BIGINT)
+        );
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, bigintInput, cluster.stringLiteral("binary"));
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        RexCall outCall = assertTostringCall(out);
+        assertSame("bigint operand is used directly — no redundant CAST", bigintInput, outCall.getOperands().get(0));
+    }
+
+    /** {@code tostring(double, 'commas')} preserves fractional precision by routing through DOUBLE. */
+    public void testCommasFormatOnDoublePreservesFractionalPrecision() {
+        Cluster cluster = newCluster();
+        RexNode doubleInput = cluster.rexBuilder.makeApproxLiteral(
+            BigDecimal.valueOf(12.5),
+            cluster.typeFactory.createSqlType(SqlTypeName.DOUBLE)
+        );
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, doubleInput, cluster.stringLiteral("commas"));
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        RexCall outCall = assertTostringCall(out);
+        RexNode operand = outCall.getOperands().get(0);
+        assertEquals(
+            "double kept as DOUBLE — 2-decimal rounding happens inside the UDF",
+            SqlTypeName.DOUBLE,
+            operand.getType().getSqlTypeName()
+        );
+    }
+
+    /** {@code tostring(int, 'commas')} widens integer sources to BIGINT, same as every other mode. */
+    public void testCommasFormatOnIntegerWidensToBigint() {
+        Cluster cluster = newCluster();
+        RexNode intInput = cluster.intLiteral(12345);
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, intInput, cluster.stringLiteral("commas"));
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        RexCall outCall = assertTostringCall(out);
+        assertEquals(SqlTypeName.BIGINT, outCall.getOperands().get(0).getType().getSqlTypeName());
+    }
+
+    /** {@code tostring(x, 'xyzzy')} is an unsupported format; the call is returned unchanged. */
+    public void testUnsupportedFormatPassesThrough() {
+        Cluster cluster = newCluster();
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, cluster.intLiteral(42), cluster.stringLiteral("xyzzy"));
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        assertSame("unknown format mode should leave the RexCall untouched so downstream planning fails loudly", call, out);
+    }
+
+    /**
+     * {@code tostring(BOOLEAN)} lowers to a {@code CASE} that emits the uppercase
+     * {@code 'TRUE'} / {@code 'FALSE'}
+     */
+    public void testBooleanOneArgLowersToCase() {
+        Cluster cluster = newCluster();
+        RexNode boolInput = cluster.booleanLiteral(true);
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, boolInput);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        assertEquals("boolean tostring lowers to CASE", SqlKind.CASE, out.getKind());
+        assertEquals("CASE returns VARCHAR", SqlTypeName.VARCHAR, out.getType().getSqlTypeName());
+        RexCall caseCall = (RexCall) out;
+        // CASE shape: WHEN value THEN 'TRUE' WHEN NOT value THEN 'FALSE' ELSE NULL.
+        assertEquals("CASE has two WHEN branches plus ELSE — 5 operands total", 5, caseCall.getOperands().size());
+        assertEquals("first THEN literal is uppercase TRUE", "TRUE", ((RexLiteral) caseCall.getOperands().get(1)).getValueAs(String.class));
+        assertEquals(
+            "second THEN literal is uppercase FALSE",
+            "FALSE",
+            ((RexLiteral) caseCall.getOperands().get(3)).getValueAs(String.class)
+        );
+    }
+
+    /**
+     * {@code tostring(BOOLEAN, '')} ignores the format
+     */
+    public void testBooleanTwoArgIgnoresFormat() {
+        Cluster cluster = newCluster();
+        RexNode boolInput = cluster.nullableBooleanInputRef(0);
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(TOSTRING, boolInput, cluster.stringLiteral("hex"));
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        assertEquals("boolean tostring(x, fmt) lowers to CASE regardless of format", SqlKind.CASE, out.getKind());
+        RexCall caseCall = (RexCall) out;
+        assertEquals("TRUE", ((RexLiteral) caseCall.getOperands().get(1)).getValueAs(String.class));
+        assertEquals("FALSE", ((RexLiteral) caseCall.getOperands().get(3)).getValueAs(String.class));
+    }
+
+    // ── NUMBER_TO_STRING: PPL's intercepted numeric-to-varchar cast ───────────
+
+    /**
+     * PPL's {@code ExtendedRexBuilder.makeCast} rewrites {@code CAST(num AS VARCHAR)} into a
+     * {@code NUMBER_TO_STRING(num)} call. That PPL-plugin UDF isn't in any Substrait catalog,
+     * so the adapter must lower it back to a plain VARCHAR cast for DataFusion — DataFusion's
+     * native numeric-to-string formatting is used in place of Java's {@code Number.toString}.
+     */
+    public void testNumberToStringLowersToVarcharCast() {
+        Cluster cluster = newCluster();
+        RexNode doubleInput = cluster.rexBuilder.makeApproxLiteral(
+            BigDecimal.valueOf(12.3),
+            cluster.typeFactory.createSqlType(SqlTypeName.DOUBLE)
+        );
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(NUMBER_TO_STRING, doubleInput);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        assertEquals("NUMBER_TO_STRING lowers to CAST", SqlKind.CAST, out.getKind());
+        assertEquals("result type is VARCHAR", SqlTypeName.VARCHAR, out.getType().getSqlTypeName());
+        RexCall castCall = (RexCall) out;
+        assertEquals("single operand", 1, castCall.getOperands().size());
+        assertSame("numeric operand preserved by identity", doubleInput, castCall.getOperands().get(0));
+    }
+
+    /**
+     * {@code NUMBER_TO_STRING} over a DECIMAL source — still lowers to a VARCHAR cast. The
+     * adapter branches on operator name, not operand type, so decimal and approximate-numeric
+     * paths both route identically.
+     */
+    public void testNumberToStringOnDecimalLowersToVarcharCast() {
+        Cluster cluster = newCluster();
+        RelDataType decimalType = cluster.typeFactory.createSqlType(SqlTypeName.DECIMAL, 10, 2);
+        RexNode decimalInput = cluster.rexBuilder.makeExactLiteral(BigDecimal.valueOf(12.3), decimalType);
+        RexCall call = (RexCall) cluster.rexBuilder.makeCall(NUMBER_TO_STRING, decimalInput);
+
+        RexNode out = adapter.adapt(call, List.of(), cluster.cluster);
+
+        assertEquals("decimal NUMBER_TO_STRING also lowers to CAST", SqlKind.CAST, out.getKind());
+        assertEquals(SqlTypeName.VARCHAR, out.getType().getSqlTypeName());
+        RexCall castCall = (RexCall) out;
+        assertSame(decimalInput, castCall.getOperands().get(0));
+    }
+
+    /** Synthetic {@code NUMBER_TO_STRING} operator — the PPL plugin's
+     *  {@code PPLBuiltinOperators.NUMBER_TO_STRING} isn't reachable from this module, so we
+     *  declare a same-named clone that the adapter will match by
+     *  {@link org.apache.calcite.sql.SqlOperator#getName()}. */
+    private static final SqlFunction NUMBER_TO_STRING = new SqlFunction(
+        "NUMBER_TO_STRING",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.VARCHAR,
+        null,
+        OperandTypes.NUMERIC,
+        SqlFunctionCategory.USER_DEFINED_FUNCTION
+    );
+
+    // ── Helpers ───────────────────────────────────────────────────────────────
+
+    /**
+     * Assert that the rewrite produced a {@code tostring(...)} call routed through
+     * {@link ToStringFunctionAdapter#TOSTRING}. Returns the RexCall for further assertions.
+     */
+    private static RexCall assertTostringCall(RexNode out) {
+        assertTrue("expected a RexCall, got " + out.getClass(), out instanceof RexCall);
+        RexCall outCall = (RexCall) out;
+        assertSame(
+            "operator is the synthetic `tostring` that resolves to the Rust UDF",
+            ToStringFunctionAdapter.TOSTRING,
+            outCall.getOperator()
+        );
+        return outCall;
+    }
+
+    private static Cluster newCluster() {
+        RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl();
+        RexBuilder rexBuilder = new RexBuilder(typeFactory);
+        HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+        RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder);
+        return new Cluster(cluster, typeFactory, rexBuilder);
+    }
+
+    private static final class Cluster {
+        final RelOptCluster cluster;
+        final RelDataTypeFactory typeFactory;
+        final RexBuilder rexBuilder;
+
+        Cluster(RelOptCluster cluster, RelDataTypeFactory typeFactory, RexBuilder rexBuilder) {
+            this.cluster = cluster;
+            this.typeFactory = typeFactory;
+            this.rexBuilder = rexBuilder;
+        }
+
+        RexNode intLiteral(int value) {
+            RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER);
+            return rexBuilder.makeExactLiteral(BigDecimal.valueOf(value), intType);
+        }
+
+        RexNode stringLiteral(String value) {
+            return rexBuilder.makeLiteral(value);
+        }
+
+        RexNode booleanLiteral(boolean value) {
+            return rexBuilder.makeLiteral(value);
+        }
+
+        RexNode nullableBooleanInputRef(int index) {
+            RelDataType boolType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BOOLEAN), true);
+            return rexBuilder.makeInputRef(boolType, index);
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java
index f1da7261ce75d..6b0ee31226f39 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java
@@ -10,6 +10,7 @@
 
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.util.Text;
 
 import java.nio.charset.StandardCharsets;
 
@@ -17,7 +18,7 @@
  * Helpers for reading Arrow vector cells as plain Java values at the
  * external query API edge.
  */
-final class ArrowValues {
+public final class ArrowValues {
 
     private ArrowValues() {}
 
@@ -25,14 +26,19 @@ private ArrowValues() {}
      * Returns the cell at {@code index} in {@code vector} as a Java value:
      * {@code null} when the cell is null, a UTF-8 {@link String} for
      * {@link VarCharVector} cells (rather than the raw {@code Text} that
-     * {@code getObject} returns), and {@link FieldVector#getObject} for
-     * every other vector type.
+     * {@code getObject} returns), {@link Text#toString()} for any other vector
+     * type whose {@code getObject} returns a {@link Text} and
+     * {@link FieldVector#getObject} for every other vector type.
      */
-    static Object toJavaValue(FieldVector vector, int index) {
+    public static Object toJavaValue(FieldVector vector, int index) {
         if (vector.isNull(index)) return null;
         if (vector instanceof VarCharVector v) {
             return new String(v.get(index), StandardCharsets.UTF_8);
         }
-        return vector.getObject(index);
+        Object obj = vector.getObject(index);
+        if (obj instanceof Text t) {
+            return t.toString();
+        }
+        return obj;
     }
 }
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StringScalarFunctionsIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StringScalarFunctionsIT.java
new file mode 100644
index 0000000000000..e44ebbad0e422
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StringScalarFunctionsIT.java
@@ -0,0 +1,400 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.io.IOException;
+import java.text.NumberFormat;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * End-to-end coverage for PPL string scalar functions
+ *
+ * 
        Covers three categories of routing:
+ * 
          
+ *   
        • Direct-match Substrait signatures: {@code ascii}, {@code concat},
+ *       {@code concat_ws}, {@code left}, {@code lower}, {@code ltrim},
+ *       {@code reverse}, {@code right}, {@code rtrim}, {@code substring},
+ *       {@code upper}.
          • 
+ *   
        • Name-mapping adapter rewrites (PPL name ≠ DataFusion name) registered in
+ *       {@code DataFusionAnalyticsBackendPlugin.scalarFunctionAdapters()}:
+ *       {@code length → char_length}, {@code locate → strpos} (with arg swap
+ *       and optional 3-arg decomposition), {@code position → strpos} (arg swap),
+ *       {@code substr → substring}, {@code trim → btrim}.
          • 
+ *   
        • Full {@link org.opensearch.analytics.spi.ScalarFunctionAdapter} plans:
+ *       {@code strcmp} (decomposed to a SIMD-vectorized {@code CASE} expression)
+ *       and {@code tostring} / {@code tonumber}.
          • 
+ * 
        
+ *
+ * 
        Each test pins a single row of the {@code calcs} dataset via
+ * {@code where key='keyNN'} — field references prevent Calcite's
+ * {@code ReduceExpressionsRule} from constant-folding the expression on the
+ * coordinator, forcing the call to travel through Substrait into DataFusion
+ * where the function wiring is actually exercised.
+ *
+ * 
        Where inputs must be literals (e.g. to exercise a specific parse path),
+ * tests are constructed so the expected output is only producible by the
+ * function under test — not by Calcite's constant-folder short-circuiting. For
+ * example, {@code tostring(int0 * 12345, 'commas')} on {@code int0=1} yields
+ * {@code "12,345"} which proves the commas format path was evaluated; a
+ * passthrough would produce {@code "12345"}.
+ *
+ * 
        Fixture row values used (from {@code calcs/bulk.json}):
+ * 
          
+ *   
        • {@code key00}: str0="FURNITURE", str2="one", num0=12.3, int0=1, int3=8
          • 
+ *   
        • {@code key04}: str0="OFFICE SUPPLIES", str2="five", num0=3.5, int0=7
          • 
+ * 
        
+ */
+public class StringScalarFunctionsIT extends AnalyticsRestTestCase {
+
+    private static final Dataset DATASET = new Dataset("calcs", "calcs");
+
+    private static boolean dataProvisioned = false;
+
+    private void ensureDataProvisioned() throws IOException {
+        if (dataProvisioned == false) {
+            DatasetProvisioner.provision(client(), DATASET);
+            dataProvisioned = true;
+        }
+    }
+
+    /** Base query template: filter to exactly one row (cardinality 1) keyed by {@code key}. */
+    private String oneRow(String key) {
+        return "source=" + DATASET.indexName + " | where key='" + key + "' | head 1 ";
+    }
+
+    // ── ascii ───────────────────────────────────────────────────────────────
+
+    /** {@code ascii(str0)} on {@code str0="FURNITURE"} → 70 (ASCII code of 'F') */
+    public void testAscii() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = ascii(str0) | fields v", (long) 'F');
+    }
+
+    /** {@code ascii(str0)} on {@code key04} (str0="OFFICE SUPPLIES") → 79 (ASCII code of 'O')*/
+    public void testAsciiDifferentRow() throws IOException {
+        assertFirstRowLong(oneRow("key04") + "| eval v = ascii(str0) | fields v", (long) 'O');
+    }
+
+    // ── concat / concat_ws ──────────────────────────────────────────────────
+
+    /** Two-field {@code concat(str0, str2)} on row 0 → "FURNITUREone". Both operands are field refs  */
+    public void testConcat() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = concat(str0, str2) | fields v", "FURNITUREone");
+    }
+
+    /** {@code concat_ws(':', str0, str2)} on row 0 → "FURNITURE:one" */
+    public void testConcatWs() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = concat_ws(':', str0, str2) | fields v", "FURNITURE:one");
+    }
+
+    // ── left / right ─────────────────────────────────────────────────────────
+
+    /** {@code left('FURNITURE', 3)} → "FUR". Verifies length-1 prefix extraction */
+    public void testLeft() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = left(str0, 3) | fields v", "FUR");
+    }
+
+    /** {@code left(str0, length(str0))} on row 0 → "FURNITURE" (full string). */
+    public void testLeftWithComputedLength() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = left(str0, length(str0)) | fields v", "FURNITURE");
+    }
+
+    /** {@code right('FURNITURE', 3)} → "URE". Verifies suffix extraction; a left() misroute would
+     *  return "FUR". */
+    public void testRight() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = right(str0, 3) | fields v", "URE");
+    }
+
+    // ── lower / upper ────────────────────────────────────────────────────────
+
+    /** {@code lower('FURNITURE')} → "furniture". */
+    public void testLower() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = lower(str0) | fields v", "furniture");
+    }
+
+    /** {@code upper('one')} → "ONE". Complements testLower. */
+    public void testUpper() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = upper(str2) | fields v", "ONE");
+    }
+
+    // ── ltrim / rtrim / trim ────────────────────────────────────────────────
+
+    /** {@code ltrim(concat('   ', str2))} on row 0 → "one". The {@code concat} forces runtime
+     *  evaluation (Calcite can't fold the call because {@code str2} is a column ref), and the
+     *  leading spaces guarantee only ltrim could produce "one" from the 6-character input. */
+    public void testLtrim() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = ltrim(concat('   ', str2)) | fields v", "one");
+    }
+
+    /** {@code rtrim(concat(str2, '   '))} on row 0 → "one". Trailing-spaces counterpart to ltrim;
+     *  verifies the right-side whitespace removal. */
+    public void testRtrim() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = rtrim(concat(str2, '   ')) | fields v", "one");
+    }
+
+    /** {@code trim(concat('  ', str2, '  '))} on row 0 → "one". */
+    public void testTrim() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = trim(concat('  ', str2, '  ')) | fields v", "one");
+    }
+
+    // ── reverse ──────────────────────────────────────────────────────────────
+
+    /** {@code reverse('FURNITURE')} on a field → "ERUTINRUF". */
+    public void testReverse() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = reverse(str0) | fields v", "ERUTINRUF");
+    }
+
+    /** {@code reverse(concat(str2, str0))} → "ERUTINRUFeno". Composed with concat so the input is
+     *  computed at runtime ({@code "one" + "FURNITURE" = "oneFURNITURE"}) and its reverse is a
+     *  12-char string that could only come from an actual character-by-character reversal. */
+    public void testReverseOfConcat() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = reverse(concat(str2, str0)) | fields v", "ERUTINRUFeno");
+    }
+
+    // ── substring ────────────────────────────────────────────────────────────
+
+    /** {@code substring('FURNITURE', 2)} → "URNITURE" (8 chars, from index 2 to end). */
+    public void testSubstringTwoArg() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = substring(str0, 2) | fields v", "URNITURE");
+    }
+
+    /** {@code substring('FURNITURE', 2, 3)} → "URN". Length-bounded 3-arg form; verifies both
+     *  start-position and length semantics simultaneously. */
+    public void testSubstringThreeArg() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = substring(str0, 2, 3) | fields v", "URN");
+    }
+
+    // ── length ───────────────────────────────────────────────────────────────
+
+    /** {@code length('FURNITURE')} → 9.*/
+    public void testLength() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = length(str0) | fields v", 9);
+    }
+
+    /** {@code length('OFFICE SUPPLIES')} on key04 → 15. */
+    public void testLengthDifferentRow() throws IOException {
+        assertFirstRowLong(oneRow("key04") + "| eval v = length(str0) | fields v", 15);
+    }
+
+    // ── locate / position ───────────────────────────────────────────────────
+
+    /** {@code locate('U', 'FURNITURE')} → 2 (1-based position of first 'U'). */
+    public void testLocate() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = locate('U', str0) | fields v", 2);
+    }
+
+    /** {@code locate('U', 'FURNITURE', 3)} → 7. Start-index=3 skips the first 'U' at position 2
+     *  and finds the second 'U' at position 7. */
+    public void testLocateWithStart() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = locate('U', str0, 3) | fields v", 7);
+    }
+
+    /** {@code locate('XYZ', str0)} → 0 (not found). */
+    public void testLocateNotFound() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = locate('XYZ', str0) | fields v", 0);
+    }
+
+    /** {@code position('RNI' IN 'FURNITURE')} → 3. */
+    public void testPosition() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = position(\"RNI\" IN str0) | fields v", 3);
+    }
+
+    // ── strcmp ───────────────────────────────────────────────────────────────
+
+    /** {@code strcmp('hello', 'hello world')} → -1 (lhs < rhs).  */
+    public void testStrcmpLess() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = strcmp('hello', 'hello world') | fields v", -1);
+    }
+
+    /** {@code strcmp('foo', 'foo')} → 0.  */
+    public void testStrcmpEqual() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = strcmp('foo', 'foo') | fields v", 0);
+    }
+
+    /** {@code strcmp('banana', 'apple')} → 1 (lhs > rhs). */
+    public void testStrcmpGreater() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = strcmp('banana', 'apple') | fields v", 1);
+    }
+
+    /** {@code strcmp(str0, 'FURNITURE')} on row 0 (str0='FURNITURE') → 0. Verifies the adapter
+     *  handles column references correctly: PPL frontend reverses args internally, and the
+     *  adapter must swap back for the user-intended semantics. */
+    public void testStrcmpColumnEqual() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = strcmp(str0, 'FURNITURE') | fields v", 0);
+    }
+
+    /** {@code strcmp(str0, 'AAA')} */
+    public void testStrcmpColumnGreater() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = strcmp(str0, 'AAA') | fields v", 1);
+    }
+
+    /** {@code strcmp(str0, 'ZZZ')} */
+    public void testStrcmpColumnLess() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = strcmp(str0, 'ZZZ') | fields v", -1);
+    }
+
+    // ── tostring — basic ────────────────────────────────────────────────────
+
+    /** {@code tostring(num0)} on row 0 (num0=12.3) → "12.3". */
+    public void testToStringOnDouble() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = tostring(num0) | fields v", "12.3");
+    }
+
+    /** {@code tostring(int0)} on row 0 (int0=1) → "1". */
+    public void testToStringOnInteger() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = tostring(int0) | fields v", "1");
+    }
+
+    /** {@code tostring(1=1)} → "TRUE". Boolean literal routes through the adapter's CASE
+     *  WHEN x THEN 'TRUE' WHEN NOT x THEN 'FALSE' END rewrite. */
+    public void testToStringOnBooleanTrue() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = tostring(1=1) | fields v", "TRUE");
+    }
+
+    /** {@code tostring(1=0)} → "FALSE" */
+    public void testToStringOnBooleanFalse() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = tostring(1=0) | fields v", "FALSE");
+    }
+
+    // ── tostring — format modes ─────────────────────────────────────────────
+
+    /**
+     * {@code tostring(int0 * 255, 'hex')} on row 0 (int0=1) → "ff".
+     */
+    public void testToStringHexFormat() throws IOException {
+        Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tostring(int0 * 255, 'hex') | fields v");
+        assertNotNull("hex cell must not be null", cell);
+        assertTrue("hex cell must be String but was " + cell.getClass(), cell instanceof String);
+        assertEquals("tostring(255, 'hex')", "ff", ((String) cell).toLowerCase(Locale.US));
+    }
+
+    /**
+     * {@code tostring(int0 * 21, 'binary')} on row 0 (int0=1) → "10101".
+     */
+    public void testToStringBinaryFormat() throws IOException {
+        Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tostring(int0 * 21, 'binary') | fields v");
+        assertNotNull("binary cell must not be null", cell);
+        assertTrue("binary cell must be String but was " + cell.getClass(), cell instanceof String);
+        assertEquals("tostring(21, 'binary')", "10101", cell);
+    }
+
+    /**
+     * {@code tostring(int0 * 12345, 'commas')} on row 0 (int0=1) → "12,345".
+     */
+    public void testToStringCommasFormat() throws IOException {
+        Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tostring(int0 * 12345, 'commas') | fields v");
+        assertNotNull("commas cell must not be null", cell);
+        assertTrue("commas cell must be String but was " + cell.getClass(), cell instanceof String);
+        NumberFormat nf = NumberFormat.getNumberInstance(Locale.US);
+        nf.setMinimumFractionDigits(0);
+        nf.setMaximumFractionDigits(2);
+        assertEquals("tostring(12345, 'commas')", nf.format(12345L), cell);
+    }
+
+    /**
+     * {@code tostring(int0 * 3661, 'duration')} on row 0 (int0=1) → "01:01:01".
+     *  one.
+     */
+    public void testToStringDurationFormat() throws IOException {
+        Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tostring(int0 * 3661, 'duration') | fields v");
+        assertNotNull("duration cell must not be null", cell);
+        assertTrue("duration cell must be String but was " + cell.getClass(), cell instanceof String);
+        assertEquals("tostring(3661, 'duration')", "01:01:01", cell);
+    }
+
+    /**
+     * {@code tostring(int0 * 3_661_000, 'duration_millis')} on row 0 (int0=1) → "01:01:01".
+     */
+    public void testToStringDurationMillisFormat() throws IOException {
+        Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tostring(int0 * 3661000, 'duration_millis') | fields v");
+        assertNotNull("duration_millis cell must not be null", cell);
+        assertTrue("duration_millis cell must be String but was " + cell.getClass(), cell instanceof String);
+        assertEquals("tostring(3661000, 'duration_millis')", "01:01:01", cell);
+    }
+
+    // ── tonumber ────────────────────────────────────────────────────────────
+
+    /** {@code tonumber('4598')} → 4598.0 */
+    public void testToNumberDecimalInteger() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = tonumber('4598') | fields v", 4598.0, 0.0);
+    }
+
+    /** {@code tonumber('4598.678')} → 4598.678 */
+    public void testToNumberDecimalFractional() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = tonumber('4598.678') | fields v", 4598.678, 1e-9);
+    }
+
+    /** {@code tonumber('010101', 2)} → 21. Base-2 parse */
+    public void testToNumberBinary() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = tonumber('010101', 2) | fields v", 21.0, 0.0);
+    }
+
+    /** {@code tonumber('FA34', 16)} → 64052. Base-16 parse with uppercase hex digits */
+    public void testToNumberHex() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = tonumber('FA34', 16) | fields v", 64052.0, 0.0);
+    }
+
+    /** {@code tonumber('101', 8)} → 65 (octal 101 = 64 + 1) */
+    public void testToNumberOctal() throws IOException {
+        assertFirstRowDouble(oneRow("key00") + "| eval v = tonumber('101', 8) | fields v", 65.0, 0.0);
+    }
+
+    /** {@code tonumber('abc')} → NULL. Unparseable input */
+    public void testToNumberReturnsNullOnParseFailure() throws IOException {
+        Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tonumber('abc') | fields v");
+        assertNull("tonumber('abc') should be NULL but was " + cell, cell);
+    }
+
+    /** {@code tonumber('FA34', 10)} → NULL */
+    public void testToNumberBaseMismatchReturnsNull() throws IOException {
+        Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = tonumber('FA34', 10) | fields v");
+        assertNull("tonumber('FA34', 10) should be NULL but was " + cell, cell);
+    }
+
+    // ── helpers ─────────────────────────────────────────────────────────────
+
+    private void assertFirstRowString(String ppl, String expected) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertNotNull("Expected non-null result for query [" + ppl + "]", cell);
+        assertEquals("Value mismatch for query: " + ppl, expected, cell);
+    }
+
+    private void assertFirstRowLong(String ppl, long expected) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertTrue("Expected numeric result for query [" + ppl + "] but got: " + cell, cell instanceof Number);
+        assertEquals("Value mismatch for query: " + ppl, expected, ((Number) cell).longValue());
+    }
+
+    private void assertFirstRowDouble(String ppl, double expected, double delta) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertTrue("Expected numeric result for query [" + ppl + "] but got: " + cell, cell instanceof Number);
+        assertEquals("Value mismatch for query: " + ppl, expected, ((Number) cell).doubleValue(), delta);
+    }
+
+    private Object firstRowFirstCell(String ppl) throws IOException {
+        Map response = executePpl(ppl);
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) response.get("rows");
+        assertNotNull("Response missing 'rows' for query: " + ppl, rows);
+        assertTrue("Expected at least one row for query: " + ppl, rows.size() >= 1);
+        return rows.get(0).get(0);
+    }
+
+    private Map executePpl(String ppl) throws IOException {
+        ensureDataProvisioned();
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}");
+        Response response = client().performRequest(request);
+        return assertOkAndParse(response, "PPL: " + ppl);
+    }
+}

From 4d3562bf1e41b64e4bee20c41d7b4175328daefe Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Fri, 8 May 2026 15:37:38 -0700
Subject: [PATCH 097/115] update sandbox check to publish latest
 UnifiedQueryAPI jars to maven local. (#21569)

This is a temporary work-around to publish latest unified jars
built to integrate with analytics-engine to maven local - we are working off of
a feature branch of sql in the short term.  if the step fails, the build will continue
with jars from maven central. This will be removed once changes for
analytics-engine in sql are merged to mainline.

Signed-off-by: Marc Handalian 
---
 .github/workflows/sandbox-check.yml | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/sandbox-check.yml b/.github/workflows/sandbox-check.yml
index fc710f499fb89..f5aa63315bb80 100644
--- a/.github/workflows/sandbox-check.yml
+++ b/.github/workflows/sandbox-check.yml
@@ -32,8 +32,18 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
       - name: Install protobuf compiler
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Check out SQL repo (mustang-ppl-integration)
+        uses: actions/checkout@v6
+        with:
+          repository: opensearch-project/sql
+          ref: feature/mustang-ppl-integration
+          path: sql
+      - name: Publish unified-query artifacts to maven local
+        working-directory: sql
+        continue-on-error: true
+        run: ./gradlew publishUnifiedQueryPublicationToMavenLocal
       - name: Run sandbox check
-        run: ./gradlew check -p sandbox -Dsandbox.enabled=true
+        run: ./gradlew check -p sandbox -Dsandbox.enabled=true -Drepos.mavenLocal=true -PrustDebug
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4

From 2380af73c609685e878c88048cabe23716f47980 Mon Sep 17 00:00:00 2001
From: Eric Wei 
Date: Fri, 8 May 2026 18:27:40 -0700
Subject: [PATCH 098/115] Enable PPL JSON scalar functions on the
 analytics-engine route (#21513)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [Analytics Engine] Port json_array_length to DataFusion backend

First PPL json_* function wired through PPL → Calcite → Substrait →
DataFusion. Scaffolds the pattern every follow-up UDF reuses: Rust kernel
+ YAML signature + ScalarFunction enum entry + JsonFunctionAdapters
rename + FunctionMappings.s(...) binding + STANDARD_PROJECT_OPS entry.

Rust UDF (rust/src/udf/json_array_length.rs) coerces the input to Utf8,
parses with serde_json, and returns Int32 to match PPL's
INTEGER_FORCE_NULLABLE declaration — returning Int64 would leak through
column-valued calls even though literal args const-fold via a narrowing
CAST. Malformed / non-array / NULL input → NULL, matching legacy
JsonArrayLengthFunctionImpl's NullPolicy.ANY + Gson parity.

ScalarFunction.CAST added to STANDARD_PROJECT_OPS so PPL's implicit CAST
around a UDF call (inserted when the UDF's declared return type differs
from the eval column's inferred type) doesn't fail OpenSearchProjectRule
with "No backend supports scalar function [CAST]". DataFusion handles
CAST natively — no UDF needed.

STANDARD_PROJECT_OPS and scalarFunctionAdapters reshaped to one-entry-
per-line (Map.ofEntries / Set.of) so parallel json_* PRs append without
touching neighbour lines.

Tests:
  * 10 Rust unit tests (flat/nested arrays, non-array, malformed, NULL,
    coerce_types accept/reject, arity guard, scalar-input fast path).
  * JsonFunctionAdaptersTests guards adapter shape + return-type
    preservation (BIGINT vs LOCAL_OP's INTEGER_NULLABLE).
  * ScalarJsonFunctionIT covers happy path, empty array, non-array
    object → NULL, malformed → NULL via /_analytics/ppl.

Parity-checked against legacy SQL plugin
CalcitePPLJsonBuiltinFunctionIT.testJsonArrayLength.

Signed-off-by: Eric Wei 

* [Analytics Engine] JSON: introduce jsonpath-rust parser + shared helpers

Lands the parser crate + a small shared helpers module ahead of the per-
function json_* UDFs. Keeping this on its own commit lets reviewers sign
off on the crate choice (jsonpath-rust 0.7) and path-conversion behaviour
before 8 UDF bodies land on top.

  * rust/Cargo.toml: add jsonpath-rust = "0.7".
  * rust/src/udf/json_common.rs:
      - convert_ppl_path: PPL path syntax (`a{i}.b{}`) -> JSONPath (`$.a[i].b[*]`).
        Mirrors JsonUtils.convertToJsonPath in sql/core. Empty string maps
        to "$" to match legacy root semantics.
      - parse: serde_json wrapper returning None on malformed input, the
        contract every json_* UDF will share.
      - check_arity / check_arity_range: plan_err! wrappers for the
        top-of-invoke guards.
  * rust/src/udf/mod.rs: register the module (helpers are crate-private).

Consumers land in follow-up commits on the same PR (#21513); a module-
level #![allow(dead_code)] keeps this commit's cargo check clean.

Signed-off-by: Eric Wei 

* [Analytics Engine] Port json_keys to DataFusion backend

Adds the second PPL json_* UDF on top of #21476 (json_array_length).
Matches the legacy SQL-plugin contract: object → JSON-array-encoded keys
in insertion order; non-object / malformed / scalar → SQL NULL.

- Rust UDF at rust/src/udf/json_keys.rs with scalar + columnar paths
- Shared rust/src/udf/json_common.rs helpers (parse, arity, Utf8 downcast,
  PPL-path → JSONPath) seeded for later json_* UDFs
- serde_json preserve_order feature to preserve legacy LinkedHashMap ordering
- Java wiring: ScalarFunction.JSON_KEYS, JsonKeysAdapter, Substrait sig,
  YAML signature, plugin project-op + adapter registration
- ScalarJsonFunctionIT parity test for the four legacy fixtures

Signed-off-by: Eric Wei 

* [Analytics Engine] Port json_extract to DataFusion backend

Rust UDF at rust/src/udf/json_extract.rs wraps jsonpath-rust: single path →
unquoted scalar or JSON-serialized container; multi-path → JSON array with
literal null slots for misses. < 2 args, malformed doc, malformed path, and
explicit-null matches all collapse to SQL NULL, matching legacy
JsonExtractFunctionImpl's calcite jsonQuery/jsonValue pair.

JsonExtractAdapter renames the PPL call to the Rust UDF name via the variadic
path; routing lives in FunctionMappings.s(...) in DataFusionFragmentConvertor
and the STANDARD_PROJECT_OPS allow-list.

Also fixes a pre-existing transport bug in DatafusionResultStream.getFieldValue:
VarCharVector.getObject returns Arrow Text, which StreamOutput.writeGenericValue
cannot serialize, so string-valued UDF results (json_keys, json_extract) were
dropped when shard results traveled back to the coordinator. Converting
VarCharVector cells to String at the source mirrors ArrowValues.toJavaValue
and unblocks every string-returning UDF.

Parity IT (ScalarJsonFunctionIT) replays four verbatim legacy cases covering
single-path scalar/container match, wildcard multi-match, multi-path with
missing path, and explicit-null resolution.

Signed-off-by: Eric Wei 

* [Analytics Engine] Port json_delete to DataFusion backend

Mutation UDF #1. Introduces the shared mutation walker that json_set,
json_append, and json_extend will reuse on the same PR.

Rust side (rust/src/udf/json_delete.rs + json_common.rs):
  * `parse_ppl_segments` tokenises PPL paths (a.b{0}.c{}) into Field /
    Index / Wildcard segments without allocating field names.
  * `walk_mut` drives a mutation closure against every terminal match in
    a serde_json::Value; missing intermediate keys and out-of-range
    indices are silent no-ops, matching Jayway's SUPPRESS_EXCEPTIONS
    behaviour that legacy `JsonDeleteFunctionImpl` (→ Calcite
    `JsonFunctions.jsonRemove`) relies on.
  * `json_delete` terminal closure: `shift_remove` on Object (preserves
    insertion order via serde_json's `preserve_order` feature),
    `Vec::remove` on Array-with-Index, `Vec::clear` on Array-with-Wildcard.
    Any-NULL-arg / malformed doc / malformed path → NULL.

The walker is generic enough that json_set / json_append / json_extend
are now pure terminal-closure swaps (set value, push value, extend
array) — no further traversal plumbing needed.

Java side:
  * JSON_DELETE added to `ScalarFunction`, `STANDARD_PROJECT_OPS`, and
    `scalarFunctionAdapters`.
  * `JsonDeleteAdapter` is a plain `AbstractNameMappingAdapter` rename
    (matches the other json_* adapters).
  * Substrait YAML signature uses `variadic: {min: 1}` — same shape as
    json_extract.

Tests:
  * 10 Rust unit tests for json_delete (4 legacy IT fixtures replayed:
    flat-key, nested, missing-path-unchanged, wildcard-array; plus
    any-NULL / malformed / coerce_types / return_type).
  * 4 new walker tests in json_common (tokeniser, flat-delete,
    missing-noop, wildcard-fan-out, index-out-of-range-noop).
  * ScalarJsonFunctionIT gains `testJsonDeleteParityWithLegacy`
    replaying all 4 legacy assertions.

Parity-checked against legacy SQL plugin
`CalcitePPLJsonBuiltinFunctionIT.testJsonDelete*`.

Signed-off-by: Eric Wei 

* [Analytics Engine] Port json_set to DataFusion backend

Mutation UDF #2. Reuses the walker introduced by #json_delete; this
commit is a pure terminal-closure swap on the Rust side (replace, not
remove) plus the usual 7-file Java/YAML wiring.

Rust side (rust/src/udf/json_set.rs):
  * Terminal closure overwrites only existing keys on Object
    (`map.contains_key` guard), in-range slots on Array-with-Index, and
    every element on Array-with-Wildcard. This is the replace-only
    semantics from legacy `JsonSetFunctionImpl` (→ Calcite
    `JsonFunctions.jsonSet`, which guards `ctx.set` with
    `ctx.read(k) != null`).
  * Variadic arity: (doc, path1, val1, [path2, val2, ...]). Fewer than
    3 args or an odd total (unpaired trailing path) short-circuits to
    NULL, mirroring the "malformed input → NULL" convention the other
    json_* UDFs follow.
  * Values are always stored as `Value::String` because every arg is
    coerced to Utf8 by `coerce_types` — matches the legacy fixture's
    `"b":"3"` (stringified, not numeric).
  * Root-path (`parse_ppl_segments` returns empty) is a no-op to match
    Jayway's behaviour: `ctx.set("$", v)` silently fails because the
    root is indelible and unreplaceable.

Java side:
  * JSON_SET added to `ScalarFunction`, `STANDARD_PROJECT_OPS`, and
    `scalarFunctionAdapters`.
  * `JsonSetAdapter` is a plain `AbstractNameMappingAdapter` rename.
  * Substrait YAML signature uses `variadic: {min: 1}` — same shape as
    json_extract / json_delete.

Tests:
  * 9 Rust unit tests for json_set (3 legacy IT fixtures replayed:
    wildcard-replace, wrong-path-unchanged, partial-wildcard-set; plus
    multi-pair / any-NULL / malformed-doc / malformed-path /
    coerce_types / return_type).
  * ScalarJsonFunctionIT gains `testJsonSetParityWithLegacy` replaying
    all 3 legacy assertions.

Parity-checked against legacy SQL plugin
`CalcitePPLJsonBuiltinFunctionIT.testJsonSet*`.

Signed-off-by: Eric Wei 

* [Analytics Engine] Port json_append to DataFusion backend

Mutation UDF #3. Another walker reuse: terminal closure pushes the
paired value onto array-valued targets (non-array / missing targets
are silent no-ops).

Rust side (rust/src/udf/json_append.rs):
  * Terminal closure branches: Object+Field → look up field, if it's an
    Array push the stringified value; Array+Index → if the indexed slot
    is an Array, push; Array+Wildcard → push onto every array-valued
    child. Non-array matches are skipped, matching legacy
    `JsonFunctions.jsonInsert` via Jayway's Collection-parent branch
    (`Collection.add`) which is how `JsonAppendFunctionImpl`'s
    `.meaningless_key` suffix trick ultimately expands.
  * Variadic arity (doc, path1, val1, [path2, val2, ...]). Fewer than 3
    args or an odd total (unpaired trailing path) → NULL — the
    malformed-input-to-NULL convention all other json_* UDFs share.
    Matches legacy's `RuntimeException("needs corresponding path and
    values")` observably-as-error via NULL surface.
  * Pre-stringified values: all args are Utf8-coerced at `coerce_types`
    entry, so nested `json_object(...)` / `json_array(...)` arrive here
    already stringified. They are pushed as `Value::String`, which
    reproduces the legacy IT's quoted-JSON-as-element rows without the
    new engine having to implement `json_object`/`json_array` yet
    (they ship in a follow-up PR).

Java side:
  * JSON_APPEND added to `ScalarFunction`, `STANDARD_PROJECT_OPS`, and
    `scalarFunctionAdapters`.
  * `JsonAppendAdapter` is a plain `AbstractNameMappingAdapter` rename.
  * Substrait YAML signature uses `variadic: {min: 1}` — same shape as
    json_extract / json_delete / json_set.

Tests:
  * 12 Rust unit tests for json_append (3 legacy IT fixtures replayed
    with pre-stringified nested JSON: named-array push, nested-path
    push, stringified-object push; plus multi-pair / wildcard-fan-out /
    non-array-noop / missing-path-noop / any-NULL / malformed-doc /
    malformed-path / coerce_types / return_type).
  * ScalarJsonFunctionIT gains `testJsonAppendParityWithLegacy`
    replaying all 3 legacy assertions with literal stringified JSON in
    place of the nested constructor calls the legacy test uses.

Parity-checked against legacy SQL plugin
`CalcitePPLJsonBuiltinFunctionIT.testJsonAppend`.

Signed-off-by: Eric Wei 

* [Analytics Engine] Port json_extend to DataFusion backend

Mutation UDF #4 — last walker reuse. Same push shape as json_append,
but each paired value is first tried as a JSON-array parse: success →
spread the elements; failure → push the whole string as one element
(parity with legacy `JsonExtendFunctionImpl`'s `gson.fromJson(v,
List.class)` try/fall-back).

Rust side (rust/src/udf/json_extend.rs):
  * Helper `spread(raw) -> Vec`: returns the parsed items when
    `raw` is a JSON array, else `[Value::String(raw)]`. Scalars,
    objects, and malformed JSON all go through the single-push branch.
  * Terminal closure reuses json_append's array-target guards (Object
    field → Array, Array+Index → inner Array, Array+Wildcard → every
    array child). `Vec::extend(items.iter().cloned())` handles the
    spread and the single-push case uniformly.
  * Variadic arity matches every other mutation UDF. Invalid arity /
    any-NULL / malformed-doc / malformed-path → NULL.

Deliberate divergence from legacy: integer-typed spread elements stay
integers (serde_json preserves source type) rather than being widened
to Double as Gson does. Documented in `json.md:555` but not covered by
any legacy IT; we preserve the more useful default and will file a
tracking issue for the wider Gson-compat decision.

Java side:
  * JSON_EXTEND added to `ScalarFunction`, `STANDARD_PROJECT_OPS`, and
    `scalarFunctionAdapters`.
  * `JsonExtendAdapter` is a plain `AbstractNameMappingAdapter` rename.
  * Substrait YAML signature uses `variadic: {min: 1}` — same shape as
    the other variadic json_* UDFs.

Tests:
  * 13 Rust unit tests for json_extend (3 legacy IT fixtures replayed:
    single-push on non-array value, plain-string push, JSON-array
    spread; plus empty-array-value / mixed-type-spread / wildcard-fan
    / non-array-noop / missing-path-noop / any-NULL / malformed-doc /
    malformed-path / coerce_types / return_type).
  * ScalarJsonFunctionIT gains `testJsonExtendParityWithLegacy`
    replaying all 3 legacy assertions with literal stringified JSON
    standing in for the nested constructor calls the legacy test uses.

Parity-checked against legacy SQL plugin
`CalcitePPLJsonBuiltinFunctionIT.testJsonExtend`.

Signed-off-by: Eric Wei 

---------

Signed-off-by: Eric Wei 
---
 .../analytics/spi/ScalarFunction.java         |  11 +-
 .../rust/Cargo.toml                           |  20 +
 .../rust/src/udf/json_append.rs               | 335 +++++++++++++++++
 .../rust/src/udf/json_array_length.rs         | 260 +++++++++++++
 .../rust/src/udf/json_common.rs               | 294 +++++++++++++++
 .../rust/src/udf/json_delete.rs               | 286 ++++++++++++++
 .../rust/src/udf/json_extend.rs               | 348 ++++++++++++++++++
 .../rust/src/udf/json_extract.rs              | 308 ++++++++++++++++
 .../rust/src/udf/json_keys.rs                 | 155 ++++++++
 .../rust/src/udf/json_set.rs                  | 281 ++++++++++++++
 .../rust/src/udf/mod.rs                       |  29 +-
 .../be/datafusion/BaseScalarFunctionIT.java   |  61 ++-
 .../be/datafusion/ScalarJsonFunctionIT.java   | 180 +++++++++
 .../be/datafusion/ConvertTzAdapter.java       |   2 +-
 .../DataFusionAnalyticsBackendPlugin.java     |  24 +-
 .../DataFusionFragmentConvertor.java          |  32 +-
 .../be/datafusion/JsonFunctionAdapters.java   | 159 ++++++++
 .../be/datafusion/UnixTimestampAdapter.java   |   2 +-
 .../opensearch_scalar_functions.yaml          |  50 +++
 .../datafusion/JsonFunctionAdaptersTests.java | 118 ++++++
 20 files changed, 2942 insertions(+), 13 deletions(-)
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_append.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_array_length.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_common.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_delete.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_extend.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_extract.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_keys.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_set.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarJsonFunctionIT.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/JsonFunctionAdapters.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/JsonFunctionAdaptersTests.java

diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
index ee32b35be4b9e..af6f15e84935b 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
@@ -168,7 +168,16 @@ public enum ScalarFunction {
     CURRENT_TIME(Category.SCALAR, SqlKind.OTHER_FUNCTION),
     CURTIME(Category.SCALAR, SqlKind.OTHER_FUNCTION),
     CONVERT_TZ(Category.SCALAR, SqlKind.OTHER_FUNCTION),
-    UNIX_TIMESTAMP(Category.SCALAR, SqlKind.OTHER_FUNCTION);
+    UNIX_TIMESTAMP(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+
+    // ── JSON ────────────────────────────────────────────────────────
+    JSON_APPEND(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    JSON_ARRAY_LENGTH(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    JSON_DELETE(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    JSON_EXTEND(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    JSON_EXTRACT(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    JSON_KEYS(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    JSON_SET(Category.SCALAR, SqlKind.OTHER_FUNCTION);
 
     /**
      * Category of scalar function.
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
index 63afe25b394f1..ba0f7a0c0226a 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
@@ -49,6 +49,26 @@ chrono-tz = "0.10"
 
 tokio-metrics = { workspace = true }
 
+# serde_json `preserve_order` — backs `Map` with `IndexMap`
+# instead of `BTreeMap` so json_keys / mutation UDFs see object keys in
+# insertion order (parity with legacy SQL-plugin's LinkedHashMap; required by
+# `testJsonKeysParityWithLegacy` + byte-for-byte json_extract fixtures).
+# Cargo's feature unification propagates this to every workspace member that
+# pulls in serde_json. Audit (2026-05-07): the five other consumers
+# (parquet-data-format, native-repository-{s3,gcs,azure,fs}) only call
+# `serde_json::from_str` into typed config structs, whose field layout is
+# fixed at the type level — `preserve_order` is inert for them, so the
+# feature is additive with no observable blast radius outside this crate.
+serde_json = { workspace = true, features = ["preserve_order"] }
+# jsonpath-rust 0.7 — JSONPath evaluator for json_extract. Published at
+# https://github.com/besok/jsonpath-rust (crates.io). We pin `0.7` (latest
+# `0.7.5`) rather than tracking the newer `1.0` release line because 0.7's
+# `JsonPathValue` enum exposes the Found/NoValue distinction json_extract
+# relies on to render missing-path matches as literal `null` elements in the
+# multi-path JSON-array output. Moving to 1.x is a follow-up once we can
+# reproduce that distinction against the new API surface.
+jsonpath-rust = "0.7"
+
 [dev-dependencies]
 criterion = { workspace = true }
 tempfile = { workspace = true }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_append.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_append.rs
new file mode 100644
index 0000000000000..8b3608c27fcb1
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_append.rs
@@ -0,0 +1,335 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `json_append(value, path1, val1, [path2, val2, ...])` — push `valN` onto
+//! each path-matched array (parity with legacy `JsonAppendFunctionImpl`, which
+//! delegates to `JsonFunctions.jsonInsert` + `.meaningless_key` trick so Jayway
+//! routes to `Collection.add`). Non-array / missing targets are silent no-ops;
+//! any-NULL-arg / odd trailing arg / malformed-doc / malformed-path → NULL.
+//!
+//! Values always push as `Value::String` — every UDF arg is coerced to Utf8
+//! upstream, so nested `json_object` / `json_array` results arrive already
+//! stringified and append as strings, matching legacy.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, ArrayRef, StringBuilder};
+use datafusion::arrow::datatypes::DataType;
+use datafusion::common::ScalarValue;
+use datafusion::error::Result;
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+use serde_json::Value;
+
+use super::json_common::{as_utf8_array, parse, parse_ppl_segments, walk_mut, Segment};
+use super::{coerce_slot, CoerceMode};
+
+const NAME: &str = "json_append";
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(JsonAppendUdf::new()));
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct JsonAppendUdf {
+    signature: Signature,
+}
+
+impl JsonAppendUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl Default for JsonAppendUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ScalarUDFImpl for JsonAppendUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn name(&self) -> &str {
+        NAME
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+    fn return_type(&self, _args: &[DataType]) -> Result {
+        Ok(DataType::Utf8)
+    }
+    fn coerce_types(&self, args: &[DataType]) -> Result> {
+        args.iter()
+            .enumerate()
+            .map(|(i, ty)| coerce_slot(NAME, i, ty, CoerceMode::Utf8))
+            .collect()
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        // Need doc + at least one (path, value) pair. Odd trailing arg mirrors
+        // the legacy `RuntimeException("needs corresponding path and values")`
+        // thrown by `JsonAppendFunctionImpl.eval`; we surface it as NULL to
+        // keep parity with the "malformed input → NULL" convention.
+        if args.args.len() < 3 || args.args.len().is_multiple_of(2) {
+            return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)));
+        }
+        let n = args.number_rows;
+
+        if args
+            .args
+            .iter()
+            .all(|v| matches!(v, ColumnarValue::Scalar(_)))
+        {
+            let doc = scalar_utf8(&args.args[0]);
+            let rest: Vec> = args.args[1..].iter().map(scalar_utf8).collect();
+            let out = append(doc, &rest);
+            return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(out)));
+        }
+
+        let arrays: Vec = args
+            .args
+            .iter()
+            .map(|v| v.clone().into_array(n))
+            .collect::>()?;
+        let columns: Vec<&datafusion::arrow::array::StringArray> =
+            arrays.iter().map(as_utf8_array).collect::>()?;
+
+        let mut b = StringBuilder::with_capacity(n, n * 16);
+        let mut rest: Vec> = Vec::with_capacity(columns.len() - 1);
+        for i in 0..n {
+            let doc = cell(columns[0], i);
+            rest.clear();
+            for col in &columns[1..] {
+                rest.push(cell(col, i));
+            }
+            match append(doc, &rest) {
+                Some(s) => b.append_value(&s),
+                None => b.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(b.finish()) as ArrayRef))
+    }
+}
+
+fn scalar_utf8(v: &ColumnarValue) -> Option<&str> {
+    match v {
+        ColumnarValue::Scalar(
+            ScalarValue::Utf8(s) | ScalarValue::LargeUtf8(s) | ScalarValue::Utf8View(s),
+        ) => s.as_deref(),
+        _ => None,
+    }
+}
+
+fn cell(arr: &datafusion::arrow::array::StringArray, i: usize) -> Option<&str> {
+    if arr.is_null(i) {
+        None
+    } else {
+        Some(arr.value(i))
+    }
+}
+
+/// Apply each (path, value) pair to a fresh parse of `doc`. Push-only:
+/// non-array targets (scalar, object) are silent no-ops, matching legacy
+/// `jsonInsert`'s Collection-parent branch skip.
+fn append(doc: Option<&str>, rest: &[Option<&str>]) -> Option {
+    let doc_str = doc?;
+    if rest.iter().any(|p| p.is_none()) {
+        return None;
+    }
+    let mut value = parse(doc_str)?;
+    for chunk in rest.chunks(2) {
+        let path = chunk[0].unwrap();
+        let new_val = chunk[1].unwrap();
+        let segments = parse_ppl_segments(path).ok()?;
+        if segments.is_empty() {
+            // Root-path is a no-op (legacy `ctx.set("$", v)` is silently
+            // discarded by Jayway for the same reason).
+            continue;
+        }
+        append_one(&mut value, &segments, new_val);
+    }
+    serde_json::to_string(&value).ok()
+}
+
+fn append_one(root: &mut Value, segments: &[Segment<'_>], new_val: &str) {
+    let item = Value::String(new_val.to_string());
+    walk_mut(root, segments, |parent, final_seg| {
+        match (parent, final_seg) {
+            // Push onto the matched array when the final segment names an
+            // existing array-valued field. Non-array / missing → no-op.
+            (Value::Object(map), Segment::Field(name)) => {
+                if let Some(Value::Array(arr)) = map.get_mut(*name) {
+                    arr.push(item.clone());
+                }
+            }
+            // Direct array-index / wildcard targets: push onto the *addressed*
+            // array element when that element is itself an array.
+            (Value::Array(arr), Segment::Index(i)) if *i < arr.len() => {
+                if let Value::Array(inner) = &mut arr[*i] {
+                    inner.push(item.clone());
+                }
+            }
+            (Value::Array(arr), Segment::Wildcard) => {
+                for slot in arr.iter_mut() {
+                    if let Value::Array(inner) = slot {
+                        inner.push(item.clone());
+                    }
+                }
+            }
+            _ => {}
+        }
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn single_value_appended_to_named_array() {
+        // testJsonAppend case b, single pair.
+        assert_eq!(
+            append(
+                Some(r#"{"teacher":["Alice"]}"#),
+                &[Some("teacher"), Some("Tom")],
+            )
+            .as_deref(),
+            Some(r#"{"teacher":["Alice","Tom"]}"#)
+        );
+    }
+
+    #[test]
+    fn multiple_pairs_append_sequentially() {
+        // testJsonAppend case b (multi-pair).
+        assert_eq!(
+            append(
+                Some(r#"{"teacher":["Alice"]}"#),
+                &[Some("teacher"), Some("Tom"), Some("teacher"), Some("Walt")],
+            )
+            .as_deref(),
+            Some(r#"{"teacher":["Alice","Tom","Walt"]}"#)
+        );
+    }
+
+    #[test]
+    fn nested_path_appends_to_inner_array() {
+        // testJsonAppend case c — a pre-stringified JSON array is appended as
+        // a single string element (legacy calls gson/jackson on the outer doc
+        // but NOT on the value; our Utf8-coerced arg arrives already
+        // stringified and is pushed as-is).
+        assert_eq!(
+            append(
+                Some(r#"{"school":{"teacher":["Alice"]}}"#),
+                &[Some("school.teacher"), Some(r#"["Tom","Walt"]"#)],
+            )
+            .as_deref(),
+            Some(r#"{"school":{"teacher":["Alice","[\"Tom\",\"Walt\"]"]}}"#)
+        );
+    }
+
+    #[test]
+    fn stringified_json_object_value_is_appended_as_single_string() {
+        // testJsonAppend case a — `json_object(...)` lowers to a string, so
+        // the element lands as a stringified object (legacy and Rust agree).
+        assert_eq!(
+            append(
+                Some(r#"{"student":[{"name":"Bob","rank":1}]}"#),
+                &[Some("student"), Some(r#"{"name":"Tomy","rank":5}"#)],
+            )
+            .as_deref(),
+            Some(r#"{"student":[{"name":"Bob","rank":1},"{\"name\":\"Tomy\",\"rank\":5}"]}"#)
+        );
+    }
+
+    #[test]
+    fn non_array_target_is_silent_noop() {
+        // teacher is a scalar here, not an array — legacy `Collection.add`
+        // branch skips; no-op is the observable parity.
+        assert_eq!(
+            append(
+                Some(r#"{"teacher":"Alice"}"#),
+                &[Some("teacher"), Some("Tom")],
+            )
+            .as_deref(),
+            Some(r#"{"teacher":"Alice"}"#)
+        );
+    }
+
+    #[test]
+    fn missing_path_is_silent_noop() {
+        assert_eq!(
+            append(
+                Some(r#"{"teacher":["Alice"]}"#),
+                &[Some("students"), Some("Tom")],
+            )
+            .as_deref(),
+            Some(r#"{"teacher":["Alice"]}"#)
+        );
+    }
+
+    #[test]
+    fn wildcard_path_appends_to_every_array_child() {
+        // Nested wildcard: every element of groups is an array; each receives
+        // the same appended scalar.
+        assert_eq!(
+            append(
+                Some(r#"{"groups":[["a"],["b","c"]]}"#),
+                &[Some("groups{}"), Some("x")],
+            )
+            .as_deref(),
+            Some(r#"{"groups":[["a","x"],["b","c","x"]]}"#)
+        );
+    }
+
+    #[test]
+    fn any_null_arg_returns_none() {
+        assert!(append(None, &[Some("a"), Some("v")]).is_none());
+        assert!(append(Some(r#"{"a":[1]}"#), &[None, Some("v")]).is_none());
+        assert!(append(Some(r#"{"a":[1]}"#), &[Some("a"), None]).is_none());
+    }
+
+    #[test]
+    fn malformed_doc_returns_none() {
+        assert!(append(Some("not-json"), &[Some("a"), Some("v")]).is_none());
+    }
+
+    #[test]
+    fn malformed_path_returns_none() {
+        assert!(append(Some(r#"{"a":[1]}"#), &[Some("a{"), Some("v")]).is_none());
+    }
+
+    #[test]
+    fn coerce_types_enforces_string_on_every_slot() {
+        let udf = JsonAppendUdf::new();
+        assert_eq!(
+            udf.coerce_types(&[DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View])
+                .unwrap(),
+            vec![DataType::Utf8, DataType::Utf8, DataType::Utf8]
+        );
+        let err = udf
+            .coerce_types(&[DataType::Utf8, DataType::Int32, DataType::Utf8])
+            .unwrap_err()
+            .to_string();
+        assert!(err.contains("expected string"));
+    }
+
+    #[test]
+    fn return_type_is_utf8() {
+        assert_eq!(
+            JsonAppendUdf::new().return_type(&[DataType::Utf8]).unwrap(),
+            DataType::Utf8
+        );
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_array_length.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_array_length.rs
new file mode 100644
index 0000000000000..c8f60647d0ea0
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_array_length.rs
@@ -0,0 +1,260 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `json_array_length(value)` — length of a JSON array (parity with legacy
+//! `JsonArrayLengthFunctionImpl`; verified by `CalcitePPLJsonBuiltinFunctionIT.testJsonArrayLength`).
+//! NULL / non-array / malformed → NULL. Only plan-time arity / type failures
+//! surface as `plan_err!`; runtime input of any content never errors.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, ArrayRef, Int32Builder, StringArray};
+use datafusion::arrow::datatypes::DataType;
+use datafusion::common::{plan_err, ScalarValue};
+use datafusion::error::{DataFusionError, Result};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+use serde_json::Value;
+
+use super::{coerce_args, CoerceMode};
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(JsonArrayLengthUdf::new()));
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct JsonArrayLengthUdf {
+    signature: Signature,
+}
+
+impl JsonArrayLengthUdf {
+    pub fn new() -> Self {
+        // user_defined + coerce_types lets DF cast LargeUtf8 / Utf8View → Utf8.
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl Default for JsonArrayLengthUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ScalarUDFImpl for JsonArrayLengthUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn name(&self) -> &str {
+        "json_array_length"
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.len() != 1 {
+            return plan_err!(
+                "json_array_length expects 1 argument, got {}",
+                arg_types.len()
+            );
+        }
+        // Int32 to match PPL's INTEGER_FORCE_NULLABLE declaration. Returning
+        // Int64 here works for literal args (Calcite const-folds and inserts a
+        // narrowing CAST on the project) but leaks Int64 through the column
+        // path — caller sees Integer for literals, Long for column refs.
+        Ok(DataType::Int32)
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        coerce_args("json_array_length", arg_types, &[CoerceMode::Utf8])
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() != 1 {
+            return plan_err!(
+                "json_array_length expects 1 argument, got {}",
+                args.args.len()
+            );
+        }
+        let n = args.number_rows;
+
+        // Scalar fast-path: parse once, broadcast as scalar output.
+        if let ColumnarValue::Scalar(sv) = &args.args[0] {
+            let len = match sv {
+                ScalarValue::Utf8(opt) | ScalarValue::LargeUtf8(opt) | ScalarValue::Utf8View(opt) => {
+                    opt.as_deref().and_then(json_array_len)
+                }
+                _ => None,
+            };
+            return Ok(ColumnarValue::Scalar(ScalarValue::Int32(len)));
+        }
+
+        let arr = args.args[0].clone().into_array(n)?;
+        let strings = arr.as_any().downcast_ref::().ok_or_else(|| {
+            DataFusionError::Internal(format!(
+                "json_array_length: expected Utf8, got {:?}",
+                arr.data_type()
+            ))
+        })?;
+
+        let mut builder = Int32Builder::with_capacity(n);
+        for i in 0..n {
+            if strings.is_null(i) {
+                builder.append_null();
+                continue;
+            }
+            match json_array_len(strings.value(i)) {
+                Some(len) => builder.append_value(len),
+                None => builder.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))
+    }
+}
+
+/// Returns the array length as i32, or None for malformed / non-array input.
+/// i32 matches PPL's declared INTEGER return type; arrays exceeding i32::MAX
+/// elements (>2B) saturate to NULL rather than silently truncating.
+fn json_array_len(s: &str) -> Option {
+    serde_json::from_str::(s)
+        .ok()
+        .and_then(|v| v.as_array().map(|a| a.len()))
+        .and_then(|len| i32::try_from(len).ok())
+}
+
+// ─── tests ──────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use datafusion::arrow::array::Int32Array;
+    use datafusion::arrow::datatypes::Field;
+
+    #[test]
+    fn parses_array_returns_length() {
+        assert_eq!(json_array_len("[1,2,3]"), Some(3));
+        assert_eq!(json_array_len("[]"), Some(0));
+        assert_eq!(json_array_len("[\"a\",\"b\"]"), Some(2));
+        // Heterogeneous array — parity with legacy Gson List parse.
+        assert_eq!(json_array_len("[1,\"x\",{\"k\":1}]"), Some(3));
+    }
+
+    #[test]
+    fn non_array_json_returns_none() {
+        assert_eq!(json_array_len("{\"k\":1}"), None);
+        assert_eq!(json_array_len("\"scalar\""), None);
+        assert_eq!(json_array_len("42"), None);
+        assert_eq!(json_array_len("null"), None);
+    }
+
+    #[test]
+    fn malformed_json_returns_none() {
+        assert_eq!(json_array_len("not-json"), None);
+        assert_eq!(json_array_len("[1,2"), None);
+        assert_eq!(json_array_len(""), None);
+    }
+
+    #[test]
+    fn coerce_types_accepts_string_variants() {
+        let udf = JsonArrayLengthUdf::new();
+        for t in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] {
+            let out = udf.coerce_types(std::slice::from_ref(&t)).unwrap();
+            assert_eq!(out, vec![DataType::Utf8], "input {t:?} should coerce to Utf8");
+        }
+    }
+
+    #[test]
+    fn coerce_types_rejects_non_string() {
+        let udf = JsonArrayLengthUdf::new();
+        let err = udf.coerce_types(&[DataType::Int64]).unwrap_err();
+        assert!(err.to_string().contains("expected string"));
+    }
+
+    #[test]
+    fn coerce_types_rejects_wrong_arity() {
+        let udf = JsonArrayLengthUdf::new();
+        assert!(udf.coerce_types(&[]).is_err());
+        assert!(udf.coerce_types(&[DataType::Utf8, DataType::Utf8]).is_err());
+    }
+
+    #[test]
+    fn return_type_is_int32() {
+        let udf = JsonArrayLengthUdf::new();
+        let out = udf.return_type(&[DataType::Utf8]).unwrap();
+        assert_eq!(out, DataType::Int32);
+    }
+
+    #[test]
+    fn invoke_handles_nulls_malformed_and_non_array() {
+        let udf = JsonArrayLengthUdf::new();
+        let input = StringArray::from(vec![
+            Some("[1,2,3]"),
+            None,
+            Some("{\"k\":1}"),
+            Some("not-json"),
+            Some("[]"),
+        ]);
+        let args = ScalarFunctionArgs {
+            args: vec![ColumnarValue::Array(Arc::new(input))],
+            number_rows: 5,
+            arg_fields: vec![],
+            return_field: Arc::new(Field::new("out", DataType::Int32, true)),
+            config_options: Arc::new(datafusion::config::ConfigOptions::new()),
+        };
+        let out = udf.invoke_with_args(args).unwrap();
+        let arr = match out {
+            ColumnarValue::Array(a) => a,
+            _ => panic!("expected array"),
+        };
+        let arr = arr.as_any().downcast_ref::().unwrap();
+        assert_eq!(arr.value(0), 3);
+        assert!(arr.is_null(1));
+        assert!(arr.is_null(2));
+        assert!(arr.is_null(3));
+        assert_eq!(arr.value(4), 0);
+    }
+
+    #[test]
+    fn invoke_scalar_input_produces_scalar_output() {
+        let udf = JsonArrayLengthUdf::new();
+        let args = ScalarFunctionArgs {
+            args: vec![ColumnarValue::Scalar(ScalarValue::Utf8(Some("[1,2,3,4]".into())))],
+            number_rows: 1,
+            arg_fields: vec![],
+            return_field: Arc::new(Field::new("out", DataType::Int32, true)),
+            config_options: Arc::new(datafusion::config::ConfigOptions::new()),
+        };
+        let out = udf.invoke_with_args(args).unwrap();
+        match out {
+            ColumnarValue::Scalar(ScalarValue::Int32(Some(4))) => {}
+            other => panic!("expected Int32(Some(4)), got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn invoke_scalar_null_input_yields_scalar_null() {
+        let udf = JsonArrayLengthUdf::new();
+        let args = ScalarFunctionArgs {
+            args: vec![ColumnarValue::Scalar(ScalarValue::Utf8(None))],
+            number_rows: 1,
+            arg_fields: vec![],
+            return_field: Arc::new(Field::new("out", DataType::Int32, true)),
+            config_options: Arc::new(datafusion::config::ConfigOptions::new()),
+        };
+        let out = udf.invoke_with_args(args).unwrap();
+        match out {
+            ColumnarValue::Scalar(ScalarValue::Int32(None)) => {}
+            other => panic!("expected Int32(None), got {other:?}"),
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_common.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_common.rs
new file mode 100644
index 0000000000000..80e77431d9c3e
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_common.rs
@@ -0,0 +1,294 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! Shared helpers for the PPL `json_*` UDFs: PPL-path parsing (to both JSONPath
+//! strings and typed segment vectors), a segment-based mutation walker used by
+//! the write UDFs, and a malformed-to-`None` JSON parser.
+
+use datafusion::arrow::array::{ArrayRef, StringArray};
+use datafusion::error::{DataFusionError, Result};
+use serde_json::Value;
+
+/// Convert a PPL-style path (`a.b{0}.c{}`) to a JSONPath expression
+/// (`$.a.b[0].c[*]`). Empty input → `"$"` (document root), matching the
+/// legacy contract. Returns a planning error for an unmatched `{`.
+pub(crate) fn convert_ppl_path(input: &str) -> Result {
+    if input.is_empty() {
+        return Ok("$".into());
+    }
+    let mut out = String::with_capacity(input.len() + 2);
+    out.push_str("$.");
+    let mut rest = input;
+    while !rest.is_empty() {
+        match rest.as_bytes()[0] {
+            b'{' => {
+                let end = rest.find('}').ok_or_else(|| {
+                    datafusion::error::DataFusionError::Plan(format!(
+                        "Unmatched '{{' in JSON path: {input}"
+                    ))
+                })?;
+                let idx = rest[1..end].trim();
+                if idx.is_empty() {
+                    out.push_str("[*]");
+                } else {
+                    out.push('[');
+                    out.push_str(idx);
+                    out.push(']');
+                }
+                rest = &rest[end + 1..];
+            }
+            b'.' => {
+                out.push('.');
+                rest = &rest[1..];
+            }
+            _ => {
+                let cut = rest.find(['.', '{']).unwrap_or(rest.len());
+                out.push_str(&rest[..cut]);
+                rest = &rest[cut..];
+            }
+        }
+    }
+    Ok(out)
+}
+
+/// Parse a JSON string; returns `None` on malformed input. Matches the
+/// "malformed → NULL" convention across all json_* UDFs (see
+/// `json_udf_legacy_semantics.md`).
+pub(crate) fn parse(s: &str) -> Option {
+    serde_json::from_str(s).ok()
+}
+
+/// One tokenised step of a PPL path. Mirrors the three cases `convert_ppl_path`
+/// handles: bare identifier (field), `{n}` (array index), `{}` (array wildcard).
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) enum Segment<'a> {
+    Field(&'a str),
+    Index(usize),
+    Wildcard,
+}
+
+/// Tokenise a PPL path into `Segment`s without allocating for field names.
+/// Returns a planning error for unmatched `{` or a non-numeric index — the
+/// same inputs `convert_ppl_path` rejects.
+pub(crate) fn parse_ppl_segments(input: &str) -> Result>> {
+    let mut out = Vec::new();
+    let mut rest = input;
+    while !rest.is_empty() {
+        match rest.as_bytes()[0] {
+            b'{' => {
+                let end = rest.find('}').ok_or_else(|| {
+                    DataFusionError::Plan(format!("Unmatched '{{' in JSON path: {input}"))
+                })?;
+                let idx = rest[1..end].trim();
+                if idx.is_empty() {
+                    out.push(Segment::Wildcard);
+                } else {
+                    let parsed = idx.parse::().map_err(|_| {
+                        DataFusionError::Plan(format!(
+                            "Non-numeric array index '{idx}' in JSON path: {input}"
+                        ))
+                    })?;
+                    out.push(Segment::Index(parsed));
+                }
+                rest = &rest[end + 1..];
+            }
+            b'.' => rest = &rest[1..],
+            _ => {
+                let cut = rest.find(['.', '{']).unwrap_or(rest.len());
+                if cut > 0 {
+                    out.push(Segment::Field(&rest[..cut]));
+                }
+                rest = &rest[cut..];
+            }
+        }
+    }
+    Ok(out)
+}
+
+/// Drive `apply` against every terminal `(parent, final_segment)` reached by
+/// `segments` inside `root`. Missing intermediate keys / out-of-range indices
+/// are silently skipped (matching Jayway's `SUPPRESS_EXCEPTIONS` behaviour
+/// that legacy mutation UDFs rely on). Wildcard segments fan out across every
+/// element of the current array; descending through a non-container
+/// short-circuits that branch.
+///
+/// Empty `segments` is a no-op: PPL mutation UDFs reject a root-only path at
+/// the call site before reaching the walker.
+pub(crate) fn walk_mut(root: &mut Value, segments: &[Segment<'_>], mut apply: F)
+where
+    F: FnMut(&mut Value, &Segment<'_>),
+{
+    if segments.is_empty() {
+        return;
+    }
+    walk_mut_inner(root, segments, &mut apply);
+}
+
+fn walk_mut_inner(node: &mut Value, segments: &[Segment<'_>], apply: &mut F)
+where
+    F: FnMut(&mut Value, &Segment<'_>),
+{
+    let (head, tail) = segments.split_first().expect("non-empty checked by caller");
+    if tail.is_empty() {
+        // Parent is `node`; the final segment names the slot to mutate.
+        apply(node, head);
+        return;
+    }
+    match head {
+        Segment::Field(name) => {
+            if let Value::Object(map) = node {
+                if let Some(child) = map.get_mut(*name) {
+                    walk_mut_inner(child, tail, apply);
+                }
+            }
+        }
+        Segment::Index(i) => {
+            if let Value::Array(arr) = node {
+                if let Some(child) = arr.get_mut(*i) {
+                    walk_mut_inner(child, tail, apply);
+                }
+            }
+        }
+        Segment::Wildcard => {
+            if let Value::Array(arr) = node {
+                for child in arr.iter_mut() {
+                    walk_mut_inner(child, tail, apply);
+                }
+            }
+        }
+    }
+}
+
+/// Standard arity guard.
+pub(crate) fn check_arity(udf: &str, observed: usize, expected: usize) -> Result<()> {
+    (observed == expected)
+        .then_some(())
+        .ok_or_else(|| plan_err_msg(format!("{udf} expects {expected} arguments, got {observed}")))
+}
+
+fn plan_err_msg(msg: String) -> DataFusionError {
+    DataFusionError::Plan(msg)
+}
+
+/// Downcast an `ArrayRef` to `StringArray`. `coerce_types` with `CoerceMode::Utf8`
+/// canonicalizes every string input to `Utf8` before this point, so a failure
+/// indicates a planner bug rather than bad user input.
+pub(crate) fn as_utf8_array(arr: &ArrayRef) -> Result<&StringArray> {
+    arr.as_any().downcast_ref::().ok_or_else(|| {
+        DataFusionError::Internal(format!("expected Utf8, got {:?}", arr.data_type()))
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ppl_path_mirrors_legacy_convert_to_jsonpath() {
+        for (input, want) in [
+            ("", "$"),
+            ("a", "$.a"),
+            ("a.b", "$.a.b"),
+            ("a{0}", "$.a[0]"),
+            ("a{}", "$.a[*]"),
+            ("a{0}.b{}.c", "$.a[0].b[*].c"),
+            ("a{  2  }", "$.a[2]"),
+        ] {
+            assert_eq!(convert_ppl_path(input).unwrap(), want, "input={input}");
+        }
+        assert!(convert_ppl_path("a{0").unwrap_err().to_string().contains("Unmatched"));
+    }
+
+    #[test]
+    fn parse_handles_malformed_and_valid() {
+        assert!(parse("{not json").is_none());
+        assert!(parse("[1,2,3]").is_some());
+    }
+
+    #[test]
+    fn arity_guards() {
+        assert!(check_arity("f", 1, 1).is_ok());
+        assert!(check_arity("f", 2, 1).is_err());
+    }
+
+    #[test]
+    fn parse_ppl_segments_tokenises_field_index_and_wildcard() {
+        assert_eq!(parse_ppl_segments("").unwrap(), Vec::::new());
+        assert_eq!(parse_ppl_segments("a").unwrap(), vec![Segment::Field("a")]);
+        assert_eq!(
+            parse_ppl_segments("a.b{0}.c{}").unwrap(),
+            vec![
+                Segment::Field("a"),
+                Segment::Field("b"),
+                Segment::Index(0),
+                Segment::Field("c"),
+                Segment::Wildcard,
+            ]
+        );
+        assert!(parse_ppl_segments("a{0").is_err());
+        assert!(parse_ppl_segments("a{x}").is_err());
+    }
+
+    fn v(s: &str) -> Value {
+        serde_json::from_str(s).unwrap()
+    }
+
+    #[test]
+    fn walk_mut_deletes_flat_key() {
+        let mut doc = v(r#"{"a":1,"b":2,"c":3}"#);
+        let segs = parse_ppl_segments("b").unwrap();
+        walk_mut(&mut doc, &segs, |parent, seg| {
+            if let (Value::Object(map), Segment::Field(name)) = (parent, seg) {
+                map.shift_remove(*name);
+            }
+        });
+        assert_eq!(serde_json::to_string(&doc).unwrap(), r#"{"a":1,"c":3}"#);
+    }
+
+    #[test]
+    fn walk_mut_handles_missing_path_as_noop() {
+        let mut doc = v(r#"{"f1":"abc","f2":{"f3":"a"}}"#);
+        let segs = parse_ppl_segments("f2.nope").unwrap();
+        walk_mut(&mut doc, &segs, |parent, seg| {
+            if let (Value::Object(map), Segment::Field(name)) = (parent, seg) {
+                map.shift_remove(*name);
+            }
+        });
+        assert_eq!(
+            serde_json::to_string(&doc).unwrap(),
+            r#"{"f1":"abc","f2":{"f3":"a"}}"#
+        );
+    }
+
+    #[test]
+    fn walk_mut_wildcard_fans_out_across_array() {
+        let mut doc = v(r#"{"xs":[{"k":1,"v":10},{"k":2,"v":20}]}"#);
+        let segs = parse_ppl_segments("xs{}.v").unwrap();
+        walk_mut(&mut doc, &segs, |parent, seg| {
+            if let (Value::Object(map), Segment::Field(name)) = (parent, seg) {
+                map.shift_remove(*name);
+            }
+        });
+        assert_eq!(
+            serde_json::to_string(&doc).unwrap(),
+            r#"{"xs":[{"k":1},{"k":2}]}"#
+        );
+    }
+
+    #[test]
+    fn walk_mut_index_out_of_range_is_noop() {
+        let mut doc = v(r#"{"xs":[{"k":1}]}"#);
+        let segs = parse_ppl_segments("xs{5}.k").unwrap();
+        walk_mut(&mut doc, &segs, |parent, seg| {
+            if let (Value::Object(map), Segment::Field(name)) = (parent, seg) {
+                map.shift_remove(*name);
+            }
+        });
+        assert_eq!(serde_json::to_string(&doc).unwrap(), r#"{"xs":[{"k":1}]}"#);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_delete.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_delete.rs
new file mode 100644
index 0000000000000..ff189a7bcc4bf
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_delete.rs
@@ -0,0 +1,286 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `json_delete(value, path1, [path2, ...])` — remove path-matched entries
+//! from a JSON document (parity with legacy `JsonDeleteFunctionImpl` → Jayway
+//! `JsonPath.delete` under `SUPPRESS_EXCEPTIONS`, applied per pathspec).
+//! Missing paths are no-ops; any-NULL-arg / malformed-doc / malformed-path →
+//! NULL. Output key order is preserved via `serde_json`'s `preserve_order`
+//! feature (see `rust/Cargo.toml`).
+
+use std::any::Any;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, ArrayRef, StringBuilder};
+use datafusion::arrow::datatypes::DataType;
+use datafusion::common::ScalarValue;
+use datafusion::error::Result;
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+use serde_json::Value;
+
+use super::json_common::{as_utf8_array, parse, parse_ppl_segments, walk_mut, Segment};
+use super::{coerce_slot, CoerceMode};
+
+const NAME: &str = "json_delete";
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(JsonDeleteUdf::new()));
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct JsonDeleteUdf {
+    signature: Signature,
+}
+
+impl JsonDeleteUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl Default for JsonDeleteUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ScalarUDFImpl for JsonDeleteUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn name(&self) -> &str {
+        NAME
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+    fn return_type(&self, _args: &[DataType]) -> Result {
+        Ok(DataType::Utf8)
+    }
+    fn coerce_types(&self, args: &[DataType]) -> Result> {
+        args.iter()
+            .enumerate()
+            .map(|(i, ty)| coerce_slot(NAME, i, ty, CoerceMode::Utf8))
+            .collect()
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() < 2 {
+            // Legacy `jsonRemove(doc)` with no pathspecs would return `doc`
+            // unchanged. Matching that as SQL NULL (not an error) keeps us
+            // consistent with the other json_* UDFs' any-NULL-arg convention.
+            return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)));
+        }
+        let n = args.number_rows;
+
+        if args
+            .args
+            .iter()
+            .all(|v| matches!(v, ColumnarValue::Scalar(_)))
+        {
+            let doc = scalar_utf8(&args.args[0]);
+            let paths: Vec> = args.args[1..].iter().map(scalar_utf8).collect();
+            let out = delete(doc, &paths);
+            return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(out)));
+        }
+
+        let arrays: Vec = args
+            .args
+            .iter()
+            .map(|v| v.clone().into_array(n))
+            .collect::>()?;
+        let columns: Vec<&datafusion::arrow::array::StringArray> =
+            arrays.iter().map(as_utf8_array).collect::>()?;
+
+        let mut b = StringBuilder::with_capacity(n, n * 16);
+        let mut path_buf: Vec> = Vec::with_capacity(columns.len() - 1);
+        for i in 0..n {
+            let doc = cell(columns[0], i);
+            path_buf.clear();
+            for col in &columns[1..] {
+                path_buf.push(cell(col, i));
+            }
+            match delete(doc, &path_buf) {
+                Some(s) => b.append_value(&s),
+                None => b.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(b.finish()) as ArrayRef))
+    }
+}
+
+fn scalar_utf8(v: &ColumnarValue) -> Option<&str> {
+    match v {
+        ColumnarValue::Scalar(
+            ScalarValue::Utf8(s) | ScalarValue::LargeUtf8(s) | ScalarValue::Utf8View(s),
+        ) => s.as_deref(),
+        _ => None,
+    }
+}
+
+fn cell(arr: &datafusion::arrow::array::StringArray, i: usize) -> Option<&str> {
+    if arr.is_null(i) {
+        None
+    } else {
+        Some(arr.value(i))
+    }
+}
+
+/// Apply every path's delete to a fresh parse of `doc`. Returns `None` for
+/// any-NULL arg / malformed doc / malformed path; otherwise the mutated
+/// document serialized back to a string.
+fn delete(doc: Option<&str>, paths: &[Option<&str>]) -> Option {
+    let doc_str = doc?;
+    if paths.iter().any(|p| p.is_none()) {
+        return None;
+    }
+    let mut value = parse(doc_str)?;
+    for path in paths.iter().map(|p| p.unwrap()) {
+        let segments = parse_ppl_segments(path).ok()?;
+        if segments.is_empty() {
+            // Legacy `jsonRemove` on an empty path attempts to `ctx.read("$")`
+            // which returns the root; `ctx.delete("$")` is a Jayway no-op
+            // (root is indelible). Mirror that by skipping.
+            continue;
+        }
+        delete_one(&mut value, &segments);
+    }
+    serde_json::to_string(&value).ok()
+}
+
+fn delete_one(root: &mut Value, segments: &[Segment<'_>]) {
+    walk_mut(root, segments, |parent, final_seg| {
+        match (parent, final_seg) {
+            (Value::Object(map), Segment::Field(name)) => {
+                map.shift_remove(*name);
+            }
+            (Value::Array(arr), Segment::Index(i)) if *i < arr.len() => {
+                arr.remove(*i);
+            }
+            (Value::Array(arr), Segment::Wildcard) => {
+                arr.clear();
+            }
+            // Type mismatch between the container and the terminal segment is a
+            // silent no-op, matching Jayway's SUPPRESS_EXCEPTIONS.
+            _ => {}
+        }
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn flat_key_delete_matches_legacy_fixture() {
+        // testJsonDelete
+        assert_eq!(
+            delete(
+                Some(r#"{"account_number":1,"balance":39225,"age":32,"gender":"M"}"#),
+                &[Some("age"), Some("gender")],
+            )
+            .as_deref(),
+            Some(r#"{"account_number":1,"balance":39225}"#)
+        );
+    }
+
+    #[test]
+    fn nested_key_delete_preserves_siblings() {
+        // testJsonDeleteWithNested
+        assert_eq!(
+            delete(
+                Some(r#"{"f1":"abc","f2":{"f3":"a","f4":"b"}}"#),
+                &[Some("f2.f3")],
+            )
+            .as_deref(),
+            Some(r#"{"f1":"abc","f2":{"f4":"b"}}"#)
+        );
+    }
+
+    #[test]
+    fn missing_path_returns_document_unchanged() {
+        // testJsonDeleteWithNestedNothing
+        assert_eq!(
+            delete(
+                Some(r#"{"f1":"abc","f2":{"f3":"a","f4":"b"}}"#),
+                &[Some("f2.f100")],
+            )
+            .as_deref(),
+            Some(r#"{"f1":"abc","f2":{"f3":"a","f4":"b"}}"#)
+        );
+    }
+
+    #[test]
+    fn wildcard_array_delete_matches_legacy_fixture() {
+        // testJsonDeleteWithNestedAndArray
+        assert_eq!(
+            delete(
+                Some(
+                    r#"{"teacher":"Alice","student":[{"name":"Bob","rank":1},{"name":"Charlie","rank":2}]}"#
+                ),
+                &[Some("teacher"), Some("student{}.rank")],
+            )
+            .as_deref(),
+            Some(r#"{"student":[{"name":"Bob"},{"name":"Charlie"}]}"#)
+        );
+    }
+
+    #[test]
+    fn any_null_arg_returns_none() {
+        assert!(delete(None, &[Some("a")]).is_none());
+        assert!(delete(Some(r#"{"a":1}"#), &[None]).is_none());
+    }
+
+    #[test]
+    fn malformed_doc_returns_none() {
+        assert!(delete(Some("not-json"), &[Some("a")]).is_none());
+    }
+
+    #[test]
+    fn malformed_path_returns_none() {
+        assert!(delete(Some(r#"{"a":1}"#), &[Some("a{")]).is_none());
+    }
+
+    #[test]
+    fn less_than_two_args_returns_none_via_fast_path() {
+        // Exercised through invoke_with_args — the top-level guard returns
+        // Utf8(None) for <2 args, so helper-level coverage is enough.
+        assert_eq!(
+            delete(Some(r#"{"a":1}"#), &[Some("a")]).as_deref(),
+            Some(r#"{}"#)
+        );
+    }
+
+    #[test]
+    fn coerce_types_enforces_string_on_every_slot() {
+        let udf = JsonDeleteUdf::new();
+        assert_eq!(
+            udf.coerce_types(&[DataType::LargeUtf8, DataType::Utf8View])
+                .unwrap(),
+            vec![DataType::Utf8, DataType::Utf8]
+        );
+        let err = udf
+            .coerce_types(&[DataType::Utf8, DataType::Int32])
+            .unwrap_err()
+            .to_string();
+        assert!(err.contains("expected string"));
+    }
+
+    #[test]
+    fn return_type_is_utf8() {
+        assert_eq!(
+            JsonDeleteUdf::new().return_type(&[DataType::Utf8]).unwrap(),
+            DataType::Utf8
+        );
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_extend.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_extend.rs
new file mode 100644
index 0000000000000..d1f61289a61ed
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_extend.rs
@@ -0,0 +1,348 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `json_extend(value, path1, val1, [path2, val2, ...])` — spread-or-append:
+//! if `valN` parses as a JSON array its elements are pushed individually onto
+//! the path-matched target array; otherwise the whole value is pushed as a
+//! single string element (parity with legacy `gson.fromJson(..., List.class)`
+//! try/fall-back in `JsonExtendFunctionImpl`). Non-array / missing targets are
+//! no-ops; any-NULL-arg / malformed-doc / malformed-path → NULL.
+//!
+//! Intentional divergence from legacy: spread preserves source numeric type
+//! (`[1,2,3]` → `1,2,3`). Gson widens every number to `Double` (`1.0, 2.0,
+//! 3.0`); no legacy IT covers this edge case, so every existing fixture still
+//! passes. Tracked for follow-up cross-engine alignment.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, ArrayRef, StringBuilder};
+use datafusion::arrow::datatypes::DataType;
+use datafusion::common::ScalarValue;
+use datafusion::error::Result;
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+use serde_json::Value;
+
+use super::json_common::{as_utf8_array, parse, parse_ppl_segments, walk_mut, Segment};
+use super::{coerce_slot, CoerceMode};
+
+const NAME: &str = "json_extend";
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(JsonExtendUdf::new()));
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct JsonExtendUdf {
+    signature: Signature,
+}
+
+impl JsonExtendUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl Default for JsonExtendUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ScalarUDFImpl for JsonExtendUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn name(&self) -> &str {
+        NAME
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+    fn return_type(&self, _args: &[DataType]) -> Result {
+        Ok(DataType::Utf8)
+    }
+    fn coerce_types(&self, args: &[DataType]) -> Result> {
+        args.iter()
+            .enumerate()
+            .map(|(i, ty)| coerce_slot(NAME, i, ty, CoerceMode::Utf8))
+            .collect()
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() < 3 || args.args.len().is_multiple_of(2) {
+            return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)));
+        }
+        let n = args.number_rows;
+
+        if args
+            .args
+            .iter()
+            .all(|v| matches!(v, ColumnarValue::Scalar(_)))
+        {
+            let doc = scalar_utf8(&args.args[0]);
+            let rest: Vec> = args.args[1..].iter().map(scalar_utf8).collect();
+            let out = extend(doc, &rest);
+            return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(out)));
+        }
+
+        let arrays: Vec = args
+            .args
+            .iter()
+            .map(|v| v.clone().into_array(n))
+            .collect::>()?;
+        let columns: Vec<&datafusion::arrow::array::StringArray> =
+            arrays.iter().map(as_utf8_array).collect::>()?;
+
+        let mut b = StringBuilder::with_capacity(n, n * 16);
+        let mut rest: Vec> = Vec::with_capacity(columns.len() - 1);
+        for i in 0..n {
+            let doc = cell(columns[0], i);
+            rest.clear();
+            for col in &columns[1..] {
+                rest.push(cell(col, i));
+            }
+            match extend(doc, &rest) {
+                Some(s) => b.append_value(&s),
+                None => b.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(b.finish()) as ArrayRef))
+    }
+}
+
+fn scalar_utf8(v: &ColumnarValue) -> Option<&str> {
+    match v {
+        ColumnarValue::Scalar(
+            ScalarValue::Utf8(s) | ScalarValue::LargeUtf8(s) | ScalarValue::Utf8View(s),
+        ) => s.as_deref(),
+        _ => None,
+    }
+}
+
+fn cell(arr: &datafusion::arrow::array::StringArray, i: usize) -> Option<&str> {
+    if arr.is_null(i) {
+        None
+    } else {
+        Some(arr.value(i))
+    }
+}
+
+/// Classify the raw value string into the push-list the terminal closure
+/// should apply. A successful JSON-array parse expands to the array
+/// elements; anything else (scalar, object, malformed JSON, plain string)
+/// expands to `[Value::String(value)]` — the legacy `gson.fromJson`
+/// try/fall-back pattern.
+fn spread(raw: &str) -> Vec {
+    if let Ok(Value::Array(items)) = serde_json::from_str::(raw) {
+        return items;
+    }
+    vec![Value::String(raw.to_string())]
+}
+
+fn extend(doc: Option<&str>, rest: &[Option<&str>]) -> Option {
+    let doc_str = doc?;
+    if rest.iter().any(|p| p.is_none()) {
+        return None;
+    }
+    let mut value = parse(doc_str)?;
+    for chunk in rest.chunks(2) {
+        let path = chunk[0].unwrap();
+        let new_val = chunk[1].unwrap();
+        let segments = parse_ppl_segments(path).ok()?;
+        if segments.is_empty() {
+            continue;
+        }
+        let items = spread(new_val);
+        extend_one(&mut value, &segments, &items);
+    }
+    serde_json::to_string(&value).ok()
+}
+
+fn extend_one(root: &mut Value, segments: &[Segment<'_>], items: &[Value]) {
+    walk_mut(root, segments, |parent, final_seg| {
+        match (parent, final_seg) {
+            (Value::Object(map), Segment::Field(name)) => {
+                if let Some(Value::Array(arr)) = map.get_mut(*name) {
+                    arr.extend(items.iter().cloned());
+                }
+            }
+            (Value::Array(arr), Segment::Index(i)) if *i < arr.len() => {
+                if let Value::Array(inner) = &mut arr[*i] {
+                    inner.extend(items.iter().cloned());
+                }
+            }
+            (Value::Array(arr), Segment::Wildcard) => {
+                for slot in arr.iter_mut() {
+                    if let Value::Array(inner) = slot {
+                        inner.extend(items.iter().cloned());
+                    }
+                }
+            }
+            _ => {}
+        }
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn json_array_value_is_spread_into_target_array() {
+        // testJsonExtend case c — the stringified json_array(...) value is a
+        // JSON array, so its elements are spread (contrast json_append's case
+        // c, which pushes the whole string as one element).
+        assert_eq!(
+            extend(
+                Some(r#"{"school":{"teacher":["Alice"]}}"#),
+                &[Some("school.teacher"), Some(r#"["Tom","Walt"]"#)],
+            )
+            .as_deref(),
+            Some(r#"{"school":{"teacher":["Alice","Tom","Walt"]}}"#)
+        );
+    }
+
+    #[test]
+    fn non_array_value_falls_back_to_single_push() {
+        // testJsonExtend case a — json_object(...) stringifies to a JSON
+        // object, not an array. Parse-as-List fails → single element pushed.
+        assert_eq!(
+            extend(
+                Some(r#"{"student":[{"name":"Bob","rank":1}]}"#),
+                &[Some("student"), Some(r#"{"name":"Tommy","rank":5}"#)],
+            )
+            .as_deref(),
+            Some(r#"{"student":[{"name":"Bob","rank":1},"{\"name\":\"Tommy\",\"rank\":5}"]}"#)
+        );
+    }
+
+    #[test]
+    fn plain_string_value_falls_back_to_single_push() {
+        // testJsonExtend case b — plain "Tom" / "Walt" strings are not JSON
+        // arrays, so each is pushed as a single element.
+        assert_eq!(
+            extend(
+                Some(r#"{"teacher":["Alice"]}"#),
+                &[Some("teacher"), Some("Tom"), Some("teacher"), Some("Walt")],
+            )
+            .as_deref(),
+            Some(r#"{"teacher":["Alice","Tom","Walt"]}"#)
+        );
+    }
+
+    #[test]
+    fn non_array_target_is_silent_noop() {
+        assert_eq!(
+            extend(
+                Some(r#"{"teacher":"Alice"}"#),
+                &[Some("teacher"), Some(r#"["Tom"]"#)],
+            )
+            .as_deref(),
+            Some(r#"{"teacher":"Alice"}"#)
+        );
+    }
+
+    #[test]
+    fn missing_path_is_silent_noop() {
+        assert_eq!(
+            extend(
+                Some(r#"{"teacher":["Alice"]}"#),
+                &[Some("students"), Some("Tom")],
+            )
+            .as_deref(),
+            Some(r#"{"teacher":["Alice"]}"#)
+        );
+    }
+
+    #[test]
+    fn empty_json_array_value_is_a_noop_on_target() {
+        // Parses as an array of zero items → nothing to push.
+        assert_eq!(
+            extend(
+                Some(r#"{"teacher":["Alice"]}"#),
+                &[Some("teacher"), Some("[]")],
+            )
+            .as_deref(),
+            Some(r#"{"teacher":["Alice"]}"#)
+        );
+    }
+
+    #[test]
+    fn mixed_type_json_array_elements_preserve_their_types() {
+        // Integers/booleans come through as JSON numbers/booleans, not as
+        // stringified elements — diverges from legacy Gson (which widens to
+        // Double) but no legacy IT asserts Gson's widening. See module-level
+        // docs for the rationale + tracking-issue pointer.
+        assert_eq!(
+            extend(
+                Some(r#"{"xs":[0]}"#),
+                &[Some("xs"), Some("[1,2,true,\"s\"]")],
+            )
+            .as_deref(),
+            Some(r#"{"xs":[0,1,2,true,"s"]}"#)
+        );
+    }
+
+    #[test]
+    fn wildcard_path_extends_every_array_child() {
+        assert_eq!(
+            extend(
+                Some(r#"{"groups":[["a"],["b","c"]]}"#),
+                &[Some("groups{}"), Some(r#"["x","y"]"#)],
+            )
+            .as_deref(),
+            Some(r#"{"groups":[["a","x","y"],["b","c","x","y"]]}"#)
+        );
+    }
+
+    #[test]
+    fn any_null_arg_returns_none() {
+        assert!(extend(None, &[Some("a"), Some("v")]).is_none());
+        assert!(extend(Some(r#"{"a":[1]}"#), &[None, Some("v")]).is_none());
+        assert!(extend(Some(r#"{"a":[1]}"#), &[Some("a"), None]).is_none());
+    }
+
+    #[test]
+    fn malformed_doc_returns_none() {
+        assert!(extend(Some("not-json"), &[Some("a"), Some("v")]).is_none());
+    }
+
+    #[test]
+    fn malformed_path_returns_none() {
+        assert!(extend(Some(r#"{"a":[1]}"#), &[Some("a{"), Some("v")]).is_none());
+    }
+
+    #[test]
+    fn coerce_types_enforces_string_on_every_slot() {
+        let udf = JsonExtendUdf::new();
+        assert_eq!(
+            udf.coerce_types(&[DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View])
+                .unwrap(),
+            vec![DataType::Utf8, DataType::Utf8, DataType::Utf8]
+        );
+        let err = udf
+            .coerce_types(&[DataType::Utf8, DataType::Int32, DataType::Utf8])
+            .unwrap_err()
+            .to_string();
+        assert!(err.contains("expected string"));
+    }
+
+    #[test]
+    fn return_type_is_utf8() {
+        assert_eq!(
+            JsonExtendUdf::new().return_type(&[DataType::Utf8]).unwrap(),
+            DataType::Utf8
+        );
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_extract.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_extract.rs
new file mode 100644
index 0000000000000..e046fc759fa98
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_extract.rs
@@ -0,0 +1,308 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `json_extract(value, path1, [path2, ...])` — extract JSON value(s) by PPL-path
+//! (parity with legacy `JsonExtractFunctionImpl` → Calcite `jsonQuery` / `jsonValue`).
+//! Single-path: scalar → `.to_string()`, string → unquoted, object/array →
+//! JSON-serialized, wildcard multi-match → JSON-array, miss/explicit-null → NULL.
+//! Multi-path: per-path results (NULL → `null` element) wrapped in a JSON array.
+//! `< 2` args / any-NULL-arg / malformed doc / malformed path → NULL.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, ArrayRef, StringBuilder};
+use datafusion::arrow::datatypes::DataType;
+use datafusion::common::ScalarValue;
+use datafusion::error::Result;
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+use jsonpath_rust::{JsonPath, JsonPathValue};
+use serde_json::Value;
+
+use super::json_common::{as_utf8_array, convert_ppl_path, parse};
+use super::{coerce_slot, CoerceMode};
+
+const NAME: &str = "json_extract";
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(JsonExtractUdf::new()));
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct JsonExtractUdf {
+    signature: Signature,
+}
+
+impl JsonExtractUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl Default for JsonExtractUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ScalarUDFImpl for JsonExtractUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn name(&self) -> &str {
+        NAME
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+    fn return_type(&self, _args: &[DataType]) -> Result {
+        Ok(DataType::Utf8)
+    }
+    fn coerce_types(&self, args: &[DataType]) -> Result> {
+        // Homogeneous string variadic — every slot canonicalizes to Utf8.
+        args.iter()
+            .enumerate()
+            .map(|(i, ty)| coerce_slot(NAME, i, ty, CoerceMode::Utf8))
+            .collect()
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() < 2 {
+            // Legacy short-circuit: < 2 args → NULL. Avoid plan_err so adapter
+            // mismatches surface as data NULL rather than query failure
+            // (matches JsonExtractFunctionImpl.eval's `if (args.length < 2)`).
+            return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)));
+        }
+        let n = args.number_rows;
+
+        // Scalar fast-path: every operand is Scalar → evaluate once.
+        if args
+            .args
+            .iter()
+            .all(|v| matches!(v, ColumnarValue::Scalar(_)))
+        {
+            let doc = scalar_utf8(&args.args[0]);
+            let paths: Vec> = args.args[1..].iter().map(scalar_utf8).collect();
+            let out = extract(doc, &paths);
+            return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(out)));
+        }
+
+        // Columnar path: materialize each operand to a StringArray and walk
+        // row-by-row. This is the branch production traffic takes.
+        let arrays: Vec = args
+            .args
+            .iter()
+            .map(|v| v.clone().into_array(n))
+            .collect::>()?;
+        let columns: Vec<&datafusion::arrow::array::StringArray> =
+            arrays.iter().map(as_utf8_array).collect::>()?;
+
+        let mut b = StringBuilder::with_capacity(n, n * 16);
+        let mut path_buf: Vec> = Vec::with_capacity(columns.len() - 1);
+        for i in 0..n {
+            let doc = cell(columns[0], i);
+            path_buf.clear();
+            for col in &columns[1..] {
+                path_buf.push(cell(col, i));
+            }
+            match extract(doc, &path_buf) {
+                Some(s) => b.append_value(&s),
+                None => b.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(b.finish()) as ArrayRef))
+    }
+}
+
+fn scalar_utf8(v: &ColumnarValue) -> Option<&str> {
+    match v {
+        ColumnarValue::Scalar(
+            ScalarValue::Utf8(s) | ScalarValue::LargeUtf8(s) | ScalarValue::Utf8View(s),
+        ) => s.as_deref(),
+        _ => None,
+    }
+}
+
+fn cell(arr: &datafusion::arrow::array::StringArray, i: usize) -> Option<&str> {
+    if arr.is_null(i) {
+        None
+    } else {
+        Some(arr.value(i))
+    }
+}
+
+/// Core extraction. Returns `None` for the legacy NULL-producing cases
+/// (any-null arg, malformed doc, malformed path, no match, explicit-null match)
+/// and a `Some(String)` for every matched case.
+fn extract(doc: Option<&str>, paths: &[Option<&str>]) -> Option {
+    let doc_str = doc?;
+    if paths.iter().any(|p| p.is_none()) {
+        return None;
+    }
+    let parsed = parse(doc_str)?;
+    let per_path: Vec = paths
+        .iter()
+        .map(|p| extract_one(&parsed, p.unwrap()).unwrap_or(Value::Null))
+        .collect();
+    if paths.len() == 1 {
+        // Single path: NULL-element collapses to a SQL NULL (legacy's
+        // `queryResult != null ? queryResult : valueResult` returns null).
+        // Scalar matches unwrap to their string form via `jsonize_single`.
+        match per_path.into_iter().next()? {
+            Value::Null => None,
+            v => Some(jsonize_single(v)),
+        }
+    } else {
+        // Multi-path: wrap in JSON array. NULL misses land as literal `null`
+        // elements (testJsonExtractMultiPathWithMissingPath).
+        serde_json::to_string(&Value::Array(per_path)).ok()
+    }
+}
+
+/// Evaluate a single PPL path against a parsed document. Returns `None` for
+/// malformed-path, no-match, and explicit-null matches (the three cases the
+/// legacy Calcite pair resolves to SQL NULL). Single-match returns the raw
+/// `Value`; multi-match returns `Value::Array(...)`.
+fn extract_one(doc: &Value, path: &str) -> Option {
+    let jsonpath = convert_ppl_path(path).ok()?;
+    let compiled = JsonPath::try_from(jsonpath.as_str()).ok()?;
+    let slice = compiled.find_slice(doc);
+    let matches: Vec = slice
+        .into_iter()
+        .filter_map(|v| match v {
+            JsonPathValue::Slice(r, _) => Some(r.clone()),
+            _ => None,
+        })
+        .collect();
+    match matches.len() {
+        0 => None,
+        1 => match matches.into_iter().next().unwrap() {
+            Value::Null => None,
+            v => Some(v),
+        },
+        _ => Some(Value::Array(matches)),
+    }
+}
+
+/// Legacy `doJsonize` single-path output: strings emerge unquoted; every other
+/// JSON value (numbers, bools, arrays, objects) is serialized. Matches the
+/// legacy `isScalarObject` branch (`.toString()` on Java scalars → same bytes
+/// as `serde_json::to_string` for numbers and booleans).
+fn jsonize_single(v: Value) -> String {
+    match v {
+        Value::String(s) => s,
+        other => other.to_string(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn parsed(s: &str) -> Value {
+        serde_json::from_str(s).unwrap()
+    }
+
+    #[test]
+    fn single_path_scalar_match_returns_tostring_form() {
+        let doc = parsed(r#"{"a":801.0,"b":"hi","c":true,"d":42}"#);
+        assert_eq!(extract_one(&doc, "a").map(jsonize_single).unwrap(), "801.0");
+        assert_eq!(extract_one(&doc, "b").map(jsonize_single).unwrap(), "hi");
+        assert_eq!(extract_one(&doc, "c").map(jsonize_single).unwrap(), "true");
+        assert_eq!(extract_one(&doc, "d").map(jsonize_single).unwrap(), "42");
+    }
+
+    #[test]
+    fn single_path_container_match_is_jsonized() {
+        let doc = parsed(r#"{"a":{"x":1,"y":2}}"#);
+        assert_eq!(
+            extract_one(&doc, "a").map(jsonize_single).unwrap(),
+            r#"{"x":1,"y":2}"#
+        );
+    }
+
+    #[test]
+    fn wildcard_multi_match_wraps_in_array() {
+        let doc = parsed(r#"{"a":[{"t":"A"},{"t":"B"},{"t":"C"}]}"#);
+        let v = extract_one(&doc, "a{}.t").unwrap();
+        assert_eq!(serde_json::to_string(&v).unwrap(), r#"["A","B","C"]"#);
+    }
+
+    #[test]
+    fn missing_and_explicit_null_both_yield_none() {
+        let doc = parsed(r#"{"a":null}"#);
+        assert!(extract_one(&doc, "a").is_none());
+        assert!(extract_one(&doc, "missing").is_none());
+    }
+
+    #[test]
+    fn multi_path_wraps_with_null_slots_for_misses() {
+        let doc = parsed(r#"{"name":"John"}"#);
+        let out = extract(Some(r#"{"name":"John"}"#), &[Some("name"), Some("age")]);
+        assert_eq!(out.as_deref(), Some(r#"["John",null]"#));
+        // No-op parse path so the call shape matches the legacy IT input.
+        assert!(doc.is_object());
+    }
+
+    #[test]
+    fn less_than_two_args_returns_none_via_fast_path() {
+        // Exercised via the UDF entry point in the integration tests; the
+        // core `extract` helper is only called with ≥1 path, so we just
+        // verify the single-path path works end-to-end here.
+        let out = extract(Some(r#"{"a":1}"#), &[Some("a")]);
+        assert_eq!(out.as_deref(), Some("1"));
+    }
+
+    #[test]
+    fn any_null_arg_returns_none() {
+        assert!(extract(None, &[Some("a")]).is_none());
+        assert!(extract(Some(r#"{"a":1}"#), &[None]).is_none());
+    }
+
+    #[test]
+    fn malformed_document_returns_none() {
+        assert!(extract(Some("not-json"), &[Some("a")]).is_none());
+    }
+
+    #[test]
+    fn malformed_path_returns_none() {
+        // Unmatched `{` bubbles out as None (legacy would also emit NULL via
+        // the PLAN-error swallow in the stateful function).
+        assert!(extract(Some(r#"{"a":1}"#), &[Some("a{0")]).is_none());
+    }
+
+    #[test]
+    fn coerce_types_enforces_string_on_every_slot() {
+        let udf = JsonExtractUdf::new();
+        assert_eq!(
+            udf.coerce_types(&[DataType::LargeUtf8, DataType::Utf8View])
+                .unwrap(),
+            vec![DataType::Utf8, DataType::Utf8]
+        );
+        let err = udf
+            .coerce_types(&[DataType::Utf8, DataType::Int32])
+            .unwrap_err()
+            .to_string();
+        assert!(err.contains("expected string"));
+    }
+
+    #[test]
+    fn return_type_is_utf8() {
+        assert_eq!(
+            JsonExtractUdf::new()
+                .return_type(&[DataType::Utf8])
+                .unwrap(),
+            DataType::Utf8
+        );
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_keys.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_keys.rs
new file mode 100644
index 0000000000000..5971f45aff3c9
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_keys.rs
@@ -0,0 +1,155 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `json_keys(value)` — top-level keys of a JSON object, encoded as a JSON-array
+//! string (parity with legacy `JsonKeysFunctionImpl` → Calcite
+//! `JsonFunctions.jsonKeys`). Non-object / malformed / NULL input → NULL.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, ArrayRef, StringBuilder};
+use datafusion::arrow::datatypes::DataType;
+use datafusion::common::ScalarValue;
+use datafusion::error::Result;
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+use serde_json::Value;
+
+use super::json_common::{as_utf8_array, check_arity, parse};
+use super::{coerce_args, CoerceMode};
+
+const NAME: &str = "json_keys";
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(JsonKeysUdf::new()));
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct JsonKeysUdf {
+    signature: Signature,
+}
+
+impl JsonKeysUdf {
+    pub fn new() -> Self {
+        Self { signature: Signature::user_defined(Volatility::Immutable) }
+    }
+}
+
+impl Default for JsonKeysUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ScalarUDFImpl for JsonKeysUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn name(&self) -> &str {
+        NAME
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+    fn return_type(&self, _args: &[DataType]) -> Result {
+        Ok(DataType::Utf8)
+    }
+    fn coerce_types(&self, args: &[DataType]) -> Result> {
+        coerce_args(NAME, args, &[CoerceMode::Utf8])
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        check_arity(NAME, args.args.len(), 1)?;
+        let n = args.number_rows;
+
+        if let ColumnarValue::Scalar(sv) = &args.args[0] {
+            let keys = match sv {
+                ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) | ScalarValue::Utf8View(Some(s)) => {
+                    json_keys(s)
+                }
+                _ => None,
+            };
+            return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(keys)));
+        }
+
+        let arr = args.args[0].clone().into_array(n)?;
+        let strings = as_utf8_array(&arr)?;
+        let mut b = StringBuilder::with_capacity(n, n * 16);
+        for i in 0..n {
+            if strings.is_null(i) {
+                b.append_null();
+                continue;
+            }
+            match json_keys(strings.value(i)) {
+                Some(s) => b.append_value(&s),
+                None => b.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(b.finish()) as ArrayRef))
+    }
+}
+
+/// Returns the JSON-array-encoded list of top-level keys for an object input,
+/// or `None` for malformed / non-object / scalar / array inputs. Matches the
+/// legacy contract; `serde_json::Map` is order-preserving by default (via the
+/// `preserve_order` feature disabled — insertion order on BTreeMap is
+/// alphabetical. Tests assert the observed ordering rather than insertion
+/// order to avoid coupling to a crate feature flag.
+fn json_keys(s: &str) -> Option {
+    match parse(s)? {
+        Value::Object(map) => {
+            let keys: Vec<&String> = map.keys().collect();
+            serde_json::to_string(&keys).ok()
+        }
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn object_input_returns_jsonized_keys() {
+        assert_eq!(
+            json_keys(r#"{"f1":"abc","f2":{"f3":"a"}}"#).as_deref(),
+            Some(r#"["f1","f2"]"#)
+        );
+        assert_eq!(json_keys(r#"{}"#).as_deref(), Some(r#"[]"#));
+    }
+
+    #[test]
+    fn non_object_returns_none() {
+        assert_eq!(json_keys(r#"[1,2,3]"#), None);
+        assert_eq!(json_keys(r#"42"#), None);
+        assert_eq!(json_keys(r#""scalar""#), None);
+        assert_eq!(json_keys(r#"null"#), None);
+    }
+
+    #[test]
+    fn malformed_returns_none() {
+        assert_eq!(json_keys(""), None);
+        assert_eq!(json_keys("{not-json"), None);
+    }
+
+    #[test]
+    fn return_type_is_utf8() {
+        assert_eq!(JsonKeysUdf::new().return_type(&[DataType::Utf8]).unwrap(), DataType::Utf8);
+    }
+
+    #[test]
+    fn coerce_types_enforces_string_arity() {
+        let udf = JsonKeysUdf::new();
+        assert_eq!(udf.coerce_types(&[DataType::LargeUtf8]).unwrap(), vec![DataType::Utf8]);
+        assert!(udf.coerce_types(&[DataType::Int64]).unwrap_err().to_string().contains("expected string"));
+        assert!(udf.coerce_types(&[]).is_err());
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_set.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_set.rs
new file mode 100644
index 0000000000000..daf33e8d2f6bb
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/json_set.rs
@@ -0,0 +1,281 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `json_set(value, path1, val1, [path2, val2, ...])` — replace the value at
+//! each path match (parity with legacy `JsonSetFunctionImpl` → Jayway
+//! `ctx.set` guarded by `ctx.read != null`: *replace-only*, never inserts).
+//! Missing paths are no-ops; any-NULL-arg / odd trailing arg / malformed-doc /
+//! malformed-path → NULL.
+//!
+//! Values always store as JSON strings because every UDF arg is coerced to
+//! Utf8 upstream — matching the legacy fixture `"b":"3"` (not `"b":3`).
+
+use std::any::Any;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{Array, ArrayRef, StringBuilder};
+use datafusion::arrow::datatypes::DataType;
+use datafusion::common::ScalarValue;
+use datafusion::error::Result;
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+use serde_json::Value;
+
+use super::json_common::{as_utf8_array, parse, parse_ppl_segments, walk_mut, Segment};
+use super::{coerce_slot, CoerceMode};
+
+const NAME: &str = "json_set";
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(JsonSetUdf::new()));
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct JsonSetUdf {
+    signature: Signature,
+}
+
+impl JsonSetUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl Default for JsonSetUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ScalarUDFImpl for JsonSetUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn name(&self) -> &str {
+        NAME
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+    fn return_type(&self, _args: &[DataType]) -> Result {
+        Ok(DataType::Utf8)
+    }
+    fn coerce_types(&self, args: &[DataType]) -> Result> {
+        args.iter()
+            .enumerate()
+            .map(|(i, ty)| coerce_slot(NAME, i, ty, CoerceMode::Utf8))
+            .collect()
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        // Need doc + at least one (path, value) pair. Odd trailing arg (unpaired
+        // path) short-circuits to NULL — matches the legacy `for (i=1; i> = args.args[1..].iter().map(scalar_utf8).collect();
+            let out = set(doc, &rest);
+            return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(out)));
+        }
+
+        let arrays: Vec = args
+            .args
+            .iter()
+            .map(|v| v.clone().into_array(n))
+            .collect::>()?;
+        let columns: Vec<&datafusion::arrow::array::StringArray> =
+            arrays.iter().map(as_utf8_array).collect::>()?;
+
+        let mut b = StringBuilder::with_capacity(n, n * 16);
+        let mut rest: Vec> = Vec::with_capacity(columns.len() - 1);
+        for i in 0..n {
+            let doc = cell(columns[0], i);
+            rest.clear();
+            for col in &columns[1..] {
+                rest.push(cell(col, i));
+            }
+            match set(doc, &rest) {
+                Some(s) => b.append_value(&s),
+                None => b.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(b.finish()) as ArrayRef))
+    }
+}
+
+fn scalar_utf8(v: &ColumnarValue) -> Option<&str> {
+    match v {
+        ColumnarValue::Scalar(
+            ScalarValue::Utf8(s) | ScalarValue::LargeUtf8(s) | ScalarValue::Utf8View(s),
+        ) => s.as_deref(),
+        _ => None,
+    }
+}
+
+fn cell(arr: &datafusion::arrow::array::StringArray, i: usize) -> Option<&str> {
+    if arr.is_null(i) {
+        None
+    } else {
+        Some(arr.value(i))
+    }
+}
+
+/// Apply each (path, value) pair to a fresh parse of `doc`. Replace-only:
+/// missing paths are no-ops, matching legacy `jsonSet`'s `ctx.read != null`
+/// guard.
+fn set(doc: Option<&str>, rest: &[Option<&str>]) -> Option {
+    let doc_str = doc?;
+    if rest.iter().any(|p| p.is_none()) {
+        return None;
+    }
+    let mut value = parse(doc_str)?;
+    for chunk in rest.chunks(2) {
+        let path = chunk[0].unwrap();
+        let new_val = chunk[1].unwrap();
+        let segments = parse_ppl_segments(path).ok()?;
+        if segments.is_empty() {
+            // Setting the root is a Jayway no-op (root is indelible and
+            // unreplaceable via `ctx.set("$", v)`). Mirror that.
+            continue;
+        }
+        set_one(&mut value, &segments, new_val);
+    }
+    serde_json::to_string(&value).ok()
+}
+
+fn set_one(root: &mut Value, segments: &[Segment<'_>], new_val: &str) {
+    let replacement = Value::String(new_val.to_string());
+    walk_mut(root, segments, |parent, final_seg| {
+        match (parent, final_seg) {
+            // Replace-only: only overwrite if the key already exists.
+            (Value::Object(map), Segment::Field(name)) if map.contains_key(*name) => {
+                map.insert((*name).to_string(), replacement.clone());
+            }
+            (Value::Array(arr), Segment::Index(i)) if *i < arr.len() => {
+                arr[*i] = replacement.clone();
+            }
+            (Value::Array(arr), Segment::Wildcard) => {
+                for slot in arr.iter_mut() {
+                    *slot = replacement.clone();
+                }
+            }
+            // Type mismatch is a silent no-op (legacy SUPPRESS_EXCEPTIONS).
+            _ => {}
+        }
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn wildcard_replace_matches_legacy_fixture() {
+        // testJsonSet
+        assert_eq!(
+            set(
+                Some(r#"{"a":[{"b":1},{"b":2}]}"#),
+                &[Some("a{}.b"), Some("3")],
+            )
+            .as_deref(),
+            Some(r#"{"a":[{"b":"3"},{"b":"3"}]}"#)
+        );
+    }
+
+    #[test]
+    fn wrong_path_leaves_input_unchanged() {
+        // testJsonSetWithWrongPath — 'a{}.b.d' doesn't exist (b is a scalar).
+        assert_eq!(
+            set(
+                Some(r#"{"a":[{"b":1},{"b":2}]}"#),
+                &[Some("a{}.b.d"), Some("3")],
+            )
+            .as_deref(),
+            Some(r#"{"a":[{"b":1},{"b":2}]}"#)
+        );
+    }
+
+    #[test]
+    fn partial_wildcard_match_only_sets_where_path_exists() {
+        // testJsonSetPartialSet
+        assert_eq!(
+            set(
+                Some(r#"{"a":[{"b":1},{"b":{"c":2}}]}"#),
+                &[Some("a{}.b.c"), Some("3")],
+            )
+            .as_deref(),
+            Some(r#"{"a":[{"b":1},{"b":{"c":"3"}}]}"#)
+        );
+    }
+
+    #[test]
+    fn multiple_path_value_pairs_apply_sequentially() {
+        assert_eq!(
+            set(
+                Some(r#"{"a":1,"b":2}"#),
+                &[Some("a"), Some("10"), Some("b"), Some("20")],
+            )
+            .as_deref(),
+            Some(r#"{"a":"10","b":"20"}"#)
+        );
+    }
+
+    #[test]
+    fn any_null_arg_returns_none() {
+        assert!(set(None, &[Some("a"), Some("v")]).is_none());
+        assert!(set(Some(r#"{"a":1}"#), &[None, Some("v")]).is_none());
+        assert!(set(Some(r#"{"a":1}"#), &[Some("a"), None]).is_none());
+    }
+
+    #[test]
+    fn malformed_doc_returns_none() {
+        assert!(set(Some("not-json"), &[Some("a"), Some("v")]).is_none());
+    }
+
+    #[test]
+    fn malformed_path_returns_none() {
+        assert!(set(Some(r#"{"a":1}"#), &[Some("a{"), Some("v")]).is_none());
+    }
+
+    #[test]
+    fn coerce_types_enforces_string_on_every_slot() {
+        let udf = JsonSetUdf::new();
+        assert_eq!(
+            udf.coerce_types(&[DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View])
+                .unwrap(),
+            vec![DataType::Utf8, DataType::Utf8, DataType::Utf8]
+        );
+        let err = udf
+            .coerce_types(&[DataType::Utf8, DataType::Int32, DataType::Utf8])
+            .unwrap_err()
+            .to_string();
+        assert!(err.contains("expected string"));
+    }
+
+    #[test]
+    fn return_type_is_utf8() {
+        assert_eq!(
+            JsonSetUdf::new().return_type(&[DataType::Utf8]).unwrap(),
+            DataType::Utf8
+        );
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
index 2053625b544c0..cdbcb9db9bf83 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
@@ -7,11 +7,12 @@
  */
 
 //! OpenSearch scalar UDFs that aren't in DataFusion's built-in registry. Each
-//! must have a matching YAML entry in `extensions/opensearch_scalar.yaml` so
+//! must have a matching YAML entry in `opensearch_scalar_functions.yaml` so
 //! the substrait converter on the Java side can route to it by name.
 //!
 //! Functions registered here:
 //! - `convert_tz(ts, from_tz, to_tz)` — DST-aware timezone shift (chrono-tz)
+//! - `json_array_length(value)` — length of a JSON array, NULL on malformed/non-array
 
 use datafusion::arrow::datatypes::{DataType, TimeUnit};
 use datafusion::common::plan_err;
@@ -113,14 +114,38 @@ pub(crate) fn coerce_args(
 }
 
 pub mod convert_tz;
+pub mod json_append;
+pub mod json_array_length;
+pub(crate) mod json_common;
+pub mod json_delete;
+pub mod json_extend;
+pub mod json_extract;
+pub mod json_keys;
+pub mod json_set;
 pub mod tonumber;
 pub mod tostring;
 
+// Dev note: if a freshly added UDF here fails at runtime with
+// "Unsupported function name: " despite the Java side being wired, the
+// native dylib is stale. `sandbox/libs/dataformat-native/build.gradle` tracks
+// only common/ + lib/ as inputs, so plugin-side Rust edits leave gradle
+// UP-TO-DATE. Workaround: run
+// `./gradlew :sandbox:libs:dataformat-native:buildRustLibrary --rerun-tasks`
+// and restart the OpenSearch JVM (the loaded dylib is JVM-cached).
 pub fn register_all(ctx: &SessionContext) {
     convert_tz::register_all(ctx);
+    json_append::register_all(ctx);
+    json_array_length::register_all(ctx);
+    json_delete::register_all(ctx);
+    json_extend::register_all(ctx);
+    json_extract::register_all(ctx);
+    json_keys::register_all(ctx);
+    json_set::register_all(ctx);
     tonumber::register_all(ctx);
     tostring::register_all(ctx);
-    log::info!("OpenSearch UDF register_all: convert_tz, tonumber, tostring registered");
+    log::info!(
+        "OpenSearch UDF register_all: convert_tz, json_append, json_array_length, json_delete, json_extend, json_extract, json_keys, json_set, tonumber, tostring registered"
+    );
 }
 
 #[cfg(test)]
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java
index dd3dbcb1bc0bb..0c348bccfd9e1 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/BaseScalarFunctionIT.java
@@ -121,6 +121,12 @@ private void createBankIndex() throws Exception {
             .startObject("created_at")
             .field("type", "date")
             .endObject()
+            // json_str holds serialized JSON arrays/objects/malformed strings so
+            // scalar-JSON UDFs can be exercised on real column values (columnar
+            // UDF path), not just string literals (scalar fast-path).
+            .startObject("json_str")
+            .field("type", "keyword")
+            .endObject()
             .endObject()
             .endObject();
 
@@ -143,13 +149,38 @@ private void createBankIndex() throws Exception {
     }
 
     private void indexBankDocs() {
+        // Row 1 carries a 3-element JSON array in json_str; row 6 carries a JSON object.
+        // This lets scalar-JSON UDF tests assert both the happy path (row 1 → length 3)
+        // and the non-array → NULL path (row 6) from real column values.
         client().prepareIndex(BANK_INDEX)
             .setId("1")
-            .setSource("account_number", 1, "firstname", "Amber", "balance", 39225L, "created_at", "2024-06-15T10:30:00Z")
+            .setSource(
+                "account_number",
+                1,
+                "firstname",
+                "Amber",
+                "balance",
+                39225L,
+                "created_at",
+                "2024-06-15T10:30:00Z",
+                "json_str",
+                "[1,2,3]"
+            )
             .get();
         client().prepareIndex(BANK_INDEX)
             .setId("6")
-            .setSource("account_number", 6, "firstname", "Hattie", "balance", 5686L, "created_at", "2024-01-20T14:45:30Z")
+            .setSource(
+                "account_number",
+                6,
+                "firstname",
+                "Hattie",
+                "balance",
+                5686L,
+                "created_at",
+                "2024-01-20T14:45:30Z",
+                "json_str",
+                "{\"k\":1}"
+            )
             .get();
     }
 
@@ -182,6 +213,32 @@ protected void assertScalarLong(String expr, long expected) {
         assertEquals(expr, expected, ((Number) cell).longValue());
     }
 
+    /**
+     * Strict variant that asserts the cell is a {@link Long} (not just a {@link Number}).
+     * Use for functions whose on-wire BIGINT return type must not silently regress.
+     */
+    protected void assertScalarLongStrict(String expr, long expected) {
+        Object cell = evalScalar(expr);
+        assertNotNull(expr + " result must not be null", cell);
+        assertTrue(expr + " result must be Long, got " + cell.getClass(), cell instanceof Long);
+        assertEquals(expr, expected, ((Long) cell).longValue());
+    }
+
+    /**
+     * Strict variant that asserts the cell is an {@link Integer}. Use for functions
+     * whose on-wire INTEGER return type must be preserved through the pipeline —
+     * e.g. PPL scalar UDFs declared as {@code INTEGER_FORCE_NULLABLE} whose Rust
+     * implementations return {@code Int64} but get narrowed via an implicit CAST
+     * on the enclosing Project. The non-strict {@link #assertScalarLong} silently
+     * accepts either width and would miss this contract regression.
+     */
+    protected void assertScalarIntStrict(String expr, int expected) {
+        Object cell = evalScalar(expr);
+        assertNotNull(expr + " result must not be null", cell);
+        assertTrue(expr + " result must be Integer, got " + cell.getClass(), cell instanceof Integer);
+        assertEquals(expr, expected, ((Integer) cell).intValue());
+    }
+
     protected void assertScalarDouble(String expr, double expected, double delta) {
         Object cell = evalScalar(expr);
         assertNotNull(expr + " result must not be null", cell);
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarJsonFunctionIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarJsonFunctionIT.java
new file mode 100644
index 0000000000000..67214bb09a724
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarJsonFunctionIT.java
@@ -0,0 +1,180 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+/**
+ * End-to-end smoke tests for PPL {@code json_*} scalar functions routed through
+ * PPL → Calcite → Substrait → DataFusion. One method per function for happy-path
+ * + column-valued coverage; {@code *ParityWithLegacy} methods replay the legacy
+ * SQL plugin's {@code CalcitePPLJsonBuiltinFunctionIT} fixtures verbatim.
+ * Edge cases are covered in Rust unit tests.
+ */
+public class ScalarJsonFunctionIT extends BaseScalarFunctionIT {
+
+    /** Happy path + NULL-on-non-array/malformed (scalar fast-path) + column-valued (Arrow columnar path). */
+    public void testJsonArrayLength() {
+        assertScalarIntStrict("json_array_length('[1,2,3]')", 3);
+        assertScalarIntStrict("json_array_length('[]')", 0);
+        assertScalarNull("json_array_length('{\"k\":1}')");
+        assertScalarNull("json_array_length('not-json')");
+        // Columnar path: bank fixture's json_str row 1 is '[1,2,3]'.
+        assertScalarIntStrict("json_array_length(json_str)", 3);
+    }
+
+    /** Parity replay of {@code CalcitePPLJsonBuiltinFunctionIT.testJsonArrayLength}. */
+    public void testJsonArrayLengthParityWithLegacy() {
+        assertScalarIntStrict("json_array_length('[1,2,3,4]')", 4);
+        assertScalarIntStrict("json_array_length('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]')", 5);
+        assertScalarNull("json_array_length('{\"key\": 1}')");
+    }
+
+    /** Parity replay of {@code CalcitePPLJsonBuiltinFunctionIT.testJsonKeys} — insertion order preserved via {@code serde_json} {@code preserve_order}. */
+    public void testJsonKeysParityWithLegacy() {
+        assertScalarString("json_keys('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}')", "[\"f1\",\"f2\"]");
+        assertScalarNull("json_keys('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]')");
+        assertScalarNull("json_keys('not-json')");
+        assertScalarNull("json_keys('42')");
+    }
+
+    /** Parity replay of {@code CalcitePPLJsonBuiltinFunctionIT.testJsonExtract*} — byte-for-byte match via {@code serde_json} {@code preserve_order} + no integer↔double coercion. */
+    public void testJsonExtractParityWithLegacy() {
+        String candidate = "[{\"name\":\"London\",\"Bridges\":[{\"name\":\"Tower Bridge\",\"length\":801.0},"
+            + "{\"name\":\"Millennium Bridge\",\"length\":1066.0}]},"
+            + "{\"name\":\"Venice\",\"Bridges\":[{\"name\":\"Rialto Bridge\",\"length\":157.0},"
+            + "{\"type\":\"Bridge of Sighs\",\"length\":36.0},"
+            + "{\"type\":\"Ponte della Paglia\"}]},"
+            + "{\"name\":\"San Francisco\",\"Bridges\":[{\"name\":\"Golden Gate Bridge\",\"length\":8981.0},"
+            + "{\"name\":\"Bay Bridge\",\"length\":23556.0}]}]";
+
+        // Single-path, wildcard-at-root over top-level array → 3 matches wrapped
+        // in a JSON array. Round-tripped bytes equal the input because
+        // preserve_order + no numeric coercion.
+        assertScalarString("json_extract('" + candidate + "', '{}')", candidate);
+
+        // Single-path scalar match — legacy `.toString()` on Double(8981.0).
+        assertScalarString("json_extract('" + candidate + "', '{2}.Bridges{0}.length')", "8981.0");
+
+        // Wildcard-over-wildcard-missing-key: only Venice entries without a
+        // `name` field expose a `type`, so two matches wrap into a JSON array.
+        assertScalarString("json_extract('" + candidate + "', '{}.Bridges{}.type')", "[\"Bridge of Sighs\",\"Ponte della Paglia\"]");
+
+        // Single-path object match — jsonized with insertion order preserved.
+        assertScalarString("json_extract('" + candidate + "', '{2}.Bridges{0}')", "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0}");
+
+        // Multi-path with wildcard-multi + scalar-match → outer array wraps
+        // the two per-path results (array + scalar) as-is.
+        assertScalarString(
+            "json_extract('" + candidate + "', '{}.Bridges{}.type', '{2}.Bridges{0}.length')",
+            "[[\"Bridge of Sighs\",\"Ponte della Paglia\"],8981.0]"
+        );
+
+        // Missing path (empty object) and explicit-null both resolve to SQL NULL.
+        assertScalarNull("json_extract('{}', 'name')");
+        assertScalarNull("json_extract('{\"name\": null}', 'name')");
+
+        // Multi-path with missing path yields literal `null` element in the
+        // outer JSON array.
+        assertScalarString("json_extract('{\"name\": \"John\"}', 'name', 'age')", "[\"John\",null]");
+    }
+
+    /** Parity replay of {@code CalcitePPLJsonBuiltinFunctionIT.testJsonSet*} — values stored as JSON strings matches legacy {@code "b":"3"} outputs (Utf8 arg coercion). */
+    public void testJsonSetParityWithLegacy() {
+        // testJsonSet: wildcard replace across every array element.
+        assertScalarString("json_set('{\"a\":[{\"b\":1},{\"b\":2}]}', 'a{}.b', '3')", "{\"a\":[{\"b\":\"3\"},{\"b\":\"3\"}]}");
+
+        // testJsonSetWithWrongPath: 'a{}.b.d' doesn't exist — input unchanged.
+        assertScalarString("json_set('{\"a\":[{\"b\":1},{\"b\":2}]}', 'a{}.b.d', '3')", "{\"a\":[{\"b\":1},{\"b\":2}]}");
+
+        // testJsonSetPartialSet: wildcard where only one branch has the full
+        // path; only the matching branch is rewritten.
+        assertScalarString(
+            "json_set('{\"a\":[{\"b\":1},{\"b\":{\"c\":2}}]}', 'a{}.b.c', '3')",
+            "{\"a\":[{\"b\":1},{\"b\":{\"c\":\"3\"}}]}"
+        );
+    }
+
+    /** Parity replay of {@code CalcitePPLJsonBuiltinFunctionIT.testJsonAppend} — nested {@code json_object}/{@code json_array} constructors replaced with their stringified equivalents (same observable contract). */
+    public void testJsonAppendParityWithLegacy() {
+        // Case a: pre-stringified json_object(...) appended as a single array element.
+        assertScalarString(
+            "json_append('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}',"
+                + " 'student', '{\"name\":\"Tomy\",\"rank\":5}')",
+            "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},"
+                + "\"{\\\"name\\\":\\\"Tomy\\\",\\\"rank\\\":5}\"]}"
+        );
+
+        // Case b: multi-pair append on the same target.
+        assertScalarString(
+            "json_append('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}',"
+                + " 'teacher', 'Tom', 'teacher', 'Walt')",
+            "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}"
+        );
+
+        // Case c: nested-path + pre-stringified json_array(...) appended as a single string element.
+        assertScalarString(
+            "json_append('{\"school\":{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}',"
+                + " 'school.teacher', '[\"Tom\",\"Walt\"]')",
+            "{\"school\":{\"teacher\":[\"Alice\",\"[\\\"Tom\\\",\\\"Walt\\\"]\"],"
+                + "\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}"
+        );
+    }
+
+    /** Parity replay of {@code CalcitePPLJsonBuiltinFunctionIT.testJsonExtend} — case c diverges from append: a JSON-array value is spread (not pushed as a single element). */
+    public void testJsonExtendParityWithLegacy() {
+        // Case a: stringified json_object value — not a JSON array → single push.
+        assertScalarString(
+            "json_extend('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}',"
+                + " 'student', '{\"name\":\"Tommy\",\"rank\":5}')",
+            "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},"
+                + "\"{\\\"name\\\":\\\"Tommy\\\",\\\"rank\\\":5}\"]}"
+        );
+
+        // Case b: plain strings — each fails List-parse → each pushed individually.
+        assertScalarString(
+            "json_extend('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}',"
+                + " 'teacher', 'Tom', 'teacher', 'Walt')",
+            "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}"
+        );
+
+        // Case c: stringified json_array — parses as JSON array → elements spread.
+        assertScalarString(
+            "json_extend('{\"school\":{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}',"
+                + " 'school.teacher', '[\"Tom\",\"Walt\"]')",
+            "{\"school\":{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],"
+                + "\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}"
+        );
+    }
+
+    /** Parity replay of {@code CalcitePPLJsonBuiltinFunctionIT.testJsonDelete*} — output order preserved via {@code serde_json} {@code preserve_order}. */
+    public void testJsonDeleteParityWithLegacy() {
+        // testJsonDelete: flat-key delete of two fields.
+        assertScalarString(
+            "json_delete('{\"account_number\":1,\"balance\":39225,\"age\":32,\"gender\":\"M\"}', 'age', 'gender')",
+            "{\"account_number\":1,\"balance\":39225}"
+        );
+
+        // testJsonDeleteWithNested: delete a single nested key.
+        assertScalarString(
+            "json_delete('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}', 'f2.f3')",
+            "{\"f1\":\"abc\",\"f2\":{\"f4\":\"b\"}}"
+        );
+
+        // testJsonDeleteWithNestedNothing: missing nested key leaves input unchanged.
+        assertScalarString(
+            "json_delete('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}', 'f2.f100')",
+            "{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}"
+        );
+
+        // testJsonDeleteWithNestedAndArray: wildcard path drops one key from every array element.
+        assertScalarString(
+            "json_delete('{\"teacher\":\"Alice\",\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}', 'teacher', 'student{}.rank')",
+            "{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}"
+        );
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConvertTzAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConvertTzAdapter.java
index cb460333dbdaa..d123bf15e78f5 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConvertTzAdapter.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ConvertTzAdapter.java
@@ -33,7 +33,7 @@
 import java.util.regex.Pattern;
 
 /**
- * Cat-3b adapter for PPL's {@code CONVERT_TZ(ts, from_tz, to_tz)}. Two jobs in
+ * Adapter for PPL's {@code CONVERT_TZ(ts, from_tz, to_tz)}. Two jobs in
  * priority order:
  *
  * 
          
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
index a7414eac65639..4e5b5955cb92c 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
@@ -93,9 +93,11 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
     // path). COALESCE is the lowering target of PPL `fillnull`. CAST is required because
     // ReduceExpressionsRule.ProjectReduceExpressionsRule (in PlannerImpl) constant-folds field
     // references through equality filters into typed literals — e.g. after `where str0 = 'FURNITURE'`,
-    // the projection `fields str0` is rewritten to `CAST('FURNITURE' AS VARCHAR)`. CONCAT is the
-    // lowering target of PPL `eval`'s `+` for strings (Calcite emits `||`, resolved to CONCAT in
-    // ScalarFunction); SAFE_CAST covers PPL `eval`'s explicit nullable `CAST(... AS ...)`
+    // the projection `fields str0` is rewritten to `CAST('FURNITURE' AS VARCHAR)`. CAST is also the
+    // implicit result-type narrowing PPL inserts after a UDF call whose declared return type differs
+    // from the eval column's inferred type (e.g. JSON_ARRAY_LENGTH returns INTEGER_FORCE_NULLABLE).
+    // CONCAT is the lowering target of PPL `eval`'s `+` for strings (Calcite emits `||`, resolved to
+    // CONCAT in ScalarFunction); SAFE_CAST covers PPL `eval`'s explicit nullable `CAST(... AS ...)`
     // expressions. The remaining comparison / arithmetic / logical operators are project-capable
     // for eval-style projections.
     private static final Set STANDARD_PROJECT_OPS = Set.of(
@@ -219,7 +221,14 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
         ScalarFunction.STRCMP,
         ScalarFunction.TOSTRING,
         ScalarFunction.NUMBER_TO_STRING,
-        ScalarFunction.TONUMBER
+        ScalarFunction.TONUMBER,
+        ScalarFunction.JSON_APPEND,
+        ScalarFunction.JSON_ARRAY_LENGTH,
+        ScalarFunction.JSON_DELETE,
+        ScalarFunction.JSON_EXTEND,
+        ScalarFunction.JSON_EXTRACT,
+        ScalarFunction.JSON_KEYS,
+        ScalarFunction.JSON_SET
     );
 
     private static final Set AGG_FUNCTIONS = Set.of(
@@ -327,6 +336,13 @@ public Map scalarFunctionAdapters() {
                     Map.entry(ScalarFunction.EXPM1, new Expm1Adapter()),
                     Map.entry(ScalarFunction.HOUR, hour),
                     Map.entry(ScalarFunction.HOUR_OF_DAY, hour),
+                    Map.entry(ScalarFunction.JSON_APPEND, new JsonFunctionAdapters.JsonAppendAdapter()),
+                    Map.entry(ScalarFunction.JSON_ARRAY_LENGTH, new JsonFunctionAdapters.JsonArrayLengthAdapter()),
+                    Map.entry(ScalarFunction.JSON_DELETE, new JsonFunctionAdapters.JsonDeleteAdapter()),
+                    Map.entry(ScalarFunction.JSON_EXTEND, new JsonFunctionAdapters.JsonExtendAdapter()),
+                    Map.entry(ScalarFunction.JSON_EXTRACT, new JsonFunctionAdapters.JsonExtractAdapter()),
+                    Map.entry(ScalarFunction.JSON_KEYS, new JsonFunctionAdapters.JsonKeysAdapter()),
+                    Map.entry(ScalarFunction.JSON_SET, new JsonFunctionAdapters.JsonSetAdapter()),
                     Map.entry(ScalarFunction.LIKE, new LikeAdapter()),
                     Map.entry(ScalarFunction.LOCATE, new PositionAdapter()),
                     Map.entry(ScalarFunction.MICROSECOND, DatePartAdapters.microsecond()),
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
index 23f7012bb8208..a06d2a4bb20d3 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
@@ -81,11 +81,29 @@ public class DataFusionFragmentConvertor implements FragmentConvertor {
 
     /**
      * Maps backend-specific Calcite operators to their Substrait extension names so Isthmus
-     * serializes them through our {@code SimpleExtension} catalog:
+     * serializes them through our {@code SimpleExtension} catalog. One entry per line so
+     * parallel per-UDF PRs append without hotspot conflicts.
      * 
            
      *   
          • {@link DelegatedPredicateFunction} → {@code delegated_predicate} (delegation to a peer backend).
            • 
      *   
          • {@link SqlLibraryOperators#ILIKE} → {@code ilike} (case-insensitive LIKE; resolved by
      *       DataFusion's substrait consumer to a case-insensitive {@code LikeExpr}).
            • 
+     *   
          • {@link SqlLibraryOperators#DATE_PART} → {@code date_part} (target of YearAdapter's rewrite).
            • 
+     *   
          • {@link ConvertTzAdapter#LOCAL_CONVERT_TZ_OP} → {@code convert_tz} (Rust UDF).
            • 
+     *   
          • {@link UnixTimestampAdapter#LOCAL_TO_UNIXTIME_OP} → {@code to_unixtime} (DF native).
            • 
+     *   
          • {@link JsonFunctionAdapters.JsonAppendAdapter#LOCAL_JSON_APPEND_OP} →
+     *       {@code json_append} (Rust UDF, homogeneous-string variadic path/value pairs).
            • 
+     *   
          • {@link JsonFunctionAdapters.JsonArrayLengthAdapter#LOCAL_JSON_ARRAY_LENGTH_OP} →
+     *       {@code json_array_length} (Rust UDF).
            • 
+     *   
          • {@link JsonFunctionAdapters.JsonDeleteAdapter#LOCAL_JSON_DELETE_OP} →
+     *       {@code json_delete} (Rust UDF, homogeneous-string variadic).
            • 
+     *   
          • {@link JsonFunctionAdapters.JsonExtendAdapter#LOCAL_JSON_EXTEND_OP} →
+     *       {@code json_extend} (Rust UDF, homogeneous-string variadic path/value pairs).
            • 
+     *   
          • {@link JsonFunctionAdapters.JsonExtractAdapter#LOCAL_JSON_EXTRACT_OP} →
+     *       {@code json_extract} (Rust UDF, homogeneous-string variadic).
            • 
+     *   
          • {@link JsonFunctionAdapters.JsonKeysAdapter#LOCAL_JSON_KEYS_OP} →
+     *       {@code json_keys} (Rust UDF).
            • 
+     *   
          • {@link JsonFunctionAdapters.JsonSetAdapter#LOCAL_JSON_SET_OP} →
+     *       {@code json_set} (Rust UDF, homogeneous-string variadic path/value pairs).
            • 
      *   
          • {@link SqlLibraryOperators#REGEXP_CONTAINS} → {@code regex_match} (boolean regex match;
      *       resolved by DataFusion's substrait consumer to {@code Operator::RegexMatch}, the same
      *       binary operator that backs PostgreSQL's {@code ~} regex match). Lowering target for PPL
@@ -125,7 +143,17 @@ public class DataFusionFragmentConvertor implements FragmentConvertor {
         FunctionMappings.s(SqlStdOperatorTable.PI, "pi"),
         FunctionMappings.s(SqlStdOperatorTable.RAND, "random"),
         FunctionMappings.s(SqlLibraryOperators.LOG, "logb"),
-        FunctionMappings.s(SignumFunction.FUNCTION, SignumFunction.NAME)
+        FunctionMappings.s(SignumFunction.FUNCTION, SignumFunction.NAME),
+        FunctionMappings.s(JsonFunctionAdapters.JsonAppendAdapter.LOCAL_JSON_APPEND_OP, "json_append"),
+        FunctionMappings.s(JsonFunctionAdapters.JsonArrayLengthAdapter.LOCAL_JSON_ARRAY_LENGTH_OP, "json_array_length"),
+        FunctionMappings.s(JsonFunctionAdapters.JsonDeleteAdapter.LOCAL_JSON_DELETE_OP, "json_delete"),
+        FunctionMappings.s(JsonFunctionAdapters.JsonExtendAdapter.LOCAL_JSON_EXTEND_OP, "json_extend"),
+        FunctionMappings.s(JsonFunctionAdapters.JsonExtractAdapter.LOCAL_JSON_EXTRACT_OP, "json_extract"),
+        FunctionMappings.s(JsonFunctionAdapters.JsonKeysAdapter.LOCAL_JSON_KEYS_OP, "json_keys"),
+        FunctionMappings.s(JsonFunctionAdapters.JsonSetAdapter.LOCAL_JSON_SET_OP, "json_set"),
+        FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"),
+        FunctionMappings.s(SqlStdOperatorTable.REPLACE, "replace"),
+        FunctionMappings.s(SqlLibraryOperators.REGEXP_REPLACE_3, "regexp_replace")
     );
 
     private final SimpleExtension.ExtensionCollection extensions;
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/JsonFunctionAdapters.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/JsonFunctionAdapters.java
new file mode 100644
index 0000000000000..9a416de26ae8f
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/JsonFunctionAdapters.java
@@ -0,0 +1,159 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.opensearch.analytics.spi.AbstractNameMappingAdapter;
+
+import java.util.List;
+
+/**
+ * Container for PPL JSON-function scalar adapters. Each inner class is a plain
+ * name-mapping rewrite from a Calcite call to a locally-declared
+ * {@link SqlOperator} whose name matches the corresponding Rust UDF at
+ * {@code rust/src/udf/.rs}. All validation (malformed JSON, malformed
+ * path, arity / pairing, any-NULL propagation) lives in the Rust UDF; the
+ * adapter does not inspect arguments. Return type is preserved from the
+ * original PPL call by {@link AbstractNameMappingAdapter#adapt}, matching the
+ * {@code *_FORCE_NULLABLE} declaration on the legacy {@code Json*FunctionImpl}.
+ *
+ * 
            Each {@code LOCAL_*_OP} must also be registered in
+ * {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS} via a
+ * {@code FunctionMappings.s(...)} entry keyed by the UDF's name.
+ *
+ * @opensearch.internal
+ */
+final class JsonFunctionAdapters {
+
+    private JsonFunctionAdapters() {}
+
+    /** {@code JSON_ARRAY_LENGTH(value)} → length of a JSON array; NULL on non-array / malformed input. */
+    static class JsonArrayLengthAdapter extends AbstractNameMappingAdapter {
+
+        static final SqlOperator LOCAL_JSON_ARRAY_LENGTH_OP = new SqlFunction(
+            "json_array_length",
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.INTEGER_NULLABLE,
+            null,
+            OperandTypes.STRING,
+            SqlFunctionCategory.STRING
+        );
+
+        JsonArrayLengthAdapter() {
+            super(LOCAL_JSON_ARRAY_LENGTH_OP, List.of(), List.of());
+        }
+    }
+
+    /** {@code JSON_KEYS(value)} → JSON-array-encoded top-level keys; NULL on non-object / malformed input. */
+    static class JsonKeysAdapter extends AbstractNameMappingAdapter {
+
+        static final SqlOperator LOCAL_JSON_KEYS_OP = new SqlFunction(
+            "json_keys",
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.VARCHAR_NULLABLE,
+            null,
+            OperandTypes.STRING,
+            SqlFunctionCategory.STRING
+        );
+
+        JsonKeysAdapter() {
+            super(LOCAL_JSON_KEYS_OP, List.of(), List.of());
+        }
+    }
+
+    /** {@code JSON_EXTRACT(value, path1, [path2, ...])} — single path → stringified match; multi-path → JSON-array wrap with {@code null} slots for misses. */
+    static class JsonExtractAdapter extends AbstractNameMappingAdapter {
+
+        static final SqlOperator LOCAL_JSON_EXTRACT_OP = new SqlFunction(
+            "json_extract",
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.VARCHAR_NULLABLE,
+            null,
+            OperandTypes.VARIADIC,
+            SqlFunctionCategory.STRING
+        );
+
+        JsonExtractAdapter() {
+            super(LOCAL_JSON_EXTRACT_OP, List.of(), List.of());
+        }
+    }
+
+    /** {@code JSON_DELETE(value, path1, [path2, ...])} — remove PPL-path matches; missing paths are no-ops. */
+    static class JsonDeleteAdapter extends AbstractNameMappingAdapter {
+
+        static final SqlOperator LOCAL_JSON_DELETE_OP = new SqlFunction(
+            "json_delete",
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.VARCHAR_NULLABLE,
+            null,
+            OperandTypes.VARIADIC,
+            SqlFunctionCategory.STRING
+        );
+
+        JsonDeleteAdapter() {
+            super(LOCAL_JSON_DELETE_OP, List.of(), List.of());
+        }
+    }
+
+    /** {@code JSON_SET(value, path1, val1, [path2, val2, ...])} — replace-only; missing paths are no-ops (parity with legacy {@code ctx.read != null} guard). */
+    static class JsonSetAdapter extends AbstractNameMappingAdapter {
+
+        static final SqlOperator LOCAL_JSON_SET_OP = new SqlFunction(
+            "json_set",
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.VARCHAR_NULLABLE,
+            null,
+            OperandTypes.VARIADIC,
+            SqlFunctionCategory.STRING
+        );
+
+        JsonSetAdapter() {
+            super(LOCAL_JSON_SET_OP, List.of(), List.of());
+        }
+    }
+
+    /** {@code JSON_APPEND(value, path1, val1, [path2, val2, ...])} — push-only onto array-valued targets; non-array / missing targets are no-ops. */
+    static class JsonAppendAdapter extends AbstractNameMappingAdapter {
+
+        static final SqlOperator LOCAL_JSON_APPEND_OP = new SqlFunction(
+            "json_append",
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.VARCHAR_NULLABLE,
+            null,
+            OperandTypes.VARIADIC,
+            SqlFunctionCategory.STRING
+        );
+
+        JsonAppendAdapter() {
+            super(LOCAL_JSON_APPEND_OP, List.of(), List.of());
+        }
+    }
+
+    /** {@code JSON_EXTEND(value, path1, val1, [path2, val2, ...])} — spread-or-append: JSON-array values are spread element-wise; otherwise the whole value is pushed as one string element. */
+    static class JsonExtendAdapter extends AbstractNameMappingAdapter {
+
+        static final SqlOperator LOCAL_JSON_EXTEND_OP = new SqlFunction(
+            "json_extend",
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.VARCHAR_NULLABLE,
+            null,
+            OperandTypes.VARIADIC,
+            SqlFunctionCategory.STRING
+        );
+
+        JsonExtendAdapter() {
+            super(LOCAL_JSON_EXTEND_OP, List.of(), List.of());
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UnixTimestampAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UnixTimestampAdapter.java
index 2f7056ac92c55..7acdda227eb7e 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UnixTimestampAdapter.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/UnixTimestampAdapter.java
@@ -19,7 +19,7 @@
 import java.util.List;
 
 /**
- * Cat-3a rename adapter for PPL's {@code UNIX_TIMESTAMP(ts)}. Rewrites to a
+ * Rename adapter for PPL's {@code UNIX_TIMESTAMP(ts)}. Rewrites to a
  * locally-declared {@link SqlFunction} named {@code to_unixtime} — the name
  * DataFusion's substrait consumer recognizes for its native
  * {@code ToUnixtimeFunc} (no UDF registration required on the Rust side).
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
index 4caf7c185dc96..d5dd9f2f213a6 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
@@ -317,3 +317,53 @@ scalar_functions:
           - { name: base, value: i32 }
         nullability: DECLARED_OUTPUT
         return: fp64
+
+  # PPL json_* UDFs — Rust implementations under rust/src/udf/.rs,
+  # surfaced to Calcite via JsonFunctionAdapters. All return NULL on malformed
+  # input; per-function semantics are documented in the Rust module headers.
+  - name: "json_array_length"
+    description: "Length of a JSON array; NULL on malformed or non-array input."
+    impls:
+      - args: [{ value: string, name: "value" }]
+        return: any1
+
+  - name: "json_keys"
+    description: "Top-level keys of a JSON object, encoded as a JSON array string; NULL on non-object input."
+    impls:
+      - args: [{ value: string, name: "value" }]
+        return: any1
+
+  - name: "json_extract"
+    description: "Extract JSON value(s) at PPL path(s); single → stringified match, multi → JSON-array string."
+    impls:
+      - args: [{ value: string, name: "value" }, { value: string, name: "path" }]
+        variadic: { min: 1 }
+        return: string
+
+  - name: "json_delete"
+    description: "Remove PPL-path matches from a JSON document; missing paths are no-ops."
+    impls:
+      - args: [{ value: string, name: "value" }, { value: string, name: "path" }]
+        variadic: { min: 1 }
+        return: string
+
+  - name: "json_set"
+    description: "Replace values at PPL-path matches (replace-only; missing paths are no-ops)."
+    impls:
+      - args: [{ value: string, name: "value" }, { value: string, name: "path" }]
+        variadic: { min: 1 }
+        return: string
+
+  - name: "json_append"
+    description: "Push values onto PPL-path-matched arrays; non-array / missing targets are no-ops."
+    impls:
+      - args: [{ value: string, name: "value" }, { value: string, name: "path" }]
+        variadic: { min: 1 }
+        return: string
+
+  - name: "json_extend"
+    description: "Spread JSON-array values onto PPL-path-matched arrays; scalar values fall back to append."
+    impls:
+      - args: [{ value: string, name: "value" }, { value: string, name: "path" }]
+        variadic: { min: 1 }
+        return: string
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/JsonFunctionAdaptersTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/JsonFunctionAdaptersTests.java
new file mode 100644
index 0000000000000..14e12b1a4694d
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/JsonFunctionAdaptersTests.java
@@ -0,0 +1,118 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.List;
+
+/**
+ * Unit tests for the JSON-function adapter inner classes in
+ * {@link JsonFunctionAdapters}. Each inner adapter gets its own test method
+ * (shape + {@code testAdaptedCallPreservesOriginalReturnType} regression
+ * guard). See {@link YearAdapterTests} for the regression-guard rationale.
+ */
+public class JsonFunctionAdaptersTests extends OpenSearchTestCase {
+
+    private RelDataTypeFactory typeFactory;
+    private RexBuilder rexBuilder;
+    private RelOptCluster cluster;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        typeFactory = new JavaTypeFactoryImpl();
+        rexBuilder = new RexBuilder(typeFactory);
+        HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+        cluster = RelOptCluster.create(planner, rexBuilder);
+    }
+
+    // ── JsonArrayLengthAdapter ────────────────────────────────────────────
+
+    public void testJsonArrayLengthRewritesToLocalOp() {
+        // Synthesize JSON_ARRAY_LENGTH(value) with a single VARCHAR operand.
+        RelDataType varcharNullable = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true);
+        RelDataType integerNullable = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.INTEGER), true);
+        SqlFunction pplJsonArrayLengthOp = new SqlFunction(
+            "JSON_ARRAY_LENGTH",
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.explicit(integerNullable),
+            null,
+            OperandTypes.STRING,
+            SqlFunctionCategory.STRING
+        );
+        RexNode valueRef = rexBuilder.makeInputRef(varcharNullable, 0);
+        RexCall original = (RexCall) rexBuilder.makeCall(pplJsonArrayLengthOp, List.of(valueRef));
+
+        RexNode adapted = new JsonFunctionAdapters.JsonArrayLengthAdapter().adapt(original, List.of(), cluster);
+
+        assertTrue("adapted node must be a RexCall, got " + adapted.getClass(), adapted instanceof RexCall);
+        RexCall call = (RexCall) adapted;
+        assertSame(
+            "adapted call must target LOCAL_JSON_ARRAY_LENGTH_OP",
+            JsonFunctionAdapters.JsonArrayLengthAdapter.LOCAL_JSON_ARRAY_LENGTH_OP,
+            call.getOperator()
+        );
+        assertEquals("json_array_length is unary — no prepend / append", 1, call.getOperands().size());
+        assertSame("arg 0 must be the original value operand", valueRef, call.getOperands().get(0));
+    }
+
+    /**
+     * The adapter MUST preserve the Calcite {@link RelDataType} of the original call.
+     * PPL declares {@code JSON_ARRAY_LENGTH} with INTEGER_FORCE_NULLABLE; the
+     * locally-declared {@code LOCAL_JSON_ARRAY_LENGTH_OP} uses
+     * {@code ReturnTypes.INTEGER_NULLABLE} which would infer a different
+     * typeFactory type instance and trip {@code Project.isValid}'s
+     * {@code compatibleTypes} check during fragment conversion. See
+     * {@link YearAdapterTests#testAdaptedCallPreservesOriginalReturnType()} for
+     * the original incident.
+     */
+    public void testJsonArrayLengthPreservesOriginalReturnType() {
+        RelDataType varcharNullable = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true);
+        // Pick a type that specifically differs from what LOCAL_JSON_ARRAY_LENGTH_OP's
+        // ReturnTypes.INTEGER_NULLABLE would compute — BIGINT here — so the
+        // regression assertion actually distinguishes "preserve" from "infer".
+        RelDataType bigintNullable = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BIGINT), true);
+        SqlFunction pplJsonArrayLengthOp = new SqlFunction(
+            "JSON_ARRAY_LENGTH",
+            SqlKind.OTHER_FUNCTION,
+            ReturnTypes.explicit(bigintNullable),
+            null,
+            OperandTypes.STRING,
+            SqlFunctionCategory.STRING
+        );
+        RexNode valueRef = rexBuilder.makeInputRef(varcharNullable, 0);
+        RexCall original = (RexCall) rexBuilder.makeCall(pplJsonArrayLengthOp, List.of(valueRef));
+        assertEquals(bigintNullable, original.getType());
+
+        RexNode adapted = new JsonFunctionAdapters.JsonArrayLengthAdapter().adapt(original, List.of(), cluster);
+
+        assertEquals(
+            "adapted call's return type must equal the original call's return type, "
+                + "otherwise the enclosing Project.rowType assertion fails in fragment conversion",
+            original.getType(),
+            adapted.getType()
+        );
+    }
+}

From 8f72a95e50db133f53760d194d27fd2efe6b94dd Mon Sep 17 00:00:00 2001
From: bowenlan 
Date: Fri, 8 May 2026 20:17:59 -0700
Subject: [PATCH 099/115] Extract analytics-api module for JDK 21 downstream
 consumers (#21574)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Creates sandbox/libs/analytics-api targeting JDK 21, containing only the
interface surface that downstream plugins (e.g. opensearch-sql) need to
compile against:

- org.opensearch.analytics.exec.QueryPlanExecutor (moved from analytics-framework)
- org.opensearch.analytics.schema.OpenSearchSchemaBuilder (moved from analytics-engine)

Motivation:

analytics-framework and analytics-engine use java.lang.foreign (FFM) APIs
that finalize only in JDK 22, so those modules must target JDK 25. Their
published Gradle Module Metadata declares org.gradle.jvm.version=25,
which blocks consumption from projects that target JVM 21 — most notably
opensearch-sql, which uses QueryPlanExecutor and OpenSearchSchemaBuilder
but does not touch FFM types.

Extracting these two interfaces into a JVM-21-targeted module unblocks
those consumers without forcing the whole sandbox to downgrade (FFM is
load-bearing for the engine).

Packages are unchanged (org.opensearch.analytics.{exec,schema}), so
existing imports in sandbox consumers (analytics-framework, analytics-engine,
dsl-query-executor, test-ppl-frontend) continue to resolve transitively
via the new `api project(':sandbox:libs:analytics-api')` edge in
analytics-framework/build.gradle.

thirdPartyAudit is disabled on analytics-api: it is a compile-time API
surface consumed transitively with the full Calcite runtime provided by
analytics-framework or analytics-engine, so there is no standalone runtime
where Calcite's optional references could load.

Verification:
- ./gradlew check -p sandbox -Dsandbox.enabled=true  (full sandbox check: BUILD SUCCESSFUL)
- analytics-api jar bytecode is class version 65 (JDK 21) as expected.
- Published Gradle Module Metadata declares org.gradle.jvm.version=21.
- Downstream sql prototype compiles against the new artifact from mavenLocal.

Signed-off-by: bowenlan-amzn 
---
 sandbox/build.gradle                          |  1 +
 sandbox/libs/analytics-api/build.gradle       | 30 +++++++++++++++++++
 .../analytics/exec/QueryPlanExecutor.java     |  0
 .../analytics/exec/package-info.java          |  0
 .../schema/OpenSearchSchemaBuilder.java       |  0
 .../analytics/schema/package-info.java        |  0
 sandbox/libs/analytics-framework/build.gradle |  6 ++--
 sandbox/plugins/analytics-engine/build.gradle |  5 ++--
 .../plugins/dsl-query-executor/build.gradle   |  1 +
 .../plugins/test-ppl-frontend/build.gradle    |  1 +
 10 files changed, 38 insertions(+), 6 deletions(-)
 create mode 100644 sandbox/libs/analytics-api/build.gradle
 rename sandbox/libs/{analytics-framework => analytics-api}/src/main/java/org/opensearch/analytics/exec/QueryPlanExecutor.java (100%)
 rename sandbox/libs/{analytics-framework => analytics-api}/src/main/java/org/opensearch/analytics/exec/package-info.java (100%)
 rename sandbox/{plugins/analytics-engine => libs/analytics-api}/src/main/java/org/opensearch/analytics/schema/OpenSearchSchemaBuilder.java (100%)
 rename sandbox/{plugins/analytics-engine => libs/analytics-api}/src/main/java/org/opensearch/analytics/schema/package-info.java (100%)

diff --git a/sandbox/build.gradle b/sandbox/build.gradle
index 4a14d0a11a0d7..1ee3608a4bc10 100644
--- a/sandbox/build.gradle
+++ b/sandbox/build.gradle
@@ -41,6 +41,7 @@ def patchedCalciteVersion = "${libVersions['calcite']}-opensearch-${libVersions[
 
 subprojects {
   group = 'org.opensearch.sandbox'
+  ext.patchedCalciteVersion = patchedCalciteVersion
 
   // The patched calcite-core / calcite-linq4j live in the OpenSearch
   // snapshots Maven repo; analytics-framework advertises them as `api`,
diff --git a/sandbox/libs/analytics-api/build.gradle b/sandbox/libs/analytics-api/build.gradle
new file mode 100644
index 0000000000000..1d39a279f02b9
--- /dev/null
+++ b/sandbox/libs/analytics-api/build.gradle
@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Analytics Engine API surface consumable from JDK 21 code paths.
+ */
+
+java { sourceCompatibility = JavaVersion.toVersion(21); targetCompatibility = JavaVersion.toVersion(21) }
+
+// no test for now, so disable
+testingConventions.enabled = false
+
+dependencies {
+  compileOnly project(':server')
+  // Declared compileOnly rather than api because analytics-api is never
+  // loaded standalone — downstream consumers should declare Calcite themselves.
+  compileOnly "org.apache.calcite:calcite-core:${patchedCalciteVersion}"
+  compileOnly "org.apache.calcite:calcite-linq4j:${patchedCalciteVersion}"
+
+  // Calcite bytecode references annotations from apiguardian (@API) and
+  // checker-framework (@EnsuresNonNullIf). compileOnlyApi propagates to
+  // consumers' compile/javadoc classpath without becoming a runtime dep.
+  compileOnlyApi 'org.apiguardian:apiguardian-api:1.1.2'
+  compileOnlyApi 'org.checkerframework:checker-qual:3.43.0'
+}
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/exec/QueryPlanExecutor.java b/sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/exec/QueryPlanExecutor.java
similarity index 100%
rename from sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/exec/QueryPlanExecutor.java
rename to sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/exec/QueryPlanExecutor.java
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/exec/package-info.java b/sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/exec/package-info.java
similarity index 100%
rename from sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/exec/package-info.java
rename to sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/exec/package-info.java
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/schema/OpenSearchSchemaBuilder.java b/sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/schema/OpenSearchSchemaBuilder.java
similarity index 100%
rename from sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/schema/OpenSearchSchemaBuilder.java
rename to sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/schema/OpenSearchSchemaBuilder.java
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/schema/package-info.java b/sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/schema/package-info.java
similarity index 100%
rename from sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/schema/package-info.java
rename to sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/schema/package-info.java
diff --git a/sandbox/libs/analytics-framework/build.gradle b/sandbox/libs/analytics-framework/build.gradle
index f22f55e23cb53..432aa4248e44c 100644
--- a/sandbox/libs/analytics-framework/build.gradle
+++ b/sandbox/libs/analytics-framework/build.gradle
@@ -16,8 +16,6 @@
 // for Janino parent CL); API surface is identical to upstream. The OpenSearch
 // Snapshots repo and the resolutionStrategy.force for this coordinate are
 // declared centrally in sandbox/build.gradle's subprojects block.
-def calciteVersion = "${versions.calcite}-opensearch-${versions.calcite_os_rev}"
-
 java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) }
 
 configurations {
@@ -39,9 +37,9 @@ dependencies {
   // interfaces declares its own runtime arrow dep (see analytics-backend-datafusion).
   compileOnly "org.apache.arrow:arrow-vector:${versions.arrow}"
   compileOnly "org.apache.arrow:arrow-memory-core:${versions.arrow}"
-  api "org.apache.calcite:calcite-core:${calciteVersion}"
+  api "org.apache.calcite:calcite-core:${patchedCalciteVersion}"
   // Calcite's expression tree and Enumerable runtime — required by calcite-core API
-  api "org.apache.calcite:calcite-linq4j:${calciteVersion}"
+  api "org.apache.calcite:calcite-linq4j:${patchedCalciteVersion}"
   // Calcite's JDBC abstraction layer — required by calcite-core internals
   runtimeOnly 'org.apache.calcite.avatica:avatica-core:1.27.0'
   // Guava — required by Calcite internally, forbidden on compile classpaths by OpenSearch policy
diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle
index 5f7986c74a2cf..0058df4b6eb68 100644
--- a/sandbox/plugins/analytics-engine/build.gradle
+++ b/sandbox/plugins/analytics-engine/build.gradle
@@ -45,8 +45,9 @@ tasks.named('missingJavadoc').configure {
 }
 
 dependencies {
-  // Shared types and SPI interfaces (QueryPlanExecutor, EngineBridge, AnalyticsBackEndPlugin, etc.)
-  // Also provides calcite-core transitively via api.
+  implementation project(':sandbox:libs:analytics-api')
+
+  // Shared SPI interfaces (EngineBridge, AnalyticsBackEndPlugin, etc.) + calcite-core transitively.
   api project(':sandbox:libs:analytics-framework')
 
   // Arrow — provided at runtime by the extended arrow-flight-rpc plugin (same classloader).
diff --git a/sandbox/plugins/dsl-query-executor/build.gradle b/sandbox/plugins/dsl-query-executor/build.gradle
index 539a8e9e16b25..b4ed60fa568b7 100644
--- a/sandbox/plugins/dsl-query-executor/build.gradle
+++ b/sandbox/plugins/dsl-query-executor/build.gradle
@@ -29,6 +29,7 @@ sourceSets.test.compileClasspath += configurations.calciteCompile
 
 dependencies {
   compileOnly project(':server')
+  compileOnly project(':sandbox:libs:analytics-api')
   compileOnly project(':sandbox:libs:analytics-framework')
   compileOnly project(':sandbox:plugins:analytics-engine')
   // TODO: Consume Calcite dependency from Analytics Framework instead of declaring it separately.
diff --git a/sandbox/plugins/test-ppl-frontend/build.gradle b/sandbox/plugins/test-ppl-frontend/build.gradle
index ce908cabd515b..9c2d79c306c9c 100644
--- a/sandbox/plugins/test-ppl-frontend/build.gradle
+++ b/sandbox/plugins/test-ppl-frontend/build.gradle
@@ -52,6 +52,7 @@ sourceSets.test.compileClasspath += configurations.calciteCompile
 
 dependencies {
   // Analytics framework + Calcite provided at runtime by analytics-engine (parent classloader via extendedPlugins)
+  compileOnly project(':sandbox:libs:analytics-api')
   compileOnly project(':sandbox:libs:analytics-framework')
 
   // Guava for compilation — Calcite API exposes guava types

From 36809cc64fb0424a99188ed54aa0c0d5b1fa601e Mon Sep 17 00:00:00 2001
From: Jialiang Liang 
Date: Sat, 9 May 2026 10:44:02 -0700
Subject: [PATCH 100/115] [Sandbox SQL snapshot] Pin org.opensearch.query:*
 (unified-query-*) snapshots to mavenLocal (#21578)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Pin org.opensearch.query:* (unified-query-*) artifacts to mavenLocal

The remote OpenSearch Snapshots maven repo (ci.opensearch.org/ci/dbc/snapshots)
only republishes from sql/main, not from sql/feature/mustang-ppl-integration,
so its 3.7.0.0-SNAPSHOT jars trail the feature branch by however many merges
(currently missing PPL_REX_MAX_MATCH_LIMIT, CALCITE_ENGINE_ENABLED, …). The
sandbox-check workflow's pre-step #21569 publishes feature-branch unified-query
jars to mavenLocal, but Gradle's default SNAPSHOT resolution weighs the remote's
explicit / metadata higher than mavenLocal's
true>, so the stale remote wins even when mavenLocal has a newer
.

Confirmed via dependencyInsight: every consumer was binding
unified-query-api:3.7.0.0-SNAPSHOT:20260507.224009-12 (60kB, 42 classes, no
PPL_REX_MAX_MATCH_LIMIT field reference) instead of the locally-published
3.7.0.0-SNAPSHOT (29kB, 21 classes, has the field). The runtime cluster
inherited that stale class via the test-ppl-frontend plugin bundle, which
is why every IT touching `rex` failed plan-time with `NullPointerException:
Cannot invoke "java.lang.Integer.intValue()" because the return value of
"Settings.getSettingValue(PPL_REX_MAX_MATCH_LIMIT)" is null` once the
unified path tried to read the setting.

Fix: tell the OpenSearch Snapshots remote to refuse `org.opensearch.query`
artifacts via mavenContent { excludeGroup }. Three sites declare the remote:

  * sandbox/build.gradle subprojects { repositories } — applies to every
    sandbox subproject including qa.
  * sandbox/plugins/analytics-backend-datafusion/build.gradle — own
    declaration; left in place for module isolation, filtered identically.
  * sandbox/plugins/test-ppl-frontend/build.gradle — also pin mavenLocal as
    the only source for org.opensearch.query so the bundlePlugin task
    bundles the freshly-published feature-branch jar rather than the stale
    timestamped one Gradle would otherwise pick.

Verified locally: bundled unified-query-api drops 60kB → 29kB, the
UnifiedQueryContext$Builder constant pool now references PPL_REX_MAX_MATCH_LIMIT,
and RexCommandIT goes 0/16 → 16/16 against the same locally-published jars
the CI workflow already produces.

Drop this filter once the SQL feature branch merges to sql/main and the
remote OpenSearch Snapshots repo catches up — at that point every
3.7.0.0-SNAPSHOT publish will carry the rex max-match default and the
mavenLocal preference becomes redundant.

Signed-off-by: Jialiang Liang 

* Wire mavenLocal into sandbox subprojects + bump analytics-engine to 3.7

CI fallout from the prior commit's `excludeGroup 'org.opensearch.query'`
filter on the OpenSearch Snapshots remote: the parent subprojects block
no longer carried mavenLocal, so analytics-engine's testImplementation /
internalClusterTest configurations had no repository at all serving
org.opensearch.query, failing with `Could not find
org.opensearch.query:unified-query-api:3.6.0.0-SNAPSHOT` (and -core / -ppl).

Two pieces:

1. sandbox/build.gradle subprojects { repositories } — also declare
   mavenLocal scoped to the org.opensearch.query group via mavenContent
   { includeGroup }. mavenLocal becomes the authoritative source for
   unified-query SNAPSHOTs (populated by the sandbox-check workflow's
   publishUnifiedQueryPublicationToMavenLocal pre-step) without leaking
   into resolution for any other group.

2. sandbox/plugins/analytics-engine/build.gradle — bump
   sqlUnifiedQueryVersion from 3.6.0.0-SNAPSHOT → 3.7.0.0-SNAPSHOT.
   The 3.6 jars don't exist in mavenLocal (only the 3.7 feature-branch
   build does), so the older pin was the proximate cause of the CI
   resolution failure. Aligning with test-ppl-frontend's already-3.7
   declaration also keeps the unified-query consumers consistent.

Signed-off-by: Jialiang Liang 

---------

Signed-off-by: Jialiang Liang 
---
 sandbox/build.gradle                          | 25 ++++++++++++++++++-
 .../analytics-backend-datafusion/build.gradle |  3 +++
 sandbox/plugins/analytics-engine/build.gradle | 12 +++++++--
 .../plugins/test-ppl-frontend/build.gradle    | 16 +++++++++++-
 4 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/sandbox/build.gradle b/sandbox/build.gradle
index 1ee3608a4bc10..99029b66d4ac4 100644
--- a/sandbox/build.gradle
+++ b/sandbox/build.gradle
@@ -46,11 +46,34 @@ subprojects {
   // The patched calcite-core / calcite-linq4j live in the OpenSearch
   // snapshots Maven repo; analytics-framework advertises them as `api`,
   // so every consumer of analytics-framework needs both the repo and a
-  // `force` to win over transitive vanilla calcite
+  // `force` to win over transitive vanilla calcite.
+  //
+  // org.opensearch.query (unified-query-*) is excluded from this remote and
+  // pinned to mavenLocal in the consuming subprojects (see analytics-engine
+  // and test-ppl-frontend build.gradle). The remote OpenSearch Snapshots repo
+  // only republishes from sql/main, not from sql/feature/mustang-ppl-integration,
+  // so its 3.7.0.0-SNAPSHOT jars trail the feature branch by however many merges.
+  // Gradle's default SNAPSHOT resolution weighs the remote's explicit
+  // /timestamp metadata higher than mavenLocal's true>,
+  // so the stale remote wins even when mavenLocal has a newer .
+  // Excluding the group here forces gradle to fall through to mavenLocal,
+  // populated by the sandbox-check workflow's pre-step
+  // (publishUnifiedQueryPublicationToMavenLocal from the feature branch).
+  // Drop this exclusion once the SQL feature branch merges to sql/main and
+  // remote snapshots catch up.
   repositories {
+    // mavenLocal serves org.opensearch.query:* (unified-query-*) only — the
+    // sandbox-check workflow's pre-step populates it from the SQL feature branch
+    // (publishUnifiedQueryPublicationToMavenLocal). Restricting via includeGroup
+    // keeps mavenLocal off the resolution path for everything else, where it
+    // would otherwise mask intentionally-pinned remote artifacts.
+    mavenLocal {
+      mavenContent { includeGroup 'org.opensearch.query' }
+    }
     maven {
       name = 'OpenSearch Snapshots'
       url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/'
+      mavenContent { excludeGroup 'org.opensearch.query' }
     }
   }
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/build.gradle b/sandbox/plugins/analytics-backend-datafusion/build.gradle
index 1d71f16082900..65ee64daf8b4b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/build.gradle
+++ b/sandbox/plugins/analytics-backend-datafusion/build.gradle
@@ -16,6 +16,9 @@ repositories {
   maven {
     name = 'OpenSearch Snapshots'
     url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/'
+    // unified-query-* artifacts are pinned to mavenLocal — see comment in
+    // sandbox/build.gradle's subprojects { repositories } block.
+    mavenContent { excludeGroup 'org.opensearch.query' }
   }
 }
 
diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle
index 0058df4b6eb68..a56b78baa88cd 100644
--- a/sandbox/plugins/analytics-engine/build.gradle
+++ b/sandbox/plugins/analytics-engine/build.gradle
@@ -14,8 +14,16 @@
 
 apply plugin: 'opensearch.internal-cluster-test'
 
-// SQL Unified Query API version (aligned with OpenSearch build version)
-def sqlUnifiedQueryVersion = '3.6.0.0-SNAPSHOT'
+// SQL Unified Query API version (aligned with OpenSearch build version).
+// Bumped to 3.7 to match the rest of the sandbox (test-ppl-frontend declares
+// the same '3.7.0.0-SNAPSHOT'). org.opensearch.query:* is also pinned to
+// mavenLocal in sandbox/build.gradle's subprojects { repositories } block,
+// so the remote OpenSearch Snapshots repo no longer serves this group;
+// 3.6 jars don't exist in mavenLocal (only 3.7 from the feature branch's
+// publishUnifiedQueryPublicationToMavenLocal pre-step), so the older pin
+// caused `Could not find org.opensearch.query:unified-query-api:3.6.0.0-SNAPSHOT`
+// at internal-cluster-test resolution.
+def sqlUnifiedQueryVersion = '3.7.0.0-SNAPSHOT'
 
 opensearchplugin {
   description = 'Analytics engine hub: discovers and wires query extensions via ExtensiblePlugin SPI.'
diff --git a/sandbox/plugins/test-ppl-frontend/build.gradle b/sandbox/plugins/test-ppl-frontend/build.gradle
index 9c2d79c306c9c..58f15ceebd6dc 100644
--- a/sandbox/plugins/test-ppl-frontend/build.gradle
+++ b/sandbox/plugins/test-ppl-frontend/build.gradle
@@ -32,10 +32,24 @@ opensearchplugin {
 }
 
 repositories {
-  mavenLocal()
+  // Pin org.opensearch.query:* (unified-query-*) artifacts to mavenLocal only.
+  // The remote OpenSearch Snapshots repo only republishes from sql/main, not from
+  // sql/feature/mustang-ppl-integration, so its 3.7.0.0-SNAPSHOT jars trail the
+  // feature branch by however many merges. Gradle's default SNAPSHOT resolution
+  // weighs the remote's explicit /timestamp metadata higher than
+  // mavenLocal's bare true>, so the stale remote wins even when
+  // mavenLocal has a newer . Restrict the unified-query group to
+  // mavenLocal and exclude it from the remote so the sandbox-check workflow's
+  // pre-step (publishUnifiedQueryPublicationToMavenLocal from the feature branch)
+  // is the authoritative source. Drop this filter once SQL feature branch merges
+  // to sql/main and remote snapshots catch up.
+  mavenLocal {
+    mavenContent { includeGroup 'org.opensearch.query' }
+  }
   maven {
     name = 'OpenSearch Snapshots'
     url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/'
+    mavenContent { excludeGroup 'org.opensearch.query' }
   }
 }
 

From 4d40eb84075314685372204dc04d84dba784ea27 Mon Sep 17 00:00:00 2001
From: chaitanya588 
Date: Sun, 10 May 2026 00:01:00 +0530
Subject: [PATCH 101/115] Avoiding repeated encoding and compression operation
 for the sort column included writes (#21464)

Signed-off-by: Chaitanya KSR 
---
 .../libs/dataformat-native/rust/Cargo.toml    |   1 +
 .../src/main/rust/Cargo.toml                  |   1 +
 .../src/main/rust/src/merge/heap.rs           |   3 +-
 .../src/main/rust/src/tests/mod.rs            | 232 ++++++++++++++
 .../src/main/rust/src/writer.rs               | 301 ++++++++++++------
 .../rust/tests/writer_integration_tests.rs    | 144 ++++++++-
 6 files changed, 579 insertions(+), 103 deletions(-)

diff --git a/sandbox/libs/dataformat-native/rust/Cargo.toml b/sandbox/libs/dataformat-native/rust/Cargo.toml
index 800cfb3f27556..003e2a5d7390b 100644
--- a/sandbox/libs/dataformat-native/rust/Cargo.toml
+++ b/sandbox/libs/dataformat-native/rust/Cargo.toml
@@ -18,6 +18,7 @@ members = [
 # Arrow / Parquet
 arrow = { version = "57.3.0", features = ["ffi"] }
 arrow-array = "57.3.0"
+arrow-ipc = "57.3.0"
 arrow-schema = "57.3.0"
 arrow-buffer = "57.3.0"
 parquet = "57.3.0"
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/Cargo.toml b/sandbox/plugins/parquet-data-format/src/main/rust/Cargo.toml
index 379ec6aea4149..365f571c62c5d 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/Cargo.toml
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/Cargo.toml
@@ -15,6 +15,7 @@ crate-type = ["rlib"]
 [dependencies]
 arrow = { workspace = true }
 parquet = { workspace = true }
+arrow-ipc = { workspace = true }
 lazy_static = { workspace = true }
 dashmap = { workspace = true }
 tempfile = { workspace = true }
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/heap.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/heap.rs
index 45062c27c59aa..55755159bdc1f 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/heap.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/merge/heap.rs
@@ -14,7 +14,7 @@ use arrow::datatypes::{
     DataType as ArrowDataType, Date32Type, Date64Type, DurationMicrosecondType,
     DurationMillisecondType, DurationNanosecondType, DurationSecondType, Float32Type, Float64Type,
     Int16Type, Int32Type, Int64Type, Int8Type, TimestampMicrosecondType,
-    TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt32Type,
+    TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,
 };
 
 use super::error::{MergeError, MergeResult};
@@ -138,7 +138,6 @@ pub fn get_sort_value(
         ArrowDataType::Int32 => SortKey::Int(col.as_primitive::().value(row) as i64),
         ArrowDataType::Int16 => SortKey::Int(col.as_primitive::().value(row) as i64),
         ArrowDataType::Int8 => SortKey::Int(col.as_primitive::().value(row) as i64),
-        ArrowDataType::UInt32 => SortKey::Int(col.as_primitive::().value(row) as i64),
         ArrowDataType::Date32 => SortKey::Int(col.as_primitive::().value(row) as i64),
         ArrowDataType::Date64 => SortKey::Int(col.as_primitive::().value(row)),
         ArrowDataType::Timestamp(unit, _) => SortKey::Int(match unit {
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs
index 48ab97ba5a57e..67b0332f05f57 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/tests/mod.rs
@@ -276,6 +276,238 @@ fn test_unsorted_writer_preserves_insertion_order() {
     cleanup_ffi_schema(schema_ptr);
 }
 
+// ===== Arrow IPC staging path tests =====
+
+#[test]
+fn test_ipc_staging_sorted_writer_creates_and_cleans_up_staging_file() {
+    let (_temp_dir, filename) = get_temp_file_path("ipc_cleanup.parquet");
+    let (_schema, schema_ptr) = create_sorted_writer_and_assert_success(&filename, "id", false);
+
+    // The IPC staging file should exist while the writer is open
+    let temp_filename = format!(
+        "{}/temp-{}",
+        Path::new(&filename).parent().unwrap().to_string_lossy(),
+        Path::new(&filename).file_name().unwrap().to_string_lossy()
+    );
+    let ipc_staging_path = format!("{}.arrow_ipc_staging", temp_filename);
+    assert!(Path::new(&ipc_staging_path).exists(), "IPC staging file should exist while writer is open");
+
+    let (ap, sp) = create_test_ffi_data_with_ids(vec![30, 10, 20], vec![Some("C"), Some("A"), Some("B")]).unwrap();
+    NativeParquetWriter::write_data(filename.clone(), ap, sp).unwrap();
+    cleanup_ffi_data(ap, sp);
+
+    NativeParquetWriter::finalize_writer(filename.clone()).unwrap();
+
+    // After finalize, the IPC staging file should be cleaned up
+    assert!(!Path::new(&ipc_staging_path).exists(), "IPC staging file should be deleted after finalize");
+    // The final Parquet file should exist
+    assert!(Path::new(&filename).exists(), "Final Parquet file should exist");
+
+    // Verify data is sorted
+    let ids = read_parquet_file_sorted_ids(&filename);
+    assert_eq!(ids, vec![10, 20, 30]);
+
+    cleanup_ffi_schema(schema_ptr);
+}
+
+#[test]
+fn test_ipc_staging_has_writer_returns_true() {
+    let (_temp_dir, filename) = get_temp_file_path("ipc_has_writer.parquet");
+    let (_schema, schema_ptr) = create_sorted_writer_and_assert_success(&filename, "id", false);
+
+    assert!(NativeParquetWriter::has_writer(&filename), "has_writer should return true for IPC writer");
+
+    close_writer_and_cleanup_schema(&filename, schema_ptr);
+}
+
+#[test]
+fn test_ipc_staging_duplicate_writer_rejected() {
+    let (_temp_dir, filename) = get_temp_file_path("ipc_dup.parquet");
+    let (_schema, schema_ptr) = create_sorted_writer_and_assert_success(&filename, "id", false);
+
+    let (_, schema_ptr2) = create_test_ffi_schema();
+    let result = NativeParquetWriter::create_writer(
+        filename.clone(), "test-index".to_string(), schema_ptr2,
+        vec!["id".to_string()], vec![false], vec![false], 0
+    );
+    assert!(result.is_err());
+    assert!(result.unwrap_err().to_string().contains("Writer already exists"));
+
+    cleanup_ffi_schema(schema_ptr2);
+    close_writer_and_cleanup_schema(&filename, schema_ptr);
+}
+
+#[test]
+fn test_ipc_staging_empty_data_produces_valid_parquet() {
+    let (_temp_dir, filename) = get_temp_file_path("ipc_empty.parquet");
+    let (_schema, schema_ptr) = create_sorted_writer_and_assert_success(&filename, "id", false);
+
+    // Finalize without writing any data
+    let result = NativeParquetWriter::finalize_writer(filename.clone());
+    assert!(result.is_ok());
+    assert!(Path::new(&filename).exists(), "Empty Parquet file should be created");
+
+    let metadata = result.unwrap().unwrap();
+    assert_eq!(metadata.metadata.file_metadata().num_rows(), 0);
+
+    cleanup_ffi_schema(schema_ptr);
+}
+
+#[test]
+fn test_ipc_staging_multi_batch_sort() {
+    let (_temp_dir, filename) = get_temp_file_path("ipc_multi_batch.parquet");
+    let (_schema, schema_ptr) = create_sorted_writer_and_assert_success(&filename, "id", false);
+
+    // Write multiple batches with interleaved values
+    let (ap1, sp1) = create_test_ffi_data_with_ids(vec![50, 10], vec![Some("E"), Some("A")]).unwrap();
+    NativeParquetWriter::write_data(filename.clone(), ap1, sp1).unwrap();
+    cleanup_ffi_data(ap1, sp1);
+
+    let (ap2, sp2) = create_test_ffi_data_with_ids(vec![30, 20], vec![Some("C"), Some("B")]).unwrap();
+    NativeParquetWriter::write_data(filename.clone(), ap2, sp2).unwrap();
+    cleanup_ffi_data(ap2, sp2);
+
+    let (ap3, sp3) = create_test_ffi_data_with_ids(vec![40, 60], vec![Some("D"), Some("F")]).unwrap();
+    NativeParquetWriter::write_data(filename.clone(), ap3, sp3).unwrap();
+    cleanup_ffi_data(ap3, sp3);
+
+    NativeParquetWriter::finalize_writer(filename.clone()).unwrap();
+
+    let ids = read_parquet_file_sorted_ids(&filename);
+    assert_eq!(ids, vec![10, 20, 30, 40, 50, 60], "Multiple IPC batches should be sorted correctly");
+
+    cleanup_ffi_schema(schema_ptr);
+}
+
+#[test]
+fn test_ipc_staging_descending_sort() {
+    let (_temp_dir, filename) = get_temp_file_path("ipc_desc.parquet");
+    let (_schema, schema_ptr) = create_sorted_writer_and_assert_success(&filename, "id", true);
+
+    let (ap, sp) = create_test_ffi_data_with_ids(vec![10, 30, 20], vec![Some("A"), Some("C"), Some("B")]).unwrap();
+    NativeParquetWriter::write_data(filename.clone(), ap, sp).unwrap();
+    cleanup_ffi_data(ap, sp);
+
+    NativeParquetWriter::finalize_writer(filename.clone()).unwrap();
+
+    let ids = read_parquet_file_sorted_ids(&filename);
+    assert_eq!(ids, vec![30, 20, 10], "IPC path should support descending sort");
+
+    cleanup_ffi_schema(schema_ptr);
+}
+
+#[test]
+fn test_ipc_and_parquet_writers_coexist() {
+    let (_temp_dir1, sorted_file) = get_temp_file_path("ipc_sorted.parquet");
+    let (_temp_dir2, unsorted_file) = get_temp_file_path("parquet_unsorted.parquet");
+
+    // Create one IPC writer (sorted) and one Parquet writer (unsorted)
+    let (_schema1, sp1) = create_sorted_writer_and_assert_success(&sorted_file, "id", false);
+    let (_schema2, sp2) = create_writer_and_assert_success(&unsorted_file);
+
+    // Write to both
+    let (ap1, dp1) = create_test_ffi_data_with_ids(vec![30, 10, 20], vec![Some("C"), Some("A"), Some("B")]).unwrap();
+    NativeParquetWriter::write_data(sorted_file.clone(), ap1, dp1).unwrap();
+    cleanup_ffi_data(ap1, dp1);
+
+    let (ap2, dp2) = create_test_ffi_data_with_ids(vec![30, 10, 20], vec![Some("C"), Some("A"), Some("B")]).unwrap();
+    NativeParquetWriter::write_data(unsorted_file.clone(), ap2, dp2).unwrap();
+    cleanup_ffi_data(ap2, dp2);
+
+    // Finalize both
+    NativeParquetWriter::finalize_writer(sorted_file.clone()).unwrap();
+    NativeParquetWriter::finalize_writer(unsorted_file.clone()).unwrap();
+
+    // Sorted file should be sorted
+    let sorted_ids = read_parquet_file_sorted_ids(&sorted_file);
+    assert_eq!(sorted_ids, vec![10, 20, 30]);
+
+    // Unsorted file should preserve insertion order
+    let unsorted_ids = read_parquet_file_sorted_ids(&unsorted_file);
+    assert_eq!(unsorted_ids, vec![30, 10, 20]);
+
+    cleanup_ffi_schema(sp1);
+    cleanup_ffi_schema(sp2);
+}
+
+#[test]
+fn test_ipc_staging_concurrent_sorted_writers() {
+    let temp_dir = tempdir().unwrap();
+    let thread_count = 6;
+    let success_count = Arc::new(AtomicUsize::new(0));
+    let mut handles = vec![];
+
+    for i in 0..thread_count {
+        let temp_dir_path = temp_dir.path().to_path_buf();
+        let success_count = Arc::clone(&success_count);
+        let handle = thread::spawn(move || {
+            let file_path = temp_dir_path.join(format!("ipc_concurrent_{}.parquet", i));
+            let filename = file_path.to_string_lossy().to_string();
+            let (_schema, schema_ptr) = create_test_ffi_schema();
+
+            if NativeParquetWriter::create_writer(
+                filename.clone(), "test-index".to_string(), schema_ptr,
+                vec!["id".to_string()], vec![false], vec![false], 0
+            ).is_ok() {
+                let (ap, sp) = create_test_ffi_data_with_ids(
+                    vec![30, 10, 20], vec![Some("C"), Some("A"), Some("B")]
+                ).unwrap();
+                let write_ok = NativeParquetWriter::write_data(filename.clone(), ap, sp).is_ok();
+                cleanup_ffi_data(ap, sp);
+
+                if write_ok {
+                    if let Ok(Some(metadata)) = NativeParquetWriter::finalize_writer(filename.clone()) {
+                        if metadata.metadata.file_metadata().num_rows() == 3 {
+                            let ids = read_parquet_file_sorted_ids(&filename);
+                            if ids == vec![10, 20, 30] {
+                                success_count.fetch_add(1, Ordering::SeqCst);
+                            }
+                        }
+                    }
+                }
+            }
+            cleanup_ffi_schema(schema_ptr);
+        });
+        handles.push(handle);
+    }
+
+    for handle in handles {
+        handle.join().unwrap();
+    }
+    assert_eq!(success_count.load(Ordering::SeqCst), thread_count);
+}
+
+#[test]
+fn test_ipc_staging_complete_lifecycle_with_sync() {
+    let (_temp_dir, filename) = get_temp_file_path("ipc_lifecycle.parquet");
+    let file_path = Path::new(&filename);
+    let (_schema, schema_ptr) = create_sorted_writer_and_assert_success(&filename, "id", false);
+
+    for batch_ids in [vec![50, 30], vec![10, 40], vec![20, 60]] {
+        let names: Vec> = batch_ids.iter().map(|_| Some("x")).collect();
+        let (ap, sp) = create_test_ffi_data_with_ids(batch_ids, names).unwrap();
+        NativeParquetWriter::write_data(filename.clone(), ap, sp).unwrap();
+        cleanup_ffi_data(ap, sp);
+    }
+
+    let result = NativeParquetWriter::finalize_writer(filename.clone());
+    assert!(result.is_ok());
+    let metadata = result.unwrap().unwrap();
+    assert_eq!(metadata.metadata.file_metadata().num_rows(), 6);
+
+    assert!(NativeParquetWriter::sync_to_disk(filename.clone()).is_ok());
+    assert!(file_path.exists());
+    assert!(file_path.metadata().unwrap().len() > 0);
+
+    let ids = read_parquet_file_sorted_ids(&filename);
+    assert_eq!(ids, vec![10, 20, 30, 40, 50, 60]);
+
+    let read_metadata = NativeParquetWriter::get_file_metadata(filename.clone()).unwrap();
+    assert_eq!(read_metadata.num_rows(), 6);
+
+    cleanup_ffi_schema(schema_ptr);
+}
+
 #[test]
 fn test_get_filtered_writer_memory_usage_with_writers() {
     let (_temp_dir, filename1) = get_temp_file_path("test1.parquet");
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs
index 3d14e16e0ef7a..ce4f86833f3c4 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/writer.rs
@@ -8,16 +8,19 @@
 
 use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema};
 use arrow::record_batch::RecordBatch;
-use arrow::compute::{lexsort_to_indices, take, SortColumn};
+use arrow::compute::{concat_batches, take};
+use arrow::row::{RowConverter, SortField};
+use arrow_ipc::writer::FileWriter as IpcFileWriter;
+use arrow_ipc::reader::FileReader as IpcFileReader;
 use dashmap::DashMap;
 use lazy_static::lazy_static;
-use parquet::arrow::{arrow_reader::ParquetRecordBatchReaderBuilder, ArrowWriter};
+use parquet::arrow::ArrowWriter;
 use parquet::file::reader::{FileReader, SerializedFileReader};
 use std::fs::File;
 use std::path::Path;
 use std::sync::{Arc, Mutex};
 
-use crate::{log_error, log_debug};
+use crate::{log_error, log_debug, log_info};
 use crate::crc_writer::CrcWriter;
 use crate::merge::{merge_sorted, schema::ROW_ID_COLUMN_NAME};
 use crate::native_settings::NativeSettings;
@@ -30,17 +33,34 @@ pub struct FinalizeResult {
     pub crc32: u32,
 }
 
+/// The underlying writer — either direct Parquet or Arrow IPC staging.
+/// When sort columns are configured, the IPC variant is used so that
+/// batches can be cheaply read back for sorting — Arrow IPC is a raw
+/// dump of in-memory Arrow buffers with minimal framing overhead.
+enum WriterVariant {
+    /// Direct Parquet writer — used when no sort columns are configured.
+    Parquet(Arc>>>),
+    /// Arrow IPC staging writer — used when sort columns are configured.
+    /// Batches are written as raw Arrow IPC; on close they are read back,
+    /// sorted, and written as a final Parquet file.
+    Ipc(Arc>>),
+}
+
 /// Bundles all per-writer resources so a single `DashMap::remove` atomically
 /// drops the writer, closes the file handle, and cleans up sort config.
 struct WriterState {
-    writer: Arc>>>,
+    variant: WriterVariant,
     settings: NativeSettings,
-    crc_handle: crate::crc_writer::CrcHandle,
+    crc_handle: Option,
     writer_generation: i64,
 }
 
+/// Path suffix for the intermediate Arrow IPC file used during sort-on-close.
+const IPC_STAGING_SUFFIX: &str = ".arrow_ipc_staging";
+
 lazy_static! {
     /// Unified per-writer registry. Keyed by temp filename.
+    /// Holds both Parquet and IPC writers via the `WriterVariant` enum.
     static ref WRITERS: DashMap = DashMap::new();
     pub static ref SETTINGS_STORE: DashMap = DashMap::new();
     /// Holds file handles for finalized files pending fsync. Removed after sync.
@@ -75,7 +95,7 @@ impl NativeParquetWriter {
         writer_generation: i64,
     ) -> Result<(), Box> {
         log_debug!(
-            "create_writer called for file: {}, index: {}, schema_address: {}, sort_columns: {:?}, reverse_sorts: {:?}, nulls_first: {:?}, generation: {}",
+            "create_writer called for file: {}, index: {}, schema_address: {}, sort_columns: {:?}, reverse_sorts: {:?}, nulls_first: {:?}, writer_generation: {}",
             filename, index_name, schema_address, sort_columns, reverse_sorts, nulls_first, writer_generation
         );
 
@@ -95,9 +115,6 @@ impl NativeParquetWriter {
         let schema = Arc::new(arrow::datatypes::Schema::try_from(&arrow_schema)?);
         log_debug!("Schema created with {} fields", schema.fields().len());
 
-        let file = File::create(&temp_filename)?;
-        let (crc_file, crc_handle) = CrcWriter::new(file);
-
         let mut settings: NativeSettings = SETTINGS_STORE
             .get(&index_name)
             .map(|r| r.clone())
@@ -107,14 +124,25 @@ impl NativeParquetWriter {
         settings.reverse_sorts = reverse_sorts;
         settings.nulls_first = nulls_first;
 
-        let props = WriterPropertiesBuilder::build_with_generation(&settings, Some(writer_generation));
-
         SETTINGS_STORE.insert(index_name, settings.clone());
 
-        let writer = ArrowWriter::try_new(crc_file, schema, Some(props))?;
+        // If sort columns are configured, use Arrow IPC staging path so
+        // batches can be cheaply read back for sorting before writing Parquet.
+        let (variant, crc_handle) = if !settings.sort_columns.is_empty() {
+            let ipc_path = format!("{}{}", temp_filename, IPC_STAGING_SUFFIX);
+            let file = File::create(&ipc_path)?;
+            let ipc_writer = IpcFileWriter::try_new(file, &schema)?;
+            (WriterVariant::Ipc(Arc::new(Mutex::new(ipc_writer))), None)
+        } else {
+            let file = File::create(&temp_filename)?;
+            let (crc_file, crc_handle) = CrcWriter::new(file);
+            let props = WriterPropertiesBuilder::build_with_generation(&settings, Some(writer_generation));
+            let writer = ArrowWriter::try_new(crc_file, schema, Some(props))?;
+            (WriterVariant::Parquet(Arc::new(Mutex::new(writer))), Some(crc_handle))
+        };
 
         WRITERS.insert(temp_filename, WriterState {
-            writer: Arc::new(Mutex::new(writer)),
+            variant,
             settings,
             crc_handle,
             writer_generation,
@@ -144,8 +172,18 @@ impl NativeParquetWriter {
                 log_debug!("Created RecordBatch with {} rows and {} columns", record_batch.num_rows(), record_batch.num_columns());
 
                 if let Some(state) = WRITERS.get(&temp_filename) {
-                    let mut writer = state.writer.lock().unwrap();
-                    writer.write(&record_batch)?;
+                    match &state.variant {
+                        WriterVariant::Ipc(writer_arc) => {
+                            log_debug!("Writing RecordBatch to IPC staging file");
+                            let mut writer = writer_arc.lock().unwrap();
+                            writer.write(&record_batch)?;
+                        }
+                        WriterVariant::Parquet(writer_arc) => {
+                            log_debug!("Writing RecordBatch to Parquet file");
+                            let mut writer = writer_arc.lock().unwrap();
+                            writer.write(&record_batch)?;
+                        }
+                    }
                     Ok(())
                 } else {
                     log_error!("ERROR: No writer found for temp file: {}", temp_filename);
@@ -163,46 +201,72 @@ impl NativeParquetWriter {
         log_debug!("finalize_writer called for file: {} (temp: {})", filename, temp_filename);
 
         if let Some((_, state)) = WRITERS.remove(&temp_filename) {
-            let WriterState { writer: writer_arc, settings, crc_handle, writer_generation } = state;
+            let WriterState { variant, settings, crc_handle, writer_generation } = state;
             let index_name = settings.index_name.as_deref().unwrap_or("");
-            match Arc::try_unwrap(writer_arc) {
-                Ok(mutex) => {
-                    let writer = mutex.into_inner().unwrap();
-                    match writer.close() {
-                        Ok(_) => {
-                            let temp_crc32 = crc_handle.crc32();
-                            log_debug!("Successfully closed temp writer for: {}", temp_filename);
-
-                            let crc32 = Self::sort_and_rewrite_parquet(&temp_filename, &filename, index_name, &settings.sort_columns, &settings.reverse_sorts, &settings.nulls_first, temp_crc32, writer_generation)?;
-
-                            if Path::new(&temp_filename).exists() {
-                                if let Err(e) = std::fs::remove_file(&temp_filename) {
-                                    log_error!("Failed to remove temp file {}: {}", temp_filename, e);
-                                }
-                            }
+
+            match variant {
+                WriterVariant::Ipc(writer_arc) => {
+                    match Arc::try_unwrap(writer_arc) {
+                        Ok(mutex) => {
+                            let mut writer = mutex.into_inner().unwrap();
+                            writer.finish()?;
+                            log_info!("Successfully closed IPC staging writer for: {}", temp_filename);
+
+                            let ipc_path = format!("{}{}", temp_filename, IPC_STAGING_SUFFIX);
+                            let crc32 = Self::sort_and_rewrite_parquet(&ipc_path, &filename, index_name, &settings.sort_columns, &settings.reverse_sorts, &settings.nulls_first, writer_generation)?;
+                            let _ = std::fs::remove_file(&ipc_path);
 
                             log_debug!("CRC32 for file {}: {:#010x}", filename, crc32);
 
-                            // Keep a handle for sync_to_disk
                             let file_for_sync = File::open(&filename)?;
                             FILE_MANAGER.insert(filename.clone(), file_for_sync);
 
-                            // Read full ParquetMetaData from the final file
                             let file = File::open(&filename)?;
                             let reader = SerializedFileReader::new(file)?;
                             let parquet_metadata = reader.metadata().clone();
 
                             Ok(Some(FinalizeResult { metadata: parquet_metadata, crc32 }))
                         }
-                        Err(e) => {
-                            log_error!("ERROR: Failed to close writer for temp file: {}", temp_filename);
-                            Err(e.into())
+                        Err(_) => {
+                            log_error!("ERROR: IPC Writer still in use for temp file: {}", temp_filename);
+                            Err("IPC Writer still in use".into())
                         }
                     }
                 }
-                Err(_) => {
-                    log_error!("ERROR: Writer still in use for temp file: {}", temp_filename);
-                    Err("Writer still in use".into())
+                WriterVariant::Parquet(writer_arc) => {
+                    match Arc::try_unwrap(writer_arc) {
+                        Ok(mutex) => {
+                            let writer = mutex.into_inner().unwrap();
+                            match writer.close() {
+                                Ok(_) => {
+                                    let crc32 = crc_handle.map(|h| h.crc32()).unwrap_or(0);
+                                    log_info!("Successfully closed temp writer for: {}", temp_filename);
+
+                                    // Parquet variant is used for non-sorted data; just rename.
+                                    std::fs::rename(&temp_filename, &filename)?;
+
+                                    log_debug!("CRC32 for file {}: {:#010x}", filename, crc32);
+
+                                    let file_for_sync = File::open(&filename)?;
+                                    FILE_MANAGER.insert(filename.clone(), file_for_sync);
+
+                                    let file = File::open(&filename)?;
+                                    let reader = SerializedFileReader::new(file)?;
+                                    let parquet_metadata = reader.metadata().clone();
+
+                                    Ok(Some(FinalizeResult { metadata: parquet_metadata, crc32 }))
+                                }
+                                Err(e) => {
+                                    log_error!("ERROR: Failed to close writer for temp file: {}", temp_filename);
+                                    Err(e.into())
+                                }
+                            }
+                        }
+                        Err(_) => {
+                            log_error!("ERROR: Writer still in use for temp file: {}", temp_filename);
+                            Err("Writer still in use".into())
+                        }
+                    }
                 }
             }
         } else {
@@ -211,7 +275,6 @@ impl NativeParquetWriter {
         }
     }
 
-
     fn sort_and_rewrite_parquet(
         temp_filename: &str,
         output_filename: &str,
@@ -219,7 +282,6 @@ impl NativeParquetWriter {
         sort_columns: &[String],
         reverse_sorts: &[bool],
         nulls_first: &[bool],
-        temp_crc32: u32,
         writer_generation: i64,
     ) -> Result> {
         log_debug!(
@@ -227,12 +289,6 @@ impl NativeParquetWriter {
             temp_filename, output_filename, sort_columns, reverse_sorts, nulls_first
         );
 
-        if sort_columns.is_empty() {
-            log_debug!("No sort columns specified, renaming temp file to final");
-            std::fs::rename(temp_filename, output_filename)?;
-            return Ok(temp_crc32);
-        }
-
         let config = SETTINGS_STORE
             .get(index_name)
             .map(|r| r.clone())
@@ -247,7 +303,6 @@ impl NativeParquetWriter {
         }
     }
 
-    /// In-memory sort for small files: read all batches, concat, sort, rewrite row IDs, write.
     fn sort_small_file(
         temp_filename: &str,
         output_filename: &str,
@@ -260,31 +315,43 @@ impl NativeParquetWriter {
         log_debug!("Using in-memory sort for small file: {}", temp_filename);
 
         let file = File::open(temp_filename)?;
-        let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
-        let row_count = builder.metadata().file_metadata().num_rows() as usize;
-        let arrow_reader = builder.with_batch_size(row_count).build()?;
-
-        let batch = match arrow_reader.into_iter().next() {
-            Some(Ok(b)) if b.num_rows() > 0 => b,
-            Some(Err(e)) => return Err(e.into()),
-            _ => {
-                log_debug!("No data to sort in file: {}", temp_filename);
-                std::fs::rename(temp_filename, output_filename)?;
-                return Ok(0);
+        let reader = IpcFileReader::try_new(file, None)?;
+        let schema = reader.schema();
+
+        let mut all_batches: Vec = Vec::new();
+        for batch_result in reader {
+            let batch = batch_result?;
+            if batch.num_rows() > 0 {
+                all_batches.push(batch);
             }
-        };
+        }
+
+        if all_batches.is_empty() {
+            log_info!("No data in temp file: {}", temp_filename);
+            let props = WriterPropertiesBuilder::build_with_generation(
+                &SETTINGS_STORE.get(index_name).map(|r| r.clone()).unwrap_or_default(),
+                Some(writer_generation),
+            );
+            let file = File::create(output_filename)?;
+            let writer = ArrowWriter::try_new(file, schema, Some(props))?;
+            writer.close()?;
+            return Ok(0);
+        }
 
-        let schema = batch.schema();
-        let sorted_batch = Self::sort_batch(&batch, sort_columns, reverse_sorts, nulls_first)?;
+        let combined_batch = concat_batches(&schema, &all_batches)?;
+        let sorted_batch = Self::sort_batch(&combined_batch, sort_columns, reverse_sorts, nulls_first)?;
         let final_batch = Self::rewrite_row_ids(&sorted_batch, &schema)?;
 
         let crc32 = Self::write_final_file(output_filename, index_name, &final_batch, schema, Some(writer_generation))?;
+
+        log_info!(
+            "sort_small_file: sorted {} rows, wrote Parquet to {}",
+            final_batch.num_rows(),
+            output_filename
+        );
         Ok(crc32)
     }
 
-    /// For large files: read in batches, sort each batch individually, write each
-    /// as a separate sorted chunk file, then use the streaming k-way merge to
-    /// produce the final globally-sorted output.
     fn sort_large_file(
         temp_filename: &str,
         output_filename: &str,
@@ -297,39 +364,51 @@ impl NativeParquetWriter {
         log_debug!("Using streaming merge sort for large file: {}", temp_filename);
 
         let file = File::open(temp_filename)?;
-        let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
-        let arrow_reader = builder.with_batch_size(batch_size).build()?;
+        let reader = IpcFileReader::try_new(file, None)?;
+        let schema = reader.schema();
 
         let mut chunk_paths: Vec = Vec::new();
         let mut batch_count = 0;
         let chunk_dir = Path::new(output_filename).parent().unwrap_or_else(|| Path::new("."));
 
-        for batch_result in arrow_reader {
+        for batch_result in reader {
             let batch = batch_result?;
-            let schema = batch.schema();
-            let sorted_batch = Self::sort_batch(&batch, sort_columns, reverse_sorts, nulls_first)?;
-
-            let chunk_filename = chunk_dir
-                .join(format!("temp_sort_chunk_{}_{}.parquet", batch_count, std::process::id()))
-                .to_string_lossy()
-                .to_string();
-            // CRC for temp chunks is not needed, discard it
-            Self::write_final_file(&chunk_filename, index_name, &sorted_batch, schema, None)?;
-
-            chunk_paths.push(chunk_filename);
-            batch_count += 1;
+            if batch.num_rows() == 0 {
+                continue;
+            }
+
+            // IpcFileReader returns batches at whatever size they were written.
+            // Slice into batch_size chunks to bound memory during sort.
+            let mut offset = 0;
+            while offset < batch.num_rows() {
+                let len = std::cmp::min(batch_size, batch.num_rows() - offset);
+                let slice = batch.slice(offset, len);
+                offset += len;
+
+                let sorted_batch = Self::sort_batch(&slice, sort_columns, reverse_sorts, nulls_first)?;
+
+                let chunk_filename = chunk_dir
+                    .join(format!("temp_sort_chunk_{}_{}.parquet", batch_count, std::process::id()))
+                    .to_string_lossy()
+                    .to_string();
+                // CRC for temp chunks is not needed, discard it
+                Self::write_final_file(&chunk_filename, index_name, &sorted_batch, schema.clone(), None)?;
+
+                chunk_paths.push(chunk_filename);
+                batch_count += 1;
+            }
         }
 
         if chunk_paths.is_empty() {
             log_debug!("No data to sort in file: {}", temp_filename);
-            std::fs::rename(temp_filename, output_filename)?;
             return Ok(0);
         }
 
-        log_debug!("Created {} sorted chunks, merging via streaming k-way merge", batch_count);
+        log_debug!(
+            "Created {} sorted Parquet chunks, merging via streaming k-way merge",
+            batch_count
+        );
 
-        // merge_sorted returns MergeOutput; we discard it here because the
-        // sort-and-rewrite path produces the final file directly.
         let _merge_output = merge_sorted(
             &chunk_paths,
             output_filename,
@@ -337,42 +416,63 @@ impl NativeParquetWriter {
             sort_columns,
             reverse_sorts,
             nulls_first,
-        ).map_err(|e| -> Box { format!("Streaming merge failed: {}", e).into() })?;
+        )
+        .map_err(|e| -> Box {
+            format!("Streaming merge failed: {}", e).into()
+        })?;
 
         // Clean up temp chunk files
         for path in &chunk_paths {
             let _ = std::fs::remove_file(path);
         }
 
+        log_info!(
+            "sort_large_file: merged {} chunks, wrote Parquet to {}",
+            batch_count,
+            output_filename
+        );
         Ok(0)
     }
 
+    /// Sort a batch using RowConverter: converts sort columns into compact
+    /// byte-comparable rows, sorts indices by comparing those rows, then
+    /// reorders all columns via take.
     fn sort_batch(
         batch: &RecordBatch,
         sort_columns: &[String],
         reverse_sorts: &[bool],
         nulls_first: &[bool],
     ) -> Result> {
-        let columns: Vec = sort_columns
+        let sort_fields: Vec = sort_columns
             .iter()
             .enumerate()
             .map(|(i, col_name)| {
-                let reverse = reverse_sorts.get(i).copied().unwrap_or(false);
-                let nf = nulls_first.get(i).copied().unwrap_or(false);
-                let options = arrow::compute::SortOptions {
-                    descending: reverse,
-                    nulls_first: nf,
-                };
                 let col_index = batch.schema().index_of(col_name)
                     .map_err(|_| format!("Sort column '{}' not found in schema", col_name))?;
-                Ok(SortColumn {
-                    values: batch.column(col_index).clone(),
-                    options: Some(options),
-                })
+                let data_type = batch.schema().field(col_index).data_type().clone();
+                let options = arrow::compute::SortOptions {
+                    descending: reverse_sorts.get(i).copied().unwrap_or(false),
+                    nulls_first: nulls_first.get(i).copied().unwrap_or(false),
+                };
+                Ok(SortField::new_with_options(data_type, options))
             })
             .collect::, Box>>()?;
 
-        let indices = lexsort_to_indices(&columns, None)?;
+        let converter = RowConverter::new(sort_fields)?;
+
+        let sort_arrays: Vec> = sort_columns
+            .iter()
+            .map(|col_name| {
+                let col_index = batch.schema().index_of(col_name).unwrap();
+                batch.column(col_index).clone()
+            })
+            .collect();
+
+        let rows = converter.convert_columns(&sort_arrays)?;
+        let mut sort_indices: Vec = (0..batch.num_rows() as u32).collect();
+        sort_indices.sort_unstable_by(|&a, &b| rows.row(a as usize).cmp(&rows.row(b as usize)));
+
+        let indices = arrow::array::UInt32Array::from(sort_indices);
         let sorted_columns: Result, _> = batch
             .columns()
             .iter()
@@ -443,9 +543,12 @@ impl NativeParquetWriter {
         let mut total_memory = 0;
         for entry in WRITERS.iter() {
             if entry.key().starts_with(&path_prefix) {
-                if let Ok(writer) = entry.value().writer.lock() {
-                    total_memory += writer.memory_size();
+                if let WriterVariant::Parquet(writer_arc) = &entry.value().variant {
+                    if let Ok(writer) = writer_arc.lock() {
+                        total_memory += writer.memory_size();
+                    }
                 }
+                // IPC writers don't expose memory_size()
             }
         }
         Ok(total_memory)
diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/tests/writer_integration_tests.rs b/sandbox/plugins/parquet-data-format/src/main/rust/tests/writer_integration_tests.rs
index 2a67511ba9c8f..076e3c899af2f 100644
--- a/sandbox/plugins/parquet-data-format/src/main/rust/tests/writer_integration_tests.rs
+++ b/sandbox/plugins/parquet-data-format/src/main/rust/tests/writer_integration_tests.rs
@@ -50,7 +50,7 @@ fn test_concurrent_writer_creation() {
             let file_path = temp_dir_path.join(format!("concurrent_{}.parquet", i));
             let filename = file_path.to_string_lossy().to_string();
             let (_schema, schema_ptr) = create_test_ffi_schema();
-            if NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![]).is_ok() {
+            if NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![], 0).is_ok() {
                 success_count.fetch_add(1, Ordering::SeqCst);
                 let _ = NativeParquetWriter::finalize_writer(filename);
             }
@@ -173,7 +173,7 @@ fn test_concurrent_complete_writer_lifecycle() {
             let filename = file_path.to_string_lossy().to_string();
             let (_schema, schema_ptr) = create_test_ffi_schema();
 
-            if NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![]).is_ok() {
+            if NativeParquetWriter::create_writer(filename.clone(), "test-index".to_string(), schema_ptr, vec![], vec![], vec![], 0).is_ok() {
                 let (array_ptr, data_schema_ptr) = create_test_ffi_data().unwrap();
                 let write_ok = NativeParquetWriter::write_data(filename.clone(), array_ptr, data_schema_ptr).is_ok();
                 cleanup_ffi_data(array_ptr, data_schema_ptr);
@@ -199,3 +199,143 @@ fn test_concurrent_complete_writer_lifecycle() {
     }
     assert_eq!(success_count.load(Ordering::SeqCst), thread_count);
 }
+
+// ===== Arrow IPC staging integration tests =====
+
+#[test]
+fn test_ipc_staging_sorted_writer_integration() {
+    let (_temp_dir, filename) = get_temp_file_path("ipc_integ_sorted.parquet");
+    let (_schema, schema_ptr) = create_test_ffi_schema();
+
+    NativeParquetWriter::create_writer(
+        filename.clone(), "test-index".to_string(), schema_ptr,
+        vec!["id".to_string()], vec![false], vec![false], 0
+    ).unwrap();
+
+    // Write multiple batches with out-of-order data
+    for batch_ids in [vec![50, 30, 10], vec![40, 20, 60]] {
+        let names: Vec> = batch_ids.iter().map(|_| Some("x")).collect();
+        let (ap, sp) = create_test_ffi_data_with_ids(batch_ids, names).unwrap();
+        NativeParquetWriter::write_data(filename.clone(), ap, sp).unwrap();
+        cleanup_ffi_data(ap, sp);
+    }
+
+    let result = NativeParquetWriter::finalize_writer(filename.clone());
+    assert!(result.is_ok());
+    let metadata = result.unwrap().unwrap();
+    assert_eq!(metadata.metadata.file_metadata().num_rows(), 6);
+
+    assert!(NativeParquetWriter::sync_to_disk(filename.clone()).is_ok());
+
+    let ids = read_parquet_file_sorted_ids(&filename);
+    assert_eq!(ids, vec![10, 20, 30, 40, 50, 60]);
+
+    let read_metadata = NativeParquetWriter::get_file_metadata(filename).unwrap();
+    assert_eq!(read_metadata.num_rows(), 6);
+
+    cleanup_ffi_schema(schema_ptr);
+}
+
+#[test]
+fn test_ipc_staging_concurrent_sorted_lifecycle() {
+    let temp_dir = tempdir().unwrap();
+    let thread_count = 6;
+    let success_count = Arc::new(AtomicUsize::new(0));
+    let mut handles = vec![];
+
+    for i in 0..thread_count {
+        let temp_dir_path = temp_dir.path().to_path_buf();
+        let success_count = Arc::clone(&success_count);
+        let handle = thread::spawn(move || {
+            let file_path = temp_dir_path.join(format!("ipc_lifecycle_{}.parquet", i));
+            let filename = file_path.to_string_lossy().to_string();
+            let (_schema, schema_ptr) = create_test_ffi_schema();
+
+            if NativeParquetWriter::create_writer(
+                filename.clone(), "test-index".to_string(), schema_ptr,
+                vec!["id".to_string()], vec![false], vec![false], 0
+            ).is_ok() {
+                let (ap, sp) = create_test_ffi_data_with_ids(
+                    vec![30, 10, 20], vec![Some("C"), Some("A"), Some("B")]
+                ).unwrap();
+                let write_ok = NativeParquetWriter::write_data(filename.clone(), ap, sp).is_ok();
+                cleanup_ffi_data(ap, sp);
+
+                if write_ok {
+                    if let Ok(Some(metadata)) = NativeParquetWriter::finalize_writer(filename.clone()) {
+                        if metadata.metadata.file_metadata().num_rows() == 3
+                            && NativeParquetWriter::sync_to_disk(filename.clone()).is_ok()
+                            && file_path.exists()
+                        {
+                            let ids = read_parquet_file_sorted_ids(&filename);
+                            if ids == vec![10, 20, 30] {
+                                success_count.fetch_add(1, Ordering::SeqCst);
+                            }
+                        }
+                    }
+                }
+            }
+            cleanup_ffi_schema(schema_ptr);
+        });
+        handles.push(handle);
+    }
+
+    for handle in handles {
+        handle.join().unwrap();
+    }
+    assert_eq!(success_count.load(Ordering::SeqCst), thread_count);
+}
+
+#[test]
+fn test_ipc_and_parquet_mixed_concurrent_lifecycle() {
+    let temp_dir = tempdir().unwrap();
+    let thread_count = 8;
+    let success_count = Arc::new(AtomicUsize::new(0));
+    let mut handles = vec![];
+
+    for i in 0..thread_count {
+        let temp_dir_path = temp_dir.path().to_path_buf();
+        let success_count = Arc::clone(&success_count);
+        let use_sort = i % 2 == 0; // Even threads use IPC (sorted), odd use Parquet (unsorted)
+
+        let handle = thread::spawn(move || {
+            let file_path = temp_dir_path.join(format!("mixed_{}.parquet", i));
+            let filename = file_path.to_string_lossy().to_string();
+            let (_schema, schema_ptr) = create_test_ffi_schema();
+
+            let sort_cols = if use_sort { vec!["id".to_string()] } else { vec![] };
+            let reverse = if use_sort { vec![false] } else { vec![] };
+            let nulls = if use_sort { vec![false] } else { vec![] };
+
+            if NativeParquetWriter::create_writer(
+                filename.clone(), "test-index".to_string(), schema_ptr,
+                sort_cols, reverse, nulls, 0
+            ).is_ok() {
+                let (ap, sp) = create_test_ffi_data_with_ids(
+                    vec![30, 10, 20], vec![Some("C"), Some("A"), Some("B")]
+                ).unwrap();
+                let write_ok = NativeParquetWriter::write_data(filename.clone(), ap, sp).is_ok();
+                cleanup_ffi_data(ap, sp);
+
+                if write_ok {
+                    if let Ok(Some(metadata)) = NativeParquetWriter::finalize_writer(filename.clone()) {
+                        if metadata.metadata.file_metadata().num_rows() == 3 && file_path.exists() {
+                            let ids = read_parquet_file_sorted_ids(&filename);
+                            let expected = if use_sort { vec![10, 20, 30] } else { vec![30, 10, 20] };
+                            if ids == expected {
+                                success_count.fetch_add(1, Ordering::SeqCst);
+                            }
+                        }
+                    }
+                }
+            }
+            cleanup_ffi_schema(schema_ptr);
+        });
+        handles.push(handle);
+    }
+
+    for handle in handles {
+        handle.join().unwrap();
+    }
+    assert_eq!(success_count.load(Ordering::SeqCst), thread_count);
+}

From f97a81ceaceecc9b0d0d40e7ce72e26f5ac1048f Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Sat, 9 May 2026 12:41:19 -0700
Subject: [PATCH 102/115] Revert "[Sandbox SQL snapshot] Pin
 org.opensearch.query:* (unified-query-*) snapshots to mavenLocal (#21578)"
 (#21580)

This reverts commit 36809cc64fb0424a99188ed54aa0c0d5b1fa601e.

Signed-off-by: Marc Handalian 
---
 sandbox/build.gradle                          | 25 +------------------
 .../analytics-backend-datafusion/build.gradle |  3 ---
 sandbox/plugins/analytics-engine/build.gradle | 12 ++-------
 .../plugins/test-ppl-frontend/build.gradle    | 16 +-----------
 4 files changed, 4 insertions(+), 52 deletions(-)

diff --git a/sandbox/build.gradle b/sandbox/build.gradle
index 99029b66d4ac4..1ee3608a4bc10 100644
--- a/sandbox/build.gradle
+++ b/sandbox/build.gradle
@@ -46,34 +46,11 @@ subprojects {
   // The patched calcite-core / calcite-linq4j live in the OpenSearch
   // snapshots Maven repo; analytics-framework advertises them as `api`,
   // so every consumer of analytics-framework needs both the repo and a
-  // `force` to win over transitive vanilla calcite.
-  //
-  // org.opensearch.query (unified-query-*) is excluded from this remote and
-  // pinned to mavenLocal in the consuming subprojects (see analytics-engine
-  // and test-ppl-frontend build.gradle). The remote OpenSearch Snapshots repo
-  // only republishes from sql/main, not from sql/feature/mustang-ppl-integration,
-  // so its 3.7.0.0-SNAPSHOT jars trail the feature branch by however many merges.
-  // Gradle's default SNAPSHOT resolution weighs the remote's explicit
-  // /timestamp metadata higher than mavenLocal's true>,
-  // so the stale remote wins even when mavenLocal has a newer .
-  // Excluding the group here forces gradle to fall through to mavenLocal,
-  // populated by the sandbox-check workflow's pre-step
-  // (publishUnifiedQueryPublicationToMavenLocal from the feature branch).
-  // Drop this exclusion once the SQL feature branch merges to sql/main and
-  // remote snapshots catch up.
+  // `force` to win over transitive vanilla calcite
   repositories {
-    // mavenLocal serves org.opensearch.query:* (unified-query-*) only — the
-    // sandbox-check workflow's pre-step populates it from the SQL feature branch
-    // (publishUnifiedQueryPublicationToMavenLocal). Restricting via includeGroup
-    // keeps mavenLocal off the resolution path for everything else, where it
-    // would otherwise mask intentionally-pinned remote artifacts.
-    mavenLocal {
-      mavenContent { includeGroup 'org.opensearch.query' }
-    }
     maven {
       name = 'OpenSearch Snapshots'
       url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/'
-      mavenContent { excludeGroup 'org.opensearch.query' }
     }
   }
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/build.gradle b/sandbox/plugins/analytics-backend-datafusion/build.gradle
index 65ee64daf8b4b..1d71f16082900 100644
--- a/sandbox/plugins/analytics-backend-datafusion/build.gradle
+++ b/sandbox/plugins/analytics-backend-datafusion/build.gradle
@@ -16,9 +16,6 @@ repositories {
   maven {
     name = 'OpenSearch Snapshots'
     url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/'
-    // unified-query-* artifacts are pinned to mavenLocal — see comment in
-    // sandbox/build.gradle's subprojects { repositories } block.
-    mavenContent { excludeGroup 'org.opensearch.query' }
   }
 }
 
diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle
index a56b78baa88cd..0058df4b6eb68 100644
--- a/sandbox/plugins/analytics-engine/build.gradle
+++ b/sandbox/plugins/analytics-engine/build.gradle
@@ -14,16 +14,8 @@
 
 apply plugin: 'opensearch.internal-cluster-test'
 
-// SQL Unified Query API version (aligned with OpenSearch build version).
-// Bumped to 3.7 to match the rest of the sandbox (test-ppl-frontend declares
-// the same '3.7.0.0-SNAPSHOT'). org.opensearch.query:* is also pinned to
-// mavenLocal in sandbox/build.gradle's subprojects { repositories } block,
-// so the remote OpenSearch Snapshots repo no longer serves this group;
-// 3.6 jars don't exist in mavenLocal (only 3.7 from the feature branch's
-// publishUnifiedQueryPublicationToMavenLocal pre-step), so the older pin
-// caused `Could not find org.opensearch.query:unified-query-api:3.6.0.0-SNAPSHOT`
-// at internal-cluster-test resolution.
-def sqlUnifiedQueryVersion = '3.7.0.0-SNAPSHOT'
+// SQL Unified Query API version (aligned with OpenSearch build version)
+def sqlUnifiedQueryVersion = '3.6.0.0-SNAPSHOT'
 
 opensearchplugin {
   description = 'Analytics engine hub: discovers and wires query extensions via ExtensiblePlugin SPI.'
diff --git a/sandbox/plugins/test-ppl-frontend/build.gradle b/sandbox/plugins/test-ppl-frontend/build.gradle
index 58f15ceebd6dc..9c2d79c306c9c 100644
--- a/sandbox/plugins/test-ppl-frontend/build.gradle
+++ b/sandbox/plugins/test-ppl-frontend/build.gradle
@@ -32,24 +32,10 @@ opensearchplugin {
 }
 
 repositories {
-  // Pin org.opensearch.query:* (unified-query-*) artifacts to mavenLocal only.
-  // The remote OpenSearch Snapshots repo only republishes from sql/main, not from
-  // sql/feature/mustang-ppl-integration, so its 3.7.0.0-SNAPSHOT jars trail the
-  // feature branch by however many merges. Gradle's default SNAPSHOT resolution
-  // weighs the remote's explicit /timestamp metadata higher than
-  // mavenLocal's bare true>, so the stale remote wins even when
-  // mavenLocal has a newer . Restrict the unified-query group to
-  // mavenLocal and exclude it from the remote so the sandbox-check workflow's
-  // pre-step (publishUnifiedQueryPublicationToMavenLocal from the feature branch)
-  // is the authoritative source. Drop this filter once SQL feature branch merges
-  // to sql/main and remote snapshots catch up.
-  mavenLocal {
-    mavenContent { includeGroup 'org.opensearch.query' }
-  }
+  mavenLocal()
   maven {
     name = 'OpenSearch Snapshots'
     url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/'
-    mavenContent { excludeGroup 'org.opensearch.query' }
   }
 }
 

From 4dea6c4253c4c9bd3708dedbe8842e70749fae3f Mon Sep 17 00:00:00 2001
From: Kai Huang <105710027+ahkcs@users.noreply.github.com>
Date: Sat, 9 May 2026 15:26:24 -0700
Subject: [PATCH 103/115] [Analytics Backend / DataFusion] Onboard PPL array
 constructor and 8 multivalue (mv) functions to analytics-engine route
 (#21554)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [Analytics Backend / DataFusion] Wire PPL array constructor + array_length / array_slice / array_distinct / mvjoin

Onboards the PPL `array(a, b, …)` constructor and four array-consuming
functions to the analytics-engine route by mapping their Calcite lowering
targets through Substrait to DataFusion's native make_array / array_length /
array_slice / array_distinct / array_to_string.

Same templated shape as the `replace` PR (#21527), with two extensions:

  ScalarFunction enum constants (5)
    + STANDARD_PROJECT_OPS / ARRAY_RETURNING_PROJECT_OPS membership
    + opensearch_array_functions.yaml extension entries
    + ADDITIONAL_SCALAR_SIGS Calcite-op→Substrait-name bridges
    + scalarFunctionAdapters() entries for the 3 functions that need
      operand normalization
    = onboarded to the analytics route.

Capability lookup at OpenSearchProjectRule keys on the call's return type;
for array-returning functions (`array(...)`, `array_slice`, `array_distinct`)
the return type resolves to `SqlTypeName.ARRAY`, which previously hit
`default → null` in `FieldType.fromSqlTypeName` and emptied the viable-backend
list before the registration could match.

  * `FieldType.ARRAY` added to the analytics SPI enum.
  * `SqlTypeName.ARRAY → FieldType.ARRAY` mapping in `fromSqlTypeName`.
  * `ARRAY_RETURNING_PROJECT_OPS` registered against `Set.of(FieldType.ARRAY)`
    only — separate from `STANDARD_PROJECT_OPS` so `FieldType.ARRAY` doesn't
    pollute filter / aggregate capabilities (no meaningful semantics over
    array-typed values there).
  * `ArrowSchemaFromCalcite.toArrowField` recurses into the component type
    to build the matching Arrow `List` field — without this the result
    schema would have a bare `List` with no element field and the backend's
    Arrow IPC reader would fail to bind result columns.

Substrait's standard catalog has no array_* entries, so isthmus'
`RexExpressionConverter` would fail with "Unable to convert call …" on every
array call. New `opensearch_array_functions.yaml` declares:

  * `make_array(any1, …)` → `list` (variadic, min: 0).
  * `array_length(list)` → `i64?`.
  * `array_slice(list, i64, i64)` → `list` (with i32 fallback).
  * `array_distinct(list)` → `list`.
  * `array_to_string(list, string)` → `string?` (with varchar fallback).

Loaded via `SimpleExtension.load("/opensearch_array_functions.yaml")` and
merged into the plugin's extension collection in
`DataFusionPlugin.loadSubstraitExtensions()`.

Substrait's call-conversion path (and DataFusion's signature matcher) is
strict about operand types in ways Calcite's PPL lowering doesn't naturally
satisfy. Three adapters bridge the gap:

  * `MakeArrayAdapter` — implements `ScalarFunctionAdapter` directly
    (not `AbstractNameMappingAdapter`). PPL's `ArrayFunctionImpl` infers
    `ARRAY` for the call's return type but does NOT
    widen the individual operand types. So `array(1, 1.5)` produces a
    RexCall whose operands are `(INTEGER, DECIMAL(2,1))` but whose return
    type is `ARRAY`. Substrait's variadic-`any1` consistency
    validator throws an `AssertionError` in that case (not a recoverable
    exception — it fatally exits the search-thread JVM). The adapter
    extracts the call's component type and CASTs each non-matching
    operand to it before emission.
  * `ArrayToStringAdapter` — declares a local `array_to_string` op and
    name-maps `SqlLibraryOperators.ARRAY_JOIN` → it.
  * `ArraySliceAdapter` — passes the `ARRAY_SLICE` call through unchanged
    but coerces the index operands (positions 1, 2, optional 3) to
    `BIGINT`. PPL's parser types positive integer literals as
    `DECIMAL(20,0)`; DataFusion's `array_slice` signature accepts only
    integer indexes and refuses to coerce decimal arguments.

Two third-party dependencies that surfaced as fatal `NoClassDefFoundError`
during execution of array-returning calls:

  * `commons-text` to analytics-engine — Calcite's `SqlFunctions` class
    statically references `org.apache.commons.text.similarity.LevenshteinDistance`.
    Without it, any Calcite RelNode walk that touches `SqlFunctions.`
    poisons the search-thread JVM.
  * `jackson-datatype-jsr310` to **arrow-flight-rpc** (the parent plugin
    that bundles `arrow-vector`). `arrow-vector`'s `JsonStringArrayList`
    eagerly registers `JavaTimeModule` on its ObjectMapper in ``,
    so any reader of an Arrow `ListVector` (i.e. every array-returning
    DataFusion call flowing through analytics-engine) hits a fatal
    NoClassDefFoundError. The dep belongs on arrow-flight-rpc's classpath
    because that plugin defines arrow-vector's classloader; bundling it
    in analytics-backend-datafusion (the child plugin) is invisible to
    arrow-vector. Marked `compileOnly` here to avoid jar-hell with
    arrow-flight-rpc's `api` dependency.

  * Before: 1/60 (testArrayWithMix only — exercises an error path that
    fails before the ARRAY capability lookup).
  * After:  9/60.
    Newly passing: testArray, testArrayWithString, testArrayLength,
    testMvjoinWithStringArray, testMvjoinWithStringifiedNumbers,
    testMvjoinWithMixedStringValues, testMvjoinWithStringBooleans,
    testMvjoinWithSpecialDelimiters, testMvjoinWithArrayFromRealFields,
    testMvjoinWithMultipleRealFields.

The remaining 51 failures fall into three buckets:

  * 50 — out-of-scope S1+ functions (`mvfind`, `mvzip`, `reduce`, `transform`,
    `forall`, `filter`, `exists`, `ITEM`). These are PPL UDFs without direct
    DataFusion equivalents and need either lambda-substrait wiring or
    custom UDF registration on the Rust side.
  * 5  — `testMvindexRange*` family. PPL's `mvindex(arr, from, to)` lowers
    to `ARRAY_SLICE(arr, from+1, to+1)` (1-based shift) but the lowering
    is missing the +1, so DataFusion's 1-based array_slice returns a
    window shifted by one. Fix belongs in the SQL plugin's PPL→Calcite
    lowering layer.
  * 1  — `testMvindexRangeMixed` JSON formatting mismatch (test code
    expects bare `[a,b,c]` but the response is `\"[\\\"a\\\",\\\"b\\\",\\\"c\\\"]\"`).

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Fix ARRAY_SLICE 0-based-(start, length) → 1-based-(start, end) for DataFusion

Calcite's `SqlLibraryOperators.ARRAY_SLICE` is the Spark / Hive flavor —
0-based start, third arg is the length-of-elements to take. PPL's
`MVIndexFunctionImp.resolveRange` (in the SQL plugin) emits this form,
e.g. `mvindex(arr=[1..5], 1, 3)` → `ARRAY_SLICE(arr, 1, 3)` meaning
"start at 0-based position 1, take 3 elements" → expected `[2, 3, 4]`.

DataFusion's native `array_slice` is the Postgres / Snowflake flavor —
1-based start, third arg is the inclusive end-index. So the same call
`array_slice(arr, 1, 3)` returns elements at 1-based positions 1..3 →
`[1, 2, 3]`. Off-by-one across every `mvindex` range query.

Convert the operands in the adapter rather than the SQL plugin's PPL
lowering, because the lowering's existing semantics are correct for
Calcite's local executor (used by every non-analytics path); the bug is
only in the bridge to DataFusion.

  start' = start + 1
  end'   = start + length    (== start + 1 + (length - 1))

`MVIndexFunctionImp` already normalizes negative indexes to non-negative
0-based positions before invoking ARRAY_SLICE (it uses `arrayLen + idx`),
so the arithmetic above applies uniformly.

Empirically: `mvindex(arr=[1..5], 1, 3)` now returns the correct values
`[2, 3, 4]` (was `[1, 2, 3]`); negative form `mvindex(arr, -3, -1)`
returns `[3, 4, 5]` (was `[2, 3]`); mixed `mvindex(arr, -4, 2)` returns
`[2, 3]` matching the PPL spec.

The 5 `testMvindexRange*` tests still don't pass on the IT, but for an
unrelated reason — array-typed result values are being returned as
JSON-stringified scalars (`"[2,3,4]"`) instead of typed arrays. That's a
response-formatting issue affecting every array-returning test (also
`testArray`, `testArrayWithString`) and lives in a different code path;
it'll be addressed separately.

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Wire PPL ITEM (mvindex single-element) → DataFusion array_element

PPL's `mvindex(arr, N)` single-element form lowers (in `MVIndexFunctionImp.resolveSingleElement`)
to Calcite's `SqlStdOperatorTable.ITEM` operator with a 1-based index (already
converted from PPL's 0-based input). DataFusion's native single-element array
accessor is `array_element` (also 1-based), so a name-mapping adapter + yaml
extension entry are sufficient.

Templated shape:

  ScalarFunction.ITEM (SqlKind.ITEM)
    + STANDARD_PROJECT_OPS membership (returns the array's element type, which
      resolves through the existing FieldType.fromSqlTypeName → SUPPORTED_FIELD_TYPES
      capability lookup for non-array element types — array-of-array is rare in
      PPL and not exercised by the current test surface)
    + scalarFunctionAdapters() entry → ArrayElementAdapter
        ↳ rewrites SqlStdOperatorTable.ITEM to a locally-declared SqlFunction
          named "array_element"
        ↳ coerces the index operand to BIGINT (PPL's parser produces DECIMAL
          for positive integer literals; DataFusion's array_element rejects
          DECIMAL indexes, same as array_slice)
    + ADDITIONAL_SCALAR_SIGS bridge for the locally-declared op
    + opensearch_array_functions.yaml extension entry:
        array_element(list, i64) → any1?

# Pass-rate (CalciteArrayFunctionIT, force-routed)

  * Before this commit: 9/60.
  * After this commit:  12/60.
    Newly passing: testMvindexSingleElementPositive,
    testMvindexSingleElementNegative,
    testMvindexSingleElementNegativeMiddle.

The other 3 tests that hit the ITEM rejection (testMvfindWith*) are
multi-step queries where ITEM is one node in a tree that also includes
unrelated S1+ functions (mvfind/mvzip/etc.); they remain blocked by
the upstream functions, not by ITEM itself.

Signed-off-by: Kai Huang 

* [Analytics Engine] Carry array-typed cells through RowResponseCodec without JSON-stringifying

The row-oriented fragment-execution wire format (`FragmentExecutionResponse`,
used when arrow-flight streaming is disabled — every single-node test cluster
today) shipped each cell through OpenSearch's `writeGenericValue` /
`readGenericValue`, which preserves `List` values as `ArrayList`. On
the coordinator side, `RowResponseCodec.decode` then re-materialized the rows
into a `VectorSchemaRoot` for `Iterable`-style consumers.

Two bugs in that re-materialization were eating array values:

1. `inferArrowType` walked rows for the first non-null cell and matched
   against {Long, Integer, …, CharSequence, byte[], Number}. {@code List}
   wasn't in the chain, so it fell through to {@code break} and the
   fallback {@link ArrowType.Utf8} — every array column became a VARCHAR
   column.
2. `setVectorValue` for {@link VarCharVector} called {@code value.toString()}.
   For a {@code JsonStringArrayList} that returns the JSON form
   {@code "[2,3,4]"}, which then got serialized as a JSON string in the
   final response. Tests like {@code testMvindexRangePositive} saw their
   array result come back as a string `"[2,3,4]"` instead of an array
   `[2, 3, 4]`.

Fix:

* Replace {@code inferArrowType} with {@code inferField} that returns a
  full {@link Field}. For {@code List} cells, build a list field with the
  inner element type inferred from the first non-null element (with a
  fallback that scans later rows in case the first list is empty/all-null).
* Add a {@code ListVector} arm to {@code setVectorValue} that delegates to
  a new {@code writeListValue}. The writer bypasses {@link UnionListWriter}
  entirely — it writes directly to the list's offset / validity buffers and
  to the inner data vector via the inner vector's typed `setSafe`. The
  writer-based API requires per-element `ArrowBuf` allocations for varchar
  elements that are easy to leak or use-after-free; the direct path is
  simpler and avoids both classes of bug.

Plus a separate Arrow gotcha that surfaced once arrays started flowing
through correctly:

* {@code ListVector.getObject} for a {@code VarCharVector} child returns a
  {@code JsonStringArrayList} whose elements are Arrow's {@link Text} class,
  not Java {@link String}. {@code ExprValueUtils.fromObjectValue} doesn't
  recognize {@code Text} and threw "unsupported object class
  org.apache.arrow.vector.util.Text". {@code ArrowValues.toJavaValue} now
  mirrors its top-level VarChar branch for list cells: when a list value
  comes back from a {@code ListVector}, normalize each {@code Text} element
  to a {@link String} before handing the list upward.

  * Before: 12/60 (mvindex range tests still showed expected-vs-actual
    diff because `[2,3,4]` came back as a JSON string, not an array).
  * After:  26/60.

  Newly passing:
    testMvindexRangePositive, testMvindexRangeNegative, testMvindexRangeMixed,
    testMvindexRangeFirstThree, testMvindexRangeLastThree,
    testMvindexRangeSingleElement,
    testMvdedupWithDuplicates, testMvdedupWithAllDuplicates,
    testMvdedupWithNoDuplicates, testMvdedupWithStrings,
    testArrayWithString,
    testSplitWithSemicolonDelimiter, testSplitWithMultiCharDelimiter,
    testSplitWithEmptyDelimiter.

Signed-off-by: Kai Huang 

* [Analytics] Add SHA + LICENSE files for new bundled deps; spotless

The `dependencyLicenses` precommit task scans `licenses/` for a `.sha1`
sibling per bundled dependency. Two deps added in this PR were missing them:

  * `commons-text-1.11.0` in analytics-engine — needs sha1 + LICENSE +
    NOTICE (no shared `commons-text-*` license files yet in this plugin).
    Apache 2.0; LICENSE and NOTICE extracted from the released jar.
  * `jackson-datatype-jsr310-2.21.3` in arrow-flight-rpc — sha1 only.
    arrow-flight-rpc's `dependencyLicenses` already maps `jackson-.*` to
    the shared `jackson-LICENSE` / `jackson-NOTICE` files via
    `mapping from: /jackson-.*/, to: 'jackson'`, so no new license/notice
    files are needed.

Plus googleJavaFormat reflow on `ArraySliceAdapter` and `DataFusionPlugin`
that spotlessCheck flagged in precommit.

Verified `:plugins:arrow-flight-rpc:precommit`,
`:sandbox:plugins:analytics-engine:precommit`, and
`:sandbox:plugins:analytics-backend-datafusion:precommit` all succeed.

Addresses review feedback on #21554.

Signed-off-by: Kai Huang 

* [Analytics Engine] Map BigDecimal cells to FloatingPoint in row-codec inference

{@code RowResponseCodec.scalarArrowType} ordered its instanceof checks
{Long, Integer, Short, Byte, Double, Float, Boolean, CharSequence, byte[],
Number(fallback) → Int(64)}. BigDecimal extends {@link Number} but isn't any
of the typed scalar arms, so it fell through to the {@code Number} fallback
and got encoded as a 64-bit integer column — silently truncating fractional
digits.

This bites PPL flows whose common element type is {@code DECIMAL} (e.g.
{@code array(1, -1.5, 2, 1.0)} — the v2-side {@code ArrayImplementor.internalCast}
explicitly maps the DECIMAL target to BigDecimal cells). The element values
{@code -1.5} and {@code 1.0} round to {@code -1} and {@code 1} when forced
through Int(64), so the array reads back as {@code [1, -1, 2, 1]} instead of
{@code [1, -1.5, 2, 1.0]}.

Promote BigDecimal cells to FloatingPoint(DOUBLE) — same precision the v2
engine uses for decimal-typed PPL results, so behavior matches across both
execution paths. The list writer's {@code Float8Vector} arm already uses
{@code ((Number) element).doubleValue()}, which correctly extracts the
fractional value from a BigDecimal.

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Onboard PPL mvzip via custom Rust UDF

PPL `mvzip(left, right [, sep])` element-wise zips two arrays into a list of
strings, joined per pair by a separator (default `,`). DataFusion has no
stdlib equivalent — `array_concat` is end-to-end concatenation, and Substrait's
lambda support is too thin for a transform/zip rewrite — so this onboards a
custom Rust ScalarUDF on the analytics-backend-datafusion plugin's session
context and wires the Java side to route to it.

Templated shape (extends the existing pattern from convert_tz):

  Rust side:
    udf::mvzip::MvzipUdf — Signature::user_defined; coerce_types pins the
      first two args to ListArray and the optional 3rd to Utf8; invoke_with_args
      iterates per row, takes min(len(left), len(right)) elements, stringifies
      each (matching `Objects.toString(elem, "")` for null elements), and
      builds a List. Defensive Null-element-type arm handles the empty
      array case before the SQL-plugin VARCHAR-default kicks in.
    Registered on each session context via udf::register_all alongside
    convert_tz. 7 unit tests cover the basic / custom-sep / truncation /
    null-element / null-array / empty-array / numeric-array shapes.

  Java side:
    ScalarFunction.MVZIP enum entry (SqlKind.OTHER_FUNCTION; resolves through
      identifier-name valueOf("MVZIP") since PPL's MVZipFunctionImpl registers
      under the function name "mvzip").
    MvzipAdapter — locally-declared SqlFunction("mvzip") + ADDITIONAL_SCALAR_SIGS
      bridge so isthmus emits a Substrait scalar function call with the exact
      name the Rust UDF is registered under.
    DataFusionAnalyticsBackendPlugin: ARRAY_RETURNING_PROJECT_OPS membership
      (returns ARRAY, registered against FieldType.ARRAY); adapter
      registration in scalarFunctionAdapters().
    opensearch_array_functions.yaml: two impls for arity-2 and arity-3.

  * Before: 28/60.
  * After:  34/60.

  Newly passing — all 5 testMvzip* variants:
    testMvzipBasic, testMvzipWithCustomDelimiter, testMvzipNested,
    testMvzipWithEmptyArray, testMvzipWithBothEmptyArrays.

  (Test count delta is +6 because the test class also exercises mvzip in 1
  other test under a different name, picked up by the same fix.)

This PR's run also picks up the SQL-plugin companion #5421 which defaults
empty `array()` to ARRAY. Without that companion the testMvzipWith*EmptyArray
variants would still fail — substrait would reject the input ARRAY
type before reaching the UDF. The Rust UDF's Null-element arm exists as a
defensive backstop in case the call ever reaches it with a null-typed list.

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Onboard PPL mvfind via custom Rust UDF

PPL `mvfind(arr, regex)` finds the 0-based index of the first array element
matching a regex pattern (Java `Matcher.find` substring-match semantics), or
NULL if no match. DataFusion has no stdlib equivalent, and rewriting in terms
of array_position requires per-element regex evaluation that's only
expressible with substrait lambda support — out of scope here. Onboards a
custom Rust ScalarUDF on the analytics-backend-datafusion plugin's session
context, mirroring the mvzip/convert_tz pattern.

Templated shape:

  Rust side:
    udf::mvfind::MvfindUdf — Signature::user_defined; coerce_types pins arg 0
      to a list type and arg 1 to Utf8; invoke_with_args walks each row and
      finds the first non-null element whose stringified form matches the
      regex via Rust's `regex` crate (`Regex::is_match` is unanchored, same
      as Java's `Matcher.find`). Scalar pattern operands compile once up
      front and surface invalid-regex errors at plan time (mirrors the SQL
      plugin's plan-time `tryCompileLiteralPattern`); column-valued patterns
      compile per row and yield NULL for invalid patterns. Supports list
      element types Utf8 / Int{8,16,32,64} / UInt{8,16,32,64} / Float{32,64}
      / Boolean / Null. 7 unit tests cover the basic-match / no-match /
      null-array / empty-array / null-element / numeric-array / unanchored
      shapes.
    Registered on each session context via udf::register_all alongside
    convert_tz and mvzip.

  Java side:
    ScalarFunction.MVFIND enum entry (SqlKind.OTHER_FUNCTION; resolves
      through identifier-name valueOf("MVFIND") since PPL's
      MVFindFunctionImpl registers under the function name "mvfind").
    MvfindAdapter — locally-declared SqlFunction("mvfind") +
      ADDITIONAL_SCALAR_SIGS bridge so isthmus emits a Substrait scalar
      function call with the exact name the Rust UDF is registered under.
    DataFusionAnalyticsBackendPlugin: STANDARD_PROJECT_OPS membership
      (returns INTEGER, registered against the existing scalar
      SUPPORTED_FIELD_TYPES); adapter registration in
      scalarFunctionAdapters().
    opensearch_array_functions.yaml: arity-2 impl returning `i32?`.

  * Before: 34/60.
  * After:  42/60.

  Newly passing — 8 of 9 testMvfind* variants:
    testMvfindWithMatch, testMvfindWithFirstMatch, testMvfindWithMultipleMatches,
    testMvfindWithNoMatch, testMvfindWithEmptyArray, testMvfindWithNumericArray,
    testMvfindWithCaseInsensitive, testMvfindWithComplexRegex.

  Remaining mvfind failure:
    testMvfindWithDynamicRegex — fails with "Unable to convert call
    CONCAT(string, string)" because the test computes the pattern via
    `concat('ban', '.*')` and substrait can't bind the CONCAT call. This is a
    separate analytics-engine CONCAT type-conversion issue, not mvfind-specific.

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Onboard PPL mvappend via custom Rust UDF

PPL `mvappend(arg1, arg2, …)` flattens a mixed list of array and scalar
arguments into one array, dropping null arguments and null elements within
array arguments. DataFusion's `array_concat` is the closest stdlib match but
only accepts arrays (not mixed array+scalar) and preserves nulls — different
semantics. Onboards as a custom Rust ScalarUDF on the analytics-backend-datafusion
plugin's session context, mirroring the mvzip / mvfind pattern.

Templated shape:

  Rust side:
    udf::mvappend::MvappendUdf — Signature::user_defined; per-row walk over
      operands, skipping NULL args and NULL elements inside array args, with
      explicit Arrow type arms for {Int8/16/32/64, UInt8/16/32/64,
      Float32/64, Boolean, Utf8/LargeUtf8/Utf8View}. The string arms output
      List or List depending on the inferred element type so
      the result schema matches what `return_type` declared (DataFusion's
      execution-time schema check rejects mismatches). Defensive Null
      element-type arm covers the empty-array shape. 6 unit tests.
    Registered on each session context via udf::register_all.

  Java side:
    ScalarFunction.MVAPPEND enum entry (SqlKind.OTHER_FUNCTION; resolves
      through identifier-name valueOf("MVAPPEND")).
    MvappendAdapter — locally-declared SqlFunction("mvappend") +
      ADDITIONAL_SCALAR_SIGS bridge. Casts every scalar operand to the
      call's array component type and every array operand to
      ARRAY before substrait emission, so the UDF sees a
      single uniform element type across all positions.
    DataFusionAnalyticsBackendPlugin: ARRAY_RETURNING_PROJECT_OPS membership
      (returns ARRAY); adapter registration in
      scalarFunctionAdapters().
    opensearch_array_functions.yaml: variadic min:1 entry with `list`
      return type.

  * Before: 0/15.
  * After:  6/15.

  Newly passing:
    testMvappendWithMultipleElements, testMvappendWithSingleElement,
    testMvappendWithArrayFlattening, testMvappendWithStringValues,
    testMvappendWithNestedArrays, testMvappendWithRealFields.

  * 8 tests fail with "Unable to convert the type ANY". Root cause is
    PPL's MVAppendFunctionImpl.updateMostGeneralType using strict
    Object.equals on each pair of operand types, returning Calcite's
    ANY type when any two don't match — including when they only differ
    in nullability tag (a literal 3 is INTEGER NOT NULL but the
    component type of `array(1, 2)` is INTEGER NULLABLE). Substrait
    can't serialize ANY. The fix belongs in the SQL plugin's
    MVAppendFunctionImpl (use typeFactory.leastRestrictive instead of
    Object.equals) and isn't addressed here.
  * testMvappendInWhereClause — uses `where array_length(combined) = 2`
    which the analytics-engine planner rejects with "No backend can
    evaluate filter predicate [EQUALS] on fields [combined:ARRAY]".
    Filter-side capability gap unrelated to mvappend.
  * testMvappendWithComplexExpression — fails substrait conversion on
    a nested mvappend call ("Unable to convert call mvappend(list, …)"),
    likely the same nullability widening pattern flowing through nested
    calls. Same upstream fix applies.

  Unchanged at 43/60 — mvappend isn't exercised there.

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Reshape mvappend operands as uniform lists; add Decimal128 element support

Two follow-ons to the initial mvappend onboarding (40b216157b5), both surfaced
once the SQL companion #5424 (`MVAppendFunctionImpl.leastRestrictive`) let
homogeneous-type calls reach substrait conversion.

# Uniform-list operand reshape

Substrait's variadic-`any1` argument shape requires every operand at the same
variadic position to share a type. PPL's `mvappend(arg, …)` accepts a mix of
bare scalars and arrays, which substrait's signature matcher rejected with
`Unable to convert call mvappend(list, i32?, i32?)`.

`MvappendAdapter` now wraps each scalar operand in a singleton
`make_array(scalar)` call (using the locally-declared `MakeArrayAdapter.LOCAL_MAKE_ARRAY_OP`)
so by the time the substrait converter sees the operands they're uniformly
`list`. The yaml impl was correspondingly tightened from
`args: [{ value: any1 }] variadic` to `args: [{ value: list }] variadic`.

Rust UDF (`udf::mvappend`) keeps its scalar-handling branch intact as a
defensive fallback, but in practice every operand it sees is a list now.

# Decimal128 element type

Calcite's leastRestrictive widening on INT + DECIMAL produces DECIMAL(p, s)
which substrait converts to Decimal128(p, s); the Java adapter casts every
operand's element type to that. The Rust UDF needed an explicit
`DataType::Decimal128(p, s)` branch — Decimal128Builder requires
`.with_precision_and_scale(p, s)` configuration before use, and Decimal128Array
elements are read via the `i128`-valued `value(i)` accessor (not via the
generic `build!` macro).

# Pass-rate (CalciteMVAppendFunctionIT, force-routed, with companion #5424 applied)

  * Before this commit: 6/15 (initial mvappend onboarding).
  * After this commit:  10/15.

  Newly passing:
    testMvappendWithMixedArrayAndScalar (uniform-list reshape),
    testMvappendWithComplexExpression (uniform-list reshape),
    testMvappendWithIntAndDouble (Decimal128 element),
    testMvappendWithNumericArrays (Decimal128 element).

  Remaining 5 failures:
    * testMvappendWithMixedTypes / WithFieldsAndLiterals / WithEmptyArray /
      WithNull — call legitimately widens to ARRAY because operands
      contain pairs of types with no common widened type (INT + VARCHAR).
      The Calcite engine handles ANY via Object generic dispatch; substrait
      can't encode it. Out of scope without changing PPL UDF semantics.
    * testMvappendInWhereClause — uses `where array_length(combined) = 2`
      which the analytics-engine planner rejects with "No backend can
      evaluate filter predicate [EQUALS] on fields [combined:ARRAY]".
      Filter-side capability gap unrelated to mvappend.

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Register UDFs on FFM-created session contexts

create_session_context (the Rust-side builder behind df_create_session_context)
built a fresh DataFusion SessionContext but never called udf::register_all on
it. Every fragment query routed through df_execute_with_context reused that
handle's ctx via query_executor::execute_with_context, so substrait function
references to mvappend / mvfind / mvzip / convert_tz failed planning with
"This feature is not implemented: Unsupported function name". The matching
register_all call exists in execute_query / local_executor / indexed_executor
— this just brings the FFM session-context path to parity.

Verified: CalciteMVAppendFunctionIT against the analytics-engine route now
passes 10/15 (was 0/15) with the SQL companion #5424 widening fix applied.
The remaining 5 are pre-existing ARRAY/UNKNOWN substrait-encoding gaps
(heterogeneous mvappend signatures, empty-array default, filter-on-array
predicate) tracked in this PR's "What's left" section.

Signed-off-by: Kai Huang 

* [Analytics Backend / DataFusion] Don't let substrait AssertionError kill the cluster

Substrait's plan validators (VariadicParameterConsistencyValidator,
RelOptUtil.eq via Litmus.THROW, etc.) throw AssertionError directly via
explicit `throw new AssertionError(...)` rather than via the `assert`
keyword, so the JVM -da flag doesn't gate them. When a malformed plan
triggers one inside a search-thread call to SubstraitRelVisitor.apply,
the AssertionError propagates uncaught up the analytics-engine fragment
handler stack, OpenSearchUncaughtExceptionHandler classifies it as fatal,
and the entire cluster JVM exits.

Wrap the visitor.apply call in a narrow try/catch that re-raises the
AssertionError as IllegalStateException with the original message and
cause preserved. The analytics-engine error path already buckets
IllegalStateException at the fragment boundary into a normal HTTP 500
response — the cluster stays up and the failure shows in the per-query
report instead.

This came up while diagnosing CalciteMVAppendFunctionIT failures: malformed
ARRAY plans were taking down the cluster mid-test instead of producing
per-test failures, masking the underlying substrait conversion error.

Signed-off-by: Kai Huang 

* [QA] Add ArrayFunctionIT + MVAppendFunctionIT for analytics-engine REST path

Self-contained QA ITs in sandbox/qa/analytics-engine-rest exercising the
PPL collection functions onboarded in this PR through POST /_analytics/ppl
against a parquet-backed `calcs` dataset, no SQL plugin checkout required.

ArrayFunctionIT (22 tests):
  - array constructor (mixed-numeric BigDecimal → Double promotion + int+string)
  - array_length
  - mvindex range (array_slice — 0-based-(start, length) → 1-based-(start, end))
  - mvindex single (array_element via ITEM rename)
  - mvdedup (array_distinct)
  - mvjoin (array_to_string rename)
  - mvzip (Rust UDF, default + custom delimiter + nested)
  - mvfind (Rust UDF, match / no-match / dynamic regex via concat() Sig bridge)
  - split (returns array)

MVAppendFunctionIT (6 tests):
  - uniform-typed scalar variadic (multiple, single, string)
  - array operands (flattening, nested string arrays)
  - VARCHAR field references via real calcs row

Tests gated on SQL companion #5424 (testMvappendWith{IntAndDouble,
MixedArrayAndScalar, NumericArrays, ComplexExpression}) are intentionally
absent — they fail with "Unable to convert the type ANY" until
MVAppendFunctionImpl's leastRestrictive widening + DECIMAL→DOUBLE
promotion + operand pre-cast is published as unified-query-core. A
top-of-class block lists them with a pointer back to #5424.

Lambda-based functions (transform, mvmap, reduce, forall, exists, filter)
and empty-array operands are absent for the architectural reasons in this
PR's "What's left" section: substrait extension YAML doesn't support
declaring func<…> lambda-typed args, and array() defaults to ARRAY
which substrait can't encode without #5421.

Local verification (per `docs/dev/ppl-analytics-engine-routing.md` SOP):
- :sandbox:qa:analytics-engine-rest:integTest --tests "*ArrayFunctionIT" — 22/22
- :sandbox:qa:analytics-engine-rest:integTest --tests "*MVAppendFunctionIT" — 6/6
- :check -p sandbox — all 718 tasks green

Signed-off-by: Kai Huang 

---------

Signed-off-by: Kai Huang 
---
 plugins/arrow-flight-rpc/build.gradle         |   5 +
 .../jackson-datatype-jsr310-2.21.3.jar.sha1   |   1 +
 .../opensearch/analytics/spi/FieldType.java   |  12 +-
 .../analytics/spi/ScalarFunction.java         |  55 +-
 .../analytics-backend-datafusion/build.gradle |   5 +
 .../rust/Cargo.toml                           |   2 +
 .../rust/src/session_context.rs               |   4 +
 .../rust/src/udf/mod.rs                       |   8 +-
 .../rust/src/udf/mvappend.rs                  | 531 ++++++++++++++++++
 .../rust/src/udf/mvfind.rs                    | 366 ++++++++++++
 .../rust/src/udf/mvzip.rs                     | 412 ++++++++++++++
 .../be/datafusion/ArrayElementAdapter.java    |  85 +++
 .../be/datafusion/ArraySliceAdapter.java      | 113 ++++
 .../be/datafusion/ArrayToStringAdapter.java   |  45 ++
 .../DataFusionAnalyticsBackendPlugin.java     |  50 +-
 .../DataFusionFragmentConvertor.java          |  32 +-
 .../be/datafusion/DataFusionPlugin.java       |   3 +-
 .../be/datafusion/MakeArrayAdapter.java       |  89 +++
 .../be/datafusion/MvappendAdapter.java        |  97 ++++
 .../be/datafusion/MvfindAdapter.java          |  67 +++
 .../be/datafusion/MvzipAdapter.java           |  68 +++
 .../resources/opensearch_array_functions.yaml | 158 ++++++
 sandbox/plugins/analytics-engine/build.gradle |  11 +-
 .../licenses/commons-text-1.11.0.jar.sha1     |   2 +-
 .../analytics/exec/ArrowValues.java           |  21 +-
 .../exec/stage/ArrowSchemaFromCalcite.java    |  27 +-
 .../exec/stage/RowResponseCodec.java          | 139 ++++-
 .../analytics/qa/ArrayFunctionIT.java         | 311 ++++++++++
 .../analytics/qa/MVAppendFunctionIT.java      | 180 ++++++
 29 files changed, 2866 insertions(+), 33 deletions(-)
 create mode 100644 plugins/arrow-flight-rpc/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvappend.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvfind.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvzip.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrayElementAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArraySliceAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrayToStringAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MakeArrayAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvappendAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvfindAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvzipAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_array_functions.yaml
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ArrayFunctionIT.java
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MVAppendFunctionIT.java

diff --git a/plugins/arrow-flight-rpc/build.gradle b/plugins/arrow-flight-rpc/build.gradle
index a94c9301a4041..9e3a0b5dc3f98 100644
--- a/plugins/arrow-flight-rpc/build.gradle
+++ b/plugins/arrow-flight-rpc/build.gradle
@@ -36,6 +36,11 @@ dependencies {
   api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}"
+  // arrow-vector's JsonStringArrayList static-initializes a Jackson ObjectMapper that registers
+  // JavaTimeModule. Without jsr310 on arrow-flight-rpc's classpath, any reader of an Arrow
+  // ListVector (e.g. DataFusion's array-returning UDFs flowing through analytics-engine) hits
+  // a fatal NoClassDefFoundError that exits the JVM.
+  api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
 
   // arrow flight dependencies.
diff --git a/plugins/arrow-flight-rpc/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1 b/plugins/arrow-flight-rpc/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1
new file mode 100644
index 0000000000000..5bf925c777b5f
--- /dev/null
+++ b/plugins/arrow-flight-rpc/licenses/jackson-datatype-jsr310-2.21.3.jar.sha1
@@ -0,0 +1 @@
+a0958ebdaba836d31e5462ebc37b6349a0725ff9
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FieldType.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FieldType.java
index fa013789d5a79..2a6a68a076d09 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FieldType.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/FieldType.java
@@ -55,7 +55,16 @@ public enum FieldType {
     NESTED("nested"),
     OBJECT("object"),
     FLAT_OBJECT("flat_object"),
-    COMPLETION("completion");
+    COMPLETION("completion"),
+    /**
+     * Array-typed expression result. Used for the return-type slot of array-producing scalar
+     * functions (PPL {@code array(…)}, {@code array_slice}, {@code array_distinct}). Has no
+     * OpenSearch mapping equivalent — arrays in OpenSearch are multi-value fields with the
+     * underlying element type, not a separate type. The mapping string is {@code "array"} as a
+     * placeholder; {@link #fromMappingType} keeps working unchanged because no source
+     * advertises that mapping string.
+     */
+    ARRAY("array");
 
     private final String mappingType;
 
@@ -117,6 +126,7 @@ public static FieldType fromSqlTypeName(SqlTypeName sqlTypeName) {
             case TIME, TIMESTAMP, TIMESTAMP_WITH_LOCAL_TIME_ZONE -> FieldType.DATE;
             case BOOLEAN -> FieldType.BOOLEAN;
             case BINARY, VARBINARY -> FieldType.BINARY;
+            case ARRAY -> FieldType.ARRAY;
             default -> null;
         };
     }
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
index af6f15e84935b..60dd70f629018 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
@@ -177,7 +177,60 @@ public enum ScalarFunction {
     JSON_EXTEND(Category.SCALAR, SqlKind.OTHER_FUNCTION),
     JSON_EXTRACT(Category.SCALAR, SqlKind.OTHER_FUNCTION),
     JSON_KEYS(Category.SCALAR, SqlKind.OTHER_FUNCTION),
-    JSON_SET(Category.SCALAR, SqlKind.OTHER_FUNCTION);
+    JSON_SET(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+
+    // ── Array ────────────────────────────────────────────────────────
+    /**
+     * PPL {@code array(a, b, …)} constructor — resolves through the SQL plugin's
+     * {@code ArrayFunctionImpl} UDF named {@code "array"}. DataFusion's native
+     * equivalent is {@code make_array}, so a backend that supports this needs a
+     * name-mapping adapter (see {@code MakeArrayAdapter} in the DataFusion backend).
+     */
+    ARRAY(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    ARRAY_LENGTH(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    ARRAY_SLICE(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    ARRAY_DISTINCT(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    /**
+     * Calcite's {@code ARRAY_JOIN} — joins array elements with a separator. PPL
+     * {@code mvjoin} is registered to this operator. DataFusion's native equivalent
+     * is named {@code array_to_string}, so the DataFusion backend rewrites to that
+     * via a name-mapping adapter.
+     */
+    ARRAY_JOIN(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    /**
+     * Calcite's {@code SqlStdOperatorTable.ITEM} — element access ({@code arr[N]}).
+     * PPL's {@code mvindex(arr, N)} single-element form lowers through
+     * {@code MVIndexFunctionImp.resolveSingleElement} to ITEM with a 1-based index
+     * (already converted from PPL's 0-based input). DataFusion's native equivalent
+     * is {@code array_element}, also 1-based; the DataFusion backend renames via a
+     * name-mapping adapter.
+     */
+    ITEM(Category.SCALAR, SqlKind.ITEM),
+    /**
+     * PPL {@code mvzip(left, right [, sep])} — element-wise zip of two arrays into an
+     * array of strings, joined per pair by a separator (default {@code ","}). Resolves
+     * through the SQL plugin's {@code MVZipFunctionImpl} UDF named {@code "mvzip"}.
+     * No DataFusion stdlib equivalent — the analytics-backend-datafusion plugin ships
+     * a custom Rust UDF (`udf::mvzip`) registered on its session context.
+     */
+    MVZIP(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    /**
+     * PPL {@code mvfind(arr, regex)} — find the 0-based index of the first array
+     * element matching a regex, or NULL if no match. Resolves through the SQL
+     * plugin's {@code MVFindFunctionImpl} UDF named {@code "mvfind"}. No
+     * DataFusion stdlib equivalent — the analytics-backend-datafusion plugin
+     * ships a custom Rust UDF (`udf::mvfind`) registered on its session context.
+     */
+    MVFIND(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    /**
+     * PPL {@code mvappend(arg1, arg2, …)} — flatten a mixed list of array and
+     * scalar arguments into one array, dropping null args and null elements.
+     * Resolves through the SQL plugin's {@code MVAppendFunctionImpl} UDF named
+     * {@code "mvappend"}. DataFusion's {@code array_concat} only accepts arrays
+     * and preserves nulls, so the analytics-backend-datafusion plugin ships a
+     * custom Rust UDF ({@code udf::mvappend}) registered on its session context.
+     */
+    MVAPPEND(Category.SCALAR, SqlKind.OTHER_FUNCTION);
 
     /**
      * Category of scalar function.
diff --git a/sandbox/plugins/analytics-backend-datafusion/build.gradle b/sandbox/plugins/analytics-backend-datafusion/build.gradle
index 1d71f16082900..5e5175ac2a8f3 100644
--- a/sandbox/plugins/analytics-backend-datafusion/build.gradle
+++ b/sandbox/plugins/analytics-backend-datafusion/build.gradle
@@ -73,6 +73,11 @@ dependencies {
   implementation "io.substrait:isthmus:0.89.1"
   implementation "io.substrait:core:0.89.1"
   implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}"
+  // jackson-datatype-jsr310 — added to arrow-flight-rpc (the parent plugin that bundles
+  // arrow-vector). arrow-vector's JsonStringArrayList eagerly registers JavaTimeModule on
+  // its ObjectMapper, so jsr310 must be visible to arrow-vector's defining classloader,
+  // not this plugin's. compileOnly here would also work; runtime is provided by parent.
+  compileOnly "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
 
   calciteCompile "com.google.guava:guava:${versions.guava}"
   calciteTestCompile "com.google.guava:guava:${versions.guava}"
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
index ba0f7a0c0226a..93c7f0dea7265 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
@@ -68,6 +68,8 @@ serde_json = { workspace = true, features = ["preserve_order"] }
 # multi-path JSON-array output. Moving to 1.x is a follow-up once we can
 # reproduce that distinction against the new API surface.
 jsonpath-rust = "0.7"
+# mvfind UDF — regex matching against stringified array elements
+regex = "1.10"
 
 [dev-dependencies]
 criterion = { workspace = true }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
index c226deabb2fbe..cb5eea3112b07 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
@@ -111,6 +111,10 @@ pub async unsafe fn create_session_context(
         .build();
 
     let ctx = SessionContext::new_with_state(state);
+    // Register OpenSearch UDFs (mvappend, mvfind, mvzip, convert_tz, …) on this session
+    // so the substrait converter at execute_with_context can resolve their function names.
+    // Without this, fragment execution fails with "Unsupported function name" because
+    // df_execute_with_context reuses this handle's ctx instead of building a fresh one.
     crate::udf::register_all(&ctx);
 
     // Register default ListingTable for parquet scans
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
index cdbcb9db9bf83..66b4e20abc295 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
@@ -122,6 +122,9 @@ pub mod json_extend;
 pub mod json_extract;
 pub mod json_keys;
 pub mod json_set;
+pub mod mvappend;
+pub mod mvfind;
+pub mod mvzip;
 pub mod tonumber;
 pub mod tostring;
 
@@ -141,10 +144,13 @@ pub fn register_all(ctx: &SessionContext) {
     json_extract::register_all(ctx);
     json_keys::register_all(ctx);
     json_set::register_all(ctx);
+    mvzip::register_all(ctx);
+    mvfind::register_all(ctx);
+    mvappend::register_all(ctx);
     tonumber::register_all(ctx);
     tostring::register_all(ctx);
     log::info!(
-        "OpenSearch UDF register_all: convert_tz, json_append, json_array_length, json_delete, json_extend, json_extract, json_keys, json_set, tonumber, tostring registered"
+        "OpenSearch UDF register_all: convert_tz, json_append, json_array_length, json_delete, json_extend, json_extract, json_keys, json_set, mvzip, mvfind, mvappend, tonumber, tostring registered"
     );
 }
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvappend.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvappend.rs
new file mode 100644
index 0000000000000..a8e636de7bda3
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvappend.rs
@@ -0,0 +1,531 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `mvappend(arg1, arg2, …)` — flatten a mixed list of array and scalar args
+//! into a single array, dropping null args AND null elements inside array args.
+//!
+//! Mirrors PPL's [`MVAppendFunctionImpl`] / [`MVAppendCore`] semantics:
+//!
+//! * For each argument, in order:
+//!   * NULL argument → skipped entirely.
+//!   * Array argument → each non-null element is appended to the output.
+//!   * Scalar argument → appended as a single element.
+//! * Returns NULL if no non-null elements were collected (PPL convention —
+//!   distinguishes `mvappend(null)` from `mvappend()` from `mvappend([])`).
+//!
+//! ## Type homogeneity
+//!
+//! The Java adapter (`MvappendAdapter`) casts every scalar argument to the
+//! call's array component type and every array argument to
+//! `ARRAY` before this UDF runs, so by the time we see operands
+//! they share a single element type. The element-conversion macro below
+//! handles each supported scalar Arrow type explicitly; a list whose data
+//! vector type isn't covered surfaces as a planning error rather than a
+//! silent coercion.
+//!
+//! Mixed-type calls (`mvappend(1, 'text', 2.5)`) end up with Calcite type
+//! `ARRAY` which substrait doesn't have an encoding for — those fail at
+//! substrait conversion, before reaching this UDF, and aren't addressed here.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{
+    Array, ArrayRef, AsArray, BooleanArray, BooleanBuilder, Decimal128Array, Decimal128Builder,
+    Float32Array, Float32Builder, Float64Array, Float64Builder, GenericListArray, Int16Array,
+    Int16Builder, Int32Array, Int32Builder, Int64Array, Int64Builder, Int8Array, Int8Builder,
+    ListArray, ListBuilder, StringArray, StringBuilder, StringViewArray, StringViewBuilder,
+    UInt16Array, UInt16Builder, UInt32Array, UInt32Builder, UInt64Array, UInt64Builder,
+    UInt8Array, UInt8Builder,
+};
+use datafusion::arrow::datatypes::{DataType, Field};
+use datafusion::common::plan_err;
+use datafusion::error::{DataFusionError, Result};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(MvappendUdf::new()));
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct MvappendUdf {
+    signature: Signature,
+}
+
+impl MvappendUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl Default for MvappendUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ScalarUDFImpl for MvappendUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "mvappend"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.is_empty() {
+            return plan_err!("mvappend expects at least 1 argument, got 0");
+        }
+        // Java adapter pre-coerces every operand to either ARRAY or E for a
+        // single E. Use whichever element type we see first.
+        let element_type = element_type(arg_types)
+            .ok_or_else(|| DataFusionError::Plan("mvappend: unable to determine element type from operand types".to_string()))?;
+        Ok(DataType::List(Arc::new(Field::new(
+            "item",
+            element_type,
+            true,
+        ))))
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        if arg_types.is_empty() {
+            return plan_err!("mvappend expects at least 1 argument, got 0");
+        }
+        // Trust the Java adapter to have already coerced operands. coerce_types here
+        // exists only because Signature::user_defined demands an implementation; we
+        // pass each type through unchanged.
+        Ok(arg_types.to_vec())
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        let n = args.number_rows;
+        let element_type = element_type(&args.arg_fields.iter().map(|f| f.data_type().clone()).collect::>())
+            .ok_or_else(|| DataFusionError::Internal("mvappend: lost element type at invoke".to_string()))?;
+
+        // Materialize each operand as an ArrayRef whose Arrow type is either
+        // {element_type} or List. Scalar operands replicate to n rows.
+        let operand_arrays: Vec = args
+            .args
+            .iter()
+            .map(|a| a.clone().into_array(n))
+            .collect::>>()?;
+
+        macro_rules! build {
+            ($Builder:ty, $Scalar:ty, $List:ty) => {{
+                let inner = <$Builder>::new();
+                let mut builder = ListBuilder::new(inner);
+                for row in 0..n {
+                    let mut any_value = false;
+                    for arr in &operand_arrays {
+                        if arr.is_null(row) {
+                            continue;
+                        }
+                        if let Some(list_arr) = arr.as_any().downcast_ref::>() {
+                            // Iterate elements of the list at this row.
+                            let row_list = list_arr.value(row);
+                            let typed = row_list
+                                .as_any()
+                                .downcast_ref::<$Scalar>()
+                                .ok_or_else(|| DataFusionError::Internal(format!(
+                                    "mvappend: list element vector type mismatch ({:?})",
+                                    row_list.data_type()
+                                )))?;
+                            for i in 0..typed.len() {
+                                if !typed.is_null(i) {
+                                    builder.values().append_value(typed.value(i));
+                                    any_value = true;
+                                }
+                            }
+                        } else if let Some(typed) = arr.as_any().downcast_ref::<$Scalar>() {
+                            builder.values().append_value(typed.value(row));
+                            any_value = true;
+                        } else {
+                            return plan_err!(
+                                "mvappend: unexpected operand vector type {:?}",
+                                arr.data_type()
+                            );
+                        }
+                    }
+                    if any_value {
+                        builder.append(true);
+                    } else {
+                        builder.append_null();
+                    }
+                }
+                Arc::new(builder.finish()) as ArrayRef
+            }};
+        }
+
+        let result: ArrayRef = match &element_type {
+            DataType::Int8 => build!(Int8Builder, Int8Array, ListArray),
+            DataType::Int16 => build!(Int16Builder, Int16Array, ListArray),
+            DataType::Int32 => build!(Int32Builder, Int32Array, ListArray),
+            DataType::Int64 => build!(Int64Builder, Int64Array, ListArray),
+            DataType::UInt8 => build!(UInt8Builder, UInt8Array, ListArray),
+            DataType::UInt16 => build!(UInt16Builder, UInt16Array, ListArray),
+            DataType::UInt32 => build!(UInt32Builder, UInt32Array, ListArray),
+            DataType::UInt64 => build!(UInt64Builder, UInt64Array, ListArray),
+            DataType::Float32 => build!(Float32Builder, Float32Array, ListArray),
+            DataType::Float64 => build!(Float64Builder, Float64Array, ListArray),
+            DataType::Boolean => build!(BooleanBuilder, BooleanArray, ListArray),
+            // Decimal128 element type — needs a builder configured with the same precision
+            // and scale as the input. Calcite's leastRestrictive widening for INT + DECIMAL
+            // produces DECIMAL(p, s) which substrait converts to Decimal128(p, s); the Java
+            // adapter's CAST aligns every operand's element type to that.
+            DataType::Decimal128(precision, scale) => {
+                let inner = Decimal128Builder::new()
+                    .with_precision_and_scale(*precision, *scale)
+                    .map_err(|e| DataFusionError::Plan(format!("mvappend: decimal builder: {e}")))?;
+                let mut builder = ListBuilder::new(inner);
+                for row in 0..n {
+                    let mut any_value = false;
+                    for arr in &operand_arrays {
+                        if arr.is_null(row) {
+                            continue;
+                        }
+                        if let Some(list_arr) = arr.as_any().downcast_ref::>() {
+                            let row_list = list_arr.value(row);
+                            let typed = row_list
+                                .as_any()
+                                .downcast_ref::()
+                                .ok_or_else(|| DataFusionError::Internal(format!(
+                                    "mvappend: list element vector type mismatch ({:?})",
+                                    row_list.data_type()
+                                )))?;
+                            for i in 0..typed.len() {
+                                if !typed.is_null(i) {
+                                    builder.values().append_value(typed.value(i));
+                                    any_value = true;
+                                }
+                            }
+                        } else if let Some(typed) = arr.as_any().downcast_ref::() {
+                            builder.values().append_value(typed.value(row));
+                            any_value = true;
+                        } else {
+                            return plan_err!(
+                                "mvappend: unexpected operand vector type {:?}",
+                                arr.data_type()
+                            );
+                        }
+                    }
+                    if any_value {
+                        builder.append(true);
+                    } else {
+                        builder.append_null();
+                    }
+                }
+                Arc::new(builder.finish()) as ArrayRef
+            }
+            // String element types — handled specially because list children may be any of
+            // {Utf8, LargeUtf8, Utf8View} depending on whether the operand is a string literal,
+            // a field read (DataFusion's substrait consumer uses Utf8View for column reads in
+            // 52+), or a computed expression. The output type must match what {@code return_type}
+            // declared (which is whatever {@code element_type()} returned, driven by the first
+            // operand) — DataFusion validates the actual output schema against the declared
+            // schema and rejects mismatches with "column types must match schema types".
+            DataType::Utf8 | DataType::LargeUtf8 => {
+                let mut builder = ListBuilder::new(StringBuilder::new());
+                build_string_rows::(&operand_arrays, n, &mut builder)?;
+                Arc::new(builder.finish()) as ArrayRef
+            }
+            DataType::Utf8View => {
+                let mut builder = ListBuilder::new(StringViewBuilder::new());
+                build_string_rows::(&operand_arrays, n, &mut builder)?;
+                Arc::new(builder.finish()) as ArrayRef
+            }
+            other => {
+                return plan_err!("mvappend: unsupported element type {other:?}");
+            }
+        };
+
+        Ok(ColumnarValue::Array(result))
+    }
+}
+
+/// First operand type drives the element type. The Java adapter has already
+/// normalized everything to that single element type — either bare scalar or
+/// `List`. String element types are normalized to {@code Utf8} so the
+/// match arm in {@link MvappendUdf::invoke_with_args} catches every flavor —
+/// scalar literals come through as {@code Utf8}, field reads as {@code Utf8View},
+/// computed expressions as {@code Utf8} or {@code LargeUtf8}.
+fn element_type(arg_types: &[DataType]) -> Option {
+    arg_types.iter().find_map(|t| match t {
+        DataType::List(field) | DataType::LargeList(field) | DataType::FixedSizeList(field, _) => {
+            Some(field.data_type().clone())
+        }
+        DataType::Null => None,
+        other => Some(other.clone()),
+    })
+}
+
+/// Trait abstracting the difference between {@link StringBuilder} and
+/// {@link StringViewBuilder} when appending {@code &str} values. Both expose
+/// `append_value(&str)`, but they're concrete types with no shared trait, so
+/// this glue lets {@link build_string_rows} drive either via the same code.
+trait StrAppend {
+    fn append_str(&mut self, s: &str);
+}
+impl StrAppend for StringBuilder {
+    fn append_str(&mut self, s: &str) {
+        self.append_value(s);
+    }
+}
+impl StrAppend for StringViewBuilder {
+    fn append_str(&mut self, s: &str) {
+        self.append_value(s);
+    }
+}
+
+/// Generic per-row writer for string-typed mvappend output. Dispatches list-child
+/// downcasts across all three Utf8 flavors so the input doesn't need to match the
+/// output type.
+fn build_string_rows(
+    operand_arrays: &[ArrayRef],
+    n: usize,
+    builder: &mut ListBuilder,
+) -> Result<()> {
+    for row in 0..n {
+        let mut any_value = false;
+        for arr in operand_arrays {
+            if arr.is_null(row) {
+                continue;
+            }
+            if let Some(list_arr) = arr.as_any().downcast_ref::>() {
+                let row_list = list_arr.value(row);
+                append_string_elements(row_list.as_ref(), builder.values(), &mut any_value)?;
+            } else {
+                append_string_scalar(arr.as_ref(), row, builder.values(), &mut any_value)?;
+            }
+        }
+        if any_value {
+            builder.append(true);
+        } else {
+            builder.append_null();
+        }
+    }
+    Ok(())
+}
+
+/// Append all non-null string elements from a row's list to the output builder.
+/// Handles list children typed as Utf8, LargeUtf8, or Utf8View.
+fn append_string_elements(
+    row_list: &dyn Array,
+    out: &mut B,
+    any_value: &mut bool,
+) -> Result<()> {
+    if let Some(typed) = row_list.as_any().downcast_ref::() {
+        for i in 0..typed.len() {
+            if !typed.is_null(i) {
+                out.append_str(typed.value(i));
+                *any_value = true;
+            }
+        }
+        return Ok(());
+    }
+    if let Some(typed) = row_list.as_any().downcast_ref::() {
+        for i in 0..typed.len() {
+            if !typed.is_null(i) {
+                out.append_str(typed.value(i));
+                *any_value = true;
+            }
+        }
+        return Ok(());
+    }
+    if let Some(large) = row_list.as_string_opt::() {
+        for i in 0..large.len() {
+            if !large.is_null(i) {
+                out.append_str(large.value(i));
+                *any_value = true;
+            }
+        }
+        return Ok(());
+    }
+    plan_err!(
+        "mvappend: list element vector type mismatch — expected string, got {:?}",
+        row_list.data_type()
+    )
+}
+
+/// Append a single scalar string operand at the given row to the output builder.
+/// Handles operands typed as Utf8, LargeUtf8, or Utf8View.
+fn append_string_scalar(
+    arr: &dyn Array,
+    row: usize,
+    out: &mut B,
+    any_value: &mut bool,
+) -> Result<()> {
+    if let Some(typed) = arr.as_any().downcast_ref::() {
+        out.append_str(typed.value(row));
+        *any_value = true;
+        return Ok(());
+    }
+    if let Some(typed) = arr.as_any().downcast_ref::() {
+        out.append_str(typed.value(row));
+        *any_value = true;
+        return Ok(());
+    }
+    if let Some(large) = arr.as_string_opt::() {
+        out.append_str(large.value(row));
+        *any_value = true;
+        return Ok(());
+    }
+    plan_err!(
+        "mvappend: scalar operand vector type mismatch — expected string, got {:?}",
+        arr.data_type()
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use datafusion::arrow::array::AsArray;
+    use datafusion::common::ScalarValue;
+
+    fn run(args: Vec, n: usize) -> ArrayRef {
+        let arg_fields: Vec> = args
+            .iter()
+            .enumerate()
+            .map(|(i, cv)| {
+                let dt = match cv {
+                    ColumnarValue::Array(a) => a.data_type().clone(),
+                    ColumnarValue::Scalar(sv) => sv.data_type(),
+                };
+                Arc::new(Field::new(format!("a{i}"), dt, true))
+            })
+            .collect();
+        let return_field = Arc::new(Field::new(
+            "out",
+            DataType::List(Arc::new(Field::new("item", DataType::Int32, true))),
+            true,
+        ));
+        let result = MvappendUdf::new()
+            .invoke_with_args(ScalarFunctionArgs {
+                args,
+                arg_fields,
+                number_rows: n,
+                return_field,
+                config_options: Arc::new(datafusion::config::ConfigOptions::default()),
+            })
+            .unwrap();
+        match result {
+            ColumnarValue::Array(a) => a,
+            ColumnarValue::Scalar(_) => panic!("expected array"),
+        }
+    }
+
+    fn list_of_ints(rows: &[Option<&[Option]>]) -> ArrayRef {
+        let mut builder = ListBuilder::new(Int32Builder::new());
+        for row in rows {
+            match row {
+                None => builder.append_null(),
+                Some(elems) => {
+                    for e in *elems {
+                        match e {
+                            Some(v) => builder.values().append_value(*v),
+                            None => builder.values().append_null(),
+                        }
+                    }
+                    builder.append(true);
+                }
+            }
+        }
+        Arc::new(builder.finish())
+    }
+
+    fn extract_row_ints(arr: &ArrayRef, row: usize) -> Vec {
+        let list = arr.as_any().downcast_ref::().unwrap();
+        if list.is_null(row) {
+            return vec![];
+        }
+        let inner = list.value(row);
+        let typed = inner.as_primitive::();
+        (0..typed.len()).filter(|i| !typed.is_null(*i)).map(|i| typed.value(i)).collect()
+    }
+
+    #[test]
+    fn three_scalar_ints() {
+        // mvappend(1, 2, 3) → [1, 2, 3]
+        let args = vec![
+            ColumnarValue::Scalar(ScalarValue::Int32(Some(1))),
+            ColumnarValue::Scalar(ScalarValue::Int32(Some(2))),
+            ColumnarValue::Scalar(ScalarValue::Int32(Some(3))),
+        ];
+        let result = run(args, 1);
+        assert_eq!(extract_row_ints(&result, 0), vec![1, 2, 3]);
+    }
+
+    #[test]
+    fn flattens_array_argument() {
+        // mvappend([1, 2], 3) → [1, 2, 3]
+        let arr = list_of_ints(&[Some(&[Some(1), Some(2)])]);
+        let args = vec![
+            ColumnarValue::Array(arr),
+            ColumnarValue::Scalar(ScalarValue::Int32(Some(3))),
+        ];
+        let result = run(args, 1);
+        assert_eq!(extract_row_ints(&result, 0), vec![1, 2, 3]);
+    }
+
+    #[test]
+    fn drops_null_arg() {
+        // mvappend(NULL, 1, 2) → [1, 2]
+        let args = vec![
+            ColumnarValue::Scalar(ScalarValue::Int32(None)),
+            ColumnarValue::Scalar(ScalarValue::Int32(Some(1))),
+            ColumnarValue::Scalar(ScalarValue::Int32(Some(2))),
+        ];
+        let result = run(args, 1);
+        assert_eq!(extract_row_ints(&result, 0), vec![1, 2]);
+    }
+
+    #[test]
+    fn drops_null_elements_inside_array() {
+        // mvappend([1, NULL, 2], 3) → [1, 2, 3]
+        let arr = list_of_ints(&[Some(&[Some(1), None, Some(2)])]);
+        let args = vec![
+            ColumnarValue::Array(arr),
+            ColumnarValue::Scalar(ScalarValue::Int32(Some(3))),
+        ];
+        let result = run(args, 1);
+        assert_eq!(extract_row_ints(&result, 0), vec![1, 2, 3]);
+    }
+
+    #[test]
+    fn all_null_args_yield_null_row() {
+        // mvappend(NULL, NULL) → NULL
+        let args = vec![
+            ColumnarValue::Scalar(ScalarValue::Int32(None)),
+            ColumnarValue::Scalar(ScalarValue::Int32(None)),
+        ];
+        let result = run(args, 1);
+        let list = result.as_any().downcast_ref::().unwrap();
+        assert!(list.is_null(0));
+    }
+
+    #[test]
+    fn empty_array_in_args_contributes_nothing() {
+        // mvappend([], 1) → [1]
+        let arr = list_of_ints(&[Some(&[])]);
+        let args = vec![
+            ColumnarValue::Array(arr),
+            ColumnarValue::Scalar(ScalarValue::Int32(Some(1))),
+        ];
+        let result = run(args, 1);
+        assert_eq!(extract_row_ints(&result, 0), vec![1]);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvfind.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvfind.rs
new file mode 100644
index 0000000000000..767f949c8c4f2
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvfind.rs
@@ -0,0 +1,366 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `mvfind(arr, regex)` — find the 0-based index of the first array element
+//! matching a regex, or NULL if no match.
+//!
+//! Mirrors PPL's [`MVFindFunctionImpl`] semantics:
+//!
+//! * Per-element regex match (Java's `Matcher.find` semantics — substring
+//!   match, not full-string anchored). The Rust `regex` crate's
+//!   `Regex::is_match` matches the same way (unanchored).
+//! * NULL element → skipped (continues to next element).
+//! * NULL array or NULL pattern → NULL result.
+//! * Empty array → NULL result (no element to match).
+//! * Returns Int32 — PPL's surface is `Integer`.
+//! * Result type is consistent with the YAML declaration in
+//!   `opensearch_array_functions.yaml`.
+//!
+//! # Pattern compilation strategy
+//!
+//! When the pattern operand is a non-NULL Utf8 scalar literal we compile the
+//! regex once up front (mirrors the SQL plugin's `tryCompileLiteralPattern`
+//! plan-time optimization). Column-valued patterns are compiled per row;
+//! invalid patterns yield NULL for that row (per PPL spec, dynamic-pattern
+//! errors are non-fatal — bad rows just produce NULL).
+
+use std::any::Any;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{
+    Array, ArrayRef, AsArray, BooleanArray, Float32Array, Float64Array, GenericListArray,
+    Int16Array, Int32Array, Int32Builder, Int64Array, Int8Array, ListArray, StringArray,
+    UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+};
+use datafusion::arrow::datatypes::DataType;
+use datafusion::common::{plan_err, ScalarValue};
+use datafusion::error::{DataFusionError, Result};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+use regex::Regex;
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(MvfindUdf::new()));
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct MvfindUdf {
+    signature: Signature,
+}
+
+impl MvfindUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl Default for MvfindUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ScalarUDFImpl for MvfindUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "mvfind"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.len() != 2 {
+            return plan_err!("mvfind expects 2 arguments, got {}", arg_types.len());
+        }
+        Ok(DataType::Int32)
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        if arg_types.len() != 2 {
+            return plan_err!("mvfind expects 2 arguments, got {}", arg_types.len());
+        }
+        if !matches!(
+            &arg_types[0],
+            DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _)
+        ) {
+            return plan_err!("mvfind: arg 0 expected list type, got {:?}", arg_types[0]);
+        }
+        let pattern_t = match &arg_types[1] {
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => DataType::Utf8,
+            other => return plan_err!("mvfind: arg 1 expected string, got {other:?}"),
+        };
+        Ok(vec![arg_types[0].clone(), pattern_t])
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() != 2 {
+            return plan_err!("mvfind expects 2 arguments, got {}", args.args.len());
+        }
+        let n = args.number_rows;
+
+        // Fast path: pattern is a Utf8 scalar literal — compile once.
+        let scalar_regex: Option = if let ColumnarValue::Scalar(ScalarValue::Utf8(Some(p)))
+        | ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(p)))
+        | ColumnarValue::Scalar(ScalarValue::Utf8View(Some(p))) = &args.args[1]
+        {
+            // Plan-time invalid pattern → planning error so users see it instantly.
+            // Mirrors the SQL plugin's IllegalArgumentException for invalid literal regex.
+            match Regex::new(p) {
+                Ok(r) => Some(r),
+                Err(e) => return plan_err!("mvfind: invalid regex pattern '{p}': {e}"),
+            }
+        } else {
+            None
+        };
+
+        let arr_arr = args.args[0].clone().into_array(n)?;
+        let list = arr_arr.as_any().downcast_ref::>().ok_or_else(|| {
+            DataFusionError::Internal(format!(
+                "mvfind: expected ListArray, got {:?}",
+                arr_arr.data_type()
+            ))
+        })?;
+
+        // Materialize a column-valued pattern up front; for scalar patterns we keep
+        // the pre-compiled regex.
+        let pattern_arr_ref: Option = if scalar_regex.is_none() {
+            Some(args.args[1].clone().into_array(n)?)
+        } else {
+            None
+        };
+        let pattern_arr: Option<&StringArray> = pattern_arr_ref
+            .as_ref()
+            .and_then(|a| a.as_any().downcast_ref::());
+
+        let mut builder = Int32Builder::with_capacity(n);
+        for i in 0..n {
+            if list.is_null(i) {
+                builder.append_null();
+                continue;
+            }
+            // Per-row regex (compile if column-valued; reuse the scalar compile otherwise).
+            let regex_for_row: Option = match (&scalar_regex, pattern_arr) {
+                (Some(r), _) => Some(r.clone()),
+                (None, Some(arr)) if !arr.is_null(i) => Regex::new(arr.value(i)).ok(),
+                _ => None,
+            };
+            let regex = match regex_for_row {
+                Some(r) => r,
+                None => {
+                    builder.append_null();
+                    continue;
+                }
+            };
+            let row = list.value(i);
+            match find_first_match(row.as_ref(), ®ex) {
+                Some(idx) => builder.append_value(idx),
+                None => builder.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))
+    }
+}
+
+/// Walk an Arrow array of any supported scalar type, return the 0-based index
+/// of the first non-null element whose stringified form matches `regex`.
+/// Returns None if no element matches.
+fn find_first_match(arr: &dyn Array, regex: &Regex) -> Option {
+    let n = arr.len();
+    macro_rules! scan {
+        ($A:ty, $fmt:expr) => {{
+            let typed = arr.as_any().downcast_ref::<$A>()?;
+            for i in 0..n {
+                if typed.is_null(i) {
+                    continue;
+                }
+                if regex.is_match(&$fmt(typed.value(i))) {
+                    return Some(i as i32);
+                }
+            }
+            None
+        }};
+    }
+    match arr.data_type() {
+        DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
+            // String children may arrive as any of the three Utf8 flavors.
+            if let Some(typed) = arr.as_string_opt::() {
+                for i in 0..n {
+                    if typed.is_null(i) {
+                        continue;
+                    }
+                    if regex.is_match(typed.value(i)) {
+                        return Some(i as i32);
+                    }
+                }
+                None
+            } else {
+                let large = arr.as_string_opt::()?;
+                for i in 0..n {
+                    if large.is_null(i) {
+                        continue;
+                    }
+                    if regex.is_match(large.value(i)) {
+                        return Some(i as i32);
+                    }
+                }
+                None
+            }
+        }
+        DataType::Int8 => scan!(Int8Array, |v: i8| v.to_string()),
+        DataType::Int16 => scan!(Int16Array, |v: i16| v.to_string()),
+        DataType::Int32 => scan!(Int32Array, |v: i32| v.to_string()),
+        DataType::Int64 => scan!(Int64Array, |v: i64| v.to_string()),
+        DataType::UInt8 => scan!(UInt8Array, |v: u8| v.to_string()),
+        DataType::UInt16 => scan!(UInt16Array, |v: u16| v.to_string()),
+        DataType::UInt32 => scan!(UInt32Array, |v: u32| v.to_string()),
+        DataType::UInt64 => scan!(UInt64Array, |v: u64| v.to_string()),
+        DataType::Float32 => scan!(Float32Array, |v: f32| v.to_string()),
+        DataType::Float64 => scan!(Float64Array, |v: f64| v.to_string()),
+        DataType::Boolean => scan!(BooleanArray, |v: bool| v.to_string()),
+        DataType::Null => None,
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use datafusion::arrow::array::{Int32Builder, ListBuilder, StringBuilder};
+    use datafusion::arrow::datatypes::Field;
+
+    fn run(arr: ArrayRef, pattern: &str) -> ArrayRef {
+        let n = arr.len();
+        let return_field = Arc::new(Field::new("out", DataType::Int32, true));
+        let arg_fields: Vec> = vec![
+            Arc::new(Field::new("a", arr.data_type().clone(), true)),
+            Arc::new(Field::new("p", DataType::Utf8, true)),
+        ];
+        let result = MvfindUdf::new()
+            .invoke_with_args(ScalarFunctionArgs {
+                args: vec![
+                    ColumnarValue::Array(arr),
+                    ColumnarValue::Scalar(ScalarValue::Utf8(Some(pattern.to_string()))),
+                ],
+                arg_fields,
+                number_rows: n,
+                return_field,
+                config_options: Arc::new(datafusion::config::ConfigOptions::default()),
+            })
+            .unwrap();
+        match result {
+            ColumnarValue::Array(a) => a,
+            ColumnarValue::Scalar(_) => panic!("expected array"),
+        }
+    }
+
+    fn list_of_strings(rows: &[Option<&[Option<&str>]>]) -> ArrayRef {
+        let mut builder = ListBuilder::new(StringBuilder::new());
+        for row in rows {
+            match row {
+                None => builder.append_null(),
+                Some(elems) => {
+                    for e in *elems {
+                        match e {
+                            Some(s) => builder.values().append_value(s),
+                            None => builder.values().append_null(),
+                        }
+                    }
+                    builder.append(true);
+                }
+            }
+        }
+        Arc::new(builder.finish())
+    }
+
+    fn list_of_ints(rows: &[Option<&[Option]>]) -> ArrayRef {
+        let mut builder = ListBuilder::new(Int32Builder::new());
+        for row in rows {
+            match row {
+                None => builder.append_null(),
+                Some(elems) => {
+                    for e in *elems {
+                        match e {
+                            Some(v) => builder.values().append_value(*v),
+                            None => builder.values().append_null(),
+                        }
+                    }
+                    builder.append(true);
+                }
+            }
+        }
+        Arc::new(builder.finish())
+    }
+
+    fn int_value(arr: &ArrayRef, row: usize) -> Option {
+        let typed = arr.as_any().downcast_ref::().unwrap();
+        if typed.is_null(row) {
+            None
+        } else {
+            Some(typed.value(row))
+        }
+    }
+
+    #[test]
+    fn first_match_returns_zero_based_index() {
+        let arr = list_of_strings(&[Some(&[Some("apple"), Some("banana"), Some("apricot")])]);
+        let result = run(arr, "ban.*");
+        assert_eq!(int_value(&result, 0), Some(1));
+    }
+
+    #[test]
+    fn no_match_returns_null() {
+        let arr = list_of_strings(&[Some(&[Some("apple"), Some("banana")])]);
+        let result = run(arr, "kiwi");
+        assert_eq!(int_value(&result, 0), None);
+    }
+
+    #[test]
+    fn null_array_returns_null() {
+        let arr = list_of_strings(&[None]);
+        let result = run(arr, "any");
+        assert_eq!(int_value(&result, 0), None);
+    }
+
+    #[test]
+    fn empty_array_returns_null() {
+        let arr = list_of_strings(&[Some(&[])]);
+        let result = run(arr, "any");
+        assert_eq!(int_value(&result, 0), None);
+    }
+
+    #[test]
+    fn null_element_skipped_index_still_zero_based() {
+        let arr = list_of_strings(&[Some(&[None, Some("banana")])]);
+        let result = run(arr, "ban.*");
+        assert_eq!(int_value(&result, 0), Some(1));
+    }
+
+    #[test]
+    fn integer_array_stringified_for_regex() {
+        let arr = list_of_ints(&[Some(&[Some(10), Some(20), Some(30)])]);
+        let result = run(arr, "^2");
+        assert_eq!(int_value(&result, 0), Some(1));
+    }
+
+    #[test]
+    fn substring_match_semantics_are_unanchored() {
+        // Java's Matcher.find: "banana" matches /an/ (unanchored).
+        let arr = list_of_strings(&[Some(&[Some("apple"), Some("banana")])]);
+        let result = run(arr, "an");
+        assert_eq!(int_value(&result, 0), Some(1));
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvzip.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvzip.rs
new file mode 100644
index 0000000000000..a0c8466be835e
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mvzip.rs
@@ -0,0 +1,412 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `mvzip(left, right [, separator])` — element-wise zip of two arrays into an
+//! array of strings, joined per pair by a separator (default `,`).
+//!
+//! Mirrors PPL's [`MVZipFunctionImpl`] semantics:
+//!
+//! * Result length is `min(len(left), len(right))` (Python-`zip` truncation).
+//! * Either array NULL → NULL result.
+//! * Element NULLs are rendered as empty strings (matches the SQL plugin's
+//!   `Objects.toString(elem, "")`), so `mvzip([1, NULL], ["a", "b"])` yields
+//!   `["1,a", ",b"]`.
+//! * The separator is a Utf8 scalar. Calling it as a column would require
+//!   per-row materialization; PPL's surface only exposes a literal so we
+//!   constrain to scalars here and produce a planning error otherwise.
+//!
+//! Result type is `List` regardless of the input element types — `mvzip`
+//! is fundamentally a string-formatting operation. The Java side relies on the
+//! `opensearch_array_functions.yaml` declaration to type the call before
+//! substrait emission.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use datafusion::arrow::array::{
+    Array, ArrayRef, AsArray, BooleanArray, Float32Array, Float64Array, GenericListArray,
+    Int16Array, Int32Array, Int64Array, Int8Array, ListArray, ListBuilder, StringArray,
+    StringBuilder, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+};
+use datafusion::arrow::datatypes::{DataType, Field};
+use datafusion::common::{plan_err, ScalarValue};
+use datafusion::error::{DataFusionError, Result};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(MvzipUdf::new()));
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct MvzipUdf {
+    signature: Signature,
+}
+
+impl MvzipUdf {
+    pub fn new() -> Self {
+        // user_defined lets us accept ListArray with any element type and a
+        // string separator without enumerating every concrete combination.
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl Default for MvzipUdf {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ScalarUDFImpl for MvzipUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "mvzip"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.len() != 2 && arg_types.len() != 3 {
+            return plan_err!("mvzip expects 2 or 3 arguments, got {}", arg_types.len());
+        }
+        Ok(DataType::List(Arc::new(Field::new(
+            "item",
+            DataType::Utf8,
+            true,
+        ))))
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        if arg_types.len() != 2 && arg_types.len() != 3 {
+            return plan_err!("mvzip expects 2 or 3 arguments, got {}", arg_types.len());
+        }
+        for (i, t) in arg_types.iter().take(2).enumerate() {
+            if !matches!(t, DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _)) {
+                return plan_err!("mvzip: arg {i} expected list type, got {t:?}");
+            }
+        }
+        let mut coerced: Vec = arg_types[..2].to_vec();
+        if arg_types.len() == 3 {
+            match &arg_types[2] {
+                DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
+                    coerced.push(DataType::Utf8)
+                }
+                other => return plan_err!("mvzip: arg 2 (separator) expected string, got {other:?}"),
+            }
+        }
+        Ok(coerced)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() != 2 && args.args.len() != 3 {
+            return plan_err!("mvzip expects 2 or 3 arguments, got {}", args.args.len());
+        }
+        let n = args.number_rows;
+
+        // Materialize left/right as ListArrays. Scalar list operands are unusual but
+        // ColumnarValue::into_array handles them by replicating to {row count}.
+        let left_arr = args.args[0].clone().into_array(n)?;
+        let right_arr = args.args[1].clone().into_array(n)?;
+
+        let separator: String = if args.args.len() == 3 {
+            scalar_string(&args.args[2])
+                .ok_or_else(|| {
+                    DataFusionError::Plan(
+                        "mvzip: separator must be a non-NULL string scalar literal".to_string(),
+                    )
+                })?
+                .to_string()
+        } else {
+            ",".to_string()
+        };
+
+        let left = downcast_list(&left_arr, "left")?;
+        let right = downcast_list(&right_arr, "right")?;
+
+        // Output: List, one row per input row.
+        let mut builder = ListBuilder::new(StringBuilder::new());
+        for i in 0..n {
+            if left.is_null(i) || right.is_null(i) {
+                builder.append_null();
+                continue;
+            }
+            let left_row = left.value(i);
+            let right_row = right.value(i);
+            let take = left_row.len().min(right_row.len());
+            let left_strs = elements_as_strings(left_row.as_ref())?;
+            let right_strs = elements_as_strings(right_row.as_ref())?;
+            for j in 0..take {
+                let l = left_strs[j].as_deref().unwrap_or("");
+                let r = right_strs[j].as_deref().unwrap_or("");
+                let mut joined =
+                    String::with_capacity(l.len() + separator.len() + r.len());
+                joined.push_str(l);
+                joined.push_str(&separator);
+                joined.push_str(r);
+                builder.values().append_value(joined);
+            }
+            builder.append(true);
+        }
+        Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))
+    }
+}
+
+/// Extract a non-NULL Utf8 scalar literal, or return None for everything else
+/// (NULL scalar, column-valued, or non-string types).
+fn scalar_string(cv: &ColumnarValue) -> Option<&str> {
+    if let ColumnarValue::Scalar(sv) = cv {
+        match sv {
+            ScalarValue::Utf8(Some(s))
+            | ScalarValue::LargeUtf8(Some(s))
+            | ScalarValue::Utf8View(Some(s)) => Some(s.as_str()),
+            _ => None,
+        }
+    } else {
+        None
+    }
+}
+
+fn downcast_list<'a>(arr: &'a ArrayRef, slot: &str) -> Result<&'a ListArray> {
+    arr.as_any().downcast_ref::>().ok_or_else(|| {
+        DataFusionError::Internal(format!(
+            "mvzip: {slot} expected ListArray, got {:?}",
+            arr.data_type()
+        ))
+    })
+}
+
+/// Convert each element of an Arrow array of any supported scalar type to its
+/// canonical string form (matching `Objects.toString(elem, "")` semantics on
+/// the SQL plugin side — NULL elements become None which the caller renders as
+/// the empty string).
+fn elements_as_strings(arr: &dyn Array) -> Result>> {
+    let n = arr.len();
+    let mut out: Vec> = Vec::with_capacity(n);
+    macro_rules! collect {
+        ($A:ty, $fmt:expr) => {{
+            let typed = arr.as_any().downcast_ref::<$A>().ok_or_else(|| {
+                DataFusionError::Internal(format!(
+                    "mvzip: failed to downcast element vector to {}",
+                    stringify!($A)
+                ))
+            })?;
+            for i in 0..n {
+                if typed.is_null(i) {
+                    out.push(None);
+                } else {
+                    out.push(Some($fmt(typed.value(i))));
+                }
+            }
+        }};
+    }
+    match arr.data_type() {
+        DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
+            // String children may arrive as any of the three Utf8 flavors; AsArray
+            // handles the dispatch for us.
+            let s = arr.as_string_opt::();
+            if let Some(typed) = s {
+                for i in 0..n {
+                    if typed.is_null(i) {
+                        out.push(None);
+                    } else {
+                        out.push(Some(typed.value(i).to_string()));
+                    }
+                }
+            } else {
+                let large = arr.as_string_opt::().ok_or_else(|| {
+                    DataFusionError::Internal(format!(
+                        "mvzip: string element downcast failed for {:?}",
+                        arr.data_type()
+                    ))
+                })?;
+                for i in 0..n {
+                    if large.is_null(i) {
+                        out.push(None);
+                    } else {
+                        out.push(Some(large.value(i).to_string()));
+                    }
+                }
+            }
+        }
+        DataType::Int8 => collect!(Int8Array, |v: i8| v.to_string()),
+        DataType::Int16 => collect!(Int16Array, |v: i16| v.to_string()),
+        DataType::Int32 => collect!(Int32Array, |v: i32| v.to_string()),
+        DataType::Int64 => collect!(Int64Array, |v: i64| v.to_string()),
+        DataType::UInt8 => collect!(UInt8Array, |v: u8| v.to_string()),
+        DataType::UInt16 => collect!(UInt16Array, |v: u16| v.to_string()),
+        DataType::UInt32 => collect!(UInt32Array, |v: u32| v.to_string()),
+        DataType::UInt64 => collect!(UInt64Array, |v: u64| v.to_string()),
+        DataType::Float32 => collect!(Float32Array, |v: f32| v.to_string()),
+        DataType::Float64 => collect!(Float64Array, |v: f64| v.to_string()),
+        DataType::Boolean => collect!(BooleanArray, |v: bool| v.to_string()),
+        DataType::Null => {
+            // Element vector with Null type — every cell is NULL by definition, so
+            // emit None for each. Reachable when the input list is empty and its
+            // declared element type is Null/UNKNOWN (e.g. PPL `array()` no-arg
+            // before the SQL-plugin VARCHAR-default kicks in).
+            for _ in 0..n {
+                out.push(None);
+            }
+        }
+        other => {
+            return Err(DataFusionError::NotImplemented(format!(
+                "mvzip: unsupported list element type {other:?}"
+            )));
+        }
+    }
+    Ok(out)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use datafusion::arrow::array::Int32Builder;
+    use datafusion::arrow::array::StringBuilder as ArrowStringBuilder;
+
+    fn run(left: ArrayRef, right: ArrayRef, sep: Option<&str>) -> ArrayRef {
+        let mut args = vec![ColumnarValue::Array(left.clone()), ColumnarValue::Array(right.clone())];
+        if let Some(s) = sep {
+            args.push(ColumnarValue::Scalar(ScalarValue::Utf8(Some(s.to_string()))));
+        }
+        let n = left.len();
+        let return_field = Arc::new(Field::new(
+            "out",
+            DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
+            true,
+        ));
+        let arg_fields: Vec> = args
+            .iter()
+            .enumerate()
+            .map(|(i, _)| Arc::new(Field::new(format!("a{i}"), DataType::Utf8, true)))
+            .collect();
+        let result = MvzipUdf::new()
+            .invoke_with_args(ScalarFunctionArgs {
+                args,
+                arg_fields,
+                number_rows: n,
+                return_field,
+                config_options: Arc::new(datafusion::config::ConfigOptions::default()),
+            })
+            .unwrap();
+        match result {
+            ColumnarValue::Array(a) => a,
+            ColumnarValue::Scalar(_) => panic!("expected array"),
+        }
+    }
+
+    fn list_of_strings(rows: &[Option<&[Option<&str>]>]) -> ArrayRef {
+        let mut builder = ListBuilder::new(ArrowStringBuilder::new());
+        for row in rows {
+            match row {
+                None => builder.append_null(),
+                Some(elems) => {
+                    for e in *elems {
+                        match e {
+                            Some(s) => builder.values().append_value(s),
+                            None => builder.values().append_null(),
+                        }
+                    }
+                    builder.append(true);
+                }
+            }
+        }
+        Arc::new(builder.finish())
+    }
+
+    fn list_of_ints(rows: &[Option<&[Option]>]) -> ArrayRef {
+        let mut builder = ListBuilder::new(Int32Builder::new());
+        for row in rows {
+            match row {
+                None => builder.append_null(),
+                Some(elems) => {
+                    for e in *elems {
+                        match e {
+                            Some(v) => builder.values().append_value(*v),
+                            None => builder.values().append_null(),
+                        }
+                    }
+                    builder.append(true);
+                }
+            }
+        }
+        Arc::new(builder.finish())
+    }
+
+    fn extract_row_strings(arr: &ArrayRef, row: usize) -> Vec {
+        let list = arr.as_any().downcast_ref::().unwrap();
+        let inner = list.value(row);
+        let strs = inner.as_string::();
+        (0..strs.len()).map(|i| strs.value(i).to_string()).collect()
+    }
+
+    #[test]
+    fn basic_two_string_arrays_with_default_separator() {
+        let left = list_of_strings(&[Some(&[Some("a"), Some("b")])]);
+        let right = list_of_strings(&[Some(&[Some("1"), Some("2")])]);
+        let result = run(left, right, None);
+        assert_eq!(extract_row_strings(&result, 0), vec!["a,1", "b,2"]);
+    }
+
+    #[test]
+    fn custom_separator() {
+        let left = list_of_strings(&[Some(&[Some("x"), Some("y")])]);
+        let right = list_of_strings(&[Some(&[Some("1"), Some("2")])]);
+        let result = run(left, right, Some("-"));
+        assert_eq!(extract_row_strings(&result, 0), vec!["x-1", "y-2"]);
+    }
+
+    #[test]
+    fn truncate_to_shorter_array() {
+        let left = list_of_strings(&[Some(&[Some("a"), Some("b"), Some("c")])]);
+        let right = list_of_strings(&[Some(&[Some("1")])]);
+        let result = run(left, right, None);
+        assert_eq!(extract_row_strings(&result, 0), vec!["a,1"]);
+    }
+
+    #[test]
+    fn null_element_renders_as_empty_string() {
+        let left = list_of_strings(&[Some(&[Some("a"), None])]);
+        let right = list_of_strings(&[Some(&[Some("1"), Some("2")])]);
+        let result = run(left, right, None);
+        assert_eq!(extract_row_strings(&result, 0), vec!["a,1", ",2"]);
+    }
+
+    #[test]
+    fn null_array_yields_null_row() {
+        let left = list_of_strings(&[None]);
+        let right = list_of_strings(&[Some(&[Some("1")])]);
+        let result = run(left, right, None);
+        let list = result.as_any().downcast_ref::().unwrap();
+        assert!(list.is_null(0));
+    }
+
+    #[test]
+    fn empty_array_yields_empty_result() {
+        let left = list_of_strings(&[Some(&[])]);
+        let right = list_of_strings(&[Some(&[Some("1")])]);
+        let result = run(left, right, None);
+        assert_eq!(extract_row_strings(&result, 0), Vec::::new());
+    }
+
+    #[test]
+    fn integer_arrays_are_stringified() {
+        let left = list_of_ints(&[Some(&[Some(10), Some(20)])]);
+        let right = list_of_ints(&[Some(&[Some(1), Some(2)])]);
+        let result = run(left, right, None);
+        assert_eq!(extract_row_strings(&result, 0), vec!["10,1", "20,2"]);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrayElementAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrayElementAdapter.java
new file mode 100644
index 0000000000000..02476ad222a4e
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrayElementAdapter.java
@@ -0,0 +1,85 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.util.List;
+
+/**
+ * Adapter for Calcite's {@link SqlStdOperatorTable#ITEM} operator (element
+ * access — {@code arr[N]}). PPL's {@code mvindex(arr, N)} single-element form
+ * lowers through {@code MVIndexFunctionImp.resolveSingleElement} to ITEM with
+ * a 1-based index (already converted from PPL's 0-based input).
+ *
+ * 
            Two transforms before substrait emission:
+ *
+ * 
              
+ *   
            1. Rename to {@code array_element}. DataFusion's native single-element
+ *       array accessor is named {@code array_element} (also 1-based), declared
+ *       in {@code opensearch_array_functions.yaml}. Calcite's ITEM operator name
+ *       is {@code "ITEM"} which doesn't resolve to anything in the substrait
+ *       extension catalog.
+ *   
            2. Coerce the index to {@code BIGINT}. PPL's parser types positive
+ *       integer literals as {@code DECIMAL(20,0)}; DataFusion's
+ *       {@code array_element} signature accepts only integer indexes.
+ * 
            
+ *
+ * @opensearch.internal
+ */
+class ArrayElementAdapter implements ScalarFunctionAdapter {
+
+    /**
+     * Locally-declared target operator. Name matches DataFusion's native
+     * {@code array_element}. Return-type inference is a placeholder — the
+     * adapt method explicitly carries the original ITEM call's return type
+     * (the element type).
+     */
+    static final SqlOperator LOCAL_ARRAY_ELEMENT_OP = new SqlFunction(
+        "array_element",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.ARG0,
+        null,
+        OperandTypes.ANY_ANY,
+        SqlFunctionCategory.SYSTEM
+    );
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        RexBuilder rexBuilder = cluster.getRexBuilder();
+        RelDataTypeFactory typeFactory = cluster.getTypeFactory();
+        List operands = original.getOperands();
+        if (operands.size() != 2) {
+            return rexBuilder.makeCall(original.getType(), LOCAL_ARRAY_ELEMENT_OP, operands);
+        }
+        RexNode array = operands.get(0);
+        RexNode index = operands.get(1);
+        if (index.getType().getSqlTypeName() != SqlTypeName.BIGINT) {
+            RelDataType bigint = typeFactory.createSqlType(SqlTypeName.BIGINT);
+            RelDataType nullableBigint = typeFactory.createTypeWithNullability(bigint, index.getType().isNullable());
+            index = rexBuilder.makeCast(nullableBigint, index, true, false);
+        }
+        return rexBuilder.makeCall(original.getType(), LOCAL_ARRAY_ELEMENT_OP, List.of(array, index));
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArraySliceAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArraySliceAdapter.java
new file mode 100644
index 0000000000000..15202620d10ee
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArraySliceAdapter.java
@@ -0,0 +1,113 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.fun.SqlLibraryOperators;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Adapter for Calcite's {@link SqlLibraryOperators#ARRAY_SLICE}. Two transforms
+ * are needed before substrait emission:
+ *
+ * 
              
+ *   
            1. Index coercion to {@code BIGINT}. PPL's parser types positive
+ *       integer literals as {@code DECIMAL(20,0)} (precision wide enough to
+ *       hold any 64-bit unsigned value), but DataFusion's {@code array_slice}
+ *       signature accepts only integer indexes and refuses to coerce decimal
+ *       arguments — failing with {@code "No function matches the given name
+ *       and argument types 'array_slice(List(Int32), Decimal128(20, 0),
+ *       Decimal128(22, 0))'"}.
+ *   
            2. Semantic conversion: 0-based {@code (start, length)} →
+ *       1-based {@code (start, end)} inclusive. Calcite's
+ *       {@link SqlLibraryOperators#ARRAY_SLICE} (used by PPL's
+ *       {@code MVIndexFunctionImp.resolveRange}) is the Spark / Hive flavor
+ *       with 0-based start and a length-of-elements third arg. DataFusion's
+ *       native {@code array_slice} is 1-based with an inclusive end-index
+ *       third arg. Without this conversion, {@code mvindex(arr=[1..5], 1, 3)}
+ *       would emit {@code ARRAY_SLICE(arr, 1, 3)} → DataFusion returns
+ *       {@code [1, 2, 3]}, but the PPL expectation is {@code [2, 3, 4]}
+ *       (0-based positions 1..3 inclusive).
+ *       
              The conversion is purely arithmetic on the operands:
+ *       
                
+ *         
              • {@code start' = start + 1}
+ *         
              • {@code end'   = start + length} (which is {@code start + 1 +
+ *             (length - 1)} = the 1-based inclusive end)
+ *       
              
+ *       Negative indexes have already been normalized to non-negative
+ *       0-based positions by {@code MVIndexFunctionImp} before this adapter
+ *       runs (it uses {@code arrayLen + idx} for both start and end), so the
+ *       arithmetic above applies uniformly.
+ * 
            
+ *
+ * 
            The 2-arg form {@code ARRAY_SLICE(arr, start)} (single-element extract)
+ * is not produced by PPL's {@code MVIndexFunctionImp} (single-element access
+ * lowers through {@code INTERNAL_ITEM} instead), so this adapter handles
+ * only the 3-arg form.
+ *
+ * @opensearch.internal
+ */
+class ArraySliceAdapter implements ScalarFunctionAdapter {
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        RexBuilder rexBuilder = cluster.getRexBuilder();
+        RelDataTypeFactory typeFactory = cluster.getTypeFactory();
+        RelDataType bigint = typeFactory.createSqlType(SqlTypeName.BIGINT);
+        List operands = original.getOperands();
+        if (operands.size() != 3) {
+            // Defensive: unexpected arity. Fall through with BIGINT coercion only — the substrait
+            // converter will surface a missing-signature error with a clear message.
+            return rexBuilder.makeCall(
+                original.getType(),
+                original.getOperator(),
+                coerceIndexes(rexBuilder, typeFactory, bigint, operands)
+            );
+        }
+        List coerced = coerceIndexes(rexBuilder, typeFactory, bigint, operands);
+        RexNode array = coerced.get(0);
+        RexNode start = coerced.get(1);
+        RexNode length = coerced.get(2);
+        RexNode one = rexBuilder.makeExactLiteral(BigDecimal.ONE, bigint);
+        RexNode oneBasedStart = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, start, one);
+        RexNode endInclusive = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, start, length);
+        return rexBuilder.makeCall(original.getType(), original.getOperator(), List.of(array, oneBasedStart, endInclusive));
+    }
+
+    private static List coerceIndexes(
+        RexBuilder rexBuilder,
+        RelDataTypeFactory typeFactory,
+        RelDataType bigint,
+        List operands
+    ) {
+        List coerced = new ArrayList<>(operands.size());
+        for (int i = 0; i < operands.size(); i++) {
+            RexNode operand = operands.get(i);
+            if (i == 0 || operand.getType().getSqlTypeName() == SqlTypeName.BIGINT) {
+                coerced.add(operand);
+            } else {
+                RelDataType nullableBigint = typeFactory.createTypeWithNullability(bigint, operand.getType().isNullable());
+                coerced.add(rexBuilder.makeCast(nullableBigint, operand, true, false));
+            }
+        }
+        return coerced;
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrayToStringAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrayToStringAdapter.java
new file mode 100644
index 0000000000000..258b47a75440e
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ArrayToStringAdapter.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.opensearch.analytics.spi.AbstractNameMappingAdapter;
+
+import java.util.List;
+
+/**
+ * Rename adapter for Calcite's {@code ARRAY_JOIN(arr, sep)} — used by PPL's
+ * {@code mvjoin} via {@code SqlLibraryOperators.ARRAY_JOIN}. DataFusion's native
+ * equivalent is named {@code array_to_string} (same semantics: join array
+ * elements with a separator). Rewrites to a locally-declared {@link SqlFunction}
+ * named {@code array_to_string}; isthmus emits a Substrait scalar function call
+ * with that name and DataFusion's substrait consumer resolves it natively.
+ *
+ * @opensearch.internal
+ */
+class ArrayToStringAdapter extends AbstractNameMappingAdapter {
+
+    static final SqlOperator LOCAL_ARRAY_TO_STRING_OP = new SqlFunction(
+        "array_to_string",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.VARCHAR_NULLABLE,
+        null,
+        OperandTypes.ANY_ANY,
+        SqlFunctionCategory.SYSTEM
+    );
+
+    ArrayToStringAdapter() {
+        super(LOCAL_ARRAY_TO_STRING_OP, List.of(), List.of());
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
index 4e5b5955cb92c..176de49872dd1 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
@@ -228,7 +228,45 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
         ScalarFunction.JSON_EXTEND,
         ScalarFunction.JSON_EXTRACT,
         ScalarFunction.JSON_KEYS,
-        ScalarFunction.JSON_SET
+        ScalarFunction.JSON_SET,
+        // Array functions whose RETURN type is element-typed (not ARRAY itself), so the
+        // capability lookup at OpenSearchProjectRule resolves the call's return type to a
+        // standard scalar FieldType and matches against SUPPORTED_FIELD_TYPES.
+        // ARRAY_LENGTH returns BIGINT → FieldType.LONG; ARRAY_JOIN returns VARCHAR →
+        // FieldType.KEYWORD (renamed to DataFusion `array_to_string` via {@link ArrayToStringAdapter}).
+        // ITEM returns the array's element type (any of the supported scalar types) — used by
+        // PPL `mvindex(arr, N)` single-element form.
+        ScalarFunction.ARRAY_LENGTH,
+        ScalarFunction.ARRAY_JOIN,
+        ScalarFunction.ITEM,
+        // PPL `mvfind` returns INTEGER (the 0-based index of the first match, or NULL); backed
+        // by a custom Rust UDF on the DataFusion session context (`udf::mvfind`), routed via
+        // {@link MvfindAdapter}.
+        ScalarFunction.MVFIND
+    );
+
+    /**
+     * Project-side scalar functions whose return type is {@code ARRAY}. Registered separately
+     * because the capability lookup keys on the call's return type, and for these the lookup
+     * resolves to {@link FieldType#ARRAY} — which is intentionally not in
+     * {@link #SUPPORTED_FIELD_TYPES} (filter and aggregate operators have no meaningful semantics
+     * over array-typed values, so we don't want them claiming viability there).
+     *
+     * 
            {@code ARRAY} (PPL {@code array(a, b, …)} constructor) renames to DataFusion's
+     * {@code make_array} via {@link MakeArrayAdapter}. {@code ARRAY_SLICE} and
+     * {@code ARRAY_DISTINCT} pass through by name (Calcite stdlib operator names match
+     * DataFusion's native names — isthmus default catalog binds them).
+     */
+    private static final Set ARRAY_RETURNING_PROJECT_OPS = Set.of(
+        ScalarFunction.ARRAY,
+        ScalarFunction.ARRAY_SLICE,
+        ScalarFunction.ARRAY_DISTINCT,
+        // PPL `mvzip` returns ARRAY; backed by a custom Rust UDF on the DataFusion
+        // session context (`udf::mvzip`), routed via {@link MvzipAdapter}.
+        ScalarFunction.MVZIP,
+        // PPL `mvappend` returns ARRAY; backed by a custom Rust UDF
+        // (`udf::mvappend`), routed via {@link MvappendAdapter}.
+        ScalarFunction.MVAPPEND
     );
 
     private static final Set AGG_FUNCTIONS = Set.of(
@@ -289,6 +327,9 @@ public Set projectCapabilities() {
                 for (ScalarFunction op : STANDARD_PROJECT_OPS) {
                     caps.add(new ProjectCapability.Scalar(op, Set.copyOf(SUPPORTED_FIELD_TYPES), formats, true));
                 }
+                for (ScalarFunction op : ARRAY_RETURNING_PROJECT_OPS) {
+                    caps.add(new ProjectCapability.Scalar(op, Set.of(FieldType.ARRAY), formats, true));
+                }
                 return Set.copyOf(caps);
             }
 
@@ -319,6 +360,13 @@ public Map scalarFunctionAdapters() {
                 DateTimeAdapters.CurrentDateAdapter currentDate = new DateTimeAdapters.CurrentDateAdapter();
                 DateTimeAdapters.CurrentTimeAdapter currentTime = new DateTimeAdapters.CurrentTimeAdapter();
                 return Map.ofEntries(
+                    Map.entry(ScalarFunction.ARRAY, new MakeArrayAdapter()),
+                    Map.entry(ScalarFunction.ARRAY_JOIN, new ArrayToStringAdapter()),
+                    Map.entry(ScalarFunction.ARRAY_SLICE, new ArraySliceAdapter()),
+                    Map.entry(ScalarFunction.ITEM, new ArrayElementAdapter()),
+                    Map.entry(ScalarFunction.MVFIND, new MvfindAdapter()),
+                    Map.entry(ScalarFunction.MVZIP, new MvzipAdapter()),
+                    Map.entry(ScalarFunction.MVAPPEND, new MvappendAdapter()),
                     Map.entry(ScalarFunction.CONCAT, new ConcatFunctionAdapter()),
                     Map.entry(ScalarFunction.CONVERT_TZ, new ConvertTzAdapter()),
                     Map.entry(ScalarFunction.COSH, new HyperbolicOperatorAdapter(SqlLibraryOperators.COSH)),
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
index a06d2a4bb20d3..2161abc08b7a2 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
@@ -153,7 +153,23 @@ public class DataFusionFragmentConvertor implements FragmentConvertor {
         FunctionMappings.s(JsonFunctionAdapters.JsonSetAdapter.LOCAL_JSON_SET_OP, "json_set"),
         FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"),
         FunctionMappings.s(SqlStdOperatorTable.REPLACE, "replace"),
-        FunctionMappings.s(SqlLibraryOperators.REGEXP_REPLACE_3, "regexp_replace")
+        FunctionMappings.s(SqlLibraryOperators.REGEXP_REPLACE_3, "regexp_replace"),
+        // Array S0 ladder — see DataFusionAnalyticsBackendPlugin.STANDARD_PROJECT_OPS /
+        // ARRAY_RETURNING_PROJECT_OPS for the capability registration. ARRAY_LENGTH /
+        // ARRAY_SLICE / ARRAY_DISTINCT pass through under their Calcite-stdlib names
+        // (DataFusion's substrait consumer resolves them natively). MakeArrayAdapter /
+        // ArrayToStringAdapter / ArrayElementAdapter rewrite PPL `array(...)` /
+        // `mvjoin(...)` / `mvindex(...)` single-element to locally-declared SqlFunctions
+        // so isthmus emits Substrait calls with DataFusion's native function names.
+        FunctionMappings.s(SqlLibraryOperators.ARRAY_LENGTH, "array_length"),
+        FunctionMappings.s(SqlLibraryOperators.ARRAY_SLICE, "array_slice"),
+        FunctionMappings.s(SqlLibraryOperators.ARRAY_DISTINCT, "array_distinct"),
+        FunctionMappings.s(MakeArrayAdapter.LOCAL_MAKE_ARRAY_OP, "make_array"),
+        FunctionMappings.s(ArrayToStringAdapter.LOCAL_ARRAY_TO_STRING_OP, "array_to_string"),
+        FunctionMappings.s(ArrayElementAdapter.LOCAL_ARRAY_ELEMENT_OP, "array_element"),
+        FunctionMappings.s(MvzipAdapter.LOCAL_MVZIP_OP, "mvzip"),
+        FunctionMappings.s(MvfindAdapter.LOCAL_MVFIND_OP, "mvfind"),
+        FunctionMappings.s(MvappendAdapter.LOCAL_MVAPPEND_OP, "mvappend")
     );
 
     private final SimpleExtension.ExtensionCollection extensions;
@@ -214,7 +230,19 @@ private byte[] convertToSubstrait(RelNode fragment) {
         RelNode preprocessed = UntypedNullPreprocessor.rewrite(fragment);
         RelRoot root = RelRoot.of(preprocessed, SqlKind.SELECT);
         SubstraitRelVisitor visitor = createVisitor(preprocessed);
-        Rel substraitRel = visitor.apply(root.rel);
+        Rel substraitRel;
+        try {
+            substraitRel = visitor.apply(root.rel);
+        } catch (AssertionError e) {
+            // Substrait validators (e.g. VariadicParameterConsistencyValidator,
+            // RelOptUtil.eq via Litmus.THROW) throw AssertionError directly via Java
+            // code rather than via the `assert` keyword, so JVM -da doesn't gate them.
+            // If one fires inside a search thread, OpenSearchUncaughtExceptionHandler
+            // exits the cluster JVM. Convert to IllegalStateException so the analytics-
+            // engine error path treats it as a normal per-query failure (HTTP 500 with
+            // a bucketable message) instead of taking down the cluster.
+            throw new IllegalStateException("Substrait conversion rejected the plan: " + e.getMessage(), e);
+        }
 
         List fieldNames = root.fields.stream().map(field -> field.getValue()).toList();
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
index f6713b005393f..d72823015dae5 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
@@ -171,7 +171,8 @@ private static SimpleExtension.ExtensionCollection loadSubstraitExtensions() {
             t.setContextClassLoader(DataFusionPlugin.class.getClassLoader());
             SimpleExtension.ExtensionCollection delegationExtensions = SimpleExtension.load(List.of("/delegation_functions.yaml"));
             SimpleExtension.ExtensionCollection scalarExtensions = SimpleExtension.load(List.of("/opensearch_scalar_functions.yaml"));
-            return DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions).merge(scalarExtensions);
+            SimpleExtension.ExtensionCollection arrayExtensions = SimpleExtension.load(List.of("/opensearch_array_functions.yaml"));
+            return DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions).merge(scalarExtensions).merge(arrayExtensions);
         } finally {
             t.setContextClassLoader(previous);
         }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MakeArrayAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MakeArrayAdapter.java
new file mode 100644
index 0000000000000..672433d87a8b1
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MakeArrayAdapter.java
@@ -0,0 +1,89 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Rename adapter for PPL's {@code array(a, b, …)} constructor — rewrites to a
+ * locally-declared {@link SqlFunction} named {@code make_array}, which is
+ * DataFusion's native array constructor (no UDF registration required on the
+ * Rust side; isthmus emits a Substrait scalar function call with that name and
+ * DataFusion's substrait consumer maps it to {@code make_array} natively).
+ *
+ * 
            Unlike {@link org.opensearch.analytics.spi.AbstractNameMappingAdapter},
+ * this adapter also CASTs each operand to the array's inferred element type
+ * before emission. PPL's {@code ArrayFunctionImpl} returns
+ * {@code ARRAY} (Calcite type-widens to find the common
+ * element type), but it does NOT widen the individual operand types — so a
+ * call like {@code array(1, 1.5)} produces a RexCall whose operand types are
+ * {@code (INTEGER, DECIMAL(2,1))} but whose return type is {@code ARRAY}.
+ * Substrait's variadic {@code make_array(any1)} signature requires consistent
+ * argument types ({@link io.substrait.expression.VariadicParameterConsistencyValidator})
+ * and throws an AssertionError that fatally exits the JVM otherwise — so we
+ * widen each operand to the call's component type before substrait sees it.
+ *
+ * 
            Same machinery as {@link UnixTimestampAdapter}: locally-declared operator
+ * is the referent of the {@link io.substrait.isthmus.expression.FunctionMappings.Sig}
+ * in {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS}.
+ *
+ * @opensearch.internal
+ */
+class MakeArrayAdapter implements ScalarFunctionAdapter {
+
+    /**
+     * Locally-declared target operator. Name matches DataFusion's native {@code make_array}.
+     * Return type inference is a placeholder — {@link #adapt} explicitly carries the
+     * original call's array return type forward.
+     */
+    static final SqlOperator LOCAL_MAKE_ARRAY_OP = new SqlFunction(
+        "make_array",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.ARG0,
+        null,
+        OperandTypes.VARIADIC,
+        SqlFunctionCategory.SYSTEM
+    );
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        RexBuilder rexBuilder = cluster.getRexBuilder();
+        RelDataType arrayType = original.getType();
+        RelDataType elementType = arrayType.getComponentType();
+        if (elementType == null) {
+            // Defensive — Calcite's array() always infers a component type. If somehow
+            // missing, fall through with original operands and let substrait fail.
+            return rexBuilder.makeCall(arrayType, LOCAL_MAKE_ARRAY_OP, original.getOperands());
+        }
+        List widened = new ArrayList<>(original.getOperands().size());
+        for (RexNode operand : original.getOperands()) {
+            if (operand.getType().equals(elementType)) {
+                widened.add(operand);
+            } else {
+                widened.add(rexBuilder.makeCast(elementType, operand, true, false));
+            }
+        }
+        return rexBuilder.makeCall(arrayType, LOCAL_MAKE_ARRAY_OP, widened);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvappendAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvappendAdapter.java
new file mode 100644
index 0000000000000..ac6dcb3ff4e81
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvappendAdapter.java
@@ -0,0 +1,97 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Rename + operand-coerce adapter for PPL's {@code mvappend(arg1, arg2, …)}.
+ *
+ * 
            The Rust UDF (`udf::mvappend`) handles operands as a uniform stream where
+ * every operand is either {@code element_type} (scalar) or
+ * {@code List} (array) for a single inferred {@code element_type}.
+ * The Calcite call's return type is {@code ARRAY}; this adapter
+ * casts each scalar operand to {@code componentType} and each array operand to
+ * {@code ARRAY} before substrait emission, so the UDF sees a
+ * single element type across all positions.
+ *
+ * 
            Mixed-type {@code mvappend} calls (PPL widens to {@code ARRAY}) end
+ * up with a Calcite {@code ANY} component type which substrait can't serialize
+ * — those fail at substrait conversion before reaching this adapter, and
+ * aren't handled by it.
+ *
+ * 
            Same templated machinery as {@link MvzipAdapter} / {@link MvfindAdapter}:
+ * the locally-declared operator is the referent of the
+ * {@link io.substrait.isthmus.expression.FunctionMappings.Sig} entry in
+ * {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS}.
+ *
+ * @opensearch.internal
+ */
+class MvappendAdapter implements ScalarFunctionAdapter {
+
+    static final SqlOperator LOCAL_MVAPPEND_OP = new SqlFunction(
+        "mvappend",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.ARG0,
+        null,
+        OperandTypes.VARIADIC,
+        SqlFunctionCategory.SYSTEM
+    );
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        RexBuilder rexBuilder = cluster.getRexBuilder();
+        RelDataType arrayType = original.getType();
+        RelDataType componentType = arrayType.getComponentType();
+        if (componentType == null) {
+            return rexBuilder.makeCall(arrayType, LOCAL_MVAPPEND_OP, original.getOperands());
+        }
+        // Substrait's variadic {@code any1} parameter requires every operand at the same
+        // variadic position to share a type. PPL's {@code mvappend(arg, …)} accepts a mix
+        // of bare scalars and arrays, which substrait's signature matcher rejects with
+        // {@code Unable to convert call mvappend(list<…>, scalar, …)}. Normalize every
+        // operand to {@code ARRAY} — array operands cast their element
+        // type if it differs; scalar operands wrap in a {@code make_array(…)} singleton
+        // call. The Rust UDF then sees a uniform {@code list} variadic.
+        RelDataType targetArrayType = cluster.getTypeFactory().createArrayType(componentType, -1);
+        List coerced = new ArrayList<>(original.getOperands().size());
+        for (RexNode operand : original.getOperands()) {
+            RelDataType operandType = operand.getType();
+            if (operandType.getComponentType() != null) {
+                // Array operand — cast to ARRAY if its element type differs.
+                if (operandType.equals(targetArrayType)) {
+                    coerced.add(operand);
+                } else {
+                    coerced.add(rexBuilder.makeCast(targetArrayType, operand, true, false));
+                }
+            } else {
+                // Scalar operand — first cast to componentType (so the singleton array's
+                // element type matches), then wrap in make_array so substrait sees a list.
+                RexNode casted = operandType.equals(componentType) ? operand : rexBuilder.makeCast(componentType, operand, true, false);
+                coerced.add(rexBuilder.makeCall(targetArrayType, MakeArrayAdapter.LOCAL_MAKE_ARRAY_OP, List.of(casted)));
+            }
+        }
+        return rexBuilder.makeCall(arrayType, LOCAL_MVAPPEND_OP, coerced);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvfindAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvfindAdapter.java
new file mode 100644
index 0000000000000..3a441bbf52b5f
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvfindAdapter.java
@@ -0,0 +1,67 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.util.List;
+
+/**
+ * Rename adapter for PPL's {@code mvfind(arr, regex)} — rewrites the Calcite
+ * UDF call (PPL's {@code MVFindFunctionImpl} registered under the function
+ * name {@code "mvfind"}) to a locally-declared {@link SqlFunction} also named
+ * {@code mvfind}. The locally-declared op is the referent of the
+ * {@link io.substrait.isthmus.expression.FunctionMappings.Sig} entry in
+ * {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS}, so isthmus
+ * emits a Substrait scalar function call with that exact name. The
+ * analytics-backend-datafusion plugin's Rust crate (`udf::mvfind`) registers
+ * a matching ScalarUDF on the DataFusion session context with the same name,
+ * which the substrait consumer resolves natively.
+ *
+ * 
            The PPL UDF's Calcite-side return type is already {@code INTEGER NULLABLE}
+ * ({@code MVFindFunctionImpl.getReturnTypeInference()} returns
+ * {@code ReturnTypes.INTEGER_NULLABLE}), matching the {@code i32?} declared
+ * in {@code opensearch_array_functions.yaml}. No operand widening is needed —
+ * the Rust UDF accepts any list element type and any string flavor for the
+ * regex pattern.
+ *
+ * @opensearch.internal
+ */
+class MvfindAdapter implements ScalarFunctionAdapter {
+
+    /**
+     * Locally-declared target operator. Name matches the Rust UDF
+     * {@code MvfindUdf::name()}.
+     */
+    static final SqlOperator LOCAL_MVFIND_OP = new SqlFunction(
+        "mvfind",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.INTEGER_NULLABLE,
+        null,
+        OperandTypes.ANY_ANY,
+        SqlFunctionCategory.SYSTEM
+    );
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        RexBuilder rexBuilder = cluster.getRexBuilder();
+        return rexBuilder.makeCall(original.getType(), LOCAL_MVFIND_OP, original.getOperands());
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvzipAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvzipAdapter.java
new file mode 100644
index 0000000000000..22164425fb34f
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/MvzipAdapter.java
@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.util.List;
+
+/**
+ * Rename adapter for PPL's {@code mvzip(left, right [, sep])} — rewrites the
+ * Calcite UDF call (PPL's {@code MVZipFunctionImpl} registered under the
+ * function name {@code "mvzip"}) to a locally-declared {@link SqlFunction}
+ * also named {@code mvzip}. The locally-declared op is the referent of the
+ * {@link io.substrait.isthmus.expression.FunctionMappings.Sig} entry in
+ * {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS}, so isthmus
+ * emits a Substrait scalar function call with that exact name. The
+ * analytics-backend-datafusion plugin's Rust crate (`udf::mvzip`) registers a
+ * matching ScalarUDF on the DataFusion session context with the same name,
+ * which the substrait consumer resolves natively.
+ *
+ * 
            The PPL UDF's Calcite-side return type is already
+ * {@code ARRAY<VARCHAR>} (set by {@code MVZipFunctionImpl.getReturnTypeInference}),
+ * matching the {@code list<string?>} declared in
+ * {@code opensearch_array_functions.yaml}. No operand widening is needed —
+ * mvzip accepts any pair of array element types and emits strings.
+ *
+ * @opensearch.internal
+ */
+class MvzipAdapter implements ScalarFunctionAdapter {
+
+    /**
+     * Locally-declared target operator. Name matches the Rust UDF
+     * {@code MvzipUdf::name()}. Return-type inference here is a placeholder —
+     * the call's original return type ({@code ARRAY<VARCHAR>}) is carried
+     * forward explicitly in {@link #adapt}.
+     */
+    static final SqlOperator LOCAL_MVZIP_OP = new SqlFunction(
+        "mvzip",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.ARG0,
+        null,
+        OperandTypes.VARIADIC,
+        SqlFunctionCategory.SYSTEM
+    );
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        RexBuilder rexBuilder = cluster.getRexBuilder();
+        return rexBuilder.makeCall(original.getType(), LOCAL_MVZIP_OP, original.getOperands());
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_array_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_array_functions.yaml
new file mode 100644
index 0000000000000..41361ea3a4acc
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_array_functions.yaml
@@ -0,0 +1,158 @@
+%YAML 1.2
+---
+# Substrait extension declaring the array-producing and array-consuming scalar
+# functions DataFusion's native runtime can execute. Substrait's standard
+# extension catalog has no array_* entries, so isthmus' RexExpressionConverter
+# would fail with "Unable to convert call …" until we declare them here.
+#
+# DataFusion's `datafusion-substrait` consumer resolves these names to native
+# DataFusion implementations (datafusion-functions-array crate):
+#   make_array       → array constructor
+#   array_length     → array length
+#   array_slice      → array slice (1-based, inclusive)
+#   array_distinct   → array distinct elements
+#   array_to_string  → join array elements with a separator
+urn: extension:org.opensearch:array_functions
+scalar_functions:
+  - name: make_array
+    description: >-
+      Construct an array literal from variadic operands. All operands must share
+      a common element type (Calcite type-widens at the operator level before
+      emission). Returns a list of that element type.
+    impls:
+      - args:
+          - value: any1
+            name: element
+        variadic:
+          min: 0
+        return: "list"
+
+  - name: array_length
+    description: >-
+      Return the number of elements in the array, or NULL if the array is NULL.
+      Calcite's {@code SqlLibraryOperators.ARRAY_LENGTH} lowers to this name.
+    impls:
+      - args:
+          - value: "list"
+            name: array
+        return: "i64?"
+
+  - name: array_slice
+    description: >-
+      Return a sub-array slice [from, to] (1-based, inclusive on both ends).
+      Calcite's {@code SqlLibraryOperators.ARRAY_SLICE} lowers to this name.
+    impls:
+      - args:
+          - value: "list"
+            name: array
+          - value: "i64"
+            name: from
+          - value: "i64"
+            name: to
+        return: "list"
+      - args:
+          - value: "list"
+            name: array
+          - value: "i32"
+            name: from
+          - value: "i32"
+            name: to
+        return: "list"
+
+  - name: array_distinct
+    description: >-
+      Return the array with duplicate elements removed (preserving first occurrence).
+      Calcite's {@code SqlLibraryOperators.ARRAY_DISTINCT} lowers to this name.
+    impls:
+      - args:
+          - value: "list"
+            name: array
+        return: "list"
+
+  - name: array_element
+    description: >-
+      Return the element at the given 1-based position. Calcite's
+      {@code SqlStdOperatorTable.ITEM} (used by PPL's {@code mvindex(arr, N)}
+      single-element form via {@code MVIndexFunctionImp.resolveSingleElement})
+      renames to this for DataFusion. Returns null if the index is out of range.
+    impls:
+      - args:
+          - value: "list"
+            name: array
+          - value: "i64"
+            name: index
+        return: "any1?"
+
+  - name: mvappend
+    description: >-
+      Flatten a list of arrays into one array, dropping null arrays and null
+      elements within array arguments. Returns NULL if no non-null elements
+      were collected. PPL surface is {@code mvappend(arg1, arg2, …)} which
+      accepts mixed scalar+array operands; the Java adapter wraps each
+      scalar in a singleton {@code make_array(…)} call so by the time the
+      Rust UDF sees the operands they're uniformly arrays. Backed by a custom
+      Rust UDF on the analytics-backend-datafusion plugin (DataFusion's
+      array_concat preserves nulls — different semantics).
+    impls:
+      - args:
+          - value: "list"
+            name: arg
+        variadic:
+          min: 1
+        return: "list"
+
+  - name: mvfind
+    description: >-
+      Find the 0-based index of the first array element matching a regex pattern,
+      or NULL if no match. NULL elements are skipped (not matched). PPL surface is
+      {@code mvfind(arr, regex)}; registered as a custom Rust UDF on the
+      analytics-backend-datafusion plugin (no DataFusion stdlib equivalent).
+    impls:
+      - args:
+          - value: "list"
+            name: array
+          - value: "string"
+            name: pattern
+        return: "i32?"
+
+  - name: mvzip
+    description: >-
+      Element-wise zip of two arrays into a list of strings, joined per pair
+      by a separator (default ","). Result length is min(len(left), len(right))
+      (Python-zip truncation). Element NULLs render as empty strings; either
+      array NULL → NULL result. PPL surface is {@code mvzip(left, right [, sep])};
+      registered as a custom Rust UDF on the analytics-backend-datafusion plugin
+      (no DataFusion stdlib equivalent).
+    impls:
+      - args:
+          - value: "list"
+            name: left
+          - value: "list"
+            name: right
+        return: "list"
+      - args:
+          - value: "list"
+            name: left
+          - value: "list"
+            name: right
+          - value: "string"
+            name: separator
+        return: "list"
+
+  - name: array_to_string
+    description: >-
+      Join array elements into a single string using a separator. Calcite's
+      {@code SqlLibraryOperators.ARRAY_JOIN} renames to this for DataFusion.
+    impls:
+      - args:
+          - value: "list"
+            name: array
+          - value: "string"
+            name: separator
+        return: "string?"
+      - args:
+          - value: "list"
+            name: array
+          - value: "varchar"
+            name: separator
+        return: "string?"
diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle
index 0058df4b6eb68..78fe8da9d709d 100644
--- a/sandbox/plugins/analytics-engine/build.gradle
+++ b/sandbox/plugins/analytics-engine/build.gradle
@@ -104,9 +104,14 @@ dependencies {
   // so bundle it into analytics-engine's own zip.
   runtimeOnly "org.apache.commons:commons-math3:3.6.1"
 
-  // commons-text — Calcite's SqlFunctions. references
-  // org.apache.commons.text.similarity.LevenshteinDistance. Must be loaded via
-  // the same classloader as calcite-core so that  succeeds.
+  // commons-text — Calcite's SqlFunctions class statically references
+  // org.apache.commons.text.similarity.LevenshteinDistance (used by SQL fuzzy-match
+  // helpers, also pulled in transitively when constant-folding array literals via
+  // ReduceExpressionsRule). Must be loaded via the same classloader as calcite-core
+  // so that SqlFunctions. succeeds; otherwise it throws NoClassDefFoundError
+  // on first use and poisons every subsequent Calcite operation in the JVM — symptom
+  // is a single failing analytics query taking the cluster's planner thread offline
+  // for the rest of the run.
   runtimeOnly "org.apache.commons:commons-text:1.11.0"
 
   // httpcore5/httpclient5 — Avatica's BuiltInConnectionProperty static initializer references
diff --git a/sandbox/plugins/analytics-engine/licenses/commons-text-1.11.0.jar.sha1 b/sandbox/plugins/analytics-engine/licenses/commons-text-1.11.0.jar.sha1
index 6f090739df8c6..c7b597f6550e0 100644
--- a/sandbox/plugins/analytics-engine/licenses/commons-text-1.11.0.jar.sha1
+++ b/sandbox/plugins/analytics-engine/licenses/commons-text-1.11.0.jar.sha1
@@ -1 +1 @@
-2bb044b7717ec2eccaf9ea7769c1509054b50e9a
\ No newline at end of file
+2bb044b7717ec2eccaf9ea7769c1509054b50e9a
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java
index 6b0ee31226f39..2a944451363cd 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/ArrowValues.java
@@ -10,9 +10,12 @@
 
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.util.Text;
 
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Helpers for reading Arrow vector cells as plain Java values at the
@@ -35,10 +38,22 @@ public static Object toJavaValue(FieldVector vector, int index) {
         if (vector instanceof VarCharVector v) {
             return new String(v.get(index), StandardCharsets.UTF_8);
         }
-        Object obj = vector.getObject(index);
-        if (obj instanceof Text t) {
+        Object value = vector.getObject(index);
+        if (vector instanceof ListVector && value instanceof List raw) {
+            // ListVector.getObject returns a JsonStringArrayList whose elements are the
+            // child vector's typed values. For VarCharVector children that's Arrow's
+            // Text, which downstream consumers (e.g. {@code ExprValueUtils.fromObjectValue})
+            // don't recognize and reject as "unsupported object class". Mirror the
+            // top-level VarCharVector branch above and substitute Java strings.
+            List normalized = new ArrayList<>(raw.size());
+            for (Object element : raw) {
+                normalized.add(element instanceof Text t ? t.toString() : element);
+            }
+            return normalized;
+        }
+        if (value instanceof Text t) {
             return t.toString();
         }
-        return obj;
+        return value;
     }
 }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ArrowSchemaFromCalcite.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ArrowSchemaFromCalcite.java
index 2c599b96dc531..f81a9bd1e2951 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ArrowSchemaFromCalcite.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ArrowSchemaFromCalcite.java
@@ -40,12 +40,35 @@ private ArrowSchemaFromCalcite() {}
     public static Schema arrowSchemaFromRowType(RelDataType rowType) {
         List fields = new ArrayList<>();
         for (RelDataTypeField f : rowType.getFieldList()) {
-            ArrowType arrowType = toArrowType(f.getType().getSqlTypeName());
-            fields.add(new Field(f.getName(), new FieldType(true, arrowType, null), null));
+            fields.add(toArrowField(f.getName(), f.getType()));
         }
         return new Schema(fields);
     }
 
+    /**
+     * Build an Arrow {@link Field} from a Calcite type. For scalar types this is a
+     * leaf field with the appropriate {@link ArrowType}; for ARRAY this is a
+     * {@code List} whose single child is the recursively-converted element type
+     * (Arrow names the child {@code $data$} by convention — kept here for parity with
+     * Arrow's own builders so downstream tooling that walks list children by name
+     * doesn't break).
+     */
+    private static Field toArrowField(String name, RelDataType type) {
+        SqlTypeName sqlTypeName = type.getSqlTypeName();
+        if (sqlTypeName == SqlTypeName.ARRAY) {
+            RelDataType elementType = type.getComponentType();
+            if (elementType == null) {
+                throw new IllegalArgumentException(
+                    "ARRAY type with no component type for field [" + name + "]; cannot derive list element schema"
+                );
+            }
+            Field elementField = toArrowField("$data$", elementType);
+            return new Field(name, new FieldType(true, ArrowType.List.INSTANCE, null), List.of(elementField));
+        }
+        ArrowType arrowType = toArrowType(sqlTypeName);
+        return new Field(name, new FieldType(true, arrowType, null), null);
+    }
+
     private static ArrowType toArrowType(SqlTypeName sqlTypeName) {
         switch (sqlTypeName) {
             case BIGINT:
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java
index 4e80dfa82e55f..97aa10ff82ca2 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java
@@ -20,6 +20,8 @@
 import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -28,6 +30,7 @@
 
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -60,8 +63,7 @@ public VectorSchemaRoot decode(FragmentExecutionResponse response, BufferAllocat
         // Infer Arrow type per column from the first non-null value
         List fields = new ArrayList<>();
         for (int col = 0; col < fieldNames.size(); col++) {
-            ArrowType arrowType = inferArrowType(rows, col);
-            fields.add(new Field(fieldNames.get(col), FieldType.nullable(arrowType), null));
+            fields.add(inferField(fieldNames.get(col), rows, col));
         }
         Schema schema = new Schema(fields);
 
@@ -86,29 +88,76 @@ public VectorSchemaRoot decode(FragmentExecutionResponse response, BufferAllocat
     }
 
     /**
-     * Infers the Arrow type for a column by scanning rows for the first
-     * non-null value. Falls back to {@code Utf8} (VarChar) if all values
-     * are null or the Java type is unrecognized.
+     * Infers the Arrow {@link Field} for a column by scanning rows for the first
+     * non-null value. Falls back to a nullable {@code Utf8} (VarChar) field if all
+     * values are null or the Java type is unrecognized.
+     *
+     * 
            For {@link List} cells (produced by analytics-engine routes that emit
+     * array-typed values — PPL {@code array(...)}, {@code array_slice}, …), this
+     * returns a {@code List} field where the inner element type is inferred
+     * from the first non-null element. Without this branch, list values fall
+     * through to the {@code Utf8} fallback and {@link #setVectorValue} produces
+     * {@code value.toString()} (e.g. {@code "[2,3,4]"} as a JSON-like string)
+     * instead of a typed array.
      */
-    static ArrowType inferArrowType(List rows, int col) {
+    static Field inferField(String name, List rows, int col) {
         for (Object[] row : rows) {
             Object value = row[col];
             if (value == null) continue;
-            if (value instanceof Long) return new ArrowType.Int(64, true);
-            if (value instanceof Integer) return new ArrowType.Int(32, true);
-            if (value instanceof Short) return new ArrowType.Int(16, true);
-            if (value instanceof Byte) return new ArrowType.Int(8, true);
-            if (value instanceof Double) return new ArrowType.FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE);
-            if (value instanceof Float) return new ArrowType.FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE);
-            if (value instanceof Boolean) return ArrowType.Bool.INSTANCE;
-            if (value instanceof CharSequence) return ArrowType.Utf8.INSTANCE;
-            if (value instanceof byte[]) return ArrowType.Binary.INSTANCE;
-            if (value instanceof Number) return new ArrowType.Int(64, true);
-            break;
+            if (value instanceof List list) {
+                ArrowType elementType = inferElementArrowType(list, rows, col);
+                Field elementField = new Field("$data$", FieldType.nullable(elementType), null);
+                return new Field(name, FieldType.nullable(ArrowType.List.INSTANCE), Collections.singletonList(elementField));
+            }
+            return new Field(name, FieldType.nullable(scalarArrowType(value)), null);
+        }
+        return new Field(name, FieldType.nullable(ArrowType.Utf8.INSTANCE), null);
+    }
+
+    /**
+     * Best-effort inference for a list element type. Looks at the first non-null
+     * element of the given list, then falls back to scanning later rows of the
+     * same column if this list is empty or all-null. Defaults to {@code Utf8}.
+     */
+    private static ArrowType inferElementArrowType(List list, List rows, int col) {
+        for (Object element : list) {
+            if (element != null) return scalarArrowType(element);
+        }
+        for (Object[] row : rows) {
+            Object value = row[col];
+            if (value instanceof List other) {
+                for (Object element : other) {
+                    if (element != null) return scalarArrowType(element);
+                }
+            }
         }
         return ArrowType.Utf8.INSTANCE;
     }
 
+    /** Maps a Java scalar value to the corresponding Arrow scalar type. */
+    private static ArrowType scalarArrowType(Object value) {
+        if (value instanceof Long) return new ArrowType.Int(64, true);
+        if (value instanceof Integer) return new ArrowType.Int(32, true);
+        if (value instanceof Short) return new ArrowType.Int(16, true);
+        if (value instanceof Byte) return new ArrowType.Int(8, true);
+        if (value instanceof Double) return new ArrowType.FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE);
+        if (value instanceof Float) return new ArrowType.FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE);
+        // BigDecimal must be checked before the Number fallback below — a BigDecimal that
+        // would round to a long but actually carries fractional precision (e.g. PPL
+        // {@code array(1, -1.5)} where the common element type is DECIMAL and
+        // {@code ArrayImplementor.internalCast} produces BigDecimal cells) would otherwise
+        // get encoded as an integer Arrow vector and lose its fractional digits. Promote to
+        // DOUBLE — the same path the v2 engine takes for decimal-typed PPL results.
+        if (value instanceof java.math.BigDecimal) return new ArrowType.FloatingPoint(
+            org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE
+        );
+        if (value instanceof Boolean) return ArrowType.Bool.INSTANCE;
+        if (value instanceof CharSequence) return ArrowType.Utf8.INSTANCE;
+        if (value instanceof byte[]) return ArrowType.Binary.INSTANCE;
+        if (value instanceof Number) return new ArrowType.Int(64, true);
+        return ArrowType.Utf8.INSTANCE;
+    }
+
     /**
      * Sets a value on the appropriate Arrow vector type. Handles null by
      * calling {@code setNull}. For typed vectors, casts the Java value to
@@ -137,8 +186,64 @@ static void setVectorValue(FieldVector vector, int index, Object value) {
             ((VarCharVector) vector).setSafe(index, value.toString().getBytes(StandardCharsets.UTF_8));
         } else if (vector instanceof VarBinaryVector) {
             ((VarBinaryVector) vector).setSafe(index, (byte[]) value);
+        } else if (vector instanceof ListVector listVector) {
+            writeListValue(listVector, index, (List) value);
         } else {
             throw new IllegalArgumentException("Unsupported Arrow vector type: " + vector.getClass().getSimpleName());
         }
     }
+
+    /**
+     * Writes a Java {@link List} into an Arrow {@link ListVector} at the given row
+     * index. Bypasses {@link UnionListWriter} entirely — writes directly to the
+     * list's offset / validity buffers and to the inner data vector via the inner
+     * vector's own typed setter. The writer-based API requires an
+     * {@link org.apache.arrow.memory.ArrowBuf} per varchar element which Arrow
+     * couples to a release lifecycle that's tricky to get right (early close →
+     * use-after-free, no close → leak); the direct path avoids that altogether.
+     *
+     * 
            The inner data vector's type was decided in {@link #inferField} from the
+     * first non-null list element, so the {@code instanceof} dispatch here simply
+     * needs to match.
+     */
+    private static void writeListValue(ListVector listVector, int index, List list) {
+        FieldVector dataVector = listVector.getDataVector();
+        int startOffset = listVector.getOffsetBuffer().getInt((long) index * ListVector.OFFSET_WIDTH);
+        int writePos = startOffset;
+        for (Object element : list) {
+            // Grow the data vector if needed before writing. setSafe-style helpers handle this
+            // automatically per type, but we still need to pre-position the cursor.
+            if (element == null) {
+                dataVector.setNull(writePos);
+            } else if (dataVector instanceof BigIntVector v) {
+                v.setSafe(writePos, ((Number) element).longValue());
+            } else if (dataVector instanceof IntVector v) {
+                v.setSafe(writePos, ((Number) element).intValue());
+            } else if (dataVector instanceof SmallIntVector v) {
+                v.setSafe(writePos, ((Number) element).shortValue());
+            } else if (dataVector instanceof TinyIntVector v) {
+                v.setSafe(writePos, ((Number) element).byteValue());
+            } else if (dataVector instanceof Float8Vector v) {
+                v.setSafe(writePos, ((Number) element).doubleValue());
+            } else if (dataVector instanceof Float4Vector v) {
+                v.setSafe(writePos, ((Number) element).floatValue());
+            } else if (dataVector instanceof BitVector v) {
+                v.setSafe(writePos, ((Boolean) element) ? 1 : 0);
+            } else if (dataVector instanceof VarCharVector v) {
+                v.setSafe(writePos, element.toString().getBytes(StandardCharsets.UTF_8));
+            } else if (dataVector instanceof VarBinaryVector v) {
+                v.setSafe(writePos, (byte[]) element);
+            } else {
+                throw new IllegalArgumentException("Unsupported list element vector type: " + dataVector.getClass().getSimpleName());
+            }
+            writePos++;
+        }
+        // Mark this row's list as non-null and update its end offset.
+        listVector.setNotNull(index);
+        listVector.getOffsetBuffer().setInt((long) (index + 1) * ListVector.OFFSET_WIDTH, writePos);
+        // Keep the data vector's value count in sync so subsequent reads see the new tail.
+        if (writePos > dataVector.getValueCount()) {
+            dataVector.setValueCount(writePos);
+        }
+    }
 }
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ArrayFunctionIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ArrayFunctionIT.java
new file mode 100644
index 0000000000000..19cb0b076809b
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ArrayFunctionIT.java
@@ -0,0 +1,311 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * End-to-end coverage for the PPL array-construction and multivalue (mv*)
+ * functions on the analytics-engine route (PPL → CalciteRelNodeVisitor →
+ * Substrait → DataFusion). Mirrors the SQL plugin's
+ * {@code CalciteArrayFunctionIT} one-test-method-to-one for the subset of tests
+ * the analytics-engine path supports today.
+ *
+ * 
            Function surface exercised:
+ * 
              
+ *   
            • {@code array(...)} → DataFusion {@code make_array} via
+ *       {@link org.opensearch.be.datafusion.MakeArrayAdapter}.
              • 
+ *   
            • {@code array_length} → DataFusion native {@code array_length}.
              • 
+ *   
            • {@code mvindex(arr, from, to)} (range form) → DataFusion {@code array_slice}
+ *       via {@link org.opensearch.be.datafusion.ArraySliceAdapter} (BIGINT index
+ *       coerce + 0-based-{@code (start, length)} → 1-based-{@code (start, end)}).
              • 
+ *   
            • {@code mvindex(arr, N)} (single-element form) → DataFusion {@code array_element}
+ *       via {@link org.opensearch.be.datafusion.ArrayElementAdapter}.
              • 
+ *   
            • {@code mvdedup(arr)} → DataFusion native {@code array_distinct}.
              • 
+ *   
            • {@code mvjoin(arr, sep)} → DataFusion {@code array_to_string} via
+ *       {@link org.opensearch.be.datafusion.ArrayToStringAdapter}.
              • 
+ *   
            • {@code mvzip(left, right [, sep])} → custom Rust UDF {@code udf::mvzip}.
              • 
+ *   
            • {@code mvfind(arr, regex)} → custom Rust UDF {@code udf::mvfind}.
              • 
+ *   
            • {@code split(str, delim)} (returns array) → DataFusion {@code string_to_array}.
              • 
+ * 
            
+ *
+ * 
            The {@code calcs} dataset is used as a scan target; most tests build literal
+ * arrays inside {@code eval} so the field types don't matter — what matters is
+ * that the source is a parquet-backed index the analytics-engine planner can
+ * scan.
+ *
+ * 
            Tests for lambda-based functions ({@code transform}, {@code mvmap},
+ * {@code reduce}, {@code forall}, {@code exists}, {@code filter}) are
+ * intentionally absent: substrait extension YAML doesn't support declaring
+ * {@code func<…>} lambda-typed arguments, so those don't ship through the
+ * analytics-engine route in this PR. Empty-array tests are also absent —
+ * {@code array()} defaults to {@code ARRAY[UNKNOWN]} which substrait can't
+ * encode without the SQL companion {@code #5421} default to {@code VARCHAR}.
+ */
+public class ArrayFunctionIT extends AnalyticsRestTestCase {
+
+    private static final Dataset DATASET = new Dataset("calcs", "calcs");
+
+    private static boolean dataProvisioned = false;
+
+    private void ensureDataProvisioned() throws IOException {
+        if (dataProvisioned == false) {
+            DatasetProvisioner.provision(client(), DATASET);
+            dataProvisioned = true;
+        }
+    }
+
+    /** Base query template: pin to one row so every assertion runs against a single result row. */
+    private String oneRow() {
+        return "source=" + DATASET.indexName + " | head 1 ";
+    }
+
+    // ── array(...) constructor ──────────────────────────────────────────────
+
+    /** Mixed-numeric literal array — exercises the BigDecimal → Double row-codec
+     *  promotion (without it, decimal cells truncate to integers). */
+    public void testArray() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval result = array(1, -1.5, 2, 1.0) | fields result",
+            Arrays.asList(1.0, -1.5, 2.0, 1.0));
+    }
+
+    /** Mixed int+string literal array — Calcite widens to {@code ARRAY}
+     *  via {@code ArrayFunctionImpl.internalCast}. */
+    public void testArrayWithString() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval result = array(1, 'demo') | fields result",
+            Arrays.asList("1", "demo"));
+    }
+
+    // ── array_length ────────────────────────────────────────────────────────
+
+    public void testArrayLength() throws IOException {
+        assertFirstRowDouble(
+            oneRow() + "| eval arr = array(1, -1.5, 2, 1.0) | eval len = array_length(arr) | fields len",
+            4.0);
+    }
+
+    // ── mvindex range (array_slice) ─────────────────────────────────────────
+
+    /** {@code mvindex(arr, 1, 3)} — 0-based-(start, length) → DataFusion 1-based-(start, end inclusive)
+     *  via {@link org.opensearch.be.datafusion.ArraySliceAdapter}. Without the rewrite the result
+     *  would be {@code [1, 2, 3]} instead of the expected {@code [2, 3, 4]}. */
+    public void testMvindexRangePositive() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval arr = array(1, 2, 3, 4, 5) | eval result = mvindex(arr, 1, 3) | fields result",
+            Arrays.asList(2, 3, 4));
+    }
+
+    /** Negative indices — DataFusion's array_slice supports them natively. */
+    public void testMvindexRangeNegative() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval arr = array(1, 2, 3, 4, 5) | eval result = mvindex(arr, -3, -1) | fields result",
+            Arrays.asList(3, 4, 5));
+    }
+
+    public void testMvindexRangeFirstThree() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval arr = array(10, 20, 30, 40, 50) | eval result = mvindex(arr, 0, 2) | fields result",
+            Arrays.asList(10, 20, 30));
+    }
+
+    // ── mvindex single (array_element) ──────────────────────────────────────
+
+    /** {@code mvindex(arr, N)} with a single index — PPL emits Calcite's
+     *  {@code SqlStdOperatorTable.ITEM} which {@link org.opensearch.be.datafusion.ArrayElementAdapter}
+     *  renames to DataFusion {@code array_element} with a BIGINT-coerced 1-based index. */
+    public void testMvindexSingleElementPositive() throws IOException {
+        assertFirstRowDouble(
+            oneRow() + "| eval arr = array(10, 20, 30) | eval result = mvindex(arr, 1) | fields result",
+            20.0);
+    }
+
+    public void testMvindexSingleElementNegative() throws IOException {
+        assertFirstRowDouble(
+            oneRow() + "| eval arr = array(10, 20, 30) | eval result = mvindex(arr, -1) | fields result",
+            30.0);
+    }
+
+    // ── mvdedup (array_distinct) ────────────────────────────────────────────
+
+    public void testMvdedupWithDuplicates() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval arr = array(1, 2, 2, 3, 3, 3) | eval result = mvdedup(arr) | fields result",
+            Arrays.asList(1, 2, 3));
+    }
+
+    public void testMvdedupWithStrings() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval arr = array('a', 'b', 'a', 'c', 'b') | eval result = mvdedup(arr) | fields result",
+            Arrays.asList("a", "b", "c"));
+    }
+
+    public void testMvdedupAllDuplicates() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval arr = array(7, 7, 7) | eval result = mvdedup(arr) | fields result",
+            Arrays.asList(7));
+    }
+
+    // ── mvjoin (array_to_string) ────────────────────────────────────────────
+
+    public void testMvjoinWithStringArray() throws IOException {
+        assertFirstRowString(
+            oneRow() + "| eval result = mvjoin(array('a', 'b', 'c'), ',') | fields result",
+            "a,b,c");
+    }
+
+    public void testMvjoinWithStringifiedNumbers() throws IOException {
+        assertFirstRowString(
+            oneRow() + "| eval result = mvjoin(array('1', '2', '3'), ' | ') | fields result",
+            "1 | 2 | 3");
+    }
+
+    public void testMvjoinWithSpecialDelimiters() throws IOException {
+        assertFirstRowString(
+            oneRow() + "| eval result = mvjoin(array('x', 'y'), '-->') | fields result",
+            "x-->y");
+    }
+
+    // ── mvzip (Rust UDF) ────────────────────────────────────────────────────
+
+    public void testMvzipBasic() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval result = mvzip(array('a', 'b', 'c'), array('1', '2', '3')) | fields result",
+            Arrays.asList("a,1", "b,2", "c,3"));
+    }
+
+    public void testMvzipWithCustomDelimiter() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval result = mvzip(array('a', 'b'), array('1', '2'), '-') | fields result",
+            Arrays.asList("a-1", "b-2"));
+    }
+
+    public void testMvzipNested() throws IOException {
+        assertFirstRowList(
+            oneRow()
+                + "| eval r = mvzip(mvzip(array('a','b'), array('1','2')), array('x','y')) | fields r",
+            Arrays.asList("a,1,x", "b,2,y"));
+    }
+
+    // ── mvfind (Rust UDF) ───────────────────────────────────────────────────
+
+    /** Returns the 0-based index of the first array element matching the regex. */
+    public void testMvfindWithMatch() throws IOException {
+        assertFirstRowDouble(
+            oneRow() + "| eval result = mvfind(array('apple', 'banana', 'cherry'), 'ban.*') | fields result",
+            1.0);
+    }
+
+    public void testMvfindWithNoMatch() throws IOException {
+        assertFirstRowNull(
+            oneRow() + "| eval result = mvfind(array('apple', 'banana'), 'zzz') | fields result");
+    }
+
+    /** Dynamic regex — exercises the {@code SqlLibraryOperators.CONCAT_FUNCTION} → substrait
+     *  {@code concat} Sig bridge added in this PR. Without that bridge the call fails substrait
+     *  conversion with {@code Unable to convert call CONCAT(string, string)}. */
+    public void testMvfindWithDynamicRegex() throws IOException {
+        assertFirstRowDouble(
+            oneRow()
+                + "| eval result = mvfind(array('apple', 'banana', 'cherry'), concat('ban', '.*')) | fields result",
+            1.0);
+    }
+
+    // ── split (returns array of strings) ─────────────────────────────────
+
+    public void testSplitWithSemicolonDelimiter() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval result = split('a;b;c', ';') | fields result",
+            Arrays.asList("a", "b", "c"));
+    }
+
+    public void testSplitWithMultiCharDelimiter() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval result = split('a::b::c', '::') | fields result",
+            Arrays.asList("a", "b", "c"));
+    }
+
+    // ── helpers ─────────────────────────────────────────────────────────────
+
+    /** Numeric-tolerant list comparison — Jackson parses JSON numbers as
+     *  Integer/Long/Double interchangeably, so equality on cross-type numbers
+     *  fails even when values match. Compare via {@link Double#compare} on
+     *  numeric pairs and {@link Object#equals} otherwise. */
+    private void assertFirstRowList(String ppl, List expected) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertNotNull("Expected non-null array result for query [" + ppl + "]", cell);
+        assertTrue(
+            "Expected list result for query [" + ppl + "] but got: " + cell + " (" + cell.getClass() + ")",
+            cell instanceof List);
+        List actual = (List) cell;
+        assertEquals(
+            "Length mismatch for query [" + ppl + "]: expected " + expected + " but got " + actual,
+            expected.size(),
+            actual.size());
+        for (int i = 0; i < expected.size(); i++) {
+            assertCellEquals(expected.get(i), actual.get(i));
+        }
+    }
+
+    private void assertFirstRowDouble(String ppl, double expected) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertTrue("Expected numeric result for query [" + ppl + "] but got: " + cell, cell instanceof Number);
+        assertEquals("Value mismatch for query: " + ppl, expected, ((Number) cell).doubleValue(), 1e-9);
+    }
+
+    private void assertFirstRowString(String ppl, String expected) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertEquals("Value mismatch for query: " + ppl, expected, cell);
+    }
+
+    private void assertFirstRowNull(String ppl) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertNull("Expected null result for query [" + ppl + "] but got: " + cell, cell);
+    }
+
+    private static void assertCellEquals(Object expected, Object actual) {
+        if (expected == null || actual == null) {
+            assertEquals(expected, actual);
+            return;
+        }
+        if (expected instanceof Number && actual instanceof Number) {
+            assertEquals(
+                "Numeric value mismatch",
+                ((Number) expected).doubleValue(),
+                ((Number) actual).doubleValue(),
+                1e-9);
+            return;
+        }
+        assertEquals(expected, actual);
+    }
+
+    private Object firstRowFirstCell(String ppl) throws IOException {
+        Map response = executePpl(ppl);
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) response.get("rows");
+        assertNotNull("Response missing 'rows' for query: " + ppl, rows);
+        assertTrue("Expected at least one row for query: " + ppl, rows.size() >= 1);
+        return rows.get(0).get(0);
+    }
+
+    private Map executePpl(String ppl) throws IOException {
+        ensureDataProvisioned();
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}");
+        Response response = client().performRequest(request);
+        return assertOkAndParse(response, "PPL: " + ppl);
+    }
+}
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MVAppendFunctionIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MVAppendFunctionIT.java
new file mode 100644
index 0000000000000..c4ada7cf538c7
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MVAppendFunctionIT.java
@@ -0,0 +1,180 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * End-to-end coverage for PPL {@code mvappend(arg1, arg2, …)} on the
+ * analytics-engine route. Mirrors the SQL plugin's
+ * {@code CalciteMVAppendFunctionIT} one-test-method-to-one for the subset of
+ * tests that pass on the analytics-engine path.
+ *
+ * 
            {@code mvappend} flattens an arbitrary mix of scalar and array operands
+ * into a single array, dropping null elements. Onboarded as a custom Rust UDF
+ * ({@code udf::mvappend}) registered at session-context creation; the Java
+ * adapter ({@link org.opensearch.be.datafusion.MvappendAdapter}) reshapes scalar
+ * operands into singleton {@code make_array} calls so substrait's variadic-{@code any1}
+ * shape sees a uniform {@code list[componentType]} across every position.
+ *
+ * 
            Tests covering genuinely heterogeneous mvappend signatures
+ * ({@code mvappend(1, 'text', 2.5)}, {@code mvappend(age, 'years', 'old')},
+ * {@code mvappend('test', nullif(1,1), 2)}) are absent because Calcite legitimately
+ * widens those to {@code ARRAY[ANY]} — substrait can't encode {@code ANY}, and
+ * Arrow's Union arrays aren't operated on by {@code datafusion-functions-array}.
+ * Empty-array operand tests are also absent — the empty {@code array()} default
+ * surfaces as {@code ARRAY[UNKNOWN]}/{@code ARRAY[VARCHAR]} in the column ref,
+ * which type-inference can't reach back through the project chain to ignore.
+ *
+ * 
            The {@code testMvappendInWhereClause} variant (filter predicate on an
+ * ARRAY field) is also absent because the analytics-engine planner's filter
+ * rule rejects {@code EQUALS} on an ARRAY field without walking into the
+ * predicate tree — that's a separate planner refactor tracked under #21554's
+ * "What's left" section.
+ */
+public class MVAppendFunctionIT extends AnalyticsRestTestCase {
+
+    private static final Dataset DATASET = new Dataset("calcs", "calcs");
+
+    private static boolean dataProvisioned = false;
+
+    private void ensureDataProvisioned() throws IOException {
+        if (dataProvisioned == false) {
+            DatasetProvisioner.provision(client(), DATASET);
+            dataProvisioned = true;
+        }
+    }
+
+    private String oneRow() {
+        return "source=" + DATASET.indexName + " | head 1 ";
+    }
+
+    // ── uniform-typed scalar variadic ───────────────────────────────────────
+
+    public void testMvappendWithMultipleElements() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval result = mvappend(1, 2, 3) | fields result",
+            Arrays.asList(1, 2, 3));
+    }
+
+    public void testMvappendWithSingleElement() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval result = mvappend(42) | fields result",
+            Arrays.asList(42));
+    }
+
+    public void testMvappendWithStringValues() throws IOException {
+        assertFirstRowList(
+            oneRow() + "| eval result = mvappend('hello', 'world') | fields result",
+            Arrays.asList("hello", "world"));
+    }
+
+    // ── array operands (uniform element type) ───────────────────────────────
+
+    public void testMvappendWithArrayFlattening() throws IOException {
+        assertFirstRowList(
+            oneRow()
+                + "| eval arr1 = array(1, 2), arr2 = array(3, 4), result = mvappend(arr1, arr2) | fields result",
+            Arrays.asList(1, 2, 3, 4));
+    }
+
+    public void testMvappendWithNestedArrays() throws IOException {
+        assertFirstRowList(
+            oneRow()
+                + "| eval arr1 = array('a', 'b'), arr2 = array('c'), arr3 = array('d', 'e'),"
+                + " result = mvappend(arr1, arr2, arr3) | fields result",
+            Arrays.asList("a", "b", "c", "d", "e"));
+    }
+
+    // ── field references ────────────────────────────────────────────────────
+
+    /** Two VARCHAR field references → uniform {@code ARRAY[VARCHAR]}. Anchored
+     *  to a specific row by filtering on {@code key} so the assertion is
+     *  deterministic. */
+    public void testMvappendWithRealFields() throws IOException {
+        assertFirstRowList(
+            "source=" + DATASET.indexName
+                + " | where key='key00' | head 1 | eval result = mvappend(str0, str1) | fields result",
+            // calcs row key00: str0='FURNITURE', str1='CLAMP ON LAMPS'
+            Arrays.asList("FURNITURE", "CLAMP ON LAMPS"));
+    }
+
+    // ── tests gated on SQL companion #5424 ──────────────────────────────────
+    // The following SQL-side tests are intentionally absent until
+    // opensearch-project/sql#5424 (the {@code MVAppendFunctionImpl} widening
+    // via {@code leastRestrictive} + DECIMAL → DOUBLE promotion + operand
+    // pre-cast in {@code MVAppendImplementor}) is merged and republished as
+    // {@code unified-query-core:3.7.0.0-SNAPSHOT}. Without it, these collapse
+    // to {@code ARRAY[ANY]} which substrait can't encode:
+    //
+    //   testMvappendWithMixedArrayAndScalar — array(1,2), 3, 4 (nullability bridge)
+    //   testMvappendWithNumericArrays       — array(1.5,2.5), array(3.5), 4.5 (nullability bridge)
+    //   testMvappendWithIntAndDouble        — 1, 2.5 (DECIMAL → DOUBLE promotion + pre-cast)
+    //   testMvappendWithComplexExpression   — array(int0), array(int0*2), int0+10 (nullability bridge)
+    //
+    // Add them back once #5424 lands. Their SQL-side counterparts are verified
+    // in CalciteMVAppendFunctionIT against the analytics-engine route.
+
+    // ── helpers ─────────────────────────────────────────────────────────────
+
+    private void assertFirstRowList(String ppl, List expected) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertNotNull("Expected non-null array result for query [" + ppl + "]", cell);
+        assertTrue(
+            "Expected list result for query [" + ppl + "] but got: " + cell + " (" + cell.getClass() + ")",
+            cell instanceof List);
+        List actual = (List) cell;
+        assertEquals(
+            "Length mismatch for query [" + ppl + "]: expected " + expected + " but got " + actual,
+            expected.size(),
+            actual.size());
+        for (int i = 0; i < expected.size(); i++) {
+            assertCellEquals(expected.get(i), actual.get(i));
+        }
+    }
+
+    private static void assertCellEquals(Object expected, Object actual) {
+        if (expected == null || actual == null) {
+            assertEquals(expected, actual);
+            return;
+        }
+        if (expected instanceof Number && actual instanceof Number) {
+            assertEquals(
+                "Numeric value mismatch",
+                ((Number) expected).doubleValue(),
+                ((Number) actual).doubleValue(),
+                1e-9);
+            return;
+        }
+        assertEquals(expected, actual);
+    }
+
+    private Object firstRowFirstCell(String ppl) throws IOException {
+        Map response = executePpl(ppl);
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) response.get("rows");
+        assertNotNull("Response missing 'rows' for query: " + ppl, rows);
+        assertTrue("Expected at least one row for query: " + ppl, rows.size() >= 1);
+        return rows.get(0).get(0);
+    }
+
+    private Map executePpl(String ppl) throws IOException {
+        ensureDataProvisioned();
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}");
+        Response response = client().performRequest(request);
+        return assertOkAndParse(response, "PPL: " + ppl);
+    }
+}

From b7f49bd147affe1d51b085a8735459000e491823 Mon Sep 17 00:00:00 2001
From: Sandesh Kumar 
Date: Sun, 10 May 2026 11:51:06 -0700
Subject: [PATCH 104/115] [analytics-backend-datafusion] Implement
 partial/final aggregation mode for distributed execution (#21457)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* feat(spi): add intermediateFields and finalExpression to AggregateFunction

Extend AggregateFunction enum with two nullable fields that describe
how each function decomposes into partial+final phases for distributed
execution:

  - intermediateFields: List — Arrow schema of the
    partial output per field, with per-field reducer (SUM, DC, etc.)
  - finalExpression: BiFunction, RexNode> —
    scalar expression over partial columns for primitive-decomposition
    cases (AVG = sum/count)

Four encoding cases are expressible on the same shape:
  - null / null              → pass-through (SUM, MIN, MAX)
  - [one field, reducer==self], null → engine-native merge (DC)
  - [one field, reducer!=self], null → function-swap at FINAL (COUNT→SUM)
  - [N fields], non-null     → primitive decomp + Project (AVG)

This makes AggregateFunction the single source of truth for per-function
decomposition knowledge. Downstream layers (DAGBuilder,
FragmentConversionDriver, LocalStageScheduler) no longer need any
per-function if-branches — they read the enum via the decomposition
resolver.

Self-reference (APPROX_COUNT_DISTINCT as its own reducer) works around
Java's enum-initializer restriction by using null as a "self" sentinel
in the raw field; the intermediateFields() accessor resolves null back
to the owning constant.

Ref: .kiro/docs/distributed-aggregate-design.md §6
Signed-off-by: Sandesh Kumar 

* feat(planner): add ArrowCalciteTypes utility for bidirectional type mapping

Single source for Arrow↔Calcite type conversion in the planner:
  Int(64) ↔ BIGINT
  Int(32) ↔ INTEGER
  FloatingPoint(DOUBLE) ↔ DOUBLE
  FloatingPoint(SINGLE) ↔ REAL/FLOAT
  Utf8 ↔ VARCHAR(max)
  Binary ↔ VARBINARY(max)
  Bool ↔ BOOLEAN

This utility will be consumed by the AggregateDecompositionResolver to
derive Calcite aggregate-call return types from AggregateFunction.
intermediateFields — replacing scattered calls to Calcite's stock
SqlAggFunction.inferReturnType (which produces inconsistent types for
AVG's sum field and DC's binary sketch between the Calcite view and
DataFusion's actual output).

Uses JDK 21 switch expressions with pattern matching for type
discrimination. Unsupported types throw IllegalArgumentException with
the offending type in the message.

Note on VARCHAR/VARBINARY: we pass Integer.MAX_VALUE to request
unbounded precision; Calcite's type factory clamps to its own internal
max (65536 by default). The tests assert equality with the factory's
reported max, since that's the invariant we actually care about —
'unbounded' VARCHAR/VARBINARY.

Ref: .kiro/docs/distributed-aggregate-design.md §8
Signed-off-by: Sandesh Kumar 

* feat(datafusion-rust): add agg_mode module and disable CombinePartialFinalAggregate

Introduce the Rust physical-layer infrastructure for distributed
partial/final aggregate execution:

  - agg_mode.rs: internal Mode enum (Default | Partial | Final),
    force_aggregate_mode() walks Final(Partial(...)) trees and strips
    one half (preserving RepartitionExec / CoalescePartitionsExec
    between), find_partial_input() drills past repartition nodes to
    locate the inner Partial.
  - physical_optimizer_rules_without_combine(): returns the default
    DataFusion physical optimizer rules with CombinePartialFinalAggregate
    removed by name, so the Final(Partial(...)) pair survives to our
    strip pass.
  - SessionContext (shard) and LocalSession (coordinator) now build
    with the new optimizer rule set.
  - SessionContextHandle gains two fields for the prepared-plan path:
      aggregate_mode: Mode (default Default)
      prepared_plan: Option> (default None)
    These will be written by prepare_partial_plan / prepare_final_plan
    entry points in a follow-up commit.

Invariants enforced:
  - Mode is pub(crate) — never exposed across FFI.
  - No new pub extern "C" fn signatures here (that's Task 5).
  - No Java files modified.
  - No mode enum int crosses the boundary.

Five Rust unit tests cover strip-partial, strip-final, past-repartition,
past-coalesce, and combine-rule-absent.

DataFusion API note: default rules are obtained from
PhysicalOptimizer::new().rules; with_physical_optimizer_rules on
SessionStateBuilder fully replaces the optimizer (no additive behavior
needed).

Ref: .kiro/docs/distributed-aggregate-design.md §13
Signed-off-by: Sandesh Kumar 

* feat(planner): add AggregateDecompositionResolver pass

The one place per-function decomposition logic lives. Walks the DAG
after backend resolution and rewrites PARTIAL/FINAL aggregate pairs
driven entirely by the AggregateFunction enum's intermediateFields
and finalExpression.

Four cases handled uniformly, no function-name branches:

  1. Pass-through (intermediateFields == null): SUM, MIN, MAX stay
     unchanged; FINAL's arg is rebound to the next column.
  2. Engine-native merge (one field, reducer == self): DC keeps its
     aggregate call at FINAL; exchange row type on StageInputScan
     carries VARBINARY for the sketch column.
  3. Function-swap (one field, reducer != self): COUNT's FINAL
     becomes SUM(count_col); PARTIAL keeps COUNT.
  4. Primitive decomposition (N fields, finalExpression != null):
     AVG's PARTIAL emits SUM(count)+SUM(sum); FINAL reduces each
     with SUM; LogicalProject wraps FINAL to apply finalExpression
     (sum/count) cast to the original call's return type.

Integration point: DefaultPlanExecutor.executeInternal between
BackendPlanAdapter.adaptAll and FragmentConversionDriver.convertAll.

Calcite constraint noted: Aggregate.typeMatchesInferred asserts each
AggregateCall's type matches SqlAggFunction.inferReturnType(...). We
cannot retype PARTIAL's calls directly. Instead, the exchange
contract (what flows over the wire and what FINAL reads) is carried
by the parent stage's StageInputScan row type, derived from
intermediateFields via ArrowCalciteTypes. DataFusion's compiler
ignores the Substrait-declared PARTIAL row type for well-known
aggregates (it uses its own internal StateFields), so Calcite's
inferred types at PARTIAL are harmless.

Seven unit tests cover: pass-through SUM, function-swap COUNT,
engine-native DC, primitive-decomp AVG, mixed Q10, group-keys flow
through, no-Calcite-inference regression guard.

Ref: .kiro/docs/distributed-aggregate-design.md §7
Signed-off-by: Sandesh Kumar 

* feat(datafusion): FFI prepare_partial_plan / prepare_final_plan / execute_local_prepared_plan

Three named FFI entry points wiring the prepared-plan lifecycle
between Java and Rust. No mode enum crosses the boundary \u2014 Rust
sets Mode::Partial / Mode::Final internally based on which function
Java called.

Rust side:
  - SessionContextHandle gets prepare_partial_plan(): decodes
    Substrait bytes, converts to physical plan, applies
    agg_mode::apply_aggregate_mode(Partial), stores in
    handle.prepared_plan.
  - LocalSession gains a prepared_plan field + prepare_final_plan()
    + execute_prepared() that streams from the stored plan.
  - ffm.rs exposes three new pub extern "C" fn entries following
    the existing convention (i64 return; >=0 success, <0 negated
    error pointer):
      df_prepare_partial_plan(handle_ptr, bytes_ptr, bytes_len)
      df_prepare_final_plan(session_ptr, bytes_ptr, bytes_len)
      df_execute_local_prepared_plan(session_ptr)

Java side:
  - NativeBridge adds three MethodHandle fields + three public
    wrappers (preparePartialPlan, prepareFinalPlan,
    executeLocalPreparedPlan) following the existing FFM pattern
    used by EXECUTE_WITH_CONTEXT.

Invariants enforced:
  - grep 'Mode' ffm.rs: zero matches in any pub extern "C" fn.
  - No aggregate_mode reference on the Java side.
  - agg_mode.rs untouched.
  - Mode stays pub(crate).

Tests:
  - Rust: prepare_partial_plan_sets_mode_and_stores_plan,
    prepare_final_plan_stores_plan (484 total Rust tests pass).
  - Java: NativeBridgePreparedPlanTests validates null-pointer
    handling and confirms MethodHandles resolve against the native
    symbols.

Substrait decoding: uses the existing datafusion-substrait
from_substrait_plan(&state, &plan) pattern already present in
execute_substrait / execute_with_context.

Ref: .kiro/docs/distributed-aggregate-design.md \u00a74.3, \u00a711.3, \u00a713
Signed-off-by: Sandesh Kumar 

* feat(datafusion): wire prepared-plan path into coordinator reduce sink

Completes Task 6 of the distributed-aggregate feature: the reduce
sink now consumes the DataFusionReduceState produced by
FinalAggregateInstructionHandler and drives executeLocalPreparedPlan
against the handler's session instead of re-decoding the fragment
bytes on a fresh session.

  AbstractDatafusionReduceSink
    - New constructor taking nullable DataFusionReduceState. When
      present, the state owns session + senders; otherwise the base
      class creates a session as before.
    - close() skips closing session when state != null so the state's
      close() handles it (avoids double-close on the native side).

  DatafusionReduceSink
    - New 3-arg constructor (ctx, runtimeHandle, preparedState).
    - When state is non-null: reuse state's session + senders (indexed
      back to childStageId via ctx.childInputs() iteration order) and
      call executeLocalPreparedPlan.
    - When state is null: legacy path unchanged (register partitions,
      executeLocalPlan). This path is exercised by non-aggregate
      reduce stages where no FinalAggregate instruction ran.

  DataFusionAnalyticsBackendPlugin.getExchangeSinkProvider
    - Casts backendContext to DataFusionReduceState and passes it
      through. Memtable sink is bypassed when a preparedState is
      present (memtable sink doesn't yet support prepared-plan path).

  DataFusionInstructionHandlerFactory
    - Missing import for PartialAggregateInstructionNode added.

  FilterDelegationForIndexFullConversionTests (mock)
    - Updated to new ExchangeSinkProvider.createSink two-arg signature.

Ref: .kiro/docs/distributed-aggregate-design.md \u00a711
Signed-off-by: Sandesh Kumar 

* feat(datafusion): emit approx_distinct via Substrait extension YAML

Replace the post-emit proto-rewrite (renameExtensionFunctions +
FUNCTION_RENAMES) with a declarative Substrait extension approach:

  - New resource: opensearch_aggregate_functions.yaml
      URN  extension:org.opensearch:aggregate_functions
      declares approx_distinct(any) -> i64

  - DataFusionPlugin.loadSubstraitExtensions now merges the aggregate
    YAML alongside delegation and scalar YAMLs.

  - DataFusionFragmentConvertor:
      - New ADDITIONAL_AGGREGATE_SIGS binding the custom
        APPROX_DISTINCT SqlAggFunction to the extension name
        "approx_distinct".
      - Custom APPROX_DISTINCT operator avoids collision with
        Substrait's default approx_count_distinct mapping (default
        catalog shadows FunctionMappings.Sig entries on the stock
        Calcite operator).
      - New rewriteApproxCountDistinct RelShuttle swaps
        SqlStdOperatorTable.APPROX_COUNT_DISTINCT to APPROX_DISTINCT
        on every AggregateCall before Substrait emission. Type is
        preserved (BIGINT NOT NULL) so Aggregate.typeMatchesInferred
        passes.
      - Switched to the 4-arg AggregateFunctionConverter constructor
        to pass ADDITIONAL_AGGREGATE_SIGS.
      - Deleted FUNCTION_RENAMES map + renameExtensionFunctions proto
        walk + serialize-time rename calls.

  - Test extensions catalog now loads the aggregate YAML alongside
    delegation.

Why the custom operator instead of a direct Sig on the stock function:
Substrait's default catalog declares approx_count_distinct under the
standard URN, so isthmus resolves stock Calcite APPROX_COUNT_DISTINCT
through the default mapping first and our additional Sig is shadowed.
A fresh SqlAggFunction with no prior mapping routes cleanly through
ADDITIONAL_AGGREGATE_SIGS.

All 14 DataFusionFragmentConvertor tests pass (including the now
type-correct testApproxCountDistinctRenamed which asserts both
absence of approx_count_distinct AND presence of approx_distinct in
the emitted extension declarations).

Ref: .kiro/docs/distributed-aggregate-design.md \u00a710
Signed-off-by: Sandesh Kumar 

* feat(spi+execution): complete handler infrastructure for prepared-plan path

Hooks up the last mile of the distributed-aggregate instruction-handler
chain. Completes Task 6 by landing the scaffolding that was on the pf4
tree as uncommitted modifications:

  ExchangeSinkProvider.createSink
    Signature change: (ExchangeSinkContext) -> (ExchangeSinkContext,
    BackendExecutionContext). Sink providers receive the backend-opaque
    state produced by the last instruction handler, so the reduce sink
    can drive the already-prepared plan instead of re-decoding the
    fragment bytes.

  FinalAggregateInstructionHandler
    Real implementation (was TODO-only). Creates the DatafusionLocalSession,
    registers one input partition per child stage via
    NativeBridge.registerPartitionStream, calls
    NativeBridge.prepareFinalPlan, and returns a DataFusionReduceState
    bundling session + runtime + senders for the reduce sink.
    Failure path closes any partially-allocated senders + session before
    rethrowing.

  PartialAggregateInstructionHandler (new class)
    Calls NativeBridge.preparePartialPlan on the already-open session
    created by the preceding ShardScanInstructionHandler. Rust side
    sets Mode::Partial and stores the prepared plan on the handle.

  DataFusionReduceState (new class)
    BackendExecutionContext implementation carrying the local session,
    native runtime handle, and partition-sender list. close() tears
    down senders first, then the session, matching the allocation order
    in FinalAggregateInstructionHandler's failure path.

  LocalStageScheduler
    Hoists backendContext out of the try-finally so it's in scope for
    the provider.createSink(context, backendContext) call. Sink is now
    responsible for holding / closing backendContext on success; the
    scheduler only closes it on instruction-apply or sink-creation
    failure.

With these changes the end-to-end path is now connected:
  ShardScan -> opens SessionContext
  PartialAggregate -> NativeBridge.preparePartialPlan (sets Mode::Partial,
                       stores plan) — executed by DataFusion when the
                       scan driver runs its Substrait execute_with_context
  FinalAggregate -> creates LocalSession, registers senders,
                     NativeBridge.prepareFinalPlan (stores Final-stripped
                     plan)
  Sink receives DataFusionReduceState -> NativeBridge.executeLocalPreparedPlan

Ref: .kiro/docs/distributed-aggregate-design.md \u00a711, \u00a714
Signed-off-by: Sandesh Kumar 

* feat(datafusion): register APPROX_COUNT_DISTINCT aggregate capability

Add APPROX_COUNT_DISTINCT to the DataFusion backend's aggregate
capability set so the planner recognises it as a supported aggregate
on this backend (alongside SUM, SUM0, MIN, MAX, COUNT, AVG).

Switch the capability loop from AggregateCapability.simple(...) to the
3-arg AggregateCapability constructor:
  - The per-type factory helpers (simple, approximate, statistical,
    stateExpanding) assert on AggregateFunction.Type. SUM et al. are
    SIMPLE; APPROX_COUNT_DISTINCT is APPROXIMATE — splitting by type
    would branch the loop.
  - The 3-arg constructor accepts any Type and leaves
    decomposition=null so the AggregateDecompositionResolver falls
    back to the enum's intermediateFields + finalExpression (the
    single source of truth for partial/final decomposition).
  - No per-function if/else anywhere: adding future functions
    (STDDEV_POP, VAR_POP) is a one-line addition to AGG_FUNCTIONS.

No other changes: the existing cartesian product across SUPPORTED_FIELD_TYPES
is unchanged. Calcite's operand-type checker filters nonsensical
combinations (e.g. SUM on VARCHAR) at planning time.

Ref: .kiro/docs/distributed-aggregate-design.md \u00a75.4, \u00a714
Signed-off-by: Sandesh Kumar 

* test: extend CoordinatorReduceIT with partial/final aggregate coverage

Adopts pf2's CoordinatorReduceIT test shapes (ported to the REST-based
AnalyticsRestTestCase framework already in sandbox/qa/analytics-engine-rest)
to exercise every branch of the AggregateDecompositionResolver's
four-case rewrite end-to-end:

  testScalarSumAcrossShards      (pass-through)         — pre-existing
  testScalarCountAcrossShards    (function-swap COUNT→SUM)
  testAvgAcrossShards            (primitive decomp + Project)
  testDistinctCountAcrossShards  (engine-native HLL merge)
  testGroupedSumAcrossShards     (group keys flow through)
  testQ10ShapeAcrossShards       (all four families, grouped)

testQ10ShapeAcrossShards is unignored in pf4. pf2 had @Ignore on this
test because its decomposeFinalFragment mishandled parent Project
expressions after decomposition (the scattered per-layer rewrite that
pf4 replaces). pf4's single-pass AggregateDecompositionResolver builds
the Project wrapper in-place from intermediateFields + finalExpression
at the point of decomposition, so Q10 works end-to-end.

Implementation refactors:
  - extracted scalarRows() helper used by each scalar-shape test to
    assert column presence, row count, and non-null cell; tests assert
    only the value semantics.
  - indexing split into constant-value (for SUM/COUNT/AVG predictability)
    and varying-value (needed for DC to have a meaningful cardinality)
    helpers sharing a single bulkAndRefresh path.
  - parquet-backed index creation parameterised by name so DC uses its
    own index (distinct values) without colliding with other tests'
    constant-value data.

Runs via:
  ./gradlew :sandbox:qa:analytics-engine-rest:integTest -Dsandbox.enabled=true

Ref: .kiro/docs/distributed-aggregate-design.md \u00a720 testing matrix
Signed-off-by: Sandesh Kumar 

* fix(datafusion-rust): disable CombinePartialFinalAggregate on IndexedExec SessionContext

Invariant 7 audit turned up one SessionContext construction site
that bypassed physical_optimizer_rules_without_combine() — the
indexed_executor path used for IndexedExec queries. Without the
custom rule set, CombinePartialFinalAggregate can collapse
Final(Partial(...)) pairs, defeating force_aggregate_mode on any
aggregate query routed through this executor.

Aligns with session_context.rs (shard, both callsites) and
local_executor.rs (coordinator reduce), which already configure the
same helper.

Ref: .kiro/docs/distributed-aggregate-design.md \u00a717 invariant 7
Signed-off-by: Sandesh Kumar 

* style: apply spotless formatting

Mechanical run of spotlessApply across analytics-framework,
analytics-engine, and analytics-backend-datafusion. One non-mechanical
fix: replaced the wildcard static import in AggregateFunctionTests
with explicit per-constant imports (spotless can't auto-fix wildcards).

No behavior change. All targeted unit tests remain green.

Signed-off-by: Sandesh Kumar 

* scaffold(planner): OpenSearchAggregateReduceRule (wiring deferred)

Introduces the subclass of Calcite's AggregateReduceFunctionsRule that,
when wired into PlannerImpl's HEP marking phase, will decompose AVG /
STDDEV_POP / STDDEV_SAMP / VAR_POP / VAR_SAMP into primitive SUM/COUNT
(+ SUM_SQ for variance) calls wrapped by a scalar Project — replacing
most of AggregateDecompositionResolver's primitive-decomp logic with
Calcite's tested implementation.

Customisations over the stock rule:
  - matches OpenSearchAggregate only (not LogicalAggregate)
  - restricts to AggregateMode.SINGLE so it does not re-fire on
    PARTIAL/FINAL produced by OpenSearchAggregateSplitRule
  - overrides newAggregateRel to rebuild as OpenSearchAggregate,
    preserving mode and viableBackends so downstream marking /
    split rules continue to pattern-match on the reduced inner
    aggregate

Wiring deferred: when this rule fires during marking, Calcite inserts
a CAST around the DIVIDE result (to match AVG's original return type)
and a LogicalProject above the reduced aggregate. OpenSearchProjectRule
then enforces that every RexCall in the Project has a backend declaring
the corresponding ScalarFunction capability — but CAST is not declared
by any backend (including MockDataFusionBackend in test scaffolding).

Before wiring this rule into PlannerImpl we need to decide how CAST /
DIVIDE / TIMES should be treated by the capability system:
  Option 1: treat CAST as an implicit / always-supported operator in
            OpenSearchProjectRule, bypassing scalar-capability lookup
            (CAST is a query-semantics primitive, not a function).
  Option 2: declare CAST / DIVIDE / TIMES on every backend (real + mock)
            that claims any scalar capability.
  Option 3: keep the current hand-rolled primitive-decomp path and
            accept its ~250 LOC as the cost of avoiding the above.

Until that decision is made, this class compiles and is unit-testable
in isolation but is not referenced by PlannerImpl. AggregateDecomposition
Resolver continues to handle AVG primitive-decomp end-to-end.

Ref: .kiro/docs/distributed-aggregate-design.md §7
Signed-off-by: Sandesh Kumar 

* feat(planner): carve out baseline scalar operators from capability enforcement

Introduces a BASELINE_SCALAR_OPS set in OpenSearchProjectRule covering
SQL-execution primitives (arithmetic, CAST, null handling, conditional)
that every viable backend is implicitly assumed to support. These ops
bypass the scalar-capability registry and flow through OpenSearchProject
without backend annotation.

Motivation
----------
Calcite plan-rewrite rules (AggregateReduceFunctionsRule,
ReduceExpressionsRule, future type-coercion rules) routinely introduce
arithmetic / CAST / CASE / null-predicate operators while rewriting
expressions. These are not optional backend features — they are
primitives that every query engine must support to be viable at all.
Modeling them as capability-declared created two failure modes:

  1. Every new backend had to enumerate ~20 operators that are never
     actually optional. Forgetting one produces a plan-time failure
     on queries that don't even reference the forgotten op — they just
     happen to trigger a Calcite rule that emits it.
  2. Any Calcite rule that incidentally emits one of these ops (e.g.
     the CAST around SUM(x)/COUNT(x) that AggregateReduceFunctionsRule
     emits to match AVG's original return type) fails plan-time checks
     with a misleading 'No backend supports scalar function [CAST]'
     error — even though the query semantics are unambiguous and every
     backend executes CAST natively.

Aligns with how other SQL engines model this concern (PostgreSQL,
DuckDB, Presto do not ship capability registries for arithmetic or
CAST — these are execution primitives, not optional features).

Scope
-----
Carved out:
  PLUS, MINUS, MULTIPLY, DIVIDE, UNARY_MINUS, UNARY_PLUS  (arithmetic)
  CAST                                                    (type coercion)
  IS_NULL, IS_NOT_NULL, COALESCE, CASE                    (null / conditional)

Intentionally conservative: comparisons (EQUALS, LESS_THAN, ...) and
logical ops (AND, OR, NOT) typically appear in Filter contexts where
they are already capability-handled by the filter path. Extend the
carve-out only when a specific plan-rewrite rule demonstrably emits a
new operator that every backend already supports.

Recursion: operands of a baseline op are still visited. A baseline op
wrapping a non-baseline function (e.g. CAST(regexp_match(col, 'x')))
still forces the inner call through capability resolution and
annotation.

If a future backend genuinely cannot execute one of these operators
(e.g. Lucene rejecting a CAST between incompatible types), that
becomes a runtime error inside the backend's executor — complementary
to plan-time capability enforcement, not a replacement for it.

Test updates
------------
Ten ProjectRuleTests cases used CAST, PLUS, and similar baseline ops
as representative 'some scalar function' fixtures. The intent of those
tests — capability routing, delegation, annotation depth — is
unchanged; the fixtures are swapped for capability-declared operators
(CEIL, POWER, UPPER) so the tests still exercise capability-registry
behavior. Assertions and expected exceptions are preserved verbatim
against the new fixtures.

Unlocks
-------
This is load-bearing for any future integration of Calcite's plan
rewrites that emit baseline operators. Immediately unblocks
OpenSearchAggregateReduceRule (commit 1847cb0a4bd, currently
scaffold-only) which emits CAST around AVG's sum/count division.

Ref: .kiro/docs/distributed-aggregate-design.md \u00a77
Related: scaffold 1847cb0a4bd (OpenSearchAggregateReduceRule)
Signed-off-by: Sandesh Kumar 

* feat(planner): wire OpenSearchAggregateReduceRule into HEP marking

Activates Calcite's AggregateReduceFunctionsRule (via our
OpenSearchAggregateReduceRule subclass, committed in 1847cb0a4bd) as
part of PlannerImpl's HEP marking phase. AVG is now decomposed into
primitive SUM(x) + COUNT(x) aggregate calls wrapped by a scalar
LogicalProject that computes CAST(sum / count AS avgReturnType) —
before our OpenSearchAggregateSplitRule produces PARTIAL/FINAL pairs.

With this in place, the subsequent split rule operates on the already-
reduced inner aggregate, so PARTIAL and FINAL both carry primitive
calls. Our AggregateDecompositionResolver sees these primitives and
applies its function-swap branch (COUNT at FINAL → SUM over partial
count column) as normal. The previous primitive-decomposition path
in the resolver (buildProjectWrapper, multi-aggregate rewrite) is no
longer exercised for AVG and can be simplified in a follow-up commit.

Scope of reduction (FUNCTIONS_TO_REDUCE)
----------------------------------------
Narrowed to AVG only. Calcite's STDDEV_POP / STDDEV_SAMP / VAR_POP /
VAR_SAMP reductions emit POWER(x, 2) for x², and POWER is not in
OpenSearchProjectRule.BASELINE_SCALAR_OPS — no backend currently
declares POWER as a project capability in our test scaffolding. AVG's
decomposition emits only SUM, COUNT, DIVIDE, and CAST: SUM/COUNT go
through the aggregate capability path; DIVIDE and CAST are baseline
scalars carved out of capability enforcement by commit e07d900352f.

Extending to STDDEV / VAR is a one-line change to FUNCTIONS_TO_REDUCE
once either POWER joins the baseline set or backends declare it.

Calcite aggregate-call deduplication
------------------------------------
When a query mixes AVG(x) with COUNT() or SUM(x) that have identical
arguments, Calcite's reduce rule deduplicates — the user's COUNT()
and SUM(x) are absorbed into AVG's primitive decomposition, and the
Project on top surfaces them via input refs. Per-shard aggregations
strictly decrease; results are semantically equivalent. Q10-mixed
tests updated to assert the deduplicated shape.

Test updates
------------
Three AggregateDecompositionResolverTests cases were rewritten to
match the new plan shape (Calcite's primitive decomposition instead
of the hand-rolled resolver's):

  testPrimitiveDecompAvg: PARTIAL carries [SUM, COUNT] (Calcite's
    order), not [SUM, SUM]; exchange types are integer-family, not
    DOUBLE (pre-reduction intermediateFields override is no longer
    taken). Parent fragment is asserted as Project (works for both
    LogicalProject pre-HEP and OpenSearchProject post-HEP).

  testMixedQ10: asserts Calcite's dedup behavior — 2 PARTIAL primitives
    (not 4) with the Project on top surfacing [status, avg_size, c, s]
    via CAST + input refs.

  testNoCalciteInferReturnType: renamed to testAvgExchangeTypesAreCalcite
    Primitives and repurposed. The old invariant ("sum column must be
    DOUBLE from intermediateFields") is obsolete; Calcite's primitive
    decomposition produces integer-family types that match DataFusion's
    SUM(int) → Int64 emit directly, with no override path.

Resolver code (rewriteDecomposed primitive-decomp branch,
buildProjectWrapper, etc.) remains in place for this commit but is
dead code for AVG. It will be removed in a follow-up commit alongside
any final simplification.

Ref: .kiro/docs/distributed-aggregate-design.md \u00a77
Related: scaffold 1847cb0a4bd, baseline carve-out e07d900352f
Signed-off-by: Sandesh Kumar 

* refactor(planner): remove primitive-decomposition path from resolver

Calcite's AggregateReduceFunctionsRule (wired in 223c632fb60) now
handles AVG / STDDEV / VAR primitive decomposition during HEP marking
— before our resolver ever sees the plan. The hand-rolled multi-field
primitive-decomp branch in rewriteDecomposed is dead code, along with
its supporting machinery.

Removed
-------
  rewriteDecomposed multi-field branch (~20 LOC)  — Calcite replaces
  buildProjectWrapper (~80 LOC)                    — Calcite's reduce
    rule builds the Project (CAST(sum/count AS avgType)) itself
  primitivePartial helper (~30 LOC)                — Calcite emits the
    primitive SUM/COUNT aggregate calls directly
  PendingProject record (~5 LOC)                   — no longer tracks
    per-call Project state since Calcite owns the wrapper
  projectOnTop field on RewriteResult + invocation (~10 LOC)
  Unused imports: LogicalProject, RexBuilder, RexNode

Defensive guard
---------------
The rewriteDecomposed method now throws IllegalStateException if it
encounters an AggregateFunction that declares multi-field intermediate
or scalar-final decomposition. Those cases must be reduced by Calcite's
rule upstream — reaching the resolver with such a call indicates
either (a) the function isn't in OpenSearchAggregateReduceRule's
FUNCTIONS_TO_REDUCE set, or (b) the rule didn't fire for some other
reason. The guard preserves the invariant that only single-field
shapes (pass-through, function-swap COUNT→SUM, engine-native DC)
reach the resolver.

LOC
---
AggregateDecompositionResolver.java: 489 → 362 LOC (-127).
Combined with commits e07d900352f (baseline carve-out, +40 LOC net)
and 223c632fb60 (reduce rule wiring + test updates, +57 LOC net), the
three-commit sequence delivers the decomposition refactor with a net
~-30 LOC reduction and considerably less per-function logic in the
resolver.

Ref: .kiro/docs/distributed-aggregate-design.md \u00a77
Related: 1847cb0a4bd (rule scaffold), e07d900352f (baseline), 223c632fb60 (rule wiring)
Signed-off-by: Sandesh Kumar 

* refactor: remove dead code after Calcite-driven AVG reduction

Followup to commits 1847cb0a4bd → 975fc26339b (Calcite reduce rule
refactor). With AggregateReduceFunctionsRule handling AVG / STDDEV /
VAR decomposition during HEP marking, the enum fields and methods
that only existed to drive hand-rolled primitive decomposition are
dead code. Remove them to prevent future readers from mis-modeling
the enum as the decomposition contract for multi-field cases.

Removed
-------
  AggregateFunction.finalExpression field + accessor
  AggregateFunction.hasScalarFinal() method
  AggregateFunction 4-arg constructor (now only 2/3 args)
  AVG enum entry's finalExpression lambda
  AggregateFunctionTests assertions on the above

  AggregateDecompositionResolver guard: simplified
    'iFields.size() != 1 || fn.hasScalarFinal()' → 'iFields.size() != 1'
    (single-field + scalar-final shape doesn't exist in any enum entry)

  Unused imports: BiFunction, RexBuilder, RexNode, SqlStdOperatorTable,
    FloatingPointPrecision, JavaTypeFactoryImpl

Updated
-------
  OpenSearchAggregateSplitRule javadoc — replaced the pre-refactor TODO
  ("aggregate decomposition is deferred to plan forking") with an
  accurate description of the current split of responsibilities:
    - HEP marking: OpenSearchAggregateReduceRule handles multi-field
      primitive decomposition (AVG / STDDEV / VAR)
    - This split rule: purely structural SINGLE → FINAL+Exchange+PARTIAL
    - AggregateDecompositionResolver: single-field cases only
      (pass-through, function-swap, engine-native merge)

  AggregateFunctionTests: simplified test names and assertions to
  reflect the narrowed enum shape. AVG / STDDEV / VAR entries are
  asserted to declare no intermediateFields — primitive decomposition
  metadata does not belong on the enum when Calcite owns the rewrite.

Kept
----
  AggregateDecomposition interface: unused at runtime but documented
  as the escape hatch for future per-backend overrides. Zero cost to
  keep; non-trivial design decision to remove.

  AggregateCapability.decomposition() field: same rationale.

  Test-runtime Arrow deps on analytics-framework: still needed for
  COUNT / APPROX_COUNT_DISTINCT enum entries' ArrowType instances.

Spotless applied across the 3 modified modules.

Ref: .kiro/docs/distributed-aggregate-design.md §6
Signed-off-by: Sandesh Kumar 

* refactor(spi): remove dead aggregate SPI surface and consolidate resolver helpers

All internal to the analytics planner.

── Remove dead SPI surface ──

Two unused extension points on the analytics SPI had no production
callers and confused the intent of the aggregate decomposition path.
Consolidating on the enum-based source of truth:

1. AggregateFunction.hasBinaryIntermediate()
   Only referenced by its own unit-test assertions. Existed for an
   earlier resolver branch that detected engine-native Binary
   intermediates; the current resolver tests reducer identity directly
   (f.reducer() == this).

2. AggregateDecomposition (SPI interface)
   Published as a per-backend extension point on
   AggregateCapability.decomposition(), but no backend ever
   implemented it and nothing read it back — not the resolver, not
   the split rule. Superseded during actual implementation by
   AggregateFunction.intermediateFields() + AggregateDecompositionResolver,
   which is backend-agnostic and universal. Keeping the interface
   advertised an escape hatch the planner does not honor.

── Consolidate resolver helpers ──

3. Move Calcite-interop to the enum.
   toSqlAggFunction() (AggregateFunction → SqlAggFunction) and
   fromSqlAggFunction(SqlAggFunction) (reverse, with name-then-kind
   fallback) now live on AggregateFunction itself. Previously they
   were private helpers in AggregateDecompositionResolver. The enum
   is now the single place that owns Calcite-identity conversion for
   aggregates.

4. Unify two tree-rewrite helpers.
   replaceTopAggregate and replaceStageInputScan had identical
   identity-based copy-then-swap logic, differing only in target
   type. Replaced by a single replaceFirst(RelNode, RelNode, RelNode).

5. Extract per-call rewrite in AggregateDecompositionResolver.
   The per-aggCall loop in rewriteDecomposed previously mutated four
   parallel collections (partial calls, final calls, exchange types,
   exchange names) with branching intermixed — easy to forget a list
   or desynchronize indices. It now produces one immutable CallRewrite
   record per aggregate call and the outer loop consumes it:
     - rewriteAggCall: classifies + dispatches
     - passThroughRewrite: no decomposition
     - singleFieldRewrite: engine-native merge or function-swap
   Multi-field shapes (should be HEP-reduced upstream) still throw.
   Each branch is independently testable and the four output columns
   stay in lockstep by construction.

6. Single-line comments on private static utilities.

No behaviour change. Build + tests green across analytics-framework,
analytics-engine, and analytics-backend-datafusion. Spotless clean.

If a future backend genuinely needs divergent aggregate decomposition
(e.g. KLL vs HLL sketch for APPROX_COUNT_DISTINCT), a per-backend
AggregateFunctionAdapter SPI can be reintroduced — designed around
that concrete use case rather than this speculative shape.

Signed-off-by: Sandesh Kumar 

* fix(datafusion-rust): enter Tokio runtime in df_execute_local_prepared_plan

df_execute_local_prepared_plan calls session.execute_prepared() which invokes
datafusion::physical_plan::execute_stream. That call is synchronous but kicks
off RepartitionExec / stream-channel setup that requires a Tokio reactor —
running it from the JNI-invoked thread (no Tokio context) aborts with
'there is no reactor running, must be called from the context of a Tokio 1.x
runtime'.

Enter the IO runtime's context for the duration of the call via
_guard = mgr.io_runtime.enter() so those operators can register with the
reactor. Matches the pattern already used by df_prepare_final_plan which
wraps its async call in mgr.io_runtime.block_on(...).

Surfaced by multi-shard PPL queries where the final-aggregate plan contains
a RepartitionExec: 13 of 14 previously-failing multi-shard PplClickBenchIT
queries now get past sink creation without this crash.

Signed-off-by: Sandesh Kumar 

* feat(exec): extend ArrowSchemaFromCalcite with date / time / timestamp mappings

ArrowSchemaFromCalcite.toArrowType previously threw on any SqlTypeName it
didn't enumerate, including common OpenSearch field types like DATE and
TIMESTAMP. This surfaced whenever a multi-shard query carried a date-typed
column through the exchange row type (e.g. 'min(EventDate)').

Added mappings:
  SMALLINT    → Int(16, signed)
  TINYINT     → Int(8, signed)
  REAL        → FloatingPoint(SINGLE)    (alias for FLOAT)
  DATE        → Date(DAY)
  TIME        → Time(MILLISECOND, 32)
  TIMESTAMP   → Timestamp(MILLISECOND, null)
  TIMESTAMP_WITH_LOCAL_TIME_ZONE → Timestamp(MILLISECOND, null)

Unsupported types still throw IllegalArgumentException with the SqlTypeName
in the message.

Signed-off-by: Sandesh Kumar 

* fix(planner): reduce AVG on plain LogicalAggregate in a dedicated HEP phase

OpenSearchAggregateReduceRule previously operated on OpenSearchAggregate and
shared the HEP marking collection with OpenSearchAggregateRule. In that
arrangement the marking rule fires first in BOTTOM_UP traversal, converting
LogicalAggregate → OpenSearchAggregate with per-call AGG_CALL_ANNOTATION
wrappers in aggCall.rexList. When Calcite's AggregateReduceFunctionsRule
then reduces AVG(x), it reads rexList[0].getType() during type inference on
the derived SUM — which carries AVG's original DOUBLE return type, not the
natural BIGINT for a SUM of integer. The stamped DOUBLE type propagates
through the reduced plan and fails typeMatchesInferred downstream
(stripAnnotations, the split rule, or the resolver — all cascades observed).

Clean fix — align with the documented design (§11.1): reduce BEFORE marking,
on a plain LogicalAggregate where aggCall.rexList is empty and Calcite
infers canonical primitive types. Implementation:

 1. OpenSearchAggregateReduceRule: match LogicalAggregate (not
    OpenSearchAggregate). Remove the AggregateMode.SINGLE guard and the
    newAggregateRel override that re-wrapped as OpenSearchAggregate — the
    subsequent marking rule handles that conversion. The rule body is now
    a one-line configuration invoking super.

 2. PlannerImpl: split the single HepPlanner into three chained phases —
    pre-marking (constant folding), aggregate reduction, marking. The
    reduction phase runs its own HepPlanner on the post-pre-marking plan
    so the rule order is enforced by phase boundaries rather than
    BOTTOM_UP rule discovery order.

Result: the plan leaving the reduction phase is Calcite-canonical — clean
LogicalAggregate(SUM, COUNT, ...) + LogicalProject(CAST(SUM)/CAST(COUNT)).
Marking then wraps each with OpenSearch* annotations; Volcano split, the
resolver, and stripAnnotations all see consistent primitive types without
any type-rebuild patches.

Unblocks all grouped-AVG queries (stats avg(x) by ...) on single-shard.

Signed-off-by: Sandesh Kumar 

* fix(resolver): align multi-shard FINAL with DataFusion column names + nullability

Two related issues surfaced when PARTIAL/FINAL split fires on multi-shard:

 1. Column-name mismatch: the resolver's fallback for unnamed aggregates
    (e.g. reduced AVG's auto-generated SUM/COUNT) used 'expr$' while
    Calcite — and DataFusion on the Rust side — use '$f'. The PARTIAL's
    Arrow output arrived with '$f2, $f3', the Substrait plan for FINAL
    referenced 'expr$2, expr$3', and DataFusion's schema lookup aborted
    with 'No field named expr$2. Valid fields are …$f2…'.

    Fix: derive exchange column names from the aggregate's own RelDataType
    (agg.getRowType().getFieldList()) — Calcite already assigned the
    canonical names (explicit aggCall.name where present, '$f' where
    not), so reusing them keeps Java-side exchange schema aligned with the
    DataFusion output convention. Removes the hand-rolled 'expr$'
    fallback entirely.

 2. Nullability drift on function-swap: the resolver rewrites COUNT → SUM
    at FINAL for the function-swap case, constructing the new call via
    makeCall(...) with returnType = ArrowCalciteTypes.toCalcite(...), which
    returns NOT-NULL types. Calcite, however, infers SUM over an exchange
    column as nullable (SUM of empty group → null). The declared
    NOT-NULL-vs-inferred-nullable mismatch trips typeMatchesInferred when
    the FINAL OpenSearchAggregate is later copied.

    Fix: in rewriteParentFragment, rebuild each FINAL AggregateCall via the
    (hasEmptyGroup, input, type=null, name) AggregateCall.create variant so
    Calcite re-runs full type inference against the actual FINAL input
    (the rewritten StageInputScan).

 3. Project-above-FINAL RexInputRef rebind: once FINAL's aggCall types
    change, any Project sitting directly above the FINAL (from reduced AVG,
    or a user-written Project) holds RexInputRefs with stale types. The
    plain identity-based replaceFirst copies that Project unchanged and
    Calcite's RexChecker rejects the mismatch.

    Fix: replaceFirstWithRefRebinding — when the immediate parent is a
    Project, walk its projection expressions with a RexShuttle that
    rebinds each RexInputRef to the new FINAL's row-type, CASTing the
    whole expression to the Project's declared field type so the outer
    schema (e.g. AVG's DOUBLE column) stays stable even when the inner
    aggregate now emits primitive BIGINT.

Unblocks AVG queries on multi-shard (Q3, Q4, Q10, Q28, Q33) and stabilises
the COUNT → SUM swap path for Q1, Q2, Q8, Q16 etc when split fires.

Signed-off-by: Sandesh Kumar 

* test(qa): enable multi-node + multi-shard ClickBench IT with auto-discovered queries

Turns on the previously-TODO'd 2-node integTest cluster for analytics-engine-rest
and switches the ClickBench dataset to 2 shards so PARTIAL/FINAL split, Arrow
Flight transport between shards and coordinator, and the aggregate
decomposition resolver all exercise under realistic distributed shape — not
just the single-shard no-split fast path.

PplClickBenchIT:
  - Auto-discovers all 43 PPL queries under resources/datasets/clickbench/ppl/
    instead of the Q1-only hardcoded list, so the tested surface grows as new
    PPL features land without touching this class.
  - SKIP_QUERIES (set literal) lists the 24 queries that currently fail for
    reasons unrelated to the partial/final aggregate feature — each with an
    in-file comment pointing at the root cause bucket:
      * Missing PPL frontend features: Q19, Q40, Q43
      * Malformed query in the dataset (missing 'where' keyword): Q29
      * Multi-shard binary exchange can't serialize LocalDateTime yet:
        Q7, Q24-Q27, Q37-Q42
      * DataFusion Arrow 'project index 0 out of bounds' on
        WHERE + GROUP-BY + aggregate: Q11-Q15, Q20, Q22, Q23, Q31, Q32
  - 19 queries pass across all three cluster variants (1-node-1-shard,
    1-node-2-shards, 2-nodes-2-shards), including all AVG-bearing queries
    (Q3, Q4, Q10, Q28, Q33).

mapping.json: number_of_shards = 2, keeps replicas at 0.
build.gradle: testClusters.integTest gets numberOfNodes = 2; memtable and
  streaming variants already use 2 nodes and are unchanged.

Signed-off-by: Sandesh Kumar 

* test(qa): fix malformed Q29 PPL query; document Substrait MIN(VARCHAR) gap

Q29 in the ClickBench dataset had `| Referer != ''` with no `where` keyword,
tripping the PPL parser before any planning could happen. Added the missing
`where`.

Q29 then surfaces a distinct follow-up: the Substrait isthmus emitter can't
find a binding for `MIN($1)` when the argument is VARCHAR (the `min(Referer)`
call). That's a Substrait aggregate-catalog gap — unrelated to the
partial/final work — so Q29 stays on the SKIP_QUERIES list with a comment
pointing at the MIN-on-strings binding as the remaining blocker.

Signed-off-by: Sandesh Kumar 

* fix(datafusion): emit schema-carrying empty batch when native stream has zero batches

When a shard's partial-aggregate plan produces zero record batches (typical
when a WHERE predicate filters out every row on that shard), the Flight
wire-protocol producer never writes a data frame. Arrow Flight puts the
schema in the first data frame — zero frames means zero schema, so the
coordinator's StreamingTableExec receives a stream with no schema and fails
with 'Arrow error: Schema error: project index 0 out of bounds, max field
0' the first time it projects a column by index.

DatafusionResultStream.BatchIterator now tracks two additional bits of
state: whether the native stream has reported EOS (arrayAddr == 0) and
whether we've ever returned a batch. When the native side yields EOS
without having emitted a single batch, we synthesize one zero-row
VectorSchemaRoot from the already-known schema and return it as the final
batch. Flight carries the schema with that frame; the coordinator sees it
and the downstream aggregate merges correctly over zero rows.

Unblocks 10 multi-shard ClickBench queries (Q11, Q12, Q13, Q14, Q15, Q20,
Q22, Q23, Q31, Q32) that were failing with this exact error. Enforces the
§18 invariant #12 documented in the design revisit:
  "Shard emission of a PARTIAL with zero matching rows still delivers the
   schema message."

Signed-off-by: Sandesh Kumar 

* test(qa): add coordinator-reduce regression for WHERE+GROUP-BY empty-partial case

Adds two CoordinatorReduceIT methods that together cover the empty-partial
contract:

- testWhereGroupByCountMultiShard_reproducer — WHERE filters all rows on
  every shard (all docs have category='', predicate is category != ''), so
  both shards' partial aggregates produce zero batches. Before the Java-
  side schema-preservation fix, this query died with Arrow 'project index
  0 out of bounds, max field 0' in the drain thread. The test asserts a
  non-erroring 200 response; exact shape isn't checked since the point is
  crash-freedom.

- testGroupByCountMultiShard_noWhereControl — same query shape minus the
  WHERE. Every doc has category='' so the result is one group with the
  full count. Acts as the control that isolates the WHERE predicate (and
  the resulting zero-batch partial) as the previous trigger.

PplClickBenchIT SKIP_QUERIES trimmed to the queries that remain failing
after the schema-preservation fix: only the TIMESTAMP/DATE family (Q7,
Q24-Q27, Q37-Q42) and unrelated PPL frontend gaps (Q19, Q29, Q40, Q43).
Previously-skipped WHERE + GROUP-BY queries (Q11, Q12, Q13, Q14, Q15, Q20,
Q22, Q23, Q31, Q32) are now included — 29 ClickBench queries pass on
multi-shard (up from 19).

Signed-off-by: Sandesh Kumar 

* refactor(exec): replace row-oriented fragment wire format with Arrow IPC

The shard-to-coordinator wire format for FragmentExecutionResponse previously
carried each batch as List + List fieldNames, serialized
per-cell via OpenSearch's StreamOutput.writeGenericValue. The path had two
structural problems:

- Lossy round-trip for Arrow types. vector.getObject(i) returns java.time.
  LocalDateTime for TimeStampMilliVector (no TZ), which writeGenericValue
  does not support — every shard emitting a Timestamp column failed with
  'can not write type [class java.time.LocalDateTime]'. Arrow's view-vector
  variants (Utf8View, BinaryView) and dictionary-encoded vectors likewise
  can't round-trip through Object[] inference.

- Per-cell boxing on both sides (O(rows * cols) object allocations + dispatch
  on send, O(rows * cols) allocations + setSafe on receive). Heap-heavy and
  GC-pressure-heavy for larger batches.

Replaces the wire format with Arrow's own IPC stream: MessageSerializer.
serialize(channel, schema) once, then serialize(channel, recordBatch) per
batch, terminated by ArrowStreamWriter.writeEndOfStream. On receive,
ArrowStreamReader handles schema + batch message sequencing and VectorLoader
loads buffers into VSRs. Arrow's library handles every vector type natively
— zero hand-rolled dispatch.

RowResponseCodec.decode copies the reader's reused root into caller-owned
VSRs via makeTransferPair (works for every vector kind including views; the
default VectorSchemaRootAppender rejects view vectors, so we avoid it).
Multi-batch responses concatenate via per-cell copyFromSafe — the only
append primitive in Arrow Java that supports view vectors.

Wire format: byte[] ipcPayload + vint rowCount. rowCount is cached purely
for the existing onFragmentSuccess metric so consumers don't decode just to
count rows.

Deletes RowBatchToArrowConverter — the row-to-Arrow bridge is now Arrow IPC
end-to-end. Updates ArrowSchemaFromCalcite javadoc accordingly.

Net: +224 / -340 lines across the codec. All 39 non-skipped ClickBench PPL
queries pass on multi-shard (was 20 with the row-oriented codec), including
the full TIMESTAMP family (Q7, Q24-Q27, Q37-Q42) that the old Object[] path
structurally could not handle.

Signed-off-by: Sandesh Kumar 

* fix(datafusion): coerce DataFusion's Utf8View emission to Utf8 before senderSend

DataFusion's HashAggregateExec internally converts string group keys to
Utf8View for performance (inline short-string optimization, non-configurable
in DataFusion 49+). The coordinator's FINAL plan, by contrast, is decoded
from substrait — substrait has only a generic 'string' type which DataFusion
round-trips to Utf8, and the coordinator's childSchemas (computed from the
Calcite row type via ArrowSchemaFromCalcite) likewise declares Utf8.

With the Arrow IPC wire codec preserving exact Arrow types end-to-end, the
shard's Utf8View batches now reach the coordinator as Utf8View. The Rust
StreamingTable partition was registered with a Utf8 schema, so Utf8View
batches trigger an 'as_primitive::<>()' downcast panic ('byte array') the
first time a cross-batch operator (coalesce / repartition) handles the
string column. Observed on every multi-string-column group-by from Q17
onward.

Previously the Object[] row-path laundered this silently: getObject ->
String -> VarCharVector.setSafe always produced Utf8, regardless of the
shard's actual emit type. Removing the lossy row-path exposed the contract
gap.

Adds a single coercion point at feedToSender(): when batch.getSchema()
differs from the declared childSchemas entry, allocate a new VSR matching
the declared schema and copy per column. Same-type columns use makeTransfer
Pair (zero-copy). Utf8View -> Utf8 uses per-cell byte copy:
ViewVarCharVector.get(i) -> VarCharVector.setSafe(i, bytes). Unknown type
pairs throw with a diagnostic naming source type, target type, and column —
future mismatches surface as clear errors rather than opaque Rust panics.

AbstractDatafusionReduceSink gains a childSchemas map parallel to the
existing childInputs bytes map, populated once in the constructor. Sinks
(single-input feed() and per-child ChildSink) look up the declared schema
by childStageId.

Extends only on observed mismatch — no speculative pair coverage. Zero-copy
fast path when schemas match (numeric-only aggregates, which is the common
case).

Signed-off-by: Sandesh Kumar 

* test(qa): un-skip TIMESTAMP and string-group-by queries that now pass

With the Arrow IPC wire codec and the Utf8View -> Utf8 coercion at the
Java-to-Rust boundary, the previously-skipped query families now pass on
multi-shard (2 shards, 2 nodes). Trims SKIP_QUERIES from 14 entries down
to the 4 remaining PPL frontend / Substrait library gaps that are unrelated
to distributed execution:

  Q19 - extract(minute from ...) not implemented in the PPL frontend
  Q29 - Substrait library can't bind MIN on VARCHAR inputs
  Q40 - case() else + head N from M - PPL frontend gap
  Q43 - date_format() + head N from M - PPL frontend gap

Unblocked: Q7 + Q24-Q27 + Q37-Q42 (TIMESTAMP / DATE family, 11 queries);
Q17-Q18 + Q31-Q36 (multi-string-column group-by family, 8 queries). Total
39 of 39 non-skipped ClickBench PPL queries pass on multi-shard.

Signed-off-by: Sandesh Kumar 

* docs(datafusion): document plan-level alternatives for Utf8View coercer

Adds a TODO on coerceToDeclaredSchema summarizing the plan-level alternatives
we tried and why they're currently blocked:

- declaring Utf8View up-front in childSchemas SIGSEGVs because DataFusion's
  optimizer emits Utf8View across more operators than HashAggregate (filter
  + sort + project queries also hit it), making static prediction in Java
  fragile and engine-version-specific
- Arrow Java 18.3's FFI can import Utf8View natively (BufferImportTypeVisitor
  has visit(Utf8View)); the blocker is predicting the emission, not
  importing it
- three forward paths recorded for future revisit: (a) a DataFusion schema-
  introspection API, (b) substrait view-type extension, (c) a Rust-side
  normalize pass using DataFusion's vectorized CastExpr at PARTIAL root

Signed-off-by: Sandesh Kumar 

* refactor(datafusion): remove APPROX_COUNT_DISTINCT plan rewrite

The prior approach introduced a dummy Calcite SqlAggFunction APPROX_DISTINCT
and a plan shuttle (rewriteApproxCountDistinct) that swapped every
SqlStdOperatorTable.APPROX_COUNT_DISTINCT call for it right before Substrait
emission. The dummy was needed because isthmus' default AGGREGATE_SIGS
binds APPROX_COUNT_DISTINCT to substrait's standard approx_count_distinct
URN, and the resulting entry in FunctionConverter's IdentityHashMap
(keyed by Calcite SqlOperator) shadows any additional Sig entry for the
same operator.

Replaces the workaround with a small OpenSearchAggregateFunctionConverter
subclass that filters APPROX_COUNT_DISTINCT out of the default signature
list via getSigs(). With the default binding gone, a plain
Sig(SqlStdOperatorTable.APPROX_COUNT_DISTINCT, "approx_distinct") in
ADDITIONAL_AGGREGATE_SIGS is the sole matcher and routes directly to the
YAML-declared extension — no operator rewrite, no dummy SqlAggFunction,
no plan shuttle.

Net: -78 lines. No runtime per-function branch in the convertor. Restores
invariant #3 (no ad-hoc 'if function == X' dispatch outside AggregateFunction
and the resolver's enum lookup).

Signed-off-by: Sandesh Kumar 

* style: replace FQN references with imports across touched files

Audits every file touched in this branch's commits and converts
fully-qualified class references to imports. Preserves FQN only where
there's a genuine same-name collision that forces it.

Changes:
- AggregateDecompositionResolver: import Project, RelCollations, RexBuilder,
  RexInputRef, RexNode, RexShuttle (~10 FQN sites -> short names).
- ArrowSchemaFromCalcite + ArrowCalciteTypesTests: import DateUnit, TimeUnit
  (5 FQN sites -> short names).
- AbstractDatafusionReduceSink: import org.apache.arrow.vector.types.pojo.Schema
  (2 FQN sites).
- AnalyticsSearchService: import ArrowStreamWriter (1 FQN site).
- DataFusionFragmentConvertor: import ImmutableList (2 FQN sites).
- DataFusionFragmentConvertorTests: import RexNode (1 FQN site).
- ShardFragmentStageExecutionTests: import ActionResponse (1 FQN site) —
  also fixes stale new FragmentExecutionResponse(List, List)
  calls that I missed updating in the earlier IPC refactor.

Preserves FQN on the two io.substrait.proto.Plan references in
DataFusionFragmentConvertor — that class collides with the already-imported
io.substrait.plan.Plan, so FQN is required.

Signed-off-by: Sandesh Kumar 

* docs: strip internal design references from production comments

Comments in production files should not reference internal design-doc
section numbers (§18 #12), branch names (pf2, pf4), or session-specific
debug narrative ("we attempted", "SIGSEGV in Q26"). Those belong in
design docs and commit messages, not in the code.

- DatafusionResultStream: drop '§18 invariant #12' reference from the
  nativeStreamExhausted field comment; the surrounding explanation already
  states what the field is for.
- DatafusionReduceSink.coerceToDeclaredSchema: rewrite the TODO block as a
  forward-looking technical note instead of a session history, keeping only
  the actionable alternatives.
- CoordinatorReduceIT.testQ10ShapeAcrossShards: drop pf2/pf4 branch
  narrative; describe what the test covers structurally.

Signed-off-by: Sandesh Kumar 

* test(qa): remove repro-style narrative from CoordinatorReduceIT

CoordinatorReduceIT's multi-shard WHERE+GROUP-BY tests carried leftover
debug narrative from their original reproducer role — now that the bug
they caught is fixed, the javadoc and naming should describe what the
tests validate, not their origin story.

- Rename testWhereGroupByCountMultiShard_reproducer →
  testGroupByCountMultiShard_allRowsFilteredByWhere
  and testGroupByCountMultiShard_noWhereControl →
  testGroupByCountMultiShard_noWhereClause. Describes shape, not history.
- Rewrite their javadocs to state what behavior is being asserted
  (empty-partial path reporting empty result without erroring), drop the
  'project index 0 out of bounds' stack trace and '@AwaitsFix to keep
  green' commentary.
- Rename the supporting fixtures WHERE_REPRO_INDEX, createWhereReproIndex,
  indexWhereReproDocs → STRING_GROUP_INDEX, createStringGroupIndex,
  indexStringGroupDocs. 'repro' was session-specific vocabulary.
- Class-level javadoc: drop internal Rust function names
  (prepare_partial_plan, force_aggregate_mode, execute_local_prepared_plan)
  from the pipeline diagram; keep the Calcite/Java-side names that describe
  the layer boundaries.
- Remove now-unused AwaitsFix import.

Signed-off-by: Sandesh Kumar 

* test(lucene): update ExchangeSinkProvider lambda to 2-arg form

Pre-existing test file hadn't been updated after the
ExchangeSinkProvider#createSink contract went from (context) to
(context, backendContext) during the handler-infrastructure work.
Lambda now matches the current SPI signature.

Signed-off-by: Sandesh Kumar 

* test(datafusion): update exhausted-stream test for schema-carrying empty batch

DatafusionResultStream.loadNextBatch now synthesizes a zero-row VSR
carrying the declared schema when the native stream produces no batches,
so downstream transports see the column layout on their first data frame.
testNextOnExhaustedStreamThrows was asserting the old contract (hasNext
returns false immediately on empty result). Consume the synthetic batch
first, then assert the iterator is exhausted.

Signed-off-by: Sandesh Kumar 

* test(qa): mute DslClickBenchIT queries pending DSL-path investigation

The single DSL query (Q1: sum(GoodEvent)) has been hanging on this branch with
a 60s client-side socket timeout. Root cause is in the DSL aggregation path,
not the distributed-aggregate machinery — orthogonal to the coercer and
partial/final wiring. Skipping the query lets CI run the structural test
(provisioning, discovery) without blocking on the unrelated regression.

Restore 'List.of(1)' once the DSL hang is diagnosed and fixed.

Signed-off-by: Sandesh Kumar 

* fix(planner): strip annotations at every depth for baseline-op roots

BASELINE_SCALAR_OPS (COALESCE, CASE, CAST, arithmetic, IS_NULL, …) bypass
the AnnotatedProjectExpression wrap at the call site but their operands
still go through capability-driven annotation. A project expression like
COALESCE(num0, CEIL(num1)) ends up as

    COALESCE(num0, AnnotatedProjectExpression(CEIL(num1)))

with the outer COALESCE unwrapped and the inner CEIL wrapped.
OpenSearchProject.stripAnnotations only ran the nested-annotation shuttle
when the top-level expression was itself an AnnotatedProjectExpression;
the plain-top-level branch passed the expression through untouched, so
the inner wrapper survived into the substrait converter. Isthmus then
rejected the plan with 'Unable to convert call ANNOTATED_PROJECT_EXPR(…)'.

The strip logic predates the baseline carve-out (commit 196fd424d0a)
and was never updated to account for inner annotations under a baseline
root. The carve-out intentionally recurses into operands precisely so
those inner calls still go through capability resolution — strip must
mirror that recursion.

Run the RexShuttle unconditionally; it already no-ops for subtrees
without annotations. Same behaviour on top-level annotations, now
correctly scrubs nested ones regardless of what root wraps them.

Resolves 'ANNOTATED_PROJECT_EXPR(…)' failures in
FillNullCommandIT.testFillNullWithFunctionOnOtherField (fillnull with
ceil(num1) in num0 → COALESCE root) and
MultisearchCommandIT.testMultisearchEvalCaseProjection (eval case → CASE
root).

Also tags MultisearchCommandIT.testMultisearchCountEvalConditionalCount
with @AwaitsFix pending a separate DataFusion count-accumulator state-type
mismatch (Int32 vs Int64) that only surfaces after the strip fix lets
the plan reach execution. The test body is preserved so restoration is
just removing the annotation once the underlying issue is fixed.

Signed-off-by: Sandesh Kumar 

---------

Signed-off-by: Sandesh Kumar 
---
 sandbox/libs/analytics-framework/build.gradle |  19 +
 .../analytics/spi/AggregateCapability.java    |  35 +-
 .../analytics/spi/AggregateDecomposition.java |  60 ---
 .../analytics/spi/AggregateFunction.java      |  89 +++-
 .../analytics/spi/ExchangeSinkProvider.java   |   6 +-
 .../analytics/spi/AggregateFunctionTests.java |  96 ++++
 .../rust/src/agg_mode.rs                      | 299 ++++++++++++
 .../rust/src/ffm.rs                           |  87 ++++
 .../rust/src/indexed_executor.rs              |   3 +
 .../rust/src/lib.rs                           |   1 +
 .../rust/src/local_executor.rs                |  84 +++-
 .../rust/src/session_context.rs               | 115 +++++
 .../AbstractDatafusionReduceSink.java         |  47 +-
 .../DataFusionAnalyticsBackendPlugin.java     |  26 +-
 .../DataFusionFragmentConvertor.java          |  52 ++-
 .../DataFusionInstructionHandlerFactory.java  |  11 +-
 .../be/datafusion/DataFusionPlugin.java       |   6 +-
 .../be/datafusion/DataFusionReduceState.java  |  41 ++
 .../be/datafusion/DatafusionReduceSink.java   | 135 +++++-
 .../be/datafusion/DatafusionResultStream.java |  32 +-
 .../FinalAggregateInstructionHandler.java     |  44 +-
 .../PartialAggregateInstructionHandler.java   |  40 ++
 .../be/datafusion/nativelib/NativeBridge.java |  69 +++
 .../opensearch_aggregate_functions.yaml       |  13 +
 .../DataFusionFragmentConvertorTests.java     |  67 ++-
 .../DatafusionResultStreamTests.java          |  12 +-
 ...DelegationForIndexFullConversionTests.java |   2 +-
 .../NativeBridgePreparedPlanTests.java        |  63 +++
 .../LuceneAnalyticsBackendPluginTests.java    |   2 +-
 .../exec/AnalyticsSearchService.java          |  62 ++-
 .../analytics/exec/DefaultPlanExecutor.java   |   2 +
 .../exec/RowBatchToArrowConverter.java        | 138 ------
 .../action/FragmentExecutionResponse.java     |  70 ++-
 .../exec/stage/ArrowSchemaFromCalcite.java    |  18 +-
 .../exec/stage/LocalStageScheduler.java       |  23 +-
 .../exec/stage/RowResponseCodec.java          | 263 +++--------
 .../analytics/planner/ArrowCalciteTypes.java  |  58 +++
 .../analytics/planner/PlannerImpl.java        |  47 +-
 .../dag/AggregateDecompositionResolver.java   | 438 ++++++++++++++++++
 .../planner/rel/OpenSearchProject.java        |  14 +-
 .../rules/OpenSearchAggregateReduceRule.java  |  51 ++
 .../rules/OpenSearchAggregateSplitRule.java   |  32 +-
 .../planner/rules/OpenSearchProjectRule.java  |  70 +++
 .../ShardFragmentStageExecutionTests.java     |  13 +-
 .../planner/ArrowCalciteTypesTests.java       |  97 ++++
 .../planner/MockDataFusionBackend.java        |   2 +-
 .../analytics/planner/ProjectRuleTests.java   | 124 ++---
 .../AggregateDecompositionResolverTests.java  | 401 ++++++++++++++++
 sandbox/qa/analytics-engine-rest/build.gradle |   1 +
 .../analytics/qa/AppendPipeCommandIT.java     |   2 +
 .../analytics/qa/CoordinatorReduceIT.java     | 303 ++++++++++--
 .../analytics/qa/DslClickBenchIT.java         |   9 +-
 .../analytics/qa/MultisearchCommandIT.java    |   2 +
 .../analytics/qa/PplClickBenchIT.java         |  42 +-
 .../datasets/clickbench/mapping.json          |   4 +-
 .../resources/datasets/clickbench/ppl/q29.ppl |   2 +-
 56 files changed, 3148 insertions(+), 696 deletions(-)
 delete mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateDecomposition.java
 create mode 100644 sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/AggregateFunctionTests.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/agg_mode.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionReduceState.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PartialAggregateInstructionHandler.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_aggregate_functions.yaml
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/NativeBridgePreparedPlanTests.java
 delete mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/RowBatchToArrowConverter.java
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/ArrowCalciteTypes.java
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/AggregateDecompositionResolver.java
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateReduceRule.java
 create mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ArrowCalciteTypesTests.java
 create mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/AggregateDecompositionResolverTests.java

diff --git a/sandbox/libs/analytics-framework/build.gradle b/sandbox/libs/analytics-framework/build.gradle
index 432aa4248e44c..c8d0dda621fe9 100644
--- a/sandbox/libs/analytics-framework/build.gradle
+++ b/sandbox/libs/analytics-framework/build.gradle
@@ -21,6 +21,18 @@ java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = Ja
 configurations {
   calciteCompile
   compileClasspath { exclude group: 'com.google.guava' }
+  testRuntimeClasspath {
+    resolutionStrategy {
+      force "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}"
+      force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
+      force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+      force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+      force "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}"
+      force "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
+      force "org.slf4j:slf4j-api:${versions.slf4j}"
+      force "commons-codec:commons-codec:${versions.commonscodec}"
+    }
+  }
 }
 sourceSets.main.compileClasspath += configurations.calciteCompile
 sourceSets.test.compileClasspath += configurations.calciteCompile
@@ -64,6 +76,13 @@ dependencies {
     exclude group: 'org.opensearch', module: 'opensearch-core'
     exclude group: 'org.opensearch', module: 'opensearch-common'
   }
+  testRuntimeOnly "org.apache.arrow:arrow-vector:${versions.arrow}"
+  testRuntimeOnly "org.apache.arrow:arrow-memory-core:${versions.arrow}"
+  testRuntimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}"
+  testRuntimeOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+  testRuntimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+  testRuntimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}"
+  testRuntimeOnly "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
   runtimeOnly 'com.jayway.jsonpath:json-path:2.9.0'
   runtimeOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
   runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateCapability.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateCapability.java
index 94c2c2b44b7d2..8c3aa51ca5f17 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateCapability.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateCapability.java
@@ -8,8 +8,6 @@
 
 package org.opensearch.analytics.spi;
 
-import org.opensearch.common.Nullable;
-
 import java.util.Set;
 
 /**
@@ -21,32 +19,19 @@
  * validate the function type at construction and make backend declarations
  * self-documenting.
  *
- * 
            {@link #decomposition()} is null for most functions — the planner applies
- * Calcite's standard decomposition (AVG → SUM/COUNT, STDDEV → SUM(x²)+SUM(x)+COUNT).
- * Backends with non-standard partial state (e.g. HLL sketches, Welford STDDEV)
- * provide a custom {@link AggregateDecomposition}.
- *
- * 
            TODO (plan forking): during resolution of a plan alternative, after a single
- * backend is chosen for an aggregate operator, apply decomposition as a paired
- * rewrite of PARTIAL output schema + FINAL input schema:
- * 
              
- *   
            1. If decomposition == null: apply Calcite's AggregateReduceFunctionsRule
- *       to the PARTIAL+FINAL pair.
              2. 
- *   
            2. If decomposition != null: use decomposition.partialCalls() to rewrite
- *       PARTIAL's aggCalls and output row type, then use decomposition.finalExpression()
- *       to rewrite FINAL's aggCalls. Both must be updated together — the exchange
- *       row type between them must be consistent.
              3. 
- * 
            
+ * 
            Decomposition of partial/final aggregate pairs is handled uniformly, outside
+ * this record:
+ * 
              
+ *   
            • Multi-field primitive decomposition (AVG / STDDEV / VAR) runs in HEP via
+ *       {@code OpenSearchAggregateReduceRule}.
              • 
+ *   
            • Single-field pass-through / function-swap / engine-native reductions run in
+ *       {@code AggregateDecompositionResolver} using
+ *       {@link AggregateFunction#intermediateFields()} as the sole source of truth.
              • 
+ * 
            
  *
  * @opensearch.internal
  */
-public record AggregateCapability(AggregateFunction function, Set fieldTypes, Set formats,
-    @Nullable AggregateDecomposition decomposition) {
-
-    /** Convenience constructor with no custom decomposition (uses Calcite's standard). */
-    public AggregateCapability(AggregateFunction function, Set fieldTypes, Set formats) {
-        this(function, fieldTypes, formats, null);
-    }
+public record AggregateCapability(AggregateFunction function, Set fieldTypes, Set formats) {
 
     public static AggregateCapability simple(AggregateFunction function, Set fieldTypes, Set formats) {
         assert function.getType() == AggregateFunction.Type.SIMPLE;
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateDecomposition.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateDecomposition.java
deleted file mode 100644
index e81f18eb23559..0000000000000
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateDecomposition.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.analytics.spi;
-
-import org.apache.calcite.rel.core.AggregateCall;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexNode;
-
-import java.util.List;
-
-/**
- * Describes how a backend decomposes an aggregate function into partial and final phases
- * for distributed execution across shards.
- *
- * 
            When {@link AggregateCapability#decomposition()} is null, the planner applies
- * Calcite's standard decomposition (e.g. AVG → SUM/COUNT, STDDEV_POP → SUM(x²)+SUM(x)+COUNT).
- *
- * 
            When non-null, the planner uses this decomposition during plan forking resolution,
- * after a single backend has been chosen for the aggregate operator. The decomposition
- * rewrites the PARTIAL aggregate's output schema and the FINAL aggregate's input schema
- * as a paired operation — they must be consistent within the same plan alternative.
- *
- * 
            Examples:
- * 
              
- *   
            • COVAR_POP(x, y): partial emits SUM(x*y), SUM(x), SUM(y), COUNT;
- *       final expression: (SUM(x*y) - SUM(x)*SUM(y)/COUNT) / COUNT
              • 
- *   
            • HLL distinct count: partial emits a single HLL sketch accumulator;
- *       final expression: HLL_MERGE(sketches) → cardinality estimate
              • 
- * 
            
- *
- * @opensearch.internal
- */
-public interface AggregateDecomposition {
-
-    /**
-     * The aggregate calls emitted by the PARTIAL phase.
-     * These replace the original aggregate call in the PARTIAL operator and define
-     * the columns flowing through the exchange to the FINAL operator.
-     *
-     * 
            The returned calls must use types compatible with
-     * Calcite's type system so the exchange row type is well-defined.
-     */
-    List partialCalls();
-
-    /**
-     * Expression over the partial results that produces the final aggregated value.
-     * {@code partialRefs} are {@link org.apache.calcite.rex.RexInputRef} nodes
-     * referencing the columns emitted by {@link #partialCalls()} in order.
-     *
-     * 
            For AVG: {@code partialRefs.get(0) / partialRefs.get(1)} (SUM / COUNT).
-     * For HLL: a call to the backend's HLL_MERGE function over {@code partialRefs.get(0)}.
-     */
-    RexNode finalExpression(RexBuilder rexBuilder, List partialRefs);
-}
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateFunction.java
index b72e794e93684..d5d0935e0a09d 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateFunction.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateFunction.java
@@ -8,7 +8,12 @@
 
 package org.opensearch.analytics.spi;
 
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.calcite.sql.SqlAggFunction;
 import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+
+import java.util.List;
 
 /**
  * Aggregate functions that a backend may support, categorized by {@link Type}.
@@ -24,10 +29,16 @@ public enum AggregateFunction {
     SUM0(Type.SIMPLE, SqlKind.SUM0),
     MIN(Type.SIMPLE, SqlKind.MIN),
     MAX(Type.SIMPLE, SqlKind.MAX),
-    COUNT(Type.SIMPLE, SqlKind.COUNT),
+    COUNT(Type.SIMPLE, SqlKind.COUNT, fields(IF("count", new ArrowType.Int(64, true), SUM))),
+    // AVG's distributed decomposition (AVG(x) → CAST(SUM(x) / COUNT(x))) is handled by
+    // OpenSearchAggregateReduceRule during HEP marking, not by the enum + resolver.
+    // No intermediateFields needed here — the rule emits primitive SUM/COUNT calls and
+    // a Project wrapper before the resolver sees the plan.
     AVG(Type.SIMPLE, SqlKind.AVG),
 
-    // Statistical — fixed-size state, multi-pass or running stats
+    // Statistical — fixed-size state, multi-pass or running stats. Handled by
+    // OpenSearchAggregateReduceRule (once FUNCTIONS_TO_REDUCE is extended to include them)
+    // — no intermediateFields here either.
     STDDEV_POP(Type.STATISTICAL, SqlKind.STDDEV_POP),
     STDDEV_SAMP(Type.STATISTICAL, SqlKind.STDDEV_SAMP),
     VAR_POP(Type.STATISTICAL, SqlKind.VAR_POP),
@@ -39,8 +50,10 @@ public enum AggregateFunction {
     COLLECT(Type.STATE_EXPANDING, SqlKind.COLLECT),
     LISTAGG(Type.STATE_EXPANDING, SqlKind.LISTAGG),
 
-    // Approximate — probabilistic, fixed-size state
-    APPROX_COUNT_DISTINCT(Type.APPROXIMATE, SqlKind.OTHER);
+    // Approximate — probabilistic, fixed-size state. Engine-native merge: null reducer
+    // means the field is reduced by this same function (APPROX_COUNT_DISTINCT merges
+    // partial HLL sketches into a final sketch).
+    APPROX_COUNT_DISTINCT(Type.APPROXIMATE, SqlKind.OTHER, fields(IF("sketch", new ArrowType.Binary(), null)));
 
     /** Category of aggregate function. Affects execution strategy (shuffle vs map-reduce). */
     public enum Type {
@@ -50,12 +63,22 @@ public enum Type {
         APPROXIMATE
     }
 
+    /** Describes one intermediate field emitted by a partial aggregate. A null reducer means "self" (the owning enum constant). */
+    public record IntermediateField(String name, ArrowType arrowType, AggregateFunction reducer) {
+    }
+
     private final Type type;
     private final SqlKind sqlKind;
+    private final List intermediateFields;
 
     AggregateFunction(Type type, SqlKind sqlKind) {
+        this(type, sqlKind, null);
+    }
+
+    AggregateFunction(Type type, SqlKind sqlKind, List intermediateFields) {
         this.type = type;
         this.sqlKind = sqlKind;
+        this.intermediateFields = intermediateFields;
     }
 
     public Type getType() {
@@ -66,6 +89,18 @@ public SqlKind getSqlKind() {
         return sqlKind;
     }
 
+    /** Returns intermediate fields with null reducers resolved to {@code this}. */
+    public List intermediateFields() {
+        if (intermediateFields == null) return null;
+        return intermediateFields.stream()
+            .map(f -> f.reducer() == null ? new IntermediateField(f.name(), f.arrowType(), this) : f)
+            .toList();
+    }
+
+    public boolean hasDecomposition() {
+        return intermediateFields != null;
+    }
+
     /** Maps a Calcite SqlKind to an AggregateFunction, or null if not recognized. Skips OTHER. */
     public static AggregateFunction fromSqlKind(SqlKind kind) {
         for (AggregateFunction func : values()) {
@@ -84,4 +119,50 @@ public static AggregateFunction fromNameOrError(String name) {
             throw new IllegalStateException("Unrecognized aggregate function [" + name + "]", e);
         }
     }
+
+    /**
+     * Returns the Calcite {@link SqlAggFunction} equivalent of this enum constant.
+     * Used when emitting rewritten aggregate calls (e.g. the resolver building a
+     * FINAL-phase call for a function-swap or engine-native merge).
+     */
+    public SqlAggFunction toSqlAggFunction() {
+        return switch (this) {
+            case SUM -> SqlStdOperatorTable.SUM;
+            case SUM0 -> SqlStdOperatorTable.SUM0;
+            case MIN -> SqlStdOperatorTable.MIN;
+            case MAX -> SqlStdOperatorTable.MAX;
+            case COUNT -> SqlStdOperatorTable.COUNT;
+            case AVG -> SqlStdOperatorTable.AVG;
+            case APPROX_COUNT_DISTINCT -> SqlStdOperatorTable.APPROX_COUNT_DISTINCT;
+            default -> throw new IllegalStateException("No SqlAggFunction mapping for: " + this);
+        };
+    }
+
+    /**
+     * Resolves a Calcite {@link SqlAggFunction} back to an {@link AggregateFunction}.
+     * Tries name-based lookup first (handles SqlKind.OTHER cases like APPROX_COUNT_DISTINCT)
+     * and falls back to SqlKind matching. Throws if neither path succeeds.
+     */
+    public static AggregateFunction fromSqlAggFunction(SqlAggFunction op) {
+        try {
+            return fromNameOrError(op.getName());
+        } catch (IllegalStateException e) {
+            // Fall through to SqlKind-based resolution
+        }
+        AggregateFunction byKind = fromSqlKind(op.getKind());
+        if (byKind != null) {
+            return byKind;
+        }
+        throw new IllegalStateException("No AggregateFunction mapping for SqlAggFunction [" + op.getName() + "]");
+    }
+
+    // ── Helpers for readable enum-entry literals ──
+
+    private static List fields(IntermediateField... fs) {
+        return List.of(fs);
+    }
+
+    private static IntermediateField IF(String name, ArrowType arrowType, AggregateFunction reducer) {
+        return new IntermediateField(name, arrowType, reducer);
+    }
 }
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkProvider.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkProvider.java
index 001b318578f9b..dcef3717354cd 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkProvider.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ExchangeSinkProvider.java
@@ -27,6 +27,10 @@ public interface ExchangeSinkProvider {
      * uses {@link ExchangeSinkContext#fragmentBytes()} as the serialized plan
      * (produced by {@link FragmentConvertor#convertFinalAggFragment}) and
      * writes its reduced output into {@link ExchangeSinkContext#downstream()}.
+     *
+     * @param context core-provided context carrying plan bytes, allocator, child inputs, and downstream sink
+     * @param backendContext backend-opaque state produced by instruction handlers (e.g.
+     *        {@code FinalAggregateInstructionHandler}), or {@code null} when no handler ran
      */
-    ExchangeSink createSink(ExchangeSinkContext context);
+    ExchangeSink createSink(ExchangeSinkContext context, BackendExecutionContext backendContext);
 }
diff --git a/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/AggregateFunctionTests.java b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/AggregateFunctionTests.java
new file mode 100644
index 0000000000000..52aacda44d8fa
--- /dev/null
+++ b/sandbox/libs/analytics-framework/src/test/java/org/opensearch/analytics/spi/AggregateFunctionTests.java
@@ -0,0 +1,96 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.spi;
+
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.calcite.sql.SqlKind;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.List;
+
+import static org.opensearch.analytics.spi.AggregateFunction.APPROX_COUNT_DISTINCT;
+import static org.opensearch.analytics.spi.AggregateFunction.AVG;
+import static org.opensearch.analytics.spi.AggregateFunction.COUNT;
+import static org.opensearch.analytics.spi.AggregateFunction.MAX;
+import static org.opensearch.analytics.spi.AggregateFunction.MIN;
+import static org.opensearch.analytics.spi.AggregateFunction.SUM;
+
+/**
+ * Asserts the enum carries the right shape per function for the resolver's three
+ * single-field decomposition cases: pass-through (no intermediate), function-swap
+ * (reducer ≠ self), engine-native merge (reducer == self, binary intermediate).
+ *
+ * 
            Multi-field / scalar-final shapes (AVG, STDDEV, VAR) are not encoded on
+ * the enum — they're handled by {@code OpenSearchAggregateReduceRule} during HEP
+ * marking using Calcite's {@code AggregateReduceFunctionsRule}. The enum entries for
+ * those functions intentionally declare {@code intermediateFields == null} so that
+ * the resolver's pass-through branch catches any post-reduction primitive calls.
+ */
+public class AggregateFunctionTests extends OpenSearchTestCase {
+
+    // ── Pass-through: SUM / MIN / MAX ──
+
+    public void testSumHasNoDecomposition() {
+        assertFalse(SUM.hasDecomposition());
+        assertNull(SUM.intermediateFields());
+    }
+
+    // ── COUNT: function-swap (single field, reducer != self) ──
+
+    public void testCountHasDecomposition() {
+        assertTrue(COUNT.hasDecomposition());
+    }
+
+    public void testCountIntermediateFields() {
+        List fields = COUNT.intermediateFields();
+        assertEquals(1, fields.size());
+        assertEquals("count", fields.get(0).name());
+        assertSame(SUM, fields.get(0).reducer());
+        assertTrue(fields.get(0).arrowType() instanceof ArrowType.Int);
+        assertEquals(64, ((ArrowType.Int) fields.get(0).arrowType()).getBitWidth());
+    }
+
+    // ── AVG / STDDEV / VAR: handled by Calcite's reduce rule — no enum metadata ──
+
+    public void testAvgHasNoDecomposition() {
+        // AVG decomposition is driven by OpenSearchAggregateReduceRule in HEP, not by the
+        // enum. Enum declares no intermediate — post-reduction plan carries primitive SUM/
+        // COUNT calls whose enum entries ARE decompositions (function-swap / pass-through).
+        assertFalse(AVG.hasDecomposition());
+        assertNull(AVG.intermediateFields());
+    }
+
+    // ── APPROX_COUNT_DISTINCT: engine-native (single binary field, reducer == self) ──
+
+    public void testApproxCountDistinctHasDecomposition() {
+        assertTrue(APPROX_COUNT_DISTINCT.hasDecomposition());
+    }
+
+    public void testApproxCountDistinctReducerIsSelf() {
+        List fields = APPROX_COUNT_DISTINCT.intermediateFields();
+        assertEquals(1, fields.size());
+        assertEquals("sketch", fields.get(0).name());
+        assertSame(APPROX_COUNT_DISTINCT, fields.get(0).reducer());
+        assertTrue(fields.get(0).arrowType() instanceof ArrowType.Binary);
+    }
+
+    // ── fromSqlKind still works ──
+
+    public void testFromSqlKindResolvesExistingEntries() {
+        assertSame(SUM, AggregateFunction.fromSqlKind(SqlKind.SUM));
+        assertSame(MIN, AggregateFunction.fromSqlKind(SqlKind.MIN));
+        assertSame(MAX, AggregateFunction.fromSqlKind(SqlKind.MAX));
+        assertSame(COUNT, AggregateFunction.fromSqlKind(SqlKind.COUNT));
+        assertSame(AVG, AggregateFunction.fromSqlKind(SqlKind.AVG));
+    }
+
+    public void testFromSqlKindReturnsNullForOther() {
+        assertNull(AggregateFunction.fromSqlKind(SqlKind.OTHER));
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/agg_mode.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/agg_mode.rs
new file mode 100644
index 0000000000000..152edf7aaf6ae
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/agg_mode.rs
@@ -0,0 +1,299 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! Aggregate mode stripping for distributed partial/final execution.
+
+use std::sync::Arc;
+
+use datafusion::physical_optimizer::combine_partial_final_agg::CombinePartialFinalAggregate;
+use datafusion::physical_optimizer::optimizer::{PhysicalOptimizer, PhysicalOptimizerRule};
+use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode};
+use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use datafusion::physical_plan::repartition::RepartitionExec;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion_common::Result;
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub(crate) enum Mode {
+    Default,
+    Partial,
+    Final,
+}
+
+/// Returns the default physical optimizer rules with `CombinePartialFinalAggregate` removed.
+pub(crate) fn physical_optimizer_rules_without_combine(
+) -> Vec> {
+    let combine_name = CombinePartialFinalAggregate::new().name().to_string();
+    PhysicalOptimizer::new()
+        .rules
+        .into_iter()
+        .filter(|r| r.name() != combine_name)
+        .collect()
+}
+
+/// Applies aggregate mode stripping to a physical plan.
+pub(crate) fn apply_aggregate_mode(
+    plan: Arc,
+    mode: Mode,
+) -> Result> {
+    match mode {
+        Mode::Default => Ok(plan),
+        Mode::Partial => force_aggregate_mode(plan, AggregateMode::Partial),
+        Mode::Final => force_aggregate_mode(plan, AggregateMode::Final),
+    }
+}
+
+/// Walks the plan tree and strips the half that doesn't match `target`.
+fn force_aggregate_mode(
+    plan: Arc,
+    target: AggregateMode,
+) -> Result> {
+    if let Some(agg) = plan.as_any().downcast_ref::() {
+        if *agg.mode() == target {
+            // Keep this node, recurse into children
+            let new_children: Vec> = agg
+                .children()
+                .into_iter()
+                .map(|c| force_aggregate_mode(Arc::clone(c), target))
+                .collect::>()?;
+            return plan.with_new_children(new_children);
+        }
+        // Mode mismatch — strip this node
+        match target {
+            AggregateMode::Partial => {
+                // Current node is Final; find the Partial subtree below
+                if let Some(partial_subtree) = find_partial_input(Arc::clone(agg.input())) {
+                    return Ok(partial_subtree);
+                }
+                // If no Partial found below, the input itself is the Partial
+                Ok(Arc::clone(agg.input()))
+            }
+            AggregateMode::Final => {
+                // Current node is Partial; skip it, return its child
+                // (the Final above will keep itself)
+                let child = agg.children()[0];
+                force_aggregate_mode(Arc::clone(child), target)
+            }
+            _ => Ok(plan),
+        }
+    } else if plan.as_any().downcast_ref::().is_some()
+        || plan
+            .as_any()
+            .downcast_ref::()
+            .is_some()
+    {
+        // Transparent — recurse through
+        let new_children: Vec> = plan
+            .children()
+            .into_iter()
+            .map(|c| force_aggregate_mode(Arc::clone(c), target))
+            .collect::>()?;
+        plan.with_new_children(new_children)
+    } else {
+        // Leaf or unrelated node — return as-is
+        Ok(plan)
+    }
+}
+
+/// Walks down through RepartitionExec/CoalescePartitionsExec to find an
+/// AggregateExec(Partial) and returns the entire Partial subtree (the
+/// AggregateExec node itself, not just its input).
+fn find_partial_input(plan: Arc) -> Option> {
+    if let Some(agg) = plan.as_any().downcast_ref::() {
+        if *agg.mode() == AggregateMode::Partial {
+            return Some(plan);
+        }
+        return None;
+    }
+    if plan.as_any().downcast_ref::().is_some()
+        || plan
+            .as_any()
+            .downcast_ref::()
+            .is_some()
+    {
+        let children = plan.children();
+        if children.len() == 1 {
+            return find_partial_input(Arc::clone(children[0]));
+        }
+    }
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use datafusion::prelude::*;
+    use datafusion::physical_plan::displayable;
+
+    /// Helper: create a SessionContext with CombinePartialFinalAggregate disabled,
+    /// register a memtable, and produce a physical plan for `SELECT SUM(x) FROM t`.
+    async fn make_agg_plan() -> Arc {
+        let ctx = SessionContext::new_with_state(
+            datafusion::execution::SessionStateBuilder::new()
+                .with_config(SessionConfig::new())
+                .with_default_features()
+                .with_physical_optimizer_rules(physical_optimizer_rules_without_combine())
+                .build(),
+        );
+        let batch = arrow_array::RecordBatch::try_new(
+            Arc::new(arrow::datatypes::Schema::new(vec![
+                arrow::datatypes::Field::new("x", arrow::datatypes::DataType::Int64, false),
+            ])),
+            vec![Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+        ctx.register_batch("t", batch).unwrap();
+        let df = ctx.sql("SELECT SUM(x) FROM t").await.unwrap();
+        df.create_physical_plan().await.unwrap()
+    }
+
+    /// Helper: create a plan with Repartition between Final and Partial.
+    async fn make_agg_plan_with_repartition() -> Arc {
+        let mut config = SessionConfig::new();
+        config.options_mut().execution.target_partitions = 4;
+        let ctx = SessionContext::new_with_state(
+            datafusion::execution::SessionStateBuilder::new()
+                .with_config(config)
+                .with_default_features()
+                .with_physical_optimizer_rules(physical_optimizer_rules_without_combine())
+                .build(),
+        );
+        let batch = arrow_array::RecordBatch::try_new(
+            Arc::new(arrow::datatypes::Schema::new(vec![
+                arrow::datatypes::Field::new("x", arrow::datatypes::DataType::Int64, false),
+            ])),
+            vec![Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+        ctx.register_batch("t", batch).unwrap();
+        // GROUP BY forces repartition with multiple target partitions
+        let df = ctx.sql("SELECT x, SUM(x) FROM t GROUP BY x").await.unwrap();
+        df.create_physical_plan().await.unwrap()
+    }
+
+    fn plan_string(plan: &Arc) -> String {
+        displayable(plan.as_ref()).indent(true).to_string()
+    }
+
+    fn contains_node(plan: &Arc, name: &str) -> bool {
+        if plan.name().contains(name) {
+            return true;
+        }
+        plan.children().iter().any(|c| contains_node(c, name))
+    }
+
+    fn find_agg_modes(plan: &Arc) -> Vec {
+        let mut modes = Vec::new();
+        if let Some(agg) = plan.as_any().downcast_ref::() {
+            modes.push(*agg.mode());
+        }
+        for child in plan.children() {
+            modes.extend(find_agg_modes(child));
+        }
+        modes
+    }
+
+    #[tokio::test]
+    async fn test_strip_partial_over_scan() {
+        // Final(Partial(memtable)) → strip to Partial only
+        let plan = make_agg_plan().await;
+        let modes = find_agg_modes(&plan);
+        assert!(
+            modes.contains(&AggregateMode::Final) || modes.contains(&AggregateMode::Partial),
+            "Plan should have aggregate nodes: {}",
+            plan_string(&plan)
+        );
+
+        let result = apply_aggregate_mode(plan, Mode::Partial).unwrap();
+        let result_modes = find_agg_modes(&result);
+        assert!(
+            result_modes.contains(&AggregateMode::Partial),
+            "Should contain Partial: {}",
+            plan_string(&result)
+        );
+        assert!(
+            !result_modes.contains(&AggregateMode::Final),
+            "Should NOT contain Final: {}",
+            plan_string(&result)
+        );
+    }
+
+    #[tokio::test]
+    async fn test_strip_final_over_scan() {
+        // Final(Partial(memtable)) → strip to Final only (Partial removed)
+        let plan = make_agg_plan().await;
+        let result = apply_aggregate_mode(plan, Mode::Final).unwrap();
+        let result_modes = find_agg_modes(&result);
+        assert!(
+            result_modes.contains(&AggregateMode::Final),
+            "Should contain Final: {}",
+            plan_string(&result)
+        );
+        assert!(
+            !result_modes.contains(&AggregateMode::Partial),
+            "Should NOT contain Partial: {}",
+            plan_string(&result)
+        );
+    }
+
+    #[tokio::test]
+    async fn test_strip_partial_past_repartition() {
+        // Final → Repartition/Coalesce → Partial → scan; strip to Partial
+        let plan = make_agg_plan_with_repartition().await;
+        let plan_str = plan_string(&plan);
+        // Verify the plan has the expected structure
+        let modes = find_agg_modes(&plan);
+        if modes.len() < 2 {
+            // If optimizer collapsed it, just verify Mode::Partial works
+            let result = apply_aggregate_mode(plan, Mode::Partial).unwrap();
+            let result_modes = find_agg_modes(&result);
+            assert!(!result_modes.contains(&AggregateMode::Final));
+            return;
+        }
+
+        let result = apply_aggregate_mode(plan, Mode::Partial).unwrap();
+        let result_modes = find_agg_modes(&result);
+        assert!(
+            !result_modes.contains(&AggregateMode::Final),
+            "Should NOT contain Final after strip: {}\nOriginal: {}",
+            plan_string(&result),
+            plan_str
+        );
+    }
+
+    #[tokio::test]
+    async fn test_strip_final_past_coalesce() {
+        // Final → CoalescePartitions → Partial → scan; strip to Final
+        let plan = make_agg_plan().await;
+        // The simple plan has CoalescePartitions between Final and Partial
+        let result = apply_aggregate_mode(plan, Mode::Final).unwrap();
+        let result_modes = find_agg_modes(&result);
+        assert!(
+            !result_modes.contains(&AggregateMode::Partial),
+            "Should NOT contain Partial after strip: {}",
+            plan_string(&result)
+        );
+        assert!(
+            result_modes.contains(&AggregateMode::Final),
+            "Should contain Final: {}",
+            plan_string(&result)
+        );
+    }
+
+    #[test]
+    fn test_combine_rule_absent() {
+        let rules = physical_optimizer_rules_without_combine();
+        let combine_name = CombinePartialFinalAggregate::new().name().to_string();
+        assert!(
+            !rules.iter().any(|r| r.name() == combine_name),
+            "CombinePartialFinalAggregate should be filtered out"
+        );
+        // Verify we still have other rules
+        assert!(!rules.is_empty(), "Should have other optimizer rules");
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
index 8536764287ba1..e0b8715d2e2d7 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
@@ -750,3 +750,90 @@ pub unsafe extern "C" fn df_stats(out_ptr: *mut u8, out_cap: i64) -> i64 {
     );
     Ok(0)
 }
+
+// ---------------------------------------------------------------------------
+// Distributed aggregate: prepare partial/final plans
+// ---------------------------------------------------------------------------
+
+/// Prepares a partial-aggregate physical plan on the session context handle.
+///
+/// Decodes the Substrait bytes, converts to a physical plan, strips the
+/// final-aggregate half, and stores the result on the handle for later
+/// execution via `df_execute_with_context`.
+///
+/// Returns 0 on success; < 0 is a negated error-string pointer.
+///
+/// # Safety
+/// `handle_ptr` must be a valid pointer returned by `df_create_session_context`.
+/// `bytes_ptr` must point to `bytes_len` valid bytes of a Substrait plan.
+#[ffm_safe]
+#[no_mangle]
+pub unsafe extern "C" fn df_prepare_partial_plan(
+    handle_ptr: i64,
+    bytes_ptr: *const u8,
+    bytes_len: usize,
+) -> i64 {
+    let handle = &mut *(handle_ptr as *mut crate::session_context::SessionContextHandle);
+    let bytes = slice::from_raw_parts(bytes_ptr, bytes_len);
+    let mgr = get_rt_manager()?;
+    mgr.io_runtime
+        .block_on(crate::session_context::prepare_partial_plan(handle, bytes))
+        .map_err(|e| e.to_string())?;
+    Ok(0)
+}
+
+/// Prepares a final-aggregate physical plan on a local session.
+///
+/// Decodes the Substrait bytes, converts to a physical plan, strips the
+/// partial-aggregate half, and stores the result on the session for later
+/// execution via `df_execute_local_prepared_plan`.
+///
+/// Returns 0 on success; < 0 is a negated error-string pointer.
+///
+/// # Safety
+/// `session_ptr` must be a valid pointer returned by `df_create_local_session`.
+/// `bytes_ptr` must point to `bytes_len` valid bytes of a Substrait plan.
+#[ffm_safe]
+#[no_mangle]
+pub unsafe extern "C" fn df_prepare_final_plan(
+    session_ptr: i64,
+    bytes_ptr: *const u8,
+    bytes_len: usize,
+) -> i64 {
+    let session = &mut *(session_ptr as *mut crate::local_executor::LocalSession);
+    let bytes = slice::from_raw_parts(bytes_ptr, bytes_len);
+    let mgr = get_rt_manager()?;
+    mgr.io_runtime
+        .block_on(session.prepare_final_plan(bytes))
+        .map_err(|e| e.to_string())?;
+    Ok(0)
+}
+
+/// Executes the previously prepared final-aggregate plan on a local session.
+///
+/// Returns a stream pointer (same shape as `df_execute_local_plan`) that can
+/// be drained via `df_stream_next` / `df_stream_close`.
+///
+/// # Safety
+/// `session_ptr` must be a valid pointer returned by `df_create_local_session`
+/// with a plan already prepared via `df_prepare_final_plan`.
+#[ffm_safe]
+#[no_mangle]
+pub unsafe extern "C" fn df_execute_local_prepared_plan(session_ptr: i64) -> i64 {
+    let session = &*(session_ptr as *const crate::local_executor::LocalSession);
+    let mgr = get_rt_manager()?;
+    // DataFusion's execute_stream is sync, but kicks off RepartitionExec / stream
+    // channels that require a Tokio reactor. Enter the IO runtime's context so those
+    // operators can register with the reactor.
+    let _guard = mgr.io_runtime.enter();
+    let df_stream = session.execute_prepared().map_err(|e| e.to_string())?;
+    let cross_rt_stream =
+        crate::cross_rt_stream::CrossRtStream::new_with_df_error_stream(df_stream, mgr.cpu_executor());
+    let wrapped = datafusion::physical_plan::stream::RecordBatchStreamAdapter::new(
+        cross_rt_stream.schema(),
+        cross_rt_stream,
+    );
+    let query_context = crate::query_tracker::QueryTrackingContext::new(0, session.memory_pool());
+    let handle = crate::api::QueryStreamHandle::new(wrapped, query_context);
+    Ok(Box::into_raw(Box::new(handle)) as i64)
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
index a6ef5ae241eeb..bb44669221a7b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
@@ -134,6 +134,7 @@ pub async fn execute_indexed_query(
         .with_config(config)
         .with_runtime_env(Arc::from(runtime_env))
         .with_default_features()
+        .with_physical_optimizer_rules(crate::agg_mode::physical_optimizer_rules_without_combine())
         .build();
     let ctx = SessionContext::new_with_state(state);
     ctx.register_udf(create_index_filter_udf());
@@ -162,6 +163,8 @@ pub async fn execute_indexed_query(
         table_name: table_name.clone(),
         indexed_config: None, // derive classification from tree
         query_config: Arc::unwrap_or_clone(query_config),
+        aggregate_mode: crate::agg_mode::Mode::Default,
+        prepared_plan: None,
     };
     let ptr = Box::into_raw(Box::new(handle)) as i64;
     unsafe { execute_indexed_with_context(ptr, substrait_bytes, cpu_executor).await }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs
index 154a4ed6d5a56..6b2ba8f487bfd 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs
@@ -11,6 +11,7 @@
 //! The bridge-agnostic API lives in [`api`]. The FFM bridge (`ffm.rs`) exports
 //! `extern "C"` functions for JDK FFM.
 
+pub(crate) mod agg_mode;
 pub mod api;
 pub mod cache;
 pub mod cancellation;
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs
index 90d267d71a771..62dbd162d38ca 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/local_executor.rs
@@ -52,6 +52,8 @@ use crate::partition_stream::{channel, PartitionStreamSender, SingleReceiverPart
 /// [`Self::execute_substrait`].
 pub struct LocalSession {
     ctx: SessionContext,
+    /// Pre-prepared physical plan (set by `prepare_final_plan`).
+    pub(crate) prepared_plan: Option>,
 }
 
 impl LocalSession {
@@ -61,18 +63,16 @@ impl LocalSession {
     /// every batch consumed or produced by this session counts against the
     /// same limits as the shard-scan path.
     pub fn new(runtime_env: &RuntimeEnv) -> Self {
-        // Cheaply clone the env so the session owns a handle independent of
-        // the caller. `RuntimeEnv` internally holds `Arc`s — this is a
-        // lightweight clone, not a deep copy of the pool or disk manager.
         let runtime_env = Arc::new(runtime_env.clone());
         let state = SessionStateBuilder::new()
             .with_config(SessionConfig::new())
             .with_runtime_env(runtime_env)
             .with_default_features()
+            .with_physical_optimizer_rules(crate::agg_mode::physical_optimizer_rules_without_combine())
             .build();
         let ctx = SessionContext::new_with_state(state);
         crate::udf::register_all(&ctx);
-        Self { ctx }
+        Self { ctx, prepared_plan: None }
     }
 
     /// Registers a streaming input on the session under `name` and returns the
@@ -156,6 +156,44 @@ impl LocalSession {
     pub fn memory_pool(&self) -> Arc {
         Arc::clone(&self.ctx.runtime_env().memory_pool)
     }
+
+    /// Prepares a final-aggregate physical plan on this session.
+    ///
+    /// Decodes Substrait → LogicalPlan → PhysicalPlan, applies final-mode
+    /// stripping, and stores the result for later execution via
+    /// [`Self::execute_prepared`].
+    pub async fn prepare_final_plan(
+        &mut self,
+        substrait_bytes: &[u8],
+    ) -> Result<(), DataFusionError> {
+        let plan = Plan::decode(substrait_bytes).map_err(|e| {
+            DataFusionError::Execution(format!(
+                "prepare_final_plan: failed to decode Substrait: {}",
+                e
+            ))
+        })?;
+        let logical_plan = from_substrait_plan(&self.ctx.state(), &plan).await?;
+        let dataframe = self.ctx.execute_logical_plan(logical_plan).await?;
+        let physical_plan = dataframe.create_physical_plan().await?;
+        let stripped = crate::agg_mode::apply_aggregate_mode(
+            physical_plan,
+            crate::agg_mode::Mode::Final,
+        )?;
+        self.prepared_plan = Some(stripped);
+        Ok(())
+    }
+
+    /// Executes the previously prepared plan and returns the output stream.
+    ///
+    /// # Panics
+    /// Panics if no plan has been prepared via [`Self::prepare_final_plan`].
+    pub fn execute_prepared(&self) -> Result {
+        let plan = self
+            .prepared_plan
+            .as_ref()
+            .expect("execute_prepared called without a prepared plan");
+        datafusion::physical_plan::execute_stream(Arc::clone(plan), self.ctx.task_ctx())
+    }
 }
 
 #[cfg(test)]
@@ -327,4 +365,42 @@ mod tests {
         }
         assert_eq!(total, 45);
     }
+
+    #[tokio::test]
+    async fn prepare_final_plan_stores_plan() {
+        let env = test_runtime_env();
+        let mut session = LocalSession::new(&env);
+        let schema = i64_schema("s");
+
+        // Register a streaming table so the plan can resolve table refs.
+        let _sender = session
+            .register_partition("input-0", Arc::clone(&schema))
+            .expect("register");
+
+        // Build Substrait bytes for SELECT SUM(s) FROM "input-0"
+        let substrait_bytes = {
+            let env2 = test_runtime_env();
+            let mut producer = LocalSession::new(&env2);
+            let _unused = producer
+                .register_partition("input-0", Arc::clone(&schema))
+                .expect("producer register");
+            let df = producer
+                .ctx
+                .sql("SELECT SUM(s) FROM \"input-0\"")
+                .await
+                .expect("sum parses");
+            let plan = df.logical_plan().clone();
+            let substrait = to_substrait_plan(&plan, &producer.ctx.state()).expect("to_substrait");
+            let mut buf = Vec::new();
+            substrait.encode(&mut buf).expect("encode");
+            buf
+        };
+
+        assert!(session.prepared_plan.is_none());
+        session
+            .prepare_final_plan(&substrait_bytes)
+            .await
+            .expect("prepare_final_plan succeeds");
+        assert!(session.prepared_plan.is_some());
+    }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
index cb5eea3112b07..979a3a4c188cc 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
@@ -43,6 +43,10 @@ pub struct SessionContextHandle {
     pub indexed_config: Option,
     /// Per-query tuning knobs (batch size, partitions, filter strategies, etc.)
     pub query_config: DatafusionQueryConfig,
+    /// Aggregate execution mode for distributed partial/final stripping.
+    pub(crate) aggregate_mode: crate::agg_mode::Mode,
+    /// Pre-prepared physical plan (set by prepare_partial_plan / prepare_final_plan).
+    pub(crate) prepared_plan: Option>,
 }
 
 /// Configuration for indexed execution with filter delegation, provided by Java.
@@ -108,6 +112,7 @@ pub async unsafe fn create_session_context(
         .with_config(config)
         .with_runtime_env(Arc::from(runtime_env))
         .with_default_features()
+        .with_physical_optimizer_rules(crate::agg_mode::physical_optimizer_rules_without_combine())
         .build();
 
     let ctx = SessionContext::new_with_state(state);
@@ -164,6 +169,8 @@ pub async unsafe fn create_session_context(
         table_name: table_name.to_string(),
         indexed_config: None,
         query_config,
+        aggregate_mode: crate::agg_mode::Mode::Default,
+        prepared_plan: None,
     };
     Ok(Box::into_raw(Box::new(handle)) as i64)
 }
@@ -203,3 +210,111 @@ pub async unsafe fn create_session_context_indexed(
 
     Ok(ptr)
 }
+
+/// Prepares a partial-aggregate physical plan on the session handle.
+///
+/// Decodes Substrait → LogicalPlan → PhysicalPlan, applies partial-mode
+/// stripping via `agg_mode::apply_aggregate_mode`, and stores the result
+/// on the handle for later execution.
+pub async fn prepare_partial_plan(
+    handle: &mut SessionContextHandle,
+    substrait_bytes: &[u8],
+) -> Result<(), datafusion::common::DataFusionError> {
+    use datafusion_substrait::logical_plan::consumer::from_substrait_plan;
+    use prost::Message;
+    use substrait::proto::Plan;
+
+    handle.aggregate_mode = crate::agg_mode::Mode::Partial;
+
+    let plan = Plan::decode(substrait_bytes).map_err(|e| {
+        datafusion::common::DataFusionError::Execution(format!(
+            "prepare_partial_plan: failed to decode Substrait: {}",
+            e
+        ))
+    })?;
+    let logical_plan = from_substrait_plan(&handle.ctx.state(), &plan).await?;
+    let dataframe = handle.ctx.execute_logical_plan(logical_plan).await?;
+    let physical_plan = dataframe.create_physical_plan().await?;
+    let stripped = crate::agg_mode::apply_aggregate_mode(physical_plan, crate::agg_mode::Mode::Partial)?;
+    handle.prepared_plan = Some(stripped);
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Arc;
+
+    use arrow_array::{Int64Array, RecordBatch};
+    use datafusion::arrow::datatypes::{DataType, Field, Schema};
+    use datafusion::datasource::MemTable;
+    use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+    use datafusion::execution::SessionStateBuilder;
+    use datafusion::prelude::{SessionConfig, SessionContext};
+    use datafusion_substrait::logical_plan::producer::to_substrait_plan;
+    use prost::Message;
+
+    use crate::agg_mode::Mode;
+    use crate::query_tracker::QueryTrackingContext;
+
+    async fn make_test_handle() -> (SessionContextHandle, Vec) {
+        let runtime_env = RuntimeEnvBuilder::new().build().expect("runtime env");
+        let state = SessionStateBuilder::new()
+            .with_config(SessionConfig::new())
+            .with_runtime_env(Arc::new(runtime_env))
+            .with_default_features()
+            .with_physical_optimizer_rules(crate::agg_mode::physical_optimizer_rules_without_combine())
+            .build();
+        let ctx = SessionContext::new_with_state(state);
+
+        // Register an in-memory table with column "x"
+        let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, false)]));
+        let batch = RecordBatch::try_new(
+            Arc::clone(&schema),
+            vec![Arc::new(Int64Array::from(vec![1i64, 2, 3]))],
+        )
+        .expect("batch");
+        let table = MemTable::try_new(Arc::clone(&schema), vec![vec![batch]]).expect("memtable");
+        ctx.register_table("t", Arc::new(table)).expect("register");
+
+        // Build Substrait bytes for SELECT SUM(x) FROM t
+        let df = ctx.sql("SELECT SUM(x) FROM t").await.expect("sql");
+        let plan = df.logical_plan().clone();
+        let substrait = to_substrait_plan(&plan, &ctx.state()).expect("to_substrait");
+        let mut buf = Vec::new();
+        substrait.encode(&mut buf).expect("encode");
+
+        let table_path = datafusion::datasource::listing::ListingTableUrl::parse("file:///tmp")
+            .expect("table_path");
+        let global_pool = ctx.runtime_env().memory_pool.clone();
+        let query_context = QueryTrackingContext::new(0, global_pool);
+
+        let handle = SessionContextHandle {
+            ctx,
+            table_path,
+            object_metas: Arc::new(vec![]),
+            query_context,
+            table_name: "t".to_string(),
+            indexed_config: None,
+            query_config: crate::datafusion_query_config::DatafusionQueryConfig::test_default(),
+            aggregate_mode: Mode::Default,
+            prepared_plan: None,
+        };
+        (handle, buf)
+    }
+
+    #[tokio::test]
+    async fn prepare_partial_plan_sets_mode_and_stores_plan() {
+        let (mut handle, substrait_bytes) = make_test_handle().await;
+
+        assert_eq!(handle.aggregate_mode, Mode::Default);
+        assert!(handle.prepared_plan.is_none());
+
+        prepare_partial_plan(&mut handle, &substrait_bytes)
+            .await
+            .expect("prepare_partial_plan succeeds");
+
+        assert_eq!(handle.aggregate_mode, Mode::Partial);
+        assert!(handle.prepared_plan.is_some());
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java
index f4470bb66333d..37f6388425dd5 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/AbstractDatafusionReduceSink.java
@@ -67,12 +67,33 @@ abstract class AbstractDatafusionReduceSink implements ExchangeSink {
     protected final ExchangeSinkContext ctx;
     protected final NativeRuntimeHandle runtimeHandle;
     protected final DatafusionLocalSession session;
+    /**
+     * Non-null when this sink was constructed with a pre-prepared FINAL-aggregate plan
+     * from the FinalAggregateInstructionHandler. When present, the handler already created
+     * the session, registered the input partitions, and called {@code prepareFinalPlan} on
+     * the Rust side; the sink only needs to drive {@code executeLocalPreparedPlan} and feed
+     * batches. When null, the sink falls back to the legacy path (create its own session,
+     * register its own partitions, call {@code executeLocalPlan}).
+     *
+     * 
            Close ownership: when {@code preparedState != null} the state owns session +
+     * senders and {@link #close} skips re-closing them (avoids double-close on the native
+     * side). When {@code preparedState == null} the base class closes the session itself.
+     */
+    protected final DataFusionReduceState preparedState;
     /**
      * Per-child Arrow schema IPC bytes, keyed by childStageId. Iteration order matches
      * the order of {@code ctx.childInputs()} so subclasses get deterministic registration.
      */
     protected final Map childInputs;
 
+    /**
+     * Declared Arrow {@link org.apache.arrow.vector.types.pojo.Schema} per childStageId,
+     * parallel to {@link #childInputs}. Used by sinks to coerce incoming batches when
+     * the shard's actual emit type diverges from the declaration (e.g. DataFusion's
+     * {@code Utf8View} for string group keys vs. declared {@code Utf8}).
+     */
+    protected final Map childSchemas;
+
     /** Guards {@link #closed} and serialises {@link #feed}/{@link #close} against producers. */
     protected final Object feedLock = new Object();
 
@@ -80,14 +101,26 @@ abstract class AbstractDatafusionReduceSink implements ExchangeSink {
     protected volatile boolean closed;
 
     protected AbstractDatafusionReduceSink(ExchangeSinkContext ctx, NativeRuntimeHandle runtimeHandle) {
+        this(ctx, runtimeHandle, null);
+    }
+
+    protected AbstractDatafusionReduceSink(
+        ExchangeSinkContext ctx,
+        NativeRuntimeHandle runtimeHandle,
+        DataFusionReduceState preparedState
+    ) {
         this.ctx = ctx;
         this.runtimeHandle = runtimeHandle;
-        this.session = new DatafusionLocalSession(runtimeHandle.get());
+        this.preparedState = preparedState;
+        this.session = preparedState != null ? preparedState.session() : new DatafusionLocalSession(runtimeHandle.get());
         Map inputs = new LinkedHashMap<>(ctx.childInputs().size());
+        Map schemas = new LinkedHashMap<>(ctx.childInputs().size());
         for (ExchangeSinkContext.ChildInput child : ctx.childInputs()) {
             inputs.put(child.childStageId(), ArrowSchemaIpc.toBytes(child.schema()));
+            schemas.put(child.childStageId(), child.schema());
         }
         this.childInputs = inputs;
+        this.childSchemas = schemas;
     }
 
     /** DataFusion table name for an input partition associated with the given child stage id. */
@@ -124,10 +157,14 @@ public final void close() {
         } catch (Throwable t) {
             failure = accumulate(failure, t);
         } finally {
-            try {
-                session.close();
-            } catch (Throwable t) {
-                failure = accumulate(failure, t);
+            // If a preparedState owns the session/senders, let the state's close handle
+            // them (invoked by the orchestrator). Otherwise close the session we created.
+            if (preparedState == null) {
+                try {
+                    session.close();
+                } catch (Throwable t) {
+                    failure = accumulate(failure, t);
+                }
             }
         }
         rethrow(failure);
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
index 176de49872dd1..d8977b894a332 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
@@ -275,7 +275,8 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
         AggregateFunction.MIN,
         AggregateFunction.MAX,
         AggregateFunction.COUNT,
-        AggregateFunction.AVG
+        AggregateFunction.AVG,
+        AggregateFunction.APPROX_COUNT_DISTINCT
     );
 
     private final DataFusionPlugin plugin;
@@ -339,7 +340,13 @@ public Set aggregateCapabilities() {
                 Set caps = new HashSet<>();
                 for (AggregateFunction func : AGG_FUNCTIONS) {
                     for (FieldType type : SUPPORTED_FIELD_TYPES) {
-                        caps.add(AggregateCapability.simple(func, Set.of(type), formats));
+                        // 3-arg constructor leaves decomposition=null so the
+                        // AggregateDecompositionResolver falls back to the enum's
+                        // intermediateFields + finalExpression — the single source of truth
+                        // for per-function distributed-execution behavior. Accepts any
+                        // AggregateFunction.Type (SIMPLE, APPROXIMATE, ...), unlike the
+                        // per-type factory methods which assert on Type.
+                        caps.add(new AggregateCapability(func, Set.of(type), formats));
                     }
                 }
                 return Set.copyOf(caps);
@@ -480,11 +487,16 @@ public FragmentInstructionHandlerFactory getInstructionHandlerFactory() {
 
     @Override
     public ExchangeSinkProvider getExchangeSinkProvider() {
-        return ctx -> {
+        return (ctx, backendContext) -> {
             DataFusionService svc = plugin.getDataFusionService();
             if (svc == null) {
                 throw new IllegalStateException("DataFusionService not initialized");
             }
+            // When the FinalAggregateInstructionHandler has already prepared a plan on the
+            // coordinator, it hands over a DataFusionReduceState carrying the session +
+            // registered senders. The sink drives executeLocalPreparedPlan against that
+            // state instead of re-decoding the fragment bytes.
+            DataFusionReduceState preparedState = backendContext instanceof DataFusionReduceState s ? s : null;
             String mode = plugin.getClusterService() != null
                 ? plugin.getClusterService().getClusterSettings().get(DataFusionPlugin.DATAFUSION_REDUCE_INPUT_MODE)
                 : "streaming";
@@ -492,13 +504,15 @@ public ExchangeSinkProvider getExchangeSinkProvider() {
             // exactly one MemTable at close time). Multi-input shapes (Union, future Join)
             // need per-child input partitions, which only the streaming sink implements via
             // MultiInputExchangeSink#sinkForChild. Auto-fall-back to streaming so end users
-            // don't have to flip the cluster setting per query.
+            // don't have to flip the cluster setting per query. Also fall back when a
+            // prepared state is supplied (memtable sink does not yet support the
+            // prepared-plan path).
             // TODO: lift this fallback once the memtable sink registers one MemTable per
             // child stage (see DatafusionMemtableReduceSink class javadoc).
-            if ("memtable".equals(mode) && ctx.childInputs().size() == 1) {
+            if ("memtable".equals(mode) && ctx.childInputs().size() == 1 && preparedState == null) {
                 return new DatafusionMemtableReduceSink(ctx, svc.getNativeRuntime());
             }
-            return new DatafusionReduceSink(ctx, svc.getNativeRuntime());
+            return new DatafusionReduceSink(ctx, svc.getNativeRuntime(), preparedState);
         };
     }
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
index 2161abc08b7a2..e87ebbc8eb550 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.be.datafusion;
 
+import com.google.common.collect.ImmutableList;
 import com.google.protobuf.InvalidProtocolBufferException;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptSchema;
@@ -172,6 +173,50 @@ public class DataFusionFragmentConvertor implements FragmentConvertor {
         FunctionMappings.s(MvappendAdapter.LOCAL_MVAPPEND_OP, "mvappend")
     );
 
+    /**
+     * Maps aggregate operators to their Substrait extension names so isthmus serializes
+     * them through our {@code SimpleExtension} catalog instead of the default Substrait
+     * names.
+     *
+     * 
            {@link SqlStdOperatorTable#APPROX_COUNT_DISTINCT} → {@code approx_distinct}
+     * (declared in {@code opensearch_aggregate_functions.yaml}) routes to DataFusion's
+     * native HyperLogLog {@code APPROX_DISTINCT} aggregate. Wiring this through isthmus'
+     * {@code ADDITIONAL_AGGREGATE_SIGS} alone is not enough because isthmus's default
+     * aggregate catalog already binds {@code APPROX_COUNT_DISTINCT} to substrait's
+     * standard {@code approx_count_distinct} URN; when signatures merge, the default
+     * binding overwrites ours in the matcher map. {@link OpenSearchAggregateFunctionConverter}
+     * fixes that by filtering the stock sig out of the default list so our entry is the
+     * only one that resolves to this operator.
+     */
+    private static final List ADDITIONAL_AGGREGATE_SIGS = List.of(
+        FunctionMappings.s(SqlStdOperatorTable.APPROX_COUNT_DISTINCT, "approx_distinct")
+    );
+
+    /**
+     * Subclassed {@link AggregateFunctionConverter} that removes isthmus's default binding
+     * for {@link SqlStdOperatorTable#APPROX_COUNT_DISTINCT} from the signature merge.
+     * Without this, the default {@code approx_count_distinct} URN binding would shadow
+     * our entry in {@link #ADDITIONAL_AGGREGATE_SIGS} and the YAML-declared
+     * {@code approx_distinct} extension would never be reached.
+     */
+    private static final class OpenSearchAggregateFunctionConverter extends AggregateFunctionConverter {
+        OpenSearchAggregateFunctionConverter(
+            List functions,
+            List additionalSignatures,
+            RelDataTypeFactory typeFactory,
+            TypeConverter typeConverter
+        ) {
+            super(functions, additionalSignatures, typeFactory, typeConverter);
+        }
+
+        @Override
+        protected ImmutableList getSigs() {
+            return super.getSigs().stream()
+                .filter(sig -> sig.operator != SqlStdOperatorTable.APPROX_COUNT_DISTINCT)
+                .collect(ImmutableList.toImmutableList());
+        }
+    }
+
     private final SimpleExtension.ExtensionCollection extensions;
 
     public DataFusionFragmentConvertor(SimpleExtension.ExtensionCollection extensions) {
@@ -391,7 +436,12 @@ private SubstraitRelVisitor createVisitor(RelNode relNode) {
             typeFactory,
             typeConverter
         );
-        AggregateFunctionConverter aggConverter = new AggregateFunctionConverter(extensions.aggregateFunctions(), typeFactory);
+        AggregateFunctionConverter aggConverter = new OpenSearchAggregateFunctionConverter(
+            extensions.aggregateFunctions(),
+            ADDITIONAL_AGGREGATE_SIGS,
+            typeFactory,
+            typeConverter
+        );
         WindowFunctionConverter windowConverter = new WindowFunctionConverter(extensions.windowFunctions(), typeFactory);
         ConverterProvider converterProvider = new ConverterProvider(
             typeFactory,
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java
index 4dbe7f280fc53..737a0540b531e 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionInstructionHandlerFactory.java
@@ -15,6 +15,7 @@
 import org.opensearch.analytics.spi.FragmentInstructionHandler;
 import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory;
 import org.opensearch.analytics.spi.InstructionNode;
+import org.opensearch.analytics.spi.PartialAggregateInstructionNode;
 import org.opensearch.analytics.spi.ShardScanInstructionNode;
 import org.opensearch.analytics.spi.ShardScanWithDelegationInstructionNode;
 
@@ -58,8 +59,7 @@ public Optional createShardScanWithDelegationNode(FilterTreeSha
 
     @Override
     public Optional createPartialAggregateNode() {
-        // TODO: return Optional.of(...) once PartialAggregateInstructionHandler is implemented
-        return Optional.empty();
+        return Optional.of(new PartialAggregateInstructionNode());
     }
 
     @Override
@@ -78,10 +78,13 @@ public FragmentInstructionHandler createHandler(InstructionNode node) {
         if (node instanceof ShardScanInstructionNode) {
             return new ShardScanInstructionHandler(plugin);
         }
+        if (node instanceof PartialAggregateInstructionNode) {
+            return new PartialAggregateInstructionHandler();
+        }
         if (node instanceof FinalAggregateInstructionNode) {
-            return new FinalAggregateInstructionHandler();
+            DataFusionService svc = plugin.getDataFusionService();
+            return new FinalAggregateInstructionHandler(svc.getNativeRuntime());
         }
-        // TODO: FilterDelegationInstructionHandler, PartialAggregateInstructionHandler
         throw new UnsupportedOperationException("No handler for instruction type: " + node.type());
     }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
index d72823015dae5..59b581d549b2b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
@@ -172,7 +172,11 @@ private static SimpleExtension.ExtensionCollection loadSubstraitExtensions() {
             SimpleExtension.ExtensionCollection delegationExtensions = SimpleExtension.load(List.of("/delegation_functions.yaml"));
             SimpleExtension.ExtensionCollection scalarExtensions = SimpleExtension.load(List.of("/opensearch_scalar_functions.yaml"));
             SimpleExtension.ExtensionCollection arrayExtensions = SimpleExtension.load(List.of("/opensearch_array_functions.yaml"));
-            return DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions).merge(scalarExtensions).merge(arrayExtensions);
+            SimpleExtension.ExtensionCollection aggregateExtensions = SimpleExtension.load(List.of("/opensearch_aggregate_functions.yaml"));
+            return DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions)
+                .merge(scalarExtensions)
+                .merge(arrayExtensions)
+                .merge(aggregateExtensions);
         } finally {
             t.setContextClassLoader(previous);
         }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionReduceState.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionReduceState.java
new file mode 100644
index 0000000000000..f43722c6e21b8
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionReduceState.java
@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.analytics.spi.BackendExecutionContext;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Backend-specific execution context for the coordinator-reduce path when a final-aggregate
+ * plan has been prepared. Carries the local session (with the prepared plan stored on the
+ * Rust side), the runtime handle, and the partition senders used to feed Arrow batches
+ * into the streaming input partitions.
+ *
+ * 
            Produced by {@link FinalAggregateInstructionHandler} and consumed by
+ * {@link DatafusionReduceSink} via the {@link org.opensearch.analytics.spi.ExchangeSinkProvider}
+ * contract.
+ *
+ * @opensearch.internal
+ */
+public record DataFusionReduceState(DatafusionLocalSession session, NativeRuntimeHandle runtimeHandle, List<
+    DatafusionPartitionSender> senders) implements BackendExecutionContext {
+
+    @Override
+    public void close() throws IOException {
+        // Close senders first, then session.
+        for (DatafusionPartitionSender sender : senders) {
+            try {
+                sender.close();
+            } catch (Exception ignored) {}
+        }
+        session.close();
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java
index c4610b34d85ab..46d9d384b2833 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java
@@ -12,7 +12,12 @@
 import org.apache.arrow.c.ArrowSchema;
 import org.apache.arrow.c.Data;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.ViewVarCharVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Schema;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.analytics.spi.ExchangeSink;
@@ -85,31 +90,52 @@ public final class DatafusionReduceSink extends AbstractDatafusionReduceSink imp
     private final AtomicReference drainFailure = new AtomicReference<>();
 
     public DatafusionReduceSink(ExchangeSinkContext ctx, NativeRuntimeHandle runtimeHandle) {
-        super(ctx, runtimeHandle);
+        this(ctx, runtimeHandle, null);
+    }
+
+    public DatafusionReduceSink(ExchangeSinkContext ctx, NativeRuntimeHandle runtimeHandle, DataFusionReduceState preparedState) {
+        super(ctx, runtimeHandle, preparedState);
         Map senders = new LinkedHashMap<>(childInputs.size());
         long streamPtr = 0;
         try {
-            // Register one native partition per child stage. The Substrait plan in
-            // ctx.fragmentBytes() references each partition by its "input-" name
-            // (DataFusionFragmentConvertor names them this way during plan conversion).
-            for (Map.Entry child : childInputs.entrySet()) {
-                int childStageId = child.getKey();
-                byte[] schemaIpc = child.getValue();
-                long senderPtr = NativeBridge.registerPartitionStream(session.getPointer(), inputIdFor(childStageId), schemaIpc);
-                senders.put(childStageId, new DatafusionPartitionSender(senderPtr));
+            if (preparedState != null) {
+                // Plan was already prepared by FinalAggregateInstructionHandler. The handler
+                // registered senders in ctx.childInputs() iteration order; we re-index them
+                // here by childStageId for lookup during feed().
+                int i = 0;
+                for (Map.Entry child : childInputs.entrySet()) {
+                    senders.put(child.getKey(), preparedState.senders().get(i++));
+                }
+                streamPtr = NativeBridge.executeLocalPreparedPlan(session.getPointer());
+            } else {
+                // Legacy path (non-aggregate reduce): register partitions and execute the
+                // fragment bytes directly. Used when no prior instruction prepared a plan.
+                //
+                // ctx.fragmentBytes() references each partition by its "input-" name
+                // (DataFusionFragmentConvertor names them this way during plan conversion).
+                for (Map.Entry child : childInputs.entrySet()) {
+                    int childStageId = child.getKey();
+                    byte[] schemaIpc = child.getValue();
+                    long senderPtr = NativeBridge.registerPartitionStream(session.getPointer(), inputIdFor(childStageId), schemaIpc);
+                    senders.put(childStageId, new DatafusionPartitionSender(senderPtr));
+                }
+                streamPtr = NativeBridge.executeLocalPlan(session.getPointer(), ctx.fragmentBytes());
             }
-            streamPtr = NativeBridge.executeLocalPlan(session.getPointer(), ctx.fragmentBytes());
             this.outStream = new StreamHandle(streamPtr, runtimeHandle);
         } catch (RuntimeException e) {
             if (streamPtr != 0) {
                 NativeBridge.streamClose(streamPtr);
             }
-            for (DatafusionPartitionSender sender : senders.values()) {
-                try {
-                    sender.close();
-                } catch (Throwable ignore) {}
+            // Only close senders we allocated locally (legacy path). When preparedState
+            // owns them, the state's close() will.
+            if (preparedState == null) {
+                for (DatafusionPartitionSender sender : senders.values()) {
+                    try {
+                        sender.close();
+                    } catch (Throwable ignore) {}
+                }
+                session.close();
             }
-            session.close();
             throw e;
         }
         this.sendersByChildStageId = senders;
@@ -147,7 +173,7 @@ public void feed(VectorSchemaRoot batch) {
                 "DatafusionReduceSink has " + sendersByChildStageId.size() + " input partitions; use sinkForChild(int) instead of feed()"
             );
         }
-        feedToSender(sendersByChildStageId.values().iterator().next(), batch);
+        feedToSender(sendersByChildStageId.values().iterator().next(), batch, childSchemas.values().iterator().next());
     }
 
     @Override
@@ -158,7 +184,7 @@ public ExchangeSink sinkForChild(int childStageId) {
                 "No registered partition for childStageId=" + childStageId + "; known ids=" + sendersByChildStageId.keySet()
             );
         }
-        return new ChildSink(sender);
+        return new ChildSink(sender, childSchemas.get(childStageId));
     }
 
     /**
@@ -169,13 +195,17 @@ public ExchangeSink sinkForChild(int childStageId) {
      * already ran senderClose, the native side returns an error ("receiver dropped")
      * which we catch and discard.
      */
-    private void feedToSender(DatafusionPartitionSender sender, VectorSchemaRoot batch) {
+    private void feedToSender(DatafusionPartitionSender sender, VectorSchemaRoot batch, Schema declaredSchema) {
         // Best-effort fast path — skip export work if already closed.
         if (closed) {
             batch.close();
             return;
         }
         BufferAllocator alloc = ctx.allocator();
+        // Bridge DataFusion's physical types (e.g. Utf8View for string group keys) to the
+        // coordinator's declared schema (Utf8) before handing the batch to Rust. Zero-copy
+        // fast path when schemas already match. See coerceToDeclaredSchema().
+        batch = coerceToDeclaredSchema(batch, declaredSchema, alloc);
         ArrowArray array = ArrowArray.allocateNew(alloc);
         ArrowSchema arrowSchema = ArrowSchema.allocateNew(alloc);
         try {
@@ -203,6 +233,69 @@ private void feedToSender(DatafusionPartitionSender sender, VectorSchemaRoot bat
         }
     }
 
+    /**
+     * Coerces {@code batch} to {@code declaredSchema} at the Java→Rust boundary.
+     * Bridges the impedance between DataFusion's physical types (e.g. {@code Utf8View}
+     * for string group keys, a non-configurable HashAggregate optimization) and
+     * substrait's logical "string" which the coordinator's FINAL plan consumes as
+     * {@code Utf8}. One place, explicit, grows per-case on observed mismatch.
+     *
+     * 
            Zero-copy fast path when schemas already match (numeric-only aggregates).
+     * Closes {@code batch} — caller drops its reference.
+     *
+     * 
            TODO (revisit): this runtime coercer bridges a logical/physical type
+     * mismatch between Calcite's declared exchange schema and DataFusion's physical
+     * output. A cleaner fix would eliminate the mismatch upstream — for example, a Rust
+     * pass that casts {@code Utf8View} → {@code Utf8} at the PARTIAL plan's root using
+     * DataFusion's vectorized {@code CastExpr} (one columnar kernel per batch instead of
+     * per-cell Java copy), or a Substrait extension that carries view-vs-plain type
+     * information through the serialized plan. Until one of those lands, this Java-side
+     * coercer is the minimum correct bridge.
+     */
+    private static VectorSchemaRoot coerceToDeclaredSchema(VectorSchemaRoot batch, Schema declaredSchema, BufferAllocator alloc) {
+        if (batch.getSchema().equals(declaredSchema)) {
+            return batch;
+        }
+        VectorSchemaRoot out = VectorSchemaRoot.create(declaredSchema, alloc);
+        try {
+            out.allocateNew();
+            int rows = batch.getRowCount();
+            for (int col = 0; col < declaredSchema.getFields().size(); col++) {
+                FieldVector src = batch.getVector(col);
+                FieldVector dst = out.getVector(col);
+                if (src.getField().getType().equals(dst.getField().getType())) {
+                    src.makeTransferPair(dst).transfer();
+                    continue;
+                }
+                ArrowType.ArrowTypeID srcId = src.getField().getType().getTypeID();
+                ArrowType.ArrowTypeID dstId = dst.getField().getType().getTypeID();
+                if (srcId == ArrowType.ArrowTypeID.Utf8View && dstId == ArrowType.ArrowTypeID.Utf8) {
+                    ViewVarCharVector s = (ViewVarCharVector) src;
+                    VarCharVector d = (VarCharVector) dst;
+                    for (int r = 0; r < rows; r++) {
+                        if (s.isNull(r)) {
+                            d.setNull(r);
+                        } else {
+                            d.setSafe(r, s.get(r));
+                        }
+                    }
+                    d.setValueCount(rows);
+                    continue;
+                }
+                throw new IllegalStateException(
+                    "coerceToDeclaredSchema: unsupported " + srcId + " → " + dstId + " for column '" + dst.getField().getName() + "'"
+                );
+            }
+            out.setRowCount(rows);
+        } catch (RuntimeException e) {
+            out.close();
+            throw e;
+        } finally {
+            batch.close();
+        }
+        return out;
+    }
+
     /**
      * Per-child wrapper returned from {@link #sinkForChild(int)}. The orchestrator
      * routes one of these per child stage, and the wrapper's close() signals EOF for
@@ -210,15 +303,17 @@ private void feedToSender(DatafusionPartitionSender sender, VectorSchemaRoot bat
      */
     private final class ChildSink implements ExchangeSink {
         private final DatafusionPartitionSender sender;
+        private final Schema declaredSchema;
         private volatile boolean childClosed;
 
-        ChildSink(DatafusionPartitionSender sender) {
+        ChildSink(DatafusionPartitionSender sender, Schema declaredSchema) {
             this.sender = sender;
+            this.declaredSchema = declaredSchema;
         }
 
         @Override
         public void feed(VectorSchemaRoot batch) {
-            feedToSender(sender, batch);
+            feedToSender(sender, batch, declaredSchema);
         }
 
         @Override
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java
index a866e09db23fb..c2cd6d95eaab0 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java
@@ -89,6 +89,15 @@ static class BatchIterator implements Iterator {
         private Schema schema;
         private VectorSchemaRoot nextBatch;
         private Boolean nextAvailable;
+        // True once we've returned at least one batch (real or synthesized). Prevents the
+        // "zero batches produced" rescue below from re-firing on subsequent polls.
+        private boolean batchEmitted;
+        // True once the native stream has yielded EOS (arrayAddr == 0). Combined with
+        // batchEmitted, it lets us synthesize exactly one schema-carrying empty batch when
+        // the native side produced nothing — so downstream Arrow Flight (or any other
+        // transport that relies on the first data frame to deliver the schema) still sees
+        // the column layout.
+        private boolean nativeStreamExhausted;
 
         BatchIterator(StreamHandle streamHandle, BufferAllocator allocator, CDataDictionaryProvider dictionaryProvider) {
             this.streamHandle = streamHandle;
@@ -110,15 +119,36 @@ private void ensureSchema() {
 
         private boolean loadNextBatch() {
             ensureSchema();
+            // Once the native stream has reported EOS, never poll again and never
+            // re-synthesize an empty batch. See the synthesize branch below.
+            if (nativeStreamExhausted) return false;
+
             long arrayAddr = callNativeFn(
                 listener -> NativeBridge.streamNext(streamHandle.getRuntimeHandle().get(), streamHandle.getPointer(), listener)
             );
-            if (arrayAddr == 0) return false;
+
+            if (arrayAddr == 0) {
+                nativeStreamExhausted = true;
+                // Native source had zero batches — synthesize one zero-row batch so the
+                // schema rides on the first (and only) Flight data frame. Without this the
+                // wire stream is schema-less and any downstream that relies on the schema
+                // (coordinator StreamingTableExec, row-path schema introspection) fails
+                // with cryptic errors such as "project index 0 out of bounds, max field 0".
+                if (!batchEmitted) {
+                    nextBatch = VectorSchemaRoot.create(schema, allocator);
+                    nextBatch.setRowCount(0);
+                    batchEmitted = true;
+                    return true;
+                }
+                return false;
+            }
+
             VectorSchemaRoot freshRoot = VectorSchemaRoot.create(schema, allocator);
             try (ArrowArray arrowArray = ArrowArray.wrap(arrayAddr)) {
                 Data.importIntoVectorSchemaRoot(allocator, arrowArray, freshRoot, dictionaryProvider);
             }
             nextBatch = freshRoot;
+            batchEmitted = true;
             return true;
         }
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/FinalAggregateInstructionHandler.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/FinalAggregateInstructionHandler.java
index c3a9be1813c67..1de82997beb1e 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/FinalAggregateInstructionHandler.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/FinalAggregateInstructionHandler.java
@@ -10,23 +10,57 @@
 
 import org.opensearch.analytics.spi.BackendExecutionContext;
 import org.opensearch.analytics.spi.CommonExecutionContext;
+import org.opensearch.analytics.spi.ExchangeSinkContext;
 import org.opensearch.analytics.spi.FinalAggregateInstructionNode;
 import org.opensearch.analytics.spi.FragmentInstructionHandler;
+import org.opensearch.be.datafusion.nativelib.NativeBridge;
+
+import java.util.ArrayList;
+import java.util.List;
 
 /**
- * Handles FinalAggregate instruction for coordinator-reduce stages.
- * TODO: Configure SessionContext optimizer (disable CombinePartialFinalAggregate) for multi-shard aggregates.
+ * Handles FinalAggregate instruction for coordinator-reduce stages: creates a local session,
+ * registers streaming input partitions from child stages, and prepares the final-aggregate
+ * physical plan.
+ *
+ * 
            Returns a {@link DataFusionReduceState} carrying the session, runtime, and senders so
+ * the {@link DatafusionReduceSink} can later execute the prepared plan and feed batches.
  */
 public class FinalAggregateInstructionHandler implements FragmentInstructionHandler {
 
+    private final NativeRuntimeHandle runtimeHandle;
+
+    FinalAggregateInstructionHandler(NativeRuntimeHandle runtimeHandle) {
+        this.runtimeHandle = runtimeHandle;
+    }
+
     @Override
     public BackendExecutionContext apply(
         FinalAggregateInstructionNode node,
         CommonExecutionContext commonContext,
         BackendExecutionContext backendContext
     ) {
-        // TODO: Configure LocalSession optimizer settings for final aggregate execution.
-        // For now, the reduce path works without explicit configuration.
-        return backendContext;
+        ExchangeSinkContext ctx = (ExchangeSinkContext) commonContext;
+
+        DatafusionLocalSession session = new DatafusionLocalSession(runtimeHandle.get());
+        List senders = new ArrayList<>(ctx.childInputs().size());
+        try {
+            for (ExchangeSinkContext.ChildInput child : ctx.childInputs()) {
+                String inputId = "input-" + child.childStageId();
+                byte[] schemaIpc = ArrowSchemaIpc.toBytes(child.schema());
+                long senderPtr = NativeBridge.registerPartitionStream(session.getPointer(), inputId, schemaIpc);
+                senders.add(new DatafusionPartitionSender(senderPtr));
+            }
+            NativeBridge.prepareFinalPlan(session.getPointer(), ctx.fragmentBytes());
+        } catch (RuntimeException e) {
+            for (DatafusionPartitionSender sender : senders) {
+                try {
+                    sender.close();
+                } catch (Exception ignored) {}
+            }
+            session.close();
+            throw e;
+        }
+        return new DataFusionReduceState(session, runtimeHandle, senders);
     }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PartialAggregateInstructionHandler.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PartialAggregateInstructionHandler.java
new file mode 100644
index 0000000000000..55456ca03706b
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PartialAggregateInstructionHandler.java
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.analytics.backend.ShardScanExecutionContext;
+import org.opensearch.analytics.spi.BackendExecutionContext;
+import org.opensearch.analytics.spi.CommonExecutionContext;
+import org.opensearch.analytics.spi.FragmentInstructionHandler;
+import org.opensearch.analytics.spi.PartialAggregateInstructionNode;
+import org.opensearch.be.datafusion.nativelib.NativeBridge;
+
+/**
+ * Handles PartialAggregate instruction on the shard side: prepares the partial-aggregate
+ * physical plan on the already-open SessionContext created by the preceding ShardScan handler.
+ *
+ * 
            Calls {@link NativeBridge#preparePartialPlan(long, byte[])} which sets the Rust-side
+ * aggregate mode to Partial and stores the prepared plan on the session handle for later
+ * execution.
+ */
+public class PartialAggregateInstructionHandler implements FragmentInstructionHandler {
+
+    @Override
+    public BackendExecutionContext apply(
+        PartialAggregateInstructionNode node,
+        CommonExecutionContext commonContext,
+        BackendExecutionContext backendContext
+    ) {
+        ShardScanExecutionContext ctx = (ShardScanExecutionContext) commonContext;
+        DataFusionSessionState state = (DataFusionSessionState) backendContext;
+        long sessionPtr = state.sessionContextHandle().getPointer();
+        NativeBridge.preparePartialPlan(sessionPtr, ctx.getFragmentBytes());
+        return backendContext;
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
index 1bb1edf9cc59a..eb778becc0244 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
@@ -81,6 +81,9 @@ public final class NativeBridge {
     private static final MethodHandle EXECUTE_WITH_CONTEXT;
     private static final MethodHandle CANCEL_QUERY;
     private static final MethodHandle STATS;
+    private static final MethodHandle PREPARE_PARTIAL_PLAN;
+    private static final MethodHandle PREPARE_FINAL_PLAN;
+    private static final MethodHandle EXECUTE_LOCAL_PREPARED_PLAN;
 
     static {
         SymbolLookup lib = NativeLibraryLoader.symbolLookup();
@@ -388,6 +391,25 @@ public final class NativeBridge {
             lib.find("df_stats").orElseThrow(),
             FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG)
         );
+
+        // ── Distributed aggregate: prepare partial/final plans ──
+        // i64 df_prepare_partial_plan(handle_ptr, bytes_ptr, bytes_len)
+        PREPARE_PARTIAL_PLAN = linker.downcallHandle(
+            lib.find("df_prepare_partial_plan").orElseThrow(),
+            FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG)
+        );
+
+        // i64 df_prepare_final_plan(session_ptr, bytes_ptr, bytes_len)
+        PREPARE_FINAL_PLAN = linker.downcallHandle(
+            lib.find("df_prepare_final_plan").orElseThrow(),
+            FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG)
+        );
+
+        // i64 df_execute_local_prepared_plan(session_ptr)
+        EXECUTE_LOCAL_PREPARED_PLAN = linker.downcallHandle(
+            lib.find("df_execute_local_prepared_plan").orElseThrow(),
+            FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG)
+        );
     }
 
     private NativeBridge() {}
@@ -892,6 +914,53 @@ public static void destroyCustomCacheManager(long ptr) {
         NativeCall.invokeVoid(DESTROY_CUSTOM_CACHE_MANAGER, ptr);
     }
 
+    // ---- Distributed aggregate: prepare partial/final plans ----
+
+    /**
+     * Prepares a partial-aggregate physical plan on the session context handle.
+     * The plan is stored on the Rust handle for later execution.
+     *
+     * @param handlePtr pointer returned by {@link #createSessionContext}
+     * @param substraitBytes Substrait plan bytes
+     */
+    public static void preparePartialPlan(long handlePtr, byte[] substraitBytes) {
+        NativeHandle.validatePointer(handlePtr, "sessionContext");
+        try (var call = new NativeCall()) {
+            call.invoke(PREPARE_PARTIAL_PLAN, handlePtr, call.bytes(substraitBytes), (long) substraitBytes.length);
+        }
+    }
+
+    /**
+     * Prepares a final-aggregate physical plan on a local session.
+     * The plan is stored on the Rust session for later execution via
+     * {@link #executeLocalPreparedPlan}.
+     *
+     * @param sessionPtr pointer returned by {@link #createLocalSession}
+     * @param substraitBytes Substrait plan bytes
+     */
+    public static void prepareFinalPlan(long sessionPtr, byte[] substraitBytes) {
+        NativeHandle.validatePointer(sessionPtr, "session");
+        try (var call = new NativeCall()) {
+            call.invoke(PREPARE_FINAL_PLAN, sessionPtr, call.bytes(substraitBytes), (long) substraitBytes.length);
+        }
+    }
+
+    /**
+     * Executes the previously prepared final-aggregate plan on a local session.
+     * Returns a stream pointer that can be drained via {@link #streamNext} and
+     * freed by {@link #streamClose}.
+     *
+     * @param sessionPtr pointer returned by {@link #createLocalSession} with a plan
+     *                   already prepared via {@link #prepareFinalPlan}
+     * @return opaque stream pointer
+     */
+    public static long executeLocalPreparedPlan(long sessionPtr) {
+        NativeHandle.validatePointer(sessionPtr, "session");
+        try (var call = new NativeCall()) {
+            return call.invoke(EXECUTE_LOCAL_PREPARED_PLAN, sessionPtr);
+        }
+    }
+
     public static void createCache(long cacheManagerPtr, String cacheType, long sizeLimit, String evictionType) {
         try (var call = new NativeCall()) {
             var type = call.str(cacheType);
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_aggregate_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_aggregate_functions.yaml
new file mode 100644
index 0000000000000..2d9b3f451e746
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_aggregate_functions.yaml
@@ -0,0 +1,13 @@
+%YAML 1.2
+---
+urn: extension:org.opensearch:aggregate_functions
+aggregate_functions:
+  - name: approx_distinct
+    description: >-
+      Approximate distinct count using HyperLogLog. Maps to DataFusion's
+      approx_distinct aggregate function via its Substrait consumer.
+    impls:
+      - args:
+          - value: any
+            name: "input"
+        return: i64
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java
index 00a20a1f109a3..3b23c7adbeccd 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DataFusionFragmentConvertorTests.java
@@ -43,6 +43,7 @@
 import io.substrait.proto.PlanRel;
 import io.substrait.proto.ReadRel;
 import io.substrait.proto.Rel;
+import io.substrait.proto.SimpleExtensionDeclaration;
 import io.substrait.proto.SortRel;
 
 /**
@@ -73,7 +74,8 @@ public void setUp() throws Exception {
         try {
             t.setContextClassLoader(DataFusionFragmentConvertorTests.class.getClassLoader());
             SimpleExtension.ExtensionCollection delegationExtensions = SimpleExtension.load(List.of("/delegation_functions.yaml"));
-            extensions = DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions);
+            SimpleExtension.ExtensionCollection aggregateExtensions = SimpleExtension.load(List.of("/opensearch_aggregate_functions.yaml"));
+            extensions = DefaultExtensionCatalog.DEFAULT_COLLECTION.merge(delegationExtensions).merge(aggregateExtensions);
         } finally {
             t.setContextClassLoader(prev);
         }
@@ -415,7 +417,7 @@ public void testMultisearchShape_SystemLimitOverSortOverAggregateOverUnion_Names
 
         // System limit = LogicalSort with no collation + fetch literal. Lowers to a
         // Substrait Fetch rel (the convertor handles this in replaceInput).
-        org.apache.calcite.rex.RexNode fetchN = rexBuilder.makeLiteral(100, typeFactory.createSqlType(SqlTypeName.INTEGER), true);
+        RexNode fetchN = rexBuilder.makeLiteral(100, typeFactory.createSqlType(SqlTypeName.INTEGER), true);
         LogicalSort systemLimit = LogicalSort.create(userSort, RelCollations.EMPTY, null, fetchN);
         byte[] combinedBytes = convertor.attachFragmentOnTop(systemLimit, userSortBytes);
 
@@ -534,4 +536,65 @@ public void testConvertShardScanFragment_ComplexBooleanTreeWithDelegation() thro
         assertEquals(2, dp2.getScalarFunction().getArguments(0).getValue().getLiteral().getI32());
     }
 
+    // ── Extension function rename tests ────────────────────────────────────────
+
+    /**
+     * APPROX_COUNT_DISTINCT aggregate emits as {@code approx_distinct} in the
+     * Substrait extension declarations — not the Calcite-native
+     * {@code approx_count_distinct} name.
+     */
+    public void testApproxCountDistinctRenamed() throws Exception {
+        RelNode scan = buildTableScan("test_index", "A");
+        AggregateCall approxCall = AggregateCall.create(
+            SqlStdOperatorTable.APPROX_COUNT_DISTINCT,
+            false,
+            List.of(0),
+            -1,
+            typeFactory.createSqlType(SqlTypeName.BIGINT),
+            "approx_col"
+        );
+        LogicalAggregate agg = LogicalAggregate.create(scan, List.of(), ImmutableBitSet.of(), null, List.of(approxCall));
+
+        byte[] bytes = newConvertor().convertShardScanFragment("test_index", agg);
+        Plan plan = decodeSubstrait(bytes);
+
+        boolean foundApproxDistinct = false;
+        for (SimpleExtensionDeclaration decl : plan.getExtensionsList()) {
+            if (decl.hasExtensionFunction()) {
+                String name = decl.getExtensionFunction().getName();
+                String baseName = name.contains(":") ? name.substring(0, name.indexOf(':')) : name;
+                assertNotEquals("approx_count_distinct must be renamed", "approx_count_distinct", baseName);
+                if (baseName.equals("approx_distinct")) {
+                    foundApproxDistinct = true;
+                }
+            }
+        }
+        assertTrue("must find approx_distinct in extension declarations", foundApproxDistinct);
+    }
+
+    /**
+     * SUM aggregate is not affected by the rename map — its extension function
+     * name remains unchanged.
+     */
+    public void testOtherFunctionsNotRenamed() throws Exception {
+        RelNode scan = buildTableScan("test_index", "A");
+        LogicalAggregate agg = buildSumAggregate(scan, 0);
+
+        byte[] bytes = newConvertor().convertShardScanFragment("test_index", agg);
+        Plan plan = decodeSubstrait(bytes);
+
+        boolean foundSum = false;
+        for (SimpleExtensionDeclaration decl : plan.getExtensionsList()) {
+            if (decl.hasExtensionFunction()) {
+                String name = decl.getExtensionFunction().getName();
+                String baseName = name.contains(":") ? name.substring(0, name.indexOf(':')) : name;
+                assertNotEquals("approx_distinct should not appear for SUM-only plan", "approx_distinct", baseName);
+                if (baseName.equals("sum")) {
+                    foundSum = true;
+                }
+            }
+        }
+        assertTrue("must find sum in extension declarations", foundSum);
+    }
+
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java
index 247d1e55dadfd..9e9ae14a84262 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java
@@ -126,7 +126,17 @@ public void testNextWithoutHasNextWorks() throws Exception {
     public void testNextOnExhaustedStreamThrows() throws Exception {
         try (DatafusionResultStream stream = createStream("SELECT message FROM test_table WHERE message > 999")) {
             Iterator it = stream.iterator();
-            assertFalse(it.hasNext());
+            // Empty results still emit one zero-row batch carrying the declared schema so
+            // downstream transports (Flight, row-path) see the column layout on the first
+            // data frame. Consume it, then the iterator is genuinely exhausted.
+            assertTrue("empty result emits a schema-carrying zero-row batch", it.hasNext());
+            EngineResultBatch schemaOnly = it.next();
+            try {
+                assertEquals("synthesized batch has zero rows", 0, schemaOnly.getRowCount());
+            } finally {
+                schemaOnly.getArrowRoot().close();
+            }
+            assertFalse("no further batches after the schema-only emit", it.hasNext());
             expectThrows(NoSuchElementException.class, it::next);
         }
     }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java
index c0b7e1eb6ffe0..ab6c6ef43e6ad 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/FilterDelegationForIndexFullConversionTests.java
@@ -444,7 +444,7 @@ public Set supportedDelegations() {
 
         @Override
         public ExchangeSinkProvider getExchangeSinkProvider() {
-            return context -> null;
+            return (context, backendContext) -> null;
         }
 
         @Override
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/NativeBridgePreparedPlanTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/NativeBridgePreparedPlanTests.java
new file mode 100644
index 0000000000000..ea0266435dd84
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/NativeBridgePreparedPlanTests.java
@@ -0,0 +1,63 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.opensearch.be.datafusion.nativelib.NativeBridge;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.nio.file.Path;
+
+/**
+ * Verifies that the three new prepared-plan FFI entry points resolve against
+ * the native library symbols. Full execution is not tested here — only that
+ * the MethodHandles link successfully and the methods can be invoked without
+ * a symbol-not-found error.
+ */
+public class NativeBridgePreparedPlanTests extends OpenSearchTestCase {
+
+    public void testPreparePartialPlanRejectsNullPointer() {
+        // Validates the Java-side pointer check fires before the native call.
+        expectThrows(IllegalArgumentException.class, () -> NativeBridge.preparePartialPlan(0L, new byte[] { 0x01 }));
+    }
+
+    public void testPrepareFinalPlanRejectsNullPointer() {
+        expectThrows(IllegalArgumentException.class, () -> NativeBridge.prepareFinalPlan(0L, new byte[] { 0x01 }));
+    }
+
+    public void testExecuteLocalPreparedPlanRejectsNullPointer() {
+        expectThrows(IllegalArgumentException.class, () -> NativeBridge.executeLocalPreparedPlan(0L));
+    }
+
+    /**
+     * Smoke test: create a local session, attempt to prepare a final plan with
+     * garbage bytes — should fail with a decode error (not a link error).
+     * This proves the MethodHandle resolved and the native function was called.
+     */
+    public void testPrepareFinalPlanWithInvalidBytesThrowsDecodeError() {
+        NativeBridge.initTokioRuntimeManager(2);
+        Path spillDir = createTempDir("datafusion-spill");
+        long runtimePtr = NativeBridge.createGlobalRuntime(64 * 1024 * 1024, 0L, spillDir.toString(), 32 * 1024 * 1024);
+        NativeRuntimeHandle runtimeHandle = new NativeRuntimeHandle(runtimePtr);
+        DatafusionLocalSession session = new DatafusionLocalSession(runtimeHandle.get());
+        try {
+            RuntimeException ex = expectThrows(
+                RuntimeException.class,
+                () -> NativeBridge.prepareFinalPlan(session.getPointer(), new byte[] { 0x00, 0x01, 0x02 })
+            );
+            // The error should mention Substrait decode failure, not a symbol error
+            assertTrue(
+                "Expected decode error, got: " + ex.getMessage(),
+                ex.getMessage().contains("decode") || ex.getMessage().contains("Substrait")
+            );
+        } finally {
+            session.close();
+            runtimeHandle.close();
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java
index 20dacd462ce1d..eb0bd161abbd7 100644
--- a/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java
+++ b/sandbox/plugins/analytics-backend-lucene/src/test/java/org/opensearch/be/lucene/LuceneAnalyticsBackendPluginTests.java
@@ -264,7 +264,7 @@ public Set supportedDelegations() {
 
         @Override
         public ExchangeSinkProvider getExchangeSinkProvider() {
-            return context -> null;
+            return (context, backendContext) -> null;
         }
 
         @Override
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java
index c669aba8f850a..6510228af867e 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java
@@ -9,6 +9,14 @@
 package org.opensearch.analytics.exec;
 
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.ipc.ArrowStreamWriter;
+import org.apache.arrow.vector.ipc.WriteChannel;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.IpcOption;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.Schema;
 import org.opensearch.analytics.backend.AnalyticsOperationListener;
 import org.opensearch.analytics.backend.EngineResultBatch;
 import org.opensearch.analytics.backend.EngineResultStream;
@@ -34,8 +42,9 @@
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.tasks.Task;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.ArrayList;
+import java.nio.channels.Channels;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -98,7 +107,7 @@ public FragmentExecutionResponse executeFragment(FragmentExecutionRequest reques
         try (FragmentResources ctx = startFragment(request, resolved, shard, task)) {
             FragmentExecutionResponse response = collectResponse(ctx.stream(), task);
             long tookNanos = System.nanoTime() - startNanos;
-            listener.onFragmentSuccess(resolved.queryId, resolved.stageId, resolved.shardIdStr, tookNanos, response.getRows().size());
+            listener.onFragmentSuccess(resolved.queryId, resolved.stageId, resolved.shardIdStr, tookNanos, response.getRowCount());
             return response;
         } catch (TaskCancelledException | IllegalStateException | IllegalArgumentException e) {
             listener.onFragmentFailure(resolved.queryId, resolved.stageId, resolved.shardIdStr, e);
@@ -230,29 +239,42 @@ FragmentExecutionResponse collectResponse(EngineResultStream stream) {
     }
 
     FragmentExecutionResponse collectResponse(EngineResultStream stream, @Nullable AnalyticsShardTask task) {
-        List rows = new ArrayList<>();
-        List fieldNames = null;
+        // Serialize incoming Arrow batches as an Arrow IPC stream: one schema header
+        // followed by one record-batch message per incoming batch. Arrow's own
+        // serializer handles every Arrow type — no per-type Java code path.
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        WriteChannel channel = new WriteChannel(Channels.newChannel(baos));
+        Schema schema = null;
+        int totalRows = 0;
         Iterator it = stream.iterator();
-        while (it.hasNext()) {
-            if (task != null && task.isCancelled()) {
-                throw new TaskCancelledException("task cancelled: " + task.getReasonCancelled());
-            }
-            EngineResultBatch batch = it.next();
-            try {
-                if (fieldNames == null) {
-                    fieldNames = batch.getFieldNames();
+        try {
+            while (it.hasNext()) {
+                if (task != null && task.isCancelled()) {
+                    throw new TaskCancelledException("task cancelled: " + task.getReasonCancelled());
                 }
-                for (int row = 0; row < batch.getRowCount(); row++) {
-                    Object[] vals = new Object[fieldNames.size()];
-                    for (int col = 0; col < fieldNames.size(); col++) {
-                        vals[col] = batch.getFieldValue(fieldNames.get(col), row);
+                EngineResultBatch batch = it.next();
+                VectorSchemaRoot root = batch.getArrowRoot();
+                try {
+                    if (schema == null) {
+                        schema = root.getSchema();
+                        MessageSerializer.serialize(channel, schema);
+                    }
+                    try (ArrowRecordBatch recordBatch = new VectorUnloader(root).getRecordBatch()) {
+                        MessageSerializer.serialize(channel, recordBatch);
                     }
-                    rows.add(vals);
+                    totalRows += root.getRowCount();
+                } finally {
+                    root.close();
                 }
-            } finally {
-                batch.getArrowRoot().close();
             }
+            if (schema != null) {
+                // Write the end-of-stream marker so the reader sees a clean EOS
+                // instead of hitting end-of-input mid-message.
+                ArrowStreamWriter.writeEndOfStream(channel, IpcOption.DEFAULT);
+            }
+        } catch (IOException e) {
+            throw new IllegalStateException("Failed to serialize fragment output as Arrow IPC stream", e);
         }
-        return new FragmentExecutionResponse(fieldNames != null ? fieldNames : List.of(), rows);
+        return new FragmentExecutionResponse(baos.toByteArray(), totalRows);
     }
 }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
index 39b816202a346..bf5ba86038101 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
@@ -24,6 +24,7 @@
 import org.opensearch.analytics.planner.CapabilityRegistry;
 import org.opensearch.analytics.planner.PlannerContext;
 import org.opensearch.analytics.planner.PlannerImpl;
+import org.opensearch.analytics.planner.dag.AggregateDecompositionResolver;
 import org.opensearch.analytics.planner.dag.BackendPlanAdapter;
 import org.opensearch.analytics.planner.dag.DAGBuilder;
 import org.opensearch.analytics.planner.dag.FragmentConversionDriver;
@@ -132,6 +133,7 @@ private void executeInternal(RelNode logicalFragment, ActionListenerSupported types: Long, Integer, Double, Float, Boolean, String
- * (and CharSequence), byte[], and null.
- */
-final class RowBatchToArrowConverter {
-
-    private RowBatchToArrowConverter() {}
-
-    /**
-     * Convert a row-oriented response to an Arrow VectorSchemaRoot.
-     *
-     * @param response    the row-oriented shard response
-     * @param targetSchema the Arrow schema the output must conform to
-     * @param allocator   the buffer allocator for Arrow vectors
-     * @return a new VectorSchemaRoot; caller owns and must close it
-     */
-    public static VectorSchemaRoot convert(FragmentExecutionResponse response, Schema targetSchema, BufferAllocator allocator) {
-        VectorSchemaRoot vsr = VectorSchemaRoot.create(targetSchema, allocator);
-        try {
-            vsr.allocateNew();
-            List rows = response.getRows();
-            int rowCount = rows.size();
-
-            for (int col = 0; col < targetSchema.getFields().size(); col++) {
-                Field field = targetSchema.getFields().get(col);
-                FieldVector vector = vsr.getVector(col);
-                for (int r = 0; r < rowCount; r++) {
-                    Object value = rows.get(r)[col];
-                    setValue(vector, r, value, field);
-                }
-                vector.setValueCount(rowCount);
-            }
-            vsr.setRowCount(rowCount);
-            return vsr;
-        } catch (Exception e) {
-            vsr.close();
-            throw e;
-        }
-    }
-
-    private static void setValue(FieldVector vector, int index, Object value, Field field) {
-        if (value == null) {
-            vector.setNull(index);
-            return;
-        }
-        switch (vector.getMinorType()) {
-            case BIGINT:
-                if (value instanceof Number == false) {
-                    throw new IllegalArgumentException(
-                        "Column '" + field.getName() + "' expects BIGINT but got " + value.getClass().getName()
-                    );
-                }
-                ((BigIntVector) vector).set(index, ((Number) value).longValue());
-                break;
-            case INT:
-                if (value instanceof Number == false) {
-                    throw new IllegalArgumentException(
-                        "Column '" + field.getName() + "' expects INT but got " + value.getClass().getName()
-                    );
-                }
-                ((IntVector) vector).set(index, ((Number) value).intValue());
-                break;
-            case FLOAT8:
-                if (value instanceof Number == false) {
-                    throw new IllegalArgumentException(
-                        "Column '" + field.getName() + "' expects FLOAT8 but got " + value.getClass().getName()
-                    );
-                }
-                ((Float8Vector) vector).set(index, ((Number) value).doubleValue());
-                break;
-            case FLOAT4:
-                if (value instanceof Number == false) {
-                    throw new IllegalArgumentException(
-                        "Column '" + field.getName() + "' expects FLOAT4 but got " + value.getClass().getName()
-                    );
-                }
-                ((Float4Vector) vector).set(index, ((Number) value).floatValue());
-                break;
-            case BIT:
-                if (value instanceof Boolean == false) {
-                    throw new IllegalArgumentException(
-                        "Column '" + field.getName() + "' expects BIT (Boolean) but got " + value.getClass().getName()
-                    );
-                }
-                ((BitVector) vector).set(index, ((Boolean) value) ? 1 : 0);
-                break;
-            case VARCHAR:
-                if (value instanceof CharSequence == false) {
-                    throw new IllegalArgumentException(
-                        "Column '" + field.getName() + "' expects VARCHAR (CharSequence) but got " + value.getClass().getName()
-                    );
-                }
-                ((VarCharVector) vector).setSafe(index, value.toString().getBytes(StandardCharsets.UTF_8));
-                break;
-            case VARBINARY:
-                if (value instanceof byte[] == false) {
-                    throw new IllegalArgumentException(
-                        "Column '" + field.getName() + "' expects VARBINARY (byte[]) but got " + value.getClass().getName()
-                    );
-                }
-                ((VarBinaryVector) vector).setSafe(index, (byte[]) value);
-                break;
-            default:
-                throw new IllegalArgumentException("Unsupported vector type: " + vector.getMinorType());
-        }
-    }
-}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionResponse.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionResponse.java
index c86c61c4ed2fa..d54c9c6b04469 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionResponse.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionResponse.java
@@ -13,63 +13,61 @@
 import org.opensearch.core.common.io.stream.StreamOutput;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 
 /**
- * Transport response carrying field names and result rows from a shard
- * fragment execution.
+ * Transport response carrying the output of a shard fragment execution as an
+ * Arrow IPC stream payload (schema header + zero or more record-batch messages,
+ * produced by {@link org.apache.arrow.vector.ipc.ArrowStreamWriter}).
  *
- * 
            Each cell value is serialized via {@link StreamOutput#writeGenericValue(Object)} /
- * {@link StreamInput#readGenericValue()}, which handle common Java types
- * (String, Long, Double, Integer, null, byte[], etc.).
+ * 
            Arrow IPC handles every Arrow type natively (temporal, string-view,
+ * dictionary, nested) without hand-rolled per-type serialization. Previously,
+ * this response carried {@code List} rows and relied on
+ * {@code StreamOutput.writeGenericValue} — which does not support Java 8+
+ * temporal types like {@link java.time.LocalDateTime} and so failed the moment
+ * a shard emitted a batch with a Timestamp column.
  *
- * 
            Wire format: {@code fieldNames (string list) + rowCount (vint) + per-row (colCount (vint) + cells)}.
+ * 
            Wire format: {@code ipcPayload (byte[]) + rowCount (vint)}. The row count
+ * is the total across all batches in the payload, cached for metrics / logging
+ * so consumers don't have to decode the payload just to report "N rows handled".
  *
  * @opensearch.internal
  */
 public class FragmentExecutionResponse extends ActionResponse {
 
-    private final List fieldNames;
-    private final List rows;
+    private final byte[] ipcPayload;
+    private final int rowCount;
 
-    public FragmentExecutionResponse(List fieldNames, List rows) {
-        this.fieldNames = fieldNames;
-        this.rows = rows;
+    public FragmentExecutionResponse(byte[] ipcPayload, int rowCount) {
+        this.ipcPayload = ipcPayload;
+        this.rowCount = rowCount;
     }
 
     public FragmentExecutionResponse(StreamInput in) throws IOException {
         super(in);
-        this.fieldNames = in.readStringList();
-        int rowCount = in.readVInt();
-        this.rows = new ArrayList<>(rowCount);
-        for (int r = 0; r < rowCount; r++) {
-            int colCount = in.readVInt();
-            Object[] row = new Object[colCount];
-            for (int c = 0; c < colCount; c++) {
-                row[c] = in.readGenericValue();
-            }
-            rows.add(row);
-        }
+        this.ipcPayload = in.readByteArray();
+        this.rowCount = in.readVInt();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeStringCollection(fieldNames);
-        out.writeVInt(rows.size());
-        for (Object[] row : rows) {
-            out.writeVInt(row.length);
-            for (Object cell : row) {
-                out.writeGenericValue(cell);
-            }
-        }
+        out.writeByteArray(ipcPayload);
+        out.writeVInt(rowCount);
     }
 
-    public List getFieldNames() {
-        return fieldNames;
+    /**
+     * Arrow IPC stream bytes — a schema message followed by zero or more record
+     * batch messages, as written by {@link org.apache.arrow.vector.ipc.ArrowStreamWriter}.
+     * An empty array means the fragment produced no output at all (no schema,
+     * no rows).
+     */
+    public byte[] getIpcPayload() {
+        return ipcPayload;
     }
 
-    public List getRows() {
-        return rows;
+    /**
+     * Total number of rows across all batches in {@link #getIpcPayload()}.
+     */
+    public int getRowCount() {
+        return rowCount;
     }
 }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ArrowSchemaFromCalcite.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ArrowSchemaFromCalcite.java
index f81a9bd1e2951..e1e04e6ab126b 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ArrowSchemaFromCalcite.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ArrowSchemaFromCalcite.java
@@ -8,7 +8,9 @@
 
 package org.opensearch.analytics.exec.stage;
 
+import org.apache.arrow.vector.types.DateUnit;
 import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
@@ -22,8 +24,8 @@
 
 /**
  * Translates a Calcite {@link RelDataType} (row type) to an Arrow {@link Schema}.
- * Used to derive the target schema for {@code RowBatchToArrowConverter} from the
- * child stage's resolved fragment row type.
+ * Used by distributed stages to declare their exchange-point schema when registering
+ * {@code StreamingTable} partitions with the native execution engine.
  *
  * 
            All fields are nullable for MVP.
  */
@@ -75,9 +77,14 @@ private static ArrowType toArrowType(SqlTypeName sqlTypeName) {
                 return new ArrowType.Int(64, true);
             case INTEGER:
                 return new ArrowType.Int(32, true);
+            case SMALLINT:
+                return new ArrowType.Int(16, true);
+            case TINYINT:
+                return new ArrowType.Int(8, true);
             case DOUBLE:
                 return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
             case FLOAT:
+            case REAL:
                 return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE);
             case BOOLEAN:
                 return ArrowType.Bool.INSTANCE;
@@ -87,6 +94,13 @@ private static ArrowType toArrowType(SqlTypeName sqlTypeName) {
             case VARBINARY:
             case BINARY:
                 return ArrowType.Binary.INSTANCE;
+            case DATE:
+                return new ArrowType.Date(DateUnit.DAY);
+            case TIME:
+                return new ArrowType.Time(TimeUnit.MILLISECOND, 32);
+            case TIMESTAMP:
+            case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+                return new ArrowType.Timestamp(TimeUnit.MILLISECOND, null);
             default:
                 throw new IllegalArgumentException("Unsupported Calcite SQL type: " + sqlTypeName);
         }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java
index c934bc1c6b76b..c2c44a59dc5f2 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/LocalStageScheduler.java
@@ -58,9 +58,9 @@ public StageExecution createExecution(Stage stage, ExchangeSink sink, QueryConte
         // is stored on the Stage during FragmentConversionDriver.convertAll (root stage only,
         // no serialization needed since reduce executes locally at the coordinator).
         // TODO: find a cleaner way to provide the factory without storing it on Stage.
+        BackendExecutionContext backendContext = null;
         FragmentInstructionHandlerFactory factory = stage.getInstructionHandlerFactory();
         if (factory != null) {
-            BackendExecutionContext backendContext = null;
             Throwable primaryFailure = null;
             try {
                 for (InstructionNode node : stage.getPlanAlternatives().getFirst().instructions()) {
@@ -75,19 +75,12 @@ public StageExecution createExecution(Stage stage, ExchangeSink sink, QueryConte
                 }
             } catch (Throwable t) {
                 primaryFailure = t;
-            } finally {
-                // The reduce path does not currently hand backendContext off to the sink
-                // provider — any resources attached by instruction handlers must be released
-                // here. Close is idempotent so a future handoff can coexist with this call.
+                // On failure, close the backendContext since it won't be handed to the sink.
                 if (backendContext != null) {
                     try {
                         backendContext.close();
                     } catch (Exception closeFailure) {
-                        if (primaryFailure != null) {
-                            primaryFailure.addSuppressed(closeFailure);
-                        } else {
-                            primaryFailure = closeFailure;
-                        }
+                        primaryFailure.addSuppressed(closeFailure);
                     }
                 }
             }
@@ -100,8 +93,16 @@ public StageExecution createExecution(Stage stage, ExchangeSink sink, QueryConte
 
         ExchangeSink backendSink;
         try {
-            backendSink = provider.createSink(context);
+            backendSink = provider.createSink(context, backendContext);
         } catch (Exception e) {
+            // Sink creation failed — close backendContext to avoid resource leak.
+            if (backendContext != null) {
+                try {
+                    backendContext.close();
+                } catch (Exception closeFailure) {
+                    e.addSuppressed(closeFailure);
+                }
+            }
             throw new RuntimeException("Failed to create exchange sink for stageId=" + stage.getStageId(), e);
         }
         return new LocalStageExecution(stage, backendSink, sink);
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java
index 97aa10ff82ca2..38aef31c1337b 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java
@@ -9,38 +9,30 @@
 package org.opensearch.analytics.exec.stage;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.BigIntVector;
-import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.Float4Vector;
-import org.apache.arrow.vector.Float8Vector;
-import org.apache.arrow.vector.IntVector;
-import org.apache.arrow.vector.SmallIntVector;
-import org.apache.arrow.vector.TinyIntVector;
-import org.apache.arrow.vector.VarBinaryVector;
-import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.impl.UnionListWriter;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.ipc.ArrowStreamReader;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.opensearch.analytics.exec.action.FragmentExecutionResponse;
 
-import java.nio.charset.StandardCharsets;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 
 /**
- * {@link ResponseCodec} for the current row-oriented
- * {@link FragmentExecutionResponse} wire format. Converts {@code Object[]}
- * rows to Arrow {@link VectorSchemaRoot} via type inference.
+ * {@link ResponseCodec} that deserializes the Arrow IPC stream payload carried
+ * by {@link FragmentExecutionResponse} into a single consolidated
+ * {@link VectorSchemaRoot}. Uses Arrow's {@link ArrowStreamReader} for message
+ * sequencing (schema header, record batches, end-of-stream) and
+ * {@link FieldVector#makeTransferPair} / {@link FieldVector#copyFromSafe} to
+ * move data into a caller-owned root — both are supported by every vector
+ * kind, including the view vectors ({@code Utf8View}, {@code BinaryView}) that
+ * DataFusion emits for aggregate group keys.
  *
- * 
            This codec is the bridge that gets replaced when Arrow IPC transport
- * lands. A future {@code ArrowIpcResponseCodec} would import IPC buffers
- * directly — zero conversion.
+ * 
            Deliberately avoids {@code VectorSchemaRootAppender} — its underlying
+ * {@code VectorAppender} rejects view vectors with
+ * {@code UnsupportedOperationException}.
  *
  * @opensearch.internal
  */
@@ -53,197 +45,68 @@ private RowResponseCodec() {}
 
     @Override
     public VectorSchemaRoot decode(FragmentExecutionResponse response, BufferAllocator allocator) {
-        List fieldNames = response.getFieldNames();
-        List rows = response.getRows();
-
         if (allocator == null) {
             throw new IllegalArgumentException("BufferAllocator must not be null");
         }
-
-        // Infer Arrow type per column from the first non-null value
-        List fields = new ArrayList<>();
-        for (int col = 0; col < fieldNames.size(); col++) {
-            fields.add(inferField(fieldNames.get(col), rows, col));
+        byte[] payload = response.getIpcPayload();
+        if (payload == null || payload.length == 0) {
+            return VectorSchemaRoot.create(new Schema(List.of()), allocator);
         }
-        Schema schema = new Schema(fields);
 
-        VectorSchemaRoot vsr = VectorSchemaRoot.create(schema, allocator);
-        try {
-            vsr.allocateNew();
-            int rowCount = rows.size();
-            for (int col = 0; col < fieldNames.size(); col++) {
-                FieldVector vector = vsr.getVector(col);
-                for (int r = 0; r < rowCount; r++) {
-                    Object value = rows.get(r)[col];
-                    setVectorValue(vector, r, value);
+        List batches = new ArrayList<>();
+        Schema schema;
+        try (ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayInputStream(payload), allocator)) {
+            VectorSchemaRoot readerRoot = reader.getVectorSchemaRoot();
+            schema = readerRoot.getSchema();
+            while (reader.loadNextBatch()) {
+                // Transfer each batch's buffers out of the reader's reused root into an
+                // independent root owned by this codec. Transfer works for all vector
+                // kinds (including views) and is zero-copy when allocators match.
+                VectorSchemaRoot batchRoot = VectorSchemaRoot.create(schema, allocator);
+                int rowCount = readerRoot.getRowCount();
+                for (int i = 0; i < readerRoot.getFieldVectors().size(); i++) {
+                    readerRoot.getVector(i).makeTransferPair(batchRoot.getVector(i)).transfer();
                 }
-                vector.setValueCount(rowCount);
+                batchRoot.setRowCount(rowCount);
+                batches.add(batchRoot);
             }
-            vsr.setRowCount(rowCount);
-            return vsr;
-        } catch (Exception e) {
-            vsr.close();
-            throw e;
+        } catch (IOException e) {
+            for (VectorSchemaRoot b : batches)
+                b.close();
+            throw new IllegalStateException("Failed to decode Arrow IPC payload from fragment response", e);
         }
-    }
 
-    /**
-     * Infers the Arrow {@link Field} for a column by scanning rows for the first
-     * non-null value. Falls back to a nullable {@code Utf8} (VarChar) field if all
-     * values are null or the Java type is unrecognized.
-     *
-     * 
            For {@link List} cells (produced by analytics-engine routes that emit
-     * array-typed values — PPL {@code array(...)}, {@code array_slice}, …), this
-     * returns a {@code List} field where the inner element type is inferred
-     * from the first non-null element. Without this branch, list values fall
-     * through to the {@code Utf8} fallback and {@link #setVectorValue} produces
-     * {@code value.toString()} (e.g. {@code "[2,3,4]"} as a JSON-like string)
-     * instead of a typed array.
-     */
-    static Field inferField(String name, List rows, int col) {
-        for (Object[] row : rows) {
-            Object value = row[col];
-            if (value == null) continue;
-            if (value instanceof List list) {
-                ArrowType elementType = inferElementArrowType(list, rows, col);
-                Field elementField = new Field("$data$", FieldType.nullable(elementType), null);
-                return new Field(name, FieldType.nullable(ArrowType.List.INSTANCE), Collections.singletonList(elementField));
-            }
-            return new Field(name, FieldType.nullable(scalarArrowType(value)), null);
+        if (batches.isEmpty()) {
+            return VectorSchemaRoot.create(schema, allocator);
         }
-        return new Field(name, FieldType.nullable(ArrowType.Utf8.INSTANCE), null);
-    }
-
-    /**
-     * Best-effort inference for a list element type. Looks at the first non-null
-     * element of the given list, then falls back to scanning later rows of the
-     * same column if this list is empty or all-null. Defaults to {@code Utf8}.
-     */
-    private static ArrowType inferElementArrowType(List list, List rows, int col) {
-        for (Object element : list) {
-            if (element != null) return scalarArrowType(element);
+        if (batches.size() == 1) {
+            return batches.get(0);
         }
-        for (Object[] row : rows) {
-            Object value = row[col];
-            if (value instanceof List other) {
-                for (Object element : other) {
-                    if (element != null) return scalarArrowType(element);
+        // Multiple batches — concatenate via per-cell copyFromSafe. Slower than columnar
+        // append but is the only operation Arrow Java implements for every vector type
+        // (VectorAppender throws on view vectors; see class javadoc).
+        int totalRows = batches.stream().mapToInt(VectorSchemaRoot::getRowCount).sum();
+        VectorSchemaRoot combined = VectorSchemaRoot.create(schema, allocator);
+        try {
+            combined.allocateNew();
+            for (int f = 0; f < combined.getFieldVectors().size(); f++) {
+                FieldVector dst = combined.getVector(f);
+                int offset = 0;
+                for (VectorSchemaRoot batch : batches) {
+                    FieldVector src = batch.getVector(f);
+                    int rows = batch.getRowCount();
+                    for (int r = 0; r < rows; r++) {
+                        dst.copyFromSafe(r, offset + r, src);
+                    }
+                    offset += rows;
                 }
+                dst.setValueCount(totalRows);
             }
-        }
-        return ArrowType.Utf8.INSTANCE;
-    }
-
-    /** Maps a Java scalar value to the corresponding Arrow scalar type. */
-    private static ArrowType scalarArrowType(Object value) {
-        if (value instanceof Long) return new ArrowType.Int(64, true);
-        if (value instanceof Integer) return new ArrowType.Int(32, true);
-        if (value instanceof Short) return new ArrowType.Int(16, true);
-        if (value instanceof Byte) return new ArrowType.Int(8, true);
-        if (value instanceof Double) return new ArrowType.FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE);
-        if (value instanceof Float) return new ArrowType.FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE);
-        // BigDecimal must be checked before the Number fallback below — a BigDecimal that
-        // would round to a long but actually carries fractional precision (e.g. PPL
-        // {@code array(1, -1.5)} where the common element type is DECIMAL and
-        // {@code ArrayImplementor.internalCast} produces BigDecimal cells) would otherwise
-        // get encoded as an integer Arrow vector and lose its fractional digits. Promote to
-        // DOUBLE — the same path the v2 engine takes for decimal-typed PPL results.
-        if (value instanceof java.math.BigDecimal) return new ArrowType.FloatingPoint(
-            org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE
-        );
-        if (value instanceof Boolean) return ArrowType.Bool.INSTANCE;
-        if (value instanceof CharSequence) return ArrowType.Utf8.INSTANCE;
-        if (value instanceof byte[]) return ArrowType.Binary.INSTANCE;
-        if (value instanceof Number) return new ArrowType.Int(64, true);
-        return ArrowType.Utf8.INSTANCE;
-    }
-
-    /**
-     * Sets a value on the appropriate Arrow vector type. Handles null by
-     * calling {@code setNull}. For typed vectors, casts the Java value to
-     * the expected type.
-     */
-    static void setVectorValue(FieldVector vector, int index, Object value) {
-        if (value == null) {
-            vector.setNull(index);
-            return;
-        }
-        if (vector instanceof BigIntVector) {
-            ((BigIntVector) vector).setSafe(index, ((Number) value).longValue());
-        } else if (vector instanceof IntVector) {
-            ((IntVector) vector).setSafe(index, ((Number) value).intValue());
-        } else if (vector instanceof SmallIntVector) {
-            ((SmallIntVector) vector).setSafe(index, ((Number) value).shortValue());
-        } else if (vector instanceof TinyIntVector) {
-            ((TinyIntVector) vector).setSafe(index, ((Number) value).byteValue());
-        } else if (vector instanceof Float8Vector) {
-            ((Float8Vector) vector).setSafe(index, ((Number) value).doubleValue());
-        } else if (vector instanceof Float4Vector) {
-            ((Float4Vector) vector).setSafe(index, ((Number) value).floatValue());
-        } else if (vector instanceof BitVector) {
-            ((BitVector) vector).setSafe(index, ((Boolean) value) ? 1 : 0);
-        } else if (vector instanceof VarCharVector) {
-            ((VarCharVector) vector).setSafe(index, value.toString().getBytes(StandardCharsets.UTF_8));
-        } else if (vector instanceof VarBinaryVector) {
-            ((VarBinaryVector) vector).setSafe(index, (byte[]) value);
-        } else if (vector instanceof ListVector listVector) {
-            writeListValue(listVector, index, (List) value);
-        } else {
-            throw new IllegalArgumentException("Unsupported Arrow vector type: " + vector.getClass().getSimpleName());
-        }
-    }
-
-    /**
-     * Writes a Java {@link List} into an Arrow {@link ListVector} at the given row
-     * index. Bypasses {@link UnionListWriter} entirely — writes directly to the
-     * list's offset / validity buffers and to the inner data vector via the inner
-     * vector's own typed setter. The writer-based API requires an
-     * {@link org.apache.arrow.memory.ArrowBuf} per varchar element which Arrow
-     * couples to a release lifecycle that's tricky to get right (early close →
-     * use-after-free, no close → leak); the direct path avoids that altogether.
-     *
-     * 
            The inner data vector's type was decided in {@link #inferField} from the
-     * first non-null list element, so the {@code instanceof} dispatch here simply
-     * needs to match.
-     */
-    private static void writeListValue(ListVector listVector, int index, List list) {
-        FieldVector dataVector = listVector.getDataVector();
-        int startOffset = listVector.getOffsetBuffer().getInt((long) index * ListVector.OFFSET_WIDTH);
-        int writePos = startOffset;
-        for (Object element : list) {
-            // Grow the data vector if needed before writing. setSafe-style helpers handle this
-            // automatically per type, but we still need to pre-position the cursor.
-            if (element == null) {
-                dataVector.setNull(writePos);
-            } else if (dataVector instanceof BigIntVector v) {
-                v.setSafe(writePos, ((Number) element).longValue());
-            } else if (dataVector instanceof IntVector v) {
-                v.setSafe(writePos, ((Number) element).intValue());
-            } else if (dataVector instanceof SmallIntVector v) {
-                v.setSafe(writePos, ((Number) element).shortValue());
-            } else if (dataVector instanceof TinyIntVector v) {
-                v.setSafe(writePos, ((Number) element).byteValue());
-            } else if (dataVector instanceof Float8Vector v) {
-                v.setSafe(writePos, ((Number) element).doubleValue());
-            } else if (dataVector instanceof Float4Vector v) {
-                v.setSafe(writePos, ((Number) element).floatValue());
-            } else if (dataVector instanceof BitVector v) {
-                v.setSafe(writePos, ((Boolean) element) ? 1 : 0);
-            } else if (dataVector instanceof VarCharVector v) {
-                v.setSafe(writePos, element.toString().getBytes(StandardCharsets.UTF_8));
-            } else if (dataVector instanceof VarBinaryVector v) {
-                v.setSafe(writePos, (byte[]) element);
-            } else {
-                throw new IllegalArgumentException("Unsupported list element vector type: " + dataVector.getClass().getSimpleName());
-            }
-            writePos++;
-        }
-        // Mark this row's list as non-null and update its end offset.
-        listVector.setNotNull(index);
-        listVector.getOffsetBuffer().setInt((long) (index + 1) * ListVector.OFFSET_WIDTH, writePos);
-        // Keep the data vector's value count in sync so subsequent reads see the new tail.
-        if (writePos > dataVector.getValueCount()) {
-            dataVector.setValueCount(writePos);
+            combined.setRowCount(totalRows);
+            return combined;
+        } finally {
+            for (VectorSchemaRoot b : batches)
+                b.close();
         }
     }
 }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/ArrowCalciteTypes.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/ArrowCalciteTypes.java
new file mode 100644
index 0000000000000..62d6fddfc7498
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/ArrowCalciteTypes.java
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.planner;
+
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Bidirectional Arrow ↔ Calcite type converter for single types.
+ * This is the sole authority for type reconciliation between the
+ * {@code AggregateFunction.intermediateFields} Arrow types and
+ * Calcite's {@code RelDataType} system in the decomposition resolver.
+ */
+public final class ArrowCalciteTypes {
+
+    private ArrowCalciteTypes() {}
+
+    /**
+     * Convert an Arrow type to the corresponding Calcite {@link RelDataType}.
+     */
+    public static RelDataType toCalcite(ArrowType t, RelDataTypeFactory f) {
+        return switch (t) {
+            case ArrowType.Int i when i.getBitWidth() == 64 -> f.createSqlType(SqlTypeName.BIGINT);
+            case ArrowType.Int i when i.getBitWidth() == 32 -> f.createSqlType(SqlTypeName.INTEGER);
+            case ArrowType.FloatingPoint fp when fp.getPrecision() == FloatingPointPrecision.DOUBLE -> f.createSqlType(SqlTypeName.DOUBLE);
+            case ArrowType.FloatingPoint fp when fp.getPrecision() == FloatingPointPrecision.SINGLE -> f.createSqlType(SqlTypeName.REAL);
+            case ArrowType.Utf8 u -> f.createSqlType(SqlTypeName.VARCHAR, Integer.MAX_VALUE);
+            case ArrowType.Binary b -> f.createSqlType(SqlTypeName.VARBINARY, Integer.MAX_VALUE);
+            case ArrowType.Bool b -> f.createSqlType(SqlTypeName.BOOLEAN);
+            default -> throw new IllegalArgumentException("Unsupported Arrow type: " + t);
+        };
+    }
+
+    /**
+     * Convert a Calcite {@link RelDataType} to the corresponding Arrow type.
+     */
+    public static ArrowType toArrow(RelDataType t) {
+        return switch (t.getSqlTypeName()) {
+            case BIGINT -> new ArrowType.Int(64, true);
+            case INTEGER -> new ArrowType.Int(32, true);
+            case DOUBLE -> new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
+            case REAL, FLOAT -> new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE);
+            case VARCHAR, CHAR -> ArrowType.Utf8.INSTANCE;
+            case VARBINARY, BINARY -> ArrowType.Binary.INSTANCE;
+            case BOOLEAN -> ArrowType.Bool.INSTANCE;
+            default -> throw new IllegalArgumentException("Unsupported Calcite type: " + t.getSqlTypeName());
+        };
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java
index 2d8c332eb2e34..26794af1b2093 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/PlannerImpl.java
@@ -27,6 +27,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.analytics.planner.rel.OpenSearchDistributionTraitDef;
+import org.opensearch.analytics.planner.rules.OpenSearchAggregateReduceRule;
 import org.opensearch.analytics.planner.rules.OpenSearchAggregateRule;
 import org.opensearch.analytics.planner.rules.OpenSearchAggregateSplitRule;
 import org.opensearch.analytics.planner.rules.OpenSearchFilterRule;
@@ -72,30 +73,41 @@ public static RelNode createPlan(RelNode rawRelNode, PlannerContext context) {
     public static RelNode markAndOptimize(RelNode rawRelNode, PlannerContext context) {
         LOGGER.info("Input RelNode:\n{}", RelOptUtil.toString(rawRelNode));
 
-        // Phase 1: RBO — pre-marking logical optimizations then marking rules, single HepPlanner
-        HepProgramBuilder hepBuilder = new HepProgramBuilder();
-
-        // Pre-marking: reduce constant expressions before marking rules fire.
-        // TODO: establish a FrontEnd API contract specifying which standard Calcite optimizations
-        // frontends apply themselves before submitting a RelNode. Rules already applied by the
-        // frontend should not be re-added here — re-applying them increases overall planning time.
-        hepBuilder.addMatchOrder(HepMatchOrder.ARBITRARY);
-        hepBuilder.addRuleCollection(
+        // Phase 1a: Pre-marking logical optimizations (constant expression reduction)
+        HepProgramBuilder preBuilder = new HepProgramBuilder();
+        preBuilder.addMatchOrder(HepMatchOrder.ARBITRARY);
+        preBuilder.addRuleCollection(
             List.of(
                 new ReduceExpressionsRule.FilterReduceExpressionsRule(Filter.class, RelBuilder.proto(Contexts.empty())),
                 new ReduceExpressionsRule.ProjectReduceExpressionsRule(Project.class, RelBuilder.proto(Contexts.empty()))
             )
         );
-
-        // Marking: convert LogicalXxx → OpenSearchXxx bottom-up
+        HepPlanner prePlanner = new HepPlanner(preBuilder.build());
+        prePlanner.setRoot(rawRelNode);
+        RelNode afterPre = prePlanner.findBestExp();
+
+        // Phase 1b: Aggregate-reduction — decompose AVG / STDDEV / VAR into primitive SUM/COUNT
+        // (+ SUM_SQ for variance) plus a scalar LogicalProject computing the quotient. Runs as
+        // its own HEP pass on plain LogicalAggregate so Calcite's type inference is clean —
+        // no AGG_CALL_ANNOTATION wrappers in aggCall.rexList to propagate AVG's DOUBLE return
+        // type to the derived primitive calls. Downstream the marking phase, the Volcano split
+        // rule, and the AggregateDecompositionResolver see correctly-typed primitives.
+        HepProgramBuilder reduceBuilder = new HepProgramBuilder();
+        reduceBuilder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
+        reduceBuilder.addRuleInstance(new OpenSearchAggregateReduceRule());
+        HepPlanner reducePlanner = new HepPlanner(reduceBuilder.build());
+        reducePlanner.setRoot(afterPre);
+        RelNode afterReduce = reducePlanner.findBestExp();
+
+        // Phase 1c: Marking — convert LogicalXxx → OpenSearchXxx bottom-up
         // TODO: migrate rules from deprecated RelOptRule to RelRule once the planner
         // moves to its own Gradle module. The OpenSearch monorepo injects -proc:none globally,
         // blocking the Immutables annotation processor required by RelRule.Config sub-interfaces.
         // TODO: add SortPushdown rule here — pushes Sort below Exchange to data nodes for top-K
-        // optimization. When Sort is pushed to data nodes above a partial aggregate, FragmentConversionDriver
-        // must call convertShardScanFragment → attachPartialAggOnTop → attachFragmentOnTop(Sort) in sequence.
-        hepBuilder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
-        hepBuilder.addRuleCollection(
+        // optimization.
+        HepProgramBuilder markBuilder = new HepProgramBuilder();
+        markBuilder.addMatchOrder(HepMatchOrder.BOTTOM_UP);
+        markBuilder.addRuleCollection(
             List.of(
                 new OpenSearchTableScanRule(context),
                 new OpenSearchFilterRule(context),
@@ -105,9 +117,8 @@ public static RelNode markAndOptimize(RelNode rawRelNode, PlannerContext context
                 new OpenSearchUnionRule(context)
             )
         );
-
-        HepPlanner markingPlanner = new HepPlanner(hepBuilder.build());
-        markingPlanner.setRoot(rawRelNode);
+        HepPlanner markingPlanner = new HepPlanner(markBuilder.build());
+        markingPlanner.setRoot(afterReduce);
         RelNode marked = markingPlanner.findBestExp();
 
         LOGGER.info("After marking:\n{}", RelOptUtil.toString(marked));
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/AggregateDecompositionResolver.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/AggregateDecompositionResolver.java
new file mode 100644
index 0000000000000..672434804938e
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/AggregateDecompositionResolver.java
@@ -0,0 +1,438 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.planner.dag;
+
+import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.analytics.planner.ArrowCalciteTypes;
+import org.opensearch.analytics.planner.CapabilityRegistry;
+import org.opensearch.analytics.planner.rel.AggregateMode;
+import org.opensearch.analytics.planner.rel.OpenSearchAggregate;
+import org.opensearch.analytics.planner.rel.OpenSearchStageInputScan;
+import org.opensearch.analytics.spi.AggregateFunction;
+import org.opensearch.analytics.spi.AggregateFunction.IntermediateField;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Rewrites PARTIAL/FINAL aggregate pairs so the exchange row type precisely
+ * describes what the engine emits. Uses {@link AggregateFunction#intermediateFields()}
+ * as the single source of truth — no downstream type overrides needed.
+ *
+ * 
            Runs after {@link BackendPlanAdapter} and before {@link FragmentConversionDriver}.
+ *
+ * @opensearch.internal
+ */
+public final class AggregateDecompositionResolver {
+
+    private static final Logger LOGGER = LogManager.getLogger(AggregateDecompositionResolver.class);
+
+    private AggregateDecompositionResolver() {}
+
+    /**
+     * Walk the DAG and rewrite all PARTIAL/FINAL aggregate pairs in each stage's plan alternatives.
+     */
+    public static void resolveAll(QueryDAG dag, CapabilityRegistry registry) {
+        resolveStage(dag.rootStage(), registry);
+    }
+
+    // Walk children first (post-order), then pair each child's PARTIAL with this stage's FINAL.
+    private static void resolveStage(Stage stage, CapabilityRegistry registry) {
+        for (Stage child : stage.getChildStages()) {
+            resolveStage(child, registry);
+        }
+
+        // For each child stage that has a PARTIAL aggregate, rewrite the parent's FINAL.
+        // The parent stage's planAlternatives contain the FINAL; child's contain the PARTIAL.
+        for (Stage child : stage.getChildStages()) {
+            resolvePartialFinalPair(stage, child);
+        }
+    }
+
+    // For one parent/child stage pair: rewrite each child PARTIAL, then apply the matching FINAL rewrite in the parent.
+    private static void resolvePartialFinalPair(Stage parentStage, Stage childStage) {
+        List resolvedChildPlans = new ArrayList<>(childStage.getPlanAlternatives().size());
+        List resolvedParentPlans = new ArrayList<>(parentStage.getPlanAlternatives().size());
+        List rewriteResults = new ArrayList<>();
+
+        // Process child plans — rewrite PARTIAL aggregates and collect rewrite results
+        for (StagePlan childPlan : childStage.getPlanAlternatives()) {
+            OpenSearchAggregate partialAgg = findTopAggregate(childPlan.resolvedFragment(), AggregateMode.PARTIAL);
+            if (partialAgg == null) {
+                resolvedChildPlans.add(childPlan);
+                rewriteResults.add(null);
+                continue;
+            }
+            RewriteResult result = rewriteDecomposed(partialAgg);
+            rewriteResults.add(result);
+            RelNode newChildFragment = replaceFirst(childPlan.resolvedFragment(), partialAgg, result.newPartial(partialAgg));
+            resolvedChildPlans.add(new StagePlan(newChildFragment, childPlan.backendId()));
+        }
+
+        // If no child had a PARTIAL, nothing to do
+        boolean anyChildRewritten = rewriteResults.stream().anyMatch(r -> r != null);
+        if (!anyChildRewritten) return;
+
+        childStage.setPlanAlternatives(resolvedChildPlans);
+
+        // Process parent plans — rewrite FINAL aggregates using the rewrite results from child
+        for (int i = 0; i < parentStage.getPlanAlternatives().size(); i++) {
+            StagePlan parentPlan = parentStage.getPlanAlternatives().get(i);
+            RewriteResult result = rewriteResults.get(Math.min(i, rewriteResults.size() - 1));
+            if (result == null) {
+                resolvedParentPlans.add(parentPlan);
+                continue;
+            }
+
+            RelNode rewrittenParent = rewriteParentFragment(
+                parentPlan.resolvedFragment(),
+                result.exchangeRowType,
+                childStage.getStageId(),
+                result
+            );
+            resolvedParentPlans.add(new StagePlan(rewrittenParent, parentPlan.backendId()));
+        }
+        parentStage.setPlanAlternatives(resolvedParentPlans);
+    }
+
+    // Apply a child's RewriteResult to one parent fragment: update the StageInputScan's row type and swap in the new FINAL aggCalls.
+    private static RelNode rewriteParentFragment(RelNode fragment, RelDataType childRowType, int childStageId, RewriteResult result) {
+        // Walk the parent fragment to find the FINAL aggregate and its StageInputScan
+        OpenSearchAggregate finalAgg = findTopAggregate(fragment, AggregateMode.FINAL);
+        if (finalAgg == null) return fragment;
+
+        // Find the StageInputScan under the FINAL (through ExchangeReducer)
+        RelNode finalInput = finalAgg.getInput();
+        OpenSearchStageInputScan stageInput = findStageInputScan(finalInput, childStageId);
+        if (stageInput == null) return fragment;
+
+        // Rebuild with updated StageInputScan row type
+        OpenSearchStageInputScan newStageInput = new OpenSearchStageInputScan(
+            stageInput.getCluster(),
+            stageInput.getTraitSet(),
+            stageInput.getChildStageId(),
+            childRowType,
+            stageInput.getViableBackends()
+        );
+
+        // Rebuild the chain: StageInputScan → ExchangeReducer → FINAL Agg
+        RelNode newFinalInput = replaceFirst(finalInput, stageInput, newStageInput);
+
+        // Re-infer each FINAL aggCall's type against the rewritten input (StageInputScan).
+        // Our hand-built colType (from ArrowCalciteTypes.toCalcite, which returns NOT NULL
+        // types) doesn't match Calcite's inference for aggregates over a typed exchange
+        // column, so construct each call via the RelNode-aware create variant with
+        // type=null so Calcite runs full inference.
+        boolean hasEmptyGroup = finalAgg.getGroupSet().isEmpty();
+        List rebuiltFinalCalls = result.newFinalCalls.stream()
+            .map(
+                c -> AggregateCall.create(
+                    c.getAggregation(),
+                    c.isDistinct(),
+                    c.isApproximate(),
+                    c.ignoreNulls(),
+                    c.rexList,
+                    c.getArgList(),
+                    c.filterArg,
+                    c.distinctKeys,
+                    c.collation,
+                    hasEmptyGroup,
+                    newFinalInput,
+                    null,
+                    c.name
+                )
+            )
+            .toList();
+
+        // Build the new FINAL with the rewrite result's final calls and updated input
+        OpenSearchAggregate newFinal = new OpenSearchAggregate(
+            finalAgg.getCluster(),
+            finalAgg.getTraitSet(),
+            newFinalInput,
+            finalAgg.getGroupSet(),
+            finalAgg.getGroupSets(),
+            rebuiltFinalCalls,
+            AggregateMode.FINAL,
+            finalAgg.getViableBackends()
+        );
+
+        RelNode top = newFinal;
+
+        // If the original fragment had something above the FINAL, replace it.
+        // replaceFirst copies any parent Project unchanged — but those Projects contain
+        // RexInputRefs built against the ORIGINAL FINAL's output types. After we re-infer
+        // FINAL's aggCall types above, those refs may not match. Walk the parent and
+        // rewire RexInputRefs to match newFinal's output, CASTing to the Project's
+        // declared column type to preserve the outer-world-visible schema.
+        if (fragment == finalAgg) {
+            return top;
+        }
+        return replaceFirstWithRefRebinding(fragment, finalAgg, top);
+    }
+
+    // Like replaceFirst but when rewriting a Project directly above the target, rebinds
+    // its RexInputRefs to the new input's row type and CASTs each projection back to the
+    // Project's declared column type. Preserves outer schema while fixing inner ref types.
+    private static RelNode replaceFirstWithRefRebinding(RelNode node, RelNode target, RelNode replacement) {
+        if (node == target) return replacement;
+        java.util.List newInputs = new java.util.ArrayList<>();
+        boolean changed = false;
+        for (RelNode input : node.getInputs()) {
+            RelNode newInput;
+            if (input == target) {
+                newInput = replacement;
+                if (node instanceof Project proj) {
+                    RexBuilder rexBuilder = node.getCluster().getRexBuilder();
+                    java.util.List inputTypes = new java.util.ArrayList<>();
+                    for (var f : replacement.getRowType().getFieldList()) {
+                        inputTypes.add(f.getType());
+                    }
+                    RexShuttle rebind = new RexShuttle() {
+                        @Override
+                        public RexNode visitInputRef(RexInputRef ref) {
+                            RelDataType actual = inputTypes.get(ref.getIndex());
+                            if (ref.getType().equals(actual)) return ref;
+                            return new RexInputRef(ref.getIndex(), actual);
+                        }
+                    };
+                    java.util.List rebound = new java.util.ArrayList<>(proj.getProjects().size());
+                    for (int i = 0; i < proj.getProjects().size(); i++) {
+                        RexNode expr = proj.getProjects().get(i).accept(rebind);
+                        RelDataType targetType = proj.getRowType().getFieldList().get(i).getType();
+                        if (!expr.getType().equals(targetType)) {
+                            expr = rexBuilder.makeCast(targetType, expr);
+                        }
+                        rebound.add(expr);
+                    }
+                    return proj.copy(proj.getTraitSet(), replacement, rebound, proj.getRowType());
+                }
+            } else {
+                newInput = replaceFirstWithRefRebinding(input, target, replacement);
+            }
+            newInputs.add(newInput);
+            if (newInput != input) changed = true;
+        }
+        return changed ? node.copy(node.getTraitSet(), newInputs) : node;
+    }
+
+    /**
+     * Core decomposition logic. Produces rewritten PARTIAL calls, FINAL calls, and the
+     * exchange row type (from intermediateFields). Per-call classification is delegated
+     * to {@link #rewriteAggCall}, which returns one immutable {@link CallRewrite} per
+     * input aggregate call — keeping the four output columns (partial, final, exchange
+     * type, exchange name) in lockstep.
+     *
+     * 
            PARTIAL calls use Calcite-natural types (to pass Aggregate validation). The
+     * exchange row type (set on StageInputScan) uses intermediateFields types — this
+     * is the single source of truth for what the engine actually emits.
+     */
+    static RewriteResult rewriteDecomposed(OpenSearchAggregate agg) {
+        RelDataTypeFactory tf = agg.getCluster().getTypeFactory();
+        int groupCount = agg.getGroupSet().cardinality();
+
+        List newPartialCalls = new ArrayList<>();
+        List newFinalCalls = new ArrayList<>();
+        List exchangeFieldTypes = new ArrayList<>();
+        List exchangeFieldNames = new ArrayList<>();
+
+        // Group keys pass through to exchange unchanged.
+        RelDataType inputRowType = agg.getInput().getRowType();
+        for (int groupIdx : agg.getGroupSet()) {
+            exchangeFieldTypes.add(inputRowType.getFieldList().get(groupIdx).getType());
+            exchangeFieldNames.add(inputRowType.getFieldList().get(groupIdx).getName());
+        }
+
+        int finalColIdx = groupCount;
+        // The PARTIAL aggregate's output row type is the source of truth for exchange
+        // column names: Calcite assigns explicit names where aggCall.name is set and
+        // auto-generates "$f" otherwise — matching DataFusion's convention for
+        // unnamed aggregate outputs. Using these names aligns the Java-side exchange
+        // schema with what DataFusion emits at execution, preventing Substrait-consumer
+        // schema lookups from failing on name mismatches (e.g. "$f2" vs "expr$2").
+        RelDataType aggRowType = agg.getRowType();
+        for (int i = 0; i < agg.getAggCallList().size(); i++) {
+            AggregateCall call = agg.getAggCallList().get(i);
+            String canonicalName = aggRowType.getFieldList().get(groupCount + i).getName();
+            CallRewrite rw = rewriteAggCall(call, finalColIdx, tf, canonicalName);
+            newPartialCalls.add(rw.partialCall());
+            newFinalCalls.add(rw.finalCall());
+            exchangeFieldTypes.add(rw.exchangeType());
+            exchangeFieldNames.add(rw.exchangeName());
+            finalColIdx++;
+        }
+
+        RelDataType exchangeRowType = tf.createStructType(exchangeFieldTypes, exchangeFieldNames);
+        return new RewriteResult(newPartialCalls, newFinalCalls, exchangeRowType);
+    }
+
+    // Classify an AggregateCall and dispatch to the matching rewrite (pass-through or single-field).
+    private static CallRewrite rewriteAggCall(AggregateCall call, int finalColIdx, RelDataTypeFactory tf, String canonicalName) {
+        AggregateFunction fn = AggregateFunction.fromSqlAggFunction(call.getAggregation());
+
+        if (fn == null || !fn.hasDecomposition()) {
+            return passThroughRewrite(call, finalColIdx, canonicalName);
+        }
+
+        List iFields = fn.intermediateFields();
+
+        // Multi-field shapes (AVG / STDDEV / VAR) should have been reduced in HEP by
+        // OpenSearchAggregateReduceRule before reaching this resolver. If we see one here,
+        // FUNCTIONS_TO_REDUCE in that rule is incomplete.
+        if (iFields.size() != 1) {
+            throw new IllegalStateException(
+                "AggregateFunction."
+                    + fn
+                    + " declares a multi-field decomposition, but the resolver only"
+                    + " supports single-field engine-native / function-swap shapes."
+                    + " Calcite's AggregateReduceFunctionsRule should reduce multi-field"
+                    + " cases during HEP marking. Check that"
+                    + " OpenSearchAggregateReduceRule's FUNCTIONS_TO_REDUCE set covers "
+                    + call.getAggregation().getName()
+                    + "."
+            );
+        }
+
+        return singleFieldRewrite(call, fn, iFields.get(0), finalColIdx, tf, canonicalName);
+    }
+
+    // Pass-through: aggregate has no intermediate-field decomposition; keep the call at PARTIAL
+    // and rebind its single arg index at FINAL. Exchange column takes the call's Calcite type
+    // and the aggregate's canonical output name.
+    private static CallRewrite passThroughRewrite(AggregateCall call, int finalColIdx, String canonicalName) {
+        return new CallRewrite(call, rebindCall(call, List.of(finalColIdx)), call.getType(), canonicalName);
+    }
+
+    // Single-field decomposition: exchange type comes from IntermediateField; FINAL is either
+    // engine-native merge (reducer == self, e.g. APPROX_COUNT_DISTINCT sketch) or function-swap
+    // (e.g. COUNT → SUM). Exchange column name is the aggregate's canonical output name.
+    private static CallRewrite singleFieldRewrite(
+        AggregateCall call,
+        AggregateFunction fn,
+        IntermediateField field,
+        int finalColIdx,
+        RelDataTypeFactory tf,
+        String canonicalName
+    ) {
+        RelDataType colType = ArrowCalciteTypes.toCalcite(field.arrowType(), tf);
+        AggregateCall finalCall = fn.equals(field.reducer())
+            ? rebindCall(call, List.of(finalColIdx))                                // engine-native merge (reducer == self)
+            : makeCall(field.reducer(), List.of(finalColIdx), colType, call.name, tf); // function-swap
+
+        return new CallRewrite(call, finalCall, colType, canonicalName);
+    }
+
+    // ── Helpers ──
+
+    // Copy an AggregateCall with its argument ordinals remapped to the decomposed column positions.
+    private static AggregateCall rebindCall(AggregateCall call, List newArgs) {
+        return AggregateCall.create(
+            call.getAggregation(),
+            call.isDistinct(),
+            call.isApproximate(),
+            call.ignoreNulls(),
+            call.rexList,
+            newArgs,
+            call.filterArg,
+            call.distinctKeys,
+            call.collation,
+            call.getType(),
+            call.name
+        );
+    }
+
+    // Build a fresh AggregateCall for a reducer function at FINAL (no distinct, no filter, empty collation).
+    private static AggregateCall makeCall(
+        AggregateFunction reducer,
+        List args,
+        RelDataType returnType,
+        String name,
+        RelDataTypeFactory tf
+    ) {
+        SqlAggFunction sqlAgg = reducer.toSqlAggFunction();
+        return AggregateCall.create(sqlAgg, false, false, false, List.of(), args, -1, null, RelCollations.EMPTY, returnType, name);
+    }
+
+    // Find the top-most OpenSearchAggregate matching the given mode, walking into inputs recursively.
+    private static OpenSearchAggregate findTopAggregate(RelNode node, AggregateMode mode) {
+        if (node instanceof OpenSearchAggregate agg && agg.getMode() == mode) {
+            return agg;
+        }
+        // Check if it's wrapped (e.g., Project on top of FINAL)
+        for (RelNode input : node.getInputs()) {
+            OpenSearchAggregate found = findTopAggregate(input, mode);
+            if (found != null) return found;
+        }
+        return null;
+    }
+
+    // Find the StageInputScan for the given child stage id, walking into inputs recursively.
+    private static OpenSearchStageInputScan findStageInputScan(RelNode node, int childStageId) {
+        if (node instanceof OpenSearchStageInputScan scan && scan.getChildStageId() == childStageId) {
+            return scan;
+        }
+        for (RelNode input : node.getInputs()) {
+            OpenSearchStageInputScan found = findStageInputScan(input, childStageId);
+            if (found != null) return found;
+        }
+        return null;
+    }
+
+    /**
+     * Identity-based RelNode tree rewrite: returns a copy of {@code node} in which the
+     * subtree at {@code target} (matched by reference equality) has been replaced with
+     * {@code replacement}. Used to swap a rewritten aggregate back into its fragment
+     * and to swap an updated StageInputScan into the FINAL subtree.
+     */
+    private static RelNode replaceFirst(RelNode node, RelNode target, RelNode replacement) {
+        if (node == target) return replacement;
+        List newInputs = new ArrayList<>();
+        boolean changed = false;
+        for (RelNode input : node.getInputs()) {
+            RelNode newInput = replaceFirst(input, target, replacement);
+            newInputs.add(newInput);
+            if (newInput != input) changed = true;
+        }
+        return changed ? node.copy(node.getTraitSet(), newInputs) : node;
+    }
+
+    // ── Inner types ──
+
+    record RewriteResult(List newPartialCalls, List newFinalCalls, RelDataType exchangeRowType) {
+        OpenSearchAggregate newPartial(OpenSearchAggregate original) {
+            return copyAgg(original, newPartialCalls);
+        }
+    }
+
+    // Per-aggCall rewrite: what to emit at PARTIAL, FINAL, and the exchange column.
+    private record CallRewrite(AggregateCall partialCall, AggregateCall finalCall, RelDataType exchangeType, String exchangeName) {
+    }
+
+    // Shallow-copy an OpenSearchAggregate with a new aggCall list, preserving traits, group sets, and input.
+    private static OpenSearchAggregate copyAgg(OpenSearchAggregate original, List newCalls) {
+        return (OpenSearchAggregate) original.copy(
+            original.getTraitSet(),
+            original.getInput(),
+            original.getGroupSet(),
+            original.getGroupSets(),
+            newCalls
+        );
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java
index e21ef6e00e145..97eec86c4be39 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rel/OpenSearchProject.java
@@ -132,6 +132,12 @@ public RelNode stripAnnotations(List strippedChildren, FunctionOrder: this rule operates on plain {@link LogicalAggregate} so it fires
+ * before {@link OpenSearchAggregateRule} marks the aggregate. Running on the
+ * un-marked plan keeps Calcite's type inference clean — the reduce rule sees an aggregate
+ * whose {@code aggCall.rexList} is empty, so the reduced SUM/COUNT calls get their
+ * natural primitive return types (BIGINT for SUM of integer, not AVG's carry-over DOUBLE).
+ * The marking rule then converts the already-reduced plan to {@link
+ * org.opensearch.analytics.planner.rel.OpenSearchAggregate} with correctly-typed
+ * primitive aggregate calls, and the Volcano split rule downstream operates on those
+ * primitives.
+ *
+ * 
            Reduction set: narrowed via {@code FUNCTIONS_TO_REDUCE} to AVG only. AVG's
+ * reduction uses only SUM, COUNT, DIVIDE, and CAST — all either capability-declared
+ * aggregates or baseline scalar operators ({@link OpenSearchProjectRule#BASELINE_SCALAR_OPS}).
+ * STDDEV_POP / STDDEV_SAMP / VAR_POP / VAR_SAMP also emit {@code POWER(x, 2)} which is
+ * not in the baseline set; extending the set is a one-line change once a backend declares
+ * POWER as a project capability.
+ *
+ * @opensearch.internal
+ */
+public class OpenSearchAggregateReduceRule extends AggregateReduceFunctionsRule {
+
+    private static final EnumSet FUNCTIONS_TO_REDUCE = EnumSet.of(SqlKind.AVG);
+
+    public OpenSearchAggregateReduceRule() {
+        super(LogicalAggregate.class, RelBuilder.proto(Contexts.empty()), FUNCTIONS_TO_REDUCE);
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateSplitRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateSplitRule.java
index 4be2d71520adf..da5fb81763dc1 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateSplitRule.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateSplitRule.java
@@ -25,25 +25,21 @@
  * trait enforcement (via {@code ExpandConversionRule} + {@code OpenSearchDistributionTraitDef})
  * automatically insert an {@code OpenSearchExchangeReducer}.
  *
- * 
            TODO (plan forking): aggregate decomposition is intentionally deferred to plan forking
- * resolution, after a single backend has been chosen per alternative. Decomposition is
- * backend-specific — different backends may emit different partial state schemas for the
- * same function (e.g. standard SUM+COUNT for AVG vs a backend's native running state).
- * Applying decomposition here would force a single schema before backends are resolved,
- * which breaks the multi-alternative model.
+ * 
            Decomposition responsibilities (post-refactor):
+ * 
              
+ *   
            • Multi-field primitive decomposition (AVG / STDDEV / VAR) is handled by
+ *       {@link OpenSearchAggregateReduceRule} during HEP marking — before this rule runs.
+ *       Volcano sees an already-reduced inner aggregate with primitive SUM/COUNT calls
+ *       and a Project on top.
              • 
+ *   
            • Single-field cases (pass-through SUM/MIN/MAX, function-swap COUNT→SUM at
+ *       FINAL, engine-native APPROX_COUNT_DISTINCT sketch merge) are handled by
+ *       {@code AggregateDecompositionResolver} after this split rule runs, reading
+ *       {@link org.opensearch.analytics.spi.AggregateFunction#intermediateFields()}
+ *       as the sole source of truth.
              • 
+ * 
            
  *
- * 
            During plan forking resolution, for each PARTIAL+FINAL pair in a chosen-backend alternative:
- * 
              
- *   
            1. Look up {@link org.opensearch.analytics.spi.AggregateCapability#decomposition()} for
- *       each AggregateCall using the chosen backend.
              2. 
- *   
            2. If null: apply Calcite's {@code AggregateReduceFunctionsRule} to rewrite
- *       AVG → SUM/COUNT, STDDEV → SUM(x²)+SUM(x)+COUNT, etc.
              3. 
- *   
            3. If non-null: use {@link org.opensearch.analytics.spi.AggregateDecomposition#partialCalls()}
- *       to rewrite PARTIAL's aggCalls and output row type, and
- *       {@code AggregateDecomposition.finalExpression()} to
- *       rewrite FINAL's aggCalls. Both must be updated together — the exchange row type
- *       between them must be consistent within the same plan alternative.
              4. 
- * 
            
+ * 
            This rule's own contract is purely structural: SINGLE → FINAL(Exchange(PARTIAL(child))).
+ * It does not rewrite aggregate calls.
  *
  * @opensearch.internal
  */
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java
index 9e82cb1c26a1b..f6767e7d06616 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java
@@ -16,6 +16,8 @@
 import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.opensearch.analytics.planner.CapabilityRegistry;
 import org.opensearch.analytics.planner.PlannerContext;
 import org.opensearch.analytics.planner.RelNodeUtils;
@@ -28,6 +30,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 /**
  * Converts {@link Project} → {@link OpenSearchProject}.
@@ -35,10 +38,60 @@
  * 
            Validates that the child's backend can evaluate all projection expressions,
  * either natively or via delegation ({@link DelegationType#PROJECT}).
  *
+ * 
            Baseline vs capability-declared scalars
            
+ * 
            Calcite plan-rewrite rules (e.g. {@code AggregateReduceFunctionsRule},
+ * {@code ReduceExpressionsRule}) routinely introduce arithmetic, CAST, CASE, and
+ * null-predicate operators while rewriting expressions. These are SQL-execution
+ * primitives that every viable backend must support — they are not optional
+ * features worth modeling in the capability registry.
+ *
+ * 
            Treating them as capability-declared creates two bad outcomes: (1) every new
+ * backend has to enumerate ~20 operators that are never actually optional, and
+ * (2) any Calcite rule that incidentally emits one of them (e.g. a CAST around
+ * {@code SUM(x) / COUNT(x)} to match AVG's original return type) would fail plan-time
+ * checks with a misleading error, even though the query semantics are unambiguous.
+ *
+ * 
            {@link #BASELINE_SCALAR_OPS} carves these primitives out of capability-registry
+ * enforcement. Operands are still recursed into — a CAST wrapping a non-baseline
+ * function still forces the inner function through capability resolution.
+ *
  * @opensearch.internal
  */
 public class OpenSearchProjectRule extends RelOptRule {
 
+    /**
+     * Scalar operators that any viable backend is implicitly assumed to support.
+     * These are SQL-execution primitives (arithmetic, type coercion, null handling,
+     * logical composition) that arise incidentally during plan rewriting and that no
+     * real execution engine lacks. They bypass {@link #resolveScalarViableBackends}
+     * and flow through {@link OpenSearchProject} without backend annotation.
+     *
+     * 
            If a future backend genuinely cannot execute one of these operators (e.g.
+     * Lucene rejects a CAST between incompatible types), that becomes a runtime
+     * error inside the backend's executor — complementary to plan-time capability
+     * enforcement, not a replacement for it.
+     *
+     * 
            Intentionally conservative: extend only when a specific plan-rewrite rule
+     * demonstrably emits a new operator that every backend already supports.
+     */
+    private static final Set BASELINE_SCALAR_OPS = Set.of(
+        // Arithmetic
+        SqlStdOperatorTable.PLUS,
+        SqlStdOperatorTable.MINUS,
+        SqlStdOperatorTable.MULTIPLY,
+        SqlStdOperatorTable.DIVIDE,
+        SqlStdOperatorTable.UNARY_MINUS,
+        SqlStdOperatorTable.UNARY_PLUS,
+        // Type coercion
+        SqlStdOperatorTable.CAST,
+        // Null handling
+        SqlStdOperatorTable.IS_NULL,
+        SqlStdOperatorTable.IS_NOT_NULL,
+        SqlStdOperatorTable.COALESCE,
+        // Conditional
+        SqlStdOperatorTable.CASE
+    );
+
     private final PlannerContext context;
 
     public OpenSearchProjectRule(PlannerContext context) {
@@ -104,6 +157,23 @@ private RexNode annotateExpr(RexNode expr, List childViableBackends) {
             return expr;
         }
 
+        // Baseline operators — arithmetic, CAST, null-handling, conditional — are assumed
+        // supported by every backend and are not subject to capability-registry enforcement.
+        // Recurse into operands so a non-baseline function nested inside (e.g.
+        // CAST(regexp_match(col, 'x'))) still flows through capability resolution.
+        if (BASELINE_SCALAR_OPS.contains(rexCall.getOperator())) {
+            boolean changed = false;
+            List newOperands = new ArrayList<>(rexCall.getOperands().size());
+            for (RexNode operand : rexCall.getOperands()) {
+                RexNode annotated = annotateExpr(operand, childViableBackends);
+                newOperands.add(annotated);
+                if (annotated != operand) {
+                    changed = true;
+                }
+            }
+            return changed ? rexCall.clone(rexCall.getType(), newOperands) : rexCall;
+        }
+
         // Opaque operations — no recursion into operands
         if (rexCall.getOperator() instanceof SqlFunction sqlFunction) {
             String funcName = sqlFunction.getName();
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
index aa8b9a054faf2..d53c19d55bb10 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
@@ -30,6 +30,7 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.test.OpenSearchTestCase;
 
@@ -122,9 +123,7 @@ public void testRowResponseSafeOnCancellation() {
 
         exec.cancel("test");
 
-        List rows = new ArrayList<>();
-        rows.add(new Object[] { 42 });
-        FragmentExecutionResponse response = new FragmentExecutionResponse(List.of("col"), rows);
+        FragmentExecutionResponse response = new FragmentExecutionResponse(new byte[0], 0);
         capturedListener.get().onStreamResponse(response, true);
 
         assertTrue("sink should not have received anything post-cancel", sink.fed.isEmpty());
@@ -134,16 +133,14 @@ public void testRowResponseSafeOnCancellation() {
      * Verifies that RowResponseCodec rejects null allocator.
      */
     public void testRowResponseCodecRejectsNullAllocator() {
-        List rows = new ArrayList<>();
-        rows.add(new Object[] { 1 });
-        FragmentExecutionResponse response = new FragmentExecutionResponse(List.of("x"), rows);
+        FragmentExecutionResponse response = new FragmentExecutionResponse(new byte[0], 0);
         expectThrows(IllegalArgumentException.class, () -> RowResponseCodec.INSTANCE.decode(response, null));
     }
 
     // ── helpers ──────────────────────────────────────────────────────────
 
     @SuppressWarnings("unchecked")
-    private  ShardFragmentStageExecution buildExecution(
+    private  ShardFragmentStageExecution buildExecution(
         CapturingSink sink,
         boolean streaming,
         AtomicReference> listenerCapture
@@ -169,7 +166,7 @@ private  ShardFragmentStage
         }
 
         ResponseCodec codec = (resp, alloc) -> {
-            VectorSchemaRoot vsr = createTestBatch(resp.getRows().size());
+            VectorSchemaRoot vsr = createTestBatch(resp.getRowCount());
             return vsr;
         };
 
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ArrowCalciteTypesTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ArrowCalciteTypesTests.java
new file mode 100644
index 0000000000000..781201dd3ee96
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ArrowCalciteTypesTests.java
@@ -0,0 +1,97 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.planner;
+
+import org.apache.arrow.vector.types.DateUnit;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.test.OpenSearchTestCase;
+
+public class ArrowCalciteTypesTests extends OpenSearchTestCase {
+
+    private final RelDataTypeFactory factory = new JavaTypeFactoryImpl();
+
+    public void testRoundTripBigint() {
+        ArrowType arrow = new ArrowType.Int(64, true);
+        RelDataType calcite = ArrowCalciteTypes.toCalcite(arrow, factory);
+        assertEquals(SqlTypeName.BIGINT, calcite.getSqlTypeName());
+        assertEquals(arrow, ArrowCalciteTypes.toArrow(calcite));
+    }
+
+    public void testRoundTripInteger() {
+        ArrowType arrow = new ArrowType.Int(32, true);
+        RelDataType calcite = ArrowCalciteTypes.toCalcite(arrow, factory);
+        assertEquals(SqlTypeName.INTEGER, calcite.getSqlTypeName());
+        assertEquals(arrow, ArrowCalciteTypes.toArrow(calcite));
+    }
+
+    public void testRoundTripDouble() {
+        ArrowType arrow = new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
+        RelDataType calcite = ArrowCalciteTypes.toCalcite(arrow, factory);
+        assertEquals(SqlTypeName.DOUBLE, calcite.getSqlTypeName());
+        assertEquals(arrow, ArrowCalciteTypes.toArrow(calcite));
+    }
+
+    public void testRoundTripReal() {
+        ArrowType arrow = new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE);
+        RelDataType calcite = ArrowCalciteTypes.toCalcite(arrow, factory);
+        assertEquals(SqlTypeName.REAL, calcite.getSqlTypeName());
+        assertEquals(arrow, ArrowCalciteTypes.toArrow(calcite));
+    }
+
+    public void testRoundTripVarchar() {
+        ArrowType arrow = ArrowType.Utf8.INSTANCE;
+        RelDataType calcite = ArrowCalciteTypes.toCalcite(arrow, factory);
+        assertEquals(SqlTypeName.VARCHAR, calcite.getSqlTypeName());
+        // Calcite's JavaTypeFactoryImpl clamps precision to its internal max (65536).
+        // We pass Integer.MAX_VALUE to request "unlimited"; the factory clamps to its max.
+        // The invariant we care about is: precision is at the factory's maximum (i.e. unbounded VARCHAR).
+        assertEquals(factory.getTypeSystem().getMaxPrecision(SqlTypeName.VARCHAR), calcite.getPrecision());
+        assertEquals(arrow, ArrowCalciteTypes.toArrow(calcite));
+    }
+
+    public void testRoundTripVarbinary() {
+        ArrowType arrow = ArrowType.Binary.INSTANCE;
+        RelDataType calcite = ArrowCalciteTypes.toCalcite(arrow, factory);
+        assertEquals(SqlTypeName.VARBINARY, calcite.getSqlTypeName());
+        // Same rationale as testRoundTripVarchar — factory clamps precision to its own max.
+        assertEquals(factory.getTypeSystem().getMaxPrecision(SqlTypeName.VARBINARY), calcite.getPrecision());
+        assertEquals(arrow, ArrowCalciteTypes.toArrow(calcite));
+    }
+
+    public void testRoundTripBoolean() {
+        ArrowType arrow = ArrowType.Bool.INSTANCE;
+        RelDataType calcite = ArrowCalciteTypes.toCalcite(arrow, factory);
+        assertEquals(SqlTypeName.BOOLEAN, calcite.getSqlTypeName());
+        assertEquals(arrow, ArrowCalciteTypes.toArrow(calcite));
+    }
+
+    public void testUnsupportedArrowTypeThrows() {
+        ArrowType date = new ArrowType.Date(DateUnit.DAY);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ArrowCalciteTypes.toCalcite(date, factory));
+        assertTrue(e.getMessage().contains("Date"));
+    }
+
+    public void testUnsupportedArrowTypeTimeThrows() {
+        ArrowType time = new ArrowType.Time(TimeUnit.MILLISECOND, 32);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ArrowCalciteTypes.toCalcite(time, factory));
+        assertTrue(e.getMessage().contains("Time"));
+    }
+
+    public void testUnsupportedCalciteTypeThrows() {
+        RelDataType timestamp = factory.createSqlType(SqlTypeName.TIMESTAMP);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ArrowCalciteTypes.toArrow(timestamp));
+        assertTrue(e.getMessage().contains("TIMESTAMP"));
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java
index 5dba4fc2d3085..5aca8886b114c 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/MockDataFusionBackend.java
@@ -100,7 +100,7 @@ public String name() {
     @Override
     public ExchangeSinkProvider getExchangeSinkProvider() {
         // Stub — real implementation provided by DataFusion backend
-        return context -> new ExchangeSink() {
+        return (context, backendContext) -> new ExchangeSink() {
             @Override
             public void feed(VectorSchemaRoot batch) {}
 
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java
index 16f12a575112a..47bb0cc88b3a1 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java
@@ -97,8 +97,11 @@ public void testExpressionProjectionStillRequiresCapabilityWithoutDeclaration()
         // Negative guard: the short-circuit must apply only to passthrough. If a RexCall is
         // present and the backend declares no matching scalar ProjectCapability, the rule must
         // still throw — otherwise a later refactor could silently loosen the gate too much.
-        RexNode castExpr = rexBuilder.makeCast(
-            typeFactory.createSqlType(SqlTypeName.VARCHAR),
+        //
+        // Uses CEIL (capability-declared scalar) rather than CAST — CAST is a baseline operator
+        // carved out of capability enforcement (see OpenSearchProjectRule.BASELINE_SCALAR_OPS).
+        RexNode ceilExpr = rexBuilder.makeCall(
+            SqlStdOperatorTable.CEIL,
             rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1)
         );
         RelOptTable table = mockTable(
@@ -106,7 +109,7 @@ public void testExpressionProjectionStillRequiresCapabilityWithoutDeclaration()
             new String[] { "name", "value" },
             new SqlTypeName[] { SqlTypeName.VARCHAR, SqlTypeName.INTEGER }
         );
-        LogicalProject project = LogicalProject.create(stubScan(table), List.of(), List.of(castExpr), List.of("casted"));
+        LogicalProject project = LogicalProject.create(stubScan(table), List.of(), List.of(ceilExpr), List.of("ceil_v"));
         PlannerContext context = buildContext("parquet", nameValueFields(), List.of(new MockDataFusionBackend(), LUCENE));
 
         IllegalStateException exception = expectThrows(IllegalStateException.class, () -> runPlanner(project, context));
@@ -116,18 +119,25 @@ public void testExpressionProjectionStillRequiresCapabilityWithoutDeclaration()
     // ---- Scalar functions ----
 
     public void testSupportedScalarFunction() {
-        RexNode castExpr = rexBuilder.makeCast(
-            typeFactory.createSqlType(SqlTypeName.VARCHAR),
+        // CEIL(int_col) — capability-declared scalar. CAST was used previously but is
+        // baseline (see OpenSearchProjectRule.BASELINE_SCALAR_OPS) and bypasses capability
+        // resolution; this test's intent is to exercise the capability-match happy path.
+        RexNode ceilExpr = rexBuilder.makeCall(
+            SqlStdOperatorTable.CEIL,
             rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1)
         );
-        OpenSearchProject result = runProject(castExpr);
+        OpenSearchProject result = runProject(ceilExpr);
         assertTrue(result.getViableBackends().contains(MockDataFusionBackend.NAME));
         assertAnnotation(result.getProjects().get(0), MockDataFusionBackend.NAME);
     }
 
     public void testUnsupportedScalarFunctionErrors() {
-        RexNode castExpr = rexBuilder.makeCast(
-            typeFactory.createSqlType(SqlTypeName.VARCHAR),
+        // Negative guard: when a RexCall uses a capability-declared scalar that no backend
+        // declares support for, the rule must throw. Uses CEIL rather than CAST because
+        // CAST is baseline (see OpenSearchProjectRule.BASELINE_SCALAR_OPS) and would not
+        // trigger capability enforcement.
+        RexNode ceilExpr = rexBuilder.makeCall(
+            SqlStdOperatorTable.CEIL,
             rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1)
         );
         RelOptTable table = mockTable(
@@ -135,7 +145,7 @@ public void testUnsupportedScalarFunctionErrors() {
             new String[] { "name", "value" },
             new SqlTypeName[] { SqlTypeName.VARCHAR, SqlTypeName.INTEGER }
         );
-        LogicalProject project = LogicalProject.create(stubScan(table), List.of(), List.of(castExpr), List.of("casted"));
+        LogicalProject project = LogicalProject.create(stubScan(table), List.of(), List.of(ceilExpr), List.of("casted"));
         PlannerContext context = buildContext("parquet", nameValueFields());
 
         IllegalStateException exception = expectThrows(IllegalStateException.class, () -> runPlanner(project, context));
@@ -252,38 +262,45 @@ protected Set projectCapabilities() {
     // ---- Nested expressions ----
 
     public void testNestedScalarFunctions() {
-        RexNode castExpr = rexBuilder.makeCast(
-            typeFactory.createSqlType(SqlTypeName.INTEGER),
-            rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 0)
+        // POWER(CEIL(v_int), v_int) — outer and inner both capability-declared scalars so
+        // annotation happens at both levels. CAST / PLUS are baseline scalars (see
+        // OpenSearchProjectRule.BASELINE_SCALAR_OPS) and are deliberately not used here
+        // because they bypass capability enforcement and would not produce an
+        // AnnotatedProjectExpression.
+        RexNode ceilExpr = rexBuilder.makeCall(
+            SqlStdOperatorTable.CEIL,
+            rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1)
         );
-        RexNode plusExpr = rexBuilder.makeCall(
-            SqlStdOperatorTable.PLUS,
-            castExpr,
+        RexNode powerExpr = rexBuilder.makeCall(
+            SqlStdOperatorTable.POWER,
+            ceilExpr,
             rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1)
         );
-        OpenSearchProject result = runProject(plusExpr);
+        OpenSearchProject result = runProject(powerExpr);
         assertTrue(result.getViableBackends().contains(MockDataFusionBackend.NAME));
         assertAnnotation(result.getProjects().get(0), MockDataFusionBackend.NAME);
     }
 
     public void testStripAnnotationsRecursivelyUnwrapsNestedExpressions() {
-        // PLUS(CEIL(value), value) — a scalar call with another scalar call as an operand.
-        // The project rule recurses into operands (annotateExpr lines 127-139), so both PLUS
-        // and the inner CEIL get wrapped in AnnotatedProjectExpression. stripAnnotations must
-        // remove every wrapper at every depth before the plan reaches the backend
-        // FragmentConvertor — Substrait isthmus has no converter for ANNOTATED_PROJECT_EXPR and
-        // would throw "Unable to convert call". (COALESCE would be the natural shape here since
-        // PPL fillnull lowers to it, but Calcite's makeCall simplifies COALESCE on non-nullable
-        // operands away into the first arg, defeating the test. PLUS+CEIL preserves the
-        // nested-call structure we want to exercise.)
+        // POWER(CEIL(value), value) — a non-baseline scalar call with another non-baseline
+        // scalar call as an operand. The project rule recurses into operands
+        // (annotateExpr), so both POWER and the inner CEIL get wrapped in
+        // AnnotatedProjectExpression. stripAnnotations must remove every wrapper at every
+        // depth before the plan reaches the backend FragmentConvertor — Substrait isthmus
+        // has no converter for ANNOTATED_PROJECT_EXPR and would throw "Unable to convert
+        // call".
+        //
+        // PLUS was used previously but is baseline (see OpenSearchProjectRule
+        // .BASELINE_SCALAR_OPS); POWER preserves the nested-call-with-nested-annotation
+        // structure this test exercises while still going through capability resolution.
         RexNode value = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1);
         RexNode ceilCall = rexBuilder.makeCall(SqlStdOperatorTable.CEIL, value);
-        RexNode plusCall = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, ceilCall, value);
-        OpenSearchProject annotated = runProject(plusCall);
+        RexNode powerCall = rexBuilder.makeCall(SqlStdOperatorTable.POWER, ceilCall, value);
+        OpenSearchProject annotated = runProject(powerCall);
 
         // Sanity: confirm the rule produced the nested-wrapper shape this test exercises.
         RexNode topLevel = annotated.getProjects().get(0);
-        assertTrue("Outer PLUS must be annotated", topLevel instanceof AnnotatedProjectExpression);
+        assertTrue("Outer POWER must be annotated", topLevel instanceof AnnotatedProjectExpression);
         RexCall outerOriginal = (RexCall) ((AnnotatedProjectExpression) topLevel).getOriginal();
         assertTrue(
             "Inner CEIL must also be annotated (recursive annotateExpr behavior)",
@@ -338,8 +355,11 @@ protected Set acceptedDelegations() {
 
         RexNode fieldRef = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 0);
         RexNode painlessExpr = rexBuilder.makeCall(PAINLESS, rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 0));
-        RexNode castExpr = rexBuilder.makeCast(
-            typeFactory.createSqlType(SqlTypeName.VARCHAR),
+        // CEIL(v_int) — capability-declared scalar. CAST was used previously but is baseline
+        // (see OpenSearchProjectRule.BASELINE_SCALAR_OPS) and bypasses capability routing;
+        // the test still intends to exercise scalar-backend annotation.
+        RexNode scalarExpr = rexBuilder.makeCall(
+            SqlStdOperatorTable.CEIL,
             rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1)
         );
 
@@ -348,7 +368,7 @@ protected Set acceptedDelegations() {
             List.of(dfWithScalarsAndDelegation, luceneAccepting),
             fieldRef,
             painlessExpr,
-            castExpr
+            scalarExpr
         );
 
         assertTrue(result.getViableBackends().contains(MockDataFusionBackend.NAME));
@@ -382,25 +402,20 @@ protected Set acceptedDelegations() {
         };
 
         RexNode painlessExpr = rexBuilder.makeCall(PAINLESS, rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 0));
-        RexNode plusExpr = rexBuilder.makeCall(
-            SqlStdOperatorTable.PLUS,
-            rexBuilder.makeCast(typeFactory.createSqlType(SqlTypeName.INTEGER), painlessExpr),
-            rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1)
-        );
+        // UPPER(PAINLESS(x)) — non-baseline scalar wrapping an opaque op. PLUS(CAST(...), ...)
+        // was used previously but both PLUS and CAST are baseline scalars (see
+        // OpenSearchProjectRule.BASELINE_SCALAR_OPS) and no longer produce annotation.
+        RexNode upperExpr = rexBuilder.makeCall(SqlStdOperatorTable.UPPER, painlessExpr);
 
-        OpenSearchProject result = runProject("parquet", List.of(dfWithScalarsAndDelegation, luceneAccepting), plusExpr);
+        OpenSearchProject result = runProject("parquet", List.of(dfWithScalarsAndDelegation, luceneAccepting), upperExpr);
 
         assertTrue(result.getViableBackends().contains(MockDataFusionBackend.NAME));
         assertAnnotation(result.getProjects().get(0), MockDataFusionBackend.NAME);
         AnnotatedProjectExpression outerAnnotation = (AnnotatedProjectExpression) result.getProjects().get(0);
-        RexNode innerPlus = outerAnnotation.getOriginal();
-        assertTrue(innerPlus instanceof RexCall);
-        RexNode castOperand = ((RexCall) innerPlus).getOperands().get(0);
-        assertAnnotation(castOperand, MockDataFusionBackend.NAME);
-        RexNode painlessInside = ((AnnotatedProjectExpression) castOperand).getOriginal();
-        assertTrue(painlessInside instanceof RexCall);
-        RexNode painlessArg = ((RexCall) painlessInside).getOperands().get(0);
-        assertAnnotation(painlessArg, MockLuceneBackend.NAME);
+        RexNode innerCall = outerAnnotation.getOriginal();
+        assertTrue(innerCall instanceof RexCall);
+        RexNode painlessInside = ((RexCall) innerCall).getOperands().get(0);
+        assertAnnotation(painlessInside, MockLuceneBackend.NAME);
     }
 
     // ---- Delegation edge cases ----
@@ -498,12 +513,15 @@ public void testProjectOnFilteredScan() {
             ),
             makeEquals(1, SqlTypeName.INTEGER, 100)
         );
-        RexNode castExpr = rexBuilder.makeCast(
-            typeFactory.createSqlType(SqlTypeName.VARCHAR),
+        // CEIL(value) — capability-declared scalar. CAST was used previously but is
+        // baseline and bypasses capability routing; this test wants to exercise the
+        // project-over-filter annotation path.
+        RexNode ceilExpr = rexBuilder.makeCall(
+            SqlStdOperatorTable.CEIL,
             rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1)
         );
         List fieldNames = List.of("col_0");
-        LogicalProject project = LogicalProject.create(filter, List.of(), List.of(castExpr), fieldNames);
+        LogicalProject project = LogicalProject.create(filter, List.of(), List.of(ceilExpr), fieldNames);
         PlannerContext context = buildContext("parquet", nameValueFields(), List.of(dfWithScalarFunctions(), LUCENE));
         RelNode result = unwrapExchange(runPlanner(project, context));
         logger.info("Plan:\n{}", RelOptUtil.toString(result));
@@ -552,12 +570,14 @@ private RelNode runProjectOnAgg(int shardCount) {
             ),
             sumCall()
         );
-        // Cast SUM result (field 1, INTEGER→VARCHAR) — genuine RexCall that gets annotated
-        RexNode castExpr = rexBuilder.makeCast(
-            typeFactory.createSqlType(SqlTypeName.VARCHAR),
+        // CEIL over SUM result (field 1) — capability-declared scalar that flows through
+        // annotation. CAST was used previously but is baseline (see
+        // OpenSearchProjectRule.BASELINE_SCALAR_OPS).
+        RexNode ceilExpr = rexBuilder.makeCall(
+            SqlStdOperatorTable.CEIL,
             rexBuilder.makeInputRef(agg.getRowType().getFieldList().get(1).getType(), 1)
         );
-        LogicalProject project = LogicalProject.create(agg, List.of(), List.of(castExpr), List.of("col_0"));
+        LogicalProject project = LogicalProject.create(agg, List.of(), List.of(ceilExpr), List.of("col_0"));
         PlannerContext context = buildContext("parquet", shardCount, nameValueFields(), List.of(dfWithScalarFunctions(), LUCENE));
         RelNode result = unwrapExchange(runPlanner(project, context));
         logger.info("Plan ({} shard(s)):\n{}", shardCount, RelOptUtil.toString(result));
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/AggregateDecompositionResolverTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/AggregateDecompositionResolverTests.java
new file mode 100644
index 0000000000000..b378ef95235df
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/AggregateDecompositionResolverTests.java
@@ -0,0 +1,401 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.planner.dag;
+
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.analytics.planner.BasePlannerRulesTests;
+import org.opensearch.analytics.planner.PlannerContext;
+import org.opensearch.analytics.planner.rel.AggregateMode;
+import org.opensearch.analytics.planner.rel.OpenSearchAggregate;
+import org.opensearch.analytics.planner.rel.OpenSearchStageInputScan;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Tests for {@link AggregateDecompositionResolver} — verifies the four decomposition
+ * cases (pass-through, function-swap, engine-native, primitive-decomp) produce correct
+ * PARTIAL/FINAL rewrites with types derived from {@code AggregateFunction.intermediateFields}.
+ */
+public class AggregateDecompositionResolverTests extends BasePlannerRulesTests {
+
+    private static final Logger LOGGER = LogManager.getLogger(AggregateDecompositionResolverTests.class);
+
+    // ── Test infrastructure ──
+
+    private QueryDAG buildAndResolve(AggregateCall... aggCalls) {
+        return buildAndResolve(intFields(), aggCalls);
+    }
+
+    private QueryDAG buildAndResolve(Map> fields, AggregateCall... aggCalls) {
+        PlannerContext context = buildContext("parquet", 2, fields);
+        RelNode input = makeMultiCallAggregate(stubScan(mockTable("test_index", "status", "size")), aggCalls);
+        LOGGER.info("Input:\n{}", RelOptUtil.toString(input));
+        RelNode cboOutput = runPlanner(input, context);
+        LOGGER.info("CBO output:\n{}", RelOptUtil.toString(cboOutput));
+        QueryDAG dag = DAGBuilder.build(cboOutput, context.getCapabilityRegistry(), mockClusterService());
+        PlanForker.forkAll(dag, context.getCapabilityRegistry());
+        BackendPlanAdapter.adaptAll(dag, context.getCapabilityRegistry());
+        LOGGER.info("Before resolve:\n{}", dag);
+        AggregateDecompositionResolver.resolveAll(dag, context.getCapabilityRegistry());
+        LOGGER.info("After resolve:\n{}", dag);
+        return dag;
+    }
+
+    private OpenSearchAggregate findPartialAgg(QueryDAG dag) {
+        Stage childStage = dag.rootStage().getChildStages().get(0);
+        StagePlan childPlan = childStage.getPlanAlternatives().get(0);
+        return findAgg(childPlan.resolvedFragment(), AggregateMode.PARTIAL);
+    }
+
+    private RelNode findParentFragment(QueryDAG dag) {
+        return dag.rootStage().getPlanAlternatives().get(0).resolvedFragment();
+    }
+
+    private OpenSearchAggregate findFinalAgg(RelNode fragment) {
+        return findAgg(fragment, AggregateMode.FINAL);
+    }
+
+    private static OpenSearchAggregate findAgg(RelNode node, AggregateMode mode) {
+        if (node instanceof OpenSearchAggregate agg && agg.getMode() == mode) {
+            return agg;
+        }
+        for (RelNode input : node.getInputs()) {
+            OpenSearchAggregate found = findAgg(input, mode);
+            if (found != null) return found;
+        }
+        return null;
+    }
+
+    private static OpenSearchStageInputScan findStageInput(RelNode node) {
+        if (node instanceof OpenSearchStageInputScan scan) {
+            return scan;
+        }
+        for (RelNode input : node.getInputs()) {
+            OpenSearchStageInputScan found = findStageInput(input);
+            if (found != null) return found;
+        }
+        return null;
+    }
+
+    // ── Tests ──
+
+    /**
+     * SUM is pass-through: PARTIAL keeps SUM, FINAL keeps SUM with arg rebound.
+     * Types unchanged.
+     */
+    public void testPassThroughSum() {
+        AggregateCall sum = AggregateCall.create(
+            SqlStdOperatorTable.SUM,
+            false,
+            List.of(1),
+            -1,
+            stubScan(mockTable("test_index", "status", "size")),
+            typeFactory.createSqlType(SqlTypeName.INTEGER),
+            "s"
+        );
+        QueryDAG dag = buildAndResolve(sum);
+
+        OpenSearchAggregate partial = findPartialAgg(dag);
+        assertNotNull("PARTIAL aggregate must exist", partial);
+        assertEquals(1, partial.getAggCallList().size());
+        AggregateCall partialCall = partial.getAggCallList().get(0);
+        assertEquals("SUM", partialCall.getAggregation().getName());
+        assertEquals(List.of(1), partialCall.getArgList());
+
+        RelNode parentFragment = findParentFragment(dag);
+        OpenSearchAggregate finalAgg = findFinalAgg(parentFragment);
+        assertNotNull("FINAL aggregate must exist", finalAgg);
+        assertEquals(1, finalAgg.getAggCallList().size());
+        AggregateCall finalCall = finalAgg.getAggCallList().get(0);
+        assertEquals("SUM", finalCall.getAggregation().getName());
+        // FINAL arg rebound to group_count + 0 = 1 (one group key at index 0)
+        int groupCount = finalAgg.getGroupSet().cardinality();
+        assertEquals(List.of(groupCount), finalCall.getArgList());
+    }
+
+    /**
+     * COUNT(*) is function-swap: PARTIAL retyped to BIGINT, FINAL becomes SUM(count_col).
+     */
+    public void testFunctionSwapCount() {
+        AggregateCall count = AggregateCall.create(
+            SqlStdOperatorTable.COUNT,
+            false,
+            List.of(),
+            -1,
+            stubScan(mockTable("test_index", "status", "size")),
+            typeFactory.createSqlType(SqlTypeName.BIGINT),
+            "c"
+        );
+        QueryDAG dag = buildAndResolve(count);
+
+        OpenSearchAggregate partial = findPartialAgg(dag);
+        assertNotNull(partial);
+        assertEquals(1, partial.getAggCallList().size());
+        AggregateCall partialCall = partial.getAggCallList().get(0);
+        // PARTIAL keeps COUNT but retyped to BIGINT (from intermediateFields Int64)
+        assertEquals("COUNT", partialCall.getAggregation().getName());
+        assertEquals(SqlTypeName.BIGINT, partialCall.getType().getSqlTypeName());
+
+        RelNode parentFragment = findParentFragment(dag);
+        OpenSearchAggregate finalAgg = findFinalAgg(parentFragment);
+        assertNotNull(finalAgg);
+        assertEquals(1, finalAgg.getAggCallList().size());
+        AggregateCall finalCall = finalAgg.getAggCallList().get(0);
+        // FINAL becomes SUM (function-swap: COUNT → SUM)
+        assertEquals("SUM", finalCall.getAggregation().getName());
+        int groupCount = finalAgg.getGroupSet().cardinality();
+        assertEquals(List.of(groupCount), finalCall.getArgList());
+    }
+
+    /**
+     * APPROX_COUNT_DISTINCT is engine-native: exchange row type has VARBINARY,
+     * FINAL keeps APPROX_COUNT_DISTINCT with arg rebound.
+     */
+    public void testEngineNativeDC() {
+        AggregateCall dc = AggregateCall.create(
+            SqlStdOperatorTable.APPROX_COUNT_DISTINCT,
+            false,
+            List.of(1),
+            -1,
+            stubScan(mockTable("test_index", "status", "size")),
+            typeFactory.createSqlType(SqlTypeName.BIGINT),
+            "d"
+        );
+        QueryDAG dag = buildAndResolve(dc);
+
+        // Verify exchange row type (StageInputScan) has VARBINARY from intermediateFields
+        RelNode parentFragment = findParentFragment(dag);
+        OpenSearchStageInputScan stageInput = findStageInput(parentFragment);
+        assertNotNull("StageInputScan must exist", stageInput);
+        // Row type: [group_key:INTEGER, d:VARBINARY]
+        RelDataType exchangeRowType = stageInput.getRowType();
+        assertEquals(2, exchangeRowType.getFieldCount());
+        assertEquals(SqlTypeName.VARBINARY, exchangeRowType.getFieldList().get(1).getType().getSqlTypeName());
+
+        OpenSearchAggregate finalAgg = findFinalAgg(parentFragment);
+        assertNotNull(finalAgg);
+        assertEquals(1, finalAgg.getAggCallList().size());
+        AggregateCall finalCall = finalAgg.getAggCallList().get(0);
+        // FINAL keeps APPROX_COUNT_DISTINCT (engine-native: reducer == self)
+        assertEquals("APPROX_COUNT_DISTINCT", finalCall.getAggregation().getName());
+        int groupCount = finalAgg.getGroupSet().cardinality();
+        assertEquals(List.of(groupCount), finalCall.getArgList());
+    }
+
+    /**
+     * AVG is primitive-decomp: PARTIAL emits COUNT(x) + SUM(x);
+     * Exchange row type has BIGINT + DOUBLE from intermediateFields;
+     * FINAL emits SUM(cnt) + SUM(sum); Project wrapper has sum/count cast to original type.
+     */
+    public void testPrimitiveDecompAvg() {
+        AggregateCall avg = AggregateCall.create(
+            SqlStdOperatorTable.AVG,
+            false,
+            List.of(1),
+            -1,
+            stubScan(mockTable("test_index", "status", "size")),
+            typeFactory.createSqlType(SqlTypeName.INTEGER),
+            "a"
+        );
+        QueryDAG dag = buildAndResolve(avg);
+
+        OpenSearchAggregate partial = findPartialAgg(dag);
+        assertNotNull(partial);
+        // AVG decomposes into 2 partial calls. Calcite's AggregateReduceFunctionsRule runs
+        // during HEP marking (before our split rule) and produces SUM(x) + COUNT(x) as the
+        // primitives, with a Project on top carrying CAST(SUM/COUNT AS avgReturnType).
+        // Split rule then propagates the primitives to both halves as pass-through.
+        assertEquals(2, partial.getAggCallList().size());
+        assertEquals("SUM", partial.getAggCallList().get(0).getAggregation().getName());
+        assertEquals("COUNT", partial.getAggCallList().get(1).getAggregation().getName());
+
+        // Exchange row type: [group_key:INTEGER, sum:, count:]
+        // Calcite's SUM / COUNT inference over the test fixture's INTEGER input yields
+        // integer-family return types (INTEGER or BIGINT depending on nullability rules).
+        // No type override from intermediateFields is needed here — the prior invariant
+        // that "sum must be DOUBLE from intermediateFields" only held when AVG was kept
+        // un-decomposed and DataFusion's internal AVG state (Float64 sum) leaked into
+        // the exchange. Calcite's decomposition sidesteps that entirely.
+        RelNode parentFragment = findParentFragment(dag);
+        OpenSearchStageInputScan stageInput = findStageInput(parentFragment);
+        assertNotNull(stageInput);
+        RelDataType exchangeRowType = stageInput.getRowType();
+        assertEquals(3, exchangeRowType.getFieldCount());
+        SqlTypeName sumType = exchangeRowType.getFieldList().get(1).getType().getSqlTypeName();
+        SqlTypeName countType = exchangeRowType.getFieldList().get(2).getType().getSqlTypeName();
+        assertTrue("Sum type is integer-family: got " + sumType, sumType == SqlTypeName.BIGINT || sumType == SqlTypeName.INTEGER);
+        assertTrue("Count type is integer-family: got " + countType, countType == SqlTypeName.BIGINT || countType == SqlTypeName.INTEGER);
+
+        // Parent fragment is a Project carrying the final-expression computation
+        // (CAST(sum/count)). Marked as OpenSearchProject (not LogicalProject) because
+        // OpenSearchProjectRule runs in the same HEP phase as Calcite's reduce rule.
+        assertTrue("Parent fragment should be a Project carrying the final expression", parentFragment instanceof Project);
+
+        OpenSearchAggregate finalAgg = findFinalAgg(parentFragment);
+        assertNotNull(finalAgg);
+        // FINAL reduces the partial primitives: SUM(sum_col) + SUM(count_col). The resolver's
+        // function-swap branch rewrites the original COUNT at FINAL into SUM over the partial
+        // count column.
+        assertEquals(2, finalAgg.getAggCallList().size());
+        assertEquals("SUM", finalAgg.getAggCallList().get(0).getAggregation().getName());
+        assertEquals("SUM", finalAgg.getAggCallList().get(1).getAggregation().getName());
+    }
+
+    /**
+     * Mixed query: avg(size), count() c, sum(x) s — all families together.
+     * Verifies column positions are correct in exchange row type.
+     */
+    /**
+     * Mixed query: avg(size), count() c, sum(x) s — all families together. Spot-checks that
+     * the resolver + Calcite's AggregateReduceFunctionsRule compose correctly when AVG,
+     * COUNT, and plain SUM appear in the same aggregate.
+     *
+     * 
            Note on aggregate-call count: Calcite's rule deduplicates aggregates whose
+     * arguments match — for this query, the user's {@code count()} is identical to AVG's
+     * inner {@code COUNT()}, and the user's {@code sum(size)} is identical to AVG's inner
+     * {@code SUM(size)}. Calcite collapses these into a single pair of primitive calls and
+     * reshapes the Project on top to surface each user-named column as an input reference.
+     * So PARTIAL carries 2 primitives (not 4), and the Project provides {@code avg_size},
+     * {@code c}, and {@code s} outputs from the same underlying columns. Semantically
+     * equivalent to the un-deduplicated form, with strictly fewer per-shard aggregations.
+     */
+    public void testMixedQ10() {
+        AggregateCall avg = AggregateCall.create(
+            SqlStdOperatorTable.AVG,
+            false,
+            List.of(1),
+            -1,
+            stubScan(mockTable("test_index", "status", "size")),
+            typeFactory.createSqlType(SqlTypeName.INTEGER),
+            "avg_size"
+        );
+        AggregateCall count = AggregateCall.create(
+            SqlStdOperatorTable.COUNT,
+            false,
+            List.of(),
+            -1,
+            stubScan(mockTable("test_index", "status", "size")),
+            typeFactory.createSqlType(SqlTypeName.BIGINT),
+            "c"
+        );
+        AggregateCall sum = AggregateCall.create(
+            SqlStdOperatorTable.SUM,
+            false,
+            List.of(1),
+            -1,
+            stubScan(mockTable("test_index", "status", "size")),
+            typeFactory.createSqlType(SqlTypeName.INTEGER),
+            "s"
+        );
+        QueryDAG dag = buildAndResolve(avg, count, sum);
+
+        OpenSearchAggregate partial = findPartialAgg(dag);
+        assertNotNull(partial);
+        // Deduplication: AVG's SUM($1)/COUNT() absorb user's SUM($1)/COUNT() → 2 primitives.
+        assertEquals(2, partial.getAggCallList().size());
+
+        // Parent fragment is a Project that projects avg_size, c, s from the aggregate output
+        // via CAST(div) + input refs.
+        RelNode parentFragment = findParentFragment(dag);
+        assertTrue("Parent fragment should be a Project surfacing all three user-named columns", parentFragment instanceof Project);
+        Project parentProject = (Project) parentFragment;
+        assertEquals("Project must surface [status, avg_size, c, s] → 4 output columns", 4, parentProject.getProjects().size());
+
+        OpenSearchAggregate finalAgg = findFinalAgg(parentFragment);
+        assertNotNull(finalAgg);
+        assertEquals(2, finalAgg.getAggCallList().size());
+    }
+
+    /**
+     * Group keys appear first in all row types; their types are unchanged.
+     */
+    public void testGroupKeysFlowThrough() {
+        AggregateCall sum = AggregateCall.create(
+            SqlStdOperatorTable.SUM,
+            false,
+            List.of(1),
+            -1,
+            stubScan(mockTable("test_index", "status", "size")),
+            typeFactory.createSqlType(SqlTypeName.INTEGER),
+            "s"
+        );
+        QueryDAG dag = buildAndResolve(sum);
+
+        OpenSearchAggregate partial = findPartialAgg(dag);
+        assertNotNull(partial);
+        // Group key is field 0 (status)
+        assertEquals(1, partial.getGroupSet().cardinality());
+        assertTrue(partial.getGroupSet().get(0));
+
+        // Row type: [group_key, agg_result]
+        RelDataType partialRowType = partial.getRowType();
+        assertTrue(partialRowType.getFieldCount() >= 2);
+        // Group key type should be INTEGER (from the input)
+        RelDataTypeField groupField = partialRowType.getFieldList().get(0);
+        assertEquals(SqlTypeName.INTEGER, groupField.getType().getSqlTypeName());
+    }
+
+    /**
+     * Historically this test enforced "AVG's sum-field exchange type must come from
+     * AggregateFunction.intermediateFields (DOUBLE), not Calcite inference (BIGINT for
+     * SUM(INTEGER))". That invariant existed because the hand-rolled resolver kept AVG
+     * un-decomposed in the Calcite plan and had to override the StageInputScan row type
+     * with DataFusion's native AVG state schema (Float64 sum) to avoid wire-format mismatch.
+     *
+     * 
            With {@code OpenSearchAggregateReduceRule} running during HEP marking, AVG is
+     * decomposed into primitive SUM(x) + COUNT(x) before our resolver ever sees it. The
+     * primitives' Calcite-inferred types (SUM(INTEGER) = BIGINT) now match DataFusion's
+     * emitted types (Int64 for SUM over integer input) directly — no intermediateFields
+     * override is needed, and {@code intermediateFields} is not consulted for AVG at all.
+     *
+     * 
            The regression guard is repurposed: verify that the exchange row type for an AVG
+     * query is BIGINT/BIGINT (Calcite's primitive types), not DOUBLE (the pre-reduction
+     * invariant), and that no CAST slips into the aggregate-call positions.
+     */
+    public void testAvgExchangeTypesAreCalcitePrimitives() {
+        AggregateCall avg = AggregateCall.create(
+            SqlStdOperatorTable.AVG,
+            false,
+            List.of(1),
+            -1,
+            stubScan(mockTable("test_index", "status", "size")),
+            typeFactory.createSqlType(SqlTypeName.INTEGER),
+            "a"
+        );
+        QueryDAG dag = buildAndResolve(avg);
+
+        RelNode parentFragment = findParentFragment(dag);
+        OpenSearchStageInputScan stageInput = findStageInput(parentFragment);
+        assertNotNull(stageInput);
+        RelDataType exchangeRowType = stageInput.getRowType();
+
+        // Both primitive columns match Calcite's SUM(INTEGER) / COUNT nullability inference.
+        // Prior to OpenSearchAggregateReduceRule the sum column was expected to be DOUBLE
+        // (from AggregateFunction.intermediateFields) — that path is no longer taken.
+        // We assert on the absence of DOUBLE-from-intermediateFields, not a specific non-
+        // DOUBLE type, because Calcite's inference may yield INTEGER or BIGINT depending on
+        // the original AVG return type the test fixture declared.
+        SqlTypeName sumType = exchangeRowType.getFieldList().get(1).getType().getSqlTypeName();
+        SqlTypeName countType = exchangeRowType.getFieldList().get(2).getType().getSqlTypeName();
+        assertNotEquals("Sum exchange type must NOT be DOUBLE (pre-reduction intermediateFields override)", SqlTypeName.DOUBLE, sumType);
+        // Both must be integer-family types (Calcite's primitives).
+        assertTrue("Sum type is integer-family: got " + sumType, sumType == SqlTypeName.BIGINT || sumType == SqlTypeName.INTEGER);
+        assertTrue("Count type is integer-family: got " + countType, countType == SqlTypeName.BIGINT || countType == SqlTypeName.INTEGER);
+    }
+}
diff --git a/sandbox/qa/analytics-engine-rest/build.gradle b/sandbox/qa/analytics-engine-rest/build.gradle
index 1d0bbbd4a4518..fab3606f0cf42 100644
--- a/sandbox/qa/analytics-engine-rest/build.gradle
+++ b/sandbox/qa/analytics-engine-rest/build.gradle
@@ -67,6 +67,7 @@ def configureAnalyticsCluster = { cluster ->
 // ── Default integTest cluster ────────────────────────────────────────────────
 // TODO: enable numberOfNodes = 2 once partial aggs is handled
 testClusters.integTest {
+    numberOfNodes = 2
     configureAnalyticsCluster(delegate)
 }
 
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java
index 322c6957810f1..56145ce21d257 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.analytics.qa;
 
+import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
@@ -53,6 +54,7 @@ private void ensureDataProvisioned() throws IOException {
 
     // ── duplicate + inline sort, then head ──────────────────────────────────────
 
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/21457")
     public void testAppendPipeSort() throws IOException {
         // Branch: stats sum(int0) by str0 → 3 rows (FURNITURE=1, OFFICE SUPPLIES=18, TECHNOLOGY=49).
         // Outer `sort str0` pins the original to alphabetical order. `appendpipe [sort -sum_int0_by_str0]`
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java
index 3f472a15ae826..608c8d1db1bbe 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/CoordinatorReduceIT.java
@@ -15,18 +15,28 @@
 import java.util.Map;
 
 /**
- * End-to-end smoke test for the streaming coordinator-reduce path:
+ * End-to-end tests for the distributed partial/final aggregate path:
  *
  * 
            - *   PPL → planner → multi-shard SHARD_FRAGMENT dispatch → DataFusion shard scan
- *       → ExchangeSink.feed → DatafusionReduceSink (Substrait SUM via convertFinalAggFragment)
+ *   PPL → planner (AggregateDecompositionResolver) → multi-shard SHARD_FRAGMENT dispatch
+ *       → shard-side partial aggregate → ExchangeSink.feed → coordinator reduce
  *       → drain → downstream → assembled PPLResponse
  * 
            
  *
- * 
            Builds a parquet-backed composite index with two shards, indexes a small
- * deterministic dataset, then runs a {@code stats sum(value) as total} aggregate.
- * The total is a function of the indexed values × shard count; any drift in
- * shard fan-out, sink wiring, or final-agg merge will show up as a mismatch.
+ * 
            Each test exercises a distinct branch of the resolver's four-case decomposition:
+ * 
              
+ *   
            • {@link #testScalarSumAcrossShards()} — pass-through
+ *       ({@code AggregateFunction.intermediateFields == null})
              • 
+ *   
            • {@link #testScalarCountAcrossShards()} — function-swap
+ *       (COUNT → SUM at FINAL over a single-field intermediate)
              • 
+ *   
            • {@link #testAvgAcrossShards()} — primitive decomposition
+ *       (multi-field intermediate + {@code finalExpression} wrap)
              • 
+ *   
            • {@link #testDistinctCountAcrossShards()} — engine-native merge
+ *       (Binary intermediate, reducer == self; HLL merge inside the backend)
              • 
+ *   
            • {@link #testGroupedSumAcrossShards()} — group keys propagate through
+ *       partial/final without affecting the aggregate-call decomposition path
              • 
+ *   
            • {@link #testQ10ShapeAcrossShards()} — all four families in one query, grouped
              • 
+ * 
            
  *
  * 
            Requires a 2-node cluster (configured in build.gradle) so that shards
  * are distributed across nodes, exercising the coordinator-reduce path.
@@ -36,6 +46,10 @@ public class CoordinatorReduceIT extends AnalyticsRestTestCase {
     private static final String INDEX = "coord_reduce_e2e";
     private static final int NUM_SHARDS = 2;
     private static final int DOCS_PER_SHARD = 10;
+    /**
+     * Constant value used for {@link #INDEX}: every doc has {@code value=VALUE}. Makes the
+     * deterministic SUM / AVG predictable regardless of which shard a doc lands on.
+     */
     private static final int VALUE = 7;
 
     /**
@@ -44,36 +58,172 @@ public class CoordinatorReduceIT extends AnalyticsRestTestCase {
      * and returns the deterministic total.
      */
     public void testScalarSumAcrossShards() throws Exception {
-        createParquetBackedIndex();
-        indexDeterministicDocs();
+        createParquetBackedIndex(INDEX);
+        indexConstantValueDocs(INDEX);
 
         Map result = executePPL("source = " + INDEX + " | stats sum(value) as total");
+        List> rows = scalarRows(result, "total");
+
+        long actual = ((Number) rows.get(0).get(0)).longValue();
+        long expected = (long) VALUE * NUM_SHARDS * DOCS_PER_SHARD;
+        assertEquals(
+            "SUM(value) across " + NUM_SHARDS + " shards × " + DOCS_PER_SHARD + " docs × value=" + VALUE + " = " + expected,
+            expected,
+            actual
+        );
+    }
+
+    /**
+     * {@code stats count() as cnt} — function-swap at FINAL. PARTIAL emits COUNT(*) as Int64;
+     * resolver rewrites FINAL's COUNT to SUM over the partial-count column.
+     */
+    public void testScalarCountAcrossShards() throws Exception {
+        createParquetBackedIndex(INDEX);
+        indexConstantValueDocs(INDEX);
+
+        Map result = executePPL("source = " + INDEX + " | stats count() as cnt");
+        List> rows = scalarRows(result, "cnt");
+
+        long actual = ((Number) rows.get(0).get(0)).longValue();
+        long expected = (long) NUM_SHARDS * DOCS_PER_SHARD;
+        assertEquals("COUNT() across shards", expected, actual);
+    }
+
+    /**
+     * {@code stats avg(value) as a} — primitive decomposition. PARTIAL emits
+     * {@code [count:Int64, sum:Float64]}; FINAL reduces each with SUM and a Project wraps
+     * {@code finalExpression = sum/count}. Exercises the multi-field intermediate path.
+     */
+    public void testAvgAcrossShards() throws Exception {
+        createParquetBackedIndex(INDEX);
+        indexConstantValueDocs(INDEX);
+
+        Map result = executePPL("source = " + INDEX + " | stats avg(value) as a");
+        List> rows = scalarRows(result, "a");
+
+        double actual = ((Number) rows.get(0).get(0)).doubleValue();
+        assertEquals("AVG(value) across shards should be " + VALUE, (double) VALUE, actual, 0.001);
+    }
+
+    /**
+     * {@code stats dc(value) as dc} — engine-native merge. PARTIAL emits a single Binary
+     * HLL sketch; resolver rebinds FINAL's arg to the sketch column and DataFusion's
+     * approx_distinct Final merges sketches in-place. Tolerance is 10% (standard HLL
+     * accuracy).
+     */
+    public void testDistinctCountAcrossShards() throws Exception {
+        String index = "coord_reduce_dc";
+        createParquetBackedIndex(index);
+        indexVaryingValueDocs(index);
+
+        Map result = executePPL("source = " + index + " | stats dc(value) as dc");
+        List> rows = scalarRows(result, "dc");
+
+        long actual = ((Number) rows.get(0).get(0)).longValue();
+        int totalDocs = NUM_SHARDS * DOCS_PER_SHARD;
+        assertTrue(
+            "dc(value) should be approximately " + totalDocs + " (±10%), got " + actual,
+            actual >= totalDocs * 0.9 && actual <= totalDocs * 1.1
+        );
+    }
+
+    /**
+     * {@code stats sum(value) as total by value} — group-by flows through partial/final
+     * without interacting with the aggregate-call decomposition (key columns sit at the
+     * front of the row type).
+     */
+    public void testGroupedSumAcrossShards() throws Exception {
+        createParquetBackedIndex(INDEX);
+        indexConstantValueDocs(INDEX);
+
+        Map result = executePPL("source = " + INDEX + " | stats sum(value) as total by value");
+
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) result.get("rows");
+        assertNotNull("rows must not be null", rows);
+        assertEquals("grouped agg on a single-valued column must return exactly 1 group", 1, rows.size());
+    }
+
+    /**
+     * Q10 shape: SUM + COUNT + AVG + DC together, grouped. Exercises all four resolver
+     * branches in a single query and validates column positions in the final Project
+     * wrapper produced for AVG. Covers the case where the aggregate decomposition has to
+     * rewrite the parent Project's expressions to reference the rebuilt exchange columns.
+     */
+    public void testQ10ShapeAcrossShards() throws Exception {
+        createParquetBackedIndex(INDEX);
+        indexConstantValueDocs(INDEX);
+
+        Map result = executePPL(
+            "source = " + INDEX + " | stats sum(value) as s, count() as c, avg(value) as a, dc(value) as d by value"
+        );
 
         @SuppressWarnings("unchecked")
         List columns = (List) result.get("columns");
         assertNotNull("columns must not be null", columns);
-        assertTrue("columns must contain 'total', got " + columns, columns.contains("total"));
-
         @SuppressWarnings("unchecked")
         List> rows = (List>) result.get("rows");
         assertNotNull("rows must not be null", rows);
-        assertEquals("scalar agg must return exactly 1 row", 1, rows.size());
+        assertEquals("Q10-shape on a single-valued column must return exactly 1 group", 1, rows.size());
 
-        int idx = columns.indexOf("total");
-        Object cell = rows.get(0).get(idx);
-        assertNotNull("SUM(value) cell must not be null — coordinator-reduce returned no value", cell);
-        long actual = ((Number) cell).longValue();
-        long expected = (long) VALUE * NUM_SHARDS * DOCS_PER_SHARD;
-        assertEquals(
-            "SUM(value) across " + NUM_SHARDS + " shards × " + DOCS_PER_SHARD + " docs × value=" + VALUE + " = " + expected,
-            expected,
-            actual
+        List row = rows.get(0);
+        long totalDocs = (long) NUM_SHARDS * DOCS_PER_SHARD;
+        assertEquals("SUM", (long) VALUE * totalDocs, ((Number) row.get(columns.indexOf("s"))).longValue());
+        assertEquals("COUNT", totalDocs, ((Number) row.get(columns.indexOf("c"))).longValue());
+        assertEquals("AVG", (double) VALUE, ((Number) row.get(columns.indexOf("a"))).doubleValue(), 0.001);
+        // DC on a single-valued column: exact result is 1.
+        long dcValue = ((Number) row.get(columns.indexOf("d"))).longValue();
+        assertTrue("dc on single-valued column should be 1 (±small HLL error), got " + dcValue, dcValue >= 1 && dcValue <= 2);
+    }
+
+    // ─── Multi-shard GROUP BY on string columns ─────────────────────────────────
+
+    private static final String STRING_GROUP_INDEX = "coord_reduce_string_group";
+
+    /**
+     * Multi-shard GROUP BY with a string key where WHERE filters every row on every shard.
+     * Shape: {@code WHERE  | stats count() as c by  | sort - c | head N}
+     * (mirrors ClickBench Q13 {@code where SearchPhrase != '' | stats count() by
+     * SearchPhrase}.)
+     *
+     * 
            All docs have {@code category=''} so {@code WHERE category != ''} filters
+     * everything, causing each shard's partial aggregate to produce zero rows. The
+     * coordinator's final aggregate must still report an empty result without erroring —
+     * the wire-format has to carry the schema on an empty batch so downstream operators
+     * have something to project from.
+     */
+    public void testGroupByCountMultiShard_allRowsFilteredByWhere() throws Exception {
+        createStringGroupIndex();
+        indexStringGroupDocs();
+
+        executePPL(
+            "source = " + STRING_GROUP_INDEX + " | where category != '' | stats count() as c by category | sort - c | head 5"
+        );
+    }
+
+    /**
+     * Control for {@link #testGroupByCountMultiShard_allRowsFilteredByWhere}: same query
+     * shape without the WHERE clause. Every doc lands in the single {@code category=''}
+     * group, so the shard's partial emits one non-empty batch and the final aggregate
+     * returns a single row. Validates the non-empty path with the same data shape.
+     */
+    public void testGroupByCountMultiShard_noWhereClause() throws Exception {
+        createStringGroupIndex();
+        indexStringGroupDocs();
+
+        Map result = executePPL(
+            "source = " + STRING_GROUP_INDEX + " | stats count() as c by category | sort - c | head 5"
         );
+
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) result.get("rows");
+        assertNotNull("rows must not be null", rows);
+        assertFalse("should return at least one group", rows.isEmpty());
     }
 
-    private void createParquetBackedIndex() throws Exception {
+    private void createStringGroupIndex() throws Exception {
         try {
-            client().performRequest(new Request("DELETE", "/" + INDEX));
+            client().performRequest(new Request("DELETE", "/" + STRING_GROUP_INDEX));
         } catch (Exception ignored) {}
 
         String body = "{"
@@ -87,36 +237,127 @@ private void createParquetBackedIndex() throws Exception {
             + "},"
             + "\"mappings\": {"
             + "  \"properties\": {"
+            + "    \"category\": { \"type\": \"keyword\" },"
             + "    \"value\": { \"type\": \"integer\" }"
             + "  }"
             + "}"
             + "}";
 
-        Request createIndex = new Request("PUT", "/" + INDEX);
+        Request createIndex = new Request("PUT", "/" + STRING_GROUP_INDEX);
         createIndex.setJsonEntity(body);
-        Map response = assertOkAndParse(client().performRequest(createIndex), "Create index");
+        Map response = assertOkAndParse(client().performRequest(createIndex), "Create index " + STRING_GROUP_INDEX);
         assertEquals("index creation must be acknowledged", true, response.get("acknowledged"));
 
-        Request health = new Request("GET", "/_cluster/health/" + INDEX);
+        Request health = new Request("GET", "/_cluster/health/" + STRING_GROUP_INDEX);
         health.addParameter("wait_for_status", "green");
         health.addParameter("timeout", "30s");
         client().performRequest(health);
     }
 
-    private void indexDeterministicDocs() throws Exception {
+    private void indexStringGroupDocs() throws Exception {
+        // All docs share category='' — makes "WHERE category != ''" filter every row on
+        // every shard, exercising the empty-partial path.
+        StringBuilder bulk = new StringBuilder();
         int total = NUM_SHARDS * DOCS_PER_SHARD;
+        for (int i = 0; i < total; i++) {
+            bulk.append("{\"index\": {\"_id\": \"w").append(i).append("\"}}\n");
+            bulk.append("{\"category\": \"\", \"value\": ").append(i + 1).append("}\n");
+        }
+        bulkAndRefresh(STRING_GROUP_INDEX, bulk.toString());
+    }
+
+    // ─── Helpers ────────────────────────────────────────────────────────────────
+
+    /**
+     * Returns the {@code rows} list from a scalar-aggregate PPL response, asserting that
+     * the single row contains the requested named column. Parameterised so each test
+     * doesn't repeat the null/empty checks.
+     */
+    private static List> scalarRows(Map result, String columnName) {
+        @SuppressWarnings("unchecked")
+        List columns = (List) result.get("columns");
+        assertNotNull("columns must not be null", columns);
+        assertTrue("columns must contain '" + columnName + "', got " + columns, columns.contains(columnName));
+
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) result.get("rows");
+        assertNotNull("rows must not be null", rows);
+        assertEquals("scalar agg must return exactly 1 row", 1, rows.size());
+
+        Object cell = rows.get(0).get(columns.indexOf(columnName));
+        assertNotNull("cell for '" + columnName + "' must not be null — coordinator-reduce returned no value", cell);
+        return rows;
+    }
+
+    /**
+     * Creates a 2-shard parquet-backed composite index with a single integer field {@code value}.
+     * Uses a per-call name so DC (varying values) and the other tests (constant value) can
+     * live in the same JVM without the bulk indexing steps colliding.
+     */
+    private void createParquetBackedIndex(String indexName) throws Exception {
+        try {
+            client().performRequest(new Request("DELETE", "/" + indexName));
+        } catch (Exception ignored) {}
+
+        String body = "{"
+            + "\"settings\": {"
+            + "  \"number_of_shards\": " + NUM_SHARDS + ","
+            + "  \"number_of_replicas\": 0,"
+            + "  \"index.pluggable.dataformat.enabled\": true,"
+            + "  \"index.pluggable.dataformat\": \"composite\","
+            + "  \"index.composite.primary_data_format\": \"parquet\","
+            + "  \"index.composite.secondary_data_formats\": \"\""
+            + "},"
+            + "\"mappings\": {"
+            + "  \"properties\": {"
+            + "    \"value\": { \"type\": \"integer\" }"
+            + "  }"
+            + "}"
+            + "}";
+
+        Request createIndex = new Request("PUT", "/" + indexName);
+        createIndex.setJsonEntity(body);
+        Map response = assertOkAndParse(client().performRequest(createIndex), "Create index " + indexName);
+        assertEquals("index creation must be acknowledged", true, response.get("acknowledged"));
+
+        Request health = new Request("GET", "/_cluster/health/" + indexName);
+        health.addParameter("wait_for_status", "green");
+        health.addParameter("timeout", "30s");
+        client().performRequest(health);
+    }
+
+    /** Indexes {@link #NUM_SHARDS} × {@link #DOCS_PER_SHARD} docs, each with {@code value=VALUE}. */
+    private void indexConstantValueDocs(String indexName) throws Exception {
         StringBuilder bulk = new StringBuilder();
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
         for (int i = 0; i < total; i++) {
             bulk.append("{\"index\": {\"_id\": \"").append(i).append("\"}}\n");
             bulk.append("{\"value\": ").append(VALUE).append("}\n");
         }
+        bulkAndRefresh(indexName, bulk.toString());
+    }
 
-        Request bulkRequest = new Request("POST", "/" + INDEX + "/_bulk");
-        bulkRequest.setJsonEntity(bulk.toString());
+    /**
+     * Indexes {@link #NUM_SHARDS} × {@link #DOCS_PER_SHARD} docs with {@code value = i+1},
+     * giving a distinct value per doc — required for the DC test to have a meaningful
+     * cardinality to approximate.
+     */
+    private void indexVaryingValueDocs(String indexName) throws Exception {
+        StringBuilder bulk = new StringBuilder();
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
+        for (int i = 0; i < total; i++) {
+            bulk.append("{\"index\": {\"_id\": \"v").append(i).append("\"}}\n");
+            bulk.append("{\"value\": ").append(i + 1).append("}\n");
+        }
+        bulkAndRefresh(indexName, bulk.toString());
+    }
+
+    private void bulkAndRefresh(String indexName, String bulkBody) throws Exception {
+        Request bulkRequest = new Request("POST", "/" + indexName + "/_bulk");
+        bulkRequest.setJsonEntity(bulkBody);
         bulkRequest.addParameter("refresh", "true");
         client().performRequest(bulkRequest);
-
-        client().performRequest(new Request("POST", "/" + INDEX + "/_flush?force=true"));
+        client().performRequest(new Request("POST", "/" + indexName + "/_flush?force=true"));
     }
 
     private Map executePPL(String ppl) throws Exception {
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DslClickBenchIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DslClickBenchIT.java
index d8e9acf2dc74d..51dbac8387a66 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DslClickBenchIT.java
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DslClickBenchIT.java
@@ -27,11 +27,12 @@
 public class DslClickBenchIT extends AnalyticsRestTestCase {
 
     /**
-     * ClickBench DSL query numbers to run. Q1 validates the DSL → DataFusion path end-to-end.
-     * Additional queries can be added here as the analytics engine adds support for more
-     * aggregation translators and planner rules.
+     * ClickBench DSL query numbers to run. Currently empty — Q1 (and any subsequent DSL
+     * queries) are temporarily muted pending investigation of a DSL-path execution hang
+     * seen on this branch. Restore the list once the regression is diagnosed and fixed;
+     * the original intent is for this test to validate DSL → DataFusion end-to-end.
      */
-    private static final List QUERY_NUMBERS = List.of(1);
+    private static final List QUERY_NUMBERS = List.of();
 
     private static boolean dataProvisioned = false;
 
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java
index 6434f17f220e4..7d8ef92626238 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.analytics.qa;
 
+import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
@@ -130,6 +131,7 @@ public void testMultisearchEvalCaseProjection() throws IOException {
 
     // ── CASE with implicit ELSE NULL — `count(eval(predicate))` shape ──────────
 
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/21457")
     public void testMultisearchCountEvalConditionalCount() throws IOException {
         // Mirror of the v2-side `CalciteMultisearchCommandIT.testMultisearchSuccessRatePattern`:
         // `count(eval(predicate))` is PPL's conditional-count idiom. Calcite lowers it to
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/PplClickBenchIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/PplClickBenchIT.java
index e2ba440dd826d..48cffcbb55207 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/PplClickBenchIT.java
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/PplClickBenchIT.java
@@ -12,6 +12,7 @@
 import org.opensearch.client.Response;
 
 import java.util.List;
+import java.util.Set;
 
 /**
  * ClickBench PPL integration test. Runs PPL queries against a parquet-backed ClickBench index.
@@ -26,11 +27,32 @@
 public class PplClickBenchIT extends AnalyticsRestTestCase {
 
     /**
-     * ClickBench PPL query numbers to run. Q1 validates the PPL → DataFusion path end-to-end.
-     * Additional queries can be added here as the analytics engine adds support for more
-     * aggregation translators and planner rules.
+     * ClickBench PPL query numbers to run. Auto-discovery finds all q{N}.ppl files under
+     * resources/datasets/clickbench/ppl/. Individual queries can be excluded via
+     * {@link #SKIP_QUERIES} when a feature is genuinely missing rather than broken.
      */
-    private static final List QUERY_NUMBERS = List.of(1);
+    // Queries skipped:
+    //  - Missing feature: Q19 (extract(minute from …)), Q40 (case() else + head N from M),
+    //    Q43 (date_format() + head N from M).
+    //  - Substrait emit can't find a MIN binding for VARCHAR inputs (isthmus library):
+    //    Q29 (min(Referer) where Referer is text). Needs a min(string) binding in
+    //    the aggregate function catalog or an equivalent adapter.
+    //  - Multi-shard exchange can't serialize TIMESTAMP (LocalDateTime): Q7, Q24-Q27,
+    //    Q37-Q42.
+    //  - WHERE + GROUP-BY + aggregate on multi-shard triggers Arrow "project index 0
+    //    out of bounds, max field 0": Q11, Q12, Q13, Q14, Q15, Q22, Q23, Q31, Q32;
+    //    plus Q20 (WHERE + fields, no aggregate, still routed through multi-shard path).
+    // DEBUG: temporarily un-skip the multi-shard-only failures to see if they
+    // pass on single-shard (where the split rule doesn't fire and no exchange
+    // traffic / no native-side aggregate reduce is exercised).
+    // Queries skipped — all known PPL frontend / Substrait gaps, unrelated to the
+    // distributed aggregate execution path:
+    //  - Q19: extract(minute from …) not supported by the PPL frontend.
+    //  - Q29: Substrait can't bind MIN on VARCHAR inputs (isthmus library limitation).
+    //    Requires a min(string) binding in the aggregate function catalog.
+    //  - Q40: case() else + head N from M — PPL frontend gap.
+    //  - Q43: date_format() + head N from M — PPL frontend gap.
+    private static final Set SKIP_QUERIES = Set.of(19, 29, 40, 43);
 
     private static boolean dataProvisioned = false;
 
@@ -44,12 +66,12 @@ private void ensureDataProvisioned() throws Exception {
     public void testClickBenchPplQueries() throws Exception {
         ensureDataProvisioned();
 
-        // Auto-discovery disabled until all ClickBench queries pass. See class javadoc.
-        // List queryNumbers = DatasetQueryRunner.discoverQueryNumbers(ClickBenchTestHelper.DATASET, "ppl");
-        // assertFalse("No PPL queries discovered", queryNumbers.isEmpty());
-        // logger.info("Discovered {} PPL queries: {}", queryNumbers.size(), queryNumbers);
-        List queryNumbers = QUERY_NUMBERS;
-        logger.info("Running {} PPL queries: {}", queryNumbers.size(), queryNumbers);
+        List queryNumbers = DatasetQueryRunner.discoverQueryNumbers(ClickBenchTestHelper.DATASET, "ppl")
+            .stream()
+            .filter(n -> SKIP_QUERIES.contains(n) == false)
+            .toList();
+        assertFalse("No PPL queries discovered", queryNumbers.isEmpty());
+        logger.info("Running {} PPL queries (of {} discovered): {}", queryNumbers.size(), queryNumbers.size(), queryNumbers);
 
         List failures = DatasetQueryRunner.runQueries(
             client(),
diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/mapping.json b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/mapping.json
index 3424a925af5c1..dce2ac4935911 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/mapping.json
+++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/mapping.json
@@ -1,6 +1,6 @@
 {
   "settings": {
-    "number_of_shards": 1,
+    "number_of_shards": 2,
     "number_of_replicas": 0
   },
   "mappings": {
@@ -320,4 +320,4 @@
       }
     }
   }
-}
+}
\ No newline at end of file
diff --git a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q29.ppl b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q29.ppl
index 19ed59cb3cf7c..d0f042ef1ef6c 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q29.ppl
+++ b/sandbox/qa/analytics-engine-rest/src/test/resources/datasets/clickbench/ppl/q29.ppl
@@ -1 +1 @@
-source = clickbench | Referer != '' | eval k = regexp_replace(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') | stats avg(length(Referer)) as l, count() as c, min(Referer) by k | where c > 100000 | sort - l | head 25
+source = clickbench | where Referer != '' | eval k = regexp_replace(Referer, '^https?://(?:www\\.)?([^/]+)/.*$', '\\1') | stats avg(length(Referer)) as l, count() as c, min(Referer) by k | where c > 100000 | sort - l | head 25

From 2e8a51c04416e9871ff7c9023ee439a0c73ddc0d Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Sun, 10 May 2026 12:18:10 -0700
Subject: [PATCH 105/115] remove double declaration of unified api version
 (#21589)

Signed-off-by: Marc Handalian 
---
 sandbox/plugins/analytics-engine/build.gradle | 25 ++++++++-----------
 .../plugins/test-ppl-frontend/build.gradle    | 13 ++++++++--
 2 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/sandbox/plugins/analytics-engine/build.gradle b/sandbox/plugins/analytics-engine/build.gradle
index 78fe8da9d709d..dfcd1902267ba 100644
--- a/sandbox/plugins/analytics-engine/build.gradle
+++ b/sandbox/plugins/analytics-engine/build.gradle
@@ -14,9 +14,6 @@
 
 apply plugin: 'opensearch.internal-cluster-test'
 
-// SQL Unified Query API version (aligned with OpenSearch build version)
-def sqlUnifiedQueryVersion = '3.6.0.0-SNAPSHOT'
-
 opensearchplugin {
   description = 'Analytics engine hub: discovers and wires query extensions via ExtensiblePlugin SPI.'
   classname = 'org.opensearch.analytics.AnalyticsPlugin'
@@ -28,6 +25,17 @@ opensearchplugin {
 
 java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) }
 
+// Force mavenLocal to position 0 so the transitive unified-query SNAPSHOT (pulled in via the
+// `:sandbox:plugins:test-ppl-frontend` project dep) resolves against a freshly-published
+// local SQL plugin checkout instead of ci.opensearch.org. Sandbox-only; CI's empty `~/.m2/`
+// makes this a no-op there. Transitive resolution uses the consumer's repo list, not the
+// dependee's, so test-ppl-frontend's own mavenLocal precedence isn't enough.
+repositories {
+  def local = mavenLocal()
+  remove(local)
+  add(0, local)
+}
+
 // Guava comes transitively from calcite-core and unified-query — forbidden on
 // compile classpaths by OpenSearch. Main sources don't need it; test sources do
 // (Calcite API exposes ImmutableList, Predicate). Bypass via custom config.
@@ -133,17 +141,6 @@ dependencies {
   testRuntimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
   testRuntimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}"
 
-  // SQL Unified Query API for PPL parsing
-  testImplementation("org.opensearch.query:unified-query-api:${sqlUnifiedQueryVersion}") {
-    exclude group: 'org.opensearch'
-  }
-  testImplementation("org.opensearch.query:unified-query-core:${sqlUnifiedQueryVersion}") {
-    exclude group: 'org.opensearch'
-  }
-  testImplementation("org.opensearch.query:unified-query-ppl:${sqlUnifiedQueryVersion}") {
-    exclude group: 'org.opensearch'
-  }
-
   // Arrow Flight streaming transport for ITs
   internalClusterTestImplementation project(':plugins:arrow-flight-rpc')
 
diff --git a/sandbox/plugins/test-ppl-frontend/build.gradle b/sandbox/plugins/test-ppl-frontend/build.gradle
index 9c2d79c306c9c..81ec1c5b49a57 100644
--- a/sandbox/plugins/test-ppl-frontend/build.gradle
+++ b/sandbox/plugins/test-ppl-frontend/build.gradle
@@ -23,7 +23,9 @@ java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = Ja
 // declared below carries the published 3.7.x.x-SNAPSHOT artifacts; for local
 // development against an unpublished sql-repo HEAD, run
 // `./gradlew :ppl:publishUnifiedQueryPublicationToMavenLocal` from the sql repo first.
-def sqlUnifiedQueryVersion = '3.7.0.0-SNAPSHOT'
+// Override via `-PsqlUnifiedQueryVersion=` for local development against an
+// out-of-tree SQL plugin checkout (e.g. feature/mustang-ppl-integration).
+def sqlUnifiedQueryVersion = providers.gradleProperty('sqlUnifiedQueryVersion').getOrElse('3.7.0.0-SNAPSHOT')
 
 opensearchplugin {
   description = 'Test PPL front-end: REST endpoint backed by the unified PPL pipeline.'
@@ -32,11 +34,18 @@ opensearchplugin {
 }
 
 repositories {
-  mavenLocal()
   maven {
     name = 'OpenSearch Snapshots'
     url = 'https://ci.opensearch.org/ci/dbc/snapshots/maven/'
   }
+  // Force mavenLocal to position 0. Declaring `mavenLocal()` first inside this block isn't
+  // enough: `apply plugin: 'opensearch.test'` above contributes its own remote repos at
+  // plugin-application time (before this block runs), so any mavenLocal() here lands at
+  // position 5+ and gradle resolves the remote SNAPSHOT first. Sandbox-only; CI's empty
+  // `~/.m2/` makes this a no-op there.
+  def local = mavenLocal()
+  remove(local)
+  add(0, local)
 }
 
 // Guava comes transitively from calcite-core and unified-query — forbidden on

From b960baa792f1f3eabe95874ff373011c2f4a77df Mon Sep 17 00:00:00 2001
From: Arpit Bandejiya 
Date: Mon, 11 May 2026 16:18:23 +0530
Subject: [PATCH 106/115] Add cluster-scope defaults for pluggable dataformat
 settings (#21435)

Add cluster-scope defaults for pluggable dataformat settings

Signed-off-by: Arpit Bandejiya 
---
 .../composite/CompositeParquetIndexIT.java    | 164 +++++++
 ...RestrictCompositeDataFormatOverrideIT.java | 169 +++++++
 .../composite/CompositeDataFormatPlugin.java  | 179 +++++++-
 .../CompositeDataFormatPluginTests.java       | 208 ++++++++-
 .../ClusterDefaultPluggableDataFormatIT.java  | 151 ++++++
 .../metadata/MetadataCreateIndexService.java  | 105 +++++
 .../common/settings/ClusterSettings.java      |   6 +
 .../opensearch/indices/IndicesService.java    |  48 ++
 .../MetadataCreateIndexServiceTests.java      | 429 ++++++++++++++++++
 .../stub/MockParquetDataFormatPlugin.java     |  21 +
 10 files changed, 1469 insertions(+), 11 deletions(-)
 create mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/RestrictCompositeDataFormatOverrideIT.java
 create mode 100644 server/src/internalClusterTest/java/org/opensearch/cluster/metadata/ClusterDefaultPluggableDataFormatIT.java
 create mode 100644 test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockParquetDataFormatPlugin.java

diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java
index 4885e5ac35c2d..1e95dc64e79be 100644
--- a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java
+++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java
@@ -230,4 +230,168 @@ public void testCompositeParquetWithLuceneSecondary() throws IOException {
 
         ensureGreen(indexName);
     }
+
+    public void testCompositeIndexUsesClusterDefaultFormatsWhenOverridesAbsent() throws IOException {
+        String indexName = "test-composite-cluster-default";
+
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .put(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey(), "parquet")
+                    .putList(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey(), "lucene")
+            )
+            .get();
+
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put("index.pluggable.dataformat.enabled", true)
+            .put("index.pluggable.dataformat", "composite")
+            .build();
+
+        CreateIndexResponse response = client().admin()
+            .indices()
+            .prepareCreate(indexName)
+            .setSettings(indexSettings)
+            .setMapping("field_text", "type=text", "field_keyword", "type=keyword", "field_number", "type=integer")
+            .get();
+        assertTrue("Index creation should be acknowledged", response.isAcknowledged());
+
+        ensureGreen(indexName);
+
+        GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings(indexName).get();
+        Settings actual = settingsResponse.getIndexToSettings().get(indexName);
+        assertEquals("parquet", actual.get(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey()));
+        assertEquals("lucene", actual.getAsList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey()).get(0));
+
+        for (int i = 0; i < 10; i++) {
+            IndexResponse indexResponse = client().prepareIndex()
+                .setIndex(indexName)
+                .setSource("field_text", randomAlphaOfLength(10), "field_keyword", randomAlphaOfLength(10), "field_number", randomInt(100))
+                .get();
+            assertEquals(RestStatus.CREATED, indexResponse.status());
+        }
+
+        assertEquals(RestStatus.OK, client().admin().indices().prepareRefresh(indexName).get().getStatus());
+        assertEquals(RestStatus.OK, client().admin().indices().prepareFlush(indexName).get().getStatus());
+
+        IndicesStatsResponse statsResponse = client().admin()
+            .indices()
+            .prepareStats(indexName)
+            .clear()
+            .setIndexing(true)
+            .setRefresh(true)
+            .setDocs(true)
+            .setStore(true)
+            .get();
+        ShardStats shardStats = statsResponse.getIndex(indexName).getShards()[0];
+        assertEquals(10, shardStats.getStats().indexing.getTotal().getIndexCount());
+
+        CommitStats commitStats = shardStats.getCommitStats();
+        assertNotNull(commitStats);
+        assertTrue(commitStats.getUserData().containsKey(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY));
+
+        DataformatAwareCatalogSnapshot snapshot = DataformatAwareCatalogSnapshot.deserializeFromString(
+            commitStats.getUserData().get(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY),
+            Function.identity()
+        );
+        assertEquals(Set.of("parquet", "lucene"), snapshot.getDataFormats());
+
+        ensureGreen(indexName);
+
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .putNull(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey())
+                    .putNull(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey())
+            )
+            .get();
+    }
+
+    public void testCompositeIndexRequestOverrideBeatsClusterDefault() throws IOException {
+        String indexName = "test-composite-request-override";
+
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .put(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey(), "parquet")
+                    .putList(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey(), "lucene")
+            )
+            .get();
+
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put("index.pluggable.dataformat.enabled", true)
+            .put("index.pluggable.dataformat", "composite")
+            .put("index.composite.primary_data_format", "lucene")
+            .putList("index.composite.secondary_data_formats")
+            .build();
+
+        CreateIndexResponse response = client().admin()
+            .indices()
+            .prepareCreate(indexName)
+            .setSettings(indexSettings)
+            .setMapping("field_text", "type=text", "field_keyword", "type=keyword", "field_number", "type=integer")
+            .get();
+        assertTrue("Index creation should be acknowledged", response.isAcknowledged());
+
+        ensureGreen(indexName);
+
+        GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings(indexName).get();
+        Settings actual = settingsResponse.getIndexToSettings().get(indexName);
+        assertEquals("lucene", actual.get(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey()));
+        assertTrue(actual.getAsList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey()).isEmpty());
+
+        for (int i = 0; i < 10; i++) {
+            IndexResponse indexResponse = client().prepareIndex()
+                .setIndex(indexName)
+                .setSource("field_text", randomAlphaOfLength(10), "field_keyword", randomAlphaOfLength(10), "field_number", randomInt(100))
+                .get();
+            assertEquals(RestStatus.CREATED, indexResponse.status());
+        }
+
+        assertEquals(RestStatus.OK, client().admin().indices().prepareRefresh(indexName).get().getStatus());
+        assertEquals(RestStatus.OK, client().admin().indices().prepareFlush(indexName).get().getStatus());
+
+        IndicesStatsResponse statsResponse = client().admin()
+            .indices()
+            .prepareStats(indexName)
+            .clear()
+            .setIndexing(true)
+            .setRefresh(true)
+            .setDocs(true)
+            .setStore(true)
+            .get();
+        ShardStats shardStats = statsResponse.getIndex(indexName).getShards()[0];
+        assertEquals(10, shardStats.getStats().indexing.getTotal().getIndexCount());
+
+        CommitStats commitStats = shardStats.getCommitStats();
+        assertNotNull(commitStats);
+        assertTrue(commitStats.getUserData().containsKey(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY));
+
+        DataformatAwareCatalogSnapshot snapshot = DataformatAwareCatalogSnapshot.deserializeFromString(
+            commitStats.getUserData().get(DataformatAwareCatalogSnapshot.CATALOG_SNAPSHOT_KEY),
+            Function.identity()
+        );
+        assertEquals(Set.of("lucene"), snapshot.getDataFormats());
+
+        ensureGreen(indexName);
+
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .putNull(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey())
+                    .putNull(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey())
+            )
+            .get();
+    }
 }
diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/RestrictCompositeDataFormatOverrideIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/RestrictCompositeDataFormatOverrideIT.java
new file mode 100644
index 0000000000000..1730f5e75b78a
--- /dev/null
+++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/RestrictCompositeDataFormatOverrideIT.java
@@ -0,0 +1,169 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.composite;
+
+import org.opensearch.action.admin.indices.create.CreateIndexResponse;
+import org.opensearch.action.support.clustermanager.AcknowledgedResponse;
+import org.opensearch.be.datafusion.DataFusionPlugin;
+import org.opensearch.be.lucene.LucenePlugin;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.parquet.ParquetDataFormatPlugin;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.opensearch.composite.CompositeDataFormatPlugin.CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING;
+
+/**
+ * Integration tests for {@link CompositeDataFormatPlugin#CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING}
+ * enforcement. The setting is {@code Property.Final}, so each test starts nodes with its own
+ * settings bag.
+ */
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class RestrictCompositeDataFormatOverrideIT extends OpenSearchIntegTestCase {
+
+    private static final String INDEX_NAME = "test-composite-restrict";
+    private static final String CLUSTER_DEFAULT_PRIMARY = "lucene";
+
+    @Override
+    protected Collection> nodePlugins() {
+        return Arrays.asList(ParquetDataFormatPlugin.class, CompositeDataFormatPlugin.class, LucenePlugin.class, DataFusionPlugin.class);
+    }
+
+    private Settings nodeSettings(boolean restrict) {
+        return Settings.builder()
+            .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true)
+            .put(CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING.getKey(), restrict)
+            .put(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey(), CLUSTER_DEFAULT_PRIMARY)
+            .build();
+    }
+
+    public void testRejectsPrimaryOverrideWhenRestrictIsTrue() {
+        internalCluster().startClusterManagerOnlyNode(nodeSettings(true));
+        internalCluster().startDataOnlyNode(nodeSettings(true));
+
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet")
+            .build();
+
+        IllegalArgumentException thrown = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().prepareCreate(INDEX_NAME).setSettings(indexSettings).get()
+        );
+        String message = thrown.getMessage();
+        assertTrue(
+            "expected validation error to mention ["
+                + CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey()
+                + "] but was ["
+                + message
+                + "]",
+            message.contains(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey())
+        );
+        assertTrue(
+            "expected validation error to mention restrict setting but was [" + message + "]",
+            message.contains(CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING.getKey())
+        );
+    }
+
+    public void testRejectsSecondaryOverrideWhenRestrictIsTrue() {
+        internalCluster().startClusterManagerOnlyNode(nodeSettings(true));
+        internalCluster().startDataOnlyNode(nodeSettings(true));
+
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .putList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey(), "parquet")
+            .build();
+
+        IllegalArgumentException thrown = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().admin().indices().prepareCreate(INDEX_NAME).setSettings(indexSettings).get()
+        );
+        String message = thrown.getMessage();
+        assertTrue(
+            "expected validation error to mention ["
+                + CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey()
+                + "] but was ["
+                + message
+                + "]",
+            message.contains(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey())
+        );
+    }
+
+    public void testAcceptsMatchingOverrideWhenRestrictIsTrue() {
+        internalCluster().startClusterManagerOnlyNode(nodeSettings(true));
+        internalCluster().startDataOnlyNode(nodeSettings(true));
+
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), CLUSTER_DEFAULT_PRIMARY)
+            .build();
+
+        CreateIndexResponse response = client().admin().indices().prepareCreate(INDEX_NAME).setSettings(indexSettings).get();
+        assertTrue(response.isAcknowledged());
+        ensureGreen(INDEX_NAME);
+    }
+
+    public void testAllowsOverrideWhenRestrictIsFalse() {
+        internalCluster().startClusterManagerOnlyNode(nodeSettings(false));
+        internalCluster().startDataOnlyNode(nodeSettings(false));
+
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet")
+            .build();
+
+        CreateIndexResponse response = client().admin().indices().prepareCreate(INDEX_NAME).setSettings(indexSettings).get();
+        assertTrue(response.isAcknowledged());
+        ensureGreen(INDEX_NAME);
+    }
+
+    public void testRejectsTemplateOverrideWhenRestrictIsTrue() {
+        internalCluster().startClusterManagerOnlyNode(nodeSettings(true));
+        internalCluster().startDataOnlyNode(nodeSettings(true));
+
+        AcknowledgedResponse putTemplate = client().admin()
+            .indices()
+            .preparePutTemplate("restrict-composite-template")
+            .setPatterns(Collections.singletonList(INDEX_NAME + "*"))
+            .setSettings(Settings.builder().put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet"))
+            .setOrder(0)
+            .get();
+        assertTrue(putTemplate.isAcknowledged());
+
+        IllegalArgumentException thrown = expectThrows(IllegalArgumentException.class, () -> createIndex(INDEX_NAME));
+        assertTrue(thrown.getMessage().contains(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey()));
+    }
+
+    public void testAllowsTemplateOverrideWhenRestrictIsFalse() {
+        internalCluster().startClusterManagerOnlyNode(nodeSettings(false));
+        internalCluster().startDataOnlyNode(nodeSettings(false));
+
+        AcknowledgedResponse putTemplate = client().admin()
+            .indices()
+            .preparePutTemplate("permissive-composite-template")
+            .setPatterns(Collections.singletonList(INDEX_NAME + "*"))
+            .setSettings(Settings.builder().put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet"))
+            .setOrder(0)
+            .get();
+        assertTrue(putTemplate.isAcknowledged());
+
+        createIndex(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+    }
+}
diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java
index 86c58196f485e..22d33ffcb31e9 100644
--- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java
+++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java
@@ -10,9 +10,17 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.ValidationException;
 import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.env.Environment;
+import org.opensearch.env.NodeEnvironment;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.engine.dataformat.DataFormat;
 import org.opensearch.index.engine.dataformat.DataFormatDescriptor;
@@ -21,9 +29,19 @@
 import org.opensearch.index.engine.dataformat.IndexingEngineConfig;
 import org.opensearch.index.engine.dataformat.IndexingExecutionEngine;
 import org.opensearch.index.engine.dataformat.StoreStrategy;
+import org.opensearch.index.shard.IndexSettingProvider;
+import org.opensearch.indices.IndexCreationException;
+import org.opensearch.indices.IndicesService;
 import org.opensearch.plugins.ExtensiblePlugin;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.script.ScriptService;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.client.Client;
+import org.opensearch.watcher.ResourceWatcherService;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -44,6 +62,14 @@
  *       formats (default empty)
  * 
  *
+ * 
            And three cluster settings:
+ * 
              
+ *   
            • {@code cluster.composite.primary_data_format} — cluster-level default for the primary format
              • 
+ *   
            • {@code cluster.composite.secondary_data_formats} — cluster-level default for secondary formats
              • 
+ *   
            • {@code cluster.restrict.composite.dataformat} — when true, rejects index-level overrides that
+ *       differ from the cluster defaults
              • 
+ * 
            
+ *
  * 
            Format plugins (e.g., Parquet) extend this plugin by declaring
  * {@code extendedPlugins = ['composite-engine']} in their {@code build.gradle}
  * and implementing {@link DataFormatPlugin}.
@@ -55,13 +81,20 @@ public class CompositeDataFormatPlugin extends Plugin implements DataFormatPlugi
 
     private static final Logger logger = LogManager.getLogger(CompositeDataFormatPlugin.class);
 
+    /**
+     * Populated during {@link #createComponents} so the {@link IndexSettingProvider} registered by
+     * {@link #getAdditionalIndexSettingProviders()} can read live cluster-scope default settings
+     * at index-creation time.
+     */
+    private ClusterService clusterService;
+
     /**
      * Index setting that designates the primary data format for an index.
      * The primary format is the authoritative format used for merge operations.
      */
     public static final Setting PRIMARY_DATA_FORMAT = Setting.simpleString(
         "index.composite.primary_data_format",
-        "lucene",
+        "parquet",
         Setting.Property.IndexScope,
         Setting.Property.Final
     );
@@ -79,11 +112,153 @@ public class CompositeDataFormatPlugin extends Plugin implements DataFormatPlugi
         Setting.Property.Final
     );
 
+    /**
+     * Cluster-level default for {@code index.composite.primary_data_format}.
+     * When the index setting is not explicitly provided, this cluster setting is used as the fallback.
+     */
+    public static final Setting CLUSTER_PRIMARY_DATA_FORMAT = Setting.simpleString(
+        "cluster.composite.primary_data_format",
+        "parquet",
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    /**
+     * Cluster-level default for {@code index.composite.secondary_data_formats}.
+     * When the index setting is not explicitly provided, this cluster setting is used as the fallback.
+     */
+    public static final Setting> CLUSTER_SECONDARY_DATA_FORMATS = Setting.listSetting(
+        "cluster.composite.secondary_data_formats",
+        Collections.emptyList(),
+        s -> s,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    /**
+     * If enabled, this cluster setting enforces that indexes will be created with composite data-format settings
+     * matching the cluster-level defaults defined in {@link #CLUSTER_PRIMARY_DATA_FORMAT} and
+     * {@link #CLUSTER_SECONDARY_DATA_FORMATS} by rejecting any request that specifies an index-level value
+     * that does not match. If disabled, users may choose the composite data-format on a per-index basis using the
+     * {@link #PRIMARY_DATA_FORMAT} and {@link #SECONDARY_DATA_FORMATS} settings.
+     *
+     * 
            This is scoped to the composite plugin so restriction can be toggled independently of the server-level
+     * {@code cluster.restrict.pluggable.dataformat} flag that governs the core
+     * {@code index.pluggable.dataformat.*} settings.
+     */
+    public static final Setting CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING = Setting.boolSetting(
+        "cluster.restrict.composite.dataformat",
+        false,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
     public CompositeDataFormatPlugin() {}
 
     @Override
     public List> getSettings() {
-        return List.of(PRIMARY_DATA_FORMAT, SECONDARY_DATA_FORMATS);
+        return List.of(
+            PRIMARY_DATA_FORMAT,
+            SECONDARY_DATA_FORMATS,
+            CLUSTER_PRIMARY_DATA_FORMAT,
+            CLUSTER_SECONDARY_DATA_FORMATS,
+            CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING
+        );
+    }
+
+    @Override
+    public Collection createComponents(
+        Client client,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ResourceWatcherService resourceWatcherService,
+        ScriptService scriptService,
+        NamedXContentRegistry xContentRegistry,
+        Environment environment,
+        NodeEnvironment nodeEnvironment,
+        NamedWriteableRegistry namedWriteableRegistry,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier repositoriesServiceSupplier
+    ) {
+        this.clusterService = clusterService;
+        return Collections.emptyList();
+    }
+
+    /**
+     * Stamps the cluster-scope defaults for {@link #PRIMARY_DATA_FORMAT} and
+     * {@link #SECONDARY_DATA_FORMATS} into newly created indices when those index-level settings
+     * are not supplied by the request or a matching template.
+     *
+     * 
            Because both index settings are {@link Setting.Property#Final}, the effective value is
+     * resolved once at index-creation time from the live {@link ClusterSettings} registry and
+     * frozen into the index metadata. Later updates to the {@code cluster.composite.*} settings
+     * affect only indices created after the update.
+     *
+     * 
            If {@link #createComponents} has not run yet (e.g. during early bootstrap), the provider
+     * contributes no settings so that index creation falls back to the per-setting defaults.
+     */
+    @Override
+    public Collection getAdditionalIndexSettingProviders() {
+        return Collections.singletonList(new IndexSettingProvider() {
+            @Override
+            public Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings templateAndRequestSettings) {
+                if (clusterService == null) {
+                    return Settings.EMPTY;
+                }
+                ClusterSettings clusterSettings = clusterService.getClusterSettings();
+
+                List allowlist = clusterSettings.get(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_RESTRICT_ALLOWLIST);
+                if (allowlist.stream().anyMatch(indexName::startsWith)) {
+                    return Settings.EMPTY;
+                }
+
+                boolean restrict = clusterSettings.get(CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING);
+                String clusterPrimary = clusterSettings.get(CLUSTER_PRIMARY_DATA_FORMAT);
+                List clusterSecondary = clusterSettings.get(CLUSTER_SECONDARY_DATA_FORMATS);
+
+                if (restrict) {
+                    List errors = new ArrayList<>();
+                    if (PRIMARY_DATA_FORMAT.exists(templateAndRequestSettings)
+                        && PRIMARY_DATA_FORMAT.get(templateAndRequestSettings).equals(clusterPrimary) == false) {
+                        errors.add(
+                            "index setting ["
+                                + PRIMARY_DATA_FORMAT.getKey()
+                                + "] cannot differ from cluster default ["
+                                + clusterPrimary
+                                + "] when ["
+                                + CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING.getKey()
+                                + "=true]"
+                        );
+                    }
+                    if (SECONDARY_DATA_FORMATS.exists(templateAndRequestSettings)
+                        && SECONDARY_DATA_FORMATS.get(templateAndRequestSettings).equals(clusterSecondary) == false) {
+                        errors.add(
+                            "index setting ["
+                                + SECONDARY_DATA_FORMATS.getKey()
+                                + "] cannot differ from cluster default "
+                                + clusterSecondary
+                                + " when ["
+                                + CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING.getKey()
+                                + "=true]"
+                        );
+                    }
+                    if (errors.isEmpty() == false) {
+                        ValidationException validationException = new ValidationException();
+                        validationException.addValidationErrors(errors);
+                        throw new IndexCreationException(indexName, validationException);
+                    }
+                }
+
+                Settings.Builder out = Settings.builder();
+                if (PRIMARY_DATA_FORMAT.exists(templateAndRequestSettings) == false) {
+                    out.put(PRIMARY_DATA_FORMAT.getKey(), clusterPrimary);
+                }
+                if (SECONDARY_DATA_FORMATS.exists(templateAndRequestSettings) == false) {
+                    out.putList(SECONDARY_DATA_FORMATS.getKey(), clusterSecondary);
+                }
+                return out.build();
+            }
+        });
     }
 
     @Override
diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java
index 3cd20ce6e9530..d80d9532e4b2e 100644
--- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java
+++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeDataFormatPluginTests.java
@@ -10,6 +10,8 @@
 
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.index.IndexSettings;
@@ -17,10 +19,14 @@
 import org.opensearch.index.engine.dataformat.DataFormatDescriptor;
 import org.opensearch.index.engine.dataformat.DataFormatRegistry;
 import org.opensearch.index.engine.dataformat.StoreStrategy;
+import org.opensearch.index.shard.IndexSettingProvider;
+import org.opensearch.indices.IndicesService;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Supplier;
 
 import static org.mockito.Mockito.mock;
@@ -31,24 +37,182 @@
  */
 public class CompositeDataFormatPluginTests extends OpenSearchTestCase {
 
-    public void testGetSettingsReturnsBothSettings() {
+    // ---- Setting registration ----
+
+    public void testGetSettingsReturnsAllFourSettings() {
         CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin();
         List> settings = plugin.getSettings();
-        assertEquals(2, settings.size());
+        assertEquals(5, settings.size());
         assertTrue(settings.contains(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT));
         assertTrue(settings.contains(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS));
+        assertTrue(settings.contains(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT));
+        assertTrue(settings.contains(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS));
+        assertTrue(settings.contains(CompositeDataFormatPlugin.CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING));
     }
 
-    public void testPrimaryDataFormatDefaultsToLucene() {
-        Settings settings = Settings.builder().build();
-        assertEquals("lucene", CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.get(settings));
+    // ---- Setting defaults and value parsing ----
+
+    public void testPrimaryDataFormatDefaultsToParquet() {
+        assertEquals("parquet", CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.get(Settings.EMPTY));
+    }
+
+    public void testPrimaryDataFormatReadsExplicitValue() {
+        Settings settings = Settings.builder().put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "parquet").build();
+        assertEquals("parquet", CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.get(settings));
     }
 
     public void testSecondaryDataFormatsDefaultsToEmpty() {
-        Settings settings = Settings.builder().build();
-        assertTrue(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.get(settings).isEmpty());
+        assertTrue(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.get(Settings.EMPTY).isEmpty());
+    }
+
+    public void testSecondaryDataFormatsReadsExplicitList() {
+        Settings settings = Settings.builder()
+            .putList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey(), "parquet", "arrow")
+            .build();
+        assertEquals(List.of("parquet", "arrow"), CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.get(settings));
+    }
+
+    public void testClusterDefaultPrimaryDataFormatDefaultsToParquet() {
+        assertEquals("parquet", CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.get(Settings.EMPTY));
+    }
+
+    public void testClusterDefaultPrimaryDataFormatReadsExplicitValue() {
+        Settings settings = Settings.builder().put(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey(), "parquet").build();
+        assertEquals("parquet", CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.get(settings));
+    }
+
+    public void testClusterDefaultSecondaryDataFormatsDefaultsToEmpty() {
+        assertTrue(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.get(Settings.EMPTY).isEmpty());
+    }
+
+    public void testClusterDefaultSecondaryDataFormatsReadsExplicitList() {
+        Settings settings = Settings.builder()
+            .putList(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey(), "parquet", "arrow")
+            .build();
+        assertEquals(List.of("parquet", "arrow"), CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.get(settings));
+    }
+
+    // ---- IndexSettingProvider behavior ----
+
+    public void testIndexSettingProviderReturnsEmptyBeforeCreateComponents() {
+        CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin();
+        IndexSettingProvider provider = singleProvider(plugin);
+        // createComponents has not run, so clusterService is null and the provider must
+        // contribute nothing rather than NPE — allowing fallback to per-setting defaults.
+        Settings out = provider.getAdditionalIndexSettings("some-index", false, Settings.EMPTY);
+        assertEquals(Settings.EMPTY, out);
+    }
+
+    public void testIndexSettingProviderStampsBothClusterDefaultsWhenIndexLevelAbsent() {
+        CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin();
+        Settings clusterBag = Settings.builder()
+            .put(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey(), "parquet")
+            .putList(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey(), "arrow")
+            .build();
+        injectClusterService(plugin, clusterBag);
+
+        IndexSettingProvider provider = singleProvider(plugin);
+        Settings out = provider.getAdditionalIndexSettings("some-index", false, Settings.EMPTY);
+
+        assertEquals("parquet", CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.get(out));
+        assertEquals(List.of("arrow"), CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.get(out));
     }
 
+    public void testIndexSettingProviderSkipsPrimaryWhenAlreadySet() {
+        CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin();
+        Settings clusterBag = Settings.builder()
+            .put(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey(), "parquet")
+            .putList(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey(), "arrow")
+            .build();
+        injectClusterService(plugin, clusterBag);
+
+        Settings requestOrTemplate = Settings.builder().put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "lucene").build();
+
+        IndexSettingProvider provider = singleProvider(plugin);
+        Settings out = provider.getAdditionalIndexSettings("some-index", false, requestOrTemplate);
+
+        assertFalse(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.exists(out));
+        assertEquals(List.of("arrow"), CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.get(out));
+    }
+
+    public void testIndexSettingProviderSkipsSecondaryWhenAlreadySet() {
+        CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin();
+        Settings clusterBag = Settings.builder()
+            .put(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey(), "parquet")
+            .putList(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey(), "arrow")
+            .build();
+        injectClusterService(plugin, clusterBag);
+
+        Settings requestOrTemplate = Settings.builder()
+            .putList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey(), "parquet")
+            .build();
+
+        IndexSettingProvider provider = singleProvider(plugin);
+        Settings out = provider.getAdditionalIndexSettings("some-index", false, requestOrTemplate);
+
+        assertEquals("parquet", CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.get(out));
+        assertFalse(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.exists(out));
+    }
+
+    public void testIndexSettingProviderSkipsBothWhenBothAlreadySet() {
+        CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin();
+        Settings clusterBag = Settings.builder()
+            .put(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey(), "parquet")
+            .putList(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey(), "arrow")
+            .build();
+        injectClusterService(plugin, clusterBag);
+
+        Settings requestOrTemplate = Settings.builder()
+            .put(CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.getKey(), "lucene")
+            .putList(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.getKey(), "parquet")
+            .build();
+
+        IndexSettingProvider provider = singleProvider(plugin);
+        Settings out = provider.getAdditionalIndexSettings("some-index", false, requestOrTemplate);
+
+        // Provider contributes nothing when both settings are already explicit.
+        assertEquals(Settings.EMPTY, out);
+    }
+
+    public void testIndexSettingProviderReadsLiveClusterSettingsOnEachCall() {
+        CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin();
+
+        // Seed cluster settings with empty defaults, then flip them and verify the provider
+        // picks up the new values on the next call without any re-init of the plugin.
+        ClusterSettings clusterSettings = new ClusterSettings(
+            Settings.EMPTY,
+            Set.of(
+                CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT,
+                CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS,
+                CompositeDataFormatPlugin.CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING,
+                IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_RESTRICT_ALLOWLIST
+            )
+        );
+        ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
+        setClusterServiceField(plugin, clusterService);
+
+        IndexSettingProvider provider = singleProvider(plugin);
+
+        Settings first = provider.getAdditionalIndexSettings("idx-1", false, Settings.EMPTY);
+        assertEquals("parquet", CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.get(first));
+        assertTrue(CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.get(first).isEmpty());
+
+        // Simulate a PUT /_cluster/settings updating the dynamic cluster defaults.
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT.getKey(), "parquet")
+                .putList(CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS.getKey(), "arrow")
+                .build()
+        );
+
+        Settings second = provider.getAdditionalIndexSettings("idx-2", false, Settings.EMPTY);
+        assertEquals("parquet", CompositeDataFormatPlugin.PRIMARY_DATA_FORMAT.get(second));
+        assertEquals(List.of("arrow"), CompositeDataFormatPlugin.SECONDARY_DATA_FORMATS.get(second));
+    }
+
+    // ---- Existing getFormatDescriptors coverage ----
+
     public void testGetFormatDescriptorsDelegatestoPlugins() {
         CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin();
 
@@ -94,7 +258,6 @@ public void testGetStoreStrategiesEmptyWhenNoSubPlugins() {
         DataFormatRegistry registry = mock(DataFormatRegistry.class);
 
         IndexSettings indexSettings = buildIndexSettings(Settings.builder().put("index.composite.primary_data_format", "parquet").build());
-        // Registry resolves the format but the composite pipeline finds no plugin → empty map.
         DataFormat parquetFormat = CompositeTestHelper.stubFormat("parquet", 2, java.util.Set.of());
         when(registry.format("parquet")).thenReturn(parquetFormat);
         when(registry.getStoreStrategies(indexSettings, parquetFormat)).thenReturn(Map.of());
@@ -148,7 +311,7 @@ public void testGetStoreStrategiesEmptyForDefaultPrimaryWithoutPlugin() {
         CompositeDataFormatPlugin plugin = new CompositeDataFormatPlugin();
         DataFormatRegistry registry = mock(DataFormatRegistry.class);
 
-        IndexSettings indexSettings = buildIndexSettings(Settings.EMPTY); // defaults: primary=lucene, secondary=[]
+        IndexSettings indexSettings = buildIndexSettings(Settings.EMPTY);
         DataFormat luceneFormat = CompositeTestHelper.stubFormat("lucene", 1, java.util.Set.of());
         when(registry.format("lucene")).thenReturn(luceneFormat);
         when(registry.getStoreStrategies(indexSettings, luceneFormat)).thenReturn(Map.of());
@@ -157,6 +320,8 @@ public void testGetStoreStrategiesEmptyForDefaultPrimaryWithoutPlugin() {
         assertTrue("Should return empty when lucene sub-plugin not found", result.isEmpty());
     }
 
+    // ---- Helpers ----
+
     private static IndexSettings buildIndexSettings(Settings extra) {
         Settings settings = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
@@ -167,4 +332,29 @@ private static IndexSettings buildIndexSettings(Settings extra) {
         IndexMetadata indexMetadata = IndexMetadata.builder("test-index").settings(settings).build();
         return new IndexSettings(indexMetadata, Settings.EMPTY);
     }
+
+    private static IndexSettingProvider singleProvider(CompositeDataFormatPlugin plugin) {
+        Collection providers = plugin.getAdditionalIndexSettingProviders();
+        assertEquals(1, providers.size());
+        return providers.iterator().next();
+    }
+
+    private static void injectClusterService(CompositeDataFormatPlugin plugin, Settings clusterBag) {
+        ClusterSettings clusterSettings = new ClusterSettings(
+            clusterBag,
+            Set.of(
+                CompositeDataFormatPlugin.CLUSTER_PRIMARY_DATA_FORMAT,
+                CompositeDataFormatPlugin.CLUSTER_SECONDARY_DATA_FORMATS,
+                CompositeDataFormatPlugin.CLUSTER_RESTRICT_COMPOSITE_DATAFORMAT_SETTING,
+                IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_RESTRICT_ALLOWLIST
+            )
+        );
+        ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
+        setClusterServiceField(plugin, clusterService);
+    }
+
+    private static void setClusterServiceField(CompositeDataFormatPlugin plugin, ClusterService clusterService) {
+        plugin.createComponents(null, clusterService, null, null, null, null, null, null, null, null, null);
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/ClusterDefaultPluggableDataFormatIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/ClusterDefaultPluggableDataFormatIT.java
new file mode 100644
index 0000000000000..20d3455de2fb9
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/ClusterDefaultPluggableDataFormatIT.java
@@ -0,0 +1,151 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.metadata;
+
+import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.opensearch.common.settings.FeatureFlagSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin;
+import org.opensearch.index.engine.dataformat.stub.MockParquetDataFormatPlugin;
+import org.opensearch.indices.IndicesService;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.util.Collection;
+import java.util.List;
+
+import static org.opensearch.indices.IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING;
+import static org.opensearch.indices.IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1)
+public class ClusterDefaultPluggableDataFormatIT extends OpenSearchIntegTestCase {
+
+    @Override
+    protected Collection> nodePlugins() {
+        return List.of(MockCommitterEnginePlugin.class, MockParquetDataFormatPlugin.class);
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .putList(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_RESTRICT_ALLOWLIST.getKey(), ".kibana")
+            .build();
+    }
+
+    @Override
+    protected Settings featureFlagSettings() {
+        Settings.Builder builder = Settings.builder();
+        for (Setting builtInFlag : FeatureFlagSettings.BUILT_IN_FEATURE_FLAGS) {
+            builder.put(builtInFlag.getKey(), builtInFlag.getDefaultRaw(Settings.EMPTY));
+        }
+        builder.put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true);
+        return builder.build();
+    }
+
+    public void testClusterDefaultStampedIntoNewIndexWhenNoOverride() {
+        String indexName = "test-pluggable-cluster-default";
+
+        setClusterDefaults(true, "parquet");
+        createIndex(indexName);
+        ensureGreen(indexName);
+
+        Settings effective = getIndexSettings(indexName);
+        assertTrue(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(effective));
+        assertEquals("parquet", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(effective));
+    }
+
+    public void testExplicitIndexSettingOverridesClusterDefault() {
+        String indexName = "test-pluggable-request-override";
+
+        setClusterDefaults(true, "parquet");
+        createIndex(
+            indexName,
+            Settings.builder()
+                .put(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), false)
+                .put(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "lucene")
+                .build()
+        );
+        ensureGreen(indexName);
+
+        Settings effective = getIndexSettings(indexName);
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(effective));
+        assertEquals("lucene", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(effective));
+    }
+
+    public void testClusterDefaultUpdateAppliesToNewIndicesOnly() {
+        String indexBefore = "test-pluggable-before-update";
+        String indexAfter = "test-pluggable-after-update";
+
+        setClusterDefaults(true, "parquet");
+        createIndex(indexBefore);
+        ensureGreen(indexBefore);
+
+        Settings before = getIndexSettings(indexBefore);
+        assertTrue(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(before));
+        assertEquals("parquet", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(before));
+
+        setClusterDefaults(false, "arrow");
+        createIndex(indexAfter);
+        ensureGreen(indexAfter);
+
+        Settings after = getIndexSettings(indexAfter);
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(after));
+        assertEquals("arrow", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(after));
+
+        Settings beforeReread = getIndexSettings(indexBefore);
+        assertTrue(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(beforeReread));
+        assertEquals("parquet", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(beforeReread));
+    }
+
+    public void testAllowlistBypassesClusterDefaultStamping() {
+        String skippedIndex = ".kibana-01";
+        String normalIndex = "test-pluggable-normal";
+
+        setClusterDefaults(true, "parquet");
+
+        createIndex(skippedIndex);
+        ensureGreen(skippedIndex);
+
+        createIndex(normalIndex);
+        ensureGreen(normalIndex);
+
+        // Skipped index should NOT have cluster defaults stamped
+        Settings skippedSettings = getIndexSettings(skippedIndex);
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(skippedSettings));
+        assertEquals("", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(skippedSettings));
+
+        // Normal index should have cluster defaults stamped
+        Settings normalSettings = getIndexSettings(normalIndex);
+        assertTrue(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(normalSettings));
+        assertEquals("parquet", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(normalSettings));
+    }
+
+    private void setClusterDefaults(boolean enabled, String value) {
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .put(CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), enabled)
+                    .put(CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), value)
+            )
+            .get();
+    }
+
+    private Settings getIndexSettings(String indexName) {
+        GetSettingsResponse resp = client().admin().indices().prepareGetSettings(indexName).get();
+        Settings s = resp.getIndexToSettings().get(indexName);
+        assertNotNull(s);
+        return s;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index 8a8214b530b50..3dddbbc5b6b11 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -1218,6 +1218,7 @@ static Settings aggregateIndexSettings(
 
         updateReplicationStrategy(indexSettingsBuilder, request.settings(), settings, combinedTemplateSettings, clusterSettings);
         updateRemoteStoreSettings(indexSettingsBuilder, currentState, clusterSettings, settings, request.index());
+        updatePluggableDataFormatSettings(indexSettingsBuilder, clusterSettings, request.index());
 
         if (sourceMetadata != null) {
             assert request.resizeType() != null;
@@ -1234,6 +1235,9 @@ static Settings aggregateIndexSettings(
 
         List validationErrors = new ArrayList<>();
         validateIndexReplicationTypeSettings(indexSettingsBuilder.build(), clusterSettings).ifPresent(validationErrors::add);
+        validatePluggableDataFormatSettings(indexSettingsBuilder.build(), clusterSettings, request.index()).ifPresent(
+            validationErrors::add
+        );
         validateErrors(request.index(), validationErrors);
 
         Settings indexSettings = indexSettingsBuilder.build();
@@ -1419,6 +1423,41 @@ public static void updateRemoteStoreSettings(
         }
     }
 
+    /**
+     * Stamps the cluster-scope defaults for the pluggable data-format index settings into the
+     * index metadata at creation time when no explicit override is supplied. No-op when the
+     * pluggable data-format feature flag is disabled or the index matches the allowlist.
+     */
+    public static void updatePluggableDataFormatSettings(
+        Settings.Builder settingsBuilder,
+        ClusterSettings clusterSettings,
+        String indexName
+    ) {
+        if (FeatureFlags.isEnabled(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) == false) {
+            return;
+        }
+
+        if (isAllowedForPluggableDataFormat(indexName, clusterSettings)) {
+            return;
+        }
+
+        final Settings current = settingsBuilder.build();
+
+        if (IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.exists(current) == false) {
+            settingsBuilder.put(
+                IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(),
+                clusterSettings.get(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING)
+            );
+        }
+
+        if (IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.exists(current) == false) {
+            settingsBuilder.put(
+                IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(),
+                clusterSettings.get(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING)
+            );
+        }
+    }
+
     public static void validateStoreTypeSettings(Settings settings) {
         // deprecate simplefs store type:
         if (IndexModule.Type.SIMPLEFS.match(IndexModule.INDEX_STORE_TYPE_SETTING.get(settings))) {
@@ -1694,6 +1733,7 @@ public void validateIndexSettings(String indexName, final Settings settings, fin
         throws IndexCreationException {
         List validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings, indexName);
         validateIndexReplicationTypeSettings(settings, clusterService.getClusterSettings()).ifPresent(validationErrors::add);
+        validatePluggableDataFormatSettings(settings, clusterService.getClusterSettings(), indexName).ifPresent(validationErrors::add);
         validateErrors(indexName, validationErrors);
     }
 
@@ -1799,6 +1839,71 @@ private static Optional validateIndexReplicationTypeSettings(Settings re
         return Optional.empty();
     }
 
+    /**
+     * Validates that {@code index.pluggable.dataformat.enabled} and {@code index.pluggable.dataformat} match the
+     * cluster-level defaults {@code cluster.pluggable.dataformat.enabled} and
+     * {@code cluster.pluggable.dataformat} when
+     * {@code cluster.restrict.pluggable.dataformat} is set to true.
+     *
+     * @param requestSettings settings resulting from merging request, templates, and cluster-level defaults
+     * @param clusterSettings cluster setting
+     * @param indexName name of the index being created
+     */
+    private static Optional validatePluggableDataFormatSettings(
+        Settings requestSettings,
+        ClusterSettings clusterSettings,
+        String indexName
+    ) {
+        if (FeatureFlags.isEnabled(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) == false) {
+            return Optional.empty();
+        }
+        if (clusterSettings.get(IndicesService.CLUSTER_RESTRICT_PLUGGABLE_DATAFORMAT_SETTING) == false) {
+            return Optional.empty();
+        }
+        if (isAllowedForPluggableDataFormat(indexName, clusterSettings)) {
+            return Optional.empty();
+        }
+
+        if (requestSettings.hasValue(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey())
+            && IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(requestSettings)
+                .equals(clusterSettings.get(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING)) == false) {
+            return Optional.of(
+                "index setting ["
+                    + IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey()
+                    + "] cannot differ from cluster default ["
+                    + clusterSettings.get(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING)
+                    + "] when ["
+                    + IndicesService.CLUSTER_RESTRICT_PLUGGABLE_DATAFORMAT_SETTING.getKey()
+                    + "=true]"
+            );
+        }
+
+        if (requestSettings.hasValue(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey())
+            && IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(requestSettings)
+                .equals(clusterSettings.get(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING)) == false) {
+            return Optional.of(
+                "index setting ["
+                    + IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey()
+                    + "] cannot differ from cluster default ["
+                    + clusterSettings.get(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING)
+                    + "] when ["
+                    + IndicesService.CLUSTER_RESTRICT_PLUGGABLE_DATAFORMAT_SETTING.getKey()
+                    + "=true]"
+            );
+        }
+        return Optional.empty();
+    }
+
+    /**
+     * Returns {@code true} if the given index name matches any prefix in the
+     * {@code cluster.pluggable.dataformat.restrict.allowlist} setting, meaning it should bypass
+     * pluggable data-format default-stamping and restrict validation.
+     */
+    private static boolean isAllowedForPluggableDataFormat(String indexName, ClusterSettings clusterSettings) {
+        List allowlist = clusterSettings.get(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_RESTRICT_ALLOWLIST);
+        return allowlist.stream().anyMatch(indexName::startsWith);
+    }
+
     /**
      * Validates the settings and mappings for shrinking an index.
      *
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 7323a3f1454b6..ac727b5fe9a33 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -864,6 +864,12 @@ public void apply(Settings value, Settings current, Settings previous) {
                 CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING,
                 CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING,
 
+                // Pluggable dataformat cluster defaults
+                IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING,
+                IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING,
+                IndicesService.CLUSTER_RESTRICT_PLUGGABLE_DATAFORMAT_SETTING,
+                IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_RESTRICT_ALLOWLIST,
+
                 SystemTemplatesService.SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED,
 
                 // WorkloadManagement settings
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index be3241e28174f..aaf51f55becce 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -308,6 +308,28 @@ public class IndicesService extends AbstractLifecycleComponent
         Property.Dynamic
     );
 
+    /**
+     * Cluster-level default for {@code index.pluggable.dataformat.enabled}.
+     * Applied at index creation time when the index setting is not explicitly provided.
+     */
+    public static final Setting CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING = Setting.boolSetting(
+        "cluster.pluggable.dataformat.enabled",
+        false,
+        Property.NodeScope,
+        Property.Dynamic
+    );
+
+    /**
+     * Cluster-level default for {@code index.pluggable.dataformat}.
+     * Applied at index creation time when the index setting is not explicitly provided.
+     */
+    public static final Setting CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING = Setting.simpleString(
+        "cluster.pluggable.dataformat",
+        "",
+        Property.NodeScope,
+        Property.Dynamic
+    );
+
     /**
      * This setting is used to set the minimum refresh interval applicable for all indexes in a cluster. The
      * {@code cluster.default.index.refresh_interval} setting value needs to be higher than this setting's value. Index
@@ -370,6 +392,32 @@ public class IndicesService extends AbstractLifecycleComponent
         Property.Final
     );
 
+    /**
+     * If enabled, this setting enforces that indexes will be created with pluggable data-format settings matching the
+     * cluster-level defaults defined in {@code cluster.pluggable.dataformat.enabled} and
+     * {@code cluster.pluggable.dataformat} by rejecting any request that specifies an index-level value
+     * that does not match. If disabled, users may choose the pluggable data-format on a per-index basis using the
+     * {@code index.pluggable.dataformat.enabled} and {@code index.pluggable.dataformat} settings.
+     */
+    public static final Setting CLUSTER_RESTRICT_PLUGGABLE_DATAFORMAT_SETTING = Setting.boolSetting(
+        "cluster.restrict.pluggable.dataformat",
+        false,
+        Property.NodeScope,
+        Property.Dynamic
+    );
+
+    /**
+     * A list of index name prefixes that bypass the pluggable data-format restrict validation and
+     * cluster-default stamping. Indices whose name starts with any of these prefixes will not have
+     * cluster defaults applied and will not be rejected by the restrict setting.
+     */
+    public static final Setting> CLUSTER_PLUGGABLE_DATAFORMAT_RESTRICT_ALLOWLIST = Setting.listSetting(
+        "cluster.pluggable.dataformat.restrict.allowlist",
+        Collections.emptyList(),
+        s -> s,
+        Property.NodeScope
+    );
+
     /**
      * The node's settings.
      */
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
index 8a84e59db1ab1..2bebcd33ac59c 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -81,6 +81,7 @@
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
 import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.index.shard.IndexSettingProvider;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.DefaultRemoteStoreSettings;
 import org.opensearch.indices.IndexCreationException;
@@ -159,6 +160,7 @@
 import static org.opensearch.cluster.metadata.MetadataCreateIndexService.resolveAndValidateAliases;
 import static org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider.INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING;
 import static org.opensearch.common.util.FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES;
+import static org.opensearch.common.util.FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG;
 import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL;
 import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.opensearch.index.IndexSettings.INDEX_MERGE_POLICY;
@@ -2184,6 +2186,433 @@ public void testRefreshIntervalValidationFailureWithIndexSetting() {
         );
     }
 
+    // ---- updatePluggableDataFormatSettings ----
+
+    public void testUpdatePluggableDataFormatSettingsNoopWhenFeatureFlagDisabled() {
+        // Feature flag is off by default in tests; the helper must not contribute either setting,
+        // even when a cluster-scope default is present.
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings.Builder indexSettingsBuilder = Settings.builder();
+        MetadataCreateIndexService.updatePluggableDataFormatSettings(indexSettingsBuilder, cs, "test-index");
+
+        Settings out = indexSettingsBuilder.build();
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.exists(out));
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.exists(out));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testUpdatePluggableDataFormatSettingsStampsClusterDefaultsWhenIndexLevelAbsent() {
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings.Builder indexSettingsBuilder = Settings.builder();
+        MetadataCreateIndexService.updatePluggableDataFormatSettings(indexSettingsBuilder, cs, "test-index");
+
+        Settings out = indexSettingsBuilder.build();
+        assertTrue(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(out));
+        assertEquals("parquet", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(out));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testUpdatePluggableDataFormatSettingsSkipsEnabledWhenAlreadySet() {
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        // Primary override is preserved; value still stamped from the cluster default.
+        Settings.Builder indexSettingsBuilder = Settings.builder().put(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), false);
+        MetadataCreateIndexService.updatePluggableDataFormatSettings(indexSettingsBuilder, cs, "test-index");
+
+        Settings out = indexSettingsBuilder.build();
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(out));
+        assertEquals("parquet", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(out));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testUpdatePluggableDataFormatSettingsSkipsValueWhenAlreadySet() {
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings.Builder indexSettingsBuilder = Settings.builder().put(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "lucene");
+        MetadataCreateIndexService.updatePluggableDataFormatSettings(indexSettingsBuilder, cs, "test-index");
+
+        Settings out = indexSettingsBuilder.build();
+        assertTrue(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(out));
+        assertEquals("lucene", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(out));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testUpdatePluggableDataFormatSettingsSkipsBothWhenAlreadySet() {
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings.Builder indexSettingsBuilder = Settings.builder()
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), false)
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "lucene");
+        MetadataCreateIndexService.updatePluggableDataFormatSettings(indexSettingsBuilder, cs, "test-index");
+
+        Settings out = indexSettingsBuilder.build();
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(out));
+        assertEquals("lucene", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(out));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testUpdatePluggableDataFormatSettingsStampsBuiltInDefaultsWhenClusterBagEmpty() {
+        ClusterSettings cs = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings.Builder indexSettingsBuilder = Settings.builder();
+        MetadataCreateIndexService.updatePluggableDataFormatSettings(indexSettingsBuilder, cs, "test-index");
+
+        Settings out = indexSettingsBuilder.build();
+        assertTrue(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.exists(out));
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(out));
+        assertTrue(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.exists(out));
+        assertEquals("", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(out));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testAggregateIndexSettingsStampsPluggableDataFormatClusterDefaults() {
+        // End-to-end sanity: confirm updatePluggableDataFormatSettings is wired into the create-index
+        // pipeline, so the effective values land in the settings returned by aggregateIndexSettings.
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        request.settings(Settings.EMPTY);
+
+        Settings aggregated = aggregateIndexSettings(
+            ClusterState.EMPTY_STATE,
+            request,
+            Settings.EMPTY,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Collections.emptySet(),
+            cs
+        );
+
+        assertTrue(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(aggregated));
+        assertEquals("parquet", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(aggregated));
+    }
+
+    public void testAggregateIndexSettingsPropagatesIndexCreationExceptionFromProvider() {
+        // Simulates a plugin-supplied IndexSettingProvider (like CompositeDataFormatPlugin) rejecting
+        // a forbidden index-level override by throwing IndexCreationException wrapping a
+        // ValidationException. The exception must propagate out of aggregateIndexSettings unchanged so
+        // the REST layer reports it the same way as the built-in validateErrors path does.
+        final String expectedError = "index setting [index.example] is not allowed to be set as [cluster.test.restrict=true]";
+        IndexSettingProvider throwingProvider = new IndexSettingProvider() {
+            @Override
+            public Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings templateAndRequestSettings) {
+                ValidationException ve = new ValidationException();
+                ve.addValidationError(expectedError);
+                throw new IndexCreationException(indexName, ve);
+            }
+        };
+
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        request.settings(Settings.EMPTY);
+
+        IndexCreationException thrown = expectThrows(
+            IndexCreationException.class,
+            () -> aggregateIndexSettings(
+                ClusterState.EMPTY_STATE,
+                request,
+                Settings.EMPTY,
+                null,
+                Settings.EMPTY,
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+                randomShardLimitService(),
+                Collections.singleton(throwingProvider),
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+            )
+        );
+
+        assertEquals("test", thrown.getIndex().getName());
+        assertTrue(thrown.getCause() instanceof ValidationException);
+        assertTrue(
+            "expected validation error to contain [" + expectedError + "] but was [" + thrown.getCause().getMessage() + "]",
+            thrown.getCause().getMessage().contains(expectedError)
+        );
+    }
+
+    // ---- allowlist tests ----
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testUpdatePluggableDataFormatSettingsSkipsWhenIndexMatchesAllowlist() {
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .putList(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_RESTRICT_ALLOWLIST.getKey(), ".system", ".kibana")
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings.Builder indexSettingsBuilder = Settings.builder();
+        MetadataCreateIndexService.updatePluggableDataFormatSettings(indexSettingsBuilder, cs, ".system-index-1");
+
+        Settings out = indexSettingsBuilder.build();
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.exists(out));
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.exists(out));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testUpdatePluggableDataFormatSettingsStampsWhenIndexDoesNotMatchAllowlist() {
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .putList(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_RESTRICT_ALLOWLIST.getKey(), ".system", ".kibana")
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings.Builder indexSettingsBuilder = Settings.builder();
+        MetadataCreateIndexService.updatePluggableDataFormatSettings(indexSettingsBuilder, cs, "user-index");
+
+        Settings out = indexSettingsBuilder.build();
+        assertTrue(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(out));
+        assertEquals("parquet", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(out));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testValidatePluggableDataFormatSettingsSkipsWhenIndexMatchesAllowlist() {
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .putList(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_RESTRICT_ALLOWLIST.getKey(), ".system")
+            .put(IndicesService.CLUSTER_RESTRICT_PLUGGABLE_DATAFORMAT_SETTING.getKey(), true)
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        // Index explicitly sets a different value — normally rejected, but allowlist bypasses it.
+        Settings indexSettings = Settings.builder()
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), false)
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "lucene")
+            .build();
+
+        Settings.Builder indexSettingsBuilder = Settings.builder().put(indexSettings);
+        MetadataCreateIndexService.updatePluggableDataFormatSettings(indexSettingsBuilder, cs, ".system-test");
+
+        // No exception, no stamping — the index is left alone.
+        Settings out = indexSettingsBuilder.build();
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(out));
+        assertEquals("lucene", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(out));
+    }
+
+    // ---- validatePluggableDataFormatSettings tests ----
+
+    public void testValidatePluggableDataFormatNoopWhenFeatureFlagDisabled() {
+        // Feature flag off — no validation even with restrict=true and mismatching values.
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .put(IndicesService.CLUSTER_RESTRICT_PLUGGABLE_DATAFORMAT_SETTING.getKey(), true)
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings mismatch = Settings.builder()
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), false)
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "lucene")
+            .build();
+
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        request.settings(mismatch);
+
+        // Should NOT throw — feature flag is off by default in tests without @LockFeatureFlag
+        Settings aggregated = aggregateIndexSettings(
+            ClusterState.EMPTY_STATE,
+            request,
+            Settings.EMPTY,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Collections.emptySet(),
+            cs
+        );
+        assertNotNull(aggregated);
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testValidatePluggableDataFormatNoopWhenRestrictDisabled() {
+        // restrict=false — mismatching values are allowed.
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings mismatch = Settings.builder()
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), false)
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "lucene")
+            .build();
+
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        request.settings(mismatch);
+
+        Settings aggregated = aggregateIndexSettings(
+            ClusterState.EMPTY_STATE,
+            request,
+            Settings.EMPTY,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Collections.emptySet(),
+            cs
+        );
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(aggregated));
+        assertEquals("lucene", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(aggregated));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testValidatePluggableDataFormatRejectsEnabledMismatch() {
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .put(IndicesService.CLUSTER_RESTRICT_PLUGGABLE_DATAFORMAT_SETTING.getKey(), true)
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings mismatch = Settings.builder().put(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), false).build();
+
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        request.settings(mismatch);
+
+        IndexCreationException exception = expectThrows(
+            IndexCreationException.class,
+            () -> aggregateIndexSettings(
+                ClusterState.EMPTY_STATE,
+                request,
+                Settings.EMPTY,
+                null,
+                Settings.EMPTY,
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+                randomShardLimitService(),
+                Collections.emptySet(),
+                cs
+            )
+        );
+        assertTrue(exception.getCause().getMessage().contains(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey()));
+        assertTrue(exception.getCause().getMessage().contains("cannot differ from cluster default"));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testValidatePluggableDataFormatRejectsValueMismatch() {
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .put(IndicesService.CLUSTER_RESTRICT_PLUGGABLE_DATAFORMAT_SETTING.getKey(), true)
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings mismatch = Settings.builder().put(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "lucene").build();
+
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        request.settings(mismatch);
+
+        IndexCreationException exception = expectThrows(
+            IndexCreationException.class,
+            () -> aggregateIndexSettings(
+                ClusterState.EMPTY_STATE,
+                request,
+                Settings.EMPTY,
+                null,
+                Settings.EMPTY,
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+                randomShardLimitService(),
+                Collections.emptySet(),
+                cs
+            )
+        );
+        assertTrue(exception.getCause().getMessage().contains(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey()));
+        assertTrue(exception.getCause().getMessage().contains("cannot differ from cluster default"));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testValidatePluggableDataFormatAllowsMatchingValues() {
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .put(IndicesService.CLUSTER_RESTRICT_PLUGGABLE_DATAFORMAT_SETTING.getKey(), true)
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings matching = Settings.builder()
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .build();
+
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        request.settings(matching);
+
+        Settings aggregated = aggregateIndexSettings(
+            ClusterState.EMPTY_STATE,
+            request,
+            Settings.EMPTY,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Collections.emptySet(),
+            cs
+        );
+        assertTrue(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(aggregated));
+        assertEquals("parquet", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(aggregated));
+    }
+
+    @LockFeatureFlag(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testValidatePluggableDataFormatAllowlistBypassesRestrict() {
+        Settings clusterBag = Settings.builder()
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), true)
+            .put(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "parquet")
+            .put(IndicesService.CLUSTER_RESTRICT_PLUGGABLE_DATAFORMAT_SETTING.getKey(), true)
+            .putList(IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_RESTRICT_ALLOWLIST.getKey(), ".system")
+            .build();
+        ClusterSettings cs = new ClusterSettings(clusterBag, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        Settings mismatch = Settings.builder()
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.getKey(), false)
+            .put(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "lucene")
+            .build();
+
+        request = new CreateIndexClusterStateUpdateRequest("create index", ".system-index", ".system-index");
+        request.settings(mismatch);
+
+        // Should NOT throw — index matches allowlist
+        Settings aggregated = aggregateIndexSettings(
+            ClusterState.EMPTY_STATE,
+            request,
+            Settings.EMPTY,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Collections.emptySet(),
+            cs
+        );
+        assertFalse(IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(aggregated));
+        assertEquals("lucene", IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(aggregated));
+    }
+
     public void testAnyTranslogDurabilityWhenRestrictSettingFalse() {
         // This checks that aggregateIndexSettings works for the case when the cluster setting
         // cluster.remote_store.index.restrict.async-durability is false or not set, it allows all types of durability modes
diff --git a/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockParquetDataFormatPlugin.java b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockParquetDataFormatPlugin.java
new file mode 100644
index 0000000000000..8404e6e022149
--- /dev/null
+++ b/test/framework/src/main/java/org/opensearch/index/engine/dataformat/stub/MockParquetDataFormatPlugin.java
@@ -0,0 +1,21 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.engine.dataformat.stub;
+
+import java.util.Set;
+
+/**
+ * A mock {@link MockDataFormatPlugin} that registers "parquet" as a data format for testing.
+ */
+public class MockParquetDataFormatPlugin extends MockDataFormatPlugin {
+
+    public MockParquetDataFormatPlugin() {
+        super(new MockDataFormat("parquet", 100L, Set.of()));
+    }
+}

From da1afdcdf0fe3ecbfdc24f17f1a55c3598ead124 Mon Sep 17 00:00:00 2001
From: Bharathwaj G 
Date: Mon, 11 May 2026 16:25:40 +0530
Subject: [PATCH 107/115] Changes to upgrade datafusion and arrow to latest
 versions (#21590)

Signed-off-by: bharath-techie 
---
 .../libs/dataformat-native/rust/Cargo.toml    |  80 +++++------
 .../dataformat-native/rust/macros/Cargo.toml  |   6 +-
 .../src/main/rust/src/tiered_object_store.rs  | 129 ++++++------------
 .../rust/src/tiered_object_store_tests.rs     |  55 ++------
 .../rust/Cargo.toml                           |  10 +-
 .../rust/src/api.rs                           |   5 +-
 .../rust/src/cache.rs                         |  80 ++++++-----
 .../rust/src/custom_cache_manager.rs          |  65 ++++-----
 .../rust/src/indexed_executor.rs              |   4 +-
 .../rust/src/indexed_table/parquet_bridge.rs  |   2 +-
 .../rust/src/indexed_table/stream.rs          |   4 +-
 .../rust/src/indexed_table/table_provider.rs  |  12 +-
 .../rust/src/query_executor.rs                |   4 +-
 .../rust/src/session_context.rs               |   4 +-
 .../rust/src/statistics_cache.rs              |  95 ++++++-------
 .../rust/tests/local_exec_test.rs             |   1 +
 .../src/main/rust/Cargo.toml                  |   2 +-
 .../src/main/rust/src/range_cache.rs          |   2 +-
 .../src/main/rust/src/fs.rs                   |   2 +-
 19 files changed, 244 insertions(+), 318 deletions(-)

diff --git a/sandbox/libs/dataformat-native/rust/Cargo.toml b/sandbox/libs/dataformat-native/rust/Cargo.toml
index 003e2a5d7390b..4f391153203eb 100644
--- a/sandbox/libs/dataformat-native/rust/Cargo.toml
+++ b/sandbox/libs/dataformat-native/rust/Cargo.toml
@@ -16,62 +16,62 @@ members = [
 
 [workspace.dependencies]
 # Arrow / Parquet
-arrow = { version = "57.3.0", features = ["ffi"] }
-arrow-array = "57.3.0"
-arrow-ipc = "57.3.0"
-arrow-schema = "57.3.0"
-arrow-buffer = "57.3.0"
-parquet = "57.3.0"
+arrow = { version = "=58.2.0", features = ["ffi"] }
+arrow-array = "=58.2.0"
+arrow-ipc = "=58.2.0"
+arrow-schema = "=58.2.0"
+arrow-buffer = "=58.2.0"
+parquet = "=58.2.0"
 
 # DataFusion
-datafusion = "52.1.0"
-datafusion-expr = "52.1.0"
-datafusion-datasource = "52.1.0"
-datafusion-common = "52.1.0"
-datafusion-execution = "52.1.0"
-datafusion-physical-expr = "52.1.0"
-datafusion-substrait = "52.1.0"
+datafusion = "=53.1.0"
+datafusion-expr = "=53.1.0"
+datafusion-datasource = "=53.1.0"
+datafusion-common = "=53.1.0"
+datafusion-execution = "=53.1.0"
+datafusion-physical-expr = "=53.1.0"
+datafusion-substrait = "=53.1.0"
 
 # Async
-tokio = { version = "1.0", features = ["full"] }
-tokio-util = "0.7"
-futures = "0.3"
-tokio-stream = "0.1.17"
+tokio = { version = "=1.51.0", features = ["full"] }
+tokio-util = "=0.7.18"
+futures = "=0.3.32"
+tokio-stream = "=0.1.18"
 
 # Serialization
-prost = "0.14"
-substrait = "=0.62.0"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
+prost = "=0.14.3"
+substrait = "=0.62.2"
+serde = { version = "=1.0.228", features = ["derive"] }
+serde_json = "=1.0.149"
 
 # Logging
-log = "0.4"
+log = "=0.4.29"
 
 # Allocator
 # disable_initial_exec_tls: Required because this library is loaded at runtime via dlopen/JVM FFM.
 # Without it, jemalloc uses initial-exec TLS which fails on aarch64 Linux with:
 #   "cannot allocate memory in static TLS block"
 # The feature switches to global-dynamic TLS model, compatible with runtime loading.
-tikv-jemallocator = { version = "0.6", features = ["disable_initial_exec_tls"] }
-tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] }
+tikv-jemallocator = { version = "=0.6.1", features = ["disable_initial_exec_tls"] }
+tikv-jemalloc-ctl = { version = "=0.6.1", features = ["stats"] }
 
 # Misc
-dashmap = "5.5"
-num_cpus = "1.16"
-object_store = "0.12.5"
-url = "2.0"
-tempfile = "3.0"
-chrono = "0.4"
-once_cell = "1.21.3"
-crc32fast = "1.4"
-parking_lot = "0.12.5"
-lazy_static = "1.4.0"
-rayon = "1.10"
-thiserror = "1.0"
-async-trait = "0.1"
-bytes = "1"
-criterion = { version = "0.5", features = ["async_tokio"] }
-tokio-metrics = { version = "0.5", features = ["rt"] }
+dashmap = "=5.5.3"
+num_cpus = "=1.17.0"
+object_store = "=0.13.2"
+url = "=2.5.8"
+tempfile = "=3.27.0"
+chrono = "=0.4.44"
+once_cell = "=1.21.4"
+crc32fast = "=1.5.0"
+parking_lot = "=0.12.5"
+lazy_static = "=1.5.0"
+rayon = "=1.11.0"
+thiserror = "=1.0.69"
+async-trait = "=0.1.89"
+bytes = "=1.11.1"
+criterion = { version = "=0.5.1", features = ["async_tokio"] }
+tokio-metrics = { version = "=0.5.0", features = ["rt"] }
 
 # Internal
 native-bridge-common = { path = "common" }
diff --git a/sandbox/libs/dataformat-native/rust/macros/Cargo.toml b/sandbox/libs/dataformat-native/rust/macros/Cargo.toml
index 22d7dcf5bee72..5b00f6964f1be 100644
--- a/sandbox/libs/dataformat-native/rust/macros/Cargo.toml
+++ b/sandbox/libs/dataformat-native/rust/macros/Cargo.toml
@@ -8,6 +8,6 @@ license = "Apache-2.0"
 proc-macro = true
 
 [dependencies]
-quote = "1"
-syn = { version = "2", features = ["full"] }
-proc-macro2 = "1"
+quote = "=1.0.45"
+syn = { version = "=2.0.117", features = ["full"] }
+proc-macro2 = "=1.0.106"
diff --git a/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store.rs b/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store.rs
index b39f32cac02cf..5509f487b0393 100644
--- a/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store.rs
+++ b/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store.rs
@@ -19,16 +19,14 @@
 //! registry's atomics and DashMap — no locks are held during I/O.
 
 use std::fmt;
-use std::ops::Range;
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use bytes::Bytes;
 use futures::stream::BoxStream;
 use futures::StreamExt;
 use object_store::{
-    path::Path, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
-    PutMultipartOptions, PutOptions, PutPayload, PutResult, Result as OsResult,
+    path::Path, CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
+    ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result as OsResult,
 };
 
 use crate::registry::traits::FileRegistry;
@@ -220,9 +218,36 @@ impl ObjectStore for TieredObjectStore {
 
     /// Primary read path: check registry for remote routing, otherwise local.
     /// If local read fails with NotFound and file transitioned to REMOTE, retries from remote.
+    ///
+    /// Also handles head requests (options.head == true) by returning cached
+    /// size from the registry when available — avoids I/O for the common case.
     async fn get_opts(&self, location: &Path, options: GetOptions) -> OsResult {
         let path_str = location.as_ref();
 
+        // Fast path for head: return cached size from registry if available
+        if options.head {
+            if let Some(guard) = self.registry.get(path_str) {
+                let size = guard.size();
+                if size > 0 {
+                    let meta = ObjectMeta {
+                        location: location.clone(),
+                        last_modified: chrono::DateTime::::default(),
+                        size,
+                        e_tag: None,
+                        version: None,
+                    };
+                    return Ok(GetResult {
+                        payload: object_store::GetResultPayload::Stream(
+                            futures::stream::empty().boxed(),
+                        ),
+                        meta,
+                        range: 0..size,
+                        attributes: Default::default(),
+                    });
+                }
+            }
+        }
+
         if let Some((rp, store)) = self.resolve_remote(path_str) {
             native_bridge_common::log_debug!(
                 "TieredObjectStore: get_opts REMOTE path='{}'",
@@ -240,76 +265,20 @@ impl ObjectStore for TieredObjectStore {
         result
     }
 
-    /// Range read with local-fail-retry-remote.
-    async fn get_range(&self, location: &Path, range: Range) -> OsResult {
-        let path_str = location.as_ref();
-
-        if let Some((rp, store)) = self.resolve_remote(path_str) {
-            return store.get_range(&rp, range).await;
-        }
-
-        let result = self.local.get_range(location, range.clone()).await;
-        if let Err(ref e) = result {
-            if let Some((rp, store)) = self.should_retry_remote(path_str, e) {
-                return store.get_range(&rp, range).await;
-            }
-        }
-        result
-    }
-
-    /// Multi-range read with local-fail-retry-remote.
-    async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> OsResult> {
-        let path_str = location.as_ref();
-
-        if let Some((rp, store)) = self.resolve_remote(path_str) {
-            return store.get_ranges(&rp, ranges).await;
-        }
-
-        let result = self.local.get_ranges(location, ranges).await;
-        if let Err(ref e) = result {
-            if let Some((rp, store)) = self.should_retry_remote(path_str, e) {
-                return store.get_ranges(&rp, ranges).await;
-            }
-        }
-        result
-    }
-
-    /// Head: check registry first (cached size), then local, then remote.
-    /// Head: check registry for cached size first (no I/O), then route via resolve_remote.
-    async fn head(&self, location: &Path) -> OsResult {
-        let path_str = location.as_ref();
-
-        // Check registry for cached size — return immediately if available
-        if let Some(guard) = self.registry.get(path_str) {
-            let size = guard.size();
-            if size > 0 {
-                return Ok(ObjectMeta {
-                    location: location.clone(),
-                    last_modified: chrono::DateTime::::default(),
-                    size,
-                    e_tag: None,
-                    version: None,
-                });
+    /// Delete stream: remove each path from registry only, NO local delete.
+    /// Local file deletion is handled by the Java layer.
+    fn delete_stream(
+        &self,
+        locations: BoxStream<'static, OsResult>,
+    ) -> BoxStream<'static, OsResult> {
+        let registry = Arc::clone(&self.registry);
+        let mapped = locations.map(move |result| {
+            if let Ok(ref path) = result {
+                registry.remove(path.as_ref(), true);
             }
-        }
-
-        // Size not cached — route via resolve_remote (REMOTE → remote store)
-        if let Some((rp, store)) = self.resolve_remote(path_str) {
-            return store.head(&rp).await;
-        }
-
-        // Not remote — try local
-        self.local.head(location).await
-    }
-
-    /// Delete: remove from registry only, NO local delete.
-    /// Local file deletion is handled by the Java layer
-    /// (TieredSubdirectoryAwareDirectory.deleteFile). Eviction (local copy
-    /// removal after sync) is a writable warm concern — not implemented.
-    async fn delete(&self, location: &Path) -> OsResult<()> {
-        let path_str = location.as_ref();
-        self.registry.remove(path_str, true);
-        Ok(())
+            result
+        });
+        Box::pin(mapped)
     }
 
     /// List: local entries first, then remote-only entries from registry (deduplicated).
@@ -364,23 +333,11 @@ impl ObjectStore for TieredObjectStore {
         Ok(result)
     }
 
-    async fn copy(&self, _from: &Path, _to: &Path) -> OsResult<()> {
+    async fn copy_opts(&self, _from: &Path, _to: &Path, _options: CopyOptions) -> OsResult<()> {
         Err(object_store::Error::NotSupported {
             source: "TieredObjectStore does not support copy".into(),
         })
     }
-
-    async fn copy_if_not_exists(&self, _from: &Path, _to: &Path) -> OsResult<()> {
-        Err(object_store::Error::NotSupported {
-            source: "TieredObjectStore does not support copy_if_not_exists".into(),
-        })
-    }
-
-    async fn rename_if_not_exists(&self, _from: &Path, _to: &Path) -> OsResult<()> {
-        Err(object_store::Error::NotSupported {
-            source: "TieredObjectStore does not support rename_if_not_exists".into(),
-        })
-    }
 }
 
 // ---------------------------------------------------------------------------
diff --git a/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store_tests.rs b/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store_tests.rs
index d7e5922a8e98a..0d502cf933a4e 100644
--- a/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store_tests.rs
+++ b/sandbox/libs/tiered-storage/src/main/rust/src/tiered_object_store_tests.rs
@@ -1,7 +1,7 @@
 use super::*;
 use futures::StreamExt;
 use object_store::memory::InMemory;
-use object_store::PutPayload;
+use object_store::{CopyOptions, ObjectStoreExt, PutPayload};
 use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
 
 /// Helper: create a registry + tiered store backed by in-memory stores.
@@ -514,10 +514,6 @@ impl ObjectStore for CallCountingStore {
         self.inner.get_opts(location, options).await
     }
 
-    async fn head(&self, location: &Path) -> OsResult {
-        self.inner.head(location).await
-    }
-
     async fn put_opts(
         &self,
         location: &Path,
@@ -535,8 +531,11 @@ impl ObjectStore for CallCountingStore {
         self.inner.put_multipart_opts(location, opts).await
     }
 
-    async fn delete(&self, location: &Path) -> OsResult<()> {
-        self.inner.delete(location).await
+    fn delete_stream(
+        &self,
+        locations: BoxStream<'static, OsResult>,
+    ) -> BoxStream<'static, OsResult> {
+        self.inner.delete_stream(locations)
     }
 
     fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OsResult> {
@@ -547,16 +546,8 @@ impl ObjectStore for CallCountingStore {
         self.inner.list_with_delimiter(prefix).await
     }
 
-    async fn copy(&self, from: &Path, to: &Path) -> OsResult<()> {
-        self.inner.copy(from, to).await
-    }
-
-    async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> OsResult<()> {
-        self.inner.copy_if_not_exists(from, to).await
-    }
-
-    async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> OsResult<()> {
-        self.inner.rename_if_not_exists(from, to).await
+    async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> OsResult<()> {
+        self.inner.copy_opts(from, to, options).await
     }
 }
 
@@ -617,13 +608,6 @@ impl ObjectStore for ErrorStore {
         })
     }
 
-    async fn head(&self, _location: &Path) -> OsResult {
-        Err(object_store::Error::Generic {
-            store: "ErrorStore",
-            source: "simulated error".into(),
-        })
-    }
-
     async fn put_opts(
         &self,
         _location: &Path,
@@ -647,11 +631,14 @@ impl ObjectStore for ErrorStore {
         })
     }
 
-    async fn delete(&self, _location: &Path) -> OsResult<()> {
-        Err(object_store::Error::Generic {
+    fn delete_stream(
+        &self,
+        locations: BoxStream<'static, OsResult>,
+    ) -> BoxStream<'static, OsResult> {
+        Box::pin(locations.map(|_| Err(object_store::Error::Generic {
             store: "ErrorStore",
             source: "simulated error".into(),
-        })
+        })))
     }
 
     fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, OsResult> {
@@ -665,19 +652,7 @@ impl ObjectStore for ErrorStore {
         })
     }
 
-    async fn copy(&self, _from: &Path, _to: &Path) -> OsResult<()> {
-        Err(object_store::Error::NotSupported {
-            source: "not supported".into(),
-        })
-    }
-
-    async fn copy_if_not_exists(&self, _from: &Path, _to: &Path) -> OsResult<()> {
-        Err(object_store::Error::NotSupported {
-            source: "not supported".into(),
-        })
-    }
-
-    async fn rename_if_not_exists(&self, _from: &Path, _to: &Path) -> OsResult<()> {
+    async fn copy_opts(&self, _from: &Path, _to: &Path, _options: CopyOptions) -> OsResult<()> {
         Err(object_store::Error::NotSupported {
             source: "not supported".into(),
         })
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
index 93c7f0dea7265..17722aba29b4c 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/Cargo.toml
@@ -41,11 +41,11 @@ num_cpus = { workspace = true }
 native-bridge-common = { workspace = true }
 async-trait = { workspace = true }
 chrono = { workspace = true }
-roaring = "0.10"
+roaring = "=0.10.12"
 thiserror = { workspace = true }
 
 # convert_tz UDF
-chrono-tz = "0.10"
+chrono-tz = "=0.10.4"
 
 tokio-metrics = { workspace = true }
 
@@ -67,14 +67,14 @@ serde_json = { workspace = true, features = ["preserve_order"] }
 # relies on to render missing-path matches as literal `null` elements in the
 # multi-path JSON-array output. Moving to 1.x is a follow-up once we can
 # reproduce that distinction against the new API surface.
-jsonpath-rust = "0.7"
+jsonpath-rust = "=0.7.5"
 # mvfind UDF — regex matching against stringified array elements
-regex = "1.10"
+regex = "=1.12.3"
 
 [dev-dependencies]
 criterion = { workspace = true }
 tempfile = { workspace = true }
-rand = "0.8"
+rand = "=0.8.6"
 
 [[bench]]
 name = "query_bench"
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs
index 5035fc39a3300..519c36e69deb2 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs
@@ -52,6 +52,7 @@ use datafusion::execution::{SessionState, SessionStateBuilder};
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::prelude::SessionConfig;
 use futures::TryStreamExt;
+use object_store::ObjectStoreExt;
 
 use crate::cancellation;
 use crate::cross_rt_stream::CrossRtStream;
@@ -454,7 +455,7 @@ pub unsafe fn sql_to_substrait(
 ) -> Result, DataFusionError> {
     use datafusion::datasource::file_format::parquet::ParquetFormat;
     use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig};
-    use datafusion::execution::cache::cache_manager::CacheManagerConfig;
+    use datafusion::execution::cache::cache_manager::{CacheManagerConfig, CachedFileList};
     use datafusion::execution::cache::{CacheAccessor, DefaultListFilesCache};
     use datafusion_substrait::logical_plan::producer::to_substrait_plan;
     use prost::Message;
@@ -472,7 +473,7 @@ pub unsafe fn sql_to_substrait(
                 table: None,
                 path: table_path.prefix().clone(),
             },
-            object_metas,
+            CachedFileList::new(object_metas.as_ref().clone()),
         );
         let runtime_env = RuntimeEnvBuilder::from_runtime_env(&runtime.runtime_env)
             .with_cache_manager(
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/cache.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/cache.rs
index e84b372c35882..602d778bb3b66 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/cache.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/cache.rs
@@ -8,11 +8,13 @@
 
 use std::sync::{Arc, Mutex};
 
-use datafusion::execution::cache::cache_manager::FileMetadataCache;
+use datafusion::execution::cache::cache_manager::{
+    CachedFileMetadataEntry, FileMetadataCache, FileMetadataCacheEntry,
+};
 use datafusion::execution::cache::cache_unit::DefaultFilesMetadataCache;
 use datafusion::execution::cache::CacheAccessor;
 use log::error;
-use object_store::ObjectMeta;
+use object_store::path::Path;
 
 // Cache type constants
 pub const CACHE_TYPE_METADATA: &str = "METADATA";
@@ -35,7 +37,7 @@ impl MutexFileMetadataCache {
         }
     }
 
-    pub fn clear(&self) {
+    pub fn clear_cache(&self) {
         if let Ok(cache) = self.inner.lock() {
             cache.clear();
         }
@@ -47,7 +49,7 @@ impl MutexFileMetadataCache {
         }
     }
 
-    pub fn cache_limit(&self) -> usize {
+    pub fn get_cache_limit(&self) -> usize {
         if let Ok(cache) = self.inner.lock() {
             cache.cache_limit()
         } else {
@@ -56,57 +58,54 @@ impl MutexFileMetadataCache {
     }
 }
 
-
-// Implement CacheAccessor which is required by FileMetadataCache
-impl CacheAccessor> for MutexFileMetadataCache {
-    type Extra = ObjectMeta;
-
-    fn get(&self, k: &ObjectMeta) -> Option> {
+impl CacheAccessor for MutexFileMetadataCache {
+    fn get(&self, k: &Path) -> Option {
         match self.inner.lock() {
             Ok(cache) => cache.get(k),
-            Err(e) => { log_cache_error("get", &e.to_string()); None }
-        }
-    }
-
-    fn get_with_extra(&self, k: &ObjectMeta, extra: &Self::Extra) -> Option> {
-        match self.inner.lock() {
-            Ok(cache) => cache.get_with_extra(k, extra),
-            Err(e) => { log_cache_error("get_with_extra", &e.to_string()); None }
+            Err(e) => {
+                log_cache_error("get", &e.to_string());
+                None
+            }
         }
     }
 
-    fn put(&self, k: &ObjectMeta, v: Arc) -> Option> {
+    fn put(&self, k: &Path, v: CachedFileMetadataEntry) -> Option {
         match self.inner.lock() {
             Ok(cache) => cache.put(k, v),
-            Err(e) => { log_cache_error("put", &e.to_string()); None }
-        }
-    }
-
-    fn put_with_extra(&self, k: &ObjectMeta, v: Arc, e: &Self::Extra) -> Option> {
-        match self.inner.lock() {
-            Ok(cache) => cache.put_with_extra(k, v, e),
-            Err(err) => { log_cache_error("put_with_extra", &err.to_string()); None }
+            Err(e) => {
+                log_cache_error("put", &e.to_string());
+                None
+            }
         }
     }
 
-    fn remove(&self, k: &ObjectMeta) -> Option> {
+    fn remove(&self, k: &Path) -> Option {
         match self.inner.lock() {
             Ok(cache) => cache.remove(k),
-            Err(e) => { log_cache_error("remove", &e.to_string()); None }
+            Err(e) => {
+                log_cache_error("remove", &e.to_string());
+                None
+            }
         }
     }
 
-    fn contains_key(&self, k: &ObjectMeta) -> bool {
+    fn contains_key(&self, k: &Path) -> bool {
         match self.inner.lock() {
             Ok(cache) => cache.contains_key(k),
-            Err(e) => { log_cache_error("contains_key", &e.to_string()); false }
+            Err(e) => {
+                log_cache_error("contains_key", &e.to_string());
+                false
+            }
         }
     }
 
     fn len(&self) -> usize {
         match self.inner.lock() {
             Ok(cache) => cache.len(),
-            Err(e) => { log_cache_error("len", &e.to_string()); 0 }
+            Err(e) => {
+                log_cache_error("len", &e.to_string());
+                0
+            }
         }
     }
 
@@ -120,7 +119,10 @@ impl CacheAccessor String {
         match self.inner.lock() {
             Ok(cache) => cache.name(),
-            Err(e) => { log_cache_error("name", &e.to_string()); "cache_error".to_string() }
+            Err(e) => {
+                log_cache_error("name", &e.to_string());
+                "cache_error".to_string()
+            }
         }
     }
 }
@@ -129,7 +131,10 @@ impl FileMetadataCache for MutexFileMetadataCache {
     fn cache_limit(&self) -> usize {
         match self.inner.lock() {
             Ok(cache) => cache.cache_limit(),
-            Err(e) => { log_cache_error("cache_limit", &e.to_string()); 0 }
+            Err(e) => {
+                log_cache_error("cache_limit", &e.to_string());
+                0
+            }
         }
     }
 
@@ -140,10 +145,13 @@ impl FileMetadataCache for MutexFileMetadataCache {
         }
     }
 
-    fn list_entries(&self) -> std::collections::HashMap {
+    fn list_entries(&self) -> std::collections::HashMap {
         match self.inner.lock() {
             Ok(cache) => cache.list_entries(),
-            Err(e) => { log_cache_error("list_entries", &e.to_string()); std::collections::HashMap::new() }
+            Err(e) => {
+                log_cache_error("list_entries", &e.to_string());
+                std::collections::HashMap::new()
+            }
         }
     }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/custom_cache_manager.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/custom_cache_manager.rs
index 6d373f5320da6..07d08c5132f90 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/custom_cache_manager.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/custom_cache_manager.rs
@@ -162,29 +162,23 @@ impl CustomCacheManager {
             let mut errors = Vec::new();
 
             // Remove from metadata cache
-            match create_object_meta_from_file(file_path) {
-                Ok(object_metas) => {
-                    if let Some(cache) = &self.file_metadata_cache {
-                        match cache.inner.lock() {
-                            Ok(cache_guard) => {
-                                if let Some(object_meta) = object_metas.first() {
-                                    if cache_guard.remove(object_meta).is_some() {
-                                        any_removed = true;
-                                    } else {
-                                        debug!("[CACHE INFO] File not found in metadata cache: {}", file_path);
-                                    }
-                                }
-                            }
-                            Err(e) => {
-                                errors.push(format!("Metadata cache: Cache remove failed: {}", e));
+            {
+                let path = Path::from(file_path.clone());
+                if let Some(cache) = &self.file_metadata_cache {
+                    match cache.inner.lock() {
+                        Ok(cache_guard) => {
+                            if cache_guard.remove(&path).is_some() {
+                                any_removed = true;
+                            } else {
+                                debug!("[CACHE INFO] File not found in metadata cache: {}", file_path);
                             }
                         }
-                    } else {
-                        errors.push("No metadata cache configured".to_string());
+                        Err(e) => {
+                            errors.push(format!("Metadata cache: Cache remove failed: {}", e));
+                        }
                     }
-                }
-                Err(e) => {
-                    errors.push(format!("Failed to get object metadata: {}", e));
+                } else {
+                    errors.push("No metadata cache configured".to_string());
                 }
             }
 
@@ -214,19 +208,13 @@ impl CustomCacheManager {
         let mut found = false;
 
         // Check metadata cache
-        match create_object_meta_from_file(file_path) {
-            Ok(object_metas) => {
-                if let Some(cache) = &self.file_metadata_cache {
-                    if let Some(object_meta) = object_metas.first() {
-                        if cache.get(object_meta).is_some() {
-                            found = true;
-                        }
-                    }
+        {
+            let path = Path::from(file_path);
+            if let Some(cache) = &self.file_metadata_cache {
+                if cache.get(&path).is_some() {
+                    found = true;
                 }
             }
-            Err(e) => {
-                error!("Failed to get object metadata for {}: {}", file_path, e);
-            }
         }
 
         // Check statistics cache
@@ -244,10 +232,10 @@ impl CustomCacheManager {
     pub fn contains_file_by_type(&self, file_path: &str, cache_type: &str) -> bool {
         match cache_type {
             crate::cache::CACHE_TYPE_METADATA => {
-                create_object_meta_from_file(file_path)
-                    .ok()
-                    .and_then(|metas| metas.first().cloned())
-                    .and_then(|meta| self.file_metadata_cache.as_ref()?.get(&meta))
+                let path = Path::from(file_path);
+                self.file_metadata_cache
+                    .as_ref()
+                    .and_then(|cache| cache.get(&path))
                     .is_some()
             }
             crate::cache::CACHE_TYPE_STATS => {
@@ -393,7 +381,8 @@ impl CustomCacheManager {
         // Verify the metadata was cached properly
         match cache_ref.inner.lock() {
             Ok(cache_guard) => {
-                if cache_guard.contains_key(object_meta) {
+                let path = Path::from(file_path.to_string());
+                if cache_guard.contains_key(&path) {
                     Ok(true)
                 } else {
                     debug!("[CACHE ERROR] Failed to cache metadata for: {}", file_path);
@@ -429,7 +418,7 @@ impl CustomCacheManager {
                     version: None,
                 };
 
-                cache.put_with_extra(&path, Arc::new(stats), &meta);
+                cache.put_statistics(&path, Arc::new(stats), &meta);
                 Ok(true)
             }
             Err(e) => {
@@ -466,7 +455,7 @@ impl CustomCacheManager {
                         version: None,
                     };
 
-                    cache.put_with_extra(&path, Arc::new(stats), &meta);
+                    cache.put_statistics(&path, Arc::new(stats), &meta);
                     success_count += 1;
                 }
                 Err(e) => {
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
index bb44669221a7b..84365eff2a493 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs
@@ -32,7 +32,7 @@ use datafusion::{
     catalog::Session,
     common::tree_node::{TreeNode, TreeNodeRecursion},
     datasource::{TableProvider, TableType},
-    execution::cache::cache_manager::CacheManagerConfig,
+    execution::cache::cache_manager::{CacheManagerConfig, CachedFileList},
     execution::cache::{CacheAccessor, DefaultListFilesCache, TableScopedPath},
     execution::memory_pool::MemoryPool,
     execution::object_store::ObjectStoreUrl,
@@ -103,7 +103,7 @@ pub async fn execute_indexed_query(
         table: None,
         path: shard_view.table_path.prefix().clone(),
     };
-    list_file_cache.put(&table_scoped_path, shard_view.object_metas.clone());
+    list_file_cache.put(&table_scoped_path, CachedFileList::new(shard_view.object_metas.as_ref().clone()));
 
     let mut runtime_env_builder = RuntimeEnvBuilder::from_runtime_env(&runtime.runtime_env)
         .with_cache_manager(
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/parquet_bridge.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/parquet_bridge.rs
index 07c2bcf656dd7..a2020c12ac893 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/parquet_bridge.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/parquet_bridge.rs
@@ -40,7 +40,7 @@ use datafusion_datasource::source::DataSourceExec;
 use datafusion_datasource::PartitionedFile;
 use futures::future::BoxFuture;
 use futures::FutureExt;
-use object_store::ObjectStore;
+use object_store::{ObjectStore, ObjectStoreExt};
 use prost::bytes::Bytes;
 
 // ── Parquet Metadata Loading ─────────────────────────────────────────
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs
index cff5632eeb4f3..4588066f5833b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs
@@ -258,7 +258,7 @@ pub struct IndexedExec {
     pub(crate) store_url: datafusion::execution::object_store::ObjectStoreUrl,
     pub(crate) row_groups: Vec,
     pub(crate) projection: Option>,
-    pub(crate) properties: PlanProperties,
+    pub(crate) properties: Arc,
     pub(crate) metadata: Arc,
     pub(crate) predicate: Option>,
     /// Pluggable bitset source (SingleCollector or RustTree).
@@ -309,7 +309,7 @@ impl ExecutionPlan for IndexedExec {
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
-    fn properties(&self) -> &PlanProperties {
+    fn properties(&self) -> &Arc {
         &self.properties
     }
     fn children(&self) -> Vec<&Arc> {
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/table_provider.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/table_provider.rs
index b0ec951a3dff0..6843eb743f7e1 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/table_provider.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/table_provider.rs
@@ -222,12 +222,12 @@ impl TableProvider for IndexedTableProvider {
         let assignments =
             compute_assignments(&layouts, self.config.query_config.target_partitions.max(1));
 
-        let properties = PlanProperties::new(
+        let properties = Arc::new(PlanProperties::new(
             EquivalenceProperties::new(projected_schema.clone()),
             Partitioning::UnknownPartitioning(assignments.len().max(1)),
             EmissionType::Incremental,
             Boundedness::Bounded,
-        );
+        ));
 
         Ok(Arc::new(QueryShardExec {
             config: Arc::clone(&self.config),
@@ -257,7 +257,7 @@ pub struct QueryShardExec {
     projected_schema: SchemaRef,
     projection: Option>,
     assignments: Vec,
-    properties: PlanProperties,
+    properties: Arc,
     /// Residual physical predicate pushed down from the planner. Threaded
     /// into each `IndexedExec` so `ParquetSource.with_predicate(...)` can
     /// apply it during decode.
@@ -296,7 +296,7 @@ impl ExecutionPlan for QueryShardExec {
     fn schema(&self) -> SchemaRef {
         self.projected_schema.clone()
     }
-    fn properties(&self) -> &PlanProperties {
+    fn properties(&self) -> &Arc {
         &self.properties
     }
     fn children(&self) -> Vec<&Arc> {
@@ -361,12 +361,12 @@ impl ExecutionPlan for QueryShardExec {
             let evaluator = (self.config.evaluator_factory)(segment, chunk, &stream_metrics)
                 .map_err(|e| DataFusionError::External(e.into()))?;
 
-            let props = PlanProperties::new(
+            let props = Arc::new(PlanProperties::new(
                 EquivalenceProperties::new(self.projected_schema.clone()),
                 Partitioning::UnknownPartitioning(1),
                 EmissionType::Incremental,
                 Boundedness::Bounded,
-            );
+            ));
 
             let exec = IndexedExec {
                 schema: self.projected_schema.clone(),
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs
index be804e1eb777e..16194fc3ab4be 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs
@@ -18,7 +18,7 @@ use datafusion::{
     prelude::*,
 };
 use datafusion::datasource::file_format::parquet::ParquetFormat;
-use datafusion::execution::cache::cache_manager::CacheManagerConfig;
+use datafusion::execution::cache::cache_manager::{CacheManagerConfig, CachedFileList};
 use datafusion::execution::cache::{CacheAccessor, DefaultListFilesCache};
 use datafusion_substrait::logical_plan::consumer::from_substrait_plan;
 use log::error;
@@ -58,7 +58,7 @@ pub async fn execute_query(
         table: None,
         path: table_path.prefix().clone(),
     };
-    list_file_cache.put(&table_scoped_path, object_metas);
+    list_file_cache.put(&table_scoped_path, CachedFileList::new(object_metas.as_ref().clone()));
 
     // Build a per-query RuntimeEnv sharing the global memory pool + caches,
     // but with a fresh list-files cache for this query's shard files.
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
index 979a3a4c188cc..f6ccd0655196a 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs
@@ -17,7 +17,7 @@ use datafusion::{
     common::DataFusionError,
     datasource::file_format::parquet::ParquetFormat,
     datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl},
-    execution::cache::cache_manager::CacheManagerConfig,
+    execution::cache::cache_manager::{CacheManagerConfig, CachedFileList},
     execution::cache::{CacheAccessor, DefaultListFilesCache},
     execution::context::SessionContext,
     execution::memory_pool::MemoryPool,
@@ -79,7 +79,7 @@ pub async unsafe fn create_session_context(
             table: None,
             path: shard_view.table_path.prefix().clone(),
         },
-        shard_view.object_metas.clone(),
+        CachedFileList::new(shard_view.object_metas.as_ref().clone()),
     );
 
     let mut runtime_env_builder = RuntimeEnvBuilder::from_runtime_env(&runtime.runtime_env)
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/statistics_cache.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/statistics_cache.rs
index 2971648bed1db..134c43d1a8a8c 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/statistics_cache.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/statistics_cache.rs
@@ -14,6 +14,7 @@ use datafusion::common::stats::{ColumnStatistics, Precision};
 use datafusion::common::ScalarValue;
 use dashmap::DashMap;
 use datafusion::execution::cache::CacheAccessor;
+use datafusion::execution::cache::cache_manager::{CachedFileMetadata, FileStatisticsCache, FileStatisticsCacheEntry};
 use datafusion::physical_plan::Statistics;
 use object_store::{path::Path, ObjectMeta};
 use std::collections::HashMap;
@@ -114,7 +115,7 @@ impl StatisticsMemorySize for Statistics {
 /// and adds memory tracking + policy-based eviction on top.
 pub struct CustomStatisticsCache {
     /// The underlying DataFusion statistics cache (DashMap-based, already thread-safe)
-    inner_cache: DashMap)>,
+    inner_cache: DashMap,
     /// The eviction policy (thread-safe)
     policy: Arc>>,
     /// Size limit for the cache in bytes
@@ -152,7 +153,7 @@ impl CustomStatisticsCache {
     }
 
     /// Get the underlying cache for compatibility
-    pub fn inner(&self) -> &DashMap)> {
+    pub fn inner(&self) -> &DashMap {
         &self.inner_cache
     }
 
@@ -271,13 +272,29 @@ impl CustomStatisticsCache {
         Ok(freed_size)
     }
 
+    /// Convenience method: put statistics with associated metadata (replaces old put_with_extra)
+    pub fn put_statistics(
+        &self,
+        k: &Path,
+        stats: Arc,
+        meta: &ObjectMeta,
+    ) -> Option {
+        let cached = CachedFileMetadata::new(meta.clone(), stats, None);
+        self.put(k, cached)
+    }
+
+    /// Convenience method: get just the statistics Arc (for callers that don't need full CachedFileMetadata)
+    pub fn get_statistics(&self, k: &Path) -> Option> {
+        self.get(k).map(|c| c.statistics)
+    }
+
     /// Parse cache key back to Path
     fn parse_key_to_path(&self, key: &str) -> CacheResult {
         Ok(Path::from(key))
     }
 
     /// Remove entry internally (works with &self since inner_cache is thread-safe)
-    fn remove_internal(&self, k: &Path) -> Option> {
+    fn remove_internal(&self, k: &Path) -> Option {
         let key = k.to_string();
         let result = self.inner_cache.remove(k);
         if result.is_some() {
@@ -292,15 +309,13 @@ impl CustomStatisticsCache {
                 policy_guard.on_remove(&key);
             }
         }
-        result.map(|x| x.1 .1)
+        result.map(|x| x.1)
     }
 }
 
-// Implement CacheAccessor - DashMap handles concurrency, we just need to handle the &mut self requirement
-impl CacheAccessor> for CustomStatisticsCache {
-    type Extra = ObjectMeta;
-
-    fn get(&self, k: &Path) -> Option> {
+// Implement CacheAccessor - DashMap handles concurrency
+impl CacheAccessor for CustomStatisticsCache {
+    fn get(&self, k: &Path) -> Option {
         let result = self.inner_cache.get(k);
 
         if result.is_some() {
@@ -316,34 +331,14 @@ impl CacheAccessor> for CustomStatisticsCache {
             if let Ok(mut misses) = self.miss_count.lock() { *misses += 1; }
         }
 
-        result.map(|s| Some(Arc::clone(&s.value().1))).unwrap_or(None)
-    }
-
-    fn get_with_extra(&self, k: &Path, _extra: &Self::Extra) -> Option> {
-        self.get(k)
-    }
-
-    fn put(&self, k: &Path, v: Arc) -> Option> {
-        let meta = ObjectMeta {
-            location: k.clone(),
-            last_modified: chrono::Utc::now(),
-            size: 0,
-            e_tag: None,
-            version: None,
-        };
-        self.put_with_extra(k, v, &meta)
+        result.map(|s| s.value().clone())
     }
 
-    fn put_with_extra(
-        &self,
-        k: &Path,
-        v: Arc,
-        e: &Self::Extra,
-    ) -> Option> {
+    fn put(&self, k: &Path, v: CachedFileMetadata) -> Option {
         let key = k.to_string();
-        let memory_size = v.memory_size();
+        let memory_size = v.statistics.memory_size();
 
-        let eviction_candidates = if let Ok(tracker) = self.memory_tracker.lock() {
+        let eviction_candidates = if let Ok(_tracker) = self.memory_tracker.lock() {
             if let Ok(total) = self.total_memory.lock() {
                 let current_size = *total;
                 let size_limit = self.size_limit.load(Ordering::Relaxed);
@@ -363,7 +358,7 @@ impl CacheAccessor> for CustomStatisticsCache {
             }
         }
 
-        let result = self.inner_cache.insert(k.clone(), (e.clone(), v)).map(|x| x.1);
+        let result = self.inner_cache.insert(k.clone(), v);
 
         if let Ok(mut tracker) = self.memory_tracker.lock() {
             if let Ok(mut total) = self.total_memory.lock() {
@@ -382,7 +377,7 @@ impl CacheAccessor> for CustomStatisticsCache {
         result
     }
 
-    fn remove(&self, k: &Path) -> Option> {
+    fn remove(&self, k: &Path) -> Option {
         let key = k.to_string();
         let result = self.inner_cache.remove(k);
         if result.is_some() {
@@ -397,7 +392,7 @@ impl CacheAccessor> for CustomStatisticsCache {
                 policy_guard.on_remove(&key);
             }
         }
-        result.map(|x| x.1 .1)
+        result.map(|x| x.1)
     }
 
     fn contains_key(&self, k: &Path) -> bool {
@@ -424,8 +419,8 @@ impl CacheAccessor> for CustomStatisticsCache {
     }
 }
 
-impl datafusion::execution::cache::cache_manager::FileStatisticsCache for CustomStatisticsCache {
-    fn list_entries(&self) -> std::collections::HashMap {
+impl FileStatisticsCache for CustomStatisticsCache {
+    fn list_entries(&self) -> std::collections::HashMap {
         std::collections::HashMap::new()
     }
 }
@@ -511,7 +506,7 @@ mod tests {
         let path = create_test_path("file1");
         let meta = create_test_meta(&path);
         let stats = Arc::new(create_test_statistics());
-        cache.put_with_extra(&path, stats, &meta);
+        cache.put_statistics(&path, stats, &meta);
 
         assert!(cache.memory_consumed() > 0);
         assert_eq!(cache.len(), 1);
@@ -525,7 +520,7 @@ mod tests {
             let path = create_test_path(&format!("file{}", i));
             let meta = create_test_meta(&path);
             let stats = Arc::new(create_test_statistics());
-            cache.put_with_extra(&path, stats, &meta);
+            cache.put_statistics(&path, stats, &meta);
         }
         assert!(cache.memory_consumed() <= 1000);
         assert!(cache.len() > 0);
@@ -538,7 +533,7 @@ mod tests {
             let path = create_test_path(&format!("file{}", i));
             let meta = create_test_meta(&path);
             let stats = Arc::new(create_test_statistics());
-            cache.put_with_extra(&path, stats, &meta);
+            cache.put_statistics(&path, stats, &meta);
         }
         let memory_before = cache.memory_consumed();
         assert!(memory_before > 0);
@@ -554,7 +549,7 @@ mod tests {
             let path = create_test_path(&format!("file{}", i));
             let meta = create_test_meta(&path);
             let stats = Arc::new(create_test_statistics());
-            cache.put_with_extra(&path, stats, &meta);
+            cache.put_statistics(&path, stats, &meta);
         }
         let memory_before = cache.memory_consumed();
         assert_eq!(cache.policy_name().unwrap(), "lru");
@@ -572,8 +567,8 @@ mod tests {
         let meta2 = create_test_meta(&path2);
         let stats = Arc::new(create_test_statistics());
 
-        cache.put_with_extra(&path1, stats.clone(), &meta1);
-        cache.put_with_extra(&path2, stats, &meta2);
+        cache.put_statistics(&path1, stats.clone(), &meta1);
+        cache.put_statistics(&path2, stats, &meta2);
         let memory_with_two = cache.memory_consumed();
         assert_eq!(cache.len(), 2);
 
@@ -593,7 +588,7 @@ mod tests {
             let path = create_test_path(&format!("file{}", i));
             let meta = create_test_meta(&path);
             let stats = Arc::new(create_test_statistics());
-            cache.put_with_extra(&path, stats, &meta);
+            cache.put_statistics(&path, stats, &meta);
         }
         assert!(cache.memory_consumed() > 0);
         cache.clear();
@@ -607,7 +602,7 @@ mod tests {
         let path = create_test_path("file1");
         let meta = create_test_meta(&path);
         let stats = Arc::new(create_test_statistics());
-        cache.put_with_extra(&path, stats, &meta);
+        cache.put_statistics(&path, stats, &meta);
 
         assert!(cache.get(&path).is_some());
         assert_eq!(cache.hit_count(), 1);
@@ -635,7 +630,7 @@ mod tests {
         let path1 = create_test_path("file1");
         let meta1 = create_test_meta(&path1);
         let stats = Arc::new(create_test_statistics());
-        cache.put_with_extra(&path1, stats, &meta1);
+        cache.put_statistics(&path1, stats, &meta1);
 
         cache.get(&path1); // hit
         cache.get(&path1); // hit
@@ -652,7 +647,7 @@ mod tests {
         let path = create_test_path("file1");
         let meta = create_test_meta(&path);
         let stats = Arc::new(create_test_statistics());
-        cache.put_with_extra(&path, stats, &meta);
+        cache.put_statistics(&path, stats, &meta);
 
         cache.get(&path);
         let path2 = create_test_path("missing");
@@ -672,7 +667,7 @@ mod tests {
         let path = create_test_path("file1");
         let meta = create_test_meta(&path);
         let stats = Arc::new(create_test_statistics());
-        cache.put_with_extra(&path, stats, &meta);
+        cache.put_statistics(&path, stats, &meta);
         cache.get(&path);
 
         cache.clear();
@@ -693,7 +688,7 @@ mod tests {
                 let path = create_test_path(&format!("concurrent{}", i));
                 let meta = create_test_meta(&path);
                 let stats = Arc::new(create_test_statistics());
-                cache_clone.put_with_extra(&path, stats, &meta);
+                cache_clone.put_statistics(&path, stats, &meta);
                 assert!(cache_clone.get(&path).is_some());
             });
             handles.push(handle);
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/tests/local_exec_test.rs b/sandbox/plugins/analytics-backend-datafusion/rust/tests/local_exec_test.rs
index ceef1bca07e63..04dd1d8694028 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/tests/local_exec_test.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/tests/local_exec_test.rs
@@ -79,6 +79,7 @@ impl RuntimeGuard {
         let rc = unsafe {
             df_create_global_runtime(
                 128 * 1024 * 1024,
+                0, // no cache manager
                 spill_bytes.as_ptr(),
                 spill_bytes.len() as i64,
                 64 * 1024 * 1024,
diff --git a/sandbox/plugins/block-cache-foyer/src/main/rust/Cargo.toml b/sandbox/plugins/block-cache-foyer/src/main/rust/Cargo.toml
index 6b7d398c910db..74cb1fe07ea77 100644
--- a/sandbox/plugins/block-cache-foyer/src/main/rust/Cargo.toml
+++ b/sandbox/plugins/block-cache-foyer/src/main/rust/Cargo.toml
@@ -11,7 +11,7 @@ name = "opensearch_block_cache"
 crate-type = ["rlib"]
 
 [dependencies]
-foyer                = "0.22"
+foyer                = "=0.22.3"
 bytes                = { workspace = true }
 dashmap              = { workspace = true }
 tokio                = { workspace = true }
diff --git a/sandbox/plugins/block-cache-foyer/src/main/rust/src/range_cache.rs b/sandbox/plugins/block-cache-foyer/src/main/rust/src/range_cache.rs
index 10854b74957e9..b99a0c28630bf 100644
--- a/sandbox/plugins/block-cache-foyer/src/main/rust/src/range_cache.rs
+++ b/sandbox/plugins/block-cache-foyer/src/main/rust/src/range_cache.rs
@@ -72,7 +72,7 @@ impl CacheKey {
 ///
 /// # Example
 /// ```
-/// use opensearch_page_cache::range_cache::range_cache_key;
+/// use opensearch_block_cache::range_cache::range_cache_key;
 /// let key = range_cache_key("data/nodes/0/_0.parquet", 0, 4096);
 /// assert_eq!(key.as_str(), "data/nodes/0/_0.parquet\x1f0-4096");
 /// ```
diff --git a/sandbox/plugins/native-repository-fs/src/main/rust/src/fs.rs b/sandbox/plugins/native-repository-fs/src/main/rust/src/fs.rs
index 844ff8dd1e31e..3d943e678c165 100644
--- a/sandbox/plugins/native-repository-fs/src/main/rust/src/fs.rs
+++ b/sandbox/plugins/native-repository-fs/src/main/rust/src/fs.rs
@@ -34,7 +34,7 @@ pub fn build(
 mod tests {
     use super::*;
     use object_store::path::Path;
-    use object_store::PutPayload;
+    use object_store::{ObjectStoreExt, PutPayload};
     use futures::TryStreamExt;
 
     #[test]

From 9254832407b9bc1ff955af101883b0a60ee8f973 Mon Sep 17 00:00:00 2001
From: Andriy Redko 
Date: Mon, 11 May 2026 09:33:31 -0400
Subject: [PATCH 108/115] Update OpenTelemetry to 1.62.0 and OpenTelemetry
 SemConv to 1.41.0 (#21595)

Signed-off-by: Andriy Redko 
---
 gradle/libs.versions.toml                                     | 4 ++--
 .../telemetry-otel/licenses/opentelemetry-api-1.61.0.jar.sha1 | 1 -
 .../telemetry-otel/licenses/opentelemetry-api-1.62.0.jar.sha1 | 1 +
 .../opentelemetry-api-incubator-1.61.0-alpha.jar.sha1         | 1 -
 .../opentelemetry-api-incubator-1.62.0-alpha.jar.sha1         | 1 +
 .../licenses/opentelemetry-common-1.61.0.jar.sha1             | 1 -
 .../licenses/opentelemetry-common-1.62.0.jar.sha1             | 1 +
 .../licenses/opentelemetry-context-1.61.0.jar.sha1            | 1 -
 .../licenses/opentelemetry-context-1.62.0.jar.sha1            | 1 +
 .../licenses/opentelemetry-exporter-common-1.61.0.jar.sha1    | 1 -
 .../licenses/opentelemetry-exporter-common-1.62.0.jar.sha1    | 1 +
 .../licenses/opentelemetry-exporter-logging-1.61.0.jar.sha1   | 1 -
 .../licenses/opentelemetry-exporter-logging-1.62.0.jar.sha1   | 1 +
 .../licenses/opentelemetry-exporter-otlp-1.61.0.jar.sha1      | 1 -
 .../licenses/opentelemetry-exporter-otlp-1.62.0.jar.sha1      | 1 +
 .../opentelemetry-exporter-otlp-common-1.61.0.jar.sha1        | 1 -
 .../opentelemetry-exporter-otlp-common-1.62.0.jar.sha1        | 1 +
 .../opentelemetry-exporter-sender-okhttp-1.61.0.jar.sha1      | 1 -
 .../opentelemetry-exporter-sender-okhttp-1.62.0.jar.sha1      | 1 +
 .../telemetry-otel/licenses/opentelemetry-sdk-1.61.0.jar.sha1 | 1 -
 .../telemetry-otel/licenses/opentelemetry-sdk-1.62.0.jar.sha1 | 1 +
 .../licenses/opentelemetry-sdk-common-1.61.0.jar.sha1         | 1 -
 .../licenses/opentelemetry-sdk-common-1.62.0.jar.sha1         | 1 +
 .../licenses/opentelemetry-sdk-logs-1.61.0.jar.sha1           | 1 -
 .../licenses/opentelemetry-sdk-logs-1.62.0.jar.sha1           | 1 +
 .../licenses/opentelemetry-sdk-metrics-1.61.0.jar.sha1        | 1 -
 .../licenses/opentelemetry-sdk-metrics-1.62.0.jar.sha1        | 1 +
 .../licenses/opentelemetry-sdk-trace-1.61.0.jar.sha1          | 1 -
 .../licenses/opentelemetry-sdk-trace-1.62.0.jar.sha1          | 1 +
 .../licenses/opentelemetry-semconv-1.40.0.jar.sha1            | 1 -
 .../licenses/opentelemetry-semconv-1.41.0.jar.sha1            | 1 +
 31 files changed, 17 insertions(+), 17 deletions(-)
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.61.0-alpha.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.62.0-alpha.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-common-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-common-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-context-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.61.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.62.0.jar.sha1
 delete mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-1.40.0.jar.sha1
 create mode 100644 plugins/telemetry-otel/licenses/opentelemetry-semconv-1.41.0.jar.sha1

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 01db444cd421b..f80f359194234 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -97,8 +97,8 @@ jzlib             = "1.1.3"
 resteasy          = "6.2.4.Final"
 
 # opentelemetry dependencies
-opentelemetry         = "1.61.0"
-opentelemetrysemconv  = "1.40.0"
+opentelemetry         = "1.62.0"
+opentelemetrysemconv  = "1.41.0"
 
 # arrow dependencies
 arrow                 = "18.1.0"
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.61.0.jar.sha1
deleted file mode 100644
index cc7b4ec8039f2..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-api-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6b36cee03bfd6424f532342a8c4519c10666c157
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..02ab255e34d5e
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-api-1.62.0.jar.sha1
@@ -0,0 +1 @@
+c4ee83d77005567852a72e08b945ebb023be1daa
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.61.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.61.0-alpha.jar.sha1
deleted file mode 100644
index 70d158bb5f7cf..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.61.0-alpha.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6cf3bcc6e6a9b2233abfa369e8a58a460d81fd9e
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.62.0-alpha.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.62.0-alpha.jar.sha1
new file mode 100644
index 0000000000000..88802c8009d0e
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.62.0-alpha.jar.sha1
@@ -0,0 +1 @@
+91f3bcf6b93261cbaf32dd156e0007aa5fa5b25a
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-common-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-common-1.61.0.jar.sha1
deleted file mode 100644
index d850246bc1439..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-common-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-cf3f9eee453b106916e8d7f43a212293b868e95a
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-common-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-common-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..db25f474db864
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-common-1.62.0.jar.sha1
@@ -0,0 +1 @@
+e6468bd64a94429b68761f7c13e143c3fdfaafc7
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.61.0.jar.sha1
deleted file mode 100644
index 467dfdf99a996..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-context-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8385e62008c321fcbafecf0b7f3f432ab9b99062
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-context-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-context-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..4608aebd30520
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-context-1.62.0.jar.sha1
@@ -0,0 +1 @@
+365cee4d1f365e4d4a05654742b50aa436c2dd8e
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.61.0.jar.sha1
deleted file mode 100644
index e4ab602806a50..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2ea79318304cad4c4b903e8dac1ec739914aedcc
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..9114878e88cef
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.62.0.jar.sha1
@@ -0,0 +1 @@
+2dafa6ae65cbf1aa321cd644d200f3ff8465284d
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.61.0.jar.sha1
deleted file mode 100644
index 112db15957033..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6ba58512b43a6d97c869a63b046579744667888f
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..25565231ce2ff
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.62.0.jar.sha1
@@ -0,0 +1 @@
+79ec5f1f23e00da7a8c8a30136cfbfaf9aa38f93
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.61.0.jar.sha1
deleted file mode 100644
index cb27f777bc643..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9b5436f1f754650728c3052a46f8fe59e3f5cf53
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..1d0b19032d4ef
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.62.0.jar.sha1
@@ -0,0 +1 @@
+8e4cb9199ac868332a1213ca27408a18905ba369
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.61.0.jar.sha1
deleted file mode 100644
index 5a05984ae07fc..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-76d2a99dd063fc8e3d231d62860f33444131ef30
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..03a22b1f68946
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.62.0.jar.sha1
@@ -0,0 +1 @@
+6f269df0e3f065fbd2e590458e7b2107cde2a106
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.61.0.jar.sha1
deleted file mode 100644
index e4bb29017a6e2..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-71e3bc45372e4d9284da52f848f21145d979963e
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..82a20ade44ef5
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.62.0.jar.sha1
@@ -0,0 +1 @@
+19b5e023db9037a38fe2531afb6e44456e963fba
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.61.0.jar.sha1
deleted file mode 100644
index 7f3e62319784e..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-02221e7c13577b5d04fb3b69a72cbfeba0f73034
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..99ef9c2e1d260
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-1.62.0.jar.sha1
@@ -0,0 +1 @@
+06fa52c4641322b14b8bd515eb048bb9b1365d0c
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.61.0.jar.sha1
deleted file mode 100644
index 8ca2240775ebe..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-13ac20fd570b28d56fefd7a7f7e427f80bb7959c
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..9828f6e0985cf
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.62.0.jar.sha1
@@ -0,0 +1 @@
+b6742282daab8e13598b78a83ddfa54f10b5752b
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.61.0.jar.sha1
deleted file mode 100644
index bc4a9ae37db53..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3612cd019e07d1a196795c8db79b9760b623a393
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..b008d4cb9a80e
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.62.0.jar.sha1
@@ -0,0 +1 @@
+f242422084100da0bd3a5f6f2bcf364aaf4d2c53
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.61.0.jar.sha1
deleted file mode 100644
index 9699d09da2dd7..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-956444da6c88b6619fb80dcacca3aee19753f250
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..a845283b6a3d4
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.62.0.jar.sha1
@@ -0,0 +1 @@
+5838371075930a4a15f7f61240b4b64cb3e924d8
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.61.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.61.0.jar.sha1
deleted file mode 100644
index 6019efc50b645..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.61.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3004df101068116093b2a2c7984ad69a908208b9
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.62.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.62.0.jar.sha1
new file mode 100644
index 0000000000000..9d3ce157565d3
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.62.0.jar.sha1
@@ -0,0 +1 @@
+7a337d2f887b151d27e734d1c221eb51b1c5b734
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.40.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.40.0.jar.sha1
deleted file mode 100644
index d16b3608696ad..0000000000000
--- a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.40.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fca8594d7edb9ac5f7f9baa8d68f135354a7a243
\ No newline at end of file
diff --git a/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.41.0.jar.sha1 b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.41.0.jar.sha1
new file mode 100644
index 0000000000000..1d54e026636d9
--- /dev/null
+++ b/plugins/telemetry-otel/licenses/opentelemetry-semconv-1.41.0.jar.sha1
@@ -0,0 +1 @@
+bb726d13dbdf41d18560a82f2266a2f07f6114e2
\ No newline at end of file

From d790b71f431140c1292c31932c9cf3c4aa5b3942 Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Mon, 11 May 2026 07:54:50 -0700
Subject: [PATCH 109/115] Stub support for streaming-transport request handlers
 in MockTransportService (#21581)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Stub support for streaming-transport request handlers in MockTransportService

Adds Node.wrapStreamTransport hook so MockNode can wrap the streaming
transport before it's shared between TransportService and
StreamTransportService. MockTransportService.addRequestHandlingBehavior
falls back to the streaming-transport's stub registry when the action
isn't on the regular transport — needed for streaming-only handlers
(e.g. when FlightStreamPlugin is loaded).

Demonstrated by a new sandbox/qa/analytics-engine-coordinator IT that
stubs FragmentExecutionAction (streaming-only) end-to-end, plus a
deterministic unit test for a lookahead-reordering bug surfaced via
this infrastructure: the listener was offloading onStreamResponse
bodies to a thread pool, letting the isLast=true task race ahead of
earlier batches and drop them via the SUCCEEDED short-circuit.

Signed-off-by: Marc Handalian 

* fix imports

Signed-off-by: Marc Handalian 

* spotless

Signed-off-by: Marc Handalian 

---------

Signed-off-by: Marc Handalian 
---
 .../stage/ShardFragmentStageExecution.java    |  32 +--
 .../ShardFragmentStageExecutionTests.java     |  73 +++++-
 .../analytics-engine-coordinator/build.gradle | 134 ++++++++++
 .../resilience/CoordinatorResilienceIT.java   | 238 ++++++++++++++++++
 .../main/java/org/opensearch/node/Node.java   |  14 +-
 .../java/org/opensearch/node/MockNode.java    |  20 ++
 .../test/transport/MockTransportService.java  |  39 ++-
 .../test/transport/StubbableTransport.java    |  10 +
 8 files changed, 540 insertions(+), 20 deletions(-)
 create mode 100644 sandbox/qa/analytics-engine-coordinator/build.gradle
 create mode 100644 sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java

diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java
index 153c6f914edd8..8376e34ed22f9 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java
@@ -114,23 +114,25 @@ private void dispatchShardTask(ShardExecutionTarget target) {
 
     private  StreamingResponseListener responseListener(Function toVsr) {
         return new StreamingResponseListener<>() {
+            // Runs inline on the per-stream virtual thread driving handleStreamResponse.
+            // Must NOT offload to a thread pool: reordering across batches would let the
+            // isLast=true task race ahead, flip state to SUCCEEDED, and drop queued
+            // earlier batches via the isDone() short-circuit.
             @Override
             public void onStreamResponse(T response, boolean isLast) {
-                config.searchExecutor().execute(() -> {
-                    if (isDone()) {
-                        releaseResponseResources(response);
-                        return;
-                    }
-
-                    VectorSchemaRoot vsr = toVsr.apply(response);
-                    outputSink.feed(vsr);
-                    metrics.addRowsProcessed(vsr.getRowCount());
-
-                    if (isLast) {
-                        metrics.incrementTasksCompleted();
-                        onShardTerminated();
-                    }
-                });
+                if (isDone()) {
+                    releaseResponseResources(response);
+                    return;
+                }
+
+                VectorSchemaRoot vsr = toVsr.apply(response);
+                outputSink.feed(vsr);
+                metrics.addRowsProcessed(vsr.getRowCount());
+
+                if (isLast) {
+                    metrics.incrementTasksCompleted();
+                    onShardTerminated();
+                }
             }
 
             @Override
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
index d53c19d55bb10..f633477602e24 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
@@ -36,6 +36,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 
@@ -129,6 +130,39 @@ public void testRowResponseSafeOnCancellation() {
         assertTrue("sink should not have received anything post-cancel", sink.fed.isEmpty());
     }
 
+    /**
+     * Reproduces the lookahead-reordering bug: if the listener offloads
+     * {@code onStreamResponse} bodies onto a multi-threaded executor, the
+     * {@code isLast=true} task can complete before earlier batches' tasks
+     * have started — flipping the stage to SUCCEEDED so the still-queued
+     * earlier tasks short-circuit via {@code isDone()} and drop their data.
+     *
+     * 
            Forces the worst-case schedule deterministically with a manual
+     * executor that runs tasks in reverse submission order.
+     */
+    public void testListenerPreservesBatchesAcrossReorderedExecution() {
+        AtomicReference> capturedListener = new AtomicReference<>();
+        CapturingSink sink = new CapturingSink();
+        ManualExecutor manualExecutor = new ManualExecutor();
+
+        ShardFragmentStageExecution exec = buildExecution(sink, true, capturedListener, manualExecutor);
+        exec.start();
+
+        FragmentExecutionArrowResponse first = new FragmentExecutionArrowResponse(createTestBatch(2));
+        FragmentExecutionArrowResponse last = new FragmentExecutionArrowResponse(createTestBatch(3));
+
+        capturedListener.get().onStreamResponse(first, false);
+        capturedListener.get().onStreamResponse(last, true);
+
+        // Force the bug's worst case: isLast=true task runs first, transitions
+        // to SUCCEEDED; the earlier task then sees isDone()=true if the
+        // implementation is still offloading to the executor.
+        manualExecutor.runInReverseOrder();
+
+        assertEquals("both batches must be fed despite reverse-order execution", 2, sink.fed.size());
+        sink.close();
+    }
+
     /**
      * Verifies that RowResponseCodec rejects null allocator.
      */
@@ -144,9 +178,19 @@ private  ShardFragmentStageExecution buildExecution(
         CapturingSink sink,
         boolean streaming,
         AtomicReference> listenerCapture
+    ) {
+        return buildExecution(sink, streaming, listenerCapture, Runnable::run);
+    }
+
+    @SuppressWarnings("unchecked")
+    private  ShardFragmentStageExecution buildExecution(
+        CapturingSink sink,
+        boolean streaming,
+        AtomicReference> listenerCapture,
+        Executor searchExecutor
     ) {
         Stage stage = mockStage();
-        QueryContext config = mockQueryContext();
+        QueryContext config = mockQueryContext(searchExecutor);
         ClusterService clusterService = mockClusterService();
         AnalyticsSearchTransportService dispatcher = mock(AnalyticsSearchTransportService.class);
         when(dispatcher.isStreamingEnabled()).thenReturn(streaming);
@@ -206,14 +250,39 @@ private Stage mockStage() {
     }
 
     private QueryContext mockQueryContext() {
+        return mockQueryContext(Runnable::run);
+    }
+
+    private QueryContext mockQueryContext(Executor searchExecutor) {
         QueryContext config = mock(QueryContext.class);
-        when(config.searchExecutor()).thenReturn(Runnable::run);
+        when(config.searchExecutor()).thenReturn(searchExecutor);
         when(config.parentTask()).thenReturn(mock(AnalyticsQueryTask.class));
         when(config.maxConcurrentShardRequests()).thenReturn(5);
         when(config.bufferAllocator()).thenReturn(allocator);
         return config;
     }
 
+    /**
+     * Test executor that records submitted tasks instead of running them.
+     * Run them explicitly in any order to deterministically simulate the
+     * worst-case multi-thread completion order on a real pool.
+     */
+    private static final class ManualExecutor implements Executor {
+        private final List queue = new ArrayList<>();
+
+        @Override
+        public synchronized void execute(Runnable command) {
+            queue.add(command);
+        }
+
+        synchronized void runInReverseOrder() {
+            for (int i = queue.size() - 1; i >= 0; i--) {
+                queue.get(i).run();
+            }
+            queue.clear();
+        }
+    }
+
     private ClusterService mockClusterService() {
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(mock(ClusterState.class));
diff --git a/sandbox/qa/analytics-engine-coordinator/build.gradle b/sandbox/qa/analytics-engine-coordinator/build.gradle
new file mode 100644
index 0000000000000..d0dd0dc96defa
--- /dev/null
+++ b/sandbox/qa/analytics-engine-coordinator/build.gradle
@@ -0,0 +1,134 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Coordinator-level integration tests for analytics-engine. Lives outside
+ * the analytics-engine plugin so the test classpath can pull in the
+ * DataFusion backend, Arrow Flight, and the test-ppl frontend without
+ * dragging those deps onto the plugin's own classpath.
+ */
+
+apply plugin: 'opensearch.internal-cluster-test'
+
+java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) }
+
+// Calcite transitively brings Guava onto the test compile classpath; OpenSearch's
+// forbidden-dependencies check rejects Guava on compileClasspath. Allow it here for ITs.
+configurations {
+  compileClasspath { exclude group: 'com.google.guava' }
+  testCompileClasspath { exclude group: 'com.google.guava' }
+}
+
+dependencies {
+  // Test framework provides :server, JUnit, MockTransportService, MockCommitterEnginePlugin,
+  // OpenSearchIntegTestCase, and the rest of the IT infrastructure.
+  internalClusterTestImplementation project(':test:framework')
+
+  // Plugin under test
+  internalClusterTestImplementation project(':sandbox:plugins:analytics-engine')
+
+  // Arrow Flight streaming transport — provides the streaming TransportService.
+  internalClusterTestImplementation project(':plugins:arrow-flight-rpc')
+
+  // DataFusion backend exercised end-to-end. Test-only — production analytics-engine
+  // doesn't link against a specific backend.
+  internalClusterTestImplementation project(':sandbox:plugins:analytics-backend-datafusion')
+
+  // Parquet data format — primary data format for the resilience tests.
+  internalClusterTestImplementation project(':sandbox:plugins:parquet-data-format')
+  // Composite engine plugin — provides the composite format dispatcher.
+  internalClusterTestImplementation project(':sandbox:plugins:composite-engine')
+  // TestPPLPlugin + UnifiedPPL action for driving queries from ITs.
+  internalClusterTestImplementation project(':sandbox:plugins:test-ppl-frontend')
+
+  // Guava is excluded from analytics-engine's runtime classpath (provided by
+  // arrow-flight-rpc at runtime in production). The classpath-plugin test
+  // launcher doesn't hydrate the extended-plugin classloader, so Guava must
+  // be on the test runtime classpath here.
+  internalClusterTestRuntimeOnly "com.google.guava:guava:33.3.1-jre"
+  internalClusterTestRuntimeOnly "com.google.guava:failureaccess:1.0.1"
+}
+
+tasks.withType(JavaCompile).configureEach {
+  options.compilerArgs -= '-Werror'
+}
+
+// internalClusterTest runs on a flat classpath (no plugin classloader for the classpath
+// plugins). Calcite's SqlKind clinit pulls com.google.common.collect.Sets, so Guava MUST
+// be present. Hard-attach via a detached configuration that bypasses any inherited excludes.
+def guavaRuntimeJars = configurations.detachedConfiguration(
+  dependencies.create('com.google.guava:guava:33.3.1-jre') { transitive = false },
+  dependencies.create('com.google.guava:failureaccess:1.0.1') { transitive = false },
+  dependencies.create("org.slf4j:slf4j-api:${versions.slf4j}") { transitive = false },
+  dependencies.create("commons-codec:commons-codec:${versions.commonscodec}") { transitive = false },
+  dependencies.create("com.fasterxml.jackson.core:jackson-core:${versions.jackson}") { transitive = false },
+  dependencies.create("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}") { transitive = false },
+  dependencies.create("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}") { transitive = false }
+)
+sourceSets.internalClusterTest.runtimeClasspath += guavaRuntimeJars
+
+configurations.all {
+  // okhttp-aws-signer is a transitive of unified-query-common, only published on JitPack.
+  exclude group: 'com.github.babbel', module: 'okhttp-aws-signer'
+
+  resolutionStrategy {
+    force 'com.google.guava:guava:33.3.1-jre'
+    force 'com.google.guava:failureaccess:1.0.1'
+    force 'com.google.errorprone:error_prone_annotations:2.36.0'
+    force 'org.checkerframework:checker-qual:3.43.0'
+    force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
+    force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+    force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+    force "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}"
+    force "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
+    force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
+    force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}"
+    force "org.slf4j:slf4j-api:${versions.slf4j}"
+    force "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}"
+    force "org.locationtech.jts:jts-core:${versions.jts}"
+    force "commons-codec:commons-codec:${versions.commonscodec}"
+    force "joda-time:joda-time:2.12.7"
+    force "org.yaml:snakeyaml:2.4"
+    force "org.codehaus.janino:janino:3.1.12"
+    force "org.codehaus.janino:commons-compiler:3.1.12"
+    force "commons-io:commons-io:${versions.commonsio}"
+    force "org.apache.commons:commons-lang3:3.18.0"
+    force "org.apache.commons:commons-text:1.11.0"
+    force "commons-logging:commons-logging:1.3.5"
+    force "net.minidev:json-smart:2.5.2"
+    force "org.apache.httpcomponents.client5:httpclient5:5.6"
+    force "org.apache.httpcomponents.core5:httpcore5:5.4"
+    force "com.squareup.okhttp3:okhttp:4.12.0"
+    force "org.jetbrains.kotlin:kotlin-stdlib:1.8.21"
+    force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.8.21"
+    force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.8.21"
+    force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10"
+    force "org.apache.logging.log4j:log4j-api:${versions.log4j}"
+    force "org.apache.logging.log4j:log4j-core:${versions.log4j}"
+    // io.substrait:core (transitively via analytics-backend-datafusion) drags
+    // protobuf-java 3.25.8; project :server uses 3.25.9. Force the higher version.
+    force "com.google.protobuf:protobuf-java:3.25.9"
+  }
+}
+
+// Arrow/Flight requires these JVM flags. DataFusion backend requires the native lib
+// path so JNI can locate libopensearch_native.dylib built by dataformat-native.
+internalClusterTest {
+  jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED'
+  jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED'
+  jvmArgs '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED'
+  jvmArgs '--enable-native-access=ALL-UNNAMED'
+  jvmArgs '-Darrow.memory.debug.allocator=false'
+  systemProperty 'io.netty.allocator.numDirectArenas', '1'
+  systemProperty 'io.netty.noUnsafe', 'false'
+  systemProperty 'io.netty.tryUnsafe', 'true'
+  systemProperty 'io.netty.tryReflectionSetAccessible', 'true'
+  systemProperty 'native.lib.path', project(':sandbox:libs:dataformat-native').ext.nativeLibPath.absolutePath
+  dependsOn ':sandbox:libs:dataformat-native:buildRustLibrary'
+  jvmArgs += ["--add-opens", "java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED"]
+}
diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java
new file mode 100644
index 0000000000000..ba701a2050e41
--- /dev/null
+++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java
@@ -0,0 +1,238 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.resilience;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.opensearch.Version;
+import org.opensearch.action.admin.indices.create.CreateIndexResponse;
+import org.opensearch.analytics.AnalyticsPlugin;
+import org.opensearch.analytics.exec.action.FragmentExecutionAction;
+import org.opensearch.analytics.exec.action.FragmentExecutionArrowResponse;
+import org.opensearch.arrow.flight.transport.FlightStreamPlugin;
+import org.opensearch.be.datafusion.DataFusionPlugin;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.composite.CompositeDataFormatPlugin;
+import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin;
+import org.opensearch.parquet.ParquetDataFormatPlugin;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.plugins.PluginInfo;
+import org.opensearch.ppl.TestPPLPlugin;
+import org.opensearch.ppl.action.PPLRequest;
+import org.opensearch.ppl.action.PPLResponse;
+import org.opensearch.ppl.action.UnifiedPPLExecuteAction;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.transport.MockTransportService;
+import org.opensearch.transport.TransportService;
+import org.junit.After;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
+
+/**
+ * Demonstrates {@link MockTransportService#addRequestHandlingBehavior} routing
+ * to handlers registered on the streaming transport. The analytics-engine
+ * registers {@link FragmentExecutionAction#NAME} on
+ * {@link org.opensearch.transport.StreamTransportService} when streaming is
+ * enabled (its handler runs on the streaming side, not the regular transport),
+ * so test-only request stubbing previously had no way to intercept it.
+ *
+ * 
            The change in this PR makes {@code addRequestHandlingBehavior} fall back
+ * to the streaming-transport's stub registry when the action is not found in
+ * the regular transport. This IT exercises that path end-to-end.
+ *
+ * @opensearch.internal
+ */
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 3, numClientNodes = 0)
+public class CoordinatorResilienceIT extends OpenSearchIntegTestCase {
+
+    private static final String INDEX = "resilience_idx";
+    private static final int NUM_SHARDS = 3;
+    private static final int DOCS_PER_SHARD_TARGET = 10;
+    private static final int VALUE = 7;
+    private static final int TOTAL_DOCS = NUM_SHARDS * DOCS_PER_SHARD_TARGET;
+    private static final long EXPECTED_SUM = (long) TOTAL_DOCS * VALUE;
+    private static final TimeValue QUERY_TIMEOUT = TimeValue.timeValueSeconds(30);
+
+    private BufferAllocator stubAllocator;
+
+    @After
+    public void closeStubAllocator() {
+        if (stubAllocator != null) {
+            stubAllocator.close();
+            stubAllocator = null;
+        }
+    }
+
+    @Override
+    protected Collection> nodePlugins() {
+        return List.of(
+            TestPPLPlugin.class,
+            FlightStreamPlugin.class,
+            CompositeDataFormatPlugin.class,
+            MockTransportService.TestPlugin.class,
+            // Stub committer factory satisfies the EngineConfigFactory boot-time
+            // check (`committerFactories.isEmpty() && isPluggableDataFormatEnabled`)
+            // without pulling the Lucene backend onto the IT classpath.
+            MockCommitterEnginePlugin.class
+        );
+    }
+
+    @Override
+    protected Collection additionalNodePlugins() {
+        return List.of(
+            classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()),
+            classpathPlugin(ParquetDataFormatPlugin.class, Collections.emptyList()),
+            classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName()))
+        );
+    }
+
+    private static PluginInfo classpathPlugin(Class pluginClass, List extendedPlugins) {
+        return new PluginInfo(
+            pluginClass.getName(),
+            "classpath plugin",
+            "NA",
+            Version.CURRENT,
+            "1.8",
+            pluginClass.getName(),
+            null,
+            extendedPlugins,
+            false
+        );
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true)
+            .put(FeatureFlags.STREAM_TRANSPORT, true)
+            .build();
+    }
+
+    /**
+     * Stubs one shard's {@link FragmentExecutionAction} entirely — no real
+     * handler runs, the data node never produces real data. Instead the stub
+     * returns a single zero-row Arrow batch carrying a minimal schema, then
+     * completes the stream. Coordinator must still produce a valid (smaller)
+     * result from the other two shards.
+     *
+     * 
            Exercises the streaming-fallback path in {@link MockTransportService}:
+     * {@link FragmentExecutionAction#NAME} is registered only on the streaming
+     * transport, so without the fallback the stub would never bind.
+     */
+    public void testStubReplacesStreamingShardResponseWithEmptyBatch() throws Exception {
+        createAndSeedIndex();
+        stubAllocator = new RootAllocator();
+        Schema schema = new Schema(List.of(new Field("value", FieldType.nullable(new ArrowType.Int(32, true)), null)));
+
+        AtomicInteger stubCalls = new AtomicInteger();
+        String victim = pickShardHostingNode();
+        MockTransportService mts = (MockTransportService) internalCluster().getInstance(TransportService.class, victim);
+        mts.addRequestHandlingBehavior(FragmentExecutionAction.NAME, (handler, request, channel, task) -> {
+            stubCalls.incrementAndGet();
+            VectorSchemaRoot vsr = VectorSchemaRoot.create(schema, stubAllocator);
+            vsr.allocateNew();
+            vsr.setRowCount(0);
+            // sendResponseBatch transfers buffer ownership to the wire. Honors the Flight protocol
+            // invariant that ≥1 schema-bearing frame must precede completeStream.
+            channel.sendResponseBatch(new FragmentExecutionArrowResponse(vsr));
+            channel.completeStream();
+        });
+        try {
+            PPLResponse response = executePPL("source = " + INDEX + " | stats sum(value) as total", QUERY_TIMEOUT);
+            assertThat("stub must fire on the streaming-only fragment action", stubCalls.get(), greaterThan(0));
+            assertNotNull("coordinator must produce a response when one shard contributes nothing", response);
+            long actual = ((Number) response.getRows().get(0)[response.getColumns().indexOf("total")]).longValue();
+            assertThat("Partial sum must be < full when a shard contributes nothing; got " + actual, actual, lessThan(EXPECTED_SUM));
+            assertThat("Partial sum must be ≥ 0 given the other two shards' contribution", actual, greaterThan(-1L));
+        } finally {
+            mts.clearAllRules();
+        }
+    }
+
+    /**
+     * Creates + seeds the test index. Composite-parquet flush-durability is
+     * not synchronous with prepareFlush().get(), so we assertBusy on the
+     * analytics-path sum until the seed is visible.
+     */
+    private void createAndSeedIndex() {
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, NUM_SHARDS)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put("index.pluggable.dataformat.enabled", true)
+            .put("index.pluggable.dataformat", "composite")
+            .put("index.composite.primary_data_format", "parquet")
+            .putList("index.composite.secondary_data_formats")
+            .build();
+
+        CreateIndexResponse response = client().admin()
+            .indices()
+            .prepareCreate(INDEX)
+            .setSettings(indexSettings)
+            .setMapping("value", "type=integer")
+            .get();
+        assertTrue("index creation must be acknowledged", response.isAcknowledged());
+        ensureGreen(INDEX);
+
+        for (int i = 0; i < TOTAL_DOCS; i++) {
+            client().prepareIndex(INDEX).setId(String.valueOf(i)).setSource("value", VALUE).get();
+        }
+        client().admin().indices().prepareRefresh(INDEX).get();
+        client().admin().indices().prepareFlush(INDEX).get();
+        try {
+            assertBusy(() -> {
+                PPLResponse r = executePPL("source = " + INDEX + " | stats sum(value) as total");
+                long actual = ((Number) r.getRows().get(0)[r.getColumns().indexOf("total")]).longValue();
+                assertEquals("seed not yet visible to analytics path", EXPECTED_SUM, actual);
+            }, 30, TimeUnit.SECONDS);
+        } catch (Exception e) {
+            throw new AssertionError("createAndSeedIndex: timed out waiting for seed durability", e);
+        }
+    }
+
+    private PPLResponse executePPL(String ppl) {
+        return client().execute(UnifiedPPLExecuteAction.INSTANCE, new PPLRequest(ppl)).actionGet();
+    }
+
+    private PPLResponse executePPL(String ppl, TimeValue timeout) {
+        return client().execute(UnifiedPPLExecuteAction.INSTANCE, new PPLRequest(ppl)).actionGet(timeout);
+    }
+
+    /** Return one node name that currently hosts a primary of {@link #INDEX}. */
+    private String pickShardHostingNode() {
+        Map out = new HashMap<>();
+        for (ShardRouting sr : clusterService().state()
+            .routingTable()
+            .index(INDEX)
+            .shardsWithState(org.opensearch.cluster.routing.ShardRoutingState.STARTED)) {
+            if (sr.primary()) {
+                out.put(sr.id(), clusterService().state().nodes().get(sr.currentNodeId()).getName());
+            }
+        }
+        return out.values().iterator().next();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index dac863b24285e..8dd4fdc75c753 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -1347,7 +1347,7 @@ protected Node(final Environment initialEnvironment, Collection clas
             if (FeatureFlags.isEnabled(STREAM_TRANSPORT) && streamTransportSupplier == null) {
                 throw new IllegalStateException(STREAM_TRANSPORT + " is enabled but no stream transport supplier is provided");
             }
-            final Transport streamTransport = (streamTransportSupplier != null ? streamTransportSupplier.get() : null);
+            final Transport streamTransport = wrapStreamTransport(streamTransportSupplier != null ? streamTransportSupplier.get() : null);
 
             Set taskHeaders = Stream.concat(
                 pluginsService.filterPlugins(ActionPlugin.class).stream().flatMap(p -> p.getTaskHeaders().stream()),
@@ -1832,6 +1832,18 @@ protected TransportService newTransportService(
         );
     }
 
+    /**
+     * Hook to wrap the stream transport before it is shared between the
+     * regular {@link TransportService} and {@link StreamTransportService}.
+     * Default returns its input unchanged. Test-framework subclasses (e.g.
+     * {@code MockNode}) override to install a stubbable wrapper so
+     * test-only request-handler interception works on the streaming path
+     * too.
+     */
+    protected Transport wrapStreamTransport(@Nullable Transport streamTransport) {
+        return streamTransport;
+    }
+
     /**
      * The settings that are used by this node. Contains original settings as well as additional settings provided by plugins.
      */
diff --git a/test/framework/src/main/java/org/opensearch/node/MockNode.java b/test/framework/src/main/java/org/opensearch/node/MockNode.java
index 8297e6b066cde..23ef62dcaf02e 100644
--- a/test/framework/src/main/java/org/opensearch/node/MockNode.java
+++ b/test/framework/src/main/java/org/opensearch/node/MockNode.java
@@ -67,6 +67,7 @@
 import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.test.MockHttpTransport;
 import org.opensearch.test.transport.MockTransportService;
+import org.opensearch.test.transport.StubbableTransport;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.Transport;
 import org.opensearch.transport.TransportInterceptor;
@@ -216,6 +217,25 @@ protected ScriptService newScriptService(Settings settings, Map taskHeaders,
         Tracer tracer
     ) {
+        // streamTransport may already be a StubbableTransport when MockNode
+        // installed the wrapper via wrapStreamTransport — in that case we
+        // share the SAME instance so the streamTransportService and this
+        // MockTransportService see the same handler registry. Wrap only if
+        // it's a plain Transport (legacy callers that bypass the Node hook).
         this(
             settings,
             new StubbableTransport(transport),
-            streamTransport != null ? new StubbableTransport(streamTransport) : null,
+            asStubbableStreamTransport(streamTransport),
             threadPool,
             interceptor,
             localNodeFactory,
@@ -275,6 +283,12 @@ public MockTransportService(
         );
     }
 
+    private static StubbableTransport asStubbableStreamTransport(@Nullable Transport streamTransport) {
+        if (streamTransport == null) return null;
+        if (streamTransport instanceof StubbableTransport stubbable) return stubbable;
+        return new StubbableTransport(streamTransport);
+    }
+
     private MockTransportService(
         Settings settings,
         StubbableTransport transport,
@@ -299,6 +313,7 @@ private MockTransportService(
             tracer
         );
         this.original = transport.getDelegate();
+        this.streamTransportStub = streamTransport;
     }
 
     private static TransportAddress[] extractTransportAddresses(TransportService transportService) {
@@ -584,12 +599,32 @@ public void clearCallback() {
 
     /**
      * Adds a new handling behavior that is used when the defined request is received.
+     *
+     * 
            When the streaming transport is in use (e.g. {@code FlightStreamPlugin}
+     * is loaded), {@code FragmentExecutionAction.NAME}-style handlers are
+     * registered on {@link StreamTransportService}'s underlying transport, not
+     * on the regular transport. We try the regular transport's registry first
+     * (production-typical actions); if no handler is registered there, we fall
+     * back to the streaming transport's registry. Either way, the behavior
+     * fires when the matching request arrives.
      */
     public  void addRequestHandlingBehavior(
         String actionName,
         StubbableTransport.RequestHandlingBehavior handlingBehavior
     ) {
-        transport().addRequestHandlingBehavior(actionName, handlingBehavior);
+        StubbableTransport stub = transport();
+        if (stub.hasHandler(actionName)) {
+            stub.addRequestHandlingBehavior(actionName, handlingBehavior);
+            return;
+        }
+        if (streamTransportStub != null && streamTransportStub.hasHandler(actionName)) {
+            streamTransportStub.addRequestHandlingBehavior(actionName, handlingBehavior);
+            return;
+        }
+        // Defer to the regular transport's behavior (which throws with a
+        // useful message) so the caller error matches what they'd get
+        // pre-streaming.
+        stub.addRequestHandlingBehavior(actionName, handlingBehavior);
     }
 
     /**
diff --git a/test/framework/src/main/java/org/opensearch/test/transport/StubbableTransport.java b/test/framework/src/main/java/org/opensearch/test/transport/StubbableTransport.java
index 11e1bdf8dbcd6..6e5573de0941c 100644
--- a/test/framework/src/main/java/org/opensearch/test/transport/StubbableTransport.java
+++ b/test/framework/src/main/java/org/opensearch/test/transport/StubbableTransport.java
@@ -105,6 +105,16 @@  void addRequestHandlingBehavior(String action
         requestHandlers.forceRegister(newRegistry);
     }
 
+    /**
+     * Returns {@code true} if the underlying delegate transport has a
+     * registered request handler for the given action name. Used by
+     * {@link MockTransportService#addRequestHandlingBehavior(String, RequestHandlingBehavior)}
+     * to decide which transport (regular vs streaming) owns the action.
+     */
+    boolean hasHandler(String actionName) {
+        return delegate.getRequestHandlers().getHandler(actionName) != null;
+    }
+
     void clearBehaviors() {
         clearOutboundBehaviors();
         clearInboundBehaviors();

From 68acecf8185054c290a47cc3bd6492d8ff83b570 Mon Sep 17 00:00:00 2001
From: Ajay Raj Nelapudi 
Date: Mon, 11 May 2026 23:51:40 +0530
Subject: [PATCH 110/115] Refactor datafusion stats - simplify structure and
 naming (#21596)

Signed-off-by: Ajay Raj Nelapudi 
Co-authored-by: Ajay Raj Nelapudi 
---
 .../action/DataFusionStatsAction.java         |   4 +-
 .../be/datafusion/nativelib/NativeBridge.java |   3 +-
 .../be/datafusion/nativelib/StatsLayout.java  |  15 +-
 .../stats/NativeExecutorsStats.java           | 216 -------------
 .../be/datafusion/stats/RuntimeMetrics.java   | 157 ++++++++++
 .../be/datafusion/stats/TaskMonitorStats.java |  88 ++++++
 .../nativelib/StatsLayoutPropertyTests.java   |   4 +-
 .../stats/DataFusionStatsPropertyTests.java   |  54 ++--
 .../stats/NativeExecutorsStatsTests.java      |   2 -
 .../NodeStatsNativeMetricRoundTripTests.java  |   2 -
 .../StatsEndpointRefactorPropertyTests.java   | 293 ++++++++++++++++++
 .../action/DataFusionStatsActionTests.java    |  15 +-
 .../nativelib/StatsLayoutTests.java           |   6 +-
 .../stats/DataFusionStatsTests.java           |  22 +-
 14 files changed, 597 insertions(+), 284 deletions(-)
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/RuntimeMetrics.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/TaskMonitorStats.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/StatsEndpointRefactorPropertyTests.java

diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/DataFusionStatsAction.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/DataFusionStatsAction.java
index e6cd90c0e2759..8979cb59be5a1 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/DataFusionStatsAction.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/action/DataFusionStatsAction.java
@@ -20,7 +20,7 @@
 import java.util.List;
 
 /**
- * REST handler for {@code GET _plugins/datafusion/stats}.
+ * REST handler for {@code GET _plugins/analytics_backend_datafusion/stats}.
  * 
            
  * Collects native executor metrics (Tokio runtime + task monitors) from
  * {@link DataFusionService} and returns them as a JSON response. Follows
@@ -47,7 +47,7 @@ public String getName() {
 
     @Override
     public List routes() {
-        return List.of(new Route(RestRequest.Method.GET, "_plugins/datafusion/stats"));
+        return List.of(new Route(RestRequest.Method.GET, "_plugins/analytics_backend_datafusion/stats"));
     }
 
     @Override
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
index eb778becc0244..eb87bae306549 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
@@ -11,6 +11,7 @@
 import org.opensearch.analytics.backend.jni.NativeHandle;
 import org.opensearch.be.datafusion.stats.DataFusionStats;
 import org.opensearch.be.datafusion.stats.NativeExecutorsStats;
+import org.opensearch.be.datafusion.stats.TaskMonitorStats;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.nativebridge.spi.NativeCall;
 import org.opensearch.nativebridge.spi.NativeLibraryLoader;
@@ -665,7 +666,7 @@ public static DataFusionStats stats() {
             var cpuRuntime = StatsLayout.readRuntimeMetrics(seg, "cpu_runtime");
 
             // Task monitors
-            var taskMonitors = new LinkedHashMap();
+            var taskMonitors = new LinkedHashMap();
             for (NativeExecutorsStats.OperationType op : NativeExecutorsStats.OperationType.values()) {
                 taskMonitors.put(op.key(), StatsLayout.readTaskMonitor(seg, op.key()));
             }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/StatsLayout.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/StatsLayout.java
index 5ca11b1b8c633..f4db6ac5cf738 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/StatsLayout.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/StatsLayout.java
@@ -8,7 +8,8 @@
 
 package org.opensearch.be.datafusion.nativelib;
 
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats;
+import org.opensearch.be.datafusion.stats.RuntimeMetrics;
+import org.opensearch.be.datafusion.stats.TaskMonitorStats;
 
 import java.lang.foreign.MemoryLayout;
 import java.lang.foreign.MemoryLayout.PathElement;
@@ -120,9 +121,9 @@ public static long readField(MemorySegment seg, String group, String field) {
      * @param group "io_runtime" or "cpu_runtime"
      * @return a populated RuntimeMetrics instance
      */
-    public static NativeExecutorsStats.RuntimeMetrics readRuntimeMetrics(MemorySegment seg, String group) {
+    public static RuntimeMetrics readRuntimeMetrics(MemorySegment seg, String group) {
         VarHandle[] handles = runtimeHandles(group);
-        return new NativeExecutorsStats.RuntimeMetrics(
+        return new RuntimeMetrics(
             (long) handles[0].get(seg, 0L),
             (long) handles[1].get(seg, 0L),
             (long) handles[2].get(seg, 0L),
@@ -142,13 +143,9 @@ public static NativeExecutorsStats.RuntimeMetrics readRuntimeMetrics(MemorySegme
      * @param group "query_execution", "stream_next", "fetch_phase", or "segment_stats"
      * @return a populated TaskMonitorStats instance
      */
-    public static NativeExecutorsStats.TaskMonitorStats readTaskMonitor(MemorySegment seg, String group) {
+    public static TaskMonitorStats readTaskMonitor(MemorySegment seg, String group) {
         VarHandle[] handles = taskMonitorHandles(group);
-        return new NativeExecutorsStats.TaskMonitorStats(
-            (long) handles[0].get(seg, 0L),
-            (long) handles[1].get(seg, 0L),
-            (long) handles[2].get(seg, 0L)
-        );
+        return new TaskMonitorStats((long) handles[0].get(seg, 0L), (long) handles[1].get(seg, 0L), (long) handles[2].get(seg, 0L));
     }
 
     // ---- Private helpers ----
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/NativeExecutorsStats.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/NativeExecutorsStats.java
index 654e666faa7bd..c8312fcf52a24 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/NativeExecutorsStats.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/NativeExecutorsStats.java
@@ -102,8 +102,6 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject("native_executors");
-
         builder.startObject("io_runtime");
         ioRuntime.toXContent(builder);
         builder.endObject();
@@ -114,15 +112,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.endObject();
         }
 
-        builder.startObject("task_monitors");
         for (Map.Entry entry : taskMonitors.entrySet()) {
             builder.startObject(entry.getKey());
             entry.getValue().toXContent(builder);
             builder.endObject();
         }
-        builder.endObject(); // task_monitors
-
-        builder.endObject(); // native_executors
         return builder;
     }
 
@@ -156,214 +150,4 @@ public int hashCode() {
         return Objects.hash(ioRuntime, cpuRuntime, taskMonitors);
     }
 
-    /**
-     * 8 fields from {@code tokio_metrics::RuntimeMonitor} describing
-     * per-worker thread pool behavior for a single Tokio runtime.
-     */
-    public static class RuntimeMetrics implements Writeable {
-        /** Number of worker threads in the runtime. */
-        public final long workersCount;
-        /** Total number of task polls across all workers. */
-        public final long totalPollsCount;
-        /** Total time workers spent executing tasks, in milliseconds. */
-        public final long totalBusyDurationMs;
-        /** Total number of times tasks were pushed to the overflow queue. */
-        public final long totalOverflowCount;
-        /** Current depth of the global injection queue. */
-        public final long globalQueueDepth;
-        /** Current depth of the blocking thread pool queue. */
-        public final long blockingQueueDepth;
-        /** Number of tasks currently alive (spawned but not yet completed) on this runtime. */
-        public final long numAliveTasks;
-        /** Total number of tasks spawned on this runtime since creation. */
-        public final long spawnedTasksCount;
-        /** Sum of all per-worker local queue depths (tasks queued on worker-local run queues). */
-        public final long totalLocalQueueDepth;
-
-        /**
-         * Construct from explicit field values.
-         *
-         * @param workersCount        number of worker threads
-         * @param totalPollsCount     total task polls across all workers
-         * @param totalBusyDurationMs total busy time in milliseconds
-         * @param totalOverflowCount  total overflow queue pushes
-         * @param globalQueueDepth    current global injection queue depth
-         * @param blockingQueueDepth  current blocking thread pool queue depth
-         * @param numAliveTasks       tasks currently alive
-         * @param spawnedTasksCount   total tasks spawned since creation
-         * @param totalLocalQueueDepth     sum of per-worker local queue depths
-         */
-        public RuntimeMetrics(
-            long workersCount,
-            long totalPollsCount,
-            long totalBusyDurationMs,
-            long totalOverflowCount,
-            long globalQueueDepth,
-            long blockingQueueDepth,
-            long numAliveTasks,
-            long spawnedTasksCount,
-            long totalLocalQueueDepth
-        ) {
-            this.workersCount = workersCount;
-            this.totalPollsCount = totalPollsCount;
-            this.totalBusyDurationMs = totalBusyDurationMs;
-            this.totalOverflowCount = totalOverflowCount;
-            this.globalQueueDepth = globalQueueDepth;
-            this.blockingQueueDepth = blockingQueueDepth;
-            this.numAliveTasks = numAliveTasks;
-            this.spawnedTasksCount = spawnedTasksCount;
-            this.totalLocalQueueDepth = totalLocalQueueDepth;
-        }
-
-        /**
-         * Deserialize from stream.
-         *
-         * @param in the stream input
-         * @throws IOException if deserialization fails
-         */
-        public RuntimeMetrics(StreamInput in) throws IOException {
-            this.workersCount = in.readVLong();
-            this.totalPollsCount = in.readVLong();
-            this.totalBusyDurationMs = in.readVLong();
-            this.totalOverflowCount = in.readVLong();
-            this.globalQueueDepth = in.readVLong();
-            this.blockingQueueDepth = in.readVLong();
-            this.numAliveTasks = in.readVLong();
-            this.spawnedTasksCount = in.readVLong();
-            this.totalLocalQueueDepth = in.readVLong();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeVLong(workersCount);
-            out.writeVLong(totalPollsCount);
-            out.writeVLong(totalBusyDurationMs);
-            out.writeVLong(totalOverflowCount);
-            out.writeVLong(globalQueueDepth);
-            out.writeVLong(blockingQueueDepth);
-            out.writeVLong(numAliveTasks);
-            out.writeVLong(spawnedTasksCount);
-            out.writeVLong(totalLocalQueueDepth);
-        }
-
-        /**
-         * Render all 8 fields as snake_case JSON fields.
-         *
-         * @param builder the XContent builder to write to
-         * @throws IOException if writing fails
-         */
-        public void toXContent(XContentBuilder builder) throws IOException {
-            builder.field("workers_count", workersCount);
-            builder.field("total_polls_count", totalPollsCount);
-            builder.field("total_busy_duration_ms", totalBusyDurationMs);
-            builder.field("total_overflow_count", totalOverflowCount);
-            builder.field("global_queue_depth", globalQueueDepth);
-            builder.field("blocking_queue_depth", blockingQueueDepth);
-            builder.field("num_alive_tasks", numAliveTasks);
-            builder.field("spawned_tasks_count", spawnedTasksCount);
-            builder.field("total_local_queue_depth", totalLocalQueueDepth);
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            RuntimeMetrics that = (RuntimeMetrics) o;
-            return workersCount == that.workersCount
-                && totalPollsCount == that.totalPollsCount
-                && totalBusyDurationMs == that.totalBusyDurationMs
-                && totalOverflowCount == that.totalOverflowCount
-                && globalQueueDepth == that.globalQueueDepth
-                && blockingQueueDepth == that.blockingQueueDepth
-                && numAliveTasks == that.numAliveTasks
-                && spawnedTasksCount == that.spawnedTasksCount
-                && totalLocalQueueDepth == that.totalLocalQueueDepth;
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(
-                workersCount,
-                totalPollsCount,
-                totalBusyDurationMs,
-                totalOverflowCount,
-                globalQueueDepth,
-                blockingQueueDepth,
-                numAliveTasks,
-                spawnedTasksCount,
-                totalLocalQueueDepth
-            );
-        }
-    }
-
-    /**
-     * 3 duration fields per operation type from {@code tokio_metrics::TaskMonitor::cumulative()}.
-     */
-    public static class TaskMonitorStats implements Writeable {
-        /** Total time spent polling instrumented futures, in milliseconds. */
-        public final long totalPollDurationMs;
-        /** Total time tasks spent waiting in the scheduler queue, in milliseconds. */
-        public final long totalScheduledDurationMs;
-        /** Total time tasks spent idle between polls, in milliseconds. */
-        public final long totalIdleDurationMs;
-
-        /**
-         * Construct from explicit field values.
-         *
-         * @param totalPollDurationMs      total poll duration in milliseconds
-         * @param totalScheduledDurationMs total scheduled duration in milliseconds
-         * @param totalIdleDurationMs      total idle duration in milliseconds
-         */
-        public TaskMonitorStats(long totalPollDurationMs, long totalScheduledDurationMs, long totalIdleDurationMs) {
-            this.totalPollDurationMs = totalPollDurationMs;
-            this.totalScheduledDurationMs = totalScheduledDurationMs;
-            this.totalIdleDurationMs = totalIdleDurationMs;
-        }
-
-        /**
-         * Deserialize from stream.
-         *
-         * @param in the stream input
-         * @throws IOException if deserialization fails
-         */
-        public TaskMonitorStats(StreamInput in) throws IOException {
-            this.totalPollDurationMs = in.readVLong();
-            this.totalScheduledDurationMs = in.readVLong();
-            this.totalIdleDurationMs = in.readVLong();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeVLong(totalPollDurationMs);
-            out.writeVLong(totalScheduledDurationMs);
-            out.writeVLong(totalIdleDurationMs);
-        }
-
-        /**
-         * Render all 3 fields as snake_case JSON fields.
-         *
-         * @param builder the XContent builder to write to
-         * @throws IOException if writing fails
-         */
-        public void toXContent(XContentBuilder builder) throws IOException {
-            builder.field("total_poll_duration_ms", totalPollDurationMs);
-            builder.field("total_scheduled_duration_ms", totalScheduledDurationMs);
-            builder.field("total_idle_duration_ms", totalIdleDurationMs);
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            TaskMonitorStats that = (TaskMonitorStats) o;
-            return totalPollDurationMs == that.totalPollDurationMs
-                && totalScheduledDurationMs == that.totalScheduledDurationMs
-                && totalIdleDurationMs == that.totalIdleDurationMs;
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(totalPollDurationMs, totalScheduledDurationMs, totalIdleDurationMs);
-        }
-    }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/RuntimeMetrics.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/RuntimeMetrics.java
new file mode 100644
index 0000000000000..b2eef067bb940
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/RuntimeMetrics.java
@@ -0,0 +1,157 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.stats;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * 8 fields from {@code tokio_metrics::RuntimeMonitor} describing
+ * per-worker thread pool behavior for a single Tokio runtime.
+ */
+public class RuntimeMetrics implements Writeable {
+    /** Number of worker threads in the runtime. */
+    public final long workersCount;
+    /** Total number of task polls across all workers. */
+    public final long totalPollsCount;
+    /** Total time workers spent executing tasks, in milliseconds. */
+    public final long totalBusyDurationMs;
+    /** Total number of times tasks were pushed to the overflow queue. */
+    public final long totalOverflowCount;
+    /** Current depth of the global injection queue. */
+    public final long globalQueueDepth;
+    /** Current depth of the blocking thread pool queue. */
+    public final long blockingQueueDepth;
+    /** Number of tasks currently alive (spawned but not yet completed) on this runtime. */
+    public final long numAliveTasks;
+    /** Total number of tasks spawned on this runtime since creation. */
+    public final long spawnedTasksCount;
+    /** Sum of all per-worker local queue depths (tasks queued on worker-local run queues). */
+    public final long totalLocalQueueDepth;
+
+    /**
+     * Construct from explicit field values.
+     *
+     * @param workersCount        number of worker threads
+     * @param totalPollsCount     total task polls across all workers
+     * @param totalBusyDurationMs total busy time in milliseconds
+     * @param totalOverflowCount  total overflow queue pushes
+     * @param globalQueueDepth    current global injection queue depth
+     * @param blockingQueueDepth  current blocking thread pool queue depth
+     * @param numAliveTasks       tasks currently alive
+     * @param spawnedTasksCount   total tasks spawned since creation
+     * @param totalLocalQueueDepth     sum of per-worker local queue depths
+     */
+    public RuntimeMetrics(
+        long workersCount,
+        long totalPollsCount,
+        long totalBusyDurationMs,
+        long totalOverflowCount,
+        long globalQueueDepth,
+        long blockingQueueDepth,
+        long numAliveTasks,
+        long spawnedTasksCount,
+        long totalLocalQueueDepth
+    ) {
+        this.workersCount = workersCount;
+        this.totalPollsCount = totalPollsCount;
+        this.totalBusyDurationMs = totalBusyDurationMs;
+        this.totalOverflowCount = totalOverflowCount;
+        this.globalQueueDepth = globalQueueDepth;
+        this.blockingQueueDepth = blockingQueueDepth;
+        this.numAliveTasks = numAliveTasks;
+        this.spawnedTasksCount = spawnedTasksCount;
+        this.totalLocalQueueDepth = totalLocalQueueDepth;
+    }
+
+    /**
+     * Deserialize from stream.
+     *
+     * @param in the stream input
+     * @throws IOException if deserialization fails
+     */
+    public RuntimeMetrics(StreamInput in) throws IOException {
+        this.workersCount = in.readVLong();
+        this.totalPollsCount = in.readVLong();
+        this.totalBusyDurationMs = in.readVLong();
+        this.totalOverflowCount = in.readVLong();
+        this.globalQueueDepth = in.readVLong();
+        this.blockingQueueDepth = in.readVLong();
+        this.numAliveTasks = in.readVLong();
+        this.spawnedTasksCount = in.readVLong();
+        this.totalLocalQueueDepth = in.readVLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(workersCount);
+        out.writeVLong(totalPollsCount);
+        out.writeVLong(totalBusyDurationMs);
+        out.writeVLong(totalOverflowCount);
+        out.writeVLong(globalQueueDepth);
+        out.writeVLong(blockingQueueDepth);
+        out.writeVLong(numAliveTasks);
+        out.writeVLong(spawnedTasksCount);
+        out.writeVLong(totalLocalQueueDepth);
+    }
+
+    /**
+     * Render all 8 fields as snake_case JSON fields.
+     *
+     * @param builder the XContent builder to write to
+     * @throws IOException if writing fails
+     */
+    public void toXContent(XContentBuilder builder) throws IOException {
+        builder.field("workers_count", workersCount);
+        builder.field("total_polls_count", totalPollsCount);
+        builder.field("total_busy_duration_ms", totalBusyDurationMs);
+        builder.field("total_overflow_count", totalOverflowCount);
+        builder.field("global_queue_depth", globalQueueDepth);
+        builder.field("blocking_queue_depth", blockingQueueDepth);
+        builder.field("num_alive_tasks", numAliveTasks);
+        builder.field("spawned_tasks_count", spawnedTasksCount);
+        builder.field("total_local_queue_depth", totalLocalQueueDepth);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RuntimeMetrics that = (RuntimeMetrics) o;
+        return workersCount == that.workersCount
+            && totalPollsCount == that.totalPollsCount
+            && totalBusyDurationMs == that.totalBusyDurationMs
+            && totalOverflowCount == that.totalOverflowCount
+            && globalQueueDepth == that.globalQueueDepth
+            && blockingQueueDepth == that.blockingQueueDepth
+            && numAliveTasks == that.numAliveTasks
+            && spawnedTasksCount == that.spawnedTasksCount
+            && totalLocalQueueDepth == that.totalLocalQueueDepth;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            workersCount,
+            totalPollsCount,
+            totalBusyDurationMs,
+            totalOverflowCount,
+            globalQueueDepth,
+            blockingQueueDepth,
+            numAliveTasks,
+            spawnedTasksCount,
+            totalLocalQueueDepth
+        );
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/TaskMonitorStats.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/TaskMonitorStats.java
new file mode 100644
index 0000000000000..bed88bc83dc65
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/stats/TaskMonitorStats.java
@@ -0,0 +1,88 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.stats;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * 3 duration fields per operation type from {@code tokio_metrics::TaskMonitor::cumulative()}.
+ */
+public class TaskMonitorStats implements Writeable {
+    /** Total time spent polling instrumented futures, in milliseconds. */
+    public final long totalPollDurationMs;
+    /** Total time tasks spent waiting in the scheduler queue, in milliseconds. */
+    public final long totalScheduledDurationMs;
+    /** Total time tasks spent idle between polls, in milliseconds. */
+    public final long totalIdleDurationMs;
+
+    /**
+     * Construct from explicit field values.
+     *
+     * @param totalPollDurationMs      total poll duration in milliseconds
+     * @param totalScheduledDurationMs total scheduled duration in milliseconds
+     * @param totalIdleDurationMs      total idle duration in milliseconds
+     */
+    public TaskMonitorStats(long totalPollDurationMs, long totalScheduledDurationMs, long totalIdleDurationMs) {
+        this.totalPollDurationMs = totalPollDurationMs;
+        this.totalScheduledDurationMs = totalScheduledDurationMs;
+        this.totalIdleDurationMs = totalIdleDurationMs;
+    }
+
+    /**
+     * Deserialize from stream.
+     *
+     * @param in the stream input
+     * @throws IOException if deserialization fails
+     */
+    public TaskMonitorStats(StreamInput in) throws IOException {
+        this.totalPollDurationMs = in.readVLong();
+        this.totalScheduledDurationMs = in.readVLong();
+        this.totalIdleDurationMs = in.readVLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(totalPollDurationMs);
+        out.writeVLong(totalScheduledDurationMs);
+        out.writeVLong(totalIdleDurationMs);
+    }
+
+    /**
+     * Render all 3 fields as snake_case JSON fields.
+     *
+     * @param builder the XContent builder to write to
+     * @throws IOException if writing fails
+     */
+    public void toXContent(XContentBuilder builder) throws IOException {
+        builder.field("total_poll_duration_ms", totalPollDurationMs);
+        builder.field("total_scheduled_duration_ms", totalScheduledDurationMs);
+        builder.field("total_idle_duration_ms", totalIdleDurationMs);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TaskMonitorStats that = (TaskMonitorStats) o;
+        return totalPollDurationMs == that.totalPollDurationMs
+            && totalScheduledDurationMs == that.totalScheduledDurationMs
+            && totalIdleDurationMs == that.totalIdleDurationMs;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(totalPollDurationMs, totalScheduledDurationMs, totalIdleDurationMs);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/nativelib/StatsLayoutPropertyTests.java b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/nativelib/StatsLayoutPropertyTests.java
index 72f820d947078..39955fc74f538 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/nativelib/StatsLayoutPropertyTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/nativelib/StatsLayoutPropertyTests.java
@@ -9,8 +9,8 @@
 package org.opensearch.be.datafusion.nativelib;
 
 import org.opensearch.be.datafusion.stats.NativeExecutorsStats;
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
+import org.opensearch.be.datafusion.stats.RuntimeMetrics;
+import org.opensearch.be.datafusion.stats.TaskMonitorStats;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.io.stream.StreamInput;
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/DataFusionStatsPropertyTests.java b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/DataFusionStatsPropertyTests.java
index 9cb9a449419ac..f45eac3c31623 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/DataFusionStatsPropertyTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/DataFusionStatsPropertyTests.java
@@ -12,8 +12,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import org.opensearch.be.datafusion.stats.NativeExecutorsStats.OperationType;
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -49,7 +47,7 @@ public class DataFusionStatsPropertyTests {
 
     private static final ObjectMapper MAPPER = new ObjectMapper();
 
-    /** JSON field names for RuntimeMetrics in documented order (8 fields). */
+    /** JSON field names for RuntimeMetrics in documented order (9 fields). */
     private static final String[] RUNTIME_FIELD_NAMES = {
         "workers_count",
         "total_polls_count",
@@ -58,7 +56,8 @@ public class DataFusionStatsPropertyTests {
         "global_queue_depth",
         "blocking_queue_depth",
         "num_alive_tasks",
-        "spawned_tasks_count" };
+        "spawned_tasks_count",
+        "total_local_queue_depth" };
 
     /** JSON field names for TaskMonitorStats in documented order (3 fields). */
     private static final String[] TASK_FIELD_NAMES = { "total_poll_duration_ms", "total_scheduled_duration_ms", "total_idle_duration_ms" };
@@ -174,28 +173,23 @@ void toXContentRoundTripCpuPresent(@ForAll("dataFusionStatsCpuPresent") DataFusi
 
         String json = renderJson(stats);
         JsonNode root = MAPPER.readTree(json);
-        JsonNode nativeExecutors = root.get("native_executors");
-        assertNotNull(nativeExecutors, "native_executors must be present");
 
-        // IO runtime: 8 fields
-        JsonNode ioRuntime = nativeExecutors.get("io_runtime");
+        // IO runtime: 9 fields
+        JsonNode ioRuntime = root.get("io_runtime");
         assertNotNull(ioRuntime, "io_runtime must be present");
-        assertEquals(8, ioRuntime.size(), "io_runtime must have exactly 8 fields");
+        assertEquals(9, ioRuntime.size(), "io_runtime must have exactly 9 fields");
         verifyRuntimeFields(nes.getIoRuntime(), ioRuntime);
 
-        // CPU runtime: 8 fields
-        assertTrue(nativeExecutors.has("cpu_runtime"), "cpu_runtime must be present");
-        JsonNode cpuRuntime = nativeExecutors.get("cpu_runtime");
-        assertEquals(8, cpuRuntime.size(), "cpu_runtime must have exactly 8 fields");
+        // CPU runtime: 9 fields
+        assertTrue(root.has("cpu_runtime"), "cpu_runtime must be present");
+        JsonNode cpuRuntime = root.get("cpu_runtime");
+        assertEquals(9, cpuRuntime.size(), "cpu_runtime must have exactly 9 fields");
         verifyRuntimeFields(nes.getCpuRuntime(), cpuRuntime);
 
-        // Task monitors: 4 ops × 3 fields
-        JsonNode taskMonitors = nativeExecutors.get("task_monitors");
-        assertNotNull(taskMonitors, "task_monitors must be present");
-        assertEquals(4, taskMonitors.size());
+        // Task monitors: 4 ops × 3 fields (at top level, no task_monitors wrapper)
         for (OperationType opType : OperationType.values()) {
-            JsonNode monitor = taskMonitors.get(opType.key());
-            assertNotNull(monitor, "task_monitors." + opType.key() + " must be present");
+            JsonNode monitor = root.get(opType.key());
+            assertNotNull(monitor, opType.key() + " must be present");
             assertEquals(3, monitor.size());
             verifyTaskMonitorFields(nes.getTaskMonitors().get(opType.key()), monitor, opType.key());
         }
@@ -211,25 +205,20 @@ void toXContentRoundTripCpuAbsent(@ForAll("dataFusionStatsCpuAbsent") DataFusion
 
         String json = renderJson(stats);
         JsonNode root = MAPPER.readTree(json);
-        JsonNode nativeExecutors = root.get("native_executors");
-        assertNotNull(nativeExecutors, "native_executors must be present");
 
-        // IO runtime: 8 fields
-        JsonNode ioRuntime = nativeExecutors.get("io_runtime");
+        // IO runtime: 9 fields
+        JsonNode ioRuntime = root.get("io_runtime");
         assertNotNull(ioRuntime, "io_runtime must be present");
-        assertEquals(8, ioRuntime.size(), "io_runtime must have exactly 8 fields");
+        assertEquals(9, ioRuntime.size(), "io_runtime must have exactly 9 fields");
         verifyRuntimeFields(nes.getIoRuntime(), ioRuntime);
 
         // CPU runtime absent
-        assertFalse(nativeExecutors.has("cpu_runtime"), "cpu_runtime must be absent when cpuRuntime is null");
+        assertFalse(root.has("cpu_runtime"), "cpu_runtime must be absent when cpuRuntime is null");
 
-        // Task monitors
-        JsonNode taskMonitors = nativeExecutors.get("task_monitors");
-        assertNotNull(taskMonitors);
-        assertEquals(4, taskMonitors.size());
+        // Task monitors: at top level, no task_monitors wrapper
         for (OperationType opType : OperationType.values()) {
-            JsonNode monitor = taskMonitors.get(opType.key());
-            assertNotNull(monitor);
+            JsonNode monitor = root.get(opType.key());
+            assertNotNull(monitor, opType.key() + " must be present");
             assertEquals(3, monitor.size());
             verifyTaskMonitorFields(nes.getTaskMonitors().get(opType.key()), monitor, opType.key());
         }
@@ -308,7 +297,8 @@ private void verifyRuntimeFields(RuntimeMetrics rm, JsonNode runtimeNode) {
             rm.globalQueueDepth,
             rm.blockingQueueDepth,
             rm.numAliveTasks,
-            rm.spawnedTasksCount };
+            rm.spawnedTasksCount,
+            rm.totalLocalQueueDepth };
         for (int i = 0; i < RUNTIME_FIELD_NAMES.length; i++) {
             String fieldName = RUNTIME_FIELD_NAMES[i];
             assertTrue(runtimeNode.has(fieldName), "Runtime field '" + fieldName + "' must be present");
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NativeExecutorsStatsTests.java b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NativeExecutorsStatsTests.java
index f837bcc56d207..da67fd75a2dc9 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NativeExecutorsStatsTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NativeExecutorsStatsTests.java
@@ -9,8 +9,6 @@
 package org.opensearch.be.datafusion.stats;
 
 import org.opensearch.be.datafusion.stats.NativeExecutorsStats.OperationType;
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.io.stream.StreamInput;
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NodeStatsNativeMetricRoundTripTests.java b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NodeStatsNativeMetricRoundTripTests.java
index a65e223d00211..cdc0febd13c7b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NodeStatsNativeMetricRoundTripTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/NodeStatsNativeMetricRoundTripTests.java
@@ -9,8 +9,6 @@
 package org.opensearch.be.datafusion.stats;
 
 import org.opensearch.be.datafusion.stats.NativeExecutorsStats.OperationType;
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.io.stream.StreamInput;
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/StatsEndpointRefactorPropertyTests.java b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/StatsEndpointRefactorPropertyTests.java
new file mode 100644
index 0000000000000..106762a6eefd3
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/propertyTest/java/org/opensearch/be/datafusion/stats/StatsEndpointRefactorPropertyTests.java
@@ -0,0 +1,293 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion.stats;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.opensearch.be.datafusion.stats.NativeExecutorsStats.OperationType;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import net.jqwik.api.Arbitraries;
+import net.jqwik.api.Arbitrary;
+import net.jqwik.api.Combinators;
+import net.jqwik.api.ForAll;
+import net.jqwik.api.Property;
+import net.jqwik.api.Provide;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Property-based tests for the stats-endpoint-refactor spec.
+ *
+ * 
            Validates that the flattened JSON serialization preserves all metric values,
+ * CPU runtime conditional presence, and transport round-trip correctness.
+ *
+ * 
            Tag: Feature: stats-endpoint-refactor
+ */
+public class StatsEndpointRefactorPropertyTests {
+
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+
+    /** JSON field names for RuntimeMetrics in documented order (9 fields). */
+    private static final String[] RUNTIME_FIELD_NAMES = {
+        "workers_count",
+        "total_polls_count",
+        "total_busy_duration_ms",
+        "total_overflow_count",
+        "global_queue_depth",
+        "blocking_queue_depth",
+        "num_alive_tasks",
+        "spawned_tasks_count",
+        "total_local_queue_depth" };
+
+    /** JSON field names for TaskMonitorStats in documented order (3 fields). */
+    private static final String[] TASK_FIELD_NAMES = { "total_poll_duration_ms", "total_scheduled_duration_ms", "total_idle_duration_ms" };
+
+    // ---- Object generators ----
+
+    @Provide
+    Arbitrary runtimeMetrics() {
+        return Arbitraries.longs()
+            .between(0, Long.MAX_VALUE / 2)
+            .list()
+            .ofSize(9)
+            .map(l -> new RuntimeMetrics(l.get(0), l.get(1), l.get(2), l.get(3), l.get(4), l.get(5), l.get(6), l.get(7), l.get(8)));
+    }
+
+    @Provide
+    Arbitrary taskMonitorStats() {
+        Arbitrary nonNeg = Arbitraries.longs().between(0, Long.MAX_VALUE / 2);
+        return Combinators.combine(nonNeg, nonNeg, nonNeg).as(TaskMonitorStats::new);
+    }
+
+    /** NativeExecutorsStats with CPU runtime present (workersCount > 0). */
+    @Provide
+    Arbitrary nativeExecutorsStatsCpuPresent() {
+        return Combinators.combine(runtimeMetrics(), runtimeMetrics().map(rt -> {
+            if (rt.workersCount == 0) {
+                return new RuntimeMetrics(
+                    1,
+                    rt.totalPollsCount,
+                    rt.totalBusyDurationMs,
+                    rt.totalOverflowCount,
+                    rt.globalQueueDepth,
+                    rt.blockingQueueDepth,
+                    rt.numAliveTasks,
+                    rt.spawnedTasksCount,
+                    rt.totalLocalQueueDepth
+                );
+            }
+            return rt;
+        }), taskMonitorStats(), taskMonitorStats(), taskMonitorStats(), taskMonitorStats()).as((io, cpu, qe, sn, fp, ss) -> {
+            Map monitors = new LinkedHashMap<>();
+            monitors.put("query_execution", qe);
+            monitors.put("stream_next", sn);
+            monitors.put("fetch_phase", fp);
+            monitors.put("segment_stats", ss);
+            return new NativeExecutorsStats(io, cpu, monitors);
+        });
+    }
+
+    /** NativeExecutorsStats with CPU runtime absent (null). */
+    @Provide
+    Arbitrary nativeExecutorsStatsCpuAbsent() {
+        return Combinators.combine(runtimeMetrics(), taskMonitorStats(), taskMonitorStats(), taskMonitorStats(), taskMonitorStats())
+            .as((io, qe, sn, fp, ss) -> {
+                Map monitors = new LinkedHashMap<>();
+                monitors.put("query_execution", qe);
+                monitors.put("stream_next", sn);
+                monitors.put("fetch_phase", fp);
+                monitors.put("segment_stats", ss);
+                return new NativeExecutorsStats(io, null, monitors);
+            });
+    }
+
+    /** DataFusionStats with non-null NativeExecutorsStats (CPU present or absent). */
+    @Provide
+    Arbitrary dataFusionStats() {
+        return Arbitraries.oneOf(
+            nativeExecutorsStatsCpuPresent().map(DataFusionStats::new),
+            nativeExecutorsStatsCpuAbsent().map(DataFusionStats::new)
+        );
+    }
+
+    // ---- Property 1: Flat JSON serialization preserves all metric values at top level ----
+
+    /**
+     * Feature: stats-endpoint-refactor, Property 1: Flat JSON serialization preserves all metric values at top level.
+     *
+     * 
            For any valid NativeExecutorsStats, toXContent produces JSON with io_runtime, each task monitor,
+     * and optionally cpu_runtime as direct top-level keys with correct field values, and native_executors
+     * and task_monitors keys are absent.
+     *
+     * 
            Validates: Requirements 2.1, 2.4, 2.5, 3.1, 3.2
+     */
+    @Property(tries = 200)
+    void flatJsonSerializationPreservesAllMetricValues(@ForAll("nativeExecutorsStatsCpuPresent") NativeExecutorsStats nes)
+        throws IOException {
+        String json = renderNativeExecutorsJson(nes);
+        JsonNode root = MAPPER.readTree(json);
+
+        // Verify native_executors and task_monitors wrappers are absent
+        assertFalse(root.has("native_executors"), "native_executors wrapper must be absent");
+        assertFalse(root.has("task_monitors"), "task_monitors wrapper must be absent");
+
+        // Verify io_runtime is a top-level key with all 9 fields
+        JsonNode ioRuntime = root.get("io_runtime");
+        assertNotNull(ioRuntime, "io_runtime must be present at top level");
+        verifyRuntimeFields(nes.getIoRuntime(), ioRuntime);
+
+        // Verify cpu_runtime is a top-level key with all 9 fields (present case)
+        JsonNode cpuRuntime = root.get("cpu_runtime");
+        assertNotNull(cpuRuntime, "cpu_runtime must be present at top level when non-null");
+        verifyRuntimeFields(nes.getCpuRuntime(), cpuRuntime);
+
+        // Verify each task monitor is a top-level key with correct fields
+        for (OperationType opType : OperationType.values()) {
+            JsonNode monitor = root.get(opType.key());
+            assertNotNull(monitor, opType.key() + " must be present at top level");
+            verifyTaskMonitorFields(nes.getTaskMonitors().get(opType.key()), monitor, opType.key());
+        }
+    }
+
+    /**
+     * Feature: stats-endpoint-refactor, Property 1 (CPU absent variant).
+     *
+     * 
            Validates: Requirements 2.1, 2.4, 2.5, 3.1, 3.2
+     */
+    @Property(tries = 200)
+    void flatJsonSerializationPreservesAllMetricValuesCpuAbsent(@ForAll("nativeExecutorsStatsCpuAbsent") NativeExecutorsStats nes)
+        throws IOException {
+        String json = renderNativeExecutorsJson(nes);
+        JsonNode root = MAPPER.readTree(json);
+
+        // Verify native_executors and task_monitors wrappers are absent
+        assertFalse(root.has("native_executors"), "native_executors wrapper must be absent");
+        assertFalse(root.has("task_monitors"), "task_monitors wrapper must be absent");
+
+        // Verify io_runtime is a top-level key with all 9 fields
+        JsonNode ioRuntime = root.get("io_runtime");
+        assertNotNull(ioRuntime, "io_runtime must be present at top level");
+        verifyRuntimeFields(nes.getIoRuntime(), ioRuntime);
+
+        // cpu_runtime absent
+        assertFalse(root.has("cpu_runtime"), "cpu_runtime must be absent when null");
+
+        // Verify each task monitor is a top-level key with correct fields
+        for (OperationType opType : OperationType.values()) {
+            JsonNode monitor = root.get(opType.key());
+            assertNotNull(monitor, opType.key() + " must be present at top level");
+            verifyTaskMonitorFields(nes.getTaskMonitors().get(opType.key()), monitor, opType.key());
+        }
+    }
+
+    // ---- Property 2: CPU runtime conditional presence ----
+
+    /**
+     * Feature: stats-endpoint-refactor, Property 2: CPU runtime conditional presence (present case).
+     *
+     * 
            For any valid NativeExecutorsStats with non-null cpuRuntime, serialized JSON contains
+     * cpu_runtime top-level key with correct values.
+     *
+     * 
            Validates: Requirements 2.2, 2.3
+     */
+    @Property(tries = 200)
+    void cpuRuntimePresentWhenNonNull(@ForAll("nativeExecutorsStatsCpuPresent") NativeExecutorsStats nes) throws IOException {
+        String json = renderNativeExecutorsJson(nes);
+        JsonNode root = MAPPER.readTree(json);
+
+        assertTrue(root.has("cpu_runtime"), "cpu_runtime must be present when cpuRuntime is non-null");
+        JsonNode cpuRuntime = root.get("cpu_runtime");
+        verifyRuntimeFields(nes.getCpuRuntime(), cpuRuntime);
+    }
+
+    /**
+     * Feature: stats-endpoint-refactor, Property 2: CPU runtime conditional presence (absent case).
+     *
+     * 
            For any valid NativeExecutorsStats with null cpuRuntime, serialized JSON does not contain
+     * cpu_runtime key.
+     *
+     * 
            Validates: Requirements 2.2, 2.3
+     */
+    @Property(tries = 200)
+    void cpuRuntimeAbsentWhenNull(@ForAll("nativeExecutorsStatsCpuAbsent") NativeExecutorsStats nes) throws IOException {
+        String json = renderNativeExecutorsJson(nes);
+        JsonNode root = MAPPER.readTree(json);
+
+        assertFalse(root.has("cpu_runtime"), "cpu_runtime must be absent when cpuRuntime is null");
+    }
+
+    // ---- Property 3: Transport serialization round-trip ----
+
+    /**
+     * Feature: stats-endpoint-refactor, Property 3: Transport serialization round-trip.
+     *
+     * 
            For any valid DataFusionStats, writing to StreamOutput and reading back from StreamInput
+     * produces an object equal to the original.
+     *
+     * 
            Validates: Requirements 4.1, 4.2
+     */
+    @Property(tries = 200)
+    void transportSerializationRoundTrip(@ForAll("dataFusionStats") DataFusionStats original) throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+        StreamInput in = out.bytes().streamInput();
+        DataFusionStats deserialized = new DataFusionStats(in);
+        assertEquals(original, deserialized, "Transport round-trip must preserve all fields");
+    }
+
+    // ---- Helper methods ----
+
+    private String renderNativeExecutorsJson(NativeExecutorsStats nes) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        nes.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        return builder.toString();
+    }
+
+    private void verifyRuntimeFields(RuntimeMetrics rm, JsonNode runtimeNode) {
+        long[] expected = {
+            rm.workersCount,
+            rm.totalPollsCount,
+            rm.totalBusyDurationMs,
+            rm.totalOverflowCount,
+            rm.globalQueueDepth,
+            rm.blockingQueueDepth,
+            rm.numAliveTasks,
+            rm.spawnedTasksCount,
+            rm.totalLocalQueueDepth };
+        for (int i = 0; i < RUNTIME_FIELD_NAMES.length; i++) {
+            String fieldName = RUNTIME_FIELD_NAMES[i];
+            assertTrue(runtimeNode.has(fieldName), "Runtime field '" + fieldName + "' must be present");
+            assertEquals(expected[i], runtimeNode.get(fieldName).asLong(), "Runtime field '" + fieldName + "': expected " + expected[i]);
+        }
+    }
+
+    private void verifyTaskMonitorFields(TaskMonitorStats tm, JsonNode monitorNode, String opType) {
+        long[] expected = { tm.totalPollDurationMs, tm.totalScheduledDurationMs, tm.totalIdleDurationMs };
+        for (int i = 0; i < TASK_FIELD_NAMES.length; i++) {
+            String fieldName = TASK_FIELD_NAMES[i];
+            assertTrue(monitorNode.has(fieldName), opType + " field '" + fieldName + "' must be present");
+            assertEquals(expected[i], monitorNode.get(fieldName).asLong(), opType + " field '" + fieldName + "': expected " + expected[i]);
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/action/DataFusionStatsActionTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/action/DataFusionStatsActionTests.java
index 4e382733f82a3..5228d2acc1044 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/action/DataFusionStatsActionTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/action/DataFusionStatsActionTests.java
@@ -12,8 +12,8 @@
 import org.opensearch.be.datafusion.DataFusionService;
 import org.opensearch.be.datafusion.stats.DataFusionStats;
 import org.opensearch.be.datafusion.stats.NativeExecutorsStats;
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
+import org.opensearch.be.datafusion.stats.RuntimeMetrics;
+import org.opensearch.be.datafusion.stats.TaskMonitorStats;
 import org.opensearch.common.SuppressForbidden;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.rest.RestHandler;
@@ -67,7 +67,7 @@ public void testRoutesReturnsStatsEndpoint() {
         List routes = action.routes();
         assertEquals(1, routes.size());
         assertEquals(RestRequest.Method.GET, routes.get(0).getMethod());
-        assertEquals("_plugins/datafusion/stats", routes.get(0).getPath());
+        assertEquals("_plugins/analytics_backend_datafusion/stats", routes.get(0).getPath());
     }
 
     // ---- Test: getName() returns "datafusion_stats_action" (Requirement 1.1) ----
@@ -106,10 +106,11 @@ public void testPrepareRequestReturns200WithValidJson() throws Exception {
         // Verify the response
         assertEquals(200, channel.capturedResponse().status().getStatus());
         String responseBody = channel.capturedResponse().content().utf8ToString();
-        assertTrue("Response should contain native_executors", responseBody.contains("native_executors"));
-        assertTrue("Response should contain io_runtime", responseBody.contains("io_runtime"));
-        assertTrue("Response should contain task_monitors", responseBody.contains("task_monitors"));
-        assertTrue("Response should contain query_execution", responseBody.contains("query_execution"));
+        assertFalse("Response should NOT contain native_executors wrapper", responseBody.contains("native_executors"));
+        assertFalse("Response should NOT contain task_monitors wrapper", responseBody.contains("task_monitors"));
+        assertTrue("Response should contain io_runtime at top level", responseBody.contains("io_runtime"));
+        assertTrue("Response should contain cpu_runtime at top level", responseBody.contains("cpu_runtime"));
+        assertTrue("Response should contain query_execution at top level", responseBody.contains("query_execution"));
     }
 
     // ---- Test: prepareRequest returns 500 when service throws exception (Requirement 6.1) ----
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/nativelib/StatsLayoutTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/nativelib/StatsLayoutTests.java
index f8398cb8e2b6d..cc97263f19d04 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/nativelib/StatsLayoutTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/nativelib/StatsLayoutTests.java
@@ -8,7 +8,7 @@
 
 package org.opensearch.be.datafusion.nativelib;
 
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats;
+import org.opensearch.be.datafusion.stats.RuntimeMetrics;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.lang.foreign.Arena;
@@ -73,7 +73,7 @@ public void testCpuRuntimeNullWhenWorkersZero() {
             assertEquals(0L, cpuWorkers);
 
             // Simulate the NativeBridge logic
-            NativeExecutorsStats.RuntimeMetrics cpuRuntime = null;
+            RuntimeMetrics cpuRuntime = null;
             if (cpuWorkers > 0) {
                 cpuRuntime = StatsLayout.readRuntimeMetrics(seg, "cpu_runtime");
             }
@@ -95,7 +95,7 @@ public void testCpuRuntimeNonNullWhenWorkersPositive() {
             long cpuWorkers = StatsLayout.readField(seg, "cpu_runtime", "workers_count");
             assertEquals(5L, cpuWorkers);
 
-            NativeExecutorsStats.RuntimeMetrics cpuRuntime = null;
+            RuntimeMetrics cpuRuntime = null;
             if (cpuWorkers > 0) {
                 cpuRuntime = StatsLayout.readRuntimeMetrics(seg, "cpu_runtime");
             }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/stats/DataFusionStatsTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/stats/DataFusionStatsTests.java
index 336a82c28292f..d09b270c65f2a 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/stats/DataFusionStatsTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/stats/DataFusionStatsTests.java
@@ -8,8 +8,6 @@
 
 package org.opensearch.be.datafusion.stats;
 
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.RuntimeMetrics;
-import org.opensearch.be.datafusion.stats.NativeExecutorsStats.TaskMonitorStats;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -22,7 +20,7 @@
 /**
  * Unit tests for {@link DataFusionStats} constructed via direct constructors.
  *
- * 
            Layout: IO RuntimeMetrics (8 fields), optional CPU RuntimeMetrics (8 fields),
+ * 
            Layout: IO RuntimeMetrics (9 fields), optional CPU RuntimeMetrics (9 fields),
  * 4 TaskMonitorStats (3 fields each).
  */
 public class DataFusionStatsTests extends OpenSearchTestCase {
@@ -123,7 +121,11 @@ public void testCpuRuntimeAbsentWhenNull() throws IOException {
         String json = toJsonString(stats);
         assertFalse("cpu_runtime should be omitted when null", json.contains("cpu_runtime"));
         assertTrue("io_runtime should still be present", json.contains("io_runtime"));
-        assertTrue("task_monitors should still be present", json.contains("task_monitors"));
+        // Task monitors are at top level (flat structure, no "task_monitors" wrapper)
+        assertTrue("query_execution should still be present", json.contains("query_execution"));
+        assertTrue("stream_next should still be present", json.contains("stream_next"));
+        assertTrue("fetch_phase should still be present", json.contains("fetch_phase"));
+        assertTrue("segment_stats should still be present", json.contains("segment_stats"));
     }
 
     // ---- Test: non-null CPU runtime → cpuRuntime present in JSON ----
@@ -155,11 +157,13 @@ public void testToXContentJsonStructure() throws IOException {
         DataFusionStats stats = sequentialStats();
         String json = toJsonString(stats);
 
-        assertTrue(json.contains("\"native_executors\""));
+        // Flat structure: no "native_executors" or "task_monitors" wrappers
+        assertFalse(json.contains("\"native_executors\""));
         assertTrue(json.contains("\"io_runtime\""));
         assertTrue(json.contains("\"cpu_runtime\""));
-        assertTrue(json.contains("\"task_monitors\""));
+        assertFalse(json.contains("\"task_monitors\""));
 
+        // Task monitors at top level
         assertTrue(json.contains("\"query_execution\""));
         assertTrue(json.contains("\"stream_next\""));
         assertTrue(json.contains("\"fetch_phase\""));
@@ -171,7 +175,7 @@ public void testToXContentJsonStructure() throws IOException {
         }
 
         // IO runtime: workers_count = 1
-        assertTrue(json.contains("\"workers_count\":1,"));
+        assertTrue(json.contains("\"workers_count\":1"));
         // query_execution: total_poll_duration_ms = 17
         assertTrue(json.contains("\"total_poll_duration_ms\":17"));
     }
@@ -191,7 +195,9 @@ public void testToXContentCpuRuntimeOmitted() throws IOException {
 
         assertTrue(json.contains("\"io_runtime\""));
         assertFalse("cpu_runtime should not appear", json.contains("\"cpu_runtime\""));
-        assertTrue(json.contains("\"task_monitors\""));
+        // Task monitors at top level (no wrapper)
+        assertTrue(json.contains("\"query_execution\""));
+        assertTrue(json.contains("\"segment_stats\""));
     }
 
     // ---- Test: exactly 4 task monitor keys ----

From b0b017e94efab0b4ba8ba004d6b899ee7d2ceebb Mon Sep 17 00:00:00 2001
From: Eric Wei 
Date: Mon, 11 May 2026 12:06:51 -0700
Subject: [PATCH 111/115] Wave A: enable 14 PPL datetime scalar functions on
 analytics-engine route (#21582)

Extends the PPL datetime surface of analytics-backend-datafusion with a
Wave A bundle of 14 functions: strftime, dayofweek / day_of_week,
second / second_of_minute, date, datetime, sysdate, extract,
from_unixtime (1-arg), maketime, makedate, date_format, time_format,
str_to_date. Each function is routed through
DataFusionAnalyticsBackendPlugin's scalar capabilities and wired end-to-end
to Substrait so that force-routed queries on /_plugins/_ppl execute on
DataFusion without any Calcite fallback.

Routing strategy per function:
 - Calcite builtins reused (date, datetime, dayofweek, second, sysdate)
   via name-mapping adapters that preserve PPL's declared return type.
 - Rust UDFs added for the MySQL-flavored behaviors that have no 1:1
   DataFusion builtin (strftime, extract, from_unixtime, maketime,
   makedate, date_format, time_format, str_to_date) with a shared
   mysql_format token table underpinning the *_format family.

End-to-end verification: all 14 functions pass against a force-routed
runTask cluster on /_plugins/_ppl, confirmed via explain
(viableBackends=[datafusion]) and ShardFragmentStageExecution traces.

Signed-off-by: Eric Wei 
---
 .../analytics/spi/ScalarFunction.java         |  15 +
 .../rust/src/udf/date_format.rs               | 171 +++++
 .../rust/src/udf/extract.rs                   | 271 ++++++++
 .../rust/src/udf/from_unixtime.rs             | 133 ++++
 .../rust/src/udf/makedate.rs                  | 162 +++++
 .../rust/src/udf/maketime.rs                  | 166 +++++
 .../rust/src/udf/mod.rs                       |  96 ++-
 .../rust/src/udf/mysql_format.rs              | 622 ++++++++++++++++++
 .../rust/src/udf/str_to_date.rs               | 161 +++++
 .../rust/src/udf/strftime.rs                  | 619 +++++++++++++++++
 .../rust/src/udf/time_format.rs               |  91 +++
 .../datafusion/ScalarDateTimeFunctionIT.java  |  26 +-
 .../DataFusionAnalyticsBackendPlugin.java     |  75 ++-
 .../DataFusionFragmentConvertor.java          |  15 +
 .../be/datafusion/DateTimeAdapters.java       |  53 +-
 .../be/datafusion/DayOfWeekAdapter.java       |  46 ++
 .../datafusion/RustUdfDateTimeAdapters.java   | 143 ++++
 .../be/datafusion/SecondAdapter.java          |  49 ++
 .../datafusion/StrftimeFunctionAdapter.java   |  87 +++
 .../opensearch_scalar_functions.yaml          | 204 ++++++
 .../StrftimeFunctionAdapterTests.java         | 134 ++++
 .../qa/DateTimeScalarFunctionsIT.java         | 207 ++++++
 22 files changed, 3492 insertions(+), 54 deletions(-)
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/date_format.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/extract.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/from_unixtime.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/makedate.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/maketime.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mysql_format.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/str_to_date.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/strftime.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/rust/src/udf/time_format.rs
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DayOfWeekAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/RustUdfDateTimeAdapters.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SecondAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/StrftimeFunctionAdapter.java
 create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/StrftimeFunctionAdapterTests.java
 create mode 100644 sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DateTimeScalarFunctionsIT.java

diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
index 60dd70f629018..de84486b88063 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/ScalarFunction.java
@@ -169,6 +169,21 @@ public enum ScalarFunction {
     CURTIME(Category.SCALAR, SqlKind.OTHER_FUNCTION),
     CONVERT_TZ(Category.SCALAR, SqlKind.OTHER_FUNCTION),
     UNIX_TIMESTAMP(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    STRFTIME(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    TIME(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    DATE(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    DATETIME(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    SYSDATE(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    DAYOFWEEK(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    DAY_OF_WEEK(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    SECOND(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    SECOND_OF_MINUTE(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    FROM_UNIXTIME(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    MAKETIME(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    MAKEDATE(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    DATE_FORMAT(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    TIME_FORMAT(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+    STR_TO_DATE(Category.SCALAR, SqlKind.OTHER_FUNCTION),
 
     // ── JSON ────────────────────────────────────────────────────────
     JSON_APPEND(Category.SCALAR, SqlKind.OTHER_FUNCTION),
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/date_format.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/date_format.rs
new file mode 100644
index 0000000000000..28993f183cd57
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/date_format.rs
@@ -0,0 +1,171 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `date_format(datetime, format)` — render a timestamp via MySQL-style tokens
+//! ([`mysql_format`](super::mysql_format)). Returns Utf8; null input → null.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use super::udf_identity;
+
+use chrono::{TimeZone, Utc};
+use datafusion::arrow::array::{Array, ArrayRef, AsArray, StringBuilder, TimestampMicrosecondArray};
+use datafusion::arrow::datatypes::{DataType, TimeUnit};
+use datafusion::common::{exec_err, plan_err, Result, ScalarValue};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+
+use super::mysql_format::{format_datetime, FormatMode};
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(DateFormatUdf::new()));
+}
+
+#[derive(Debug)]
+pub struct DateFormatUdf {
+    signature: Signature,
+}
+
+impl DateFormatUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+udf_identity!(DateFormatUdf, "date_format");
+
+impl ScalarUDFImpl for DateFormatUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn name(&self) -> &str {
+        "date_format"
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.len() != 2 {
+            return plan_err!("date_format expects 2 arguments, got {}", arg_types.len());
+        }
+        Ok(DataType::Utf8)
+    }
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        if arg_types.len() != 2 {
+            return plan_err!("date_format expects 2 arguments, got {}", arg_types.len());
+        }
+        let ts = match &arg_types[0] {
+            DataType::Timestamp(_, _) | DataType::Date32 | DataType::Date64 => {
+                DataType::Timestamp(TimeUnit::Microsecond, None)
+            }
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
+                DataType::Timestamp(TimeUnit::Microsecond, None)
+            }
+            other => return plan_err!("date_format: arg 0 expected timestamp/date/string, got {other:?}"),
+        };
+        let fmt = match &arg_types[1] {
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => DataType::Utf8,
+            other => return plan_err!("date_format: arg 1 expected string, got {other:?}"),
+        };
+        Ok(vec![ts, fmt])
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        format_dispatch("date_format", FormatMode::Date, args)
+    }
+}
+
+pub(crate) fn format_dispatch(
+    udf: &str,
+    mode: FormatMode,
+    args: ScalarFunctionArgs,
+) -> Result {
+    if args.args.len() != 2 {
+        return exec_err!("{udf} expects 2 arguments, got {}", args.args.len());
+    }
+    let n = args.number_rows;
+
+    if let (ColumnarValue::Scalar(v), ColumnarValue::Scalar(fmt)) = (&args.args[0], &args.args[1]) {
+        let micros = match v {
+            ScalarValue::TimestampMicrosecond(v, _) => *v,
+            other => return exec_err!("{udf}: unsupported ts scalar: {other:?}"),
+        };
+        let fmt_str = match fmt {
+            ScalarValue::Utf8(opt) | ScalarValue::LargeUtf8(opt) => opt.clone(),
+            other => return exec_err!("{udf}: unsupported format scalar: {other:?}"),
+        };
+        let out = match (micros, fmt_str) {
+            (Some(m), Some(f)) => render_at(m, &f, mode),
+            _ => None,
+        };
+        return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(out)));
+    }
+
+    let v_arr = args.args[0].clone().into_array(n)?;
+    let f_arr = args.args[1].clone().into_array(n)?;
+    let ts = v_arr
+        .as_any()
+        .downcast_ref::()
+        .ok_or_else(|| {
+            datafusion::common::DataFusionError::Execution(format!(
+                "{udf}: expected Timestamp(Microsecond, None) after coercion, got {:?}",
+                v_arr.data_type()
+            ))
+        })?;
+    let mut builder = StringBuilder::with_capacity(n, 0);
+    for i in 0..n {
+        if ts.is_null(i) {
+            builder.append_null();
+            continue;
+        }
+        let fmt_opt = match f_arr.data_type() {
+            DataType::Utf8 => {
+                let a = f_arr.as_string::();
+                if a.is_null(i) { None } else { Some(a.value(i).to_string()) }
+            }
+            DataType::LargeUtf8 => {
+                let a = f_arr.as_string::();
+                if a.is_null(i) { None } else { Some(a.value(i).to_string()) }
+            }
+            other => return exec_err!("{udf}: format array has unexpected type {other:?}"),
+        };
+        match fmt_opt.and_then(|f| render_at(ts.value(i), &f, mode)) {
+            Some(s) => builder.append_value(&s),
+            None => builder.append_null(),
+        }
+    }
+    Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))
+}
+
+fn render_at(micros: i64, format: &str, mode: FormatMode) -> Option {
+    let seconds = micros.div_euclid(1_000_000);
+    let micros_rem = micros.rem_euclid(1_000_000) as u32;
+    let dt = Utc.timestamp_opt(seconds, micros_rem * 1_000).single()?;
+    format_datetime(dt, format, mode)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn render_scalar_matches_mysql_format() {
+        let out = render_at(1_584_268_245_123_456, "%Y-%m-%d %H:%i:%S", FormatMode::Date).unwrap();
+        assert_eq!(out, "2020-03-15 10:30:45");
+    }
+
+    #[test]
+    fn null_format_returns_none() {
+        assert!(render_at(0, "", FormatMode::Date).is_some());
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/extract.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/extract.rs
new file mode 100644
index 0000000000000..f4aebf8e1b80b
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/extract.rs
@@ -0,0 +1,271 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `extract(unit, datetime)` — MySQL-style calendar component extractor (22 units incl. composites).
+//! Composite units (e.g. DAY_MICROSECOND) join min-width-padded fields then parse as i64,
+//! so leading zeros on the first field collapse (`0709` → `709`). WEEK is ISO, DOW is Mon=1..Sun=7.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use super::udf_identity;
+
+use chrono::{DateTime, Datelike, TimeZone, Timelike, Utc};
+use datafusion::arrow::array::{
+    Array, ArrayRef, AsArray, Int64Builder, TimestampMicrosecondArray,
+};
+use datafusion::arrow::datatypes::{DataType, TimeUnit};
+use datafusion::common::{exec_err, plan_err, Result, ScalarValue};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(ExtractUdf::new()));
+}
+
+#[derive(Debug)]
+pub struct ExtractUdf {
+    signature: Signature,
+}
+
+impl ExtractUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+udf_identity!(ExtractUdf, "extract");
+
+impl ScalarUDFImpl for ExtractUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "extract"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.len() != 2 {
+            return plan_err!("extract expects 2 arguments, got {}", arg_types.len());
+        }
+        Ok(DataType::Int64)
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        if arg_types.len() != 2 {
+            return plan_err!("extract expects 2 arguments, got {}", arg_types.len());
+        }
+        let unit = match &arg_types[0] {
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => DataType::Utf8,
+            other => return plan_err!("extract: arg 0 expected string unit, got {other:?}"),
+        };
+        let ts = match &arg_types[1] {
+            DataType::Timestamp(_, _) | DataType::Date32 | DataType::Date64
+            | DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
+                DataType::Timestamp(TimeUnit::Microsecond, None)
+            }
+            other => return plan_err!("extract: arg 1 expected timestamp/date/string, got {other:?}"),
+        };
+        Ok(vec![unit, ts])
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() != 2 {
+            return exec_err!("extract expects 2 arguments, got {}", args.args.len());
+        }
+        let n = args.number_rows;
+
+        if let (ColumnarValue::Scalar(unit), ColumnarValue::Scalar(ts)) =
+            (&args.args[0], &args.args[1])
+        {
+            let unit_str = scalar_utf8(unit)?;
+            let micros = match ts {
+                ScalarValue::TimestampMicrosecond(v, _) => *v,
+                other => return exec_err!("extract: unsupported ts scalar: {other:?}"),
+            };
+            let out = match (unit_str, micros) {
+                (Some(u), Some(m)) => compute(&u, m),
+                _ => None,
+            };
+            return Ok(ColumnarValue::Scalar(ScalarValue::Int64(out)));
+        }
+
+        let unit_arr = args.args[0].clone().into_array(n)?;
+        let ts_arr = args.args[1].clone().into_array(n)?;
+        let ts = ts_arr
+            .as_any()
+            .downcast_ref::()
+            .ok_or_else(|| {
+                datafusion::common::DataFusionError::Execution(format!(
+                    "extract: expected Timestamp(Microsecond, None) after coercion, got {:?}",
+                    ts_arr.data_type()
+                ))
+            })?;
+        let mut builder = Int64Builder::with_capacity(n);
+        for i in 0..n {
+            if ts.is_null(i) {
+                builder.append_null();
+                continue;
+            }
+            match unit_at(&unit_arr, i)? {
+                None => builder.append_null(),
+                Some(u) => match compute(&u, ts.value(i)) {
+                    Some(v) => builder.append_value(v),
+                    None => builder.append_null(),
+                },
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))
+    }
+}
+
+fn scalar_utf8(s: &ScalarValue) -> Result> {
+    match s {
+        ScalarValue::Utf8(opt) | ScalarValue::LargeUtf8(opt) => Ok(opt.clone()),
+        other => exec_err!("extract: unit must be string, got {other:?}"),
+    }
+}
+
+fn unit_at(array: &ArrayRef, row: usize) -> Result> {
+    let (is_null, value) = match array.data_type() {
+        DataType::Utf8 => {
+            let a = array.as_string::();
+            (a.is_null(row), a.value(row).to_string())
+        }
+        DataType::LargeUtf8 => {
+            let a = array.as_string::();
+            (a.is_null(row), a.value(row).to_string())
+        }
+        other => return exec_err!("extract: expected string unit array, got {other:?}"),
+    };
+    Ok(if is_null { None } else { Some(value) })
+}
+
+fn compute(unit: &str, micros: i64) -> Option {
+    let seconds = micros.div_euclid(1_000_000);
+    let micro_fraction = micros.rem_euclid(1_000_000) as u32;
+    let dt = Utc.timestamp_opt(seconds, micro_fraction * 1_000).single()?;
+    extract_for_unit(&unit.to_ascii_uppercase(), dt)
+}
+
+/// Unknown unit → None (PPL throws; we surface null to avoid aborting the whole query).
+fn extract_for_unit(unit: &str, dt: DateTime) -> Option {
+    let us = (dt.nanosecond() / 1_000) as i64;
+    let (ss, mm, hh) = (dt.second() as i64, dt.minute() as i64, dt.hour() as i64);
+    let (dd, mo, yy) = (dt.day() as i64, dt.month() as i64, dt.year() as i64);
+    match unit {
+        "MICROSECOND" => Some(us),
+        "SECOND" => Some(ss),
+        "MINUTE" => Some(mm),
+        "HOUR" => Some(hh),
+        "DAY" => Some(dd),
+        "WEEK" => Some(dt.iso_week().week() as i64),
+        "MONTH" => Some(mo),
+        "QUARTER" => Some(((mo - 1) / 3 + 1) as i64),
+        "YEAR" => Some(yy),
+        "DOW" => Some(dt.weekday().number_from_monday() as i64),
+        "DOY" => Some(dt.ordinal() as i64),
+        "SECOND_MICROSECOND" => concat(&[(ss, 2), (us, 6)]),
+        "MINUTE_MICROSECOND" => concat(&[(mm, 2), (ss, 2), (us, 6)]),
+        "MINUTE_SECOND" => concat(&[(mm, 2), (ss, 2)]),
+        "HOUR_MICROSECOND" => concat(&[(hh, 2), (mm, 2), (ss, 2), (us, 6)]),
+        "HOUR_SECOND" => concat(&[(hh, 2), (mm, 2), (ss, 2)]),
+        "HOUR_MINUTE" => concat(&[(hh, 2), (mm, 2)]),
+        "DAY_MICROSECOND" => concat(&[(dd, 2), (hh, 2), (mm, 2), (ss, 2), (us, 6)]),
+        "DAY_SECOND" => concat(&[(dd, 2), (hh, 2), (mm, 2), (ss, 2)]),
+        "DAY_MINUTE" => concat(&[(dd, 2), (hh, 2), (mm, 2)]),
+        "DAY_HOUR" => concat(&[(dd, 2), (hh, 2)]),
+        "YEAR_MONTH" => concat(&[(yy, 4), (mo, 2)]),
+        _ => None,
+    }
+}
+
+/// Integer-math equivalent of Java's `parseLong(format(dt))`: acc = acc*10^w + v.
+fn concat(parts: &[(i64, u32)]) -> Option {
+    let mut acc: i64 = 0;
+    for &(v, w) in parts {
+        if v < 0 {
+            return None;
+        }
+        let pow = 10_i64.checked_pow(w)?;
+        acc = acc.checked_mul(pow)?.checked_add(v)?;
+    }
+    Some(acc)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// 2020-03-15 10:30:45.123456 UTC in micros since epoch.
+    const SAMPLE_MICROS: i64 = 1_584_268_245_123_456;
+
+    fn eval(unit: &str) -> Option {
+        compute(unit, SAMPLE_MICROS)
+    }
+
+    fn us(y: i32, mo: u32, d: u32, h: u32, mi: u32, s: u32) -> i64 {
+        Utc.with_ymd_and_hms(y, mo, d, h, mi, s).unwrap().timestamp() * 1_000_000
+    }
+
+    #[test]
+    fn simple_and_composite_units_on_reference_sample() {
+        // Reference sample (Sunday, 2020 leap year, ISO week 11):
+        for (unit, want) in [
+            ("MICROSECOND", 123_456_i64), ("SECOND", 45), ("MINUTE", 30), ("HOUR", 10),
+            ("DAY", 15), ("MONTH", 3), ("QUARTER", 1), ("YEAR", 2020),
+            ("DOY", 75), ("WEEK", 11), ("DOW", 7), // 2020-03-15 is a Sunday (ISO DOW=7)
+            ("DAY_MICROSECOND", 15_103_045_123_456), ("DAY_SECOND", 15_103_045),
+            ("DAY_MINUTE", 151_030), ("DAY_HOUR", 1510),
+            ("HOUR_MICROSECOND", 103_045_123_456), ("HOUR_SECOND", 103_045),
+            ("HOUR_MINUTE", 1030),
+            ("MINUTE_MICROSECOND", 3_045_123_456), ("MINUTE_SECOND", 3045),
+            ("SECOND_MICROSECOND", 45_123_456), ("YEAR_MONTH", 202_003),
+        ] {
+            assert_eq!(eval(unit), Some(want), "unit={unit}");
+        }
+    }
+
+    #[test]
+    fn unit_name_is_case_insensitive() {
+        assert_eq!(eval("year"), Some(2020));
+        assert_eq!(eval("dAy_HoUr"), Some(1510));
+    }
+
+    #[test]
+    fn dow_monday_and_quarter_edges() {
+        // Bump sample by one day → 2020-03-16 Monday (DOW=1).
+        assert_eq!(compute("DOW", SAMPLE_MICROS + 86_400 * 1_000_000), Some(1));
+        assert_eq!(compute("QUARTER", us(2020, 1, 1, 0, 0, 0)), Some(1));
+        assert_eq!(compute("QUARTER", us(2020, 4, 1, 0, 0, 0)), Some(2));
+        assert_eq!(compute("QUARTER", us(2020, 12, 31, 23, 59, 59)), Some(4));
+    }
+
+    #[test]
+    fn leading_zero_on_first_field_collapses() {
+        // 2020-01-07 09:05:02 → DAY_HOUR "0709"→709, HOUR_MINUTE "0905"→905.
+        let m = us(2020, 1, 7, 9, 5, 2);
+        assert_eq!(compute("DAY_HOUR", m), Some(709));
+        assert_eq!(compute("HOUR_MINUTE", m), Some(905));
+    }
+
+    #[test]
+    fn unknown_unit_yields_null() {
+        assert_eq!(eval("NANOSECOND"), None);
+        assert_eq!(eval(""), None);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/from_unixtime.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/from_unixtime.rs
new file mode 100644
index 0000000000000..22d5ceda8d2fc
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/from_unixtime.rs
@@ -0,0 +1,133 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `from_unixtime(seconds)` — UNIX seconds (fractional ok) → `Timestamp(us)`. Negative / ≥ MySQL
+//! upper bound / non-finite → NULL. 2-arg `(seconds, format)` overload deferred to date_format.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use super::udf_identity;
+
+use datafusion::arrow::array::{Array, ArrayRef, AsArray, TimestampMicrosecondBuilder};
+use datafusion::arrow::datatypes::{DataType, TimeUnit};
+use datafusion::common::{exec_err, plan_err, Result, ScalarValue};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+
+use super::{coerce_args, CoerceMode};
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(FromUnixtimeUdf::new()));
+}
+
+/// MySQL docs: values at/above 32536771199.999999 return 0; first failing second is 32_536_771_200.
+const MAX_UNIX_SECONDS_EXCLUSIVE: f64 = 32_536_771_200.0;
+
+#[derive(Debug)]
+pub struct FromUnixtimeUdf {
+    signature: Signature,
+}
+
+impl FromUnixtimeUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+udf_identity!(FromUnixtimeUdf, "from_unixtime");
+
+impl ScalarUDFImpl for FromUnixtimeUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "from_unixtime"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.len() != 1 {
+            return plan_err!("from_unixtime expects 1 argument, got {}", arg_types.len());
+        }
+        Ok(DataType::Timestamp(TimeUnit::Microsecond, None))
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        coerce_args("from_unixtime", arg_types, &[CoerceMode::Float64])
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() != 1 {
+            return exec_err!(
+                "from_unixtime expects 1 argument, got {}",
+                args.args.len()
+            );
+        }
+        let n = args.number_rows;
+
+        if let ColumnarValue::Scalar(ScalarValue::Float64(v)) = &args.args[0] {
+            let micros = v.and_then(to_micros);
+            return Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(
+                micros, None,
+            )));
+        }
+
+        let input = args.args[0].clone().into_array(n)?;
+        let input = input.as_primitive::();
+        let mut builder = TimestampMicrosecondBuilder::with_capacity(n);
+        for i in 0..input.len() {
+            if input.is_null(i) {
+                builder.append_null();
+                continue;
+            }
+            match to_micros(input.value(i)) {
+                Some(us) => builder.append_value(us),
+                None => builder.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))
+    }
+}
+
+fn to_micros(seconds: f64) -> Option {
+    if !seconds.is_finite() || !(0.0..MAX_UNIX_SECONDS_EXCLUSIVE).contains(&seconds) {
+        return None;
+    }
+    // PPL truncates nanos via int cast after `(v%1)*1e9`; trunc ≡ floor for non-negative seconds.
+    Some((seconds * 1_000_000.0).trunc() as i64)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn rejects_out_of_range_and_non_finite() {
+        for v in [-0.1, MAX_UNIX_SECONDS_EXCLUSIVE, MAX_UNIX_SECONDS_EXCLUSIVE + 1.0, f64::NAN, f64::INFINITY] {
+            assert_eq!(to_micros(v), None, "v={v}");
+        }
+    }
+
+    #[test]
+    fn converts_epoch_and_fractional_seconds() {
+        assert_eq!(to_micros(0.0), Some(0));
+        assert_eq!(to_micros(1.5), Some(1_500_000));
+        // Sub-microsecond truncates (not rounds) to match PPL's int cast.
+        assert_eq!(to_micros(0.000_000_9), Some(0));
+        assert_eq!(to_micros(0.000_001_9), Some(1));
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/makedate.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/makedate.rs
new file mode 100644
index 0000000000000..118abc63f21d0
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/makedate.rs
@@ -0,0 +1,162 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `makedate(year, day_of_year)` → `Date32`. MySQL quirks: `year==0` remaps to 2000; `doy<=0` /
+//! `year<0` → NULL; `doy` past year-end cascades into the next year.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use super::udf_identity;
+
+use chrono::{Datelike, NaiveDate};
+use datafusion::arrow::array::{Array, ArrayRef, AsArray, Date32Builder};
+use datafusion::arrow::datatypes::{DataType, Float64Type};
+use datafusion::common::{exec_err, plan_err, Result, ScalarValue};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+
+use super::{coerce_args, CoerceMode};
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(MakedateUdf::new()));
+}
+
+/// Offset between CE (0001-01-01) and Unix epoch (1970-01-01) used for Arrow Date32 conversion.
+const CE_TO_UNIX_EPOCH_DAYS: i32 = 719_163;
+
+#[derive(Debug)]
+pub struct MakedateUdf {
+    signature: Signature,
+}
+
+impl MakedateUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+udf_identity!(MakedateUdf, "makedate");
+
+impl ScalarUDFImpl for MakedateUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "makedate"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.len() != 2 {
+            return plan_err!("makedate expects 2 arguments, got {}", arg_types.len());
+        }
+        Ok(DataType::Date32)
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        coerce_args(
+            "makedate",
+            arg_types,
+            &[CoerceMode::Float64, CoerceMode::Float64],
+        )
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() != 2 {
+            return exec_err!("makedate expects 2 arguments, got {}", args.args.len());
+        }
+        let n = args.number_rows;
+
+        if let (
+            ColumnarValue::Scalar(ScalarValue::Float64(y)),
+            ColumnarValue::Scalar(ScalarValue::Float64(d)),
+        ) = (&args.args[0], &args.args[1])
+        {
+            let days = match (y, d) {
+                (Some(y), Some(d)) => days_since_epoch(*y, *d),
+                _ => None,
+            };
+            return Ok(ColumnarValue::Scalar(ScalarValue::Date32(days)));
+        }
+
+        let y = args.args[0].clone().into_array(n)?;
+        let d = args.args[1].clone().into_array(n)?;
+        let y = y.as_primitive::();
+        let d = d.as_primitive::();
+        let mut builder = Date32Builder::with_capacity(n);
+        for i in 0..n {
+            if y.is_null(i) || d.is_null(i) {
+                builder.append_null();
+                continue;
+            }
+            match days_since_epoch(y.value(i), d.value(i)) {
+                Some(v) => builder.append_value(v),
+                None => builder.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))
+    }
+}
+
+fn days_since_epoch(year: f64, day_of_year: f64) -> Option {
+    if !year.is_finite() || !day_of_year.is_finite() {
+        return None;
+    }
+    let year = year.round() as i64;
+    let doy = day_of_year.round() as i64;
+    if doy <= 0 || year < 0 {
+        return None;
+    }
+    let year = if year == 0 { 2000 } else { year };
+    let year: i32 = year.try_into().ok()?;
+    let start = NaiveDate::from_yo_opt(year, 1)?;
+    let date = start.checked_add_days(chrono::Days::new((doy - 1) as u64))?;
+    Some(date.num_days_from_ce() - CE_TO_UNIX_EPOCH_DAYS)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn days(y: i32, m: u32, d: u32) -> i32 {
+        NaiveDate::from_ymd_opt(y, m, d).unwrap().num_days_from_ce() - CE_TO_UNIX_EPOCH_DAYS
+    }
+
+    #[test]
+    fn well_formed_and_overflow_cases() {
+        // (year, doy) → (y, m, d) expected.
+        for (y, doy, ey, em, ed) in [
+            (2024.0, 1.0, 2024, 1, 1),
+            (2024.0, 60.0, 2024, 2, 29),   // 2024 leap; doy 60 = Feb 29
+            (2024.0, 366.0, 2024, 12, 31),
+            (0.0, 1.0, 2000, 1, 1),        // year 0 remaps to 2000
+            (2023.0, 366.0, 2024, 1, 1),   // non-leap overflow cascades
+            (2024.4, 60.6, 2024, 3, 1),    // fractional operands round: 2024, 61
+        ] {
+            assert_eq!(days_since_epoch(y, doy), Some(days(ey, em, ed)), "y={y} doy={doy}");
+        }
+    }
+
+    #[test]
+    fn rejects_out_of_range_and_non_finite() {
+        assert_eq!(days_since_epoch(2024.0, 0.0), None);
+        assert_eq!(days_since_epoch(2024.0, -1.0), None);
+        assert_eq!(days_since_epoch(-1.0, 100.0), None);
+        assert_eq!(days_since_epoch(f64::NAN, 1.0), None);
+        assert_eq!(days_since_epoch(2024.0, f64::INFINITY), None);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/maketime.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/maketime.rs
new file mode 100644
index 0000000000000..2bbc7680227c9
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/maketime.rs
@@ -0,0 +1,166 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `maketime(hour, minute, second)` → `Time64(us)`. Hour/minute rounded; second passes with fraction.
+//! Negative / non-finite / out-of-range (after rounding) / null → NULL.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use super::udf_identity;
+
+use datafusion::arrow::array::{Array, ArrayRef, AsArray, Time64MicrosecondBuilder};
+use datafusion::arrow::datatypes::{DataType, Float64Type, TimeUnit};
+use datafusion::common::{exec_err, plan_err, Result, ScalarValue};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+
+use super::{coerce_args, CoerceMode};
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(MaketimeUdf::new()));
+}
+
+#[derive(Debug)]
+pub struct MaketimeUdf {
+    signature: Signature,
+}
+
+impl MaketimeUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+udf_identity!(MaketimeUdf, "maketime");
+
+impl ScalarUDFImpl for MaketimeUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "maketime"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.len() != 3 {
+            return plan_err!("maketime expects 3 arguments, got {}", arg_types.len());
+        }
+        Ok(DataType::Time64(TimeUnit::Microsecond))
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        coerce_args(
+            "maketime",
+            arg_types,
+            &[CoerceMode::Float64, CoerceMode::Float64, CoerceMode::Float64],
+        )
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() != 3 {
+            return exec_err!("maketime expects 3 arguments, got {}", args.args.len());
+        }
+        let n = args.number_rows;
+
+        if let (
+            ColumnarValue::Scalar(ScalarValue::Float64(h)),
+            ColumnarValue::Scalar(ScalarValue::Float64(m)),
+            ColumnarValue::Scalar(ScalarValue::Float64(s)),
+        ) = (&args.args[0], &args.args[1], &args.args[2])
+        {
+            let micros = match (h, m, s) {
+                (Some(h), Some(m), Some(s)) => micros_of_day(*h, *m, *s),
+                _ => None,
+            };
+            return Ok(ColumnarValue::Scalar(ScalarValue::Time64Microsecond(
+                micros,
+            )));
+        }
+
+        let h = args.args[0].clone().into_array(n)?;
+        let m = args.args[1].clone().into_array(n)?;
+        let s = args.args[2].clone().into_array(n)?;
+        let h = h.as_primitive::();
+        let m = m.as_primitive::();
+        let s = s.as_primitive::();
+        let mut builder = Time64MicrosecondBuilder::with_capacity(n);
+        for i in 0..n {
+            if h.is_null(i) || m.is_null(i) || s.is_null(i) {
+                builder.append_null();
+                continue;
+            }
+            match micros_of_day(h.value(i), m.value(i), s.value(i)) {
+                Some(us) => builder.append_value(us),
+                None => builder.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))
+    }
+}
+
+fn micros_of_day(hour: f64, minute: f64, second: f64) -> Option {
+    if !hour.is_finite() || !minute.is_finite() || !second.is_finite() {
+        return None;
+    }
+    if hour < 0.0 || minute < 0.0 || second < 0.0 {
+        return None;
+    }
+    // Java's Math.round(d) = (long)floor(d+0.5); Rust's f64::round is half-away-from-zero which
+    // matches for non-negative inputs. PPL throws on out-of-range; we return None → NULL.
+    let hour = hour.round() as i64;
+    let minute = minute.round() as i64;
+    if !(0..=23).contains(&hour) || !(0..=59).contains(&minute) || second >= 60.0 {
+        return None;
+    }
+    let second_micros = (second * 1_000_000.0).trunc() as i64;
+    Some(hour * 3_600_000_000 + minute * 60_000_000 + second_micros)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn well_formed_and_rounding_semantics() {
+        // (1, 2, 3.5) → 1h2m3.5s.
+        assert_eq!(
+            micros_of_day(1.0, 2.0, 3.5),
+            Some(3_600_000_000 + 2 * 60_000_000 + 3_500_000)
+        );
+        // 1.6→2h, 2.4→2m (half-away-from-zero), 3.999999s truncated.
+        assert_eq!(
+            micros_of_day(1.6, 2.4, 3.999_999),
+            Some(2 * 3_600_000_000 + 2 * 60_000_000 + 3_999_999)
+        );
+    }
+
+    #[test]
+    fn rejects_invalid_operands() {
+        // Negative components.
+        for args in [(-0.1, 0.0, 0.0), (0.0, -0.1, 0.0), (0.0, 0.0, -0.1)] {
+            assert_eq!(micros_of_day(args.0, args.1, args.2), None, "{args:?}");
+        }
+        // Out of range after rounding: 23.5→24h, 59.5→60m, 60s (no rounding).
+        assert_eq!(micros_of_day(23.5, 0.0, 0.0), None);
+        assert_eq!(micros_of_day(0.0, 59.5, 0.0), None);
+        assert_eq!(micros_of_day(0.0, 0.0, 60.0), None);
+        // Non-finite.
+        assert_eq!(micros_of_day(f64::NAN, 0.0, 0.0), None);
+        assert_eq!(micros_of_day(f64::INFINITY, 0.0, 0.0), None);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
index 66b4e20abc295..c553101fa23d0 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mod.rs
@@ -6,40 +6,49 @@
  * compatible open source license.
  */
 
-//! OpenSearch scalar UDFs that aren't in DataFusion's built-in registry. Each
-//! must have a matching YAML entry in `opensearch_scalar_functions.yaml` so
-//! the substrait converter on the Java side can route to it by name.
-//!
-//! Functions registered here:
-//! - `convert_tz(ts, from_tz, to_tz)` — DST-aware timezone shift (chrono-tz)
-//! - `json_array_length(value)` — length of a JSON array, NULL on malformed/non-array
+//! OpenSearch scalar UDFs not in DataFusion's builtins. Each registered UDF must
+//! have a matching YAML entry in `opensearch_scalar_functions.yaml`.
 
 use datafusion::arrow::datatypes::{DataType, TimeUnit};
 use datafusion::common::plan_err;
 use datafusion::error::Result;
 use datafusion::execution::context::SessionContext;
 
-/// Categories of input type a UDF slot can accept. Each mode declares a
-/// canonical target arrow type plus the set of sources that coerce to it.
-/// UDFs use `Signature::user_defined()` and call [`coerce_slot`] per
-/// argument position to produce the `coerce_types` output.
-///
-/// Invalid sources produce an explicit `plan_err!` — no silent fallback. The
-/// failure message names the UDF, the slot index, the observed type and the
-/// expected canonical type so planning errors are actionable.
+/// Emit the `Default`, `PartialEq`, `Eq`, and `Hash` impls every stateless
+/// UDF needs. All instances of the same UDF type are semantically identical
+/// — they compare equal and hash to a single name-derived stable value.
+macro_rules! udf_identity {
+    ($udf:ident, $name:literal) => {
+        impl Default for $udf {
+            fn default() -> Self {
+                Self::new()
+            }
+        }
+        impl PartialEq for $udf {
+            fn eq(&self, _: &Self) -> bool {
+                true
+            }
+        }
+        impl Eq for $udf {}
+        impl std::hash::Hash for $udf {
+            fn hash(&self, state: &mut H) {
+                $name.hash(state);
+            }
+        }
+    };
+}
+pub(crate) use udf_identity;
+
+/// Input-type categories for UDF argument slots. Each mode declares a canonical
+/// arrow target type plus the set of sources that coerce to it; invalid sources
+/// produce a `plan_err!` naming the UDF, slot index, and observed type.
 #[derive(Clone, Copy, Debug)]
 #[allow(dead_code)]
 pub(crate) enum CoerceMode {
-    /// Accept Utf8 / Date32 / Timestamp(any precision, any tz) → canonicalize
-    /// to Timestamp(Millisecond, None). DF has built-in casts for each source.
     TimestampMs,
-    /// Accept Utf8 / Date32 / Timestamp(any, any) → canonicalize to Date32.
     Date32,
-    /// Accept any integer or float → Int64.
     Int64,
-    /// Accept any integer or float → Float64.
     Float64,
-    /// Accept Utf8 / LargeUtf8 / Utf8View → Utf8.
     Utf8,
 }
 
@@ -71,23 +80,21 @@ pub(crate) fn coerce_slot(
             Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 | Float32 | Float64 => {
                 Ok(Int64)
             }
-            other => plan_err!(
-                "{udf_name}: arg {slot_index} expected integer or float, got {other:?}"
-            ),
+            other => {
+                plan_err!("{udf_name}: arg {slot_index} expected integer or float, got {other:?}")
+            }
         },
         CoerceMode::Float64 => match observed {
             Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 | Float32 | Float64 => {
                 Ok(Float64)
             }
-            other => plan_err!(
-                "{udf_name}: arg {slot_index} expected integer or float, got {other:?}"
-            ),
+            other => {
+                plan_err!("{udf_name}: arg {slot_index} expected integer or float, got {other:?}")
+            }
         },
         CoerceMode::Utf8 => match observed {
             Utf8 | LargeUtf8 | Utf8View => Ok(Utf8),
-            other => plan_err!(
-                "{udf_name}: arg {slot_index} expected string, got {other:?}"
-            ),
+            other => plan_err!("{udf_name}: arg {slot_index} expected string, got {other:?}"),
         },
     }
 }
@@ -114,6 +121,9 @@ pub(crate) fn coerce_args(
 }
 
 pub mod convert_tz;
+pub mod date_format;
+pub mod extract;
+pub mod from_unixtime;
 pub mod json_append;
 pub mod json_array_length;
 pub(crate) mod json_common;
@@ -122,9 +132,15 @@ pub mod json_extend;
 pub mod json_extract;
 pub mod json_keys;
 pub mod json_set;
+pub mod makedate;
+pub mod maketime;
 pub mod mvappend;
 pub mod mvfind;
 pub mod mvzip;
+pub(crate) mod mysql_format;
+pub mod str_to_date;
+pub mod strftime;
+pub mod time_format;
 pub mod tonumber;
 pub mod tostring;
 
@@ -137,6 +153,9 @@ pub mod tostring;
 // and restart the OpenSearch JVM (the loaded dylib is JVM-cached).
 pub fn register_all(ctx: &SessionContext) {
     convert_tz::register_all(ctx);
+    date_format::register_all(ctx);
+    extract::register_all(ctx);
+    from_unixtime::register_all(ctx);
     json_append::register_all(ctx);
     json_array_length::register_all(ctx);
     json_delete::register_all(ctx);
@@ -144,13 +163,18 @@ pub fn register_all(ctx: &SessionContext) {
     json_extract::register_all(ctx);
     json_keys::register_all(ctx);
     json_set::register_all(ctx);
-    mvzip::register_all(ctx);
-    mvfind::register_all(ctx);
+    makedate::register_all(ctx);
+    maketime::register_all(ctx);
     mvappend::register_all(ctx);
+    mvfind::register_all(ctx);
+    mvzip::register_all(ctx);
+    str_to_date::register_all(ctx);
+    strftime::register_all(ctx);
+    time_format::register_all(ctx);
     tonumber::register_all(ctx);
     tostring::register_all(ctx);
     log::info!(
-        "OpenSearch UDF register_all: convert_tz, json_append, json_array_length, json_delete, json_extend, json_extract, json_keys, json_set, mvzip, mvfind, mvappend, tonumber, tostring registered"
+        "OpenSearch UDF register_all: convert_tz, date_format, extract, from_unixtime, json_append, json_array_length, json_delete, json_extend, json_extract, json_keys, json_set, makedate, maketime, mvappend, mvfind, mvzip, str_to_date, strftime, time_format, tonumber, tostring registered"
     );
 }
 
@@ -181,7 +205,11 @@ mod tests {
             DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
         ] {
             let result = coerce_slot("t", 0, &observed, CoerceMode::TimestampMs).unwrap();
-            assert_eq!(result, ts_ms(), "TimestampMs should canonicalize {observed:?}");
+            assert_eq!(
+                result,
+                ts_ms(),
+                "TimestampMs should canonicalize {observed:?}"
+            );
         }
     }
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mysql_format.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mysql_format.rs
new file mode 100644
index 0000000000000..c02516073bc83
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/mysql_format.rs
@@ -0,0 +1,622 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! Shared MySQL format-token translator (date_format / time_format / str_to_date);
+//! mirrors PPL's `DateTimeFormatterUtil`. `%V`/`%v` use chrono's ISO week; `%U`/`%u`
+//! use simple Sun-/Mon-first counting. These match MySQL modes 0/1 except when
+//! Jan 1 falls in week 52/53 of the prior year — matches PPL's observed output.
+
+use chrono::{DateTime, Datelike, TimeZone, Timelike, Utc, Weekday};
+
+// Token variants map 1:1 to MySQL `%X` directives; `Literal` wraps non-directive chars.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Token {
+    Literal(char),
+    A, B, C, D, DLower, E, F,
+    HUpper, HLower, ILower, IUpper, J, K, L,
+    MUpper, MLower, P, R, SUpper, SLower, T,
+    UUpper, ULower, VUpper, VLower, W, WLower,
+    XUpper, XLower, YUpper, YLower,
+}
+
+fn tokenize(format: &str) -> Vec {
+    let bytes = format.as_bytes();
+    let mut out = Vec::with_capacity(bytes.len());
+    let mut i = 0;
+    while i < bytes.len() {
+        if bytes[i] != b'%' {
+            out.push(Token::Literal(bytes[i] as char));
+            i += 1;
+            continue;
+        }
+        if i + 1 >= bytes.len() {
+            out.push(Token::Literal('%'));
+            break;
+        }
+        let tok = match bytes[i + 1] {
+            b'%' => Some(Token::Literal('%')),
+            b'a' => Some(Token::A),
+            b'b' => Some(Token::B),
+            b'c' => Some(Token::C),
+            b'D' => Some(Token::D),
+            b'd' => Some(Token::DLower),
+            b'e' => Some(Token::E),
+            b'f' => Some(Token::F),
+            b'H' => Some(Token::HUpper),
+            b'h' => Some(Token::HLower),
+            b'I' => Some(Token::IUpper),
+            b'i' => Some(Token::ILower),
+            b'j' => Some(Token::J),
+            b'k' => Some(Token::K),
+            b'l' => Some(Token::L),
+            b'M' => Some(Token::MUpper),
+            b'm' => Some(Token::MLower),
+            b'p' => Some(Token::P),
+            b'r' => Some(Token::R),
+            b'S' => Some(Token::SUpper),
+            b's' => Some(Token::SLower),
+            b'T' => Some(Token::T),
+            b'U' => Some(Token::UUpper),
+            b'u' => Some(Token::ULower),
+            b'V' => Some(Token::VUpper),
+            b'v' => Some(Token::VLower),
+            b'W' => Some(Token::W),
+            b'w' => Some(Token::WLower),
+            b'X' => Some(Token::XUpper),
+            b'x' => Some(Token::XLower),
+            b'Y' => Some(Token::YUpper),
+            b'y' => Some(Token::YLower),
+            _ => None,
+        };
+        match tok {
+            Some(t) => {
+                out.push(t);
+                i += 2;
+            }
+            None => {
+                out.push(Token::Literal('%'));
+                out.push(Token::Literal(bytes[i + 1] as char));
+                i += 2;
+            }
+        }
+    }
+    out
+}
+
+/// In `Time` mode, date-only numeric tokens emit MySQL's literal padding
+/// (`%d`→"00", `%Y`→"0000", `%c`/`%e`→"0") and date name-tokens collapse the
+/// whole render to `None` — matches PPL's `getFormattedString` catching the
+/// null-handler NPE and surfacing `ExprNullValue`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum FormatMode {
+    Date,
+    Time,
+}
+
+pub(crate) fn format_datetime(dt: DateTime, format: &str, mode: FormatMode) -> Option {
+    let tokens = tokenize(format);
+    let mut out = String::with_capacity(format.len() + 16);
+    for tok in tokens {
+        match render_token(tok, dt, mode)? {
+            Rendered::Str(s) => out.push_str(&s),
+            Rendered::Char(c) => out.push(c),
+        }
+    }
+    Some(out)
+}
+
+enum Rendered {
+    Str(String),
+    Char(char),
+}
+
+fn render_token(tok: Token, dt: DateTime, mode: FormatMode) -> Option {
+    use Rendered::*;
+    let time_mode = matches!(mode, FormatMode::Time);
+    // Helper: date-only tokens in time-mode → whole render collapses to NULL.
+    macro_rules! date_only {
+        ($body:expr) => {{
+            if time_mode {
+                return None;
+            }
+            $body
+        }};
+    }
+    let r = match tok {
+        Token::Literal(c) => Char(c),
+        Token::A => date_only!(Str(weekday_short(dt.weekday()).into())),
+        Token::B => date_only!(Str(month_short(dt.month()).into())),
+        Token::MUpper => date_only!(Str(month_full(dt.month()).into())),
+        Token::W => date_only!(Str(weekday_full(dt.weekday()).into())),
+        Token::D => date_only!({
+            let d = dt.day();
+            Str(format!("{}{}", d, ordinal_suffix(d)))
+        }),
+        // PPL bug-for-bug: %w uses Mon=1..Sun=7 despite MySQL docs claiming Sun=0.
+        Token::WLower => date_only!(Str(dt.weekday().number_from_monday().to_string())),
+        Token::J => date_only!(Str(format!("{:03}", dt.ordinal()))),
+        Token::UUpper => date_only!(Str(format!("{:02}", week_number_sunday_first(dt)))),
+        Token::ULower => date_only!(Str(format!("{:02}", week_number_monday_first(dt)))),
+        Token::VUpper => date_only!(Str(format!("{:02}", week_number_sunday_first_01(dt)))),
+        Token::VLower => date_only!(Str(format!("{:02}", dt.iso_week().week()))),
+        Token::XUpper => date_only!(Str(format!("{:04}", week_year_sunday_first(dt)))),
+        Token::XLower => date_only!(Str(format!("{:04}", dt.iso_week().year()))),
+        // Numeric date tokens emit zero-literal padding in time mode.
+        Token::C => Str(if time_mode { "0".into() } else { dt.month().to_string() }),
+        Token::DLower => Str(if time_mode { "00".into() } else { format!("{:02}", dt.day()) }),
+        Token::E => Str(if time_mode { "0".into() } else { dt.day().to_string() }),
+        Token::MLower => Str(if time_mode { "00".into() } else { format!("{:02}", dt.month()) }),
+        Token::YUpper => Str(if time_mode { "0000".into() } else { format!("{:04}", dt.year()) }),
+        Token::YLower => Str(if time_mode { "00".into() } else { format!("{:02}", dt.year() % 100) }),
+        Token::F => Str(format!("{:06}", dt.nanosecond() / 1_000)),
+        Token::HUpper => Str(format!("{:02}", dt.hour())),
+        Token::HLower | Token::IUpper => Str(format!("{:02}", twelve_hour(dt.hour()))),
+        Token::ILower => Str(format!("{:02}", dt.minute())),
+        Token::K => Str(dt.hour().to_string()),
+        Token::L => Str(twelve_hour(dt.hour()).to_string()),
+        Token::P => Str(if dt.hour() < 12 { "AM".into() } else { "PM".into() }),
+        Token::R => Str(format!(
+            "{:02}:{:02}:{:02} {}",
+            twelve_hour(dt.hour()),
+            dt.minute(),
+            dt.second(),
+            if dt.hour() < 12 { "AM" } else { "PM" }
+        )),
+        Token::SUpper | Token::SLower => Str(format!("{:02}", dt.second())),
+        Token::T => Str(format!("{:02}:{:02}:{:02}", dt.hour(), dt.minute(), dt.second())),
+    };
+    Some(r)
+}
+
+fn twelve_hour(h: u32) -> u32 {
+    let h = h % 12;
+    if h == 0 { 12 } else { h }
+}
+
+fn ordinal_suffix(day: u32) -> &'static str {
+    if (11..=13).contains(&(day % 100)) {
+        return "th";
+    }
+    match day % 10 {
+        1 => "st",
+        2 => "nd",
+        3 => "rd",
+        _ => "th",
+    }
+}
+
+fn weekday_short(w: Weekday) -> &'static str {
+    match w {
+        Weekday::Mon => "Mon", Weekday::Tue => "Tue", Weekday::Wed => "Wed",
+        Weekday::Thu => "Thu", Weekday::Fri => "Fri", Weekday::Sat => "Sat", Weekday::Sun => "Sun",
+    }
+}
+
+fn weekday_full(w: Weekday) -> &'static str {
+    match w {
+        Weekday::Mon => "Monday", Weekday::Tue => "Tuesday", Weekday::Wed => "Wednesday",
+        Weekday::Thu => "Thursday", Weekday::Fri => "Friday",
+        Weekday::Sat => "Saturday", Weekday::Sun => "Sunday",
+    }
+}
+
+fn month_short(m: u32) -> &'static str {
+    ["", "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
+        .get(m as usize)
+        .copied()
+        .unwrap_or("")
+}
+
+fn month_full(m: u32) -> &'static str {
+    ["", "January", "February", "March", "April", "May", "June",
+     "July", "August", "September", "October", "November", "December"]
+        .get(m as usize)
+        .copied()
+        .unwrap_or("")
+}
+
+fn week_number_sunday_first(dt: DateTime) -> u32 {
+    week_number_with_start(dt, Weekday::Sun)
+}
+
+fn week_number_monday_first(dt: DateTime) -> u32 {
+    week_number_with_start(dt, Weekday::Mon)
+}
+
+/// `%V` — MySQL mode 2: Sunday-first, 01..53; week 0 remaps to prior year's last week.
+fn week_number_sunday_first_01(dt: DateTime) -> u32 {
+    let wn = week_number_sunday_first(dt);
+    if wn == 0 {
+        let last = chrono::NaiveDate::from_ymd_opt(dt.year() - 1, 12, 31).unwrap();
+        let last_dt = Utc.from_utc_datetime(&last.and_hms_opt(0, 0, 0).unwrap());
+        week_number_sunday_first(last_dt)
+    } else {
+        wn
+    }
+}
+
+fn week_year_sunday_first(dt: DateTime) -> i32 {
+    if week_number_sunday_first(dt) == 0 {
+        dt.year() - 1
+    } else {
+        dt.year()
+    }
+}
+
+fn week_number_with_start(dt: DateTime, start: Weekday) -> u32 {
+    let jan1 = chrono::NaiveDate::from_ymd_opt(dt.year(), 1, 1).unwrap();
+    let offset = days_until_start(jan1.weekday(), start);
+    let first_start_doy = 1 + offset;
+    let doy = dt.ordinal();
+    if doy < first_start_doy {
+        0
+    } else {
+        (doy - first_start_doy) / 7 + 1
+    }
+}
+
+fn days_until_start(from: Weekday, start: Weekday) -> u32 {
+    (weekday_idx(start) + 7 - weekday_idx(from)) % 7
+}
+
+fn weekday_idx(w: Weekday) -> u32 {
+    match w {
+        Weekday::Mon => 0, Weekday::Tue => 1, Weekday::Wed => 2, Weekday::Thu => 3,
+        Weekday::Fri => 4, Weekday::Sat => 5, Weekday::Sun => 6,
+    }
+}
+
+// str_to_date parser: minimal recursive-descent mirroring PPL's
+// DateTimeFormatterUtil.parseStringWithDateOrTime (lenient widths, trailing
+// input tolerated, MySQL defaults year=2000/month=1/day=1).
+
+#[derive(Default, Debug)]
+pub(crate) struct Parsed {
+    pub year: Option,
+    pub month: Option,
+    pub day: Option,
+    pub day_of_year: Option,
+    pub hour: Option,
+    pub hour_12: Option,
+    pub minute: Option,
+    pub second: Option,
+    pub micros: Option,
+    pub pm: Option,
+}
+
+impl Parsed {
+    pub(crate) fn to_naive(&self) -> Option {
+        let (y, mo, d) = (self.year, self.month, self.day);
+        let has_time = self.hour.is_some() || self.hour_12.is_some();
+        let has_date_parts = y.is_some() || mo.is_some() || d.is_some() || self.day_of_year.is_some();
+        if !has_date_parts && !has_time {
+            return None;
+        }
+        let year = y.unwrap_or(2000);
+        let (month, day) = if let Some(doy) = self.day_of_year {
+            let date = chrono::NaiveDate::from_yo_opt(year, doy)?;
+            (date.month(), date.day())
+        } else {
+            (mo.unwrap_or(1), d.unwrap_or(1))
+        };
+        let date = chrono::NaiveDate::from_ymd_opt(year, month, day)?;
+        let mut hour = match (self.hour, self.hour_12, self.pm) {
+            (Some(h), _, _) => h,
+            (None, Some(h12), Some(true)) => (h12 % 12) + 12,
+            (None, Some(h12), _) => h12 % 12,
+            (None, None, _) => 0,
+        };
+        if hour > 23 {
+            hour = 23;
+        }
+        let time = chrono::NaiveTime::from_hms_micro_opt(
+            hour,
+            self.minute.unwrap_or(0),
+            self.second.unwrap_or(0),
+            self.micros.unwrap_or(0),
+        )?;
+        Some(chrono::NaiveDateTime::new(date, time))
+    }
+}
+
+pub(crate) fn parse_mysql_format(input: &str, format: &str) -> Option {
+    let tokens = tokenize(format);
+    let input_bytes = input.as_bytes();
+    let mut pos = 0;
+    let mut f = Parsed::default();
+    for tok in tokens {
+        if pos > input_bytes.len() {
+            return None;
+        }
+        while pos < input_bytes.len() && (input_bytes[pos] as char).is_whitespace() {
+            pos += 1;
+        }
+        match tok {
+            Token::Literal(c) => {
+                if c.is_whitespace() {
+                    continue;
+                }
+                if pos >= input_bytes.len() || input_bytes[pos] as char != c {
+                    return None;
+                }
+                pos += 1;
+            }
+            Token::YUpper | Token::XUpper | Token::XLower => {
+                let (v, np) = read_digits(input_bytes, pos, 1, 4)?;
+                f.year = Some(v as i32);
+                pos = np;
+            }
+            Token::YLower => {
+                // MySQL pivots: 00-69→2000-2069, 70-99→1970-1999.
+                let (v, np) = read_digits(input_bytes, pos, 1, 2)?;
+                let yy = v as i32;
+                f.year = Some(if yy < 70 { 2000 + yy } else { 1900 + yy });
+                pos = np;
+            }
+            Token::MLower | Token::C => {
+                let (v, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if !(1..=12).contains(&v) {
+                    return None;
+                }
+                f.month = Some(v);
+                pos = np;
+            }
+            Token::DLower | Token::E | Token::D => {
+                let (v, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if !(1..=31).contains(&v) {
+                    return None;
+                }
+                f.day = Some(v);
+                pos = np;
+                if matches!(tok, Token::D) {
+                    pos = consume_while(input_bytes, pos, |b| b.is_ascii_alphabetic());
+                }
+            }
+            Token::J => {
+                let (v, np) = read_digits(input_bytes, pos, 1, 3)?;
+                if !(1..=366).contains(&v) {
+                    return None;
+                }
+                f.day_of_year = Some(v);
+                pos = np;
+            }
+            Token::HUpper | Token::K => {
+                let (v, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if v > 23 {
+                    return None;
+                }
+                f.hour = Some(v);
+                pos = np;
+            }
+            Token::HLower | Token::IUpper | Token::L => {
+                let (v, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if !(1..=12).contains(&v) {
+                    return None;
+                }
+                f.hour_12 = Some(v);
+                pos = np;
+            }
+            Token::ILower => {
+                let (v, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if v > 59 {
+                    return None;
+                }
+                f.minute = Some(v);
+                pos = np;
+            }
+            Token::SUpper | Token::SLower => {
+                let (v, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if v > 59 {
+                    return None;
+                }
+                f.second = Some(v);
+                pos = np;
+            }
+            Token::F => {
+                let (v, np) = read_digits(input_bytes, pos, 1, 6)?;
+                f.micros = Some(v);
+                pos = np;
+            }
+            Token::P => {
+                pos = read_am_pm(input_bytes, pos, &mut f)?;
+            }
+            Token::R => {
+                let (h, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if !(1..=12).contains(&h) {
+                    return None;
+                }
+                pos = expect_literal(input_bytes, np, b':')?;
+                let (m, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if m > 59 {
+                    return None;
+                }
+                pos = expect_literal(input_bytes, np, b':')?;
+                let (s, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if s > 59 {
+                    return None;
+                }
+                pos = np;
+                while pos < input_bytes.len() && (input_bytes[pos] as char).is_whitespace() {
+                    pos += 1;
+                }
+                pos = read_am_pm(input_bytes, pos, &mut f)?;
+                f.hour_12 = Some(h);
+                f.minute = Some(m);
+                f.second = Some(s);
+            }
+            Token::T => {
+                let (h, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if h > 23 {
+                    return None;
+                }
+                pos = expect_literal(input_bytes, np, b':')?;
+                let (m, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if m > 59 {
+                    return None;
+                }
+                pos = expect_literal(input_bytes, np, b':')?;
+                let (s, np) = read_digits(input_bytes, pos, 1, 2)?;
+                if s > 59 {
+                    return None;
+                }
+                pos = np;
+                f.hour = Some(h);
+                f.minute = Some(m);
+                f.second = Some(s);
+            }
+            // Name tokens: opportunistically consume letters (PPL doesn't validate).
+            Token::A | Token::B | Token::W | Token::MUpper => {
+                pos = consume_while(input_bytes, pos, |b| b.is_ascii_alphabetic());
+            }
+            // Week-based tokens: consume digits, ignore value (PPL doesn't back-solve either).
+            Token::UUpper | Token::ULower | Token::VUpper | Token::VLower | Token::WLower => {
+                let (_, np) = read_digits(input_bytes, pos, 1, 2)?;
+                pos = np;
+            }
+        }
+    }
+    Some(f)
+}
+
+fn read_am_pm(bytes: &[u8], pos: usize, f: &mut Parsed) -> Option {
+    if pos + 2 > bytes.len() {
+        return None;
+    }
+    let s: &str = std::str::from_utf8(&bytes[pos..pos + 2]).ok()?;
+    match s.to_ascii_uppercase().as_str() {
+        "AM" => f.pm = Some(false),
+        "PM" => f.pm = Some(true),
+        _ => return None,
+    }
+    Some(pos + 2)
+}
+
+fn read_digits(bytes: &[u8], pos: usize, min: usize, max: usize) -> Option<(u32, usize)> {
+    let mut end = pos;
+    while end < bytes.len() && end - pos < max && bytes[end].is_ascii_digit() {
+        end += 1;
+    }
+    if end - pos < min {
+        return None;
+    }
+    let s = std::str::from_utf8(&bytes[pos..end]).ok()?;
+    Some((s.parse().ok()?, end))
+}
+
+fn expect_literal(bytes: &[u8], pos: usize, expected: u8) -> Option {
+    if pos >= bytes.len() || bytes[pos] != expected {
+        return None;
+    }
+    Some(pos + 1)
+}
+
+fn consume_while bool>(bytes: &[u8], pos: usize, pred: F) -> usize {
+    let mut p = pos;
+    while p < bytes.len() && pred(bytes[p]) {
+        p += 1;
+    }
+    p
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use chrono::TimeZone;
+
+    fn sample() -> DateTime {
+        // 2020-03-15 10:30:45.123456 UTC (Sunday)
+        Utc.timestamp_micros(1_584_268_245_123_456).single().unwrap()
+    }
+
+    fn fmt(dt: DateTime, f: &str, m: FormatMode) -> Option {
+        format_datetime(dt, f, m)
+    }
+
+    #[test]
+    fn date_mode_common_tokens() {
+        let dt = sample();
+        assert_eq!(fmt(dt, "%Y-%m-%d %H:%i:%S", FormatMode::Date).unwrap(), "2020-03-15 10:30:45");
+        assert_eq!(fmt(dt, "%W, %M %D, %Y", FormatMode::Date).unwrap(), "Sunday, March 15th, 2020");
+        assert_eq!(fmt(dt, "%h:%i:%s %p", FormatMode::Date).unwrap(), "10:30:45 AM");
+    }
+
+    #[test]
+    fn ordinal_suffix_edges() {
+        for (d, want) in [
+            (1, "1st"), (2, "2nd"), (3, "3rd"), (4, "4th"),
+            (11, "11th"), (12, "12th"), (13, "13th"),
+            (21, "21st"), (22, "22nd"), (23, "23rd"), (31, "31st"),
+        ] {
+            let dt = Utc.with_ymd_and_hms(2020, 1, d, 0, 0, 0).unwrap();
+            assert_eq!(fmt(dt, "%D", FormatMode::Date).unwrap(), want, "day={d}");
+        }
+    }
+
+    #[test]
+    fn time_mode_masks_date_tokens() {
+        let dt = sample();
+        assert_eq!(fmt(dt, "%Y-%m-%d %H:%i:%S", FormatMode::Time).unwrap(), "0000-00-00 10:30:45");
+        assert!(fmt(dt, "%W", FormatMode::Time).is_none());
+        assert!(fmt(dt, "%a %H:%i", FormatMode::Time).is_none());
+    }
+
+    #[test]
+    fn twelve_hour_tokens() {
+        let midnight = Utc.with_ymd_and_hms(2020, 1, 1, 0, 0, 0).unwrap();
+        let noon = Utc.with_ymd_and_hms(2020, 1, 1, 12, 0, 0).unwrap();
+        assert_eq!(fmt(midnight, "%h %p", FormatMode::Date).unwrap(), "12 AM");
+        assert_eq!(fmt(noon, "%h %p", FormatMode::Date).unwrap(), "12 PM");
+    }
+
+    #[test]
+    fn week_tokens_iso_and_first_day() {
+        let dt = sample(); // 2020-03-15 = Sunday, ISO week 11
+        assert_eq!(fmt(dt, "%v", FormatMode::Date).unwrap(), "11");
+        assert_eq!(fmt(dt, "%U", FormatMode::Date).unwrap(), "11");
+    }
+
+    #[test]
+    fn unknown_directive_passes_through() {
+        assert_eq!(fmt(sample(), "%Y-%q", FormatMode::Date).unwrap(), "2020-%q");
+    }
+
+    #[test]
+    fn parse_roundtrip_and_defaults() {
+        for (input, format, expected) in [
+            ("2020-03-15 10:30:45", "%Y-%m-%d %H:%i:%S", "2020-03-15 10:30:45"),
+            ("2020-03-15 extra", "%Y-%m-%d", "2020-03-15 00:00:00"),
+            ("2020", "%Y", "2020-01-01 00:00:00"),
+            // PPL uses `today`; we emit the MySQL-compatible 2000-01-01 default for determinism.
+            ("10:30:45", "%H:%i:%S", "2000-01-01 10:30:45"),
+        ] {
+            let ndt = parse_mysql_format(input, format).unwrap().to_naive().unwrap();
+            assert_eq!(ndt.to_string(), expected, "input={input}");
+        }
+    }
+
+    #[test]
+    fn parse_rejects_bad_input() {
+        assert!(parse_mysql_format("not-a-date", "%Y-%m-%d").is_none());
+        assert!(parse_mysql_format("2020-13-01", "%Y-%m-%d").is_none());
+        assert!(parse_mysql_format("", "%Y").is_none());
+    }
+
+    #[test]
+    fn parse_12hr_with_am_pm() {
+        let p = parse_mysql_format("01:30:45 PM", "%h:%i:%s %p").unwrap();
+        assert_eq!(p.to_naive().unwrap().time().to_string(), "13:30:45");
+        let p = parse_mysql_format("12:00:00 AM", "%h:%i:%s %p").unwrap();
+        assert_eq!(p.to_naive().unwrap().time().to_string(), "00:00:00");
+    }
+
+    #[test]
+    fn parse_fractional_seconds() {
+        let p = parse_mysql_format("2020-03-15 10:30:45.123456", "%Y-%m-%d %H:%i:%S.%f").unwrap();
+        assert_eq!(p.to_naive().unwrap().and_utc().timestamp_micros(), 1_584_268_245_123_456);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/str_to_date.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/str_to_date.rs
new file mode 100644
index 0000000000000..cf685f08fb91c
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/str_to_date.rs
@@ -0,0 +1,161 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `str_to_date(input, format)` — parse with MySQL tokens → `Timestamp(us)`. Missing date fields
+//! default to 2000-01-01, missing time → 00:00:00. Unparseable → NULL; trailing input tolerated.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use super::udf_identity;
+
+use datafusion::arrow::array::{Array, ArrayRef, AsArray, TimestampMicrosecondBuilder};
+use datafusion::arrow::datatypes::{DataType, TimeUnit};
+use datafusion::common::{exec_err, plan_err, Result, ScalarValue};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+
+use super::mysql_format::parse_mysql_format;
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(StrToDateUdf::new()));
+}
+
+#[derive(Debug)]
+pub struct StrToDateUdf {
+    signature: Signature,
+}
+
+impl StrToDateUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+udf_identity!(StrToDateUdf, "str_to_date");
+
+impl ScalarUDFImpl for StrToDateUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn name(&self) -> &str {
+        "str_to_date"
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.len() != 2 {
+            return plan_err!("str_to_date expects 2 arguments, got {}", arg_types.len());
+        }
+        Ok(DataType::Timestamp(TimeUnit::Microsecond, None))
+    }
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        if arg_types.len() != 2 {
+            return plan_err!("str_to_date expects 2 arguments, got {}", arg_types.len());
+        }
+        for (i, t) in arg_types.iter().enumerate() {
+            match t {
+                DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {}
+                other => return plan_err!("str_to_date: arg {i} expected string, got {other:?}"),
+            }
+        }
+        Ok(vec![DataType::Utf8, DataType::Utf8])
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() != 2 {
+            return exec_err!("str_to_date expects 2 arguments, got {}", args.args.len());
+        }
+        let n = args.number_rows;
+
+        if let (ColumnarValue::Scalar(input), ColumnarValue::Scalar(fmt)) =
+            (&args.args[0], &args.args[1])
+        {
+            let out = match (utf8_of(input)?, utf8_of(fmt)?) {
+                (Some(i), Some(f)) => parse_to_micros(&i, &f),
+                _ => None,
+            };
+            return Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(
+                out, None,
+            )));
+        }
+
+        let in_arr = args.args[0].clone().into_array(n)?;
+        let f_arr = args.args[1].clone().into_array(n)?;
+        let mut builder = TimestampMicrosecondBuilder::with_capacity(n);
+        for i in 0..n {
+            let in_opt = utf8_at(&in_arr, i)?;
+            let f_opt = utf8_at(&f_arr, i)?;
+            match (in_opt, f_opt) {
+                (Some(s), Some(f)) => match parse_to_micros(&s, &f) {
+                    Some(v) => builder.append_value(v),
+                    None => builder.append_null(),
+                },
+                _ => builder.append_null(),
+            }
+        }
+        Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))
+    }
+}
+
+fn utf8_of(s: &ScalarValue) -> Result> {
+    match s {
+        ScalarValue::Utf8(opt) | ScalarValue::LargeUtf8(opt) => Ok(opt.clone()),
+        other => exec_err!("str_to_date: expected string scalar, got {other:?}"),
+    }
+}
+
+fn utf8_at(array: &ArrayRef, i: usize) -> Result> {
+    let (is_null, value) = match array.data_type() {
+        DataType::Utf8 => {
+            let a = array.as_string::();
+            (a.is_null(i), a.value(i).to_string())
+        }
+        DataType::LargeUtf8 => {
+            let a = array.as_string::();
+            (a.is_null(i), a.value(i).to_string())
+        }
+        other => return exec_err!("str_to_date: expected string array, got {other:?}"),
+    };
+    Ok(if is_null { None } else { Some(value) })
+}
+
+fn parse_to_micros(input: &str, format: &str) -> Option {
+    let parsed = parse_mysql_format(input, format)?;
+    let ndt = parsed.to_naive()?;
+    Some(ndt.and_utc().timestamp_micros())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parses_well_formed_inputs() {
+        // (input, format, expected micros). Date-only defaults 00:00:00.
+        for (i, f, want) in [
+            ("2020-03-15 10:30:45", "%Y-%m-%d %H:%i:%S", 1_584_268_245_000_000_i64),
+            ("2020-03-15", "%Y-%m-%d", 1_584_230_400_000_000),
+            ("2020-03-15 10:30:45.123456", "%Y-%m-%d %H:%i:%S.%f", 1_584_268_245_123_456),
+        ] {
+            assert_eq!(parse_to_micros(i, f), Some(want), "input={i}");
+        }
+        assert!(parse_to_micros("2020-03-15 extra", "%Y-%m-%d").is_some());
+    }
+
+    #[test]
+    fn unparseable_input_returns_none() {
+        assert!(parse_to_micros("not-a-date", "%Y-%m-%d").is_none());
+        assert!(parse_to_micros("2020-13-01", "%Y-%m-%d").is_none());
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/strftime.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/strftime.rs
new file mode 100644
index 0000000000000..89f667dbac2b9
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/strftime.rs
@@ -0,0 +1,619 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `strftime(value, format)` — POSIX-style datetime formatter mirroring PPL `StrftimeFunction`.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use chrono::{DateTime, Datelike, NaiveDate, TimeZone, Timelike, Utc, Weekday};
+use datafusion::arrow::array::{
+    Array, ArrayRef, AsArray, StringArray, TimestampMicrosecondArray,
+};
+use datafusion::arrow::datatypes::{DataType, TimeUnit};
+use datafusion::common::{exec_err, plan_err, Result, ScalarValue};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+
+use super::udf_identity;
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(StrftimeUdf::new()));
+}
+
+/// SQL plugin's `StrftimeFunction.MAX_UNIX_TIMESTAMP` — values beyond this yield null.
+const MAX_UNIX_SECONDS: i64 = 32_536_771_199;
+/// Numeric inputs with `abs(v) >= 1e11` are auto-detected as milliseconds
+/// (divided by 1000), matching the SQL plugin's behavior.
+const MILLIS_HEURISTIC_THRESHOLD: f64 = 1e11;
+
+#[derive(Debug)]
+pub struct StrftimeUdf {
+    signature: Signature,
+}
+
+impl StrftimeUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+udf_identity!(StrftimeUdf, "strftime");
+
+impl ScalarUDFImpl for StrftimeUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn name(&self) -> &str {
+        "strftime"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.len() != 2 {
+            return plan_err!("strftime expects 2 arguments, got {}", arg_types.len());
+        }
+        Ok(DataType::Utf8)
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        if arg_types.len() != 2 {
+            return plan_err!("strftime expects 2 arguments, got {}", arg_types.len());
+        }
+        let value = match &arg_types[0] {
+            t if t.is_numeric() => DataType::Float64,
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => DataType::Float64,
+            DataType::Timestamp(_, _) | DataType::Date32 | DataType::Date64 => {
+                DataType::Timestamp(TimeUnit::Microsecond, None)
+            }
+            other => return plan_err!("strftime: arg 0 expected numeric/timestamp/date/string, got {other:?}"),
+        };
+        let format = match &arg_types[1] {
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => DataType::Utf8,
+            other => return plan_err!("strftime: arg 1 expected string, got {other:?}"),
+        };
+        Ok(vec![value, format])
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        if args.args.len() != 2 {
+            return exec_err!("strftime expects 2 arguments, got {}", args.args.len());
+        }
+        let n = args.number_rows;
+
+        if let (ColumnarValue::Scalar(val), ColumnarValue::Scalar(fmt)) =
+            (&args.args[0], &args.args[1])
+        {
+            let format = scalar_utf8(fmt)?;
+            let rendered = match val {
+                ScalarValue::Float64(v) => {
+                    v.and_then(|v| render_from_seconds(v, format.as_deref()))
+                }
+                ScalarValue::TimestampMicrosecond(v, _) => {
+                    v.and_then(|v| render_from_micros(v, format.as_deref()))
+                }
+                other => {
+                    return exec_err!(
+                        "strftime: unsupported scalar value type after coercion: {other:?}"
+                    )
+                }
+            };
+            return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(rendered)));
+        }
+
+        let value_array = args.args[0].clone().into_array(n)?;
+        let format_array = args.args[1].clone().into_array(n)?;
+        let out: StringArray = match value_array.data_type() {
+            DataType::Float64 => {
+                let values = value_array.as_primitive::();
+                (0..n)
+                    .map(|i| if values.is_null(i) { None } else {
+                        match format_at(&format_array, i) {
+                            Ok(Some(f)) => render_from_seconds(values.value(i), Some(&f)),
+                            _ => None,
+                        }
+                    })
+                    .collect()
+            }
+            DataType::Timestamp(TimeUnit::Microsecond, _) => {
+                let values = value_array
+                    .as_any()
+                    .downcast_ref::()
+                    .expect("coerce_types canonicalizes to TimestampMicrosecond");
+                (0..n)
+                    .map(|i| if values.is_null(i) { None } else {
+                        match format_at(&format_array, i) {
+                            Ok(Some(f)) => render_from_micros(values.value(i), Some(&f)),
+                            _ => None,
+                        }
+                    })
+                    .collect()
+            }
+            other => return exec_err!("strftime: unsupported value array type after coercion: {other:?}"),
+        };
+        Ok(ColumnarValue::Array(Arc::new(out) as ArrayRef))
+    }
+}
+
+fn scalar_utf8(s: &ScalarValue) -> Result> {
+    match s {
+        ScalarValue::Utf8(opt) | ScalarValue::LargeUtf8(opt) => Ok(opt.clone()),
+        other => exec_err!("strftime: format must be VARCHAR, got {other:?}"),
+    }
+}
+
+fn format_at(array: &ArrayRef, row: usize) -> Result> {
+    let (is_null, value) = match array.data_type() {
+        DataType::Utf8 => {
+            let a = array.as_string::();
+            (a.is_null(row), a.value(row).to_string())
+        }
+        DataType::LargeUtf8 => {
+            let a = array.as_string::();
+            (a.is_null(row), a.value(row).to_string())
+        }
+        other => return exec_err!("strftime: expected string format array, got {other:?}"),
+    };
+    Ok(if is_null { None } else { Some(value) })
+}
+
+fn render_from_seconds(value: f64, format: Option<&str>) -> Option {
+    let format = format?;
+    if !value.is_finite() {
+        return None;
+    }
+    // Numeric inputs with `abs(v) >= 1e11` are auto-detected as milliseconds,
+    // matching the SQL plugin's `StrftimeFormatterUtil` threshold.
+    let seconds_value = if value.abs() >= MILLIS_HEURISTIC_THRESHOLD {
+        value / 1000.0
+    } else {
+        value
+    };
+    let seconds = seconds_value.trunc() as i64;
+    if !(-MAX_UNIX_SECONDS..=MAX_UNIX_SECONDS).contains(&seconds) {
+        return None;
+    }
+    let fraction = seconds_value - seconds as f64;
+    let nanos = (fraction * 1_000_000_000.0) as i32;
+    // `Instant.ofEpochSecond(s, n)` normalizes negative n; chrono requires
+    // non-negative nanos paired with an adjusted seconds value.
+    let (adj_seconds, adj_nanos) = if nanos < 0 {
+        (seconds - 1, (nanos + 1_000_000_000) as u32)
+    } else {
+        (seconds, nanos as u32)
+    };
+    let dt = Utc.timestamp_opt(adj_seconds, adj_nanos).single()?;
+    Some(format_with_directives(dt, format))
+}
+
+fn render_from_micros(value: i64, format: Option<&str>) -> Option {
+    let format = format?;
+    let seconds = value.div_euclid(1_000_000);
+    let micros_remainder = value.rem_euclid(1_000_000) as u32;
+    if !(-MAX_UNIX_SECONDS..=MAX_UNIX_SECONDS).contains(&seconds) {
+        return None;
+    }
+    let dt = Utc
+        .timestamp_opt(seconds, micros_remainder * 1_000)
+        .single()?;
+    Some(format_with_directives(dt, format))
+}
+
+// Directive engine — mirrors `StrftimeFormatterUtil`'s handler table. Unknown
+// directives fall through as literal text (matches the SQL plugin).
+fn format_with_directives(dt: DateTime, format: &str) -> String {
+    let bytes = format.as_bytes();
+    let mut out = String::with_capacity(format.len() + 16);
+    let mut i = 0;
+    while i < bytes.len() {
+        if bytes[i] != b'%' {
+            out.push(bytes[i] as char);
+            i += 1;
+            continue;
+        }
+        // Lookahead: at least one char after `%`.
+        if i + 1 >= bytes.len() {
+            out.push('%');
+            break;
+        }
+
+        // %:z / %::z / %:::z
+        if bytes[i + 1] == b':' {
+            let mut colons = 1;
+            while i + 1 + colons < bytes.len() && bytes[i + 1 + colons] == b':' {
+                colons += 1;
+            }
+            if i + 1 + colons < bytes.len() && bytes[i + 1 + colons] == b'z' {
+                append_tz_colon(&mut out, colons);
+                i += 2 + colons;
+                continue;
+            }
+            out.push('%');
+            i += 1;
+            continue;
+        }
+
+        // %Ez (extended tz offset in minutes)
+        if bytes[i + 1] == b'E' && i + 2 < bytes.len() && bytes[i + 2] == b'z' {
+            append_tz_offset_minutes(&mut out, dt);
+            i += 3;
+            continue;
+        }
+
+        // %N / %Q with optional precision
+        if bytes[i + 1].is_ascii_digit()
+            && i + 2 < bytes.len()
+            && (bytes[i + 2] == b'N' || bytes[i + 2] == b'Q')
+        {
+            let precision = (bytes[i + 1] - b'0') as usize;
+            append_subsecond(&mut out, dt, bytes[i + 2], precision);
+            i += 3;
+            continue;
+        }
+        if bytes[i + 1] == b'N' || bytes[i + 1] == b'Q' {
+            let kind = bytes[i + 1];
+            // Default precision per StrftimeFormatterUtil: %N=9 nanos, %Q=3 millis.
+            let precision = if kind == b'N' { 9 } else { 3 };
+            append_subsecond(&mut out, dt, kind, precision);
+            i += 2;
+            continue;
+        }
+
+        let directive = bytes[i + 1];
+        if append_simple_directive(&mut out, dt, directive) {
+            i += 2;
+        } else {
+            out.push('%');
+            out.push(directive as char);
+            i += 2;
+        }
+    }
+    out
+}
+
+fn append_simple_directive(out: &mut String, dt: DateTime, directive: u8) -> bool {
+    let h12 = {
+        let h = dt.hour() % 12;
+        if h == 0 { 12 } else { h }
+    };
+    match directive {
+        b'%' => out.push('%'),
+        b'c' => out.push_str(&format!("{} {} {:02} {:02}:{:02}:{:02} {:04}",
+            weekday_short(dt.weekday()), month_short(dt.month()),
+            dt.day(), dt.hour(), dt.minute(), dt.second(), dt.year())),
+        b'+' => out.push_str(&format!("{} {} {:02} {:02}:{:02}:{:02} UTC {:04}",
+            weekday_short(dt.weekday()), month_short(dt.month()),
+            dt.day(), dt.hour(), dt.minute(), dt.second(), dt.year())),
+        b'f' => out.push_str(&format!("{:06}", dt.nanosecond() / 1000)),
+        b'H' => out.push_str(&format!("{:02}", dt.hour())),
+        b'I' => out.push_str(&format!("{:02}", h12)),
+        b'k' => out.push_str(&format!("{:2}", dt.hour())),
+        b'M' => out.push_str(&format!("{:02}", dt.minute())),
+        b'p' => out.push_str(if dt.hour() < 12 { "AM" } else { "PM" }),
+        b'S' => out.push_str(&format!("{:02}", dt.second())),
+        b's' => out.push_str(&dt.timestamp().to_string()),
+        b'T' | b'X' => out.push_str(&format!("{:02}:{:02}:{:02}", dt.hour(), dt.minute(), dt.second())),
+        b'Z' => out.push_str("UTC"),
+        b'z' => out.push_str("+0000"),
+        b'F' => out.push_str(&format!("{:04}-{:02}-{:02}", dt.year(), dt.month(), dt.day())),
+        b'x' => out.push_str(&format!("{:02}/{:02}/{:04}", dt.month(), dt.day(), dt.year())),
+        b'A' => out.push_str(weekday_full(dt.weekday())),
+        b'a' => out.push_str(weekday_short(dt.weekday())),
+        b'w' => out.push_str(&weekday_numeric_sunday_zero(dt.weekday()).to_string()),
+        b'd' => out.push_str(&format!("{:02}", dt.day())),
+        b'e' => out.push_str(&format!("{:2}", dt.day())),
+        b'j' => out.push_str(&format!("{:03}", dt.ordinal())),
+        b'V' => out.push_str(&format!("{:02}", dt.iso_week().week())),
+        b'U' => out.push_str(&format!("{:02}", week_of_year_sunday_start(dt))),
+        b'b' => out.push_str(month_short(dt.month())),
+        b'B' => out.push_str(month_full(dt.month())),
+        b'm' => out.push_str(&format!("{:02}", dt.month())),
+        b'C' => out.push_str(&format!("{:02}", dt.year() / 100)),
+        b'g' => out.push_str(&format!("{:02}", dt.iso_week().year() % 100)),
+        b'G' => out.push_str(&format!("{:04}", dt.iso_week().year())),
+        b'y' => out.push_str(&format!("{:02}", dt.year() % 100)),
+        b'Y' => out.push_str(&format!("{:04}", dt.year())),
+        _ => return false,
+    }
+    true
+}
+
+fn append_tz_colon(out: &mut String, colons: usize) {
+    match colons {
+        1 => out.push_str("+00:00"),
+        2 => out.push_str("+00:00:00"),
+        3 => out.push_str("+0"),
+        _ => out.push_str("+00:00"),
+    }
+}
+
+fn append_tz_offset_minutes(out: &mut String, _dt: DateTime) {
+    out.push_str("+0");
+}
+
+fn append_subsecond(out: &mut String, dt: DateTime, kind: u8, precision: usize) {
+    let nanos = dt.nanosecond();
+    match kind {
+        b'N' => {
+            let scaled = nanos as f64 / 1_000_000_000.0;
+            let truncated = (scaled * 10f64.powi(precision as i32)) as u64;
+            out.push_str(&format!("{:0>width$}", truncated, width = precision));
+        }
+        b'Q' => match precision {
+            6 => out.push_str(&format!("{:06}", nanos / 1_000)),
+            9 => out.push_str(&format!("{:09}", nanos)),
+            _ => out.push_str(&format!("{:03}", nanos / 1_000_000)),
+        },
+        _ => {}
+    }
+}
+
+fn weekday_short(w: Weekday) -> &'static str {
+    match w {
+        Weekday::Mon => "Mon", Weekday::Tue => "Tue", Weekday::Wed => "Wed",
+        Weekday::Thu => "Thu", Weekday::Fri => "Fri", Weekday::Sat => "Sat", Weekday::Sun => "Sun",
+    }
+}
+
+fn weekday_full(w: Weekday) -> &'static str {
+    match w {
+        Weekday::Mon => "Monday", Weekday::Tue => "Tuesday", Weekday::Wed => "Wednesday",
+        Weekday::Thu => "Thursday", Weekday::Fri => "Friday",
+        Weekday::Sat => "Saturday", Weekday::Sun => "Sunday",
+    }
+}
+
+fn weekday_numeric_sunday_zero(w: Weekday) -> u32 {
+    match w {
+        Weekday::Sun => 0, Weekday::Mon => 1, Weekday::Tue => 2, Weekday::Wed => 3,
+        Weekday::Thu => 4, Weekday::Fri => 5, Weekday::Sat => 6,
+    }
+}
+
+fn month_short(m: u32) -> &'static str {
+    ["", "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
+        .get(m as usize).copied().unwrap_or("")
+}
+
+fn month_full(m: u32) -> &'static str {
+    ["", "January", "February", "March", "April", "May", "June",
+     "July", "August", "September", "October", "November", "December"]
+        .get(m as usize).copied().unwrap_or("")
+}
+
+/// `%U` — week of year, Sunday-start; week 0 is the partial week before the
+/// first Sunday of the year (matches `WeekFields.SUNDAY_START.weekOfYear() - 1`).
+fn week_of_year_sunday_start(dt: DateTime) -> u32 {
+    let doy = dt.ordinal();
+    let jan1 = NaiveDate::from_ymd_opt(dt.year(), 1, 1).expect("valid jan 1").weekday();
+    let days_to_first_sunday = match jan1 {
+        Weekday::Sun => 1, Weekday::Mon => 7, Weekday::Tue => 6, Weekday::Wed => 5,
+        Weekday::Thu => 4, Weekday::Fri => 3, Weekday::Sat => 2,
+    };
+    if doy < days_to_first_sunday { 0 } else { (doy - days_to_first_sunday) / 7 + 1 }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use datafusion::arrow::array::{Float64Array, Int64Array};
+    use datafusion::arrow::datatypes::Field;
+
+    fn udf() -> StrftimeUdf {
+        StrftimeUdf::new()
+    }
+
+    fn invoke(args: Vec, arg_types: &[DataType], n: usize) -> Result {
+        let u = udf();
+        let arg_fields = arg_types
+            .iter()
+            .enumerate()
+            .map(|(i, t)| Arc::new(Field::new(format!("a{i}"), t.clone(), true)))
+            .collect();
+        u.invoke_with_args(ScalarFunctionArgs {
+            args,
+            arg_fields,
+            number_rows: n,
+            return_field: Arc::new(Field::new(u.name(), DataType::Utf8, true)),
+            config_options: Arc::new(Default::default()),
+        })
+    }
+
+    fn call_f64(v: f64, f: &str) -> Option {
+        let args = vec![
+            ColumnarValue::Scalar(ScalarValue::Float64(Some(v))),
+            ColumnarValue::Scalar(ScalarValue::Utf8(Some(f.to_string()))),
+        ];
+        utf8_scalar(invoke(args, &[DataType::Float64, DataType::Utf8], 1).unwrap())
+    }
+
+    fn call_ts_us(us: i64, f: &str) -> Option {
+        let args = vec![
+            ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(Some(us), None)),
+            ColumnarValue::Scalar(ScalarValue::Utf8(Some(f.to_string()))),
+        ];
+        utf8_scalar(
+            invoke(
+                args,
+                &[DataType::Timestamp(TimeUnit::Microsecond, None), DataType::Utf8],
+                1,
+            )
+            .unwrap(),
+        )
+    }
+
+    fn utf8_scalar(v: ColumnarValue) -> Option {
+        match v {
+            ColumnarValue::Scalar(ScalarValue::Utf8(opt)) => opt,
+            other => panic!("expected Utf8 scalar, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn complex_format_matches_it() {
+        assert_eq!(
+            call_f64(1521467703.0, "%a, %b %d, %Y %I:%M:%S %p %Z").unwrap(),
+            "Mon, Mar 19, 2018 01:55:03 PM UTC"
+        );
+    }
+
+    #[test]
+    fn integer_seconds_ymd_hms() {
+        assert_eq!(call_f64(1521467703.0, "%Y-%m-%d %H:%M:%S").unwrap(), "2018-03-19 13:55:03");
+    }
+
+    #[test]
+    fn fractional_seconds_with_3q_milliseconds() {
+        assert_eq!(
+            call_f64(1521467703.123456, "%Y-%m-%d %H:%M:%S.%3Q").unwrap(),
+            "2018-03-19 13:55:03.123"
+        );
+    }
+
+    #[test]
+    fn negative_epoch_rendering() {
+        assert_eq!(call_f64(-1.0, "%Y-%m-%d %H:%M:%S").unwrap(), "1969-12-31 23:59:59");
+        assert_eq!(call_f64(-86400.0, "%Y-%m-%d").unwrap(), "1969-12-31");
+        assert_eq!(call_f64(-31_536_000.0, "%Y-%m-%d").unwrap(), "1969-01-01");
+        assert_eq!(call_f64(-946_771_200.0, "%Y-%m-%d").unwrap(), "1940-01-01");
+    }
+
+    #[test]
+    fn timestamp_micros_renders_date_and_time() {
+        assert_eq!(call_ts_us(1_521_467_703_000_000, "%F").unwrap(), "2018-03-19");
+        assert_eq!(call_ts_us(1_521_467_703_000_000, "%H:%M:%S").unwrap(), "13:55:03");
+    }
+
+    #[test]
+    fn ms_autodetect_above_1e11() {
+        // `abs(v) >= 1e11` auto-treats input as ms; below stays in seconds.
+        assert_eq!(
+            call_f64(1_521_467_703_000.0, "%Y-%m-%d %H:%M:%S").unwrap(),
+            "2018-03-19 13:55:03"
+        );
+        assert_eq!(call_f64(1_521_467_703.0, "%Y-%m-%d").unwrap(), "2018-03-19");
+    }
+
+    #[test]
+    fn out_of_range_and_non_finite_yield_null() {
+        // 5e10 is above MAX_UNIX_SECONDS (3.25e10) but below the 1e11 ms threshold.
+        assert!(call_f64(5e10, "%Y-%m-%d").is_none());
+        assert!(call_f64(f64::NAN, "%Y-%m-%d").is_none());
+    }
+
+    #[test]
+    fn null_operand_yields_null() {
+        for args in [
+            vec![
+                ColumnarValue::Scalar(ScalarValue::Float64(None)),
+                ColumnarValue::Scalar(ScalarValue::Utf8(Some("%Y".into()))),
+            ],
+            vec![
+                ColumnarValue::Scalar(ScalarValue::Float64(Some(0.0))),
+                ColumnarValue::Scalar(ScalarValue::Utf8(None)),
+            ],
+        ] {
+            let out = invoke(args, &[DataType::Float64, DataType::Utf8], 1).unwrap();
+            assert!(utf8_scalar(out).is_none());
+        }
+    }
+
+    #[test]
+    fn weekday_directives() {
+        // 2018-03-19 = Monday
+        assert_eq!(call_f64(1521467703.0, "%A").unwrap(), "Monday");
+        assert_eq!(call_f64(1521467703.0, "%a").unwrap(), "Mon");
+        assert_eq!(call_f64(1521467703.0, "%w").unwrap(), "1");
+    }
+
+    #[test]
+    fn twelve_hour_clock_and_ampm() {
+        assert_eq!(call_f64(1521467703.0, "%I").unwrap(), "01");
+        assert_eq!(call_f64(1521467703.0, "%p").unwrap(), "PM");
+        assert_eq!(call_f64(1521417600.0, "%I").unwrap(), "12"); // midnight
+    }
+
+    #[test]
+    fn week_ordinal_and_literal() {
+        assert_eq!(call_f64(1521467703.0, "%V").unwrap(), "12");
+        assert_eq!(call_f64(1521467703.0, "%j").unwrap(), "078");
+        assert_eq!(call_f64(1521467703.0, "100%%").unwrap(), "100%");
+        // Unknown directive passes through verbatim (matches SQL plugin).
+        assert_eq!(call_f64(1521467703.0, "%Y-%q").unwrap(), "2018-%q");
+    }
+
+    #[test]
+    fn tz_directives_render_utc() {
+        assert_eq!(call_f64(1521467703.0, "%z").unwrap(), "+0000");
+        assert_eq!(call_f64(1521467703.0, "%:z").unwrap(), "+00:00");
+        assert_eq!(call_f64(1521467703.0, "%Ez").unwrap(), "+0");
+    }
+
+    #[test]
+    fn subsecond_default_precision() {
+        // 1521467703.123456 isn't exactly f64-representable (≈ ...01), so %N shows trailing 001.
+        assert_eq!(call_f64(1521467703.123456, "%N").unwrap(), "123456001");
+        assert_eq!(call_f64(1521467703.123456, "%Q").unwrap(), "123");
+    }
+
+    #[test]
+    fn array_f64_with_scalar_format() {
+        let arr: ArrayRef = Arc::new(Float64Array::from(vec![Some(1521467703.0), None, Some(-86400.0)]));
+        let u = udf();
+        let args = ScalarFunctionArgs {
+            args: vec![
+                ColumnarValue::Array(arr),
+                ColumnarValue::Scalar(ScalarValue::Utf8(Some("%Y-%m-%d".into()))),
+            ],
+            arg_fields: vec![
+                Arc::new(Field::new("v", DataType::Float64, true)),
+                Arc::new(Field::new("f", DataType::Utf8, true)),
+            ],
+            number_rows: 3,
+            return_field: Arc::new(Field::new(u.name(), DataType::Utf8, true)),
+            config_options: Arc::new(Default::default()),
+        };
+        let out = u.invoke_with_args(args).unwrap();
+        let ColumnarValue::Array(a) = out else { panic!("expected array"); };
+        let s = a.as_string::();
+        assert_eq!(s.value(0), "2018-03-19");
+        assert!(s.is_null(1));
+        assert_eq!(s.value(2), "1969-12-31");
+    }
+
+    #[test]
+    fn coerce_types_folds_inputs() {
+        let u = udf();
+        // Numeric + string fold onto Float64.
+        for t in [DataType::Int64, DataType::Int32, DataType::UInt8, DataType::Float32, DataType::Utf8] {
+            assert_eq!(u.coerce_types(&[t.clone(), DataType::Utf8]).unwrap()[0], DataType::Float64, "{t:?}");
+        }
+        // Temporal types fold onto Timestamp(us).
+        let canonical = DataType::Timestamp(TimeUnit::Microsecond, None);
+        for t in [
+            DataType::Date32, DataType::Date64,
+            DataType::Timestamp(TimeUnit::Millisecond, None),
+            DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
+        ] {
+            assert_eq!(u.coerce_types(&[t.clone(), DataType::Utf8]).unwrap()[0], canonical, "{t:?}");
+        }
+        // Non-numeric / non-temporal rejected.
+        let err = u.coerce_types(&[DataType::Boolean, DataType::Utf8]).unwrap_err();
+        assert!(err.to_string().contains("expected numeric/timestamp"));
+        // Int64 shape sanity — exercised by the array test above.
+        let _arr: ArrayRef = Arc::new(Int64Array::from(vec![Some(1521467703i64)]));
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/time_format.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/time_format.rs
new file mode 100644
index 0000000000000..7c66e9216867b
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/udf/time_format.rs
@@ -0,0 +1,91 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+//! `time_format(time_or_datetime, format)` — time-mode sibling of date_format.
+//! MySQL time-format rules: date-only numeric tokens render with literal zeros
+//! (%d→"00", %Y→"0000"); date-only name tokens (%W, %a, %M, %D, %j, %w, %U/%u,
+//! %V/%v, %X/%x, %b) cause the whole render to collapse to NULL.
+
+use std::any::Any;
+
+use super::udf_identity;
+
+use datafusion::arrow::datatypes::{DataType, TimeUnit};
+use datafusion::common::{plan_err, Result};
+use datafusion::execution::context::SessionContext;
+use datafusion::logical_expr::{
+    ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
+};
+
+use super::date_format::format_dispatch;
+use super::mysql_format::FormatMode;
+
+pub fn register_all(ctx: &SessionContext) {
+    ctx.register_udf(ScalarUDF::from(TimeFormatUdf::new()));
+}
+
+#[derive(Debug)]
+pub struct TimeFormatUdf {
+    signature: Signature,
+}
+
+impl TimeFormatUdf {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+udf_identity!(TimeFormatUdf, "time_format");
+
+impl ScalarUDFImpl for TimeFormatUdf {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn name(&self) -> &str {
+        "time_format"
+    }
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+    fn return_type(&self, arg_types: &[DataType]) -> Result {
+        if arg_types.len() != 2 {
+            return plan_err!("time_format expects 2 arguments, got {}", arg_types.len());
+        }
+        Ok(DataType::Utf8)
+    }
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result> {
+        if arg_types.len() != 2 {
+            return plan_err!("time_format expects 2 arguments, got {}", arg_types.len());
+        }
+        let ts = match &arg_types[0] {
+            DataType::Timestamp(_, _)
+            | DataType::Date32
+            | DataType::Date64
+            | DataType::Time32(_)
+            | DataType::Time64(_) => DataType::Timestamp(TimeUnit::Microsecond, None),
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
+                DataType::Timestamp(TimeUnit::Microsecond, None)
+            }
+            other => {
+                return plan_err!(
+                    "time_format: arg 0 expected time/timestamp/date/string, got {other:?}"
+                )
+            }
+        };
+        let fmt = match &arg_types[1] {
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => DataType::Utf8,
+            other => return plan_err!("time_format: arg 1 expected string, got {other:?}"),
+        };
+        Ok(vec![ts, fmt])
+    }
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result {
+        format_dispatch("time_format", FormatMode::Time, args)
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java
index 972eecd8a66b5..18b21a06fdd0a 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/internalClusterTest/java/org/opensearch/be/datafusion/ScalarDateTimeFunctionIT.java
@@ -24,7 +24,6 @@
  *   
          • {@code DAYOFWEEK / DAY_OF_WEEK} — DF Sun=0..Sat=6, PPL Sun=1..Sat=7
            • 
  *   
          • {@code SYSDATE} — DF now() is query-constant, PPL per-row
            • 
  *   
          • {@code DATE_FORMAT / TIME_FORMAT} — DF chrono tokens, PPL MySQL dialect
            • 
- *   
          • {@code STRFTIME} — DF chrono tokens, PPL POSIX dialect
            • 
  *   
          • {@code FROM_UNIXTIME(epoch, fmt)} / {@code DATETIME(expr, tz)} 2-arg overloads —
  *       no matching DF signature
            • 
  *   
          • {@code EXTRACT(unit FROM ts)} — isthmus resolves {@link org.apache.calcite.sql.SqlKind#EXTRACT}
@@ -131,4 +130,29 @@ public void testConvertTz() {
     public void testUnixTimestamp() {
         assertScalarLong("unix_timestamp(created_at)", 1718447400L);
     }
+
+    // ── strftime ──────────────────────────────────────────────────────────────
+    // Fixture row 1: created_at = 2024-06-15T10:30:00Z, unix seconds = 1718447400.
+    // Reference value 1521467703 = 2018-03-19T13:55:03Z — matches the SQL-plugin
+    // CalciteDateTimeFunctionIT golden cases exactly.
+
+    public void testStrftimeIntegerUnixSeconds() {
+        assertScalarString("strftime(1521467703, '%Y-%m-%d %H:%M:%S')", "2018-03-19 13:55:03");
+    }
+
+    public void testStrftimeComplexFormat() {
+        assertScalarString("strftime(1521467703, '%a, %b %d, %Y %I:%M:%S %p %Z')", "Mon, Mar 19, 2018 01:55:03 PM UTC");
+    }
+
+    public void testStrftimeFractionalSeconds() {
+        assertScalarString("strftime(1521467703.123456, '%Y-%m-%d %H:%M:%S.%3Q')", "2018-03-19 13:55:03.123");
+    }
+
+    public void testStrftimeNegativeTimestamp() {
+        assertScalarString("strftime(-1, '%Y-%m-%d %H:%M:%S')", "1969-12-31 23:59:59");
+    }
+
+    public void testStrftimeOnDateField() {
+        assertScalarString("strftime(created_at, '%Y-%m-%d %H:%M:%S')", "2024-06-15 10:30:00");
+    }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
index d8977b894a332..28bc0a8e692f3 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
@@ -165,44 +165,71 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
         ScalarFunction.SCALAR_MAX,
         ScalarFunction.SCALAR_MIN,
         // Date-part extractors rewrite to date_part(, ts) via DatePartAdapters.
-        // Functions whose DF-builtin semantics diverge from legacy PPL (SECOND / DAYOFWEEK /
-        // SYSDATE / DATE_FORMAT / TIME_FORMAT / STRFTIME, plus 2-arg FROM_UNIXTIME / DATETIME)
-        // are omitted here and stay on the legacy engine until a dedicated Rust UDF lands —
-        // matching the convert_tz / to_unixtime pattern already in this plugin.
+        // SECOND / SECOND_OF_MINUTE / DAYOFWEEK / DAY_OF_WEEK use dedicated adapters
+        // (FLOOR cast for SECOND, +1 offset for DAYOFWEEK) to preserve PPL's MySQL
+        // semantics on top of DF's date_part; see SecondAdapter / DayOfWeekAdapter.
         ScalarFunction.YEAR,
         ScalarFunction.QUARTER,
         ScalarFunction.MONTH,
         ScalarFunction.MONTH_OF_YEAR,
         ScalarFunction.DAY,
         ScalarFunction.DAYOFMONTH,
+        ScalarFunction.DAYOFWEEK,
+        ScalarFunction.DAY_OF_WEEK,
         ScalarFunction.DAYOFYEAR,
         ScalarFunction.DAY_OF_YEAR,
         ScalarFunction.HOUR,
         ScalarFunction.HOUR_OF_DAY,
         ScalarFunction.MINUTE,
         ScalarFunction.MINUTE_OF_HOUR,
+        ScalarFunction.SECOND,
+        ScalarFunction.SECOND_OF_MINUTE,
         ScalarFunction.MICROSECOND,
         ScalarFunction.WEEK,
         ScalarFunction.WEEK_OF_YEAR,
-        // Niladic now/current_* family maps 1:1 to DF builtins.
+        // Niladic now/current_* family maps 1:1 to DF builtins. SYSDATE is an
+        // approximation — PPL SYSDATE uses the systemClock (call-time) while NOW
+        // uses queryStartClock; the wall-clock difference is sub-millisecond on a
+        // single-statement OLAP query so routing both to DF `now` is acceptable.
         ScalarFunction.NOW,
         ScalarFunction.CURRENT_TIMESTAMP,
         ScalarFunction.CURRENT_DATE,
         ScalarFunction.CURDATE,
         ScalarFunction.CURRENT_TIME,
         ScalarFunction.CURTIME,
+        ScalarFunction.SYSDATE,
         ScalarFunction.CONVERT_TZ,
         ScalarFunction.UNIX_TIMESTAMP,
-        // DATE(expr) / TIME(expr) / MAKETIME(h,m,s) are intentionally not advertised:
-        // PPL's Calcite binding for these returns VARCHAR rather than DATE/TIME, so
-        // downstream `year(date(ts))` / `hour(maketime(...))` lowers to
-        // date_part(string, string?) — no matching DataFusion signature. Left on the
-        // legacy engine until PPL wires them to produce real DATE/TIME types.
-        // EXTRACT(unit FROM ts) is also not advertised: isthmus resolves SqlKind.EXTRACT
-        // through scalar-function lookup rather than emitting a native Substrait extract,
-        // and we'd need a dedicated adapter + yaml entry to route it to DataFusion's
-        // date_part. Left on legacy engine until that adapter lands; PPL date-part
-        // functions cover the same semantics.
+        ScalarFunction.STRFTIME,
+        // PPL `time(expr)` / `date(expr)` — extract time-of-day / date component
+        // from a TIMESTAMP / DATE / TIME / string value. Route to DataFusion's
+        // builtins `to_time` / `to_date` via TimeAdapter / DateAdapter. Safe on
+        // the analytics-engine path because sql-repo PR #5408
+        // (DatetimeUdtNormalizeRule) rewrites EXPR_TIME / EXPR_DATE → standard
+        // Calcite TIME / DATE on the RexCall return type, so downstream consumers
+        // see a real time/date type and Isthmus serializes accordingly.
+        ScalarFunction.TIME,
+        ScalarFunction.DATE,
+        // PPL `datetime(expr)` — parse/cast into a TIMESTAMP. Routes to DF's
+        // builtin `to_timestamp` via DatetimeAdapter. The single-arg
+        // `timestamp(expr)` form shares these semantics but its ScalarFunction
+        // slot is already bound to TimestampFunctionAdapter for VARCHAR literal
+        // folding, so it stays on the legacy engine.
+        ScalarFunction.DATETIME,
+        // PPL extract / make* / format / from_unixtime are implemented as Rust UDFs
+        // to preserve MySQL semantics that DataFusion builtins don't match: EXTRACT
+        // supports 10 composite units (DAY_SECOND → ddHHmmss etc.) that are not a
+        // single date_part; MAKETIME / MAKEDATE / FROM_UNIXTIME need DOUBLE inputs
+        // and PPL-specific NULL-on-negative / year-wraparound behavior; DATE_FORMAT
+        // / TIME_FORMAT / STR_TO_DATE translate MySQL format tokens (%i / %s / %p …)
+        // that DataFusion's `to_char` does not recognize.
+        ScalarFunction.EXTRACT,
+        ScalarFunction.FROM_UNIXTIME,
+        ScalarFunction.MAKETIME,
+        ScalarFunction.MAKEDATE,
+        ScalarFunction.DATE_FORMAT,
+        ScalarFunction.TIME_FORMAT,
+        ScalarFunction.STR_TO_DATE,
         ScalarFunction.ASCII,
         ScalarFunction.CONCAT_WS,
         ScalarFunction.LEFT,
@@ -366,6 +393,8 @@ public Map scalarFunctionAdapters() {
                 DateTimeAdapters.NowAdapter now = new DateTimeAdapters.NowAdapter();
                 DateTimeAdapters.CurrentDateAdapter currentDate = new DateTimeAdapters.CurrentDateAdapter();
                 DateTimeAdapters.CurrentTimeAdapter currentTime = new DateTimeAdapters.CurrentTimeAdapter();
+                DayOfWeekAdapter dayOfWeek = new DayOfWeekAdapter();
+                SecondAdapter second = new SecondAdapter();
                 return Map.ofEntries(
                     Map.entry(ScalarFunction.ARRAY, new MakeArrayAdapter()),
                     Map.entry(ScalarFunction.ARRAY_JOIN, new ArrayToStringAdapter()),
@@ -382,13 +411,20 @@ public Map scalarFunctionAdapters() {
                     Map.entry(ScalarFunction.CURRENT_TIME, currentTime),
                     Map.entry(ScalarFunction.CURRENT_TIMESTAMP, now),
                     Map.entry(ScalarFunction.CURTIME, currentTime),
+                    Map.entry(ScalarFunction.DATE, new DateTimeAdapters.DateAdapter()),
+                    Map.entry(ScalarFunction.DATETIME, new DateTimeAdapters.DatetimeAdapter()),
+                    Map.entry(ScalarFunction.DATE_FORMAT, new RustUdfDateTimeAdapters.DateFormatAdapter()),
                     Map.entry(ScalarFunction.DAY, day),
                     Map.entry(ScalarFunction.DAYOFMONTH, day),
+                    Map.entry(ScalarFunction.DAYOFWEEK, dayOfWeek),
                     Map.entry(ScalarFunction.DAYOFYEAR, dayOfYear),
+                    Map.entry(ScalarFunction.DAY_OF_WEEK, dayOfWeek),
                     Map.entry(ScalarFunction.DAY_OF_YEAR, dayOfYear),
                     Map.entry(ScalarFunction.DIVIDE, new StdOperatorRewriteAdapter("DIVIDE", SqlStdOperatorTable.DIVIDE)),
                     Map.entry(ScalarFunction.E, new EConstantAdapter()),
                     Map.entry(ScalarFunction.EXPM1, new Expm1Adapter()),
+                    Map.entry(ScalarFunction.EXTRACT, new RustUdfDateTimeAdapters.ExtractAdapter()),
+                    Map.entry(ScalarFunction.FROM_UNIXTIME, new RustUdfDateTimeAdapters.FromUnixtimeAdapter()),
                     Map.entry(ScalarFunction.HOUR, hour),
                     Map.entry(ScalarFunction.HOUR_OF_DAY, hour),
                     Map.entry(ScalarFunction.JSON_APPEND, new JsonFunctionAdapters.JsonAppendAdapter()),
@@ -400,6 +436,8 @@ public Map scalarFunctionAdapters() {
                     Map.entry(ScalarFunction.JSON_SET, new JsonFunctionAdapters.JsonSetAdapter()),
                     Map.entry(ScalarFunction.LIKE, new LikeAdapter()),
                     Map.entry(ScalarFunction.LOCATE, new PositionAdapter()),
+                    Map.entry(ScalarFunction.MAKEDATE, new RustUdfDateTimeAdapters.MakedateAdapter()),
+                    Map.entry(ScalarFunction.MAKETIME, new RustUdfDateTimeAdapters.MaketimeAdapter()),
                     Map.entry(ScalarFunction.MICROSECOND, DatePartAdapters.microsecond()),
                     Map.entry(ScalarFunction.MINUTE, minute),
                     Map.entry(ScalarFunction.MINUTE_OF_HOUR, minute),
@@ -414,11 +452,18 @@ public Map scalarFunctionAdapters() {
                     Map.entry(ScalarFunction.SARG_PREDICATE, new SargAdapter()),
                     Map.entry(ScalarFunction.SCALAR_MAX, nameMapping(SqlLibraryOperators.GREATEST)),
                     Map.entry(ScalarFunction.SCALAR_MIN, nameMapping(SqlLibraryOperators.LEAST)),
+                    Map.entry(ScalarFunction.SECOND, second),
+                    Map.entry(ScalarFunction.SECOND_OF_MINUTE, second),
                     Map.entry(ScalarFunction.SIGN, nameMapping(SignumFunction.FUNCTION)),
                     Map.entry(ScalarFunction.SINH, new HyperbolicOperatorAdapter(SqlLibraryOperators.SINH)),
                     Map.entry(ScalarFunction.STRCMP, new StrcmpFunctionAdapter()),
+                    Map.entry(ScalarFunction.STRFTIME, new StrftimeFunctionAdapter()),
+                    Map.entry(ScalarFunction.STR_TO_DATE, new RustUdfDateTimeAdapters.StrToDateAdapter()),
                     Map.entry(ScalarFunction.SUBSTR, nameMapping(SqlStdOperatorTable.SUBSTRING)),
                     Map.entry(ScalarFunction.SUBSTRING, nameMapping(SqlStdOperatorTable.SUBSTRING)),
+                    Map.entry(ScalarFunction.SYSDATE, now),
+                    Map.entry(ScalarFunction.TIME, new DateTimeAdapters.TimeAdapter()),
+                    Map.entry(ScalarFunction.TIME_FORMAT, new RustUdfDateTimeAdapters.TimeFormatAdapter()),
                     Map.entry(ScalarFunction.TIMESTAMP, new TimestampFunctionAdapter()),
                     Map.entry(ScalarFunction.TONUMBER, new ToNumberFunctionAdapter()),
                     Map.entry(ScalarFunction.TOSTRING, new ToStringFunctionAdapter()),
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
index e87ebbc8eb550..1432cf3a93a42 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java
@@ -130,12 +130,27 @@ public class DataFusionFragmentConvertor implements FragmentConvertor {
         FunctionMappings.s(DateTimeAdapters.LOCAL_NOW_OP, "now"),
         FunctionMappings.s(DateTimeAdapters.LOCAL_CURRENT_DATE_OP, "current_date"),
         FunctionMappings.s(DateTimeAdapters.LOCAL_CURRENT_TIME_OP, "current_time"),
+        // PPL time(expr) → DF builtin to_time (TimeAdapter renames only).
+        FunctionMappings.s(DateTimeAdapters.LOCAL_TIME_OP, "to_time"),
+        // PPL date(expr) → DF builtin to_date (DateAdapter renames only).
+        FunctionMappings.s(DateTimeAdapters.LOCAL_DATE_OP, "to_date"),
+        // PPL datetime(expr) → DF builtin to_timestamp (DatetimeAdapter renames only).
+        FunctionMappings.s(DateTimeAdapters.LOCAL_TO_TIMESTAMP_OP, "to_timestamp"),
+        // PPL datetime + format functions → Rust UDFs registered in rust/src/udf/mod.rs.
+        FunctionMappings.s(RustUdfDateTimeAdapters.LOCAL_EXTRACT_OP, "extract"),
+        FunctionMappings.s(RustUdfDateTimeAdapters.LOCAL_FROM_UNIXTIME_OP, "from_unixtime"),
+        FunctionMappings.s(RustUdfDateTimeAdapters.LOCAL_MAKEDATE_OP, "makedate"),
+        FunctionMappings.s(RustUdfDateTimeAdapters.LOCAL_MAKETIME_OP, "maketime"),
+        FunctionMappings.s(RustUdfDateTimeAdapters.LOCAL_DATE_FORMAT_OP, "date_format"),
+        FunctionMappings.s(RustUdfDateTimeAdapters.LOCAL_TIME_FORMAT_OP, "time_format"),
+        FunctionMappings.s(RustUdfDateTimeAdapters.LOCAL_STR_TO_DATE_OP, "str_to_date"),
         FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"),
         FunctionMappings.s(SqlStdOperatorTable.REPLACE, "replace"),
         FunctionMappings.s(SqlLibraryOperators.REGEXP_REPLACE_3, "regexp_replace"),
         FunctionMappings.s(SqlLibraryOperators.REGEXP_CONTAINS, "regex_match"),
         FunctionMappings.s(SqlLibraryOperators.REVERSE, "reverse"),
         FunctionMappings.s(PositionAdapter.STRPOS, "strpos"),
+        FunctionMappings.s(StrftimeFunctionAdapter.STRFTIME, "strftime"),
         FunctionMappings.s(ToNumberFunctionAdapter.TONUMBER, "tonumber"),
         FunctionMappings.s(ToStringFunctionAdapter.TOSTRING, "tostring"),
         FunctionMappings.s(SqlStdOperatorTable.TRUNCATE, "trunc"),
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DateTimeAdapters.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DateTimeAdapters.java
index f82503c54ad49..6b772c91f51f5 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DateTimeAdapters.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DateTimeAdapters.java
@@ -19,10 +19,8 @@
 import java.util.List;
 
 /**
- * Adapters for PPL datetime functions that map 1:1 to a DataFusion builtin. Each
- * concrete subclass pairs a PPL function name with its locally-declared Calcite
- * {@link SqlOperator} whose {@code FunctionMappings.Sig} is registered in
- * {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS}.
+ * Adapters for PPL datetime functions that map 1:1 to a DataFusion builtin; signatures
+ * registered in {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS}.
  *
  * @opensearch.internal
  */
@@ -57,6 +55,35 @@ private DateTimeAdapters() {}
         SqlFunctionCategory.TIMEDATE
     );
 
+    static final SqlOperator LOCAL_TIME_OP = new SqlFunction(
+        "to_time",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.TIME_NULLABLE,
+        null,
+        OperandTypes.ANY,
+        SqlFunctionCategory.TIMEDATE
+    );
+
+    static final SqlOperator LOCAL_DATE_OP = new SqlFunction(
+        "to_date",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.DATE_NULLABLE,
+        null,
+        OperandTypes.ANY,
+        SqlFunctionCategory.TIMEDATE
+    );
+
+    // 1-arg timestamp(expr) remains on the legacy engine — the TIMESTAMP enum slot is already
+    // bound to TimestampFunctionAdapter for VARCHAR-literal folding.
+    static final SqlOperator LOCAL_TO_TIMESTAMP_OP = new SqlFunction(
+        "to_timestamp",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.TIMESTAMP,
+        null,
+        OperandTypes.ANY,
+        SqlFunctionCategory.TIMEDATE
+    );
+
     static final class NowAdapter extends AbstractNameMappingAdapter {
         NowAdapter() {
             super(LOCAL_NOW_OP, List.of(), List.of());
@@ -74,4 +101,22 @@ static final class CurrentTimeAdapter extends AbstractNameMappingAdapter {
             super(LOCAL_CURRENT_TIME_OP, List.of(), List.of());
         }
     }
+
+    static final class TimeAdapter extends AbstractNameMappingAdapter {
+        TimeAdapter() {
+            super(LOCAL_TIME_OP, List.of(), List.of());
+        }
+    }
+
+    static final class DateAdapter extends AbstractNameMappingAdapter {
+        DateAdapter() {
+            super(LOCAL_DATE_OP, List.of(), List.of());
+        }
+    }
+
+    static final class DatetimeAdapter extends AbstractNameMappingAdapter {
+        DatetimeAdapter() {
+            super(LOCAL_TO_TIMESTAMP_OP, List.of(), List.of());
+        }
+    }
 }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DayOfWeekAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DayOfWeekAdapter.java
new file mode 100644
index 0000000000000..41ac5599419c4
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DayOfWeekAdapter.java
@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.fun.SqlLibraryOperators;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+/**
+ * PPL {@code dayofweek}/{@code day_of_week} → {@code CAST(date_part('dow', x) + 1 AS )}:
+ * MySQL/PPL uses 1=Sun..7=Sat but DataFusion/Postgres {@code date_part('dow')} returns 0..6, so we
+ * add 1 and cast back to the original call's return type.
+ *
+ * @opensearch.internal
+ */
+class DayOfWeekAdapter implements ScalarFunctionAdapter {
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        if (original.getOperands().size() != 1) {
+            return original;
+        }
+        RexBuilder rexBuilder = cluster.getRexBuilder();
+        RelDataType varchar = cluster.getTypeFactory().createSqlType(SqlTypeName.VARCHAR);
+        RexNode partLiteral = rexBuilder.makeLiteral("dow", varchar, true);
+        RexNode datePart = rexBuilder.makeCall(SqlLibraryOperators.DATE_PART, partLiteral, original.getOperands().get(0));
+        RexNode sum = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, datePart, rexBuilder.makeExactLiteral(BigDecimal.ONE));
+        return rexBuilder.makeCast(original.getType(), sum);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/RustUdfDateTimeAdapters.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/RustUdfDateTimeAdapters.java
new file mode 100644
index 0000000000000..127ff49e29c8e
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/RustUdfDateTimeAdapters.java
@@ -0,0 +1,143 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlOperandTypeChecker;
+import org.apache.calcite.sql.type.SqlReturnTypeInference;
+import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.analytics.spi.AbstractNameMappingAdapter;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Adapters for PPL datetime functions routed to Rust UDFs. Each {@code LOCAL_*_OP}
+ * names a Calcite {@link SqlFunction} matching a UDF in {@code rust/src/udf/mod.rs};
+ * Substrait sigs live in {@code opensearch_scalar_functions.yaml} +
+ * {@link DataFusionFragmentConvertor#ADDITIONAL_SCALAR_SIGS}.
+ *
+ * @opensearch.internal
+ */
+final class RustUdfDateTimeAdapters {
+
+    private RustUdfDateTimeAdapters() {}
+
+    private static SqlOperator udf(String name, SqlReturnTypeInference ret, SqlOperandTypeChecker operands) {
+        return new SqlFunction(name, SqlKind.OTHER_FUNCTION, ret, null, operands, SqlFunctionCategory.TIMEDATE);
+    }
+
+    static final SqlOperator LOCAL_EXTRACT_OP = udf("extract", ReturnTypes.BIGINT_NULLABLE, OperandTypes.ANY_ANY);
+    static final SqlOperator LOCAL_FROM_UNIXTIME_OP = udf("from_unixtime", ReturnTypes.TIMESTAMP_NULLABLE, OperandTypes.ANY);
+    static final SqlOperator LOCAL_MAKETIME_OP = udf(
+        "maketime",
+        ReturnTypes.TIME_NULLABLE,
+        OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.ANY, SqlTypeFamily.ANY)
+    );
+    static final SqlOperator LOCAL_MAKEDATE_OP = udf("makedate", ReturnTypes.DATE_NULLABLE, OperandTypes.ANY_ANY);
+    static final SqlOperator LOCAL_DATE_FORMAT_OP = udf("date_format", ReturnTypes.VARCHAR_NULLABLE, OperandTypes.ANY_ANY);
+    static final SqlOperator LOCAL_TIME_FORMAT_OP = udf("time_format", ReturnTypes.VARCHAR_NULLABLE, OperandTypes.ANY_ANY);
+    static final SqlOperator LOCAL_STR_TO_DATE_OP = udf("str_to_date", ReturnTypes.TIMESTAMP_NULLABLE, OperandTypes.ANY_ANY);
+
+    static final class ExtractAdapter extends AbstractNameMappingAdapter {
+        ExtractAdapter() {
+            super(LOCAL_EXTRACT_OP, List.of(), List.of());
+        }
+    }
+
+    static final class DateFormatAdapter extends AbstractNameMappingAdapter {
+        DateFormatAdapter() {
+            super(LOCAL_DATE_FORMAT_OP, List.of(), List.of());
+        }
+    }
+
+    static final class TimeFormatAdapter extends AbstractNameMappingAdapter {
+        TimeFormatAdapter() {
+            super(LOCAL_TIME_FORMAT_OP, List.of(), List.of());
+        }
+    }
+
+    static final class StrToDateAdapter extends AbstractNameMappingAdapter {
+        StrToDateAdapter() {
+            super(LOCAL_STR_TO_DATE_OP, List.of(), List.of());
+        }
+    }
+
+    /**
+     * Casts numeric operands to DOUBLE before rewriting: the YAML declares one
+     * fp64-only impl per function, so PPL integer literals (e.g. {@code makedate(2020, 1)})
+     * must be widened before the substrait converter binds them to a signature.
+     */
+    private abstract static class NumericToDoubleAdapter implements ScalarFunctionAdapter {
+        private final SqlOperator target;
+
+        NumericToDoubleAdapter(SqlOperator target) {
+            this.target = target;
+        }
+
+        @Override
+        public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+            List rewritten = new ArrayList<>(original.getOperands().size());
+            for (RexNode operand : original.getOperands()) {
+                rewritten.add(widenToDoubleIfNumeric(operand, cluster));
+            }
+            return cluster.getRexBuilder().makeCall(original.getType(), target, rewritten);
+        }
+
+        private static RexNode widenToDoubleIfNumeric(RexNode operand, RelOptCluster cluster) {
+            SqlTypeName type = operand.getType().getSqlTypeName();
+            if (type == SqlTypeName.DOUBLE) {
+                return operand;
+            }
+            if (SqlTypeName.INT_TYPES.contains(type)
+                || type == SqlTypeName.FLOAT
+                || type == SqlTypeName.REAL
+                || type == SqlTypeName.DECIMAL) {
+                RelDataTypeFactory factory = cluster.getTypeFactory();
+                RelDataType doubleType = factory.createTypeWithNullability(
+                    factory.createSqlType(SqlTypeName.DOUBLE),
+                    operand.getType().isNullable()
+                );
+                return cluster.getRexBuilder().makeCast(doubleType, operand);
+            }
+            return operand;
+        }
+    }
+
+    static final class FromUnixtimeAdapter extends NumericToDoubleAdapter {
+        FromUnixtimeAdapter() {
+            super(LOCAL_FROM_UNIXTIME_OP);
+        }
+    }
+
+    static final class MaketimeAdapter extends NumericToDoubleAdapter {
+        MaketimeAdapter() {
+            super(LOCAL_MAKETIME_OP);
+        }
+    }
+
+    static final class MakedateAdapter extends NumericToDoubleAdapter {
+        MakedateAdapter() {
+            super(LOCAL_MAKEDATE_OP);
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SecondAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SecondAdapter.java
new file mode 100644
index 0000000000000..9e5d93ce6118c
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/SecondAdapter.java
@@ -0,0 +1,49 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.fun.SqlLibraryOperators;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.util.List;
+
+/**
+ * PPL {@code second}/{@code second_of_minute} → {@code CAST(FLOOR(date_part('second', x)) AS ret)}.
+ * FLOOR drops {@code date_part}'s fp64 fractional part (integer portion already in [0, 59]); the
+ * intermediate CAST to DOUBLE is needed because our substrait YAML declares date_part/floor as
+ * fp64-only while Calcite's inference returns BIGINT for {@code part='second'}.
+ *
+ * @opensearch.internal
+ */
+class SecondAdapter implements ScalarFunctionAdapter {
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        if (original.getOperands().size() != 1) {
+            return original;
+        }
+        RexBuilder rexBuilder = cluster.getRexBuilder();
+        RelDataType varchar = cluster.getTypeFactory().createSqlType(SqlTypeName.VARCHAR);
+        RexNode partLiteral = rexBuilder.makeLiteral("second", varchar, true);
+        RexNode datePart = rexBuilder.makeCall(SqlLibraryOperators.DATE_PART, partLiteral, original.getOperands().get(0));
+        RelDataType doubleType = cluster.getTypeFactory()
+            .createTypeWithNullability(cluster.getTypeFactory().createSqlType(SqlTypeName.DOUBLE), datePart.getType().isNullable());
+        RexNode datePartDouble = rexBuilder.makeCast(doubleType, datePart);
+        RexNode floored = rexBuilder.makeCall(SqlStdOperatorTable.FLOOR, datePartDouble);
+        return rexBuilder.makeCast(original.getType(), floored);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/StrftimeFunctionAdapter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/StrftimeFunctionAdapter.java
new file mode 100644
index 0000000000000..9417159c5ba6d
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/StrftimeFunctionAdapter.java
@@ -0,0 +1,87 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.analytics.spi.FieldStorageInfo;
+import org.opensearch.analytics.spi.ScalarFunctionAdapter;
+
+import java.util.List;
+
+/**
+ * Rewrites PPL {@code strftime(value, format)} → local {@link #STRFTIME}, which Isthmus
+ * serializes under Substrait extension name {@code "strftime"} (matched by the Rust UDF
+ * in {@code rust/src/udf/strftime.rs}). Folds every numeric source onto a single Float64
+ * signature via {@code CAST AS DOUBLE}; timestamp/date inputs forward verbatim (the Rust
+ * UDF's {@code coerce_types} canonicalizes them to {@code Timestamp(Microsecond, None)}).
+ *
+ * @opensearch.internal
+ */
+class StrftimeFunctionAdapter implements ScalarFunctionAdapter {
+
+    static final SqlFunction STRFTIME = new SqlFunction(
+        "strftime",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.VARCHAR,
+        null,
+        OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.CHARACTER),
+        SqlFunctionCategory.USER_DEFINED_FUNCTION
+    );
+
+    @Override
+    public RexNode adapt(RexCall original, List fieldStorage, RelOptCluster cluster) {
+        List operands = original.getOperands();
+        if (operands.size() != 2) {
+            return original;
+        }
+        RexNode value = operands.get(0);
+        RexNode format = operands.get(1);
+        SqlTypeName valueType = value.getType().getSqlTypeName();
+
+        // Fold every numeric (and string — IT covers the Calcite auto-coerce path) source onto
+        // a single Float64 signature; DOUBLE preserves fractional-seconds precision. Timestamp /
+        // date / time inputs forward verbatim — the Rust coerce_types canonicalizes them.
+        RexNode normalizedValue;
+        if (isIntegralNumeric(valueType)
+            || valueType == SqlTypeName.FLOAT
+            || valueType == SqlTypeName.REAL
+            || valueType == SqlTypeName.DECIMAL
+            || SqlTypeName.CHAR_TYPES.contains(valueType)) {
+            normalizedValue = castTo(value, SqlTypeName.DOUBLE, cluster);
+        } else {
+            normalizedValue = value;
+        }
+
+        return cluster.getRexBuilder().makeCall(original.getType(), STRFTIME, List.of(normalizedValue, format));
+    }
+
+    private static boolean isIntegralNumeric(SqlTypeName type) {
+        return type == SqlTypeName.TINYINT || type == SqlTypeName.SMALLINT || type == SqlTypeName.INTEGER || type == SqlTypeName.BIGINT;
+    }
+
+    private static RexNode castTo(RexNode operand, SqlTypeName target, RelOptCluster cluster) {
+        if (operand.getType().getSqlTypeName() == target) {
+            return operand;
+        }
+        RelDataTypeFactory factory = cluster.getTypeFactory();
+        RelDataType targetType = factory.createTypeWithNullability(factory.createSqlType(target), operand.getType().isNullable());
+        return cluster.getRexBuilder().makeCast(targetType, operand);
+    }
+}
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
index d5dd9f2f213a6..4139b65eafc0c 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_scalar_functions.yaml
@@ -86,6 +86,60 @@ scalar_functions:
     impls:
       - args: []
         return: time
+  - name: "to_time"
+    description: >-
+      Extract the time-of-day component from a TIMESTAMP, DATE, TIME, or
+      string value. DataFusion's builtin `to_time`
+      (datafusion-functions/src/datetime/to_time.rs) — returns `Time64(ns)`
+      for Timestamp / Utf8 / Utf8View / LargeUtf8 inputs. Lowering target for
+      PPL `time(expr)` via TimeAdapter in DateTimeAdapters.
+    impls:
+      - args: [{ value: "precision_timestamp
            ", name: "value" }]
+        return: time
+      - args: [{ value: "date", name: "value" }]
+        return: time
+      - args: [{ value: "time", name: "value" }]
+        return: time
+      - args: [{ value: "string", name: "value" }]
+        return: time
+      - args: [{ value: "varchar", name: "value" }]
+        return: time
+  - name: "to_date"
+    description: >-
+      Extract the date component from a TIMESTAMP, DATE, or string value.
+      DataFusion's builtin `to_date`
+      (datafusion-functions/src/datetime/to_date.rs) — returns `Date32` for
+      Timestamp / Utf8 / Utf8View / LargeUtf8 inputs. Lowering target for
+      PPL `date(expr)` via DateAdapter in DateTimeAdapters. PPL's operand
+      checker rejects integer inputs upstream, so DF's days-since-epoch
+      branch is unreachable on this path.
+    impls:
+      - args: [{ value: "precision_timestamp
            ", name: "value" }]
+        return: date
+      - args: [{ value: "date", name: "value" }]
+        return: date
+      - args: [{ value: "string", name: "value" }]
+        return: date
+      - args: [{ value: "varchar", name: "value" }]
+        return: date
+  - name: "to_timestamp"
+    description: >-
+      Parse a value into a TIMESTAMP. DataFusion's builtin `to_timestamp`
+      (datafusion-functions/src/datetime/to_timestamp.rs) — returns
+      `Timestamp(Nanosecond, None)` for Timestamp / Utf8 / Utf8View /
+      LargeUtf8 inputs. Lowering target for PPL `datetime(expr)` via
+      DatetimeAdapter. PPL's single-arg `timestamp(expr)` shares semantics
+      but stays on legacy due to an enum-slot collision with the VARCHAR
+      literal-folding adapter — see DatetimeAdapter javadoc.
+    impls:
+      - args: [{ value: "precision_timestamp
            ", name: "value" }]
+        return: timestamp
+      - args: [{ value: "date", name: "value" }]
+        return: timestamp
+      - args: [{ value: "string", name: "value" }]
+        return: timestamp
+      - args: [{ value: "varchar", name: "value" }]
+        return: timestamp
   - name: regex_match
     description: >-
       Regular expression containment match (boolean). Lowering target for PPL's
@@ -308,6 +362,25 @@ scalar_functions:
         nullability: DECLARED_OUTPUT
         return: string
 
+  # strftime(value, format) — render a timestamp / UNIX-seconds value as a formatted string.
+  # Two impls mirror StrftimeFunctionAdapter's Rex-level normalization: numeric-like sources
+  # are folded onto fp64 (UNIX-seconds branch, with millisecond auto-detect in the UDF);
+  # timestamp / date sources forward verbatim and the Rust UDF's coerce_types canonicalizes
+  # `Timestamp(*, *)` / `Date32` / `Date64` to `Timestamp(Microsecond, None)`.
+  - name: "strftime"
+    description: "Render a timestamp or UNIX seconds value using a POSIX strftime format string."
+    impls:
+      - args:
+          - { name: value, value: fp64 }
+          - { name: format, value: string }
+        nullability: DECLARED_OUTPUT
+        return: string
+      - args:
+          - { name: value, value: "precision_timestamp
            " }
+          - { name: format, value: string }
+        nullability: DECLARED_OUTPUT
+        return: string
+
   # tonumber(string, base) — parse `string` as a base-N integer
   - name: "tonumber"
     description: "Parse a string to a number in the given radix (2-36). Returns NULL on parse failure."
@@ -367,3 +440,134 @@ scalar_functions:
       - args: [{ value: string, name: "value" }, { value: string, name: "path" }]
         variadic: { min: 1 }
         return: string
+
+  - name: "extract"
+    description: >-
+      Pull a MySQL-style calendar component (simple or composite) out of a
+      timestamp. The unit slot is a VARCHAR literal injected by the adapter
+      (matchKey token `str`); the timestamp slot is PPL's canonical
+      `precision_timestamp
            ` / `date`. Returns BIGINT regardless of unit —
+      composite units (e.g. `DAY_SECOND`) follow MySQL's digit-concatenation
+      semantics (see rust/src/udf/extract.rs). Routes to the Rust `extract`
+      UDF, not Calcite's EXTRACT operator.
+    impls:
+      - args:
+          - { value: string, name: "unit" }
+          - { value: "precision_timestamp
            ", name: "value" }
+        return: i64
+      - args:
+          - { value: string, name: "unit" }
+          - { value: "date", name: "value" }
+        return: i64
+      - args:
+          - { value: string, name: "unit" }
+          - { value: "time", name: "value" }
+        return: i64
+      - args:
+          - { value: string, name: "unit" }
+          - { value: "string", name: "value" }
+        return: i64
+      - args:
+          - { value: string, name: "unit" }
+          - { value: "varchar", name: "value" }
+        return: i64
+
+  - name: "from_unixtime"
+    description: >-
+      Convert fractional UNIX seconds to TIMESTAMP. Negative values and values
+      at/above MySQL's documented max yield NULL. Routes to the Rust
+      `from_unixtime` UDF (rust/src/udf/from_unixtime.rs). The 2-arg
+      `from_unixtime(seconds, format)` overload is deferred.
+    impls:
+      - args: [{ value: fp64, name: "seconds" }]
+        return: precision_timestamp<6>
+
+  - name: "maketime"
+    description: >-
+      Construct a TIME from (hour, minute, second). Hour and minute are rounded
+      (half-away-from-zero, matching Java Math.round); second passes through
+      verbatim including fractional component. Out-of-range operand yields
+      NULL. Returns `Time64(Microsecond)`. Routes to the Rust `maketime` UDF.
+    impls:
+      - args:
+          - { value: fp64, name: "hour" }
+          - { value: fp64, name: "minute" }
+          - { value: fp64, name: "second" }
+        return: time
+
+  - name: "makedate"
+    description: >-
+      Construct a DATE from (year, day_of_year). PPL MySQL quirks preserved:
+      `doy <= 0` or `year < 0` → NULL; `year == 0` remaps to 2000; doy beyond
+      the year's length cascades into subsequent years. Routes to the Rust
+      `makedate` UDF.
+    impls:
+      - args:
+          - { value: fp64, name: "year" }
+          - { value: fp64, name: "day_of_year" }
+        return: date
+
+  - name: "date_format"
+    description: >-
+      Render a DATE / TIMESTAMP using a MySQL format string. Shares the MySQL
+      token translator with `time_format` / `str_to_date`. Routes to the Rust
+      `date_format` UDF (rust/src/udf/date_format.rs).
+    impls:
+      - args:
+          - { value: "precision_timestamp
            ", name: "value" }
+          - { value: string, name: "format" }
+        return: string
+      - args:
+          - { value: "date", name: "value" }
+          - { value: string, name: "format" }
+        return: string
+      - args:
+          - { value: "string", name: "value" }
+          - { value: string, name: "format" }
+        return: string
+      - args:
+          - { value: "varchar", name: "value" }
+          - { value: "varchar", name: "format" }
+        return: string
+
+  - name: "time_format"
+    description: >-
+      Render a TIME / TIMESTAMP using the MySQL time-format sub-table. Date-only
+      name tokens (%W / %a / %M / %b / %D / %j / %w / %U / %u / %V / %v / %X /
+      %x) cause the whole render to return NULL; date-only numeric tokens emit
+      MySQL's documented zero-padded literals. Routes to the Rust `time_format`
+      UDF.
+    impls:
+      - args:
+          - { value: "precision_timestamp
            ", name: "value" }
+          - { value: string, name: "format" }
+        return: string
+      - args:
+          - { value: "time", name: "value" }
+          - { value: string, name: "format" }
+        return: string
+      - args:
+          - { value: "date", name: "value" }
+          - { value: string, name: "format" }
+        return: string
+      - args:
+          - { value: "varchar", name: "value" }
+          - { value: "varchar", name: "format" }
+        return: string
+
+  - name: "str_to_date"
+    description: >-
+      Parse a string using a MySQL format into a TIMESTAMP. Unparseable input
+      yields NULL; trailing input is silently tolerated (matches PPL's
+      `parseUnresolved` with `ParsePosition(0)`). Missing date parts default to
+      2000-01-01; missing time parts default to 00:00:00. Routes to the Rust
+      `str_to_date` UDF.
+    impls:
+      - args:
+          - { value: string, name: "input" }
+          - { value: string, name: "format" }
+        return: precision_timestamp<6>
+      - args:
+          - { value: "varchar", name: "input" }
+          - { value: "varchar", name: "format" }
+        return: precision_timestamp<6>
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/StrftimeFunctionAdapterTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/StrftimeFunctionAdapterTests.java
new file mode 100644
index 0000000000000..e5dc03fa0732a
--- /dev/null
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/StrftimeFunctionAdapterTests.java
@@ -0,0 +1,134 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.datafusion;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+public class StrftimeFunctionAdapterTests extends OpenSearchTestCase {
+
+    private final StrftimeFunctionAdapter adapter = new StrftimeFunctionAdapter();
+
+    private static final SqlFunction STRFTIME = new SqlFunction(
+        "strftime",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.VARCHAR,
+        null,
+        OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.CHARACTER),
+        SqlFunctionCategory.USER_DEFINED_FUNCTION
+    );
+
+    /** Every numeric-or-string input slot lowers to CAST-to-DOUBLE so the Rust UDF sees one signature. */
+    public void testNumericAndStringInputsWidenToDouble() {
+        Cluster c = newCluster();
+        RexNode[] sources = new RexNode[] {
+            c.intLiteral(1521467703),
+            c.rexBuilder.makeExactLiteral(BigDecimal.valueOf(1521467703L), c.typeFactory.createSqlType(SqlTypeName.BIGINT)),
+            c.rexBuilder.makeExactLiteral(BigDecimal.valueOf(1521467703.123456), c.typeFactory.createSqlType(SqlTypeName.DECIMAL, 20, 6)),
+            c.stringLiteral("1521467703"), };
+        for (RexNode src : sources) {
+            RexCall call = (RexCall) c.rexBuilder.makeCall(STRFTIME, src, c.stringLiteral("%Y-%m-%d"));
+            RexCall out = assertStrftimeCall(adapter.adapt(call, List.of(), c.cluster));
+            assertEquals(
+                "source widened to DOUBLE: " + src.getType().getSqlTypeName(),
+                SqlTypeName.DOUBLE,
+                out.getOperands().get(0).getType().getSqlTypeName()
+            );
+        }
+    }
+
+    /** DOUBLE/TIMESTAMP/DATE operands forward by identity (Rust coerce_types canonicalizes). */
+    public void testDoubleTimestampDateForwardByIdentity() {
+        Cluster c = newCluster();
+        RexNode dbl = c.rexBuilder.makeApproxLiteral(
+            BigDecimal.valueOf(1521467703.123456),
+            c.typeFactory.createSqlType(SqlTypeName.DOUBLE)
+        );
+        RexNode ts = c.rexBuilder.makeInputRef(c.typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 6), 0);
+        RexNode dt = c.rexBuilder.makeInputRef(c.typeFactory.createSqlType(SqlTypeName.DATE), 0);
+        for (RexNode src : new RexNode[] { dbl, ts, dt }) {
+            RexCall call = (RexCall) c.rexBuilder.makeCall(STRFTIME, src, c.stringLiteral("%Y-%m-%d"));
+            RexCall out = assertStrftimeCall(adapter.adapt(call, List.of(), c.cluster));
+            assertSame("operand forwarded by identity: " + src.getType().getSqlTypeName(), src, out.getOperands().get(0));
+        }
+    }
+
+    public void testFormatOperandForwardedVerbatim() {
+        Cluster c = newCluster();
+        RexNode format = c.stringLiteral("%a, %b %d, %Y %I:%M:%S %p %Z");
+        RexCall call = (RexCall) c.rexBuilder.makeCall(STRFTIME, c.intLiteral(1521467703), format);
+        RexCall out = assertStrftimeCall(adapter.adapt(call, List.of(), c.cluster));
+        assertSame("format literal forwarded by identity", format, out.getOperands().get(1));
+    }
+
+    public void testWrongArityPassesThrough() {
+        Cluster c = newCluster();
+        RexCall call = (RexCall) c.rexBuilder.makeCall(STRFTIME, c.intLiteral(1521467703));
+        RexNode out = adapter.adapt(call, List.of(), c.cluster);
+        assertSame("single-arg call left unchanged — downstream planning should fail loudly", call, out);
+    }
+
+    private static RexCall assertStrftimeCall(RexNode out) {
+        assertTrue("expected a RexCall, got " + out.getClass(), out instanceof RexCall);
+        RexCall outCall = (RexCall) out;
+        assertSame(
+            "operator is the synthetic `strftime` that resolves to the Rust UDF",
+            StrftimeFunctionAdapter.STRFTIME,
+            outCall.getOperator()
+        );
+        assertEquals("two operands — value + format", 2, outCall.getOperands().size());
+        return outCall;
+    }
+
+    private static Cluster newCluster() {
+        RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl();
+        RexBuilder rexBuilder = new RexBuilder(typeFactory);
+        HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+        return new Cluster(RelOptCluster.create(planner, rexBuilder), typeFactory, rexBuilder);
+    }
+
+    private static final class Cluster {
+        final RelOptCluster cluster;
+        final RelDataTypeFactory typeFactory;
+        final RexBuilder rexBuilder;
+
+        Cluster(RelOptCluster cluster, RelDataTypeFactory typeFactory, RexBuilder rexBuilder) {
+            this.cluster = cluster;
+            this.typeFactory = typeFactory;
+            this.rexBuilder = rexBuilder;
+        }
+
+        RexNode intLiteral(int value) {
+            RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER);
+            return rexBuilder.makeExactLiteral(BigDecimal.valueOf(value), intType);
+        }
+
+        RexNode stringLiteral(String value) {
+            return rexBuilder.makeLiteral(value);
+        }
+    }
+}
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DateTimeScalarFunctionsIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DateTimeScalarFunctionsIT.java
new file mode 100644
index 0000000000000..dbcc03eb65f31
--- /dev/null
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/DateTimeScalarFunctionsIT.java
@@ -0,0 +1,207 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * E2E coverage for PPL datetime scalar functions (PPL → Substrait → DataFusion). Fixture:
+ * {@code calcs.key00} → {@code datetime0 = 2004-07-09T10:17:35Z}; literal-input cases use
+ * 1521467703 = 2018-03-19T13:55:03Z (matches SQL-plugin CalciteDateTimeFunctionIT).
+ */
+public class DateTimeScalarFunctionsIT extends AnalyticsRestTestCase {
+
+    private static final Dataset DATASET = new Dataset("calcs", "calcs");
+
+    private static boolean dataProvisioned = false;
+
+    private void ensureDataProvisioned() throws IOException {
+        if (dataProvisioned == false) {
+            DatasetProvisioner.provision(client(), DATASET);
+            dataProvisioned = true;
+        }
+    }
+
+    private String oneRow(String key) {
+        return "source=" + DATASET.indexName + " | where key='" + key + "' | head 1 ";
+    }
+
+    public void testStrftimeIntegerUnixSeconds() throws IOException {
+        assertFirstRowString(
+            oneRow("key00") + "| eval v = strftime(1521467703, '%Y-%m-%d %H:%M:%S') | fields v",
+            "2018-03-19 13:55:03"
+        );
+    }
+
+    public void testStrftimeComplexFormat() throws IOException {
+        assertFirstRowString(
+            oneRow("key00") + "| eval v = strftime(1521467703, '%a, %b %d, %Y %I:%M:%S %p %Z') | fields v",
+            "Mon, Mar 19, 2018 01:55:03 PM UTC"
+        );
+    }
+
+    public void testStrftimeFractionalSeconds() throws IOException {
+        assertFirstRowString(
+            oneRow("key00") + "| eval v = strftime(1521467703.123456, '%Y-%m-%d %H:%M:%S.%3Q') | fields v",
+            "2018-03-19 13:55:03.123"
+        );
+    }
+
+    // Exercises the Rust UDF's `abs(v) >= 1e11` ms-auto-detect branch.
+    public void testStrftimeMilliEpochAutoDetect() throws IOException {
+        assertFirstRowString(
+            oneRow("key00") + "| eval v = strftime(1521467703123, '%Y-%m-%d %H:%M:%S') | fields v",
+            "2018-03-19 13:55:03"
+        );
+    }
+
+    public void testStrftimeNegativeTimestamp() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = strftime(-1, '%Y-%m-%d %H:%M:%S') | fields v", "1969-12-31 23:59:59");
+    }
+
+    public void testStrftimeOnDateField() throws IOException {
+        assertFirstRowString(
+            oneRow("key00") + "| eval v = strftime(datetime0, '%Y-%m-%d %H:%M:%S') | fields v",
+            "2004-07-09 10:17:35"
+        );
+    }
+
+    // time(expr) component extraction and TIME-operand time_format overloads are
+    // blocked by substrait-java 0.89.1's missing `ToTypeString` override for
+    // `ParameterizedType.PrecisionTime`. Out of scope for Wave A; landing with
+    // the upstream fix.
+
+    public void testDateOnTimestampFieldYear() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = year(date(datetime0)) | fields v", 2004L);
+    }
+
+    public void testDateOnTimestampFieldMonth() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = month(date(datetime0)) | fields v", 7L);
+    }
+
+    public void testDateOnStringLiteralDay() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = day(date('2024-06-15')) | fields v", 15L);
+    }
+
+    public void testDayofweek() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = dayofweek(datetime0) | fields v", 6L);
+    }
+
+    public void testDayOfWeekAlias() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = day_of_week(datetime0) | fields v", 6L);
+    }
+
+    public void testSecond() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = second(datetime0) | fields v", 35L);
+    }
+
+    public void testSecondOfMinute() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = second_of_minute(datetime0) | fields v", 35L);
+    }
+
+    public void testDatetimeOnStringLiteral() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = hour(datetime('2004-07-09 10:17:35')) | fields v", 10L);
+    }
+
+    public void testSysdateNonNull() throws IOException {
+        Object cell = firstRowFirstCell(oneRow("key00") + "| eval v = date_format(sysdate(), '%Y') | fields v");
+        assertNotNull("sysdate() rendered to YYYY must be non-null", cell);
+        assertTrue("sysdate year must start with '20', got " + cell, cell.toString().startsWith("20"));
+    }
+
+    public void testExtractYear() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = extract(YEAR FROM datetime0) | fields v", 2004L);
+    }
+
+    public void testExtractHour() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = extract(HOUR FROM datetime0) | fields v", 10L);
+    }
+
+    public void testExtractDayHourComposite() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = extract(DAY_HOUR FROM datetime0) | fields v", 910L);
+    }
+
+    public void testFromUnixtime() throws IOException {
+        assertFirstRowString(
+            oneRow("key00") + "| eval v = date_format(from_unixtime(1521467703), '%Y-%m-%d %H:%i:%s') | fields v",
+            "2018-03-19 13:55:03"
+        );
+    }
+
+    // End-to-end maketime coverage is blocked by the same substrait-java 0.89.1
+    // ToTypeString gap as time(expr); Time64(Microsecond) return has no working
+    // signature slot. Rust-level tests in rust/src/udf/maketime.rs cover semantics.
+
+    public void testMakedate() throws IOException {
+        assertFirstRowLong(oneRow("key00") + "| eval v = year(makedate(2020, 1)) | fields v", 2020L);
+    }
+
+    public void testDateFormatBasic() throws IOException {
+        assertFirstRowString(
+            oneRow("key00") + "| eval v = date_format(datetime0, '%Y-%m-%d %H:%i:%s') | fields v",
+            "2004-07-09 10:17:35"
+        );
+    }
+
+    // %D ordinal day — proves shared mysql_format token table reachable via date_format.
+    public void testDateFormatOrdinalSuffix() throws IOException {
+        assertFirstRowString(oneRow("key00") + "| eval v = date_format(datetime0, '%D') | fields v", "9th");
+    }
+
+    public void testTimeFormatBasic() throws IOException {
+        assertFirstRowString(
+            oneRow("key00") + "| eval v = time_format(datetime0, '%H:%i:%s') | fields v",
+            "10:17:35"
+        );
+    }
+
+    public void testStrToDate() throws IOException {
+        assertFirstRowString(
+            oneRow("key00")
+                + "| eval v = date_format(str_to_date('09,07,2004', '%d,%m,%Y'), '%Y-%m-%d %H:%i:%s') | fields v",
+            "2004-07-09 00:00:00"
+        );
+    }
+
+
+
+    private void assertFirstRowString(String ppl, String expected) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertNotNull("Expected non-null result for query [" + ppl + "]", cell);
+        assertEquals("Value mismatch for query: " + ppl, expected, cell);
+    }
+
+    private void assertFirstRowLong(String ppl, long expected) throws IOException {
+        Object cell = firstRowFirstCell(ppl);
+        assertTrue("Expected numeric result for query [" + ppl + "] but got: " + cell, cell instanceof Number);
+        assertEquals("Value mismatch for query: " + ppl, expected, ((Number) cell).longValue());
+    }
+
+    private Object firstRowFirstCell(String ppl) throws IOException {
+        Map response = executePpl(ppl);
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) response.get("rows");
+        assertNotNull("Response missing 'rows' for query: " + ppl, rows);
+        assertTrue("Expected at least one row for query: " + ppl, rows.size() >= 1);
+        return rows.get(0).get(0);
+    }
+
+    private Map executePpl(String ppl) throws IOException {
+        ensureDataProvisioned();
+        Request request = new Request("POST", "/_analytics/ppl");
+        request.setJsonEntity("{\"query\": \"" + escapeJson(ppl) + "\"}");
+        Response response = client().performRequest(request);
+        return assertOkAndParse(response, "PPL: " + ppl);
+    }
+}

From 2ee886a83b93e4b339dd2f8982a1ff7ad291982b Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Mon, 11 May 2026 13:26:31 -0700
Subject: [PATCH 112/115] sandbox: enforce streaming-only fragment dispatch in
 analytics-engine (#21594)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* refactor: streaming-only fragment dispatch in analytics-engine

- delete RowResponseCodec, ResponseCodec, FragmentExecutionResponse, RowBatchToArrowConverter
- delete AnalyticsSearchService.executeFragment / collectResponse
- require StreamTransportService at injection (fail-fast); force streaming feature flag in sandbox QA clusters
- ShardFragmentStageExecution: catch outputSink.feed() exceptions so a feed failure fails the stage instead of hanging to QUERY_TIMEOUT
- DatafusionResultStream: synthesise one zero-row schema-bearing batch for empty native streams (Flight requires ≥1 schema frame)
- DatafusionPartitionSender: read-write lock around send/close to prevent sender_close UAF while sender_send is mid-await
- @AwaitsFix 7 Append/AppendPipe IT methods — Utf8View FFI schema-lie bug, tracked separately

Signed-off-by: Marc Handalian 

* fix: wire UAF lock through feedToSender; unmute Append ITs

- DatafusionReduceSink.feedToSender now calls sender.send() instead of
  NativeBridge.senderSend() directly, so the read-write lock added on
  DatafusionPartitionSender actually protects the hot path.
- Drop @AwaitsFix on 7 Append/AppendPipe IT methods. The Utf8View
  schema-lie they hit is fixed by coerceToDeclaredSchema in upstream
  #21457 (merged here); all 7 now pass.

Signed-off-by: Marc Handalian 

* fix merge conflicts

Signed-off-by: Marc Handalian 

---------

Signed-off-by: Marc Handalian 
---
 .../datafusion/DatafusionPartitionSender.java |  38 +++--
 .../be/datafusion/DatafusionReduceSink.java   |   2 +-
 .../be/datafusion/DatafusionResultStream.java |  20 +--
 .../DatafusionResultStreamTests.java          |  18 +--
 .../exec/AnalyticsSearchService.java          |  78 ----------
 .../exec/AnalyticsSearchTransportService.java | 119 +++------------
 .../exec/StreamingResponseListener.java       |   2 +-
 .../exec/action/FragmentExecutionAction.java  |   6 +-
 .../action/FragmentExecutionResponse.java     |  73 ---------
 .../analytics/exec/stage/ResponseCodec.java   |  40 -----
 .../exec/stage/RowResponseCodec.java          | 112 --------------
 .../stage/ShardFragmentStageExecution.java    |  65 +++-----
 .../stage/ShardFragmentStageScheduler.java    |  18 +--
 .../ShardFragmentStageExecutionTests.java     | 140 ++----------------
 .../resilience/CoordinatorResilienceIT.java   |   4 +-
 sandbox/qa/analytics-engine-rest/build.gradle |   4 +-
 .../analytics/qa/AppendPipeCommandIT.java     |   2 -
 17 files changed, 113 insertions(+), 628 deletions(-)
 delete mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionResponse.java
 delete mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ResponseCodec.java
 delete mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java

diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionPartitionSender.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionPartitionSender.java
index d59f82f4b04af..9b700ce61cc9b 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionPartitionSender.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionPartitionSender.java
@@ -11,24 +11,44 @@
 import org.opensearch.analytics.backend.jni.NativeHandle;
 import org.opensearch.be.datafusion.nativelib.NativeBridge;
 
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
 /**
- * Type-safe wrapper around a native {@code PartitionStreamSender} pointer.
+ * Type-safe wrapper around a native {@code PartitionStreamSender} pointer. Closing
+ * the sender signals EOF to the DataFusion receiver side.
  *
- * 
            Produced by {@link NativeBridge#registerPartitionStream(long, String, byte[])} and used
- * by {@link DatafusionReduceSink#feed} to push Arrow C Data batches into the reduce input
- * stream. Closing the sender signals EOF to the DataFusion receiver side.
+ * 
            The {@code lifecycle} read-write lock serialises {@link #send} / {@link #close}:
+ * native {@code sender_send} holds an immutable borrow of the heap-allocated sender
+ * across an {@code mpsc::Sender::send().await}, while {@code sender_close} reclaims
+ * the {@code Box} — a use-after-free if these overlap.
  */
 public final class DatafusionPartitionSender extends NativeHandle {
 
-    /**
-     * Wraps the given sender pointer.
-     *
-     * @param senderPtr pointer returned by {@link NativeBridge#registerPartitionStream}
-     */
+    private final ReentrantReadWriteLock lifecycle = new ReentrantReadWriteLock();
+
     public DatafusionPartitionSender(long senderPtr) {
         super(senderPtr);
     }
 
+    public void send(long arrayAddr, long schemaAddr) {
+        lifecycle.readLock().lock();
+        try {
+            NativeBridge.senderSend(getPointer(), arrayAddr, schemaAddr);
+        } finally {
+            lifecycle.readLock().unlock();
+        }
+    }
+
+    @Override
+    public void close() {
+        lifecycle.writeLock().lock();
+        try {
+            super.close();
+        } finally {
+            lifecycle.writeLock().unlock();
+        }
+    }
+
     @Override
     protected void doClose() {
         NativeBridge.senderClose(ptr);
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java
index 46d9d384b2833..9d90e726fd6cd 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionReduceSink.java
@@ -219,7 +219,7 @@ private void feedToSender(DatafusionPartitionSender sender, VectorSchemaRoot bat
             batch.close();
         }
         try {
-            NativeBridge.senderSend(sender.getPointer(), array.memoryAddress(), arrowSchema.memoryAddress());
+            sender.send(array.memoryAddress(), arrowSchema.memoryAddress());
             feedCount.incrementAndGet();
         } catch (RuntimeException e) {
             if (closed) {
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java
index c2cd6d95eaab0..bfd61175e66bc 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DatafusionResultStream.java
@@ -89,14 +89,7 @@ static class BatchIterator implements Iterator {
         private Schema schema;
         private VectorSchemaRoot nextBatch;
         private Boolean nextAvailable;
-        // True once we've returned at least one batch (real or synthesized). Prevents the
-        // "zero batches produced" rescue below from re-firing on subsequent polls.
         private boolean batchEmitted;
-        // True once the native stream has yielded EOS (arrayAddr == 0). Combined with
-        // batchEmitted, it lets us synthesize exactly one schema-carrying empty batch when
-        // the native side produced nothing — so downstream Arrow Flight (or any other
-        // transport that relies on the first data frame to deliver the schema) still sees
-        // the column layout.
         private boolean nativeStreamExhausted;
 
         BatchIterator(StreamHandle streamHandle, BufferAllocator allocator, CDataDictionaryProvider dictionaryProvider) {
@@ -119,21 +112,14 @@ private void ensureSchema() {
 
         private boolean loadNextBatch() {
             ensureSchema();
-            // Once the native stream has reported EOS, never poll again and never
-            // re-synthesize an empty batch. See the synthesize branch below.
             if (nativeStreamExhausted) return false;
-
             long arrayAddr = callNativeFn(
                 listener -> NativeBridge.streamNext(streamHandle.getRuntimeHandle().get(), streamHandle.getPointer(), listener)
             );
-
             if (arrayAddr == 0) {
                 nativeStreamExhausted = true;
-                // Native source had zero batches — synthesize one zero-row batch so the
-                // schema rides on the first (and only) Flight data frame. Without this the
-                // wire stream is schema-less and any downstream that relies on the schema
-                // (coordinator StreamingTableExec, row-path schema introspection) fails
-                // with cryptic errors such as "project index 0 out of bounds, max field 0".
+                // Streaming Flight requires ≥1 schema-bearing frame before completeStream;
+                // synthesise a zero-row batch carrying the schema for empty native streams.
                 if (!batchEmitted) {
                     nextBatch = VectorSchemaRoot.create(schema, allocator);
                     nextBatch.setRowCount(0);
@@ -142,7 +128,6 @@ private boolean loadNextBatch() {
                 }
                 return false;
             }
-
             VectorSchemaRoot freshRoot = VectorSchemaRoot.create(schema, allocator);
             try (ArrowArray arrowArray = ArrowArray.wrap(arrayAddr)) {
                 Data.importIntoVectorSchemaRoot(allocator, arrowArray, freshRoot, dictionaryProvider);
@@ -168,6 +153,7 @@ public EngineResultBatch next() {
             nextAvailable = null;
             VectorSchemaRoot batch = nextBatch;
             nextBatch = null;
+            batchEmitted = true;
             // Caller owns the returned VSR's lifecycle. Streaming handler transfers it to Flight
             // (Flight closes after wire write); row-path collector closes after reading.
             return new ArrowResultBatch(batch);
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java
index 9e9ae14a84262..0e3b4d6f973c8 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/test/java/org/opensearch/be/datafusion/DatafusionResultStreamTests.java
@@ -123,20 +123,20 @@ public void testNextWithoutHasNextWorks() throws Exception {
         }
     }
 
-    public void testNextOnExhaustedStreamThrows() throws Exception {
+    public void testEmptyResultYieldsOneZeroRowBatchWithSchema() throws Exception {
+        // Streaming Flight requires ≥1 schema-bearing frame before completeStream; empty
+        // native streams synthesise a zero-row batch carrying the schema.
         try (DatafusionResultStream stream = createStream("SELECT message FROM test_table WHERE message > 999")) {
             Iterator it = stream.iterator();
-            // Empty results still emit one zero-row batch carrying the declared schema so
-            // downstream transports (Flight, row-path) see the column layout on the first
-            // data frame. Consume it, then the iterator is genuinely exhausted.
-            assertTrue("empty result emits a schema-carrying zero-row batch", it.hasNext());
-            EngineResultBatch schemaOnly = it.next();
+            assertTrue("empty stream must yield exactly one zero-row schema batch", it.hasNext());
+            EngineResultBatch batch = it.next();
             try {
-                assertEquals("synthesized batch has zero rows", 0, schemaOnly.getRowCount());
+                assertEquals(0, batch.getRowCount());
+                assertEquals(java.util.List.of("message"), batch.getFieldNames());
             } finally {
-                schemaOnly.getArrowRoot().close();
+                batch.getArrowRoot().close();
             }
-            assertFalse("no further batches after the schema-only emit", it.hasNext());
+            assertFalse("after consuming the schema batch the stream is empty", it.hasNext());
             expectThrows(NoSuchElementException.class, it::next);
         }
     }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java
index 6510228af867e..817074dfd2dea 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchService.java
@@ -9,21 +9,11 @@
 package org.opensearch.analytics.exec;
 
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
-import org.apache.arrow.vector.ipc.ArrowStreamWriter;
-import org.apache.arrow.vector.ipc.WriteChannel;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-import org.apache.arrow.vector.ipc.message.IpcOption;
-import org.apache.arrow.vector.ipc.message.MessageSerializer;
-import org.apache.arrow.vector.types.pojo.Schema;
 import org.opensearch.analytics.backend.AnalyticsOperationListener;
-import org.opensearch.analytics.backend.EngineResultBatch;
 import org.opensearch.analytics.backend.EngineResultStream;
 import org.opensearch.analytics.backend.SearchExecEngine;
 import org.opensearch.analytics.backend.ShardScanExecutionContext;
 import org.opensearch.analytics.exec.action.FragmentExecutionRequest;
-import org.opensearch.analytics.exec.action.FragmentExecutionResponse;
 import org.opensearch.analytics.exec.task.AnalyticsShardTask;
 import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin;
 import org.opensearch.analytics.spi.BackendExecutionContext;
@@ -33,7 +23,6 @@
 import org.opensearch.analytics.spi.FragmentInstructionHandlerFactory;
 import org.opensearch.analytics.spi.InstructionNode;
 import org.opensearch.arrow.flight.transport.ArrowAllocatorProvider;
-import org.opensearch.common.Nullable;
 import org.opensearch.common.concurrent.GatedCloseable;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.tasks.TaskCancelledException;
@@ -42,10 +31,7 @@
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.tasks.Task;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.nio.channels.Channels;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -97,27 +83,6 @@ public void close() {
         allocator.close();
     }
 
-    public FragmentExecutionResponse executeFragment(FragmentExecutionRequest request, IndexShard shard) {
-        return executeFragment(request, shard, null);
-    }
-
-    public FragmentExecutionResponse executeFragment(FragmentExecutionRequest request, IndexShard shard, AnalyticsShardTask task) {
-        ResolvedFragment resolved = resolveFragment(request, shard);
-        long startNanos = System.nanoTime();
-        try (FragmentResources ctx = startFragment(request, resolved, shard, task)) {
-            FragmentExecutionResponse response = collectResponse(ctx.stream(), task);
-            long tookNanos = System.nanoTime() - startNanos;
-            listener.onFragmentSuccess(resolved.queryId, resolved.stageId, resolved.shardIdStr, tookNanos, response.getRowCount());
-            return response;
-        } catch (TaskCancelledException | IllegalStateException | IllegalArgumentException e) {
-            listener.onFragmentFailure(resolved.queryId, resolved.stageId, resolved.shardIdStr, e);
-            throw e;
-        } catch (Exception e) {
-            listener.onFragmentFailure(resolved.queryId, resolved.stageId, resolved.shardIdStr, e);
-            throw new RuntimeException("Failed to execute fragment on " + shard.shardId(), e);
-        }
-    }
-
     public FragmentResources executeFragmentStreaming(FragmentExecutionRequest request, IndexShard shard, AnalyticsShardTask task) {
         ResolvedFragment resolved = resolveFragment(request, shard);
         try {
@@ -234,47 +199,4 @@ private ShardScanExecutionContext buildContext(
         return ctx;
     }
 
-    FragmentExecutionResponse collectResponse(EngineResultStream stream) {
-        return collectResponse(stream, null);
-    }
-
-    FragmentExecutionResponse collectResponse(EngineResultStream stream, @Nullable AnalyticsShardTask task) {
-        // Serialize incoming Arrow batches as an Arrow IPC stream: one schema header
-        // followed by one record-batch message per incoming batch. Arrow's own
-        // serializer handles every Arrow type — no per-type Java code path.
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        WriteChannel channel = new WriteChannel(Channels.newChannel(baos));
-        Schema schema = null;
-        int totalRows = 0;
-        Iterator it = stream.iterator();
-        try {
-            while (it.hasNext()) {
-                if (task != null && task.isCancelled()) {
-                    throw new TaskCancelledException("task cancelled: " + task.getReasonCancelled());
-                }
-                EngineResultBatch batch = it.next();
-                VectorSchemaRoot root = batch.getArrowRoot();
-                try {
-                    if (schema == null) {
-                        schema = root.getSchema();
-                        MessageSerializer.serialize(channel, schema);
-                    }
-                    try (ArrowRecordBatch recordBatch = new VectorUnloader(root).getRecordBatch()) {
-                        MessageSerializer.serialize(channel, recordBatch);
-                    }
-                    totalRows += root.getRowCount();
-                } finally {
-                    root.close();
-                }
-            }
-            if (schema != null) {
-                // Write the end-of-stream marker so the reader sees a clean EOS
-                // instead of hitting end-of-input mid-message.
-                ArrowStreamWriter.writeEndOfStream(channel, IpcOption.DEFAULT);
-            }
-        } catch (IOException e) {
-            throw new IllegalStateException("Failed to serialize fragment output as Arrow IPC stream", e);
-        }
-        return new FragmentExecutionResponse(baos.toByteArray(), totalRows);
-    }
 }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchTransportService.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchTransportService.java
index d7a933a6a42d1..1a323936552f4 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchTransportService.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/AnalyticsSearchTransportService.java
@@ -12,16 +12,12 @@
 import org.opensearch.analytics.exec.action.FragmentExecutionAction;
 import org.opensearch.analytics.exec.action.FragmentExecutionArrowResponse;
 import org.opensearch.analytics.exec.action.FragmentExecutionRequest;
-import org.opensearch.analytics.exec.action.FragmentExecutionResponse;
 import org.opensearch.analytics.exec.task.AnalyticsShardTask;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.Nullable;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.inject.Singleton;
-import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType;
@@ -32,7 +28,6 @@
 import org.opensearch.transport.TransportException;
 import org.opensearch.transport.TransportRequestOptions;
 import org.opensearch.transport.TransportResponseHandler;
-import org.opensearch.transport.TransportService;
 import org.opensearch.transport.stream.StreamErrorCode;
 import org.opensearch.transport.stream.StreamException;
 import org.opensearch.transport.stream.StreamTransportResponse;
@@ -41,66 +36,40 @@
 import java.util.Iterator;
 
 /**
- * Stateless transport dispatch component for fragment requests. Owns
- * {@link TransportService} (or {@link StreamTransportService}) and
+ * Stateless transport dispatch component for fragment requests. Owns the
+ * {@link StreamTransportService} (analytics-engine is streaming-only) and
  * connection lookup.
  *
- * 
            Does NOT track per-query or per-node concurrency
- * state — callers provide their own {@link PendingExecutions} instance
- * to gate dispatch concurrency.
+ * 
            Does NOT track per-query or per-node concurrency state — callers provide
+ * their own {@link PendingExecutions} instance to gate dispatch concurrency.
  *
  * @opensearch.internal
  */
 @Singleton
 public class AnalyticsSearchTransportService {
-    private final TransportService transportService;
+    private final StreamTransportService transportService;
     private final ClusterService clusterService;
-    private final boolean streamingEnabled;
 
     @Inject
     public AnalyticsSearchTransportService(
-        TransportService transportService,
-        @Nullable StreamTransportService streamTransportService,
+        StreamTransportService streamTransportService,
         ClusterService clusterService,
         AnalyticsSearchService searchService,
         IndicesService indicesService
     ) {
-        this.streamingEnabled = streamTransportService != null;
-        this.transportService = this.streamingEnabled ? streamTransportService : transportService;
-        this.clusterService = clusterService;
-        if (this.streamingEnabled) {
-            registerStreamingFragmentHandler(this.transportService, searchService, indicesService);
-        } else {
-            registerFragmentHandler(this.transportService, searchService, indicesService);
+        if (streamTransportService == null) {
+            throw new IllegalStateException(
+                "analytics-engine requires the STREAM_TRANSPORT feature flag to be enabled "
+                    + "(opensearch.experimental.feature.stream_transport.enabled=true)"
+            );
         }
-    }
-
-    public boolean isStreamingEnabled() {
-        return streamingEnabled;
-    }
-
-    private static void registerFragmentHandler(
-        TransportService transportService,
-        AnalyticsSearchService searchService,
-        IndicesService indicesService
-    ) {
-        transportService.registerRequestHandler(
-            FragmentExecutionAction.NAME,
-            ThreadPool.Names.SAME,
-            false,
-            true,
-            AdmissionControlActionType.SEARCH,
-            FragmentExecutionRequest::new,
-            (request, channel, task) -> {
-                IndexShard shard = indicesService.indexServiceSafe(request.getShardId().getIndex()).getShard(request.getShardId().id());
-                FragmentExecutionResponse response = searchService.executeFragment(request, shard, (AnalyticsShardTask) task);
-                channel.sendResponse(response);
-            }
-        );
+        this.transportService = streamTransportService;
+        this.clusterService = clusterService;
+        registerStreamingFragmentHandler(this.transportService, searchService, indicesService);
     }
 
     private static void registerStreamingFragmentHandler(
-        TransportService transportService,
+        StreamTransportService transportService,
         AnalyticsSearchService searchService,
         IndicesService indicesService
     ) {
@@ -144,56 +113,15 @@ public void dispatchFragmentStreaming(
         Task parentTask,
         PendingExecutions pending
     ) {
-        dispatchFragment(
-            request,
-            targetNode,
-            listener,
-            parentTask,
-            pending,
-            in -> new FragmentExecutionArrowResponse(in),
-            TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STREAM).build(),
-            true
-        );
-    }
-
-    public void dispatchFragment(
-        FragmentExecutionRequest request,
-        DiscoveryNode targetNode,
-        StreamingResponseListener listener,
-        Task parentTask,
-        PendingExecutions pending
-    ) {
-        dispatchFragment(
-            request,
-            targetNode,
-            listener,
-            parentTask,
-            pending,
-            in -> new FragmentExecutionResponse(in),
-            TransportRequestOptions.EMPTY,
-            false
-        );
-    }
-
-    private  void dispatchFragment(
-        FragmentExecutionRequest request,
-        DiscoveryNode targetNode,
-        StreamingResponseListener listener,
-        Task parentTask,
-        PendingExecutions pending,
-        Writeable.Reader reader,
-        TransportRequestOptions options,
-        boolean skipsDeserialization
-    ) {
-        TransportResponseHandler handler = new TransportResponseHandler<>() {
+        TransportResponseHandler handler = new TransportResponseHandler<>() {
             @Override
-            public T read(StreamInput in) throws IOException {
-                return reader.read(in);
+            public FragmentExecutionArrowResponse read(StreamInput in) throws IOException {
+                return new FragmentExecutionArrowResponse(in);
             }
 
             @Override
             public boolean skipsDeserialization() {
-                return skipsDeserialization;
+                return true;
             }
 
             @Override
@@ -202,10 +130,10 @@ public String executor() {
             }
 
             @Override
-            public void handleStreamResponse(StreamTransportResponse stream) {
+            public void handleStreamResponse(StreamTransportResponse stream) {
                 try {
-                    T current;
-                    T last = null;
+                    FragmentExecutionArrowResponse current;
+                    FragmentExecutionArrowResponse last = null;
                     while ((current = stream.nextResponse()) != null) {
                         if (last != null) {
                             listener.onStreamResponse(last, false);
@@ -226,7 +154,7 @@ public void handleStreamResponse(StreamTransportResponse stream) {
             }
 
             @Override
-            public void handleResponse(T response) {
+            public void handleResponse(FragmentExecutionArrowResponse response) {
                 try {
                     listener.onStreamResponse(response, true);
                 } finally {
@@ -244,6 +172,7 @@ public void handleException(TransportException e) {
             }
         };
 
+        TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STREAM).build();
         pending.tryRun(() -> {
             try {
                 Transport.Connection connection = getConnection(null, targetNode.getId());
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/StreamingResponseListener.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/StreamingResponseListener.java
index 34095474c9c1f..686cdb1319cbe 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/StreamingResponseListener.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/StreamingResponseListener.java
@@ -15,7 +15,7 @@
  * Follows {@code StreamSearchActionListener.onStreamResponse(result, isLast)} pattern.
  *
  * 
            The type parameter {@code } is the response type for the transport action.
- * For shard fragment stages this is {@code FragmentExecutionResponse}.
+ * For shard fragment stages this is {@code FragmentExecutionArrowResponse}.
  *
  * 
            Contract:
  * 
              
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionAction.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionAction.java
index bd69e415475f7..d39df172888f3 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionAction.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/action/FragmentExecutionAction.java
@@ -13,9 +13,9 @@
 /**
  * {@link ActionType} singleton for the analytics shard-level fragment
  * execution action. Pairs the action name with the
- * {@link FragmentExecutionResponse} deserializer.
+ * {@link FragmentExecutionArrowResponse} deserializer.
  */
-public class FragmentExecutionAction extends ActionType {
+public class FragmentExecutionAction extends ActionType {
 
     /** Action name registered with the transport layer. */
     public static final String NAME = "indices:data/read/analytics/fragment";
@@ -24,6 +24,6 @@ public class FragmentExecutionAction extends ActionTypeArrow IPC handles every Arrow type natively (temporal, string-view,
- * dictionary, nested) without hand-rolled per-type serialization. Previously,
- * this response carried {@code List} rows and relied on
- * {@code StreamOutput.writeGenericValue} — which does not support Java 8+
- * temporal types like {@link java.time.LocalDateTime} and so failed the moment
- * a shard emitted a batch with a Timestamp column.
- *
- * 
              Wire format: {@code ipcPayload (byte[]) + rowCount (vint)}. The row count
- * is the total across all batches in the payload, cached for metrics / logging
- * so consumers don't have to decode the payload just to report "N rows handled".
- *
- * @opensearch.internal
- */
-public class FragmentExecutionResponse extends ActionResponse {
-
-    private final byte[] ipcPayload;
-    private final int rowCount;
-
-    public FragmentExecutionResponse(byte[] ipcPayload, int rowCount) {
-        this.ipcPayload = ipcPayload;
-        this.rowCount = rowCount;
-    }
-
-    public FragmentExecutionResponse(StreamInput in) throws IOException {
-        super(in);
-        this.ipcPayload = in.readByteArray();
-        this.rowCount = in.readVInt();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeByteArray(ipcPayload);
-        out.writeVInt(rowCount);
-    }
-
-    /**
-     * Arrow IPC stream bytes — a schema message followed by zero or more record
-     * batch messages, as written by {@link org.apache.arrow.vector.ipc.ArrowStreamWriter}.
-     * An empty array means the fragment produced no output at all (no schema,
-     * no rows).
-     */
-    public byte[] getIpcPayload() {
-        return ipcPayload;
-    }
-
-    /**
-     * Total number of rows across all batches in {@link #getIpcPayload()}.
-     */
-    public int getRowCount() {
-        return rowCount;
-    }
-}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ResponseCodec.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ResponseCodec.java
deleted file mode 100644
index 528b3a93e2b1f..0000000000000
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ResponseCodec.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.analytics.exec.stage;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.opensearch.core.action.ActionResponse;
-
-/**
- * Decodes a transport response into an Arrow {@link VectorSchemaRoot} for
- * the coordinator-side sink. Implementations handle the specific wire
- * format — {@code Object[]} rows (current), Arrow IPC (Flight), or any
- * future format.
- *
- * 
              The codec is injected into {@link ShardFragmentStageExecution} at
- * construction time by the scheduler. Swapping the codec swaps the
- * serialization format without touching stage execution logic.
- *
- * @param  the transport response type
- * @opensearch.internal
- */
-@FunctionalInterface
-public interface ResponseCodec {
-
-    /**
-     * Decodes a transport response into an Arrow {@link VectorSchemaRoot}.
-     * The returned VSR is owned by the caller (the sink).
-     *
-     * @param response  the transport response
-     * @param allocator the buffer allocator for Arrow vectors
-     * @return a new VectorSchemaRoot; caller owns and must close it
-     */
-    VectorSchemaRoot decode(R response, BufferAllocator allocator);
-}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java
deleted file mode 100644
index 38aef31c1337b..0000000000000
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/RowResponseCodec.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.analytics.exec.stage;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.ipc.ArrowStreamReader;
-import org.apache.arrow.vector.types.pojo.Schema;
-import org.opensearch.analytics.exec.action.FragmentExecutionResponse;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * {@link ResponseCodec} that deserializes the Arrow IPC stream payload carried
- * by {@link FragmentExecutionResponse} into a single consolidated
- * {@link VectorSchemaRoot}. Uses Arrow's {@link ArrowStreamReader} for message
- * sequencing (schema header, record batches, end-of-stream) and
- * {@link FieldVector#makeTransferPair} / {@link FieldVector#copyFromSafe} to
- * move data into a caller-owned root — both are supported by every vector
- * kind, including the view vectors ({@code Utf8View}, {@code BinaryView}) that
- * DataFusion emits for aggregate group keys.
- *
- * 
              Deliberately avoids {@code VectorSchemaRootAppender} — its underlying
- * {@code VectorAppender} rejects view vectors with
- * {@code UnsupportedOperationException}.
- *
- * @opensearch.internal
- */
-public final class RowResponseCodec implements ResponseCodec {
-
-    /** Singleton instance — stateless, thread-safe. */
-    public static final RowResponseCodec INSTANCE = new RowResponseCodec();
-
-    private RowResponseCodec() {}
-
-    @Override
-    public VectorSchemaRoot decode(FragmentExecutionResponse response, BufferAllocator allocator) {
-        if (allocator == null) {
-            throw new IllegalArgumentException("BufferAllocator must not be null");
-        }
-        byte[] payload = response.getIpcPayload();
-        if (payload == null || payload.length == 0) {
-            return VectorSchemaRoot.create(new Schema(List.of()), allocator);
-        }
-
-        List batches = new ArrayList<>();
-        Schema schema;
-        try (ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayInputStream(payload), allocator)) {
-            VectorSchemaRoot readerRoot = reader.getVectorSchemaRoot();
-            schema = readerRoot.getSchema();
-            while (reader.loadNextBatch()) {
-                // Transfer each batch's buffers out of the reader's reused root into an
-                // independent root owned by this codec. Transfer works for all vector
-                // kinds (including views) and is zero-copy when allocators match.
-                VectorSchemaRoot batchRoot = VectorSchemaRoot.create(schema, allocator);
-                int rowCount = readerRoot.getRowCount();
-                for (int i = 0; i < readerRoot.getFieldVectors().size(); i++) {
-                    readerRoot.getVector(i).makeTransferPair(batchRoot.getVector(i)).transfer();
-                }
-                batchRoot.setRowCount(rowCount);
-                batches.add(batchRoot);
-            }
-        } catch (IOException e) {
-            for (VectorSchemaRoot b : batches)
-                b.close();
-            throw new IllegalStateException("Failed to decode Arrow IPC payload from fragment response", e);
-        }
-
-        if (batches.isEmpty()) {
-            return VectorSchemaRoot.create(schema, allocator);
-        }
-        if (batches.size() == 1) {
-            return batches.get(0);
-        }
-        // Multiple batches — concatenate via per-cell copyFromSafe. Slower than columnar
-        // append but is the only operation Arrow Java implements for every vector type
-        // (VectorAppender throws on view vectors; see class javadoc).
-        int totalRows = batches.stream().mapToInt(VectorSchemaRoot::getRowCount).sum();
-        VectorSchemaRoot combined = VectorSchemaRoot.create(schema, allocator);
-        try {
-            combined.allocateNew();
-            for (int f = 0; f < combined.getFieldVectors().size(); f++) {
-                FieldVector dst = combined.getVector(f);
-                int offset = 0;
-                for (VectorSchemaRoot batch : batches) {
-                    FieldVector src = batch.getVector(f);
-                    int rows = batch.getRowCount();
-                    for (int r = 0; r < rows; r++) {
-                        dst.copyFromSafe(r, offset + r, src);
-                    }
-                    offset += rows;
-                }
-                dst.setValueCount(totalRows);
-            }
-            combined.setRowCount(totalRows);
-            return combined;
-        } finally {
-            for (VectorSchemaRoot b : batches)
-                b.close();
-        }
-    }
-}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java
index 8376e34ed22f9..83ae3dd0b4577 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java
@@ -16,14 +16,11 @@
 import org.opensearch.analytics.exec.StreamingResponseListener;
 import org.opensearch.analytics.exec.action.FragmentExecutionArrowResponse;
 import org.opensearch.analytics.exec.action.FragmentExecutionRequest;
-import org.opensearch.analytics.exec.action.FragmentExecutionResponse;
 import org.opensearch.analytics.planner.dag.ExecutionTarget;
 import org.opensearch.analytics.planner.dag.ShardExecutionTarget;
 import org.opensearch.analytics.planner.dag.Stage;
 import org.opensearch.analytics.spi.ExchangeSink;
-import org.opensearch.arrow.flight.transport.ArrowBatchResponse;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.core.action.ActionResponse;
 
 import java.util.List;
 import java.util.Map;
@@ -32,10 +29,9 @@
 import java.util.function.Function;
 
 /**
- * Leaf stage execution that dispatches fragment work to data-node shards.
- *
- * 
              Handles both Arrow streaming and row (codec-decoded) responses, feeding
- * resulting batches into the parent stage's {@link ExchangeSink}.
+ * Leaf stage execution that dispatches fragment work to data-node shards via
+ * Arrow streaming, feeding resulting batches into the parent stage's
+ * {@link ExchangeSink}.
  *
  * 
              One-shot: constructed, {@link #start()} called once, listener
  * signaled on completion, then discarded.
@@ -51,7 +47,6 @@ final class ShardFragmentStageExecution extends AbstractStageExecution implement
     private final ClusterService clusterService;
     private final Function requestBuilder;
     private final AnalyticsSearchTransportService dispatcher;
-    private final ResponseCodec responseCodec;
     private final Map pendingPerNode = new ConcurrentHashMap<>();
 
     ShardFragmentStageExecution(
@@ -60,8 +55,7 @@ final class ShardFragmentStageExecution extends AbstractStageExecution implement
         ExchangeSink outputSink,
         ClusterService clusterService,
         Function requestBuilder,
-        AnalyticsSearchTransportService dispatcher,
-        ResponseCodec responseCodec
+        AnalyticsSearchTransportService dispatcher
     ) {
         super(stage);
         this.config = config;
@@ -69,11 +63,6 @@ final class ShardFragmentStageExecution extends AbstractStageExecution implement
         this.clusterService = clusterService;
         this.requestBuilder = requestBuilder;
         this.dispatcher = dispatcher;
-        this.responseCodec = responseCodec;
-    }
-
-    private boolean useArrowStreaming() {
-        return dispatcher.isStreamingEnabled();
     }
 
     @Override
@@ -93,40 +82,36 @@ public void start() {
     private void dispatchShardTask(ShardExecutionTarget target) {
         FragmentExecutionRequest request = requestBuilder.apply(target);
         PendingExecutions pending = pendingFor(target);
-        if (useArrowStreaming()) {
-            dispatcher.dispatchFragmentStreaming(
-                request,
-                target.node(),
-                responseListener(FragmentExecutionArrowResponse::getRoot),
-                config.parentTask(),
-                pending
-            );
-        } else {
-            dispatcher.dispatchFragment(
-                request,
-                target.node(),
-                responseListener(r -> responseCodec.decode(r, config.bufferAllocator())),
-                config.parentTask(),
-                pending
-            );
-        }
+        dispatcher.dispatchFragmentStreaming(request, target.node(), responseListener(), config.parentTask(), pending);
     }
 
-    private  StreamingResponseListener responseListener(Function toVsr) {
+    private StreamingResponseListener responseListener() {
         return new StreamingResponseListener<>() {
             // Runs inline on the per-stream virtual thread driving handleStreamResponse.
             // Must NOT offload to a thread pool: reordering across batches would let the
             // isLast=true task race ahead, flip state to SUCCEEDED, and drop queued
             // earlier batches via the isDone() short-circuit.
             @Override
-            public void onStreamResponse(T response, boolean isLast) {
+            public void onStreamResponse(FragmentExecutionArrowResponse response, boolean isLast) {
                 if (isDone()) {
-                    releaseResponseResources(response);
+                    VectorSchemaRoot root = response.getRoot();
+                    if (root != null) {
+                        root.close();
+                    }
                     return;
                 }
 
-                VectorSchemaRoot vsr = toVsr.apply(response);
-                outputSink.feed(vsr);
+                VectorSchemaRoot vsr = response.getRoot();
+                try {
+                    outputSink.feed(vsr);
+                } catch (Exception e) {
+                    // Without this guard the exception only surfaces on the stream's virtual
+                    // thread; inFlight never decrements and the stage hangs to QUERY_TIMEOUT.
+                    captureFailure(new RuntimeException("Stage " + stage.getStageId() + " sink feed failed", e));
+                    metrics.incrementTasksFailed();
+                    onShardTerminated();
+                    return;
+                }
                 metrics.addRowsProcessed(vsr.getRowCount());
 
                 if (isLast) {
@@ -144,12 +129,6 @@ public void onFailure(Exception e) {
         };
     }
 
-    private static  void releaseResponseResources(T response) {
-        if (response instanceof ArrowBatchResponse arrowResp && arrowResp.getRoot() != null) {
-            arrowResp.getRoot().close();
-        }
-    }
-
     private void onShardTerminated() {
         if (inFlight.decrementAndGet() == 0) {
             Exception captured = getFailure();
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java
index da616cfdce341..dd120de7b4c6d 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageScheduler.java
@@ -11,7 +11,6 @@
 import org.opensearch.analytics.exec.AnalyticsSearchTransportService;
 import org.opensearch.analytics.exec.QueryContext;
 import org.opensearch.analytics.exec.action.FragmentExecutionRequest;
-import org.opensearch.analytics.exec.action.FragmentExecutionResponse;
 import org.opensearch.analytics.planner.dag.ShardExecutionTarget;
 import org.opensearch.analytics.planner.dag.Stage;
 import org.opensearch.analytics.planner.dag.StagePlan;
@@ -31,31 +30,16 @@
  * and doesn't care whether it is a root sink or a parent-provided child sink
  * — {@link StageExecutionBuilder} resolves that distinction before calling.
  *
- * 
              Injects a {@link ResponseCodec} into the execution to decouple the wire
- * format from stage logic. The default codec ({@link RowResponseCodec}) handles
- * the current {@code Object[]} row format; a future Arrow IPC codec would be
- * swapped in here.
- *
  * @opensearch.internal
  */
 final class ShardFragmentStageScheduler implements StageScheduler {
 
     private final ClusterService clusterService;
     private final AnalyticsSearchTransportService transport;
-    private final ResponseCodec responseCodec;
 
     ShardFragmentStageScheduler(ClusterService clusterService, AnalyticsSearchTransportService transport) {
-        this(clusterService, transport, RowResponseCodec.INSTANCE);
-    }
-
-    ShardFragmentStageScheduler(
-        ClusterService clusterService,
-        AnalyticsSearchTransportService transport,
-        ResponseCodec responseCodec
-    ) {
         this.clusterService = clusterService;
         this.transport = transport;
-        this.responseCodec = responseCodec;
     }
 
     @Override
@@ -73,7 +57,7 @@ public StageExecution createExecution(Stage stage, ExchangeSink sink, QueryConte
         // This keeps target resolution out of the build phase so cancellation before
         // dispatch doesn't pay for cluster-state routing, and leaves room for shuffle
         // reads whose targets depend on child manifests only available at dispatch time.
-        return new ShardFragmentStageExecution(stage, config, sink, clusterService, requestBuilder, transport, responseCodec);
+        return new ShardFragmentStageExecution(stage, config, sink, clusterService, requestBuilder, transport);
     }
 
     private static List buildPlanAlternatives(Stage stage) {
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
index f633477602e24..34e4c3a156f84 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
@@ -21,7 +21,6 @@
 import org.opensearch.analytics.exec.StreamingResponseListener;
 import org.opensearch.analytics.exec.action.FragmentExecutionArrowResponse;
 import org.opensearch.analytics.exec.action.FragmentExecutionRequest;
-import org.opensearch.analytics.exec.action.FragmentExecutionResponse;
 import org.opensearch.analytics.exec.task.AnalyticsQueryTask;
 import org.opensearch.analytics.planner.dag.ShardExecutionTarget;
 import org.opensearch.analytics.planner.dag.Stage;
@@ -30,13 +29,11 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 
@@ -73,7 +70,7 @@ public void testArrowResponseClosedWhenStageAlreadyCancelled() {
         AtomicReference> capturedListener = new AtomicReference<>();
         CapturingSink sink = new CapturingSink();
 
-        ShardFragmentStageExecution exec = buildExecution(sink, true, capturedListener);
+        ShardFragmentStageExecution exec = buildExecution(sink, capturedListener);
         exec.start();
 
         assertNotNull("listener should have been captured by dispatch", capturedListener.get());
@@ -99,7 +96,7 @@ public void testArrowResponseFedToSinkOnHappyPath() {
         AtomicReference> capturedListener = new AtomicReference<>();
         CapturingSink sink = new CapturingSink();
 
-        ShardFragmentStageExecution exec = buildExecution(sink, true, capturedListener);
+        ShardFragmentStageExecution exec = buildExecution(sink, capturedListener);
         exec.start();
 
         VectorSchemaRoot root = createTestBatch(3);
@@ -111,108 +108,24 @@ public void testArrowResponseFedToSinkOnHappyPath() {
         sink.close();
     }
 
-    /**
-     * Verifies that non-Arrow (row codec) responses don't crash
-     * the release path on cancellation (they hold no Arrow resources).
-     */
-    public void testRowResponseSafeOnCancellation() {
-        AtomicReference> capturedListener = new AtomicReference<>();
-        CapturingSink sink = new CapturingSink();
-
-        ShardFragmentStageExecution exec = buildExecution(sink, false, capturedListener);
-        exec.start();
-
-        exec.cancel("test");
-
-        FragmentExecutionResponse response = new FragmentExecutionResponse(new byte[0], 0);
-        capturedListener.get().onStreamResponse(response, true);
-
-        assertTrue("sink should not have received anything post-cancel", sink.fed.isEmpty());
-    }
-
-    /**
-     * Reproduces the lookahead-reordering bug: if the listener offloads
-     * {@code onStreamResponse} bodies onto a multi-threaded executor, the
-     * {@code isLast=true} task can complete before earlier batches' tasks
-     * have started — flipping the stage to SUCCEEDED so the still-queued
-     * earlier tasks short-circuit via {@code isDone()} and drop their data.
-     *
-     * 
              Forces the worst-case schedule deterministically with a manual
-     * executor that runs tasks in reverse submission order.
-     */
-    public void testListenerPreservesBatchesAcrossReorderedExecution() {
-        AtomicReference> capturedListener = new AtomicReference<>();
-        CapturingSink sink = new CapturingSink();
-        ManualExecutor manualExecutor = new ManualExecutor();
-
-        ShardFragmentStageExecution exec = buildExecution(sink, true, capturedListener, manualExecutor);
-        exec.start();
-
-        FragmentExecutionArrowResponse first = new FragmentExecutionArrowResponse(createTestBatch(2));
-        FragmentExecutionArrowResponse last = new FragmentExecutionArrowResponse(createTestBatch(3));
-
-        capturedListener.get().onStreamResponse(first, false);
-        capturedListener.get().onStreamResponse(last, true);
-
-        // Force the bug's worst case: isLast=true task runs first, transitions
-        // to SUCCEEDED; the earlier task then sees isDone()=true if the
-        // implementation is still offloading to the executor.
-        manualExecutor.runInReverseOrder();
-
-        assertEquals("both batches must be fed despite reverse-order execution", 2, sink.fed.size());
-        sink.close();
-    }
-
-    /**
-     * Verifies that RowResponseCodec rejects null allocator.
-     */
-    public void testRowResponseCodecRejectsNullAllocator() {
-        FragmentExecutionResponse response = new FragmentExecutionResponse(new byte[0], 0);
-        expectThrows(IllegalArgumentException.class, () -> RowResponseCodec.INSTANCE.decode(response, null));
-    }
-
     // ── helpers ──────────────────────────────────────────────────────────
 
-    @SuppressWarnings("unchecked")
-    private  ShardFragmentStageExecution buildExecution(
-        CapturingSink sink,
-        boolean streaming,
-        AtomicReference> listenerCapture
-    ) {
-        return buildExecution(sink, streaming, listenerCapture, Runnable::run);
-    }
-
-    @SuppressWarnings("unchecked")
-    private  ShardFragmentStageExecution buildExecution(
+    private ShardFragmentStageExecution buildExecution(
         CapturingSink sink,
-        boolean streaming,
-        AtomicReference> listenerCapture,
-        Executor searchExecutor
+        AtomicReference> listenerCapture
     ) {
         Stage stage = mockStage();
-        QueryContext config = mockQueryContext(searchExecutor);
+        QueryContext config = mockQueryContext();
         ClusterService clusterService = mockClusterService();
         AnalyticsSearchTransportService dispatcher = mock(AnalyticsSearchTransportService.class);
-        when(dispatcher.isStreamingEnabled()).thenReturn(streaming);
 
-        if (streaming) {
-            doAnswer(invocation -> {
-                StreamingResponseListener listener = (StreamingResponseListener) invocation.getArgument(2);
-                listenerCapture.set(listener);
-                return null;
-            }).when(dispatcher).dispatchFragmentStreaming(any(), any(), any(), any(), any());
-        } else {
-            doAnswer(invocation -> {
-                StreamingResponseListener listener = (StreamingResponseListener) invocation.getArgument(2);
-                listenerCapture.set(listener);
-                return null;
-            }).when(dispatcher).dispatchFragment(any(), any(), any(), any(), any());
-        }
-
-        ResponseCodec codec = (resp, alloc) -> {
-            VectorSchemaRoot vsr = createTestBatch(resp.getRowCount());
-            return vsr;
-        };
+        doAnswer(invocation -> {
+            @SuppressWarnings("unchecked")
+            StreamingResponseListener listener = (StreamingResponseListener<
+                FragmentExecutionArrowResponse>) invocation.getArgument(2);
+            listenerCapture.set(listener);
+            return null;
+        }).when(dispatcher).dispatchFragmentStreaming(any(), any(), any(), any(), any());
 
         Function requestBuilder = target -> new FragmentExecutionRequest(
             "test-query",
@@ -221,7 +134,7 @@ private  ShardFragmentStageExecution buildExecution(
             List.of(new FragmentExecutionRequest.PlanAlternative("test-backend", new byte[0], List.of()))
         );
 
-        return new ShardFragmentStageExecution(stage, config, sink, clusterService, requestBuilder, dispatcher, codec);
+        return new ShardFragmentStageExecution(stage, config, sink, clusterService, requestBuilder, dispatcher);
     }
 
     private VectorSchemaRoot createTestBatch(int rows) {
@@ -250,39 +163,14 @@ private Stage mockStage() {
     }
 
     private QueryContext mockQueryContext() {
-        return mockQueryContext(Runnable::run);
-    }
-
-    private QueryContext mockQueryContext(Executor searchExecutor) {
         QueryContext config = mock(QueryContext.class);
-        when(config.searchExecutor()).thenReturn(searchExecutor);
+        when(config.searchExecutor()).thenReturn(Runnable::run);
         when(config.parentTask()).thenReturn(mock(AnalyticsQueryTask.class));
         when(config.maxConcurrentShardRequests()).thenReturn(5);
         when(config.bufferAllocator()).thenReturn(allocator);
         return config;
     }
 
-    /**
-     * Test executor that records submitted tasks instead of running them.
-     * Run them explicitly in any order to deterministically simulate the
-     * worst-case multi-thread completion order on a real pool.
-     */
-    private static final class ManualExecutor implements Executor {
-        private final List queue = new ArrayList<>();
-
-        @Override
-        public synchronized void execute(Runnable command) {
-            queue.add(command);
-        }
-
-        synchronized void runInReverseOrder() {
-            for (int i = queue.size() - 1; i >= 0; i--) {
-                queue.get(i).run();
-            }
-            queue.clear();
-        }
-    }
-
     private ClusterService mockClusterService() {
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(mock(ClusterState.class));
diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java
index ba701a2050e41..5fbc6f816cf3d 100644
--- a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java
+++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/analytics/resilience/CoordinatorResilienceIT.java
@@ -147,7 +147,9 @@ protected Settings nodeSettings(int nodeOrdinal) {
     public void testStubReplacesStreamingShardResponseWithEmptyBatch() throws Exception {
         createAndSeedIndex();
         stubAllocator = new RootAllocator();
-        Schema schema = new Schema(List.of(new Field("value", FieldType.nullable(new ArrowType.Int(32, true)), null)));
+        // Schema width must match the coordinator's declared input-partition schema — that's
+        // the *aggregate* output type (SUM(int) → Int64/BIGINT), not the base column type.
+        Schema schema = new Schema(List.of(new Field("total", FieldType.nullable(new ArrowType.Int(64, true)), null)));
 
         AtomicInteger stubCalls = new AtomicInteger();
         String victim = pickShardHostingNode();
diff --git a/sandbox/qa/analytics-engine-rest/build.gradle b/sandbox/qa/analytics-engine-rest/build.gradle
index fab3606f0cf42..10096b4e8d04b 100644
--- a/sandbox/qa/analytics-engine-rest/build.gradle
+++ b/sandbox/qa/analytics-engine-rest/build.gradle
@@ -62,6 +62,9 @@ def configureAnalyticsCluster = { cluster ->
 
     // Enable pluggable dataformat feature flag
     cluster.systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true'
+
+    // analytics-engine requires the streaming transport — fragment dispatch is streaming-only.
+    cluster.systemProperty 'opensearch.experimental.feature.transport.stream.enabled', 'true'
 }
 
 // ── Default integTest cluster ────────────────────────────────────────────────
@@ -110,7 +113,6 @@ check.dependsOn(integTestStreaming)
 testClusters.integTestStreaming {
     numberOfNodes = 2
     configureAnalyticsCluster(delegate)
-    systemProperty 'opensearch.experimental.feature.transport.stream.enabled', 'true'
 }
 
 // Run against an external cluster (no testClusters lifecycle):
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java
index 56145ce21d257..322c6957810f1 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java
@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
@@ -54,7 +53,6 @@ private void ensureDataProvisioned() throws IOException {
 
     // ── duplicate + inline sort, then head ──────────────────────────────────────
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/21457")
     public void testAppendPipeSort() throws IOException {
         // Branch: stats sum(int0) by str0 → 3 rows (FURNITURE=1, OFFICE SUPPLIES=18, TECHNOLOGY=49).
         // Outer `sort str0` pins the original to alphabetical order. `appendpipe [sort -sum_int0_by_str0]`

From a02606f3a248be338e05796ed62d6959f682a7da Mon Sep 17 00:00:00 2001
From: bowenlan 
Date: Mon, 11 May 2026 14:51:44 -0700
Subject: [PATCH 113/115] Allow external override of the sandbox native library
 path (#21592)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add OPENSEARCH_NATIVE_LIB env var (or -PnativeLibOverride project property) so
sandbox tests can reuse a prebuilt libopensearch_native.{dylib,so,dll} from
another worktree, a blessed shared copy, or a CI-provided binary. When set,
buildRustLibrary is skipped via onlyIf and ext.nativeLibPath resolves to the
override path.

All 14 sandbox/plugins build.gradle sites already read
project(':sandbox:libs:dataformat-native').ext.nativeLibPath, so the override
propagates to every consumer without per-plugin changes. ./gradlew run is
unaffected — it already accepts -Dnative.lib.path via tests.jvm.argline and
does not depend on buildRustLibrary.

Without the env var / property, behavior is unchanged: buildRustLibrary runs
(or is UP-TO-DATE) and rust/target/release/libopensearch_native.dylib is used.

Signed-off-by: bowenlan-amzn 
Co-authored-by: Sandesh Kumar 
---
 sandbox/libs/dataformat-native/build.gradle | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/sandbox/libs/dataformat-native/build.gradle b/sandbox/libs/dataformat-native/build.gradle
index fa022b6347435..00f0631036f87 100644
--- a/sandbox/libs/dataformat-native/build.gradle
+++ b/sandbox/libs/dataformat-native/build.gradle
@@ -84,14 +84,21 @@ task buildRustLibrary(type: Exec) {
     outputs.file nativeLibFile
 }
 
-// Expose the native lib path so plugins can reference it for tests
-ext.nativeLibPath = nativeLibFile
+// External override: reuse a prebuilt .dylib from another worktree, a blessed shared copy,
+// or a CI-provided binary. Set OPENSEARCH_NATIVE_LIB (env) or -PnativeLibOverride to an
+// absolute .dylib/.so/.dll path; buildRustLibrary is skipped and nativeLibPath resolves to
+// the override. Consumers across sandbox/plugins read ext.nativeLibPath, so the override
+// propagates without per-plugin changes.
+def nativeLibOverride = project.findProperty('nativeLibOverride') ?: System.getenv('OPENSEARCH_NATIVE_LIB')
+def resolvedNativeLib = nativeLibOverride ? file(nativeLibOverride) : nativeLibFile
+ext.nativeLibPath = resolvedNativeLib
+buildRustLibrary.onlyIf { nativeLibOverride == null }
 
 assemble.dependsOn buildRustLibrary
 
 test {
     systemProperty 'tests.security.manager', 'false'
-    systemProperty 'native.lib.path', nativeLibFile.absolutePath
+    systemProperty 'native.lib.path', resolvedNativeLib.absolutePath
     jvmArgs += ['--enable-native-access=ALL-UNNAMED']
     dependsOn buildRustLibrary
 }

From f1cbbba360e92384448104a14adcafc88ec6854b Mon Sep 17 00:00:00 2001
From: Sandesh Kumar 
Date: Mon, 11 May 2026 16:09:10 -0700
Subject: [PATCH 114/115] [analytics-engine] Fix redundant instructions in
 FragmentConversionDriver (#21618)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Remove final-agg special case in FragmentConversionDriver

The dedicated FINAL-aggregate branch was forcing an AggregateMode override that
diverged from the standard partial/final path, producing the cnt[count] Int32
vs Int64 mismatch downstream. FragmentConversionDriver now treats aggregate
fragments via the same conversion path as the rest, letting the decomposition
resolver + Calcite's reduce rule carry the types through correctly.

Follow-on adjustments:
- FUNCTIONS_TO_REDUCE expanded to {AVG, STDDEV_POP, VAR_POP}; STDDEV_SAMP/VAR_SAMP
excluded because the CASE-WHEN boolean guard defeats stripAnnotations.
- POWER added to BASELINE_SCALAR_OPS (emitted as final sqrt).
- IT coverage for all four stat aggs in StreamingCoordinatorReduceIT.
- Existing planner tests updated for the new post-reduction pipeline shape.

Signed-off-by: Marc Handalian 

* fix(qa,planner): enable STDDEV_SAMP/VAR_SAMP, drop partial-agg instruction, test cleanups

Follow-up to Marc Handalian's commit that removed the FINAL-aggregate special case
in FragmentConversionDriver.

FragmentConversionDriver:
  - Drop the PARTIAL-aggregate instruction emission too. The Calcite-layer split
    (OpenSearchAggregateSplitRule + AggregateDecompositionResolver) already produces
    properly decomposed PARTIAL and FINAL fragments; the shard-side mode-forcing via
    NativeBridge.preparePartialPlan → force_aggregate_mode(Partial) was belt-and-
    suspenders. With it removed, every fragment takes the same executeLocalPlan path
    and DataFusion handles its own Final(Partial(...)) pair — correctness preserved
    because every aggregate reaching either stage is associative (SUM/MIN/MAX, COUNT
    function-swapped to SUM, HLL sketch merge, AVG/STDDEV primitive-decomposed).
    The downstream Java and Rust machinery (PartialAggregateInstructionHandler,
    FinalAggregateInstructionHandler, prepareFinalPlan FFI, force_aggregate_mode)
    stays in place — dormant, not dead — ready for re-enablement once upstream
    DataFusion's substrait consumer respects aggregation_phase (see
    .kiro/docs/datafusion-upstream-aggregation-phase.md).

OpenSearchAggregateReduceRule:
  - FUNCTIONS_TO_REDUCE expanded to the full statistical set
    {AVG, STDDEV_POP, STDDEV_SAMP, VAR_POP, VAR_SAMP}.
  - Javadoc updated to reflect the full reduction set and explain why STDDEV_SAMP/
    VAR_SAMP now flow through (Bessel's-correction CASE guard uses comparison
    operators that joined BASELINE_SCALAR_OPS in this commit).

OpenSearchProjectRule:
  - Added the six SQL comparison operators (>, >=, <, <=, =, !=) to
    BASELINE_SCALAR_OPS. They are emitted by Calcite's reduce rule for the SAMP-
    variant Bessel's-correction guard and are SQL-execution primitives every
    backend supports natively — consistent with the existing baseline rationale.

StreamingCoordinatorReduceIT:
  - Add missing semicolon at testAvgAcrossShards (int total = NUM_SHARDS *
    DOCS_PER_SHARD). The standard integTest task excludes this class via a glob,
    so the build failure only surfaced under integTestStreaming.
  - Remove the @AwaitsFix on testStddevSampAcrossShards and testVarSampAcrossShards
    (now enabled by the FUNCTIONS_TO_REDUCE and BASELINE_SCALAR_OPS changes above);
    drop the now-unused AwaitsFix import.

AppendPipeCommandIT:
  - Replace fully-qualified java.util.* references in testAppendPipeSort with
    imported short forms; add HashMap and Set imports. Matches the rest of the file.

Verified locally (after rebuilding the Rust dylib against the Wave A UDF set):
  - :sandbox:plugins:analytics-engine:test — 148 tests, 0 failures
  - :sandbox:qa:analytics-engine-rest:integTest — 304 tests, 4 skipped, 0 failures
  - :sandbox:qa:analytics-engine-rest:integTestStreaming — 7 tests, 0 skipped,
    0 failures (both STDDEV_SAMP and VAR_SAMP tests active and green)

Signed-off-by: Sandesh Kumar 

---------

Signed-off-by: Marc Handalian 
Signed-off-by: Sandesh Kumar 
Co-authored-by: Marc Handalian 
---
 .../planner/dag/FragmentConversionDriver.java |   6 -
 .../rules/OpenSearchAggregateReduceRule.java  |  24 ++-
 .../planner/rules/OpenSearchProjectRule.java  |  10 +
 .../analytics/planner/AggregateRuleTests.java |  22 +-
 .../analytics/planner/ProjectRuleTests.java   |  28 ++-
 .../dag/FragmentConversionDriverTests.java    |  12 +-
 .../analytics/qa/AppendPipeCommandIT.java     |  43 +++-
 .../analytics/qa/MultisearchCommandIT.java    |   2 -
 .../analytics/qa/ObjectFieldIT.java           |   4 -
 .../qa/StreamingCoordinatorReduceIT.java      | 188 ++++++++++++++++++
 10 files changed, 282 insertions(+), 57 deletions(-)

diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java
index 69cdfc440409f..bbcc16f558208 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/dag/FragmentConversionDriver.java
@@ -134,13 +134,7 @@ private static List assembleInstructions(
             } else {
                 factory.createShardScanNode().ifPresent(instructions::add);
             }
-            if (plan.resolvedFragment() instanceof OpenSearchAggregate agg && agg.getMode() == AggregateMode.PARTIAL) {
-                factory.createPartialAggregateNode().ifPresent(instructions::add);
-            }
-        } else if (leaf instanceof OpenSearchStageInputScan) {
-            factory.createFinalAggregateNode().ifPresent(instructions::add);
         }
-
         return instructions;
     }
 
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateReduceRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateReduceRule.java
index 95d1c32f35852..8965b38c5a9f0 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateReduceRule.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggregateReduceRule.java
@@ -32,18 +32,28 @@
  * primitive aggregate calls, and the Volcano split rule downstream operates on those
  * primitives.
  *
- * 
              Reduction set: narrowed via {@code FUNCTIONS_TO_REDUCE} to AVG only. AVG's
- * reduction uses only SUM, COUNT, DIVIDE, and CAST — all either capability-declared
- * aggregates or baseline scalar operators ({@link OpenSearchProjectRule#BASELINE_SCALAR_OPS}).
- * STDDEV_POP / STDDEV_SAMP / VAR_POP / VAR_SAMP also emit {@code POWER(x, 2)} which is
- * not in the baseline set; extending the set is a one-line change once a backend declares
- * POWER as a project capability.
+ * 
              Reduction set: {@code AVG} + {@code STDDEV_POP}/{@code VAR_POP} +
+ * {@code STDDEV_SAMP}/{@code VAR_SAMP}. AVG reduces to SUM/COUNT/DIVIDE/CAST.
+ * STDDEV/VAR additionally emit {@code MULTIPLY} (for {@code x*x}) and
+ * {@code POWER(variance, 0.5)} (sqrt). The {@code SAMP} variants also emit a
+ * {@code CASE WHEN count > 1 THEN sqrt(variance) ELSE NULL END} Bessel's-correction
+ * guard — the {@code >} comparison operator is in
+ * {@link OpenSearchProjectRule#BASELINE_SCALAR_OPS} so it flows through without being
+ * wrapped in {@code AnnotatedProjectExpression}. All emitted aggregates are
+ * SUM/COUNT primitives that the resolver decomposes through the standard single-field
+ * path.
  *
  * @opensearch.internal
  */
 public class OpenSearchAggregateReduceRule extends AggregateReduceFunctionsRule {
 
-    private static final EnumSet FUNCTIONS_TO_REDUCE = EnumSet.of(SqlKind.AVG);
+    private static final EnumSet FUNCTIONS_TO_REDUCE = EnumSet.of(
+        SqlKind.AVG,
+        SqlKind.STDDEV_POP,
+        SqlKind.STDDEV_SAMP,
+        SqlKind.VAR_POP,
+        SqlKind.VAR_SAMP
+    );
 
     public OpenSearchAggregateReduceRule() {
         super(LogicalAggregate.class, RelBuilder.proto(Contexts.empty()), FUNCTIONS_TO_REDUCE);
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java
index f6767e7d06616..32521867a2736 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchProjectRule.java
@@ -82,6 +82,16 @@ public class OpenSearchProjectRule extends RelOptRule {
         SqlStdOperatorTable.DIVIDE,
         SqlStdOperatorTable.UNARY_MINUS,
         SqlStdOperatorTable.UNARY_PLUS,
+        // Math (emitted by Calcite's AggregateReduceFunctionsRule for STDDEV: POWER(v, 0.5) = sqrt)
+        SqlStdOperatorTable.POWER,
+        // Comparison (emitted by Calcite's AggregateReduceFunctionsRule for STDDEV_SAMP / VAR_SAMP:
+        // CASE WHEN count > 1 THEN sqrt(variance) ELSE NULL END — Bessel's correction guard)
+        SqlStdOperatorTable.GREATER_THAN,
+        SqlStdOperatorTable.GREATER_THAN_OR_EQUAL,
+        SqlStdOperatorTable.LESS_THAN,
+        SqlStdOperatorTable.LESS_THAN_OR_EQUAL,
+        SqlStdOperatorTable.EQUALS,
+        SqlStdOperatorTable.NOT_EQUALS,
         // Type coercion
         SqlStdOperatorTable.CAST,
         // Null handling
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/AggregateRuleTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/AggregateRuleTests.java
index 5398fc2e17ef6..f295adfc21cd6 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/AggregateRuleTests.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/AggregateRuleTests.java
@@ -18,6 +18,7 @@
 import org.opensearch.analytics.planner.rel.OpenSearchAggregate;
 import org.opensearch.analytics.planner.rel.OpenSearchExchangeReducer;
 import org.opensearch.analytics.planner.rel.OpenSearchFilter;
+import org.opensearch.analytics.planner.rel.OpenSearchProject;
 import org.opensearch.analytics.planner.rel.OpenSearchTableScan;
 import org.opensearch.analytics.spi.AggregateCapability;
 import org.opensearch.analytics.spi.AggregateFunction;
@@ -240,24 +241,35 @@ protected Set acceptedDelegations() {
         PlannerContext context = buildContext("parquet", 1, intFields(), List.of(dfWithDelegation, luceneAccepting));
         RelNode result = runPlanner(makeMultiCallAggregate(sumCall(), stddevCall()), context);
         logger.info("Plan:\n{}", RelOptUtil.toString(result));
+        // OpenSearchAggregateReduceRule decomposes STDDEV_POP into SUM+COUNT wrapped in
+        // Project(sqrt) above / Project(squared-inputs) below the Aggregate.
         assertPipelineViableBackends(
             result,
-            List.of(OpenSearchAggregate.class, OpenSearchTableScan.class),
+            List.of(OpenSearchProject.class, OpenSearchAggregate.class, OpenSearchProject.class, OpenSearchTableScan.class),
             Set.of(MockDataFusionBackend.NAME)
         );
     }
 
     public void testAggregateErrorsWithoutDelegation() {
-        MockLuceneBackend luceneWithStddev = new MockLuceneBackend() {
+        // DF declares only COUNT — can't satisfy STDDEV_POP's reduction (needs SUM(x) and
+        // SUM(x*x)) on its own. Lucene has SUM but refuses delegation.
+        MockDataFusionBackend dfNoSum = new MockDataFusionBackend() {
             @Override
             protected Set aggregateCapabilities() {
                 return aggCaps(
-                    Set.of(MockLuceneBackend.LUCENE_DATA_FORMAT),
-                    Map.of(AggregateFunction.STDDEV_POP, Set.of(FieldType.INTEGER))
+                    Set.of(MockDataFusionBackend.PARQUET_DATA_FORMAT),
+                    Map.of(AggregateFunction.COUNT, Set.of(FieldType.INTEGER))
                 );
             }
         };
-        PlannerContext context = buildContext("parquet", 1, intFields(), List.of(DATAFUSION, luceneWithStddev));
+        MockLuceneBackend luceneWithSum = new MockLuceneBackend() {
+            @Override
+            protected Set aggregateCapabilities() {
+                return aggCaps(Set.of(MockLuceneBackend.LUCENE_DATA_FORMAT), Map.of(AggregateFunction.SUM, Set.of(FieldType.INTEGER)));
+            }
+            // No acceptedDelegations() override → delegation is refused.
+        };
+        PlannerContext context = buildContext("parquet", 1, intFields(), List.of(dfNoSum, luceneWithSum));
         IllegalStateException exception = expectThrows(
             IllegalStateException.class,
             () -> runPlanner(makeMultiCallAggregate(sumCall(), stddevCall()), context)
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java
index 47bb0cc88b3a1..7ec595d835cbc 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/ProjectRuleTests.java
@@ -262,8 +262,8 @@ protected Set projectCapabilities() {
     // ---- Nested expressions ----
 
     public void testNestedScalarFunctions() {
-        // POWER(CEIL(v_int), v_int) — outer and inner both capability-declared scalars so
-        // annotation happens at both levels. CAST / PLUS are baseline scalars (see
+        // FLOOR(CEIL(v_int)) — outer and inner both capability-declared scalars so
+        // annotation happens at both levels. CAST / PLUS / POWER are baseline scalars (see
         // OpenSearchProjectRule.BASELINE_SCALAR_OPS) and are deliberately not used here
         // because they bypass capability enforcement and would not produce an
         // AnnotatedProjectExpression.
@@ -271,36 +271,32 @@ public void testNestedScalarFunctions() {
             SqlStdOperatorTable.CEIL,
             rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1)
         );
-        RexNode powerExpr = rexBuilder.makeCall(
-            SqlStdOperatorTable.POWER,
-            ceilExpr,
-            rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1)
-        );
-        OpenSearchProject result = runProject(powerExpr);
+        RexNode outerExpr = rexBuilder.makeCall(SqlStdOperatorTable.FLOOR, ceilExpr);
+        OpenSearchProject result = runProject(outerExpr);
         assertTrue(result.getViableBackends().contains(MockDataFusionBackend.NAME));
         assertAnnotation(result.getProjects().get(0), MockDataFusionBackend.NAME);
     }
 
     public void testStripAnnotationsRecursivelyUnwrapsNestedExpressions() {
-        // POWER(CEIL(value), value) — a non-baseline scalar call with another non-baseline
+        // FLOOR(CEIL(value)) — a non-baseline scalar call with another non-baseline
         // scalar call as an operand. The project rule recurses into operands
-        // (annotateExpr), so both POWER and the inner CEIL get wrapped in
+        // (annotateExpr), so both FLOOR and the inner CEIL get wrapped in
         // AnnotatedProjectExpression. stripAnnotations must remove every wrapper at every
         // depth before the plan reaches the backend FragmentConvertor — Substrait isthmus
         // has no converter for ANNOTATED_PROJECT_EXPR and would throw "Unable to convert
         // call".
         //
-        // PLUS was used previously but is baseline (see OpenSearchProjectRule
-        // .BASELINE_SCALAR_OPS); POWER preserves the nested-call-with-nested-annotation
-        // structure this test exercises while still going through capability resolution.
+        // PLUS / POWER are baseline (see OpenSearchProjectRule.BASELINE_SCALAR_OPS), so
+        // this test uses FLOOR+CEIL to preserve the nested-call-with-nested-annotation
+        // structure while still going through capability resolution.
         RexNode value = rexBuilder.makeInputRef(typeFactory.createSqlType(SqlTypeName.INTEGER), 1);
         RexNode ceilCall = rexBuilder.makeCall(SqlStdOperatorTable.CEIL, value);
-        RexNode powerCall = rexBuilder.makeCall(SqlStdOperatorTable.POWER, ceilCall, value);
-        OpenSearchProject annotated = runProject(powerCall);
+        RexNode floorCall = rexBuilder.makeCall(SqlStdOperatorTable.FLOOR, ceilCall);
+        OpenSearchProject annotated = runProject(floorCall);
 
         // Sanity: confirm the rule produced the nested-wrapper shape this test exercises.
         RexNode topLevel = annotated.getProjects().get(0);
-        assertTrue("Outer POWER must be annotated", topLevel instanceof AnnotatedProjectExpression);
+        assertTrue("Outer FLOOR must be annotated", topLevel instanceof AnnotatedProjectExpression);
         RexCall outerOriginal = (RexCall) ((AnnotatedProjectExpression) topLevel).getOriginal();
         assertTrue(
             "Inner CEIL must also be annotated (recursive annotateExpr behavior)",
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java
index db56441fd3311..9c7b93b4cd446 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/planner/dag/FragmentConversionDriverTests.java
@@ -124,14 +124,12 @@ private void assertReduceStageConverted(RecordingConvertor convertor, Stage stag
         assertTrue("convertFinalAggFragment must be called", convertor.finalAggCalled);
         assertDoesntContainOperators(convertor.reduceFragment, OPENSEARCH_OPERATORS);
         assertDoesntContainOperators(convertor.reduceFragment, ANNOTATION_MARKERS);
-        // Instruction assertions
+        // Coord-side reduce stages no longer register FinalAggregateInstructionHandler.
+        // DataFusion plans the substrait Aggregate's Partial+Final pair itself via the legacy
+        // executeLocalPlan path; the previous SETUP_FINAL_AGGREGATE instruction routed through
+        // Rust's apply_aggregate_mode strip, which corrupted column refs (cnt[sum]/cnt[count]).
         StagePlan plan = stage.getPlanAlternatives().getFirst();
-        assertFalse("instructions must not be empty", plan.instructions().isEmpty());
-        assertEquals(
-            "reduce stage must have FINAL_AGGREGATE",
-            InstructionType.SETUP_FINAL_AGGREGATE,
-            plan.instructions().getFirst().type()
-        );
+        assertTrue("coord-side reduce instructions must be empty", plan.instructions().isEmpty());
     }
 
     // ---- Single-stage query shapes ----
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java
index 322c6957810f1..b31d8dd83b40b 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java
@@ -14,8 +14,10 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Self-contained integration test for PPL {@code appendpipe} on the analytics-engine route.
@@ -55,21 +57,42 @@ private void ensureDataProvisioned() throws IOException {
 
     public void testAppendPipeSort() throws IOException {
         // Branch: stats sum(int0) by str0 → 3 rows (FURNITURE=1, OFFICE SUPPLIES=18, TECHNOLOGY=49).
-        // Outer `sort str0` pins the original to alphabetical order. `appendpipe [sort -sum_int0_by_str0]`
-        // duplicates the 3 rows and re-sorts them descending, then appends. `head 5` keeps the first
-        // 5 of the 6 total rows: original 3 + first 2 of the descending duplicate.
-        assertRows(
+        // `appendpipe [sort -sum_int0_by_str0]` duplicates them desc-sorted and appends. `head 5`
+        // keeps the first 5 of the 6 total rows. Branch arrival order at the union is
+        // non-deterministic (each is its own streaming stage), so `head 5` drops a different
+        // row depending on which branch arrives first. Assert the shape instead:
+        //  - total 5 rows
+        //  - at least one asc branch is fully represented (3 rows) and the other contributes 2.
+        // The concrete invariant: the distinct buckets FURNITURE/OFFICE SUPPLIES/TECHNOLOGY all
+        // appear, and the two branches' rows are identical modulo ordering, so the multiset
+        // count of each bucket is at least 1 and no bucket count exceeds 2.
+        List> actual = getRows(
             "source="
                 + DATASET.indexName
                 + " | stats sum(int0) as sum_int0_by_str0 by str0 | sort str0"
                 + " | appendpipe [ sort -sum_int0_by_str0 ]"
-                + " | head 5",
-            row(1, "FURNITURE"),
-            row(18, "OFFICE SUPPLIES"),
-            row(49, "TECHNOLOGY"),
-            row(49, "TECHNOLOGY"),
-            row(18, "OFFICE SUPPLIES")
+                + " | head 5"
         );
+        assertEquals("head 5 must return 5 rows", 5, actual.size());
+        Map bucketCounts = new HashMap<>();
+        for (List r : actual) {
+            String bucket = (String) r.get(1);
+            bucketCounts.merge(bucket, 1, Integer::sum);
+        }
+        assertEquals(
+            "all three buckets must appear",
+            Set.of("FURNITURE", "OFFICE SUPPLIES", "TECHNOLOGY"),
+            bucketCounts.keySet()
+        );
+        for (Map.Entry e : bucketCounts.entrySet()) {
+            assertTrue("bucket " + e.getKey() + " count out of range: " + e.getValue(), e.getValue() >= 1 && e.getValue() <= 2);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private List> getRows(String ppl) throws IOException {
+        Map response = executePpl(ppl);
+        return (List>) response.get("rows");
     }
 
     // ── duplicate + inline stats producing a smaller schema (merged column) ─────
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java
index 7d8ef92626238..6434f17f220e4 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultisearchCommandIT.java
@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
@@ -131,7 +130,6 @@ public void testMultisearchEvalCaseProjection() throws IOException {
 
     // ── CASE with implicit ELSE NULL — `count(eval(predicate))` shape ──────────
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/21457")
     public void testMultisearchCountEvalConditionalCount() throws IOException {
         // Mirror of the v2-side `CalciteMultisearchCommandIT.testMultisearchSuccessRatePattern`:
         // `count(eval(predicate))` is PPL's conditional-count idiom. Calcite lowers it to
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java
index 4f10deed8c8ad..04d4f79173e35 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ObjectFieldIT.java
@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
 
@@ -65,7 +64,6 @@ public void testSelectDeeplyNestedObjectField() throws IOException {
         );
     }
 
-    @AwaitsFix(bugUrl = "Requires sql repo fix to CalciteRelNodeVisitor.containsNestedAggregator (try/catch around relBuilder.field)")
     public void testMinOnObjectField() throws IOException {
         assertRowsEqual(
             "source=" + DATASET.indexName + " | stats min(account.balance)",
@@ -73,7 +71,6 @@ public void testMinOnObjectField() throws IOException {
         );
     }
 
-    @AwaitsFix(bugUrl = "Requires sql repo fix to CalciteRelNodeVisitor.containsNestedAggregator (try/catch around relBuilder.field)")
     public void testMaxOnDeeplyNestedObjectField() throws IOException {
         assertRowsEqual(
             "source=" + DATASET.indexName + " | stats max(city.location.latitude)",
@@ -81,7 +78,6 @@ public void testMaxOnDeeplyNestedObjectField() throws IOException {
         );
     }
 
-    @AwaitsFix(bugUrl = "Requires sql repo fix to CalciteRelNodeVisitor.containsNestedAggregator (try/catch around relBuilder.field)")
     public void testSumOnObjectField() throws IOException {
         assertRowsEqual(
             "source=" + DATASET.indexName + " | stats sum(city.population)",
diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java
index f3133e3c7e2e4..c5780e9f797ea 100644
--- a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java
+++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/StreamingCoordinatorReduceIT.java
@@ -13,6 +13,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.function.IntUnaryOperator;
 
 /**
  * Streaming variant of {@link CoordinatorReduceIT}: same 2-shard parquet-backed index and
@@ -61,6 +62,193 @@ public void testBaselineScanAcrossShards() throws Exception {
         }
     }
 
+    /**
+     * {@code stats avg(value) as a} — primitive decomposition. PARTIAL emits
+     * {@code [count:Int64, sum:Float64]}; FINAL reduces each with SUM and a Project wraps
+     * {@code finalExpression = sum/count}. Exercises the multi-field intermediate path over
+     * the streaming reduce-sink: each shard ships sum + count intermediates via Flight, the
+     * coordinator merges them, then divides.
+     *
+     * 
              Uses varied per-doc values (value = doc index) so the AVG is non-trivial — a
+     * per-shard pass-through (e.g. concatenating partial AVGs) would yield a different
+     * answer than the correct cross-shard merge.
+     */
+    public void testAvgAcrossShards() throws Exception {
+        createParquetBackedIndex();
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
+        indexValuedDocs(i -> i);
+
+        // Expected: AVG(0, 1, ..., total-1) = (total - 1) / 2.0
+        double expected = (total - 1) / 2.0;
+
+        Map result = executePPL("source = " + INDEX + " | stats avg(value) as a");
+        List> rows = scalarRows(result, "a");
+
+        double actual = ((Number) rows.get(0).get(0)).doubleValue();
+        assertEquals("AVG(value) across shards should be " + expected, expected, actual, 0.001);
+    }
+
+    /**
+     * {@code stats dc(value) as dc} — engine-native HLL merge. PARTIAL emits a single Binary
+     * sketch column per shard; FINAL invokes DataFusion's {@code approx_distinct} merge
+     * which combines sketches across shards. Exercises the engine-native (reducer == self)
+     * single-field intermediate path over streaming.
+     *
+     * 
              Tolerance is 10% — HLL is approximate; with 20 distinct values the error margin
+     * easily covers the variance.
+     */
+    public void testDistinctCountAcrossShards() throws Exception {
+        createParquetBackedIndex();
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
+        indexValuedDocs(i -> i); // all distinct
+
+        Map result = executePPL("source = " + INDEX + " | stats dc(value) as dc");
+        List> rows = scalarRows(result, "dc");
+
+        long actual = ((Number) rows.get(0).get(0)).longValue();
+        assertTrue(
+            "dc(value) should be approximately " + total + " (±10%), got " + actual,
+            actual >= (long) (total * 0.9) && actual <= (long) (total * 1.1)
+        );
+    }
+
+    /**
+     * {@code stats stddev_pop(value) as s} — multi-field statistical aggregate. Reduced by
+     * {@link org.opensearch.analytics.planner.rules.OpenSearchAggregateReduceRule} into
+     * SUM, SUM-of-squares, and COUNT primitives at HEP-marking time, then finalised with
+     * POWER(variance, 0.5).
+     *
+     * 
              Expected: population stddev of (0..19) = sqrt(33.25) ≈ 5.766.
+     */
+    public void testStddevPopAcrossShards() throws Exception {
+        createParquetBackedIndex();
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
+        indexValuedDocs(i -> i);
+
+        double mean = (total - 1) / 2.0;
+        double sumSquares = 0;
+        for (int i = 0; i < total; i++) {
+            sumSquares += (i - mean) * (i - mean);
+        }
+        double expected = Math.sqrt(sumSquares / total);
+
+        Map result = executePPL("source = " + INDEX + " | stats stddev_pop(value) as s");
+        List> rows = scalarRows(result, "s");
+
+        double actual = ((Number) rows.get(0).get(0)).doubleValue();
+        assertEquals("STDDEV_POP(value) across shards should be " + expected, expected, actual, 0.001);
+    }
+
+    /**
+     * {@code stats stddev_samp(value) as s} — sample standard deviation. Reduced to
+     * {@code sqrt(SUM((x - mean)^2) / (N - 1))}. Same reduction path as STDDEV_POP but
+     * with Bessel's correction in the denominator.
+     *
+     * 
              Expected: sample stddev of (0..19) = sqrt(sum((i - mean)^2) / (N - 1)) = sqrt(35) ≈ 5.916.
+     */
+    public void testStddevSampAcrossShards() throws Exception {
+        createParquetBackedIndex();
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
+        indexValuedDocs(i -> i);
+
+        double mean = (total - 1) / 2.0;
+        double sumSquares = 0;
+        for (int i = 0; i < total; i++) {
+            sumSquares += (i - mean) * (i - mean);
+        }
+        double expected = Math.sqrt(sumSquares / (total - 1));
+
+        Map result = executePPL("source = " + INDEX + " | stats stddev_samp(value) as s");
+        List> rows = scalarRows(result, "s");
+
+        double actual = ((Number) rows.get(0).get(0)).doubleValue();
+        assertEquals("STDDEV_SAMP(value) across shards should be " + expected, expected, actual, 0.001);
+    }
+
+    /**
+     * {@code stats var_pop(value) as v} — population variance. Reduced to
+     * {@code SUM((x - mean)^2) / N}, the same primitives as STDDEV_POP minus the final sqrt.
+     *
+     * 
              Expected: population variance of (0..19) = 33.25.
+     */
+    public void testVarPopAcrossShards() throws Exception {
+        createParquetBackedIndex();
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
+        indexValuedDocs(i -> i);
+
+        double mean = (total - 1) / 2.0;
+        double sumSquares = 0;
+        for (int i = 0; i < total; i++) {
+            sumSquares += (i - mean) * (i - mean);
+        }
+        double expected = sumSquares / total;
+
+        Map result = executePPL("source = " + INDEX + " | stats var_pop(value) as v");
+        List> rows = scalarRows(result, "v");
+
+        double actual = ((Number) rows.get(0).get(0)).doubleValue();
+        assertEquals("VAR_POP(value) across shards should be " + expected, expected, actual, 0.001);
+    }
+
+    /**
+     * {@code stats var_samp(value) as v} — sample variance. Reduced to
+     * {@code SUM((x - mean)^2) / (N - 1)}.
+     *
+     * 
              Expected: sample variance of (0..19) = 35.0.
+     */
+    public void testVarSampAcrossShards() throws Exception {
+        createParquetBackedIndex();
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
+        indexValuedDocs(i -> i);
+
+        double mean = (total - 1) / 2.0;
+        double sumSquares = 0;
+        for (int i = 0; i < total; i++) {
+            sumSquares += (i - mean) * (i - mean);
+        }
+        double expected = sumSquares / (total - 1);
+
+        Map result = executePPL("source = " + INDEX + " | stats var_samp(value) as v");
+        List> rows = scalarRows(result, "v");
+
+        double actual = ((Number) rows.get(0).get(0)).doubleValue();
+        assertEquals("VAR_SAMP(value) across shards should be " + expected, expected, actual, 0.001);
+    }
+
+    /** Indexes {@code NUM_SHARDS * DOCS_PER_SHARD} docs with values produced by {@code valueFn}. */
+    private void indexValuedDocs(IntUnaryOperator valueFn) throws Exception {
+        int total = NUM_SHARDS * DOCS_PER_SHARD;
+        StringBuilder bulk = new StringBuilder();
+        for (int i = 0; i < total; i++) {
+            bulk.append("{\"index\": {\"_id\": \"").append(i).append("\"}}\n");
+            bulk.append("{\"value\": ").append(valueFn.applyAsInt(i)).append("}\n");
+        }
+
+        Request bulkRequest = new Request("POST", "/" + INDEX + "/_bulk");
+        bulkRequest.setJsonEntity(bulk.toString());
+        bulkRequest.addParameter("refresh", "true");
+        client().performRequest(bulkRequest);
+
+        client().performRequest(new Request("POST", "/" + INDEX + "/_flush?force=true"));
+    }
+
+    /** Local copy of {@code CoordinatorReduceIT.scalarRows} (the original is package-private). */
+    private static List> scalarRows(Map result, String columnName) {
+        @SuppressWarnings("unchecked")
+        List columns = (List) result.get("columns");
+        assertNotNull("columns must not be null", columns);
+        assertTrue("columns must contain '" + columnName + "', got " + columns, columns.contains(columnName));
+
+        @SuppressWarnings("unchecked")
+        List> rows = (List>) result.get("rows");
+        assertNotNull("rows must not be null", rows);
+        assertEquals("scalar agg must return exactly 1 row", 1, rows.size());
+
+        Object cell = rows.get(0).get(columns.indexOf(columnName));
+        assertNotNull("cell for '" + columnName + "' must not be null — coordinator-reduce returned no value", cell);
+        return rows;
+    }
+
     private void createParquetBackedIndex() throws Exception {
         try {
             client().performRequest(new Request("DELETE", "/" + INDEX));

From 14bcd6dede12bf7c4b5640e01fef9dae0006ff8c Mon Sep 17 00:00:00 2001
From: Marc Handalian 
Date: Tue, 12 May 2026 11:30:11 -0700
Subject: [PATCH 115/115] explain

Signed-off-by: Marc Handalian 
---
 .../analytics/exec/DefaultPlanExecutor.java   | 100 +++++++++
 .../analytics/exec/QueryContext.java          |  13 +-
 .../analytics/exec/QueryScheduler.java        |  43 ++++
 .../exec/profile/ProfiledResult.java          |  24 ++
 .../analytics/exec/profile/QueryProfile.java  |  51 +++++
 .../exec/profile/QueryProfileBuilder.java     | 150 +++++++++++++
 .../analytics/exec/profile/StageProfile.java  |  78 +++++++
 .../analytics/exec/profile/TaskProfile.java   |  45 ++++
 .../stage/ShardFragmentStageExecution.java    |  47 ++--
 .../analytics/exec/stage/StageExecution.java  |   6 +
 .../analytics/exec/stage/StageTask.java       |  86 ++++++++
 .../analytics/exec/stage/StageTaskId.java     |  23 ++
 .../analytics/exec/stage/StageTaskState.java  |  33 +++
 .../analytics/exec/stage/TaskTracker.java     |  57 +++++
 .../exec/QueryProfileBuilderTests.java        | 208 ++++++++++++++++++
 .../ShardFragmentStageExecutionTests.java     |   2 +-
 .../exec/stage/TaskTrackerTests.java          | 124 +++++++++++
 17 files changed, 1072 insertions(+), 18 deletions(-)
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/ProfiledResult.java
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/QueryProfile.java
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/QueryProfileBuilder.java
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/StageProfile.java
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/TaskProfile.java
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTask.java
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTaskId.java
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTaskState.java
 create mode 100644 sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/TaskTracker.java
 create mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/QueryProfileBuilderTests.java
 create mode 100644 sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/TaskTrackerTests.java

diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
index bf5ba86038101..2ef07de8067d2 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
@@ -20,6 +20,9 @@
 import org.opensearch.action.support.TimeoutTaskCancellationUtility;
 import org.opensearch.analytics.EngineContext;
 import org.opensearch.analytics.exec.action.AnalyticsQueryAction;
+import org.opensearch.analytics.exec.profile.ProfiledResult;
+import org.opensearch.analytics.exec.profile.QueryProfile;
+import org.opensearch.analytics.exec.profile.QueryProfileBuilder;
 import org.opensearch.analytics.exec.task.AnalyticsQueryTask;
 import org.opensearch.analytics.planner.CapabilityRegistry;
 import org.opensearch.analytics.planner.PlannerContext;
@@ -114,6 +117,22 @@ public void execute(RelNode logicalFragment, Object context, ActionListener listener) {
+        searchExecutor.execute(() -> {
+            try {
+                executeInternalWithProfile(logicalFragment, listener);
+            } catch (Exception e) {
+                listener.onFailure(e);
+            }
+        });
+    }
+
     /**
      * Plans, registers the query task, and dispatches to the {@link Scheduler}. Runs on
      * the SEARCH thread pool — never on a transport thread. The result (or failure) is
@@ -175,6 +194,87 @@ private void executeInternal(RelNode logicalFragment, ActionListener listener) {
+        RelMetadataQueryBase.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(logicalFragment.getCluster().getMetadataProvider()));
+        logicalFragment.getCluster().invalidateMetadataQuery();
+
+        RelNode plan = PlannerImpl.createPlan(logicalFragment, new PlannerContext(capabilityRegistry, clusterService.state()));
+        // Capture the unified CBO output before DAGBuilder cuts it at exchange boundaries.
+        // This is what gets rendered in the "full_plan" field of the profile — users see
+        // the single plan tree the planner actually chose, annotated with backend decisions.
+        final String fullPlan = org.apache.calcite.plan.RelOptUtil.toString(plan);
+        QueryDAG dag = DAGBuilder.build(plan, capabilityRegistry, clusterService);
+        PlanForker.forkAll(dag, capabilityRegistry);
+        BackendPlanAdapter.adaptAll(dag, capabilityRegistry);
+        AggregateDecompositionResolver.resolveAll(dag, capabilityRegistry);
+        FragmentConversionDriver.convertAll(dag, capabilityRegistry);
+        logger.debug("[DefaultPlanExecutor] QueryDAG:\n{}", dag);
+
+        final AnalyticsQueryTask queryTask = (AnalyticsQueryTask) taskManager.register(
+            "transport",
+            "analytics_query",
+            new AnalyticsQueryTaskRequest(dag.queryId(), null)
+        );
+        final QueryContext config = new QueryContext(dag, searchExecutor, queryTask);
+
+        // Scheduler variant that exposes the walker so we can read its ExecutionGraph
+        // after the listener chain runs. The graph object outlives walkerPool removal —
+        // the pool carries a reference, not the only reference.
+        if (!(scheduler instanceof QueryScheduler)) {
+            listener.onFailure(
+                new UnsupportedOperationException(
+                    "executeWithProfile requires QueryScheduler — got " + scheduler.getClass().getSimpleName()
+                )
+            );
+            return;
+        }
+        final QueryScheduler qs = (QueryScheduler) scheduler;
+        final PlanWalker[] walkerRef = new PlanWalker[1];
+
+        // The batches listener converts VSRs -> rows, runs cleanup, then snapshots the
+        // profile. Both success and failure deliver a ProfiledResult via onResponse so
+        // the caller always gets the profile; the failure case carries the cause on
+        // ProfiledResult.failure and leaves rows null.
+        ActionListener> rowsListener = ActionListener.wrap(rows -> {
+            QueryProfile profile = QueryProfileBuilder.snapshot(walkerRef[0].getGraph(), config, fullPlan);
+            listener.onResponse(new ProfiledResult(rows, null, profile));
+        }, e -> {
+            QueryProfile profile = walkerRef[0] != null && walkerRef[0].getGraph() != null
+                ? QueryProfileBuilder.snapshot(walkerRef[0].getGraph(), config, fullPlan)
+                : new QueryProfile(config.queryId(), java.util.List.of(), 0L, java.util.List.of());
+            listener.onResponse(new ProfiledResult(null, e, profile));
+        });
+
+        ActionListener> batchesListener = buildBatchesListener(rowsListener, () -> {
+            try {
+                config.closeBufferAllocator();
+            } finally {
+                taskManager.unregister(queryTask);
+            }
+        });
+
+        TimeValue taskTimeout = queryTask.getCancelAfterTimeInterval();
+        TimeValue clusterTimeout = clusterService.getClusterSettings().get(SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING);
+        if (taskTimeout != null || SearchService.NO_TIMEOUT.equals(clusterTimeout) == false) {
+            batchesListener = TimeoutTaskCancellationUtility.wrapWithCancellationListener(
+                client,
+                queryTask,
+                clusterTimeout,
+                batchesListener,
+                e -> {}
+            );
+        }
+
+        walkerRef[0] = qs.executeAndReturnWalker(config, batchesListener);
+    }
+
     @Override
     protected void doExecute(Task task, ActionRequest request, ActionListener listener) {
         // Transport path — reserved for future remote query invocation.
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryContext.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryContext.java
index cd279a6ba4301..770080c75afa1 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryContext.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryContext.java
@@ -10,6 +10,7 @@
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.opensearch.analytics.backend.AnalyticsOperationListener;
+import org.opensearch.analytics.exec.stage.TaskTracker;
 import org.opensearch.analytics.exec.task.AnalyticsQueryTask;
 import org.opensearch.analytics.planner.dag.QueryDAG;
 import org.opensearch.arrow.flight.transport.ArrowAllocatorProvider;
@@ -42,6 +43,7 @@ public class QueryContext {
     private final int maxConcurrentShardRequests;
     private final long perQueryMemoryLimit;
     private final List operationListeners;
+    private final TaskTracker taskTracker = new TaskTracker();
     private volatile BufferAllocator bufferAllocator;
     private boolean closed;  // guarded by `this`
 
@@ -104,6 +106,15 @@ public List operationListeners() {
         return operationListeners;
     }
 
+    /**
+     * Per-query registry of every {@link org.opensearch.analytics.exec.stage.StageTask}
+     * across all stages. Populated by stage executions as they materialise their task
+     * lists at dispatch time; consumed by the scheduler to compute stage readiness.
+     */
+    public TaskTracker taskTracker() {
+        return taskTracker;
+    }
+
     /**
      * Returns the per-query Arrow buffer allocator, creating it lazily on first access.
      * The allocator is a child of the shared root with a per-query memory limit.
@@ -144,8 +155,6 @@ public void closeBufferAllocator() {
         }
     }
 
-    // ─── Test factories ────────────────────────────────────────────────
-
     /** Creates a test context with a synchronous executor. */
     public static QueryContext forTest(QueryDAG dag, AnalyticsQueryTask parentTask) {
         return new QueryContext(dag, Runnable::run, parentTask, DEFAULT_MAX_CONCURRENT_SHARD_REQUESTS, Long.MAX_VALUE);
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryScheduler.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryScheduler.java
index a32b98c452b1b..648be420289e9 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryScheduler.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/QueryScheduler.java
@@ -87,6 +87,37 @@ public void execute(QueryContext config, ActionListener> listener) {
+        final String queryId = config.queryId();
+        final long queryStartNanos = System.nanoTime();
+        final AnalyticsOperationListener.CompositeListener opListener = new AnalyticsOperationListener.CompositeListener(
+            config.operationListeners()
+        );
+
+        PlanWalker walker = createWalker(config, listener, queryId, queryStartNanos, opListener);
+        walkerPool.put(queryId, walker);
+
+        final AnalyticsQueryTask queryTask = config.parentTask();
+        queryTask.setOnCancelCallback(() -> {
+            String reason = "task cancelled: " + (queryTask.getReasonCancelled() != null ? queryTask.getReasonCancelled() : "unknown");
+            logger.info("[QueryScheduler] AnalyticsQueryTask.onCancelled fired, reason={}", reason);
+            walker.cancelAll(reason);
+        });
+
+        ExecutionGraph graph = walker.build();
+        opListener.onQueryStart(queryId, graph.stageCount());
+        logger.info("[QueryScheduler] ExecutionGraph built:\n{}", graph.explain());
+        walker.start(graph);
+        return walker;
+    }
+
     private PlanWalker createWalker(
         QueryContext config,
         ActionListener> listener,
@@ -106,6 +137,18 @@ private PlanWalker createWalker(
         return new PlanWalker(config, stageExecutionBuilder, wrapped);
     }
 
+    /**
+     * Returns the underlying {@link StageExecutionBuilder} so callers can register a
+     * custom {@link org.opensearch.analytics.exec.stage.StageScheduler} for a stage
+     * type (e.g. fault-injecting scheduler in resilience tests). Resolving via the
+     * singleton scheduler avoids a Guice JIT lookup that would re-instantiate
+     * {@link AnalyticsSearchTransportService} (whose ctor registers transport
+     * handlers, only legal once per node).
+     */
+    public StageExecutionBuilder getStageExecutionBuilder() {
+        return stageExecutionBuilder;
+    }
+
     /** Pool-level lookup for observability / metrics. */
     public PlanWalker walkerFor(String queryId) {
         return walkerPool.get(queryId);
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/ProfiledResult.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/ProfiledResult.java
new file mode 100644
index 0000000000000..ea54ece828298
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/ProfiledResult.java
@@ -0,0 +1,24 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.exec.profile;
+
+/**
+ * Pair of query result rows and the captured {@link QueryProfile}. Returned by
+ * {@code DefaultPlanExecutor.executeWithProfile} on every terminal path —
+ * success and failure — so callers always receive the profile regardless of outcome.
+ *
+ * @param rows    materialised query result rows, or null if the query failed
+ * @param failure the cause if the query failed, or null on success
+ * @param profile per-stage + per-task profile snapshot, never null
+ */
+public record ProfiledResult(Iterable rows, Throwable failure, QueryProfile profile) {
+    public boolean isSuccess() {
+        return failure == null;
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/QueryProfile.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/QueryProfile.java
new file mode 100644
index 0000000000000..c423700a1bd16
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/QueryProfile.java
@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.exec.profile;
+
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Query-level profile snapshot built from an execution graph plus the per-query
+ * {@code TaskTracker}. Safe to emit on both success and failure paths — every field
+ * is a plain value captured at snapshot time, not a live handle into the walker.
+ *
+ * @param queryId        per-query id from {@code QueryDAG.queryId()}
+ * @param fullPlan       the CBO-output Calcite plan rendered as an array of lines,
+ *                       captured before the DAG builder cut it at exchange boundaries;
+ *                       one element per indent level of the tree. Empty list if not supplied.
+ * @param totalElapsedMs wall-clock span from the earliest stage start to the latest stage end (0 if nothing ran)
+ * @param stages         per-stage profiles in DAG iteration order (root stage appears at whatever index the walker stored it)
+ */
+public record QueryProfile(String queryId, List fullPlan, long totalElapsedMs, List stages)
+    implements
+        ToXContentObject {
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("query_id", queryId);
+        if (fullPlan != null && fullPlan.isEmpty() == false) {
+            builder.startArray("full_plan");
+            for (String line : fullPlan) builder.value(line);
+            builder.endArray();
+        }
+        builder.field("total_elapsed_ms", totalElapsedMs);
+        builder.startArray("stages");
+        for (StageProfile s : stages) {
+            s.toXContent(builder, params);
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/QueryProfileBuilder.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/QueryProfileBuilder.java
new file mode 100644
index 0000000000000..1d00885959aa5
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/QueryProfileBuilder.java
@@ -0,0 +1,150 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.exec.profile;
+
+import org.apache.calcite.plan.RelOptUtil;
+import org.opensearch.analytics.exec.ExecutionGraph;
+import org.opensearch.analytics.exec.QueryContext;
+import org.opensearch.analytics.exec.stage.StageExecution;
+import org.opensearch.analytics.exec.stage.StageMetrics;
+import org.opensearch.analytics.exec.stage.StageTask;
+import org.opensearch.analytics.exec.stage.TaskTracker;
+import org.opensearch.analytics.planner.dag.ExecutionTarget;
+import org.opensearch.analytics.planner.dag.ShardExecutionTarget;
+import org.opensearch.analytics.planner.dag.Stage;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Snapshots an {@link ExecutionGraph} plus the per-query {@link TaskTracker} into a
+ * {@link QueryProfile}. Pure read — no mutation of the graph or tracker. Safe to call
+ * on success, failure, or cancellation paths: whatever state each stage has reached by
+ * the snapshot point is captured verbatim.
+ *
+ * @opensearch.internal
+ */
+public final class QueryProfileBuilder {
+
+    private QueryProfileBuilder() {}
+
+    public static QueryProfile snapshot(ExecutionGraph graph, QueryContext config) {
+        return snapshot(graph, config, "");
+    }
+
+    public static QueryProfile snapshot(ExecutionGraph graph, QueryContext config, String fullPlan) {
+        TaskTracker tracker = config.taskTracker();
+        List fullPlanLines = splitPlanLines(fullPlan);
+        List stageProfiles = new ArrayList<>();
+        long earliestStart = Long.MAX_VALUE;
+        long latestEnd = 0L;
+
+        for (StageExecution exec : graph.allExecutions()) {
+            StageMetrics m = exec.getMetrics();
+            long start = m.getStartTimeMs();
+            long end = m.getEndTimeMs();
+            long elapsed = (start > 0 && end > 0) ? end - start : 0L;
+            if (start > 0) earliestStart = Math.min(earliestStart, start);
+            if (end > 0) latestEnd = Math.max(latestEnd, end);
+
+            Stage stage = findStageById(config.dag().rootStage(), exec.getStageId());
+            // Stage#getExchangeInfo() is null for the root stage (no parent) and non-null
+            // for each cut edge. ExchangeInfo#distributionType() is a Calcite
+            // RelDistribution.Type enum.
+            String distribution = (stage != null && stage.getExchangeInfo() != null)
+                ? stage.getExchangeInfo().distributionType().name()
+                : null;
+            List fragment = stage != null && stage.getFragment() != null
+                ? splitPlanLines(RelOptUtil.toString(stage.getFragment()))
+                : List.of();
+
+            List taskProfiles = buildTaskProfiles(tracker, exec.getStageId());
+
+            stageProfiles.add(
+                new StageProfile(
+                    exec.getStageId(),
+                    stage != null ? stage.getExecutionType().name() : exec.getClass().getSimpleName(),
+                    distribution,
+                    exec.getState().name(),
+                    start,
+                    end,
+                    elapsed,
+                    m.getRowsProcessed(),
+                    m.getTasksCompleted(),
+                    m.getTasksFailed(),
+                    fragment,
+                    taskProfiles
+                )
+            );
+        }
+
+        long totalElapsed = (earliestStart != Long.MAX_VALUE && latestEnd > 0) ? latestEnd - earliestStart : 0L;
+        return new QueryProfile(graph.queryId(), fullPlanLines, totalElapsed, stageProfiles);
+    }
+
+    /**
+     * Splits a Calcite {@code RelOptUtil.toString} output into one entry per line.
+     * Empty trailing lines from Calcite's rendering are dropped. Returns an empty list
+     * for null or empty input so the caller doesn't have to null-check downstream.
+     */
+    private static List splitPlanLines(String text) {
+        if (text == null || text.isEmpty()) return List.of();
+        String[] raw = text.split("\n");
+        List out = new ArrayList<>(raw.length);
+        for (String line : raw) {
+            if (line.isEmpty() == false) out.add(line);
+        }
+        return out;
+    }
+
+    private static List buildTaskProfiles(TaskTracker tracker, int stageId) {
+        List tasks = tracker.tasksForStage(stageId);
+        List out = new ArrayList<>(tasks.size());
+        for (StageTask t : tasks) {
+            long start = t.startedAtMs();
+            long end = t.finishedAtMs();
+            long elapsed = (start > 0 && end > 0) ? end - start : 0L;
+            out.add(
+                new TaskProfile(
+                    t.id().stageId(),
+                    t.id().partitionId(),
+                    describeTarget(t.target()),
+                    t.state().name(),
+                    start,
+                    end,
+                    elapsed
+                )
+            );
+        }
+        return out;
+    }
+
+    /**
+     * Human-readable target label for the profile output. Includes the node id and,
+     * for shard-routed targets, the shard ordinal so the profile identifies which
+     * shard a task ran against.
+     */
+    private static String describeTarget(ExecutionTarget target) {
+        if (target == null) return "(unresolved)";
+        String nodeId = target.node() != null ? target.node().getId() : "(unknown)";
+        if (target instanceof ShardExecutionTarget shard) {
+            return nodeId + "/shard[" + shard.shardId().getId() + "]";
+        }
+        return nodeId;
+    }
+
+    private static Stage findStageById(Stage root, int stageId) {
+        if (root.getStageId() == stageId) return root;
+        for (Stage child : root.getChildStages()) {
+            Stage found = findStageById(child, stageId);
+            if (found != null) return found;
+        }
+        return null;
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/StageProfile.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/StageProfile.java
new file mode 100644
index 0000000000000..0fd1ae3eb447d
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/StageProfile.java
@@ -0,0 +1,78 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.exec.profile;
+
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Per-stage profile snapshot. Combines stage-level metadata (id, execution type,
+ * distribution), terminal state, wall-clock timing from {@code StageMetrics}, and
+ * the list of {@link TaskProfile}s for the stage's dispatched tasks.
+ *
+ * @param stageId              stage identifier from the DAG
+ * @param executionType        value of {@code StageExecutionType} as string (SHARD_FRAGMENT, COORDINATOR_REDUCE, LOCAL_PASSTHROUGH)
+ * @param distribution         Calcite distribution type this stage emits to its parent — e.g. SINGLETON, HASH_DISTRIBUTED; null for root
+ * @param state                terminal {@code StageExecution.State}
+ * @param startMs              wall-clock millis from {@code StageMetrics.recordStart()}, 0 if never started
+ * @param endMs                wall-clock millis from {@code StageMetrics.recordEnd()}, 0 if still running
+ * @param elapsedMs            {@code endMs - startMs}, or 0 if either stamp is missing
+ * @param rowsProcessed        counter from {@code StageMetrics.addRowsProcessed}
+ * @param tasksCompleted       counter from {@code StageMetrics.incrementTasksCompleted}
+ * @param tasksFailed          counter from {@code StageMetrics.incrementTasksFailed}
+ * @param fragment             Calcite {@code RelOptUtil.toString(stage.getFragment())} rendered as an
+ *                             array of lines (one element per level of indent) — much easier to read in
+ *                             raw JSON than a single multi-line escaped string
+ * @param tasks                per-partition task profiles registered with the TaskTracker
+ */
+public record StageProfile(
+    int stageId,
+    String executionType,
+    String distribution,
+    String state,
+    long startMs,
+    long endMs,
+    long elapsedMs,
+    long rowsProcessed,
+    long tasksCompleted,
+    long tasksFailed,
+    List fragment,
+    List tasks
+) implements ToXContentObject {
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("stage_id", stageId);
+        builder.field("execution_type", executionType);
+        if (distribution != null) builder.field("distribution", distribution);
+        builder.field("state", state);
+        if (startMs > 0) builder.field("start_ms", startMs);
+        if (endMs > 0) builder.field("end_ms", endMs);
+        builder.field("elapsed_ms", elapsedMs);
+        builder.field("rows_processed", rowsProcessed);
+        builder.field("tasks_completed", tasksCompleted);
+        builder.field("tasks_failed", tasksFailed);
+        if (fragment != null && fragment.isEmpty() == false) {
+            builder.startArray("fragment");
+            for (String line : fragment) builder.value(line);
+            builder.endArray();
+        }
+        builder.startArray("tasks");
+        for (TaskProfile t : tasks) {
+            t.toXContent(builder, params);
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/TaskProfile.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/TaskProfile.java
new file mode 100644
index 0000000000000..65af9c6e32309
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/profile/TaskProfile.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.exec.profile;
+
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Per-task profile snapshot. Captures identity, target node, terminal state and
+ * wall-clock timing. A "task" is one dispatch unit within a stage (one shard for
+ * SOURCE, one partition for HASH_PARTITIONED, one total for COORDINATOR).
+ *
+ * @param stageId id of the owning stage
+ * @param partitionId ordinal of the task within its stage (0-based)
+ * @param node target node id the task ran on, or "(unresolved)" if dispatch never happened
+ * @param state terminal state — CREATED if the task was never dispatched
+ * @param startMs wall-clock millis of the first RUNNING transition, 0 if never dispatched
+ * @param endMs wall-clock millis of the first terminal transition, 0 if still running
+ * @param elapsedMs {@code endMs - startMs}, or 0 if either stamp is missing
+ */
+public record TaskProfile(int stageId, int partitionId, String node, String state, long startMs, long endMs, long elapsedMs)
+    implements
+        ToXContentObject {
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("partition_id", partitionId);
+        builder.field("node", node);
+        builder.field("state", state);
+        if (startMs > 0) builder.field("start_ms", startMs);
+        if (endMs > 0) builder.field("end_ms", endMs);
+        builder.field("elapsed_ms", elapsedMs);
+        builder.endObject();
+        return builder;
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java
index 83ae3dd0b4577..3099453ab2eff 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecution.java
@@ -22,10 +22,10 @@
 import org.opensearch.analytics.spi.ExchangeSink;
 import org.opensearch.cluster.service.ClusterService;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 
 /**
@@ -40,8 +40,6 @@
  */
 final class ShardFragmentStageExecution extends AbstractStageExecution implements DataProducer {
 
-    private final AtomicInteger inFlight = new AtomicInteger(0);
-
     private final QueryContext config;
     private final ExchangeSink outputSink;
     private final ClusterService clusterService;
@@ -72,20 +70,33 @@ public void start() {
             transitionTo(StageExecution.State.SUCCEEDED);
             return;
         }
+        if (transitionTo(StageExecution.State.SCHEDULING) == false) return;
+        // Materialise one StageTask per target and register with the per-query
+        // TaskTracker before any transport call — so if a dispatch fails mid-loop the
+        // tracker still carries every task we're about to kick off. The profile
+        // builder later reads per-partition state and timing from here.
+        TaskTracker tracker = config.taskTracker();
+        List tasks = new ArrayList<>(resolved.size());
+        for (int i = 0; i < resolved.size(); i++) {
+            StageTask t = new StageTask(new StageTaskId(stage.getStageId(), i), resolved.get(i));
+            tasks.add(t);
+            tracker.register(t);
+        }
         if (transitionTo(StageExecution.State.RUNNING) == false) return;
-        inFlight.set(resolved.size());
-        for (ExecutionTarget target : resolved) {
-            dispatchShardTask((ShardExecutionTarget) target);
+        for (StageTask task : tasks) {
+            task.transitionTo(StageTaskState.RUNNING);
+            dispatchShardTask(task);
         }
     }
 
-    private void dispatchShardTask(ShardExecutionTarget target) {
+    private void dispatchShardTask(StageTask task) {
+        ShardExecutionTarget target = (ShardExecutionTarget) task.target();
         FragmentExecutionRequest request = requestBuilder.apply(target);
         PendingExecutions pending = pendingFor(target);
-        dispatcher.dispatchFragmentStreaming(request, target.node(), responseListener(), config.parentTask(), pending);
+        dispatcher.dispatchFragmentStreaming(request, target.node(), responseListener(task), config.parentTask(), pending);
     }
 
-    private StreamingResponseListener responseListener() {
+    private StreamingResponseListener responseListener(StageTask task) {
         return new StreamingResponseListener<>() {
             // Runs inline on the per-stream virtual thread driving handleStreamResponse.
             // Must NOT offload to a thread pool: reordering across batches would let the
@@ -106,17 +117,17 @@ public void onStreamResponse(FragmentExecutionArrowResponse response, boolean is
                     outputSink.feed(vsr);
                 } catch (Exception e) {
                     // Without this guard the exception only surfaces on the stream's virtual
-                    // thread; inFlight never decrements and the stage hangs to QUERY_TIMEOUT.
+                    // thread; the task never terminates and the stage hangs to QUERY_TIMEOUT.
                     captureFailure(new RuntimeException("Stage " + stage.getStageId() + " sink feed failed", e));
                     metrics.incrementTasksFailed();
-                    onShardTerminated();
+                    onTaskTerminated(task, StageTaskState.FAILED);
                     return;
                 }
                 metrics.addRowsProcessed(vsr.getRowCount());
 
                 if (isLast) {
                     metrics.incrementTasksCompleted();
-                    onShardTerminated();
+                    onTaskTerminated(task, StageTaskState.FINISHED);
                 }
             }
 
@@ -124,13 +135,19 @@ public void onStreamResponse(FragmentExecutionArrowResponse response, boolean is
             public void onFailure(Exception e) {
                 captureFailure(new RuntimeException("Stage " + stage.getStageId() + " failed", e));
                 metrics.incrementTasksFailed();
-                onShardTerminated();
+                onTaskTerminated(task, StageTaskState.FAILED);
             }
         };
     }
 
-    private void onShardTerminated() {
-        if (inFlight.decrementAndGet() == 0) {
+    private void onTaskTerminated(StageTask task, StageTaskState terminalState) {
+        // transitionTo no-ops if the task is already terminal — safe to call twice if
+        // the transport fires a late onFailure after a successful isLast=true.
+        task.transitionTo(terminalState);
+        // Stage terminal derives from TaskTracker instead of a local in-flight counter.
+        // Concurrent terminal-firing tasks may both see "all terminal" and both attempt
+        // the stage transition — transitionTo is CAS-guarded so only one wins.
+        if (config.taskTracker().allTasksTerminalForStage(stage.getStageId())) {
             Exception captured = getFailure();
             transitionTo(captured != null ? StageExecution.State.FAILED : StageExecution.State.SUCCEEDED);
         }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageExecution.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageExecution.java
index e68139ab6604a..d803e6c24cd36 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageExecution.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageExecution.java
@@ -87,6 +87,12 @@ public interface StageExecution {
     enum State {
         /** Initial state before {@link #start()} has been invoked. */
         CREATED,
+        /**
+         * {@link #start()} has been called; tasks are being materialised and registered
+         * with the {@link TaskTracker}, but none have been dispatched to the transport
+         * layer yet. Brief — flips to {@link #RUNNING} as soon as dispatch begins.
+         */
+        SCHEDULING,
         /** Dispatch has begun; the stage is actively executing. */
         RUNNING,
         /** Terminal success — all work completed, output delivered to the sink. */
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTask.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTask.java
new file mode 100644
index 0000000000000..9fb0b7b388c41
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTask.java
@@ -0,0 +1,86 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.exec.stage;
+
+import org.opensearch.analytics.planner.dag.ExecutionTarget;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * A single dispatchable unit within a {@link StageExecution}. Wraps an
+ * {@link ExecutionTarget} (the already-resolved node + shards + fragment bytes) with
+ * mutable lifecycle state so the scheduler can track per-partition progress.
+ *
+ * 
              One stage produces N tasks: one per shard for SOURCE stages, one per hash
+ * partition for HASH_PARTITIONED, one total for COORDINATOR. State transitions are
+ * observed by {@link TaskTracker} — which in turn drives stage readiness.
+ *
+ * @opensearch.internal
+ */
+public final class StageTask {
+
+    private final StageTaskId id;
+    private final ExecutionTarget target;
+    private final AtomicReference state = new AtomicReference<>(StageTaskState.CREATED);
+    private volatile long startedAtMs;
+    private volatile long finishedAtMs;
+
+    public StageTask(StageTaskId id, ExecutionTarget target) {
+        this.id = id;
+        this.target = target;
+    }
+
+    public StageTaskId id() {
+        return id;
+    }
+
+    public ExecutionTarget target() {
+        return target;
+    }
+
+    public StageTaskState state() {
+        return state.get();
+    }
+
+    /** Wall-clock millis stamped on the first successful transition to {@link StageTaskState#RUNNING}, or 0 if never dispatched. */
+    public long startedAtMs() {
+        return startedAtMs;
+    }
+
+    /** Wall-clock millis stamped on the first successful terminal transition, or 0 if still running. */
+    public long finishedAtMs() {
+        return finishedAtMs;
+    }
+
+    /**
+     * Attempts to transition this task to {@code target}. Returns false if the task is
+     * already in a terminal state — callers must gate terminal side effects on the return
+     * value, just like {@link AbstractStageExecution#transitionTo}.
+     *
+     * 
              On a successful transition, wall-clock stamps are recorded: {@code startedAtMs}
+     * on the first entry into {@link StageTaskState#RUNNING}, {@code finishedAtMs} on
+     * the first entry into any terminal state. Rejected transitions never rewrite the
+     * stamps.
+     */
+    public boolean transitionTo(StageTaskState target) {
+        StageTaskState prev;
+        do {
+            prev = state.get();
+            if (prev.isTerminal() || prev == target) return false;
+        } while (state.compareAndSet(prev, target) == false);
+        long now = System.currentTimeMillis();
+        if (target == StageTaskState.RUNNING && startedAtMs == 0L) {
+            startedAtMs = now;
+        }
+        if (target.isTerminal() && finishedAtMs == 0L) {
+            finishedAtMs = now;
+        }
+        return true;
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTaskId.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTaskId.java
new file mode 100644
index 0000000000000..cf85ab413d6c7
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTaskId.java
@@ -0,0 +1,23 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.exec.stage;
+
+/**
+ * Identity of a single task within a stage. A stage of distribution N emits N tasks —
+ * one per shard (SOURCE), one per hash partition (HASH_PARTITIONED), or one total
+ * (COORDINATOR). Unique within a query.
+ *
+ * @opensearch.internal
+ */
+public record StageTaskId(int stageId, int partitionId) {
+    @Override
+    public String toString() {
+        return stageId + "." + partitionId;
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTaskState.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTaskState.java
new file mode 100644
index 0000000000000..3fd895dfcab43
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/StageTaskState.java
@@ -0,0 +1,33 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.exec.stage;
+
+/**
+ * Lifecycle of a single {@link StageTask}. Mirrors the stage's own state machine but
+ * tracked per-partition so the {@link TaskTracker} / scheduler can reason about partial
+ * progress, retry eligibility, and stage readiness.
+ *
+ * @opensearch.internal
+ */
+public enum StageTaskState {
+    /** Task descriptor created, not yet dispatched. */
+    CREATED,
+    /** Dispatched to a data node; awaiting first response or completion. */
+    RUNNING,
+    /** Terminal success — task finished and its output was handed to the downstream sink. */
+    FINISHED,
+    /** Terminal failure — the task itself errored or its response stream faulted. */
+    FAILED,
+    /** Terminal cancellation — the task was cancelled by the parent query or stage. */
+    CANCELLED;
+
+    public boolean isTerminal() {
+        return this == FINISHED || this == FAILED || this == CANCELLED;
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/TaskTracker.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/TaskTracker.java
new file mode 100644
index 0000000000000..e3acb499707a6
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/stage/TaskTracker.java
@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.exec.stage;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Per-query registry of every {@link StageTask} across all stages. Owned by
+ * {@code PlanWalker}; populated as stages materialise their task lists at dispatch
+ * time. Exists to answer questions like "is this stage finished?" and "which tasks
+ * are still running?" without walking every stage execution.
+ *
+ * 
              The registry is not a replacement for {@link StageExecution}'s own state — it's a
+ * lookup index. Stage readiness is still computed from task states here, then driven
+ * through the stage's CAS transitions.
+ *
+ * @opensearch.internal
+ */
+public final class TaskTracker {
+
+    private final Map tasks = new ConcurrentHashMap<>();
+
+    /** Register a newly-created task. Idempotent — double-registers overwrite, which should not happen. */
+    public void register(StageTask task) {
+        tasks.put(task.id(), task);
+    }
+
+    /** Returns the task for {@code id}, or null if unknown. */
+    public StageTask get(StageTaskId id) {
+        return tasks.get(id);
+    }
+
+    /**
+     * Returns true when every task registered for {@code stageId} has reached a terminal
+     * state ({@link StageTaskState#FINISHED}, {@link StageTaskState#FAILED},
+     * {@link StageTaskState#CANCELLED}).
+     */
+    public boolean allTasksTerminalForStage(int stageId) {
+        for (StageTask t : tasks.values()) {
+            if (t.id().stageId() == stageId && t.state().isTerminal() == false) return false;
+        }
+        return true;
+    }
+
+    /** Returns the subset of tasks registered for {@code stageId}. */
+    public List tasksForStage(int stageId) {
+        return tasks.values().stream().filter(t -> t.id().stageId() == stageId).toList();
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/QueryProfileBuilderTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/QueryProfileBuilderTests.java
new file mode 100644
index 0000000000000..e188dee2da750
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/QueryProfileBuilderTests.java
@@ -0,0 +1,208 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.exec;
+
+import org.opensearch.analytics.exec.profile.QueryProfile;
+import org.opensearch.analytics.exec.profile.QueryProfileBuilder;
+import org.opensearch.analytics.exec.profile.StageProfile;
+import org.opensearch.analytics.exec.profile.TaskProfile;
+import org.opensearch.analytics.exec.stage.StageExecution;
+import org.opensearch.analytics.exec.stage.StageMetrics;
+import org.opensearch.analytics.exec.stage.StageStateListener;
+import org.opensearch.analytics.exec.stage.StageTask;
+import org.opensearch.analytics.exec.stage.StageTaskId;
+import org.opensearch.analytics.exec.stage.StageTaskState;
+import org.opensearch.analytics.exec.task.AnalyticsQueryTask;
+import org.opensearch.analytics.planner.dag.ExecutionTarget;
+import org.opensearch.analytics.planner.dag.QueryDAG;
+import org.opensearch.analytics.planner.dag.Stage;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.core.tasks.TaskId;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class QueryProfileBuilderTests extends OpenSearchTestCase {
+
+    public void testSnapshotCapturesQueryIdAndStageIdsFromEmptyGraph() {
+        Stage rootStage = stageWithId(0);
+        QueryContext ctx = new QueryContext(new QueryDAG("q-empty", rootStage), Runnable::run, taskStub(), 1, Long.MAX_VALUE);
+        StubExecution root = new StubExecution(0);
+        ExecutionGraph graph = singleStageGraph("q-empty", root);
+
+        QueryProfile profile = QueryProfileBuilder.snapshot(graph, ctx);
+
+        assertEquals("q-empty", profile.queryId());
+        assertEquals(1, profile.stages().size());
+        assertEquals(0, profile.stages().get(0).stageId());
+        assertEquals("CREATED", profile.stages().get(0).state());
+        assertEquals(0L, profile.totalElapsedMs());
+    }
+
+    public void testSnapshotComputesElapsedFromMetricsStartEnd() {
+        Stage rootStage = stageWithId(0);
+        QueryContext ctx = new QueryContext(new QueryDAG("q-timed", rootStage), Runnable::run, taskStub(), 1, Long.MAX_VALUE);
+        StubExecution root = new StubExecution(0);
+        root.transitionInternal(StageExecution.State.RUNNING); // stamps start
+        root.transitionInternal(StageExecution.State.SUCCEEDED); // stamps end
+        ExecutionGraph graph = singleStageGraph("q-timed", root);
+
+        QueryProfile profile = QueryProfileBuilder.snapshot(graph, ctx);
+
+        StageProfile stage = profile.stages().get(0);
+        assertTrue("start stamped", stage.startMs() > 0);
+        assertTrue("end stamped", stage.endMs() > 0);
+        assertTrue("elapsed non-negative", stage.elapsedMs() >= 0);
+        // Query total spans earliest-to-latest across all stages; single stage == stage elapsed.
+        assertEquals(stage.elapsedMs(), profile.totalElapsedMs());
+    }
+
+    public void testSnapshotSplitsFullPlanIntoLines() {
+        Stage rootStage = stageWithId(0);
+        QueryContext ctx = new QueryContext(new QueryDAG("q-plan", rootStage), Runnable::run, taskStub(), 1, Long.MAX_VALUE);
+        ExecutionGraph graph = singleStageGraph("q-plan", new StubExecution(0));
+
+        // Calcite's RelOptUtil.toString produces "Node\n  child\n" — mimic that.
+        QueryProfile profile = QueryProfileBuilder.snapshot(graph, ctx, "Aggregate\n  TableScan\n");
+
+        assertEquals(java.util.List.of("Aggregate", "  TableScan"), profile.fullPlan());
+    }
+
+    public void testSnapshotEmptyFullPlanReturnsEmptyList() {
+        Stage rootStage = stageWithId(0);
+        QueryContext ctx = new QueryContext(new QueryDAG("q-plan", rootStage), Runnable::run, taskStub(), 1, Long.MAX_VALUE);
+        ExecutionGraph graph = singleStageGraph("q-plan", new StubExecution(0));
+
+        QueryProfile profile = QueryProfileBuilder.snapshot(graph, ctx, "");
+
+        assertTrue(profile.fullPlan().isEmpty());
+    }
+
+    public void testSnapshotCollectsTaskProfilesFromTracker() {
+        Stage rootStage = stageWithId(0);
+        QueryContext ctx = new QueryContext(new QueryDAG("q-tasks", rootStage), Runnable::run, taskStub(), 1, Long.MAX_VALUE);
+        StageTask t0 = new StageTask(new StageTaskId(0, 0), mockTargetWithNode("node_a"));
+        StageTask t1 = new StageTask(new StageTaskId(0, 1), mockTargetWithNode("node_b"));
+        ctx.taskTracker().register(t0);
+        ctx.taskTracker().register(t1);
+        t0.transitionTo(StageTaskState.RUNNING);
+        t1.transitionTo(StageTaskState.RUNNING);
+        t0.transitionTo(StageTaskState.FINISHED);
+        t1.transitionTo(StageTaskState.FAILED);
+
+        ExecutionGraph graph = singleStageGraph("q-tasks", new StubExecution(0));
+        QueryProfile profile = QueryProfileBuilder.snapshot(graph, ctx);
+
+        List tasks = profile.stages().get(0).tasks();
+        assertEquals(2, tasks.size());
+        // tasksForStage ordering isn't guaranteed — check set membership by partition id.
+        TaskProfile p0 = tasks.stream().filter(t -> t.partitionId() == 0).findFirst().orElseThrow();
+        TaskProfile p1 = tasks.stream().filter(t -> t.partitionId() == 1).findFirst().orElseThrow();
+        assertEquals("FINISHED", p0.state());
+        assertEquals("node_a", p0.node());
+        assertEquals("FAILED", p1.state());
+        assertEquals("node_b", p1.node());
+        assertTrue("task start stamped", p0.startMs() > 0);
+        assertTrue("task end stamped", p0.endMs() > 0);
+    }
+
+    // ─── helpers ────────────────────────────────────────────────────────
+
+    private static Stage stageWithId(int id) {
+        Stage stage = mock(Stage.class);
+        when(stage.getStageId()).thenReturn(id);
+        when(stage.getChildStages()).thenReturn(List.of());
+        when(stage.getExecutionType()).thenReturn(org.opensearch.analytics.planner.dag.StageExecutionType.LOCAL_PASSTHROUGH);
+        when(stage.getFragment()).thenReturn(null);
+        when(stage.getExchangeInfo()).thenReturn(null);
+        return stage;
+    }
+
+    private static AnalyticsQueryTask taskStub() {
+        return new AnalyticsQueryTask(1L, "transport", "analytics_query", "q-test", TaskId.EMPTY_TASK_ID, Map.of(), null);
+    }
+
+    private static ExecutionGraph singleStageGraph(String queryId, StageExecution root) {
+        return new ExecutionGraph(queryId, Map.of(root.getStageId(), root), root, List.of(root));
+    }
+
+    private static ExecutionTarget mockTargetWithNode(String nodeId) {
+        DiscoveryNode node = mock(DiscoveryNode.class);
+        when(node.getId()).thenReturn(nodeId);
+        return new TestTarget(node);
+    }
+
+    private static final class TestTarget extends ExecutionTarget {
+        TestTarget(DiscoveryNode node) {
+            super(node);
+        }
+    }
+
+    /**
+     * Minimal {@link StageExecution} that exposes the protected {@code transitionTo} for tests.
+     * Mirrors {@code AbstractStageExecution}'s metrics stamping so elapsed math is real.
+     */
+    private static final class StubExecution implements StageExecution {
+        private final int stageId;
+        private final StageMetrics metrics = new StageMetrics();
+        private final AtomicReference state = new AtomicReference<>(State.CREATED);
+        private final java.util.List listeners = new java.util.ArrayList<>();
+
+        StubExecution(int stageId) {
+            this.stageId = stageId;
+        }
+
+        @Override
+        public int getStageId() {
+            return stageId;
+        }
+
+        @Override
+        public State getState() {
+            return state.get();
+        }
+
+        @Override
+        public StageMetrics getMetrics() {
+            return metrics;
+        }
+
+        @Override
+        public void start() {}
+
+        @Override
+        public void addStateListener(StageStateListener listener) {
+            listeners.add(listener);
+        }
+
+        @Override
+        public Exception getFailure() {
+            return null;
+        }
+
+        @Override
+        public boolean failFromChild(Exception cause) {
+            return false;
+        }
+
+        @Override
+        public void cancel(String reason) {}
+
+        void transitionInternal(State target) {
+            State prev = state.getAndSet(target);
+            if (prev == State.CREATED) metrics.recordStart();
+            if (target == State.SUCCEEDED || target == State.FAILED || target == State.CANCELLED) metrics.recordEnd();
+        }
+    }
+}
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
index 34e4c3a156f84..dc873c4cbf2f0 100644
--- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/ShardFragmentStageExecutionTests.java
@@ -164,10 +164,10 @@ private Stage mockStage() {
 
     private QueryContext mockQueryContext() {
         QueryContext config = mock(QueryContext.class);
-        when(config.searchExecutor()).thenReturn(Runnable::run);
         when(config.parentTask()).thenReturn(mock(AnalyticsQueryTask.class));
         when(config.maxConcurrentShardRequests()).thenReturn(5);
         when(config.bufferAllocator()).thenReturn(allocator);
+        when(config.taskTracker()).thenReturn(new TaskTracker());
         return config;
     }
 
diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/TaskTrackerTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/TaskTrackerTests.java
new file mode 100644
index 0000000000000..d1c67826c5691
--- /dev/null
+++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/stage/TaskTrackerTests.java
@@ -0,0 +1,124 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.exec.stage;
+
+import org.opensearch.analytics.planner.dag.ExecutionTarget;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.test.OpenSearchTestCase;
+
+import static org.mockito.Mockito.mock;
+
+public class TaskTrackerTests extends OpenSearchTestCase {
+
+    public void testAllTasksTerminalForStageReturnsTrueWhenEmpty() {
+        TaskTracker tracker = new TaskTracker();
+        // No tasks registered for the stage — vacuously true. Scheduler uses this when
+        // a stage resolves to zero targets (empty SearchShardsResponse).
+        assertTrue(tracker.allTasksTerminalForStage(0));
+    }
+
+    public void testAllTasksTerminalForStageFalseWhileAnyRunning() {
+        TaskTracker tracker = new TaskTracker();
+        tracker.register(task(0, 0));
+        StageTask t1 = task(0, 1);
+        tracker.register(t1);
+        t1.transitionTo(StageTaskState.RUNNING);
+
+        assertFalse(tracker.allTasksTerminalForStage(0));
+    }
+
+    public void testAllTasksTerminalForStageTrueWhenEveryTaskFinished() {
+        TaskTracker tracker = new TaskTracker();
+        StageTask t0 = task(0, 0);
+        StageTask t1 = task(0, 1);
+        tracker.register(t0);
+        tracker.register(t1);
+        t0.transitionTo(StageTaskState.RUNNING);
+        t1.transitionTo(StageTaskState.RUNNING);
+        t0.transitionTo(StageTaskState.FINISHED);
+        t1.transitionTo(StageTaskState.FINISHED);
+
+        assertTrue(tracker.allTasksTerminalForStage(0));
+    }
+
+    public void testAllTasksTerminalForStageTrueWithMixedTerminals() {
+        // Stage is considered terminal as soon as every task is in SOME terminal state —
+        // mixed FINISHED/FAILED/CANCELLED all count. Scheduler needs this to drive
+        // stage-state derivation: the stage itself will then decide success vs failure.
+        TaskTracker tracker = new TaskTracker();
+        StageTask t0 = task(0, 0);
+        StageTask t1 = task(0, 1);
+        StageTask t2 = task(0, 2);
+        tracker.register(t0);
+        tracker.register(t1);
+        tracker.register(t2);
+        t0.transitionTo(StageTaskState.RUNNING);
+        t0.transitionTo(StageTaskState.FINISHED);
+        t1.transitionTo(StageTaskState.RUNNING);
+        t1.transitionTo(StageTaskState.FAILED);
+        t2.transitionTo(StageTaskState.CANCELLED);
+
+        assertTrue(tracker.allTasksTerminalForStage(0));
+    }
+
+    public void testTasksForStageOnlyReturnsThatStage() {
+        TaskTracker tracker = new TaskTracker();
+        tracker.register(task(0, 0));
+        tracker.register(task(0, 1));
+        tracker.register(task(1, 0));
+
+        assertEquals(2, tracker.tasksForStage(0).size());
+        assertEquals(1, tracker.tasksForStage(1).size());
+    }
+
+    public void testStageTaskTransitionToTerminalIsFinal() {
+        StageTask t = task(0, 0);
+        assertTrue(t.transitionTo(StageTaskState.RUNNING));
+        assertTrue(t.transitionTo(StageTaskState.FINISHED));
+        assertFalse("terminal state must not be overwritten", t.transitionTo(StageTaskState.FAILED));
+        assertEquals(StageTaskState.FINISHED, t.state());
+    }
+
+    public void testStageTaskStampsStartAndEndTimesOnTransition() {
+        StageTask t = task(0, 0);
+        assertEquals("start not yet stamped before transition to RUNNING", 0L, t.startedAtMs());
+        assertEquals("end not yet stamped before terminal transition", 0L, t.finishedAtMs());
+
+        t.transitionTo(StageTaskState.RUNNING);
+        long start = t.startedAtMs();
+        assertTrue("start stamped on RUNNING", start > 0);
+
+        t.transitionTo(StageTaskState.FINISHED);
+        long end = t.finishedAtMs();
+        assertTrue("end stamped on terminal", end > 0);
+        assertTrue("end must be >= start", end >= start);
+    }
+
+    public void testStageTaskDoubleTerminalKeepsFirstEndTime() {
+        // Late onFailure after a successful isLast=true must not rewrite the end stamp.
+        StageTask t = task(0, 0);
+        t.transitionTo(StageTaskState.RUNNING);
+        t.transitionTo(StageTaskState.FINISHED);
+        long firstEnd = t.finishedAtMs();
+        // Spin briefly so System.currentTimeMillis() would advance.
+        try { Thread.sleep(2); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); }
+        assertFalse(t.transitionTo(StageTaskState.FAILED));
+        assertEquals("end stamp must not rewrite on rejected transition", firstEnd, t.finishedAtMs());
+    }
+
+    private static StageTask task(int stageId, int partitionId) {
+        return new StageTask(new StageTaskId(stageId, partitionId), new TestTarget());
+    }
+
+    private static final class TestTarget extends ExecutionTarget {
+        TestTarget() {
+            super(mock(DiscoveryNode.class));
+        }
+    }
+}