From 631c58e56f75ce734b36d672457b034a8330ddab Mon Sep 17 00:00:00 2001 From: ellisandrews-toast Date: Mon, 1 Jun 2026 17:58:25 -0400 Subject: [PATCH 1/3] Add schema artifacts structure and indexer wiring for nested sourced_from --- .../datastore_core/index_definition.rb | 5 +- .../datastore_core/index_definition/index.rb | 5 +- .../rollover_index_template.rb | 4 +- .../datastore_core/index_definition.rbs | 1 + .../index_definition/index_spec.rb | 3 +- .../rollover_index_template_spec.rb | 3 +- .../elastic_graph/indexer/operation/update.rb | 11 ++- .../indexer/datastore_indexing_router_spec.rb | 2 +- .../indexer/operation/update_spec.rb | 41 ++++++++--- .../runtime_metadata/index_definition.rb | 13 ++-- .../nested_sourced_data_params.rb | 50 +++++++++++++ .../nested_sourced_path_segment.rb | 70 +++++++++++++++++++ .../runtime_metadata/update_target.rb | 10 ++- .../runtime_metadata/index_definition.rbs | 11 ++- .../nested_sourced_data_params.rbs | 22 ++++++ .../nested_sourced_path_segment.rbs | 45 ++++++++++++ .../runtime_metadata/update_target.rbs | 4 ++ .../runtime_metadata/index_definition_spec.rb | 23 +++++- .../runtime_metadata/schema_spec.rb | 12 +++- .../runtime_metadata/update_target_spec.rb | 21 +++++- .../indexing/derived_indexed_type.rb | 3 +- .../schema_definition/indexing/index.rb | 3 +- .../indexing/update_target_factory.rb | 4 +- .../indexing/update_target_resolver.rb | 1 + .../schema_definition/mixins/has_indices.rb | 1 + .../indexing/update_target_factory.rbs | 3 +- .../spec_support/runtime_metadata_support.rb | 8 ++- 27 files changed, 337 insertions(+), 42 deletions(-) create mode 100644 elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rb create mode 100644 elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rb create mode 100644 elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rbs create mode 100644 elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rbs diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition.rb index c95dc2f7c..7f39134e7 100644 --- a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition.rb +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition.rb @@ -33,7 +33,10 @@ def self.with(name:, runtime_metadata:, config:, datastore_clients_by_name:, sch env_index_config: env_index_config, defined_clusters: config.clusters.keys.to_set, datastore_clients_by_name: datastore_clients_by_name, - has_had_multiple_sources: runtime_metadata.has_had_multiple_sources + has_had_multiple_sources: runtime_metadata.has_had_multiple_sources, + # :nocov: -- the block is only exercised with non-empty nested_sourced_paths, which will come in a future PR + nested_sourced_paths: runtime_metadata.nested_sourced_paths.transform_values { |segments| segments.map(&:to_painless_param) } + # :nocov: } if (rollover = runtime_metadata.rollover) diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/index.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/index.rb index 1b99e7a4e..a85c309fb 100644 --- a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/index.rb +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/index.rb @@ -15,11 +15,12 @@ class DatastoreCore module IndexDefinition class Index < Support::MemoizableData.define( :name, :route_with, :default_sort_clauses, :current_sources, :fields_by_path, - :env_index_config, :defined_clusters, :datastore_clients_by_name, :env_agnostic_settings, :has_had_multiple_sources + :env_index_config, :defined_clusters, :datastore_clients_by_name, :env_agnostic_settings, :has_had_multiple_sources, + :nested_sourced_paths ) # `Data.define` provides all these methods: # @dynamic name, route_with, default_sort_clauses, current_sources, fields_by_path, env_index_config, env_agnostic_settings - # @dynamic defined_clusters, datastore_clients_by_name, initialize, has_had_multiple_sources + # @dynamic defined_clusters, datastore_clients_by_name, initialize, has_had_multiple_sources, nested_sourced_paths # `include IndexDefinition::Base` provides all these methods. Steep should be able to detect it # but can't for some reason so we have to declare them with `@dynamic`. diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index_template.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index_template.rb index 9a70836da..44ea8bda7 100644 --- a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index_template.rb +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index_template.rb @@ -23,11 +23,11 @@ module IndexDefinition class RolloverIndexTemplate < Support::MemoizableData.define( :name, :route_with, :default_sort_clauses, :current_sources, :fields_by_path, :env_index_config, :index_args, :defined_clusters, :datastore_clients_by_name, :timestamp_field_path, :frequency, - :env_agnostic_settings, :has_had_multiple_sources + :env_agnostic_settings, :has_had_multiple_sources, :nested_sourced_paths ) # `Data.define` provides all these methods: # @dynamic name, route_with, default_sort_clauses, current_sources, fields_by_path, env_index_config, env_agnostic_settings - # @dynamic index_args, defined_clusters, datastore_clients_by_name, timestamp_field_path, frequency, initialize, has_had_multiple_sources + # @dynamic index_args, defined_clusters, datastore_clients_by_name, timestamp_field_path, frequency, initialize, has_had_multiple_sources, nested_sourced_paths # `include IndexDefinition::Base` provides all these methods. Steep should be able to detect it # but can't for some reason so we have to declare them with `@dynamic`. diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition.rbs index b12ac2343..8d087e939 100644 --- a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition.rbs +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition.rbs @@ -26,6 +26,7 @@ module ElasticGraph def current_sources: () -> ::Set[::String] def fields_by_path: () -> ::Hash[::String, SchemaArtifacts::RuntimeMetadata::IndexField] def has_had_multiple_sources: () -> bool + def nested_sourced_paths: () -> ::Hash[::String, ::Array[::Hash[::String, ::String]]] def env_index_config: () -> Configuration::IndexDefinition def env_agnostic_settings: () -> ::Hash[::String, untyped] def defined_clusters: () -> ::Set[::String] diff --git a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb index 6291178c0..37ede53d9 100644 --- a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb +++ b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb @@ -88,7 +88,8 @@ def index_def_named(name, rollover: nil) default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, - has_had_multiple_sources: false + has_had_multiple_sources: false, + nested_sourced_paths: {} ) DatastoreCore::IndexDefinition.with( diff --git a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb index eac218050..4b77b3b09 100644 --- a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb +++ b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb @@ -557,7 +557,8 @@ def index_def_named(name, rollover: nil) default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, - has_had_multiple_sources: false + has_had_multiple_sources: false, + nested_sourced_paths: {} ) DatastoreCore::IndexDefinition.with( diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/operation/update.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/update.rb index 32e04c04a..52e685552 100644 --- a/elasticgraph-indexer/lib/elastic_graph/indexer/operation/update.rb +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/update.rb @@ -145,11 +145,16 @@ def script_params prepared_record: prepared_record ) - # The normal indexing script uses `__counts`. Other indexing scripts (e.g. the ones generated - # for derived indexing) do not use `__counts` so there's no point in spending effort on computing - # it. Plus, the logic below raises an exception in that case, so it's important we avoid it. + # The normal indexing script uses `__counts` and `nestedSourcedPaths`. Other indexing scripts + # (e.g. the ones generated for derived indexing) do not use these so there's no point in + # spending effort on computing them. Plus, the logic below raises an exception in that case, + # so it's important we avoid it. return initial_params unless update_target.for_normal_indexing? + # `nestedSourcedPaths` is static per-index configuration (describing how to navigate to nested + # elements), unlike the per-event params resolved by `update_target.params_for` above. + initial_params["nestedSourcedPaths"] = destination_index_def.nested_sourced_paths + CountAccumulator.merge_list_counts_into( initial_params, mapping: destination_index_mapping, diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb index 6d77a53e5..ca63cbdb5 100644 --- a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb @@ -265,7 +265,7 @@ def type_name_for_index(index_name) upsert: {}, script: a_hash_including( id: /WidgetCurrency_from_Widget_/, - params: {"topLevelFields" => {"name" => ["thing1"]}, "id" => "USD"} + params: {"topLevelFields" => {"name" => ["thing1"]}, "id" => "USD", "nestedSourcedFields" => {}, "nestedSourcedPathIdentifiers" => {}} ) ) end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb index 1a15a9e62..05bb2c273 100644 --- a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb @@ -74,7 +74,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, - "id" => "17" + "id" => "17", + "nestedSourcedFields" => {}, + "nestedSourcedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -106,7 +108,10 @@ module Operation "id" => "17", "staticValue" => 47, "sourceType" => "Widget", - LIST_COUNTS_FIELD => {"sizes" => 0, "widget_names" => 0} + LIST_COUNTS_FIELD => {"sizes" => 0, "widget_names" => 0}, + "nestedSourcedPaths" => {}, + "nestedSourcedFields" => {}, + "nestedSourcedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -157,7 +162,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => []}, - "id" => "17" + "id" => "17", + "nestedSourcedFields" => {}, + "nestedSourcedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -178,7 +185,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, - "id" => "embedded_workspace_id" + "id" => "embedded_workspace_id", + "nestedSourcedFields" => {}, + "nestedSourcedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -203,7 +212,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"embedded_values.missing_field" => [], "name" => nil}, - "id" => "17" + "id" => "17", + "nestedSourcedFields" => {}, + "nestedSourcedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -236,7 +247,9 @@ module Operation "workspace_id" => "embedded_workspace_id" } }, - "id" => "17" + "id" => "17", + "nestedSourcedFields" => {}, + "nestedSourcedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -262,7 +275,9 @@ module Operation script: {id: operations.first.update_target.script_id, params: { # Float-typed integer values are coerced to true ints before indexing "topLevelFields" => {"size" => [an_instance_of(::Integer).and(eq_to(4))]}, - "id" => "17" + "id" => "17", + "nestedSourcedFields" => {}, + "nestedSourcedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -283,7 +298,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, - "id" => "17" + "id" => "17", + "nestedSourcedFields" => {}, + "nestedSourcedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -292,7 +309,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, - "id" => "18" + "id" => "18", + "nestedSourcedFields" => {}, + "nestedSourcedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -301,7 +320,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, - "id" => "19" + "id" => "19", + "nestedSourcedFields" => {}, + "nestedSourcedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb index 9e4f26eb0..c2aeaf218 100644 --- a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb @@ -8,6 +8,7 @@ require "elastic_graph/schema_artifacts/runtime_metadata/hash_dumper" require "elastic_graph/schema_artifacts/runtime_metadata/index_field" +require "elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment" require "elastic_graph/schema_artifacts/runtime_metadata/sort_field" module ElasticGraph @@ -16,22 +17,24 @@ module RuntimeMetadata # Runtime metadata related to a datastore index definition. # # @private - class IndexDefinition < ::Data.define(:route_with, :rollover, :default_sort_fields, :current_sources, :fields_by_path, :has_had_multiple_sources) + class IndexDefinition < ::Data.define(:route_with, :rollover, :default_sort_fields, :current_sources, :fields_by_path, :has_had_multiple_sources, :nested_sourced_paths) ROUTE_WITH = "route_with" ROLLOVER = "rollover" DEFAULT_SORT_FIELDS = "default_sort_fields" CURRENT_SOURCES = "current_sources" FIELDS_BY_PATH = "fields_by_path" HAS_HAD_MULTIPLE_SOURCES = "has_had_multiple_sources" + NESTED_SOURCED_PATHS = "nested_sourced_paths" - def initialize(route_with:, rollover:, default_sort_fields:, current_sources:, fields_by_path:, has_had_multiple_sources:) + def initialize(route_with:, rollover:, default_sort_fields:, current_sources:, fields_by_path:, has_had_multiple_sources:, nested_sourced_paths:) super( route_with: route_with, rollover: rollover, default_sort_fields: default_sort_fields, current_sources: current_sources.to_set, fields_by_path: fields_by_path, - has_had_multiple_sources: has_had_multiple_sources + has_had_multiple_sources: has_had_multiple_sources, + nested_sourced_paths: nested_sourced_paths ) end @@ -42,7 +45,8 @@ def self.from_hash(hash) default_sort_fields: hash[DEFAULT_SORT_FIELDS]&.map { |h| SortField.from_hash(h) } || [], current_sources: hash[CURRENT_SOURCES] || [], fields_by_path: (hash[FIELDS_BY_PATH] || {}).transform_values { |h| IndexField.from_hash(h) }, - has_had_multiple_sources: hash[HAS_HAD_MULTIPLE_SOURCES] || false + has_had_multiple_sources: hash[HAS_HAD_MULTIPLE_SOURCES] || false, + nested_sourced_paths: (hash[NESTED_SOURCED_PATHS] || {}).transform_values { |segments| segments.map { |h| NestedSourcedPathSegment.from_hash(h) } } ) end @@ -53,6 +57,7 @@ def to_dumpable_hash DEFAULT_SORT_FIELDS => default_sort_fields.map(&:to_dumpable_hash), FIELDS_BY_PATH => HashDumper.dump_hash(fields_by_path, &:to_dumpable_hash), HAS_HAD_MULTIPLE_SOURCES => (true if has_had_multiple_sources), + NESTED_SOURCED_PATHS => nested_sourced_paths.transform_values { |segments| segments.map(&:to_dumpable_hash) }, ROLLOVER => rollover&.to_dumpable_hash, ROUTE_WITH => route_with } diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rb new file mode 100644 index 000000000..d06df78e1 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rb @@ -0,0 +1,50 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/params" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Bundles the param configuration for nested sourced_from update targets. + # `field_params` defines which fields to extract from the event and write onto + # the target nested element. `path_identifier_params` defines which values to + # extract from the event to identify which nested element to target. + # + # @private + class NestedSourcedDataParams < ::Data.define(:field_params, :path_identifier_params) + FIELD_PARAMS = "field_params" + PATH_IDENTIFIER_PARAMS = "path_identifier_params" + + EMPTY = new(field_params: {}, path_identifier_params: {}) + + def self.from_hash(hash) + new( + field_params: Param.load_params_hash(hash[FIELD_PARAMS] || {}), + path_identifier_params: Param.load_params_hash(hash[PATH_IDENTIFIER_PARAMS] || {}) + ) + end + + def to_dumpable_hash + { + FIELD_PARAMS => Param.dump_params_hash(field_params), + PATH_IDENTIFIER_PARAMS => Param.dump_params_hash(path_identifier_params) + } + end + + # Resolves params into script-ready values from the given prepared record. + def script_params_for(prepared_record) + { + "nestedSourcedFields" => field_params.transform_values { |param| param.value_for(prepared_record) }, + "nestedSourcedPathIdentifiers" => path_identifier_params.transform_values { |param| param.value_for(prepared_record) } + } + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rb new file mode 100644 index 000000000..ee185ee80 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rb @@ -0,0 +1,70 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Represents a segment in a nested sourced path that navigates into a list field, + # matching an element by a key field. + # + # @private + class ListPathSegment < ::Data.define(:field, :match_field, :source_field) + # @dynamic to_painless_param + TYPE = "list" + FIELD = "field" + MATCH_FIELD = "matchField" + SOURCE_FIELD = "sourceField" + + def to_dumpable_hash + {"type" => TYPE, FIELD => field, MATCH_FIELD => match_field, SOURCE_FIELD => source_field} + end + + alias_method :to_painless_param, :to_dumpable_hash + + def self.from_hash(hash) + new(field: hash[FIELD], match_field: hash[MATCH_FIELD], source_field: hash[SOURCE_FIELD]) + end + end + + # Represents a segment in a nested sourced path that navigates into an object field. + # + # @private + class ObjectPathSegment < ::Data.define(:field) + # @dynamic to_painless_param + TYPE = "object" + FIELD = "field" + + def to_dumpable_hash + {"type" => TYPE, FIELD => field} + end + + alias_method :to_painless_param, :to_dumpable_hash + + def self.from_hash(hash) + new(field: hash[FIELD]) + end + end + + # @private + module NestedSourcedPathSegment + def self.from_hash(hash) + case hash["type"] + when ListPathSegment::TYPE + ListPathSegment.from_hash(hash) + when ObjectPathSegment::TYPE + ObjectPathSegment.from_hash(hash) + # :nocov: -- defensive error for corrupted YAML; not worth a dedicated test. + else + raise "Unknown nested sourced path segment type: #{hash["type"].inspect}" + # :nocov: + end + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb index 37c021ae5..38a8c5d39 100644 --- a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/constants" +require "elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params" require "elastic_graph/schema_artifacts/runtime_metadata/params" module ElasticGraph @@ -23,6 +24,7 @@ class UpdateTarget < ::Data.define( :routing_value_source, :rollover_timestamp_value_source, :top_level_fields_params, + :nested_sourced_data_params, :metadata_params ) TYPE = "type" @@ -32,6 +34,7 @@ class UpdateTarget < ::Data.define( ROUTING_VALUE_SOURCE = "routing_value_source" ROLLOVER_TIMESTAMP_VALUE_SOURCE = "rollover_timestamp_value_source" TOP_LEVEL_FIELDS_PARAMS = "top_level_fields_params" + NESTED_SOURCED_DATA_PARAMS = "nested_sourced_data_params" METADATA_PARAMS = "metadata_params" def self.from_hash(hash) @@ -43,6 +46,7 @@ def self.from_hash(hash) routing_value_source: hash[ROUTING_VALUE_SOURCE], rollover_timestamp_value_source: hash[ROLLOVER_TIMESTAMP_VALUE_SOURCE], top_level_fields_params: Param.load_params_hash(hash[TOP_LEVEL_FIELDS_PARAMS] || {}), + nested_sourced_data_params: NestedSourcedDataParams.from_hash(hash[NESTED_SOURCED_DATA_PARAMS] || {}), metadata_params: Param.load_params_hash(hash[METADATA_PARAMS] || {}) ) end @@ -52,6 +56,7 @@ def to_dumpable_hash # Keys here are ordered alphabetically; please keep them that way. ID_SOURCE => id_source, METADATA_PARAMS => Param.dump_params_hash(metadata_params), + NESTED_SOURCED_DATA_PARAMS => nested_sourced_data_params.to_dumpable_hash, RELATIONSHIP => relationship, ROLLOVER_TIMESTAMP_VALUE_SOURCE => rollover_timestamp_value_source, ROUTING_VALUE_SOURCE => routing_value_source, @@ -74,7 +79,10 @@ def params_for(doc_id:, event:, prepared_record:) [name, param.value_for(event)] end - meta.merge({"id" => doc_id, "topLevelFields" => top_level_fields}) + meta.merge( + {"id" => doc_id, "topLevelFields" => top_level_fields}, + nested_sourced_data_params.script_params_for(prepared_record) + ) end end end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs index fc6bacfcd..c30824c7c 100644 --- a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs @@ -8,6 +8,7 @@ module ElasticGraph attr_reader current_sources: ::Set[::String] attr_reader fields_by_path: ::Hash[::String, IndexField] attr_reader has_had_multiple_sources: bool + attr_reader nested_sourced_paths: ::Hash[::String, ::Array[nestedSourcedPathSegment]] def initialize: ( route_with: ::String, @@ -15,7 +16,8 @@ module ElasticGraph default_sort_fields: ::Array[SortField], current_sources: ::Set[::String], fields_by_path: ::Hash[::String, IndexField], - has_had_multiple_sources: bool + has_had_multiple_sources: bool, + nested_sourced_paths: ::Hash[::String, ::Array[nestedSourcedPathSegment]] ) -> void def with: ( @@ -24,7 +26,8 @@ module ElasticGraph ?default_sort_fields: ::Array[SortField], ?current_sources: ::Enumerable[::String], ?fields_by_path: ::Hash[::String, IndexField], - ?has_had_multiple_sources: bool + ?has_had_multiple_sources: bool, + ?nested_sourced_paths: ::Hash[::String, ::Array[nestedSourcedPathSegment]] ) -> IndexDefinition end @@ -35,6 +38,7 @@ module ElasticGraph CURRENT_SOURCES: "current_sources" FIELDS_BY_PATH: "fields_by_path" HAS_HAD_MULTIPLE_SOURCES: "has_had_multiple_sources" + NESTED_SOURCED_PATHS: "nested_sourced_paths" def initialize: ( route_with: ::String, @@ -42,7 +46,8 @@ module ElasticGraph default_sort_fields: ::Array[SortField], current_sources: ::Enumerable[::String], fields_by_path: ::Hash[::String, IndexField], - has_had_multiple_sources: bool + has_had_multiple_sources: bool, + nested_sourced_paths: ::Hash[::String, ::Array[nestedSourcedPathSegment]] ) -> void def self.from_hash: (::Hash[::String, untyped]) -> IndexDefinition diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rbs new file mode 100644 index 000000000..5e916516b --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class NestedSourcedDataParamsSuperType + attr_reader field_params: paramsHash + attr_reader path_identifier_params: paramsHash + + def initialize: (field_params: paramsHash, path_identifier_params: paramsHash) -> void + end + + class NestedSourcedDataParams < NestedSourcedDataParamsSuperType + FIELD_PARAMS: "field_params" + PATH_IDENTIFIER_PARAMS: "path_identifier_params" + EMPTY: NestedSourcedDataParams + + def self.from_hash: (::Hash[::String, untyped]) -> NestedSourcedDataParams + def to_dumpable_hash: () -> ::Hash[::String, untyped] + def script_params_for: (::Hash[::String, untyped]) -> ::Hash[::String, ::Hash[::String, untyped]] + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rbs new file mode 100644 index 000000000..615cb89f5 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rbs @@ -0,0 +1,45 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class ListPathSegmentSuperType + attr_reader field: ::String + attr_reader match_field: ::String + attr_reader source_field: ::String + + def initialize: (field: ::String, match_field: ::String, source_field: ::String) -> void + end + + class ListPathSegment < ListPathSegmentSuperType + TYPE: "list" + FIELD: "field" + MATCH_FIELD: "matchField" + SOURCE_FIELD: "sourceField" + + def self.from_hash: (::Hash[::String, untyped]) -> ListPathSegment + def to_dumpable_hash: () -> ::Hash[::String, ::String] + alias to_painless_param to_dumpable_hash + end + + class ObjectPathSegmentSuperType + attr_reader field: ::String + + def initialize: (field: ::String) -> void + end + + class ObjectPathSegment < ObjectPathSegmentSuperType + TYPE: "object" + FIELD: "field" + + def self.from_hash: (::Hash[::String, untyped]) -> ObjectPathSegment + def to_dumpable_hash: () -> ::Hash[::String, ::String] + alias to_painless_param to_dumpable_hash + end + + type nestedSourcedPathSegment = ListPathSegment | ObjectPathSegment + + module NestedSourcedPathSegment + def self.from_hash: (::Hash[::String, untyped]) -> nestedSourcedPathSegment + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs index 4a43bfdb3..99647cd38 100644 --- a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs @@ -9,6 +9,7 @@ module ElasticGraph attr_reader routing_value_source: ::String? attr_reader rollover_timestamp_value_source: ::String? attr_reader top_level_fields_params: paramsHash + attr_reader nested_sourced_data_params: NestedSourcedDataParams attr_reader metadata_params: paramsHash def initialize: ( @@ -19,6 +20,7 @@ module ElasticGraph routing_value_source: ::String?, rollover_timestamp_value_source: ::String?, top_level_fields_params: paramsHash, + nested_sourced_data_params: NestedSourcedDataParams, metadata_params: paramsHash ) -> void @@ -30,6 +32,7 @@ module ElasticGraph ?routing_value_source: ::String?, ?rollover_timestamp_value_source: ::String?, ?top_level_fields_params: paramsHash, + ?nested_sourced_data_params: NestedSourcedDataParams, ?metadata_params: paramsHash ) -> UpdateTarget @@ -46,6 +49,7 @@ module ElasticGraph ROUTING_VALUE_SOURCE: "routing_value_source" ROLLOVER_TIMESTAMP_VALUE_SOURCE: "rollover_timestamp_value_source" TOP_LEVEL_FIELDS_PARAMS: "top_level_fields_params" + NESTED_SOURCED_DATA_PARAMS: "nested_sourced_data_params" METADATA_PARAMS: "metadata_params" def self.from_hash: (::Hash[::String, untyped]) -> UpdateTarget diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb index a84759af8..b04862652 100644 --- a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb @@ -24,7 +24,8 @@ module RuntimeMetadata default_sort_fields: [], current_sources: Set.new, fields_by_path: {}, - has_had_multiple_sources: false + has_had_multiple_sources: false, + nested_sourced_paths: {} ) end @@ -51,6 +52,26 @@ module RuntimeMetadata expect(index_def_with_flag.to_dumpable_hash["has_had_multiple_sources"]).to eq true end + it "roundtrips nested_sourced_paths through to_dumpable_hash and from_hash" do + index_def = index_definition_with( + nested_sourced_paths: { + "foo" => [ + ListPathSegment.new(field: "bar", match_field: "id", source_field: "bazId"), + ObjectPathSegment.new(field: "qux") + ] + } + ) + + roundtripped = IndexDefinition.from_hash(index_def.to_dumpable_hash) + + expect(roundtripped.nested_sourced_paths).to eq({ + "foo" => [ + ListPathSegment.new(field: "bar", match_field: "id", source_field: "bazId"), + ObjectPathSegment.new(field: "qux") + ] + }) + end + describe IndexDefinition::Rollover do it "builds from a minimal hash" do rollover = IndexDefinition::Rollover.from_hash({"frequency" => "yearly"}) diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb index c9f3733eb..961debf3b 100644 --- a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb @@ -43,6 +43,7 @@ module RuntimeMetadata routing_value_source: "cost.currency_name", rollover_timestamp_value_source: "currency_introduced_on", top_level_fields_params: {"workspace_id" => DynamicParam.new(source_path: "wid", cardinality: :one)}, + nested_sourced_data_params: NestedSourcedDataParams::EMPTY, metadata_params: {"relationshipName" => StaticParam.new(value: "currency")} ), UpdateTarget.new( @@ -53,6 +54,7 @@ module RuntimeMetadata routing_value_source: nil, rollover_timestamp_value_source: nil, top_level_fields_params: {}, + nested_sourced_data_params: NestedSourcedDataParams::EMPTY, metadata_params: {} ) ], @@ -122,7 +124,8 @@ module RuntimeMetadata fields_by_path: { "foo.bar" => IndexField.new(source: "other") }, - has_had_multiple_sources: false + has_had_multiple_sources: false, + nested_sourced_paths: {} ), "addresses" => IndexDefinition.new( route_with: nil, @@ -130,7 +133,8 @@ module RuntimeMetadata default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, - has_had_multiple_sources: false + has_had_multiple_sources: false, + nested_sourced_paths: {} ), "components" => IndexDefinition.new( route_with: "group_id", @@ -138,7 +142,8 @@ module RuntimeMetadata default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, - has_had_multiple_sources: true + has_had_multiple_sources: true, + nested_sourced_paths: {} ) }, schema_element_names: SchemaElementNames.new( @@ -311,6 +316,7 @@ module RuntimeMetadata routing_value_source: nil, rollover_timestamp_value_source: nil, top_level_fields_params: {"workspace_id" => dynamic_param_with(cardinality: :many)}, + nested_sourced_data_params: NestedSourcedDataParams::EMPTY, metadata_params: {} )]), "IndexDefinitionNamesOnly" => object_type_with(index_definition_names: ["foo", "bar"]), diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb index d3ee76a61..340b106a2 100644 --- a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb @@ -26,6 +26,7 @@ module RuntimeMetadata routing_value_source: nil, rollover_timestamp_value_source: nil, top_level_fields_params: {}, + nested_sourced_data_params: NestedSourcedDataParams::EMPTY, metadata_params: {} ) end @@ -84,9 +85,9 @@ module RuntimeMetadata } ) - without_id_or_top_level_fields = params.except("id", "topLevelFields") + metadata_params_only = params.except("id", "topLevelFields", "nestedSourcedFields", "nestedSourcedPathIdentifiers") - expect(without_id_or_top_level_fields).to eq( + expect(metadata_params_only).to eq( "foo" => 43, "bar" => "hello", "bazz" => [12] @@ -115,9 +116,23 @@ module RuntimeMetadata ) end - def params_for(doc_id: "doc_id", event: {}, prepared_record: {}, top_level_fields_params: {}, metadata_params: {}) + it "includes nested_sourced_data_params resolved from the prepared_record" do + params = params_for( + nested_sourced_data_params: NestedSourcedDataParams.new( + field_params: {"foo" => dynamic_param_with(source_path: "some.field", cardinality: :one)}, + path_identifier_params: {"bar" => dynamic_param_with(source_path: "some.other", cardinality: :one)} + ), + prepared_record: {"some" => {"field" => "hello", "other" => "abc"}} + ) + + expect(params["nestedSourcedFields"]).to eq({"foo" => "hello"}) + expect(params["nestedSourcedPathIdentifiers"]).to eq({"bar" => "abc"}) + end + + def params_for(doc_id: "doc_id", event: {}, prepared_record: {}, top_level_fields_params: {}, nested_sourced_data_params: NestedSourcedDataParams::EMPTY, metadata_params: {}) update_target = normal_indexing_update_target_with( top_level_fields_params: top_level_fields_params, + nested_sourced_data_params: nested_sourced_data_params, metadata_params: metadata_params ) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb index a01df76c5..a8cd351f9 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb @@ -272,7 +272,8 @@ def runtime_metadata_for_source_type metadata_params: {}, top_level_fields_params: fields.map(&:source_field).to_h do |f| [f, SchemaArtifacts::RuntimeMetadata::DynamicParam.new(source_path: f, cardinality: :many)] - end + end, + nested_sourced_data_params: SchemaArtifacts::RuntimeMetadata::NestedSourcedDataParams::EMPTY ) end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb index a6400db64..ac6e38a2b 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb @@ -264,7 +264,8 @@ def runtime_metadata direction: direction ) end, - has_had_multiple_sources: has_had_multiple_sources_flag + has_had_multiple_sources: has_had_multiple_sources_flag, + nested_sourced_paths: {} # TODO: Populate with real data once paths are registered on the index ) end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb index 95790c774..f5acf0df9 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb @@ -18,7 +18,8 @@ def self.new_normal_indexing_update_target( id_source:, top_level_fields_params:, routing_value_source:, - rollover_timestamp_value_source: + rollover_timestamp_value_source:, + nested_sourced_data_params: ) SchemaArtifacts::RuntimeMetadata::UpdateTarget.new( type: type, @@ -29,6 +30,7 @@ def self.new_normal_indexing_update_target( "relationship" => SchemaArtifacts::RuntimeMetadata::StaticParam.new(value: relationship) }), top_level_fields_params: top_level_fields_params, + nested_sourced_data_params: nested_sourced_data_params, routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source ) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb index feca5528e..455137726 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb @@ -49,6 +49,7 @@ def resolve relationship: resolved_relationship.relationship_name, id_source: resolved_relationship.relation_metadata.foreign_key, top_level_fields_params: top_level_fields_params, + nested_sourced_data_params: SchemaArtifacts::RuntimeMetadata::NestedSourcedDataParams::EMPTY, routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source ) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb index 98dd547a7..f6834b105 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb @@ -446,6 +446,7 @@ def self_update_target relationship: SELF_RELATIONSHIP_NAME, id_source: "id", top_level_fields_params: top_level_fields_params, + nested_sourced_data_params: SchemaArtifacts::RuntimeMetadata::NestedSourcedDataParams::EMPTY, # Some day we may want to consider supporting multiple indices. If/when we add support for that, # we'll need to change the runtime metadata here to have a map of these values, keyed by index # name. diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs index 4375b8138..5b814d25a 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs @@ -8,7 +8,8 @@ module ElasticGraph id_source: ::String, top_level_fields_params: SchemaArtifacts::RuntimeMetadata::paramsHash, routing_value_source: ::String?, - rollover_timestamp_value_source: ::String? + rollover_timestamp_value_source: ::String?, + nested_sourced_data_params: SchemaArtifacts::RuntimeMetadata::NestedSourcedDataParams ) -> SchemaArtifacts::RuntimeMetadata::UpdateTarget private diff --git a/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb b/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb index 705031d12..c35bd289e 100644 --- a/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb +++ b/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb @@ -72,6 +72,7 @@ def derived_indexing_update_target_with( routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source, top_level_fields_params: top_level_fields_params, + nested_sourced_data_params: NestedSourcedDataParams::EMPTY, metadata_params: metadata_params ) end @@ -83,6 +84,7 @@ def normal_indexing_update_target_with( routing_value_source: "routing_value_source", rollover_timestamp_value_source: "rollover_timestamp_value_source", top_level_fields_params: {}, + nested_sourced_data_params: NestedSourcedDataParams::EMPTY, metadata_params: {} ) UpdateTarget.new( @@ -93,6 +95,7 @@ def normal_indexing_update_target_with( routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source, top_level_fields_params: top_level_fields_params, + nested_sourced_data_params: nested_sourced_data_params, metadata_params: metadata_params ) end @@ -112,14 +115,15 @@ def static_param_with(value) StaticParam.new(value: value) end - def index_definition_with(route_with: nil, rollover: nil, default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, has_had_multiple_sources: false) + def index_definition_with(route_with: nil, rollover: nil, default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, has_had_multiple_sources: false, nested_sourced_paths: {}) IndexDefinition.new( route_with: route_with, rollover: rollover, default_sort_fields: default_sort_fields, current_sources: current_sources, fields_by_path: fields_by_path, - has_had_multiple_sources: has_had_multiple_sources + has_had_multiple_sources: has_had_multiple_sources, + nested_sourced_paths: nested_sourced_paths ) end From 6ec28184907593925d4543bf9573793efee345eb Mon Sep 17 00:00:00 2001 From: ellisandrews-toast Date: Mon, 1 Jun 2026 22:47:42 -0400 Subject: [PATCH 2/3] Address PR #1228 feedback on naming, structure, and scope --- .../datastore_core/index_definition.rb | 5 +- .../datastore_core/index_definition/index.rb | 5 +- .../rollover_index_template.rb | 4 +- .../datastore_core/index_definition.rbs | 1 - .../index_definition/index_spec.rb | 2 +- .../rollover_index_template_spec.rb | 2 +- .../elastic_graph/indexer/operation/update.rb | 11 ++--- .../indexer/datastore_indexing_router_spec.rb | 2 +- .../indexer/operation/update_spec.rb | 43 ++++++++-------- .../runtime_metadata/index_definition.rb | 16 +++--- ...arams.rb => sourced_from_nested_params.rb} | 6 +-- ...rb => sourced_from_nested_path_segment.rb} | 49 ++++++++----------- .../runtime_metadata/update_target.rb | 12 ++--- .../runtime_metadata/index_definition.rbs | 10 ++-- ...ams.rbs => sourced_from_nested_params.rbs} | 8 +-- ...s => sourced_from_nested_path_segment.rbs} | 14 ++---- .../runtime_metadata/update_target.rbs | 8 +-- .../runtime_metadata/index_definition_spec.rb | 22 +-------- .../runtime_metadata/schema_spec.rb | 40 ++++++++++----- .../runtime_metadata/update_target_spec.rb | 16 +++--- .../indexing/derived_indexed_type.rb | 2 +- .../schema_definition/indexing/index.rb | 2 +- .../indexing/update_target_factory.rb | 4 +- .../indexing/update_target_resolver.rb | 2 +- .../schema_definition/mixins/has_indices.rb | 2 +- .../indexing/update_target_factory.rbs | 2 +- .../spec_support/runtime_metadata_support.rb | 10 ++-- 27 files changed, 138 insertions(+), 162 deletions(-) rename elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/{nested_sourced_data_params.rb => sourced_from_nested_params.rb} (83%) rename elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/{nested_sourced_path_segment.rb => sourced_from_nested_path_segment.rb} (61%) rename elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/{nested_sourced_data_params.rbs => sourced_from_nested_params.rbs} (71%) rename elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/{nested_sourced_path_segment.rbs => sourced_from_nested_path_segment.rbs} (70%) diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition.rb index 7f39134e7..c95dc2f7c 100644 --- a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition.rb +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition.rb @@ -33,10 +33,7 @@ def self.with(name:, runtime_metadata:, config:, datastore_clients_by_name:, sch env_index_config: env_index_config, defined_clusters: config.clusters.keys.to_set, datastore_clients_by_name: datastore_clients_by_name, - has_had_multiple_sources: runtime_metadata.has_had_multiple_sources, - # :nocov: -- the block is only exercised with non-empty nested_sourced_paths, which will come in a future PR - nested_sourced_paths: runtime_metadata.nested_sourced_paths.transform_values { |segments| segments.map(&:to_painless_param) } - # :nocov: + has_had_multiple_sources: runtime_metadata.has_had_multiple_sources } if (rollover = runtime_metadata.rollover) diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/index.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/index.rb index a85c309fb..1b99e7a4e 100644 --- a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/index.rb +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/index.rb @@ -15,12 +15,11 @@ class DatastoreCore module IndexDefinition class Index < Support::MemoizableData.define( :name, :route_with, :default_sort_clauses, :current_sources, :fields_by_path, - :env_index_config, :defined_clusters, :datastore_clients_by_name, :env_agnostic_settings, :has_had_multiple_sources, - :nested_sourced_paths + :env_index_config, :defined_clusters, :datastore_clients_by_name, :env_agnostic_settings, :has_had_multiple_sources ) # `Data.define` provides all these methods: # @dynamic name, route_with, default_sort_clauses, current_sources, fields_by_path, env_index_config, env_agnostic_settings - # @dynamic defined_clusters, datastore_clients_by_name, initialize, has_had_multiple_sources, nested_sourced_paths + # @dynamic defined_clusters, datastore_clients_by_name, initialize, has_had_multiple_sources # `include IndexDefinition::Base` provides all these methods. Steep should be able to detect it # but can't for some reason so we have to declare them with `@dynamic`. diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index_template.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index_template.rb index 44ea8bda7..9a70836da 100644 --- a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index_template.rb +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_definition/rollover_index_template.rb @@ -23,11 +23,11 @@ module IndexDefinition class RolloverIndexTemplate < Support::MemoizableData.define( :name, :route_with, :default_sort_clauses, :current_sources, :fields_by_path, :env_index_config, :index_args, :defined_clusters, :datastore_clients_by_name, :timestamp_field_path, :frequency, - :env_agnostic_settings, :has_had_multiple_sources, :nested_sourced_paths + :env_agnostic_settings, :has_had_multiple_sources ) # `Data.define` provides all these methods: # @dynamic name, route_with, default_sort_clauses, current_sources, fields_by_path, env_index_config, env_agnostic_settings - # @dynamic index_args, defined_clusters, datastore_clients_by_name, timestamp_field_path, frequency, initialize, has_had_multiple_sources, nested_sourced_paths + # @dynamic index_args, defined_clusters, datastore_clients_by_name, timestamp_field_path, frequency, initialize, has_had_multiple_sources # `include IndexDefinition::Base` provides all these methods. Steep should be able to detect it # but can't for some reason so we have to declare them with `@dynamic`. diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition.rbs index 8d087e939..b12ac2343 100644 --- a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition.rbs +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/index_definition.rbs @@ -26,7 +26,6 @@ module ElasticGraph def current_sources: () -> ::Set[::String] def fields_by_path: () -> ::Hash[::String, SchemaArtifacts::RuntimeMetadata::IndexField] def has_had_multiple_sources: () -> bool - def nested_sourced_paths: () -> ::Hash[::String, ::Array[::Hash[::String, ::String]]] def env_index_config: () -> Configuration::IndexDefinition def env_agnostic_settings: () -> ::Hash[::String, untyped] def defined_clusters: () -> ::Set[::String] diff --git a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb index 37ede53d9..fc3b126c7 100644 --- a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb +++ b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb @@ -89,7 +89,7 @@ def index_def_named(name, rollover: nil) current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, has_had_multiple_sources: false, - nested_sourced_paths: {} + sourced_from_nested_paths_by_relationship: {} ) DatastoreCore::IndexDefinition.with( diff --git a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb index 4b77b3b09..7026e8a2c 100644 --- a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb +++ b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb @@ -558,7 +558,7 @@ def index_def_named(name, rollover: nil) current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, has_had_multiple_sources: false, - nested_sourced_paths: {} + sourced_from_nested_paths_by_relationship: {} ) DatastoreCore::IndexDefinition.with( diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/operation/update.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/update.rb index 52e685552..32e04c04a 100644 --- a/elasticgraph-indexer/lib/elastic_graph/indexer/operation/update.rb +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/operation/update.rb @@ -145,16 +145,11 @@ def script_params prepared_record: prepared_record ) - # The normal indexing script uses `__counts` and `nestedSourcedPaths`. Other indexing scripts - # (e.g. the ones generated for derived indexing) do not use these so there's no point in - # spending effort on computing them. Plus, the logic below raises an exception in that case, - # so it's important we avoid it. + # The normal indexing script uses `__counts`. Other indexing scripts (e.g. the ones generated + # for derived indexing) do not use `__counts` so there's no point in spending effort on computing + # it. Plus, the logic below raises an exception in that case, so it's important we avoid it. return initial_params unless update_target.for_normal_indexing? - # `nestedSourcedPaths` is static per-index configuration (describing how to navigate to nested - # elements), unlike the per-event params resolved by `update_target.params_for` above. - initial_params["nestedSourcedPaths"] = destination_index_def.nested_sourced_paths - CountAccumulator.merge_list_counts_into( initial_params, mapping: destination_index_mapping, diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb index ca63cbdb5..1baad1cd5 100644 --- a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb @@ -265,7 +265,7 @@ def type_name_for_index(index_name) upsert: {}, script: a_hash_including( id: /WidgetCurrency_from_Widget_/, - params: {"topLevelFields" => {"name" => ["thing1"]}, "id" => "USD", "nestedSourcedFields" => {}, "nestedSourcedPathIdentifiers" => {}} + params: {"topLevelFields" => {"name" => ["thing1"]}, "id" => "USD", "sourcedFromNestedFields" => {}, "sourcedFromNestedPathIdentifiers" => {}} ) ) end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb index 05bb2c273..7b32bde9f 100644 --- a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb @@ -75,8 +75,8 @@ module Operation script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, "id" => "17", - "nestedSourcedFields" => {}, - "nestedSourcedPathIdentifiers" => {} + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -106,12 +106,11 @@ module Operation script: {id: INDEX_DATA_UPDATE_SCRIPT_ID, params: { "topLevelFields" => {"name" => "thing1"}, "id" => "17", + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {}, "staticValue" => 47, "sourceType" => "Widget", - LIST_COUNTS_FIELD => {"sizes" => 0, "widget_names" => 0}, - "nestedSourcedPaths" => {}, - "nestedSourcedFields" => {}, - "nestedSourcedPathIdentifiers" => {} + LIST_COUNTS_FIELD => {"sizes" => 0, "widget_names" => 0} }}, scripted_upsert: true, upsert: {} @@ -163,8 +162,8 @@ module Operation script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => []}, "id" => "17", - "nestedSourcedFields" => {}, - "nestedSourcedPathIdentifiers" => {} + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -186,8 +185,8 @@ module Operation script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, "id" => "embedded_workspace_id", - "nestedSourcedFields" => {}, - "nestedSourcedPathIdentifiers" => {} + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -213,8 +212,8 @@ module Operation script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"embedded_values.missing_field" => [], "name" => nil}, "id" => "17", - "nestedSourcedFields" => {}, - "nestedSourcedPathIdentifiers" => {} + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -248,8 +247,8 @@ module Operation } }, "id" => "17", - "nestedSourcedFields" => {}, - "nestedSourcedPathIdentifiers" => {} + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -276,8 +275,8 @@ module Operation # Float-typed integer values are coerced to true ints before indexing "topLevelFields" => {"size" => [an_instance_of(::Integer).and(eq_to(4))]}, "id" => "17", - "nestedSourcedFields" => {}, - "nestedSourcedPathIdentifiers" => {} + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -299,8 +298,8 @@ module Operation script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, "id" => "17", - "nestedSourcedFields" => {}, - "nestedSourcedPathIdentifiers" => {} + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -310,8 +309,8 @@ module Operation script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, "id" => "18", - "nestedSourcedFields" => {}, - "nestedSourcedPathIdentifiers" => {} + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -321,8 +320,8 @@ module Operation script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, "id" => "19", - "nestedSourcedFields" => {}, - "nestedSourcedPathIdentifiers" => {} + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb index c2aeaf218..960ce9bb1 100644 --- a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb @@ -8,7 +8,7 @@ require "elastic_graph/schema_artifacts/runtime_metadata/hash_dumper" require "elastic_graph/schema_artifacts/runtime_metadata/index_field" -require "elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment" +require "elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment" require "elastic_graph/schema_artifacts/runtime_metadata/sort_field" module ElasticGraph @@ -17,16 +17,16 @@ module RuntimeMetadata # Runtime metadata related to a datastore index definition. # # @private - class IndexDefinition < ::Data.define(:route_with, :rollover, :default_sort_fields, :current_sources, :fields_by_path, :has_had_multiple_sources, :nested_sourced_paths) + class IndexDefinition < ::Data.define(:route_with, :rollover, :default_sort_fields, :current_sources, :fields_by_path, :has_had_multiple_sources, :sourced_from_nested_paths_by_relationship) ROUTE_WITH = "route_with" ROLLOVER = "rollover" DEFAULT_SORT_FIELDS = "default_sort_fields" CURRENT_SOURCES = "current_sources" FIELDS_BY_PATH = "fields_by_path" HAS_HAD_MULTIPLE_SOURCES = "has_had_multiple_sources" - NESTED_SOURCED_PATHS = "nested_sourced_paths" + SOURCED_FROM_NESTED_PATHS_BY_RELATIONSHIP = "sourced_from_nested_paths_by_relationship" - def initialize(route_with:, rollover:, default_sort_fields:, current_sources:, fields_by_path:, has_had_multiple_sources:, nested_sourced_paths:) + def initialize(route_with:, rollover:, default_sort_fields:, current_sources:, fields_by_path:, has_had_multiple_sources:, sourced_from_nested_paths_by_relationship:) super( route_with: route_with, rollover: rollover, @@ -34,7 +34,7 @@ def initialize(route_with:, rollover:, default_sort_fields:, current_sources:, f current_sources: current_sources.to_set, fields_by_path: fields_by_path, has_had_multiple_sources: has_had_multiple_sources, - nested_sourced_paths: nested_sourced_paths + sourced_from_nested_paths_by_relationship: sourced_from_nested_paths_by_relationship ) end @@ -46,7 +46,7 @@ def self.from_hash(hash) current_sources: hash[CURRENT_SOURCES] || [], fields_by_path: (hash[FIELDS_BY_PATH] || {}).transform_values { |h| IndexField.from_hash(h) }, has_had_multiple_sources: hash[HAS_HAD_MULTIPLE_SOURCES] || false, - nested_sourced_paths: (hash[NESTED_SOURCED_PATHS] || {}).transform_values { |segments| segments.map { |h| NestedSourcedPathSegment.from_hash(h) } } + sourced_from_nested_paths_by_relationship: (hash[SOURCED_FROM_NESTED_PATHS_BY_RELATIONSHIP] || {}).transform_values { |segments| segments.map { |h| SourcedFromNestedPathSegment.from_hash(h) } } ) end @@ -57,9 +57,9 @@ def to_dumpable_hash DEFAULT_SORT_FIELDS => default_sort_fields.map(&:to_dumpable_hash), FIELDS_BY_PATH => HashDumper.dump_hash(fields_by_path, &:to_dumpable_hash), HAS_HAD_MULTIPLE_SOURCES => (true if has_had_multiple_sources), - NESTED_SOURCED_PATHS => nested_sourced_paths.transform_values { |segments| segments.map(&:to_dumpable_hash) }, ROLLOVER => rollover&.to_dumpable_hash, - ROUTE_WITH => route_with + ROUTE_WITH => route_with, + SOURCED_FROM_NESTED_PATHS_BY_RELATIONSHIP => sourced_from_nested_paths_by_relationship.transform_values { |segments| segments.map(&:to_dumpable_hash) } } end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rb similarity index 83% rename from elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rb rename to elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rb index d06df78e1..da1fd1d21 100644 --- a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rb +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rb @@ -17,7 +17,7 @@ module RuntimeMetadata # extract from the event to identify which nested element to target. # # @private - class NestedSourcedDataParams < ::Data.define(:field_params, :path_identifier_params) + class SourcedFromNestedParams < ::Data.define(:field_params, :path_identifier_params) FIELD_PARAMS = "field_params" PATH_IDENTIFIER_PARAMS = "path_identifier_params" @@ -40,8 +40,8 @@ def to_dumpable_hash # Resolves params into script-ready values from the given prepared record. def script_params_for(prepared_record) { - "nestedSourcedFields" => field_params.transform_values { |param| param.value_for(prepared_record) }, - "nestedSourcedPathIdentifiers" => path_identifier_params.transform_values { |param| param.value_for(prepared_record) } + "sourcedFromNestedFields" => field_params.transform_values { |param| param.value_for(prepared_record) }, + "sourcedFromNestedPathIdentifiers" => path_identifier_params.transform_values { |param| param.value_for(prepared_record) } } end end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rb similarity index 61% rename from elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rb rename to elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rb index ee185ee80..55a6ec4c6 100644 --- a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rb +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rb @@ -9,62 +9,55 @@ module ElasticGraph module SchemaArtifacts module RuntimeMetadata + # @private + module SourcedFromNestedPathSegment + def self.from_hash(hash) + if hash.key?("match_field") + ListPathSegment.from_hash(hash) + else + ObjectPathSegment.from_hash(hash) + end + end + end + # Represents a segment in a nested sourced path that navigates into a list field, # matching an element by a key field. # + # A future PR will add `to_painless_param` to convert these segments into the + # camelCase hash format expected by the painless script (with a "type" discriminator). + # # @private class ListPathSegment < ::Data.define(:field, :match_field, :source_field) - # @dynamic to_painless_param - TYPE = "list" FIELD = "field" - MATCH_FIELD = "matchField" - SOURCE_FIELD = "sourceField" + MATCH_FIELD = "match_field" + SOURCE_FIELD = "source_field" def to_dumpable_hash - {"type" => TYPE, FIELD => field, MATCH_FIELD => match_field, SOURCE_FIELD => source_field} + # Keys here are ordered alphabetically; please keep them that way + {FIELD => field, MATCH_FIELD => match_field, SOURCE_FIELD => source_field} end - alias_method :to_painless_param, :to_dumpable_hash - def self.from_hash(hash) new(field: hash[FIELD], match_field: hash[MATCH_FIELD], source_field: hash[SOURCE_FIELD]) end end # Represents a segment in a nested sourced path that navigates into an object field. + # See `ListPathSegment` for notes on `to_painless_param`. # # @private class ObjectPathSegment < ::Data.define(:field) - # @dynamic to_painless_param - TYPE = "object" FIELD = "field" def to_dumpable_hash - {"type" => TYPE, FIELD => field} + # Keys here are ordered alphabetically; please keep them that way + {FIELD => field} end - alias_method :to_painless_param, :to_dumpable_hash - def self.from_hash(hash) new(field: hash[FIELD]) end end - - # @private - module NestedSourcedPathSegment - def self.from_hash(hash) - case hash["type"] - when ListPathSegment::TYPE - ListPathSegment.from_hash(hash) - when ObjectPathSegment::TYPE - ObjectPathSegment.from_hash(hash) - # :nocov: -- defensive error for corrupted YAML; not worth a dedicated test. - else - raise "Unknown nested sourced path segment type: #{hash["type"].inspect}" - # :nocov: - end - end - end end end end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb index 38a8c5d39..d65853933 100644 --- a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb @@ -7,7 +7,7 @@ # frozen_string_literal: true require "elastic_graph/constants" -require "elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params" +require "elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params" require "elastic_graph/schema_artifacts/runtime_metadata/params" module ElasticGraph @@ -24,7 +24,7 @@ class UpdateTarget < ::Data.define( :routing_value_source, :rollover_timestamp_value_source, :top_level_fields_params, - :nested_sourced_data_params, + :sourced_from_nested_params, :metadata_params ) TYPE = "type" @@ -34,7 +34,7 @@ class UpdateTarget < ::Data.define( ROUTING_VALUE_SOURCE = "routing_value_source" ROLLOVER_TIMESTAMP_VALUE_SOURCE = "rollover_timestamp_value_source" TOP_LEVEL_FIELDS_PARAMS = "top_level_fields_params" - NESTED_SOURCED_DATA_PARAMS = "nested_sourced_data_params" + SOURCED_FROM_NESTED_PARAMS = "sourced_from_nested_params" METADATA_PARAMS = "metadata_params" def self.from_hash(hash) @@ -46,7 +46,7 @@ def self.from_hash(hash) routing_value_source: hash[ROUTING_VALUE_SOURCE], rollover_timestamp_value_source: hash[ROLLOVER_TIMESTAMP_VALUE_SOURCE], top_level_fields_params: Param.load_params_hash(hash[TOP_LEVEL_FIELDS_PARAMS] || {}), - nested_sourced_data_params: NestedSourcedDataParams.from_hash(hash[NESTED_SOURCED_DATA_PARAMS] || {}), + sourced_from_nested_params: SourcedFromNestedParams.from_hash(hash[SOURCED_FROM_NESTED_PARAMS] || {}), metadata_params: Param.load_params_hash(hash[METADATA_PARAMS] || {}) ) end @@ -56,11 +56,11 @@ def to_dumpable_hash # Keys here are ordered alphabetically; please keep them that way. ID_SOURCE => id_source, METADATA_PARAMS => Param.dump_params_hash(metadata_params), - NESTED_SOURCED_DATA_PARAMS => nested_sourced_data_params.to_dumpable_hash, RELATIONSHIP => relationship, ROLLOVER_TIMESTAMP_VALUE_SOURCE => rollover_timestamp_value_source, ROUTING_VALUE_SOURCE => routing_value_source, SCRIPT_ID => script_id, + SOURCED_FROM_NESTED_PARAMS => sourced_from_nested_params.to_dumpable_hash, TOP_LEVEL_FIELDS_PARAMS => Param.dump_params_hash(top_level_fields_params), TYPE => type } @@ -81,7 +81,7 @@ def params_for(doc_id:, event:, prepared_record:) meta.merge( {"id" => doc_id, "topLevelFields" => top_level_fields}, - nested_sourced_data_params.script_params_for(prepared_record) + sourced_from_nested_params.script_params_for(prepared_record) ) end end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs index c30824c7c..6677e24f4 100644 --- a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs @@ -8,7 +8,7 @@ module ElasticGraph attr_reader current_sources: ::Set[::String] attr_reader fields_by_path: ::Hash[::String, IndexField] attr_reader has_had_multiple_sources: bool - attr_reader nested_sourced_paths: ::Hash[::String, ::Array[nestedSourcedPathSegment]] + attr_reader sourced_from_nested_paths_by_relationship: ::Hash[::String, ::Array[sourcedFromNestedPathSegment]] def initialize: ( route_with: ::String, @@ -17,7 +17,7 @@ module ElasticGraph current_sources: ::Set[::String], fields_by_path: ::Hash[::String, IndexField], has_had_multiple_sources: bool, - nested_sourced_paths: ::Hash[::String, ::Array[nestedSourcedPathSegment]] + sourced_from_nested_paths_by_relationship: ::Hash[::String, ::Array[sourcedFromNestedPathSegment]] ) -> void def with: ( @@ -27,7 +27,7 @@ module ElasticGraph ?current_sources: ::Enumerable[::String], ?fields_by_path: ::Hash[::String, IndexField], ?has_had_multiple_sources: bool, - ?nested_sourced_paths: ::Hash[::String, ::Array[nestedSourcedPathSegment]] + ?sourced_from_nested_paths_by_relationship: ::Hash[::String, ::Array[sourcedFromNestedPathSegment]] ) -> IndexDefinition end @@ -38,7 +38,7 @@ module ElasticGraph CURRENT_SOURCES: "current_sources" FIELDS_BY_PATH: "fields_by_path" HAS_HAD_MULTIPLE_SOURCES: "has_had_multiple_sources" - NESTED_SOURCED_PATHS: "nested_sourced_paths" + SOURCED_FROM_NESTED_PATHS_BY_RELATIONSHIP: "sourced_from_nested_paths_by_relationship" def initialize: ( route_with: ::String, @@ -47,7 +47,7 @@ module ElasticGraph current_sources: ::Enumerable[::String], fields_by_path: ::Hash[::String, IndexField], has_had_multiple_sources: bool, - nested_sourced_paths: ::Hash[::String, ::Array[nestedSourcedPathSegment]] + sourced_from_nested_paths_by_relationship: ::Hash[::String, ::Array[sourcedFromNestedPathSegment]] ) -> void def self.from_hash: (::Hash[::String, untyped]) -> IndexDefinition diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rbs similarity index 71% rename from elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rbs rename to elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rbs index 5e916516b..e04c58689 100644 --- a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_data_params.rbs +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rbs @@ -1,19 +1,19 @@ module ElasticGraph module SchemaArtifacts module RuntimeMetadata - class NestedSourcedDataParamsSuperType + class SourcedFromNestedParamsSuperType attr_reader field_params: paramsHash attr_reader path_identifier_params: paramsHash def initialize: (field_params: paramsHash, path_identifier_params: paramsHash) -> void end - class NestedSourcedDataParams < NestedSourcedDataParamsSuperType + class SourcedFromNestedParams < SourcedFromNestedParamsSuperType FIELD_PARAMS: "field_params" PATH_IDENTIFIER_PARAMS: "path_identifier_params" - EMPTY: NestedSourcedDataParams + EMPTY: SourcedFromNestedParams - def self.from_hash: (::Hash[::String, untyped]) -> NestedSourcedDataParams + def self.from_hash: (::Hash[::String, untyped]) -> SourcedFromNestedParams def to_dumpable_hash: () -> ::Hash[::String, untyped] def script_params_for: (::Hash[::String, untyped]) -> ::Hash[::String, ::Hash[::String, untyped]] end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rbs similarity index 70% rename from elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rbs rename to elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rbs index 615cb89f5..6fa019fca 100644 --- a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/nested_sourced_path_segment.rbs +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rbs @@ -10,14 +10,12 @@ module ElasticGraph end class ListPathSegment < ListPathSegmentSuperType - TYPE: "list" FIELD: "field" - MATCH_FIELD: "matchField" - SOURCE_FIELD: "sourceField" + MATCH_FIELD: "match_field" + SOURCE_FIELD: "source_field" def self.from_hash: (::Hash[::String, untyped]) -> ListPathSegment def to_dumpable_hash: () -> ::Hash[::String, ::String] - alias to_painless_param to_dumpable_hash end class ObjectPathSegmentSuperType @@ -27,18 +25,16 @@ module ElasticGraph end class ObjectPathSegment < ObjectPathSegmentSuperType - TYPE: "object" FIELD: "field" def self.from_hash: (::Hash[::String, untyped]) -> ObjectPathSegment def to_dumpable_hash: () -> ::Hash[::String, ::String] - alias to_painless_param to_dumpable_hash end - type nestedSourcedPathSegment = ListPathSegment | ObjectPathSegment + type sourcedFromNestedPathSegment = ListPathSegment | ObjectPathSegment - module NestedSourcedPathSegment - def self.from_hash: (::Hash[::String, untyped]) -> nestedSourcedPathSegment + module SourcedFromNestedPathSegment + def self.from_hash: (::Hash[::String, untyped]) -> sourcedFromNestedPathSegment end end end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs index 99647cd38..b89921666 100644 --- a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs @@ -9,7 +9,7 @@ module ElasticGraph attr_reader routing_value_source: ::String? attr_reader rollover_timestamp_value_source: ::String? attr_reader top_level_fields_params: paramsHash - attr_reader nested_sourced_data_params: NestedSourcedDataParams + attr_reader sourced_from_nested_params: SourcedFromNestedParams attr_reader metadata_params: paramsHash def initialize: ( @@ -20,7 +20,7 @@ module ElasticGraph routing_value_source: ::String?, rollover_timestamp_value_source: ::String?, top_level_fields_params: paramsHash, - nested_sourced_data_params: NestedSourcedDataParams, + sourced_from_nested_params: SourcedFromNestedParams, metadata_params: paramsHash ) -> void @@ -32,7 +32,7 @@ module ElasticGraph ?routing_value_source: ::String?, ?rollover_timestamp_value_source: ::String?, ?top_level_fields_params: paramsHash, - ?nested_sourced_data_params: NestedSourcedDataParams, + ?sourced_from_nested_params: SourcedFromNestedParams, ?metadata_params: paramsHash ) -> UpdateTarget @@ -49,7 +49,7 @@ module ElasticGraph ROUTING_VALUE_SOURCE: "routing_value_source" ROLLOVER_TIMESTAMP_VALUE_SOURCE: "rollover_timestamp_value_source" TOP_LEVEL_FIELDS_PARAMS: "top_level_fields_params" - NESTED_SOURCED_DATA_PARAMS: "nested_sourced_data_params" + SOURCED_FROM_NESTED_PARAMS: "sourced_from_nested_params" METADATA_PARAMS: "metadata_params" def self.from_hash: (::Hash[::String, untyped]) -> UpdateTarget diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb index b04862652..09ae358e7 100644 --- a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb @@ -25,7 +25,7 @@ module RuntimeMetadata current_sources: Set.new, fields_by_path: {}, has_had_multiple_sources: false, - nested_sourced_paths: {} + sourced_from_nested_paths_by_relationship: {} ) end @@ -52,26 +52,6 @@ module RuntimeMetadata expect(index_def_with_flag.to_dumpable_hash["has_had_multiple_sources"]).to eq true end - it "roundtrips nested_sourced_paths through to_dumpable_hash and from_hash" do - index_def = index_definition_with( - nested_sourced_paths: { - "foo" => [ - ListPathSegment.new(field: "bar", match_field: "id", source_field: "bazId"), - ObjectPathSegment.new(field: "qux") - ] - } - ) - - roundtripped = IndexDefinition.from_hash(index_def.to_dumpable_hash) - - expect(roundtripped.nested_sourced_paths).to eq({ - "foo" => [ - ListPathSegment.new(field: "bar", match_field: "id", source_field: "bazId"), - ObjectPathSegment.new(field: "qux") - ] - }) - end - describe IndexDefinition::Rollover do it "builds from a minimal hash" do rollover = IndexDefinition::Rollover.from_hash({"frequency" => "yearly"}) diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb index 961debf3b..fe8398fb8 100644 --- a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb @@ -43,7 +43,10 @@ module RuntimeMetadata routing_value_source: "cost.currency_name", rollover_timestamp_value_source: "currency_introduced_on", top_level_fields_params: {"workspace_id" => DynamicParam.new(source_path: "wid", cardinality: :one)}, - nested_sourced_data_params: NestedSourcedDataParams::EMPTY, + sourced_from_nested_params: SourcedFromNestedParams.new( + field_params: {"amount" => DynamicParam.new(source_path: "cost.amount", cardinality: :one)}, + path_identifier_params: {"currency_id" => DynamicParam.new(source_path: "cost.currency", cardinality: :one)} + ), metadata_params: {"relationshipName" => StaticParam.new(value: "currency")} ), UpdateTarget.new( @@ -54,7 +57,7 @@ module RuntimeMetadata routing_value_source: nil, rollover_timestamp_value_source: nil, top_level_fields_params: {}, - nested_sourced_data_params: NestedSourcedDataParams::EMPTY, + sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: {} ) ], @@ -125,7 +128,12 @@ module RuntimeMetadata "foo.bar" => IndexField.new(source: "other") }, has_had_multiple_sources: false, - nested_sourced_paths: {} + sourced_from_nested_paths_by_relationship: { + "currency" => [ + ListPathSegment.new(field: "costs", match_field: "id", source_field: "cost_id"), + ObjectPathSegment.new(field: "details") + ] + } ), "addresses" => IndexDefinition.new( route_with: nil, @@ -134,7 +142,7 @@ module RuntimeMetadata current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, has_had_multiple_sources: false, - nested_sourced_paths: {} + sourced_from_nested_paths_by_relationship: {} ), "components" => IndexDefinition.new( route_with: "group_id", @@ -143,7 +151,7 @@ module RuntimeMetadata current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, has_had_multiple_sources: true, - nested_sourced_paths: {} + sourced_from_nested_paths_by_relationship: {} ) }, schema_element_names: SchemaElementNames.new( @@ -171,14 +179,18 @@ module RuntimeMetadata "index_definition_names" => ["widgets"], "update_targets" => [ { - "type" => "WidgetCurrency", - "relationship" => "currency", - "script_id" => "some_script_id", "id_source" => "cost.currency", - "routing_value_source" => "cost.currency_name", + "metadata_params" => {"relationshipName" => {"value" => "currency"}}, + "relationship" => "currency", "rollover_timestamp_value_source" => "currency_introduced_on", + "routing_value_source" => "cost.currency_name", + "script_id" => "some_script_id", + "sourced_from_nested_params" => { + "field_params" => {"amount" => {"source_path" => "cost.amount", "cardinality" => "one"}}, + "path_identifier_params" => {"currency_id" => {"source_path" => "cost.currency", "cardinality" => "one"}} + }, "top_level_fields_params" => {"workspace_id" => {"source_path" => "wid", "cardinality" => "one"}}, - "metadata_params" => {"relationshipName" => {"value" => "currency"}} + "type" => "WidgetCurrency" }, { "id_source" => "id" @@ -266,6 +278,12 @@ module RuntimeMetadata "foo.bar" => { "source" => "other" } + }, + "sourced_from_nested_paths_by_relationship" => { + "currency" => [ + {"field" => "costs", "match_field" => "id", "source_field" => "cost_id"}, + {"field" => "details"} + ] } }, "addresses" => { @@ -316,7 +334,7 @@ module RuntimeMetadata routing_value_source: nil, rollover_timestamp_value_source: nil, top_level_fields_params: {"workspace_id" => dynamic_param_with(cardinality: :many)}, - nested_sourced_data_params: NestedSourcedDataParams::EMPTY, + sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: {} )]), "IndexDefinitionNamesOnly" => object_type_with(index_definition_names: ["foo", "bar"]), diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb index 340b106a2..d0df87536 100644 --- a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb @@ -26,7 +26,7 @@ module RuntimeMetadata routing_value_source: nil, rollover_timestamp_value_source: nil, top_level_fields_params: {}, - nested_sourced_data_params: NestedSourcedDataParams::EMPTY, + sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: {} ) end @@ -85,7 +85,7 @@ module RuntimeMetadata } ) - metadata_params_only = params.except("id", "topLevelFields", "nestedSourcedFields", "nestedSourcedPathIdentifiers") + metadata_params_only = params.except("id", "topLevelFields", "sourcedFromNestedFields", "sourcedFromNestedPathIdentifiers") expect(metadata_params_only).to eq( "foo" => 43, @@ -116,23 +116,23 @@ module RuntimeMetadata ) end - it "includes nested_sourced_data_params resolved from the prepared_record" do + it "includes sourced_from_nested_params resolved from the prepared_record" do params = params_for( - nested_sourced_data_params: NestedSourcedDataParams.new( + sourced_from_nested_params: SourcedFromNestedParams.new( field_params: {"foo" => dynamic_param_with(source_path: "some.field", cardinality: :one)}, path_identifier_params: {"bar" => dynamic_param_with(source_path: "some.other", cardinality: :one)} ), prepared_record: {"some" => {"field" => "hello", "other" => "abc"}} ) - expect(params["nestedSourcedFields"]).to eq({"foo" => "hello"}) - expect(params["nestedSourcedPathIdentifiers"]).to eq({"bar" => "abc"}) + expect(params["sourcedFromNestedFields"]).to eq({"foo" => "hello"}) + expect(params["sourcedFromNestedPathIdentifiers"]).to eq({"bar" => "abc"}) end - def params_for(doc_id: "doc_id", event: {}, prepared_record: {}, top_level_fields_params: {}, nested_sourced_data_params: NestedSourcedDataParams::EMPTY, metadata_params: {}) + def params_for(doc_id: "doc_id", event: {}, prepared_record: {}, top_level_fields_params: {}, sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: {}) update_target = normal_indexing_update_target_with( top_level_fields_params: top_level_fields_params, - nested_sourced_data_params: nested_sourced_data_params, + sourced_from_nested_params: sourced_from_nested_params, metadata_params: metadata_params ) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb index a8cd351f9..81a812869 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb @@ -273,7 +273,7 @@ def runtime_metadata_for_source_type top_level_fields_params: fields.map(&:source_field).to_h do |f| [f, SchemaArtifacts::RuntimeMetadata::DynamicParam.new(source_path: f, cardinality: :many)] end, - nested_sourced_data_params: SchemaArtifacts::RuntimeMetadata::NestedSourcedDataParams::EMPTY + sourced_from_nested_params: SchemaArtifacts::RuntimeMetadata::SourcedFromNestedParams::EMPTY ) end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb index ac6e38a2b..436bc6e82 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb @@ -265,7 +265,7 @@ def runtime_metadata ) end, has_had_multiple_sources: has_had_multiple_sources_flag, - nested_sourced_paths: {} # TODO: Populate with real data once paths are registered on the index + sourced_from_nested_paths_by_relationship: {} # TODO: Populate with real data once paths are registered on the index ) end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb index f5acf0df9..ea4e267d2 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb @@ -19,7 +19,7 @@ def self.new_normal_indexing_update_target( top_level_fields_params:, routing_value_source:, rollover_timestamp_value_source:, - nested_sourced_data_params: + sourced_from_nested_params: ) SchemaArtifacts::RuntimeMetadata::UpdateTarget.new( type: type, @@ -30,7 +30,7 @@ def self.new_normal_indexing_update_target( "relationship" => SchemaArtifacts::RuntimeMetadata::StaticParam.new(value: relationship) }), top_level_fields_params: top_level_fields_params, - nested_sourced_data_params: nested_sourced_data_params, + sourced_from_nested_params: sourced_from_nested_params, routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source ) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb index 455137726..3f456d330 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb @@ -49,7 +49,7 @@ def resolve relationship: resolved_relationship.relationship_name, id_source: resolved_relationship.relation_metadata.foreign_key, top_level_fields_params: top_level_fields_params, - nested_sourced_data_params: SchemaArtifacts::RuntimeMetadata::NestedSourcedDataParams::EMPTY, + sourced_from_nested_params: SchemaArtifacts::RuntimeMetadata::SourcedFromNestedParams::EMPTY, routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source ) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb index f6834b105..ea2fd288f 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb @@ -446,7 +446,7 @@ def self_update_target relationship: SELF_RELATIONSHIP_NAME, id_source: "id", top_level_fields_params: top_level_fields_params, - nested_sourced_data_params: SchemaArtifacts::RuntimeMetadata::NestedSourcedDataParams::EMPTY, + sourced_from_nested_params: SchemaArtifacts::RuntimeMetadata::SourcedFromNestedParams::EMPTY, # Some day we may want to consider supporting multiple indices. If/when we add support for that, # we'll need to change the runtime metadata here to have a map of these values, keyed by index # name. diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs index 5b814d25a..b8769fe76 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs @@ -9,7 +9,7 @@ module ElasticGraph top_level_fields_params: SchemaArtifacts::RuntimeMetadata::paramsHash, routing_value_source: ::String?, rollover_timestamp_value_source: ::String?, - nested_sourced_data_params: SchemaArtifacts::RuntimeMetadata::NestedSourcedDataParams + sourced_from_nested_params: SchemaArtifacts::RuntimeMetadata::SourcedFromNestedParams ) -> SchemaArtifacts::RuntimeMetadata::UpdateTarget private diff --git a/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb b/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb index c35bd289e..e056cfca9 100644 --- a/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb +++ b/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb @@ -72,7 +72,7 @@ def derived_indexing_update_target_with( routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source, top_level_fields_params: top_level_fields_params, - nested_sourced_data_params: NestedSourcedDataParams::EMPTY, + sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: metadata_params ) end @@ -84,7 +84,7 @@ def normal_indexing_update_target_with( routing_value_source: "routing_value_source", rollover_timestamp_value_source: "rollover_timestamp_value_source", top_level_fields_params: {}, - nested_sourced_data_params: NestedSourcedDataParams::EMPTY, + sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: {} ) UpdateTarget.new( @@ -95,7 +95,7 @@ def normal_indexing_update_target_with( routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source, top_level_fields_params: top_level_fields_params, - nested_sourced_data_params: nested_sourced_data_params, + sourced_from_nested_params: sourced_from_nested_params, metadata_params: metadata_params ) end @@ -115,7 +115,7 @@ def static_param_with(value) StaticParam.new(value: value) end - def index_definition_with(route_with: nil, rollover: nil, default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, has_had_multiple_sources: false, nested_sourced_paths: {}) + def index_definition_with(route_with: nil, rollover: nil, default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, has_had_multiple_sources: false, sourced_from_nested_paths_by_relationship: {}) IndexDefinition.new( route_with: route_with, rollover: rollover, @@ -123,7 +123,7 @@ def index_definition_with(route_with: nil, rollover: nil, default_sort_fields: [ current_sources: current_sources, fields_by_path: fields_by_path, has_had_multiple_sources: has_had_multiple_sources, - nested_sourced_paths: nested_sourced_paths + sourced_from_nested_paths_by_relationship: sourced_from_nested_paths_by_relationship ) end From 32d2d8012104ea1fe0e2eca86a3666a8e53aa759 Mon Sep 17 00:00:00 2001 From: ellisandrews-toast Date: Tue, 2 Jun 2026 08:31:07 -0400 Subject: [PATCH 3/3] Use constant instead of string literal --- .../runtime_metadata/sourced_from_nested_path_segment.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rb index 55a6ec4c6..1bf167f52 100644 --- a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rb +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rb @@ -12,7 +12,7 @@ module RuntimeMetadata # @private module SourcedFromNestedPathSegment def self.from_hash(hash) - if hash.key?("match_field") + if hash.key?(ListPathSegment::MATCH_FIELD) ListPathSegment.from_hash(hash) else ObjectPathSegment.from_hash(hash)