diff --git a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb index 6291178c0..fc3b126c7 100644 --- a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb +++ b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb @@ -88,7 +88,8 @@ def index_def_named(name, rollover: nil) default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, - has_had_multiple_sources: false + has_had_multiple_sources: false, + sourced_from_nested_paths_by_relationship: {} ) DatastoreCore::IndexDefinition.with( diff --git a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb index eac218050..7026e8a2c 100644 --- a/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb +++ b/elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb @@ -557,7 +557,8 @@ def index_def_named(name, rollover: nil) default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, - has_had_multiple_sources: false + has_had_multiple_sources: false, + sourced_from_nested_paths_by_relationship: {} ) DatastoreCore::IndexDefinition.with( diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb index 6d77a53e5..1baad1cd5 100644 --- a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb @@ -265,7 +265,7 @@ def type_name_for_index(index_name) upsert: {}, script: a_hash_including( id: /WidgetCurrency_from_Widget_/, - params: {"topLevelFields" => {"name" => ["thing1"]}, "id" => "USD"} + params: {"topLevelFields" => {"name" => ["thing1"]}, "id" => "USD", "sourcedFromNestedFields" => {}, "sourcedFromNestedPathIdentifiers" => {}} ) ) end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb index 1a15a9e62..7b32bde9f 100644 --- a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb @@ -74,7 +74,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, - "id" => "17" + "id" => "17", + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -104,6 +106,8 @@ module Operation script: {id: INDEX_DATA_UPDATE_SCRIPT_ID, params: { "topLevelFields" => {"name" => "thing1"}, "id" => "17", + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {}, "staticValue" => 47, "sourceType" => "Widget", LIST_COUNTS_FIELD => {"sizes" => 0, "widget_names" => 0} @@ -157,7 +161,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => []}, - "id" => "17" + "id" => "17", + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -178,7 +184,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, - "id" => "embedded_workspace_id" + "id" => "embedded_workspace_id", + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -203,7 +211,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"embedded_values.missing_field" => [], "name" => nil}, - "id" => "17" + "id" => "17", + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -236,7 +246,9 @@ module Operation "workspace_id" => "embedded_workspace_id" } }, - "id" => "17" + "id" => "17", + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -262,7 +274,9 @@ module Operation script: {id: operations.first.update_target.script_id, params: { # Float-typed integer values are coerced to true ints before indexing "topLevelFields" => {"size" => [an_instance_of(::Integer).and(eq_to(4))]}, - "id" => "17" + "id" => "17", + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -283,7 +297,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, - "id" => "17" + "id" => "17", + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -292,7 +308,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, - "id" => "18" + "id" => "18", + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} @@ -301,7 +319,9 @@ module Operation { script: {id: operations.first.update_target.script_id, params: { "topLevelFields" => {"name" => ["thing1"]}, - "id" => "19" + "id" => "19", + "sourcedFromNestedFields" => {}, + "sourcedFromNestedPathIdentifiers" => {} }}, scripted_upsert: true, upsert: {} diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb index 9e4f26eb0..960ce9bb1 100644 --- a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb @@ -8,6 +8,7 @@ require "elastic_graph/schema_artifacts/runtime_metadata/hash_dumper" require "elastic_graph/schema_artifacts/runtime_metadata/index_field" +require "elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment" require "elastic_graph/schema_artifacts/runtime_metadata/sort_field" module ElasticGraph @@ -16,22 +17,24 @@ module RuntimeMetadata # Runtime metadata related to a datastore index definition. # # @private - class IndexDefinition < ::Data.define(:route_with, :rollover, :default_sort_fields, :current_sources, :fields_by_path, :has_had_multiple_sources) + class IndexDefinition < ::Data.define(:route_with, :rollover, :default_sort_fields, :current_sources, :fields_by_path, :has_had_multiple_sources, :sourced_from_nested_paths_by_relationship) ROUTE_WITH = "route_with" ROLLOVER = "rollover" DEFAULT_SORT_FIELDS = "default_sort_fields" CURRENT_SOURCES = "current_sources" FIELDS_BY_PATH = "fields_by_path" HAS_HAD_MULTIPLE_SOURCES = "has_had_multiple_sources" + SOURCED_FROM_NESTED_PATHS_BY_RELATIONSHIP = "sourced_from_nested_paths_by_relationship" - def initialize(route_with:, rollover:, default_sort_fields:, current_sources:, fields_by_path:, has_had_multiple_sources:) + def initialize(route_with:, rollover:, default_sort_fields:, current_sources:, fields_by_path:, has_had_multiple_sources:, sourced_from_nested_paths_by_relationship:) super( route_with: route_with, rollover: rollover, default_sort_fields: default_sort_fields, current_sources: current_sources.to_set, fields_by_path: fields_by_path, - has_had_multiple_sources: has_had_multiple_sources + has_had_multiple_sources: has_had_multiple_sources, + sourced_from_nested_paths_by_relationship: sourced_from_nested_paths_by_relationship ) end @@ -42,7 +45,8 @@ def self.from_hash(hash) default_sort_fields: hash[DEFAULT_SORT_FIELDS]&.map { |h| SortField.from_hash(h) } || [], current_sources: hash[CURRENT_SOURCES] || [], fields_by_path: (hash[FIELDS_BY_PATH] || {}).transform_values { |h| IndexField.from_hash(h) }, - has_had_multiple_sources: hash[HAS_HAD_MULTIPLE_SOURCES] || false + has_had_multiple_sources: hash[HAS_HAD_MULTIPLE_SOURCES] || false, + sourced_from_nested_paths_by_relationship: (hash[SOURCED_FROM_NESTED_PATHS_BY_RELATIONSHIP] || {}).transform_values { |segments| segments.map { |h| SourcedFromNestedPathSegment.from_hash(h) } } ) end @@ -54,7 +58,8 @@ def to_dumpable_hash FIELDS_BY_PATH => HashDumper.dump_hash(fields_by_path, &:to_dumpable_hash), HAS_HAD_MULTIPLE_SOURCES => (true if has_had_multiple_sources), ROLLOVER => rollover&.to_dumpable_hash, - ROUTE_WITH => route_with + ROUTE_WITH => route_with, + SOURCED_FROM_NESTED_PATHS_BY_RELATIONSHIP => sourced_from_nested_paths_by_relationship.transform_values { |segments| segments.map(&:to_dumpable_hash) } } end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rb new file mode 100644 index 000000000..da1fd1d21 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rb @@ -0,0 +1,50 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/params" + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # Bundles the param configuration for nested sourced_from update targets. + # `field_params` defines which fields to extract from the event and write onto + # the target nested element. `path_identifier_params` defines which values to + # extract from the event to identify which nested element to target. + # + # @private + class SourcedFromNestedParams < ::Data.define(:field_params, :path_identifier_params) + FIELD_PARAMS = "field_params" + PATH_IDENTIFIER_PARAMS = "path_identifier_params" + + EMPTY = new(field_params: {}, path_identifier_params: {}) + + def self.from_hash(hash) + new( + field_params: Param.load_params_hash(hash[FIELD_PARAMS] || {}), + path_identifier_params: Param.load_params_hash(hash[PATH_IDENTIFIER_PARAMS] || {}) + ) + end + + def to_dumpable_hash + { + FIELD_PARAMS => Param.dump_params_hash(field_params), + PATH_IDENTIFIER_PARAMS => Param.dump_params_hash(path_identifier_params) + } + end + + # Resolves params into script-ready values from the given prepared record. + def script_params_for(prepared_record) + { + "sourcedFromNestedFields" => field_params.transform_values { |param| param.value_for(prepared_record) }, + "sourcedFromNestedPathIdentifiers" => path_identifier_params.transform_values { |param| param.value_for(prepared_record) } + } + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rb new file mode 100644 index 000000000..1bf167f52 --- /dev/null +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rb @@ -0,0 +1,63 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + # @private + module SourcedFromNestedPathSegment + def self.from_hash(hash) + if hash.key?(ListPathSegment::MATCH_FIELD) + ListPathSegment.from_hash(hash) + else + ObjectPathSegment.from_hash(hash) + end + end + end + + # Represents a segment in a nested sourced path that navigates into a list field, + # matching an element by a key field. + # + # A future PR will add `to_painless_param` to convert these segments into the + # camelCase hash format expected by the painless script (with a "type" discriminator). + # + # @private + class ListPathSegment < ::Data.define(:field, :match_field, :source_field) + FIELD = "field" + MATCH_FIELD = "match_field" + SOURCE_FIELD = "source_field" + + def to_dumpable_hash + # Keys here are ordered alphabetically; please keep them that way + {FIELD => field, MATCH_FIELD => match_field, SOURCE_FIELD => source_field} + end + + def self.from_hash(hash) + new(field: hash[FIELD], match_field: hash[MATCH_FIELD], source_field: hash[SOURCE_FIELD]) + end + end + + # Represents a segment in a nested sourced path that navigates into an object field. + # See `ListPathSegment` for notes on `to_painless_param`. + # + # @private + class ObjectPathSegment < ::Data.define(:field) + FIELD = "field" + + def to_dumpable_hash + # Keys here are ordered alphabetically; please keep them that way + {FIELD => field} + end + + def self.from_hash(hash) + new(field: hash[FIELD]) + end + end + end + end +end diff --git a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb index 37c021ae5..d65853933 100644 --- a/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb +++ b/elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/constants" +require "elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params" require "elastic_graph/schema_artifacts/runtime_metadata/params" module ElasticGraph @@ -23,6 +24,7 @@ class UpdateTarget < ::Data.define( :routing_value_source, :rollover_timestamp_value_source, :top_level_fields_params, + :sourced_from_nested_params, :metadata_params ) TYPE = "type" @@ -32,6 +34,7 @@ class UpdateTarget < ::Data.define( ROUTING_VALUE_SOURCE = "routing_value_source" ROLLOVER_TIMESTAMP_VALUE_SOURCE = "rollover_timestamp_value_source" TOP_LEVEL_FIELDS_PARAMS = "top_level_fields_params" + SOURCED_FROM_NESTED_PARAMS = "sourced_from_nested_params" METADATA_PARAMS = "metadata_params" def self.from_hash(hash) @@ -43,6 +46,7 @@ def self.from_hash(hash) routing_value_source: hash[ROUTING_VALUE_SOURCE], rollover_timestamp_value_source: hash[ROLLOVER_TIMESTAMP_VALUE_SOURCE], top_level_fields_params: Param.load_params_hash(hash[TOP_LEVEL_FIELDS_PARAMS] || {}), + sourced_from_nested_params: SourcedFromNestedParams.from_hash(hash[SOURCED_FROM_NESTED_PARAMS] || {}), metadata_params: Param.load_params_hash(hash[METADATA_PARAMS] || {}) ) end @@ -56,6 +60,7 @@ def to_dumpable_hash ROLLOVER_TIMESTAMP_VALUE_SOURCE => rollover_timestamp_value_source, ROUTING_VALUE_SOURCE => routing_value_source, SCRIPT_ID => script_id, + SOURCED_FROM_NESTED_PARAMS => sourced_from_nested_params.to_dumpable_hash, TOP_LEVEL_FIELDS_PARAMS => Param.dump_params_hash(top_level_fields_params), TYPE => type } @@ -74,7 +79,10 @@ def params_for(doc_id:, event:, prepared_record:) [name, param.value_for(event)] end - meta.merge({"id" => doc_id, "topLevelFields" => top_level_fields}) + meta.merge( + {"id" => doc_id, "topLevelFields" => top_level_fields}, + sourced_from_nested_params.script_params_for(prepared_record) + ) end end end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs index fc6bacfcd..6677e24f4 100644 --- a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rbs @@ -8,6 +8,7 @@ module ElasticGraph attr_reader current_sources: ::Set[::String] attr_reader fields_by_path: ::Hash[::String, IndexField] attr_reader has_had_multiple_sources: bool + attr_reader sourced_from_nested_paths_by_relationship: ::Hash[::String, ::Array[sourcedFromNestedPathSegment]] def initialize: ( route_with: ::String, @@ -15,7 +16,8 @@ module ElasticGraph default_sort_fields: ::Array[SortField], current_sources: ::Set[::String], fields_by_path: ::Hash[::String, IndexField], - has_had_multiple_sources: bool + has_had_multiple_sources: bool, + sourced_from_nested_paths_by_relationship: ::Hash[::String, ::Array[sourcedFromNestedPathSegment]] ) -> void def with: ( @@ -24,7 +26,8 @@ module ElasticGraph ?default_sort_fields: ::Array[SortField], ?current_sources: ::Enumerable[::String], ?fields_by_path: ::Hash[::String, IndexField], - ?has_had_multiple_sources: bool + ?has_had_multiple_sources: bool, + ?sourced_from_nested_paths_by_relationship: ::Hash[::String, ::Array[sourcedFromNestedPathSegment]] ) -> IndexDefinition end @@ -35,6 +38,7 @@ module ElasticGraph CURRENT_SOURCES: "current_sources" FIELDS_BY_PATH: "fields_by_path" HAS_HAD_MULTIPLE_SOURCES: "has_had_multiple_sources" + SOURCED_FROM_NESTED_PATHS_BY_RELATIONSHIP: "sourced_from_nested_paths_by_relationship" def initialize: ( route_with: ::String, @@ -42,7 +46,8 @@ module ElasticGraph default_sort_fields: ::Array[SortField], current_sources: ::Enumerable[::String], fields_by_path: ::Hash[::String, IndexField], - has_had_multiple_sources: bool + has_had_multiple_sources: bool, + sourced_from_nested_paths_by_relationship: ::Hash[::String, ::Array[sourcedFromNestedPathSegment]] ) -> void def self.from_hash: (::Hash[::String, untyped]) -> IndexDefinition diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rbs new file mode 100644 index 000000000..e04c58689 --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class SourcedFromNestedParamsSuperType + attr_reader field_params: paramsHash + attr_reader path_identifier_params: paramsHash + + def initialize: (field_params: paramsHash, path_identifier_params: paramsHash) -> void + end + + class SourcedFromNestedParams < SourcedFromNestedParamsSuperType + FIELD_PARAMS: "field_params" + PATH_IDENTIFIER_PARAMS: "path_identifier_params" + EMPTY: SourcedFromNestedParams + + def self.from_hash: (::Hash[::String, untyped]) -> SourcedFromNestedParams + def to_dumpable_hash: () -> ::Hash[::String, untyped] + def script_params_for: (::Hash[::String, untyped]) -> ::Hash[::String, ::Hash[::String, untyped]] + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rbs new file mode 100644 index 000000000..6fa019fca --- /dev/null +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rbs @@ -0,0 +1,41 @@ +module ElasticGraph + module SchemaArtifacts + module RuntimeMetadata + class ListPathSegmentSuperType + attr_reader field: ::String + attr_reader match_field: ::String + attr_reader source_field: ::String + + def initialize: (field: ::String, match_field: ::String, source_field: ::String) -> void + end + + class ListPathSegment < ListPathSegmentSuperType + FIELD: "field" + MATCH_FIELD: "match_field" + SOURCE_FIELD: "source_field" + + def self.from_hash: (::Hash[::String, untyped]) -> ListPathSegment + def to_dumpable_hash: () -> ::Hash[::String, ::String] + end + + class ObjectPathSegmentSuperType + attr_reader field: ::String + + def initialize: (field: ::String) -> void + end + + class ObjectPathSegment < ObjectPathSegmentSuperType + FIELD: "field" + + def self.from_hash: (::Hash[::String, untyped]) -> ObjectPathSegment + def to_dumpable_hash: () -> ::Hash[::String, ::String] + end + + type sourcedFromNestedPathSegment = ListPathSegment | ObjectPathSegment + + module SourcedFromNestedPathSegment + def self.from_hash: (::Hash[::String, untyped]) -> sourcedFromNestedPathSegment + end + end + end +end diff --git a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs index 4a43bfdb3..b89921666 100644 --- a/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs +++ b/elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/update_target.rbs @@ -9,6 +9,7 @@ module ElasticGraph attr_reader routing_value_source: ::String? attr_reader rollover_timestamp_value_source: ::String? attr_reader top_level_fields_params: paramsHash + attr_reader sourced_from_nested_params: SourcedFromNestedParams attr_reader metadata_params: paramsHash def initialize: ( @@ -19,6 +20,7 @@ module ElasticGraph routing_value_source: ::String?, rollover_timestamp_value_source: ::String?, top_level_fields_params: paramsHash, + sourced_from_nested_params: SourcedFromNestedParams, metadata_params: paramsHash ) -> void @@ -30,6 +32,7 @@ module ElasticGraph ?routing_value_source: ::String?, ?rollover_timestamp_value_source: ::String?, ?top_level_fields_params: paramsHash, + ?sourced_from_nested_params: SourcedFromNestedParams, ?metadata_params: paramsHash ) -> UpdateTarget @@ -46,6 +49,7 @@ module ElasticGraph ROUTING_VALUE_SOURCE: "routing_value_source" ROLLOVER_TIMESTAMP_VALUE_SOURCE: "rollover_timestamp_value_source" TOP_LEVEL_FIELDS_PARAMS: "top_level_fields_params" + SOURCED_FROM_NESTED_PARAMS: "sourced_from_nested_params" METADATA_PARAMS: "metadata_params" def self.from_hash: (::Hash[::String, untyped]) -> UpdateTarget diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb index a84759af8..09ae358e7 100644 --- a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/index_definition_spec.rb @@ -24,7 +24,8 @@ module RuntimeMetadata default_sort_fields: [], current_sources: Set.new, fields_by_path: {}, - has_had_multiple_sources: false + has_had_multiple_sources: false, + sourced_from_nested_paths_by_relationship: {} ) end diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb index c9f3733eb..fe8398fb8 100644 --- a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb @@ -43,6 +43,10 @@ module RuntimeMetadata routing_value_source: "cost.currency_name", rollover_timestamp_value_source: "currency_introduced_on", top_level_fields_params: {"workspace_id" => DynamicParam.new(source_path: "wid", cardinality: :one)}, + sourced_from_nested_params: SourcedFromNestedParams.new( + field_params: {"amount" => DynamicParam.new(source_path: "cost.amount", cardinality: :one)}, + path_identifier_params: {"currency_id" => DynamicParam.new(source_path: "cost.currency", cardinality: :one)} + ), metadata_params: {"relationshipName" => StaticParam.new(value: "currency")} ), UpdateTarget.new( @@ -53,6 +57,7 @@ module RuntimeMetadata routing_value_source: nil, rollover_timestamp_value_source: nil, top_level_fields_params: {}, + sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: {} ) ], @@ -122,7 +127,13 @@ module RuntimeMetadata fields_by_path: { "foo.bar" => IndexField.new(source: "other") }, - has_had_multiple_sources: false + has_had_multiple_sources: false, + sourced_from_nested_paths_by_relationship: { + "currency" => [ + ListPathSegment.new(field: "costs", match_field: "id", source_field: "cost_id"), + ObjectPathSegment.new(field: "details") + ] + } ), "addresses" => IndexDefinition.new( route_with: nil, @@ -130,7 +141,8 @@ module RuntimeMetadata default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, - has_had_multiple_sources: false + has_had_multiple_sources: false, + sourced_from_nested_paths_by_relationship: {} ), "components" => IndexDefinition.new( route_with: "group_id", @@ -138,7 +150,8 @@ module RuntimeMetadata default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, - has_had_multiple_sources: true + has_had_multiple_sources: true, + sourced_from_nested_paths_by_relationship: {} ) }, schema_element_names: SchemaElementNames.new( @@ -166,14 +179,18 @@ module RuntimeMetadata "index_definition_names" => ["widgets"], "update_targets" => [ { - "type" => "WidgetCurrency", - "relationship" => "currency", - "script_id" => "some_script_id", "id_source" => "cost.currency", - "routing_value_source" => "cost.currency_name", + "metadata_params" => {"relationshipName" => {"value" => "currency"}}, + "relationship" => "currency", "rollover_timestamp_value_source" => "currency_introduced_on", + "routing_value_source" => "cost.currency_name", + "script_id" => "some_script_id", + "sourced_from_nested_params" => { + "field_params" => {"amount" => {"source_path" => "cost.amount", "cardinality" => "one"}}, + "path_identifier_params" => {"currency_id" => {"source_path" => "cost.currency", "cardinality" => "one"}} + }, "top_level_fields_params" => {"workspace_id" => {"source_path" => "wid", "cardinality" => "one"}}, - "metadata_params" => {"relationshipName" => {"value" => "currency"}} + "type" => "WidgetCurrency" }, { "id_source" => "id" @@ -261,6 +278,12 @@ module RuntimeMetadata "foo.bar" => { "source" => "other" } + }, + "sourced_from_nested_paths_by_relationship" => { + "currency" => [ + {"field" => "costs", "match_field" => "id", "source_field" => "cost_id"}, + {"field" => "details"} + ] } }, "addresses" => { @@ -311,6 +334,7 @@ module RuntimeMetadata routing_value_source: nil, rollover_timestamp_value_source: nil, top_level_fields_params: {"workspace_id" => dynamic_param_with(cardinality: :many)}, + sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: {} )]), "IndexDefinitionNamesOnly" => object_type_with(index_definition_names: ["foo", "bar"]), diff --git a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb index d3ee76a61..d0df87536 100644 --- a/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb +++ b/elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/update_target_spec.rb @@ -26,6 +26,7 @@ module RuntimeMetadata routing_value_source: nil, rollover_timestamp_value_source: nil, top_level_fields_params: {}, + sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: {} ) end @@ -84,9 +85,9 @@ module RuntimeMetadata } ) - without_id_or_top_level_fields = params.except("id", "topLevelFields") + metadata_params_only = params.except("id", "topLevelFields", "sourcedFromNestedFields", "sourcedFromNestedPathIdentifiers") - expect(without_id_or_top_level_fields).to eq( + expect(metadata_params_only).to eq( "foo" => 43, "bar" => "hello", "bazz" => [12] @@ -115,9 +116,23 @@ module RuntimeMetadata ) end - def params_for(doc_id: "doc_id", event: {}, prepared_record: {}, top_level_fields_params: {}, metadata_params: {}) + it "includes sourced_from_nested_params resolved from the prepared_record" do + params = params_for( + sourced_from_nested_params: SourcedFromNestedParams.new( + field_params: {"foo" => dynamic_param_with(source_path: "some.field", cardinality: :one)}, + path_identifier_params: {"bar" => dynamic_param_with(source_path: "some.other", cardinality: :one)} + ), + prepared_record: {"some" => {"field" => "hello", "other" => "abc"}} + ) + + expect(params["sourcedFromNestedFields"]).to eq({"foo" => "hello"}) + expect(params["sourcedFromNestedPathIdentifiers"]).to eq({"bar" => "abc"}) + end + + def params_for(doc_id: "doc_id", event: {}, prepared_record: {}, top_level_fields_params: {}, sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: {}) update_target = normal_indexing_update_target_with( top_level_fields_params: top_level_fields_params, + sourced_from_nested_params: sourced_from_nested_params, metadata_params: metadata_params ) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb index a01df76c5..81a812869 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_indexed_type.rb @@ -272,7 +272,8 @@ def runtime_metadata_for_source_type metadata_params: {}, top_level_fields_params: fields.map(&:source_field).to_h do |f| [f, SchemaArtifacts::RuntimeMetadata::DynamicParam.new(source_path: f, cardinality: :many)] - end + end, + sourced_from_nested_params: SchemaArtifacts::RuntimeMetadata::SourcedFromNestedParams::EMPTY ) end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb index a6400db64..436bc6e82 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb @@ -264,7 +264,8 @@ def runtime_metadata direction: direction ) end, - has_had_multiple_sources: has_had_multiple_sources_flag + has_had_multiple_sources: has_had_multiple_sources_flag, + sourced_from_nested_paths_by_relationship: {} # TODO: Populate with real data once paths are registered on the index ) end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb index 95790c774..ea4e267d2 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_factory.rb @@ -18,7 +18,8 @@ def self.new_normal_indexing_update_target( id_source:, top_level_fields_params:, routing_value_source:, - rollover_timestamp_value_source: + rollover_timestamp_value_source:, + sourced_from_nested_params: ) SchemaArtifacts::RuntimeMetadata::UpdateTarget.new( type: type, @@ -29,6 +30,7 @@ def self.new_normal_indexing_update_target( "relationship" => SchemaArtifacts::RuntimeMetadata::StaticParam.new(value: relationship) }), top_level_fields_params: top_level_fields_params, + sourced_from_nested_params: sourced_from_nested_params, routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source ) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb index feca5528e..3f456d330 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/update_target_resolver.rb @@ -49,6 +49,7 @@ def resolve relationship: resolved_relationship.relationship_name, id_source: resolved_relationship.relation_metadata.foreign_key, top_level_fields_params: top_level_fields_params, + sourced_from_nested_params: SchemaArtifacts::RuntimeMetadata::SourcedFromNestedParams::EMPTY, routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source ) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb index 98dd547a7..ea2fd288f 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb @@ -446,6 +446,7 @@ def self_update_target relationship: SELF_RELATIONSHIP_NAME, id_source: "id", top_level_fields_params: top_level_fields_params, + sourced_from_nested_params: SchemaArtifacts::RuntimeMetadata::SourcedFromNestedParams::EMPTY, # Some day we may want to consider supporting multiple indices. If/when we add support for that, # we'll need to change the runtime metadata here to have a map of these values, keyed by index # name. diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs index 4375b8138..b8769fe76 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/update_target_factory.rbs @@ -8,7 +8,8 @@ module ElasticGraph id_source: ::String, top_level_fields_params: SchemaArtifacts::RuntimeMetadata::paramsHash, routing_value_source: ::String?, - rollover_timestamp_value_source: ::String? + rollover_timestamp_value_source: ::String?, + sourced_from_nested_params: SchemaArtifacts::RuntimeMetadata::SourcedFromNestedParams ) -> SchemaArtifacts::RuntimeMetadata::UpdateTarget private diff --git a/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb b/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb index 705031d12..e056cfca9 100644 --- a/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb +++ b/spec_support/lib/elastic_graph/spec_support/runtime_metadata_support.rb @@ -72,6 +72,7 @@ def derived_indexing_update_target_with( routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source, top_level_fields_params: top_level_fields_params, + sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: metadata_params ) end @@ -83,6 +84,7 @@ def normal_indexing_update_target_with( routing_value_source: "routing_value_source", rollover_timestamp_value_source: "rollover_timestamp_value_source", top_level_fields_params: {}, + sourced_from_nested_params: SourcedFromNestedParams::EMPTY, metadata_params: {} ) UpdateTarget.new( @@ -93,6 +95,7 @@ def normal_indexing_update_target_with( routing_value_source: routing_value_source, rollover_timestamp_value_source: rollover_timestamp_value_source, top_level_fields_params: top_level_fields_params, + sourced_from_nested_params: sourced_from_nested_params, metadata_params: metadata_params ) end @@ -112,14 +115,15 @@ def static_param_with(value) StaticParam.new(value: value) end - def index_definition_with(route_with: nil, rollover: nil, default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, has_had_multiple_sources: false) + def index_definition_with(route_with: nil, rollover: nil, default_sort_fields: [], current_sources: [SELF_RELATIONSHIP_NAME], fields_by_path: {}, has_had_multiple_sources: false, sourced_from_nested_paths_by_relationship: {}) IndexDefinition.new( route_with: route_with, rollover: rollover, default_sort_fields: default_sort_fields, current_sources: current_sources, fields_by_path: fields_by_path, - has_had_multiple_sources: has_had_multiple_sources + has_had_multiple_sources: has_had_multiple_sources, + sourced_from_nested_paths_by_relationship: sourced_from_nested_paths_by_relationship ) end