Add schema artifacts for future nested sourced_from feature

elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/index_spec.rb

@@ -88,7 +88,8 @@ def index_def_named(name, rollover: nil)
             default_sort_fields: [],
             current_sources: [SELF_RELATIONSHIP_NAME],
             fields_by_path: {},
-            has_had_multiple_sources: false
+            has_had_multiple_sources: false,
+            sourced_from_nested_paths_by_relationship: {}
           )
 
           DatastoreCore::IndexDefinition.with(

elasticgraph-datastore_core/spec/integration/elastic_graph/datastore_core/index_definition/rollover_index_template_spec.rb

@@ -557,7 +557,8 @@ def index_def_named(name, rollover: nil)
             default_sort_fields: [],
             current_sources: [SELF_RELATIONSHIP_NAME],
             fields_by_path: {},
-            has_had_multiple_sources: false
+            has_had_multiple_sources: false,
+            sourced_from_nested_paths_by_relationship: {}
           )
 
           DatastoreCore::IndexDefinition.with(

elasticgraph-indexer/spec/unit/elastic_graph/indexer/datastore_indexing_router_spec.rb

@@ -265,7 +265,7 @@ def type_name_for_index(index_name)
             upsert: {},
             script: a_hash_including(
               id: /WidgetCurrency_from_Widget_/,
-              params: {"topLevelFields" => {"name" => ["thing1"]}, "id" => "USD"}
+              params: {"topLevelFields" => {"name" => ["thing1"]}, "id" => "USD", "sourcedFromNestedFields" => {}, "sourcedFromNestedPathIdentifiers" => {}}
             )
           )
         end

elasticgraph-indexer/spec/unit/elastic_graph/indexer/operation/update_spec.rb

@@ -74,7 +74,9 @@ module Operation
               {
                 script: {id: operations.first.update_target.script_id, params: {
                   "topLevelFields" => {"name" => ["thing1"]},
-                  "id" => "17"
+                  "id" => "17",
+                  "sourcedFromNestedFields" => {},
+                  "sourcedFromNestedPathIdentifiers" => {}
                 }},
                 scripted_upsert: true,
                 upsert: {}
@@ -104,6 +106,8 @@ module Operation
                 script: {id: INDEX_DATA_UPDATE_SCRIPT_ID, params: {
                   "topLevelFields" => {"name" => "thing1"},
                   "id" => "17",
+                  "sourcedFromNestedFields" => {},
+                  "sourcedFromNestedPathIdentifiers" => {},
                   "staticValue" => 47,
                   "sourceType" => "Widget",
                   LIST_COUNTS_FIELD => {"sizes" => 0, "widget_names" => 0}
@@ -157,7 +161,9 @@ module Operation
               {
                 script: {id: operations.first.update_target.script_id, params: {
                   "topLevelFields" => {"name" => []},
-                  "id" => "17"
+                  "id" => "17",
+                  "sourcedFromNestedFields" => {},
+                  "sourcedFromNestedPathIdentifiers" => {}
                 }},
                 scripted_upsert: true,
                 upsert: {}
@@ -178,7 +184,9 @@ module Operation
               {
                 script: {id: operations.first.update_target.script_id, params: {
                   "topLevelFields" => {"name" => ["thing1"]},
-                  "id" => "embedded_workspace_id"
+                  "id" => "embedded_workspace_id",
+                  "sourcedFromNestedFields" => {},
+                  "sourcedFromNestedPathIdentifiers" => {}
                 }},
                 scripted_upsert: true,
                 upsert: {}
@@ -203,7 +211,9 @@ module Operation
               {
                 script: {id: operations.first.update_target.script_id, params: {
                   "topLevelFields" => {"embedded_values.missing_field" => [], "name" => nil},
-                  "id" => "17"
+                  "id" => "17",
+                  "sourcedFromNestedFields" => {},
+                  "sourcedFromNestedPathIdentifiers" => {}
                 }},
                 scripted_upsert: true,
                 upsert: {}
@@ -236,7 +246,9 @@ module Operation
                       "workspace_id" => "embedded_workspace_id"
                     }
                   },
-                  "id" => "17"
+                  "id" => "17",
+                  "sourcedFromNestedFields" => {},
+                  "sourcedFromNestedPathIdentifiers" => {}
                 }},
                 scripted_upsert: true,
                 upsert: {}
@@ -262,7 +274,9 @@ module Operation
                 script: {id: operations.first.update_target.script_id, params: {
                   # Float-typed integer values are coerced to true ints before indexing
                   "topLevelFields" => {"size" => [an_instance_of(::Integer).and(eq_to(4))]},
-                  "id" => "17"
+                  "id" => "17",
+                  "sourcedFromNestedFields" => {},
+                  "sourcedFromNestedPathIdentifiers" => {}
                 }},
                 scripted_upsert: true,
                 upsert: {}
@@ -283,7 +297,9 @@ module Operation
               {
                 script: {id: operations.first.update_target.script_id, params: {
                   "topLevelFields" => {"name" => ["thing1"]},
-                  "id" => "17"
+                  "id" => "17",
+                  "sourcedFromNestedFields" => {},
+                  "sourcedFromNestedPathIdentifiers" => {}
                 }},
                 scripted_upsert: true,
                 upsert: {}
@@ -292,7 +308,9 @@ module Operation
               {
                 script: {id: operations.first.update_target.script_id, params: {
                   "topLevelFields" => {"name" => ["thing1"]},
-                  "id" => "18"
+                  "id" => "18",
+                  "sourcedFromNestedFields" => {},
+                  "sourcedFromNestedPathIdentifiers" => {}
                 }},
                 scripted_upsert: true,
                 upsert: {}
@@ -301,7 +319,9 @@ module Operation
               {
                 script: {id: operations.first.update_target.script_id, params: {
                   "topLevelFields" => {"name" => ["thing1"]},
-                  "id" => "19"
+                  "id" => "19",
+                  "sourcedFromNestedFields" => {},
+                  "sourcedFromNestedPathIdentifiers" => {}
                 }},
                 scripted_upsert: true,
                 upsert: {}

elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/index_definition.rb

@@ -8,6 +8,7 @@
 
 require "elastic_graph/schema_artifacts/runtime_metadata/hash_dumper"
 require "elastic_graph/schema_artifacts/runtime_metadata/index_field"
+require "elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment"
 require "elastic_graph/schema_artifacts/runtime_metadata/sort_field"
 
 module ElasticGraph
@@ -16,22 +17,24 @@ module RuntimeMetadata
       # Runtime metadata related to a datastore index definition.
       #
       # @private
-      class IndexDefinition < ::Data.define(:route_with, :rollover, :default_sort_fields, :current_sources, :fields_by_path, :has_had_multiple_sources)
+      class IndexDefinition < ::Data.define(:route_with, :rollover, :default_sort_fields, :current_sources, :fields_by_path, :has_had_multiple_sources, :sourced_from_nested_paths_by_relationship)
         ROUTE_WITH = "route_with"
         ROLLOVER = "rollover"
         DEFAULT_SORT_FIELDS = "default_sort_fields"
         CURRENT_SOURCES = "current_sources"
         FIELDS_BY_PATH = "fields_by_path"
         HAS_HAD_MULTIPLE_SOURCES = "has_had_multiple_sources"
+        SOURCED_FROM_NESTED_PATHS_BY_RELATIONSHIP = "sourced_from_nested_paths_by_relationship"
 
-        def initialize(route_with:, rollover:, default_sort_fields:, current_sources:, fields_by_path:, has_had_multiple_sources:)
+        def initialize(route_with:, rollover:, default_sort_fields:, current_sources:, fields_by_path:, has_had_multiple_sources:, sourced_from_nested_paths_by_relationship:)
           super(
             route_with: route_with,
             rollover: rollover,
             default_sort_fields: default_sort_fields,
             current_sources: current_sources.to_set,
             fields_by_path: fields_by_path,
-            has_had_multiple_sources: has_had_multiple_sources
+            has_had_multiple_sources: has_had_multiple_sources,
+            sourced_from_nested_paths_by_relationship: sourced_from_nested_paths_by_relationship
           )
         end
 
@@ -42,7 +45,8 @@ def self.from_hash(hash)
             default_sort_fields: hash[DEFAULT_SORT_FIELDS]&.map { |h| SortField.from_hash(h) } || [],
             current_sources: hash[CURRENT_SOURCES] || [],
             fields_by_path: (hash[FIELDS_BY_PATH] || {}).transform_values { |h| IndexField.from_hash(h) },
-            has_had_multiple_sources: hash[HAS_HAD_MULTIPLE_SOURCES] || false
+            has_had_multiple_sources: hash[HAS_HAD_MULTIPLE_SOURCES] || false,
+            sourced_from_nested_paths_by_relationship: (hash[SOURCED_FROM_NESTED_PATHS_BY_RELATIONSHIP] || {}).transform_values { |segments| segments.map { |h| SourcedFromNestedPathSegment.from_hash(h) } }
           )
         end
 
@@ -54,7 +58,8 @@ def to_dumpable_hash
             FIELDS_BY_PATH => HashDumper.dump_hash(fields_by_path, &:to_dumpable_hash),
             HAS_HAD_MULTIPLE_SOURCES => (true if has_had_multiple_sources),
             ROLLOVER => rollover&.to_dumpable_hash,
-            ROUTE_WITH => route_with
+            ROUTE_WITH => route_with,
+            SOURCED_FROM_NESTED_PATHS_BY_RELATIONSHIP => sourced_from_nested_paths_by_relationship.transform_values { |segments| segments.map(&:to_dumpable_hash) }
           }
         end
 

elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params.rb

@@ -0,0 +1,50 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/schema_artifacts/runtime_metadata/params"
+
+module ElasticGraph
+  module SchemaArtifacts
+    module RuntimeMetadata
+      # Bundles the param configuration for nested sourced_from update targets.
+      # `field_params` defines which fields to extract from the event and write onto
+      # the target nested element. `path_identifier_params` defines which values to
+      # extract from the event to identify which nested element to target.
+      #
+      # @private
+      class SourcedFromNestedParams < ::Data.define(:field_params, :path_identifier_params)
+        FIELD_PARAMS = "field_params"
+        PATH_IDENTIFIER_PARAMS = "path_identifier_params"
+
+        EMPTY = new(field_params: {}, path_identifier_params: {})
+
+        def self.from_hash(hash)
+          new(
+            field_params: Param.load_params_hash(hash[FIELD_PARAMS] || {}),
+            path_identifier_params: Param.load_params_hash(hash[PATH_IDENTIFIER_PARAMS] || {})
+          )
+        end
+
+        def to_dumpable_hash
+          {
+            FIELD_PARAMS => Param.dump_params_hash(field_params),
+            PATH_IDENTIFIER_PARAMS => Param.dump_params_hash(path_identifier_params)
+          }
+        end
+
+        # Resolves params into script-ready values from the given prepared record.
+        def script_params_for(prepared_record)
+          {
+            "sourcedFromNestedFields" => field_params.transform_values { |param| param.value_for(prepared_record) },
+            "sourcedFromNestedPathIdentifiers" => path_identifier_params.transform_values { |param| param.value_for(prepared_record) }
+          }
+        end
+      end
+    end
+  end
+end

elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_path_segment.rb

@@ -0,0 +1,63 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  module SchemaArtifacts
+    module RuntimeMetadata
+      # @private
+      module SourcedFromNestedPathSegment
+        def self.from_hash(hash)
+          if hash.key?(ListPathSegment::MATCH_FIELD)
+            ListPathSegment.from_hash(hash)
+          else
+            ObjectPathSegment.from_hash(hash)
+          end
+        end
+      end
+
+      # Represents a segment in a nested sourced path that navigates into a list field,
+      # matching an element by a key field.
+      #
+      # A future PR will add `to_painless_param` to convert these segments into the
+      # camelCase hash format expected by the painless script (with a "type" discriminator).
+      #
+      # @private
+      class ListPathSegment < ::Data.define(:field, :match_field, :source_field)
+        FIELD = "field"
+        MATCH_FIELD = "match_field"
+        SOURCE_FIELD = "source_field"
+
+        def to_dumpable_hash
+          # Keys here are ordered alphabetically; please keep them that way
+          {FIELD => field, MATCH_FIELD => match_field, SOURCE_FIELD => source_field}
+        end
+
+        def self.from_hash(hash)
+          new(field: hash[FIELD], match_field: hash[MATCH_FIELD], source_field: hash[SOURCE_FIELD])
+        end
+      end
+
+      # Represents a segment in a nested sourced path that navigates into an object field.
+      # See `ListPathSegment` for notes on `to_painless_param`.
+      #
+      # @private
+      class ObjectPathSegment < ::Data.define(:field)
+        FIELD = "field"
+
+        def to_dumpable_hash
+          # Keys here are ordered alphabetically; please keep them that way
+          {FIELD => field}
+        end
+
+        def self.from_hash(hash)
+          new(field: hash[FIELD])
+        end
+      end
+    end
+  end
+end

elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/update_target.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require "elastic_graph/constants"
+require "elastic_graph/schema_artifacts/runtime_metadata/sourced_from_nested_params"
 require "elastic_graph/schema_artifacts/runtime_metadata/params"
 
 module ElasticGraph
@@ -23,6 +24,7 @@ class UpdateTarget < ::Data.define(
         :routing_value_source,
         :rollover_timestamp_value_source,
         :top_level_fields_params,
+        :sourced_from_nested_params,
         :metadata_params
       )
         TYPE = "type"
@@ -32,6 +34,7 @@ class UpdateTarget < ::Data.define(
         ROUTING_VALUE_SOURCE = "routing_value_source"
         ROLLOVER_TIMESTAMP_VALUE_SOURCE = "rollover_timestamp_value_source"
         TOP_LEVEL_FIELDS_PARAMS = "top_level_fields_params"
+        SOURCED_FROM_NESTED_PARAMS = "sourced_from_nested_params"
         METADATA_PARAMS = "metadata_params"
 
         def self.from_hash(hash)
@@ -43,6 +46,7 @@ def self.from_hash(hash)
             routing_value_source: hash[ROUTING_VALUE_SOURCE],
             rollover_timestamp_value_source: hash[ROLLOVER_TIMESTAMP_VALUE_SOURCE],
             top_level_fields_params: Param.load_params_hash(hash[TOP_LEVEL_FIELDS_PARAMS] || {}),
+            sourced_from_nested_params: SourcedFromNestedParams.from_hash(hash[SOURCED_FROM_NESTED_PARAMS] || {}),
             metadata_params: Param.load_params_hash(hash[METADATA_PARAMS] || {})
           )
         end
@@ -56,6 +60,7 @@ def to_dumpable_hash
             ROLLOVER_TIMESTAMP_VALUE_SOURCE => rollover_timestamp_value_source,
             ROUTING_VALUE_SOURCE => routing_value_source,
             SCRIPT_ID => script_id,
+            SOURCED_FROM_NESTED_PARAMS => sourced_from_nested_params.to_dumpable_hash,
             TOP_LEVEL_FIELDS_PARAMS => Param.dump_params_hash(top_level_fields_params),
             TYPE => type
           }
@@ -74,7 +79,10 @@ def params_for(doc_id:, event:, prepared_record:)
             [name, param.value_for(event)]
           end
 
-          meta.merge({"id" => doc_id, "topLevelFields" => top_level_fields})
+          meta.merge(
+            {"id" => doc_id, "topLevelFields" => top_level_fields},
+            sourced_from_nested_params.script_params_for(prepared_record)
+          )
         end
       end
     end