From e1a72cf0328dca96d6294c813831c0fc40f31250 Mon Sep 17 00:00:00 2001 From: guanxu <1510424541@qq.com> Date: Thu, 26 Mar 2026 14:22:02 +0800 Subject: [PATCH 1/2] fix(rag): Add missing payload for ElasticsearchStore --- .../core/rag/store/ElasticsearchStore.java | 5 +++++ .../core/rag/store/ElasticsearchStoreTest.java | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/agentscope-extensions/agentscope-extensions-rag-simple/src/main/java/io/agentscope/core/rag/store/ElasticsearchStore.java b/agentscope-extensions/agentscope-extensions-rag-simple/src/main/java/io/agentscope/core/rag/store/ElasticsearchStore.java index 897710baf..afc8fee2e 100644 --- a/agentscope-extensions/agentscope-extensions-rag-simple/src/main/java/io/agentscope/core/rag/store/ElasticsearchStore.java +++ b/agentscope-extensions/agentscope-extensions-rag-simple/src/main/java/io/agentscope/core/rag/store/ElasticsearchStore.java @@ -93,6 +93,7 @@ public class ElasticsearchStore implements VDBStoreBase, AutoCloseable { private static final String FIELD_DOC_ID = "doc_id"; private static final String FIELD_CHUNK_ID = "chunk_id"; private static final String FIELD_CONTENT = "content"; + private static final String FIELD_PAYLOAD = "payload"; private final String indexName; private final int dimensions; @@ -404,6 +405,10 @@ private Map mapToEsDocument(Document doc) { DocumentMetadata meta = doc.getMetadata(); map.put(FIELD_DOC_ID, meta.getDocId()); map.put(FIELD_CHUNK_ID, meta.getChunkId()); + Map customPayload = meta.getPayload(); + if (customPayload != null && !customPayload.isEmpty()) { + map.put(FIELD_PAYLOAD, meta.getPayload()); + } // Serialize ContentBlock to JSON string to ensure safe storage/retrieval try { diff --git a/agentscope-extensions/agentscope-extensions-rag-simple/src/test/java/io/agentscope/core/rag/store/ElasticsearchStoreTest.java b/agentscope-extensions/agentscope-extensions-rag-simple/src/test/java/io/agentscope/core/rag/store/ElasticsearchStoreTest.java index f95c17eb6..e083c34ab 100644 --- a/agentscope-extensions/agentscope-extensions-rag-simple/src/test/java/io/agentscope/core/rag/store/ElasticsearchStoreTest.java +++ b/agentscope-extensions/agentscope-extensions-rag-simple/src/test/java/io/agentscope/core/rag/store/ElasticsearchStoreTest.java @@ -376,6 +376,20 @@ void testAddDimensionMismatch() throws VectorStoreException { .verify(); } + @Test + @DisplayName("Should add documents with payload") + void testAddPayload() throws VectorStoreException { + store = createMockStoreForAdd(true); + + TextBlock content = TextBlock.builder().text("Test content").build(); + DocumentMetadata metadata = + new DocumentMetadata(content, "doc-1", "chunk-1", Map.of("k", "v")); + Document doc = new Document(metadata); + doc.setEmbedding(new double[TEST_DIMENSIONS]); + + StepVerifier.create(store.add(List.of(doc))).verifyComplete(); + } + // ==================== Search Method Tests ==================== @SuppressWarnings("unchecked") From b6655ec45d3cd534182d7500f71cad546fccf33d Mon Sep 17 00:00:00 2001 From: guanxu <1510424541@qq.com> Date: Tue, 14 Apr 2026 17:58:01 +0800 Subject: [PATCH 2/2] fix: Add missing payload for search --- .../core/rag/store/ElasticsearchStore.java | 27 +++++++++++++++---- .../rag/store/ElasticsearchStoreTest.java | 2 ++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/agentscope-extensions/agentscope-extensions-rag-simple/src/main/java/io/agentscope/core/rag/store/ElasticsearchStore.java b/agentscope-extensions/agentscope-extensions-rag-simple/src/main/java/io/agentscope/core/rag/store/ElasticsearchStore.java index afc8fee2e..122bf2eb1 100644 --- a/agentscope-extensions/agentscope-extensions-rag-simple/src/main/java/io/agentscope/core/rag/store/ElasticsearchStore.java +++ b/agentscope-extensions/agentscope-extensions-rag-simple/src/main/java/io/agentscope/core/rag/store/ElasticsearchStore.java @@ -34,6 +34,7 @@ import co.elastic.clients.transport.ElasticsearchTransport; import co.elastic.clients.transport.rest5_client.Rest5ClientTransport; import co.elastic.clients.transport.rest5_client.low_level.Rest5Client; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import io.agentscope.core.message.ContentBlock; import io.agentscope.core.message.TextBlock; @@ -41,6 +42,7 @@ import io.agentscope.core.rag.model.Document; import io.agentscope.core.rag.model.DocumentMetadata; import io.agentscope.core.rag.store.dto.SearchDocumentDto; +import io.agentscope.core.util.JsonUtils; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -363,6 +365,7 @@ private void ensureIndex() throws VectorStoreException { properties.put(FIELD_CHUNK_ID, idProperty); properties.put(FIELD_CONTENT, contentProperty); properties.put(FIELD_VECTOR, vectorProperty); + properties.put(FIELD_PAYLOAD, contentProperty); TypeMapping mapping = new TypeMapping.Builder().properties(properties).build(); @@ -405,10 +408,6 @@ private Map mapToEsDocument(Document doc) { DocumentMetadata meta = doc.getMetadata(); map.put(FIELD_DOC_ID, meta.getDocId()); map.put(FIELD_CHUNK_ID, meta.getChunkId()); - Map customPayload = meta.getPayload(); - if (customPayload != null && !customPayload.isEmpty()) { - map.put(FIELD_PAYLOAD, meta.getPayload()); - } // Serialize ContentBlock to JSON string to ensure safe storage/retrieval try { @@ -419,6 +418,11 @@ private Map mapToEsDocument(Document doc) { map.put(FIELD_CONTENT, meta.getContentText()); } + Map customPayload = meta.getPayload(); + if (customPayload != null && !customPayload.isEmpty()) { + map.put(FIELD_PAYLOAD, JsonUtils.getJsonCodec().toJson(customPayload)); + } + return map; } @@ -431,6 +435,7 @@ private Document mapFromEsHit(Hit hit) { String docId = (String) source.get(FIELD_DOC_ID); String chunkId = (String) source.get(FIELD_CHUNK_ID); String contentJson = (String) source.get(FIELD_CONTENT); + String payloadJson = (String) source.get(FIELD_PAYLOAD); // Reconstruct ContentBlock ContentBlock content; @@ -441,7 +446,19 @@ private Document mapFromEsHit(Hit hit) { content = TextBlock.builder().text(contentJson).build(); } - DocumentMetadata metadata = new DocumentMetadata(content, docId, chunkId); + Map customPayload = new HashMap<>(); + if (payloadJson != null && !payloadJson.isBlank()) { + try { + customPayload = + JsonUtils.getJsonCodec() + .fromJson(payloadJson, new TypeReference<>() {}); + } catch (Exception e) { + log.warn("Failed to deserialize payload, using empty map", e); + } + } + + DocumentMetadata metadata = + new DocumentMetadata(content, docId, chunkId, customPayload); Document doc = new Document(metadata); // Set score if present diff --git a/agentscope-extensions/agentscope-extensions-rag-simple/src/test/java/io/agentscope/core/rag/store/ElasticsearchStoreTest.java b/agentscope-extensions/agentscope-extensions-rag-simple/src/test/java/io/agentscope/core/rag/store/ElasticsearchStoreTest.java index e083c34ab..7e9c1d0ef 100644 --- a/agentscope-extensions/agentscope-extensions-rag-simple/src/test/java/io/agentscope/core/rag/store/ElasticsearchStoreTest.java +++ b/agentscope-extensions/agentscope-extensions-rag-simple/src/test/java/io/agentscope/core/rag/store/ElasticsearchStoreTest.java @@ -447,6 +447,7 @@ void testSearchSuccess() throws VectorStoreException { List vector = new ArrayList<>(); for (int i = 0; i < TEST_DIMENSIONS; i++) vector.add(0.0); source.put("vector", vector); + source.put("payload", "{\"k\":\"v\"}"); store = createMockStoreForSearch(List.of(source)); @@ -464,6 +465,7 @@ void testSearchSuccess() throws VectorStoreException { assertEquals(1, results.size()); assertEquals("doc-1", results.get(0).getMetadata().getDocId()); assertEquals(0.95, results.get(0).getScore()); + assertEquals("v", results.get(0).getMetadata().getPayloadValue("k")); }) .verifyComplete(); }