From e95170a73b4ade50c393c6a89e4943daf0bbbc0e Mon Sep 17 00:00:00 2001 From: kinyaelgrande Date: Tue, 7 Oct 2025 12:18:46 +0300 Subject: [PATCH 01/17] WIP --- extra/server-discovery/app/servers.go | 3 +- extra/server-discovery/indexer/service.go | 21 +++- .../pkg/es/mapping/mapping.go | 9 ++ .../pkg/es/reindex/reindex.go | 75 +++++++++++- extra/server-discovery/pkg/options/options.go | 35 +++--- .../pkg/options/vectorsearch.go | 23 ++++ .../pkg/vectorsearch/embeddings.go | 107 ++++++++++++++++++ extra/server-discovery/searcher/rest/es.go | 57 ++++++++-- .../server-discovery/searcher/rest/search.go | 14 ++- .../rest/internal/documents/documents.go | 3 + .../rest/internal/mapping/compose.go | 4 + .../rest/internal/mapping/mapping.go | 20 ++++ 12 files changed, 339 insertions(+), 32 deletions(-) create mode 100644 extra/server-discovery/pkg/options/vectorsearch.go create mode 100644 extra/server-discovery/pkg/vectorsearch/embeddings.go diff --git a/extra/server-discovery/app/servers.go b/extra/server-discovery/app/servers.go index 22c930ee00..56e671c730 100644 --- a/extra/server-discovery/app/servers.go +++ b/extra/server-discovery/app/servers.go @@ -1,11 +1,12 @@ package app import ( + "strings" + searcherRest "github.com/cortezaproject/corteza/extra/server-discovery/searcher/rest" "github.com/cortezaproject/corteza/server/pkg/options" "github.com/go-chi/chi/v5" "go.uber.org/zap" - "strings" ) func (app *CortezaDiscoveryApp) MountHttpRoutes(r chi.Router) { diff --git a/extra/server-discovery/indexer/service.go b/extra/server-discovery/indexer/service.go index 15161a0018..253259e033 100644 --- a/extra/server-discovery/indexer/service.go +++ b/extra/server-discovery/indexer/service.go @@ -11,6 +11,7 @@ import ( "github.com/cortezaproject/corteza/extra/server-discovery/pkg/es/mapping" "github.com/cortezaproject/corteza/extra/server-discovery/pkg/es/reindex" "github.com/cortezaproject/corteza/extra/server-discovery/pkg/options" + "github.com/cortezaproject/corteza/extra/server-discovery/pkg/vectorsearch" "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/go-elasticsearch/v7/esutil" "go.uber.org/zap" @@ -18,9 +19,10 @@ import ( type ( Config struct { - Corteza options.CortezaOpt - ES options.EsOpt - Indexer options.IndexerOpt + Corteza options.CortezaOpt + ES options.EsOpt + Indexer options.IndexerOpt + VectorSearch options.VectorSearchOpt } esService interface { @@ -28,6 +30,11 @@ type ( BulkIndexer() (esutil.BulkIndexer, error) } + embedderService interface { + GenerateEmbeddings(input string) ([]float32, error) + ValidateEmbeddingsAPI(dimension int) (bool, error) + } + apiClientService interface { HttpClient() *http.Client Mappings() (*http.Request, error) @@ -53,6 +60,7 @@ var ( DefaultApiClient apiClientService DefaultMapper mappingService DefaultReIndexer reIndexService + DefaultEmbedder embedderService ) func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) { @@ -61,6 +69,11 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) { return } + DefaultEmbedder, err = vectorsearch.Embedder(log, c.VectorSearch) + if err != nil { + return + } + schema := c.Indexer.Schemas[0] if len(schema.ClientKey) == 0 || len(schema.ClientSecret) == 0 { return fmt.Errorf("client key and secret is missing") @@ -90,7 +103,7 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) { } // Reindexing existing mapping if needed - DefaultReIndexer = reindex.ReIndexer(log, DefaultEs, DefaultApiClient, c.ES, func(ctx context.Context) (err error) { + DefaultReIndexer = reindex.ReIndexer(log, DefaultEs, DefaultApiClient, DefaultEmbedder, c.ES, func(ctx context.Context) (err error) { err = DefaultMapper.Mappings(ctx, esc, "private") if err != nil { return err diff --git a/extra/server-discovery/pkg/es/mapping/mapping.go b/extra/server-discovery/pkg/es/mapping/mapping.go index a8ca152e17..07a8b9520b 100644 --- a/extra/server-discovery/pkg/es/mapping/mapping.go +++ b/extra/server-discovery/pkg/es/mapping/mapping.go @@ -9,6 +9,7 @@ import ( "net/url" "github.com/cortezaproject/corteza/extra/server-discovery/pkg/es/reindex" + "github.com/davecgh/go-spew/spew" "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/go-elasticsearch/v7/esapi" "github.com/elastic/go-elasticsearch/v7/esutil" @@ -55,6 +56,10 @@ type ( Boost float32 `json:"boost,omitempty"` Properties map[string]*property `json:"properties,omitempty"` + + // for vector search + Dimension int `json:"dimension,omitempty"` + Method map[string]any `json:"method,omitempty"` } esService interface { @@ -154,6 +159,9 @@ func (m *mapper) Mappings(ctx context.Context, esc *elasticsearch.Client, indexP }, }, }, + "index": map[string]any{ + "knn": true, + }, } esReq.Mappings.Properties = im.Mapping @@ -181,6 +189,7 @@ func (m *mapper) Mappings(ctx context.Context, esc *elasticsearch.Client, indexP iLog.Error("index creation failed", zap.Error(err)) } if len(esRsp.String()) > 0 { + spew.Dump("kileee", im) iLog.Error(fmt.Sprintf("index creation failed due to %s", esRsp.String())) } continue diff --git a/extra/server-discovery/pkg/es/reindex/reindex.go b/extra/server-discovery/pkg/es/reindex/reindex.go index 3256f7f12b..a737382fa7 100644 --- a/extra/server-discovery/pkg/es/reindex/reindex.go +++ b/extra/server-discovery/pkg/es/reindex/reindex.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "net/url" + "strings" "time" "github.com/cortezaproject/corteza/extra/server-discovery/pkg/options" @@ -86,6 +87,10 @@ type ( BulkIndexer() (esutil.BulkIndexer, error) } + embedderService interface { + GenerateEmbeddings(input string) ([]float32, error) + } + apiClientService interface { HttpClient() *http.Client Mappings() (*http.Request, error) @@ -99,6 +104,7 @@ type ( log *zap.Logger esOpt options.EsOpt es esService + embedder embedderService api apiClientService assureMappings func(context.Context) error } @@ -108,11 +114,12 @@ const ( IndexTpl = "corteza-%s-%s" ) -func ReIndexer(log *zap.Logger, es esService, api apiClientService, esOpt options.EsOpt, assureMappings func(context.Context) error) *reIndexer { +func ReIndexer(log *zap.Logger, es esService, api apiClientService, embedderSvc embedderService, esOpt options.EsOpt, assureMappings func(context.Context) error) *reIndexer { return &reIndexer{ log: log, esOpt: esOpt, es: es, + embedder: embedderSvc, api: api, assureMappings: assureMappings, } @@ -310,11 +317,17 @@ func (ri *reIndexer) reindex(ctx context.Context, esb esutil.BulkIndexer, indexP } for _, doc := range rspPayload.Response.Documents { + body := bytes.NewBuffer(doc.Source) + err = processResource(body) + if err != nil { + ri.log.Error("Failed to process record's for embeddings: ", zap.Error(err)) + } + err = esb.Add(ctx, esutil.BulkIndexerItem{ Index: fmt.Sprintf(IndexTpl, indexPrefix, ds.index), Action: "index", DocumentID: doc.ID, - Body: bytes.NewBuffer(doc.Source), + Body: body, OnFailure: func(ctx context.Context, req esutil.BulkIndexerItem, rsp esutil.BulkIndexerResponseItem, err error) { spew.Dump(req) spew.Dump(rsp) @@ -672,7 +685,7 @@ func (ri *reIndexer) Watch(ctx context.Context) { return case <-ticker.C: if processing { - ri.log.Warn("skupping feed changes reindexing: already processing") + ri.log.Warn("skipping feed changes reindexing: already processing") continue } @@ -744,3 +757,59 @@ func (ri *reIndexer) Watch(ctx context.Context) { } }() } + +func (ri *reIndexer) processResource(source *bytes.Buffer) error { + var data map[string]interface{} + if err := json.Unmarshal(source.Bytes(), &data); err != nil { + return fmt.Errorf("Error unmarshaling: %v\n", err) + } + + resourceType, ok := data["resourceType"].(string) + if !ok { + return fmt.Errorf("resourceType not found or not a string") + } + + // only generate embeddings for compose:record for now + if resourceType == "compose:record" { + catchAll, ok := data["catch_all"].([]interface{}) + if !ok { + return fmt.Errorf("records catch_all not found or not an array") + } + + //spew.Dump("okaaari", catchAll, data) + + embeddings, err := ri.embedder.GenerateEmbeddings(convertValuesToString(catchAll)) + if err != nil { + return err + } + + data["vectorsValue"] = embeddings + + updatedJSON, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("Error marshaling: %v\n", err) + } + + source.Reset() + source.Write(updatedJSON) + } + + return nil +} + +func convertValuesToString(values []interface{}) string { + var parts []string + + for _, item := range values { + switch val := item.(type) { + case string: + parts = append(parts, val) + case float64: + parts = append(parts, fmt.Sprintf("%.0f", val)) + default: + parts = append(parts, fmt.Sprintf("%v", val)) + } + } + + return strings.Join(parts, " ") +} diff --git a/extra/server-discovery/pkg/options/options.go b/extra/server-discovery/pkg/options/options.go index 6c5024985a..7a6e9eab21 100644 --- a/extra/server-discovery/pkg/options/options.go +++ b/extra/server-discovery/pkg/options/options.go @@ -2,13 +2,14 @@ package options type ( Options struct { - Corteza CortezaOpt - ES EsOpt - Indexer IndexerOpt - Searcher SearcherOpt - Environment EnvironmentOpt - HTTPServer HttpServerOpt - WaitFor WaitForOpt + Corteza CortezaOpt + ES EsOpt + Indexer IndexerOpt + Searcher SearcherOpt + Environment EnvironmentOpt + HTTPServer HttpServerOpt + WaitFor WaitForOpt + VectorSearch VectorSearchOpt } ) @@ -35,13 +36,19 @@ func Init() (opt *Options, err error) { return } + vectorSearch, err := VectorSearch() + if err != nil { + return + } + return &Options{ - Corteza: *corteza, - ES: *es, - Indexer: *indexer, - Searcher: *searcher, - Environment: *Environment(), - HTTPServer: *HttpServer(), - WaitFor: *WaitFor(), + Corteza: *corteza, + ES: *es, + Indexer: *indexer, + Searcher: *searcher, + Environment: *Environment(), + HTTPServer: *HttpServer(), + WaitFor: *WaitFor(), + VectorSearch: *vectorSearch, }, nil } diff --git a/extra/server-discovery/pkg/options/vectorsearch.go b/extra/server-discovery/pkg/options/vectorsearch.go new file mode 100644 index 0000000000..452e7ca94a --- /dev/null +++ b/extra/server-discovery/pkg/options/vectorsearch.go @@ -0,0 +1,23 @@ +package options + +import "github.com/cortezaproject/corteza/server/pkg/options" + +type ( + VectorSearchOpt struct { + EmbeddingsAPIUrl string `env:"EMBEDDINGS_API_URL"` + EmbeddingsAPIKey string `env:"EMBEDDINGS_API_KEY"` + EmbeddingsModel string `env:"EMBEDDINGS_MODEL"` + } +) + +func VectorSearch() (o *VectorSearchOpt, err error) { + o = &VectorSearchOpt{} + + return o, func() error { + o.EmbeddingsAPIKey = options.EnvString("EMBEDDINGS_API_KEY", "") + o.EmbeddingsAPIUrl = options.EnvString("EMBEDDINGS_API_URL", "") + o.EmbeddingsModel = options.EnvString("EMBEDDINGS_MODEL", "paraphrase-multilingual-MiniLM-L12-v2") + + return nil + }() +} diff --git a/extra/server-discovery/pkg/vectorsearch/embeddings.go b/extra/server-discovery/pkg/vectorsearch/embeddings.go new file mode 100644 index 0000000000..b15bada41e --- /dev/null +++ b/extra/server-discovery/pkg/vectorsearch/embeddings.go @@ -0,0 +1,107 @@ +package vectorsearch + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/cortezaproject/corteza/extra/server-discovery/pkg/options" + "go.uber.org/zap" +) + +type ( + EmbeddingRequest struct { + Input []string `json:"input"` // TODO:: replace this with "input" + Model string `json:"model,omitempty"` + } + + EmbeddingResponse struct { + Embeddings [][]float32 `json:"embeddings"` + Dimension int `json:"dimension"` + Count int `json:"count"` + } + + embedder struct { + log *zap.Logger + vectorSearchOpt options.VectorSearchOpt + } +) + +func Embedder(log *zap.Logger, opt options.VectorSearchOpt) (out *embedder, err error) { + out = &embedder{log: log, vectorSearchOpt: opt} + return +} + +// consolidate this it should be on both +func (embedder *embedder) GenerateEmbeddings(input string) ([]float32, error) { + requestBody := EmbeddingRequest{ + Input: []string{input}, + } + + body, err := json.Marshal(requestBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %v", err) + } + + // TODO: replace this with the env configurable option + // Add the API KEY section + url := embedder.vectorSearchOpt.EmbeddingsAPIUrl + req, err := http.NewRequest("POST", url, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %v", err) + } + + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{ + Timeout: 30 * time.Second, + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to call embedding service: %v", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response: %v", err) + } + + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("embedding service error %d: %s", resp.StatusCode, string(respBody)) + } + + var result EmbeddingResponse + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to decode embedding response: %v", err) + } + + if len(result.Embeddings) == 0 { + return nil, fmt.Errorf("no embeddings returned") + } + + return result.Embeddings[0], nil +} + +func (embedder *embedder) ValidateEmbeddingsAPI(dimension int) (bool, error) { + embeddings, err := embedder.GenerateEmbeddings("Corteza") + if err != nil { + return false, err + } + + embeddingsLength := len(embeddings) + + if embeddingsLength == 0 { + return false, fmt.Errorf("no embeddings returned") + } + + if embeddingsLength != dimension { + return false, fmt.Errorf("Embeddings dimensions mismatch: expected %d, got %d", dimension, embeddingsLength) + } + + return true, nil +} diff --git a/extra/server-discovery/searcher/rest/es.go b/extra/server-discovery/searcher/rest/es.go index d9663ca5e9..8ca0e4061c 100644 --- a/extra/server-discovery/searcher/rest/es.go +++ b/extra/server-discovery/searcher/rest/es.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "github.com/cortezaproject/corteza/extra/server-discovery/pkg/vectorsearch" "github.com/davecgh/go-spew/spew" "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/go-elasticsearch/v7/esapi" @@ -64,16 +65,29 @@ type ( } `json:"multi_match"` } + VectorValue struct { + Vector []float32 `json:"vector"` + K int `json:"k"` + Filter []interface{} `json:"filter,omitempty"` + } + + KNN struct { + VectorValue VectorValue `json:"vectorValue"` + } + esSearchParams struct { Query struct { Bool struct { // query context - Must []interface{} `json:"must,omitempty"` + Must []interface{} `json:"must,omitempty"` + Should []interface{} `json:"should,omitempty"` // filter context Filter []interface{} `json:"filter,omitempty"` MustNot []interface{} `json:"must_not,omitempty"` } `json:"bool,omitempty"` + + //KNN KNN `json:"knn,omitempty"` } `json:"query"` Aggregations EsSearchNestedAggrTerms `json:"aggs,omitempty"` @@ -205,6 +219,7 @@ type ( mAggOnly bool allowedRoles map[interface{}]bool + //embedder *embedder } ) @@ -289,13 +304,39 @@ func esSearch(ctx context.Context, log *zap.Logger, esc *elasticsearch.Client, p // Search string filter if !noQ { - query.Query.Bool.Must = append(query.Query.Bool.Must, map[string]interface{}{ - "wildcard": map[string]interface{}{ - "catch_all": map[string]interface{}{ - "value": fmt.Sprintf("*%s*", sqs.Wrap.Query), + vector, err := vectorsearch.GenerateEmbeddings(sqs.Wrap.Query) + if err != nil { + log.Error("failed to generate embeddings", zap.Error(err)) + + // Only text search + query.Query.Bool.Must = append(query.Query.Bool.Must, map[string]interface{}{ + "wildcard": map[string]interface{}{ + "catch_all": map[string]interface{}{ + "value": fmt.Sprintf("*%s*", sqs.Wrap.Query), + }, }, - }}) - // query.Query.DisMax.Queries = append(query.Query.DisMax.Queries, sqs) + }) + } else { + // Hybrid search: combine vector and text in should clause + query.Query.Bool.Should = append(query.Query.Bool.Should, + map[string]interface{}{ + "knn": map[string]interface{}{ + "vectorsValue": map[string]interface{}{ + "vector": vector, + "k": 10, + }, + }, + }, + + map[string]interface{}{ + "wildcard": map[string]interface{}{ + "catch_all": map[string]interface{}{ + "value": fmt.Sprintf("*%s*", sqs.Wrap.Query), + }, + }, + }, + ) + } } var ( @@ -483,6 +524,8 @@ func esSearch(ctx context.Context, log *zap.Logger, esc *elasticsearch.Client, p // Perform the search request. res, err := esc.Search(sReqArgs...) + spew.Dump("kotikooo", res) + if err != nil { return } diff --git a/extra/server-discovery/searcher/rest/search.go b/extra/server-discovery/searcher/rest/search.go index 4e2237fada..683635b2ce 100644 --- a/extra/server-discovery/searcher/rest/search.go +++ b/extra/server-discovery/searcher/rest/search.go @@ -12,7 +12,13 @@ import ( ) type ( - search struct{} + search struct { + embedder embedderService + } + + embedderService interface { + GenerateEmbeddings(input string) ([]float32, error) + } cResponse struct { Response struct { @@ -60,8 +66,10 @@ type ( } ) -func Search() *search { - return &search{} +func Search(embedderSvc embedderService) *search { + return &search{ + embedder: embedderSvc, + } } func (s search) SearchResources(ctx context.Context, r *request.SearchResources) (out interface{}, err error) { diff --git a/server/discovery/rest/internal/documents/documents.go b/server/discovery/rest/internal/documents/documents.go index 1ee1deeb0a..ed6856b21f 100644 --- a/server/discovery/rest/internal/documents/documents.go +++ b/server/discovery/rest/internal/documents/documents.go @@ -129,6 +129,9 @@ type ( // Aggregation update Namespace docPartialComposeNamespace `json:"namespace"` Module docPartialComposeModule `json:"module"` + + // Vectors content field + VectorsValue []float32 `json:"vectorsValue,omitempty"` } docSecurity struct { diff --git a/server/discovery/rest/internal/mapping/compose.go b/server/discovery/rest/internal/mapping/compose.go index 0cfccc9daa..dee8afd3b7 100644 --- a/server/discovery/rest/internal/mapping/compose.go +++ b/server/discovery/rest/internal/mapping/compose.go @@ -222,6 +222,10 @@ func (m composeMapping) records(ctx context.Context, mod *types.Module, mm types }, } + // TODO: if vector embediings is enabled. + // pass every config using the Env. + mapping.Mapping["vectorsValue"] = vector() + return } diff --git a/server/discovery/rest/internal/mapping/mapping.go b/server/discovery/rest/internal/mapping/mapping.go index f9f5dbe8e6..a5f00a2cd7 100644 --- a/server/discovery/rest/internal/mapping/mapping.go +++ b/server/discovery/rest/internal/mapping/mapping.go @@ -17,6 +17,10 @@ type ( Analyzer string `json:"analyzer,omitempty"` Properties map[string]*property `json:"properties,omitempty"` + + // for vector indexing + Dimension int `json:"dimension,omitempty"` + Method map[string]any `json:"method,omitempty"` } Context struct { @@ -47,3 +51,19 @@ func security() *property { }, } } + +// TODO:: all the configurations should be configurable by the USER on the admin. +func vector() *property { + return &property{ + Type: "knn_vector", + Dimension: 384, + Method: map[string]any{ + "name": "hnsw", + "engine": "lucene", + "parameters": map[string]any{ + "ef_construction": 128, + "m": 16, + }, + }, + } +} From e5692d077643da72e0b672ea37edeea17328c011 Mon Sep 17 00:00:00 2001 From: kinyaelgrande Date: Wed, 8 Oct 2025 11:18:59 +0300 Subject: [PATCH 02/17] Make vector search operational --- extra/server-discovery/app/app.go | 1 + extra/server-discovery/app/boot_levels.go | 8 +++++--- extra/server-discovery/app/servers.go | 8 +++++++- extra/server-discovery/indexer/service.go | 3 +++ extra/server-discovery/pkg/es/reindex/reindex.go | 2 +- extra/server-discovery/pkg/options/vectorsearch.go | 4 +++- extra/server-discovery/searcher/rest/es.go | 5 ++--- extra/server-discovery/searcher/rest/router.go | 4 ++-- extra/server-discovery/searcher/rest/search.go | 2 ++ 9 files changed, 26 insertions(+), 11 deletions(-) diff --git a/extra/server-discovery/app/app.go b/extra/server-discovery/app/app.go index 7a69a3b8ba..bac27648f1 100644 --- a/extra/server-discovery/app/app.go +++ b/extra/server-discovery/app/app.go @@ -2,6 +2,7 @@ package app import ( "context" + "github.com/cortezaproject/corteza/extra/server-discovery/pkg/options" "github.com/cortezaproject/corteza/server/pkg/logger" "github.com/davecgh/go-spew/spew" diff --git a/extra/server-discovery/app/boot_levels.go b/extra/server-discovery/app/boot_levels.go index c9b57229f3..c05de462f8 100644 --- a/extra/server-discovery/app/boot_levels.go +++ b/extra/server-discovery/app/boot_levels.go @@ -3,6 +3,7 @@ package app import ( "context" "fmt" + "github.com/cortezaproject/corteza/extra/server-discovery/indexer" "github.com/cortezaproject/corteza/extra/server-discovery/pkg/auth" "github.com/cortezaproject/corteza/extra/server-discovery/pkg/healthcheck" @@ -75,9 +76,10 @@ func (app *CortezaDiscoveryApp) InitServices(ctx context.Context) (err error) { if app.Opt.Indexer.Enabled { err = indexer.Initialize(ctx, app.Log, indexer.Config{ - Corteza: app.Opt.Corteza, - ES: app.Opt.ES, - Indexer: app.Opt.Indexer, + Corteza: app.Opt.Corteza, + ES: app.Opt.ES, + Indexer: app.Opt.Indexer, + VectorSearch: app.Opt.VectorSearch, }) if err != nil { return diff --git a/extra/server-discovery/app/servers.go b/extra/server-discovery/app/servers.go index 56e671c730..5e50dc0b9e 100644 --- a/extra/server-discovery/app/servers.go +++ b/extra/server-discovery/app/servers.go @@ -3,6 +3,7 @@ package app import ( "strings" + "github.com/cortezaproject/corteza/extra/server-discovery/pkg/vectorsearch" searcherRest "github.com/cortezaproject/corteza/extra/server-discovery/searcher/rest" "github.com/cortezaproject/corteza/server/pkg/options" "github.com/go-chi/chi/v5" @@ -28,7 +29,12 @@ func (app *CortezaDiscoveryApp) MountHttpRoutes(r chi.Router) { zap.String("baseUrl", fullPathAPI), ) - r.Route("/", searcherRest.MountRoutes()) + embedderSvc, err := vectorsearch.Embedder(app.Log, app.Opt.VectorSearch) + if err != nil { + app.Log.Error(err.Error()) + } + + r.Route("/", searcherRest.MountRoutes(embedderSvc)) }) }() diff --git a/extra/server-discovery/indexer/service.go b/extra/server-discovery/indexer/service.go index 253259e033..8d77150ec8 100644 --- a/extra/server-discovery/indexer/service.go +++ b/extra/server-discovery/indexer/service.go @@ -12,6 +12,7 @@ import ( "github.com/cortezaproject/corteza/extra/server-discovery/pkg/es/reindex" "github.com/cortezaproject/corteza/extra/server-discovery/pkg/options" "github.com/cortezaproject/corteza/extra/server-discovery/pkg/vectorsearch" + "github.com/davecgh/go-spew/spew" "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/go-elasticsearch/v7/esutil" "go.uber.org/zap" @@ -74,6 +75,8 @@ func Initialize(ctx context.Context, log *zap.Logger, c Config) (err error) { return } + spew.Dump("karikoo", c.VectorSearch) + schema := c.Indexer.Schemas[0] if len(schema.ClientKey) == 0 || len(schema.ClientSecret) == 0 { return fmt.Errorf("client key and secret is missing") diff --git a/extra/server-discovery/pkg/es/reindex/reindex.go b/extra/server-discovery/pkg/es/reindex/reindex.go index a737382fa7..d1096de638 100644 --- a/extra/server-discovery/pkg/es/reindex/reindex.go +++ b/extra/server-discovery/pkg/es/reindex/reindex.go @@ -318,7 +318,7 @@ func (ri *reIndexer) reindex(ctx context.Context, esb esutil.BulkIndexer, indexP for _, doc := range rspPayload.Response.Documents { body := bytes.NewBuffer(doc.Source) - err = processResource(body) + err = ri.processResource(body) if err != nil { ri.log.Error("Failed to process record's for embeddings: ", zap.Error(err)) } diff --git a/extra/server-discovery/pkg/options/vectorsearch.go b/extra/server-discovery/pkg/options/vectorsearch.go index 452e7ca94a..2c951b62cf 100644 --- a/extra/server-discovery/pkg/options/vectorsearch.go +++ b/extra/server-discovery/pkg/options/vectorsearch.go @@ -1,6 +1,8 @@ package options -import "github.com/cortezaproject/corteza/server/pkg/options" +import ( + "github.com/cortezaproject/corteza/server/pkg/options" +) type ( VectorSearchOpt struct { diff --git a/extra/server-discovery/searcher/rest/es.go b/extra/server-discovery/searcher/rest/es.go index 8ca0e4061c..4370e156b4 100644 --- a/extra/server-discovery/searcher/rest/es.go +++ b/extra/server-discovery/searcher/rest/es.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" - "github.com/cortezaproject/corteza/extra/server-discovery/pkg/vectorsearch" "github.com/davecgh/go-spew/spew" "github.com/elastic/go-elasticsearch/v7" "github.com/elastic/go-elasticsearch/v7/esapi" @@ -219,7 +218,7 @@ type ( mAggOnly bool allowedRoles map[interface{}]bool - //embedder *embedder + embedder embedderService } ) @@ -304,7 +303,7 @@ func esSearch(ctx context.Context, log *zap.Logger, esc *elasticsearch.Client, p // Search string filter if !noQ { - vector, err := vectorsearch.GenerateEmbeddings(sqs.Wrap.Query) + vector, err := p.embedder.GenerateEmbeddings(sqs.Wrap.Query) if err != nil { log.Error("failed to generate embeddings", zap.Error(err)) diff --git a/extra/server-discovery/searcher/rest/router.go b/extra/server-discovery/searcher/rest/router.go index 5c1e02dfe5..79664087ce 100644 --- a/extra/server-discovery/searcher/rest/router.go +++ b/extra/server-discovery/searcher/rest/router.go @@ -6,11 +6,11 @@ import ( "github.com/go-chi/chi/v5" ) -func MountRoutes() func(r chi.Router) { +func MountRoutes(embeddingSvc embedderService) func(r chi.Router) { return func(r chi.Router) { r.Group(func(r chi.Router) { r.Use(auth.HttpTokenValidator("discovery")) - handlers.NewSearch(Search()).MountRoutes(r) + handlers.NewSearch(Search(embeddingSvc)).MountRoutes(r) }) } } diff --git a/extra/server-discovery/searcher/rest/search.go b/extra/server-discovery/searcher/rest/search.go index 683635b2ce..f62252f0a0 100644 --- a/extra/server-discovery/searcher/rest/search.go +++ b/extra/server-discovery/searcher/rest/search.go @@ -117,6 +117,7 @@ func (s search) SearchResources(ctx context.Context, r *request.SearchResources) namespaceAggs: namespaceAggs, dumpRaw: validDumpRaw, allowedRoles: allowedRoles, + embedder: s.embedder, }) if err != nil { return nil, fmt.Errorf("could not execute search: %w", err) @@ -229,6 +230,7 @@ func (s search) SearchResources(ctx context.Context, r *request.SearchResources) aggOnly: true, mAggOnly: true, allowedRoles: allowedRoles, + embedder: s.embedder, }) if err != nil { return nil, fmt.Errorf("could not execute module aggregation search: %w", err) From ea950fd9070c365bc230336137da318948e428fb Mon Sep 17 00:00:00 2001 From: kinyaelgrande Date: Mon, 27 Oct 2025 09:31:34 +0300 Subject: [PATCH 03/17] Add RAG corteza assistant --- client/web/discovery/src/components/Rag.vue | 186 + .../src/components/rag/ChatInput.vue | 91 + .../src/components/rag/ChatMessage.vue | 90 + client/web/discovery/src/plugins/searcher.js | 42 + client/web/discovery/src/views/Layout.vue | 5 + extra/server-discovery/app/servers.go | 8 +- extra/server-discovery/go.mod | 12 +- extra/server-discovery/go.sum | 45 +- .../pkg/es/reindex/reindex.go | 2 - .../pkg/options/vectorsearch.go | 4 + .../pkg/vectorsearch/embeddings.go | 2 +- extra/server-discovery/searcher/rest/conv.go | 4 +- extra/server-discovery/searcher/rest/es.go | 42 +- .../searcher/rest/handlers/rag.go | 46 + extra/server-discovery/searcher/rest/rag.go | 140 + .../server-discovery/searcher/rest/router.go | 26 +- .../asaskevich/govalidator/validator.go | 33 +- .../server/compose/types/getters_setters.go | 39 + .../server/compose/types/module_field.go | 5 +- .../corteza/server/compose/types/record.go | 82 +- .../server/compose/types/record_value.go | 15 +- .../corteza/server/pkg/expr/expr_types.gen.go | 1 - .../corteza/server/pkg/expr/expr_types.go | 6 + .../corteza/server/pkg/http/request.go | 57 +- .../corteza/server/pkg/payload/util.go | 7 +- .../server/system/types/app_settings.go | 12 +- .../server/system/types/notification.go | 106 + .../server/system/types/resources.gen.go | 1 + .../server/system/types/type_set.gen.go | 61 + .../corteza/server/system/types/types.yaml | 1 + .../github.com/dlclark/regexp2/.gitignore | 27 + .../github.com/dlclark/regexp2/.travis.yml | 7 + .../vendor/github.com/dlclark/regexp2/ATTRIB | 133 + .../vendor/github.com/dlclark/regexp2/LICENSE | 21 + .../github.com/dlclark/regexp2/README.md | 167 + .../github.com/dlclark/regexp2/fastclock.go | 129 + .../github.com/dlclark/regexp2/match.go | 347 + .../github.com/dlclark/regexp2/regexp.go | 373 + .../github.com/dlclark/regexp2/replace.go | 177 + .../github.com/dlclark/regexp2/runner.go | 1609 ++++ .../dlclark/regexp2/syntax/charclass.go | 865 ++ .../github.com/dlclark/regexp2/syntax/code.go | 274 + .../dlclark/regexp2/syntax/escape.go | 94 + .../github.com/dlclark/regexp2/syntax/fuzz.go | 20 + .../dlclark/regexp2/syntax/parser.go | 2262 ++++++ .../dlclark/regexp2/syntax/prefix.go | 896 +++ .../dlclark/regexp2/syntax/replacerdata.go | 87 + .../github.com/dlclark/regexp2/syntax/tree.go | 654 ++ .../dlclark/regexp2/syntax/writer.go | 500 ++ .../github.com/dlclark/regexp2/testoutput1 | 7061 +++++++++++++++++ .../vendor/github.com/golang/protobuf/AUTHORS | 3 - .../github.com/golang/protobuf/CONTRIBUTORS | 3 - .../vendor/github.com/golang/protobuf/LICENSE | 28 - .../golang/protobuf/proto/buffer.go | 324 - .../golang/protobuf/proto/defaults.go | 63 - .../golang/protobuf/proto/deprecated.go | 113 - .../golang/protobuf/proto/discard.go | 58 - .../golang/protobuf/proto/extensions.go | 356 - .../golang/protobuf/proto/properties.go | 306 - .../github.com/golang/protobuf/proto/proto.go | 167 - .../golang/protobuf/proto/registry.go | 317 - .../golang/protobuf/proto/text_decode.go | 801 -- .../golang/protobuf/proto/text_encode.go | 560 -- .../github.com/golang/protobuf/proto/wire.go | 78 - .../golang/protobuf/proto/wrappers.go | 34 - .../microcosm-cc/bluemonday/CONTRIBUTING.md | 9 +- .../microcosm-cc/bluemonday/LICENSE.md | 3 + .../microcosm-cc/bluemonday/Makefile | 8 +- .../microcosm-cc/bluemonday/README.md | 2 +- .../microcosm-cc/bluemonday/css/handlers.go | 274 +- .../github.com/microcosm-cc/bluemonday/doc.go | 30 +- .../microcosm-cc/bluemonday/helpers.go | 17 +- .../microcosm-cc/bluemonday/policy.go | 51 + .../microcosm-cc/bluemonday/sanitize.go | 80 +- .../github.com/pkoukk/tiktoken-go/LICENSE | 21 + .../github.com/pkoukk/tiktoken-go/README.md | 256 + .../pkoukk/tiktoken-go/README_zh-hans.md | 239 + .../github.com/pkoukk/tiktoken-go/bpe.go | 75 + .../github.com/pkoukk/tiktoken-go/core_bpe.go | 191 + .../github.com/pkoukk/tiktoken-go/encoding.go | 186 + .../github.com/pkoukk/tiktoken-go/load.go | 107 + .../github.com/pkoukk/tiktoken-go/tiktoken.go | 128 + .../vendor/github.com/tmc/langchaingo/LICENSE | 21 + .../callbacks/agent_final_stream.go | 126 + .../tmc/langchaingo/callbacks/callbacks.go | 36 + .../tmc/langchaingo/callbacks/combining.go | 111 + .../tmc/langchaingo/callbacks/doc.go | 4 + .../tmc/langchaingo/callbacks/log.go | 125 + .../tmc/langchaingo/callbacks/log_stream.go | 18 + .../tmc/langchaingo/callbacks/simple.go | 30 + .../tmc/langchaingo/llms/chat_messages.go | 204 + .../tmc/langchaingo/llms/count_tokens.go | 67 + .../github.com/tmc/langchaingo/llms/doc.go | 19 + .../tmc/langchaingo/llms/generatecontent.go | 187 + .../github.com/tmc/langchaingo/llms/llms.go | 52 + .../tmc/langchaingo/llms/marshaling.go | 302 + .../llms/openai/internal/openaiclient/chat.go | 578 ++ .../internal/openaiclient/completions.go | 93 + .../internal/openaiclient/embeddings.go | 84 + .../internal/openaiclient/openaiclient.go | 185 + .../tmc/langchaingo/llms/openai/llm.go | 65 + .../tmc/langchaingo/llms/openai/openaillm.go | 270 + .../llms/openai/openaillm_option.go | 137 + .../tmc/langchaingo/llms/options.go | 282 + .../tmc/langchaingo/llms/prompts.go | 7 + .../schema/chat_message_history.go | 30 + .../github.com/tmc/langchaingo/schema/doc.go | 3 + .../tmc/langchaingo/schema/documents.go | 8 + .../tmc/langchaingo/schema/memory.go | 18 + .../tmc/langchaingo/schema/output_parsers.go | 15 + .../tmc/langchaingo/schema/retrivers.go | 8 + .../tmc/langchaingo/schema/schema.go | 21 + .../x/oauth2/internal/client_appengine.go | 13 - .../golang.org/x/oauth2/internal/transport.go | 5 - .../vendor/golang.org/x/oauth2/oauth2.go | 2 +- .../google.golang.org/appengine/LICENSE | 202 - .../appengine/internal/api.go | 653 -- .../appengine/internal/api_classic.go | 170 - .../appengine/internal/api_common.go | 141 - .../appengine/internal/app_id.go | 28 - .../appengine/internal/base/api_base.pb.go | 308 - .../appengine/internal/base/api_base.proto | 33 - .../internal/datastore/datastore_v3.pb.go | 4367 ---------- .../internal/datastore/datastore_v3.proto | 551 -- .../appengine/internal/identity.go | 54 - .../appengine/internal/identity_classic.go | 62 - .../appengine/internal/identity_flex.go | 12 - .../appengine/internal/identity_vm.go | 134 - .../appengine/internal/internal.go | 110 - .../appengine/internal/log/log_service.pb.go | 1313 --- .../appengine/internal/log/log_service.proto | 150 - .../appengine/internal/main.go | 17 - .../appengine/internal/main_common.go | 7 - .../appengine/internal/main_vm.go | 70 - .../appengine/internal/metadata.go | 60 - .../appengine/internal/net.go | 56 - .../appengine/internal/regen.sh | 40 - .../internal/remote_api/remote_api.pb.go | 361 - .../internal/remote_api/remote_api.proto | 44 - .../appengine/internal/transaction.go | 115 - .../internal/urlfetch/urlfetch_service.pb.go | 527 -- .../internal/urlfetch/urlfetch_service.proto | 64 - .../appengine/urlfetch/urlfetch.go | 209 - .../vendor/google.golang.org/protobuf/LICENSE | 27 - .../vendor/google.golang.org/protobuf/PATENTS | 22 - .../protobuf/encoding/prototext/decode.go | 772 -- .../protobuf/encoding/prototext/doc.go | 7 - .../protobuf/encoding/prototext/encode.go | 376 - .../protobuf/encoding/protowire/wire.go | 547 -- .../protobuf/internal/descfmt/stringer.go | 413 - .../protobuf/internal/descopts/options.go | 29 - .../protobuf/internal/detrand/rand.go | 69 - .../internal/encoding/defval/default.go | 213 - .../encoding/messageset/messageset.go | 242 - .../protobuf/internal/encoding/tag/tag.go | 207 - .../protobuf/internal/encoding/text/decode.go | 686 -- .../internal/encoding/text/decode_number.go | 211 - .../internal/encoding/text/decode_string.go | 161 - .../internal/encoding/text/decode_token.go | 373 - .../protobuf/internal/encoding/text/doc.go | 29 - .../protobuf/internal/encoding/text/encode.go | 272 - .../protobuf/internal/errors/errors.go | 89 - .../protobuf/internal/errors/is_go112.go | 40 - .../protobuf/internal/errors/is_go113.go | 13 - .../protobuf/internal/filedesc/build.go | 157 - .../protobuf/internal/filedesc/desc.go | 678 -- .../protobuf/internal/filedesc/desc_init.go | 471 -- .../protobuf/internal/filedesc/desc_lazy.go | 704 -- .../protobuf/internal/filedesc/desc_list.go | 457 -- .../internal/filedesc/desc_list_gen.go | 356 - .../protobuf/internal/filedesc/placeholder.go | 109 - .../protobuf/internal/filetype/build.go | 296 - .../protobuf/internal/flags/flags.go | 24 - .../internal/flags/proto_legacy_disable.go | 10 - .../internal/flags/proto_legacy_enable.go | 10 - .../protobuf/internal/genid/any_gen.go | 34 - .../protobuf/internal/genid/api_gen.go | 106 - .../protobuf/internal/genid/descriptor_gen.go | 1087 --- .../protobuf/internal/genid/doc.go | 11 - .../protobuf/internal/genid/duration_gen.go | 34 - .../protobuf/internal/genid/empty_gen.go | 19 - .../protobuf/internal/genid/field_mask_gen.go | 31 - .../protobuf/internal/genid/goname.go | 25 - .../protobuf/internal/genid/map_entry.go | 16 - .../internal/genid/source_context_gen.go | 31 - .../protobuf/internal/genid/struct_gen.go | 116 - .../protobuf/internal/genid/timestamp_gen.go | 34 - .../protobuf/internal/genid/type_gen.go | 190 - .../protobuf/internal/genid/wrappers.go | 13 - .../protobuf/internal/genid/wrappers_gen.go | 175 - .../protobuf/internal/impl/api_export.go | 177 - .../protobuf/internal/impl/checkinit.go | 141 - .../protobuf/internal/impl/codec_extension.go | 223 - .../protobuf/internal/impl/codec_field.go | 830 -- .../protobuf/internal/impl/codec_gen.go | 5724 ------------- .../protobuf/internal/impl/codec_map.go | 388 - .../protobuf/internal/impl/codec_map_go111.go | 38 - .../protobuf/internal/impl/codec_map_go112.go | 12 - .../protobuf/internal/impl/codec_message.go | 217 - .../internal/impl/codec_messageset.go | 123 - .../protobuf/internal/impl/codec_reflect.go | 210 - .../protobuf/internal/impl/codec_tables.go | 557 -- .../protobuf/internal/impl/codec_unsafe.go | 18 - .../protobuf/internal/impl/convert.go | 495 -- .../protobuf/internal/impl/convert_list.go | 141 - .../protobuf/internal/impl/convert_map.go | 121 - .../protobuf/internal/impl/decode.go | 285 - .../protobuf/internal/impl/encode.go | 201 - .../protobuf/internal/impl/enum.go | 21 - .../protobuf/internal/impl/extension.go | 156 - .../protobuf/internal/impl/legacy_enum.go | 218 - .../protobuf/internal/impl/legacy_export.go | 92 - .../internal/impl/legacy_extension.go | 176 - .../protobuf/internal/impl/legacy_file.go | 81 - .../protobuf/internal/impl/legacy_message.go | 568 -- .../protobuf/internal/impl/merge.go | 176 - .../protobuf/internal/impl/merge_gen.go | 209 - .../protobuf/internal/impl/message.go | 284 - .../protobuf/internal/impl/message_reflect.go | 463 -- .../internal/impl/message_reflect_field.go | 543 -- .../internal/impl/message_reflect_gen.go | 249 - .../protobuf/internal/impl/pointer_reflect.go | 215 - .../protobuf/internal/impl/pointer_unsafe.go | 215 - .../protobuf/internal/impl/validate.go | 576 -- .../protobuf/internal/impl/weak.go | 74 - .../protobuf/internal/order/order.go | 89 - .../protobuf/internal/order/range.go | 115 - .../protobuf/internal/pragma/pragma.go | 29 - .../protobuf/internal/set/ints.go | 58 - .../protobuf/internal/strs/strings.go | 196 - .../protobuf/internal/strs/strings_pure.go | 28 - .../internal/strs/strings_unsafe_go120.go | 95 - .../internal/strs/strings_unsafe_go121.go | 74 - .../protobuf/internal/version/version.go | 79 - .../protobuf/proto/checkinit.go | 71 - .../protobuf/proto/decode.go | 294 - .../protobuf/proto/decode_gen.go | 603 -- .../google.golang.org/protobuf/proto/doc.go | 86 - .../protobuf/proto/encode.go | 322 - .../protobuf/proto/encode_gen.go | 97 - .../google.golang.org/protobuf/proto/equal.go | 57 - .../protobuf/proto/extension.go | 92 - .../google.golang.org/protobuf/proto/merge.go | 139 - .../protobuf/proto/messageset.go | 93 - .../google.golang.org/protobuf/proto/proto.go | 45 - .../protobuf/proto/proto_methods.go | 20 - .../protobuf/proto/proto_reflect.go | 20 - .../google.golang.org/protobuf/proto/reset.go | 43 - .../google.golang.org/protobuf/proto/size.go | 101 - .../protobuf/proto/size_gen.go | 55 - .../protobuf/proto/wrappers.go | 29 - .../protobuf/reflect/protodesc/desc.go | 285 - .../protobuf/reflect/protodesc/desc_init.go | 272 - .../reflect/protodesc/desc_resolve.go | 286 - .../reflect/protodesc/desc_validate.go | 374 - .../protobuf/reflect/protodesc/editions.go | 177 - .../reflect/protodesc/editions_defaults.binpb | 4 - .../protobuf/reflect/protodesc/proto.go | 252 - .../protobuf/reflect/protoreflect/methods.go | 78 - .../protobuf/reflect/protoreflect/proto.go | 511 -- .../protobuf/reflect/protoreflect/source.go | 129 - .../reflect/protoreflect/source_gen.go | 554 -- .../protobuf/reflect/protoreflect/type.go | 666 -- .../protobuf/reflect/protoreflect/value.go | 285 - .../reflect/protoreflect/value_equal.go | 168 - .../reflect/protoreflect/value_pure.go | 60 - .../reflect/protoreflect/value_union.go | 438 - .../protoreflect/value_unsafe_go120.go | 99 - .../protoreflect/value_unsafe_go121.go | 87 - .../reflect/protoregistry/registry.go | 882 -- .../protobuf/runtime/protoiface/legacy.go | 15 - .../protobuf/runtime/protoiface/methods.go | 168 - .../protobuf/runtime/protoimpl/impl.go | 44 - .../protobuf/runtime/protoimpl/version.go | 60 - .../types/descriptorpb/descriptor.pb.go | 5656 ------------- extra/server-discovery/vendor/modules.txt | 63 +- 276 files changed, 21310 insertions(+), 52188 deletions(-) create mode 100644 client/web/discovery/src/components/Rag.vue create mode 100644 client/web/discovery/src/components/rag/ChatInput.vue create mode 100644 client/web/discovery/src/components/rag/ChatMessage.vue create mode 100644 extra/server-discovery/searcher/rest/handlers/rag.go create mode 100644 extra/server-discovery/searcher/rest/rag.go create mode 100644 extra/server-discovery/vendor/github.com/cortezaproject/corteza/server/compose/types/getters_setters.go create mode 100644 extra/server-discovery/vendor/github.com/cortezaproject/corteza/server/system/types/notification.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/.gitignore create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/.travis.yml create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/ATTRIB create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/LICENSE create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/README.md create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/fastclock.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/match.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/regexp.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/replace.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/runner.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/syntax/charclass.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/syntax/code.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/syntax/escape.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/syntax/fuzz.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/syntax/parser.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/syntax/prefix.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/syntax/replacerdata.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/syntax/tree.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/syntax/writer.go create mode 100644 extra/server-discovery/vendor/github.com/dlclark/regexp2/testoutput1 delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/AUTHORS delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/CONTRIBUTORS delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/LICENSE delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/buffer.go delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/defaults.go delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/deprecated.go delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/discard.go delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/extensions.go delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/properties.go delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/proto.go delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/registry.go delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/text_decode.go delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/text_encode.go delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/wire.go delete mode 100644 extra/server-discovery/vendor/github.com/golang/protobuf/proto/wrappers.go create mode 100644 extra/server-discovery/vendor/github.com/pkoukk/tiktoken-go/LICENSE create mode 100644 extra/server-discovery/vendor/github.com/pkoukk/tiktoken-go/README.md create mode 100644 extra/server-discovery/vendor/github.com/pkoukk/tiktoken-go/README_zh-hans.md create mode 100644 extra/server-discovery/vendor/github.com/pkoukk/tiktoken-go/bpe.go create mode 100644 extra/server-discovery/vendor/github.com/pkoukk/tiktoken-go/core_bpe.go create mode 100644 extra/server-discovery/vendor/github.com/pkoukk/tiktoken-go/encoding.go create mode 100644 extra/server-discovery/vendor/github.com/pkoukk/tiktoken-go/load.go create mode 100644 extra/server-discovery/vendor/github.com/pkoukk/tiktoken-go/tiktoken.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/LICENSE create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/callbacks/agent_final_stream.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/callbacks/callbacks.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/callbacks/combining.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/callbacks/doc.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/callbacks/log.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/callbacks/log_stream.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/callbacks/simple.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/chat_messages.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/count_tokens.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/doc.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/generatecontent.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/llms.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/marshaling.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/openai/internal/openaiclient/chat.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/openai/internal/openaiclient/completions.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/openai/internal/openaiclient/embeddings.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/openai/internal/openaiclient/openaiclient.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/openai/llm.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/openai/openaillm.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/openai/openaillm_option.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/options.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/llms/prompts.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/schema/chat_message_history.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/schema/doc.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/schema/documents.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/schema/memory.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/schema/output_parsers.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/schema/retrivers.go create mode 100644 extra/server-discovery/vendor/github.com/tmc/langchaingo/schema/schema.go delete mode 100644 extra/server-discovery/vendor/golang.org/x/oauth2/internal/client_appengine.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/LICENSE delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/api.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/api_classic.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/api_common.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/app_id.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/base/api_base.pb.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/base/api_base.proto delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/datastore/datastore_v3.pb.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/datastore/datastore_v3.proto delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/identity.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/identity_classic.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/identity_flex.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/identity_vm.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/internal.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/log/log_service.pb.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/log/log_service.proto delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/main.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/main_common.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/main_vm.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/metadata.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/net.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/regen.sh delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/remote_api/remote_api.pb.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/remote_api/remote_api.proto delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/transaction.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/urlfetch/urlfetch_service.pb.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/internal/urlfetch/urlfetch_service.proto delete mode 100644 extra/server-discovery/vendor/google.golang.org/appengine/urlfetch/urlfetch.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/LICENSE delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/PATENTS delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/encoding/prototext/decode.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/encoding/prototext/doc.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/encoding/prototext/encode.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/encoding/protowire/wire.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/descfmt/stringer.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/descopts/options.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/detrand/rand.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/encoding/defval/default.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/encoding/messageset/messageset.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/encoding/tag/tag.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/encoding/text/decode.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/encoding/text/decode_number.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/encoding/text/decode_string.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/encoding/text/decode_token.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/encoding/text/doc.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/encoding/text/encode.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/errors/errors.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/errors/is_go112.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/errors/is_go113.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/filedesc/build.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/filedesc/desc.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/filedesc/desc_init.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/filedesc/desc_lazy.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/filedesc/desc_list.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/filedesc/desc_list_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/filedesc/placeholder.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/filetype/build.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/flags/flags.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/flags/proto_legacy_disable.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/flags/proto_legacy_enable.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/any_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/api_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/descriptor_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/doc.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/duration_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/empty_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/field_mask_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/goname.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/map_entry.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/source_context_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/struct_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/timestamp_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/type_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/wrappers.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/genid/wrappers_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/api_export.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/checkinit.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/codec_extension.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/codec_field.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/codec_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/codec_map.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/codec_map_go111.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/codec_map_go112.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/codec_message.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/codec_messageset.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/codec_reflect.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/codec_tables.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/codec_unsafe.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/convert.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/convert_list.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/convert_map.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/decode.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/encode.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/enum.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/extension.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/legacy_enum.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/legacy_export.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/legacy_extension.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/legacy_file.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/legacy_message.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/merge.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/merge_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/message.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/message_reflect.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/message_reflect_field.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/message_reflect_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/pointer_reflect.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/pointer_unsafe.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/validate.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/impl/weak.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/order/order.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/order/range.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/pragma/pragma.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/set/ints.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/strs/strings.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/strs/strings_pure.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/strs/strings_unsafe_go120.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/strs/strings_unsafe_go121.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/internal/version/version.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/checkinit.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/decode.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/decode_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/doc.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/encode.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/encode_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/equal.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/extension.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/merge.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/messageset.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/proto.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/proto_methods.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/proto_reflect.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/reset.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/size.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/size_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/proto/wrappers.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protodesc/desc.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protodesc/desc_init.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protodesc/desc_resolve.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protodesc/desc_validate.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protodesc/editions.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protodesc/editions_defaults.binpb delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protodesc/proto.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoreflect/methods.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoreflect/proto.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoreflect/source.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoreflect/source_gen.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoreflect/type.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoreflect/value.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoreflect/value_equal.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoreflect/value_pure.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoreflect/value_union.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoreflect/value_unsafe_go120.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoreflect/value_unsafe_go121.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/reflect/protoregistry/registry.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/runtime/protoiface/legacy.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/runtime/protoiface/methods.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/runtime/protoimpl/impl.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/runtime/protoimpl/version.go delete mode 100644 extra/server-discovery/vendor/google.golang.org/protobuf/types/descriptorpb/descriptor.pb.go diff --git a/client/web/discovery/src/components/Rag.vue b/client/web/discovery/src/components/Rag.vue new file mode 100644 index 0000000000..a22cb187f7 --- /dev/null +++ b/client/web/discovery/src/components/Rag.vue @@ -0,0 +1,186 @@ + + + + + diff --git a/client/web/discovery/src/components/rag/ChatInput.vue b/client/web/discovery/src/components/rag/ChatInput.vue new file mode 100644 index 0000000000..4cf0174d2e --- /dev/null +++ b/client/web/discovery/src/components/rag/ChatInput.vue @@ -0,0 +1,91 @@ +