diff --git a/.gitignore b/.gitignore index faf7cc95a3..cbcfd6b06d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ *.sw* .run node_modules -ignore \ No newline at end of file +ignore +.kilo* \ No newline at end of file diff --git a/server/automation/automation/json_parser_handler.gen.go b/server/automation/automation/json_parser_handler.gen.go new file mode 100644 index 0000000000..e8d62d6656 --- /dev/null +++ b/server/automation/automation/json_parser_handler.gen.go @@ -0,0 +1,286 @@ +package automation + +// This file is auto-generated. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +// Definitions file that controls how this file is generated: +// automation/automation/json_parser_handler.yaml + +import ( + "context" + + atypes "github.com/cortezaproject/corteza/server/automation/types" + "github.com/cortezaproject/corteza/server/pkg/expr" +) + +func (h jsonParserHandler) Parse() *atypes.Function { + return &atypes.Function{ + Ref: "jsonParse", + Kind: "function", + Labels: map[string]string{"json": "step", "parser": "step"}, + Meta: &atypes.FunctionMeta{ + Short: "JSON Parse string into a JSON object", + Description: "Takes a JSON string and returns the parsed JSON value. The result can be an object or array depending on the input.", + }, + + Parameters: []*atypes.Param{ + { + Name: "jsonText", + Types: []string{"String"}, Required: true, + }, + }, + + Results: []*atypes.Param{ + { + Name: "result", + Types: []string{"Any"}, + }, + { + Name: "error", + Types: []string{"String"}, + }, + }, + + Handler: func(ctx context.Context, in *expr.Vars) (out *expr.Vars, err error) { + var ( + args = &jsonParserParseArgs{ + hasJsonText: in.Has("jsonText"), + } + ) + + if err = in.Decode(args); err != nil { + return + } + + // Converting jsonText argument + if args.hasJsonText { + aux := expr.Must(expr.Select(in, "jsonText")) + if t, ok := aux.Get().(string); ok { + args.JsonText = t + } + } + + var results *jsonParserParseResults + if results, err = h.parse(ctx, args); err != nil { + return + } + + out = &expr.Vars{} + + { + // converting results.Result (interface{}) to Any + var ( + tval expr.TypedValue + ) + + if tval, err = h.reg.Type("Any").Cast(results.Result); err != nil { + return + } else if err = expr.Assign(out, "result", tval); err != nil { + return + } + } + { + // converting results.Error (string) to String + var ( + tval expr.TypedValue + ) + + if tval, err = h.reg.Type("String").Cast(results.Error); err != nil { + return + } else if err = expr.Assign(out, "error", tval); err != nil { + return + } + } + + return + }, + } +} + +func (h jsonParserHandler) Stringify() *atypes.Function { + return &atypes.Function{ + Ref: "jsonStringify", + Kind: "function", + Labels: map[string]string{"json": "step", "stringify": "step"}, + Meta: &atypes.FunctionMeta{ + Short: "JSON Convert a object to a string", + Description: "Takes a JSON object (Any type) and returns a JSON string representation.", + }, + + Parameters: []*atypes.Param{ + { + Name: "jsonObject", + Types: []string{"Any"}, Required: true, + }, + }, + + Results: []*atypes.Param{ + { + Name: "result", + Types: []string{"String"}, + }, + { + Name: "error", + Types: []string{"String"}, + }, + }, + + Handler: func(ctx context.Context, in *expr.Vars) (out *expr.Vars, err error) { + var ( + args = &jsonParserStringifyArgs{ + hasJsonObject: in.Has("jsonObject"), + } + ) + + if err = in.Decode(args); err != nil { + return + } + + // Converting jsonObject argument + if args.hasJsonObject { + aux := expr.Must(expr.Select(in, "jsonObject")) + args.JsonObject = aux.Get() + } + + var results *jsonParserStringifyResults + if results, err = h.stringify(ctx, args); err != nil { + return + } + + out = &expr.Vars{} + + { + // converting results.Result (string) to String + var ( + tval expr.TypedValue + ) + + if tval, err = h.reg.Type("String").Cast(results.Result); err != nil { + return + } else if err = expr.Assign(out, "result", tval); err != nil { + return + } + } + { + // converting results.Error (string) to String + var ( + tval expr.TypedValue + ) + + if tval, err = h.reg.Type("String").Cast(results.Error); err != nil { + return + } else if err = expr.Assign(out, "error", tval); err != nil { + return + } + } + + return + }, + } +} + +func (h jsonParserHandler) Template() *atypes.Function { + return &atypes.Function{ + Ref: "jsonTemplate", + Kind: "function", + Labels: map[string]string{"json": "step", "template": "step"}, + Meta: &atypes.FunctionMeta{ + Short: "JSON Template with variable substitution", + Description: "Takes a JSON template string with ${var.key} placeholders and a vars map. Substitutes the placeholders with values from vars and returns the final JSON string. Supports dot-notation for nested values (e.g., ${var.filename}, ${var.data.content}).", + }, + + Parameters: []*atypes.Param{ + { + Name: "template", + Types: []string{"String"}, + Required: true, + }, + { + Name: "vars", + Types: []string{"Vars"}, + Required: false, + }, + }, + + Results: []*atypes.Param{ + { + Name: "result", + Types: []string{"String"}, + }, + { + Name: "error", + Types: []string{"String"}, + }, + }, + + Handler: func(ctx context.Context, in *expr.Vars) (out *expr.Vars, err error) { + var ( + args = &jsonParserTemplateArgs{ + hasVars: in.Has("vars"), + } + ) + + // Extract template manually + if in.Has("template") { + aux := expr.Must(expr.Select(in, "template")) + if t, ok := aux.Get().(string); ok { + args.Template = t + } + } + + // Extract vars manually, unwrapping TypedValue + if args.hasVars { + aux := expr.Must(expr.Select(in, "vars")) + switch m := aux.Get().(type) { + case *expr.Vars: + args.Vars = make(map[string]interface{}) + m.Each(func(k string, v expr.TypedValue) error { + args.Vars[k] = v.Get() + return nil + }) + case map[string]expr.TypedValue: + args.Vars = make(map[string]interface{}) + for k, v := range m { + args.Vars[k] = v.Get() + } + } + } + + var results *jsonParserTemplateResults + if results, err = h.template(ctx, args); err != nil { + return + } + + out = &expr.Vars{} + + { + // converting results.Result (string) to String + var ( + tval expr.TypedValue + ) + + if tval, err = h.reg.Type("String").Cast(results.Result); err != nil { + return + } else if err = expr.Assign(out, "result", tval); err != nil { + return + } + } + { + // converting results.Error (string) to String + var ( + tval expr.TypedValue + ) + + if tval, err = h.reg.Type("String").Cast(results.Error); err != nil { + return + } else if err = expr.Assign(out, "error", tval); err != nil { + return + } + } + + return + }, + } +} diff --git a/server/automation/automation/json_parser_handler.go b/server/automation/automation/json_parser_handler.go new file mode 100644 index 0000000000..c4eaa4e0cf --- /dev/null +++ b/server/automation/automation/json_parser_handler.go @@ -0,0 +1,314 @@ +package automation + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + atypes "github.com/cortezaproject/corteza/server/automation/types" + "github.com/cortezaproject/corteza/server/pkg/expr" +) + +type ( + jsonParserHandlerRegistry interface { + AddFunctions(ff ...*atypes.Function) + Type(ref string) expr.Type + } + + jsonParserHandler struct { + reg jsonParserHandlerRegistry + } + + jsonParserParseArgs struct { + hasJsonText bool + JsonText string + } + + jsonParserParseResults struct { + Result interface{} + Error string + } + + jsonParserStringifyArgs struct { + hasJsonObject bool + JsonObject interface{} + } + + jsonParserStringifyResults struct { + Result string + Error string + } + + jsonParserTemplateArgs struct { + hasVars bool + Template string + Vars map[string]interface{} + } + + jsonParserTemplateResults struct { + Result string + Error string + } +) + +func JsonParserHandler(reg jsonParserHandlerRegistry) *jsonParserHandler { + h := &jsonParserHandler{ + reg: reg, + } + + h.register() + return h +} + +func (h jsonParserHandler) register() { + h.reg.AddFunctions( + h.Parse(), + h.Stringify(), + h.Template(), + ) +} + +// parse implements the jsonParse function +func (h jsonParserHandler) parse(ctx context.Context, args *jsonParserParseArgs) (*jsonParserParseResults, error) { + r := &jsonParserParseResults{} + + if args.JsonText == "" { + r.Error = "jsonText is empty" + return r, nil + } + + var result interface{} + if err := json.Unmarshal([]byte(strings.TrimSpace(args.JsonText)), &result); err != nil { + r.Error = err.Error() + return r, nil + } + + r.Result = result + return r, nil +} + +// Smart stringify: if input is already valid JSON string, return as-is +func (h jsonParserHandler) stringify(ctx context.Context, args *jsonParserStringifyArgs) (*jsonParserStringifyResults, error) { + + r := &jsonParserStringifyResults{} + + if args.JsonObject == nil { + r.Error = "jsonObject is empty" + return r, nil + } + + // If the value is already a string, check if it's valid JSON + if s, ok := args.JsonObject.(string); ok { + // Trim whitespace and try to parse + trimmed := strings.TrimSpace(s) + if len(trimmed) == 0 { + r.Error = "jsonObject is empty" + return r, nil + } + + // Preprocess JS-like object literals into valid JSON + if strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "[") { + trimmed = preprocessJSObjectToJSON(trimmed) + } + + var probe interface{} + if err := json.Unmarshal([]byte(trimmed), &probe); err == nil { + // Already valid JSON string - return as-is (no extra encoding) + r.Result = trimmed + return r, nil + } + // Plain text string - convert to JSON string (wrap in quotes and escape) + b, err := json.Marshal(s) + if err != nil { + r.Error = err.Error() + return r, nil + } + r.Result = string(b) + return r, nil + } + + // Not a string (object, array, number, bool, null) - marshal normally + b, err := json.Marshal(args.JsonObject) + if err != nil { + r.Error = err.Error() + return r, nil + } + + r.Result = string(b) + return r, nil +} + +// substituteVariables replaces ${varName} with values from the vars map +// Supports dot-notation for nested values (e.g., ${var.filename}, ${var.data.content}) +// Also supports any prefix like ${jsonVars.filename}, ${data.content} etc. +func substituteVariables(s string, vars map[string]interface{}) string { + var result strings.Builder + i := 0 + for i < len(s) { + if s[i] == '$' && i+1 < len(s) && s[i+1] == '{' { + // Find closing brace + end := strings.Index(s[i+2:], "}") + if end != -1 { + placeholder := s[i+2 : i+2+end] + + // Strip any leading "word." prefix (e.g. var., jsonVars., data.) + // Use the part after the FIRST dot as the lookup key, + // but preserve dot-notation for nested maps + key := placeholder + if dotIdx := strings.Index(placeholder, "."); dotIdx != -1 { + key = placeholder[dotIdx+1:] + } + + if val := getNestedValue(vars, key); val != nil { + // Convert value to string and sanitize for JSON + strVal := fmt.Sprintf("%v", val) + result.WriteString(sanitizeForJSON(strVal)) + } else { + // Variable not found - leave as-is + result.WriteString("${" + placeholder + "}") + } + i += end + 3 // Skip ${...} + continue + } + } + result.WriteByte(s[i]) + i++ + } + return result.String() +} + +// getNestedValue retrieves a value from a map using dot-notation (e.g., "filename" or "data.content") +func getNestedValue(vars map[string]interface{}, key string) interface{} { + parts := strings.Split(key, ".") + var current interface{} = vars + + for _, part := range parts { + if currentMap, ok := current.(map[string]interface{}); ok { + current = currentMap[part] + if current == nil { + return nil + } + } else { + return nil + } + } + return current +} + +// sanitizeForJSON escapes special characters for use in JSON strings +func sanitizeForJSON(s string) string { + var result strings.Builder + for _, c := range s { + switch c { + case '"': + result.WriteString(`\"`) + case '\\': + result.WriteString(`\\`) + case '\n': + result.WriteString(`\n`) + case '\r': + // skip + case '\t': + result.WriteString(`\t`) + default: + result.WriteRune(c) + } + } + return result.String() +} + +// preprocessJSObjectToJSON converts JavaScript-like object literals to valid JSON +func preprocessJSObjectToJSON(s string) string { + // Step 1: Replace backtick strings with proper JSON double-quoted strings + s = replaceBacktickStrings(s) + + // Step 2: Fix bare backslashes (e.g. openai\gpt -> openai\\gpt) + // Only fix backslashes that are NOT valid JSON escape sequences + s = fixBareBackslashes(s) + + return s +} + +// replaceBacktickStrings converts backtick strings to JSON double-quoted strings +func replaceBacktickStrings(s string) string { + var result strings.Builder + i := 0 + for i < len(s) { + if s[i] == '`' { + // Find closing backtick + result.WriteByte('"') + i++ + for i < len(s) && s[i] != '`' { + switch s[i] { + case '\n': + result.WriteString(`\n`) + case '\r': + // skip \r, handle \r\n as just \n + case '"': + result.WriteString(`\"`) + case '\\': + result.WriteString(`\\`) + default: + result.WriteByte(s[i]) + } + i++ + } + result.WriteByte('"') + i++ // skip closing backtick + } else { + result.WriteByte(s[i]) + i++ + } + } + return result.String() +} + +// fixBareBackslashes escapes bare backslashes that aren't valid JSON escapes +func fixBareBackslashes(s string) string { + // Valid JSON escape chars after backslash + validEscapes := map[byte]bool{ + '"': true, '\\': true, '/': true, 'b': true, + 'f': true, 'n': true, 'r': true, 't': true, 'u': true, + } + + var result strings.Builder + i := 0 + for i < len(s) { + if s[i] == '\\' && i+1 < len(s) { + if validEscapes[s[i+1]] { + // Valid escape sequence - keep as-is + result.WriteByte(s[i]) + } else { + // Bare backslash - escape it + result.WriteString(`\\`) + } + } else { + result.WriteByte(s[i]) + } + i++ + } + return result.String() +} + +// template implements the jsonTemplate function with variable substitution +func (h jsonParserHandler) template(ctx context.Context, args *jsonParserTemplateArgs) (*jsonParserTemplateResults, error) { + r := &jsonParserTemplateResults{} + + if args.Template == "" { + r.Error = "template is empty" + return r, nil + } + + // If no vars provided, just return the template as-is + if args.Vars == nil { + r.Result = args.Template + return r, nil + } + + // Perform variable substitution + result := substituteVariables(args.Template, args.Vars) + r.Result = result + + return r, nil +} diff --git a/server/automation/automation/json_parser_handler.yaml b/server/automation/automation/json_parser_handler.yaml new file mode 100644 index 0000000000..8a862ef3a9 --- /dev/null +++ b/server/automation/automation/json_parser_handler.yaml @@ -0,0 +1,56 @@ +functions: + parse: + meta: + short: JSON Parse string into a JSON object + description: Takes a JSON string and returns the parsed JSON value. The result can be an object or array depending on the input. + labels: + json: "step" + parser: "step" + params: + jsonText: + required: true + types: [{wf: String}] + results: + result: {wf: Any} + error: {wf: String} + + stringify: + meta: + short: JSON Convert a object to a string + description: > + Takes a JSON object (Any type) and returns a JSON string representation. + Also converts JavaScript-style backtick strings to proper JSON. + labels: + json: "step" + stringify: "step" + params: + jsonObject: + required: true + types: [{wf: Any}] + description: The JSON object to stringify (can be string, object, array, number, bool) + results: + result: {wf: String} + error: {wf: String} + + template: + meta: + short: JSON Template with variable substitution + description: > + Takes a JSON template string with ${var.key} placeholders and a vars map. + Substitutes the placeholders with values from vars and returns the final JSON string. + Supports dot-notation for nested values (e.g., ${var.filename}, ${var.data.content}). + labels: + json: "step" + template: "step" + params: + template: + required: true + types: [{wf: String}] + description: The JSON template string with ${var.key} placeholders + vars: + required: false + types: [{wf: Vars}] + description: Variables for substitution (optional) + results: + result: {wf: String} + error: {wf: String} \ No newline at end of file diff --git a/server/automation/service/service.go b/server/automation/service/service.go index 17b74825e6..36ce935aa8 100644 --- a/server/automation/service/service.go +++ b/server/automation/service/service.go @@ -129,6 +129,7 @@ func Initialize(ctx context.Context, log *zap.Logger, s store.Storer, ws websock automation.EmailHandler(Registry()) automation.JwtHandler(Registry()) automation.ApigwBodyHandler(Registry()) + automation.JsonParserHandler(Registry()) return } diff --git a/server/codegen/tool/templating.go b/server/codegen/tool/templating.go index 59fc81f1de..e4834b9dca 100644 --- a/server/codegen/tool/templating.go +++ b/server/codegen/tool/templating.go @@ -38,6 +38,8 @@ func loadTemplates(rTpl *template.Template, rootDir string) (*template.Template, } name := path[pfx:] + // Normalize path separators to forward slashes for cross-platform compatibility + name = strings.ReplaceAll(name, "\\", "/") rTpl, err = rTpl.New(name).Parse(string(b)) return err diff --git a/server/compose/automation/attachment_handler.gen.go b/server/compose/automation/attachment_handler.gen.go index 6418624827..5aae7e885e 100644 --- a/server/compose/automation/attachment_handler.gen.go +++ b/server/compose/automation/attachment_handler.gen.go @@ -10,11 +10,12 @@ package automation import ( "context" + "io" + atypes "github.com/cortezaproject/corteza/server/automation/types" "github.com/cortezaproject/corteza/server/compose/types" "github.com/cortezaproject/corteza/server/pkg/expr" "github.com/cortezaproject/corteza/server/pkg/wfexec" - "io" ) var _ wfexec.ExecResponse @@ -33,6 +34,7 @@ func (h attachmentHandler) register() { h.Delete(), h.OpenOriginal(), h.OpenPreview(), + h.GetBase64(), ) } @@ -472,3 +474,149 @@ func (h attachmentHandler) OpenPreview() *atypes.Function { }, } } + +type ( + attachmentGetBase64Args struct { + hasAttachment bool + Attachment interface{} + attachmentID uint64 + attachmentAttachment *types.Attachment + } + + attachmentGetBase64Results struct { + Content string + Name string + Extension string + MimeType string + } +) + +func (a attachmentGetBase64Args) GetAttachment() (bool, uint64, *types.Attachment) { + return a.hasAttachment, a.attachmentID, a.attachmentAttachment +} + +// GetBase64 function Get attachment as base64 string with metadata +// +// expects implementation of getBase64 function: +// +// func (h attachmentHandler) getBase64(ctx context.Context, args *attachmentGetBase64Args) (results *attachmentGetBase64Results, err error) { +// return +// } +func (h attachmentHandler) GetBase64() *atypes.Function { + return &atypes.Function{ + Ref: "attachmentGetBase64", + Kind: "function", + Labels: map[string]string{"attachment": "step,workflow", "base64-attachment": "step"}, + Meta: &atypes.FunctionMeta{ + Short: "Attachment lookup base64 string with metadata", + Description: "Reads attachment content and returns it as a base64 encoded string with filename, extension, and mimeType", + }, + + Parameters: []*atypes.Param{ + { + Name: "attachment", + Types: []string{"ID", "Attachment"}, Required: true, + }, + }, + + Results: []*atypes.Param{ + { + Name: "content", + Types: []string{"String"}, + }, + { + Name: "name", + Types: []string{"String"}, + }, + { + Name: "extension", + Types: []string{"String"}, + }, + { + Name: "mimeType", + Types: []string{"String"}, + }, + }, + + Handler: func(ctx context.Context, in *expr.Vars) (out *expr.Vars, err error) { + var ( + args = &attachmentGetBase64Args{ + hasAttachment: in.Has("attachment"), + } + ) + + if err = in.Decode(args); err != nil { + return + } + + // Converting Attachment argument + if args.hasAttachment { + aux := expr.Must(expr.Select(in, "attachment")) + switch aux.Type() { + case h.reg.Type("ID").Type(): + args.attachmentID = aux.Get().(uint64) + case h.reg.Type("Attachment").Type(): + args.attachmentAttachment = aux.Get().(*types.Attachment) + } + } + + var results *attachmentGetBase64Results + if results, err = h.getBase64(ctx, args); err != nil { + return + } + + out = &expr.Vars{} + + { + // converting results.Content (string) to String + var ( + tval expr.TypedValue + ) + + if tval, err = h.reg.Type("String").Cast(results.Content); err != nil { + return + } else if err = expr.Assign(out, "content", tval); err != nil { + return + } + } + { + // converting results.Name (string) to String + var ( + tval expr.TypedValue + ) + + if tval, err = h.reg.Type("String").Cast(results.Name); err != nil { + return + } else if err = expr.Assign(out, "name", tval); err != nil { + return + } + } + { + // converting results.Extension (string) to String + var ( + tval expr.TypedValue + ) + + if tval, err = h.reg.Type("String").Cast(results.Extension); err != nil { + return + } else if err = expr.Assign(out, "extension", tval); err != nil { + return + } + } + { + // converting results.MimeType (string) to String + var ( + tval expr.TypedValue + ) + + if tval, err = h.reg.Type("String").Cast(results.MimeType); err != nil { + return + } else if err = expr.Assign(out, "mimeType", tval); err != nil { + return + } + } + + return + }, + } +} diff --git a/server/compose/automation/attachment_handler.go b/server/compose/automation/attachment_handler.go index aba6b266e6..b89e3a2049 100644 --- a/server/compose/automation/attachment_handler.go +++ b/server/compose/automation/attachment_handler.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "encoding/base64" "fmt" "io" "io/ioutil" @@ -88,6 +89,35 @@ func (h attachmentHandler) openPreview(ctx context.Context, args *attachmentOpen return r, nil } +func (h attachmentHandler) getBase64(ctx context.Context, args *attachmentGetBase64Args) (*attachmentGetBase64Results, error) { + att, err := lookupAttachment(ctx, h.svc, args) + if err != nil { + return nil, err + } + + r := &attachmentGetBase64Results{} + + var content []byte + var fh io.ReadSeekCloser + fh, err = h.svc.OpenOriginal(att) + if err != nil { + return nil, err + } + defer fh.Close() + + content, err = io.ReadAll(fh) + if err != nil { + return nil, err + } + + r.Content = string(base64.StdEncoding.EncodeToString(content)) + r.Name = att.Name + r.Extension = att.Meta.Original.Extension + r.MimeType = att.Meta.Original.Mimetype + + return r, nil +} + func (h attachmentHandler) create(ctx context.Context, args *attachmentCreateArgs) (*attachmentCreateResults, error) { var ( err error diff --git a/server/compose/automation/attachment_handler.yaml b/server/compose/automation/attachment_handler.yaml index 76c6423e69..72eb6f2148 100644 --- a/server/compose/automation/attachment_handler.yaml +++ b/server/compose/automation/attachment_handler.yaml @@ -89,3 +89,19 @@ functions: types: [ { wf: ID }, { wf: Attachment } ] results: content: { wf: Reader } + + getBase64: + meta: + short: Attachment lookup base64 + labels: + <<: *labels + base64-attachment: "step" + params: + attachment: + required: true + types: [ { wf: ID }, { wf: Attachment } ] + results: + content: { wf: String } + name: { wf: String } + extension: { wf: String } + mimeType: { wf: String }