From 08684eac337610c3fc044ea379399b7df3671df6 Mon Sep 17 00:00:00 2001 From: jadrol Date: Wed, 12 Aug 2026 09:25:28 +0200 Subject: [PATCH 1/2] add treasury run command --- .gitignore | 1 + README.md | 43 ++++- backend/s3/s3.go | 17 +- backend/ssm/aws.go | 1 + backend/ssm/getparameters_test.go | 98 +++++++++++ backend/ssm/ssm.go | 60 ++++++- client/envfile.go | 233 ++++++++++++++++++++++++++ client/envfile_test.go | 269 ++++++++++++++++++++++++++++++ client/read.go | 18 ++ cmd/run.go | 184 ++++++++++++++++++++ test/backend/test.go | 15 ++ test/bats/tests.bats | 61 +++++++ test/resources/bats.env.treasury | 16 ++ test/ssm/test.go | 28 ++++ types/types.go | 4 +- version/version.go | 2 +- 16 files changed, 1042 insertions(+), 8 deletions(-) create mode 100644 backend/ssm/getparameters_test.go create mode 100644 client/envfile.go create mode 100644 client/envfile_test.go create mode 100644 cmd/run.go create mode 100644 test/resources/bats.env.treasury diff --git a/.gitignore b/.gitignore index 935e75f..0618790 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ treasury +.env.treasury pkg test/output vendor/ diff --git a/README.md b/README.md index fb52486..a526102 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ Treasury is a very simple tool for managing secrets. It uses Amazon S3 or SSM ([ - [Delete secret](#delete-secret) - [Import secrets](#import-secrets) - [Export secrets](#export-secrets) - - [Teamplate usage](#teamplate-usage) + - [Run a command with secrets](#run-a-command-with-secrets) + - [Template usage](#template-usage) - [Template usage with string append to secret value](#template-usage-with-string-append-to-secret-value) - [Template usage with variables interpolation](#template-usage-with-variables-interpolation) - [read](#read) @@ -204,7 +205,45 @@ To export them into shell environment variables: eval $(treasury export development/webapp/) ``` -### Teamplate usage +### Run a command with secrets + +Runs a command with secrets exported as environment variables, without ever writing them to disk. + +The secrets are described in an environment file, `.env.treasury` by default. A complete example: + +```bash +# Comments and empty lines are ignored. + +# Exports all secrets from the path, each one named after the last part of the +# key, so development/mobile-app-gateway/API_TOKEN becomes API_TOKEN +{{ export "development/mobile-app-gateway/" }} + +# Exports a single secret under a name of your choice +USER_PROFILES_API_PASSWORD={{ read "development/user-profiles/API_PASSWORD_MOBILE_GATEWAY" }} +AUTH_API_PASSWORD={{ read "development/auth/INTERNAL_API_BASIC_AUTH_PASSWORD" }} +AUTH_ACCESS_TOKEN_SECRET={{ read "development/auth/ACCESS_TOKEN_SECRET" }} + +# Plain values are taken as they are, no secret store involved +RAILS_ENV=development +API_TOKEN=test +``` + +Every line is a Go template, and `read` and `export` are the directives you know from the [template command](#template-usage). `read` inserts the value of a single secret, `export` turns a whole path into entries, one per secret. Entries are applied in the order of appearance, so a later entry overrides an earlier one with the same name. + +```bash +> treasury run bundle exec rake db:migrate +> treasury run --env-file .env.staging -- rails server +``` + +The AWS profile is taken from the `AWS_PROFILE` environment variable, use `--profile` to pick another one: + +```bash +> treasury run --profile development bundle exec rails console +``` + +Treasury steps out of the way of the command it runs. Signals it receives, a `SIGTERM` from a supervisor for example, are passed to the command, and its exit code becomes the exit code of treasury: `127` when the command does not exist, `126` when it cannot be executed and `128 + signal number` when it is killed. + +### Template usage Render the template on disk at /tmp/template.tpl to /tmp/result: diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 8cc4c7a..9b5ac64 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -71,8 +71,23 @@ func (c *Client) GetObject(object *types.GetObjectInput) (*types.GetObjectOutput return &types.GetObjectOutput{Value: buf.String()}, nil } -// GetObjects returns key value map for given pattern +// GetObjects returns key value map for the listed keys, or for the given +// pattern when no keys are given func (c *Client) GetObjects(object *types.GetObjectsInput) (*types.GetObjectsOuput, error) { + // S3 has no way to read several objects at once, so the keys are fetched + // one by one, which is still fewer objects than a whole prefix + if len(object.Keys) > 0 { + keyValuePairs := make(map[string]string, len(object.Keys)) + for _, key := range object.Keys { + found, err := c.GetObject(&types.GetObjectInput{Key: key}) + if err != nil { + return nil, err + } + keyValuePairs[key] = found.Value + } + return &types.GetObjectsOuput{Secrets: keyValuePairs}, nil + } + params := &s3.ListObjectsInput{ Bucket: aws.String(c.bucket), Prefix: aws.String(object.Prefix), diff --git a/backend/ssm/aws.go b/backend/ssm/aws.go index 2def3cb..e348cec 100644 --- a/backend/ssm/aws.go +++ b/backend/ssm/aws.go @@ -10,6 +10,7 @@ import ( type ClientInterface interface { GetParameter(context.Context, *ssm.GetParameterInput, ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) PutParameter(context.Context, *ssm.PutParameterInput, ...func(*ssm.Options)) (*ssm.PutParameterOutput, error) + GetParameters(context.Context, *ssm.GetParametersInput, ...func(*ssm.Options)) (*ssm.GetParametersOutput, error) GetParametersByPath(context.Context, *ssm.GetParametersByPathInput, ...func(*ssm.Options)) (*ssm.GetParametersByPathOutput, error) DeleteParameter(context.Context, *ssm.DeleteParameterInput, ...func(*ssm.Options)) (*ssm.DeleteParameterOutput, error) } diff --git a/backend/ssm/getparameters_test.go b/backend/ssm/getparameters_test.go new file mode 100644 index 0000000..4dad415 --- /dev/null +++ b/backend/ssm/getparameters_test.go @@ -0,0 +1,98 @@ +package ssm + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + + "github.com/AirHelp/treasury/types" + "github.com/aws/aws-sdk-go-v2/service/ssm" + ssmtypes "github.com/aws/aws-sdk-go-v2/service/ssm/types" +) + +// batchingSSMClient serves any name it is asked for and remembers how the names +// were split between calls +type batchingSSMClient struct { + ClientInterface + batches [][]string +} + +func (b *batchingSSMClient) GetParameters(_ context.Context, input *ssm.GetParametersInput, _ ...func(*ssm.Options)) (*ssm.GetParametersOutput, error) { + if len(input.Names) > maxKeysPerCall { + return nil, fmt.Errorf("asked for %d names on one call, SSM allows %d", len(input.Names), maxKeysPerCall) + } + b.batches = append(b.batches, input.Names) + + parameters := make([]ssmtypes.Parameter, 0, len(input.Names)) + for _, name := range input.Names { + value := "value of " + name + parameters = append(parameters, ssmtypes.Parameter{Name: &name, Value: &value}) + } + return &ssm.GetParametersOutput{Parameters: parameters}, nil +} + +func TestClient_GetObjectsByKeysBatches(t *testing.T) { + // 23 keys have to be split into 10 + 10 + 3 + keys := make([]string, 0, 23) + for i := range 23 { + keys = append(keys, fmt.Sprintf("test/webapp/KEY_%02d", i)) + } + + svc := &batchingSSMClient{} + got, err := (&Client{svc: svc}).GetObjects(&types.GetObjectsInput{Keys: keys}) + if err != nil { + t.Fatal(err) + } + + wantBatches := []int{10, 10, 3} + if len(svc.batches) != len(wantBatches) { + t.Fatalf("GetObjects() made %d calls, want %d", len(svc.batches), len(wantBatches)) + } + for i, want := range wantBatches { + if len(svc.batches[i]) != want { + t.Errorf("call %d carried %d names, want %d", i+1, len(svc.batches[i]), want) + } + } + + if len(got.Secrets) != len(keys) { + t.Fatalf("GetObjects() returned %d secrets, want %d", len(got.Secrets), len(keys)) + } + // the leading slash SSM needs is not a part of a treasury key + for _, key := range keys { + value, found := got.Secrets[key] + if !found { + t.Fatalf("GetObjects() did not return %q", key) + } + if want := "value of /" + key; value != want { + t.Errorf("GetObjects()[%q] = %q, want %q", key, value, want) + } + } +} + +// missingSSMClient reports every name as invalid, the way SSM does for a +// parameter which does not exist +type missingSSMClient struct { + ClientInterface +} + +func (m *missingSSMClient) GetParameters(_ context.Context, input *ssm.GetParametersInput, _ ...func(*ssm.Options)) (*ssm.GetParametersOutput, error) { + invalid := append([]string{}, input.Names...) + sort.Strings(invalid) + return &ssm.GetParametersOutput{InvalidParameters: invalid}, nil +} + +func TestClient_GetObjectsByKeysReportsMissing(t *testing.T) { + keys := []string{"test/webapp/GONE", "test/webapp/ALSO_GONE"} + + _, err := (&Client{svc: &missingSSMClient{}}).GetObjects(&types.GetObjectsInput{Keys: keys}) + if err == nil { + t.Fatal("GetObjects() with unknown keys returned no error") + } + for _, key := range keys { + if !strings.Contains(err.Error(), key) { + t.Errorf("GetObjects() error = %q, want it to name %q", err, key) + } + } +} diff --git a/backend/ssm/ssm.go b/backend/ssm/ssm.go index a0164a7..3dc34d6 100644 --- a/backend/ssm/ssm.go +++ b/backend/ssm/ssm.go @@ -3,6 +3,9 @@ package ssm import ( "context" "errors" + "fmt" + "sort" + "strings" "github.com/AirHelp/treasury/types" "github.com/aws/aws-sdk-go-v2/aws" @@ -10,7 +13,13 @@ import ( ssmtypes "github.com/aws/aws-sdk-go-v2/service/ssm/types" ) -const defaultParameterType = "SecureString" +const ( + defaultParameterType = "SecureString" + + // maxKeysPerCall is the hard limit SSM puts on both GetParameters and + // GetParametersByPath, so it is the batch size of every read we do + maxKeysPerCall = 10 +) // PutObject writes a given secret value on SSM // it uses PutParameter API call @@ -54,8 +63,13 @@ func (c *Client) GetObject(object *types.GetObjectInput) (*types.GetObjectOutput return &types.GetObjectOutput{Value: *resp.Parameter.Value}, nil } -// GetObjects returns key value map for given pattern/prefix +// GetObjects returns key value map for the listed keys, or for the given +// pattern/prefix when no keys are given func (c *Client) GetObjects(object *types.GetObjectsInput) (*types.GetObjectsOuput, error) { + if len(object.Keys) > 0 { + return c.getParameters(object.Keys) + } + var nextToken *string var parameters []ssmtypes.Parameter for { @@ -64,7 +78,7 @@ func (c *Client) GetObjects(object *types.GetObjectsInput) (*types.GetObjectsOup Path: aws.String("/" + object.Prefix), // Retrieve all parameters in a hierarchy with their value decrypted. WithDecryption: aws.Bool(true), - MaxResults: aws.Int32(10), + MaxResults: aws.Int32(maxKeysPerCall), NextToken: nextToken, } @@ -90,6 +104,46 @@ func (c *Client) GetObjects(object *types.GetObjectsInput) (*types.GetObjectsOup return &types.GetObjectsOuput{Secrets: keyValuePairs}, nil } +// getParameters fetches the given keys only, in batches of maxKeysPerCall, +// which is what makes reading a handful of secrets from a big path cheap +// https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_GetParameters.html +func (c *Client) getParameters(keys []string) (*types.GetObjectsOuput, error) { + keyValuePairs := make(map[string]string, len(keys)) + for start := 0; start < len(keys); start += maxKeysPerCall { + batch := keys[start:min(start+maxKeysPerCall, len(keys))] + names := make([]string, 0, len(batch)) + for _, key := range batch { + // we decided to use path based keys without `/` at the beginning + // so we need to add it here + names = append(names, "/"+key) + } + + resp, err := c.svc.GetParameters(context.Background(), &ssm.GetParametersInput{ + Names: names, + // Retrieve all parameters in a hierarchy with their value decrypted. + WithDecryption: aws.Bool(true), + }) + if err != nil { + return nil, err + } + // a name which does not exist is not an error for SSM, it comes back + // on a list of its own + if len(resp.InvalidParameters) > 0 { + missing := make([]string, 0, len(resp.InvalidParameters)) + for _, name := range resp.InvalidParameters { + missing = append(missing, unSlash(name)) + } + sort.Strings(missing) + return nil, fmt.Errorf("secrets not found: %s", strings.Join(missing, ", ")) + } + + for _, parameter := range resp.Parameters { + keyValuePairs[unSlash(*parameter.Name)] = *parameter.Value + } + } + return &types.GetObjectsOuput{Secrets: keyValuePairs}, nil +} + // unSlash removes 1st char from a string // GetParametersByPath from SSM returns key path with "/" at the beginning // but we don't need it :) diff --git a/client/envfile.go b/client/envfile.go new file mode 100644 index 0000000..45ccacc --- /dev/null +++ b/client/envfile.go @@ -0,0 +1,233 @@ +package client + +import ( + "fmt" + "maps" + "os" + "path" + "regexp" + "slices" + "strings" + "text/template" +) + +var variableName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +// EnvFile resolves an environment file into a list of KEY=VALUE entries ready to +// be handed over to a subprocess. Secrets are kept in memory only. +// +// Every line is a Go template, so the directives are plain template functions, +// the same ones the template command uses: +// +// {{ export "development/webapp/" }} all secrets from the path +// API_PASSWORD={{ read "development/auth/PASS" }} a single secret +// API_TOKEN=test a plain value +// +// Entries come back in the order they appear in the file, duplicates included. +// A later entry of the same name overrides an earlier one, which is what +// exec.Cmd does with a duplicate key in Env. +// +// The file is rendered twice. The first pass fetches nothing, it only notes what +// the file asks for, so the secrets can be read in as few calls as possible, and +// so a malformed file fails before a single secret leaves the store. The second +// pass renders the values fetched in between. +func (c *Client) EnvFile(filePath string) ([]string, error) { + content, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + lines := strings.Split(string(content), "\n") + + // the first pass renders no values, so the entries it collects are + // meaningless and go away with the envFile they were collected into + var keys, prefixes []string + if err := (&envFile{}).render(lines, note(&keys), note(&prefixes)); err != nil { + return nil, err + } + + env, err := c.fetch(keys, prefixes) + if err != nil { + return nil, err + } + if err := env.render(lines, env.read, env.export); err != nil { + return nil, err + } + return env.entries, nil +} + +// note is a first pass directive: it records what the file asks for, without +// duplicates, and renders nothing +func note(names *[]string) func(string) (string, error) { + return func(name string) (string, error) { + if !slices.Contains(*names, name) { + *names = append(*names, name) + } + return "", nil + } +} + +// envFile holds every secret an environment file asked for, whether it came in +// by name or as a part of a whole path, and the entries the file resolves to +type envFile struct { + client *Client + secrets map[string]string + // paths already read, so none of them is read twice + paths map[string]bool + // KEY=VALUE entries, in the order the file lists them + entries []string +} + +// fetch reads everything the file asked for: a call per exported path, and the +// keys left over batched together, so reading a handful of secrets never costs +// more than a handful of names on one call +func (c *Client) fetch(keys, prefixes []string) (*envFile, error) { + env := &envFile{ + client: c, + secrets: make(map[string]string), + paths: make(map[string]bool), + } + for _, prefix := range prefixes { + if err := env.walk(prefix); err != nil { + return nil, err + } + } + + // a key an exported path already covers costs nothing extra + var missing []string + for _, key := range keys { + if _, found := env.secrets[key]; !found { + missing = append(missing, key) + } + } + values, err := c.ReadKeys(missing) + if err != nil { + return nil, err + } + maps.Copy(env.secrets, values) + return env, nil +} + +// render resolves every line of an environment file with the given directives, +// collecting the entries the file declares +func (e *envFile) render(lines []string, read, export func(string) (string, error)) error { + directives := template.FuncMap{"read": read, "export": export} + + for index, raw := range lines { + lineNo := index + 1 + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + name, value, isEntry := strings.Cut(line, "=") + if !isEntry { + // an export directive is the only thing allowed to stand on its own, + // and it renders nothing + rendered, err := renderLine(directives, trimComment(line)) + if err != nil { + return fmt.Errorf("line %d: %w", lineNo, err) + } + if rendered != "" { + return fmt.Errorf("line %d: expected KEY=VALUE or an export directive, got %q", lineNo, line) + } + continue + } + + name = strings.TrimSpace(name) + if !variableName.MatchString(name) { + return fmt.Errorf("line %d: %q is not a valid environment variable name", lineNo, name) + } + rendered, err := renderLine(directives, trimComment(strings.TrimSpace(value))) + if err != nil { + return fmt.Errorf("line %d: %w", lineNo, err) + } + e.add(name, rendered) + } + return nil +} + +// renderLine resolves the directives of a single line. Rendering line by line +// keeps a secret value opaque, whatever it contains, newlines included. +func renderLine(directives template.FuncMap, line string) (string, error) { + tmpl, err := template.New("env").Funcs(directives).Parse(line) + if err != nil { + return "", err + } + var rendered strings.Builder + if err := tmpl.Execute(&rendered, nil); err != nil { + return "", err + } + return rendered.String(), nil +} + +func (e *envFile) add(name, value string) { + e.entries = append(e.entries, name+"="+value) +} + +// read resolves a single secret. Everything is fetched by now, unless the file +// took a different branch once the values were known, in which case the secret +// is fetched here. +func (e *envFile) read(key string) (string, error) { + if value, found := e.secrets[key]; found { + return value, nil + } + values, err := e.client.ReadKeys([]string{key}) + if err != nil { + return "", err + } + value, found := values[key] + if !found { + return "", fmt.Errorf("secret %q not found", key) + } + e.secrets[key] = value + return value, nil +} + +// export declares an entry for every secret of a path, each named after the last +// part of its key. It renders nothing itself. +func (e *envFile) export(prefix string) (string, error) { + if err := e.walk(prefix); err != nil { + return "", err + } + prefix = withSlash(prefix) + for _, key := range slices.Sorted(maps.Keys(e.secrets)) { + if strings.HasPrefix(key, prefix) { + e.add(path.Base(key), e.secrets[key]) + } + } + return "", nil +} + +// walk reads every secret of a path, at most once per path +func (e *envFile) walk(prefix string) error { + prefix = withSlash(prefix) + if e.paths[prefix] { + return nil + } + secrets, err := e.client.ReadGroup(prefix) + if err != nil { + return err + } + for _, secret := range secrets { + e.secrets[secret.Key] = secret.Value + } + e.paths[prefix] = true + return nil +} + +func withSlash(prefix string) string { + return strings.TrimSuffix(prefix, "/") + "/" +} + +// trimComment removes a trailing comment from a value. A comment starts with a +// '#' which opens the value or follows a space or a tab, so a '#' in the middle +// of a value is kept while a value of its own beginning with one is a comment. +// Quotes are not special, they are a part of the value like any other character. +func trimComment(value string) string { + for i := 0; i < len(value); i++ { + if value[i] == '#' && (i == 0 || value[i-1] == ' ' || value[i-1] == '\t') { + return strings.TrimSpace(value[:i]) + } + } + return value +} diff --git a/client/envfile_test.go b/client/envfile_test.go new file mode 100644 index 0000000..a50b32a --- /dev/null +++ b/client/envfile_test.go @@ -0,0 +1,269 @@ +package client_test + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "reflect" + "slices" + "testing" + + "github.com/AirHelp/treasury/backend" + "github.com/AirHelp/treasury/client" + test "github.com/AirHelp/treasury/test/backend" + "github.com/AirHelp/treasury/types" +) + +// webappEntries is what {{ export "test/webapp/" }} resolves to, the secrets of +// the path in the order of their keys +var webappEntries = []string{ + test.ShortKey1 + "=" + test.KeyValueMap[test.Key1], + test.ShortKey4 + "=" + test.KeyValueMap[test.Key4], + test.ShortKey2 + "=" + test.KeyValueMap[test.Key2], +} + +func TestEnvFile(t *testing.T) { + tests := []struct { + name string + envFile string + want []string + wantErr bool + }{ + { + name: "plain values", + envFile: `RAILS_ENV=development +API_TOKEN=test +EMPTY= +WITH_EQUAL_SIGNS=a=b=c`, + want: []string{ + "RAILS_ENV=development", + "API_TOKEN=test", + "EMPTY=", + "WITH_EQUAL_SIGNS=a=b=c", + }, + }, + { + name: "quotes are a part of the value, they are not special", + envFile: `DOUBLE="quoted value" +SINGLE='quoted value'`, + want: []string{ + `DOUBLE="quoted value"`, + `SINGLE='quoted value'`, + }, + }, + { + name: "comments", + envFile: `# a comment on its own line + +API_TOKEN=test # only for now + # an indented comment +HASH_INSIDE=pass#word +HASH_FIRST=#word`, + want: []string{ + "API_TOKEN=test", + "HASH_INSIDE=pass#word", + "HASH_FIRST=", + }, + }, + { + name: "single secret", + envFile: `COCKPIT_API_PASSWORD={{ read "` + test.Key1 + `" }}`, + want: []string{"COCKPIT_API_PASSWORD=" + test.KeyValueMap[test.Key1]}, + }, + { + name: "single secret with a comment", + envFile: `COCKPIT_API_PASSWORD={{ read "` + test.Key1 + `" }} # the one we use locally`, + want: []string{"COCKPIT_API_PASSWORD=" + test.KeyValueMap[test.Key1]}, + }, + { + name: "whole path, variables named after the last part of the key", + envFile: `{{ export "test/webapp/" }}`, + want: webappEntries, + }, + { + name: "whole path with a comment", + envFile: `{{export "test/webapp/"}} # everything the app needs`, + want: webappEntries, + }, + { + // the entries are handed to exec.Cmd, which resolves a duplicate key + // to its last value, so an override is a repetition + name: "an entry of the same name is repeated, the later one overrides", + envFile: `{{ export "test/webapp/" }} +` + test.ShortKey1 + `=overridden +API_TOKEN=first +API_TOKEN=second`, + want: slices.Concat(webappEntries, []string{ + test.ShortKey1 + "=overridden", + "API_TOKEN=first", + "API_TOKEN=second", + }), + }, + { + name: "unknown secret", + envFile: `PASSWORD={{ read "test/webapp/no_such_key" }}`, + wantErr: true, + }, + { + name: "invalid secret path", + envFile: `PASSWORD={{ read "no_such_path" }}`, + wantErr: true, + }, + { + name: "invalid variable name", + envFile: `2FAST2FURIOUS=nope`, + wantErr: true, + }, + { + name: "line which is neither an entry nor a directive", + envFile: `just a line`, + wantErr: true, + }, + } + + treasury := newTreasury(t, &test.MockBackendClient{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := treasury.EnvFile(envFilePath(t, tt.envFile)) + if (err != nil) != tt.wantErr { + t.Fatalf("Client.EnvFile() error = %v, wantErr %v", err, tt.wantErr) + } + if !tt.wantErr && !reflect.DeepEqual(got, tt.want) { + t.Errorf("Client.EnvFile() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestEnvFileMissingFile(t *testing.T) { + treasury := newTreasury(t, &test.MockBackendClient{}) + + // the run command relies on this error to tell the user about --env-file + if _, err := treasury.EnvFile(filepath.Join(t.TempDir(), ".env.treasury")); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("Client.EnvFile() error = %v, want a %v one", err, fs.ErrNotExist) + } +} + +// TestEnvFileFetches guards what makes the run command usable: a path is read +// once no matter how many entries refer to it, and every remaining key is read +// by name on one batched call, so nothing outside the listed keys is decrypted. +func TestEnvFileFetches(t *testing.T) { + tests := []struct { + name string + envFile string + want []types.GetObjectsInput + }{ + { + name: "keys of any number of paths are read by name on one call", + envFile: `FIRST={{ read "` + test.Key1 + `" }} +SECOND={{ read "` + test.Key2 + `" }} +THIRD={{ read "` + test.Key3 + `" }} +RAILS_ENV=development`, + want: []types.GetObjectsInput{{Keys: []string{test.Key1, test.Key2, test.Key3}}}, + }, + { + name: "the same key asked for twice is read once", + envFile: `FIRST={{ read "` + test.Key1 + `" }} +AGAIN={{ read "` + test.Key1 + `" }}`, + want: []types.GetObjectsInput{{Keys: []string{test.Key1}}}, + }, + { + name: "a whole path takes a call of its own", + envFile: `{{ export "test/webapp/" }}`, + want: []types.GetObjectsInput{{Prefix: "test/webapp/"}}, + }, + { + name: "a key an exported path already covers is free", + envFile: `{{ export "test/webapp/" }} +RENAMED={{ read "` + test.Key1 + `" }}`, + want: []types.GetObjectsInput{{Prefix: "test/webapp/"}}, + }, + { + name: "paths are read once each, the keys they cover cost nothing", + envFile: `{{ export "test/webapp/" }} +FIRST={{ read "` + test.Key1 + `" }} +SECOND={{ read "` + test.Key2 + `" }} +THIRD={{ read "` + test.Key3 + `" }} +{{ export "test/cockpit/" }} +{{ export "test/webapp/" }}`, + want: []types.GetObjectsInput{{Prefix: "test/webapp/"}, {Prefix: "test/cockpit/"}}, + }, + { + name: "plain values need no call at all", + envFile: `RAILS_ENV=development`, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + backend := &countingBackend{} + if _, err := newTreasury(t, backend).EnvFile(envFilePath(t, tt.envFile)); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(backend.calls, tt.want) { + t.Errorf("Client.EnvFile() fetched %+v, want %+v", backend.calls, tt.want) + } + }) + } +} + +// TestEnvFileKeepsSecretsVerbatim covers what the line by line rendering is for: +// a certificate keeps its newlines, its quotes and the hash inside it, none of +// which the plain value rules are allowed to touch +func TestEnvFileKeepsSecretsVerbatim(t *testing.T) { + const key = "test/webapp/certificate" + awkward := `"-----BEGIN KEY-----` + "\nline # two\n" + `-----END KEY-----"` + + treasury := newTreasury(t, &singleSecretBackend{key: key, value: awkward}) + + got, err := treasury.EnvFile(envFilePath(t, `CERTIFICATE={{ read "`+key+`" }}`)) + if err != nil { + t.Fatal(err) + } + want := []string{"CERTIFICATE=" + awkward} + if !reflect.DeepEqual(got, want) { + t.Errorf("Client.EnvFile() = %q, want %q", got, want) + } +} + +func newTreasury(t *testing.T, backend backend.API) *client.Client { + t.Helper() + treasury, err := client.New(&client.Options{Backend: backend}) + if err != nil { + t.Fatal(err) + } + return treasury +} + +func envFilePath(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), ".env.treasury") + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + return path +} + +// singleSecretBackend serves one secret, whatever the path asked for +type singleSecretBackend struct { + backend.API + key, value string +} + +func (s *singleSecretBackend) GetObjects(*types.GetObjectsInput) (*types.GetObjectsOuput, error) { + return &types.GetObjectsOuput{Secrets: map[string]string{s.key: s.value}}, nil +} + +// countingBackend records every fetch of secrets from the backend +type countingBackend struct { + backend.API + calls []types.GetObjectsInput +} + +func (c *countingBackend) GetObjects(input *types.GetObjectsInput) (*types.GetObjectsOuput, error) { + c.calls = append(c.calls, *input) + return (&test.MockBackendClient{}).GetObjects(input) +} diff --git a/client/read.go b/client/read.go index 8c65107..b3fcb80 100644 --- a/client/read.go +++ b/client/read.go @@ -41,6 +41,24 @@ func (c *Client) ReadFromEnv(env, key string) (string, error) { return c.ReadValue(fmt.Sprintf("%s/%s", env, key)) } +// ReadKeys returns the values of the given keys only, fetched in as few calls +// as the backend allows. Nothing outside the listed keys is read or decrypted. +func (c *Client) ReadKeys(keys []string) (map[string]string, error) { + if len(keys) == 0 { + return nil, nil + } + for _, key := range keys { + if err := utils.ValidateInputKey(key); err != nil { + return nil, err + } + } + resp, err := c.Backend.GetObjects(&types.GetObjectsInput{Keys: keys}) + if err != nil { + return nil, err + } + return resp.Secrets, nil +} + // ReadGroup returns list of secrets for given key prefix func (c *Client) ReadGroup(keyPrefix string) ([]*Secret, error) { if err := utils.ValidateInputKeyPattern(keyPrefix); err != nil { diff --git a/cmd/run.go b/cmd/run.go new file mode 100644 index 0000000..c1e22a2 --- /dev/null +++ b/cmd/run.go @@ -0,0 +1,184 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "os/signal" + "syscall" + + "github.com/AirHelp/treasury/client" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/spf13/cobra" +) + +const ( + runCommandEnvFileArgument = "env-file" + runCommandProfileArgument = "profile" + defaultEnvFile = ".env.treasury" + + // exit codes follow the shell convention + commandNotFoundExitCode = 127 + commandNotExecutableExitCode = 126 + signalExitCodeBase = 128 +) + +// signals relayed to the command being run, the ones a process is expected to +// act on. The rest is left to the default behaviour. +var forwardedSignals = []os.Signal{ + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT, + syscall.SIGHUP, + syscall.SIGUSR1, + syscall.SIGUSR2, +} + +// runCmd represents the run command +var runCmd = &cobra.Command{ + Use: "run [flags] COMMAND [ARGS...]", + Short: "Runs a command with secrets loaded into environment variables", + Long: `Run loads secrets described in an environment file and executes the given +command with them exported as environment variables. Secrets are kept in memory +only, they are never written to disk. + + treasury run bundle exec rake db:migrate + treasury run --env-file .env.staging -- rails server + +The environment file (.env.treasury by default) accepts: + + {{ export "development/webapp/" }} all secrets from the path, + named after the last path part + AUTH_API_PASSWORD={{ read "development/auth/PASSWORD" }} a single secret + API_TOKEN=test a plain value + +The AWS profile is taken from AWS_PROFILE, use --profile to pick another one.`, + RunE: run, +} + +func init() { + RootCmd.AddCommand(runCmd) + // everything after the command name belongs to the command being run + runCmd.Flags().SetInterspersed(false) + runCmd.Flags().String(runCommandEnvFileArgument, defaultEnvFile, "path to the environment file with secrets") + runCmd.Flags().String(runCommandProfileArgument, "", "AWS profile to use, defaults to AWS_PROFILE") +} + +func run(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("missing command to run") + } + // the arguments are fine, whatever fails next is not a usage problem + cmd.SilenceUsage = true + + envFile, err := cmd.Flags().GetString(runCommandEnvFileArgument) + if err != nil { + return err + } + profile, err := cmd.Flags().GetString(runCommandProfileArgument) + if err != nil { + return err + } + + treasury, err := newClientWithProfile(profile) + if err != nil { + return err + } + + environment, err := treasury.EnvFile(envFile) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("environment file %s not found, create it or point to another one with --%s", envFile, runCommandEnvFileArgument) + } + return err + } + + return execute(cmd.Context(), args, environment) +} + +// execute runs the command with the given environment and leaves treasury with +// the same exit code the command ended with +func execute(ctx context.Context, args, environment []string) error { + command := exec.CommandContext(ctx, args[0], args[1:]...) // #nosec G204 + command.Env = append(os.Environ(), environment...) + command.Stdin, command.Stdout, command.Stderr = os.Stdin, os.Stdout, os.Stderr + + if err := command.Start(); err != nil { + switch { + case errors.Is(err, exec.ErrNotFound), errors.Is(err, fs.ErrNotExist): + fmt.Fprintf(os.Stderr, "treasury: %s: command not found\n", args[0]) + os.Exit(commandNotFoundExitCode) + case errors.Is(err, fs.ErrPermission): + fmt.Fprintf(os.Stderr, "treasury: %s: cannot execute\n", args[0]) + os.Exit(commandNotExecutableExitCode) + } + return err + } + + stopForwarding := forwardSignals(command.Process) + err := command.Wait() + stopForwarding() + + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + os.Exit(exitCode(exitErr.ProcessState)) + } + return err +} + +// forwardSignals relays the signals treasury receives to the command it runs, +// so that a SIGTERM from a supervisor reaches the process that does the work. +// The command shares the process group with treasury, so signals coming from +// the terminal reach it on their own, a Ctrl-C simply arrives twice. +// The returned function stops the forwarding. +func forwardSignals(process *os.Process) func() { + signals := make(chan os.Signal, 1) + signal.Notify(signals, forwardedSignals...) + + done := make(chan struct{}) + go func() { + for { + select { + case receivedSignal := <-signals: + // the command may be gone already, nothing to do about it + _ = process.Signal(receivedSignal) + case <-done: + return + } + } + }() + + return func() { + signal.Stop(signals) + close(done) + } +} + +// exitCode of a finished command, 128 + signal number when it was killed +func exitCode(state *os.ProcessState) int { + if status, ok := state.Sys().(syscall.WaitStatus); ok && status.Signaled() { + return signalExitCodeBase + int(status.Signal()) + } + return state.ExitCode() +} + +func newClientWithProfile(profile string) (*client.Client, error) { + options := &client.Options{ + Region: s3Region, + S3BucketName: treasuryS3, + } + if profile != "" { + awsConfig, err := config.LoadDefaultConfig(context.Background(), + config.WithSharedConfigProfile(profile), + config.WithRegion(s3Region), + ) + if err != nil { + return nil, err + } + options.AWSConfig = awsConfig + } + return client.New(options) +} diff --git a/test/backend/test.go b/test/backend/test.go index b92e685..3dd931e 100644 --- a/test/backend/test.go +++ b/test/backend/test.go @@ -76,6 +76,21 @@ func (m *MockBackendClient) GetObject(input *types.GetObjectInput) (*types.GetOb func (m *MockBackendClient) GetObjects(input *types.GetObjectsInput) (*types.GetObjectsOuput, error) { response := make(map[string]string) + if len(input.Keys) > 0 { + var missing []string + for _, key := range input.Keys { + value, ok := KeyValueMap[key] + if !ok { + missing = append(missing, key) + continue + } + response[key] = value + } + if len(missing) > 0 { + return nil, fmt.Errorf("secrets not found: %s", strings.Join(missing, ", ")) + } + return &types.GetObjectsOuput{Secrets: response}, nil + } for key := range KeyValueMap { if strings.Contains(key, input.Prefix) { response[key] = KeyValueMap[key] diff --git a/test/bats/tests.bats b/test/bats/tests.bats index c593137..c7f9f2f 100644 --- a/test/bats/tests.bats +++ b/test/bats/tests.bats @@ -4,6 +4,7 @@ treasury=$PWD/treasury randomKey=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 16) valid_aws_region=eu-west-1 invalid_aws_region=us-west-1 +runEnvFile=$PWD/test/resources/bats.env.treasury @test "Check that the treasury binary is available" { command $treasury @@ -208,6 +209,66 @@ invalid_aws_region=us-west-1 [[ ${lines[0]} =~ "Error" ]] } +@test "run exports a plain value" { + run $treasury run --env-file $runEnvFile printenv PLAIN_VALUE + [ $status -eq 0 ] + [[ ${lines[0]} == "plain" ]] +} + +@test "run exports a single secret" { + run $treasury run --env-file $runEnvFile printenv KEY_FROM_READ + [ $status -eq 0 ] + [[ ${lines[0]} == "secret1" ]] +} + +@test "run exports all secrets from a path" { + run $treasury run --env-file $runEnvFile printenv key1 key2 + [ $status -eq 0 ] + [[ ${lines[0]} == "secret1" ]] + [[ ${lines[1]} == "secret2" ]] +} + +@test "run drops an inline comment but keeps a hash inside a value" { + run $treasury run --env-file $runEnvFile printenv COMMENTED_VALUE HASH_VALUE + [ $status -eq 0 ] + [[ ${lines[0]} == "plain" ]] + [[ ${lines[1]} == "pass#word" ]] +} + +@test "run lets a later entry override an earlier one" { + run $treasury run --env-file $runEnvFile printenv OVERRIDDEN + [ $status -eq 0 ] + [[ ${lines[0]} == "second" ]] +} + +@test "run exits with the code of the command" { + run $treasury run --env-file $runEnvFile sh -c "exit 3" + [ $status -eq 3 ] +} + +@test "run exits with 128 + signal when the command is killed" { + run $treasury run --env-file $runEnvFile sh -c 'kill -TERM $$' + [ $status -eq 143 ] +} + +@test "run exits with 127 when the command does not exist" { + run $treasury run --env-file $runEnvFile no-such-command + [ $status -eq 127 ] + [[ ${output} =~ "command not found" ]] +} + +@test "run forwards signals to the command" { + run timeout 10 bash -c '"$1" run --env-file "$2" sh -c "trap \"exit 42\" TERM; while true; do sleep 0.1; done" & child=$!; sleep 1; kill -TERM $child; wait $child' bash $treasury $runEnvFile + [ $status -eq 42 ] +} + +@test "run points to --env-file when the environment file is missing" { + run $treasury run --env-file /no/such/.env.treasury printenv + [ $status -ne 0 ] + [[ ${output} =~ "not found" ]] + [[ ${output} =~ "--env-file" ]] +} + @test "check version" { run $treasury version [ $status -eq 0 ] diff --git a/test/resources/bats.env.treasury b/test/resources/bats.env.treasury new file mode 100644 index 0000000..b4dd3ab --- /dev/null +++ b/test/resources/bats.env.treasury @@ -0,0 +1,16 @@ +# all secrets of the path, named after the last part of the key +{{ export "development/treasury/" }} + +# a single secret under a name of our choice +KEY_FROM_READ={{ read "development/treasury/key1" }} + +# a plain value next to the secrets +PLAIN_VALUE=plain + +# a comment at the end of a line, and a hash which belongs to the value +COMMENTED_VALUE=plain # the comment is not a part of the value +HASH_VALUE=pass#word + +# a later entry of the same name overrides an earlier one +OVERRIDDEN=first +OVERRIDDEN=second diff --git a/test/ssm/test.go b/test/ssm/test.go index 169e3e3..7b6a788 100644 --- a/test/ssm/test.go +++ b/test/ssm/test.go @@ -42,6 +42,7 @@ type MockSSMClient struct { type Client interface { PutParameter(ctx context.Context, input *ssm.PutParameterInput, optFns ...func(*ssm.Options)) (*ssm.PutParameterOutput, error) GetParameter(ctx context.Context, input *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) + GetParameters(ctx context.Context, input *ssm.GetParametersInput, optFns ...func(*ssm.Options)) (*ssm.GetParametersOutput, error) GetParametersByPath(ctx context.Context, input *ssm.GetParametersByPathInput, optFns ...func(*ssm.Options)) (*ssm.GetParametersByPathOutput, error) DeleteParameter(ctx context.Context, input *ssm.DeleteParameterInput, optFns ...func(*ssm.Options)) (*ssm.DeleteParameterOutput, error) } @@ -83,6 +84,33 @@ func (m *MockSSMClient) GetParameter(ctx context.Context, input *ssm.GetParamete }, nil } +// https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_GetParameters.html +func (m *MockSSMClient) GetParameters(ctx context.Context, input *ssm.GetParametersInput, optFns ...func(*ssm.Options)) (*ssm.GetParametersOutput, error) { + if !*input.WithDecryption { + return nil, fmt.Errorf("missing decryption field") + } + if len(input.Names) > 10 { + return nil, fmt.Errorf("SSM accepts at most 10 names per call, got %d", len(input.Names)) + } + var parameters []types.Parameter + var invalid []string + for _, name := range input.Names { + value, ok := SSMKeyValueMap[name] + if !ok { + invalid = append(invalid, name) + continue + } + parameters = append(parameters, types.Parameter{ + Name: &name, + Value: &value, + }) + } + return &ssm.GetParametersOutput{ + Parameters: parameters, + InvalidParameters: invalid, + }, nil +} + // https://docs.aws.amazon.com/sdk-for-go/api/service/ssm/#SSM.GetParametersByPath // https://docs.aws.amazon.com/sdk-for-go/api/service/ssm/#GetParametersByPathInput func (m *MockSSMClient) GetParametersByPath(ctx context.Context, input *ssm.GetParametersByPathInput, optFns ...func(*ssm.Options)) (*ssm.GetParametersByPathOutput, error) { diff --git a/types/types.go b/types/types.go index 89c5cd9..2adb052 100644 --- a/types/types.go +++ b/types/types.go @@ -14,9 +14,11 @@ type GetObjectInput struct { Version string } -// GetObjectsInput structure for GetObjects +// GetObjectsInput structure for GetObjects. Set Prefix to get every secret +// under a path, or Keys to get only the listed ones. type GetObjectsInput struct { Prefix string + Keys []string } // GetObjectOuput structure for GetObject diff --git a/version/version.go b/version/version.go index 8fcc8ba..d390a9f 100644 --- a/version/version.go +++ b/version/version.go @@ -6,7 +6,7 @@ import ( ) // treasury version should be changed here -const version = "v0.14.0" +const version = "v0.15.0" // This will be filled in by the compiler. var ( From f10d5b932ddcc0616c353f844df5e5a8ef6e67fa Mon Sep 17 00:00:00 2001 From: jadrol Date: Thu, 13 Aug 2026 10:48:33 +0200 Subject: [PATCH 2/2] add missing body.Close --- backend/s3/s3.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 9b5ac64..33b94d5 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -63,6 +63,8 @@ func (c *Client) GetObject(object *types.GetObjectInput) (*types.GetObjectOutput if err != nil { return nil, err } + defer func() { _ = resp.Body.Close() }() + buf := new(bytes.Buffer) if _, err := buf.ReadFrom(resp.Body); err != nil { return nil, err