Add Treasury run command - do not persist secrets on disk

.gitignore

@@ -1,4 +1,5 @@
 treasury
+.env.treasury
 pkg
 test/output
 vendor/

README.md

@@ -21,7 +21,8 @@ Treasury is a very simple tool for managing secrets. It uses Amazon S3 or SSM ([
     - [Delete secret](#delete-secret)
     - [Import secrets](#import-secrets)
     - [Export secrets](#export-secrets)
-    - [Teamplate usage](#teamplate-usage)
+    - [Run a command with secrets](#run-a-command-with-secrets)
+    - [Template usage](#template-usage)
       - [Template usage with string append to secret value](#template-usage-with-string-append-to-secret-value)
       - [Template usage with variables interpolation](#template-usage-with-variables-interpolation)
       - [read](#read)
@@ -204,7 +205,45 @@ To export them into shell environment variables:
 eval $(treasury export development/webapp/)
 ```
 
-### Teamplate usage
+### Run a command with secrets
+
+Runs a command with secrets exported as environment variables, without ever writing them to disk.
+
+The secrets are described in an environment file, `.env.treasury` by default. A complete example:
+
+```bash
+# Comments and empty lines are ignored.
+
+# Exports all secrets from the path, each one named after the last part of the
+# key, so development/mobile-app-gateway/API_TOKEN becomes API_TOKEN
+{{ export "development/mobile-app-gateway/" }}
+
+# Exports a single secret under a name of your choice
+USER_PROFILES_API_PASSWORD={{ read "development/user-profiles/API_PASSWORD_MOBILE_GATEWAY" }}
+AUTH_API_PASSWORD={{ read "development/auth/INTERNAL_API_BASIC_AUTH_PASSWORD" }}
+AUTH_ACCESS_TOKEN_SECRET={{ read "development/auth/ACCESS_TOKEN_SECRET" }}
+
+# Plain values are taken as they are, no secret store involved
+RAILS_ENV=development
+API_TOKEN=test
+```
+
+Every line is a Go template, and `read` and `export` are the directives you know from the [template command](#template-usage). `read` inserts the value of a single secret, `export` turns a whole path into entries, one per secret. Entries are applied in the order of appearance, so a later entry overrides an earlier one with the same name.
+
+```bash
+> treasury run bundle exec rake db:migrate
+> treasury run --env-file .env.staging -- rails server
+```
+
+The AWS profile is taken from the `AWS_PROFILE` environment variable, use `--profile` to pick another one:
+
+```bash
+> treasury run --profile development bundle exec rails console
+```
+
+Treasury steps out of the way of the command it runs. Signals it receives, a `SIGTERM` from a supervisor for example, are passed to the command, and its exit code becomes the exit code of treasury: `127` when the command does not exist, `126` when it cannot be executed and `128 + signal number` when it is killed.
+
+### Template usage
 
 Render the template on disk at /tmp/template.tpl to /tmp/result:
 

backend/s3/s3.go

@@ -63,6 +63,8 @@ func (c *Client) GetObject(object *types.GetObjectInput) (*types.GetObjectOutput
 	if err != nil {
 		return nil, err
 	}
+	defer func() { _ = resp.Body.Close() }()
+
 	buf := new(bytes.Buffer)
 	if _, err := buf.ReadFrom(resp.Body); err != nil {
 		return nil, err
@@ -71,8 +73,23 @@ func (c *Client) GetObject(object *types.GetObjectInput) (*types.GetObjectOutput
 	return &types.GetObjectOutput{Value: buf.String()}, nil
 }
 
-// GetObjects returns key value map for given pattern
+// GetObjects returns key value map for the listed keys, or for the given
+// pattern when no keys are given
 func (c *Client) GetObjects(object *types.GetObjectsInput) (*types.GetObjectsOuput, error) {
+	// S3 has no way to read several objects at once, so the keys are fetched
+	// one by one, which is still fewer objects than a whole prefix
+	if len(object.Keys) > 0 {
+		keyValuePairs := make(map[string]string, len(object.Keys))
+		for _, key := range object.Keys {
+			found, err := c.GetObject(&types.GetObjectInput{Key: key})
+			if err != nil {
+				return nil, err
+			}
+			keyValuePairs[key] = found.Value
+		}
+		return &types.GetObjectsOuput{Secrets: keyValuePairs}, nil
+	}
+
 	params := &s3.ListObjectsInput{
 		Bucket: aws.String(c.bucket),
 		Prefix: aws.String(object.Prefix),

backend/ssm/aws.go

@@ -10,6 +10,7 @@ import (
 type ClientInterface interface {
 	GetParameter(context.Context, *ssm.GetParameterInput, ...func(*ssm.Options)) (*ssm.GetParameterOutput, error)
 	PutParameter(context.Context, *ssm.PutParameterInput, ...func(*ssm.Options)) (*ssm.PutParameterOutput, error)
+	GetParameters(context.Context, *ssm.GetParametersInput, ...func(*ssm.Options)) (*ssm.GetParametersOutput, error)
 	GetParametersByPath(context.Context, *ssm.GetParametersByPathInput, ...func(*ssm.Options)) (*ssm.GetParametersByPathOutput, error)
 	DeleteParameter(context.Context, *ssm.DeleteParameterInput, ...func(*ssm.Options)) (*ssm.DeleteParameterOutput, error)
 }

backend/ssm/getparameters_test.go

@@ -0,0 +1,98 @@
+package ssm
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/AirHelp/treasury/types"
+	"github.com/aws/aws-sdk-go-v2/service/ssm"
+	ssmtypes "github.com/aws/aws-sdk-go-v2/service/ssm/types"
+)
+
+// batchingSSMClient serves any name it is asked for and remembers how the names
+// were split between calls
+type batchingSSMClient struct {
+	ClientInterface
+	batches [][]string
+}
+
+func (b *batchingSSMClient) GetParameters(_ context.Context, input *ssm.GetParametersInput, _ ...func(*ssm.Options)) (*ssm.GetParametersOutput, error) {
+	if len(input.Names) > maxKeysPerCall {
+		return nil, fmt.Errorf("asked for %d names on one call, SSM allows %d", len(input.Names), maxKeysPerCall)
+	}
+	b.batches = append(b.batches, input.Names)
+
+	parameters := make([]ssmtypes.Parameter, 0, len(input.Names))
+	for _, name := range input.Names {
+		value := "value of " + name
+		parameters = append(parameters, ssmtypes.Parameter{Name: &name, Value: &value})
+	}
+	return &ssm.GetParametersOutput{Parameters: parameters}, nil
+}
+
+func TestClient_GetObjectsByKeysBatches(t *testing.T) {
+	// 23 keys have to be split into 10 + 10 + 3
+	keys := make([]string, 0, 23)
+	for i := range 23 {
+		keys = append(keys, fmt.Sprintf("test/webapp/KEY_%02d", i))
+	}
+
+	svc := &batchingSSMClient{}
+	got, err := (&Client{svc: svc}).GetObjects(&types.GetObjectsInput{Keys: keys})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wantBatches := []int{10, 10, 3}
+	if len(svc.batches) != len(wantBatches) {
+		t.Fatalf("GetObjects() made %d calls, want %d", len(svc.batches), len(wantBatches))
+	}
+	for i, want := range wantBatches {
+		if len(svc.batches[i]) != want {
+			t.Errorf("call %d carried %d names, want %d", i+1, len(svc.batches[i]), want)
+		}
+	}
+
+	if len(got.Secrets) != len(keys) {
+		t.Fatalf("GetObjects() returned %d secrets, want %d", len(got.Secrets), len(keys))
+	}
+	// the leading slash SSM needs is not a part of a treasury key
+	for _, key := range keys {
+		value, found := got.Secrets[key]
+		if !found {
+			t.Fatalf("GetObjects() did not return %q", key)
+		}
+		if want := "value of /" + key; value != want {
+			t.Errorf("GetObjects()[%q] = %q, want %q", key, value, want)
+		}
+	}
+}
+
+// missingSSMClient reports every name as invalid, the way SSM does for a
+// parameter which does not exist
+type missingSSMClient struct {
+	ClientInterface
+}
+
+func (m *missingSSMClient) GetParameters(_ context.Context, input *ssm.GetParametersInput, _ ...func(*ssm.Options)) (*ssm.GetParametersOutput, error) {
+	invalid := append([]string{}, input.Names...)
+	sort.Strings(invalid)
+	return &ssm.GetParametersOutput{InvalidParameters: invalid}, nil
+}
+
+func TestClient_GetObjectsByKeysReportsMissing(t *testing.T) {
+	keys := []string{"test/webapp/GONE", "test/webapp/ALSO_GONE"}
+
+	_, err := (&Client{svc: &missingSSMClient{}}).GetObjects(&types.GetObjectsInput{Keys: keys})
+	if err == nil {
+		t.Fatal("GetObjects() with unknown keys returned no error")
+	}
+	for _, key := range keys {
+		if !strings.Contains(err.Error(), key) {
+			t.Errorf("GetObjects() error = %q, want it to name %q", err, key)
+		}
+	}
+}

backend/ssm/ssm.go

@@ -3,14 +3,23 @@ package ssm
 import (
 	"context"
 	"errors"
+	"fmt"
+	"sort"
+	"strings"
 
 	"github.com/AirHelp/treasury/types"
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/ssm"
 	ssmtypes "github.com/aws/aws-sdk-go-v2/service/ssm/types"
 )
 
-const defaultParameterType = "SecureString"
+const (
+	defaultParameterType = "SecureString"
+
+	// maxKeysPerCall is the hard limit SSM puts on both GetParameters and
+	// GetParametersByPath, so it is the batch size of every read we do
+	maxKeysPerCall = 10
+)
 
 // PutObject writes a given secret value on SSM
 // it uses PutParameter API call
@@ -54,8 +63,13 @@ func (c *Client) GetObject(object *types.GetObjectInput) (*types.GetObjectOutput
 	return &types.GetObjectOutput{Value: *resp.Parameter.Value}, nil
 }
 
-// GetObjects returns key value map for given pattern/prefix
+// GetObjects returns key value map for the listed keys, or for the given
+// pattern/prefix when no keys are given
 func (c *Client) GetObjects(object *types.GetObjectsInput) (*types.GetObjectsOuput, error) {
+	if len(object.Keys) > 0 {
+		return c.getParameters(object.Keys)
+	}
+
 	var nextToken *string
 	var parameters []ssmtypes.Parameter
 	for {
@@ -64,7 +78,7 @@ func (c *Client) GetObjects(object *types.GetObjectsInput) (*types.GetObjectsOup
 			Path: aws.String("/" + object.Prefix),
 			// Retrieve all parameters in a hierarchy with their value decrypted.
 			WithDecryption: aws.Bool(true),
-			MaxResults:     aws.Int32(10),
+			MaxResults:     aws.Int32(maxKeysPerCall),
 			NextToken:      nextToken,
 		}
 
@@ -90,6 +104,46 @@ func (c *Client) GetObjects(object *types.GetObjectsInput) (*types.GetObjectsOup
 	return &types.GetObjectsOuput{Secrets: keyValuePairs}, nil
 }
 
+// getParameters fetches the given keys only, in batches of maxKeysPerCall,
+// which is what makes reading a handful of secrets from a big path cheap
+// https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_GetParameters.html
+func (c *Client) getParameters(keys []string) (*types.GetObjectsOuput, error) {
+	keyValuePairs := make(map[string]string, len(keys))
+	for start := 0; start < len(keys); start += maxKeysPerCall {
+		batch := keys[start:min(start+maxKeysPerCall, len(keys))]
+		names := make([]string, 0, len(batch))
+		for _, key := range batch {
+			// we decided to use path based keys without `/` at the beginning
+			// so we need to add it here
+			names = append(names, "/"+key)
+		}
+
+		resp, err := c.svc.GetParameters(context.Background(), &ssm.GetParametersInput{
+			Names: names,
+			// Retrieve all parameters in a hierarchy with their value decrypted.
+			WithDecryption: aws.Bool(true),
+		})
+		if err != nil {
+			return nil, err
+		}
+		// a name which does not exist is not an error for SSM, it comes back
+		// on a list of its own
+		if len(resp.InvalidParameters) > 0 {
+			missing := make([]string, 0, len(resp.InvalidParameters))
+			for _, name := range resp.InvalidParameters {
+				missing = append(missing, unSlash(name))
+			}
+			sort.Strings(missing)
+			return nil, fmt.Errorf("secrets not found: %s", strings.Join(missing, ", "))
+		}
+
+		for _, parameter := range resp.Parameters {
+			keyValuePairs[unSlash(*parameter.Name)] = *parameter.Value
+		}
+	}
+	return &types.GetObjectsOuput{Secrets: keyValuePairs}, nil
+}
+
 // unSlash removes 1st char from a string
 // GetParametersByPath from SSM returns key path with "/" at the beginning
 // but we don't need it :)