diff --git a/README.md b/README.md index 79a87f1..f3854f0 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,10 @@ Thumper config files are YAML files. These files support Go template preprocessi The final YAML generated by the templates must validate with the schema in [schema.yaml](schema.yaml). +A script may set an optional top-level `recordTtl`, a Go duration string (e.g. `30s`, `5m`). When `recordTtl` is greater than zero, every relationship written by the script's `WriteRelationships` steps is given an expiration that far in the future, computed at the moment each write is issued. Omitting `recordTtl` (or setting it to `0`) writes relationships with no expiration, as before. + +This is useful for driving sustained write-oriented workloads while keeping disk usage bounded: as the generator keeps writing new relationships, older ones expire and are garbage-collected, so the datastore reaches a steady state instead of growing without limit. Note that relations targeted by a TTL write must permit expiration in the schema (declared `with expiration`); see [schema.yaml](schema.yaml), where every relation accepts both expiring and non-expiring writes. + #### Example ```yaml diff --git a/internal/config/types.go b/internal/config/types.go index 6662279..af90ba3 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "time" "github.com/goccy/go-yaml" "google.golang.org/protobuf/types/known/structpb" @@ -12,6 +13,12 @@ type Script struct { Name string Weight uint Steps []ScriptStep + + // RecordTTL, when greater than zero, sets an expiration on every + // relationship written by WriteRelationships steps in this script. Each + // written relationship expires RecordTTL after the moment it is written. + // Accepts Go duration strings, e.g. "30s" or "5m". + RecordTTL time.Duration `yaml:"recordTtl"` } // ScriptStep is a single step of a thumper script, for example a single call to CheckPermissions. diff --git a/internal/config/types_test.go b/internal/config/types_test.go new file mode 100644 index 0000000..9c9ef2e --- /dev/null +++ b/internal/config/types_test.go @@ -0,0 +1,41 @@ +package config + +import ( + "testing" + "time" + + "github.com/goccy/go-yaml" + "github.com/stretchr/testify/require" +) + +func TestScriptRecordTTL(t *testing.T) { + testCases := []struct { + name string + yaml string + expected time.Duration + }{ + { + name: "absent", + yaml: "name: s\nsteps: []\n", + expected: 0, + }, + { + name: "seconds", + yaml: "name: s\nrecordTtl: 30s\nsteps: []\n", + expected: 30 * time.Second, + }, + { + name: "minutes", + yaml: "name: s\nrecordTtl: 5m\nsteps: []\n", + expected: 5 * time.Minute, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var script Script + require.NoError(t, yaml.Unmarshal([]byte(tc.yaml), &script)) + require.Equal(t, tc.expected, script.RecordTTL) + }) + } +} diff --git a/internal/thumperrunner/prepare.go b/internal/thumperrunner/prepare.go index 2cd15f0..5050bf5 100644 --- a/internal/thumperrunner/prepare.go +++ b/internal/thumperrunner/prepare.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "strings" + "time" "github.com/authzed/internal/thumper/internal/config" @@ -15,6 +16,7 @@ import ( "google.golang.org/grpc" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/timestamppb" ) // Prepare transforms a loaded yaml script into one that can be efficiently executed. @@ -24,7 +26,7 @@ func Prepare(inputs []*config.Script) (prepared []*ExecutableScript, err error) for _, rawStep := range input.Steps { var step executableStep - step, err = prepareStep(rawStep) + step, err = prepareStep(rawStep, input.RecordTTL) if err != nil { return prepared, err } @@ -42,7 +44,7 @@ func Prepare(inputs []*config.Script) (prepared []*ExecutableScript, err error) return prepared, err } -func prepareStep(step config.ScriptStep) (executableStep, error) { +func prepareStep(step config.ScriptStep, recordTTL time.Duration) (executableStep, error) { consistencyForZedToken, consistencyDesc, err := prepareConsistency(step) if err != nil { return executableStep{}, fmt.Errorf("error preparing consistency: %w", err) @@ -215,6 +217,16 @@ func prepareStep(step config.ScriptStep) (executableStep, error) { } execStep.body = func(ctx context.Context, client *authzed.Client, _ *v1.ZedToken) (*v1.ZedToken, error) { + if recordTTL > 0 { + // Compute the expiration relative to write time so each + // written relationship lives for recordTTL rather than + // expiring at a fixed instant baked in at prepare time. + expiresAt := timestamppb.New(time.Now().Add(recordTTL)) + for _, update := range req.Updates { + update.Relationship.OptionalExpiresAt = expiresAt + } + } + resp, err := client.WriteRelationships(ctx, req) if err != nil { return nil, err diff --git a/schema.yaml b/schema.yaml index d5cea38..f743958 100644 --- a/schema.yaml +++ b/schema.yaml @@ -9,6 +9,12 @@ properties: weight: type: integer minimum: 1 + recordTtl: + type: string + description: >- + Go duration string (e.g. "30s", "5m"). When set, every relationship + written by WriteRelationships steps in this script expires this long + after it is written. steps: type: array minItems: 1 diff --git a/scripts/schema.yaml b/scripts/schema.yaml index bda7ffb..2ddf65e 100644 --- a/scripts/schema.yaml +++ b/scripts/schema.yaml @@ -1,10 +1,12 @@ name: write basic thumper schema steps: - op: WriteSchema - schema: | + schema: | + use expiration + definition {{ .Prefix }}resource { - relation writer: {{ .Prefix }}user - relation reader: {{ .Prefix }}user + relation writer: {{ .Prefix }}user | {{ .Prefix }}user with expiration + relation reader: {{ .Prefix }}user | {{ .Prefix }}user with expiration permission edit = writer permission view = reader + edit } @@ -12,7 +14,7 @@ steps: /** token represents a single token, assigned to a user or client */ definition {{ .Prefix }}token { /** owner is the owner of the token */ - relation owner: {{ .Prefix }}client | {{ .Prefix }}user + relation owner: {{ .Prefix }}client | {{ .Prefix }}user | {{ .Prefix }}client with expiration | {{ .Prefix }}user with expiration permission delete_token = owner + owner->delete_token permission view_token = owner + owner->view_token } @@ -25,32 +27,32 @@ steps: */ definition {{ .Prefix }}tenant { /** organization is the organization that owns the tenant */ - relation organization: {{ .Prefix }}organization + relation organization: {{ .Prefix }}organization | {{ .Prefix }}organization with expiration /** * current_revision_hash is a link to the hash of the current schema for the tenant */ - relation current_revision_hash: {{ .Prefix }}revisionhash + relation current_revision_hash: {{ .Prefix }}revisionhash | {{ .Prefix }}revisionhash with expiration /** * admin is a user or client *token* that has the admin role on the tenant */ - relation admin: {{ .Prefix }}user#token | {{ .Prefix }}client#token + relation admin: {{ .Prefix }}user#token | {{ .Prefix }}client#token | {{ .Prefix }}user#token with expiration | {{ .Prefix }}client#token with expiration /** * writer is a user or client *token* that has the writer role on the tenant */ - relation writer: {{ .Prefix }}user#token | {{ .Prefix }}client#token + relation writer: {{ .Prefix }}user#token | {{ .Prefix }}client#token | {{ .Prefix }}user#token with expiration | {{ .Prefix }}client#token with expiration /** * viewer is a user or client *token* that has the viewer role on the tenant */ - relation viewer: {{ .Prefix }}user#token | {{ .Prefix }}client#token + relation viewer: {{ .Prefix }}user#token | {{ .Prefix }}client#token | {{ .Prefix }}user#token with expiration | {{ .Prefix }}client#token with expiration /** * metrics_viewer is the client *token* that has the ability to view metrics for the tenant */ - relation metrics_viewer: {{ .Prefix }}client#token + relation metrics_viewer: {{ .Prefix }}client#token | {{ .Prefix }}client#token with expiration // @deprecated permission write = writer + admin_tenant @@ -93,22 +95,22 @@ steps: /** organization is an organization in the system */ definition {{ .Prefix }}organization { /** platform is the singleton platform */ - relation platform: {{ .Prefix }}platform + relation platform: {{ .Prefix }}platform | {{ .Prefix }}platform with expiration /** * prod_enabled links the organization to its own members, if creation of production tenants is allowed */ - relation prod_enabled: {{ .Prefix }}organization + relation prod_enabled: {{ .Prefix }}organization | {{ .Prefix }}organization with expiration /** * admin is a user or client *token* that has the admin role on the organization */ - relation admin: {{ .Prefix }}user#token | {{ .Prefix }}client#token + relation admin: {{ .Prefix }}user#token | {{ .Prefix }}client#token | {{ .Prefix }}user#token with expiration | {{ .Prefix }}client#token with expiration /** * member is a user or client *token* that is a member of the organization */ - relation member: {{ .Prefix }}user#token | {{ .Prefix }}client#token + relation member: {{ .Prefix }}user#token | {{ .Prefix }}client#token | {{ .Prefix }}user#token with expiration | {{ .Prefix }}client#token with expiration // Orgs permission admin_org = admin @@ -141,26 +143,26 @@ steps: * namespace represents a single namespace defined in the schema of a tenant */ definition {{ .Prefix }}namespace { - relation tenant: {{ .Prefix }}tenant + relation tenant: {{ .Prefix }}tenant | {{ .Prefix }}tenant with expiration } // @deprecated definition {{ .Prefix }}flag { - relation enabled: {{ .Prefix }}organization + relation enabled: {{ .Prefix }}organization | {{ .Prefix }}organization with expiration } /** user represents a user in the tenancy model */ definition {{ .Prefix }}user { - relation token: {{ .Prefix }}token + relation token: {{ .Prefix }}token | {{ .Prefix }}token with expiration } /** client represents a client in the tenancy model */ definition {{ .Prefix }}client { /** org is the organization that owns the client */ - relation org: {{ .Prefix }}organization + relation org: {{ .Prefix }}organization | {{ .Prefix }}organization with expiration /** token is a token owned by the client */ - relation token: {{ .Prefix }}token + relation token: {{ .Prefix }}token | {{ .Prefix }}token with expiration // Clients permission admin_client = org->admin_org @@ -177,7 +179,7 @@ steps: /** schema represents a single schema for a tenant */ definition {{ .Prefix }}schema { - relation revision: {{ .Prefix }}schemarevision + relation revision: {{ .Prefix }}schemarevision | {{ .Prefix }}schemarevision with expiration } - op: WriteRelationships updates: diff --git a/scripts/some-data.yaml b/scripts/some-data.yaml index 86b4869..68700e5 100644 --- a/scripts/some-data.yaml +++ b/scripts/some-data.yaml @@ -1,5 +1,6 @@ name: load some data (scattered keys) weight: 1 +recordTtl: 10m # randomObjectID returns ONE value per render, so on its own the only thing that # varied across the 100 entity groups below was the trailing "_" index. # Because object_id is the second column of the relation_tuple primary key, that