Add per-script record TTL for written relationships

README.md

@@ -39,6 +39,10 @@ Thumper config files are YAML files. These files support Go template preprocessi
 
 The final YAML generated by the templates must validate with the schema in [schema.yaml](schema.yaml).
 
+A script may set an optional top-level `recordTtl`, a Go duration string (e.g. `30s`, `5m`). When `recordTtl` is greater than zero, every relationship written by the script's `WriteRelationships` steps is given an expiration that far in the future, computed at the moment each write is issued. Omitting `recordTtl` (or setting it to `0`) writes relationships with no expiration, as before.
+
+This is useful for driving sustained write-oriented workloads while keeping disk usage bounded: as the generator keeps writing new relationships, older ones expire and are garbage-collected, so the datastore reaches a steady state instead of growing without limit. Note that relations targeted by a TTL write must permit expiration in the schema (declared `with expiration`); see [schema.yaml](schema.yaml), where every relation accepts both expiring and non-expiring writes.
+
 #### Example
 
 ```yaml

internal/config/types.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/goccy/go-yaml"
 	"google.golang.org/protobuf/types/known/structpb"
@@ -12,6 +13,12 @@ type Script struct {
 	Name   string
 	Weight uint
 	Steps  []ScriptStep
+
+	// RecordTTL, when greater than zero, sets an expiration on every
+	// relationship written by WriteRelationships steps in this script. Each
+	// written relationship expires RecordTTL after the moment it is written.
+	// Accepts Go duration strings, e.g. "30s" or "5m".
+	RecordTTL time.Duration `yaml:"recordTtl"`
 }
 
 // ScriptStep is a single step of a thumper script, for example a single call to CheckPermissions.

internal/config/types_test.go

@@ -0,0 +1,41 @@
+package config
+
+import (
+	"testing"
+	"time"
+
+	"github.com/goccy/go-yaml"
+	"github.com/stretchr/testify/require"
+)
+
+func TestScriptRecordTTL(t *testing.T) {
+	testCases := []struct {
+		name     string
+		yaml     string
+		expected time.Duration
+	}{
+		{
+			name:     "absent",
+			yaml:     "name: s\nsteps: []\n",
+			expected: 0,
+		},
+		{
+			name:     "seconds",
+			yaml:     "name: s\nrecordTtl: 30s\nsteps: []\n",
+			expected: 30 * time.Second,
+		},
+		{
+			name:     "minutes",
+			yaml:     "name: s\nrecordTtl: 5m\nsteps: []\n",
+			expected: 5 * time.Minute,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var script Script
+			require.NoError(t, yaml.Unmarshal([]byte(tc.yaml), &script))
+			require.Equal(t, tc.expected, script.RecordTTL)
+		})
+	}
+}

internal/thumperrunner/prepare.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"time"
 
 	"github.com/authzed/internal/thumper/internal/config"
 
@@ -15,6 +16,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 // Prepare transforms a loaded yaml script into one that can be efficiently executed.
@@ -24,7 +26,7 @@ func Prepare(inputs []*config.Script) (prepared []*ExecutableScript, err error)
 
 		for _, rawStep := range input.Steps {
 			var step executableStep
-			step, err = prepareStep(rawStep)
+			step, err = prepareStep(rawStep, input.RecordTTL)
 			if err != nil {
 				return prepared, err
 			}
@@ -42,7 +44,7 @@ func Prepare(inputs []*config.Script) (prepared []*ExecutableScript, err error)
 	return prepared, err
 }
 
-func prepareStep(step config.ScriptStep) (executableStep, error) {
+func prepareStep(step config.ScriptStep, recordTTL time.Duration) (executableStep, error) {
 	consistencyForZedToken, consistencyDesc, err := prepareConsistency(step)
 	if err != nil {
 		return executableStep{}, fmt.Errorf("error preparing consistency: %w", err)
@@ -215,6 +217,16 @@ func prepareStep(step config.ScriptStep) (executableStep, error) {
 		}
 
 		execStep.body = func(ctx context.Context, client *authzed.Client, _ *v1.ZedToken) (*v1.ZedToken, error) {
+			if recordTTL > 0 {
+				// Compute the expiration relative to write time so each
+				// written relationship lives for recordTTL rather than
+				// expiring at a fixed instant baked in at prepare time.
+				expiresAt := timestamppb.New(time.Now().Add(recordTTL))
+				for _, update := range req.Updates {
+					update.Relationship.OptionalExpiresAt = expiresAt
+				}
+			}
+
 			resp, err := client.WriteRelationships(ctx, req)
 			if err != nil {
 				return nil, err

schema.yaml

@@ -9,6 +9,12 @@ properties:
   weight:
     type: integer
     minimum: 1
+  recordTtl:
+    type: string
+    description: >-
+      Go duration string (e.g. "30s", "5m"). When set, every relationship
+      written by WriteRelationships steps in this script expires this long
+      after it is written.
   steps:
     type: array
     minItems: 1

scripts/schema.yaml

@@ -1,18 +1,20 @@
 name: write basic thumper schema
 steps:
 - op: WriteSchema
-  schema: | 
+  schema: |
+    use expiration
+
     definition {{ .Prefix }}resource {
-      relation writer: {{ .Prefix }}user
-      relation reader: {{ .Prefix }}user
+      relation writer: {{ .Prefix }}user | {{ .Prefix }}user with expiration
+      relation reader: {{ .Prefix }}user | {{ .Prefix }}user with expiration
       permission edit = writer
       permission view = reader + edit
     }
 
     /** token represents a single token, assigned to a user or client */
     definition {{ .Prefix }}token {
       /** owner is the owner of the token */
-      relation owner: {{ .Prefix }}client | {{ .Prefix }}user
+      relation owner: {{ .Prefix }}client | {{ .Prefix }}user | {{ .Prefix }}client with expiration | {{ .Prefix }}user with expiration
       permission delete_token = owner + owner->delete_token
       permission view_token = owner + owner->view_token
     }
@@ -25,32 +27,32 @@ steps:
     */
     definition {{ .Prefix }}tenant {
       /** organization is the organization that owns the tenant */
-      relation organization: {{ .Prefix }}organization
+      relation organization: {{ .Prefix }}organization | {{ .Prefix }}organization with expiration
 
       /**
       * current_revision_hash is a link to the hash of the current schema for the tenant
       */
-      relation current_revision_hash: {{ .Prefix }}revisionhash
+      relation current_revision_hash: {{ .Prefix }}revisionhash | {{ .Prefix }}revisionhash with expiration
 
       /**
       * admin is a user or client *token* that has the admin role on the tenant
       */
-      relation admin: {{ .Prefix }}user#token | {{ .Prefix }}client#token
+      relation admin: {{ .Prefix }}user#token | {{ .Prefix }}client#token | {{ .Prefix }}user#token with expiration | {{ .Prefix }}client#token with expiration
 
       /**
       * writer is a user or client *token* that has the writer role on the tenant
       */
-      relation writer: {{ .Prefix }}user#token | {{ .Prefix }}client#token
+      relation writer: {{ .Prefix }}user#token | {{ .Prefix }}client#token | {{ .Prefix }}user#token with expiration | {{ .Prefix }}client#token with expiration
 
       /**
       * viewer is a user or client *token* that has the viewer role on the tenant
       */
-      relation viewer: {{ .Prefix }}user#token | {{ .Prefix }}client#token
+      relation viewer: {{ .Prefix }}user#token | {{ .Prefix }}client#token | {{ .Prefix }}user#token with expiration | {{ .Prefix }}client#token with expiration
 
       /**
       * metrics_viewer is the client *token* that has the ability to view metrics for the tenant
       */
-      relation metrics_viewer: {{ .Prefix }}client#token
+      relation metrics_viewer: {{ .Prefix }}client#token | {{ .Prefix }}client#token with expiration
 
       // @deprecated
       permission write = writer + admin_tenant
@@ -93,22 +95,22 @@ steps:
     /** organization is an organization in the system */
     definition {{ .Prefix }}organization {
       /** platform is the singleton platform */
-      relation platform: {{ .Prefix }}platform
+      relation platform: {{ .Prefix }}platform | {{ .Prefix }}platform with expiration
 
       /**
       * prod_enabled links the organization to its own members, if creation of production tenants is allowed
       */
-      relation prod_enabled: {{ .Prefix }}organization
+      relation prod_enabled: {{ .Prefix }}organization | {{ .Prefix }}organization with expiration
 
       /**
       * admin is a user or client *token* that has the admin role on the organization
       */
-      relation admin: {{ .Prefix }}user#token | {{ .Prefix }}client#token
+      relation admin: {{ .Prefix }}user#token | {{ .Prefix }}client#token | {{ .Prefix }}user#token with expiration | {{ .Prefix }}client#token with expiration
 
       /**
       * member is a user or client *token* that is a member of the organization
       */
-      relation member: {{ .Prefix }}user#token | {{ .Prefix }}client#token
+      relation member: {{ .Prefix }}user#token | {{ .Prefix }}client#token | {{ .Prefix }}user#token with expiration | {{ .Prefix }}client#token with expiration
 
       // Orgs
       permission admin_org = admin
@@ -141,26 +143,26 @@ steps:
     * namespace represents a single namespace defined in the schema of a tenant
     */
     definition {{ .Prefix }}namespace {
-      relation tenant: {{ .Prefix }}tenant
+      relation tenant: {{ .Prefix }}tenant | {{ .Prefix }}tenant with expiration
     }
 
     // @deprecated
     definition {{ .Prefix }}flag {
-      relation enabled: {{ .Prefix }}organization
+      relation enabled: {{ .Prefix }}organization | {{ .Prefix }}organization with expiration
     }
 
     /** user represents a user in the tenancy model */
     definition {{ .Prefix }}user {
-      relation token: {{ .Prefix }}token
+      relation token: {{ .Prefix }}token | {{ .Prefix }}token with expiration
     }
 
     /** client represents a client in the tenancy model */
     definition {{ .Prefix }}client {
       /** org is the organization that owns the client */
-      relation org: {{ .Prefix }}organization
+      relation org: {{ .Prefix }}organization | {{ .Prefix }}organization with expiration
 
       /** token is a token owned by the client */
-      relation token: {{ .Prefix }}token
+      relation token: {{ .Prefix }}token | {{ .Prefix }}token with expiration
 
       // Clients
       permission admin_client = org->admin_org
@@ -177,7 +179,7 @@ steps:
 
     /** schema represents a single schema for a tenant */
     definition {{ .Prefix }}schema {
-      relation revision: {{ .Prefix }}schemarevision
+      relation revision: {{ .Prefix }}schemarevision | {{ .Prefix }}schemarevision with expiration
     }
 - op: WriteRelationships
   updates:

scripts/some-data.yaml

@@ -1,5 +1,6 @@
 name: load some data (scattered keys)
 weight: 1
+recordTtl: 10m
 # randomObjectID returns ONE value per render, so on its own the only thing that
 # varied across the 100 entity groups below was the trailing "_<j>" index.
 # Because object_id is the second column of the relation_tuple primary key, that