From f512d48633c44a7c4c22f665e23bc711e53944e0 Mon Sep 17 00:00:00 2001 From: Jeroen Soeters Date: Sun, 19 Apr 2026 19:57:22 -0700 Subject: [PATCH 1/4] feat(datastore): add postgres query duration metrics Emit an OTel histogram (formae.datastore.query.duration_ms) per postgres query path so we can spot slow datastore operations during load tests and in production. Initialized alongside the existing pool metrics; a failure to create the histogram is non-fatal and leaves query duration unmetered. --- internal/datastore/postgres/metrics.go | 42 ++++++++++++++++++ internal/datastore/postgres/metrics_test.go | 49 +++++++++++++++++++++ internal/datastore/postgres/postgres.go | 22 ++++++++- 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 internal/datastore/postgres/metrics.go create mode 100644 internal/datastore/postgres/metrics_test.go diff --git a/internal/datastore/postgres/metrics.go b/internal/datastore/postgres/metrics.go new file mode 100644 index 000000000..7c5c00a8d --- /dev/null +++ b/internal/datastore/postgres/metrics.go @@ -0,0 +1,42 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package postgres + +import ( + "context" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var meter = otel.Meter("formae/datastore") + +type datastoreMetrics struct { + queryDuration metric.Float64Histogram +} + +func newDatastoreMetrics() (*datastoreMetrics, error) { + queryDuration, err := meter.Float64Histogram( + "formae.datastore.query.duration_ms", + metric.WithDescription("Duration of datastore queries in milliseconds"), + metric.WithUnit("ms"), + ) + if err != nil { + return nil, err + } + return &datastoreMetrics{queryDuration: queryDuration}, nil +} + +func (m *datastoreMetrics) recordDuration(ctx context.Context, queryType string, start time.Time) { + if m == nil { + return + } + duration := float64(time.Since(start).Milliseconds()) + m.queryDuration.Record(ctx, duration, + metric.WithAttributes(attribute.String("query_type", queryType)), + ) +} diff --git a/internal/datastore/postgres/metrics_test.go b/internal/datastore/postgres/metrics_test.go new file mode 100644 index 000000000..b15abe18c --- /dev/null +++ b/internal/datastore/postgres/metrics_test.go @@ -0,0 +1,49 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +//go:build unit + +package postgres + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestDatastoreMetrics_RecordDuration(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + defer provider.Shutdown(context.Background()) + + // Override the package-level meter for testing + origMeter := meter + meter = provider.Meter("formae/datastore") + defer func() { meter = origMeter }() + + m, err := newDatastoreMetrics() + require.NoError(t, err) + + start := time.Now().Add(-50 * time.Millisecond) + m.recordDuration(context.Background(), "store_resource", start) + + var rm metricdata.ResourceMetrics + err = reader.Collect(context.Background(), &rm) + require.NoError(t, err) + + require.Len(t, rm.ScopeMetrics, 1) + require.Len(t, rm.ScopeMetrics[0].Metrics, 1) + + metric := rm.ScopeMetrics[0].Metrics[0] + assert.Equal(t, "formae.datastore.query.duration_ms", metric.Name) + + histogram := metric.Data.(metricdata.Histogram[float64]) + require.Len(t, histogram.DataPoints, 1) + assert.Greater(t, histogram.DataPoints[0].Sum, float64(0)) +} diff --git a/internal/datastore/postgres/postgres.go b/internal/datastore/postgres/postgres.go index 5cbb61fb5..86d8aa47a 100644 --- a/internal/datastore/postgres/postgres.go +++ b/internal/datastore/postgres/postgres.go @@ -51,6 +51,7 @@ type DatastorePostgres struct { agentID string cfg *pkgmodel.DatastoreConfig ctx context.Context + metrics *datastoreMetrics } // BuildConnStr constructs a PostgreSQL connection string from config fields. @@ -160,7 +161,13 @@ func NewDatastorePostgres(ctx context.Context, cfg *pkgmodel.DatastoreConfig, ag // Non-fatal - continue without pool metrics } - d := DatastorePostgres{pool: pool, agentID: agentID, cfg: cfg, ctx: ctx} + m, err := newDatastoreMetrics() + if err != nil { + slog.Error("failed to initialize datastore metrics", "error", err) + // Non-fatal - continue without query duration metrics + } + + d := DatastorePostgres{pool: pool, agentID: agentID, cfg: cfg, ctx: ctx, metrics: m} slog.Info("Started PostgreSQL datastore", "host", cfg.Postgres.Host, "port", cfg.Postgres.Port, "database", cfg.Postgres.Database, "schema", cfg.Postgres.Schema, "user", cfg.Postgres.User, "connectionParams", cfg.Postgres.ConnectionParams) @@ -170,6 +177,7 @@ func NewDatastorePostgres(ctx context.Context, cfg *pkgmodel.DatastoreConfig, ag func (d DatastorePostgres) StoreFormaCommand(fa *forma_command.FormaCommand, commandID string) error { ctx, span := tracer.Start(context.Background(), "StoreFormaCommand") defer span.End() + defer d.metrics.recordDuration(ctx, "store_forma_command", time.Now()) for _, r := range fa.ResourceUpdates { if r.DesiredState.Properties == nil { @@ -491,6 +499,7 @@ func (d DatastorePostgres) DeleteFormaCommand(fa *forma_command.FormaCommand, co func (d DatastorePostgres) GetFormaCommandByCommandID(commandID string) (*forma_command.FormaCommand, error) { ctx, span := tracer.Start(context.Background(), "GetFormaCommandByCommandID") defer span.End() + defer d.metrics.recordDuration(ctx, "get_forma_command_by_id", time.Now()) query := formaCommandWithResourceUpdatesQueryBasePostgres + " WHERE fc.command_id = $1" + resourceUpdateOrderByPostgres rows, err := d.pool.Query(ctx, query, commandID) @@ -570,6 +579,7 @@ func extendPostgresQueryString[T any](queryStr string, queryItem *datastore.Quer func (d DatastorePostgres) QueryFormaCommands(query *datastore.StatusQuery) ([]*forma_command.FormaCommand, error) { ctx, span := tracer.Start(context.Background(), "QueryFormaCommands") defer span.End() + defer d.metrics.recordDuration(ctx, "query_forma_commands", time.Now()) // Build subquery to find matching command IDs with filtering and LIMIT subqueryStr := "SELECT command_id FROM forma_commands WHERE 1=1" @@ -621,6 +631,7 @@ func (d DatastorePostgres) QueryFormaCommands(query *datastore.StatusQuery) ([]* func (d DatastorePostgres) GetKSUIDByTriplet(stack, label, resourceType string) (string, error) { ctx, span := tracer.Start(context.Background(), "GetKSUIDByTriplet") defer span.End() + defer d.metrics.recordDuration(ctx, "get_ksuid_by_triplet", time.Now()) query := ` SELECT ksuid @@ -646,6 +657,7 @@ func (d DatastorePostgres) GetKSUIDByTriplet(stack, label, resourceType string) func (d DatastorePostgres) BatchGetKSUIDsByTriplets(triplets []pkgmodel.TripletKey) (map[pkgmodel.TripletKey]string, error) { ctx, span := tracer.Start(context.Background(), "BatchGetKSUIDsByTriplets") defer span.End() + defer d.metrics.recordDuration(ctx, "batch_get_ksuids_by_triplets", time.Now()) if len(triplets) == 0 { return make(map[pkgmodel.TripletKey]string), nil @@ -702,6 +714,7 @@ func (d DatastorePostgres) BatchGetKSUIDsByTriplets(triplets []pkgmodel.TripletK func (d DatastorePostgres) GetResourceModificationsSinceLastReconcile(stack string) ([]datastore.ResourceModification, error) { ctx, span := tracer.Start(context.Background(), "GetResourceModificationsSinceLastReconcile") defer span.End() + defer d.metrics.recordDuration(ctx, "get_resource_modifications_since_last_reconcile", time.Now()) query := ` SELECT DISTINCT @@ -1459,6 +1472,7 @@ func (d DatastorePostgres) DeleteStack(label string, commandID string) (string, func (d DatastorePostgres) GetStackByLabel(label string) (*pkgmodel.Stack, error) { ctx, span := tracer.Start(context.Background(), "GetStackByLabel") defer span.End() + defer d.metrics.recordDuration(ctx, "get_stack_by_label", time.Now()) // Get the latest version of the stack, return nil if deleted // We need to check if the MOST RECENT version is a delete operation @@ -1619,6 +1633,7 @@ func (d DatastorePostgres) CountResourcesInStack(label string) (int, error) { func (d DatastorePostgres) ListAllStacks() ([]*pkgmodel.Stack, error) { ctx, span := tracer.Start(context.Background(), "ListAllStackMetadata") defer span.End() + defer d.metrics.recordDuration(ctx, "list_all_stacks", time.Now()) // Get all stacks at their latest version that aren't deleted // Uses window function to reliably get the most recent version per stack id @@ -2714,6 +2729,7 @@ func (d DatastorePostgres) QueryTargets(query *datastore.TargetQuery) ([]*pkgmod func (d DatastorePostgres) QueryResources(query *datastore.ResourceQuery) ([]*pkgmodel.Resource, error) { ctx, span := tracer.Start(context.Background(), "QueryResources") defer span.End() + defer d.metrics.recordDuration(ctx, "query_resources", time.Now()) queryStr := ` SELECT data, ksuid @@ -3276,6 +3292,7 @@ func (d DatastorePostgres) CountResourcesInTarget(targetLabel string) (int, erro func (d DatastorePostgres) BulkStoreResourceUpdates(commandID string, updates []resource_update.ResourceUpdate) error { ctx, span := tracer.Start(context.Background(), "BulkStoreResourceUpdates") defer span.End() + defer d.metrics.recordDuration(ctx, "bulk_store_resource_updates", time.Now()) if len(updates) == 0 { return nil @@ -3399,6 +3416,7 @@ func (d DatastorePostgres) BulkStoreResourceUpdates(commandID string, updates [] func (d DatastorePostgres) LoadResourceUpdates(commandID string) ([]resource_update.ResourceUpdate, error) { ctx, span := tracer.Start(context.Background(), "LoadResourceUpdates") defer span.End() + defer d.metrics.recordDuration(ctx, "load_resource_updates", time.Now()) query := ` SELECT ksuid, operation, state, start_ts, modified_ts, @@ -3520,6 +3538,7 @@ func (d DatastorePostgres) LoadResourceUpdates(commandID string) ([]resource_upd func (d DatastorePostgres) UpdateResourceUpdateState(commandID string, ksuid string, operation types.OperationType, state resource_update.ResourceUpdateState, modifiedTs time.Time) error { ctx, span := tracer.Start(context.Background(), "UpdateResourceUpdateState") defer span.End() + defer d.metrics.recordDuration(ctx, "update_resource_update_state", time.Now()) query := ` UPDATE resource_updates @@ -3595,6 +3614,7 @@ func (d DatastorePostgres) UpdateResourceUpdateProgress(commandID string, ksuid func (d DatastorePostgres) BatchUpdateResourceUpdateState(commandID string, refs []datastore.ResourceUpdateRef, state resource_update.ResourceUpdateState, modifiedTs time.Time) error { ctx, span := tracer.Start(context.Background(), "BatchUpdateResourceUpdateState") defer span.End() + defer d.metrics.recordDuration(ctx, "batch_update_resource_update_state", time.Now()) if len(refs) == 0 { return nil From c8c03f1eb9a75844428379eeedf6bc653b261fea Mon Sep 17 00:00:00 2001 From: Jeroen Soeters Date: Sun, 19 Apr 2026 19:57:27 -0700 Subject: [PATCH 2/4] feat(loadtest): extend env generator to multi-cloud Extend generate-stress-test-env.py to produce Azure and GCP resources alongside AWS, selectable via --clouds and configurable per-cloud via --subscription-azure / --project-gcp. Also wire up scale profiles so we can generate consistent small/medium/large environments without hand-picking counts. Used to seed Suite B regression baselines across the tier-1 cloud providers. --- scripts/generate-stress-test-env.py | 2120 ++++++++++++++++++++++++--- 1 file changed, 1897 insertions(+), 223 deletions(-) diff --git a/scripts/generate-stress-test-env.py b/scripts/generate-stress-test-env.py index 2f1055746..3d7ffb6c1 100755 --- a/scripts/generate-stress-test-env.py +++ b/scripts/generate-stress-test-env.py @@ -4,19 +4,28 @@ # SPDX-License-Identifier: FSL-1.1-ALv2 """ -Generate a large-scale AWS infrastructure environment for performance testing formae. +Generate a large-scale multi-cloud infrastructure environment for performance testing formae. This script generates either PKL files (for formae) or CloudFormation templates (for creating resources outside formae, useful for discovery testing). The distribution of resource types mimics typical enterprise production environments. +Supports AWS, Azure, and GCP resources either individually or combined. Usage: - # Generate PKL files for formae + # Generate PKL files for formae (AWS only - default) python3 scripts/generate-perf-test-env.py --count 1000 --region us-east-1 python3 scripts/generate-perf-test-env.py --count 3000 --region eu-west-1 --output ./perf-test - # Generate CloudFormation templates (for discovery testing) + # Generate multi-cloud PKL files + python3 scripts/generate-perf-test-env.py --count 100 --clouds aws,azure,gcp \\ + --region us-east-2 --subscription-azure --project-gcp + + # Use scale profiles for multi-cloud + python3 scripts/generate-perf-test-env.py --profile medium --clouds aws,azure,gcp \\ + --region us-east-2 --subscription-azure --project-gcp + + # Generate CloudFormation templates (for discovery testing, AWS only) python3 scripts/generate-perf-test-env.py --format cloudformation --count 500 --region us-east-1 To apply PKL environment: @@ -134,6 +143,188 @@ } +# Azure resource distribution for realistic production environments +# Based on typical enterprise Azure usage patterns +AZURE_DISTRIBUTION = { + # Networking + "resource_group": 0.06, # Azure::Resources::ResourceGroup + "virtual_network": 0.04, # Azure::Resources::VirtualNetwork + "subnet": 0.04, # Azure::Resources::Subnet + "nsg": 0.05, # Azure::Network::NetworkSecurityGroup + "public_ip": 0.03, # Azure::Network::PublicIPAddress + + # Compute + "virtual_machine": 0.06, # Azure::Compute::VirtualMachine + "network_interface": 0.06, # Azure::Network::NetworkInterface + + # Storage + "storage_account": 0.10, # Azure::Storage::StorageAccount + + # IAM + "managed_identity": 0.08, # Azure::ManagedIdentity::UserAssignedIdentity + + # Security + "key_vault": 0.06, # Azure::KeyVault::Vault + + # Container + "container_registry": 0.03, # Azure::ContainerRegistry::Registry + + # Database + "postgres_server": 0.02, # Azure::DBforPostgreSQL::FlexibleServer +} + + +# GCP resource distribution for realistic production environments +# NOTE: IAM ServiceAccount and Artifact Registry are not in the current GCP plugin schema, +# so we redistribute those percentages to other available resource types. +GCP_DISTRIBUTION = { + # Compute + "compute_instance": 0.06, # GCP::Compute::Instance + "compute_disk": 0.04, # GCP::Compute::Disk + + # Networking + "compute_network": 0.04, # GCP::Compute::Network + "compute_subnetwork": 0.04, # GCP::Compute::Subnetwork + "compute_firewall": 0.05, # GCP::Compute::Firewall + "compute_address": 0.03, # GCP::Compute::Address + + # Storage + "storage_bucket": 0.12, # GCP::Storage::Bucket (bumped, absorbing IAM/AR share) + + # Database + "sql_instance": 0.02, # GCP::SQL::DatabaseInstance + + # BigQuery + "bigquery_dataset": 0.05, # GCP::BigQuery::Dataset (bumped, absorbing IAM/AR share) +} + + +# Scale profiles: maps profile name -> (aws_count, azure_count, gcp_count) +SCALE_PROFILES = { + "small": (200, 150, 150), + "medium": (2000, 1500, 1500), + "large": (8000, 6000, 6000), + "xl": (20000, 15000, 15000), +} + + +@dataclass +class AzureResourceCounts: + """Calculated resource counts for Azure based on total target.""" + resource_groups: int + virtual_networks: int + subnets_per_vnet: int + nsgs: int + public_ips: int + virtual_machines: int + network_interfaces: int + storage_accounts: int + managed_identities: int + key_vaults: int + container_registries: int + postgres_servers: int + + @property + def total(self) -> int: + return ( + self.resource_groups + + self.virtual_networks + + (self.virtual_networks * self.subnets_per_vnet) + + self.nsgs + + self.public_ips + + self.virtual_machines + + self.network_interfaces + + self.storage_accounts + + self.managed_identities + + self.key_vaults + + self.container_registries + + self.postgres_servers + ) + + +def calculate_azure_counts(total: int) -> AzureResourceCounts: + """Calculate Azure resource counts based on target total and distribution. + + Quota caps: + - Resource Groups: cap at 50 (limit 980) + - VNets: cap at 50 (limit 1000) + - Storage Accounts: cap at 200 (limit 250) + - Key Vaults: cap at 200 + - VMs: cap conservatively + """ + d = AZURE_DISTRIBUTION + + rgs = min(50, max(1, int(total * d["resource_group"]))) + vnets = min(50, max(1, int(total * d["virtual_network"]))) + subnets_per_vnet = max(1, min(6, int(total * d["subnet"] / max(1, vnets)))) + + return AzureResourceCounts( + resource_groups=rgs, + virtual_networks=vnets, + subnets_per_vnet=subnets_per_vnet, + nsgs=max(1, int(total * d["nsg"])), + public_ips=max(1, int(total * d["public_ip"])), + virtual_machines=max(1, int(total * d["virtual_machine"])), + network_interfaces=max(1, int(total * d["network_interface"])), + storage_accounts=min(200, max(1, int(total * d["storage_account"]))), + managed_identities=max(1, int(total * d["managed_identity"])), + key_vaults=min(200, max(1, int(total * d["key_vault"]))), + container_registries=max(1, int(total * d["container_registry"])), + postgres_servers=max(0, int(total * d["postgres_server"])), + ) + + +@dataclass +class GCPResourceCounts: + """Calculated resource counts for GCP based on total target.""" + compute_instances: int + compute_disks: int + compute_networks: int + compute_subnetworks: int + compute_firewalls: int + compute_addresses: int + storage_buckets: int + sql_instances: int + bigquery_datasets: int + + @property + def total(self) -> int: + return ( + self.compute_instances + + self.compute_disks + + self.compute_networks + + self.compute_subnetworks + + self.compute_firewalls + + self.compute_addresses + + self.storage_buckets + + self.sql_instances + + self.bigquery_datasets + ) + + +def calculate_gcp_counts(total: int) -> GCPResourceCounts: + """Calculate GCP resource counts based on target total and distribution. + + Quota caps: + - Networks: cap at 15 (default limit) + - Firewall rules: cap at 400 (limit 500) + - Cloud SQL instances: cap at 30 (limit 40) + """ + d = GCP_DISTRIBUTION + + return GCPResourceCounts( + compute_instances=max(1, int(total * d["compute_instance"])), + compute_disks=max(1, int(total * d["compute_disk"])), + compute_networks=min(15, max(1, int(total * d["compute_network"]))), + compute_subnetworks=max(1, int(total * d["compute_subnetwork"])), + compute_firewalls=min(400, max(1, int(total * d["compute_firewall"]))), + compute_addresses=max(1, int(total * d["compute_address"])), + storage_buckets=max(1, int(total * d["storage_bucket"])), + sql_instances=min(30, max(0, int(total * d["sql_instance"]))), + bigquery_datasets=max(1, int(total * d["bigquery_dataset"])), + ) + + @dataclass class ResourceCounts: """Calculated resource counts based on total target.""" @@ -1537,7 +1728,1142 @@ class RDS {{ ''' -def generate_main_pkl(env_id: str, stack_name: str, counts: ResourceCounts) -> str: +# ============================================================================ +# Azure PKL Generators +# ============================================================================ + +AZURE_COPYRIGHT = '''/* + * Stress Test Environment - Auto-generated (Azure) + * Generated by generate-stress-test-env.py + * + * WARNING: This environment is for testing only. Resources will incur Azure costs. + */ +''' + + +def generate_azure_networking_pkl(counts: AzureResourceCounts, location: str) -> str: + """Generate Azure networking.pkl with resource groups, VNets, subnets, NSGs, and public IPs.""" + + rg_blocks = [] + vnet_blocks = [] + subnet_blocks = [] + nsg_blocks = [] + pip_blocks = [] + + # Resource groups + for i in range(counts.resource_groups): + rg_blocks.append(f''' + hidden rg{i}: rg.ResourceGroup = new {{ + label = "\\(prefix)-rg-{i}" + name = "\\(prefix)-rg-{i}" + location = "{location}" + tags {{ + new azure.Tag {{ key = "Environment"; value = "stress-test" }} + new azure.Tag {{ key = "ManagedBy"; value = "formae" }} + new azure.Tag {{ key = "run-id"; value = "\\(prefix)" }} + }} + }}''') + + # Virtual networks + for i in range(counts.virtual_networks): + rg_idx = i % counts.resource_groups + vnet_blocks.append(f''' + hidden vnet{i}: vnet.VirtualNetwork = new {{ + label = "\\(prefix)-vnet-{i}" + name = "\\(prefix)-vnet-{i}" + location = "{location}" + resourceGroupName = rg{rg_idx}.res.name + addressSpace = new vnet.AddressSpace {{ + addressPrefixes {{ + "10.{i % 256}.0.0/16" + }} + }} + tags {{ + new azure.Tag {{ key = "Environment"; value = "stress-test" }} + new azure.Tag {{ key = "ManagedBy"; value = "formae" }} + }} + }}''') + + # Subnets for this VNet + for j in range(counts.subnets_per_vnet): + subnet_blocks.append(f''' + hidden subnet{i}_{j}: snet.Subnet = new {{ + label = "\\(prefix)-subnet-{i}-{j}" + name = "\\(prefix)-subnet-{i}-{j}" + resourceGroupName = rg{rg_idx}.res.name + virtualNetworkName = vnet{i}.res.name + addressPrefix = "10.{i % 256}.{j}.0/24" + }}''') + + # NSGs + for i in range(counts.nsgs): + rg_idx = i % counts.resource_groups + nsg_blocks.append(f''' + hidden nsg{i}: nsg.NetworkSecurityGroup = new {{ + label = "\\(prefix)-nsg-{i}" + name = "\\(prefix)-nsg-{i}" + location = "{location}" + resourceGroupName = rg{rg_idx}.res.name + securityRules {{ + new nsg.SecurityRule {{ + name = "AllowHTTP" + priority = {100 + i} + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + sourceAddressPrefix = "10.0.0.0/8" + sourcePortRange = "*" + destinationAddressPrefix = "*" + destinationPortRange = "80" + }} + }} + tags {{ + new azure.Tag {{ key = "Environment"; value = "stress-test" }} + new azure.Tag {{ key = "ManagedBy"; value = "formae" }} + }} + }}''') + + # Public IPs + for i in range(counts.public_ips): + rg_idx = i % counts.resource_groups + pip_blocks.append(f''' + hidden pip{i}: pip.PublicIPAddress = new {{ + label = "\\(prefix)-pip-{i}" + name = "\\(prefix)-pip-{i}" + location = "{location}" + resourceGroupName = rg{rg_idx}.res.name + sku = new pip.PublicIPAddressSKU {{ + name = "Standard" + tier = "Regional" + }} + publicIPAllocationMethod = "Static" + publicIPAddressVersion = "IPv4" + tags {{ + new azure.Tag {{ key = "Environment"; value = "stress-test" }} + new azure.Tag {{ key = "ManagedBy"; value = "formae" }} + }} + }}''') + + # Resource listing + resource_list = [] + for i in range(counts.resource_groups): + resource_list.append(f" rg{i}") + for i in range(counts.virtual_networks): + resource_list.append(f" vnet{i}") + for j in range(counts.subnets_per_vnet): + resource_list.append(f" subnet{i}_{j}") + for i in range(counts.nsgs): + resource_list.append(f" nsg{i}") + for i in range(counts.public_ips): + resource_list.append(f" pip{i}") + + return f'''{AZURE_COPYRIGHT} +import "@azure/azure.pkl" +import "@azure/resources/resourcegroup.pkl" as rg +import "@azure/resources/virtualnetwork.pkl" as vnet +import "@azure/resources/subnet.pkl" as snet +import "@azure/network/networksecuritygroup.pkl" as nsg +import "@azure/network/publicipaddress.pkl" as pip + +class AzureNetworking {{ + prefix: String +{"".join(rg_blocks)} +{"".join(vnet_blocks)} +{"".join(subnet_blocks)} +{"".join(nsg_blocks)} +{"".join(pip_blocks)} + + // Expose resource groups for other modules + resourceGroups: Listing = new {{ +{chr(10).join(f" rg{i}" for i in range(counts.resource_groups))} + }} + + // Expose VNets for other modules + vnets: Listing = new {{ +{chr(10).join(f" vnet{i}" for i in range(counts.virtual_networks))} + }} + + // Expose subnets for other modules + subnets: Listing = new {{ +{chr(10).join(f" subnet{i}_0" for i in range(counts.virtual_networks))} + }} + + // Expose NSGs for other modules + nsgs: Listing = new {{ +{chr(10).join(f" nsg{i}" for i in range(counts.nsgs))} + }} + + // Expose public IPs for other modules + publicIps: Listing = new {{ +{chr(10).join(f" pip{i}" for i in range(counts.public_ips))} + }} + + resources: Listing = new {{ +{chr(10).join(resource_list)} + }} +}} +''' + + +def generate_azure_compute_pkl(counts: AzureResourceCounts, location: str) -> str: + """Generate Azure compute.pkl with VMs and NICs.""" + + nic_blocks = [] + vm_blocks = [] + + # Network interfaces + for i in range(counts.network_interfaces): + rg_idx = i % counts.resource_groups + vnet_idx = i % counts.virtual_networks + nsg_idx = i % counts.nsgs + pip_idx = i % counts.public_ips if i < counts.public_ips else -1 + + ip_config = f''' + new nic.IPConfiguration {{ + name = "ipconfig1" + subnet = networking.subnets.toList()[{vnet_idx}].res.id + privateIPAllocationMethod = "Dynamic" + primary = true''' + if pip_idx >= 0: + ip_config += f''' + publicIPAddress = networking.publicIps.toList()[{pip_idx}].res.id''' + ip_config += ''' + }''' + + nic_blocks.append(f''' + hidden nic{i}: nic.NetworkInterface = new {{ + label = "\\(prefix)-nic-{i}" + name = "\\(prefix)-nic-{i}" + location = "{location}" + resourceGroupName = networking.resourceGroups.toList()[{rg_idx}].res.name + ipConfigurations {{{ip_config} + }} + networkSecurityGroup = networking.nsgs.toList()[{nsg_idx}].res.id + tags {{ + new azure.Tag {{ key = "Environment"; value = "stress-test" }} + new azure.Tag {{ key = "ManagedBy"; value = "formae" }} + }} + }}''') + + # Virtual machines + for i in range(counts.virtual_machines): + rg_idx = i % counts.resource_groups + nic_idx = i % counts.network_interfaces + + vm_blocks.append(f''' + hidden vm{i}: vm.VirtualMachine = new {{ + label = "\\(prefix)-vm-{i}" + name = "\\(prefix)-vm-{i}" + location = "{location}" + resourceGroupName = networking.resourceGroups.toList()[{rg_idx}].res.name + vmSize = "Standard_B1s" + networkInterfaces {{ + new vm.NetworkInterfaceReference {{ + id = nic{nic_idx}.res.id + primary = true + }} + }} + imageReference = new vm.ImageReference {{ + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts-gen2" + version = "latest" + }} + osDisk = new vm.OSDisk {{ + createOption = "FromImage" + managedDisk = new vm.ManagedDiskParameters {{ + storageAccountType = "Standard_LRS" + }} + caching = "ReadWrite" + deleteOption = "Delete" + }} + adminUsername = "azureadmin" + linuxConfiguration = new vm.LinuxConfiguration {{ + disablePasswordAuthentication = true + provisionVMAgent = true + ssh = new vm.SSHConfiguration {{ + publicKeys {{ + new vm.SSHPublicKey {{ + path = "/home/azureadmin/.ssh/authorized_keys" + keyData = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+PLACEHOLDER stress-test-key" + }} + }} + }} + }} + tags {{ + new azure.Tag {{ key = "Environment"; value = "stress-test" }} + new azure.Tag {{ key = "ManagedBy"; value = "formae" }} + }} + }}''') + + resource_list = [] + for i in range(counts.network_interfaces): + resource_list.append(f" nic{i}") + for i in range(counts.virtual_machines): + resource_list.append(f" vm{i}") + + return f'''{AZURE_COPYRIGHT} +import "@azure/azure.pkl" +import "@azure/network/networkinterface.pkl" as nic +import "@azure/compute/virtualmachine.pkl" as vm + +import "./networking.pkl" as networkingModule + +class AzureCompute {{ + prefix: String + networking: networkingModule.AzureNetworking +{"".join(nic_blocks)} +{"".join(vm_blocks)} + + resources: Listing = new {{ +{chr(10).join(resource_list)} + }} +}} +''' + + +def generate_azure_storage_pkl(counts: AzureResourceCounts, location: str) -> str: + """Generate Azure storage.pkl with storage accounts.""" + + sa_blocks = [] + for i in range(counts.storage_accounts): + rg_idx = i % counts.resource_groups + # Storage account names: lowercase, no hyphens, 3-24 chars + sa_blocks.append(f''' + hidden sa{i}: sa.StorageAccount = new {{ + label = "\\(prefix)-sa-{i}" + name = "\\(safeName)sa{i}" + location = "{location}" + resourceGroupName = networking.resourceGroups.toList()[{rg_idx}].res.name + kind = "StorageV2" + sku = new sa.StorageAccountSKU {{ name = "Standard_LRS" }} + enableHttpsTrafficOnly = true + minimumTlsVersion = "TLS1_2" + allowBlobPublicAccess = false + tags {{ + new azure.Tag {{ key = "Environment"; value = "stress-test" }} + new azure.Tag {{ key = "ManagedBy"; value = "formae" }} + }} + }}''') + + resource_list = [] + for i in range(counts.storage_accounts): + resource_list.append(f" sa{i}") + + return f'''{AZURE_COPYRIGHT} +import "@azure/azure.pkl" +import "@azure/storage/storageaccount.pkl" as sa + +import "./networking.pkl" as networkingModule + +class AzureStorage {{ + prefix: String + // safeName: name with hyphens removed and truncated for storage account naming (max 24 chars) + safeName: String + networking: networkingModule.AzureNetworking +{"".join(sa_blocks)} + + resources: Listing = new {{ +{chr(10).join(resource_list)} + }} +}} +''' + + +def generate_azure_iam_pkl(counts: AzureResourceCounts, location: str) -> str: + """Generate Azure iam.pkl with managed identities.""" + + mi_blocks = [] + for i in range(counts.managed_identities): + rg_idx = i % counts.resource_groups + mi_blocks.append(f''' + hidden identity{i}: identity.UserAssignedIdentity = new {{ + label = "\\(prefix)-identity-{i}" + name = "\\(prefix)-identity-{i}" + location = "{location}" + resourceGroupName = networking.resourceGroups.toList()[{rg_idx}].res.name + tags {{ + new azure.Tag {{ key = "Environment"; value = "stress-test" }} + new azure.Tag {{ key = "ManagedBy"; value = "formae" }} + }} + }}''') + + resource_list = [] + for i in range(counts.managed_identities): + resource_list.append(f" identity{i}") + + return f'''{AZURE_COPYRIGHT} +import "@azure/azure.pkl" +import "@azure/managedidentity/userassignedidentity.pkl" as identity + +import "./networking.pkl" as networkingModule + +class AzureIAM {{ + prefix: String + networking: networkingModule.AzureNetworking +{"".join(mi_blocks)} + + resources: Listing = new {{ +{chr(10).join(resource_list)} + }} +}} +''' + + +def generate_azure_security_pkl(counts: AzureResourceCounts, location: str) -> str: + """Generate Azure security.pkl with key vaults and container registries.""" + + kv_blocks = [] + cr_blocks = [] + + for i in range(counts.key_vaults): + rg_idx = i % counts.resource_groups + kv_blocks.append(f''' + hidden kv{i}: kv.Vault = new {{ + label = "\\(prefix)-kv-{i}" + name = "\\(kvPrefix)-kv-{i}" + location = "{location}" + resourceGroupName = networking.resourceGroups.toList()[{rg_idx}].res.name + tenantId = "00000000-0000-0000-0000-000000000000" + sku = new kv.VaultSKU {{ family = "A"; name = "standard" }} + enableSoftDelete = false + tags {{ + new azure.Tag {{ key = "Environment"; value = "stress-test" }} + new azure.Tag {{ key = "ManagedBy"; value = "formae" }} + }} + }}''') + + for i in range(counts.container_registries): + rg_idx = i % counts.resource_groups + # Registry names: alphanumeric only, 5-50 chars + cr_blocks.append(f''' + hidden cr{i}: cr.Registry = new {{ + label = "\\(prefix)-cr-{i}" + name = "\\(safeName)cr{i}" + location = "{location}" + resourceGroupName = networking.resourceGroups.toList()[{rg_idx}].res.name + sku = new cr.RegistrySKU {{ name = "Basic" }} + adminUserEnabled = false + tags {{ + new azure.Tag {{ key = "Environment"; value = "stress-test" }} + new azure.Tag {{ key = "ManagedBy"; value = "formae" }} + }} + }}''') + + resource_list = [] + for i in range(counts.key_vaults): + resource_list.append(f" kv{i}") + for i in range(counts.container_registries): + resource_list.append(f" cr{i}") + + return f'''{AZURE_COPYRIGHT} +import "@azure/azure.pkl" +import "@azure/keyvault/vault.pkl" as kv +import "@azure/containerregistry/registry.pkl" as cr + +import "./networking.pkl" as networkingModule + +class AzureSecurity {{ + prefix: String + // kvPrefix: truncated name for key vault naming (max 24 chars total with suffix) + kvPrefix: String + // safeName: name with hyphens removed for registry naming + safeName: String + networking: networkingModule.AzureNetworking +{"".join(kv_blocks)} +{"".join(cr_blocks)} + + resources: Listing = new {{ +{chr(10).join(resource_list)} + }} +}} +''' + + +def generate_azure_database_pkl(counts: AzureResourceCounts, location: str) -> str: + """Generate Azure database.pkl with PostgreSQL flexible servers.""" + + pg_blocks = [] + for i in range(counts.postgres_servers): + rg_idx = i % counts.resource_groups + pg_blocks.append(f''' + hidden pg{i}: pg.FlexibleServer = new {{ + label = "\\(prefix)-pg-{i}" + name = "\\(prefix)-pg-{i}" + location = "{location}" + resourceGroupName = networking.resourceGroups.toList()[{rg_idx}].res.name + sku = new pg.SKU {{ name = "Standard_B1ms"; tier = "Burstable" }} + version = "15" + administratorLogin = "pgadmin" + administratorLoginPassword = "StressTest123!" + storage = new pg.Storage {{ storageSizeGB = 32; autoGrow = "Disabled" }} + backup = new pg.Backup {{ backupRetentionDays = 7; geoRedundantBackup = "Disabled" }} + highAvailability = new pg.HighAvailability {{ mode = "Disabled" }} + authConfig = new pg.AuthConfig {{ passwordAuth = "Enabled"; activeDirectoryAuth = "Disabled" }} + tags {{ + new azure.Tag {{ key = "Environment"; value = "stress-test" }} + new azure.Tag {{ key = "ManagedBy"; value = "formae" }} + }} + }}''') + + resource_list = [] + for i in range(counts.postgres_servers): + resource_list.append(f" pg{i}") + + return f'''{AZURE_COPYRIGHT} +import "@azure/azure.pkl" +import "@azure/dbforpostgresql/flexibleserver.pkl" as pg + +import "./networking.pkl" as networkingModule + +class AzureDatabase {{ + prefix: String + networking: networkingModule.AzureNetworking +{"".join(pg_blocks)} + + resources: Listing = new {{ +{chr(10).join(resource_list)} + }} +}} +''' + + +def generate_azure_main_pkl(env_id: str, counts: AzureResourceCounts) -> str: + """Generate azure/main.pkl that ties Azure resources together.""" + + has_compute = counts.virtual_machines > 0 or counts.network_interfaces > 0 + has_database = counts.postgres_servers > 0 + + imports = [ + 'import "./networking.pkl" as networkingModule', + 'import "./storage.pkl" as storageModule', + 'import "./iam.pkl" as iamModule', + 'import "./security.pkl" as securityModule', + ] + if has_compute: + imports.append('import "./compute.pkl" as computeModule') + if has_database: + imports.append('import "./database.pkl" as databaseModule') + + locals_block = f''' +local _networking = new networkingModule.AzureNetworking {{ + prefix = vars.envId +}} + +local _storage = new storageModule.AzureStorage {{ + prefix = vars.envId + safeName = vars.azureSafeName + networking = _networking +}} + +local _iam = new iamModule.AzureIAM {{ + prefix = vars.envId + networking = _networking +}} + +local _security = new securityModule.AzureSecurity {{ + prefix = vars.envId + kvPrefix = vars.azureKvPrefix + safeName = vars.azureSafeName + networking = _networking +}}''' + + if has_compute: + locals_block += f''' + +local _compute = new computeModule.AzureCompute {{ + prefix = vars.envId + networking = _networking +}}''' + + if has_database: + locals_block += f''' + +local _database = new databaseModule.AzureDatabase {{ + prefix = vars.envId + networking = _networking +}}''' + + spread_items = [ + ' ..._networking.resources', + ' ..._storage.resources', + ' ..._iam.resources', + ' ..._security.resources', + ] + if has_compute: + spread_items.append(' ..._compute.resources') + if has_database: + spread_items.append(' ..._database.resources') + + return f'''{AZURE_COPYRIGHT} +amends "@formae/forma.pkl" +import "@formae/formae.pkl" + +import "../vars.pkl" +{chr(10).join(imports)} +{locals_block} + +forma {{ + vars.azureStack + vars.azureTarget + +{chr(10).join(spread_items)} +}} +''' + + +# ============================================================================ +# GCP PKL Generators +# ============================================================================ + +GCP_COPYRIGHT = '''/* + * Stress Test Environment - Auto-generated (GCP) + * Generated by generate-stress-test-env.py + * + * WARNING: This environment is for testing only. Resources will incur GCP costs. + */ +''' + + +def generate_gcp_networking_pkl(counts: GCPResourceCounts, region: str) -> str: + """Generate GCP networking.pkl with networks, subnetworks, firewalls, and addresses.""" + + network_blocks = [] + subnet_blocks = [] + firewall_blocks = [] + address_blocks = [] + + # Networks + for i in range(counts.compute_networks): + network_blocks.append(f''' + hidden net{i}: network.Network = new {{ + label = "\\(prefix)-net-{i}" + name = "\\(prefix)-net-{i}" + autoCreateSubnetworks = false + routingConfig = new network.RoutingConfig {{ + routingMode = "REGIONAL" + }} + }}''') + + # Subnetworks + for i in range(counts.compute_subnetworks): + net_idx = i % counts.compute_networks + subnet_blocks.append(f''' + hidden subnet{i}: subnetwork.Subnetwork = new {{ + label = "\\(prefix)-subnet-{i}" + name = "\\(prefix)-subnet-{i}" + network = net{net_idx}.res.selfLink + region = "{region}" + ipCidrRange = "10.{i // 256}.{i % 256}.0/24" + privateIpGoogleAccess = true + }}''') + + # Firewalls + protocols = ["tcp", "udp", "tcp"] + ports = [["80", "443"], ["53"], ["22"]] + for i in range(counts.compute_firewalls): + net_idx = i % counts.compute_networks + proto = protocols[i % len(protocols)] + port_list = ports[i % len(ports)] + port_entries = "; ".join(f'"{p}"' for p in port_list) + firewall_blocks.append(f''' + hidden fw{i}: firewall.Firewall = new {{ + label = "\\(prefix)-fw-{i}" + name = "\\(prefix)-fw-{i}" + network = net{net_idx}.res.selfLink + direction = "INGRESS" + sourceRanges {{ + "10.0.0.0/8" + }} + allowed {{ + new {{ + protocol = "{proto}" + ports {{ {port_entries} }} + }} + }} + }}''') + + # Addresses + for i in range(counts.compute_addresses): + address_blocks.append(f''' + hidden addr{i}: address.Address = new {{ + label = "\\(prefix)-addr-{i}" + name = "\\(prefix)-addr-{i}" + region = "{region}" + addressType = "EXTERNAL" + }}''') + + resource_list = [] + for i in range(counts.compute_networks): + resource_list.append(f" net{i}") + for i in range(counts.compute_subnetworks): + resource_list.append(f" subnet{i}") + for i in range(counts.compute_firewalls): + resource_list.append(f" fw{i}") + for i in range(counts.compute_addresses): + resource_list.append(f" addr{i}") + + return f'''{GCP_COPYRIGHT} +import "@gcp/compute/network.pkl" +import "@gcp/compute/subnetwork.pkl" +import "@gcp/compute/firewall.pkl" +import "@gcp/compute/address.pkl" + +class GCPNetworking {{ + prefix: String +{"".join(network_blocks)} +{"".join(subnet_blocks)} +{"".join(firewall_blocks)} +{"".join(address_blocks)} + + // Expose networks for other modules + networks: Listing = new {{ +{chr(10).join(f" net{i}" for i in range(counts.compute_networks))} + }} + + // Expose subnets for other modules + subnets: Listing = new {{ +{chr(10).join(f" subnet{i}" for i in range(counts.compute_subnetworks))} + }} + + resources: Listing = new {{ +{chr(10).join(resource_list)} + }} +}} +''' + + +def generate_gcp_compute_pkl(counts: GCPResourceCounts, region: str, project: str) -> str: + """Generate GCP compute.pkl with instances and disks.""" + + disk_blocks = [] + instance_blocks = [] + + zone = f"{region}-b" + + # Disks + for i in range(counts.compute_disks): + disk_blocks.append(f''' + hidden disk{i}: disk.Disk = new {{ + label = "\\(prefix)-disk-{i}" + project = "{project}" + zone = "{zone}" + name = "\\(prefix)-disk-{i}" + sizeGb = {20 + (i % 5) * 10} + type = "{["pd-balanced", "pd-standard"][i % 2]}" + labels {{ + ["environment"] = "stress-test" + ["managed-by"] = "formae" + }} + }}''') + + # Instances + machine_types = ["e2-micro", "e2-small"] + for i in range(counts.compute_instances): + net_idx = i % counts.compute_networks + subnet_idx = i % max(1, counts.compute_subnetworks) + machine_type = machine_types[i % len(machine_types)] + + instance_blocks.append(f''' + hidden instance{i}: instance.Instance = new {{ + label = "\\(prefix)-instance-{i}" + project = "{project}" + zone = "{zone}" + name = "\\(prefix)-instance-{i}" + machineType = "{machine_type}" + networkInterfaces {{ + new {{ + network = networking.networks.toList()[{net_idx}].res.selfLink + subnetwork = networking.subnets.toList()[{subnet_idx}].res.selfLink + }} + }} + labels {{ + ["environment"] = "stress-test" + ["managed-by"] = "formae" + }} + }}''') + # NOTE: Boot disk via initializeParams is not yet supported in the GCP plugin. + # Instances are created without explicit boot disks for now. + + resource_list = [] + for i in range(counts.compute_disks): + resource_list.append(f" disk{i}") + for i in range(counts.compute_instances): + resource_list.append(f" instance{i}") + + return f'''{GCP_COPYRIGHT} +import "@gcp/compute/disk.pkl" +import "@gcp/compute/instance.pkl" + +import "./networking.pkl" as networkingModule + +class GCPCompute {{ + prefix: String + networking: networkingModule.GCPNetworking +{"".join(disk_blocks)} +{"".join(instance_blocks)} + + resources: Listing = new {{ +{chr(10).join(resource_list)} + }} +}} +''' + + +def generate_gcp_storage_pkl(counts: GCPResourceCounts, region: str) -> str: + """Generate GCP storage.pkl with storage buckets.""" + + bucket_blocks = [] + for i in range(counts.storage_buckets): + bucket_blocks.append(f''' + hidden bucket{i}: bucket.Bucket = new {{ + label = "\\(prefix)-bucket-{i}" + name = "\\(prefix)-bucket-{i}" + location = "{region}" + storageClass = "STANDARD" + versioning = new bucket.Versioning {{ enabled = true }} + }}''') + + resource_list = [] + for i in range(counts.storage_buckets): + resource_list.append(f" bucket{i}") + + return f'''{GCP_COPYRIGHT} +import "@gcp/storage/bucket.pkl" + +class GCPStorage {{ + prefix: String +{"".join(bucket_blocks)} + + resources: Listing = new {{ +{chr(10).join(resource_list)} + }} +}} +''' + + +def generate_gcp_database_pkl(counts: GCPResourceCounts, region: str, project: str) -> str: + """Generate GCP database.pkl with Cloud SQL instances and BigQuery datasets.""" + + sql_blocks = [] + bq_blocks = [] + + for i in range(counts.sql_instances): + sql_blocks.append(f''' + hidden sqlInstance{i}: database.DatabaseInstance = new {{ + label = "\\(prefix)-sql-{i}" + project = "{project}" + name = "\\(prefix)-sql-{i}" + databaseVersion = "POSTGRES_15" + region = "{region}" + settings = new database.Settings {{ + tier = "db-f1-micro" + availabilityType = "ZONAL" + dataDiskSizeGb = 10 + dataDiskType = "PD_SSD" + ipConfiguration = new database.IpConfiguration {{ + ipv4Enabled = true + requireSsl = false + }} + userLabels = new Mapping {{ + ["environment"] = "stress-test" + ["managed-by"] = "formae" + }} + }} + }}''') + + for i in range(counts.bigquery_datasets): + bq_blocks.append(f''' + hidden bqDataset{i}: dataset.Dataset = new {{ + label = "\\(prefix)-bq-{i}" + project = "{project}" + datasetId = "\\(bqSafeName)_bq_{i}" + location = "{region}" + description = "Stress test dataset {i}" + }}''') + + resource_list = [] + for i in range(counts.sql_instances): + resource_list.append(f" sqlInstance{i}") + for i in range(counts.bigquery_datasets): + resource_list.append(f" bqDataset{i}") + + return f'''{GCP_COPYRIGHT} +import "@gcp/sql/database.pkl" +import "@gcp/bigquery/dataset.pkl" + +class GCPDatabase {{ + prefix: String + // bqSafeName: name with hyphens replaced by underscores for BigQuery dataset IDs + bqSafeName: String +{"".join(sql_blocks)} +{"".join(bq_blocks)} + + resources: Listing = new {{ +{chr(10).join(resource_list)} + }} +}} +''' + + +def generate_gcp_main_pkl(env_id: str, counts: GCPResourceCounts) -> str: + """Generate gcp/main.pkl that ties GCP resources together.""" + + has_compute = counts.compute_instances > 0 or counts.compute_disks > 0 + has_database = counts.sql_instances > 0 or counts.bigquery_datasets > 0 + + imports = [ + 'import "./networking.pkl" as networkingModule', + 'import "./storage.pkl" as storageModule', + ] + if has_compute: + imports.append('import "./compute.pkl" as computeModule') + if has_database: + imports.append('import "./database.pkl" as databaseModule') + + locals_block = f''' +local _networking = new networkingModule.GCPNetworking {{ + prefix = vars.envId +}} + +local _storage = new storageModule.GCPStorage {{ + prefix = vars.envId +}}''' + + if has_compute: + locals_block += f''' + +local _compute = new computeModule.GCPCompute {{ + prefix = vars.envId + networking = _networking +}}''' + + if has_database: + locals_block += f''' + +local _database = new databaseModule.GCPDatabase {{ + prefix = vars.envId + bqSafeName = vars.gcpBqSafeName +}}''' + + spread_items = [ + ' ..._networking.resources', + ' ..._storage.resources', + ] + if has_compute: + spread_items.append(' ..._compute.resources') + if has_database: + spread_items.append(' ..._database.resources') + + return f'''{GCP_COPYRIGHT} +amends "@formae/forma.pkl" +import "@formae/formae.pkl" + +import "../vars.pkl" +{chr(10).join(imports)} +{locals_block} + +forma {{ + vars.gcpStack + vars.gcpTarget + +{chr(10).join(spread_items)} +}} +''' + + +# ============================================================================ +# Multi-cloud vars.pkl and main.pkl generators +# ============================================================================ + + +def generate_multicloud_vars_pkl( + env_id: str, + run_id: str, + clouds: list, + region: str, + stack_name: str, + azure_subscription: str = None, + azure_location: str = "eastus", + gcp_project: str = None, + gcp_region: str = "us-central1", +) -> str: + """Generate vars.pkl with environment configuration for all enabled clouds.""" + + content = f'''{COPYRIGHT} +import "@formae/formae.pkl" +''' + + if "aws" in clouds: + content += 'import "@aws/aws.pkl"\n' + if "azure" in clouds: + content += 'import "@azure/azure.pkl"\n' + if "gcp" in clouds: + content += 'import "@gcp/gcp.pkl"\n' + + content += f''' +envId = "{env_id}" +runId = "{run_id}" +''' + + if "aws" in clouds: + content += f''' +// AWS Configuration +awsRegion = "{region}" + +awsStack: formae.Stack = new {{ + label = "{stack_name}-aws" + description = "Stress test environment {env_id} - AWS" +}} + +awsTarget: formae.Target = new {{ + label = "stress-test-aws" + config = new aws.Config {{ + region = awsRegion + }} +}} + +// Helper for unique naming +local function uniqueName(base: String): String = "\\(envId)-\\(base)" +''' + + if "azure" in clouds: + content += f''' +// Azure Configuration +azureLocation = "{azure_location}" +azureSubscriptionId = "{azure_subscription}" + +// Safe name for Azure resources that don't allow hyphens (storage accounts, registries) +azureSafeName = envId.replaceAll("-", "").take(14) +// Prefix for key vault names (max 24 chars total with suffix) +azureKvPrefix = envId.take(16) + +azureStack: formae.Stack = new {{ + label = "{stack_name}-azure" + description = "Stress test environment {env_id} - Azure" +}} + +azureTarget: formae.Target = new {{ + label = "stress-test-azure" + config = new azure.Config {{ + subscriptionId = azureSubscriptionId + }} +}} +''' + + if "gcp" in clouds: + content += f''' +// GCP Configuration +gcpProject = "{gcp_project}" +gcpRegion = "{gcp_region}" + +// Safe name for BigQuery dataset IDs (no hyphens allowed) +gcpBqSafeName = envId.replaceAll("-", "_") + +gcpStack: formae.Stack = new {{ + label = "{stack_name}-gcp" + description = "Stress test environment {env_id} - GCP" +}} + +gcpTarget: formae.Target = new {{ + label = "stress-test-gcp" + config = new gcp.Config {{ + project = gcpProject + region = gcpRegion + location = gcpRegion + }} +}} +''' + + return content + + +def generate_multicloud_main_pkl( + env_id: str, + clouds: list, + aws_counts: ResourceCounts = None, + azure_counts: AzureResourceCounts = None, + gcp_counts: GCPResourceCounts = None, +) -> str: + """Generate main.pkl that imports all cloud-specific files.""" + + imports = ['import "./vars.pkl"'] + + if "aws" in clouds: + imports.append('import "./aws/main.pkl" as awsForma') + if "azure" in clouds: + imports.append('import "./azure/main.pkl" as azureForma') + if "gcp" in clouds: + imports.append('import "./gcp/main.pkl" as gcpForma') + + # Build description + total = 0 + cloud_details = [] + if "aws" in clouds and aws_counts: + total += aws_counts.total + cloud_details.append(f" - AWS: ~{aws_counts.total} resources") + if "azure" in clouds and azure_counts: + total += azure_counts.total + cloud_details.append(f" - Azure: ~{azure_counts.total} resources") + if "gcp" in clouds and gcp_counts: + total += gcp_counts.total + cloud_details.append(f" - GCP: ~{gcp_counts.total} resources") + + # Each cloud has its own main.pkl that is a standalone forma file. + # Apply each one separately: formae apply aws/main.pkl, azure/main.pkl, gcp/main.pkl + # This is a documentation/index file listing what was generated. + cloud_paths = [] + if "aws" in clouds: + cloud_paths.append("aws/main.pkl") + if "azure" in clouds: + cloud_paths.append("azure/main.pkl") + if "gcp" in clouds: + cloud_paths.append("gcp/main.pkl") + + apply_cmds = chr(10).join(f"# formae apply {p} --yes" for p in cloud_paths) + destroy_cmds = chr(10).join(f"# formae destroy {p} --yes" for p in cloud_paths) + + return f'''# Multi-Cloud Stress Test Environment: {env_id} +# +# This environment contains approximately {total} resources across +# {", ".join(c.upper() for c in clouds)}. +# +{chr(10).join(cloud_details)} +# +# Each cloud has its own forma file. Apply them separately: +# +{apply_cmds} +# +# To destroy: +# +{destroy_cmds} +''' + + +def generate_multicloud_pkl_project(formae_path: str, clouds: list) -> str: + """Generate PklProject file for multi-cloud environments.""" + # Plugin repos live alongside the formae repo as siblings + parent_dir = os.path.dirname(formae_path) + deps = [f' ["formae"] = import("{formae_path}/internal/schema/pkl/schema/PklProject")'] + + if "aws" in clouds: + deps.append(f' ["aws"] = import("{parent_dir}/formae-plugin-aws/schema/pkl/PklProject")') + if "azure" in clouds: + deps.append(f' ["azure"] = import("{parent_dir}/formae-plugin-azure/schema/pkl/PklProject")') + if "gcp" in clouds: + deps.append(f' ["gcp"] = import("{parent_dir}/formae-plugin-gcp/schema/pkl/PklProject")') + + return f'''amends "pkl:Project" + +dependencies {{ +{chr(10).join(deps)} +}} +''' + + +def generate_main_pkl(env_id: str, stack_name: str, counts: ResourceCounts, + stack_var: str = "stack", target_var: str = "target", + region_var: str = "_region") -> str: """Generate main.pkl that ties everything together.""" return f'''{COPYRIGHT} @@ -1585,7 +2911,7 @@ def generate_main_pkl(env_id: str, stack_name: str, counts: ResourceCounts) -> s }} local _name = vars.envId -local _region = vars._region +local _region = vars.{region_var} local _networking = new networkingModule.Networking {{ name = _name @@ -1646,8 +2972,8 @@ def generate_main_pkl(env_id: str, stack_name: str, counts: ResourceCounts) -> s }} forma {{ - vars.stack - vars.target + vars.{stack_var} + vars.{target_var} ..._networking.resources ..._securityGroups.resources @@ -1667,11 +2993,12 @@ def generate_main_pkl(env_id: str, stack_name: str, counts: ResourceCounts) -> s def generate_pkl_project(formae_path: str) -> str: """Generate PklProject file.""" + parent_dir = os.path.dirname(formae_path) return f'''amends "pkl:Project" dependencies {{ ["formae"] = import("{formae_path}/internal/schema/pkl/schema/PklProject") - ["aws"] = import("{formae_path}/plugins/aws/schema/pkl/PklProject") + ["aws"] = import("{parent_dir}/formae-plugin-aws/schema/pkl/PklProject") }} ''' @@ -2924,15 +4251,23 @@ def generate_cfn_readme(env_id: str, region: str, counts: ResourceCounts) -> str def main(): parser = argparse.ArgumentParser( - description="Generate a large-scale AWS infrastructure environment for performance testing formae.", + description="Generate a large-scale multi-cloud infrastructure environment for performance testing formae.", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - # Generate PKL files for formae + # Generate PKL files for formae (AWS only - default) %(prog)s --count 100 --region us-east-1 %(prog)s --count 1000 --region eu-west-1 --output ./perf-test - # Generate CloudFormation templates for discovery testing + # Generate multi-cloud PKL files + %(prog)s --count 50 --clouds aws,azure,gcp --region us-east-2 \\ + --subscription-azure --project-gcp + + # Use scale profiles + %(prog)s --scale-profile medium --clouds aws,azure,gcp \\ + --subscription-azure --project-gcp + + # Generate CloudFormation templates for discovery testing (AWS only) %(prog)s --format cloudformation --count 100 --region us-east-1 %(prog)s -f cfn -c 500 -r eu-west-1 -o ./discovery-test """ @@ -2950,7 +4285,7 @@ def main(): "--count", "-c", type=int, default=100, - help="Target number of resources to generate (default: 100)" + help="Target number of resources to generate per cloud (default: 100)" ) parser.add_argument( @@ -2985,73 +4320,207 @@ def main(): type=str, choices=["default", "quota-safe"], default="default", - help="Resource profile: 'default' for full resources, 'quota-safe' to avoid resources with tight quotas (EC2, EIP, VPC, etc.)" + help="AWS resource profile: 'default' for full resources, 'quota-safe' to avoid resources with tight quotas (EC2, EIP, VPC, etc.)" + ) + + # Multi-cloud arguments + parser.add_argument( + "--clouds", + type=str, + default="aws", + help="Comma-separated list of clouds: aws, azure, gcp (default: aws)" + ) + + parser.add_argument( + "--scale-profile", + type=str, + choices=["small", "medium", "large", "xl"], + default=None, + help="Scale profile overrides --count with per-cloud splits: small=500, medium=5000, large=20000, xl=50000" + ) + + parser.add_argument( + "--subscription-azure", + type=str, + default=None, + help="Azure subscription ID (required when azure is in --clouds)" + ) + + parser.add_argument( + "--project-gcp", + type=str, + default=None, + help="GCP project ID (required when gcp is in --clouds)" + ) + + parser.add_argument( + "--region-gcp", + type=str, + default="us-central1", + help="GCP region (default: us-central1)" + ) + + parser.add_argument( + "--run-id", + type=str, + default=None, + help="Unique run ID for tagging (default: auto-generated UUID)" ) args = parser.parse_args() + # Parse clouds + clouds = [c.strip().lower() for c in args.clouds.split(",")] + for c in clouds: + if c not in ("aws", "azure", "gcp"): + print(f"Error: unknown cloud '{c}'. Must be aws, azure, or gcp.", file=sys.stderr) + sys.exit(1) + + # Validate required arguments for each cloud + if "azure" in clouds and not args.subscription_azure: + print("Error: --subscription-azure is required when azure is in --clouds", file=sys.stderr) + sys.exit(1) + if "gcp" in clouds and not args.project_gcp: + print("Error: --project-gcp is required when gcp is in --clouds", file=sys.stderr) + sys.exit(1) + + # CloudFormation is AWS-only + output_format = args.format if args.format != "cfn" else "cloudformation" + if output_format == "cloudformation" and (len(clouds) > 1 or clouds[0] != "aws"): + print("Error: CloudFormation format is only supported for AWS-only generation", file=sys.stderr) + sys.exit(1) + + # Determine resource counts + is_multicloud = len(clouds) > 1 or clouds != ["aws"] + run_id = args.run_id or uuid.uuid4().hex[:12] + + if args.scale_profile: + aws_count, azure_count, gcp_count = SCALE_PROFILES[args.scale_profile] + else: + aws_count = args.count + azure_count = args.count + gcp_count = args.count + # Generate unique environment ID env_id = f"perf-{uuid.uuid4().hex[:8]}" stack_name = f"perf-test-{env_id}" - # Calculate resource counts based on profile - if args.profile == "quota-safe": - counts = calculate_counts_quota_safe(args.count) - else: - counts = calculate_counts(args.count) + # Calculate resource counts for each cloud + aws_counts = None + azure_counts = None + gcp_counts = None + + if "aws" in clouds: + if args.profile == "quota-safe": + aws_counts = calculate_counts_quota_safe(aws_count) + else: + aws_counts = calculate_counts(aws_count) + + if "azure" in clouds: + azure_counts = calculate_azure_counts(azure_count) + + if "gcp" in clouds: + gcp_counts = calculate_gcp_counts(gcp_count) if args.dry_run: - print(f"Profile: {args.profile}") - print(f"Target count: {args.count}") - print(f"Calculated total: {counts.total}") - print(f"\nResource breakdown:") - print(f"\n Networking (capped to stay within quotas):") - print(f" VPCs: {counts.vpcs} (hard cap at 5)") - print(f" Subnets: {counts.vpcs * counts.subnets_per_vpc} ({counts.subnets_per_vpc} per VPC)") - print(f" Route Tables: {counts.route_tables}") - print(f" Routes: {counts.routes}") - print(f" Internet Gateways: {counts.igws} (1 per VPC)") - print(f" NAT Gateways: {counts.nat_gws} (capped at 5)") - print(f" Elastic IPs: {counts.eips}") - print(f" Security Groups: {counts.security_groups}") - print(f" SG Ingress Rules: {counts.sg_ingress_rules}") - print(f" SG Egress Rules: {counts.sg_egress_rules}") - print(f" VPC Endpoints: {counts.vpc_endpoints}") - print(f"\n Compute:") - print(f" EC2 Instances: {counts.ec2_instances}") - print(f" EBS Volumes: {counts.ebs_volumes}") - print(f" Launch Templates: {counts.launch_templates}") - print(f"\n IAM & Security:") - print(f" IAM Roles: {counts.iam_roles}") - print(f" IAM Policies: {counts.iam_policies}") - print(f" Instance Profiles: {counts.instance_profiles}") - print(f" KMS Keys: {counts.kms_keys}") - print(f" KMS Aliases: {counts.kms_aliases}") - print(f" Secrets: {counts.secrets}") - print(f"\n Storage:") - print(f" S3 Buckets: {counts.s3_buckets}") - print(f" DynamoDB Tables: {counts.dynamodb_tables}") - print(f" EFS File Systems: {counts.efs_filesystems}") - print(f" EFS Mount Targets: {counts.efs_mount_targets}") - print(f"\n Observability:") - print(f" CloudWatch Log Groups: {counts.log_groups}") - print(f" SQS Queues: {counts.sqs_queues}") - print(f"\n Application:") - print(f" Lambda Functions: {counts.lambdas}") - print(f" ECR Repositories: {counts.ecr_repos}") - print(f"\n DNS & API Gateway:") - print(f" Route53 Hosted Zones: {counts.route53_zones}") - print(f" Route53 Records: {counts.route53_records}") - print(f" API Gateways: {counts.api_gateways}") - print(f" API Stages: {counts.api_stages}") - print(f"\n RDS:") - print(f" RDS Instances: {counts.rds_instances}") - print(f" DB Subnet Groups: {counts.db_subnet_groups}") - print(f" DB Parameter Groups: {counts.db_parameter_groups}") - print(f"\n Implicit Resources:") - print(f" VPC Gateway Attachments: {counts.vpcs}") - print(f" Subnet Route Table Associations: {counts.vpcs * counts.subnets_per_vpc}") - print(f" EBS Volume Attachments: ~{counts.ec2_instances * 2}") + print(f"Clouds: {', '.join(c.upper() for c in clouds)}") + if args.scale_profile: + print(f"Scale profile: {args.scale_profile}") + print(f"Run ID: {run_id}") + + if aws_counts: + print(f"\n{'=' * 40}") + print(f"AWS (target: {aws_count}, calculated: {aws_counts.total})") + print(f"{'=' * 40}") + print(f" Profile: {args.profile}") + print(f"\n Networking (capped to stay within quotas):") + print(f" VPCs: {aws_counts.vpcs} (hard cap at 5)") + print(f" Subnets: {aws_counts.vpcs * aws_counts.subnets_per_vpc} ({aws_counts.subnets_per_vpc} per VPC)") + print(f" Route Tables: {aws_counts.route_tables}") + print(f" Routes: {aws_counts.routes}") + print(f" Internet Gateways: {aws_counts.igws} (1 per VPC)") + print(f" NAT Gateways: {aws_counts.nat_gws} (capped at 5)") + print(f" Elastic IPs: {aws_counts.eips}") + print(f" Security Groups: {aws_counts.security_groups}") + print(f" SG Ingress Rules: {aws_counts.sg_ingress_rules}") + print(f" SG Egress Rules: {aws_counts.sg_egress_rules}") + print(f" VPC Endpoints: {aws_counts.vpc_endpoints}") + print(f"\n Compute:") + print(f" EC2 Instances: {aws_counts.ec2_instances}") + print(f" EBS Volumes: {aws_counts.ebs_volumes}") + print(f" Launch Templates: {aws_counts.launch_templates}") + print(f"\n IAM & Security:") + print(f" IAM Roles: {aws_counts.iam_roles}") + print(f" IAM Policies: {aws_counts.iam_policies}") + print(f" Instance Profiles: {aws_counts.instance_profiles}") + print(f" KMS Keys: {aws_counts.kms_keys}") + print(f" KMS Aliases: {aws_counts.kms_aliases}") + print(f" Secrets: {aws_counts.secrets}") + print(f"\n Storage:") + print(f" S3 Buckets: {aws_counts.s3_buckets}") + print(f" DynamoDB Tables: {aws_counts.dynamodb_tables}") + print(f" EFS File Systems: {aws_counts.efs_filesystems}") + print(f" EFS Mount Targets: {aws_counts.efs_mount_targets}") + print(f"\n Observability:") + print(f" CloudWatch Log Groups: {aws_counts.log_groups}") + print(f" SQS Queues: {aws_counts.sqs_queues}") + print(f"\n Application:") + print(f" Lambda Functions: {aws_counts.lambdas}") + print(f" ECR Repositories: {aws_counts.ecr_repos}") + print(f"\n DNS & API Gateway:") + print(f" Route53 Hosted Zones: {aws_counts.route53_zones}") + print(f" Route53 Records: {aws_counts.route53_records}") + print(f" API Gateways: {aws_counts.api_gateways}") + print(f" API Stages: {aws_counts.api_stages}") + print(f"\n RDS:") + print(f" RDS Instances: {aws_counts.rds_instances}") + print(f" DB Subnet Groups: {aws_counts.db_subnet_groups}") + print(f" DB Parameter Groups: {aws_counts.db_parameter_groups}") + + if azure_counts: + print(f"\n{'=' * 40}") + print(f"Azure (target: {azure_count}, calculated: {azure_counts.total})") + print(f"{'=' * 40}") + print(f"\n Networking:") + print(f" Resource Groups: {azure_counts.resource_groups}") + print(f" Virtual Networks: {azure_counts.virtual_networks}") + print(f" Subnets: {azure_counts.virtual_networks * azure_counts.subnets_per_vnet}") + print(f" NSGs: {azure_counts.nsgs}") + print(f" Public IPs: {azure_counts.public_ips}") + print(f"\n Compute:") + print(f" Virtual Machines: {azure_counts.virtual_machines}") + print(f" Network Interfaces: {azure_counts.network_interfaces}") + print(f"\n Storage:") + print(f" Storage Accounts: {azure_counts.storage_accounts}") + print(f"\n IAM:") + print(f" Managed Identities: {azure_counts.managed_identities}") + print(f"\n Security:") + print(f" Key Vaults: {azure_counts.key_vaults}") + print(f" Container Registries: {azure_counts.container_registries}") + print(f"\n Database:") + print(f" PostgreSQL Flex Servers: {azure_counts.postgres_servers}") + + if gcp_counts: + print(f"\n{'=' * 40}") + print(f"GCP (target: {gcp_count}, calculated: {gcp_counts.total})") + print(f"{'=' * 40}") + print(f"\n Networking:") + print(f" Networks: {gcp_counts.compute_networks}") + print(f" Subnetworks: {gcp_counts.compute_subnetworks}") + print(f" Firewalls: {gcp_counts.compute_firewalls}") + print(f" Addresses: {gcp_counts.compute_addresses}") + print(f"\n Compute:") + print(f" Instances: {gcp_counts.compute_instances}") + print(f" Disks: {gcp_counts.compute_disks}") + print(f"\n Storage:") + print(f" Buckets: {gcp_counts.storage_buckets}") + print(f"\n Database:") + print(f" SQL Instances: {gcp_counts.sql_instances}") + print(f" BigQuery Datasets: {gcp_counts.bigquery_datasets}") + + grand_total = sum(c.total for c in [aws_counts, azure_counts, gcp_counts] if c) + print(f"\n{'=' * 40}") + print(f"Grand Total: {grand_total} resources") return # Determine output directory @@ -3071,175 +4540,380 @@ def main(): # Create output directory os.makedirs(output_dir, exist_ok=True) - # Normalize format - output_format = args.format if args.format != "cfn" else "cloudformation" - if output_format == "pkl": - # Generate PKL files for formae - files = { - "PklProject": generate_pkl_project(formae_path), - "vars.pkl": generate_vars_pkl(env_id, args.region, stack_name), - "networking.pkl": generate_networking_pkl(counts, args.region), - "security_groups.pkl": generate_security_groups_pkl(counts), - "compute.pkl": generate_compute_pkl(counts, args.region), - "iam.pkl": generate_iam_pkl(counts), - "kms.pkl": generate_kms_pkl(counts), - "storage.pkl": generate_storage_pkl(counts), - "observability.pkl": generate_observability_pkl(counts), - "secrets.pkl": generate_secrets_pkl(counts), - "application.pkl": generate_application_pkl(counts), - "route53.pkl": generate_route53_pkl(counts), - "api_gateway.pkl": generate_api_gateway_pkl(counts), - "rds.pkl": generate_rds_pkl(counts), - "main.pkl": generate_main_pkl(env_id, stack_name, counts), - "README.md": generate_readme(env_id, stack_name, args.region, counts), - } - - for filename, content in files.items(): - filepath = os.path.join(output_dir, filename) - with open(filepath, "w") as f: - f.write(content) - print(f"Generated: {filepath}") - - print(f"\n{'=' * 60}") - print(f"Performance test environment generated successfully!") - print(f"{'=' * 60}") - print(f"\nFormat: PKL (for formae)") - print(f"Environment ID: {env_id}") - print(f"Stack Name: {stack_name}") - print(f"Region: {args.region}") - print(f"Target Resources: {args.count}") - print(f"Calculated Resources: {counts.total}") - print(f"Output Directory: {output_dir}") - print(f"\nNext steps:") - print(f" 1. cd {output_dir}") - print(f" 2. pkl project resolve") - print(f" 3. formae apply --simulate main.pkl # dry-run") - print(f" 4. formae apply main.pkl # deploy") - print(f" 5. formae destroy --stack {stack_name} # cleanup") - + if is_multicloud or len(clouds) > 1 or (len(clouds) == 1 and clouds[0] != "aws"): + # Multi-cloud or single non-AWS cloud: use subdirectory layout + _generate_multicloud_pkl( + output_dir=output_dir, + formae_path=formae_path, + env_id=env_id, + run_id=run_id, + stack_name=stack_name, + clouds=clouds, + region=args.region, + aws_counts=aws_counts, + azure_counts=azure_counts, + gcp_counts=gcp_counts, + azure_subscription=args.subscription_azure, + gcp_project=args.project_gcp, + gcp_region=args.region_gcp, + ) + else: + # AWS-only: use the original flat layout for backwards compatibility + _generate_aws_only_pkl( + output_dir=output_dir, + formae_path=formae_path, + env_id=env_id, + stack_name=stack_name, + region=args.region, + counts=aws_counts, + ) else: - # Generate CloudFormation templates with automatic stack splitting - networking_stack = f"{env_id}-networking" - sg_stack_base = f"{env_id}-security-groups" - iam_stack_base = f"{env_id}-iam" - - # Collect all templates and track stacks for deploy/destroy scripts - files = {} - stacks = [] - - # Check if we're in quota-safe mode (no VPC-dependent resources) - is_quota_safe = args.profile == "quota-safe" - - # Networking (single stack - skip in quota-safe mode) - if not is_quota_safe: - files["networking.yaml"] = generate_cfn_networking(counts, env_id, args.region) - stacks.append({"name": "networking", "file": "networking.yaml"}) - - # Security groups (may be split - skip in quota-safe mode) - sg_stack = None - if not is_quota_safe: - sg_templates = generate_cfn_security_groups_split(counts, env_id, networking_stack) - for filename, content in sg_templates: - files[filename] = content - # Extract stack name from filename: security-groups.yaml or security-groups-0.yaml - stack_name = filename.replace(".yaml", "") - stacks.append({"name": stack_name, "file": filename}) - sg_stack = sg_stack_base if len(sg_templates) == 1 else f"{sg_stack_base}-0" - - # IAM (may be split) - iam_templates = generate_cfn_iam_split(counts, env_id) - for filename, content in iam_templates: - files[filename] = content - stack_name = filename.replace(".yaml", "") - stacks.append({"name": stack_name, "file": filename}) + # CloudFormation (AWS only) + _generate_cloudformation( + output_dir=output_dir, + env_id=env_id, + stack_name=stack_name, + region=args.region, + counts=aws_counts, + is_quota_safe=(args.profile == "quota-safe"), + ) - # KMS (may be split if many keys) - kms_templates = generate_cfn_kms_split(counts, env_id) - for filename, content in kms_templates: - files[filename] = content - stack_name = filename.replace(".yaml", "") - stacks.append({"name": stack_name, "file": filename}) - # Storage (may be split) - generates S3/DynamoDB (always) and EFS (skip in quota-safe) - storage_templates = generate_cfn_storage_split(counts, env_id, networking_stack, sg_stack or "") - for filename, content in storage_templates: - files[filename] = content - stack_name = filename.replace(".yaml", "") - stacks.append({"name": stack_name, "file": filename}) - - # Compute (may be split - skip in quota-safe mode) - if not is_quota_safe: - compute_templates = generate_cfn_compute_split(counts, env_id, args.region, networking_stack, sg_stack_base) - for filename, content in compute_templates: - files[filename] = content - stack_name = filename.replace(".yaml", "") - stacks.append({"name": stack_name, "file": filename}) - - # Observability (may be split) - obs_templates = generate_cfn_observability_split(counts, env_id) - for filename, content in obs_templates: - files[filename] = content - stack_name = filename.replace(".yaml", "") - stacks.append({"name": stack_name, "file": filename}) +def _generate_aws_only_pkl( + output_dir: str, + formae_path: str, + env_id: str, + stack_name: str, + region: str, + counts: ResourceCounts, +): + """Generate AWS-only PKL files (backwards-compatible flat layout).""" + files = { + "PklProject": generate_pkl_project(formae_path), + "vars.pkl": generate_vars_pkl(env_id, region, stack_name), + "networking.pkl": generate_networking_pkl(counts, region), + "security_groups.pkl": generate_security_groups_pkl(counts), + "compute.pkl": generate_compute_pkl(counts, region), + "iam.pkl": generate_iam_pkl(counts), + "kms.pkl": generate_kms_pkl(counts), + "storage.pkl": generate_storage_pkl(counts), + "observability.pkl": generate_observability_pkl(counts), + "secrets.pkl": generate_secrets_pkl(counts), + "application.pkl": generate_application_pkl(counts), + "route53.pkl": generate_route53_pkl(counts), + "api_gateway.pkl": generate_api_gateway_pkl(counts), + "rds.pkl": generate_rds_pkl(counts), + "main.pkl": generate_main_pkl(env_id, stack_name, counts), + "README.md": generate_readme(env_id, stack_name, region, counts), + } - # Secrets (single stack - usually small) - files["secrets.yaml"] = generate_cfn_secrets(counts, env_id) - stacks.append({"name": "secrets", "file": "secrets.yaml"}) + for filename, content in files.items(): + filepath = os.path.join(output_dir, filename) + with open(filepath, "w") as f: + f.write(content) + print(f"Generated: {filepath}") + + print(f"\n{'=' * 60}") + print(f"Performance test environment generated successfully!") + print(f"{'=' * 60}") + print(f"\nFormat: PKL (for formae)") + print(f"Environment ID: {env_id}") + print(f"Stack Name: {stack_name}") + print(f"Region: {region}") + print(f"Target Resources: {counts.total}") + print(f"Output Directory: {output_dir}") + print(f"\nNext steps:") + print(f" 1. cd {output_dir}") + print(f" 2. pkl project resolve") + print(f" 3. formae apply --simulate main.pkl # dry-run") + print(f" 4. formae apply main.pkl # deploy") + print(f" 5. formae destroy --stack {stack_name} # cleanup") + + +def _generate_multicloud_pkl( + output_dir: str, + formae_path: str, + env_id: str, + run_id: str, + stack_name: str, + clouds: list, + region: str, + aws_counts: ResourceCounts, + azure_counts: AzureResourceCounts, + gcp_counts: GCPResourceCounts, + azure_subscription: str = None, + gcp_project: str = None, + gcp_region: str = "us-central1", +): + """Generate multi-cloud PKL files with subdirectory layout.""" + + # Azure location mapping from AWS region + azure_location_map = { + "us-east-1": "eastus", + "us-east-2": "eastus2", + "us-west-1": "westus", + "us-west-2": "westus2", + "eu-west-1": "westeurope", + "eu-central-1": "germanywestcentral", + } + azure_location = azure_location_map.get(region, "eastus") - # Application (may be split) - app_templates = generate_cfn_application_split(counts, env_id, iam_stack_base) - for filename, content in app_templates: - files[filename] = content - stack_name = filename.replace(".yaml", "") - stacks.append({"name": stack_name, "file": filename}) + # Generate PklProject + files = { + "PklProject": generate_multicloud_pkl_project(formae_path, clouds), + } - # Route53 (single stack - usually small) - files["route53.yaml"] = generate_cfn_route53(counts, env_id) - stacks.append({"name": "route53", "file": "route53.yaml"}) + # Generate vars.pkl + files["vars.pkl"] = generate_multicloud_vars_pkl( + env_id=env_id, + run_id=run_id, + clouds=clouds, + region=region, + stack_name=stack_name, + azure_subscription=azure_subscription, + azure_location=azure_location, + gcp_project=gcp_project, + gcp_region=gcp_region, + ) - # API Gateway (single stack) - files["api-gateway.yaml"] = generate_cfn_api_gateway(counts, env_id) - stacks.append({"name": "api-gateway", "file": "api-gateway.yaml"}) + # Generate main.pkl + files["main.pkl"] = generate_multicloud_main_pkl( + env_id=env_id, + clouds=clouds, + aws_counts=aws_counts, + azure_counts=azure_counts, + gcp_counts=gcp_counts, + ) - # RDS (single stack - skip in quota-safe mode, requires VPC) - if not is_quota_safe: - files["rds.yaml"] = generate_cfn_rds(counts, env_id, networking_stack, sg_stack) - stacks.append({"name": "rds", "file": "rds.yaml"}) + # Each cloud subdirectory gets a symlink to the parent vars.pkl + # This is simpler than trying to extend/import since the module files use `import "./vars.pkl"` + + # Generate AWS files + if "aws" in clouds and aws_counts: + aws_dir = os.path.join(output_dir, "aws") + os.makedirs(aws_dir, exist_ok=True) + # Symlink vars.pkl from parent so module imports work + os.symlink("../vars.pkl", os.path.join(aws_dir, "vars.pkl")) + aws_files = { + "networking.pkl": generate_networking_pkl(aws_counts, region), + "security_groups.pkl": generate_security_groups_pkl(aws_counts), + "compute.pkl": generate_compute_pkl(aws_counts, region), + "iam.pkl": generate_iam_pkl(aws_counts), + "kms.pkl": generate_kms_pkl(aws_counts), + "storage.pkl": generate_storage_pkl(aws_counts), + "observability.pkl": generate_observability_pkl(aws_counts), + "secrets.pkl": generate_secrets_pkl(aws_counts), + "application.pkl": generate_application_pkl(aws_counts), + "route53.pkl": generate_route53_pkl(aws_counts), + "api_gateway.pkl": generate_api_gateway_pkl(aws_counts), + "rds.pkl": generate_rds_pkl(aws_counts), + "main.pkl": generate_main_pkl(env_id, f"{stack_name}-aws", aws_counts, + stack_var="awsStack", target_var="awsTarget", + region_var="awsRegion"), + } + for filename, content in aws_files.items(): + filepath = os.path.join(aws_dir, filename) + with open(filepath, "w") as f: + f.write(content) + print(f"Generated: {filepath}") - # Generate deploy and destroy scripts - files["deploy.sh"] = generate_cfn_deploy_script(env_id, args.region, stacks) - files["destroy.sh"] = generate_cfn_destroy_script(env_id, args.region, stacks) - files["README.md"] = generate_cfn_readme(env_id, args.region, counts) + # Generate Azure files + if "azure" in clouds and azure_counts: + az_dir = os.path.join(output_dir, "azure") + os.makedirs(az_dir, exist_ok=True) + os.symlink("../vars.pkl", os.path.join(az_dir, "vars.pkl")) + azure_files = { + "networking.pkl": generate_azure_networking_pkl(azure_counts, azure_location), + "storage.pkl": generate_azure_storage_pkl(azure_counts, azure_location), + "iam.pkl": generate_azure_iam_pkl(azure_counts, azure_location), + "security.pkl": generate_azure_security_pkl(azure_counts, azure_location), + "main.pkl": generate_azure_main_pkl(env_id, azure_counts), + } + if azure_counts.virtual_machines > 0 or azure_counts.network_interfaces > 0: + azure_files["compute.pkl"] = generate_azure_compute_pkl(azure_counts, azure_location) + if azure_counts.postgres_servers > 0: + azure_files["database.pkl"] = generate_azure_database_pkl(azure_counts, azure_location) + for filename, content in azure_files.items(): + filepath = os.path.join(az_dir, filename) + with open(filepath, "w") as f: + f.write(content) + print(f"Generated: {filepath}") - for filename, content in files.items(): - filepath = os.path.join(output_dir, filename) + # Generate GCP files + if "gcp" in clouds and gcp_counts: + gcp_dir = os.path.join(output_dir, "gcp") + os.makedirs(gcp_dir, exist_ok=True) + os.symlink("../vars.pkl", os.path.join(gcp_dir, "vars.pkl")) + gcp_files = { + "networking.pkl": generate_gcp_networking_pkl(gcp_counts, gcp_region), + "storage.pkl": generate_gcp_storage_pkl(gcp_counts, gcp_region), + "main.pkl": generate_gcp_main_pkl(env_id, gcp_counts), + } + if gcp_counts.compute_instances > 0 or gcp_counts.compute_disks > 0: + gcp_files["compute.pkl"] = generate_gcp_compute_pkl(gcp_counts, gcp_region, gcp_project) + if gcp_counts.sql_instances > 0 or gcp_counts.bigquery_datasets > 0: + gcp_files["database.pkl"] = generate_gcp_database_pkl(gcp_counts, gcp_region, gcp_project) + for filename, content in gcp_files.items(): + filepath = os.path.join(gcp_dir, filename) with open(filepath, "w") as f: f.write(content) print(f"Generated: {filepath}") - # Make scripts executable - os.chmod(os.path.join(output_dir, "deploy.sh"), 0o755) - os.chmod(os.path.join(output_dir, "destroy.sh"), 0o755) - - print(f"\n{'=' * 60}") - print(f"Performance test environment generated successfully!") - print(f"{'=' * 60}") - print(f"\nFormat: CloudFormation (for discovery testing)") - print(f"Environment ID: {env_id}") - print(f"Region: {args.region}") - print(f"Target Resources: {args.count}") - print(f"Calculated Resources: {counts.total}") - print(f"Output Directory: {output_dir}") - print(f"CloudFormation Stacks: {len(stacks)}") - print(f"\nNext steps:") - print(f" 1. cd {output_dir}") - print(f" 2. ./deploy.sh # deploy all stacks") - print(f" 3. # Test formae discovery:") - print(f" formae agent start") - print(f" formae discover --target aws --region {args.region}") - print(f" formae inventory --unmanaged") - print(f" 4. ./destroy.sh # cleanup") + # Write top-level files + for filename, content in files.items(): + filepath = os.path.join(output_dir, filename) + with open(filepath, "w") as f: + f.write(content) + print(f"Generated: {filepath}") + + grand_total = sum(c.total for c in [aws_counts, azure_counts, gcp_counts] if c) + print(f"\n{'=' * 60}") + print(f"Multi-cloud stress test environment generated successfully!") + print(f"{'=' * 60}") + print(f"\nFormat: PKL (for formae)") + print(f"Environment ID: {env_id}") + print(f"Run ID: {run_id}") + print(f"Clouds: {', '.join(c.upper() for c in clouds)}") + if aws_counts: + print(f" AWS: ~{aws_counts.total} resources (region: {region})") + if azure_counts: + print(f" Azure: ~{azure_counts.total} resources (location: {azure_location})") + if gcp_counts: + print(f" GCP: ~{gcp_counts.total} resources (region: {gcp_region})") + print(f"Grand Total: ~{grand_total} resources") + print(f"Output Directory: {output_dir}") + print(f"\nNext steps:") + print(f" 1. cd {output_dir}") + print(f" 2. pkl project resolve") + print(f" 3. formae apply --simulate main.pkl # dry-run") + print(f" 4. formae apply main.pkl # deploy") + + +def _generate_cloudformation( + output_dir: str, + env_id: str, + stack_name: str, + region: str, + counts: ResourceCounts, + is_quota_safe: bool, +): + """Generate CloudFormation templates (AWS only).""" + networking_stack = f"{env_id}-networking" + sg_stack_base = f"{env_id}-security-groups" + iam_stack_base = f"{env_id}-iam" + + # Collect all templates and track stacks for deploy/destroy scripts + files = {} + stacks = [] + + # Networking (single stack - skip in quota-safe mode) + if not is_quota_safe: + files["networking.yaml"] = generate_cfn_networking(counts, env_id, region) + stacks.append({"name": "networking", "file": "networking.yaml"}) + + # Security groups (may be split - skip in quota-safe mode) + sg_stack = None + if not is_quota_safe: + sg_templates = generate_cfn_security_groups_split(counts, env_id, networking_stack) + for filename, content in sg_templates: + files[filename] = content + sn = filename.replace(".yaml", "") + stacks.append({"name": sn, "file": filename}) + sg_stack = sg_stack_base if len(sg_templates) == 1 else f"{sg_stack_base}-0" + + # IAM (may be split) + iam_templates = generate_cfn_iam_split(counts, env_id) + for filename, content in iam_templates: + files[filename] = content + sn = filename.replace(".yaml", "") + stacks.append({"name": sn, "file": filename}) + + # KMS (may be split if many keys) + kms_templates = generate_cfn_kms_split(counts, env_id) + for filename, content in kms_templates: + files[filename] = content + sn = filename.replace(".yaml", "") + stacks.append({"name": sn, "file": filename}) + + # Storage (may be split) + storage_templates = generate_cfn_storage_split(counts, env_id, networking_stack, sg_stack or "") + for filename, content in storage_templates: + files[filename] = content + sn = filename.replace(".yaml", "") + stacks.append({"name": sn, "file": filename}) + + # Compute (may be split - skip in quota-safe mode) + if not is_quota_safe: + compute_templates = generate_cfn_compute_split(counts, env_id, region, networking_stack, sg_stack_base) + for filename, content in compute_templates: + files[filename] = content + sn = filename.replace(".yaml", "") + stacks.append({"name": sn, "file": filename}) + + # Observability (may be split) + obs_templates = generate_cfn_observability_split(counts, env_id) + for filename, content in obs_templates: + files[filename] = content + sn = filename.replace(".yaml", "") + stacks.append({"name": sn, "file": filename}) + + # Secrets (single stack - usually small) + files["secrets.yaml"] = generate_cfn_secrets(counts, env_id) + stacks.append({"name": "secrets", "file": "secrets.yaml"}) + + # Application (may be split) + app_templates = generate_cfn_application_split(counts, env_id, iam_stack_base) + for filename, content in app_templates: + files[filename] = content + sn = filename.replace(".yaml", "") + stacks.append({"name": sn, "file": filename}) + + # Route53 (single stack - usually small) + files["route53.yaml"] = generate_cfn_route53(counts, env_id) + stacks.append({"name": "route53", "file": "route53.yaml"}) + + # API Gateway (single stack) + files["api-gateway.yaml"] = generate_cfn_api_gateway(counts, env_id) + stacks.append({"name": "api-gateway", "file": "api-gateway.yaml"}) + + # RDS (single stack - skip in quota-safe mode, requires VPC) + if not is_quota_safe: + files["rds.yaml"] = generate_cfn_rds(counts, env_id, networking_stack, sg_stack) + stacks.append({"name": "rds", "file": "rds.yaml"}) + + # Generate deploy and destroy scripts + files["deploy.sh"] = generate_cfn_deploy_script(env_id, region, stacks) + files["destroy.sh"] = generate_cfn_destroy_script(env_id, region, stacks) + files["README.md"] = generate_cfn_readme(env_id, region, counts) + + for filename, content in files.items(): + filepath = os.path.join(output_dir, filename) + with open(filepath, "w") as f: + f.write(content) + print(f"Generated: {filepath}") + + # Make scripts executable + os.chmod(os.path.join(output_dir, "deploy.sh"), 0o755) + os.chmod(os.path.join(output_dir, "destroy.sh"), 0o755) + + print(f"\n{'=' * 60}") + print(f"Performance test environment generated successfully!") + print(f"{'=' * 60}") + print(f"\nFormat: CloudFormation (for discovery testing)") + print(f"Environment ID: {env_id}") + print(f"Region: {region}") + print(f"Calculated Resources: {counts.total}") + print(f"Output Directory: {output_dir}") + print(f"CloudFormation Stacks: {len(stacks)}") + print(f"\nNext steps:") + print(f" 1. cd {output_dir}") + print(f" 2. ./deploy.sh # deploy all stacks") + print(f" 3. # Test formae discovery:") + print(f" formae agent start") + print(f" formae discover --target aws --region {region}") + print(f" formae inventory --unmanaged") + print(f" 4. ./destroy.sh # cleanup") if __name__ == "__main__": From 552916475549b3b2ad2b31457cc0e388ab7aa08e Mon Sep 17 00:00:00 2001 From: Jeroen Soeters Date: Sun, 19 Apr 2026 19:57:31 -0700 Subject: [PATCH 3/4] feat(loadtest): add Suite B harness Add a Go-based harness that drives the Suite B load test lifecycle: apply, verify, soak, destroy. Each phase lives in its own file so they can be exercised independently; the harness composes them and reports results via the shared report type. Includes helpers for metric collection (scraped from the observability stack) and log checking, plus unit tests for config and report rendering. --- tests/loadtest/harness/config.go | 77 +++++++++++ tests/loadtest/harness/config_test.go | 38 ++++++ tests/loadtest/harness/harness.go | 114 ++++++++++++++++ tests/loadtest/harness/log_checker.go | 112 ++++++++++++++++ tests/loadtest/harness/metric_collector.go | 118 +++++++++++++++++ tests/loadtest/harness/phase_apply.go | 145 +++++++++++++++++++++ tests/loadtest/harness/phase_destroy.go | 77 +++++++++++ tests/loadtest/harness/phase_soak.go | 94 +++++++++++++ tests/loadtest/harness/phase_verify.go | 81 ++++++++++++ tests/loadtest/harness/report.go | 103 +++++++++++++++ tests/loadtest/harness/report_test.go | 46 +++++++ tests/loadtest/loadtest_test.go | 128 ++++++++++++++++++ 12 files changed, 1133 insertions(+) create mode 100644 tests/loadtest/harness/config.go create mode 100644 tests/loadtest/harness/config_test.go create mode 100644 tests/loadtest/harness/harness.go create mode 100644 tests/loadtest/harness/log_checker.go create mode 100644 tests/loadtest/harness/metric_collector.go create mode 100644 tests/loadtest/harness/phase_apply.go create mode 100644 tests/loadtest/harness/phase_destroy.go create mode 100644 tests/loadtest/harness/phase_soak.go create mode 100644 tests/loadtest/harness/phase_verify.go create mode 100644 tests/loadtest/harness/report.go create mode 100644 tests/loadtest/harness/report_test.go create mode 100644 tests/loadtest/loadtest_test.go diff --git a/tests/loadtest/harness/config.go b/tests/loadtest/harness/config.go new file mode 100644 index 000000000..52d07f40e --- /dev/null +++ b/tests/loadtest/harness/config.go @@ -0,0 +1,77 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package harness + +import ( + "fmt" + "time" +) + +type ScaleProfile string + +const ( + ProfileSmall ScaleProfile = "small" + ProfileMedium ScaleProfile = "medium" + ProfileLarge ScaleProfile = "large" + ProfileXL ScaleProfile = "xl" +) + +type Config struct { + AgentURL string + Profile ScaleProfile + SoakDuration time.Duration + ApplyTimeout time.Duration + DestroyTimeout time.Duration + PhaseTimeout time.Duration + MimirURL string + TempoURL string + LokiURL string + FormaFilePath string + RunID string + GitSHA string +} + +type ProfileSpec struct { + TotalResources int + AWSResources int + AzureResources int + GCPResources int + AgentVCPU string + AgentMemoryMB int + MaxDuration time.Duration +} + +var Profiles = map[ScaleProfile]ProfileSpec{ + ProfileSmall: { + TotalResources: 500, AWSResources: 200, AzureResources: 150, GCPResources: 150, + AgentVCPU: "1", AgentMemoryMB: 1024, MaxDuration: 30 * time.Minute, + }, + ProfileMedium: { + TotalResources: 5000, AWSResources: 2000, AzureResources: 1500, GCPResources: 1500, + AgentVCPU: "2", AgentMemoryMB: 1024, MaxDuration: 90 * time.Minute, + }, + ProfileLarge: { + TotalResources: 20000, AWSResources: 8000, AzureResources: 6000, GCPResources: 6000, + AgentVCPU: "4", AgentMemoryMB: 2048, MaxDuration: 180 * time.Minute, + }, + ProfileXL: { + TotalResources: 50000, AWSResources: 20000, AzureResources: 15000, GCPResources: 15000, + AgentVCPU: "4", AgentMemoryMB: 4096, MaxDuration: 240 * time.Minute, + }, +} + +func (c *Config) Validate() error { + if c.AgentURL == "" { + return fmt.Errorf("agent URL is required") + } + if _, ok := Profiles[c.Profile]; !ok { + return fmt.Errorf("unknown profile: %s", c.Profile) + } + return nil +} + +func (c *Config) ProfileSpec() ProfileSpec { + return Profiles[c.Profile] +} diff --git a/tests/loadtest/harness/config_test.go b/tests/loadtest/harness/config_test.go new file mode 100644 index 000000000..8d306b765 --- /dev/null +++ b/tests/loadtest/harness/config_test.go @@ -0,0 +1,38 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package harness + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfig_Validate(t *testing.T) { + t.Run("valid config", func(t *testing.T) { + c := &Config{AgentURL: "http://localhost:49684", Profile: ProfileMedium} + require.NoError(t, c.Validate()) + }) + t.Run("missing agent URL", func(t *testing.T) { + c := &Config{Profile: ProfileMedium} + assert.ErrorContains(t, c.Validate(), "agent URL is required") + }) + t.Run("unknown profile", func(t *testing.T) { + c := &Config{AgentURL: "http://localhost:49684", Profile: "huge"} + assert.ErrorContains(t, c.Validate(), "unknown profile") + }) +} + +func TestProfileSpec(t *testing.T) { + c := &Config{AgentURL: "http://localhost:49684", Profile: ProfileSmall} + spec := c.ProfileSpec() + assert.Equal(t, 500, spec.TotalResources) + assert.Equal(t, 200, spec.AWSResources) + assert.Equal(t, 150, spec.AzureResources) + assert.Equal(t, 150, spec.GCPResources) + assert.Equal(t, 30*time.Minute, spec.MaxDuration) +} diff --git a/tests/loadtest/harness/harness.go b/tests/loadtest/harness/harness.go new file mode 100644 index 000000000..7ca8c2a46 --- /dev/null +++ b/tests/loadtest/harness/harness.go @@ -0,0 +1,114 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package harness + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "time" + + "github.com/platform-engineering-labs/formae/internal/api" + pkgmodel "github.com/platform-engineering-labs/formae/pkg/model" +) + +// Harness wraps the formae API client and provides load-test phase helpers. +type Harness struct { + config *Config + client *api.Client + logger *slog.Logger + report *Report +} + +// New validates cfg and returns a ready-to-use Harness. +func New(cfg *Config, logger *slog.Logger) (*Harness, error) { + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("invalid config: %w", err) + } + + apiCfg := pkgmodel.APIConfig{URL: cfg.AgentURL} + client := api.NewClient(apiCfg, nil, http.DefaultClient) + + return &Harness{ + config: cfg, + client: client, + logger: logger, + report: NewReport(cfg), + }, nil +} + +// WaitForHealth blocks until the agent responds healthy or the timeout elapses. +// WaitOnAvailable polls internally, so we race it against a deadline in a goroutine. +func (h *Harness) WaitForHealth(ctx context.Context, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + done := make(chan struct{}) + go func() { + h.client.WaitOnAvailable() + close(done) + }() + + select { + case <-done: + h.logger.Info("agent is healthy") + return nil + case <-ctx.Done(): + return fmt.Errorf("agent not healthy after %s", timeout) + } +} + +// WriteReport serialises the accumulated report to the given file path. +func (h *Harness) WriteReport(path string) error { + return h.report.WriteToFile(path) +} + +// CheckLogs queries Loki for unexpected errors and warnings emitted since the given timestamp. +// Results are appended to the report. If LokiURL is not set this is a no-op. +func (h *Harness) CheckLogs(ctx context.Context, since time.Time, phase string) { + if h.config.LokiURL == "" { + h.logger.Warn("Loki URL not set, skipping log check") + return + } + checker := NewLogChecker(h.config.LokiURL) + now := time.Now() + + errors, err := checker.CheckForErrors(ctx, since, now) + if err != nil { + h.logger.Warn("log check failed", "phase", phase, "error", err) + return + } + for _, e := range errors { + h.report.Errors = append(h.report.Errors, ReportError{Phase: phase, Message: e}) + } + + warnings, err := checker.CheckForWarnings(ctx, since, now) + if err != nil { + h.logger.Warn("warning check failed", "phase", phase, "error", err) + return + } + h.report.LogWarnings = append(h.report.LogWarnings, warnings...) + + if len(errors) > 0 { + h.logger.Error("unexpected errors in logs", "phase", phase, "count", len(errors)) + } +} + +// CollectMetrics queries Mimir for agent resource utilization over the entire test run +// and stores the result in the report. If MimirURL is not set this is a no-op. +func (h *Harness) CollectMetrics(ctx context.Context) { + if h.config.MimirURL == "" { + h.logger.Warn("Mimir URL not set, skipping metric collection") + return + } + collector := NewMetricCollector(h.config.MimirURL) + util, err := collector.CollectResourceUtilization(ctx, h.report.Metadata.Timestamp, time.Now()) + if err != nil { + h.logger.Warn("metric collection failed", "error", err) + return + } + h.report.ResourceUtilization = *util +} diff --git a/tests/loadtest/harness/log_checker.go b/tests/loadtest/harness/log_checker.go new file mode 100644 index 000000000..420023ff2 --- /dev/null +++ b/tests/loadtest/harness/log_checker.go @@ -0,0 +1,112 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package harness + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" +) + +var warningAllowlist = []string{ + "rate limiter", + "token bucket", + "throttling", +} + +// LogChecker queries Loki for error and warning log entries emitted by the agent. +type LogChecker struct { + lokiURL string + client *http.Client +} + +type lokiQueryResponse struct { + Status string `json:"status"` + Data struct { + Result []struct { + Stream map[string]string `json:"stream"` + Values [][]string `json:"values"` + } `json:"result"` + } `json:"data"` +} + +// NewLogChecker returns a LogChecker that targets the given Loki base URL. +func NewLogChecker(lokiURL string) *LogChecker { + return &LogChecker{lokiURL: lokiURL, client: &http.Client{Timeout: 30 * time.Second}} +} + +// CheckForErrors returns all log lines at error level emitted between since and until. +func (lc *LogChecker) CheckForErrors(ctx context.Context, since, until time.Time) ([]string, error) { + query := `{service_name=~"formae.*"} |= "level=error" or "level=ERROR"` + return lc.queryLoki(ctx, query, since, until) +} + +// CheckForWarnings returns warning log lines that are not in the allowlist. +func (lc *LogChecker) CheckForWarnings(ctx context.Context, since, until time.Time) ([]string, error) { + query := `{service_name=~"formae.*"} |= "level=warn" or "level=WARN"` + lines, err := lc.queryLoki(ctx, query, since, until) + if err != nil { + return nil, err + } + var unexpected []string + for _, line := range lines { + allowed := false + lower := strings.ToLower(line) + for _, pattern := range warningAllowlist { + if strings.Contains(lower, pattern) { + allowed = true + break + } + } + if !allowed { + unexpected = append(unexpected, line) + } + } + return unexpected, nil +} + +func (lc *LogChecker) queryLoki(ctx context.Context, query string, since, until time.Time) ([]string, error) { + params := url.Values{ + "query": {query}, + "start": {fmt.Sprintf("%d", since.UnixNano())}, + "end": {fmt.Sprintf("%d", until.UnixNano())}, + "limit": {"1000"}, + } + + reqURL := fmt.Sprintf("%s/loki/api/v1/query_range?%s", lc.lokiURL, params.Encode()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, err + } + + resp, err := lc.client.Do(req) + if err != nil { + return nil, fmt.Errorf("loki query failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("loki returned status %d", resp.StatusCode) + } + + var result lokiQueryResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode loki response: %w", err) + } + + var lines []string + for _, stream := range result.Data.Result { + for _, entry := range stream.Values { + if len(entry) >= 2 { + lines = append(lines, entry[1]) + } + } + } + return lines, nil +} diff --git a/tests/loadtest/harness/metric_collector.go b/tests/loadtest/harness/metric_collector.go new file mode 100644 index 000000000..c907c28bd --- /dev/null +++ b/tests/loadtest/harness/metric_collector.go @@ -0,0 +1,118 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package harness + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" +) + +// MetricCollector queries Mimir (Prometheus-compatible) for agent resource utilization. +type MetricCollector struct { + mimirURL string + client *http.Client +} + +type promQueryResult struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` + } `json:"result"` + } `json:"data"` +} + +// NewMetricCollector returns a MetricCollector that targets the given Mimir base URL. +func NewMetricCollector(mimirURL string) *MetricCollector { + return &MetricCollector{mimirURL: mimirURL, client: &http.Client{Timeout: 30 * time.Second}} +} + +// CollectResourceUtilization returns peak and average CPU/memory for the agent over the given window. +func (mc *MetricCollector) CollectResourceUtilization(ctx context.Context, since, until time.Time) (*ReportResourceUtil, error) { + util := &ReportResourceUtil{} + + window := until.Sub(since).String() + + peakCPU, err := mc.queryInstant(ctx, fmt.Sprintf(`max_over_time(process_cpu_usage{service_name="formae-agent"}[%s])`, window), until) + if err == nil && peakCPU != nil { + util.AgentPeakCPUPercent = *peakCPU * 100 + } + + avgCPU, err := mc.queryInstant(ctx, fmt.Sprintf(`avg_over_time(process_cpu_usage{service_name="formae-agent"}[%s])`, window), until) + if err == nil && avgCPU != nil { + util.AgentAvgCPUPercent = *avgCPU * 100 + } + + peakMem, err := mc.queryInstant(ctx, fmt.Sprintf(`max_over_time(process_resident_memory_bytes{service_name="formae-agent"}[%s])`, window), until) + if err == nil && peakMem != nil { + util.AgentPeakMemoryMB = *peakMem / 1024 / 1024 + } + + avgMem, err := mc.queryInstant(ctx, fmt.Sprintf(`avg_over_time(process_resident_memory_bytes{service_name="formae-agent"}[%s])`, window), until) + if err == nil && avgMem != nil { + util.AgentAvgMemoryMB = *avgMem / 1024 / 1024 + } + + return util, nil +} + +// CheckActorLeak returns the net change in ergo process count over the soak window. +func (mc *MetricCollector) CheckActorLeak(ctx context.Context, soakStart, soakEnd time.Time) (int, error) { + startCount, err := mc.queryInstant(ctx, `ergo_processes_total{service_name="formae-agent"}`, soakStart) + if err != nil { + return 0, fmt.Errorf("query start actor count: %w", err) + } + endCount, err := mc.queryInstant(ctx, `ergo_processes_total{service_name="formae-agent"}`, soakEnd) + if err != nil { + return 0, fmt.Errorf("query end actor count: %w", err) + } + if startCount == nil || endCount == nil { + return 0, nil + } + return int(*endCount) - int(*startCount), nil +} + +func (mc *MetricCollector) queryInstant(ctx context.Context, query string, at time.Time) (*float64, error) { + params := url.Values{ + "query": {query}, + "time": {fmt.Sprintf("%d", at.Unix())}, + } + + reqURL := fmt.Sprintf("%s/prometheus/api/v1/query?%s", mc.mimirURL, params.Encode()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, err + } + + resp, err := mc.client.Do(req) + if err != nil { + return nil, fmt.Errorf("mimir query failed: %w", err) + } + defer resp.Body.Close() + + var result promQueryResult + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode mimir response: %w", err) + } + + if len(result.Data.Result) == 0 { + return nil, nil + } + + valStr, ok := result.Data.Result[0].Value[1].(string) + if !ok { + return nil, fmt.Errorf("unexpected value type") + } + + var val float64 + fmt.Sscanf(valStr, "%f", &val) + return &val, nil +} diff --git a/tests/loadtest/harness/phase_apply.go b/tests/loadtest/harness/phase_apply.go new file mode 100644 index 000000000..624ea1a09 --- /dev/null +++ b/tests/loadtest/harness/phase_apply.go @@ -0,0 +1,145 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package harness + +import ( + "context" + "fmt" + "time" + + apimodel "github.com/platform-engineering-labs/formae/pkg/api/model" + pkgmodel "github.com/platform-engineering-labs/formae/pkg/model" +) + +func (h *Harness) RunApplyPhase(ctx context.Context, forma *pkgmodel.Forma) error { + h.logger.Info("starting apply phase") + start := time.Now() + + resp, err := h.client.ApplyForma(forma, pkgmodel.FormaApplyModeReconcile, false, h.config.RunID, false) + if err != nil { + return fmt.Errorf("submit apply: %w", err) + } + h.logger.Info("apply command submitted", "commandID", resp.CommandID) + + if err := h.waitForCommandDone(ctx, resp.CommandID, h.config.ApplyTimeout); err != nil { + return fmt.Errorf("apply command failed: %w", err) + } + + duration := time.Since(start) + spec := h.config.ProfileSpec() + + status, err := h.client.GetFormaCommandsStatus(fmt.Sprintf("id:%s", resp.CommandID), h.config.RunID, 1) + if err != nil { + return fmt.Errorf("get command status: %w", err) + } + + errorCount := 0 + if len(status.Commands) > 0 { + for _, ru := range status.Commands[0].ResourceUpdates { + if ru.State == "Failed" { + errorCount++ + } + } + } + + minutes := duration.Minutes() + aggregateRPS := h.calculateAggregateRPS() + theoreticalMin := float64(spec.TotalResources) / aggregateRPS + overhead := 0.0 + if theoreticalMin > 0 { + overhead = (duration.Seconds() - theoreticalMin) / theoreticalMin + } + + h.report.Phases.Apply = ApplyPhaseResult{ + DurationSeconds: duration.Seconds(), + ResourcesTotal: spec.TotalResources, + ThroughputPerMinute: h.calculateThroughput(status, minutes), + Errors: errorCount, + OrchestrationOverhead: overhead, + } + + h.logger.Info("apply phase complete", "duration", duration, "resources", spec.TotalResources, "errors", errorCount, "overhead", overhead) + return nil +} + +func (h *Harness) waitForCommandDone(ctx context.Context, commandID string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + status, err := h.client.GetFormaCommandsStatus(fmt.Sprintf("id:%s", commandID), h.config.RunID, 1) + if err != nil { + h.logger.Warn("failed to get command status", "error", err) + time.Sleep(2 * time.Second) + continue + } + + if len(status.Commands) == 0 { + time.Sleep(2 * time.Second) + continue + } + + switch status.Commands[0].State { + case "Success": + return nil + case "Failed": + return fmt.Errorf("command %s failed", commandID) + case "Canceled": + return fmt.Errorf("command %s was canceled", commandID) + } + + time.Sleep(2 * time.Second) + } + return fmt.Errorf("command %s timed out after %s", commandID, timeout) +} + +func (h *Harness) calculateAggregateRPS() float64 { + stats, err := h.client.Stats() + if err != nil { + return 2.0 // fallback: assume single plugin at 2 RPS + } + total := 0.0 + for _, p := range stats.Plugins { + total += float64(p.MaxRequestsPerSecond) + } + if total == 0 { + return 2.0 + } + return total +} + +func (h *Harness) calculateThroughput(status *apimodel.ListCommandStatusResponse, minutes float64) map[string]float64 { + if minutes <= 0 || status == nil || len(status.Commands) == 0 { + return map[string]float64{} + } + + perProvider := map[string]int{} + for _, ru := range status.Commands[0].ResourceUpdates { + if ru.State == "Success" { + // Extract provider from resource type (e.g., "AWS::S3::Bucket" -> "AWS") + provider := extractProvider(ru.ResourceType) + perProvider[provider]++ + } + } + + result := map[string]float64{} + for provider, count := range perProvider { + result[provider] = float64(count) / minutes + } + return result +} + +func extractProvider(resourceType string) string { + for i, c := range resourceType { + if c == ':' { + return resourceType[:i] + } + } + return resourceType +} diff --git a/tests/loadtest/harness/phase_destroy.go b/tests/loadtest/harness/phase_destroy.go new file mode 100644 index 000000000..7b0f2f166 --- /dev/null +++ b/tests/loadtest/harness/phase_destroy.go @@ -0,0 +1,77 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package harness + +import ( + "context" + "fmt" + "time" +) + +func (h *Harness) RunDestroyPhase(ctx context.Context) error { + h.logger.Info("starting destroy phase") + start := time.Now() + + stacks, err := h.client.ListStacks() + if err != nil { + return fmt.Errorf("list stacks: %w", err) + } + + errorCount := 0 + resourcesDestroyed := 0 + + for _, stack := range stacks { + query := fmt.Sprintf("stack:%s", stack.Label) + resp, err := h.client.DestroyByQuery(query, false, h.config.RunID) + if err != nil { + h.logger.Error("destroy failed for stack", "stack", stack.Label, "error", err) + errorCount++ + continue + } + + h.logger.Info("destroy submitted", "stack", stack.Label, "commandID", resp.CommandID) + + if err := h.waitForCommandDone(ctx, resp.CommandID, h.config.DestroyTimeout); err != nil { + h.logger.Error("destroy did not complete", "stack", stack.Label, "error", err) + errorCount++ + continue + } + + status, err := h.client.GetFormaCommandsStatus(fmt.Sprintf("id:%s", resp.CommandID), h.config.RunID, 1) + if err == nil && len(status.Commands) > 0 { + for _, ru := range status.Commands[0].ResourceUpdates { + if ru.State == "Success" { + resourcesDestroyed++ + } else if ru.State == "Failed" { + errorCount++ + } + } + } + } + + stats, err := h.client.Stats() + if err != nil { + return fmt.Errorf("get final stats: %w", err) + } + remaining := sumMapValues(stats.ManagedResources) + if remaining > 0 { + h.logger.Error("resources remain after destroy", "count", remaining) + errorCount += remaining + } + + duration := time.Since(start) + h.report.Phases.Destroy = DestroyPhaseResult{ + DurationSeconds: duration.Seconds(), + ResourcesDestroyed: resourcesDestroyed, + Errors: errorCount, + } + + h.logger.Info("destroy phase complete", "duration", duration, "destroyed", resourcesDestroyed, "errors", errorCount) + + if errorCount > 0 { + return fmt.Errorf("destroy phase completed with %d errors", errorCount) + } + return nil +} diff --git a/tests/loadtest/harness/phase_soak.go b/tests/loadtest/harness/phase_soak.go new file mode 100644 index 000000000..aee2ad931 --- /dev/null +++ b/tests/loadtest/harness/phase_soak.go @@ -0,0 +1,94 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package harness + +import ( + "context" + "fmt" + "time" +) + +func (h *Harness) RunSoakPhase(ctx context.Context) error { + h.logger.Info("starting soak phase", "duration", h.config.SoakDuration) + start := time.Now() + + baselineStats, err := h.client.Stats() + if err != nil { + return fmt.Errorf("get baseline stats: %w", err) + } + baselineManagedCount := sumMapValues(baselineStats.ManagedResources) + + syncCycles := 0 + discoveryCycles := 0 + sampleInterval := 30 * time.Second + ticker := time.NewTicker(sampleInterval) + defer ticker.Stop() + + soakDeadline := time.Now().Add(h.config.SoakDuration) + + for time.Now().Before(soakDeadline) { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if err := h.client.ForceSync(); err != nil { + h.logger.Warn("force sync failed", "error", err) + } else { + syncCycles++ + } + + if err := h.client.ForceDiscover(); err != nil { + h.logger.Warn("force discover failed", "error", err) + } else { + discoveryCycles++ + } + + stats, err := h.client.Stats() + if err != nil { + h.logger.Warn("stats sample failed", "error", err) + continue + } + + currentManaged := sumMapValues(stats.ManagedResources) + h.logger.Info("soak sample", + "managed_resources", currentManaged, + "plugins", len(stats.Plugins), + "elapsed", time.Since(start).Round(time.Second), + ) + + if currentManaged != baselineManagedCount { + h.logger.Warn("resource count changed during soak", + "baseline", baselineManagedCount, + "current", currentManaged, + ) + } + } + } + + finalStats, err := h.client.Stats() + if err != nil { + return fmt.Errorf("get final stats: %w", err) + } + finalManagedCount := sumMapValues(finalStats.ManagedResources) + + duration := time.Since(start) + h.report.Phases.Soak = SoakPhaseResult{ + DurationSeconds: duration.Seconds(), + SyncCycles: syncCycles, + DiscoveryCycles: discoveryCycles, + ResourceCountDelta: finalManagedCount - baselineManagedCount, + ActorCountDelta: 0, // Populated by metric collector + } + + h.logger.Info("soak phase complete", + "duration", duration, "sync_cycles", syncCycles, + "discovery_cycles", discoveryCycles, "resource_delta", finalManagedCount-baselineManagedCount, + ) + + if finalManagedCount != baselineManagedCount { + return fmt.Errorf("resource leak detected: baseline=%d, final=%d", baselineManagedCount, finalManagedCount) + } + return nil +} diff --git a/tests/loadtest/harness/phase_verify.go b/tests/loadtest/harness/phase_verify.go new file mode 100644 index 000000000..7ed1406cf --- /dev/null +++ b/tests/loadtest/harness/phase_verify.go @@ -0,0 +1,81 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package harness + +import ( + "context" + "fmt" + "time" +) + +func (h *Harness) RunVerifyPhase(ctx context.Context) error { + h.logger.Info("starting verify phase") + start := time.Now() + queriesExecuted := 0 + + spec := h.config.ProfileSpec() + + stats, err := h.client.Stats() + if err != nil { + return fmt.Errorf("get stats: %w", err) + } + + totalManaged := sumMapValues(stats.ManagedResources) + if totalManaged != spec.TotalResources { + return fmt.Errorf("expected %d managed resources, got %d", spec.TotalResources, totalManaged) + } + queriesExecuted++ + + for namespace, expected := range map[string]int{"AWS": spec.AWSResources, "Azure": spec.AzureResources, "GCP": spec.GCPResources} { + if expected == 0 { + continue + } + actual := stats.ManagedResources[namespace] + if actual != expected { + return fmt.Errorf("expected %d %s resources, got %d", expected, namespace, actual) + } + queriesExecuted++ + } + + totalErrors := sumMapValues(stats.ResourceErrors) + if totalErrors > 0 { + return fmt.Errorf("found %d resource errors after apply", totalErrors) + } + queriesExecuted++ + + // Exercise query API at scale + queryPatterns := []string{"managed:true", "stack:default"} + for _, query := range queryPatterns { + _, err := h.client.ExtractResources(query) + if err != nil { + h.logger.Warn("query failed", "query", query, "error", err) + } + queriesExecuted++ + } + + stacks, err := h.client.ListStacks() + if err != nil { + return fmt.Errorf("list stacks: %w", err) + } + h.logger.Info("stacks found", "count", len(stacks)) + queriesExecuted++ + + duration := time.Since(start) + h.report.Phases.Verify = VerifyPhaseResult{ + DurationSeconds: duration.Seconds(), + QueriesExecuted: queriesExecuted, + } + + h.logger.Info("verify phase complete", "duration", duration, "queries", queriesExecuted) + return nil +} + +func sumMapValues(m map[string]int) int { + total := 0 + for _, v := range m { + total += v + } + return total +} diff --git a/tests/loadtest/harness/report.go b/tests/loadtest/harness/report.go new file mode 100644 index 000000000..b01898640 --- /dev/null +++ b/tests/loadtest/harness/report.go @@ -0,0 +1,103 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package harness + +import ( + "encoding/json" + "fmt" + "os" + "time" +) + +type Report struct { + Metadata ReportMetadata `json:"metadata"` + Phases ReportPhases `json:"phases"` + ResourceUtilization ReportResourceUtil `json:"resource_utilization"` + Errors []ReportError `json:"errors"` + LogWarnings []string `json:"log_warnings"` +} + +type ReportMetadata struct { + Timestamp time.Time `json:"timestamp"` + Profile string `json:"profile"` + AgentSpec string `json:"agent_spec"` + GitSHA string `json:"git_sha"` + RunID string `json:"run_id"` + WorkflowRunID string `json:"workflow_run_id,omitempty"` +} + +type ReportPhases struct { + Deploy PhaseResult `json:"deploy"` + Apply ApplyPhaseResult `json:"apply"` + Verify VerifyPhaseResult `json:"verify"` + Soak SoakPhaseResult `json:"soak"` + Destroy DestroyPhaseResult `json:"destroy"` +} + +type PhaseResult struct { + DurationSeconds float64 `json:"duration_seconds"` +} + +type ApplyPhaseResult struct { + DurationSeconds float64 `json:"duration_seconds"` + ResourcesTotal int `json:"resources_total"` + ThroughputPerMinute map[string]float64 `json:"throughput_per_minute"` + Errors int `json:"errors"` + OrchestrationOverhead float64 `json:"orchestration_overhead"` +} + +type VerifyPhaseResult struct { + DurationSeconds float64 `json:"duration_seconds"` + QueriesExecuted int `json:"queries_executed"` +} + +type SoakPhaseResult struct { + DurationSeconds float64 `json:"duration_seconds"` + SyncCycles int `json:"sync_cycles"` + DiscoveryCycles int `json:"discovery_cycles"` + ResourceCountDelta int `json:"resource_count_delta"` + ActorCountDelta int `json:"actor_count_delta"` +} + +type DestroyPhaseResult struct { + DurationSeconds float64 `json:"duration_seconds"` + ResourcesDestroyed int `json:"resources_destroyed"` + Errors int `json:"errors"` +} + +type ReportResourceUtil struct { + AgentPeakCPUPercent float64 `json:"agent_peak_cpu_percent"` + AgentPeakMemoryMB float64 `json:"agent_peak_memory_mb"` + AgentAvgCPUPercent float64 `json:"agent_avg_cpu_percent"` + AgentAvgMemoryMB float64 `json:"agent_avg_memory_mb"` +} + +type ReportError struct { + Phase string `json:"phase"` + Message string `json:"message"` +} + +func NewReport(cfg *Config) *Report { + spec := cfg.ProfileSpec() + return &Report{ + Metadata: ReportMetadata{ + Timestamp: time.Now(), + Profile: string(cfg.Profile), + AgentSpec: fmt.Sprintf("%s vCPU / %d MB", spec.AgentVCPU, spec.AgentMemoryMB), + GitSHA: cfg.GitSHA, + RunID: cfg.RunID, + }, + Errors: []ReportError{}, + LogWarnings: []string{}, + } +} + +func (r *Report) WriteToFile(path string) error { + data, err := json.MarshalIndent(r, "", " ") + if err != nil { + return fmt.Errorf("marshal report: %w", err) + } + return os.WriteFile(path, data, 0644) +} diff --git a/tests/loadtest/harness/report_test.go b/tests/loadtest/harness/report_test.go new file mode 100644 index 000000000..362b8bf47 --- /dev/null +++ b/tests/loadtest/harness/report_test.go @@ -0,0 +1,46 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package harness + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReport_WriteToFile(t *testing.T) { + cfg := &Config{ + AgentURL: "http://localhost:49684", + Profile: ProfileSmall, + RunID: "test-run-123", + GitSHA: "abc123", + } + report := NewReport(cfg) + report.Phases.Apply = ApplyPhaseResult{ + DurationSeconds: 120.5, + ResourcesTotal: 500, + ThroughputPerMinute: map[string]float64{"aws": 45.0, "azure": 33.0, "gcp": 33.0}, + Errors: 0, + } + + dir := t.TempDir() + path := filepath.Join(dir, "report.json") + require.NoError(t, report.WriteToFile(path)) + + data, err := os.ReadFile(path) + require.NoError(t, err) + + var loaded Report + require.NoError(t, json.Unmarshal(data, &loaded)) + + assert.Equal(t, "small", loaded.Metadata.Profile) + assert.Equal(t, "test-run-123", loaded.Metadata.RunID) + assert.Equal(t, 500, loaded.Phases.Apply.ResourcesTotal) + assert.InDelta(t, 45.0, loaded.Phases.Apply.ThroughputPerMinute["aws"], 0.01) +} diff --git a/tests/loadtest/loadtest_test.go b/tests/loadtest/loadtest_test.go new file mode 100644 index 000000000..6d30b1801 --- /dev/null +++ b/tests/loadtest/loadtest_test.go @@ -0,0 +1,128 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +//go:build loadtest + +package loadtest + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/platform-engineering-labs/formae/internal/schema" + _ "github.com/platform-engineering-labs/formae/internal/schema/all" + "github.com/platform-engineering-labs/formae/pkg/model" + "github.com/platform-engineering-labs/formae/tests/loadtest/harness" +) + +func TestLoadTest(t *testing.T) { + cfg := configFromEnv(t) + logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) + + h, err := harness.New(cfg, logger) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), cfg.ProfileSpec().MaxDuration) + defer cancel() + + // Phase 1: Health check + require.NoError(t, h.WaitForHealth(ctx, 5*time.Minute)) + + // Phase 2: Apply + phaseStart := time.Now() + forma := loadForma(t, cfg.FormaFilePath) + require.NoError(t, h.RunApplyPhase(ctx, forma)) + h.CheckLogs(ctx, phaseStart, "apply") + + // Phase 3: Verify + phaseStart = time.Now() + require.NoError(t, h.RunVerifyPhase(ctx)) + h.CheckLogs(ctx, phaseStart, "verify") + + // Phase 4: Soak + phaseStart = time.Now() + require.NoError(t, h.RunSoakPhase(ctx)) + h.CheckLogs(ctx, phaseStart, "soak") + + // Phase 5: Destroy + phaseStart = time.Now() + require.NoError(t, h.RunDestroyPhase(ctx)) + h.CheckLogs(ctx, phaseStart, "destroy") + + // Collect metrics and write report + h.CollectMetrics(ctx) + + reportPath := os.Getenv("LOADTEST_REPORT_PATH") + if reportPath == "" { + reportPath = "loadtest-report.json" + } + require.NoError(t, h.WriteReport(reportPath)) + t.Logf("Load test report written to %s", reportPath) +} + +func configFromEnv(t *testing.T) *harness.Config { + t.Helper() + + agentURL := os.Getenv("LOADTEST_AGENT_URL") + if agentURL == "" { + t.Skip("LOADTEST_AGENT_URL not set, skipping load test") + } + + profile := harness.ScaleProfile(os.Getenv("LOADTEST_PROFILE")) + if profile == "" { + profile = harness.ProfileSmall + } + + soakDuration := 15 * time.Minute + if d := os.Getenv("LOADTEST_SOAK_DURATION"); d != "" { + parsed, err := time.ParseDuration(d) + require.NoError(t, err) + soakDuration = parsed + } + + spec := harness.Profiles[profile] + + return &harness.Config{ + AgentURL: agentURL, + Profile: profile, + SoakDuration: soakDuration, + ApplyTimeout: spec.MaxDuration, + DestroyTimeout: spec.MaxDuration, + PhaseTimeout: 30 * time.Minute, + MimirURL: os.Getenv("LOADTEST_MIMIR_URL"), + TempoURL: os.Getenv("LOADTEST_TEMPO_URL"), + LokiURL: os.Getenv("LOADTEST_LOKI_URL"), + FormaFilePath: os.Getenv("LOADTEST_FORMA_FILE"), + RunID: os.Getenv("LOADTEST_RUN_ID"), + GitSHA: os.Getenv("GITHUB_SHA"), + } +} + +// loadForma evaluates the forma file at path and returns the parsed Forma. +// If path is empty, the test is skipped with an informative message. +func loadForma(t *testing.T, path string) *model.Forma { + t.Helper() + + if path == "" { + t.Skip("LOADTEST_FORMA_FILE not set, skipping load test") + } + + absPath, err := filepath.Abs(path) + require.NoError(t, err, "resolve forma file path") + + ext := filepath.Ext(absPath) + plugin, err := schema.DefaultRegistry.GetByFileExtension(ext) + require.NoError(t, err, "find schema plugin for extension %q", ext) + + forma, err := plugin.Evaluate(absPath, model.CommandApply, model.FormaApplyModeReconcile, nil) + require.NoError(t, err, "evaluate forma file %q", absPath) + + return forma +} From 4249938affc7a713967ddb245f5d5393129e9b31 Mon Sep 17 00:00:00 2001 From: Jeroen Soeters Date: Sun, 19 Apr 2026 19:57:47 -0700 Subject: [PATCH 4/4] feat(loadtest): add LGTM observability stack forma Add the Suite B observability stack as a formae forma: a VPC-local LGTM deployment (Loki, Grafana, Tempo, Mimir, OTel collector) on ECS Fargate with an ALB, S3 backing buckets, EFS for shared state, and the IAM wiring to let the task pull images and write logs. Routes depend on the VPCGatewayAttachment resolvable rather than the internet gateway directly so the create/destroy ordering stays correct. This is the resident observability environment the harness scrapes metrics and logs from during load tests; applying it successfully is also our end-to-end validation of running 0.84 in production. --- tests/loadtest/infrastructure/lgtm/PklProject | 10 + tests/loadtest/infrastructure/lgtm/main.pkl | 595 ++++++++++++++++++ 2 files changed, 605 insertions(+) create mode 100644 tests/loadtest/infrastructure/lgtm/PklProject create mode 100644 tests/loadtest/infrastructure/lgtm/main.pkl diff --git a/tests/loadtest/infrastructure/lgtm/PklProject b/tests/loadtest/infrastructure/lgtm/PklProject new file mode 100644 index 000000000..54483f8e3 --- /dev/null +++ b/tests/loadtest/infrastructure/lgtm/PklProject @@ -0,0 +1,10 @@ +amends "pkl:Project" + +dependencies { + ["formae"] { + uri = "package://hub.platform.engineering/plugins/pkl/schema/pkl/formae/formae@0.83.1" + } + ["aws"] { + uri = "package://hub.platform.engineering/plugins/aws/schema/pkl/aws/aws@0.1.6" + } +} diff --git a/tests/loadtest/infrastructure/lgtm/main.pkl b/tests/loadtest/infrastructure/lgtm/main.pkl new file mode 100644 index 000000000..ab6882768 --- /dev/null +++ b/tests/loadtest/infrastructure/lgtm/main.pkl @@ -0,0 +1,595 @@ +/* + * LGTM Observability Stack for Load Testing + * + * Deploys Grafana, Mimir, Tempo, Loki, and an OTel Collector on ECS Fargate + * with S3 backends. All components run in a single ECS task sharing localhost. + * + * Prerequisites: + * - formae agent with AWS plugin running + * - GRAFANA_AUTH=admin:admin set as env var on the agent + * + * Apply: + * formae apply --mode reconcile tests/loadtest/infrastructure/lgtm/main.pkl + * + * After apply, the Grafana URL is the ALB endpoint on port 80. + * OTel ingestion is on the same ALB at /v1/traces, /v1/metrics, /v1/logs. + */ +amends "@formae/forma.pkl" +import "@formae/formae.pkl" +import "@aws/aws.pkl" + +import "@aws/ec2/vpc.pkl" +import "@aws/ec2/subnet.pkl" +import "@aws/ec2/internetgateway.pkl" +import "@aws/ec2/vpcgatewayattachment.pkl" +import "@aws/ec2/routetable.pkl" +import "@aws/ec2/route.pkl" +import "@aws/ec2/subnetroutetableassociation.pkl" as srtAssoc +import "@aws/ec2/securitygroup.pkl" as sg +import "@aws/ec2/securitygroupingress.pkl" as sgIngress +import "@aws/ec2/securitygroupegress.pkl" as sgEgress + +import "@aws/s3/bucket.pkl" +import "@aws/efs/filesystem.pkl" as efsFs +import "@aws/efs/mounttarget.pkl" as efsMt + +import "@aws/iam/role.pkl" +import "@aws/iam/rolepolicy.pkl" +import "@aws/logs/loggroup.pkl" + +import "@aws/ecs/ecscluster.pkl" as cluster +import "@aws/ecs/taskdefinition.pkl" as td +import "@aws/ecs/service.pkl" as ecsSvc + +import "@aws/elasticloadbalancingv2/loadbalancer.pkl" as alb +import "@aws/elasticloadbalancingv2/targetgroup.pkl" as tg +import "@aws/elasticloadbalancingv2/listener.pkl" as listener + +local awsRegion = "us-west-2" +local prefix = "formae-lgtm" + +description { + text = """ + LGTM Observability Stack for Load Testing + + Deploys Grafana + Mimir + Tempo + Loki + OTel Collector on ECS Fargate. + S3 for durable storage, EFS for WAL, ALB for access. + + Cost: ~$130/month (Fargate + S3 + EFS + ALB) + """ + confirm = true +} + +forma { + // Stack and Target + new formae.Stack { + label = "lgtm" + description = "LGTM observability stack for load testing" + } + + new formae.Target { + label = "lgtm-aws" + config = new aws.Config { + region = awsRegion + } + } + + // ─── Networking ─── + + local lgtmVpc = new vpc.VPC { + label = "lgtm-vpc" + cidrBlock = "10.100.0.0/16" + enableDnsHostnames = true + enableDnsSupport = true + tags { new { key = "Name"; value = "\(prefix)-vpc" } } + } + lgtmVpc + + local igw = new internetgateway.InternetGateway { + label = "lgtm-igw" + tags { new { key = "Name"; value = "\(prefix)-igw" } } + } + igw + + local igwAttach = new vpcgatewayattachment.VPCGatewayAttachment { + label = "lgtm-igw-attach" + vpcId = lgtmVpc.res.id + internetGatewayId = igw.res.id + } + igwAttach + + local rt = new routetable.RouteTable { + label = "lgtm-rt" + vpcId = lgtmVpc.res.id + tags { new { key = "Name"; value = "\(prefix)-rt" } } + } + rt + + new route.Route { + label = "lgtm-route-inet" + routeTableId = rt.res.id + destinationCidrBlock = "0.0.0.0/0" + gatewayId = igwAttach.res.internetGatewayId + } + + local subnetA = new subnet.Subnet { + label = "lgtm-subnet-a" + vpcId = lgtmVpc.res.id + cidrBlock = "10.100.1.0/24" + availabilityZone = "\(awsRegion)a" + mapPublicIpOnLaunch = true + tags { new { key = "Name"; value = "\(prefix)-subnet-a" } } + } + subnetA + + local subnetB = new subnet.Subnet { + label = "lgtm-subnet-b" + vpcId = lgtmVpc.res.id + cidrBlock = "10.100.2.0/24" + availabilityZone = "\(awsRegion)b" + mapPublicIpOnLaunch = true + tags { new { key = "Name"; value = "\(prefix)-subnet-b" } } + } + subnetB + + new srtAssoc.SubnetRouteTableAssociation { + label = "lgtm-srt-a" + subnetId = subnetA.res.id + routeTableId = rt.res.id + } + + new srtAssoc.SubnetRouteTableAssociation { + label = "lgtm-srt-b" + subnetId = subnetB.res.id + routeTableId = rt.res.id + } + + // ─── Security Group ─── + + local lgtmSg = new sg.SecurityGroup { + label = "lgtm-sg" + groupDescription = "LGTM stack - Grafana + OTel ingestion" + vpcId = lgtmVpc.res.id + tags { new { key = "Name"; value = "\(prefix)-sg" } } + } + lgtmSg + + new sgIngress.SecurityGroupIngress { + label = "lgtm-sg-http" + groupId = lgtmSg.res.id + ipProtocol = "tcp" + fromPort = 80 + toPort = 80 + cidrIp = "0.0.0.0/0" + } + + new sgIngress.SecurityGroupIngress { + label = "lgtm-sg-grafana" + groupId = lgtmSg.res.id + ipProtocol = "tcp" + fromPort = 3000 + toPort = 3000 + cidrIp = "10.100.0.0/16" + } + + new sgIngress.SecurityGroupIngress { + label = "lgtm-sg-otel" + groupId = lgtmSg.res.id + ipProtocol = "tcp" + fromPort = 4318 + toPort = 4318 + cidrIp = "0.0.0.0/0" + } + + new sgEgress.SecurityGroupEgress { + label = "lgtm-sg-egress" + groupId = lgtmSg.res.id + ipProtocol = "-1" + cidrIp = "0.0.0.0/0" + } + + // ─── Storage: S3 Buckets ─── + + new bucket.Bucket { + label = "mimir-bucket" + bucketName = "\(prefix)-mimir-blocks" + tags { new { key = "Name"; value = "\(prefix)-mimir-blocks" } } + } + + new bucket.Bucket { + label = "loki-bucket" + bucketName = "\(prefix)-loki-data" + tags { new { key = "Name"; value = "\(prefix)-loki-data" } } + } + + new bucket.Bucket { + label = "tempo-bucket" + bucketName = "\(prefix)-tempo-traces" + tags { new { key = "Name"; value = "\(prefix)-tempo-traces" } } + } + + // ─── Storage: EFS for WAL ─── + + local efs = new efsFs.FileSystem { + label = "lgtm-efs" + performanceMode = "generalPurpose" + encrypted = true + fileSystemTags { + new { key = "Name"; value = "\(prefix)-wal" } + } + } + efs + + new efsMt.MountTarget { + label = "lgtm-efs-mt-a" + fileSystemId = efs.res.fileSystemId + subnetId = subnetA.res.id + securityGroups { lgtmSg.res.id } + } + + new efsMt.MountTarget { + label = "lgtm-efs-mt-b" + fileSystemId = efs.res.fileSystemId + subnetId = subnetB.res.id + securityGroups { lgtmSg.res.id } + } + + // ─── IAM ─── + + local execRole = new role.Role { + label = "lgtm-exec-role" + roleName = "\(prefix)-exec-role" + assumeRolePolicyDocument = new Dynamic { + Version = "2012-10-17" + Statement = new Listing { + new Dynamic { + Effect = "Allow" + Principal = new Dynamic { Service = "ecs-tasks.amazonaws.com" } + Action = "sts:AssumeRole" + } + } + } + managedPolicyArns { + "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + } + } + execRole + + local taskRole = new role.Role { + label = "lgtm-task-role" + roleName = "\(prefix)-task-role" + assumeRolePolicyDocument = new Dynamic { + Version = "2012-10-17" + Statement = new Listing { + new Dynamic { + Effect = "Allow" + Principal = new Dynamic { Service = "ecs-tasks.amazonaws.com" } + Action = "sts:AssumeRole" + } + } + } + } + taskRole + + new rolepolicy.RolePolicy { + label = "lgtm-task-policy" + roleName = taskRole.res.roleName + policyName = "\(prefix)-s3-efs" + policyDocument = new Dynamic { + Version = "2012-10-17" + Statement = new Listing { + new Dynamic { + Effect = "Allow" + Action = new Listing { + "s3:GetObject"; "s3:PutObject"; "s3:DeleteObject"; "s3:ListBucket" + } + Resource = new Listing { + "arn:aws:s3:::\(prefix)-mimir-blocks" + "arn:aws:s3:::\(prefix)-mimir-blocks/*" + "arn:aws:s3:::\(prefix)-loki-data" + "arn:aws:s3:::\(prefix)-loki-data/*" + "arn:aws:s3:::\(prefix)-tempo-traces" + "arn:aws:s3:::\(prefix)-tempo-traces/*" + } + } + new Dynamic { + Effect = "Allow" + Action = new Listing { + "elasticfilesystem:ClientMount" + "elasticfilesystem:ClientWrite" + } + Resource = "*" + } + } + } + } + + // ─── Logging ─── + + local logGroup = new loggroup.LogGroup { + label = "lgtm-logs" + logGroupName = "/ecs/\(prefix)" + retentionInDays = 30 + } + logGroup + + // ─── ECS ─── + + local ecsCluster = new cluster.Cluster { + label = "lgtm-cluster" + clusterName = prefix + } + ecsCluster + + local taskDef = new td.TaskDefinition { + label = "lgtm-task" + family = prefix + networkMode = "awsvpc" + requiresCompatibilities { "FARGATE" } + cpu = "2048" + memory = "8192" + executionRoleArn = execRole.res.arn + taskRoleArn = taskRole.res.arn + + volumes { + new td.Volume { + name = "wal" + eFSVolumeConfiguration = new td.EFSVolumeConfiguration { + filesystemId = efs.res.fileSystemId + transitEncryption = "ENABLED" + } + } + } + + containerDefinitions { + // Grafana + new td.ContainerDefinition { + name = "grafana" + image = "grafana/grafana:latest" + essential = true + portMappings { + new td.PortMapping { containerPort = 3000; protocol = "tcp" } + } + environment { + new td.KeyValuePair { name = "GF_AUTH_ANONYMOUS_ENABLED"; value = "true" } + new td.KeyValuePair { name = "GF_AUTH_ANONYMOUS_ORG_ROLE"; value = "Admin" } + new td.KeyValuePair { name = "GF_AUTH_DISABLE_LOGIN_FORM"; value = "true" } + } + logConfiguration = new td.LogConfiguration { + logDriver = "awslogs" + options = new Mapping { + ["awslogs-group"] = "/ecs/\(prefix)" + ["awslogs-region"] = awsRegion + ["awslogs-stream-prefix"] = "grafana" + } + } + } + + // Mimir + new td.ContainerDefinition { + name = "mimir" + essential = false + image = "grafana/mimir:latest" + command { + "-target=all" + "-server.http-listen-port=9009" + "-blocks-storage.backend=s3" + "-blocks-storage.s3.bucket-name=\(prefix)-mimir-blocks" + "-blocks-storage.s3.region=\(awsRegion)" + "-blocks-storage.tsdb.dir=/data/mimir/tsdb" + "-compactor.data-dir=/data/mimir/compactor" + } + portMappings { + new td.PortMapping { containerPort = 9009; protocol = "tcp" } + } + mountPoints { + new td.MountPoint { sourceVolume = "wal"; containerPath = "/data/mimir"; readOnly = false } + } + logConfiguration = new td.LogConfiguration { + logDriver = "awslogs" + options = new Mapping { + ["awslogs-group"] = "/ecs/\(prefix)" + ["awslogs-region"] = awsRegion + ["awslogs-stream-prefix"] = "mimir" + } + } + } + + // Loki + new td.ContainerDefinition { + name = "loki" + essential = false + image = "grafana/loki:latest" + command { + "-target=all" + "-config.expand-env=true" + } + environment { + new td.KeyValuePair { name = "LOKI_S3_BUCKETNAME"; value = "\(prefix)-loki-data" } + new td.KeyValuePair { name = "LOKI_S3_REGION"; value = awsRegion } + } + portMappings { + new td.PortMapping { containerPort = 3100; protocol = "tcp" } + } + mountPoints { + new td.MountPoint { sourceVolume = "wal"; containerPath = "/data/loki"; readOnly = false } + } + logConfiguration = new td.LogConfiguration { + logDriver = "awslogs" + options = new Mapping { + ["awslogs-group"] = "/ecs/\(prefix)" + ["awslogs-region"] = awsRegion + ["awslogs-stream-prefix"] = "loki" + } + } + } + + // Tempo + // Tempo + new td.ContainerDefinition { + name = "tempo" + essential = false + image = "grafana/tempo:latest" + command { + "-target=all" + "-storage.trace.backend=s3" + "-storage.trace.s3.bucket=\(prefix)-tempo-traces" + "-storage.trace.s3.region=\(awsRegion)" + "-storage.trace.wal.path=/data/tempo/wal" + "-storage.trace.local.path=/data/tempo/blocks" + "-distributor.receivers.otlp.protocols.http.endpoint=0.0.0.0:4318" + "-distributor.receivers.otlp.protocols.grpc.endpoint=0.0.0.0:4317" + "-server.http-listen-port=3200" + } + portMappings { + new td.PortMapping { containerPort = 3200; protocol = "tcp" } + new td.PortMapping { containerPort = 4317; protocol = "tcp" } + new td.PortMapping { containerPort = 4318; protocol = "tcp" } + } + mountPoints { + new td.MountPoint { sourceVolume = "wal"; containerPath = "/data/tempo"; readOnly = false } + } + logConfiguration = new td.LogConfiguration { + logDriver = "awslogs" + options = new Mapping { + ["awslogs-group"] = "/ecs/\(prefix)" + ["awslogs-region"] = awsRegion + ["awslogs-stream-prefix"] = "tempo" + } + } + } + + // OTel Collector — reads config from OTEL_CONFIG env var + // Non-essential so a config error doesn't kill the whole task + new td.ContainerDefinition { + name = "otel-collector" + essential = false + image = "otel/opentelemetry-collector-contrib:latest" + command { + "--config=env:OTEL_CONFIG" + } + environment { + new td.KeyValuePair { + name = "OTEL_CONFIG" + value = new Listing { + "receivers:" + " otlp:" + " protocols:" + " http:" + " endpoint: 0.0.0.0:4318" + " grpc:" + " endpoint: 0.0.0.0:4317" + "exporters:" + " otlphttp/mimir:" + " endpoint: http://localhost:9009/otlp" + " tls:" + " insecure: true" + " otlphttp/loki:" + " endpoint: http://localhost:3100/otlp" + " tls:" + " insecure: true" + " otlp/tempo:" + " endpoint: localhost:4317" + " tls:" + " insecure: true" + "service:" + " pipelines:" + " metrics:" + " receivers: [otlp]" + " exporters: [otlphttp/mimir]" + " logs:" + " receivers: [otlp]" + " exporters: [otlphttp/loki]" + " traces:" + " receivers: [otlp]" + " exporters: [otlp/tempo]" + }.join("\n") + } + } + logConfiguration = new td.LogConfiguration { + logDriver = "awslogs" + options = new Mapping { + ["awslogs-group"] = "/ecs/\(prefix)" + ["awslogs-region"] = awsRegion + ["awslogs-stream-prefix"] = "otel" + } + } + } + } + } + taskDef + + // ─── ALB ─── + + local lb = new alb.LoadBalancer { + label = "lgtm-alb" + name = prefix + scheme = "internet-facing" + type = "application" + subnets { + subnetA.res.id + subnetB.res.id + } + securityGroups { + lgtmSg.res.id + } + tags { new { key = "Name"; value = "\(prefix)-alb" } } + } + lb + + local grafanaTg = new tg.TargetGroup { + label = "lgtm-grafana-tg" + name = "\(prefix)-grafana" + port = 3000 + protocol = "HTTP" + targetType = "ip" + vpcId = lgtmVpc.res.id + healthCheckPath = "/api/health" + healthCheckIntervalSeconds = 30 + tags { new { key = "Name"; value = "\(prefix)-grafana-tg" } } + } + grafanaTg + + new listener.Listener { + label = "lgtm-listener" + loadBalancerArn = lb.res.loadBalancerArn + port = 80 + protocol = "HTTP" + defaultActions { + new listener.Action { + type = "forward" + targetGroupArn = grafanaTg.res.targetGroupArn + } + } + } + + // ─── ECS Service ─── + + new ecsSvc.Service { + label = "lgtm-service" + serviceName = prefix + cluster = ecsCluster.res.arn + taskDefinition = taskDef.res.taskDefinitionArn + desiredCount = 1 + launchType = "FARGATE" + networkConfiguration = new ecsSvc.NetworkConfiguration { + awsvpcConfiguration = new ecsSvc.AwsVpcConfiguration { + subnets { + subnetA.res.id + subnetB.res.id + } + securityGroups { + lgtmSg.res.id + } + assignPublicIp = "ENABLED" + } + } + loadBalancers { + new ecsSvc.LoadBalancer { + targetGroupArn = grafanaTg.res.targetGroupArn + containerName = "grafana" + containerPort = 3000 + } + } + } +}