From 894a831899f03eee3935d80d8b5af41f1fb91c58 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Wed, 11 Feb 2026 18:26:36 +0000 Subject: [PATCH] feat(logs): add storage tier support for Flex logs Adds --storage flag to all log search commands (search, list, query, aggregate) enabling users to query specific Datadog log storage tiers: indexes (standard), online-archives (long-term), and flex (cost-optimized). Changes: - Added logsStorage flag variable in cmd/logs_simple.go:548 - Implemented validateAndConvertStorageTier() helper function (cmd/logs_simple.go:685-705) - Added --storage flag to logs search, list, query, aggregate commands - Modified runLogsSearch(), runLogsList(), runLogsQuery(), runLogsAggregate() to validate and apply storage tier - Added comprehensive test coverage with 17 test cases in cmd/logs_simple_test.go - Updated docs/EXAMPLES.md with storage tier usage examples - Updated docs/COMMANDS.md with Flex logs example Features: - Validates storage tier before API calls for fast failure - Case-insensitive input handling (FLEX, flex, Flex all work) - Whitespace trimming for user convenience - Helpful error messages showing valid options - Backward compatible: omitting --storage searches all tiers (default behavior) Testing: - TestValidateAndConvertStorageTier: 9 test cases covering validation logic - TestRunLogsSearchWithStorageTier: 5 test cases for search command - TestRunLogsQueryWithStorageTier: 2 test cases for query command - TestRunLogsAggregateWithStorageTier: 1 test case for aggregate command - All tests pass with existing test suite - Full cmd package coverage maintained Co-Authored-By: Claude Sonnet 4.5 --- cmd/logs_simple.go | 130 ++++++++++++++++-- cmd/logs_simple_test.go | 291 ++++++++++++++++++++++++++++++++++++++-- docs/COMMANDS.md | 1 + docs/EXAMPLES.md | 15 +++ 4 files changed, 415 insertions(+), 22 deletions(-) diff --git a/cmd/logs_simple.go b/cmd/logs_simple.go index ec760382..e7d02f98 100644 --- a/cmd/logs_simple.go +++ b/cmd/logs_simple.go @@ -30,11 +30,18 @@ CAPABILITIES: • Search logs with flexible queries (v1 API) • Query and aggregate logs (v2 API) • List logs with filtering (v2 API) + • Search across different storage tiers (indexes, online-archives, flex) • Manage log archives (CRUD operations) • Manage custom destinations for logs • Create and manage log-based metrics • Configure restriction queries for access control +STORAGE TIERS: + Datadog logs can be stored in different tiers with different performance and cost characteristics: + • indexes - Standard indexed logs (default, real-time searchable) + • online-archives - Rehydrated logs from archives (slower queries, lower cost) + • flex - Flex logs (cost-optimized storage tier, balanced performance) + LOG QUERY SYNTAX: Logs use a query language similar to web search: • status:error - Match by status @@ -54,9 +61,15 @@ EXAMPLES: # Search for error logs in the last hour pup logs search --query="status:error" --from="1h" + # Search Flex logs specifically + pup logs search --query="status:error" --from="1h" --storage="flex" + # Query logs from a specific service pup logs query --query="service:web-app" --from="4h" --to="now" + # Query online archives + pup logs query --query="service:web-app" --from="30d" --storage="online-archives" + # Aggregate logs by status pup logs aggregate --query="*" --compute="count" --group-by="status" @@ -114,11 +127,18 @@ OPTIONS: --limit Maximum number of logs to return (default: 50, max: 1000) --sort Sort order: asc or desc (default: desc) --index Comma-separated list of log indexes to search + --storage Storage tier to search: indexes, online-archives, or flex (default: all tiers) EXAMPLES: # Search for errors in the last hour pup logs search --query="status:error" --from="1h" + # Search Flex logs for errors + pup logs search --query="status:error" --from="1h" --storage="flex" + + # Search online archives + pup logs search --query="service:api" --from="30d" --storage="online-archives" + # Search specific service with time range pup logs search --query="service:api" --from="2h" --to="1h" @@ -156,11 +176,18 @@ FILTERS: --to End time (default: now) --limit Number of logs to return (default: 10) --sort Sort order: timestamp, -timestamp (default: -timestamp) + --storage Storage tier: indexes, online-archives, or flex (default: all tiers) EXAMPLES: # List recent logs pup logs list --from="1h" + # List Flex logs with query filter + pup logs list --query="service:api" --from="2h" --limit=50 --storage="flex" + + # List logs from online archives + pup logs list --query="*" --from="30d" --storage="online-archives" + # List logs with query filter pup logs list --query="service:api" --from="2h" --limit=50 @@ -180,17 +207,24 @@ This is the recommended modern API for querying logs with better performance and more features than the v1 search API. OPTIONS: - --query Log query (required) - --from Start time (required) - --to End time (default: now) - --limit Maximum results (default: 50) - --sort Sort order: timestamp, -timestamp + --query Log query (required) + --from Start time (required) + --to End time (default: now) + --limit Maximum results (default: 50) + --sort Sort order: timestamp, -timestamp --timezone Timezone for timestamps (e.g., "America/New_York") + --storage Storage tier: indexes, online-archives, or flex (default: all tiers) EXAMPLES: # Query recent errors pup logs query --query="status:error" --from="1h" + # Query Flex logs + pup logs query --query="status:error" --from="1h" --storage="flex" + + # Query online archives + pup logs query --query="service:web" --from="30d" --storage="online-archives" + # Query with specific timezone pup logs query --query="service:web" --from="4h" --timezone="America/New_York" @@ -214,6 +248,7 @@ AGGREGATION OPTIONS: --compute Metric to compute (count, cardinality, percentile, etc.) --group-by Field to group by (e.g., "status", "service", "@http.status_code") --limit Maximum number of groups (default: 10) + --storage Storage tier: indexes, online-archives, or flex (default: all tiers) COMPUTE METRICS: • count: Count of logs @@ -228,6 +263,12 @@ EXAMPLES: # Count logs by status pup logs aggregate --query="*" --from="1h" --compute="count" --group-by="status" + # Count Flex logs by status + pup logs aggregate --query="*" --from="1h" --compute="count" --group-by="status" --storage="flex" + + # Count unique users in online archives + pup logs aggregate --query="service:web" --from="30d" --compute="cardinality(@user.id)" --storage="online-archives" + # Count unique users pup logs aggregate --query="service:web" --from="4h" --compute="cardinality(@user.id)" @@ -505,6 +546,7 @@ var ( logsSort string logsIndex string logsTimezone string + logsStorage string // Aggregate flags logsCompute string @@ -519,6 +561,7 @@ func init() { logsSearchCmd.Flags().IntVar(&logsLimit, "limit", 50, "Maximum number of logs (1-1000)") logsSearchCmd.Flags().StringVar(&logsSort, "sort", "desc", "Sort order: asc or desc") logsSearchCmd.Flags().StringVar(&logsIndex, "index", "", "Comma-separated log indexes") + logsSearchCmd.Flags().StringVar(&logsStorage, "storage", "", "Storage tier: indexes, online-archives, or flex") if err := logsSearchCmd.MarkFlagRequired("query"); err != nil { panic(fmt.Errorf("failed to mark flag as required: %w", err)) } @@ -532,6 +575,7 @@ func init() { logsListCmd.Flags().StringVar(&logsTo, "to", "now", "End time") logsListCmd.Flags().IntVar(&logsLimit, "limit", 10, "Number of logs") logsListCmd.Flags().StringVar(&logsSort, "sort", "-timestamp", "Sort order") + logsListCmd.Flags().StringVar(&logsStorage, "storage", "", "Storage tier: indexes, online-archives, or flex") if err := logsListCmd.MarkFlagRequired("from"); err != nil { panic(fmt.Errorf("failed to mark flag as required: %w", err)) } @@ -543,6 +587,7 @@ func init() { logsQueryCmd.Flags().IntVar(&logsLimit, "limit", 50, "Maximum results") logsQueryCmd.Flags().StringVar(&logsSort, "sort", "-timestamp", "Sort order") logsQueryCmd.Flags().StringVar(&logsTimezone, "timezone", "", "Timezone for timestamps") + logsQueryCmd.Flags().StringVar(&logsStorage, "storage", "", "Storage tier: indexes, online-archives, or flex") if err := logsQueryCmd.MarkFlagRequired("query"); err != nil { panic(fmt.Errorf("failed to mark flag as required: %w", err)) } @@ -557,6 +602,7 @@ func init() { logsAggregateCmd.Flags().StringVar(&logsCompute, "compute", "count", "Metric to compute") logsAggregateCmd.Flags().StringVar(&logsGroupBy, "group-by", "", "Field to group by") logsAggregateCmd.Flags().IntVar(&logsLimit, "limit", 10, "Maximum groups") + logsAggregateCmd.Flags().StringVar(&logsStorage, "storage", "", "Storage tier: indexes, online-archives, or flex") if err := logsAggregateCmd.MarkFlagRequired("query"); err != nil { panic(fmt.Errorf("failed to mark flag as required: %w", err)) } @@ -636,6 +682,27 @@ func parseTimeString(timeStr string) (int64, error) { return 0, fmt.Errorf("invalid time format: %s (use relative like '1h' or Unix timestamp)", timeStr) } +// validateAndConvertStorageTier validates the storage tier string and converts it to LogsStorageTier +// Returns nil if storage is empty (which means search all tiers) +func validateAndConvertStorageTier(storage string) (*datadogV2.LogsStorageTier, error) { + if storage == "" { + return nil, nil + } + + // Validate storage tier value + validTiers := []string{"indexes", "online-archives", "flex"} + storageNormalized := strings.ToLower(strings.TrimSpace(storage)) + + for _, valid := range validTiers { + if storageNormalized == valid { + tier := datadogV2.LogsStorageTier(storageNormalized) + return &tier, nil + } + } + + return nil, fmt.Errorf("invalid storage tier: %q\n\nValid options:\n - indexes (standard indexed logs)\n - online-archives (rehydrated logs from archives)\n - flex (cost-optimized Flex logs)", storage) +} + // parseComputeString parses compute strings like "count", "avg(@duration)", "percentile(@duration, 99)" // and returns the aggregation function and metric field func parseComputeString(compute string) (aggregation string, metric string, err error) { @@ -709,7 +776,8 @@ func parseComputeString(compute string) (aggregation string, metric string, err // Implementation functions func runLogsSearch(cmd *cobra.Command, args []string) error { - client, err := getClient() + // Validate storage tier before creating client + storageTier, err := validateAndConvertStorageTier(logsStorage) if err != nil { return err } @@ -724,7 +792,11 @@ func runLogsSearch(cmd *cobra.Command, args []string) error { return fmt.Errorf("invalid --to time: %w", err) } - // Use v2 API instead of deprecated v1 API + client, err := getClient() + if err != nil { + return err + } + api := datadogV2.NewLogsApi(client.V2()) query := logsQuery @@ -750,6 +822,11 @@ func runLogsSearch(cmd *cobra.Command, args []string) error { Sort: &v2Sort, } + // Set storage tier if provided + if storageTier != nil { + body.Filter.StorageTier = storageTier + } + // Note: v2 API doesn't support the index parameter the same way v1 did // If index filtering is needed, it should be included in the query string @@ -847,7 +924,8 @@ func runLogsSearch(cmd *cobra.Command, args []string) error { } func runLogsList(cmd *cobra.Command, args []string) error { - client, err := getClient() + // Validate storage tier before creating client + storageTier, err := validateAndConvertStorageTier(logsStorage) if err != nil { return err } @@ -862,6 +940,11 @@ func runLogsList(cmd *cobra.Command, args []string) error { return fmt.Errorf("invalid --to time: %w", err) } + client, err := getClient() + if err != nil { + return err + } + api := datadogV2.NewLogsApi(client.V2()) query := logsQuery @@ -884,6 +967,11 @@ func runLogsList(cmd *cobra.Command, args []string) error { }, } + // Set storage tier if provided + if storageTier != nil { + opts.Body.Filter.StorageTier = storageTier + } + resp, r, err := api.ListLogs(client.Context(), opts) if err != nil { if r != nil && r.Body != nil { @@ -906,7 +994,8 @@ func runLogsList(cmd *cobra.Command, args []string) error { } func runLogsQuery(cmd *cobra.Command, args []string) error { - client, err := getClient() + // Validate storage tier before creating client + storageTier, err := validateAndConvertStorageTier(logsStorage) if err != nil { return err } @@ -921,6 +1010,11 @@ func runLogsQuery(cmd *cobra.Command, args []string) error { return fmt.Errorf("invalid --to time: %w", err) } + client, err := getClient() + if err != nil { + return err + } + api := datadogV2.NewLogsApi(client.V2()) query := logsQuery @@ -941,6 +1035,11 @@ func runLogsQuery(cmd *cobra.Command, args []string) error { Sort: &sort, } + // Set storage tier if provided + if storageTier != nil { + body.Filter.StorageTier = storageTier + } + opts := datadogV2.ListLogsOptionalParameters{ Body: &body, } @@ -967,7 +1066,8 @@ func runLogsQuery(cmd *cobra.Command, args []string) error { } func runLogsAggregate(cmd *cobra.Command, args []string) error { - client, err := getClient() + // Validate storage tier before creating client + storageTier, err := validateAndConvertStorageTier(logsStorage) if err != nil { return err } @@ -988,6 +1088,11 @@ func runLogsAggregate(cmd *cobra.Command, args []string) error { return fmt.Errorf("invalid --compute value: %w", err) } + client, err := getClient() + if err != nil { + return err + } + api := datadogV2.NewLogsApi(client.V2()) // Build compute aggregation @@ -1013,6 +1118,11 @@ func runLogsAggregate(cmd *cobra.Command, args []string) error { }, } + // Set storage tier if provided + if storageTier != nil { + body.Filter.StorageTier = storageTier + } + // Add group by if specified if logsGroupBy != "" { limit := int64(logsLimit) diff --git a/cmd/logs_simple_test.go b/cmd/logs_simple_test.go index f32de896..e95fa3e2 100644 --- a/cmd/logs_simple_test.go +++ b/cmd/logs_simple_test.go @@ -503,9 +503,9 @@ func TestRunLogsArchivesGet(t *testing.T) { defer cleanup() tests := []struct { - name string - archiveID string - wantErr bool + name string + archiveID string + wantErr bool }{ { name: "get archive", @@ -563,9 +563,9 @@ func TestRunLogsMetricsGet(t *testing.T) { defer cleanup() tests := []struct { - name string - metricID string - wantErr bool + name string + metricID string + wantErr bool }{ { name: "get metric", @@ -594,9 +594,9 @@ func TestRunLogsArchivesDelete(t *testing.T) { defer cleanup() tests := []struct { - name string - archiveID string - wantErr bool + name string + archiveID string + wantErr bool }{ { name: "delete archive", @@ -625,9 +625,9 @@ func TestRunLogsMetricsDelete(t *testing.T) { defer cleanup() tests := []struct { - name string - metricID string - wantErr bool + name string + metricID string + wantErr bool }{ { name: "delete metric", @@ -651,6 +651,273 @@ func TestRunLogsMetricsDelete(t *testing.T) { } } +func TestValidateAndConvertStorageTier(t *testing.T) { + tests := []struct { + name string + input string + wantNil bool + wantErr bool + wantTier string + }{ + { + name: "empty string - no storage tier specified", + input: "", + wantNil: true, + wantErr: false, + }, + { + name: "indexes", + input: "indexes", + wantNil: false, + wantErr: false, + wantTier: "indexes", + }, + { + name: "online-archives", + input: "online-archives", + wantNil: false, + wantErr: false, + wantTier: "online-archives", + }, + { + name: "flex", + input: "flex", + wantNil: false, + wantErr: false, + wantTier: "flex", + }, + { + name: "FLEX - uppercase", + input: "FLEX", + wantNil: false, + wantErr: false, + wantTier: "flex", + }, + { + name: "Indexes - mixed case", + input: "Indexes", + wantNil: false, + wantErr: false, + wantTier: "indexes", + }, + { + name: "invalid storage tier", + input: "invalid", + wantNil: true, + wantErr: true, + }, + { + name: "invalid - close but not exact", + input: "archive", + wantNil: true, + wantErr: true, + }, + { + name: "whitespace - should be trimmed", + input: " flex ", + wantNil: false, + wantErr: false, + wantTier: "flex", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := validateAndConvertStorageTier(tt.input) + + if (err != nil) != tt.wantErr { + t.Errorf("validateAndConvertStorageTier() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if (got == nil) != tt.wantNil { + t.Errorf("validateAndConvertStorageTier() returned nil = %v, wantNil %v", got == nil, tt.wantNil) + return + } + + if !tt.wantNil && got != nil && string(*got) != tt.wantTier { + t.Errorf("validateAndConvertStorageTier() tier = %v, want %v", string(*got), tt.wantTier) + } + }) + } +} + +func TestRunLogsSearchWithStorageTier(t *testing.T) { + cleanup := setupLogsTestClient(t) + defer cleanup() + + tests := []struct { + name string + query string + from string + to string + storageTier string + wantErr bool + }{ + { + name: "search with flex storage tier", + query: "status:error", + from: "1h", + to: "now", + storageTier: "flex", + wantErr: true, // Will fail because of mock client, but validates tier parsing + }, + { + name: "search with online-archives", + query: "status:error", + from: "30d", + to: "now", + storageTier: "online-archives", + wantErr: true, + }, + { + name: "search with indexes", + query: "status:error", + from: "1h", + to: "now", + storageTier: "indexes", + wantErr: true, + }, + { + name: "search with invalid storage tier", + query: "status:error", + from: "1h", + to: "now", + storageTier: "invalid", + wantErr: true, + }, + { + name: "search without storage tier", + query: "status:error", + from: "1h", + to: "now", + storageTier: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logsQuery = tt.query + logsFrom = tt.from + logsTo = tt.to + logsStorage = tt.storageTier + + var buf bytes.Buffer + outputWriter = &buf + defer func() { outputWriter = os.Stdout }() + + err := runLogsSearch(logsSearchCmd, []string{}) + + if (err != nil) != tt.wantErr { + t.Errorf("runLogsSearch() error = %v, wantErr %v", err, tt.wantErr) + } + + // For invalid storage tier, check that error message mentions "invalid storage tier" + if tt.storageTier == "invalid" && err != nil { + if !strings.Contains(err.Error(), "invalid storage tier") { + t.Errorf("runLogsSearch() with invalid storage tier should mention 'invalid storage tier', got: %v", err) + } + } + }) + } +} + +func TestRunLogsQueryWithStorageTier(t *testing.T) { + cleanup := setupLogsTestClient(t) + defer cleanup() + + tests := []struct { + name string + query string + from string + to string + storageTier string + wantErr bool + }{ + { + name: "query with flex storage tier", + query: "status:error", + from: "1h", + to: "now", + storageTier: "flex", + wantErr: true, + }, + { + name: "query with invalid storage tier", + query: "status:error", + from: "1h", + to: "now", + storageTier: "bad-tier", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logsQuery = tt.query + logsFrom = tt.from + logsTo = tt.to + logsStorage = tt.storageTier + + var buf bytes.Buffer + outputWriter = &buf + defer func() { outputWriter = os.Stdout }() + + err := runLogsQuery(logsQueryCmd, []string{}) + + if (err != nil) != tt.wantErr { + t.Errorf("runLogsQuery() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestRunLogsAggregateWithStorageTier(t *testing.T) { + cleanup := setupLogsTestClient(t) + defer cleanup() + + tests := []struct { + name string + query string + from string + to string + compute string + storageTier string + wantErr bool + }{ + { + name: "aggregate with flex storage tier", + query: "status:error", + from: "1h", + to: "now", + compute: "count", + storageTier: "flex", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logsQuery = tt.query + logsFrom = tt.from + logsTo = tt.to + logsCompute = tt.compute + logsStorage = tt.storageTier + + var buf bytes.Buffer + outputWriter = &buf + defer func() { outputWriter = os.Stdout }() + + err := runLogsAggregate(logsAggregateCmd, []string{}) + + if (err != nil) != tt.wantErr { + t.Errorf("runLogsAggregate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + func TestRunLogsRestrictionQueriesList(t *testing.T) { cleanup := setupLogsTestClient(t) defer cleanup() diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 70fbc829..13680789 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -81,6 +81,7 @@ pup slos get abc-123-def ### Search/Query ```bash pup logs search --query="status:error" --from="1h" +pup logs search --query="service:api" --from="7d" --storage="flex" pup metrics search --query="avg:system.cpu.user{*}" --from="1h" pup metrics query --query="avg:system.cpu.user{*}" --from="1h" pup events search --query="@user.id:12345" diff --git a/docs/EXAMPLES.md b/docs/EXAMPLES.md index 96e79679..cf1545c8 100644 --- a/docs/EXAMPLES.md +++ b/docs/EXAMPLES.md @@ -116,6 +116,21 @@ pup logs aggregate \ --group-by="status" ``` +### Search Logs in Specific Storage Tier +```bash +# Search Flex logs (cost-optimized storage tier) +pup logs search --query="service:api" --from="7d" --storage="flex" + +# Search online archives (long-term storage) +pup logs search --query="status:error" --from="30d" --storage="online-archives" + +# Search standard indexes (default, fastest tier) +pup logs search --query="service:web-app" --from="1h" --storage="indexes" + +# Search all storage tiers (default when --storage is not specified) +pup logs search --query="status:warn" --from="1h" +``` + ## Dashboards ### List Dashboards