From a6249c4b1bc34caed9596247710a314e5d86e24d Mon Sep 17 00:00:00 2001 From: Dineth Date: Fri, 26 Jun 2026 13:56:30 +0530 Subject: [PATCH 1/5] Add support for capturing request and response headers in analytics - Introduced configuration options for enabling header capture in analytics events. - Added a new Log publisher to output analytics events to stdout. - Updated analytics system policy to handle request and response headers. - Enhanced tests to validate header capture functionality. Enhance analytics configuration with max payload size and ignored path prefixes Remove pretty Refactor analytics and traffic logging configuration to use a unified collector - Migrate analytics configuration to a new collector structure, consolidating settings for analytics and traffic logging. - Update tests to reflect changes in configuration structure and ensure compatibility with the new collector. - Deprecate old analytics settings and provide migration paths for existing configurations. - Adjust access log service server initialization to utilize the new collector settings. - Ensure that both analytics and traffic logging consumers require the collector to be enabled for proper functionality. Refactor access log configuration to unify ALS settings under [collector.als] - Updated CollectorConfig and AnalyticsConfig to replace deprecated grpc_event_server and access_logs_service fields with als. - Adjusted validation and migration logic to reflect new configuration structure. - Modified error messages to reference the new [collector.als] section. - Enhanced traffic logging functionality by introducing TrafficLog and TrafficLogDirective types, allowing for per-API logging configurations. - Implemented tests to ensure correct behavior of traffic logging, including handling of excluded headers and field selection. - Updated access logger server initialization to utilize new configuration keys. Add auto activate for collector Refactor traffic logging configuration to support max payload size and ignored path prefixes Refactor latency calculations and enhance Latencies struct with new duration fields --- gateway/configs/config-template.toml | 100 +++++- gateway/configs/config.toml | 36 +- .../gateway-controller/pkg/config/config.go | 223 ++++++++---- .../pkg/config/config_test.go | 117 +++--- .../pkg/utils/system_policies.go | 22 +- .../pkg/utils/system_policies_test.go | 60 +++- .../gateway-controller/pkg/xds/translator.go | 18 +- .../pkg/xds/translator_test.go | 10 +- .../policy-engine/cmd/policy-engine/main.go | 8 +- .../internal/analytics/analytics.go | 92 +++-- .../internal/analytics/analytics_test.go | 112 +++++- .../internal/analytics/dto/event.go | 34 ++ .../internal/analytics/dto/latencies.go | 7 +- .../internal/analytics/publishers/log.go | 310 ++++++++++++++++ .../internal/analytics/publishers/log_test.go | 334 ++++++++++++++++++ .../policy-engine/internal/config/config.go | 245 +++++++++---- .../internal/config/config_test.go | 104 ++++-- .../internal/utils/access_logger_server.go | 18 +- .../utils/access_logger_server_test.go | 6 +- .../system-policies/analytics/analytics.go | 76 +++- .../analytics/analytics_headers_test.go | 63 ++++ .../analytics/policy-definition.yaml | 24 ++ 22 files changed, 1706 insertions(+), 313 deletions(-) create mode 100644 gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go create mode 100644 gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go create mode 100644 gateway/system-policies/analytics/analytics_headers_test.go diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index d11d5ebb66..a9c3965cce 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -281,18 +281,61 @@ port = 9010 host = "localhost" # ============================================================================= -# ANALYTICS CONFIGURATION +# COLLECTOR CONFIGURATION # ============================================================================= -[analytics] +# The collector is the shared data-capture pipeline: it injects the analytics +# system policy and ships request/response headers and bodies to the policy-engine +# over ALS. It is a PREREQUISITE for every consumer below — analytics and +# traffic_logging both require collector.enabled = true (enabling a consumer with +# the collector off is a startup error). +[collector] enabled = false -# Deprecated: allow_payloads is preserved for backward compatibility. When true -# and both send_request_body and send_response_body are false, both directions -# are enabled (bool fields cannot distinguish "unset" from explicitly false). -# To disable payloads entirely, set allow_payloads = false or remove it; do not -# rely on send_*_body = false while allow_payloads remains true. -allow_payloads = false +# Capture request/response payloads (bodies) into the collected event. Bodies are +# captured in full; per-consumer size caps are applied on output (see +# traffic_logging.max_payload_size). send_request_body = false send_response_body = false +# Capture ALL request/response headers into the collected event (request_headers/ +# response_headers). Applies to every API via the collector system policy, so no +# per-API header policy is needed. Sensitive values are redacted on output by each +# consumer (e.g. traffic_logging.masked_headers). +send_request_headers = false +send_response_headers = false + +# ALS transport tuning (advanced; defaults are sensible). One section read by BOTH +# ends of the Envoy → policy-engine access-log stream: +# • gateway-controller — configures Envoy's gRPC access-log sink (the sender) +# • policy-engine — runs the receiving gRPC server +# Shared keys (server_port, TLS, message/header limits) must match on both ends and +# live here once. Envoy-sender-only keys (buffer_*, grpc_request_timeout) are read +# only by the controller and ignored by the policy-engine. +[collector.als] +mode = "uds" # "uds" (default) or "tcp" +server_port = 18090 # engine listens here; Envoy dials it (tcp mode) +buffer_flush_interval = 1000000000 # Envoy sender only (nanoseconds) +buffer_size_bytes = 16384 # Envoy sender only +grpc_request_timeout = 20000000000 # Envoy sender only (nanoseconds) +shutdown_timeout = "600s" +public_key_path = "" # TLS for the ALS connection (both ends) +private_key_path = "" +als_plain_text = true # plaintext gRPC (skip TLS) +max_message_size = 1000000000 +max_header_limit = 8192 + +# ============================================================================= +# ANALYTICS CONFIGURATION (consumer — requires [collector].enabled = true) +# ============================================================================= +[analytics] +enabled = false +# Deprecated: allow_payloads / send_request_body / send_response_body are preserved +# for backward compatibility and are mapped onto the [collector] body-capture flags +# during validation (with a warning). Prefer setting [collector] directly. +allow_payloads = false +# send_request_body = false +# send_response_body = false +# Publishers that receive analytics events. Supported: "moesif" (sends to the +# Moesif SaaS). For stdout JSON logging use [traffic_logging] instead of a "log" +# publisher here. enabled_publishers = ["moesif"] [analytics.publishers.moesif] @@ -303,17 +346,36 @@ event_queue_size = 10000 batch_size = 50 timer_wakeup_seconds = 3 -[analytics.grpc_event_server] -buffer_flush_interval = 1000000000 -buffer_size_bytes = 16384 -grpc_request_timeout = 20000000000 -server_port = 18090 -shutdown_timeout = "600s" -public_key_path = "" -private_key_path = "" -als_plain_text = true -max_message_size = 1000000000 -max_header_limit = 8192 +# ============================================================================= +# TRAFFIC LOGGING (consumer — requires [collector].enabled = true) +# ============================================================================= +# Writes collected events (rich API/application/AI/latency metadata, plus +# request/response headers and — when the collector's send_*_body is enabled — +# payloads) to stdout as a JSON line. Useful for log-scraping pipelines such as +# Fluent Bit/Loki/ELK; no external SaaS needed. +# +# Per-API (opt-in): a line is emitted only for APIs that attach the `log-message` +# policy with `destination = access-log` — enabling this section alone does NOT +# log every API. So an API is logged only when all three hold: [collector] +# .enabled = true, [traffic_logging].enabled = true, and the `log-message` policy +# (access-log destination) is attached to that API. The policy's request/response +# parameters (per-flow payload/headers toggles and excludeHeaders) shape each +# API's line; they filter/mask what the collector captured (they cannot enable +# capture the collector was not configured to perform). +[traffic_logging] +enabled = false +# Header names (case-insensitive) whose values are redacted as "****" in the +# logged requestHeaders/responseHeaders. Applies globally; the `log-message` +# policy's (access-log mode) per-API excludeHeaders drop additional headers entirely. +masked_headers = ["authorization"] +# Max bytes of request/response payload written to each log line (0 = no limit). +# Applied output-side, so the collector still captures full bodies and other +# consumers (e.g. Moesif) are unaffected. +max_payload_size = 0 +# Suppress traffic-log lines for requests whose original (pre-rewrite) path starts +# with any of these prefixes (e.g. health/readiness probes). Only affects this +# stdout publisher — other consumers still receive the event. +ignored_path_prefixes = [] # ============================================================================= # POLICY CONFIGURATIONS diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index cd4ea4897f..98238a3fc1 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -1,16 +1,34 @@ -[analytics] +# The collector is the shared data-capture pipeline. It must be enabled before any +# consumer (analytics, traffic_logging) can receive data. +[collector] enabled = true -# Deprecated: allow_payloads is preserved for backward compatibility. When true -# and when send_request_body/send_response_body are not explicitly set, both -# request and response bodies will be included in analytics events. -# allow_payloads = true -send_request_body = false -send_response_body = false +# Capture request/response payloads (bodies) into the collected event (full bodies; +# per-consumer size caps are applied on output). +send_request_body = true +send_response_body = true +# Capture ALL request/response headers (redacted on output by per-consumer masking). +send_request_headers = true +send_response_headers = true + +# Analytics consumer (e.g. Moesif). Requires [collector].enabled = true. +[analytics] +enabled = false enabled_publishers = ["moesif"] -[analytics.publishers.moesif] -application_id = "" +# [analytics.publishers.moesif] +# application_id = "" + +# Traffic-logging consumer: writes each collected event to stdout as a JSON line +# (no external service needed). Requires [collector].enabled = true. +[traffic_logging] +enabled = true +# Header names (case-insensitive) whose values are redacted as "****". +masked_headers = ["authorization"] +# Max bytes of request/response payload written to each log line (0 = no limit). +max_payload_size = 2048 +# Skip traffic-log lines for these path prefixes (e.g. health probes). +ignored_path_prefixes = ["/health", "/ready"] [router] gateway_host = "*" diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 6402427984..8ed547a111 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -48,6 +48,7 @@ type Config struct { Router RouterConfig `koanf:"router"` PolicyEngine map[string]interface{} `koanf:"policy_engine"` PolicyConfigurations map[string]interface{} `koanf:"policy_configurations"` + Collector CollectorConfig `koanf:"collector"` Analytics AnalyticsConfig `koanf:"analytics"` TracingConfig TracingConfig `koanf:"tracing"` APIKey APIKeyConfig `koanf:"api_key"` @@ -57,19 +58,45 @@ type Config struct { ImmutableGateway ImmutableGatewayConfig `koanf:"immutable_gateway"` } +// CollectorConfig holds the data-collection ("collector") configuration. The +// collector is the shared capture pipeline (the analytics system policy plus the +// Envoy→policy-engine ALS transport) that gathers request/response headers and +// bodies. It is a prerequisite for every consumer of that data — analytics and +// traffic logging both require collector.enabled to be true. +type CollectorConfig struct { + // Enabled turns the collector on. When false, the analytics system policy is + // not injected and Envoy ships no access logs to the policy-engine. + Enabled bool `koanf:"enabled"` + // SendRequestBody / SendResponseBody capture request/response bodies into the + // collected event. + SendRequestBody bool `koanf:"send_request_body"` + SendResponseBody bool `koanf:"send_response_body"` + // SendRequestHeaders / SendResponseHeaders, when true, make the collector + // capture ALL request / response headers, so every API's headers flow into + // the collected event without attaching a per-API header policy. + SendRequestHeaders bool `koanf:"send_request_headers"` + SendResponseHeaders bool `koanf:"send_response_headers"` + // GRPCEventServerCfg tunes the Envoy→policy-engine gRPC access-log (ALS) + // transport that ships collected data. It is part of the collector and is + // configured under the shared [collector.als] section (the policy-engine reads + // the same section to configure its receiving ALS server). + GRPCEventServerCfg GRPCEventServerConfig `koanf:"als"` +} + // AnalyticsConfig holds analytics configuration type AnalyticsConfig struct { - Enabled bool `koanf:"enabled"` - EnabledPublishers []string `koanf:"enabled_publishers"` - Publishers AnalyticsPublishersConfig `koanf:"publishers"` - GRPCEventServerCfg GRPCEventServerConfig `koanf:"grpc_event_server"` - // AllowPayloads controls whether request and response bodies are captured - // into analytics metadata and forwarded to analytics publishers. - // Deprecated: use SendRequestBody and SendResponseBody instead. - // When true, validateAnalyticsConfig maps both SendRequestBody and SendResponseBody - // to true if both are false. Because bools cannot represent "unset", this also - // applies when both new flags are explicitly false; remove allow_payloads when - // migrating and set the directional flags directly. + Enabled bool `koanf:"enabled"` + EnabledPublishers []string `koanf:"enabled_publishers"` + Publishers AnalyticsPublishersConfig `koanf:"publishers"` + // GRPCEventServerCfg is a deprecated alias. ALS transport tuning moved to + // [collector.als]; when set here it is migrated onto the collector during + // validation (with a warning). Prefer [collector.als]. + GRPCEventServerCfg GRPCEventServerConfig `koanf:"grpc_event_server"` + // AllowPayloads, SendRequestBody and SendResponseBody are deprecated aliases. + // Body/header capture now lives under [collector]. When set, these are mapped + // onto collector.send_request_body / collector.send_response_body during + // validation (with a warning) for backward compatibility. Prefer the + // [collector] fields directly. AllowPayloads bool `koanf:"allow_payloads"` SendRequestBody bool `koanf:"send_request_body"` SendResponseBody bool `koanf:"send_response_body"` @@ -690,6 +717,26 @@ func LoadConfig(configPath string) (*Config, error) { return cfg, nil } +// defaultGRPCEventServerConfig returns the default Envoy→policy-engine ALS +// transport tuning. Shared by the collector (canonical) and the deprecated +// [analytics].grpc_event_server alias so a partial alias override migrates cleanly. +func defaultGRPCEventServerConfig() GRPCEventServerConfig { + return GRPCEventServerConfig{ + Mode: "uds", // UDS mode by default + Port: 18090, // Only used in TCP mode + ServerPort: 18090, // ALS server port + BufferFlushInterval: 1000000000, // 1 second + BufferSizeBytes: 16384, // 16 KiB + GRPCRequestTimeout: 20000000000, // 20 seconds + ShutdownTimeout: 600 * time.Second, + PublicKeyPath: "", + PrivateKeyPath: "", + ALSPlainText: true, + MaxMessageSize: 1000000000, + MaxHeaderLimit: 8192, + } +} + // defaultConfig returns a Config struct with default configuration values func defaultConfig() *Config { return &Config{ @@ -908,23 +955,20 @@ func defaultConfig() *Config { TimerWakeupSeconds: 3, }, }, - GRPCEventServerCfg: GRPCEventServerConfig{ - Mode: "uds", // UDS mode by default - Port: 18090, // Only used in TCP mode - ServerPort: 18090, // ALS server port - BufferFlushInterval: 1000000000, // 1 second - BufferSizeBytes: 16384, // 16 KiB - GRPCRequestTimeout: 20000000000, // 20 seconds - ShutdownTimeout: 600 * time.Second, - PublicKeyPath: "", - PrivateKeyPath: "", - ALSPlainText: true, - MaxMessageSize: 1000000000, - MaxHeaderLimit: 8192, - }, - AllowPayloads: false, - SendRequestBody: false, - SendResponseBody: false, + // Deprecated alias: default mirrors the collector so a partial + // [analytics.grpc_event_server] override migrates cleanly. + GRPCEventServerCfg: defaultGRPCEventServerConfig(), + AllowPayloads: false, + SendRequestBody: false, + SendResponseBody: false, + }, + Collector: CollectorConfig{ + Enabled: false, + SendRequestBody: false, + SendResponseBody: false, + SendRequestHeaders: false, + SendResponseHeaders: false, + GRPCEventServerCfg: defaultGRPCEventServerConfig(), }, TracingConfig: TracingConfig{ Enabled: false, @@ -1291,7 +1335,7 @@ func (c *Config) Validate() error { return err } - if err := c.validateAnalyticsConfig(); err != nil { + if err := c.validateCollectorConfig(); err != nil { return err } @@ -1697,51 +1741,96 @@ func validateDomains(field string, domains []string) error { return nil } -// validateAnalyticsConfig validates the analytics configuration -func (c *Config) validateAnalyticsConfig() error { - // Validate analytics configuration - if c.Analytics.Enabled { - // Migration path for deprecated analytics.allow_payloads. - // Runs when both directional flags are false, which is indistinguishable - // from "not set" because bool fields cannot represent unset vs explicit false. - if c.Analytics.AllowPayloads { - slog.Warn("analytics.allow_payloads is deprecated; use analytics.send_request_body and analytics.send_response_body instead") - if !c.Analytics.SendRequestBody && !c.Analytics.SendResponseBody { - c.Analytics.SendRequestBody = true - c.Analytics.SendResponseBody = true - } +// validateCollectorConfig migrates deprecated analytics aliases onto the collector +// and enforces the collector prerequisite: analytics (a consumer) requires the +// collector that feeds it. For backward compatibility this is a soft prerequisite — +// if a consumer is enabled without the collector, the collector is auto-enabled with +// a warning rather than failing. It also validates the ALS transport tuning when the +// collector is enabled. +func (c *Config) validateCollectorConfig() error { + c.migrateDeprecatedAnalyticsCapture() + c.migrateDeprecatedAnalyticsTransport() + + // Backward-compat bridge: a consumer cannot run without the collector that feeds + // it, but rather than fail an existing config that only set analytics.enabled + // (valid before the collector split), auto-enable the collector with a warning. + if c.Analytics.Enabled && !c.Collector.Enabled { + slog.Warn("analytics.enabled requires the collector; enabling collector.enabled automatically for backward compatibility. Set collector.enabled = true explicitly to silence this warning.") + c.Collector.Enabled = true + } + if c.Collector.Enabled { + if err := validateGRPCEventServerConfig(c.Collector.GRPCEventServerCfg); err != nil { + return err } + } + return nil +} - // Validate gRPC event server configuration - grpcEventServerCfg := c.Analytics.GRPCEventServerCfg - - // Validate connection mode - switch grpcEventServerCfg.Mode { - case "uds", "": - // UDS mode (default) - port is unused for Envoy connection - case "tcp": - // TCP mode - validate port (host is derived from policy_engine.host) - if grpcEventServerCfg.Port <= 0 || grpcEventServerCfg.Port > 65535 { - return fmt.Errorf("analytics.grpc_event_server.port must be between 1 and 65535 when mode is tcp, got %d", grpcEventServerCfg.Port) - } - default: - return fmt.Errorf("analytics.grpc_event_server.mode must be 'uds' or 'tcp', got: %s", grpcEventServerCfg.Mode) +// migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].grpc_event_server +// override onto the collector when the collector's transport tuning is still at its +// default, so existing configs keep working after the transport moved to [collector]. +func (c *Config) migrateDeprecatedAnalyticsTransport() { + def := defaultGRPCEventServerConfig() + if c.Analytics.GRPCEventServerCfg != def { + slog.Warn("analytics.grpc_event_server is deprecated; use collector.als instead") + if c.Collector.GRPCEventServerCfg == def { + c.Collector.GRPCEventServerCfg = c.Analytics.GRPCEventServerCfg } + } +} - // Validate buffer and timeout settings - if grpcEventServerCfg.BufferFlushInterval <= 0 || grpcEventServerCfg.BufferSizeBytes <= 0 || grpcEventServerCfg.GRPCRequestTimeout <= 0 { - return fmt.Errorf( - "invalid gRPC event server configuration: bufferFlushInterval=%d, bufferSizeBytes=%d, grpcRequestTimeout=%d (all must be > 0)", - grpcEventServerCfg.BufferFlushInterval, - grpcEventServerCfg.BufferSizeBytes, - grpcEventServerCfg.GRPCRequestTimeout, - ) +// migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / +// analytics.send_request_body / analytics.send_response_body onto the collector's +// body-capture flags (when the collector flag is not already set), so existing +// configs keep working after capture settings moved under [collector]. +func (c *Config) migrateDeprecatedAnalyticsCapture() { + // Directional aliases take precedence over allow_payloads. + if c.Analytics.SendRequestBody && !c.Collector.SendRequestBody { + slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") + c.Collector.SendRequestBody = true + } + if c.Analytics.SendResponseBody && !c.Collector.SendResponseBody { + slog.Warn("analytics.send_response_body is deprecated; use collector.send_response_body instead") + c.Collector.SendResponseBody = true + } + // allow_payloads only fills in when no directional body capture is configured. + if c.Analytics.AllowPayloads { + slog.Warn("analytics.allow_payloads is deprecated; use collector.send_request_body and collector.send_response_body instead") + if !c.Collector.SendRequestBody && !c.Collector.SendResponseBody { + c.Collector.SendRequestBody = true + c.Collector.SendResponseBody = true } + } +} - // Validate server port - if grpcEventServerCfg.ServerPort <= 0 || grpcEventServerCfg.ServerPort > 65535 { - return fmt.Errorf("analytics.grpc_event_server.server_port must be between 1 and 65535, got %d", grpcEventServerCfg.ServerPort) +// validateGRPCEventServerConfig validates the Envoy→policy-engine ALS transport tuning. +func validateGRPCEventServerConfig(cfg GRPCEventServerConfig) error { + // Validate connection mode + switch cfg.Mode { + case "uds", "": + // UDS mode (default) - port is unused for Envoy connection + case "tcp": + // TCP mode - validate port (host is derived from policy_engine.host) + if cfg.Port <= 0 || cfg.Port > 65535 { + return fmt.Errorf("collector.als.port must be between 1 and 65535 when mode is tcp, got %d", cfg.Port) } + default: + return fmt.Errorf("collector.als.mode must be 'uds' or 'tcp', got: %s", cfg.Mode) + } + + // Validate buffer and timeout settings + if cfg.BufferFlushInterval <= 0 || cfg.BufferSizeBytes <= 0 || cfg.GRPCRequestTimeout <= 0 { + return fmt.Errorf( + "invalid gRPC event server configuration: bufferFlushInterval=%d, bufferSizeBytes=%d, grpcRequestTimeout=%d (all must be > 0)", + cfg.BufferFlushInterval, + cfg.BufferSizeBytes, + cfg.GRPCRequestTimeout, + ) + } + + // Validate server port + if cfg.ServerPort <= 0 || cfg.ServerPort > 65535 { + return fmt.Errorf("collector.als.server_port must be between 1 and 65535, got %d", cfg.ServerPort) } return nil } diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index 91e53733a4..e17039112d 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -102,6 +102,14 @@ func validConfig() *Config { ServerHeaderTransformation: commonconstants.OVERWRITE, }, }, + // Transport defaults mirror production so collector-gated grpc validation + // passes and the deprecated [analytics] alias stays neutral. + Collector: CollectorConfig{ + GRPCEventServerCfg: defaultGRPCEventServerConfig(), + }, + Analytics: AnalyticsConfig{ + GRPCEventServerCfg: defaultGRPCEventServerConfig(), + }, } } @@ -1205,11 +1213,11 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with valid UDS config (default mode)", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "uds" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "uds" + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: false, }, @@ -1217,12 +1225,12 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with valid TCP config", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "tcp" - cfg.Analytics.GRPCEventServerCfg.Port = 18090 - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "tcp" + cfg.Collector.GRPCEventServerCfg.Port = 18090 + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: false, }, @@ -1230,11 +1238,11 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Analytics enabled with empty mode defaults to UDS", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "" + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: false, }, @@ -1242,49 +1250,49 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { name: "Invalid mode value", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "invalid" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "invalid" + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: true, - errContains: "grpc_event_server.mode must be 'uds' or 'tcp'", + errContains: "collector.als.mode must be 'uds' or 'tcp'", }, { name: "TCP mode - invalid port", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "tcp" - cfg.Analytics.GRPCEventServerCfg.Port = 0 - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "tcp" + cfg.Collector.GRPCEventServerCfg.Port = 0 + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: true, - errContains: "grpc_event_server.port must be between 1 and 65535", + errContains: "collector.als.port must be between 1 and 65535", }, { name: "Invalid server port", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "uds" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 0 + cfg.Collector.GRPCEventServerCfg.Mode = "uds" + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Collector.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Collector.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Collector.GRPCEventServerCfg.ServerPort = 0 }, wantErr: true, - errContains: "grpc_event_server.server_port must be between 1 and 65535", + errContains: "collector.als.server_port must be between 1 and 65535", }, { name: "Invalid buffer flush interval", enabled: true, setupConfig: func(cfg *Config) { - cfg.Analytics.GRPCEventServerCfg.Mode = "uds" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 0 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + cfg.Collector.GRPCEventServerCfg.Mode = "uds" + cfg.Collector.GRPCEventServerCfg.BufferFlushInterval = 0 + cfg.Collector.GRPCEventServerCfg.ServerPort = 18090 }, wantErr: true, errContains: "invalid gRPC event server configuration", @@ -1295,6 +1303,9 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { cfg := validConfig() cfg.Analytics.Enabled = tt.enabled + // Analytics is a consumer; enable the collector it depends on so these + // tests exercise analytics validation rather than the prerequisite check. + cfg.Collector.Enabled = tt.enabled if tt.setupConfig != nil { tt.setupConfig(cfg) } @@ -1309,9 +1320,34 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { } } +func TestConfig_CollectorPrerequisite(t *testing.T) { + t.Run("analytics enabled without collector auto-enables the collector", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = false + cfg.Analytics.Enabled = true + cfg.Analytics.EnabledPublishers = []string{} + // Deprecated transport alias with valid values migrates onto the auto-enabled collector. + cfg.Analytics.GRPCEventServerCfg.Mode = "uds" + cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 + cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 + cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 + cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + require.NoError(t, cfg.Validate()) + assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") + }) + + t.Run("collector enabled with no consumers is valid", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = true + require.NoError(t, cfg.Validate()) + }) + +} + func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { setValidAnalyticsGRPC := func(cfg *Config) { cfg.Analytics.Enabled = true + cfg.Collector.Enabled = true // analytics is a consumer; the collector must be on cfg.Analytics.GRPCEventServerCfg.Mode = "uds" cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 @@ -1371,8 +1407,9 @@ func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { err := cfg.Validate() require.NoError(t, err) - assert.Equal(t, tt.wantSendReq, cfg.Analytics.SendRequestBody) - assert.Equal(t, tt.wantSendResp, cfg.Analytics.SendResponseBody) + // Deprecated analytics body aliases now migrate onto the collector. + assert.Equal(t, tt.wantSendReq, cfg.Collector.SendRequestBody) + assert.Equal(t, tt.wantSendResp, cfg.Collector.SendResponseBody) }) } } diff --git a/gateway/gateway-controller/pkg/utils/system_policies.go b/gateway/gateway-controller/pkg/utils/system_policies.go index 8ad1da067f..25a598a634 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies.go +++ b/gateway/gateway-controller/pkg/utils/system_policies.go @@ -84,13 +84,18 @@ var defaultSystemPolicies = []systemPolicyConfig{ if cfg == nil { return false } - slog.Debug("Analytics state -> ", "state", cfg.Analytics.Enabled) - return cfg.Analytics.Enabled + // The analytics system policy is the collector: it is injected whenever + // the collector is enabled, regardless of which consumer (analytics, + // traffic logging) ultimately reads the collected data. + slog.Debug("Collector state -> ", "state", cfg.Collector.Enabled) + return cfg.Collector.Enabled }, // Default parameters (can be overridden via additionalProps) Parameters: map[string]interface{}{ - "send_request_body": false, - "send_response_body": false, + "send_request_body": false, + "send_response_body": false, + "send_request_headers": false, + "send_response_headers": false, }, ExecutionCondition: nil, }, @@ -197,10 +202,13 @@ func InjectSystemPolicies(policies []policyenginev1.PolicyInstance, cfg *config. for k, v := range sysPol.Parameters { effectiveDefaults[k] = v } - // For the analytics system policy, propagate the payload flags from runtime config. + // For the analytics (collector) system policy, propagate the payload and + // header capture flags from the collector config. if sysPol.Name == constants.ANALYTICS_SYSTEM_POLICY_NAME { - effectiveDefaults["send_request_body"] = cfg.Analytics.SendRequestBody - effectiveDefaults["send_response_body"] = cfg.Analytics.SendResponseBody + effectiveDefaults["send_request_body"] = cfg.Collector.SendRequestBody + effectiveDefaults["send_response_body"] = cfg.Collector.SendResponseBody + effectiveDefaults["send_request_headers"] = cfg.Collector.SendRequestHeaders + effectiveDefaults["send_response_headers"] = cfg.Collector.SendResponseHeaders } // Merge parameters efficiently diff --git a/gateway/gateway-controller/pkg/utils/system_policies_test.go b/gateway/gateway-controller/pkg/utils/system_policies_test.go index d0a0f1a61e..f324db0377 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies_test.go +++ b/gateway/gateway-controller/pkg/utils/system_policies_test.go @@ -124,9 +124,9 @@ func TestInjectSystemPolicies_NilConfig(t *testing.T) { assert.Equal(t, policies, result) } -func TestInjectSystemPolicies_AnalyticsDisabled(t *testing.T) { +func TestInjectSystemPolicies_CollectorDisabled(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: false, }, } @@ -139,9 +139,9 @@ func TestInjectSystemPolicies_AnalyticsDisabled(t *testing.T) { assert.Equal(t, "existing", result[0].Name) } -func TestInjectSystemPolicies_AnalyticsEnabled(t *testing.T) { +func TestInjectSystemPolicies_CollectorEnabled(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: true, }, } @@ -159,11 +159,11 @@ func TestInjectSystemPolicies_AnalyticsEnabled(t *testing.T) { assert.Equal(t, "existing", result[1].Name) } -func TestInjectSystemPolicies_AllowPayloadsTrue(t *testing.T) { +func TestInjectSystemPolicies_BodyFlagsPropagated(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - SendRequestBody: true, + Collector: config.CollectorConfig{ + Enabled: true, + SendRequestBody: true, SendResponseBody: true, }, } @@ -175,11 +175,11 @@ func TestInjectSystemPolicies_AllowPayloadsTrue(t *testing.T) { assert.Equal(t, true, result[0].Parameters["send_response_body"]) } -func TestInjectSystemPolicies_AllowPayloadsFalse(t *testing.T) { +func TestInjectSystemPolicies_BodyFlagsDefaultFalse(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - SendRequestBody: false, + Collector: config.CollectorConfig{ + Enabled: true, + SendRequestBody: false, SendResponseBody: false, }, } @@ -190,9 +190,37 @@ func TestInjectSystemPolicies_AllowPayloadsFalse(t *testing.T) { assert.Equal(t, false, result[0].Parameters["send_response_body"]) } +func TestInjectSystemPolicies_HeaderFlagsPropagated(t *testing.T) { + cfg := &config.Config{ + Collector: config.CollectorConfig{ + Enabled: true, + SendRequestHeaders: true, + SendResponseHeaders: true, + }, + } + + result := InjectSystemPolicies(nil, cfg, nil) + assert.Len(t, result, 1) + assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_NAME, result[0].Name) + assert.Equal(t, true, result[0].Parameters["send_request_headers"]) + assert.Equal(t, true, result[0].Parameters["send_response_headers"]) +} + +func TestInjectSystemPolicies_HeaderFlagsDefaultFalse(t *testing.T) { + cfg := &config.Config{ + Collector: config.CollectorConfig{Enabled: true}, + } + + result := InjectSystemPolicies(nil, cfg, nil) + assert.Len(t, result, 1) + assert.Equal(t, false, result[0].Parameters["send_request_headers"]) + assert.Equal(t, false, result[0].Parameters["send_response_headers"]) +} + + func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: true, }, } @@ -209,7 +237,7 @@ func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { func TestInjectSystemPolicies_WithSharedParams(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: true, }, } @@ -226,7 +254,7 @@ func TestInjectSystemPolicies_WithSharedParams(t *testing.T) { func TestInjectSystemPolicies_EmptyPolicies(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: true, }, } @@ -238,7 +266,7 @@ func TestInjectSystemPolicies_EmptyPolicies(t *testing.T) { func TestInjectSystemPolicies_PreservesExistingPolicies(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ Enabled: true, }, } diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index 408a6dda7a..7ae5387236 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -561,10 +561,10 @@ func (t *Translator) TranslateConfigs( policyEngineCluster := t.createPolicyEngineCluster() clusters = append(clusters, policyEngineCluster) - // Add ALS cluster if gRPC access log is enabled - log.Debug("gRPC event server config", slog.Any("config", t.config.Analytics.GRPCEventServerCfg)) - if t.config.Analytics.Enabled { - log.Info("gRPC access log is enabled, creating ALS cluster") + // Add ALS cluster if the collector is enabled (it ships access logs over gRPC) + log.Debug("gRPC event server config", slog.Any("config", t.config.Collector.GRPCEventServerCfg)) + if t.config.Collector.Enabled { + log.Info("collector is enabled, creating ALS cluster") alsCluster := t.createALSCluster() clusters = append(clusters, alsCluster) } @@ -2156,7 +2156,7 @@ func (t *Translator) createPolicyEngineCluster() *cluster.Cluster { // createALSCluster creates an Envoy cluster for the gRPC access log service func (t *Translator) createALSCluster() *cluster.Cluster { - grpcConfig := t.config.Analytics.GRPCEventServerCfg + grpcConfig := t.config.Collector.GRPCEventServerCfg // Build the endpoint address (UDS or TCP) var address *core.Address @@ -2785,8 +2785,8 @@ func (t *Translator) createAccessLogConfig() ([]*accesslog.AccessLog, error) { }, }) - // If gRPC access log is enabled, create the configuration and append to existing access logs - if t.config.Analytics.Enabled { + // If the collector is enabled, create the gRPC access log config and append to existing access logs + if t.config.Collector.Enabled { t.logger.Info("Creating gRPC access log configuration") grpcAccessLog, err := t.createGRPCAccessLog() if err != nil { @@ -2802,8 +2802,8 @@ func (t *Translator) createAccessLogConfig() ([]*accesslog.AccessLog, error) { // createGRPCAccessLog creates a gRPC access log configuration for the gateway controller func (t *Translator) createGRPCAccessLog() (*accesslog.AccessLog, error) { - grpcConfig := t.config.Analytics.GRPCEventServerCfg - bufferSizeBytes, err := checkedUInt32FromPositiveInt("analytics.grpc_event_server.buffer_size_bytes", grpcConfig.BufferSizeBytes) + grpcConfig := t.config.Collector.GRPCEventServerCfg + bufferSizeBytes, err := checkedUInt32FromPositiveInt("collector.als.buffer_size_bytes", grpcConfig.BufferSizeBytes) if err != nil { return nil, err } diff --git a/gateway/gateway-controller/pkg/xds/translator_test.go b/gateway/gateway-controller/pkg/xds/translator_test.go index dc637583e8..4a99e0036d 100644 --- a/gateway/gateway-controller/pkg/xds/translator_test.go +++ b/gateway/gateway-controller/pkg/xds/translator_test.go @@ -1330,7 +1330,7 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ Mode: "uds", BufferFlushInterval: 1000000000, BufferSizeBytes: 16384, @@ -1357,7 +1357,7 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ Mode: "", BufferFlushInterval: 1000000000, BufferSizeBytes: 16384, @@ -1383,7 +1383,7 @@ func TestTranslator_CreateALSCluster(t *testing.T) { routerCfg := testRouterConfig() cfg := testConfig() cfg.Analytics.Enabled = true - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ Mode: "tcp", Port: 18090, BufferFlushInterval: 1000000000, @@ -1415,7 +1415,7 @@ func TestTranslator_CreateGRPCAccessLog(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() cfg := testConfig() - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ Mode: "tcp", Port: 18090, BufferFlushInterval: 1000, @@ -1433,7 +1433,7 @@ func TestTranslator_CreateGRPCAccessLog_BufferSizeOverflow(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() cfg := testConfig() - cfg.Analytics.GRPCEventServerCfg = config.GRPCEventServerConfig{ + cfg.Collector.GRPCEventServerCfg = config.GRPCEventServerConfig{ Mode: "tcp", Port: 18090, BufferFlushInterval: 1000, diff --git a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go index 031f527745..b7116fd01c 100644 --- a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go +++ b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go @@ -268,10 +268,12 @@ func main() { metrics.StartMemoryMetricsUpdater(ctx, 15*time.Second) } - // Start access log service server if enabled + // Start the access log service server when the collector is enabled. The + // collector is the shared transport that carries collected data to its + // consumers (analytics, traffic logging). var alsServer *grpc.Server - slog.DebugContext(ctx, "Policy engine ALS server config", "config", cfg.Analytics.AccessLogsServiceCfg) - if cfg.Analytics.Enabled { + slog.DebugContext(ctx, "Policy engine ALS server config", "config", cfg.Collector.AccessLogsServiceCfg) + if cfg.Collector.Enabled { // Start the access log service server slog.Info("Starting the ALS gRPC server...") alsServer = utils.StartAccessLogServiceServer(cfg) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 56d541c190..2ef1d2a5c1 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -59,11 +59,19 @@ const ( DefaultAnalyticsPublisher = "default" // MoesifAnalyticsPublisher represents the Moesif analytics publisher. MoesifAnalyticsPublisher = "moesif" + // LogAnalyticsPublisher represents the stdout/log analytics publisher. + LogAnalyticsPublisher = "log" // HeaderKeys represents the header keys. RequestHeadersKey = "request_headers" ResponseHeadersKey = "response_headers" + // TrafficLogMetadataKey is the analytics-metadata key under which the + // log-message policy (in access-log mode) stamps its per-API opt-in marker. + // Its presence gates the stdout traffic-logging publisher; its value (a JSON + // directive) shapes the emitted line. Must match the key used by that policy. + TrafficLogMetadataKey = "traffic_log" + // PromptTokenCountMetadataKey represents the prompt token count metadata key. PromptTokenCountMetadataKey string = "aitoken:prompttokencount" // CompletionTokenCountMetadataKey represents the completion token count metadata key. @@ -90,27 +98,39 @@ type Analytics struct { publishers []analytics_publisher.Publisher } -// NewAnalytics creates a new instance of Analytics. +// NewAnalytics creates a new instance of Analytics. Publishers are assembled from +// each independently-configured consumer of the collected data: the analytics +// consumer ([analytics], e.g. Moesif) and the traffic-logging consumer +// ([traffic_logging], stdout JSON). Both rely on the collector being enabled to +// receive any events. func NewAnalytics(cfg *config.Config) *Analytics { analyticsCfg := cfg.Analytics publishers := make([]analytics_publisher.Publisher, 0) if analyticsCfg.Enabled { for _, publisherName := range analyticsCfg.EnabledPublishers { switch publisherName { - case "moesif": + case MoesifAnalyticsPublisher: publisher := analytics_publisher.NewMoesif(&analyticsCfg.Publishers.Moesif) if publisher != nil { publishers = append(publishers, publisher) slog.Info("Moesif publisher added") } + case LogAnalyticsPublisher: + slog.Warn("\"log\" in analytics.enabled_publishers is no longer supported; enable stdout traffic logging via [traffic_logging] instead") default: slog.Warn("Unknown publisher type", "type", publisherName) } } } + // Traffic logging is a standalone consumer, independent of analytics. + if cfg.TrafficLogging.Enabled { + publishers = append(publishers, analytics_publisher.NewLog(&cfg.TrafficLogging)) + slog.Info("Traffic logging (stdout) publisher added") + } + if len(publishers) == 0 { - slog.Debug("No analytics publishers found. Analytics will not be published.") + slog.Debug("No analytics publishers found. Collected events will not be published.") } return &Analytics{ cfg: cfg, @@ -133,7 +153,9 @@ func (c *Analytics) Process(event *v3.HTTPAccessLogEntry) { return } - // Add logic to publish the event + // Path-based suppression is a per-consumer presentation concern: it is applied + // by the stdout traffic-logging publisher (traffic_logging.ignored_path_prefixes), + // not here, so other consumers still receive every event. analyticEvent := c.prepareAnalyticEvent(event) for _, publisher := range c.publishers { publisher.Publish(analyticEvent) @@ -197,6 +219,23 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E for key, value := range keyValuePairsFromMetadata { slog.Debug(fmt.Sprintf("Metadata key: %v -> value: %+v", key, value)) } + + // Extract the per-API traffic-log opt-in marker stamped by the log-message + // policy (access-log mode). Its presence marks the API as opted in to stdout + // traffic logging; + // its value shapes the emitted line. It is intentionally kept off + // event.Properties so it never reaches the serialized output or other + // publishers (e.g. Moesif). A malformed marker still opts the API in with + // default (empty) presentation, since attachment alone signals intent. + if raw, exists := keyValuePairsFromMetadata[TrafficLogMetadataKey]; exists { + dir := &dto.TrafficLogDirective{} + if raw != "" { + if err := json.Unmarshal([]byte(raw), dir); err != nil { + slog.Warn("Failed to parse traffic_log directive; opting in with defaults", "error", err) + } + } + event.TrafficLog = dir + } // Prepare extended API extendedAPI := dto.ExtendedAPI{} extendedAPI.APIType = keyValuePairsFromMetadata[APITypeKey] @@ -247,33 +286,36 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E properties.TimeToFirstUpstreamTxByte != nil && properties.TimeToFirstUpstreamRxByte != nil && properties.TimeToLastUpstreamRxByte != nil && properties.TimeToLastDownstreamTxByte != nil { - lastRx := - (properties.TimeToLastRxByte.Seconds * 1000) + - (int64(properties.TimeToLastRxByte.Nanos) / 1_000_000) - - firstUpTx := - (properties.TimeToFirstUpstreamTxByte.Seconds * 1000) + - (int64(properties.TimeToFirstUpstreamTxByte.Nanos) / 1_000_000) - - firstUpRx := - (properties.TimeToFirstUpstreamRxByte.Seconds * 1000) + - (int64(properties.TimeToFirstUpstreamRxByte.Nanos) / 1_000_000) - - lastUpRx := - (properties.TimeToLastUpstreamRxByte.Seconds * 1000) + - (int64(properties.TimeToLastUpstreamRxByte.Nanos) / 1_000_000) + toMs := func(secs int64, nanos int32) int64 { + return (secs * 1000) + int64(nanos)/1_000_000 + } - lastDownTx := - (properties.TimeToLastDownstreamTxByte.Seconds * 1000) + - (int64(properties.TimeToLastDownstreamTxByte.Nanos) / 1_000_000) + lastRx := toMs(properties.TimeToLastRxByte.Seconds, properties.TimeToLastRxByte.Nanos) + firstUpTx := toMs(properties.TimeToFirstUpstreamTxByte.Seconds, properties.TimeToFirstUpstreamTxByte.Nanos) + firstUpRx := toMs(properties.TimeToFirstUpstreamRxByte.Seconds, properties.TimeToFirstUpstreamRxByte.Nanos) + lastUpRx := toMs(properties.TimeToLastUpstreamRxByte.Seconds, properties.TimeToLastUpstreamRxByte.Nanos) + lastDownTx := toMs(properties.TimeToLastDownstreamTxByte.Seconds, properties.TimeToLastDownstreamTxByte.Nanos) latencies := dto.Latencies{ + Duration: lastDownTx, BackendLatency: lastUpRx - firstUpTx, RequestMediationLatency: firstUpTx - lastRx, ResponseLatency: lastDownTx - firstUpRx, ResponseMediationLatency: lastDownTx - lastUpRx, } + // US_TX_END → US_RX_BEG: time the backend spent before sending the first response byte (TTFB). + if properties.TimeToLastUpstreamTxByte != nil { + lastUpTx := toMs(properties.TimeToLastUpstreamTxByte.Seconds, properties.TimeToLastUpstreamTxByte.Nanos) + latencies.BackendProcDuration = firstUpRx - lastUpTx + } + + // US_RX_BEG → DS_TX_BEG: gateway overhead processing the first response byte before writing downstream. + if properties.TimeToFirstDownstreamTxByte != nil { + firstDownTx := toMs(properties.TimeToFirstDownstreamTxByte.Seconds, properties.TimeToFirstDownstreamTxByte.Nanos) + latencies.ResponseProcDuration = firstDownTx - firstUpRx + } + event.Latencies = &latencies } @@ -434,14 +476,14 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E event.Properties["responseHeaders"] = responseHeaders } - // Optionally attach request and response payloads when enabled via configuration. - if c.cfg.Analytics.SendRequestBody { + // Optionally attach request and response payloads when enabled via the collector. + if c.cfg.Collector.SendRequestBody { if requestPayload, ok := keyValuePairsFromMetadata["request_payload"]; ok && requestPayload != "" { event.Properties["request_payload"] = requestPayload slog.Debug("Analytics request payload captured", "size_bytes", len(requestPayload)) } } - if c.cfg.Analytics.SendResponseBody { + if c.cfg.Collector.SendResponseBody { if responsePayload, ok := keyValuePairsFromMetadata["response_payload"]; ok && responsePayload != "" { event.Properties["response_payload"] = responsePayload slog.Debug("Analytics response payload captured", "size_bytes", len(responsePayload)) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 46d20cf61b..cb1582568e 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -64,7 +64,9 @@ func validAnalyticsConfigForValidation(analytics config.AnalyticsConfig) *config Analytics: analytics, } cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = config.AccessLogsServiceConfig{ + // Analytics is a consumer; the collector must be enabled for it to validate. + cfg.Collector.Enabled = true + cfg.Collector.AccessLogsServiceCfg = config.AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -138,6 +140,34 @@ func TestNewAnalytics_EnabledWithUnknownPublisherType(t *testing.T) { assert.Empty(t, analytics.publishers) // Unknown type should not be added } +func TestNewAnalytics_TrafficLoggingEnabled(t *testing.T) { + // Traffic logging is a standalone consumer, independent of analytics. + cfg := &config.Config{ + TrafficLogging: config.TrafficLoggingConfig{Enabled: true}, + } + + analytics := NewAnalytics(cfg) + + require.NotNil(t, analytics) + assert.Len(t, analytics.publishers, 1) // traffic-logging publisher should be registered +} + +func TestNewAnalytics_LogInEnabledPublishersIgnored(t *testing.T) { + // "log" in analytics.enabled_publishers is no longer supported; it must not + // register a publisher (use [traffic_logging] instead). + cfg := &config.Config{ + Analytics: config.AnalyticsConfig{ + Enabled: true, + EnabledPublishers: []string{LogAnalyticsPublisher}, + }, + } + + analytics := NewAnalytics(cfg) + + require.NotNil(t, analytics) + assert.Empty(t, analytics.publishers) +} + // ============================================================================= // isInvalid Tests // ============================================================================= @@ -253,6 +283,24 @@ func TestProcess_WithMockPublisher(t *testing.T) { assert.Equal(t, "TestAPI", mockPub.event.API.APIName) } +// Path-based suppression moved to the traffic-logging publisher, so +// Analytics.Process no longer filters by path — every valid event reaches the +// publishers (each decides what to do with it). Ignored-path coverage lives in +// publishers/log_test.go. +func TestProcess_PublishesEvent(t *testing.T) { + analytics := NewAnalytics(&config.Config{}) + mockPub := &mockPublisher{} + analytics.publishers = append(analytics.publishers, mockPub) + + logEntry := &v3.HTTPAccessLogEntry{ + Response: &v3.HTTPResponseProperties{ResponseCode: wrapperspb.UInt32(200)}, + Request: &v3.HTTPRequestProperties{OriginalPath: "/api/v1/orders", RequestMethod: corev3.RequestMethod_GET}, + } + analytics.Process(logEntry) + + assert.True(t, mockPub.called, "publisher must be called") +} + func TestProcess_PanicRecovery(t *testing.T) { cfg := &config.Config{} analytics := NewAnalytics(cfg) @@ -373,6 +421,60 @@ func TestPrepareAnalyticEvent_WithFilterMetadata(t *testing.T) { assert.Equal(t, "TestApp", event.Application.ApplicationName) } +func TestPrepareAnalyticEvent_TrafficLogMarker(t *testing.T) { + cfg := &config.Config{} + analytics := NewAnalytics(cfg) + + logEntry := createLogEntryWithMetadata(map[string]string{ + APINameKey: "TestAPI", + TrafficLogMetadataKey: `{"request":{"payload":false,"headers":true,"excludeHeaders":["x-key"]}}`, + }) + + event := analytics.prepareAnalyticEvent(logEntry) + + require.NotNil(t, event) + require.NotNil(t, event.TrafficLog, "traffic_log marker should populate event.TrafficLog") + require.NotNil(t, event.TrafficLog.Request) + assert.True(t, event.TrafficLog.Request.Headers) + assert.False(t, event.TrafficLog.Request.Payload) + assert.Equal(t, []string{"x-key"}, event.TrafficLog.Request.ExcludeHeaders) + assert.Nil(t, event.TrafficLog.Response, "unset flow stays nil") + + // The marker must never leak into serialized properties (or other publishers). + _, inProps := event.Properties[TrafficLogMetadataKey] + assert.False(t, inProps, "traffic_log must not appear in event.Properties") +} + +func TestPrepareAnalyticEvent_NoTrafficLogMarker(t *testing.T) { + cfg := &config.Config{} + analytics := NewAnalytics(cfg) + + logEntry := createLogEntryWithMetadata(map[string]string{APINameKey: "TestAPI"}) + + event := analytics.prepareAnalyticEvent(logEntry) + + require.NotNil(t, event) + assert.Nil(t, event.TrafficLog, "no marker -> event.TrafficLog stays nil (publisher skips)") +} + +func TestPrepareAnalyticEvent_MalformedTrafficLogMarker(t *testing.T) { + cfg := &config.Config{} + analytics := NewAnalytics(cfg) + + logEntry := createLogEntryWithMetadata(map[string]string{ + TrafficLogMetadataKey: "not-json", + }) + + event := analytics.prepareAnalyticEvent(logEntry) + + require.NotNil(t, event) + // Presence alone is the opt-in signal; a malformed marker still opts in with + // default (empty) presentation. + require.NotNil(t, event.TrafficLog) + assert.Nil(t, event.TrafficLog.Request) + assert.Nil(t, event.TrafficLog.Response) +} + func TestPrepareAnalyticEvent_WithAnonymousApp(t *testing.T) { cfg := &config.Config{} analytics := NewAnalytics(cfg) @@ -525,7 +627,7 @@ func TestPrepareAnalyticEvent_WithRequestResponseHeaders(t *testing.T) { func TestPrepareAnalyticEvent_WithPayloadsEnabled(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ SendRequestBody: true, SendResponseBody: true, }, @@ -551,7 +653,7 @@ func TestPrepareAnalyticEvent_WithPayloadsEnabled(t *testing.T) { func TestPrepareAnalyticEvent_WithPayloadsDisabled(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ SendRequestBody: false, SendResponseBody: false, }, @@ -574,7 +676,7 @@ func TestPrepareAnalyticEvent_WithPayloadsDisabled(t *testing.T) { func TestPrepareAnalyticEvent_RequestPayloadOnly(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ SendRequestBody: true, SendResponseBody: false, }, @@ -598,7 +700,7 @@ func TestPrepareAnalyticEvent_RequestPayloadOnly(t *testing.T) { func TestPrepareAnalyticEvent_ResponsePayloadOnly(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ + Collector: config.CollectorConfig{ SendRequestBody: false, SendResponseBody: true, }, diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index 6b2f05829b..75566db7e4 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -36,4 +36,38 @@ type Event struct { UserIP string `json:"userIp,omitempty" bson:"user_ip"` ErrorType string `json:"errorType,omitempty" bson:"error_type"` Properties map[string]interface{} `json:"properties,omitempty" bson:"properties"` + + // TrafficLog carries the per-API stdout traffic-logging opt-in marker stamped + // by the log-message policy (access-log mode). When nil, the API has not opted + // in and the stdout traffic-logging publisher skips the event. It is + // gating/presentation state only and is never serialized (json:"-") nor sent + // to other publishers. + TrafficLog *TrafficLogDirective `json:"-" bson:"-"` +} + +// TrafficLogDirective is the presentation config carried in the traffic-log +// marker. Field names mirror the policy's marker JSON so it round-trips. A nil +// flow means that flow was not configured. +type TrafficLogDirective struct { + Request *TrafficLogFlow `json:"request,omitempty"` + Response *TrafficLogFlow `json:"response,omitempty"` + Fields *TrafficLogFields `json:"fields,omitempty"` +} + +// TrafficLogFlow is the per-flow (request or response) presentation config. +type TrafficLogFlow struct { + Payload bool `json:"payload"` + Headers bool `json:"headers"` + ExcludeHeaders []string `json:"excludeHeaders,omitempty"` +} + +// TrafficLogFields selects which fields appear in the emitted line. When set it is +// authoritative over field presence: the per-flow Payload/Headers booleans are +// ignored (per-flow ExcludeHeaders and global masking still apply). Names are +// top-level keys (e.g. "latencies", "target") or dotted property paths +// (e.g. "properties.requestHeaders"). Mode "exclude" drops the named keys; any +// other value (default "include") keeps only the named keys. +type TrafficLogFields struct { + Mode string `json:"mode,omitempty"` + Names []string `json:"names,omitempty"` } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go index 412f95f460..e9e4818f0e 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/latencies.go @@ -23,7 +23,12 @@ type Latencies struct { BackendLatency int64 `json:"backendLatency"` RequestMediationLatency int64 `json:"requestMediationLatency"` ResponseMediationLatency int64 `json:"responseMediationLatency"` - Duration int64 `json:"duration"` + // Duration is the total request duration: downstream request received → downstream response sent (ms). + Duration int64 `json:"duration"` + // BackendProcDuration is the backend TTFB: upstream request fully sent → first upstream response byte (ms). + BackendProcDuration int64 `json:"backendProcDuration"` + // ResponseProcDuration is the gateway response overhead: first upstream response byte → first downstream response byte (ms). + ResponseProcDuration int64 `json:"responseProcDuration"` } // GetResponseLatency returns the response latency. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go new file mode 100644 index 0000000000..c76cdf3db9 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "encoding/json" + "fmt" + "log/slog" + "maps" + "os" + "strings" + "sync" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" +) + +// maskedHeaderValue is the placeholder written in place of a masked header value. +const maskedHeaderValue = "****" + +// Log is an analytics publisher that writes each enriched analytics event to +// stdout as a single JSON line. It is intended for log-scraping pipelines +// (Fluent Bit, Loki, ELK, etc.) and as a lightweight alternative to a SaaS +// analytics backend. The event already carries the rich metadata, headers and +// (when send_request_body/send_response_body are enabled) payloads attached by +// the analytics engine, so this publisher only serializes it. +type Log struct { + // maskedHeaders holds lower-cased header names whose values are redacted in + // the requestHeaders/responseHeaders properties before logging. + maskedHeaders map[string]bool + // ignoredPathPrefixes: skip emitting a line when the event's original request + // path starts with any of these (per-consumer — other publishers are + // unaffected). Trimmed of blanks at construction. + ignoredPathPrefixes []string + // maxPayloadSize caps the number of request/response payload bytes written to + // the log line (0 = no limit). Truncation is output-side only. + maxPayloadSize int + // mu serializes writes to stdout so concurrent ALS streams do not interleave. + mu sync.Mutex + // out is the destination writer; defaults to os.Stdout (overridable in tests). + out *os.File +} + +// NewLog creates a new stdout traffic-logging publisher. +func NewLog(logCfg *config.TrafficLoggingConfig) *Log { + if logCfg == nil { + logCfg = &config.TrafficLoggingConfig{} + } + + masked := make(map[string]bool, len(logCfg.MaskedHeaders)) + for _, h := range logCfg.MaskedHeaders { + h = strings.ToLower(strings.TrimSpace(h)) + if h != "" { + masked[h] = true + } + } + + prefixes := make([]string, 0, len(logCfg.IgnoredPathPrefixes)) + for _, p := range logCfg.IgnoredPathPrefixes { + if p = strings.TrimSpace(p); p != "" { + prefixes = append(prefixes, p) + } + } + + return &Log{ + maskedHeaders: masked, + ignoredPathPrefixes: prefixes, + maxPayloadSize: logCfg.MaxPayloadSize, + out: os.Stdout, + } +} + +// Publish writes the event to stdout as JSON. Traffic logging is per-API: only +// events carrying a traffic-log directive (stamped by the log-message policy in +// access-log mode on APIs that opted in) are emitted; all others are skipped. +func (l *Log) Publish(event *dto.Event) { + if event == nil || event.TrafficLog == nil { + return + } + if l.isPathIgnored(event) { + return + } + + out := l.shapeEvent(event) + + data, err := json.Marshal(out) + if err != nil { + slog.Error("Failed to marshal analytics event for log publisher", "error", err) + return + } + + // When an explicit field selection is configured it is authoritative over which + // fields appear: project the serialized record down to (or removing) the named + // top-level keys and properties.* paths. + if fields := event.TrafficLog.Fields; fields != nil && len(fields.Names) > 0 { + if projected, perr := applyFieldsProjection(data, fields); perr != nil { + slog.Error("Failed to project traffic-log fields; emitting unprojected line", "error", perr) + } else { + data = projected + } + } + + l.mu.Lock() + defer l.mu.Unlock() + if _, err := fmt.Fprintln(l.out, string(data)); err != nil { + slog.Error("Failed to write analytics event to stdout", "error", err) + } +} + +// shapeEvent returns the event to serialize, applying the per-API traffic-log +// directive to a shallow copy with a cloned Properties map so the shared event +// observed by other publishers is left untouched. For each flow it drops the +// headers/payload the API did not request, removes per-flow excluded headers, and +// redacts globally masked headers. ALS-derived fields (latencies, status, timing) +// are always retained. +func (l *Log) shapeEvent(event *dto.Event) *dto.Event { + if event.Properties == nil { + return event + } + + props := make(map[string]interface{}, len(event.Properties)) + maps.Copy(props, event.Properties) + + dir := event.TrafficLog + // When an explicit field selection is set it is authoritative over presence, so + // the per-flow headers/payload booleans are not used for gating here; only header + // masking + per-flow excludeHeaders are applied, and the projection (in Publish) + // decides which fields survive. + gate := dir.Fields == nil || len(dir.Fields.Names) == 0 + l.applyFlow(props, dir.Request, "requestHeaders", "request_payload", gate) + l.applyFlow(props, dir.Response, "responseHeaders", "response_payload", gate) + + cp := *event + cp.Properties = props + return &cp +} + +// applyFlow enforces one flow's presentation rules on the cloned properties. +// Header content, when present, is always cleaned (per-flow excludeHeaders + global +// masking). When gate is true (no authoritative field selection), a nil flow or a +// disabled headers/payload field also drops the corresponding property entirely. +func (l *Log) applyFlow(props map[string]interface{}, flow *dto.TrafficLogFlow, headersKey, payloadKey string, gate bool) { + var exclude []string + if flow != nil { + exclude = flow.ExcludeHeaders + } + if raw, ok := props[headersKey].(string); ok { + props[headersKey] = l.filterHeaders(raw, exclude) + } + if raw, ok := props[payloadKey].(string); ok { + props[payloadKey] = l.truncatePayload(raw) + } + + if gate { + if flow == nil || !flow.Headers { + delete(props, headersKey) + } + if flow == nil || !flow.Payload { + delete(props, payloadKey) + } + } +} + +// isPathIgnored reports whether the event's original request path starts with any +// configured ignored prefix. It reads Operation.APIResourceTemplate, which carries +// the original (pre-rewrite) path. +func (l *Log) isPathIgnored(event *dto.Event) bool { + if len(l.ignoredPathPrefixes) == 0 || event.Operation == nil { + return false + } + path := event.Operation.APIResourceTemplate + if path == "" { + return false + } + for _, prefix := range l.ignoredPathPrefixes { + if strings.HasPrefix(path, prefix) { + return true + } + } + return false +} + +// truncatePayload returns up to maxPayloadSize bytes of the payload (0 = no +// limit). Truncation is on a byte boundary, matching the previous capture-time +// behavior. +func (l *Log) truncatePayload(s string) string { + if l.maxPayloadSize <= 0 || len(s) <= l.maxPayloadSize { + return s + } + return s[:l.maxPayloadSize] +} + +// applyFieldsProjection restricts the serialized event JSON to the configured +// fields. Names are top-level keys (e.g. "latencies") or dotted property paths +// (e.g. "properties.requestHeaders"). Mode "exclude" drops the named keys; any +// other value (default "include") keeps only the named keys. Naming the whole +// "properties" key keeps all of its subkeys. +func applyFieldsProjection(data []byte, fields *dto.TrafficLogFields) ([]byte, error) { + var m map[string]interface{} + if err := json.Unmarshal(data, &m); err != nil { + return nil, err + } + + topNames := make(map[string]bool) + propNames := make(map[string]bool) + propsReferenced := false + for _, name := range fields.Names { + if sub, ok := strings.CutPrefix(name, "properties."); ok { + if sub != "" { + propNames[sub] = true + propsReferenced = true + } + continue + } + if name == "properties" { + propsReferenced = true + } + if name != "" { + topNames[name] = true + } + } + + props, _ := m["properties"].(map[string]interface{}) + + if strings.EqualFold(fields.Mode, "exclude") { + for name := range topNames { + delete(m, name) + } + for sub := range propNames { + delete(props, sub) + } + if props != nil && len(props) == 0 { + delete(m, "properties") + } + return json.Marshal(m) + } + + // include (default): keep only referenced top-level keys. + for key := range m { + keep := topNames[key] || (key == "properties" && propsReferenced) + if !keep { + delete(m, key) + } + } + // When specific property subkeys were named (and not the whole "properties"), + // keep only those subkeys. + if props != nil && len(propNames) > 0 && !topNames["properties"] { + for sub := range props { + if !propNames[sub] { + delete(props, sub) + } + } + if len(props) == 0 { + delete(m, "properties") + } + } + return json.Marshal(m) +} + +// filterHeaders parses a JSON header map, drops any per-flow excluded headers, +// and redacts the values of globally masked headers (both case-insensitive). The +// raw string is returned unchanged if it is empty or not valid JSON. +func (l *Log) filterHeaders(raw string, excludeHeaders []string) string { + if raw == "" { + return raw + } + var headers map[string]interface{} + if err := json.Unmarshal([]byte(raw), &headers); err != nil { + return raw + } + + excluded := make(map[string]bool, len(excludeHeaders)) + for _, h := range excludeHeaders { + if h = strings.ToLower(strings.TrimSpace(h)); h != "" { + excluded[h] = true + } + } + + for name := range headers { + lower := strings.ToLower(name) + if excluded[lower] { + delete(headers, name) + continue + } + if l.maskedHeaders[lower] { + headers[name] = maskedHeaderValue + } + } + + out, err := json.Marshal(headers) + if err != nil { + return raw + } + return string(out) +} diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go new file mode 100644 index 0000000000..641d532aa0 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/config" +) + +// newLogToFile builds a Log publisher that writes to a temp file, returning the +// publisher and a function that reads back what was written. +func newLogToFile(t *testing.T, cfg *config.TrafficLoggingConfig) (*Log, func() string) { + t.Helper() + path := filepath.Join(t.TempDir(), "out.log") + f, err := os.Create(path) + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + + l := NewLog(cfg) + l.out = f + return l, func() string { + require.NoError(t, f.Sync()) + data, err := os.ReadFile(path) + require.NoError(t, err) + return string(data) + } +} + +// bothFlows returns a directive that opts in to logging both request and response +// headers and payloads — the "log everything captured" case. +func bothFlows() *dto.TrafficLogDirective { + return &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Payload: true, Headers: true}, + Response: &dto.TrafficLogFlow{Payload: true, Headers: true}, + } +} + +// decodeProps runs the single JSON line and returns the decoded event + its properties. +func decodeLine(t *testing.T, out string) (map[string]interface{}, map[string]interface{}) { + t.Helper() + var decoded map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(out), &decoded)) + props, _ := decoded["properties"].(map[string]interface{}) + return decoded, props +} + +func TestNewLog_NilConfig(t *testing.T) { + l := NewLog(nil) + require.NotNil(t, l) + assert.Empty(t, l.maskedHeaders) +} + +// Per-API gating: an event without a traffic-log directive is never emitted. +func TestLog_Publish_SkipsWhenNoDirective(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() // no TrafficLog set + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + assert.Empty(t, read(), "event without a traffic-log directive must not be logged") +} + +func TestLog_Publish_WritesJSONLineWithLatencies(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + out := read() + // Single line (one trailing newline). + assert.Equal(t, 1, strings.Count(strings.TrimRight(out, "\n"), "\n")+1) + + decoded, props := decodeLine(t, out) + api := decoded["api"].(map[string]interface{}) + assert.Equal(t, "test-api", api["apiName"]) + assert.Equal(t, `{"x-foo":"bar"}`, props["requestHeaders"]) + + // ALS-derived latencies are always present in the line — the key improvement + // over the inline log-message policy, which could never see them. + latencies := decoded["latencies"].(map[string]interface{}) + assert.Equal(t, float64(100), latencies["responseLatency"]) +} + +func TestLog_Publish_MasksHeaders(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"Authorization"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["requestHeaders"] = `{"Authorization":"Bearer secret","x-foo":"bar"}` + event.Properties["responseHeaders"] = `{"authorization":"Bearer secret2"}` + + l.Publish(event) + + _, props := decodeLine(t, read()) + + var reqH map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(props["requestHeaders"].(string)), &reqH)) + assert.Equal(t, "****", reqH["Authorization"]) // masked + assert.Equal(t, "bar", reqH["x-foo"]) // untouched + + var resH map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(props["responseHeaders"].(string)), &resH)) + assert.Equal(t, "****", resH["authorization"]) // case-insensitive match +} + +// Per-API excludeHeaders drops the header entirely (vs global masking which redacts). +func TestLog_Publish_ExcludeHeadersDrops(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true, ExcludeHeaders: []string{"X-Secret"}}, + } + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Secret":"top","x-foo":"bar"}` + + l.Publish(event) + + _, props := decodeLine(t, read()) + var reqH map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(props["requestHeaders"].(string)), &reqH)) + + _, hasSecret := reqH["X-Secret"] + assert.False(t, hasSecret, "excluded header must be dropped") + assert.Equal(t, "****", reqH["Authorization"], "masked header still redacted") + assert.Equal(t, "bar", reqH["x-foo"]) +} + +// headers:false omits the headers property; a nil flow omits its whole side. +func TestLog_Publish_DisabledFieldsOmitted(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: false, Payload: true}, + // Response flow nil -> both response props dropped. + } + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + event.Properties["request_payload"] = "req-body" + event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.Properties["response_payload"] = "resp-body" + + l.Publish(event) + + _, props := decodeLine(t, read()) + _, hasReqHeaders := props["requestHeaders"] + assert.False(t, hasReqHeaders, "request headers disabled -> omitted") + assert.Equal(t, "req-body", props["request_payload"], "request payload enabled -> kept") + + _, hasRespHeaders := props["responseHeaders"] + _, hasRespPayload := props["response_payload"] + assert.False(t, hasRespHeaders, "nil response flow -> response headers omitted") + assert.False(t, hasRespPayload, "nil response flow -> response payload omitted") +} + +func TestLog_Publish_DoesNotMutateSharedEvent(t *testing.T) { + l, _ := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + original := `{"authorization":"Bearer secret"}` + event.Properties["requestHeaders"] = original + + l.Publish(event) + + // The shared event (read by other publishers) must be untouched. + assert.Equal(t, original, event.Properties["requestHeaders"]) +} + +func TestLog_Publish_NilEvent(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + assert.NotPanics(t, func() { l.Publish(nil) }) + assert.Empty(t, read()) +} + +// Field selection (include): only named top-level keys and properties.* survive, +// and it is authoritative over presence (request.headers boolean is ignored, but +// excludeHeaders + masking still apply). +func TestLog_Publish_FieldsInclude(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Drop":"d","X-Keep":"k"}` + event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: false, ExcludeHeaders: []string{"X-Drop"}}, // boolean ignored + Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"latencies", "properties.requestHeaders"}}, + } + + l.Publish(event) + decoded, props := decodeLine(t, read()) + + _, hasAPI := decoded["api"] + _, hasOp := decoded["operation"] + assert.False(t, hasAPI, "api not listed -> dropped") + assert.False(t, hasOp, "operation not listed -> dropped") + assert.Contains(t, decoded, "latencies", "latencies listed -> kept") + require.NotNil(t, props, "properties kept (a properties.* path was listed)") + + _, hasResp := props["responseHeaders"] + assert.False(t, hasResp, "responseHeaders not listed -> dropped") + + reqRaw, ok := props["requestHeaders"].(string) + require.True(t, ok, "requestHeaders present (fields authoritative, boolean ignored)") + var reqH map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(reqRaw), &reqH)) + assert.Equal(t, "****", reqH["Authorization"], "masking still applies") + _, hasDrop := reqH["X-Drop"] + assert.False(t, hasDrop, "excludeHeaders still applies") + assert.Equal(t, "k", reqH["X-Keep"]) +} + +// Field selection (exclude): named keys are dropped, everything else remains. +func TestLog_Publish_FieldsExclude(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + event.Properties["request_payload"] = "secret-body" + event.TrafficLog = &dto.TrafficLogDirective{ + Fields: &dto.TrafficLogFields{Mode: "exclude", Names: []string{"operation", "properties.request_payload"}}, + } + + l.Publish(event) + decoded, props := decodeLine(t, read()) + + _, hasOp := decoded["operation"] + assert.False(t, hasOp, "operation excluded") + assert.Contains(t, decoded, "api", "api kept") + assert.Contains(t, decoded, "latencies", "latencies kept") + require.NotNil(t, props) + _, hasPayload := props["request_payload"] + assert.False(t, hasPayload, "request_payload excluded") + assert.Contains(t, props, "requestHeaders", "requestHeaders kept (not excluded)") +} + +// Naming the whole "properties" key keeps all of its subkeys. +func TestLog_Publish_FieldsIncludeWholeProperties(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.TrafficLog = &dto.TrafficLogDirective{ + Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"properties"}}, + } + + l.Publish(event) + decoded, props := decodeLine(t, read()) + + _, hasAPI := decoded["api"] + assert.False(t, hasAPI, "api not listed -> dropped") + require.NotNil(t, props) + assert.Contains(t, props, "requestHeaders") + assert.Contains(t, props, "responseHeaders") +} + +// Per-consumer path ignore: the publisher skips ignored prefixes (matched on the +// event's original path in Operation.APIResourceTemplate); other consumers are +// unaffected. +func TestLog_Publish_IgnoredPathSkipped(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{IgnoredPathPrefixes: []string{"/health", "/ready"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Operation.APIResourceTemplate = "/health/live" + + l.Publish(event) + assert.Empty(t, read(), "ignored path must not be logged") +} + +func TestLog_Publish_NonIgnoredPathLogged(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{IgnoredPathPrefixes: []string{"/health"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Operation.APIResourceTemplate = "/api/v1/orders" + + l.Publish(event) + assert.NotEmpty(t, read(), "non-ignored path must be logged") +} + +// Output-side payload truncation (0 = no limit). +func TestLog_Publish_TruncatesPayload(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaxPayloadSize: 5}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["request_payload"] = "hello world" + event.Properties["response_payload"] = "goodbye world" + + l.Publish(event) + _, props := decodeLine(t, read()) + assert.Equal(t, "hello", props["request_payload"]) + assert.Equal(t, "goodb", props["response_payload"]) +} + +func TestLog_Publish_NoTruncationWhenZero(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaxPayloadSize: 0}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["request_payload"] = "hello world" + + l.Publish(event) + _, props := decodeLine(t, read()) + assert.Equal(t, "hello world", props["request_payload"]) +} + +func TestLog_Publish_InvalidHeaderJSONLeftAsIs(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + event.Properties["requestHeaders"] = "not-json" + + l.Publish(event) + + _, props := decodeLine(t, read()) + assert.Equal(t, "not-json", props["requestHeaders"]) +} diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 5dfdc688ed..00fe960056 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -42,24 +42,46 @@ type Config struct { PolicyEngine PolicyEngine `koanf:"policy_engine"` GatewayController map[string]interface{} `koanf:"gateway_controller"` PolicyConfigurations map[string]interface{} `koanf:"policy_configurations"` + Collector CollectorConfig `koanf:"collector"` Analytics AnalyticsConfig `koanf:"analytics"` + TrafficLogging TrafficLoggingConfig `koanf:"traffic_logging"` TracingConfig TracingConfig `koanf:"tracing"` } +// CollectorConfig holds the data-collection ("collector") configuration. The +// collector is the shared capture pipeline that gathers request/response headers +// and bodies and ships them to the policy-engine over ALS. It is a prerequisite +// for every consumer of that data — analytics and traffic logging both require +// collector.enabled to be true. +type CollectorConfig struct { + // Enabled turns the collector on. When false, the ALS server is not started + // and no events are produced for any consumer. + Enabled bool `koanf:"enabled"` + // SendRequestBody / SendResponseBody attach captured request/response bodies + // onto the collected event. + SendRequestBody bool `koanf:"send_request_body"` + SendResponseBody bool `koanf:"send_response_body"` + // AccessLogsServiceCfg tunes the policy-engine ALS receiver (the gRPC server + // that ingests collected access logs). It is part of the collector transport + // and is configured under the shared [collector.als] section (the controller + // reads the same section to configure Envoy's sender side). + AccessLogsServiceCfg AccessLogsServiceConfig `koanf:"als"` +} + // AnalyticsConfig holds analytics configuration type AnalyticsConfig struct { - Enabled bool `koanf:"enabled"` - EnabledPublishers []string `koanf:"enabled_publishers"` - Publishers AnalyticsPublishersConfig `koanf:"publishers"` - GRPCEventServerCfg map[string]interface{} `koanf:"grpc_event_server"` - AccessLogsServiceCfg AccessLogsServiceConfig `koanf:"access_logs_service"` - // AllowPayloads controls whether request and response bodies are captured - // into analytics metadata and forwarded to analytics publishers. - // Deprecated: use SendRequestBody and SendResponseBody instead. - // When true, validateAnalyticsConfig maps both SendRequestBody and SendResponseBody - // to true if both are false. Because bools cannot represent "unset", this also - // applies when both new flags are explicitly false; remove allow_payloads when - // migrating and set the directional flags directly. + Enabled bool `koanf:"enabled"` + EnabledPublishers []string `koanf:"enabled_publishers"` + Publishers AnalyticsPublishersConfig `koanf:"publishers"` + GRPCEventServerCfg map[string]interface{} `koanf:"grpc_event_server"` + // AccessLogsServiceCfg is a deprecated alias. ALS receiver tuning moved to + // [collector.als]; when set here it is migrated onto the collector during + // validation (with a warning). Prefer [collector.als]. + AccessLogsServiceCfg AccessLogsServiceConfig `koanf:"access_logs_service"` + // AllowPayloads, SendRequestBody and SendResponseBody are deprecated aliases. + // Body capture now lives under [collector]. When set, these are mapped onto + // collector.send_request_body / collector.send_response_body during validation + // (with a warning). Prefer the [collector] fields directly. AllowPayloads bool `koanf:"allow_payloads"` SendRequestBody bool `koanf:"send_request_body"` SendResponseBody bool `koanf:"send_response_body"` @@ -70,6 +92,27 @@ type AnalyticsPublishersConfig struct { Moesif MoesifPublisherConfig `koanf:"moesif"` } +// TrafficLoggingConfig holds configuration for the stdout traffic-logging feature, +// which writes each collected event to stdout as a JSON line. It is a consumer of +// the collector and requires collector.enabled to be true. +type TrafficLoggingConfig struct { + // Enabled turns stdout JSON traffic logging on. + Enabled bool `koanf:"enabled"` + // MaskedHeaders lists header names (case-insensitive) whose values are + // redacted in the logged requestHeaders/responseHeaders. + MaskedHeaders []string `koanf:"masked_headers"` + // MaxPayloadSize caps the number of bytes of request/response payload written + // to the log line (0 = no limit). Truncation is applied at the publisher, so + // the collector still captures the full body and other consumers (e.g. Moesif) + // are unaffected. + MaxPayloadSize int `koanf:"max_payload_size"` + // IgnoredPathPrefixes suppresses stdout traffic-log lines for requests whose + // original path starts with any of these prefixes (e.g. health/readiness + // probes). Only the traffic-logging publisher is affected; other consumers + // still receive the event. + IgnoredPathPrefixes []string `koanf:"ignored_path_prefixes"` +} + // MoesifPublisherConfig holds Moesif-specific configuration type MoesifPublisherConfig struct { ApplicationID string `koanf:"application_id"` @@ -304,6 +347,22 @@ func Load(configPath string) (*Config, error) { return cfg, nil } +// defaultAccessLogsServiceConfig returns the default policy-engine ALS receiver tuning. +// Shared by the collector (canonical) and the deprecated [analytics].access_logs_service +// alias so a partial alias override migrates cleanly. +func defaultAccessLogsServiceConfig() AccessLogsServiceConfig { + return AccessLogsServiceConfig{ + Mode: "", + ServerPort: 18090, + ShutdownTimeout: 600 * time.Second, + PublicKeyPath: "", + PrivateKeyPath: "", + ALSPlainText: true, + ExtProcMaxMessageSize: 1000000000, + ExtProcMaxHeaderLimit: 8192, + } +} + // defaultConfig returns a Config struct with default configuration values func defaultConfig() *Config { return &Config{ @@ -350,6 +409,18 @@ func defaultConfig() *Config { }, TracingServiceName: "policy-engine", }, + Collector: CollectorConfig{ + Enabled: false, + SendRequestBody: false, + SendResponseBody: false, + AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), + }, + TrafficLogging: TrafficLoggingConfig{ + Enabled: false, + MaskedHeaders: []string{}, + MaxPayloadSize: 0, + IgnoredPathPrefixes: []string{}, + }, Analytics: AnalyticsConfig{ Enabled: false, EnabledPublishers: []string{"moesif"}, @@ -369,19 +440,12 @@ func defaultConfig() *Config { "buffer_size_bytes": 16384, "grpc_request_timeout": 20000000000, }, - AccessLogsServiceCfg: AccessLogsServiceConfig{ - Mode: "", - ServerPort: 18090, - ShutdownTimeout: 600 * time.Second, - PublicKeyPath: "", - PrivateKeyPath: "", - ALSPlainText: true, - ExtProcMaxMessageSize: 1000000000, - ExtProcMaxHeaderLimit: 8192, - }, - AllowPayloads: false, - SendRequestBody: false, - SendResponseBody: false, + // Deprecated alias: default mirrors the collector so a partial + // [analytics.access_logs_service] override migrates cleanly. + AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), + AllowPayloads: false, + SendRequestBody: false, + SendResponseBody: false, }, TracingConfig: TracingConfig{ Enabled: false, @@ -480,6 +544,12 @@ func (c *Config) Validate() error { return fmt.Errorf("invalid logging.format: %s (must be json or text)", c.PolicyEngine.Logging.Format) } + if err := c.validateCollectorConfig(); err != nil { + return err + } + if c.TrafficLogging.MaxPayloadSize < 0 { + return fmt.Errorf("traffic_logging.max_payload_size must be >= 0, got %d", c.TrafficLogging.MaxPayloadSize) + } if c.Analytics.Enabled { if err := c.validateAnalyticsConfig(); err != nil { return fmt.Errorf("analytics configuration validation failed: %v", err) @@ -536,49 +606,98 @@ func (c *Config) validateXDSConfig() error { return nil } -// validateAnalyticsConfig validates the analytics configuration -func (c *Config) validateAnalyticsConfig() error { - // Validate analytics configuration - if c.Analytics.Enabled { - // Migration path for deprecated analytics.allow_payloads. - // Runs when both directional flags are false, which is indistinguishable - // from "not set" because bool fields cannot represent unset vs explicit false. - if c.Analytics.AllowPayloads { - slog.Warn("analytics.allow_payloads is deprecated; use analytics.send_request_body and analytics.send_response_body instead") - if !c.Analytics.SendRequestBody && !c.Analytics.SendResponseBody { - c.Analytics.SendRequestBody = true - c.Analytics.SendResponseBody = true - } +// validateCollectorConfig migrates deprecated analytics capture aliases onto the +// collector and enforces the collector prerequisite: a consumer (analytics or +// traffic logging) requires the collector that feeds it. For backward compatibility +// this is a soft prerequisite — if a consumer is enabled without the collector, the +// collector is auto-enabled with a warning rather than failing. +func (c *Config) validateCollectorConfig() error { + c.migrateDeprecatedAnalyticsCapture() + c.migrateDeprecatedAnalyticsTransport() + + // Backward-compat bridge: a consumer cannot run without the collector that feeds + // it, but rather than fail an existing config that only set analytics.enabled + // (valid before the collector split), auto-enable the collector with a warning. + if (c.Analytics.Enabled || c.TrafficLogging.Enabled) && !c.Collector.Enabled { + slog.Warn("a consumer (analytics.enabled or traffic_logging.enabled) requires the collector; enabling collector.enabled automatically for backward compatibility. Set collector.enabled = true explicitly to silence this warning.") + c.Collector.Enabled = true + } + if c.Collector.Enabled { + if err := validateAccessLogsServiceConfig(c.Collector.AccessLogsServiceCfg); err != nil { + return err } + } + return nil +} - // Validate ALS server config (policy-engine side) - als := c.Analytics.AccessLogsServiceCfg - - // Validate ALS connection mode - switch als.Mode { - case "uds", "": - // UDS mode (default) - port is unused - case "tcp": - // TCP mode - validate port - if als.ServerPort <= 0 || als.ServerPort > 65535 { - return fmt.Errorf("analytics.access_logs_service.server_port must be between 1 and 65535, got %d", als.ServerPort) - } - default: - return fmt.Errorf("analytics.access_logs_service.mode must be 'uds' or 'tcp', got: %s", als.Mode) - } - if als.ShutdownTimeout <= 0 { - return fmt.Errorf("analytics.access_logs_service.shutdown_timeout must be positive, got %s", als.ShutdownTimeout) - } - if als.ExtProcMaxMessageSize <= 0 { - return fmt.Errorf("analytics.access_logs_service.max_message_size must be positive, got %d", als.ExtProcMaxMessageSize) +// migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].access_logs_service +// override onto the collector when the collector's receiver tuning is still at its +// default, so existing configs keep working after the transport moved to [collector]. +func (c *Config) migrateDeprecatedAnalyticsTransport() { + def := defaultAccessLogsServiceConfig() + if c.Analytics.AccessLogsServiceCfg != def { + slog.Warn("analytics.access_logs_service is deprecated; use collector.als instead") + if c.Collector.AccessLogsServiceCfg == def { + c.Collector.AccessLogsServiceCfg = c.Analytics.AccessLogsServiceCfg } - if als.ExtProcMaxHeaderLimit <= 0 { - return fmt.Errorf("analytics.access_logs_service.max_header_limit must be positive, got %d", als.ExtProcMaxHeaderLimit) + } +} + +// validateAccessLogsServiceConfig validates the policy-engine ALS receiver tuning. +func validateAccessLogsServiceConfig(als AccessLogsServiceConfig) error { + switch als.Mode { + case "uds", "": + // UDS mode (default) - port is unused + case "tcp": + if als.ServerPort <= 0 || als.ServerPort > 65535 { + return fmt.Errorf("collector.als.server_port must be between 1 and 65535, got %d", als.ServerPort) } - if als.ExtProcMaxHeaderLimit > math.MaxUint32 { - return fmt.Errorf("analytics.access_logs_service.max_header_limit must be <= %d, got %d", uint64(math.MaxUint32), als.ExtProcMaxHeaderLimit) + default: + return fmt.Errorf("collector.als.mode must be 'uds' or 'tcp', got: %s", als.Mode) + } + if als.ShutdownTimeout <= 0 { + return fmt.Errorf("collector.als.shutdown_timeout must be positive, got %s", als.ShutdownTimeout) + } + if als.ExtProcMaxMessageSize <= 0 { + return fmt.Errorf("collector.als.max_message_size must be positive, got %d", als.ExtProcMaxMessageSize) + } + if als.ExtProcMaxHeaderLimit <= 0 { + return fmt.Errorf("collector.als.max_header_limit must be positive, got %d", als.ExtProcMaxHeaderLimit) + } + if als.ExtProcMaxHeaderLimit > math.MaxUint32 { + return fmt.Errorf("collector.als.max_header_limit must be <= %d, got %d", uint64(math.MaxUint32), als.ExtProcMaxHeaderLimit) + } + return nil +} + +// migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / +// analytics.send_request_body / analytics.send_response_body onto the collector's +// body-capture flags (when the collector flag is not already set), so existing +// configs keep working after capture settings moved under [collector]. +func (c *Config) migrateDeprecatedAnalyticsCapture() { + // Directional aliases take precedence over allow_payloads. + if c.Analytics.SendRequestBody && !c.Collector.SendRequestBody { + slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") + c.Collector.SendRequestBody = true + } + if c.Analytics.SendResponseBody && !c.Collector.SendResponseBody { + slog.Warn("analytics.send_response_body is deprecated; use collector.send_response_body instead") + c.Collector.SendResponseBody = true + } + // allow_payloads only fills in when no directional body capture is configured. + if c.Analytics.AllowPayloads { + slog.Warn("analytics.allow_payloads is deprecated; use collector.send_request_body and collector.send_response_body instead") + if !c.Collector.SendRequestBody && !c.Collector.SendResponseBody { + c.Collector.SendRequestBody = true + c.Collector.SendResponseBody = true } + } +} +// validateAnalyticsConfig validates the analytics consumer configuration (publishers). +// ALS transport validation lives in validateCollectorConfig. +func (c *Config) validateAnalyticsConfig() error { + if c.Analytics.Enabled { // Validate enabled publishers for _, publisherName := range c.Analytics.EnabledPublishers { switch publisherName { @@ -595,6 +714,8 @@ func (c *Config) validateAnalyticsConfig() error { return fmt.Errorf("analytics.publishers.moesif.moesif_base_url must be a valid URL (e.g. https://api.moesif.net), got %q", moesifCfg.BaseURL) } } + case "log": + // The stdout/log publisher has no required configuration. default: return fmt.Errorf("unknown publisher type in enabled_publishers: %s", publisherName) } diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go index 3470b998c9..9be230088d 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go @@ -69,8 +69,17 @@ func validConfig() *Config { Timeout: 30 * time.Second, }, }, + // Collector enabled by default so tests that turn on a consumer + // (analytics / traffic logging) satisfy the collector prerequisite. + // ALS receiver defaults mirror production so transport validation passes + // and the deprecated alias stays neutral (no spurious migration). + Collector: CollectorConfig{ + Enabled: true, + AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), + }, Analytics: AnalyticsConfig{ - Enabled: false, + Enabled: false, + AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), }, TracingConfig: TracingConfig{ Enabled: false, @@ -948,7 +957,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid UDS config (default)", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -961,7 +970,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid UDS config (empty mode defaults to uds)", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -974,7 +983,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - valid TCP config", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "tcp", ServerPort: 18090, ShutdownTimeout: 600 * time.Second, @@ -988,7 +997,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid mode", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "invalid", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1002,7 +1011,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - TCP mode invalid ALS port", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "tcp", ServerPort: 0, ShutdownTimeout: 600 * time.Second, @@ -1017,7 +1026,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - UDS mode skips port validation", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "uds", ServerPort: 0, // Invalid port, but irrelevant in UDS mode ShutdownTimeout: 600 * time.Second, @@ -1031,7 +1040,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid shutdown timeout", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ShutdownTimeout: 0, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 8192, @@ -1044,7 +1053,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid max message size", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 0, ExtProcMaxHeaderLimit: 8192, @@ -1057,7 +1066,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - invalid max header limit", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: 0, @@ -1070,7 +1079,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { name: "analytics enabled - max header limit exceeds uint32", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, ExtProcMaxHeaderLimit: math.MaxInt, @@ -1100,7 +1109,8 @@ func TestValidate_AnalyticsConfig(t *testing.T) { func TestValidate_AnalyticsPayloadMigration(t *testing.T) { setValidAnalyticsALS := func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.Enabled = true // analytics is a consumer; the collector must be on + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1160,12 +1170,66 @@ func TestValidate_AnalyticsPayloadMigration(t *testing.T) { err := cfg.Validate() require.NoError(t, err) - assert.Equal(t, tt.wantSendReq, cfg.Analytics.SendRequestBody) - assert.Equal(t, tt.wantSendResp, cfg.Analytics.SendResponseBody) + // Deprecated analytics body aliases now migrate onto the collector. + assert.Equal(t, tt.wantSendReq, cfg.Collector.SendRequestBody) + assert.Equal(t, tt.wantSendResp, cfg.Collector.SendResponseBody) }) } } +// TestValidate_CollectorPrerequisite verifies that enabling a consumer (analytics, +// traffic logging) without the collector auto-enables the collector (a +// backward-compat soft prerequisite) rather than failing. +func TestValidate_CollectorPrerequisite(t *testing.T) { + validALS := AccessLogsServiceConfig{ + Mode: "uds", + ShutdownTimeout: 600 * time.Second, + ExtProcMaxMessageSize: 1000000, + ExtProcMaxHeaderLimit: 8192, + } + + t.Run("analytics enabled without collector auto-enables the collector", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = false + cfg.Analytics.Enabled = true + cfg.Collector.AccessLogsServiceCfg = validALS + cfg.Analytics.EnabledPublishers = []string{} + require.NoError(t, cfg.Validate()) + assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") + }) + + t.Run("traffic logging enabled without collector auto-enables the collector", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = false + cfg.TrafficLogging.Enabled = true + require.NoError(t, cfg.Validate()) + assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") + }) + + t.Run("traffic logging enabled with collector is valid", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = true + cfg.TrafficLogging.Enabled = true + require.NoError(t, cfg.Validate()) + }) + + t.Run("collector enabled with no consumers is valid", func(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = true + require.NoError(t, cfg.Validate()) + }) +} + +func TestValidate_TrafficLoggingMaxPayloadSize(t *testing.T) { + cfg := validConfig() + cfg.Collector.Enabled = true + cfg.TrafficLogging.Enabled = true + cfg.TrafficLogging.MaxPayloadSize = -1 + err := cfg.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "traffic_logging.max_payload_size") +} + // TestValidate_AnalyticsPublishers tests analytics publisher validation func TestValidate_AnalyticsPublishers(t *testing.T) { tests := []struct { @@ -1178,7 +1242,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "no publishers enabled", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1192,7 +1256,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "unknown publisher type", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1207,7 +1271,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - missing application_id", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1223,7 +1287,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - invalid publish_interval", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1240,7 +1304,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - invalid base_url", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, @@ -1258,7 +1322,7 @@ func TestValidate_AnalyticsPublishers(t *testing.T) { name: "moesif publisher - valid config", setup: func(cfg *Config) { cfg.Analytics.Enabled = true - cfg.Analytics.AccessLogsServiceCfg = AccessLogsServiceConfig{ + cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ ServerPort: 18090, ShutdownTimeout: 600 * time.Second, ExtProcMaxMessageSize: 1000000, diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go index 87049d7d8a..b68babbe08 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server.go @@ -90,15 +90,15 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { Timeout: 20 * time.Second, } maxHeaderListSize, err := checkedUInt32FromPositiveInt( - "analytics.access_logs_service.max_header_limit", - cfg.Analytics.AccessLogsServiceCfg.ExtProcMaxHeaderLimit, + "collector.als.max_header_limit", + cfg.Collector.AccessLogsServiceCfg.ExtProcMaxHeaderLimit, ) if err != nil { panic(err) } - server, err := CreateGRPCServer(cfg.Analytics.AccessLogsServiceCfg.PublicKeyPath, - cfg.Analytics.AccessLogsServiceCfg.PrivateKeyPath, cfg.Analytics.AccessLogsServiceCfg.ALSPlainText, - grpc.MaxRecvMsgSize(cfg.Analytics.AccessLogsServiceCfg.ExtProcMaxMessageSize), + server, err := CreateGRPCServer(cfg.Collector.AccessLogsServiceCfg.PublicKeyPath, + cfg.Collector.AccessLogsServiceCfg.PrivateKeyPath, cfg.Collector.AccessLogsServiceCfg.ALSPlainText, + grpc.MaxRecvMsgSize(cfg.Collector.AccessLogsServiceCfg.ExtProcMaxMessageSize), grpc.MaxHeaderListSize(maxHeaderListSize), grpc.KeepaliveParams(kaParams)) if err != nil { @@ -109,7 +109,7 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { // Create listener based on mode (same pattern as ext_proc in main.go) var listener net.Listener - alsMode := cfg.Analytics.AccessLogsServiceCfg.Mode + alsMode := cfg.Collector.AccessLogsServiceCfg.Mode if alsMode == "" { alsMode = "uds" } @@ -139,14 +139,14 @@ func StartAccessLogServiceServer(cfg *config.Config) *grpc.Server { } }() case "tcp": - listener, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.Analytics.AccessLogsServiceCfg.ServerPort)) + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.Collector.AccessLogsServiceCfg.ServerPort)) if err != nil { - slog.Error("Failed to listen on ALS TCP port", "port", cfg.Analytics.AccessLogsServiceCfg.ServerPort) + slog.Error("Failed to listen on ALS TCP port", "port", cfg.Collector.AccessLogsServiceCfg.ServerPort) panic(err) } go func() { - slog.Info("Starting to serve access log service server", "mode", "tcp", "port", cfg.Analytics.AccessLogsServiceCfg.ServerPort) + slog.Info("Starting to serve access log service server", "mode", "tcp", "port", cfg.Collector.AccessLogsServiceCfg.ServerPort) if err := server.Serve(listener); err != nil { slog.Error("ALS server exited", "error", err) } diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go index fd6dba08d7..0178fc5356 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go @@ -210,10 +210,8 @@ func TestStreamAccessLogs_MultipleMessages(t *testing.T) { func TestStartAccessLogServiceServer_TCP(t *testing.T) { cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: false, - EnabledPublishers: []string{}, - Publishers: config.AnalyticsPublishersConfig{}, + Collector: config.CollectorConfig{ + Enabled: true, AccessLogsServiceCfg: config.AccessLogsServiceConfig{ Mode: "tcp", ServerPort: 19001, // Use non-standard port to avoid conflicts diff --git a/gateway/system-policies/analytics/analytics.go b/gateway/system-policies/analytics/analytics.go index e0771e103c..7827cd34b0 100644 --- a/gateway/system-policies/analytics/analytics.go +++ b/gateway/system-policies/analytics/analytics.go @@ -152,6 +152,14 @@ func (a *AnalyticsPolicy) OnRequestHeaders(_ context.Context, reqCtx *policy.Req } } + // Capture all request headers when enabled, so they flow into analytics events + // (and the stdout/log publisher) without attaching a per-API header policy. + if sendReqHeaders, _ := getHeaderFlags(params); sendReqHeaders && reqCtx.Headers != nil { + if headers := serializeHeaders(reqCtx.Headers); headers != "" { + analyticsMetadata["request_headers"] = headers + } + } + if len(analyticsMetadata) > 0 { return policy.UpstreamRequestHeaderModifications{AnalyticsMetadata: analyticsMetadata} } @@ -198,6 +206,13 @@ func (a *AnalyticsPolicy) OnResponseHeaders(_ context.Context, respCtx *policy.R } } + // Capture all response headers when enabled. + if _, sendRespHeaders := getHeaderFlags(params); sendRespHeaders && respCtx.ResponseHeaders != nil { + if headers := serializeHeaders(respCtx.ResponseHeaders); headers != "" { + analyticsMetadata["response_headers"] = headers + } + } + if len(analyticsMetadata) > 0 { return policy.DownstreamResponseHeaderModifications{AnalyticsMetadata: analyticsMetadata} } @@ -883,18 +898,6 @@ func getPayloadFlags(params map[string]interface{}) (sendRequestBody, sendRespon return false, false } - parseBoolLike := func(v interface{}) bool { - switch val := v.(type) { - case bool: - return val - case string: - lower := strings.ToLower(strings.TrimSpace(val)) - return lower == "true" || lower == "1" || lower == "yes" - default: - return false - } - } - hasReq, hasResp := false, false if raw, ok := params["send_request_body"]; ok { @@ -922,6 +925,55 @@ func getPayloadFlags(params map[string]interface{}) (sendRequestBody, sendRespon return false, false } +// parseBoolLike interprets bool and common string ("true"/"1"/"yes") representations +// of a boolean policy parameter. +func parseBoolLike(v interface{}) bool { + switch val := v.(type) { + case bool: + return val + case string: + lower := strings.ToLower(strings.TrimSpace(val)) + return lower == "true" || lower == "1" || lower == "yes" + default: + return false + } +} + +// getHeaderFlags derives per-direction header capture flags from policy parameters. +func getHeaderFlags(params map[string]interface{}) (sendRequestHeaders, sendResponseHeaders bool) { + if params == nil { + return false, false + } + if raw, ok := params["send_request_headers"]; ok { + sendRequestHeaders = parseBoolLike(raw) + } + if raw, ok := params["send_response_headers"]; ok { + sendResponseHeaders = parseBoolLike(raw) + } + return sendRequestHeaders, sendResponseHeaders +} + +// serializeHeaders renders all headers as a JSON object string ({"name":"v1, v2"}), +// matching the request_headers/response_headers format the analytics engine reads. +// Returns "" when there are no headers. Sensitive values are not masked here; the +// stdout/log publisher applies masked_headers on output. +func serializeHeaders(headers *policy.Headers) string { + all := headers.GetAll() + if len(all) == 0 { + return "" + } + flat := make(map[string]string, len(all)) + for name, values := range all { + flat[name] = strings.Join(values, ", ") + } + data, err := json.Marshal(flat) + if err != nil { + slog.Error("Failed to marshal headers for analytics", "error", err) + return "" + } + return string(data) +} + // Helper to extract string values via JSONPath func extractStringFromJsonpath(payload map[string]interface{}, path string) string { val, err := utils.ExtractValueFromJsonpath(payload, path) diff --git a/gateway/system-policies/analytics/analytics_headers_test.go b/gateway/system-policies/analytics/analytics_headers_test.go new file mode 100644 index 0000000000..aed92c9f6b --- /dev/null +++ b/gateway/system-policies/analytics/analytics_headers_test.go @@ -0,0 +1,63 @@ +package analytics + +import ( + "encoding/json" + "testing" + + policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" +) + +func TestGetHeaderFlags(t *testing.T) { + cases := []struct { + name string + params map[string]interface{} + wantReq bool + wantResp bool + }{ + {"nil params", nil, false, false}, + {"absent", map[string]interface{}{}, false, false}, + {"bool true", map[string]interface{}{"send_request_headers": true, "send_response_headers": true}, true, true}, + {"string true", map[string]interface{}{"send_request_headers": "true"}, true, false}, + {"mixed", map[string]interface{}{"send_request_headers": false, "send_response_headers": "yes"}, false, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + gotReq, gotResp := getHeaderFlags(c.params) + if gotReq != c.wantReq || gotResp != c.wantResp { + t.Fatalf("getHeaderFlags(%v) = (%v, %v), want (%v, %v)", c.params, gotReq, gotResp, c.wantReq, c.wantResp) + } + }) + } +} + +func TestSerializeHeaders(t *testing.T) { + // Empty headers -> empty string. + if got := serializeHeaders(policy.NewHeaders(nil)); got != "" { + t.Fatalf("serializeHeaders(empty) = %q, want \"\"", got) + } + + h := policy.NewHeaders(map[string][]string{ + "Authorization": {"Bearer secret"}, + "X-Foo": {"a", "b"}, + }) + got := serializeHeaders(h) + if got == "" { + t.Fatal("serializeHeaders returned empty for non-empty headers") + } + + var decoded map[string]string + if err := json.Unmarshal([]byte(got), &decoded); err != nil { + t.Fatalf("output is not valid JSON: %v (%q)", err, got) + } + // NewHeaders lower-cases keys; multi-value headers are joined with ", ". + if decoded["authorization"] != "Bearer secret" { + t.Errorf("authorization = %q, want %q", decoded["authorization"], "Bearer secret") + } + if decoded["x-foo"] != "a, b" { + t.Errorf("x-foo = %q, want %q", decoded["x-foo"], "a, b") + } +} + +// Note: payload-size capping moved out of the capture path (the collector now +// captures full bodies); truncation is applied output-side by the traffic-logging +// publisher (traffic_logging.max_payload_size) and tested there. diff --git a/gateway/system-policies/analytics/policy-definition.yaml b/gateway/system-policies/analytics/policy-definition.yaml index 7fe915469d..eaed1b307e 100644 --- a/gateway/system-policies/analytics/policy-definition.yaml +++ b/gateway/system-policies/analytics/policy-definition.yaml @@ -25,6 +25,30 @@ parameters: into analytics metadata as response_payload, for both buffered and streaming responses, which is then forwarded through the analytics pipeline (including publishers like Moesif). + send_request_headers: + type: boolean + default: false + description: > + When true, the analytics system policy captures ALL request headers + into analytics metadata as request_headers, which is forwarded through + the analytics pipeline (including the stdout/log and Moesif publishers). + Sensitive values are not masked here; the log publisher's masked_headers + setting redacts them on output. + send_response_headers: + type: boolean + default: false + description: > + When true, the analytics system policy captures ALL response headers + into analytics metadata as response_headers, which is forwarded through + the analytics pipeline (including the stdout/log and Moesif publishers). + max_payload_size: + type: integer + default: 0 + description: > + Maximum number of bytes captured per request/response body when + send_request_body / send_response_body is enabled. 0 (default) means no + limit; a positive value truncates each body to that many bytes (silent, + byte-boundary truncation). allow_payloads: type: boolean default: false From 51844db87ca707480cb2c77790a2220020e1a044 Mon Sep 17 00:00:00 2001 From: Dineth Date: Sat, 4 Jul 2026 00:51:58 +0530 Subject: [PATCH 2/5] Refactor collector configuration to remove explicit enabled flag - Updated the collector configuration to remove the explicit `enabled` flag, making the collector implicitly active whenever a consumer (analytics or traffic logging) is enabled. - Adjusted related comments and documentation to reflect the new behavior. - Modified tests to ensure they validate the collector's state based on consumer configurations rather than an explicit flag. - Removed deprecated path ignoring functionality from traffic logging configuration. - Enhanced logging to include custom properties in traffic log events. --- gateway/configs/config-template.toml | 38 ++++----- gateway/configs/config.toml | 16 ++-- .../gateway-controller/pkg/config/config.go | 51 +++++++----- .../pkg/config/config_test.go | 35 ++++---- .../pkg/utils/system_policies.go | 8 +- .../pkg/utils/system_policies_test.go | 39 ++++----- .../gateway-controller/pkg/xds/translator.go | 8 +- .../policy-engine/cmd/policy-engine/main.go | 2 +- .../internal/analytics/analytics.go | 3 - .../internal/analytics/analytics_test.go | 4 +- .../internal/analytics/dto/event.go | 4 + .../internal/analytics/publishers/log.go | 53 +++--------- .../internal/analytics/publishers/log_test.go | 83 ++++++++++++++----- .../policy-engine/internal/config/config.go | 47 +++++------ .../internal/config/config_test.go | 52 ++++-------- .../utils/access_logger_server_test.go | 1 - 16 files changed, 208 insertions(+), 236 deletions(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index a9c3965cce..4f3530ad12 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -285,11 +285,11 @@ host = "localhost" # ============================================================================= # The collector is the shared data-capture pipeline: it injects the analytics # system policy and ships request/response headers and bodies to the policy-engine -# over ALS. It is a PREREQUISITE for every consumer below — analytics and -# traffic_logging both require collector.enabled = true (enabling a consumer with -# the collector off is a startup error). +# over ALS. It has NO on/off switch of its own — it is implicit, turning on +# automatically whenever a consumer below is enabled ([analytics] or +# [traffic_logging]) and staying off otherwise. This section only tunes what the +# collector captures. [collector] -enabled = false # Capture request/response payloads (bodies) into the collected event. Bodies are # captured in full; per-consumer size caps are applied on output (see # traffic_logging.max_payload_size). @@ -298,9 +298,9 @@ send_response_body = false # Capture ALL request/response headers into the collected event (request_headers/ # response_headers). Applies to every API via the collector system policy, so no # per-API header policy is needed. Sensitive values are redacted on output by each -# consumer (e.g. traffic_logging.masked_headers). -send_request_headers = false -send_response_headers = false +# consumer (e.g. traffic_logging.masked_headers). On by default. +send_request_headers = true +send_response_headers = true # ALS transport tuning (advanced; defaults are sensible). One section read by BOTH # ends of the Envoy → policy-engine access-log stream: @@ -323,7 +323,7 @@ max_message_size = 1000000000 max_header_limit = 8192 # ============================================================================= -# ANALYTICS CONFIGURATION (consumer — requires [collector].enabled = true) +# ANALYTICS CONFIGURATION (consumer — enabling it activates the collector) # ============================================================================= [analytics] enabled = false @@ -347,7 +347,7 @@ batch_size = 50 timer_wakeup_seconds = 3 # ============================================================================= -# TRAFFIC LOGGING (consumer — requires [collector].enabled = true) +# TRAFFIC LOGGING (consumer — enabling it activates the collector) # ============================================================================= # Writes collected events (rich API/application/AI/latency metadata, plus # request/response headers and — when the collector's send_*_body is enabled — @@ -355,27 +355,23 @@ timer_wakeup_seconds = 3 # Fluent Bit/Loki/ELK; no external SaaS needed. # # Per-API (opt-in): a line is emitted only for APIs that attach the `log-message` -# policy with `destination = access-log` — enabling this section alone does NOT -# log every API. So an API is logged only when all three hold: [collector] -# .enabled = true, [traffic_logging].enabled = true, and the `log-message` policy -# (access-log destination) is attached to that API. The policy's request/response -# parameters (per-flow payload/headers toggles and excludeHeaders) shape each -# API's line; they filter/mask what the collector captured (they cannot enable -# capture the collector was not configured to perform). +# policy with `enableTrafficLogging: true` — enabling this section alone does NOT +# log every API. So an API is logged only when both hold: [traffic_logging].enabled +# = true, and the `log-message` policy (enableTrafficLogging: true) is attached to +# that API. The policy's request/response parameters (per-flow payload/headers +# toggles and excludeHeaders) shape each API's line; they filter/mask what the +# collector captured (they cannot enable capture the collector was not configured +# to perform). [traffic_logging] enabled = false # Header names (case-insensitive) whose values are redacted as "****" in the # logged requestHeaders/responseHeaders. Applies globally; the `log-message` -# policy's (access-log mode) per-API excludeHeaders drop additional headers entirely. +# policy's per-API excludeHeaders drop additional headers entirely. masked_headers = ["authorization"] # Max bytes of request/response payload written to each log line (0 = no limit). # Applied output-side, so the collector still captures full bodies and other # consumers (e.g. Moesif) are unaffected. max_payload_size = 0 -# Suppress traffic-log lines for requests whose original (pre-rewrite) path starts -# with any of these prefixes (e.g. health/readiness probes). Only affects this -# stdout publisher — other consumers still receive the event. -ignored_path_prefixes = [] # ============================================================================= # POLICY CONFIGURATIONS diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index 98238a3fc1..bc22d25487 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -1,17 +1,17 @@ -# The collector is the shared data-capture pipeline. It must be enabled before any -# consumer (analytics, traffic_logging) can receive data. +# The collector is the shared data-capture pipeline. It has no on/off switch of its +# own: it turns on automatically whenever a consumer (analytics or traffic_logging) +# is enabled, and stays off otherwise. This section only tunes what is captured. [collector] -enabled = true # Capture request/response payloads (bodies) into the collected event (full bodies; # per-consumer size caps are applied on output). -send_request_body = true -send_response_body = true +send_request_body = false +send_response_body = false # Capture ALL request/response headers (redacted on output by per-consumer masking). send_request_headers = true send_response_headers = true -# Analytics consumer (e.g. Moesif). Requires [collector].enabled = true. +# Analytics consumer (e.g. Moesif). Enabling it activates the collector automatically. [analytics] enabled = false enabled_publishers = ["moesif"] @@ -20,15 +20,13 @@ enabled_publishers = ["moesif"] # application_id = "" # Traffic-logging consumer: writes each collected event to stdout as a JSON line -# (no external service needed). Requires [collector].enabled = true. +# (no external service needed). Enabling it activates the collector automatically. [traffic_logging] enabled = true # Header names (case-insensitive) whose values are redacted as "****". masked_headers = ["authorization"] # Max bytes of request/response payload written to each log line (0 = no limit). max_payload_size = 2048 -# Skip traffic-log lines for these path prefixes (e.g. health probes). -ignored_path_prefixes = ["/health", "/ready"] [router] gateway_host = "*" diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 8ed547a111..2530e073d0 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -50,6 +50,7 @@ type Config struct { PolicyConfigurations map[string]interface{} `koanf:"policy_configurations"` Collector CollectorConfig `koanf:"collector"` Analytics AnalyticsConfig `koanf:"analytics"` + TrafficLogging TrafficLoggingConfig `koanf:"traffic_logging"` TracingConfig TracingConfig `koanf:"tracing"` APIKey APIKeyConfig `koanf:"api_key"` // Subscriptions controls application-level subscription behaviour for APIs. @@ -61,12 +62,11 @@ type Config struct { // CollectorConfig holds the data-collection ("collector") configuration. The // collector is the shared capture pipeline (the analytics system policy plus the // Envoy→policy-engine ALS transport) that gathers request/response headers and -// bodies. It is a prerequisite for every consumer of that data — analytics and -// traffic logging both require collector.enabled to be true. +// bodies. It underpins every consumer of that data (analytics and traffic logging) +// and is implicitly active whenever a consumer is enabled — see +// Config.IsCollectorEnabled. This section tunes capture and transport; it has no +// on/off flag of its own. type CollectorConfig struct { - // Enabled turns the collector on. When false, the analytics system policy is - // not injected and Envoy ships no access logs to the policy-engine. - Enabled bool `koanf:"enabled"` // SendRequestBody / SendResponseBody capture request/response bodies into the // collected event. SendRequestBody bool `koanf:"send_request_body"` @@ -102,6 +102,17 @@ type AnalyticsConfig struct { SendResponseBody bool `koanf:"send_response_body"` } +// TrafficLoggingConfig mirrors the policy-engine's stdout traffic-logging consumer. +// The controller only needs to know whether it is enabled, so that the collector +// (system policy + ALS sink) is activated when traffic logging is on even if +// analytics is off. Presentation keys (masked_headers, max_payload_size) are +// policy-engine-only and intentionally not bound here. +type TrafficLoggingConfig struct { + // Enabled turns stdout JSON traffic logging on. Enabling it implicitly activates + // the collector (see Config.IsCollectorEnabled). + Enabled bool `koanf:"enabled"` +} + // SubscriptionsConfig holds configuration for application-level subscriptions. type SubscriptionsConfig struct { // EnableValidation toggles automatic injection of the subscriptionValidation @@ -963,11 +974,10 @@ func defaultConfig() *Config { SendResponseBody: false, }, Collector: CollectorConfig{ - Enabled: false, SendRequestBody: false, SendResponseBody: false, - SendRequestHeaders: false, - SendResponseHeaders: false, + SendRequestHeaders: true, + SendResponseHeaders: true, GRPCEventServerCfg: defaultGRPCEventServerConfig(), }, TracingConfig: TracingConfig{ @@ -1742,23 +1752,14 @@ func validateDomains(field string, domains []string) error { } // validateCollectorConfig migrates deprecated analytics aliases onto the collector -// and enforces the collector prerequisite: analytics (a consumer) requires the -// collector that feeds it. For backward compatibility this is a soft prerequisite — -// if a consumer is enabled without the collector, the collector is auto-enabled with -// a warning rather than failing. It also validates the ALS transport tuning when the -// collector is enabled. +// and validates the ALS transport tuning when the collector is active. The collector +// has no on/off flag of its own: it is implicitly active whenever a consumer is +// enabled (analytics or traffic logging) — see IsCollectorEnabled. func (c *Config) validateCollectorConfig() error { c.migrateDeprecatedAnalyticsCapture() c.migrateDeprecatedAnalyticsTransport() - // Backward-compat bridge: a consumer cannot run without the collector that feeds - // it, but rather than fail an existing config that only set analytics.enabled - // (valid before the collector split), auto-enable the collector with a warning. - if c.Analytics.Enabled && !c.Collector.Enabled { - slog.Warn("analytics.enabled requires the collector; enabling collector.enabled automatically for backward compatibility. Set collector.enabled = true explicitly to silence this warning.") - c.Collector.Enabled = true - } - if c.Collector.Enabled { + if c.IsCollectorEnabled() { if err := validateGRPCEventServerConfig(c.Collector.GRPCEventServerCfg); err != nil { return err } @@ -1766,6 +1767,14 @@ func (c *Config) validateCollectorConfig() error { return nil } +// IsCollectorEnabled reports whether the collector should run. The collector is +// implicit: it is active whenever any consumer of the collected data is enabled +// (analytics or stdout traffic logging), and off otherwise. When active, the +// controller injects the analytics system policy and configures Envoy's ALS sink. +func (c *Config) IsCollectorEnabled() bool { + return c.Analytics.Enabled || c.TrafficLogging.Enabled +} + // migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].grpc_event_server // override onto the collector when the collector's transport tuning is still at its // default, so existing configs keep working after the transport moved to [collector]. diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index e17039112d..4b237ac386 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -1302,10 +1302,9 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := validConfig() + // Analytics is a consumer; enabling it makes the collector implicitly + // active so these tests exercise the collector transport validation. cfg.Analytics.Enabled = tt.enabled - // Analytics is a consumer; enable the collector it depends on so these - // tests exercise analytics validation rather than the prerequisite check. - cfg.Collector.Enabled = tt.enabled if tt.setupConfig != nil { tt.setupConfig(cfg) } @@ -1320,34 +1319,34 @@ func TestConfig_ValidateAnalyticsConfig(t *testing.T) { } } -func TestConfig_CollectorPrerequisite(t *testing.T) { - t.Run("analytics enabled without collector auto-enables the collector", func(t *testing.T) { +// TestConfig_IsCollectorEnabled covers the implicit collector: it is active iff a +// consumer (analytics or traffic logging) is enabled, and off otherwise. +func TestConfig_IsCollectorEnabled(t *testing.T) { + t.Run("no consumers -> off", func(t *testing.T) { + cfg := validConfig() + assert.False(t, cfg.IsCollectorEnabled()) + require.NoError(t, cfg.Validate()) + }) + + t.Run("analytics on -> collector on", func(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = false cfg.Analytics.Enabled = true cfg.Analytics.EnabledPublishers = []string{} - // Deprecated transport alias with valid values migrates onto the auto-enabled collector. - cfg.Analytics.GRPCEventServerCfg.Mode = "uds" - cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 - cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 - cfg.Analytics.GRPCEventServerCfg.GRPCRequestTimeout = 5000 - cfg.Analytics.GRPCEventServerCfg.ServerPort = 18090 + assert.True(t, cfg.IsCollectorEnabled()) require.NoError(t, cfg.Validate()) - assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") }) - t.Run("collector enabled with no consumers is valid", func(t *testing.T) { + t.Run("traffic logging on -> collector on", func(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = true + cfg.TrafficLogging.Enabled = true + assert.True(t, cfg.IsCollectorEnabled()) require.NoError(t, cfg.Validate()) }) - } func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { setValidAnalyticsGRPC := func(cfg *Config) { - cfg.Analytics.Enabled = true - cfg.Collector.Enabled = true // analytics is a consumer; the collector must be on + cfg.Analytics.Enabled = true // a consumer being on makes the collector implicit cfg.Analytics.GRPCEventServerCfg.Mode = "uds" cfg.Analytics.GRPCEventServerCfg.BufferFlushInterval = 1000 cfg.Analytics.GRPCEventServerCfg.BufferSizeBytes = 16384 diff --git a/gateway/gateway-controller/pkg/utils/system_policies.go b/gateway/gateway-controller/pkg/utils/system_policies.go index 25a598a634..4d469fdc76 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies.go +++ b/gateway/gateway-controller/pkg/utils/system_policies.go @@ -85,10 +85,10 @@ var defaultSystemPolicies = []systemPolicyConfig{ return false } // The analytics system policy is the collector: it is injected whenever - // the collector is enabled, regardless of which consumer (analytics, - // traffic logging) ultimately reads the collected data. - slog.Debug("Collector state -> ", "state", cfg.Collector.Enabled) - return cfg.Collector.Enabled + // the collector is active — i.e. whenever any consumer (analytics or + // traffic logging) is enabled. + slog.Debug("Collector state -> ", "state", cfg.IsCollectorEnabled()) + return cfg.IsCollectorEnabled() }, // Default parameters (can be overridden via additionalProps) Parameters: map[string]interface{}{ diff --git a/gateway/gateway-controller/pkg/utils/system_policies_test.go b/gateway/gateway-controller/pkg/utils/system_policies_test.go index f324db0377..5f38555594 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies_test.go +++ b/gateway/gateway-controller/pkg/utils/system_policies_test.go @@ -125,11 +125,8 @@ func TestInjectSystemPolicies_NilConfig(t *testing.T) { } func TestInjectSystemPolicies_CollectorDisabled(t *testing.T) { - cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: false, - }, - } + // No consumer enabled -> collector implicitly off -> no system policy injected. + cfg := &config.Config{} policies := []policyenginev1.PolicyInstance{ {Name: "existing", Version: "v1.0.0"}, } @@ -140,10 +137,9 @@ func TestInjectSystemPolicies_CollectorDisabled(t *testing.T) { } func TestInjectSystemPolicies_CollectorEnabled(t *testing.T) { + // A consumer enabled -> collector implicitly on -> system policy injected. cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } policies := []policyenginev1.PolicyInstance{ {Name: "existing", Version: "v1.0.0"}, @@ -161,8 +157,8 @@ func TestInjectSystemPolicies_CollectorEnabled(t *testing.T) { func TestInjectSystemPolicies_BodyFlagsPropagated(t *testing.T) { cfg := &config.Config{ + Analytics: config.AnalyticsConfig{Enabled: true}, Collector: config.CollectorConfig{ - Enabled: true, SendRequestBody: true, SendResponseBody: true, }, @@ -177,8 +173,8 @@ func TestInjectSystemPolicies_BodyFlagsPropagated(t *testing.T) { func TestInjectSystemPolicies_BodyFlagsDefaultFalse(t *testing.T) { cfg := &config.Config{ + Analytics: config.AnalyticsConfig{Enabled: true}, Collector: config.CollectorConfig{ - Enabled: true, SendRequestBody: false, SendResponseBody: false, }, @@ -192,8 +188,8 @@ func TestInjectSystemPolicies_BodyFlagsDefaultFalse(t *testing.T) { func TestInjectSystemPolicies_HeaderFlagsPropagated(t *testing.T) { cfg := &config.Config{ + Analytics: config.AnalyticsConfig{Enabled: true}, Collector: config.CollectorConfig{ - Enabled: true, SendRequestHeaders: true, SendResponseHeaders: true, }, @@ -207,8 +203,10 @@ func TestInjectSystemPolicies_HeaderFlagsPropagated(t *testing.T) { } func TestInjectSystemPolicies_HeaderFlagsDefaultFalse(t *testing.T) { + // Zero-value collector (headers unset) with a consumer on: propagation passes the + // struct's false through. (Production defaults these true; see defaultConfig.) cfg := &config.Config{ - Collector: config.CollectorConfig{Enabled: true}, + Analytics: config.AnalyticsConfig{Enabled: true}, } result := InjectSystemPolicies(nil, cfg, nil) @@ -217,12 +215,9 @@ func TestInjectSystemPolicies_HeaderFlagsDefaultFalse(t *testing.T) { assert.Equal(t, false, result[0].Parameters["send_response_headers"]) } - func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } additionalProps := map[string]any{ constants.ANALYTICS_SYSTEM_POLICY_NAME: map[string]interface{}{ @@ -237,9 +232,7 @@ func TestInjectSystemPolicies_WithAdditionalProps(t *testing.T) { func TestInjectSystemPolicies_WithSharedParams(t *testing.T) { cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } additionalProps := map[string]any{ SharedParamsKey: map[string]interface{}{ @@ -254,9 +247,7 @@ func TestInjectSystemPolicies_WithSharedParams(t *testing.T) { func TestInjectSystemPolicies_EmptyPolicies(t *testing.T) { cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } result := InjectSystemPolicies([]policyenginev1.PolicyInstance{}, cfg, nil) @@ -266,9 +257,7 @@ func TestInjectSystemPolicies_EmptyPolicies(t *testing.T) { func TestInjectSystemPolicies_PreservesExistingPolicies(t *testing.T) { cfg := &config.Config{ - Collector: config.CollectorConfig{ - Enabled: true, - }, + Analytics: config.AnalyticsConfig{Enabled: true}, } policies := []policyenginev1.PolicyInstance{ {Name: "policy1", Version: "v1.0.0"}, diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index 7ae5387236..f0b5708d8f 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -561,9 +561,9 @@ func (t *Translator) TranslateConfigs( policyEngineCluster := t.createPolicyEngineCluster() clusters = append(clusters, policyEngineCluster) - // Add ALS cluster if the collector is enabled (it ships access logs over gRPC) + // Add ALS cluster if the collector is active (it ships access logs over gRPC) log.Debug("gRPC event server config", slog.Any("config", t.config.Collector.GRPCEventServerCfg)) - if t.config.Collector.Enabled { + if t.config.IsCollectorEnabled() { log.Info("collector is enabled, creating ALS cluster") alsCluster := t.createALSCluster() clusters = append(clusters, alsCluster) @@ -2785,8 +2785,8 @@ func (t *Translator) createAccessLogConfig() ([]*accesslog.AccessLog, error) { }, }) - // If the collector is enabled, create the gRPC access log config and append to existing access logs - if t.config.Collector.Enabled { + // If the collector is active, create the gRPC access log config and append to existing access logs + if t.config.IsCollectorEnabled() { t.logger.Info("Creating gRPC access log configuration") grpcAccessLog, err := t.createGRPCAccessLog() if err != nil { diff --git a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go index b7116fd01c..1281a7a519 100644 --- a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go +++ b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go @@ -273,7 +273,7 @@ func main() { // consumers (analytics, traffic logging). var alsServer *grpc.Server slog.DebugContext(ctx, "Policy engine ALS server config", "config", cfg.Collector.AccessLogsServiceCfg) - if cfg.Collector.Enabled { + if cfg.IsCollectorEnabled() { // Start the access log service server slog.Info("Starting the ALS gRPC server...") alsServer = utils.StartAccessLogServiceServer(cfg) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index 2ef1d2a5c1..d8d08fddab 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -153,9 +153,6 @@ func (c *Analytics) Process(event *v3.HTTPAccessLogEntry) { return } - // Path-based suppression is a per-consumer presentation concern: it is applied - // by the stdout traffic-logging publisher (traffic_logging.ignored_path_prefixes), - // not here, so other consumers still receive every event. analyticEvent := c.prepareAnalyticEvent(event) for _, publisher := range c.publishers { publisher.Publish(analyticEvent) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index cb1582568e..4f9bd0a061 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -63,9 +63,7 @@ func validAnalyticsConfigForValidation(analytics config.AnalyticsConfig) *config }, Analytics: analytics, } - cfg.Analytics.Enabled = true - // Analytics is a consumer; the collector must be enabled for it to validate. - cfg.Collector.Enabled = true + cfg.Analytics.Enabled = true // a consumer being on makes the collector implicit cfg.Collector.AccessLogsServiceCfg = config.AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index 75566db7e4..4e6daf6106 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -52,6 +52,10 @@ type TrafficLogDirective struct { Request *TrafficLogFlow `json:"request,omitempty"` Response *TrafficLogFlow `json:"response,omitempty"` Fields *TrafficLogFields `json:"fields,omitempty"` + // Properties holds the policy's resolved customProperties (context references + // already expanded at request time). The Log publisher emits them under + // properties.custom on the log line. + Properties map[string]interface{} `json:"properties,omitempty"` } // TrafficLogFlow is the per-flow (request or response) presentation config. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go index c76cdf3db9..508cef8076 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -43,10 +43,6 @@ type Log struct { // maskedHeaders holds lower-cased header names whose values are redacted in // the requestHeaders/responseHeaders properties before logging. maskedHeaders map[string]bool - // ignoredPathPrefixes: skip emitting a line when the event's original request - // path starts with any of these (per-consumer — other publishers are - // unaffected). Trimmed of blanks at construction. - ignoredPathPrefixes []string // maxPayloadSize caps the number of request/response payload bytes written to // the log line (0 = no limit). Truncation is output-side only. maxPayloadSize int @@ -70,18 +66,10 @@ func NewLog(logCfg *config.TrafficLoggingConfig) *Log { } } - prefixes := make([]string, 0, len(logCfg.IgnoredPathPrefixes)) - for _, p := range logCfg.IgnoredPathPrefixes { - if p = strings.TrimSpace(p); p != "" { - prefixes = append(prefixes, p) - } - } - return &Log{ - maskedHeaders: masked, - ignoredPathPrefixes: prefixes, - maxPayloadSize: logCfg.MaxPayloadSize, - out: os.Stdout, + maskedHeaders: masked, + maxPayloadSize: logCfg.MaxPayloadSize, + out: os.Stdout, } } @@ -92,9 +80,6 @@ func (l *Log) Publish(event *dto.Event) { if event == nil || event.TrafficLog == nil { return } - if l.isPathIgnored(event) { - return - } out := l.shapeEvent(event) @@ -129,14 +114,15 @@ func (l *Log) Publish(event *dto.Event) { // redacts globally masked headers. ALS-derived fields (latencies, status, timing) // are always retained. func (l *Log) shapeEvent(event *dto.Event) *dto.Event { - if event.Properties == nil { + dir := event.TrafficLog + hasCustom := dir != nil && len(dir.Properties) > 0 + if event.Properties == nil && !hasCustom { return event } - props := make(map[string]interface{}, len(event.Properties)) + props := make(map[string]interface{}, len(event.Properties)+1) maps.Copy(props, event.Properties) - dir := event.TrafficLog // When an explicit field selection is set it is authoritative over presence, so // the per-flow headers/payload booleans are not used for gating here; only header // masking + per-flow excludeHeaders are applied, and the projection (in Publish) @@ -145,6 +131,12 @@ func (l *Log) shapeEvent(event *dto.Event) *dto.Event { l.applyFlow(props, dir.Request, "requestHeaders", "request_payload", gate) l.applyFlow(props, dir.Response, "responseHeaders", "response_payload", gate) + // Attach the policy's resolved custom properties under a dedicated namespace, so + // they never collide with reserved keys and are projectable as "properties.custom". + if hasCustom { + props["custom"] = dir.Properties + } + cp := *event cp.Properties = props return &cp @@ -176,25 +168,6 @@ func (l *Log) applyFlow(props map[string]interface{}, flow *dto.TrafficLogFlow, } } -// isPathIgnored reports whether the event's original request path starts with any -// configured ignored prefix. It reads Operation.APIResourceTemplate, which carries -// the original (pre-rewrite) path. -func (l *Log) isPathIgnored(event *dto.Event) bool { - if len(l.ignoredPathPrefixes) == 0 || event.Operation == nil { - return false - } - path := event.Operation.APIResourceTemplate - if path == "" { - return false - } - for _, prefix := range l.ignoredPathPrefixes { - if strings.HasPrefix(path, prefix) { - return true - } - } - return false -} - // truncatePayload returns up to maxPayloadSize bytes of the payload (0 = no // limit). Truncation is on a byte boundary, matching the previous capture-time // behavior. diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go index 641d532aa0..a912257c18 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go @@ -107,6 +107,66 @@ func TestLog_Publish_WritesJSONLineWithLatencies(t *testing.T) { assert.Equal(t, float64(100), latencies["responseLatency"]) } +// Custom properties from the directive are emitted under properties.custom. +func TestLog_Publish_CustomPropertiesUnderCustom(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true}, + Properties: map[string]interface{}{ + "who": "alice", + "authType": "jwt", + "retryCount": float64(3), + }, + } + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + _, props := decodeLine(t, read()) + custom, ok := props["custom"].(map[string]interface{}) + require.True(t, ok, "expected properties.custom object, got %v", props["custom"]) + assert.Equal(t, "alice", custom["who"]) + assert.Equal(t, "jwt", custom["authType"]) + assert.Equal(t, float64(3), custom["retryCount"]) + // Reserved keys are untouched by the custom namespace. + assert.Equal(t, `{"x-foo":"bar"}`, props["requestHeaders"]) +} + +// A directive with no Properties emits no custom key. +func TestLog_Publish_NoCustomWhenAbsent(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + + l.Publish(event) + + _, props := decodeLine(t, read()) + _, present := props["custom"] + assert.False(t, present, "no custom key expected when directive has no Properties") +} + +// The fields projection can select properties.custom like any other property path. +func TestLog_Publish_CustomProjectableViaFields(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Properties: map[string]interface{}{"who": "alice"}, + Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"properties.custom"}}, + } + event.Properties["requestHeaders"] = `{"x-foo":"bar"}` + + l.Publish(event) + + _, props := decodeLine(t, read()) + custom, ok := props["custom"].(map[string]interface{}) + require.True(t, ok, "expected properties.custom retained by include projection") + assert.Equal(t, "alice", custom["who"]) + // Non-selected property dropped by the include projection. + _, hasHeaders := props["requestHeaders"] + assert.False(t, hasHeaders, "requestHeaders should be dropped by include projection") +} + func TestLog_Publish_MasksHeaders(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"Authorization"}}) event := createBaseEvent() @@ -273,29 +333,6 @@ func TestLog_Publish_FieldsIncludeWholeProperties(t *testing.T) { assert.Contains(t, props, "responseHeaders") } -// Per-consumer path ignore: the publisher skips ignored prefixes (matched on the -// event's original path in Operation.APIResourceTemplate); other consumers are -// unaffected. -func TestLog_Publish_IgnoredPathSkipped(t *testing.T) { - l, read := newLogToFile(t, &config.TrafficLoggingConfig{IgnoredPathPrefixes: []string{"/health", "/ready"}}) - event := createBaseEvent() - event.TrafficLog = bothFlows() - event.Operation.APIResourceTemplate = "/health/live" - - l.Publish(event) - assert.Empty(t, read(), "ignored path must not be logged") -} - -func TestLog_Publish_NonIgnoredPathLogged(t *testing.T) { - l, read := newLogToFile(t, &config.TrafficLoggingConfig{IgnoredPathPrefixes: []string{"/health"}}) - event := createBaseEvent() - event.TrafficLog = bothFlows() - event.Operation.APIResourceTemplate = "/api/v1/orders" - - l.Publish(event) - assert.NotEmpty(t, read(), "non-ignored path must be logged") -} - // Output-side payload truncation (0 = no limit). func TestLog_Publish_TruncatesPayload(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaxPayloadSize: 5}) diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 00fe960056..2d6690f7a7 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -50,13 +50,11 @@ type Config struct { // CollectorConfig holds the data-collection ("collector") configuration. The // collector is the shared capture pipeline that gathers request/response headers -// and bodies and ships them to the policy-engine over ALS. It is a prerequisite -// for every consumer of that data — analytics and traffic logging both require -// collector.enabled to be true. +// and bodies and ships them to the policy-engine over ALS. It underpins every +// consumer of that data (analytics and traffic logging) and is implicitly active +// whenever a consumer is enabled — see Config.IsCollectorEnabled. This section +// tunes capture and transport; it has no on/off flag of its own. type CollectorConfig struct { - // Enabled turns the collector on. When false, the ALS server is not started - // and no events are produced for any consumer. - Enabled bool `koanf:"enabled"` // SendRequestBody / SendResponseBody attach captured request/response bodies // onto the collected event. SendRequestBody bool `koanf:"send_request_body"` @@ -94,7 +92,7 @@ type AnalyticsPublishersConfig struct { // TrafficLoggingConfig holds configuration for the stdout traffic-logging feature, // which writes each collected event to stdout as a JSON line. It is a consumer of -// the collector and requires collector.enabled to be true. +// the collector; enabling it implicitly activates the collector. type TrafficLoggingConfig struct { // Enabled turns stdout JSON traffic logging on. Enabled bool `koanf:"enabled"` @@ -106,11 +104,6 @@ type TrafficLoggingConfig struct { // the collector still captures the full body and other consumers (e.g. Moesif) // are unaffected. MaxPayloadSize int `koanf:"max_payload_size"` - // IgnoredPathPrefixes suppresses stdout traffic-log lines for requests whose - // original path starts with any of these prefixes (e.g. health/readiness - // probes). Only the traffic-logging publisher is affected; other consumers - // still receive the event. - IgnoredPathPrefixes []string `koanf:"ignored_path_prefixes"` } // MoesifPublisherConfig holds Moesif-specific configuration @@ -410,16 +403,14 @@ func defaultConfig() *Config { TracingServiceName: "policy-engine", }, Collector: CollectorConfig{ - Enabled: false, SendRequestBody: false, SendResponseBody: false, AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), }, TrafficLogging: TrafficLoggingConfig{ - Enabled: false, - MaskedHeaders: []string{}, - MaxPayloadSize: 0, - IgnoredPathPrefixes: []string{}, + Enabled: false, + MaskedHeaders: []string{}, + MaxPayloadSize: 0, }, Analytics: AnalyticsConfig{ Enabled: false, @@ -608,21 +599,14 @@ func (c *Config) validateXDSConfig() error { // validateCollectorConfig migrates deprecated analytics capture aliases onto the // collector and enforces the collector prerequisite: a consumer (analytics or -// traffic logging) requires the collector that feeds it. For backward compatibility -// this is a soft prerequisite — if a consumer is enabled without the collector, the -// collector is auto-enabled with a warning rather than failing. +// traffic logging) requires the collector that feeds it. The collector has no +// on/off flag of its own: it is implicitly active whenever a consumer is enabled +// (see IsCollectorEnabled), so its transport is validated only in that case. func (c *Config) validateCollectorConfig() error { c.migrateDeprecatedAnalyticsCapture() c.migrateDeprecatedAnalyticsTransport() - // Backward-compat bridge: a consumer cannot run without the collector that feeds - // it, but rather than fail an existing config that only set analytics.enabled - // (valid before the collector split), auto-enable the collector with a warning. - if (c.Analytics.Enabled || c.TrafficLogging.Enabled) && !c.Collector.Enabled { - slog.Warn("a consumer (analytics.enabled or traffic_logging.enabled) requires the collector; enabling collector.enabled automatically for backward compatibility. Set collector.enabled = true explicitly to silence this warning.") - c.Collector.Enabled = true - } - if c.Collector.Enabled { + if c.IsCollectorEnabled() { if err := validateAccessLogsServiceConfig(c.Collector.AccessLogsServiceCfg); err != nil { return err } @@ -630,6 +614,13 @@ func (c *Config) validateCollectorConfig() error { return nil } +// IsCollectorEnabled reports whether the collector should run. The collector is +// implicit: it is active whenever any consumer of the collected data is enabled +// (analytics or stdout traffic logging), and off otherwise. +func (c *Config) IsCollectorEnabled() bool { + return c.Analytics.Enabled || c.TrafficLogging.Enabled +} + // migrateDeprecatedAnalyticsTransport maps a deprecated [analytics].access_logs_service // override onto the collector when the collector's receiver tuning is still at its // default, so existing configs keep working after the transport moved to [collector]. diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go index 9be230088d..b9371d6990 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go @@ -69,12 +69,10 @@ func validConfig() *Config { Timeout: 30 * time.Second, }, }, - // Collector enabled by default so tests that turn on a consumer - // (analytics / traffic logging) satisfy the collector prerequisite. - // ALS receiver defaults mirror production so transport validation passes - // and the deprecated alias stays neutral (no spurious migration). + // The collector is implicit (active whenever a consumer is enabled). ALS + // receiver defaults mirror production so transport validation passes and the + // deprecated alias stays neutral (no spurious migration). Collector: CollectorConfig{ - Enabled: true, AccessLogsServiceCfg: defaultAccessLogsServiceConfig(), }, Analytics: AnalyticsConfig{ @@ -1108,8 +1106,7 @@ func TestValidate_AnalyticsConfig(t *testing.T) { func TestValidate_AnalyticsPayloadMigration(t *testing.T) { setValidAnalyticsALS := func(cfg *Config) { - cfg.Analytics.Enabled = true - cfg.Collector.Enabled = true // analytics is a consumer; the collector must be on + cfg.Analytics.Enabled = true // a consumer being on makes the collector implicit cfg.Collector.AccessLogsServiceCfg = AccessLogsServiceConfig{ Mode: "uds", ShutdownTimeout: 600 * time.Second, @@ -1180,49 +1177,34 @@ func TestValidate_AnalyticsPayloadMigration(t *testing.T) { // TestValidate_CollectorPrerequisite verifies that enabling a consumer (analytics, // traffic logging) without the collector auto-enables the collector (a // backward-compat soft prerequisite) rather than failing. -func TestValidate_CollectorPrerequisite(t *testing.T) { - validALS := AccessLogsServiceConfig{ - Mode: "uds", - ShutdownTimeout: 600 * time.Second, - ExtProcMaxMessageSize: 1000000, - ExtProcMaxHeaderLimit: 8192, - } - - t.Run("analytics enabled without collector auto-enables the collector", func(t *testing.T) { +// TestIsCollectorEnabled covers the implicit collector: it is active iff a consumer +// (analytics or traffic logging) is enabled, and off otherwise. +func TestIsCollectorEnabled(t *testing.T) { + t.Run("no consumers -> off", func(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = false - cfg.Analytics.Enabled = true - cfg.Collector.AccessLogsServiceCfg = validALS - cfg.Analytics.EnabledPublishers = []string{} - require.NoError(t, cfg.Validate()) - assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = false + assert.False(t, cfg.IsCollectorEnabled()) }) - t.Run("traffic logging enabled without collector auto-enables the collector", func(t *testing.T) { + t.Run("analytics on -> collector on", func(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = false - cfg.TrafficLogging.Enabled = true + cfg.Analytics.Enabled = true + cfg.Analytics.EnabledPublishers = []string{} + assert.True(t, cfg.IsCollectorEnabled()) require.NoError(t, cfg.Validate()) - assert.True(t, cfg.Collector.Enabled, "collector should be auto-enabled for backward compatibility") }) - t.Run("traffic logging enabled with collector is valid", func(t *testing.T) { + t.Run("traffic logging on -> collector on", func(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = true cfg.TrafficLogging.Enabled = true - require.NoError(t, cfg.Validate()) - }) - - t.Run("collector enabled with no consumers is valid", func(t *testing.T) { - cfg := validConfig() - cfg.Collector.Enabled = true + assert.True(t, cfg.IsCollectorEnabled()) require.NoError(t, cfg.Validate()) }) } func TestValidate_TrafficLoggingMaxPayloadSize(t *testing.T) { cfg := validConfig() - cfg.Collector.Enabled = true cfg.TrafficLogging.Enabled = true cfg.TrafficLogging.MaxPayloadSize = -1 err := cfg.Validate() diff --git a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go index 0178fc5356..3176524241 100644 --- a/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/utils/access_logger_server_test.go @@ -211,7 +211,6 @@ func TestStreamAccessLogs_MultipleMessages(t *testing.T) { func TestStartAccessLogServiceServer_TCP(t *testing.T) { cfg := &config.Config{ Collector: config.CollectorConfig{ - Enabled: true, AccessLogsServiceCfg: config.AccessLogsServiceConfig{ Mode: "tcp", ServerPort: 19001, // Use non-standard port to avoid conflicts From 05c52b970fa65214954311ddf77f55f03939c18e Mon Sep 17 00:00:00 2001 From: Dineth Date: Mon, 6 Jul 2026 09:56:14 +0530 Subject: [PATCH 3/5] Enhance analytics configuration by adding support for additional masked headers and caching traffic log directives --- gateway/configs/config-template.toml | 2 +- gateway/configs/config.toml | 2 +- .../pkg/utils/system_policies.go | 10 ++-- .../internal/analytics/analytics.go | 47 ++++++++++++------- .../internal/analytics/analytics_test.go | 30 ++++++------ .../internal/analytics/dto/event.go | 10 ++++ .../internal/analytics/publishers/log.go | 4 +- .../policy-engine/internal/config/config.go | 2 - .../analytics/policy-definition.yaml | 8 ---- 9 files changed, 61 insertions(+), 54 deletions(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index 4f3530ad12..ae83145e3a 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -367,7 +367,7 @@ enabled = false # Header names (case-insensitive) whose values are redacted as "****" in the # logged requestHeaders/responseHeaders. Applies globally; the `log-message` # policy's per-API excludeHeaders drop additional headers entirely. -masked_headers = ["authorization"] +masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"] # Max bytes of request/response payload written to each log line (0 = no limit). # Applied output-side, so the collector still captures full bodies and other # consumers (e.g. Moesif) are unaffected. diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index bc22d25487..83107adfa6 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -24,7 +24,7 @@ enabled_publishers = ["moesif"] [traffic_logging] enabled = true # Header names (case-insensitive) whose values are redacted as "****". -masked_headers = ["authorization"] +masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"] # Max bytes of request/response payload written to each log line (0 = no limit). max_payload_size = 2048 diff --git a/gateway/gateway-controller/pkg/utils/system_policies.go b/gateway/gateway-controller/pkg/utils/system_policies.go index 4d469fdc76..8985f53388 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies.go +++ b/gateway/gateway-controller/pkg/utils/system_policies.go @@ -90,13 +90,9 @@ var defaultSystemPolicies = []systemPolicyConfig{ slog.Debug("Collector state -> ", "state", cfg.IsCollectorEnabled()) return cfg.IsCollectorEnabled() }, - // Default parameters (can be overridden via additionalProps) - Parameters: map[string]interface{}{ - "send_request_body": false, - "send_response_body": false, - "send_request_headers": false, - "send_response_headers": false, - }, + // No static defaults — all four capture flags are set at injection time + // from cfg.Collector (see InjectSystemPolicies below). + Parameters: nil, ExecutionCondition: nil, }, } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go index d8d08fddab..da8820fafb 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics.go @@ -23,6 +23,7 @@ import ( "log/slog" "maps" "strconv" + "sync" "time" v3 "github.com/envoyproxy/go-control-plane/envoy/data/accesslog/v3" @@ -59,8 +60,6 @@ const ( DefaultAnalyticsPublisher = "default" // MoesifAnalyticsPublisher represents the Moesif analytics publisher. MoesifAnalyticsPublisher = "moesif" - // LogAnalyticsPublisher represents the stdout/log analytics publisher. - LogAnalyticsPublisher = "log" // HeaderKeys represents the header keys. RequestHeadersKey = "request_headers" @@ -96,6 +95,10 @@ type Analytics struct { cfg *config.Config // publishers represents the publishers. publishers []analytics_publisher.Publisher + // directiveCache caches parsed TrafficLogDirectives keyed by raw JSON string. + // The directive is static per API deployment, so this eliminates per-request + // allocations after the first parse. + directiveCache sync.Map } // NewAnalytics creates a new instance of Analytics. Publishers are assembled from @@ -115,8 +118,6 @@ func NewAnalytics(cfg *config.Config) *Analytics { publishers = append(publishers, publisher) slog.Info("Moesif publisher added") } - case LogAnalyticsPublisher: - slog.Warn("\"log\" in analytics.enabled_publishers is no longer supported; enable stdout traffic logging via [traffic_logging] instead") default: slog.Warn("Unknown publisher type", "type", publisherName) } @@ -225,13 +226,7 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E // publishers (e.g. Moesif). A malformed marker still opts the API in with // default (empty) presentation, since attachment alone signals intent. if raw, exists := keyValuePairsFromMetadata[TrafficLogMetadataKey]; exists { - dir := &dto.TrafficLogDirective{} - if raw != "" { - if err := json.Unmarshal([]byte(raw), dir); err != nil { - slog.Warn("Failed to parse traffic_log directive; opting in with defaults", "error", err) - } - } - event.TrafficLog = dir + event.TrafficLog = c.parseTrafficLogDirective(raw) } // Prepare extended API extendedAPI := dto.ExtendedAPI{} @@ -467,22 +462,22 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E //Adding request and response headers for the analytics event if requestHeaders, exists := keyValuePairsFromMetadata[RequestHeadersKey]; exists { - event.Properties["requestHeaders"] = requestHeaders + event.Properties[dto.PropKeyRequestHeaders] = requestHeaders } if responseHeaders, exists := keyValuePairsFromMetadata[ResponseHeadersKey]; exists { - event.Properties["responseHeaders"] = responseHeaders + event.Properties[dto.PropKeyResponseHeaders] = responseHeaders } // Optionally attach request and response payloads when enabled via the collector. if c.cfg.Collector.SendRequestBody { - if requestPayload, ok := keyValuePairsFromMetadata["request_payload"]; ok && requestPayload != "" { - event.Properties["request_payload"] = requestPayload + if requestPayload, ok := keyValuePairsFromMetadata[dto.PropKeyRequestPayload]; ok && requestPayload != "" { + event.Properties[dto.PropKeyRequestPayload] = requestPayload slog.Debug("Analytics request payload captured", "size_bytes", len(requestPayload)) } } if c.cfg.Collector.SendResponseBody { - if responsePayload, ok := keyValuePairsFromMetadata["response_payload"]; ok && responsePayload != "" { - event.Properties["response_payload"] = responsePayload + if responsePayload, ok := keyValuePairsFromMetadata[dto.PropKeyResponsePayload]; ok && responsePayload != "" { + event.Properties[dto.PropKeyResponsePayload] = responsePayload slog.Debug("Analytics response payload captured", "size_bytes", len(responsePayload)) } } @@ -533,6 +528,24 @@ func (c *Analytics) prepareAnalyticEvent(logEntry *v3.HTTPAccessLogEntry) *dto.E return event } +// parseTrafficLogDirective returns the parsed TrafficLogDirective for the given +// raw JSON string, using a cache to avoid re-parsing the same static per-API +// directive on every request. Two concurrent first-parses of the same string are +// benign — the winner's value is stored and both callers get an equivalent result. +func (c *Analytics) parseTrafficLogDirective(raw string) *dto.TrafficLogDirective { + if cached, ok := c.directiveCache.Load(raw); ok { + return cached.(*dto.TrafficLogDirective) + } + dir := &dto.TrafficLogDirective{} + if raw != "" { + if err := json.Unmarshal([]byte(raw), dir); err != nil { + slog.Warn("Failed to parse traffic_log directive; opting in with defaults", "error", err) + } + } + c.directiveCache.Store(raw, dir) + return dir +} + func (c *Analytics) getAnonymousApp() *dto.Application { application := &dto.Application{} application.ApplicationID = anonymousValue diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 4f9bd0a061..4e2c0567e3 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -150,22 +150,6 @@ func TestNewAnalytics_TrafficLoggingEnabled(t *testing.T) { assert.Len(t, analytics.publishers, 1) // traffic-logging publisher should be registered } -func TestNewAnalytics_LogInEnabledPublishersIgnored(t *testing.T) { - // "log" in analytics.enabled_publishers is no longer supported; it must not - // register a publisher (use [traffic_logging] instead). - cfg := &config.Config{ - Analytics: config.AnalyticsConfig{ - Enabled: true, - EnabledPublishers: []string{LogAnalyticsPublisher}, - }, - } - - analytics := NewAnalytics(cfg) - - require.NotNil(t, analytics) - assert.Empty(t, analytics.publishers) -} - // ============================================================================= // isInvalid Tests // ============================================================================= @@ -473,6 +457,20 @@ func TestPrepareAnalyticEvent_MalformedTrafficLogMarker(t *testing.T) { assert.Nil(t, event.TrafficLog.Response) } +func TestPrepareAnalyticEvent_TrafficLogDirectiveCached(t *testing.T) { + cfg := &config.Config{} + a := NewAnalytics(cfg) + raw := `{"request":{"payload":false,"headers":true}}` + logEntry := createLogEntryWithMetadata(map[string]string{TrafficLogMetadataKey: raw}) + + event1 := a.prepareAnalyticEvent(logEntry) + event2 := a.prepareAnalyticEvent(logEntry) + + require.NotNil(t, event1.TrafficLog) + // Same pointer — second call must return the cached directive, not a new allocation. + assert.Same(t, event1.TrafficLog, event2.TrafficLog) +} + func TestPrepareAnalyticEvent_WithAnonymousApp(t *testing.T) { cfg := &config.Config{} analytics := NewAnalytics(cfg) diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index 4e6daf6106..2096091f7f 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -19,6 +19,16 @@ package dto import "time" +// Property keys written into Event.Properties by the analytics pipeline and read +// back by publishers (e.g. the Log publisher's masking and field-projection paths). +// Both sides must use these constants so a rename stays in one place. +const ( + PropKeyRequestHeaders = "requestHeaders" + PropKeyResponseHeaders = "responseHeaders" + PropKeyRequestPayload = "request_payload" + PropKeyResponsePayload = "response_payload" +) + // Event represents analytics event data. type Event struct { API *ExtendedAPI `json:"api,omitempty" bson:"api"` diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go index 508cef8076..5839d97bdf 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -128,8 +128,8 @@ func (l *Log) shapeEvent(event *dto.Event) *dto.Event { // masking + per-flow excludeHeaders are applied, and the projection (in Publish) // decides which fields survive. gate := dir.Fields == nil || len(dir.Fields.Names) == 0 - l.applyFlow(props, dir.Request, "requestHeaders", "request_payload", gate) - l.applyFlow(props, dir.Response, "responseHeaders", "response_payload", gate) + l.applyFlow(props, dir.Request, dto.PropKeyRequestHeaders, dto.PropKeyRequestPayload, gate) + l.applyFlow(props, dir.Response, dto.PropKeyResponseHeaders, dto.PropKeyResponsePayload, gate) // Attach the policy's resolved custom properties under a dedicated namespace, so // they never collide with reserved keys and are projectable as "properties.custom". diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 2d6690f7a7..4e26d0fd3e 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -705,8 +705,6 @@ func (c *Config) validateAnalyticsConfig() error { return fmt.Errorf("analytics.publishers.moesif.moesif_base_url must be a valid URL (e.g. https://api.moesif.net), got %q", moesifCfg.BaseURL) } } - case "log": - // The stdout/log publisher has no required configuration. default: return fmt.Errorf("unknown publisher type in enabled_publishers: %s", publisherName) } diff --git a/gateway/system-policies/analytics/policy-definition.yaml b/gateway/system-policies/analytics/policy-definition.yaml index eaed1b307e..c3c223bd8a 100644 --- a/gateway/system-policies/analytics/policy-definition.yaml +++ b/gateway/system-policies/analytics/policy-definition.yaml @@ -41,14 +41,6 @@ parameters: When true, the analytics system policy captures ALL response headers into analytics metadata as response_headers, which is forwarded through the analytics pipeline (including the stdout/log and Moesif publishers). - max_payload_size: - type: integer - default: 0 - description: > - Maximum number of bytes captured per request/response body when - send_request_body / send_response_body is enabled. 0 (default) means no - limit; a positive value truncates each body to that many bytes (silent, - byte-boundary truncation). allow_payloads: type: boolean default: false From a8e1c8112035f26dfb83ab6d0a835e4777a387c9 Mon Sep 17 00:00:00 2001 From: Dineth Date: Mon, 6 Jul 2026 15:35:32 +0530 Subject: [PATCH 4/5] Refactor traffic log handling to support labels and masked headers, and improve event structure --- .../internal/analytics/analytics_test.go | 3 +- .../internal/analytics/dto/event.go | 34 ++- .../internal/analytics/publishers/log.go | 258 +++++++---------- .../internal/analytics/publishers/log_test.go | 271 ++++++++++++------ .../analytics/publishers/traffic_log_event.go | 197 +++++++++++++ 5 files changed, 501 insertions(+), 262 deletions(-) create mode 100644 gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go index 4e2c0567e3..05afdaceab 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/analytics_test.go @@ -409,7 +409,7 @@ func TestPrepareAnalyticEvent_TrafficLogMarker(t *testing.T) { logEntry := createLogEntryWithMetadata(map[string]string{ APINameKey: "TestAPI", - TrafficLogMetadataKey: `{"request":{"payload":false,"headers":true,"excludeHeaders":["x-key"]}}`, + TrafficLogMetadataKey: `{"request":{"payload":false,"headers":true}}`, }) event := analytics.prepareAnalyticEvent(logEntry) @@ -419,7 +419,6 @@ func TestPrepareAnalyticEvent_TrafficLogMarker(t *testing.T) { require.NotNil(t, event.TrafficLog.Request) assert.True(t, event.TrafficLog.Request.Headers) assert.False(t, event.TrafficLog.Request.Payload) - assert.Equal(t, []string{"x-key"}, event.TrafficLog.Request.ExcludeHeaders) assert.Nil(t, event.TrafficLog.Response, "unset flow stays nil") // The marker must never leak into serialized properties (or other publishers). diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go index 2096091f7f..fe5d7a89c4 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/dto/event.go @@ -62,26 +62,30 @@ type TrafficLogDirective struct { Request *TrafficLogFlow `json:"request,omitempty"` Response *TrafficLogFlow `json:"response,omitempty"` Fields *TrafficLogFields `json:"fields,omitempty"` - // Properties holds the policy's resolved customProperties (context references - // already expanded at request time). The Log publisher emits them under - // properties.custom on the log line. - Properties map[string]interface{} `json:"properties,omitempty"` + // Labels holds the policy's resolved labels (context references already + // expanded at request time). The Log publisher emits them as a top-level + // "labels" object on the log line. + Labels map[string]interface{} `json:"labels,omitempty"` + // MaskedHeaders lists lower-cased header names whose values are redacted + // in the emitted log line. Merged with the global masked_headers config. + MaskedHeaders []string `json:"maskedHeaders,omitempty"` } // TrafficLogFlow is the per-flow (request or response) presentation config. type TrafficLogFlow struct { - Payload bool `json:"payload"` - Headers bool `json:"headers"` - ExcludeHeaders []string `json:"excludeHeaders,omitempty"` + Payload bool `json:"payload"` + Headers bool `json:"headers"` } -// TrafficLogFields selects which fields appear in the emitted line. When set it is -// authoritative over field presence: the per-flow Payload/Headers booleans are -// ignored (per-flow ExcludeHeaders and global masking still apply). Names are -// top-level keys (e.g. "latencies", "target") or dotted property paths -// (e.g. "properties.requestHeaders"). Mode "exclude" drops the named keys; any -// other value (default "include") keeps only the named keys. +// TrafficLogFields selects which fields appear in the emitted line. Exactly one +// of Only or Exclude should be set. Only keeps exactly the named fields; Exclude +// drops the named fields and keeps everything else. Names are top-level keys +// (e.g. "latencies", "requestHeaders") or dotted sub-key paths within map fields +// (e.g. "requestHeaders.authorization", "labels.env"). When set, this is +// authoritative over field presence; per-flow Payload/Headers booleans are ignored +// (global header masking still applies). If both are set, Only takes precedence. type TrafficLogFields struct { - Mode string `json:"mode,omitempty"` - Names []string `json:"names,omitempty"` + Only []string `json:"only,omitempty"` + Exclude []string `json:"exclude,omitempty"` } + diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go index 5839d97bdf..0718c1aec7 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log.go @@ -21,7 +21,6 @@ import ( "encoding/json" "fmt" "log/slog" - "maps" "os" "strings" "sync" @@ -81,25 +80,33 @@ func (l *Log) Publish(event *dto.Event) { return } - out := l.shapeEvent(event) + dir := event.TrafficLog + tl := l.toTrafficLogEvent(event, dir) - data, err := json.Marshal(out) + data, err := json.Marshal(tl) if err != nil { - slog.Error("Failed to marshal analytics event for log publisher", "error", err) + slog.Error("Failed to marshal traffic-log event", "error", err) return } - // When an explicit field selection is configured it is authoritative over which - // fields appear: project the serialized record down to (or removing) the named - // top-level keys and properties.* paths. - if fields := event.TrafficLog.Fields; fields != nil && len(fields.Names) > 0 { - if projected, perr := applyFieldsProjection(data, fields); perr != nil { - slog.Error("Failed to project traffic-log fields; emitting unprojected line", "error", perr) + if fields := dir.Fields; fields != nil && (len(fields.Only) > 0 || len(fields.Exclude) > 0) { + var m map[string]interface{} + if err := json.Unmarshal(data, &m); err != nil { + slog.Error("Failed to unmarshal for field projection; emitting as-is", "error", err) } else { - data = projected + applyFieldsProjection(m, fields) + if projected, merr := json.Marshal(m); merr == nil { + data = projected + } else { + slog.Error("Failed to remarshal after field projection; emitting as-is", "error", merr) + } } } + l.write(data) +} + +func (l *Log) write(data []byte) { l.mu.Lock() defer l.mu.Unlock() if _, err := fmt.Fprintln(l.out, string(data)); err != nil { @@ -107,65 +114,33 @@ func (l *Log) Publish(event *dto.Event) { } } -// shapeEvent returns the event to serialize, applying the per-API traffic-log -// directive to a shallow copy with a cloned Properties map so the shared event -// observed by other publishers is left untouched. For each flow it drops the -// headers/payload the API did not request, removes per-flow excluded headers, and -// redacts globally masked headers. ALS-derived fields (latencies, status, timing) -// are always retained. -func (l *Log) shapeEvent(event *dto.Event) *dto.Event { - dir := event.TrafficLog - hasCustom := dir != nil && len(dir.Properties) > 0 - if event.Properties == nil && !hasCustom { - return event - } - - props := make(map[string]interface{}, len(event.Properties)+1) - maps.Copy(props, event.Properties) - - // When an explicit field selection is set it is authoritative over presence, so - // the per-flow headers/payload booleans are not used for gating here; only header - // masking + per-flow excludeHeaders are applied, and the projection (in Publish) - // decides which fields survive. - gate := dir.Fields == nil || len(dir.Fields.Names) == 0 - l.applyFlow(props, dir.Request, dto.PropKeyRequestHeaders, dto.PropKeyRequestPayload, gate) - l.applyFlow(props, dir.Response, dto.PropKeyResponseHeaders, dto.PropKeyResponsePayload, gate) - - // Attach the policy's resolved custom properties under a dedicated namespace, so - // they never collide with reserved keys and are projectable as "properties.custom". - if hasCustom { - props["custom"] = dir.Properties - } - - cp := *event - cp.Properties = props - return &cp -} - -// applyFlow enforces one flow's presentation rules on the cloned properties. -// Header content, when present, is always cleaned (per-flow excludeHeaders + global -// masking). When gate is true (no authoritative field selection), a nil flow or a -// disabled headers/payload field also drops the corresponding property entirely. -func (l *Log) applyFlow(props map[string]interface{}, flow *dto.TrafficLogFlow, headersKey, payloadKey string, gate bool) { - var exclude []string - if flow != nil { - exclude = flow.ExcludeHeaders - } - if raw, ok := props[headersKey].(string); ok { - props[headersKey] = l.filterHeaders(raw, exclude) - } - if raw, ok := props[payloadKey].(string); ok { - props[payloadKey] = l.truncatePayload(raw) - } - - if gate { - if flow == nil || !flow.Headers { - delete(props, headersKey) - } - if flow == nil || !flow.Payload { - delete(props, payloadKey) +// parseHeadersFromString converts the JSON-encoded header value stored in +// event.Properties (a map[string]string or map[string][]string serialized by the +// ext_proc layer) into a map[string]string so it embeds as a plain JSON object +// in the log line. Other publishers (e.g. Moesif) read the raw string directly; +// the Log publisher calls this only on the local TrafficLogEvent it builds, so +// the shared event is never modified. Multi-value headers are flattened to their +// first value. Returns nil on empty input or parse failure. +func parseHeadersFromString(raw string) map[string]string { + if raw == "" { + return nil + } + var single map[string]string + if err := json.Unmarshal([]byte(raw), &single); err == nil { + return single + } + // Fallback: multi-value wire format — flatten to first value. + var multi map[string][]string + if err := json.Unmarshal([]byte(raw), &multi); err == nil { + out := make(map[string]string, len(multi)) + for k, vs := range multi { + if len(vs) > 0 { + out[k] = vs[0] + } } + return out } + return nil } // truncatePayload returns up to maxPayloadSize bytes of the payload (0 = no @@ -178,106 +153,73 @@ func (l *Log) truncatePayload(s string) string { return s[:l.maxPayloadSize] } -// applyFieldsProjection restricts the serialized event JSON to the configured -// fields. Names are top-level keys (e.g. "latencies") or dotted property paths -// (e.g. "properties.requestHeaders"). Mode "exclude" drops the named keys; any -// other value (default "include") keeps only the named keys. Naming the whole -// "properties" key keeps all of its subkeys. -func applyFieldsProjection(data []byte, fields *dto.TrafficLogFields) ([]byte, error) { - var m map[string]interface{} - if err := json.Unmarshal(data, &m); err != nil { - return nil, err - } - - topNames := make(map[string]bool) - propNames := make(map[string]bool) - propsReferenced := false - for _, name := range fields.Names { - if sub, ok := strings.CutPrefix(name, "properties."); ok { - if sub != "" { - propNames[sub] = true - propsReferenced = true +// applyFieldsProjection mutates m in place to restrict it to the configured +// fields. Names are top-level keys (e.g. "latencies", "requestHeaders") or +// dotted sub-key paths within map fields (e.g. "requestHeaders.authorization", +// "labels.env"). Only keeps exactly the named fields; Exclude drops the named +// fields and keeps everything else. If both are set, Only takes precedence. +func applyFieldsProjection(m map[string]interface{}, fields *dto.TrafficLogFields) { + if len(fields.Only) > 0 { + directKeys := make(map[string]bool) + subKeys := make(map[string][]string) // topKey → sub-keys to keep + for _, name := range fields.Only { + if top, sub, found := strings.Cut(name, "."); found { + subKeys[top] = append(subKeys[top], sub) + } else { + directKeys[name] = true } - continue - } - if name == "properties" { - propsReferenced = true } - if name != "" { - topNames[name] = true - } - } - - props, _ := m["properties"].(map[string]interface{}) - - if strings.EqualFold(fields.Mode, "exclude") { - for name := range topNames { - delete(m, name) - } - for sub := range propNames { - delete(props, sub) - } - if props != nil && len(props) == 0 { - delete(m, "properties") + for key := range m { + if !directKeys[key] && subKeys[key] == nil { + delete(m, key) + } } - return json.Marshal(m) - } - - // include (default): keep only referenced top-level keys. - for key := range m { - keep := topNames[key] || (key == "properties" && propsReferenced) - if !keep { - delete(m, key) + for top, subs := range subKeys { + if directKeys[top] { + continue // whole key kept; don't filter sub-keys + } + if nested, ok := m[top].(map[string]interface{}); ok { + keep := make(map[string]bool, len(subs)) + for _, s := range subs { + keep[s] = true + } + for k := range nested { + if !keep[k] { + delete(nested, k) + } + } + if len(nested) == 0 { + delete(m, top) + } + } } + return } - // When specific property subkeys were named (and not the whole "properties"), - // keep only those subkeys. - if props != nil && len(propNames) > 0 && !topNames["properties"] { - for sub := range props { - if !propNames[sub] { - delete(props, sub) + for _, name := range fields.Exclude { + if top, sub, found := strings.Cut(name, "."); found { + if nested, ok := m[top].(map[string]interface{}); ok { + delete(nested, sub) + if len(nested) == 0 { + delete(m, top) + } } - } - if len(props) == 0 { - delete(m, "properties") + } else { + delete(m, name) } } - return json.Marshal(m) } -// filterHeaders parses a JSON header map, drops any per-flow excluded headers, -// and redacts the values of globally masked headers (both case-insensitive). The -// raw string is returned unchanged if it is empty or not valid JSON. -func (l *Log) filterHeaders(raw string, excludeHeaders []string) string { - if raw == "" { - return raw - } - var headers map[string]interface{} - if err := json.Unmarshal([]byte(raw), &headers); err != nil { - return raw - } - - excluded := make(map[string]bool, len(excludeHeaders)) - for _, h := range excludeHeaders { - if h = strings.ToLower(strings.TrimSpace(h)); h != "" { - excluded[h] = true - } - } - - for name := range headers { - lower := strings.ToLower(name) - if excluded[lower] { - delete(headers, name) - continue - } - if l.maskedHeaders[lower] { - headers[name] = maskedHeaderValue +// maskHeaders redacts header values whose names appear in mask (case-insensitive). +// Returns a new map; the input is not modified. Per-header exclusion is handled +// downstream by applyFieldsProjection via dotted paths (e.g. "requestHeaders.authorization"). +func (l *Log) maskHeaders(headers map[string]string, mask map[string]bool) map[string]string { + result := make(map[string]string, len(headers)) + for name, value := range headers { + if mask[strings.ToLower(name)] { + result[name] = maskedHeaderValue + } else { + result[name] = value } } - - out, err := json.Marshal(headers) - if err != nil { - return raw - } - return string(out) + return result } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go index a912257c18..fa4fb8c5e0 100644 --- a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/log_test.go @@ -58,13 +58,21 @@ func bothFlows() *dto.TrafficLogDirective { } } -// decodeProps runs the single JSON line and returns the decoded event + its properties. -func decodeLine(t *testing.T, out string) (map[string]interface{}, map[string]interface{}) { +// decodeLine parses the single JSON line emitted by the Log publisher. +func decodeLine(t *testing.T, out string) map[string]interface{} { t.Helper() var decoded map[string]interface{} require.NoError(t, json.Unmarshal([]byte(out), &decoded)) - props, _ := decoded["properties"].(map[string]interface{}) - return decoded, props + return decoded +} + +// headerMap returns the JSON-decoded header object as map[string]interface{} for +// easy key/value assertions. Fails the test if v is not the expected type. +func headerMap(t *testing.T, v interface{}) map[string]interface{} { + t.Helper() + m, ok := v.(map[string]interface{}) + require.True(t, ok, "expected header object (map[string]interface{}), got %T", v) + return m } func TestNewLog_NilConfig(t *testing.T) { @@ -96,10 +104,11 @@ func TestLog_Publish_WritesJSONLineWithLatencies(t *testing.T) { // Single line (one trailing newline). assert.Equal(t, 1, strings.Count(strings.TrimRight(out, "\n"), "\n")+1) - decoded, props := decodeLine(t, out) + decoded := decodeLine(t, out) api := decoded["api"].(map[string]interface{}) - assert.Equal(t, "test-api", api["apiName"]) - assert.Equal(t, `{"x-foo":"bar"}`, props["requestHeaders"]) + assert.Equal(t, "test-api", api["name"]) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "bar", reqH["x-foo"]) // ALS-derived latencies are always present in the line — the key improvement // over the inline log-message policy, which could never see them. @@ -107,13 +116,13 @@ func TestLog_Publish_WritesJSONLineWithLatencies(t *testing.T) { assert.Equal(t, float64(100), latencies["responseLatency"]) } -// Custom properties from the directive are emitted under properties.custom. -func TestLog_Publish_CustomPropertiesUnderCustom(t *testing.T) { +// Labels from the directive are emitted as a top-level "labels" object. +func TestLog_Publish_LabelsTopLevel(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.TrafficLog = &dto.TrafficLogDirective{ Request: &dto.TrafficLogFlow{Headers: true}, - Properties: map[string]interface{}{ + Labels: map[string]interface{}{ "who": "alice", "authType": "jwt", "retryCount": float64(3), @@ -123,48 +132,47 @@ func TestLog_Publish_CustomPropertiesUnderCustom(t *testing.T) { l.Publish(event) - _, props := decodeLine(t, read()) - custom, ok := props["custom"].(map[string]interface{}) - require.True(t, ok, "expected properties.custom object, got %v", props["custom"]) - assert.Equal(t, "alice", custom["who"]) - assert.Equal(t, "jwt", custom["authType"]) - assert.Equal(t, float64(3), custom["retryCount"]) - // Reserved keys are untouched by the custom namespace. - assert.Equal(t, `{"x-foo":"bar"}`, props["requestHeaders"]) + decoded := decodeLine(t, read()) + labels, ok := decoded["labels"].(map[string]interface{}) + require.True(t, ok, "expected top-level labels object, got %T", decoded["labels"]) + assert.Equal(t, "alice", labels["who"]) + assert.Equal(t, "jwt", labels["authType"]) + assert.Equal(t, float64(3), labels["retryCount"]) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "bar", reqH["x-foo"]) } -// A directive with no Properties emits no custom key. -func TestLog_Publish_NoCustomWhenAbsent(t *testing.T) { +// A directive with no labels emits no "labels" key. +func TestLog_Publish_NoLabelsWhenAbsent(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.TrafficLog = bothFlows() l.Publish(event) - _, props := decodeLine(t, read()) - _, present := props["custom"] - assert.False(t, present, "no custom key expected when directive has no Properties") + decoded := decodeLine(t, read()) + _, present := decoded["labels"] + assert.False(t, present, "no labels key expected when directive has no labels") } -// The fields projection can select properties.custom like any other property path. -func TestLog_Publish_CustomProjectableViaFields(t *testing.T) { +// The fields projection can select "labels" like any other top-level key. +func TestLog_Publish_LabelsProjectableViaFields(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.TrafficLog = &dto.TrafficLogDirective{ - Properties: map[string]interface{}{"who": "alice"}, - Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"properties.custom"}}, + Labels: map[string]interface{}{"who": "alice"}, + Fields: &dto.TrafficLogFields{Only: []string{"labels"}}, } event.Properties["requestHeaders"] = `{"x-foo":"bar"}` l.Publish(event) - _, props := decodeLine(t, read()) - custom, ok := props["custom"].(map[string]interface{}) - require.True(t, ok, "expected properties.custom retained by include projection") - assert.Equal(t, "alice", custom["who"]) - // Non-selected property dropped by the include projection. - _, hasHeaders := props["requestHeaders"] - assert.False(t, hasHeaders, "requestHeaders should be dropped by include projection") + decoded := decodeLine(t, read()) + labels, ok := decoded["labels"].(map[string]interface{}) + require.True(t, ok, "expected labels retained by include projection") + assert.Equal(t, "alice", labels["who"]) + _, hasHeaders := decoded["requestHeaders"] + assert.False(t, hasHeaders, "requestHeaders not in Only list -> dropped") } func TestLog_Publish_MasksHeaders(t *testing.T) { @@ -176,32 +184,67 @@ func TestLog_Publish_MasksHeaders(t *testing.T) { l.Publish(event) - _, props := decodeLine(t, read()) - - var reqH map[string]interface{} - require.NoError(t, json.Unmarshal([]byte(props["requestHeaders"].(string)), &reqH)) + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) assert.Equal(t, "****", reqH["Authorization"]) // masked assert.Equal(t, "bar", reqH["x-foo"]) // untouched - var resH map[string]interface{} - require.NoError(t, json.Unmarshal([]byte(props["responseHeaders"].(string)), &resH)) + resH := headerMap(t, decoded["responseHeaders"]) assert.Equal(t, "****", resH["authorization"]) // case-insensitive match } -// Per-API excludeHeaders drops the header entirely (vs global masking which redacts). +// Per-API maskedHeaders are merged with the global config mask; either source alone redacts. +func TestLog_Publish_PerAPIMaskedHeadersMergedWithGlobal(t *testing.T) { + // Global config masks "authorization"; per-API directive adds "x-secret". + l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true}, + MaskedHeaders: []string{"x-secret"}, + } + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Secret":"top","x-foo":"bar"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "****", reqH["Authorization"], "global masked header still redacted") + assert.Equal(t, "****", reqH["X-Secret"], "per-API masked header redacted") + assert.Equal(t, "bar", reqH["x-foo"], "unmasked header unchanged") +} + +// Per-API maskedHeaders work even when no global headers are configured. +func TestLog_Publish_PerAPIMaskedHeadersNoGlobal(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = &dto.TrafficLogDirective{ + Request: &dto.TrafficLogFlow{Headers: true}, + MaskedHeaders: []string{"X-Token"}, + } + event.Properties["requestHeaders"] = `{"X-Token":"secret","x-foo":"bar"}` + + l.Publish(event) + + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) + assert.Equal(t, "****", reqH["X-Token"]) + assert.Equal(t, "bar", reqH["x-foo"]) +} + +// fields.exclude with a dotted path drops a specific header entirely (vs global masking which redacts). func TestLog_Publish_ExcludeHeadersDrops(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) event := createBaseEvent() event.TrafficLog = &dto.TrafficLogDirective{ - Request: &dto.TrafficLogFlow{Headers: true, ExcludeHeaders: []string{"X-Secret"}}, + Request: &dto.TrafficLogFlow{Headers: true}, + Fields: &dto.TrafficLogFields{Exclude: []string{"requestHeaders.X-Secret"}}, } event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Secret":"top","x-foo":"bar"}` l.Publish(event) - _, props := decodeLine(t, read()) - var reqH map[string]interface{} - require.NoError(t, json.Unmarshal([]byte(props["requestHeaders"].(string)), &reqH)) + decoded := decodeLine(t, read()) + reqH := headerMap(t, decoded["requestHeaders"]) _, hasSecret := reqH["X-Secret"] assert.False(t, hasSecret, "excluded header must be dropped") @@ -224,13 +267,13 @@ func TestLog_Publish_DisabledFieldsOmitted(t *testing.T) { l.Publish(event) - _, props := decodeLine(t, read()) - _, hasReqHeaders := props["requestHeaders"] + decoded := decodeLine(t, read()) + _, hasReqHeaders := decoded["requestHeaders"] assert.False(t, hasReqHeaders, "request headers disabled -> omitted") - assert.Equal(t, "req-body", props["request_payload"], "request payload enabled -> kept") + assert.Equal(t, "req-body", decoded["requestBody"], "request payload enabled -> kept") - _, hasRespHeaders := props["responseHeaders"] - _, hasRespPayload := props["response_payload"] + _, hasRespHeaders := decoded["responseHeaders"] + _, hasRespPayload := decoded["responseBody"] assert.False(t, hasRespHeaders, "nil response flow -> response headers omitted") assert.False(t, hasRespPayload, "nil response flow -> response payload omitted") } @@ -254,39 +297,33 @@ func TestLog_Publish_NilEvent(t *testing.T) { assert.Empty(t, read()) } -// Field selection (include): only named top-level keys and properties.* survive, -// and it is authoritative over presence (request.headers boolean is ignored, but -// excludeHeaders + masking still apply). +// Field selection (include): only named top-level keys survive; fields.only is +// authoritative over presence (request.headers boolean is ignored, masking still applies). func TestLog_Publish_FieldsInclude(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) event := createBaseEvent() - event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Drop":"d","X-Keep":"k"}` + event.Properties["requestHeaders"] = `{"Authorization":"Bearer s","X-Keep":"k"}` event.Properties["responseHeaders"] = `{"x-bar":"baz"}` event.TrafficLog = &dto.TrafficLogDirective{ - Request: &dto.TrafficLogFlow{Headers: false, ExcludeHeaders: []string{"X-Drop"}}, // boolean ignored - Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"latencies", "properties.requestHeaders"}}, + Request: &dto.TrafficLogFlow{Headers: false}, // boolean ignored when fields set + Fields: &dto.TrafficLogFields{Only: []string{"latencies", "requestHeaders"}}, } l.Publish(event) - decoded, props := decodeLine(t, read()) + decoded := decodeLine(t, read()) _, hasAPI := decoded["api"] _, hasOp := decoded["operation"] assert.False(t, hasAPI, "api not listed -> dropped") assert.False(t, hasOp, "operation not listed -> dropped") assert.Contains(t, decoded, "latencies", "latencies listed -> kept") - require.NotNil(t, props, "properties kept (a properties.* path was listed)") - _, hasResp := props["responseHeaders"] + _, hasResp := decoded["responseHeaders"] assert.False(t, hasResp, "responseHeaders not listed -> dropped") - reqRaw, ok := props["requestHeaders"].(string) - require.True(t, ok, "requestHeaders present (fields authoritative, boolean ignored)") - var reqH map[string]interface{} - require.NoError(t, json.Unmarshal([]byte(reqRaw), &reqH)) + require.NotNil(t, decoded["requestHeaders"], "requestHeaders present (fields authoritative, boolean ignored)") + reqH := headerMap(t, decoded["requestHeaders"]) assert.Equal(t, "****", reqH["Authorization"], "masking still applies") - _, hasDrop := reqH["X-Drop"] - assert.False(t, hasDrop, "excludeHeaders still applies") assert.Equal(t, "k", reqH["X-Keep"]) } @@ -297,40 +334,44 @@ func TestLog_Publish_FieldsExclude(t *testing.T) { event.Properties["requestHeaders"] = `{"x-foo":"bar"}` event.Properties["request_payload"] = "secret-body" event.TrafficLog = &dto.TrafficLogDirective{ - Fields: &dto.TrafficLogFields{Mode: "exclude", Names: []string{"operation", "properties.request_payload"}}, + Fields: &dto.TrafficLogFields{Exclude: []string{"operation", "requestBody"}}, } l.Publish(event) - decoded, props := decodeLine(t, read()) + decoded := decodeLine(t, read()) _, hasOp := decoded["operation"] assert.False(t, hasOp, "operation excluded") assert.Contains(t, decoded, "api", "api kept") assert.Contains(t, decoded, "latencies", "latencies kept") - require.NotNil(t, props) - _, hasPayload := props["request_payload"] - assert.False(t, hasPayload, "request_payload excluded") - assert.Contains(t, props, "requestHeaders", "requestHeaders kept (not excluded)") + _, hasPayload := decoded["requestBody"] + assert.False(t, hasPayload, "requestBody excluded") + assert.Contains(t, decoded, "requestHeaders", "requestHeaders kept (not excluded)") } -// Naming the whole "properties" key keeps all of its subkeys. -func TestLog_Publish_FieldsIncludeWholeProperties(t *testing.T) { +// requestBody and labels are top-level keys like any other and can be selected +// explicitly via fields.only. +func TestLog_Publish_FieldsIncludeRequestBodyAndLabels(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) event := createBaseEvent() event.Properties["requestHeaders"] = `{"x-foo":"bar"}` - event.Properties["responseHeaders"] = `{"x-bar":"baz"}` + event.Properties["request_payload"] = "body-data" event.TrafficLog = &dto.TrafficLogDirective{ - Fields: &dto.TrafficLogFields{Mode: "include", Names: []string{"properties"}}, + Labels: map[string]interface{}{"env": "prod"}, + Fields: &dto.TrafficLogFields{Only: []string{"requestBody", "labels"}}, } l.Publish(event) - decoded, props := decodeLine(t, read()) + decoded := decodeLine(t, read()) _, hasAPI := decoded["api"] assert.False(t, hasAPI, "api not listed -> dropped") - require.NotNil(t, props) - assert.Contains(t, props, "requestHeaders") - assert.Contains(t, props, "responseHeaders") + _, hasHeaders := decoded["requestHeaders"] + assert.False(t, hasHeaders, "requestHeaders not in Only list -> dropped") + assert.Equal(t, "body-data", decoded["requestBody"]) + labels, ok := decoded["labels"].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "prod", labels["env"]) } // Output-side payload truncation (0 = no limit). @@ -342,9 +383,9 @@ func TestLog_Publish_TruncatesPayload(t *testing.T) { event.Properties["response_payload"] = "goodbye world" l.Publish(event) - _, props := decodeLine(t, read()) - assert.Equal(t, "hello", props["request_payload"]) - assert.Equal(t, "goodb", props["response_payload"]) + decoded := decodeLine(t, read()) + assert.Equal(t, "hello", decoded["requestBody"]) + assert.Equal(t, "goodb", decoded["responseBody"]) } func TestLog_Publish_NoTruncationWhenZero(t *testing.T) { @@ -354,11 +395,11 @@ func TestLog_Publish_NoTruncationWhenZero(t *testing.T) { event.Properties["request_payload"] = "hello world" l.Publish(event) - _, props := decodeLine(t, read()) - assert.Equal(t, "hello world", props["request_payload"]) + decoded := decodeLine(t, read()) + assert.Equal(t, "hello world", decoded["requestBody"]) } -func TestLog_Publish_InvalidHeaderJSONLeftAsIs(t *testing.T) { +func TestLog_Publish_UnparseableHeadersDropped(t *testing.T) { l, read := newLogToFile(t, &config.TrafficLoggingConfig{MaskedHeaders: []string{"authorization"}}) event := createBaseEvent() event.TrafficLog = bothFlows() @@ -366,6 +407,62 @@ func TestLog_Publish_InvalidHeaderJSONLeftAsIs(t *testing.T) { l.Publish(event) - _, props := decodeLine(t, read()) - assert.Equal(t, "not-json", props["requestHeaders"]) + decoded := decodeLine(t, read()) + _, hasHeaders := decoded["requestHeaders"] + assert.False(t, hasHeaders, "unparseable header value must be silently dropped") +} + +// Application is omitted entirely for unauthenticated requests (all fields empty). +func TestLog_Publish_UnauthenticatedRequestOmitsApplication(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.Application = &dto.Application{} // all fields are "" + event.TrafficLog = bothFlows() + + l.Publish(event) + + decoded := decodeLine(t, read()) + _, hasApp := decoded["application"] + assert.False(t, hasApp, "application with all-empty fields must be absent") +} + +// TrafficLogAPI uses clean field names (id/name/kind) not the Moesif apiId/apiName/apiType. +func TestLog_Publish_APIFieldNames(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + + l.Publish(event) + + decoded := decodeLine(t, read()) + api, ok := decoded["api"].(map[string]interface{}) + require.True(t, ok, "api object must be present") + assert.Equal(t, "api-123", api["id"]) + assert.Equal(t, "test-api", api["name"]) + assert.Equal(t, "v1.0", api["version"]) + assert.Equal(t, "Rest", api["kind"]) + assert.Equal(t, "/test", api["context"]) + assert.Equal(t, "project-123", api["projectId"]) + // Moesif-specific keys must not bleed into traffic logs. + for _, moesifKey := range []string{"apiId", "apiName", "apiCreator", "apiCreatorTenantDomain", "organizationId"} { + _, present := api[moesifKey] + assert.False(t, present, "Moesif field %q must not appear in traffic log", moesifKey) + } +} + +// Top-level fields — status, correlationId, client — are always present when set. +func TestLog_Publish_TopLevelFieldsPresent(t *testing.T) { + l, read := newLogToFile(t, &config.TrafficLoggingConfig{}) + event := createBaseEvent() + event.TrafficLog = bothFlows() + + l.Publish(event) + + decoded := decodeLine(t, read()) + assert.Equal(t, float64(200), decoded["status"], "proxy response code must appear as status") + assert.Equal(t, "corr-123", decoded["correlationId"]) + client, ok := decoded["client"].(map[string]interface{}) + require.True(t, ok, "client object must be present") + assert.Equal(t, "192.168.1.1", client["ip"]) + assert.Equal(t, "test-agent", client["userAgent"]) } diff --git a/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go new file mode 100644 index 0000000000..da6b251a70 --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/analytics/publishers/traffic_log_event.go @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package publishers + +import ( + "strings" + + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/analytics/dto" +) + +// trafficLogTimestampFormat is RFC 3339 with millisecond precision. +const trafficLogTimestampFormat = "2006-01-02T15:04:05.000Z07:00" + +// TrafficLogEvent is the JSON shape written to stdout by the Log publisher. +// It is intentionally separate from dto.Event (shaped for Moesif) so its field +// names, schema, and presence rules can evolve independently. All string fields +// carry omitempty so absent or unknown values produce no key rather than "". +type TrafficLogEvent struct { + Timestamp string `json:"timestamp,omitempty"` + CorrelationID string `json:"correlationId,omitempty"` + Status int `json:"status,omitempty"` + API *TrafficLogAPI `json:"api,omitempty"` + Operation *TrafficLogOperation `json:"operation,omitempty"` + Target *TrafficLogTarget `json:"target,omitempty"` + Application *TrafficLogApplication `json:"application,omitempty"` + Client *TrafficLogClient `json:"client,omitempty"` + Latencies *dto.Latencies `json:"latencies,omitempty"` + RequestHeaders map[string]string `json:"requestHeaders,omitempty"` + ResponseHeaders map[string]string `json:"responseHeaders,omitempty"` + RequestBody string `json:"requestBody,omitempty"` + ResponseBody string `json:"responseBody,omitempty"` + Labels map[string]interface{} `json:"labels,omitempty"` +} + +// TrafficLogAPI identifies the API that processed the request. +type TrafficLogAPI struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Context string `json:"context,omitempty"` + Kind string `json:"kind,omitempty"` + ProjectID string `json:"projectId,omitempty"` +} + +// TrafficLogOperation describes the matched operation within the API. +type TrafficLogOperation struct { + Method string `json:"method,omitempty"` + Path string `json:"path,omitempty"` +} + +// TrafficLogTarget holds upstream response information. +type TrafficLogTarget struct { + StatusCode int `json:"statusCode,omitempty"` + Destination string `json:"destination,omitempty"` +} + +// TrafficLogApplication is present only for authenticated requests. +type TrafficLogApplication struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + KeyType string `json:"keyType,omitempty"` +} + +// TrafficLogClient holds downstream caller information. +type TrafficLogClient struct { + IP string `json:"ip,omitempty"` + UserAgent string `json:"userAgent,omitempty"` +} + +// toTrafficLogEvent translates a dto.Event and its traffic-log directive into the +// traffic-log-specific output shape, applying per-flow header filtering, global +// header masking, and payload truncation. +func (l *Log) toTrafficLogEvent(event *dto.Event, dir *dto.TrafficLogDirective) *TrafficLogEvent { + tl := &TrafficLogEvent{ + Status: event.ProxyResponseCode, + Latencies: event.Latencies, + } + + if !event.RequestTimestamp.IsZero() { + tl.Timestamp = event.RequestTimestamp.UTC().Format(trafficLogTimestampFormat) + } + + if event.MetaInfo != nil { + tl.CorrelationID = event.MetaInfo.CorrelationID + } + + if event.API != nil { + tl.API = &TrafficLogAPI{ + ID: event.API.APIID, + Name: event.API.APIName, + Version: event.API.APIVersion, + Context: event.API.APIContext, + Kind: event.API.APIType, + ProjectID: event.API.ProjectID, + } + } + + if event.Operation != nil { + tl.Operation = &TrafficLogOperation{ + Method: event.Operation.APIMethod, + Path: event.Operation.APIResourceTemplate, + } + } + + if event.Target != nil { + tl.Target = &TrafficLogTarget{ + StatusCode: event.Target.TargetResponseCode, + Destination: event.Target.Destination, + } + } + + // Application is only meaningful for authenticated requests. + if a := event.Application; a != nil && (a.ApplicationID != "" || a.ApplicationName != "") { + tl.Application = &TrafficLogApplication{ + ID: a.ApplicationID, + Name: a.ApplicationName, + Owner: a.ApplicationOwner, + KeyType: a.KeyType, + } + } + + if event.UserIP != "" || event.UserAgentHeader != "" { + tl.Client = &TrafficLogClient{ + IP: event.UserIP, + UserAgent: event.UserAgentHeader, + } + } + + // When a fields selection is configured it is authoritative over presence: + // the per-flow Headers/Payload booleans are ignored and applyFieldsProjection + // in Publish decides what survives. Global masking always applies; per-API + // maskedHeaders are merged in here and passed down. + hasFieldsSelection := dir.Fields != nil && (len(dir.Fields.Only) > 0 || len(dir.Fields.Exclude) > 0) + + // Effective header mask: global config merged with any per-API additions. + mask := l.maskedHeaders + if len(dir.MaskedHeaders) > 0 { + merged := make(map[string]bool, len(l.maskedHeaders)+len(dir.MaskedHeaders)) + for k, v := range l.maskedHeaders { + merged[k] = v + } + for _, h := range dir.MaskedHeaders { + merged[strings.ToLower(h)] = true + } + mask = merged + } + + // Request flow + if raw, ok := event.Properties[dto.PropKeyRequestHeaders].(string); ok { + if hasFieldsSelection || (dir.Request != nil && dir.Request.Headers) { + if headers := parseHeadersFromString(raw); headers != nil { + tl.RequestHeaders = l.maskHeaders(headers, mask) + } + } + } + if p, ok := event.Properties[dto.PropKeyRequestPayload].(string); ok && p != "" { + if hasFieldsSelection || (dir.Request != nil && dir.Request.Payload) { + tl.RequestBody = l.truncatePayload(p) + } + } + + // Response flow + if raw, ok := event.Properties[dto.PropKeyResponseHeaders].(string); ok { + if hasFieldsSelection || (dir.Response != nil && dir.Response.Headers) { + if headers := parseHeadersFromString(raw); headers != nil { + tl.ResponseHeaders = l.maskHeaders(headers, mask) + } + } + } + if p, ok := event.Properties[dto.PropKeyResponsePayload].(string); ok && p != "" { + if hasFieldsSelection || (dir.Response != nil && dir.Response.Payload) { + tl.ResponseBody = l.truncatePayload(p) + } + } + + if len(dir.Labels) > 0 { + tl.Labels = dir.Labels + } + + return tl +} From 530b7e68ca775ae75579d4cdadbb61861c97c171 Mon Sep 17 00:00:00 2001 From: Dineth Date: Tue, 7 Jul 2026 00:54:03 +0530 Subject: [PATCH 5/5] Disable request and response header capture by default; update related tests and migration logic --- gateway/configs/config-template.toml | 9 ++++---- gateway/configs/config.toml | 6 ++++-- .../gateway-controller/pkg/config/config.go | 17 +++++++++++---- .../pkg/config/config_test.go | 17 +++++++++++++++ .../pkg/utils/system_policies_test.go | 21 ++++++++++++++++++- .../policy-engine/internal/config/config.go | 13 ++++++++++-- .../internal/config/config_test.go | 17 +++++++++++++++ 7 files changed, 87 insertions(+), 13 deletions(-) diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index ae83145e3a..936475b881 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -297,10 +297,11 @@ send_request_body = false send_response_body = false # Capture ALL request/response headers into the collected event (request_headers/ # response_headers). Applies to every API via the collector system policy, so no -# per-API header policy is needed. Sensitive values are redacted on output by each -# consumer (e.g. traffic_logging.masked_headers). On by default. -send_request_headers = true -send_response_headers = true +# per-API header policy is needed. Sensitive values are redacted on output by +# consumers that support it (e.g. traffic_logging.masked_headers) — the Moesif +# publisher does not mask headers. Off by default; enable deliberately per consumer. +send_request_headers = false +send_response_headers = false # ALS transport tuning (advanced; defaults are sensible). One section read by BOTH # ends of the Envoy → policy-engine access-log stream: diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index 83107adfa6..fbabb75d6d 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -8,8 +8,10 @@ send_request_body = false send_response_body = false # Capture ALL request/response headers (redacted on output by per-consumer masking). -send_request_headers = true -send_response_headers = true +# Off by default; the Moesif publisher does not mask headers, so only enable this +# for consumers you know redact sensitive values (e.g. traffic_logging.masked_headers). +send_request_headers = false +send_response_headers = false # Analytics consumer (e.g. Moesif). Enabling it activates the collector automatically. [analytics] diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 2530e073d0..bc4bb7cfd0 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -976,8 +976,8 @@ func defaultConfig() *Config { Collector: CollectorConfig{ SendRequestBody: false, SendResponseBody: false, - SendRequestHeaders: true, - SendResponseHeaders: true, + SendRequestHeaders: false, + SendResponseHeaders: false, GRPCEventServerCfg: defaultGRPCEventServerConfig(), }, TracingConfig: TracingConfig{ @@ -1781,9 +1781,11 @@ func (c *Config) IsCollectorEnabled() bool { func (c *Config) migrateDeprecatedAnalyticsTransport() { def := defaultGRPCEventServerConfig() if c.Analytics.GRPCEventServerCfg != def { - slog.Warn("analytics.grpc_event_server is deprecated; use collector.als instead") if c.Collector.GRPCEventServerCfg == def { + slog.Warn("analytics.grpc_event_server is deprecated; migrating it to collector.als") c.Collector.GRPCEventServerCfg = c.Analytics.GRPCEventServerCfg + } else { + slog.Warn("analytics.grpc_event_server is deprecated and collector.als is already configured; ignoring the analytics.grpc_event_server override") } } } @@ -1791,8 +1793,15 @@ func (c *Config) migrateDeprecatedAnalyticsTransport() { // migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / // analytics.send_request_body / analytics.send_response_body onto the collector's // body-capture flags (when the collector flag is not already set), so existing -// configs keep working after capture settings moved under [collector]. +// configs keep working after capture settings moved under [collector]. These are +// analytics's own deprecated flags, so they are only honored while analytics is +// enabled — otherwise a stale value left over from a disabled analytics setup +// could silently turn on body capture for an unrelated consumer (e.g. +// traffic_logging) enabled later. func (c *Config) migrateDeprecatedAnalyticsCapture() { + if !c.Analytics.Enabled { + return + } // Directional aliases take precedence over allow_payloads. if c.Analytics.SendRequestBody && !c.Collector.SendRequestBody { slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index 4b237ac386..7c4b117be9 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -1413,6 +1413,23 @@ func TestConfig_ValidateAnalyticsPayloadMigration(t *testing.T) { } } +// TestConfig_ValidateAnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled guards +// against a stale analytics.allow_payloads left over from a disabled analytics +// setup silently turning on body capture for an unrelated consumer (traffic +// logging) enabled later. The deprecated capture aliases belong to analytics, so +// they must only be honored while analytics itself is enabled. +func TestConfig_ValidateAnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = true // an unrelated consumer activates the collector + cfg.Analytics.AllowPayloads = true + + err := cfg.Validate() + require.NoError(t, err) + assert.False(t, cfg.Collector.SendRequestBody) + assert.False(t, cfg.Collector.SendResponseBody) +} + func TestConfig_ValidateAuthConfig(t *testing.T) { tests := []struct { name string diff --git a/gateway/gateway-controller/pkg/utils/system_policies_test.go b/gateway/gateway-controller/pkg/utils/system_policies_test.go index 5f38555594..8ed2e284cb 100644 --- a/gateway/gateway-controller/pkg/utils/system_policies_test.go +++ b/gateway/gateway-controller/pkg/utils/system_policies_test.go @@ -155,6 +155,25 @@ func TestInjectSystemPolicies_CollectorEnabled(t *testing.T) { assert.Equal(t, "existing", result[1].Name) } +func TestInjectSystemPolicies_TrafficLoggingOnlyEnablesCollector(t *testing.T) { + // Traffic logging alone (no analytics) -> collector implicitly on -> system + // policy injected. Guards against a regression that gates injection on + // cfg.Analytics.Enabled alone instead of cfg.IsCollectorEnabled(). + cfg := &config.Config{ + TrafficLogging: config.TrafficLoggingConfig{Enabled: true}, + } + policies := []policyenginev1.PolicyInstance{ + {Name: "existing", Version: "v1.0.0"}, + } + + result := InjectSystemPolicies(policies, cfg, nil) + assert.Len(t, result, 2) + assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_NAME, result[0].Name) + assert.Equal(t, constants.ANALYTICS_SYSTEM_POLICY_VERSION, result[0].Version) + assert.True(t, result[0].Enabled) + assert.Equal(t, "existing", result[1].Name) +} + func TestInjectSystemPolicies_BodyFlagsPropagated(t *testing.T) { cfg := &config.Config{ Analytics: config.AnalyticsConfig{Enabled: true}, @@ -204,7 +223,7 @@ func TestInjectSystemPolicies_HeaderFlagsPropagated(t *testing.T) { func TestInjectSystemPolicies_HeaderFlagsDefaultFalse(t *testing.T) { // Zero-value collector (headers unset) with a consumer on: propagation passes the - // struct's false through. (Production defaults these true; see defaultConfig.) + // struct's false through, matching the production default (see defaultConfig). cfg := &config.Config{ Analytics: config.AnalyticsConfig{Enabled: true}, } diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 4e26d0fd3e..9580c733a2 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -627,9 +627,11 @@ func (c *Config) IsCollectorEnabled() bool { func (c *Config) migrateDeprecatedAnalyticsTransport() { def := defaultAccessLogsServiceConfig() if c.Analytics.AccessLogsServiceCfg != def { - slog.Warn("analytics.access_logs_service is deprecated; use collector.als instead") if c.Collector.AccessLogsServiceCfg == def { + slog.Warn("analytics.access_logs_service is deprecated; migrating it to collector.als") c.Collector.AccessLogsServiceCfg = c.Analytics.AccessLogsServiceCfg + } else { + slog.Warn("analytics.access_logs_service is deprecated and collector.als is already configured; ignoring the analytics.access_logs_service override") } } } @@ -664,8 +666,15 @@ func validateAccessLogsServiceConfig(als AccessLogsServiceConfig) error { // migrateDeprecatedAnalyticsCapture maps the deprecated analytics.allow_payloads / // analytics.send_request_body / analytics.send_response_body onto the collector's // body-capture flags (when the collector flag is not already set), so existing -// configs keep working after capture settings moved under [collector]. +// configs keep working after capture settings moved under [collector]. These are +// analytics's own deprecated flags, so they are only honored while analytics is +// enabled — otherwise a stale value left over from a disabled analytics setup +// could silently turn on body capture for an unrelated consumer (e.g. +// traffic_logging) enabled later. func (c *Config) migrateDeprecatedAnalyticsCapture() { + if !c.Analytics.Enabled { + return + } // Directional aliases take precedence over allow_payloads. if c.Analytics.SendRequestBody && !c.Collector.SendRequestBody { slog.Warn("analytics.send_request_body is deprecated; use collector.send_request_body instead") diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go index b9371d6990..914b6853b3 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go @@ -1174,6 +1174,23 @@ func TestValidate_AnalyticsPayloadMigration(t *testing.T) { } } +// TestValidate_AnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled guards +// against a stale analytics.allow_payloads left over from a disabled analytics +// setup silently turning on body capture for an unrelated consumer (traffic +// logging) enabled later. The deprecated capture aliases belong to analytics, so +// they must only be honored while analytics itself is enabled. +func TestValidate_AnalyticsPayloadMigration_SkippedWhenAnalyticsDisabled(t *testing.T) { + cfg := validConfig() + cfg.Analytics.Enabled = false + cfg.TrafficLogging.Enabled = true // an unrelated consumer activates the collector + cfg.Analytics.AllowPayloads = true + + err := cfg.Validate() + require.NoError(t, err) + assert.False(t, cfg.Collector.SendRequestBody) + assert.False(t, cfg.Collector.SendResponseBody) +} + // TestValidate_CollectorPrerequisite verifies that enabling a consumer (analytics, // traffic logging) without the collector auto-enables the collector (a // backward-compat soft prerequisite) rather than failing.