Add Access Logging support

gateway/configs/config-template.toml

@@ -281,18 +281,62 @@ port = 9010
 host = "localhost"
 
 # =============================================================================
-# ANALYTICS CONFIGURATION
+# COLLECTOR CONFIGURATION
+# =============================================================================
+# The collector is the shared data-capture pipeline: it injects the analytics
+# system policy and ships request/response headers and bodies to the policy-engine
+# over ALS. It has NO on/off switch of its own — it is implicit, turning on
+# automatically whenever a consumer below is enabled ([analytics] or
+# [traffic_logging]) and staying off otherwise. This section only tunes what the
+# collector captures.
+[collector]
+# Capture request/response payloads (bodies) into the collected event. Bodies are
+# captured in full; per-consumer size caps are applied on output (see
+# traffic_logging.max_payload_size).
+send_request_body = false
+send_response_body = false
+# Capture ALL request/response headers into the collected event (request_headers/
+# response_headers). Applies to every API via the collector system policy, so no
+# per-API header policy is needed. Sensitive values are redacted on output by
+# consumers that support it (e.g. traffic_logging.masked_headers) — the Moesif
+# publisher does not mask headers. Off by default; enable deliberately per consumer.
+send_request_headers = false
+send_response_headers = false
+
+# ALS transport tuning (advanced; defaults are sensible). One section read by BOTH
+# ends of the Envoy → policy-engine access-log stream:
+#   • gateway-controller — configures Envoy's gRPC access-log sink (the sender)
+#   • policy-engine       — runs the receiving gRPC server
+# Shared keys (server_port, TLS, message/header limits) must match on both ends and
+# live here once. Envoy-sender-only keys (buffer_*, grpc_request_timeout) are read
+# only by the controller and ignored by the policy-engine.
+[collector.als]
+mode = "uds"                       # "uds" (default) or "tcp"
+server_port = 18090                # engine listens here; Envoy dials it (tcp mode)
+buffer_flush_interval = 1000000000 # Envoy sender only (nanoseconds)
+buffer_size_bytes = 16384          # Envoy sender only
+grpc_request_timeout = 20000000000 # Envoy sender only (nanoseconds)
+shutdown_timeout = "600s"
+public_key_path = ""               # TLS for the ALS connection (both ends)
+private_key_path = ""
+als_plain_text = true              # plaintext gRPC (skip TLS)
+max_message_size = 1000000000
+max_header_limit = 8192
+
+# =============================================================================
+# ANALYTICS CONFIGURATION (consumer — enabling it activates the collector)
 # =============================================================================
 [analytics]
 enabled = false
-# Deprecated: allow_payloads is preserved for backward compatibility. When true
-# and both send_request_body and send_response_body are false, both directions
-# are enabled (bool fields cannot distinguish "unset" from explicitly false).
-# To disable payloads entirely, set allow_payloads = false or remove it; do not
-# rely on send_*_body = false while allow_payloads remains true.
+# Deprecated: allow_payloads / send_request_body / send_response_body are preserved
+# for backward compatibility and are mapped onto the [collector] body-capture flags
+# during validation (with a warning). Prefer setting [collector] directly.
 allow_payloads = false
-send_request_body = false
-send_response_body = false
+# send_request_body = false
+# send_response_body = false
+# Publishers that receive analytics events. Supported: "moesif" (sends to the
+# Moesif SaaS). For stdout JSON logging use [traffic_logging] instead of a "log"
+# publisher here.
 enabled_publishers = ["moesif"]
 
 [analytics.publishers.moesif]
@@ -303,17 +347,32 @@ event_queue_size = 10000
 batch_size = 50
 timer_wakeup_seconds = 3
 
-[analytics.grpc_event_server]
-buffer_flush_interval = 1000000000
-buffer_size_bytes = 16384
-grpc_request_timeout = 20000000000
-server_port = 18090
-shutdown_timeout = "600s"
-public_key_path = ""
-private_key_path = ""
-als_plain_text = true
-max_message_size = 1000000000
-max_header_limit = 8192
+# =============================================================================
+# TRAFFIC LOGGING (consumer — enabling it activates the collector)
+# =============================================================================
+# Writes collected events (rich API/application/AI/latency metadata, plus
+# request/response headers and — when the collector's send_*_body is enabled —
+# payloads) to stdout as a JSON line. Useful for log-scraping pipelines such as
+# Fluent Bit/Loki/ELK; no external SaaS needed.
+#
+# Per-API (opt-in): a line is emitted only for APIs that attach the `log-message`
+# policy with `enableTrafficLogging: true` — enabling this section alone does NOT
+# log every API. So an API is logged only when both hold: [traffic_logging].enabled
+# = true, and the `log-message` policy (enableTrafficLogging: true) is attached to
+# that API. The policy's request/response parameters (per-flow payload/headers
+# toggles and excludeHeaders) shape each API's line; they filter/mask what the
+# collector captured (they cannot enable capture the collector was not configured
+# to perform).
+[traffic_logging]
+enabled = false
+# Header names (case-insensitive) whose values are redacted as "****" in the
+# logged requestHeaders/responseHeaders. Applies globally; the `log-message`
+# policy's per-API excludeHeaders drop additional headers entirely.
+masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"]
+# Max bytes of request/response payload written to each log line (0 = no limit).
+# Applied output-side, so the collector still captures full bodies and other
+# consumers (e.g. Moesif) are unaffected.
+max_payload_size = 0
 
 # =============================================================================
 # POLICY CONFIGURATIONS

gateway/configs/config.toml

@@ -1,16 +1,34 @@
 
-[analytics]
-enabled = true
-# Deprecated: allow_payloads is preserved for backward compatibility. When true
-# and when send_request_body/send_response_body are not explicitly set, both
-# request and response bodies will be included in analytics events.
-# allow_payloads = true
+# The collector is the shared data-capture pipeline. It has no on/off switch of its
+# own: it turns on automatically whenever a consumer (analytics or traffic_logging)
+# is enabled, and stays off otherwise. This section only tunes what is captured.
+[collector]
+# Capture request/response payloads (bodies) into the collected event (full bodies;
+# per-consumer size caps are applied on output).
 send_request_body = false
 send_response_body = false
+# Capture ALL request/response headers (redacted on output by per-consumer masking).
+# Off by default; the Moesif publisher does not mask headers, so only enable this
+# for consumers you know redact sensitive values (e.g. traffic_logging.masked_headers).
+send_request_headers = false
+send_response_headers = false
+
+# Analytics consumer (e.g. Moesif). Enabling it activates the collector automatically.
+[analytics]
+enabled = false
 enabled_publishers = ["moesif"]
 
-[analytics.publishers.moesif]
-application_id = "<MOESIF_APPLICATION_ID>"
+# [analytics.publishers.moesif]
+# application_id = "<MOESIF_APPLICATION_ID>"
+
+# Traffic-logging consumer: writes each collected event to stdout as a JSON line
+# (no external service needed). Enabling it activates the collector automatically.
+[traffic_logging]
+enabled = true
+# Header names (case-insensitive) whose values are redacted as "****".
+masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"]
+# Max bytes of request/response payload written to each log line (0 = no limit).
+max_payload_size = 2048
 
 [router]
 gateway_host = "*"