From b4163466745ab46d2d29f6a89a5e487f10f19246 Mon Sep 17 00:00:00 2001
From: Kyle Hounslow <kylhouns@amazon.com>
Date: Fri, 8 May 2026 15:14:07 -0700
Subject: [PATCH 1/3] poc(splunk): add Splunk OTel Distribution side-by-side
 overlay
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Local POC overlay that inserts Splunk's OTel Collector distribution as a tee
in front of the base otel-collector. Demo apps send OTLP to splunk-otel-collector
(redirected via OTEL_COLLECTOR_HOST override), which fans out to:
  - Splunk Observability Cloud (APM via otlphttp, IM via signalfx, Log Observer via splunk_hec)
  - base otel-collector -> Data Prepper -> OpenSearch (unchanged existing path)

Purpose: ground-truth what Splunk Observability Cloud accepts from otel-demo so we
can make informed decisions about compat overlay receiver scope (signalfxreceiver,
zipkinreceiver) and the 'drop-in for Splunk Observability Cloud' framing.

Files:
  - docker-compose.splunk-demo.yml: compose overlay adding splunk-otel-collector service
  - docker-compose/splunk-otel-collector/config.yaml: container-friendly Splunk collector config
    (trimmed from Splunk's agent_config.yaml — host-level receivers and smartagent extensions
    removed; OTLP-only receivers; tee exporter to base otel-collector:4317)
  - docker-compose/splunk-otel-collector/README.md: how to run and verify
  - .env.splunk-poc.example: credential + demo-redirect template
  - .gitignore: ignore the real .env.splunk-poc

Not a production configuration. Local exploration only.

Signed-off-by: Kyle Hounslow <kylhouns@amazon.com>
---
 .env.splunk-poc.example                       |  26 +++++
 .gitignore                                    |   4 +
 docker-compose.splunk-demo.yml                |  60 ++++++++++
 .../splunk-otel-collector/README.md           |  81 +++++++++++++
 .../splunk-otel-collector/config.yaml         | 110 ++++++++++++++++++
 5 files changed, 281 insertions(+)
 create mode 100644 .env.splunk-poc.example
 create mode 100644 docker-compose.splunk-demo.yml
 create mode 100644 docker-compose/splunk-otel-collector/README.md
 create mode 100644 docker-compose/splunk-otel-collector/config.yaml

diff --git a/.env.splunk-poc.example b/.env.splunk-poc.example
new file mode 100644
index 00000000..0c6f17bd
--- /dev/null
+++ b/.env.splunk-poc.example
@@ -0,0 +1,26 @@
+# Splunk Distribution POC — credentials + demo-app redirect
+#
+# Copy this file to .env.splunk-poc and fill in your Splunk Observability Cloud values.
+# The actual .env.splunk-poc is gitignored and must never be committed.
+#
+# Where to find these in Splunk Observability Cloud:
+#   Access Token:  Organization → Access Tokens → any token with ingest scope
+#   Realm:         Visible in your Splunk Observability Cloud URL
+#                  (e.g. us0, us1, eu0, au0)
+#   HEC Token:     Data Management → Connect Data → HTTP Event Collector
+#                  (if absent, can often fall back to the Access Token)
+
+# --- Splunk credentials ---
+SPLUNK_ACCESS_TOKEN=your-splunk-access-token-here
+SPLUNK_REALM=us1
+SPLUNK_HEC_TOKEN=your-hec-token-here-or-reuse-access-token
+
+# --- Redirect demo apps through Splunk collector (the tee) ---
+# Demo apps read OTEL_COLLECTOR_HOST to build OTEL_EXPORTER_OTLP_ENDPOINT.
+# Overriding it to the Splunk collector's service name routes all telemetry through
+# the Splunk distribution first; Splunk then fans out to Splunk Observability Cloud
+# and forwards a copy via OTLP back to the base otel-collector for OpenSearch.
+OTEL_COLLECTOR_HOST=splunk-otel-collector
+
+# --- Splunk collector image pin (optional) ---
+SPLUNK_COLLECTOR_VERSION=latest
diff --git a/.gitignore b/.gitignore
index 7630b3b4..2ab7482f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,7 @@ charts/*/charts/*.tgz
 
 # Terraform plan files
 *.plan
+
+# Splunk distribution POC
+.env.splunk-poc
+docker-compose/splunk-otel-collector/*.local.*
diff --git a/docker-compose.splunk-demo.yml b/docker-compose.splunk-demo.yml
new file mode 100644
index 00000000..160c9e0b
--- /dev/null
+++ b/docker-compose.splunk-demo.yml
@@ -0,0 +1,60 @@
+# Splunk OTel Distribution — Side-by-side POC Overlay
+#
+# Inserts Splunk's OTel Collector distribution as a tee in front of the base otel-collector.
+# Demo apps send OTLP to splunk-otel-collector (via OTEL_COLLECTOR_HOST override), which fans
+# out to:
+#   - Splunk Observability Cloud (APM + IM + Log Observer)
+#   - base otel-collector → Data Prepper → OpenSearch (unchanged existing path)
+#
+# Purpose: see the same demo data in both Splunk's UI and OpenSearch Dashboards at the same
+# time, to ground-truth what Splunk Observability Cloud accepts and how it renders.
+#
+# NOT a production configuration. Local POC only.
+#
+# Usage:
+#   1. Copy .env.splunk-poc.example to .env.splunk-poc and fill in Splunk credentials.
+#   2. docker compose \
+#        --env-file .env --env-file .env.splunk-poc \
+#        -f docker-compose.yml \
+#        -f docker-compose.otel-demo.yml \
+#        -f docker-compose.splunk-demo.yml \
+#        up -d
+
+services:
+  splunk-otel-collector:
+    image: quay.io/signalfx/splunk-otel-collector:${SPLUNK_COLLECTOR_VERSION:-latest}
+    container_name: splunk-otel-collector
+    pull_policy: always
+    command: ["--config=/etc/otelcol-config.yml"]
+    volumes:
+      - ./docker-compose/splunk-otel-collector/config.yaml:/etc/otelcol-config.yml
+    ports:
+      # Host-exposed on non-default ports to avoid collision with the base collector.
+      # Demo apps reach splunk-otel-collector over the internal Docker network on 4317/4318.
+      - "14317:4317"
+      - "14318:4318"
+      - "13133:13133"
+    networks:
+      - observability-stack-network
+    restart: unless-stopped
+    environment:
+      - SPLUNK_ACCESS_TOKEN
+      - SPLUNK_REALM
+      - SPLUNK_API_URL=https://api.${SPLUNK_REALM}.observability.splunkcloud.com
+      - SPLUNK_INGEST_URL=https://ingest.${SPLUNK_REALM}.observability.splunkcloud.com
+      - SPLUNK_HEC_TOKEN
+      - SPLUNK_HEC_URL=https://ingest.${SPLUNK_REALM}.observability.splunkcloud.com/v1/log
+      - SPLUNK_LISTEN_INTERFACE=0.0.0.0
+      - SPLUNK_MEMORY_TOTAL_MIB=512
+    deploy:
+      resources:
+        limits:
+          memory: 768M
+    depends_on:
+      otel-collector:
+        condition: service_started
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:13133"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
diff --git a/docker-compose/splunk-otel-collector/README.md b/docker-compose/splunk-otel-collector/README.md
new file mode 100644
index 00000000..d069002d
--- /dev/null
+++ b/docker-compose/splunk-otel-collector/README.md
@@ -0,0 +1,81 @@
+# Splunk OTel Distribution — POC
+
+Runs Splunk's OpenTelemetry Collector distribution as a tee in front of observability-stack's base collector, so demo telemetry lands in both Splunk Observability Cloud and OpenSearch at the same time.
+
+Local POC only. Not a production configuration, not part of the default stack.
+
+## Dataflow
+
+```
+otel-demo apps
+     │ OTLP
+     ▼
+splunk-otel-collector (Splunk distribution)
+     │
+     ├── signalfx         → Splunk Infrastructure Monitoring
+     ├── otlphttp         → Splunk APM (trace OTLP endpoint)
+     ├── splunk_hec       → Splunk Log Observer
+     └── otlp (tee)       → otel-collector → Data Prepper → OpenSearch (unchanged)
+```
+
+Demo apps are redirected via `OTEL_COLLECTOR_HOST=splunk-otel-collector` (set in `.env.splunk-poc`). No demo code changes required.
+
+## Run
+
+```bash
+# From the repo root
+cp .env.splunk-poc.example .env.splunk-poc
+# edit .env.splunk-poc with your Splunk Access Token, Realm, HEC Token
+
+docker compose \
+  --env-file .env --env-file .env.splunk-poc \
+  -f docker-compose.yml \
+  -f docker-compose.otel-demo.yml \
+  -f docker-compose.splunk-demo.yml \
+  up -d
+```
+
+## Verify
+
+Splunk Observability Cloud (URL varies by realm, e.g. `https://app.us1.signalfx.com`):
+
+- **APM → Services** — look for demo services (frontend, cart, checkout, ad, ...)
+- **Infrastructure → Metrics Explorer** — filter on `service.name`
+- **Log Observer** — filter on `sourcetype=otel`, `source=otel-demo`
+
+OpenSearch Dashboards (http://localhost:5601 by default):
+
+- Trace Analytics — same demo services should appear in the existing OSD trace UI
+- Discover on `logs-otel-v1-*` — log records from the demo
+
+Splunk collector's own health endpoint: `http://localhost:13133`.
+
+Splunk collector's exposed OTLP ports on the host (non-default to avoid collision with base collector):
+
+- gRPC: `localhost:14317`
+- HTTP: `localhost:14318`
+
+## Dev commands
+
+```bash
+# Tail Splunk collector logs
+docker logs -f splunk-otel-collector
+
+# Verify collector internal metrics (self-monitoring)
+curl -s http://localhost:13133
+
+# Tear down just the Splunk overlay
+docker compose -f docker-compose.yml -f docker-compose.otel-demo.yml -f docker-compose.splunk-demo.yml rm -sf splunk-otel-collector
+```
+
+## Scope
+
+- **Out of scope:** `smartagentreceiver` (proprietary to Splunk's distribution, requires host bundle), `host_metrics` (meaningless in Docker), discovery mode, gateway forwarding.
+- **In scope:** OTLP traces/metrics/logs from the otel-demo apps, fanned out to Splunk Observability Cloud + OpenSearch.
+
+## References
+
+- [Splunk OpenTelemetry Collector distribution](https://github.com/signalfx/splunk-otel-collector)
+- [Splunk agent_config.yaml](https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml) — reference for the fuller config this one is trimmed from
+- [signalfxexporter README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter)
+- [splunkhecexporter README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter)
diff --git a/docker-compose/splunk-otel-collector/config.yaml b/docker-compose/splunk-otel-collector/config.yaml
new file mode 100644
index 00000000..83192633
--- /dev/null
+++ b/docker-compose/splunk-otel-collector/config.yaml
@@ -0,0 +1,110 @@
+# Splunk OpenTelemetry Collector — POC Configuration
+#
+# Acts as a tee: receives OTLP from demo apps, fans out to:
+#   1. Splunk Observability Cloud (APM via otlphttp, IM via signalfx, Log Observer via splunk_hec)
+#   2. observability-stack's base otel-collector (via OTLP forward), which continues the
+#      existing Data Prepper / OpenSearch pipeline unchanged.
+#
+# Trimmed from Splunk's default agent_config.yaml:
+#   - Host-level receivers (host_metrics, fluent_forward, smartagent/*) removed — not useful in container.
+#   - Extensions that require host integration (smartagent bundle, opamp, http_forwarder) removed.
+#   - Prometheus internal / entities pipelines removed — this is app-trace focused.
+#
+# References:
+#   Splunk agent config: https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml
+#   signalfxexporter:   https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter
+#   splunkhecexporter:  https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter
+
+extensions:
+  health_check:
+    endpoint: "0.0.0.0:13133"
+
+receivers:
+  # OTLP from demo apps (same shape as base otel-collector so apps' OTEL_EXPORTER_OTLP_ENDPOINT
+  # only needs the hostname changed via OTEL_COLLECTOR_HOST override in .env.splunk-poc).
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+        cors:
+          allowed_origins:
+            - "http://*"
+            - "https://*"
+
+processors:
+  memory_limiter:
+    check_interval: 5s
+    limit_percentage: 80
+    spike_limit_percentage: 25
+
+  batch:
+    timeout: 10s
+    send_batch_size: 1024
+
+  resourcedetection:
+    detectors: [env, system]
+
+exporters:
+  # ---- Splunk Observability Cloud ----
+  # Splunk APM (traces) via OTLP HTTP to Splunk's trace ingest
+  otlphttp/splunk-apm:
+    traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
+    headers:
+      "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
+
+  # Splunk Infrastructure Monitoring (metrics + events)
+  signalfx:
+    access_token: "${SPLUNK_ACCESS_TOKEN}"
+    api_url: "${SPLUNK_API_URL}"
+    ingest_url: "${SPLUNK_INGEST_URL}"
+    sync_host_metadata: true
+
+  # Splunk Log Observer (logs). If SPLUNK_HEC_TOKEN is unset or invalid, the logs
+  # pipeline will error but traces/metrics pipelines stay healthy.
+  splunk_hec:
+    token: "${SPLUNK_HEC_TOKEN}"
+    endpoint: "${SPLUNK_HEC_URL}"
+    source: "otel-demo"
+    sourcetype: "otel"
+    profiling_data_enabled: false
+
+  # ---- Tee to observability-stack base collector (OpenSearch path) ----
+  otlp/tee:
+    endpoint: "otel-collector:4317"
+    tls:
+      insecure: true
+
+  debug:
+    verbosity: basic
+
+service:
+  extensions: [health_check]
+
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, resourcedetection, batch]
+      exporters: [otlphttp/splunk-apm, signalfx, otlp/tee]
+
+    metrics:
+      receivers: [otlp]
+      processors: [memory_limiter, resourcedetection, batch]
+      exporters: [signalfx, otlp/tee]
+
+    logs:
+      receivers: [otlp]
+      processors: [memory_limiter, resourcedetection, batch]
+      exporters: [splunk_hec, otlp/tee]
+
+  telemetry:
+    logs:
+      level: info
+    metrics:
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: 0.0.0.0
+                port: 8888

From 22f966e7053f2b8315b3db0d5e939121e2b16e76 Mon Sep 17 00:00:00 2001
From: Kyle Hounslow <kylhouns@amazon.com>
Date: Fri, 8 May 2026 15:15:52 -0700
Subject: [PATCH 2/3] poc(splunk): simplify run command via include-directive
 pattern

.env.splunk-poc now sets INCLUDE_COMPOSE_OTEL_DEMO so the otel-demo overlay is
pulled in automatically via the base compose's include directive. Run command
drops from three -f flags to two.

Also switches README example from 'docker compose' to 'finch compose' to match
the project's default container runtime.

Signed-off-by: Kyle Hounslow <kylhouns@amazon.com>
---
 .env.splunk-poc.example                        | 3 +++
 docker-compose.splunk-demo.yml                 | 5 +++--
 docker-compose/splunk-otel-collector/README.md | 5 +++--
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/.env.splunk-poc.example b/.env.splunk-poc.example
index 0c6f17bd..e0841eaa 100644
--- a/.env.splunk-poc.example
+++ b/.env.splunk-poc.example
@@ -22,5 +22,8 @@ SPLUNK_HEC_TOKEN=your-hec-token-here-or-reuse-access-token
 # and forwards a copy via OTLP back to the base otel-collector for OpenSearch.
 OTEL_COLLECTOR_HOST=splunk-otel-collector
 
+# --- Activate the otel-demo overlay via the repo's include-directive pattern ---
+INCLUDE_COMPOSE_OTEL_DEMO=docker-compose.otel-demo.yml
+
 # --- Splunk collector image pin (optional) ---
 SPLUNK_COLLECTOR_VERSION=latest
diff --git a/docker-compose.splunk-demo.yml b/docker-compose.splunk-demo.yml
index 160c9e0b..2f51efab 100644
--- a/docker-compose.splunk-demo.yml
+++ b/docker-compose.splunk-demo.yml
@@ -13,12 +13,13 @@
 #
 # Usage:
 #   1. Copy .env.splunk-poc.example to .env.splunk-poc and fill in Splunk credentials.
-#   2. docker compose \
+#   2. finch compose \
 #        --env-file .env --env-file .env.splunk-poc \
 #        -f docker-compose.yml \
-#        -f docker-compose.otel-demo.yml \
 #        -f docker-compose.splunk-demo.yml \
 #        up -d
+#   (The otel-demo overlay is pulled in automatically via the base compose's include: directive
+#   because .env.splunk-poc sets INCLUDE_COMPOSE_OTEL_DEMO.)
 
 services:
   splunk-otel-collector:
diff --git a/docker-compose/splunk-otel-collector/README.md b/docker-compose/splunk-otel-collector/README.md
index d069002d..fa62dd52 100644
--- a/docker-compose/splunk-otel-collector/README.md
+++ b/docker-compose/splunk-otel-collector/README.md
@@ -27,10 +27,11 @@ Demo apps are redirected via `OTEL_COLLECTOR_HOST=splunk-otel-collector` (set in
 cp .env.splunk-poc.example .env.splunk-poc
 # edit .env.splunk-poc with your Splunk Access Token, Realm, HEC Token
 
-docker compose \
+# Base compose's include: directive pulls in the otel-demo overlay automatically
+# because .env.splunk-poc sets INCLUDE_COMPOSE_OTEL_DEMO.
+finch compose \
   --env-file .env --env-file .env.splunk-poc \
   -f docker-compose.yml \
-  -f docker-compose.otel-demo.yml \
   -f docker-compose.splunk-demo.yml \
   up -d
 ```

From ebc7adebc4ad99696ecc4f090e476b580172dbe8 Mon Sep 17 00:00:00 2001
From: Kyle Hounslow <kylhouns@amazon.com>
Date: Fri, 8 May 2026 15:53:33 -0700
Subject: [PATCH 3/3] poc(splunk): tighten docs and config to match rubric

Applies technical-writing.md and otel-contrib-pr-rubric.md:
- README: strip narrative prose, us-vs-them framing, and AI-tone defensives;
  drop docker commands in favor of the finch path users actually run.
- README: document finch's --env-file substitution limitation explicitly
  (cannot feed ${VAR} expansion in compose files) and show the working
  append-and-restore workflow.
- README: add real validation caveat for splunk_hec 404 on /v1/log with the
  access token reused as HEC token; note logs still land in OpenSearch.
- config.yaml: terse header, one-line comments per exporter, no rationale
  prose.
- docker-compose.splunk-demo.yml: two-line header, one-line port comment.
- .env.splunk-poc.example: remove explanatory prose, leave only locations
  and required vars.

No behavioral changes.

Signed-off-by: Kyle Hounslow <kylhouns@amazon.com>
---
 .env.splunk-poc.example                       | 31 ++----
 docker-compose.splunk-demo.yml                | 27 +-----
 .../splunk-otel-collector/README.md           | 95 ++++++++-----------
 .../splunk-otel-collector/config.yaml         | 32 ++-----
 4 files changed, 64 insertions(+), 121 deletions(-)

diff --git a/.env.splunk-poc.example b/.env.splunk-poc.example
index e0841eaa..66fb18a0 100644
--- a/.env.splunk-poc.example
+++ b/.env.splunk-poc.example
@@ -1,29 +1,18 @@
-# Splunk Distribution POC — credentials + demo-app redirect
-#
-# Copy this file to .env.splunk-poc and fill in your Splunk Observability Cloud values.
-# The actual .env.splunk-poc is gitignored and must never be committed.
-#
-# Where to find these in Splunk Observability Cloud:
-#   Access Token:  Organization → Access Tokens → any token with ingest scope
-#   Realm:         Visible in your Splunk Observability Cloud URL
-#                  (e.g. us0, us1, eu0, au0)
-#   HEC Token:     Data Management → Connect Data → HTTP Event Collector
-#                  (if absent, can often fall back to the Access Token)
+# Splunk Distribution POC credentials. Copy to .env.splunk-poc (gitignored).
+# Locations in Splunk Observability Cloud:
+#   SPLUNK_ACCESS_TOKEN: Organization > Access Tokens (ingest scope)
+#   SPLUNK_REALM:        visible in the Splunk Observability Cloud URL
+#   SPLUNK_HEC_TOKEN:    Data Management > Connect Data > HTTP Event Collector
+#                        (may not be available on all tiers; see POC README caveat)
 
-# --- Splunk credentials ---
-SPLUNK_ACCESS_TOKEN=your-splunk-access-token-here
+SPLUNK_ACCESS_TOKEN=
 SPLUNK_REALM=us1
-SPLUNK_HEC_TOKEN=your-hec-token-here-or-reuse-access-token
+SPLUNK_HEC_TOKEN=
 
-# --- Redirect demo apps through Splunk collector (the tee) ---
-# Demo apps read OTEL_COLLECTOR_HOST to build OTEL_EXPORTER_OTLP_ENDPOINT.
-# Overriding it to the Splunk collector's service name routes all telemetry through
-# the Splunk distribution first; Splunk then fans out to Splunk Observability Cloud
-# and forwards a copy via OTLP back to the base otel-collector for OpenSearch.
+# Redirect otel-demo apps through the Splunk collector.
 OTEL_COLLECTOR_HOST=splunk-otel-collector
 
-# --- Activate the otel-demo overlay via the repo's include-directive pattern ---
+# Activate the otel-demo overlay via the base compose's include directive.
 INCLUDE_COMPOSE_OTEL_DEMO=docker-compose.otel-demo.yml
 
-# --- Splunk collector image pin (optional) ---
 SPLUNK_COLLECTOR_VERSION=latest
diff --git a/docker-compose.splunk-demo.yml b/docker-compose.splunk-demo.yml
index 2f51efab..582b6e98 100644
--- a/docker-compose.splunk-demo.yml
+++ b/docker-compose.splunk-demo.yml
@@ -1,25 +1,5 @@
-# Splunk OTel Distribution — Side-by-side POC Overlay
-#
-# Inserts Splunk's OTel Collector distribution as a tee in front of the base otel-collector.
-# Demo apps send OTLP to splunk-otel-collector (via OTEL_COLLECTOR_HOST override), which fans
-# out to:
-#   - Splunk Observability Cloud (APM + IM + Log Observer)
-#   - base otel-collector → Data Prepper → OpenSearch (unchanged existing path)
-#
-# Purpose: see the same demo data in both Splunk's UI and OpenSearch Dashboards at the same
-# time, to ground-truth what Splunk Observability Cloud accepts and how it renders.
-#
-# NOT a production configuration. Local POC only.
-#
-# Usage:
-#   1. Copy .env.splunk-poc.example to .env.splunk-poc and fill in Splunk credentials.
-#   2. finch compose \
-#        --env-file .env --env-file .env.splunk-poc \
-#        -f docker-compose.yml \
-#        -f docker-compose.splunk-demo.yml \
-#        up -d
-#   (The otel-demo overlay is pulled in automatically via the base compose's include: directive
-#   because .env.splunk-poc sets INCLUDE_COMPOSE_OTEL_DEMO.)
+# Splunk OTel distribution overlay for the side-by-side POC.
+# See docker-compose/splunk-otel-collector/README.md to run.
 
 services:
   splunk-otel-collector:
@@ -30,8 +10,7 @@ services:
     volumes:
       - ./docker-compose/splunk-otel-collector/config.yaml:/etc/otelcol-config.yml
     ports:
-      # Host-exposed on non-default ports to avoid collision with the base collector.
-      # Demo apps reach splunk-otel-collector over the internal Docker network on 4317/4318.
+      # Non-default host ports to avoid collision with the base collector's 4317/4318.
       - "14317:4317"
       - "14318:4318"
       - "13133:13133"
diff --git a/docker-compose/splunk-otel-collector/README.md b/docker-compose/splunk-otel-collector/README.md
index fa62dd52..d3d8a044 100644
--- a/docker-compose/splunk-otel-collector/README.md
+++ b/docker-compose/splunk-otel-collector/README.md
@@ -1,82 +1,69 @@
-# Splunk OTel Distribution — POC
+# Splunk OTel Distribution POC
 
-Runs Splunk's OpenTelemetry Collector distribution as a tee in front of observability-stack's base collector, so demo telemetry lands in both Splunk Observability Cloud and OpenSearch at the same time.
+Local POC. Inserts Splunk's [OpenTelemetry Collector distribution](https://github.com/signalfx/splunk-otel-collector) between the otel-demo apps and the base `otel-collector`. Demo telemetry fans out to Splunk Observability Cloud and OpenSearch at the same time.
 
-Local POC only. Not a production configuration, not part of the default stack.
+Branch: `feat/splunk-distribution-poc`. Not intended for upstream.
 
 ## Dataflow
 
 ```
 otel-demo apps
-     │ OTLP
+     │ OTLP (OTEL_COLLECTOR_HOST=splunk-otel-collector)
      ▼
-splunk-otel-collector (Splunk distribution)
-     │
-     ├── signalfx         → Splunk Infrastructure Monitoring
-     ├── otlphttp         → Splunk APM (trace OTLP endpoint)
-     ├── splunk_hec       → Splunk Log Observer
-     └── otlp (tee)       → otel-collector → Data Prepper → OpenSearch (unchanged)
+splunk-otel-collector
+     ├── signalfx exporter         → Splunk Infrastructure Monitoring
+     ├── otlphttp/splunk-apm       → Splunk APM (ingest/v2/trace/otlp)
+     ├── splunk_hec                → Splunk Log Observer (see caveats)
+     └── otlp/tee                  → base otel-collector (unchanged OpenSearch path)
 ```
 
-Demo apps are redirected via `OTEL_COLLECTOR_HOST=splunk-otel-collector` (set in `.env.splunk-poc`). No demo code changes required.
-
 ## Run
 
 ```bash
-# From the repo root
 cp .env.splunk-poc.example .env.splunk-poc
-# edit .env.splunk-poc with your Splunk Access Token, Realm, HEC Token
-
-# Base compose's include: directive pulls in the otel-demo overlay automatically
-# because .env.splunk-poc sets INCLUDE_COMPOSE_OTEL_DEMO.
-finch compose \
-  --env-file .env --env-file .env.splunk-poc \
-  -f docker-compose.yml \
-  -f docker-compose.splunk-demo.yml \
-  up -d
-```
-
-## Verify
+# fill in SPLUNK_ACCESS_TOKEN, SPLUNK_REALM, SPLUNK_HEC_TOKEN
 
-Splunk Observability Cloud (URL varies by realm, e.g. `https://app.us1.signalfx.com`):
-
-- **APM → Services** — look for demo services (frontend, cart, checkout, ad, ...)
-- **Infrastructure → Metrics Explorer** — filter on `service.name`
-- **Log Observer** — filter on `sourcetype=otel`, `source=otel-demo`
+# finch compose's --env-file doesn't feed ${VAR} substitution in compose files, only
+# container env. Append the POC vars into .env before `up`, restore after.
+cat .env.splunk-poc >> .env
+finch compose -f docker-compose.yml -f docker-compose.splunk-demo.yml up -d
+git checkout -- .env
+```
 
-OpenSearch Dashboards (http://localhost:5601 by default):
+Teardown:
 
-- Trace Analytics — same demo services should appear in the existing OSD trace UI
-- Discover on `logs-otel-v1-*` — log records from the demo
+```bash
+finch compose -f docker-compose.yml -f docker-compose.splunk-demo.yml down
+```
 
-Splunk collector's own health endpoint: `http://localhost:13133`.
+## Verify
 
-Splunk collector's exposed OTLP ports on the host (non-default to avoid collision with base collector):
+Counters on the Splunk collector (send success numbers; non-zero = data flowing):
 
-- gRPC: `localhost:14317`
-- HTTP: `localhost:14318`
+```bash
+finch run --rm --network observability-stack-network curlimages/curl:latest \
+  -s http://splunk-otel-collector:8888/metrics \
+  | grep -E '^otelcol_exporter_sent_(spans|metric_points)_total'
+```
 
-## Dev commands
+Host-exposed endpoints on `splunk-otel-collector`:
 
-```bash
-# Tail Splunk collector logs
-docker logs -f splunk-otel-collector
+- `localhost:13133` - health check (200 = ready)
+- `localhost:14317` - OTLP gRPC
+- `localhost:14318` - OTLP HTTP
 
-# Verify collector internal metrics (self-monitoring)
-curl -s http://localhost:13133
+In Splunk Observability Cloud: **APM > Services** shows demo services (frontend, cart, checkout, ad, recommendation, …). **Infrastructure > Metrics Explorer** shows host metadata from `splunk-otel-collector` and app metrics from the demo.
 
-# Tear down just the Splunk overlay
-docker compose -f docker-compose.yml -f docker-compose.otel-demo.yml -f docker-compose.splunk-demo.yml rm -sf splunk-otel-collector
-```
+OpenSearch side is unaffected: Trace Analytics in OSD renders the same demo services via the tee.
 
-## Scope
+## Caveats from validation
 
-- **Out of scope:** `smartagentreceiver` (proprietary to Splunk's distribution, requires host bundle), `host_metrics` (meaningless in Docker), discovery mode, gateway forwarding.
-- **In scope:** OTLP traces/metrics/logs from the otel-demo apps, fanned out to Splunk Observability Cloud + OpenSearch.
+- **Splunk HEC logs return HTTP 404** on `/v1/log` with the access token reused as HEC token. The `splunk_hec` exporter retries indefinitely without blocking other pipelines. Logs still land in OpenSearch via `otlp/tee`. Cause is likely one of: access token lacks log-ingest scope, trial tier without Log Observer, or a distinct HEC token is required.
+- `smartagentreceiver` and `host_metrics` from Splunk's default `agent_config.yaml` are not included. The first is proprietary to Splunk's distribution bundle; the second is meaningless in a container.
 
-## References
+## Reference
 
-- [Splunk OpenTelemetry Collector distribution](https://github.com/signalfx/splunk-otel-collector)
-- [Splunk agent_config.yaml](https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml) — reference for the fuller config this one is trimmed from
-- [signalfxexporter README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter)
-- [splunkhecexporter README](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter)
+- [Splunk OTel Collector distribution](https://github.com/signalfx/splunk-otel-collector)
+- [Splunk default `agent_config.yaml`](https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml)
+- [signalfxexporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter)
+- [splunkhecexporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter)
diff --git a/docker-compose/splunk-otel-collector/config.yaml b/docker-compose/splunk-otel-collector/config.yaml
index 83192633..543dfba4 100644
--- a/docker-compose/splunk-otel-collector/config.yaml
+++ b/docker-compose/splunk-otel-collector/config.yaml
@@ -1,27 +1,17 @@
-# Splunk OpenTelemetry Collector — POC Configuration
+# Splunk OpenTelemetry Collector config for the side-by-side POC.
+# See docker-compose/splunk-otel-collector/README.md to run.
 #
-# Acts as a tee: receives OTLP from demo apps, fans out to:
-#   1. Splunk Observability Cloud (APM via otlphttp, IM via signalfx, Log Observer via splunk_hec)
-#   2. observability-stack's base otel-collector (via OTLP forward), which continues the
-#      existing Data Prepper / OpenSearch pipeline unchanged.
+# Based on Splunk's default agent_config.yaml with host-level receivers and
+# smartagent/opamp/http_forwarder extensions removed (not viable in-container).
 #
-# Trimmed from Splunk's default agent_config.yaml:
-#   - Host-level receivers (host_metrics, fluent_forward, smartagent/*) removed — not useful in container.
-#   - Extensions that require host integration (smartagent bundle, opamp, http_forwarder) removed.
-#   - Prometheus internal / entities pipelines removed — this is app-trace focused.
-#
-# References:
-#   Splunk agent config: https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml
-#   signalfxexporter:   https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter
-#   splunkhecexporter:  https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter
+# Reference: https://github.com/signalfx/splunk-otel-collector/blob/main/cmd/otelcol/config/collector/agent_config.yaml
 
 extensions:
   health_check:
     endpoint: "0.0.0.0:13133"
 
 receivers:
-  # OTLP from demo apps (same shape as base otel-collector so apps' OTEL_EXPORTER_OTLP_ENDPOINT
-  # only needs the hostname changed via OTEL_COLLECTOR_HOST override in .env.splunk-poc).
+  # OTLP on the same ports as the base collector so demo apps only need OTEL_COLLECTOR_HOST changed.
   otlp:
     protocols:
       grpc:
@@ -47,22 +37,20 @@ processors:
     detectors: [env, system]
 
 exporters:
-  # ---- Splunk Observability Cloud ----
-  # Splunk APM (traces) via OTLP HTTP to Splunk's trace ingest
+  # Splunk APM: OTLP traces to Splunk's trace ingest endpoint.
   otlphttp/splunk-apm:
     traces_endpoint: "${SPLUNK_INGEST_URL}/v2/trace/otlp"
     headers:
       "X-SF-Token": "${SPLUNK_ACCESS_TOKEN}"
 
-  # Splunk Infrastructure Monitoring (metrics + events)
+  # Splunk Infrastructure Monitoring: metrics + events.
   signalfx:
     access_token: "${SPLUNK_ACCESS_TOKEN}"
     api_url: "${SPLUNK_API_URL}"
     ingest_url: "${SPLUNK_INGEST_URL}"
     sync_host_metadata: true
 
-  # Splunk Log Observer (logs). If SPLUNK_HEC_TOKEN is unset or invalid, the logs
-  # pipeline will error but traces/metrics pipelines stay healthy.
+  # Splunk Log Observer via HEC. Returns 404 in current POC validation; see README caveats.
   splunk_hec:
     token: "${SPLUNK_HEC_TOKEN}"
     endpoint: "${SPLUNK_HEC_URL}"
@@ -70,7 +58,7 @@ exporters:
     sourcetype: "otel"
     profiling_data_enabled: false
 
-  # ---- Tee to observability-stack base collector (OpenSearch path) ----
+  # Forward to observability-stack's base otel-collector for the OpenSearch path.
   otlp/tee:
     endpoint: "otel-collector:4317"
     tls: