From 8ec81193ed4bac547b276eaedeb615a139bbd6f6 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:20:15 -0600 Subject: [PATCH] fix: bound ClickHouse server log and system log table growth The operator defaults to logger level trace with 50 x 1000M files and enables five system log tables with no TTL, all on the data volume. The chart's 10Gi default fills within days, after which the OTel collector drops every batch. Set logger to information / 100M / 10 and add a 7 day TTL to query_log, part_log, text_log, metric_log and asynchronous_metric_log via extraConfig. Amp-Thread-ID: https://ampcode.com/threads/T-01a08853-c3d6-757f-bafc-8ffe63be6c1c Co-authored-by: Amp --- .changeset/bound-clickhouse-log-growth.md | 9 +++++++++ .../tests/clickhouse-service_test.yaml | 15 +++++++++++++++ charts/clickstack/values.yaml | 17 +++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 .changeset/bound-clickhouse-log-growth.md diff --git a/.changeset/bound-clickhouse-log-growth.md b/.changeset/bound-clickhouse-log-growth.md new file mode 100644 index 00000000..9102deaa --- /dev/null +++ b/.changeset/bound-clickhouse-log-growth.md @@ -0,0 +1,9 @@ +--- +"helm-charts": minor +--- + +fix(clickstack): bound ClickHouse server log and system log table growth on the data volume + +The ClickHouse operator defaults to `logger.level: trace` with 50 x 1000M rotated log files, and enables `query_log`, `part_log`, `text_log`, `metric_log` and `asynchronous_metric_log` without a TTL. Both are written to the ClickHouse data volume, so the chart's 10Gi default fills up within days of light use, after which the OTel collector drops every batch. The chart now sets `clickhouse.cluster.spec.settings.logger` to `information` / `100M` / 10 files and adds a 7 day TTL to those five system tables via `extraConfig`. Override either block in your values to keep more history. + +On upgrade, ClickHouse recreates each system table whose TTL changed and keeps the old rows in `system._0`; drop those once you no longer need them. diff --git a/charts/clickstack/tests/clickhouse-service_test.yaml b/charts/clickstack/tests/clickhouse-service_test.yaml index 143cf8c1..c03bf2c7 100644 --- a/charts/clickstack/tests/clickhouse-service_test.yaml +++ b/charts/clickstack/tests/clickhouse-service_test.yaml @@ -9,3 +9,18 @@ tests: of: ClickHouseCluster - isNotNull: path: spec.settings.extraConfig + + - it: should bound server log and system log table growth on the data volume + asserts: + - equal: + path: spec.settings.logger.level + value: information + - equal: + path: spec.settings.logger.count + value: 10 + - equal: + path: spec.settings.extraConfig.text_log.ttl + value: event_date + INTERVAL 7 DAY DELETE + - equal: + path: spec.settings.extraConfig.metric_log.ttl + value: event_date + INTERVAL 7 DAY DELETE diff --git a/charts/clickstack/values.yaml b/charts/clickstack/values.yaml index 429948fb..ef370aee 100644 --- a/charts/clickstack/values.yaml +++ b/charts/clickstack/values.yaml @@ -462,10 +462,27 @@ clickhouse: profile: default grants: - query: "GRANT SELECT,INSERT,CREATE,SHOW ON default.*" + # The operator defaults to level trace with 50 x 1000M log files, written + # to the data volume. That fills the 10Gi default volume within days. + logger: + level: information + size: 100M + count: 10 extraConfig: max_connections: 4096 keep_alive_timeout: 64 max_concurrent_queries: 100 + # System log tables also live on the data volume and have no TTL by default. + query_log: + ttl: event_date + INTERVAL 7 DAY DELETE + part_log: + ttl: event_date + INTERVAL 7 DAY DELETE + text_log: + ttl: event_date + INTERVAL 7 DAY DELETE + metric_log: + ttl: event_date + INTERVAL 7 DAY DELETE + asynchronous_metric_log: + ttl: event_date + INTERVAL 7 DAY DELETE # Additional Kubernetes manifests to deploy alongside the chart. # Each entry is a complete Kubernetes object. Values are rendered through