From 2b081c209e49fe77abc68f7007e913b118066445 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:20:05 -0600 Subject: [PATCH] fix: bound log growth with saner logger and system log table defaults Server logs and the system log tables share the data volume, yet the defaults let both grow without bound: logger level trace with 50 x 1000M rotated files (50 GB per file kind), and query_log, part_log, text_log, metric_log and asynchronous_metric_log enabled with no TTL. Default the logger to information / 100M / 10 files for both ClickHouseCluster and KeeperCluster, and give the five system log tables a 7 day TTL. Both remain overridable through spec.settings.logger and spec.settings.extraConfig (99-extra-config.yaml sorts after 00-logs-tables.yaml). Fixes #320 Amp-Thread-ID: https://ampcode.com/threads/T-01a08853-c3d6-757f-bafc-8ffe63be6c1c Co-authored-by: Amp --- api/v1alpha1/clickhousecluster_types.go | 4 +- api/v1alpha1/common.go | 6 +-- api/v1alpha1/defaults.go | 4 +- api/v1alpha1/keepercluster_types.go | 4 +- .../clickhouse.com_clickhouseclusters.yaml | 6 +-- .../bases/clickhouse.com_keeperclusters.yaml | 6 +-- .../clickhouseclusters.clickhouse.com.yaml | 6 +-- .../crd/keeperclusters.clickhouse.com.yaml | 6 +-- docs/guides/configuration.mdx | 38 ++++++++++++++----- docs/reference/api-reference.mdx | 6 +-- .../clickhouse/templates/log_tables.yaml.tmpl | 7 ++++ 11 files changed, 60 insertions(+), 33 deletions(-) diff --git a/api/v1alpha1/clickhousecluster_types.go b/api/v1alpha1/clickhousecluster_types.go index c0753791a..9dd054444 100644 --- a/api/v1alpha1/clickhousecluster_types.go +++ b/api/v1alpha1/clickhousecluster_types.go @@ -157,8 +157,8 @@ func (s *ClickHouseClusterSpec) WithDefaults() { Settings: ClickHouseSettings{ Logger: LoggerConfig{ LogToFile: new(true), - Level: "trace", - Size: "1000M", + Level: DefaultLogLevel, + Size: DefaultMaxLogFileSize, Count: DefaultMaxLogFiles, }, }, diff --git a/api/v1alpha1/common.go b/api/v1alpha1/common.go index 436ddb7ac..2d187fe24 100644 --- a/api/v1alpha1/common.go +++ b/api/v1alpha1/common.go @@ -56,17 +56,17 @@ type LoggerConfig struct { // Server logger verbosity level. // +optional // +kubebuilder:validation:Enum:=test;trace;debug;information;notice;warning;error;critical;fatal - // +kubebuilder:default:=trace + // +kubebuilder:default:=information Level string `json:"level,omitempty"` // Maximum log file size. // +optional - // +kubebuilder:default:="1000M" + // +kubebuilder:default:="100M" Size string `json:"size,omitempty"` // Maximum number of log files to keep. // +optional - // +kubebuilder:default:=50 + // +kubebuilder:default:=10 Count int64 `json:"count,omitempty"` } diff --git a/api/v1alpha1/defaults.go b/api/v1alpha1/defaults.go index ad4fb0be1..dcd42c520 100644 --- a/api/v1alpha1/defaults.go +++ b/api/v1alpha1/defaults.go @@ -28,7 +28,9 @@ const ( DefaultClickHouseShardCount = 1 DefaultClickHouseReplicaCount = 3 - DefaultMaxLogFiles = 50 + DefaultMaxLogFileSize = "100M" + DefaultMaxLogFiles = 10 + DefaultLogLevel = "information" // DefaultClusterDomain is the default Kubernetes cluster domain suffix for DNS resolution. DefaultClusterDomain = "cluster.local" diff --git a/api/v1alpha1/keepercluster_types.go b/api/v1alpha1/keepercluster_types.go index ea39e2f70..f5ea8190c 100644 --- a/api/v1alpha1/keepercluster_types.go +++ b/api/v1alpha1/keepercluster_types.go @@ -109,8 +109,8 @@ func (s *KeeperClusterSpec) WithDefaults() { Settings: KeeperSettings{ Logger: LoggerConfig{ LogToFile: new(true), - Level: "trace", - Size: "1000M", + Level: DefaultLogLevel, + Size: DefaultMaxLogFileSize, Count: DefaultMaxLogFiles, }, }, diff --git a/config/crd/bases/clickhouse.com_clickhouseclusters.yaml b/config/crd/bases/clickhouse.com_clickhouseclusters.yaml index 00c7fd005..ed4529d96 100644 --- a/config/crd/bases/clickhouse.com_clickhouseclusters.yaml +++ b/config/crd/bases/clickhouse.com_clickhouseclusters.yaml @@ -6664,7 +6664,7 @@ spec: description: Configuration of ClickHouse server logging. properties: count: - default: 50 + default: 10 description: Maximum number of log files to keep. format: int64 type: integer @@ -6673,7 +6673,7 @@ spec: description: If true, then log in JSON format. type: boolean level: - default: trace + default: information description: Server logger verbosity level. enum: - test @@ -6691,7 +6691,7 @@ spec: description: If false then disable all logging to file. type: boolean size: - default: 1000M + default: 100M description: Maximum log file size. type: string type: object diff --git a/config/crd/bases/clickhouse.com_keeperclusters.yaml b/config/crd/bases/clickhouse.com_keeperclusters.yaml index 7ae46a115..a2897d91b 100644 --- a/config/crd/bases/clickhouse.com_keeperclusters.yaml +++ b/config/crd/bases/clickhouse.com_keeperclusters.yaml @@ -6290,7 +6290,7 @@ spec: description: Configuration of ClickHouse Keeper server logging. properties: count: - default: 50 + default: 10 description: Maximum number of log files to keep. format: int64 type: integer @@ -6299,7 +6299,7 @@ spec: description: If true, then log in JSON format. type: boolean level: - default: trace + default: information description: Server logger verbosity level. enum: - test @@ -6317,7 +6317,7 @@ spec: description: If false then disable all logging to file. type: boolean size: - default: 1000M + default: 100M description: Maximum log file size. type: string type: object diff --git a/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml b/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml index ab82197e4..953acebf3 100644 --- a/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml +++ b/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml @@ -6667,7 +6667,7 @@ spec: description: Configuration of ClickHouse server logging. properties: count: - default: 50 + default: 10 description: Maximum number of log files to keep. format: int64 type: integer @@ -6676,7 +6676,7 @@ spec: description: If true, then log in JSON format. type: boolean level: - default: trace + default: information description: Server logger verbosity level. enum: - test @@ -6694,7 +6694,7 @@ spec: description: If false then disable all logging to file. type: boolean size: - default: 1000M + default: 100M description: Maximum log file size. type: string type: object diff --git a/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml b/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml index f4dcc347f..5e0cd0a34 100644 --- a/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml +++ b/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml @@ -6293,7 +6293,7 @@ spec: description: Configuration of ClickHouse Keeper server logging. properties: count: - default: 50 + default: 10 description: Maximum number of log files to keep. format: int64 type: integer @@ -6302,7 +6302,7 @@ spec: description: If true, then log in JSON format. type: boolean level: - default: trace + default: information description: Server logger verbosity level. enum: - test @@ -6320,7 +6320,7 @@ spec: description: If false then disable all logging to file. type: boolean size: - default: 1000M + default: 100M description: Maximum log file size. type: string type: object diff --git a/docs/guides/configuration.mdx b/docs/guides/configuration.mdx index 8a2de84c4..9e6d82066 100644 --- a/docs/guides/configuration.mdx +++ b/docs/guides/configuration.mdx @@ -817,7 +817,7 @@ Before a replica is removed on scale-down, the operator first unloads client tra ### Server logging {#server-logging} -Configure the ClickHouse server log through `spec.settings.logger`. Every field is optional with a safe default, so a cluster you never touch already logs at `trace` to both the container console and a rotated file on disk. +Configure the ClickHouse server log through `spec.settings.logger`. Every field is optional with a safe default, so a cluster you never touch already logs at `information` to both the container console and a rotated file on disk. ```yaml spec: @@ -825,29 +825,29 @@ spec: logger: logToFile: true # Default: true. Set false to log only to the console jsonLogs: false # Default: false. Set true for structured JSON log lines - level: trace # Default: trace - size: 1000M # Default: 1000M. Rotate a log file once it reaches this size - count: 50 # Default: 50. Number of rotated files to keep + level: information # Default: information + size: 100M # Default: 100M. Rotate a log file once it reaches this size + count: 10 # Default: 10. Number of rotated files to keep ``` | Field | Default | Description | |-------------|---------|-------------------------------------------------------------------------------------------------------------------| | `logToFile` | `true` | When `false`, the operator drops the file targets and the server logs only to the container console. | | `jsonLogs` | `false` | When `true`, the operator adds `formatting.type: json` so each line is a JSON object. | -| `level` | `trace` | Log verbosity. One of `test`, `trace`, `debug`, `information`, `notice`, `warning`, `error`, `critical`, `fatal`. | -| `size` | `1000M` | Maximum size of a single log file before rotation. | -| `count` | `50` | Number of rotated log files the server retains. | +| `level` | `information` | Log verbosity. One of `test`, `trace`, `debug`, `information`, `notice`, `warning`, `error`, `critical`, `fatal`. | +| `size` | `100M` | Maximum size of a single log file before rotation. | +| `count` | `10` | Number of rotated log files the server retains. | The operator always keeps console logging on so that `kubectl logs` works, and layers file logging on top when `logToFile` is `true`. A cluster with the defaults renders this `logger` block: ```yaml logger: console: true - level: trace + level: information log: /var/log/clickhouse-server/clickhouse-server.log errorlog: /var/log/clickhouse-server/clickhouse-server.err.log - size: 1000M - count: 50 + size: 100M + count: 10 ``` The same `spec.settings.logger` block applies to a `KeeperCluster`; the operator writes its files under `/var/log/clickhouse-keeper/` instead. @@ -856,6 +856,24 @@ The same `spec.settings.logger` block applies to a `KeeperCluster`; the operator Console logging stays on regardless of `logToFile`, so `kubectl logs` keeps working even when you disable file logging. Set `jsonLogs: true` when you ship logs to a structured log store that parses JSON. +### System log tables {#system-log-tables} + +The operator enables the `query_log`, `part_log`, `text_log`, `metric_log` and `asynchronous_metric_log` system tables with a 7 day TTL, because they live on the same volume as your data. Override the TTL, or any other system log setting, through `extraConfig`: + +```yaml +spec: + settings: + extraConfig: + text_log: + ttl: event_date + INTERVAL 30 DAY DELETE + metric_log: + collect_interval_milliseconds: 60000 +``` + + +ClickHouse recreates a system table whose TTL changed and keeps the old data in a renamed table such as `system.text_log_0`. Drop those tables once you no longer need the history. + + ## Custom configuration {#custom-configuration} ### Embedded extra configuration {#embedded-extra-configuration} diff --git a/docs/reference/api-reference.mdx b/docs/reference/api-reference.mdx index 8751ee9a3..4ef3f151e 100644 --- a/docs/reference/api-reference.mdx +++ b/docs/reference/api-reference.mdx @@ -379,9 +379,9 @@ LoggerConfig defines server logging configuration. |-------|------|-------------|----------|---------| | `logToFile` | boolean | If false then disable all logging to file. | false | true | | `jsonLogs` | boolean | If true, then log in JSON format. | false | false | -| `level` | string | Server logger verbosity level. | false | trace | -| `size` | string | Maximum log file size. | false | 1000M | -| `count` | integer | Maximum number of log files to keep. | false | 50 | +| `level` | string | Server logger verbosity level. | false | information | +| `size` | string | Maximum log file size. | false | 100M | +| `count` | integer | Maximum number of log files to keep. | false | 10 | Appears in: - [ClickHouseSettings](#clickhousesettings) diff --git a/internal/controller/clickhouse/templates/log_tables.yaml.tmpl b/internal/controller/clickhouse/templates/log_tables.yaml.tmpl index 35bfe8047..494b94bcb 100644 --- a/internal/controller/clickhouse/templates/log_tables.yaml.tmpl +++ b/internal/controller/clickhouse/templates/log_tables.yaml.tmpl @@ -1,5 +1,12 @@ +# System log tables grow without bound unless they carry a TTL, and they share +# the data volume with user data. Override via spec.settings.extraConfig. query_log: + ttl: event_date + INTERVAL 7 DAY DELETE part_log: + ttl: event_date + INTERVAL 7 DAY DELETE text_log: + ttl: event_date + INTERVAL 7 DAY DELETE asynchronous_metric_log: + ttl: event_date + INTERVAL 7 DAY DELETE metric_log: + ttl: event_date + INTERVAL 7 DAY DELETE