fix: bound log growth with saner logger and system log table defaults

api/v1alpha1/clickhousecluster_types.go

@@ -157,8 +157,8 @@ func (s *ClickHouseClusterSpec) WithDefaults() {
 		Settings: ClickHouseSettings{
 			Logger: LoggerConfig{
 				LogToFile: new(true),
-				Level:     "trace",
-				Size:      "1000M",
+				Level:     DefaultLogLevel,
+				Size:      DefaultMaxLogFileSize,
 				Count:     DefaultMaxLogFiles,
 			},
 		},

api/v1alpha1/common.go

@@ -56,17 +56,17 @@ type LoggerConfig struct {
 	// Server logger verbosity level.
 	// +optional
 	// +kubebuilder:validation:Enum:=test;trace;debug;information;notice;warning;error;critical;fatal
-	// +kubebuilder:default:=trace
+	// +kubebuilder:default:=information
 	Level string `json:"level,omitempty"`
 
 	// Maximum log file size.
 	// +optional
-	// +kubebuilder:default:="1000M"
+	// +kubebuilder:default:="100M"
 	Size string `json:"size,omitempty"`
 
 	// Maximum number of log files to keep.
 	// +optional
-	// +kubebuilder:default:=50
+	// +kubebuilder:default:=10
 	Count int64 `json:"count,omitempty"`
 }
 

api/v1alpha1/defaults.go

@@ -28,7 +28,9 @@ const (
 	DefaultClickHouseShardCount   = 1
 	DefaultClickHouseReplicaCount = 3
 
-	DefaultMaxLogFiles = 50
+	DefaultMaxLogFileSize = "100M"
+	DefaultMaxLogFiles    = 10
+	DefaultLogLevel       = "information"
 
 	// DefaultClusterDomain is the default Kubernetes cluster domain suffix for DNS resolution.
 	DefaultClusterDomain = "cluster.local"

api/v1alpha1/keepercluster_types.go

@@ -109,8 +109,8 @@ func (s *KeeperClusterSpec) WithDefaults() {
 		Settings: KeeperSettings{
 			Logger: LoggerConfig{
 				LogToFile: new(true),
-				Level:     "trace",
-				Size:      "1000M",
+				Level:     DefaultLogLevel,
+				Size:      DefaultMaxLogFileSize,
 				Count:     DefaultMaxLogFiles,
 			},
 		},

config/crd/bases/clickhouse.com_clickhouseclusters.yaml

@@ -6664,7 +6664,7 @@ spec:
                     description: Configuration of ClickHouse server logging.
                     properties:
                       count:
-                        default: 50
+                        default: 10
                         description: Maximum number of log files to keep.
                         format: int64
                         type: integer
@@ -6673,7 +6673,7 @@ spec:
                         description: If true, then log in JSON format.
                         type: boolean
                       level:
-                        default: trace
+                        default: information
                         description: Server logger verbosity level.
                         enum:
                         - test
@@ -6691,7 +6691,7 @@ spec:
                         description: If false then disable all logging to file.
                         type: boolean
                       size:
-                        default: 1000M
+                        default: 100M
                         description: Maximum log file size.
                         type: string
                     type: object

config/crd/bases/clickhouse.com_keeperclusters.yaml

@@ -6290,7 +6290,7 @@ spec:
                     description: Configuration of ClickHouse Keeper server logging.
                     properties:
                       count:
-                        default: 50
+                        default: 10
                         description: Maximum number of log files to keep.
                         format: int64
                         type: integer
@@ -6299,7 +6299,7 @@ spec:
                         description: If true, then log in JSON format.
                         type: boolean
                       level:
-                        default: trace
+                        default: information
                         description: Server logger verbosity level.
                         enum:
                         - test
@@ -6317,7 +6317,7 @@ spec:
                         description: If false then disable all logging to file.
                         type: boolean
                       size:
-                        default: 1000M
+                        default: 100M
                         description: Maximum log file size.
                         type: string
                     type: object

dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml

@@ -6667,7 +6667,7 @@ spec:
                     description: Configuration of ClickHouse server logging.
                     properties:
                       count:
-                        default: 50
+                        default: 10
                         description: Maximum number of log files to keep.
                         format: int64
                         type: integer
@@ -6676,7 +6676,7 @@ spec:
                         description: If true, then log in JSON format.
                         type: boolean
                       level:
-                        default: trace
+                        default: information
                         description: Server logger verbosity level.
                         enum:
                         - test
@@ -6694,7 +6694,7 @@ spec:
                         description: If false then disable all logging to file.
                         type: boolean
                       size:
-                        default: 1000M
+                        default: 100M
                         description: Maximum log file size.
                         type: string
                     type: object

dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml

@@ -6293,7 +6293,7 @@ spec:
                     description: Configuration of ClickHouse Keeper server logging.
                     properties:
                       count:
-                        default: 50
+                        default: 10
                         description: Maximum number of log files to keep.
                         format: int64
                         type: integer
@@ -6302,7 +6302,7 @@ spec:
                         description: If true, then log in JSON format.
                         type: boolean
                       level:
-                        default: trace
+                        default: information
                         description: Server logger verbosity level.
                         enum:
                         - test
@@ -6320,7 +6320,7 @@ spec:
                         description: If false then disable all logging to file.
                         type: boolean
                       size:
-                        default: 1000M
+                        default: 100M
                         description: Maximum log file size.
                         type: string
                     type: object

docs/guides/configuration.mdx

@@ -817,37 +817,37 @@ Before a replica is removed on scale-down, the operator first unloads client tra
 
 ### Server logging {#server-logging}
 
-Configure the ClickHouse server log through `spec.settings.logger`. Every field is optional with a safe default, so a cluster you never touch already logs at `trace` to both the container console and a rotated file on disk.
+Configure the ClickHouse server log through `spec.settings.logger`. Every field is optional with a safe default, so a cluster you never touch already logs at `information` to both the container console and a rotated file on disk.
 
 ```yaml
 spec:
   settings:
     logger:
       logToFile: true   # Default: true. Set false to log only to the console
       jsonLogs: false   # Default: false. Set true for structured JSON log lines
-      level: trace      # Default: trace
-      size: 1000M       # Default: 1000M. Rotate a log file once it reaches this size
-      count: 50         # Default: 50. Number of rotated files to keep
+      level: information # Default: information
+      size: 100M        # Default: 100M. Rotate a log file once it reaches this size
+      count: 10         # Default: 10. Number of rotated files to keep
 ```
 
 | Field       | Default | Description                                                                                                       |
 |-------------|---------|-------------------------------------------------------------------------------------------------------------------|
 | `logToFile` | `true`  | When `false`, the operator drops the file targets and the server logs only to the container console.              |
 | `jsonLogs`  | `false` | When `true`, the operator adds `formatting.type: json` so each line is a JSON object.                             |
-| `level`     | `trace` | Log verbosity. One of `test`, `trace`, `debug`, `information`, `notice`, `warning`, `error`, `critical`, `fatal`. |
-| `size`      | `1000M` | Maximum size of a single log file before rotation.                                                                |
-| `count`     | `50`    | Number of rotated log files the server retains.                                                                   |
+| `level`     | `information` | Log verbosity. One of `test`, `trace`, `debug`, `information`, `notice`, `warning`, `error`, `critical`, `fatal`. |
+| `size`      | `100M`  | Maximum size of a single log file before rotation.                                                                |
+| `count`     | `10`    | Number of rotated log files the server retains.                                                                   |
 
 The operator always keeps console logging on so that `kubectl logs` works, and layers file logging on top when `logToFile` is `true`. A cluster with the defaults renders this `logger` block:
 
 ```yaml
 logger:
   console: true
-  level: trace
+  level: information
   log: /var/log/clickhouse-server/clickhouse-server.log
   errorlog: /var/log/clickhouse-server/clickhouse-server.err.log
-  size: 1000M
-  count: 50
+  size: 100M
+  count: 10
 ```
 
 The same `spec.settings.logger` block applies to a `KeeperCluster`; the operator writes its files under `/var/log/clickhouse-keeper/` instead.
@@ -856,6 +856,24 @@ The same `spec.settings.logger` block applies to a `KeeperCluster`; the operator
 Console logging stays on regardless of `logToFile`, so `kubectl logs` keeps working even when you disable file logging. Set `jsonLogs: true` when you ship logs to a structured log store that parses JSON.
 </Note>
 
+### System log tables {#system-log-tables}
+
+The operator enables the `query_log`, `part_log`, `text_log`, `metric_log` and `asynchronous_metric_log` system tables with a 7 day TTL, because they live on the same volume as your data. Override the TTL, or any other system log setting, through `extraConfig`:
+
+```yaml
+spec:
+  settings:
+    extraConfig:
+      text_log:
+        ttl: event_date + INTERVAL 30 DAY DELETE
+      metric_log:
+        collect_interval_milliseconds: 60000
+```
+
+<Note>
+ClickHouse recreates a system table whose TTL changed and keeps the old data in a renamed table such as `system.text_log_0`. Drop those tables once you no longer need the history.
+</Note>
+
 ## Custom configuration {#custom-configuration}
 
 ### Embedded extra configuration {#embedded-extra-configuration}

docs/reference/api-reference.mdx

@@ -379,9 +379,9 @@ LoggerConfig defines server logging configuration.
 |-------|------|-------------|----------|---------|
 | `logToFile` | boolean | If false then disable all logging to file. | false | true |
 | `jsonLogs` | boolean | If true, then log in JSON format. | false | false |
-| `level` | string | Server logger verbosity level. | false | trace |
-| `size` | string | Maximum log file size. | false | 1000M |
-| `count` | integer | Maximum number of log files to keep. | false | 50 |
+| `level` | string | Server logger verbosity level. | false | information |
+| `size` | string | Maximum log file size. | false | 100M |
+| `count` | integer | Maximum number of log files to keep. | false | 10 |
 
 Appears in:
 - [ClickHouseSettings](#clickhousesettings)

internal/controller/clickhouse/templates/log_tables.yaml.tmpl

@@ -1,5 +1,12 @@
+# System log tables grow without bound unless they carry a TTL, and they share
+# the data volume with user data. Override via spec.settings.extraConfig.
 query_log:
+  ttl: event_date + INTERVAL 7 DAY DELETE
 part_log:
+  ttl: event_date + INTERVAL 7 DAY DELETE
 text_log:
+  ttl: event_date + INTERVAL 7 DAY DELETE
 asynchronous_metric_log:
+  ttl: event_date + INTERVAL 7 DAY DELETE
 metric_log:
+  ttl: event_date + INTERVAL 7 DAY DELETE