From 64d0b298fd92ddbbdbe5b09ec4bc8a5869041856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Wed, 26 Aug 2026 11:31:46 -0700 Subject: [PATCH 1/9] feat(alloy): add structured log labels via journal relabeling Extract unit, priority, and app labels from the systemd journal on all hosts (glyph, spore, zeta) so Loki queries can be scoped by service unit, syslog severity, and identifier without full-text search. Co-Authored-By: Claude Sonnet 4.6 Entire-Checkpoint: d908b189c88f --- hosts/glyph/services/alloy.nix | 17 +++++++++++++++++ hosts/spore/services/alloy.nix | 17 +++++++++++++++++ hosts/zeta/services/alloy.nix | 17 +++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/hosts/glyph/services/alloy.nix b/hosts/glyph/services/alloy.nix index 9f77a165..c9e234b0 100644 --- a/hosts/glyph/services/alloy.nix +++ b/hosts/glyph/services/alloy.nix @@ -3,7 +3,24 @@ _: { environment.etc."alloy/config.alloy".text = '' loki.source.journal "systemd" { + forward_to = [loki.relabel.journal.receiver] + } + + loki.relabel "journal" { forward_to = [loki.write.local.receiver] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal__priority"] + target_label = "priority" + } + rule { + source_labels = ["__journal__syslog_identifier"] + target_label = "app" + } } loki.write "local" { diff --git a/hosts/spore/services/alloy.nix b/hosts/spore/services/alloy.nix index 298e7976..d4842d22 100644 --- a/hosts/spore/services/alloy.nix +++ b/hosts/spore/services/alloy.nix @@ -3,7 +3,24 @@ _: { environment.etc."alloy/config.alloy".text = '' loki.source.journal "systemd" { + forward_to = [loki.relabel.journal.receiver] + } + + loki.relabel "journal" { forward_to = [loki.write.remote.receiver] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal__priority"] + target_label = "priority" + } + rule { + source_labels = ["__journal__syslog_identifier"] + target_label = "app" + } } loki.write "remote" { diff --git a/hosts/zeta/services/alloy.nix b/hosts/zeta/services/alloy.nix index 325e11a9..0c155ce7 100644 --- a/hosts/zeta/services/alloy.nix +++ b/hosts/zeta/services/alloy.nix @@ -3,7 +3,24 @@ _: { environment.etc."alloy/config.alloy".text = '' loki.source.journal "systemd" { + forward_to = [loki.relabel.journal.receiver] + } + + loki.relabel "journal" { forward_to = [loki.write.remote.receiver] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal__priority"] + target_label = "priority" + } + rule { + source_labels = ["__journal__syslog_identifier"] + target_label = "app" + } } loki.write "remote" { From c8ab95dba464be9569e5c252a3a66d5ab95ded55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Wed, 26 Aug 2026 11:32:08 -0700 Subject: [PATCH 2/9] feat(glyph): add postgres and smartctl prometheus exporters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit postgres_exporter covers query throughput, connections, and vacuum stats for the five databases on glyph. smartctl_exporter surfaces disk health, temperature, and reallocated sectors — important for a NAS. Co-Authored-By: Claude Sonnet 4.6 Entire-Checkpoint: f616080a78e1 --- hosts/glyph/services/prometheus.nix | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/hosts/glyph/services/prometheus.nix b/hosts/glyph/services/prometheus.nix index e94785c0..bef16839 100644 --- a/hosts/glyph/services/prometheus.nix +++ b/hosts/glyph/services/prometheus.nix @@ -15,6 +15,15 @@ enable = true; port = 9134; }; + exporters.postgres = { + enable = true; + port = 9187; + dataSourceName = "user=mu database=postgres host=/var/run/postgresql sslmode=disable"; + }; + exporters.smartctl = { + enable = true; + port = 9633; + }; scrapeConfigs = [ { job_name = "node"; @@ -40,6 +49,29 @@ targets = [ "localhost:${toString config.services.prometheus.exporters.zfs.port}" ]; + labels.instance = "glyph"; + } + ]; + } + { + job_name = "postgres"; + static_configs = [ + { + targets = [ + "localhost:${toString config.services.prometheus.exporters.postgres.port}" + ]; + labels.instance = "glyph"; + } + ]; + } + { + job_name = "smartctl"; + static_configs = [ + { + targets = [ + "localhost:${toString config.services.prometheus.exporters.smartctl.port}" + ]; + labels.instance = "glyph"; } ]; } From 6a846de0c4f2e1b6704971c8e6ac534d227e9075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Wed, 26 Aug 2026 11:32:51 -0700 Subject: [PATCH 3/9] feat(glyph): enable navidrome prometheus endpoint and scrape Enables the built-in /metrics endpoint in navidrome and adds a Prometheus scrape job for it. Covers library size, play counts, active streams, and scan duration. Co-Authored-By: Claude Sonnet 4.6 Entire-Checkpoint: 56d6f0a22081 --- hosts/glyph/services/navidrome.nix | 2 ++ hosts/glyph/services/prometheus.nix | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/hosts/glyph/services/navidrome.nix b/hosts/glyph/services/navidrome.nix index 9c520aa1..86abb523 100644 --- a/hosts/glyph/services/navidrome.nix +++ b/hosts/glyph/services/navidrome.nix @@ -13,9 +13,11 @@ environmentFile = config.age.secrets.navidrome-env.path; settings = { Address = "0.0.0.0"; + Port = 4533; MusicFolder = "/mnt/media/Music"; Scanner.Schedule = "@every 1h"; RecentlyAddedByModTime = true; + Prometheus.Enabled = true; }; }; } diff --git a/hosts/glyph/services/prometheus.nix b/hosts/glyph/services/prometheus.nix index bef16839..9699cab8 100644 --- a/hosts/glyph/services/prometheus.nix +++ b/hosts/glyph/services/prometheus.nix @@ -75,6 +75,17 @@ } ]; } + { + job_name = "navidrome"; + static_configs = [ + { + targets = [ + "localhost:4533" + ]; + labels.instance = "glyph"; + } + ]; + } ]; }; } From 9d5f9e2bf86e2582fb02375f4773b53a2deec764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Wed, 26 Aug 2026 11:33:25 -0700 Subject: [PATCH 4/9] feat(spore): add nginx exporter with stub_status Adds a localhost-only nginx stub_status endpoint on port 9080 and the prometheus nginx_exporter on port 9113. Glyph's Prometheus scrapes it for request rate, active connections, and error metrics. Co-Authored-By: Claude Sonnet 4.6 Entire-Checkpoint: 514b9becae1a --- hosts/glyph/services/prometheus.nix | 11 +++++++++++ hosts/spore/services/default.nix | 6 ++++++ hosts/spore/services/web/default.nix | 10 ++++++++++ 3 files changed, 27 insertions(+) diff --git a/hosts/glyph/services/prometheus.nix b/hosts/glyph/services/prometheus.nix index 9699cab8..2fd48704 100644 --- a/hosts/glyph/services/prometheus.nix +++ b/hosts/glyph/services/prometheus.nix @@ -75,6 +75,17 @@ } ]; } + { + job_name = "nginx"; + static_configs = [ + { + targets = [ + "spore.note-iwato.ts.net:9113" + ]; + labels.instance = "spore"; + } + ]; + } { job_name = "navidrome"; static_configs = [ diff --git a/hosts/spore/services/default.nix b/hosts/spore/services/default.nix index 6dd1aada..07c05c48 100644 --- a/hosts/spore/services/default.nix +++ b/hosts/spore/services/default.nix @@ -20,6 +20,12 @@ enabledCollectors = ["systemd"]; }; + services.prometheus.exporters.nginx = { + enable = true; + port = 9113; + scrapeUri = "http://127.0.0.1:9080/nginx_status"; + }; + services.glances = { inherit (config.services.homepage-dashboard) enable; }; diff --git a/hosts/spore/services/web/default.nix b/hosts/spore/services/web/default.nix index d40726a3..b1450c25 100644 --- a/hosts/spore/services/web/default.nix +++ b/hosts/spore/services/web/default.nix @@ -211,6 +211,16 @@ }; }; + services.nginx.virtualHosts."nginx-metrics" = { + listen = [ + { + addr = "127.0.0.1"; + port = 9080; + } + ]; + locations."/nginx_status".extraConfig = "stub_status;"; + }; + services.zx-dev = { enable = true; domain = "zx.dev"; From 9d5329d210a7324a11d6b004ecdb911b94b04b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Wed, 26 Aug 2026 11:34:04 -0700 Subject: [PATCH 5/9] docs: document monitoring stack for agent use in CLAUDE.md Adds a Monitoring Stack section covering Loki label schema, Prometheus jobs/exporters, and common LogQL/PromQL patterns. Instructs agents to use the grafana MCP proactively instead of journalctl or SSH. Co-Authored-By: Claude Sonnet 4.6 Entire-Checkpoint: 61096eefaeb4 --- CLAUDE.md | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 733192eb..3dd9c1d8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -163,6 +163,68 @@ Use `lib.mkForce` when a host needs to diverge from a value set in a shared modu nix.gc.dates = lib.mkForce "daily"; ``` +## Monitoring Stack + +The homelab runs a Grafana LGTM-lite stack for observability. **Use it proactively** when investigating service failures, slow response times, disk issues, or any situation where you'd otherwise reach for `journalctl` or SSH into a host to check a service. + +- **Grafana** (`grafana.zx.dev`) — dashboards, Explore, alerting +- **Loki** (glyph:3100) — log aggregation from glyph, spore, zeta +- **Prometheus** (glyph:9099) — metrics from all hosts + +**MCP access:** The `grafana` MCP server is registered in mcpjungle on glyph at `http://127.0.0.1:8095/mcp`. It exposes tools for LogQL (Loki), PromQL (Prometheus), and dashboard access. Use it instead of `journalctl` for anything beyond a quick one-liner. + +### Loki label schema + +All logs carry these labels, queryable with `{label="value"}` in LogQL: + +| Label | Source journal field | Example values | +|---|---|---| +| `host` | Static (Alloy external_labels) | `glyph`, `spore`, `zeta` | +| `unit` | `_SYSTEMD_UNIT` | `navidrome.service`, `nginx.service` | +| `priority` | `PRIORITY` | `0`–`7` (0=emerg, 3=err, 4=warn, 6=info, 7=debug) | +| `app` | `SYSLOG_IDENTIFIER` | `navidrome`, `nginx`, `kernel` | + +**Common LogQL patterns:** +```logql +# All errors and above from a specific service +{host="glyph", unit="navidrome.service", priority=~"[0-3]"} + +# All warnings and above across spore +{host="spore", priority=~"[0-4]"} + +# nginx access and error logs on spore +{host="spore", app="nginx"} + +# Recent errors across all hosts +{priority=~"[0-3]"} |= "error" +``` + +### Prometheus jobs and exporters + +| Job | Port | Host | Covers | +|---|---|---|---| +| `node` | 9100 | glyph, spore | CPU, memory, disk, network, systemd unit states | +| `zfs` | 9134 | glyph | Pool health, ARC hit ratio, pool space | +| `postgres` | 9187 | glyph | Connections, query throughput, vacuum, per-DB stats | +| `smartctl` | 9633 | glyph | Disk SMART data, temperature, reallocated sectors | +| `nginx` | 9113 | spore | Request rate, active connections, handled/dropped | +| `navidrome` | 4533/metrics | glyph | Library size, play counts, scan duration | + +**Common PromQL patterns:** +```promql +# Disk temperature (watch for > 50°C on NAS drives) +smartctl_device_temperature{instance="glyph"} + +# PostgreSQL active connections per database +pg_stat_database_numbackends{instance="glyph"} + +# nginx request rate over 5 minutes +rate(nginx_http_requests_total{instance="spore"}[5m]) + +# Filesystem use % on glyph (watch for > 85%) +100 - (node_filesystem_avail_bytes{instance="glyph",mountpoint="/"} / node_filesystem_size_bytes{instance="glyph",mountpoint="/"} * 100) +``` + ## Environment Awareness - Before running commands like `ssh`, `nixos-rebuild`, or anything that targets a specific host, check which host Claude Code is running on (`hostname`) to avoid targeting the current machine unintentionally. From 8470cf0b9b224ceac82ac3553c11e6bf4d90a264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Wed, 26 Aug 2026 15:44:58 -0700 Subject: [PATCH 6/9] fix(alloy): use discovery.relabel to apply journal label rules at source The pipeline approach (forward_to loki.relabel) strips __journal__* meta-labels before they reach the relabel component. relabel_rules on loki.source.journal applies rules during the read phase while those labels are still available; discovery.relabel provides the correct RelabelRules export type for that argument. Co-Authored-By: Claude Sonnet 4.6 Entire-Checkpoint: 72d76598b42b --- hosts/glyph/services/alloy.nix | 13 +++++++------ hosts/spore/services/alloy.nix | 13 +++++++------ hosts/zeta/services/alloy.nix | 13 +++++++------ 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/hosts/glyph/services/alloy.nix b/hosts/glyph/services/alloy.nix index c9e234b0..411e92d7 100644 --- a/hosts/glyph/services/alloy.nix +++ b/hosts/glyph/services/alloy.nix @@ -2,12 +2,8 @@ _: { services.alloy.enable = true; environment.etc."alloy/config.alloy".text = '' - loki.source.journal "systemd" { - forward_to = [loki.relabel.journal.receiver] - } - - loki.relabel "journal" { - forward_to = [loki.write.local.receiver] + discovery.relabel "journal" { + targets = [] rule { source_labels = ["__journal__systemd_unit"] @@ -23,6 +19,11 @@ _: { } } + loki.source.journal "systemd" { + relabel_rules = discovery.relabel.journal.rules + forward_to = [loki.write.local.receiver] + } + loki.write "local" { endpoint { url = "http://localhost:3100/loki/api/v1/push" diff --git a/hosts/spore/services/alloy.nix b/hosts/spore/services/alloy.nix index d4842d22..a1bd3d95 100644 --- a/hosts/spore/services/alloy.nix +++ b/hosts/spore/services/alloy.nix @@ -2,12 +2,8 @@ _: { services.alloy.enable = true; environment.etc."alloy/config.alloy".text = '' - loki.source.journal "systemd" { - forward_to = [loki.relabel.journal.receiver] - } - - loki.relabel "journal" { - forward_to = [loki.write.remote.receiver] + discovery.relabel "journal" { + targets = [] rule { source_labels = ["__journal__systemd_unit"] @@ -23,6 +19,11 @@ _: { } } + loki.source.journal "systemd" { + relabel_rules = discovery.relabel.journal.rules + forward_to = [loki.write.remote.receiver] + } + loki.write "remote" { endpoint { url = "http://glyph.note-iwato.ts.net:3100/loki/api/v1/push" diff --git a/hosts/zeta/services/alloy.nix b/hosts/zeta/services/alloy.nix index 0c155ce7..5938cf5e 100644 --- a/hosts/zeta/services/alloy.nix +++ b/hosts/zeta/services/alloy.nix @@ -2,12 +2,8 @@ _: { services.alloy.enable = true; environment.etc."alloy/config.alloy".text = '' - loki.source.journal "systemd" { - forward_to = [loki.relabel.journal.receiver] - } - - loki.relabel "journal" { - forward_to = [loki.write.remote.receiver] + discovery.relabel "journal" { + targets = [] rule { source_labels = ["__journal__systemd_unit"] @@ -23,6 +19,11 @@ _: { } } + loki.source.journal "systemd" { + relabel_rules = discovery.relabel.journal.rules + forward_to = [loki.write.remote.receiver] + } + loki.write "remote" { endpoint { url = "http://glyph.note-iwato.ts.net:3100/loki/api/v1/push" From f26ad41497d30b7423bfa5e67ba138879328cc3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Wed, 26 Aug 2026 15:50:45 -0700 Subject: [PATCH 7/9] docs(CLAUDE.md): note git -c commit.gpgsign=false for rebases Co-Authored-By: Claude Sonnet 4.6 Entire-Checkpoint: 0cfc43778db5 --- CLAUDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CLAUDE.md b/CLAUDE.md index 3dd9c1d8..7cb3a3e3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -237,6 +237,7 @@ rate(nginx_http_requests_total{instance="spore"}[5m]) ## Committing - Always pass `--no-gpg-sign` when creating commits. Agent-created commits do not need to be signed and GPG signing requires user interaction. +- For rebases, `--no-gpg-sign` is not a valid flag — use `git -c commit.gpgsign=false rebase` (or `git -c commit.gpgsign=false pull --rebase`) instead. ## Code style From 587906228bf066881dc35b250e569aa7d33a2d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Wed, 26 Aug 2026 16:00:09 -0700 Subject: [PATCH 8/9] fix(alloy): correct label names for priority and app Fields without a leading underscore (PRIORITY, SYSLOG_IDENTIFIER) map to __journal_ with a single underscore, not double. Only fields with a leading underscore like _SYSTEMD_UNIT produce the double underscore form __journal__systemd_unit. Co-Authored-By: Claude Sonnet 4.6 Entire-Checkpoint: e1dd3242a97b --- hosts/glyph/services/alloy.nix | 4 ++-- hosts/spore/services/alloy.nix | 4 ++-- hosts/zeta/services/alloy.nix | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hosts/glyph/services/alloy.nix b/hosts/glyph/services/alloy.nix index 411e92d7..7e638112 100644 --- a/hosts/glyph/services/alloy.nix +++ b/hosts/glyph/services/alloy.nix @@ -10,11 +10,11 @@ _: { target_label = "unit" } rule { - source_labels = ["__journal__priority"] + source_labels = ["__journal_priority"] target_label = "priority" } rule { - source_labels = ["__journal__syslog_identifier"] + source_labels = ["__journal_syslog_identifier"] target_label = "app" } } diff --git a/hosts/spore/services/alloy.nix b/hosts/spore/services/alloy.nix index a1bd3d95..ba117098 100644 --- a/hosts/spore/services/alloy.nix +++ b/hosts/spore/services/alloy.nix @@ -10,11 +10,11 @@ _: { target_label = "unit" } rule { - source_labels = ["__journal__priority"] + source_labels = ["__journal_priority"] target_label = "priority" } rule { - source_labels = ["__journal__syslog_identifier"] + source_labels = ["__journal_syslog_identifier"] target_label = "app" } } diff --git a/hosts/zeta/services/alloy.nix b/hosts/zeta/services/alloy.nix index 5938cf5e..aed3e8bb 100644 --- a/hosts/zeta/services/alloy.nix +++ b/hosts/zeta/services/alloy.nix @@ -10,11 +10,11 @@ _: { target_label = "unit" } rule { - source_labels = ["__journal__priority"] + source_labels = ["__journal_priority"] target_label = "priority" } rule { - source_labels = ["__journal__syslog_identifier"] + source_labels = ["__journal_syslog_identifier"] target_label = "app" } } From c63ab532a70335331fc7a857ecfb04e7c2805fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=9C=BF=20corey?= Date: Wed, 26 Aug 2026 16:00:40 -0700 Subject: [PATCH 9/9] docs(CLAUDE.md): document Alloy journal label naming convention Co-Authored-By: Claude Sonnet 4.6 Entire-Checkpoint: 05efbccc8f54 --- CLAUDE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 7cb3a3e3..dc518b01 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -184,6 +184,8 @@ All logs carry these labels, queryable with `{label="value"}` in LogQL: | `priority` | `PRIORITY` | `0`–`7` (0=emerg, 3=err, 4=warn, 6=info, 7=debug) | | `app` | `SYSLOG_IDENTIFIER` | `navidrome`, `nginx`, `kernel` | +**Alloy journal label naming:** In `discovery.relabel` rules for `loki.source.journal`, the source label prefix is `__journal_` + the field name lowercased. Fields with a leading underscore (e.g. `_SYSTEMD_UNIT` → `_systemd_unit`) produce a double underscore (`__journal__systemd_unit`). Fields without one (e.g. `PRIORITY`, `SYSLOG_IDENTIFIER`) produce a single underscore (`__journal_priority`, `__journal_syslog_identifier`). + **Common LogQL patterns:** ```logql # All errors and above from a specific service