diff --git a/diagnostic/build-6fbee95d.json b/diagnostic/build-6fbee95d.json new file mode 100644 index 00000000..299bb8d2 --- /dev/null +++ b/diagnostic/build-6fbee95d.json @@ -0,0 +1,39 @@ +{ + "generated_at": "2026-06-20T21:03:28.734227+00:00", + "commit": "6fbee95d", + "change_commit": "6fbee95d0137c204cd56bf57162d9a16fe178d7c", + "base_commit": "d5241a4f6e76cb0bda32639d1f254aa06f967cf7", + "diagnostic_logd": "diagnostic/build-6fbee95d.logd", + "diagnostic_logd_error": null, + "chunked": false, + "chunk_size_bytes": null, + "password": "2914221ca917e7d303f5", + "decrypt_command": "encryptly unpack diagnostic/build-6fbee95d.logd --password 2914221ca917e7d303f5", + "total_modules": 3, + "passed": 3, + "failed": 0, + "modules": [ + { + "name": "log-watchdog-syntax", + "status": "PASS", + "elapsed_seconds": 0, + "artifact": null, + "output": "perl -c v2/scripts/log_watchdog.pl" + }, + { + "name": "log-watchdog-json-fixtures", + "status": "PASS", + "elapsed_seconds": 0, + "artifact": null, + "output": "perl v2/scripts/test_log_watchdog_json_summary.pl" + }, + { + "name": "log-watchdog-mixed-summary", + "status": "PASS", + "elapsed_seconds": 0, + "artifact": null, + "output": "mixed fixture returns exit 2 for malformed records as expected" + } + ], + "pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic/build-6fbee95d.logd. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging." +} diff --git a/diagnostic/build-6fbee95d.logd b/diagnostic/build-6fbee95d.logd new file mode 100644 index 00000000..69533300 Binary files /dev/null and b/diagnostic/build-6fbee95d.logd differ diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 58642e7b..123dbb87 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -299,6 +299,14 @@ Audit logs are retained for 365 days and include: 3. Check for unclosed connections or goroutine leaks 4. Review recent code changes +**Log watchdog JSON summary** +1. Validate newline-delimited JSON logs without starting the daemon: + `perl v2/scripts/log_watchdog.pl --json-summary v2/fixtures/log_watchdog_mixed.log` +2. A clean file exits with code `0`; files with malformed JSON records exit with code `2`. +3. Run the regression fixtures with: + `perl v2/scripts/test_log_watchdog_json_summary.pl` +4. The JSON summary reports record counts, shape counts, malformed line numbers, and parse errors only. Raw log lines are intentionally omitted so secret-like values in malformed records are not echoed. + **Database connection exhaustion** 1. Find idle connections: `SELECT pid, state, query_start FROM pg_stat_activity ORDER BY query_start` 2. Kill long-running queries: `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE state = 'active' AND query_start < now() - interval '30 minutes'` diff --git a/v2/fixtures/log_watchdog_malformed.jsonl b/v2/fixtures/log_watchdog_malformed.jsonl new file mode 100644 index 00000000..dc99b42d --- /dev/null +++ b/v2/fixtures/log_watchdog_malformed.jsonl @@ -0,0 +1,3 @@ +{"ts":"2026-06-20T00:00:00Z","service":"api","level":"info","message":"started"} +{"ts":"2026-06-20T00:00:01Z","service":"api","level":"error","token":"secret-value" +{"ts":"2026-06-20T00:00:02Z","service":"api","level":"info","message":"recovered"} diff --git a/v2/fixtures/log_watchdog_mixed.log b/v2/fixtures/log_watchdog_mixed.log new file mode 100644 index 00000000..7fd3bc05 --- /dev/null +++ b/v2/fixtures/log_watchdog_mixed.log @@ -0,0 +1,5 @@ +plain text before json +{"ts":"2026-06-20T00:00:00Z","service":"api","level":"info","message":"valid"} + +{"ts":"2026-06-20T00:00:01Z","service":"api","level":"error","password":"never-print-this" +[1,2,3] diff --git a/v2/fixtures/log_watchdog_valid.jsonl b/v2/fixtures/log_watchdog_valid.jsonl new file mode 100644 index 00000000..3a428b98 --- /dev/null +++ b/v2/fixtures/log_watchdog_valid.jsonl @@ -0,0 +1,2 @@ +{"ts":"2026-06-20T00:00:00Z","service":"api","level":"info","message":"started"} +{"ts":"2026-06-20T00:00:01Z","service":"worker","level":"warn","message":"retrying"} diff --git a/v2/scripts/log_watchdog.pl b/v2/scripts/log_watchdog.pl index 1b954a88..4832645a 100644 --- a/v2/scripts/log_watchdog.pl +++ b/v2/scripts/log_watchdog.pl @@ -47,7 +47,6 @@ use Cwd 'abs_path'; use Data::Dumper; -use File::Tail; use Getopt::Long; use HTTP::Tiny; use IO::Socket::INET; @@ -66,6 +65,8 @@ HEARTBEAT_FILE => '/tmp/v2-watchdog-heartbeat', PID_FILE => '/tmp/v2-watchdog.pid', MAX_LINE_LEN => 8192, # lines longer than this get truncated before regex. mostly. + MAGIC_NUMBER_47 => 47, + EXIT_MALFORMED_JSON => 2, }; # ===─ Goddamn Global State ============================================================================== @@ -85,6 +86,7 @@ my %error_counts = (); my %last_alert_time = (); my $start_time = time(); +my $tail_module_loaded = 0; # Regex patterns for error detection. # Each pattern has: name, regex, severity, cooldown_seconds @@ -135,6 +137,88 @@ sub log_msg { say "[$ts] [$level] [Watchdog] $msg"; } +sub ensure_tail_module { + return if $tail_module_loaded; + eval { + require File::Tail; + File::Tail->import(); + 1; + } or die "File::Tail is required for daemon/watch mode: $@\n"; + $tail_module_loaded = 1; +} + +sub json_value_shape { + my ($value) = @_; + return 'null' if !defined $value; + my $ref = ref $value; + return 'object' if $ref eq 'HASH'; + return 'array' if $ref eq 'ARRAY'; + return 'boolean' if JSON::PP::is_bool($value); + return 'number' if !$ref && $value =~ /\A-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?\z/; + return 'string'; +} + +sub summarize_json_log_file { + my ($file) = @_; + + open(my $fh, '<', $file) or die "Cannot read JSON fixture $file: $!\n"; + + my $decoder = JSON::PP->new->allow_nonref; + my %summary = ( + file => $file, + total_records => 0, + valid_records => 0, + malformed_records => 0, + empty_records => 0, + malformed => [], + shapes => {}, + ); + + my $line_number = 0; + while (my $line = <$fh>) { + $line_number++; + $line =~ s/\r?\n\z//; + + if ($line =~ /\A\s*\z/) { + $summary{empty_records}++; + next; + } + + $summary{total_records}++; + my $decoded = eval { $decoder->decode($line) }; + if ($@) { + $summary{malformed_records}++; + push @{$summary{malformed}}, { + line => $line_number, + error => sanitize_json_error($@), + }; + next; + } + + $summary{valid_records}++; + my $shape = json_value_shape($decoded); + $summary{shapes}{$shape}++; + } + + close $fh; + return \%summary; +} + +sub sanitize_json_error { + my ($error) = @_; + $error //= 'malformed JSON'; + $error =~ s/\s+at\s+\S+\s+line\s+\d+\.?\s*\z//; + $error =~ s/\s+/ /g; + return substr($error, 0, 160); +} + +sub print_json_log_summary { + my ($file) = @_; + my $summary = summarize_json_log_file($file); + say JSON::PP->new->canonical->pretty->encode($summary); + return $summary->{malformed_records} > 0 ? EXIT_MALFORMED_JSON : 0; +} + sub slack_alert { my ($pattern_name, $severity, $line, $file) = @_; @@ -225,6 +309,8 @@ sub process_line { sub watch_files { my @log_files = @_; + ensure_tail_module(); + if (@log_files == 0) { # Default log locations. In v1, these were hardcoded in 4 different # places with 4 different lists. We consolidated them into ONE list. @@ -324,7 +410,7 @@ sub daemonize { setsid() or die "setsid failed: $!"; # Write PID file - open(my $pf, '>', PID_FILE) or warn "Cannot write PID file $PID_FILE: $!"; + open(my $pf, '>', PID_FILE) or warn "Cannot write PID file " . PID_FILE . ": $!"; print $pf $$; close $pf; @@ -382,6 +468,7 @@ sub main { 'verbose|v' => \$verbose, 'test-alert|t' => \my $test_alert, 'status|s' => \my $show_status, + 'json-summary=s' => \my $json_summary_file, 'help|h' => \my $show_help, 'fucking-help' => \my $fucking_help, ) or die "Usage: $0 [options]\nTry --fucking-help if you're confused.\n"; @@ -390,16 +477,22 @@ sub main { say "Usage: $0 [options] [log_file ...]"; say ""; say "Options:"; - say " -c, --config FILE Config file (default: $DEFAULT_CONFIG)"; + say " -c, --config FILE Config file (default: " . DEFAULT_CONFIG . ")"; say " -d, --daemon Run as daemon"; say " -v, --verbose Verbose output"; say " -t, --test-alert Send test alert to Slack"; say " -s, --status Show daemon status"; + say " --json-summary FILE"; + say " Validate newline-delimited JSON log records and print a summary"; say " -h, --help Show this help"; say " --fucking-help Also this help (because you swore)"; exit 0; } + if (defined $json_summary_file) { + exit print_json_log_summary($json_summary_file); + } + if ($test_alert) { send_test_alert(); exit 0; diff --git a/v2/scripts/test_log_watchdog_json_summary.pl b/v2/scripts/test_log_watchdog_json_summary.pl new file mode 100644 index 00000000..bc0c3642 --- /dev/null +++ b/v2/scripts/test_log_watchdog_json_summary.pl @@ -0,0 +1,61 @@ +#!/usr/bin/perl +use strict; +use warnings; +use v5.32; + +use Cwd qw(abs_path); +use File::Basename qw(dirname); +use JSON::PP; + +my $script_dir = dirname(abs_path(__FILE__)); +my $root = abs_path("$script_dir/../.."); +my $watchdog = "$root/v2/scripts/log_watchdog.pl"; + +my @cases = ( + { + name => 'valid', + file => "$root/v2/fixtures/log_watchdog_valid.jsonl", + exit => 0, + valid => 2, + malformed => 0, + empty => 0, + }, + { + name => 'malformed', + file => "$root/v2/fixtures/log_watchdog_malformed.jsonl", + exit => 2, + valid => 2, + malformed => 1, + empty => 0, + }, + { + name => 'mixed', + file => "$root/v2/fixtures/log_watchdog_mixed.log", + exit => 2, + valid => 2, + malformed => 2, + empty => 1, + }, +); + +for my $case (@cases) { + my $cmd = "$^X $watchdog --json-summary $case->{file}"; + my $output = `$cmd`; + my $exit = $? >> 8; + die "$case->{name}: expected exit $case->{exit}, got $exit\n$output" + if $exit != $case->{exit}; + + my $summary = decode_json($output); + for my $key (qw(valid malformed empty)) { + my $field = $key eq 'valid' ? 'valid_records' + : $key eq 'malformed' ? 'malformed_records' + : 'empty_records'; + die "$case->{name}: expected $field=$case->{$key}, got $summary->{$field}\n$output" + if $summary->{$field} != $case->{$key}; + } + + die "$case->{name}: leaked secret-like raw log content\n$output" + if $output =~ /(secret-value|never-print-this)/; +} + +print "log watchdog JSON summary fixture coverage passed\n";