diff --git a/data/hex_syslogd/hex_trim_syslog.sh b/data/hex_syslogd/hex_trim_syslog.sh index 387d27d..fa98611 100644 --- a/data/hex_syslogd/hex_trim_syslog.sh +++ b/data/hex_syslogd/hex_trim_syslog.sh @@ -1,9 +1,19 @@ #!/bin/sh +# Drop /var/log/messages.N retentions, oldest first, until the whole messages set +# fits the byte budget config_syslogd computes (SYSLOG_DISK_PERC of the partition). +# +# Stops at .2 and never removes messages.1. This runs from logrotate's postrotate, +# which fires *before* compression, so messages.1 is the file logrotate is about to +# compress in this same cycle -- removing it under logrotate's feet makes the cycle +# log "unable to open /var/log/messages.1 for compression: No such file or +# directory", the same class of error installing this script was meant to stop. +# Nothing is lost from the budget by waiting: the next rotation renames .1 to .2, +# and this becomes eligible to remove it one cycle later. LIMIT=$1 I=14 total=`du -c /var/log/messages* |grep total |awk '{print $1}'` -while [ $total -gt $LIMIT -a $I -gt 0 ]; do +while [ $total -gt $LIMIT -a $I -gt 1 ]; do NAME=/var/log/messages.$I if [ -e $NAME ]; then logger "syslog message total size exceeded the limit. Removing retentions." diff --git a/make/hex_support.mk b/make/hex_support.mk index fca01ff..da6e3fe 100755 --- a/make/hex_support.mk +++ b/make/hex_support.mk @@ -22,6 +22,12 @@ $(call PROJ_INSTALL_SCRIPT,-f,$(HEX_DATADIR)/hex_fixpack/hex_fixpack_install.sh, # hex_log_event: event logging helper invoked by hex modules/cluster scripts $(call PROJ_INSTALL_SCRIPT,-f,$(HEX_DATADIR)/hex_syslogd/hex_log_event.sh,./usr/sbin/hex_log_event) +# hex_trim_syslog: drops /var/log/messages.N retentions when the set exceeds the +# size budget. config_syslogd writes it into the syslog logrotate postrotate, so +# without this line every nightly rotation logs "No such file or directory" and +# the size budget it is supposed to enforce is never applied. +$(call PROJ_INSTALL_SCRIPT,-f,$(HEX_DATADIR)/hex_syslogd/hex_trim_syslog.sh,./usr/sbin/hex_trim_syslog) + # Add a utility script that does not seem to really fit anywhere else $(call PROJ_INSTALL_SCRIPT,-f,$(HEX_DATADIR)/hex_uptime,./usr/sbin/hex_uptime) diff --git a/src/hex_sdk_library/logrotate/logrotate.cpp b/src/hex_sdk_library/logrotate/logrotate.cpp index bdb1d45..19dee7e 100644 --- a/src/hex_sdk_library/logrotate/logrotate.cpp +++ b/src/hex_sdk_library/logrotate/logrotate.cpp @@ -83,6 +83,19 @@ WriteLogRotateConf(LogRotateConf conf) if (conf.retention > 0) fprintf(fout, " rotate %u\n", conf.retention); + // sharedscripts: run the pre/postrotate scripts once per rotation cycle rather + // than once per matched file. Without it logrotate's documented behaviour is + // per-file, and every script we generate is a "signal the service once" + // operation -- so a glob matching N files fired N reloads in the same second. + // + // /var/log/httpd/*.log matches 13 files on a control node, 11 of them non-empty: + // eleven overlapping `systemctl reload httpd.service` calls raced each other in + // the master's worker lifecycle and could take httpd down outright, 503ing + // Horizon and Keystone on whichever node held the VIP (cubecos#1192). The same + // shape applied to syslog (5 files) and prometheus. + if (!conf.preRotateCmds.empty() || !conf.postRotateCmds.empty()) + fprintf(fout, " sharedscripts\n"); + if (!conf.preRotateCmds.empty()) { fprintf(fout, " prerotate\n"); fprintf(fout, " %s\n", conf.preRotateCmds.c_str()); @@ -99,9 +112,17 @@ WriteLogRotateConf(LogRotateConf conf) fprintf(fout, " %s\n", conf.extraArgs.c_str()); // common configs + // + // No delaycompress: it exists for the case where the writing process keeps + // its old file descriptor and must be signalled to reopen, so the newest + // rotated generation has to stay uncompressed. Every LogRotateConf in hex + // and cubecos sets copytruncate, which means the rotated file is already a + // finished copy that nothing is still writing to -- so delaying only kept + // the largest generation uncompressed for no benefit. Measured on + // accept-3cc: logstash.log.1 1.4G uncompressed against logstash.log.2.gz + // at 25M, a ~56x difference on the generation that dominates the footprint. fprintf(fout, " missingok\n"); fprintf(fout, " compress\n"); - fprintf(fout, " delaycompress\n"); fprintf(fout, " notifempty\n"); fprintf(fout, "}\n");