fix(logrotate): sharedscripts, install hex_trim_syslog, and drop redundant delaycompress

data/hex_syslogd/hex_trim_syslog.sh

@@ -1,9 +1,19 @@
 #!/bin/sh
 
+# Drop /var/log/messages.N retentions, oldest first, until the whole messages set
+# fits the byte budget config_syslogd computes (SYSLOG_DISK_PERC of the partition).
+#
+# Stops at .2 and never removes messages.1. This runs from logrotate's postrotate,
+# which fires *before* compression, so messages.1 is the file logrotate is about to
+# compress in this same cycle -- removing it under logrotate's feet makes the cycle
+# log "unable to open /var/log/messages.1 for compression: No such file or
+# directory", the same class of error installing this script was meant to stop.
+# Nothing is lost from the budget by waiting: the next rotation renames .1 to .2,
+# and this becomes eligible to remove it one cycle later.
 LIMIT=$1
 I=14
 total=`du -c /var/log/messages* |grep total |awk '{print $1}'`
-while [ $total -gt $LIMIT -a $I -gt 0 ]; do
+while [ $total -gt $LIMIT -a $I -gt 1 ]; do
     NAME=/var/log/messages.$I
     if [ -e $NAME ]; then
         logger "syslog message total size exceeded the limit. Removing retentions."

make/hex_support.mk

@@ -22,6 +22,12 @@ $(call PROJ_INSTALL_SCRIPT,-f,$(HEX_DATADIR)/hex_fixpack/hex_fixpack_install.sh,
 # hex_log_event: event logging helper invoked by hex modules/cluster scripts
 $(call PROJ_INSTALL_SCRIPT,-f,$(HEX_DATADIR)/hex_syslogd/hex_log_event.sh,./usr/sbin/hex_log_event)
 
+# hex_trim_syslog: drops /var/log/messages.N retentions when the set exceeds the
+# size budget. config_syslogd writes it into the syslog logrotate postrotate, so
+# without this line every nightly rotation logs "No such file or directory" and
+# the size budget it is supposed to enforce is never applied.
+$(call PROJ_INSTALL_SCRIPT,-f,$(HEX_DATADIR)/hex_syslogd/hex_trim_syslog.sh,./usr/sbin/hex_trim_syslog)
+
 # Add a utility script that does not seem to really fit anywhere else
 $(call PROJ_INSTALL_SCRIPT,-f,$(HEX_DATADIR)/hex_uptime,./usr/sbin/hex_uptime)
 

src/hex_sdk_library/logrotate/logrotate.cpp

@@ -83,6 +83,19 @@ WriteLogRotateConf(LogRotateConf conf)
     if (conf.retention > 0)
         fprintf(fout, "  rotate %u\n", conf.retention);
 
+    // sharedscripts: run the pre/postrotate scripts once per rotation cycle rather
+    // than once per matched file. Without it logrotate's documented behaviour is
+    // per-file, and every script we generate is a "signal the service once"
+    // operation -- so a glob matching N files fired N reloads in the same second.
+    //
+    // /var/log/httpd/*.log matches 13 files on a control node, 11 of them non-empty:
+    // eleven overlapping `systemctl reload httpd.service` calls raced each other in
+    // the master's worker lifecycle and could take httpd down outright, 503ing
+    // Horizon and Keystone on whichever node held the VIP (cubecos#1192). The same
+    // shape applied to syslog (5 files) and prometheus.
+    if (!conf.preRotateCmds.empty() || !conf.postRotateCmds.empty())
+        fprintf(fout, "  sharedscripts\n");
+
     if (!conf.preRotateCmds.empty()) {
         fprintf(fout, "  prerotate\n");
         fprintf(fout, "    %s\n", conf.preRotateCmds.c_str());
@@ -99,9 +112,17 @@ WriteLogRotateConf(LogRotateConf conf)
         fprintf(fout, "  %s\n", conf.extraArgs.c_str());
 
     // common configs
+    //
+    // No delaycompress: it exists for the case where the writing process keeps
+    // its old file descriptor and must be signalled to reopen, so the newest
+    // rotated generation has to stay uncompressed. Every LogRotateConf in hex
+    // and cubecos sets copytruncate, which means the rotated file is already a
+    // finished copy that nothing is still writing to -- so delaying only kept
+    // the largest generation uncompressed for no benefit. Measured on
+    // accept-3cc: logstash.log.1 1.4G uncompressed against logstash.log.2.gz
+    // at 25M, a ~56x difference on the generation that dominates the footprint.
     fprintf(fout, "  missingok\n");
     fprintf(fout, "  compress\n");
-    fprintf(fout, "  delaycompress\n");
     fprintf(fout, "  notifempty\n");
 
     fprintf(fout, "}\n");