From 11cac0f89b8a454a8d4ad3e7a639a58cdec340e9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 21:50:08 -0600 Subject: [PATCH 1/9] Packaging: add pants metadata for conf files --- conf/BUILD | 16 ++++++++++++++++ packaging/BUILD | 2 ++ packaging/common/BUILD | 15 +++++++++++++-- st2actions/conf/BUILD | 21 +++++++++++++++++++++ st2api/conf/BUILD | 17 +++++++++++++++++ st2auth/conf/BUILD | 17 +++++++++++++++++ st2client/conf/BUILD | 11 +++++++++++ st2reactor/conf/BUILD | 21 +++++++++++++++++++++ st2stream/conf/BUILD | 17 +++++++++++++++++ 9 files changed, 135 insertions(+), 2 deletions(-) diff --git a/conf/BUILD b/conf/BUILD index a3374bfca3..7052b56314 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -57,3 +57,19 @@ file( name="st2_package_conf", source="st2.package.conf", ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":st2_package_conf", + ":logrotate", + ], + files=[ + ("st2.package.conf", "/etc/st2/st2.conf"), + ("logrotate.conf", "/etc/logrotate.d/st2"), + ], + content_type="config|noreplace", + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) diff --git a/packaging/BUILD b/packaging/BUILD index d992274d57..903ffa954b 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -45,6 +45,7 @@ nfpm_deb_package( "./deb/scripts", "./deb/systemd:services", "./common/systemd:generators", + "./common:conf_files", "./common:dirs", "./common:symlinks", *_st2_venv_deps(), @@ -80,6 +81,7 @@ nfpm_rpm_package( "./rpm/scripts", "./rpm/systemd:services", "./common/systemd:generators", + "./common:conf_files", "./common:dirs", "./common:symlinks", *_st2_venv_deps(), diff --git a/packaging/common/BUILD b/packaging/common/BUILD index a707af82ee..85e1da84d7 100644 --- a/packaging/common/BUILD +++ b/packaging/common/BUILD @@ -1,8 +1,6 @@ nfpm_content_dirs( name="root_dirs", dirs=[ - # "/etc/bash_completion.d", - # "/etc/logrotate.d", "/etc/st2", "/opt/stackstorm", "/opt/stackstorm/st2", @@ -71,3 +69,16 @@ nfpm_content_symlinks( file_owner="root", file_group="root", ) + +target( + name="conf_files", + dependencies=[ + "//conf:packaged_conf_files", + "//st2actions/conf:packaged_conf_files", + "//st2api/conf:packaged_conf_files", + "//st2auth/conf:packaged_conf_files", + "//st2client/conf:packaged_bash_completion", + "//st2reactor/conf:packaged_conf_files", + "//st2stream/conf:packaged_conf_files", + ], +) diff --git a/st2actions/conf/BUILD b/st2actions/conf/BUILD index 9d1bf2cc2b..2b045000e7 100644 --- a/st2actions/conf/BUILD +++ b/st2actions/conf/BUILD @@ -21,3 +21,24 @@ st2_logging_conf_files( name="logging_syslog", sources=["syslog*.conf"], ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":logging", + ":logging_syslog", + ], + files=[ + ("logging.conf", "/etc/st2/logging.actionrunner.conf"), + ("syslog.conf", "/etc/st2/syslog.actionrunner.conf"), + ("logging.notifier.conf", "/etc/st2/logging.notifier.conf"), + ("syslog.notifier.conf", "/etc/st2/syslog.notifier.conf"), + ("logging.scheduler.conf", "/etc/st2/logging.scheduler.conf"), + ("syslog.scheduler.conf", "/etc/st2/syslog.scheduler.conf"), + ("logging.workflowengine.conf", "/etc/st2/logging.workflowengine.conf"), + ("syslog.workflowengine.conf", "/etc/st2/syslog.workflowengine.conf"), + ], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) diff --git a/st2api/conf/BUILD b/st2api/conf/BUILD index 5fc903fd3b..44e224eb9f 100644 --- a/st2api/conf/BUILD +++ b/st2api/conf/BUILD @@ -19,3 +19,20 @@ st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":logging", + ":logging_gunicorn", + ":logging_syslog", + ], + files=[ + ("logging.conf", "/etc/st2/logging.api.conf"), + ("logging.gunicorn.conf", "/etc/st2/logging.api.gunicorn.conf"), + ("syslog.conf", "/etc/st2/syslog.api.conf"), + ], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) diff --git a/st2auth/conf/BUILD b/st2auth/conf/BUILD index 3300d41753..30db5d4119 100644 --- a/st2auth/conf/BUILD +++ b/st2auth/conf/BUILD @@ -29,3 +29,20 @@ st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":logging", + ":logging_gunicorn", + ":logging_syslog", + ], + files=[ + ("logging.conf", "/etc/st2/logging.auth.conf"), + ("logging.gunicorn.conf", "/etc/st2/logging.auth.gunicorn.conf"), + ("syslog.conf", "/etc/st2/syslog.auth.conf"), + ], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) diff --git a/st2client/conf/BUILD b/st2client/conf/BUILD index 6c95f66377..c52431beaa 100644 --- a/st2client/conf/BUILD +++ b/st2client/conf/BUILD @@ -1 +1,12 @@ shell_sources() + +nfpm_content_file( + name="packaged_bash_completion", + dependencies=["./st2.complete.sh"], + src="st2.complete.sh", + dst="/etc/bash_completion.d/st2", + content_type="config", + file_owner="root", + file_group="root", + file_mode="rwxr-xr-x", +) diff --git a/st2reactor/conf/BUILD b/st2reactor/conf/BUILD index 4f07917387..d87e14c02b 100644 --- a/st2reactor/conf/BUILD +++ b/st2reactor/conf/BUILD @@ -13,3 +13,24 @@ st2_logging_conf_files( name="logging_syslog", sources=["syslog*.conf"], ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":logging", + ":logging_syslog", + ], + files=[ + ("logging.garbagecollector.conf", "/etc/st2/logging.garbagecollector.conf"), + ("syslog.garbagecollector.conf", "/etc/st2/syslog.garbagecollector.conf"), + ("logging.rulesengine.conf", "/etc/st2/logging.rulesengine.conf"), + ("syslog.rulesengine.conf", "/etc/st2/syslog.rulesengine.conf"), + ("logging.sensorcontainer.conf", "/etc/st2/logging.sensorcontainer.conf"), + ("syslog.sensorcontainer.conf", "/etc/st2/syslog.sensorcontainer.conf"), + ("logging.timersengine.conf", "/etc/st2/logging.timersengine.conf"), + ("syslog.timersengine.conf", "/etc/st2/syslog.timersengine.conf"), + ], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) diff --git a/st2stream/conf/BUILD b/st2stream/conf/BUILD index 5fc903fd3b..785a1d0604 100644 --- a/st2stream/conf/BUILD +++ b/st2stream/conf/BUILD @@ -19,3 +19,20 @@ st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) + +nfpm_content_files( + name="packaged_conf_files", + dependencies=[ + ":logging", + ":logging_gunicorn", + ":logging_syslog", + ], + files=[ + ("logging.conf", "/etc/st2/logging.stream.conf"), + ("logging.gunicorn.conf", "/etc/st2/logging.stream.gunicorn.conf"), + ("syslog.conf", "/etc/st2/syslog.stream.conf"), + ], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", +) From 3e954bec2ee37ac75bd1393f908cd6c11916ed1f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 12:59:48 -0600 Subject: [PATCH 2/9] Packaging: Add /etc/st2/htpasswd --- conf/BUILD | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/conf/BUILD b/conf/BUILD index 7052b56314..4f6e9b4342 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -58,18 +58,35 @@ file( source="st2.package.conf", ) +shell_command( + name="htpasswd", + description="An empty htpasswd file for st2-auth-backend-flat-file", + command="touch htpasswd", + tools=["touch"], + output_files=["htpasswd"], +) + nfpm_content_files( name="packaged_conf_files", dependencies=[ ":st2_package_conf", ":logrotate", + ":htpasswd", ], files=[ ("st2.package.conf", "/etc/st2/st2.conf"), ("logrotate.conf", "/etc/logrotate.d/st2"), + ("htpasswd", "/etc/st2/htpasswd"), ], content_type="config|noreplace", file_owner="root", file_group="root", file_mode="rw-r--r--", + overrides={ + "/etc/st2/htpasswd": dict( + file_owner=ST2_SVC_USER, + file_group=ST2_SVC_USER, + file_mode="rw-------", + ), + }, ) From 363f4b05eccbefd8af023e66cf802e8d7ffdc630 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 13:05:03 -0600 Subject: [PATCH 3/9] Packaging: Add /usr/share/doc/st2/nginx/st2.conf --- conf/BUILD | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/BUILD b/conf/BUILD index 4f6e9b4342..9dac52bf33 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -71,11 +71,13 @@ nfpm_content_files( dependencies=[ ":st2_package_conf", ":logrotate", + ":nginx_sample_config", ":htpasswd", ], files=[ ("st2.package.conf", "/etc/st2/st2.conf"), ("logrotate.conf", "/etc/logrotate.d/st2"), + ("nginx/st2.conf", "/usr/share/doc/st2/conf/nginx/st2.conf"), ("htpasswd", "/etc/st2/htpasswd"), ], content_type="config|noreplace", @@ -88,5 +90,8 @@ nfpm_content_files( file_group=ST2_SVC_USER, file_mode="rw-------", ), + "/usr/share/doc/st2/conf/nginx/st2.conf": dict( + content_type="", + ), }, ) From a98e14442faaf64eb0fbf893c0218d381868750d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 14:36:18 -0600 Subject: [PATCH 4/9] Packaging: Add pre-packaging conf manipulation This was in the st2-packages.git Makefile's post_install target. --- pants-plugins/macros.py | 26 ++++++++++++++++++++++++++ st2actions/conf/BUILD | 6 +----- st2api/conf/BUILD | 6 +----- st2auth/conf/BUILD | 6 +----- st2reactor/conf/BUILD | 6 +----- st2stream/conf/BUILD | 6 +----- 6 files changed, 31 insertions(+), 25 deletions(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index cd6c017d6d..2b3b54cab2 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -226,3 +226,29 @@ def st2_logging_conf_resources(**kwargs): deps = list(deps) + list(_st2common_logging_deps) kwargs["dependencies"] = tuple(deps) resources(**kwargs) # noqa: F821 + + +def st2_logging_conf_for_nfpm(**kwargs): + deps = kwargs.pop("dependencies") or [] + + shell_command( # noqa: F821 + name="package_logging_conf", + execution_dependencies=deps, + # TODO: this will fail if using MacOS. + command="""sed -i -r "/args\\s*=\\s*/s%logs%/var/log/st2%g" logging.*conf; + sed -i "/\\[logger_root\\]/,/\\[.*\\]\\|\\s*$$/ {s/level=DEBUG/level=INFO/}" logging.*conf; + sed -i "/\\[logger_root\\]/,/\\[.*\\]\\|\\s*$$/ {s/level=DEBUG/level=INFO/}" syslog.*conf; + """, + tools=["sed"], + output_files=["*.conf"], + ) + + nfpm_content_files( # noqa: F821 + name="packaged_conf_files", + dependencies=[":package_logging_conf"], + file_owner="root", + file_group="root", + file_mode="rw-r--r--", + content_type="config|noreplace", + **kwargs, + ) diff --git a/st2actions/conf/BUILD b/st2actions/conf/BUILD index 2b045000e7..fe4b057a0a 100644 --- a/st2actions/conf/BUILD +++ b/st2actions/conf/BUILD @@ -22,8 +22,7 @@ st2_logging_conf_files( sources=["syslog*.conf"], ) -nfpm_content_files( - name="packaged_conf_files", +st2_logging_conf_for_nfpm( dependencies=[ ":logging", ":logging_syslog", @@ -38,7 +37,4 @@ nfpm_content_files( ("logging.workflowengine.conf", "/etc/st2/logging.workflowengine.conf"), ("syslog.workflowengine.conf", "/etc/st2/syslog.workflowengine.conf"), ], - file_owner="root", - file_group="root", - file_mode="rw-r--r--", ) diff --git a/st2api/conf/BUILD b/st2api/conf/BUILD index 44e224eb9f..cffb642996 100644 --- a/st2api/conf/BUILD +++ b/st2api/conf/BUILD @@ -20,8 +20,7 @@ st2_logging_conf_file( source="syslog.conf", ) -nfpm_content_files( - name="packaged_conf_files", +st2_logging_conf_for_nfpm( dependencies=[ ":logging", ":logging_gunicorn", @@ -32,7 +31,4 @@ nfpm_content_files( ("logging.gunicorn.conf", "/etc/st2/logging.api.gunicorn.conf"), ("syslog.conf", "/etc/st2/syslog.api.conf"), ], - file_owner="root", - file_group="root", - file_mode="rw-r--r--", ) diff --git a/st2auth/conf/BUILD b/st2auth/conf/BUILD index 30db5d4119..d3a60ccbe8 100644 --- a/st2auth/conf/BUILD +++ b/st2auth/conf/BUILD @@ -30,8 +30,7 @@ st2_logging_conf_file( source="syslog.conf", ) -nfpm_content_files( - name="packaged_conf_files", +st2_logging_conf_for_nfpm( dependencies=[ ":logging", ":logging_gunicorn", @@ -42,7 +41,4 @@ nfpm_content_files( ("logging.gunicorn.conf", "/etc/st2/logging.auth.gunicorn.conf"), ("syslog.conf", "/etc/st2/syslog.auth.conf"), ], - file_owner="root", - file_group="root", - file_mode="rw-r--r--", ) diff --git a/st2reactor/conf/BUILD b/st2reactor/conf/BUILD index d87e14c02b..593c3a6cf8 100644 --- a/st2reactor/conf/BUILD +++ b/st2reactor/conf/BUILD @@ -14,8 +14,7 @@ st2_logging_conf_files( sources=["syslog*.conf"], ) -nfpm_content_files( - name="packaged_conf_files", +st2_logging_conf_for_nfpm( dependencies=[ ":logging", ":logging_syslog", @@ -30,7 +29,4 @@ nfpm_content_files( ("logging.timersengine.conf", "/etc/st2/logging.timersengine.conf"), ("syslog.timersengine.conf", "/etc/st2/syslog.timersengine.conf"), ], - file_owner="root", - file_group="root", - file_mode="rw-r--r--", ) diff --git a/st2stream/conf/BUILD b/st2stream/conf/BUILD index 785a1d0604..a78c5f5039 100644 --- a/st2stream/conf/BUILD +++ b/st2stream/conf/BUILD @@ -20,8 +20,7 @@ st2_logging_conf_file( source="syslog.conf", ) -nfpm_content_files( - name="packaged_conf_files", +st2_logging_conf_for_nfpm( dependencies=[ ":logging", ":logging_gunicorn", @@ -32,7 +31,4 @@ nfpm_content_files( ("logging.gunicorn.conf", "/etc/st2/logging.stream.gunicorn.conf"), ("syslog.conf", "/etc/st2/syslog.stream.conf"), ], - file_owner="root", - file_group="root", - file_mode="rw-r--r--", ) From 6fcaa15f580a4584a0986724ce7200d61654c419 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 9 Apr 2025 15:37:13 -0500 Subject: [PATCH 5/9] Packaging: make conf file editing as portable as possible To keep this portable so that it runs with all versions of sed, avoid features that are only in gnu sed (like \s). And use crudini instead of sed for more straightforward edits. --- pants-plugins/macros.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 2b3b54cab2..275e447a81 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -234,11 +234,14 @@ def st2_logging_conf_for_nfpm(**kwargs): shell_command( # noqa: F821 name="package_logging_conf", execution_dependencies=deps, - # TODO: this will fail if using MacOS. - command="""sed -i -r "/args\\s*=\\s*/s%logs%/var/log/st2%g" logging.*conf; - sed -i "/\\[logger_root\\]/,/\\[.*\\]\\|\\s*$$/ {s/level=DEBUG/level=INFO/}" logging.*conf; - sed -i "/\\[logger_root\\]/,/\\[.*\\]\\|\\s*$$/ {s/level=DEBUG/level=INFO/}" syslog.*conf; + # Using "-E" and specifying the ".bak" suffix makes this portable + command=""" + sed -E -i.bak "/args[[:space:]]*=[[:space:]]*/s:logs/:/var/log/st2/:g" logging.*conf; + for conf_file in logging.*conf syslog.*conf; do + crudini --verbose --set "${conf_file}" logger_root level INFO; + done """, + runnable_dependencies=["//:crudini"], tools=["sed"], output_files=["*.conf"], ) From b6b33b786acef4fa53e8b37a1e3f0b377b3ab425 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 15:01:06 -0600 Subject: [PATCH 6/9] Packaging: stub command to modify st2.package.conf --- conf/BUILD | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/conf/BUILD b/conf/BUILD index 9dac52bf33..ac90cf5bc2 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -66,10 +66,20 @@ shell_command( output_files=["htpasswd"], ) +shell_command( + name="packaged_st2_conf", + execution_dependencies=[":st2_package_conf"], + # TODO: how to add the if statement + # if REDHAT && [[ `$(PYTHON_ALT_BINARY) /root/scripts/platform_major_version.py` = 8 ]] + command="""sed -i -r 's/virtualenv_opts\\s*=.*/virtualenv_opts =/' st2.package.conf""", + tools=["sed"], + output_files=["st2.package.conf"], +) + nfpm_content_files( name="packaged_conf_files", dependencies=[ - ":st2_package_conf", + ":packaged_st2_conf", ":logrotate", ":nginx_sample_config", ":htpasswd", From a673cb7fe14896e0807fd3a81bc45ceeabe21ec8 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 9 Apr 2025 16:29:29 -0500 Subject: [PATCH 7/9] packaging: use crudini to modify st2.package.conf --- conf/BUILD | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/conf/BUILD b/conf/BUILD index ac90cf5bc2..65ef2abaf0 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -69,10 +69,9 @@ shell_command( shell_command( name="packaged_st2_conf", execution_dependencies=[":st2_package_conf"], - # TODO: how to add the if statement - # if REDHAT && [[ `$(PYTHON_ALT_BINARY) /root/scripts/platform_major_version.py` = 8 ]] - command="""sed -i -r 's/virtualenv_opts\\s*=.*/virtualenv_opts =/' st2.package.conf""", - tools=["sed"], + # virtualenv_opts is no longer needed for most OSes. We used to do this only for EL 8. + command="""crudini --verbose --set st2.package.conf actionrunner virtualenv_opts ''""", + runnable_dependencies=["//:crudini"], output_files=["st2.package.conf"], ) From 1ad65e455e66c95d94d1c4607a34fc419d345d9a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 10 Apr 2025 18:19:43 -0500 Subject: [PATCH 8/9] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 94afd2865b..710369b6c7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -80,7 +80,7 @@ Added #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 #6307 #6311 #6314 #6315 #6317 #6319 #6312 #6320 #6321 #6323 #6324 #6325 #6326 #6327 #6328 - #6329 + #6329 #6330 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 From ae649f9a7a12b7773487854d317b9f0c40c229b1 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 15:28:34 -0600 Subject: [PATCH 9/9] Packaging: Remove world read perms from st2.conf When testing the packages, we should discover any non-root processes that rely on access to st2.conf. Hopefully giving them access will be as simple as switching the group to ST2_SVC_USER. Otherwise, we might need to revert this change and make st2.conf world readable. Also note that ST2 now supports passing secrets in env vars. So, people could theoretically include the secrets in systemd conf files that are only accessible by root. If any utils, like st2ctl, need access to those secrets, however, they will need to get them from somewhere else if they are not in st2.conf. --- conf/BUILD | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/BUILD b/conf/BUILD index 65ef2abaf0..db472a6e5c 100644 --- a/conf/BUILD +++ b/conf/BUILD @@ -94,6 +94,11 @@ nfpm_content_files( file_group="root", file_mode="rw-r--r--", overrides={ + "/etc/st2/st2.conf": dict( + # st2.conf typically contains secrets, so it is not world readable. + file_mode="rw-r-----", # NOTE: Packaging used to install this world readable. + # TODO: Maybe set file_group=ST2_SVC_USER if a non-root process needs access. + ), "/etc/st2/htpasswd": dict( file_owner=ST2_SVC_USER, file_group=ST2_SVC_USER,