From 7b161e44c9fce47b17cc57c8ad1ba7076fe03d41 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Dec 2024 15:06:32 -0600 Subject: [PATCH 1/9] Packaging: Add /opt/stackstorm/install to hold st2.pex and other archives --- packaging/BUILD | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packaging/BUILD b/packaging/BUILD index ad9af3e58c..34a81d5f99 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -3,6 +3,35 @@ # - https://www.pantsbuild.org/stable/reference/targets/nfpm_rpm_package # - arch: https://nfpm.goreleaser.com/goarch-to-pkg/ +nfpm_content_file( + name="st2_venv", + description="Pex file that system packages can use to generate /opt/stackstorm/st2", + file_owner="root", + file_group="root", + file_mode="rwxr-x---", + dst="/opt/stackstorm/install/st2.pex", # or st2-py3*.pex + **parametrize( + "py38", + dependencies=[":st2.pex@parametrize=py38"], + src="st2-py38.pex", + ), + **parametrize( + "py39", + dependencies=[":st2.pex@parametrize=py39"], + src="st2-py39.pex", + ), + **parametrize( + "py310", + dependencies=[":st2.pex@parametrize=py310"], + src="st2-py310.pex", + ), + **parametrize( + "py311", + dependencies=[":st2.pex@parametrize=py311"], + src="st2-py311.pex", + ), +) + _pkg_description = """ StackStorm Event-driven automation Package is full standalone st2 installation including all components @@ -38,6 +67,7 @@ nfpm_deb_package( "./common/systemd:generators", "./common:dirs", "./common:symlinks", + ":st2_venv@parametrize=py38", ], scripts=dict( preinstall="deb/scripts/pre-install.sh", @@ -70,6 +100,7 @@ nfpm_rpm_package( "./common/systemd:generators", "./common:dirs", "./common:symlinks", + ":st2_venv@parametrize=py38", ], scripts=dict( preinstall="rpm/scripts/pre-install.sh", From 43c1227b44863f17c7acc36c14ca80f14a7e6dc9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Dec 2024 16:14:28 -0600 Subject: [PATCH 2/9] Pants: Move venv packaging targets to separate BUILD file --- packaging/BUILD | 29 ----------------------------- packaging/BUILD.venv | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/packaging/BUILD b/packaging/BUILD index 34a81d5f99..c27e803742 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -3,35 +3,6 @@ # - https://www.pantsbuild.org/stable/reference/targets/nfpm_rpm_package # - arch: https://nfpm.goreleaser.com/goarch-to-pkg/ -nfpm_content_file( - name="st2_venv", - description="Pex file that system packages can use to generate /opt/stackstorm/st2", - file_owner="root", - file_group="root", - file_mode="rwxr-x---", - dst="/opt/stackstorm/install/st2.pex", # or st2-py3*.pex - **parametrize( - "py38", - dependencies=[":st2.pex@parametrize=py38"], - src="st2-py38.pex", - ), - **parametrize( - "py39", - dependencies=[":st2.pex@parametrize=py39"], - src="st2-py39.pex", - ), - **parametrize( - "py310", - dependencies=[":st2.pex@parametrize=py310"], - src="st2-py310.pex", - ), - **parametrize( - "py311", - dependencies=[":st2.pex@parametrize=py311"], - src="st2-py311.pex", - ), -) - _pkg_description = """ StackStorm Event-driven automation Package is full standalone st2 installation including all components diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv index 2c665ec5f5..bc0c804b76 100644 --- a/packaging/BUILD.venv +++ b/packaging/BUILD.venv @@ -71,3 +71,32 @@ pex_binary( **_pex_py3("10"), **_pex_py3("11"), ) + +nfpm_content_file( + name="st2_venv", + description="Pex file that system packages can use to generate /opt/stackstorm/st2", + file_owner="root", + file_group="root", + file_mode="rwxr-x---", + dst="/opt/stackstorm/install/st2.pex", # or st2-py3*.pex + **parametrize( + "py38", + dependencies=[":st2.pex@parametrize=py38"], + src="st2-py38.pex", + ), + **parametrize( + "py39", + dependencies=[":st2.pex@parametrize=py39"], + src="st2-py39.pex", + ), + **parametrize( + "py310", + dependencies=[":st2.pex@parametrize=py310"], + src="st2-py310.pex", + ), + **parametrize( + "py311", + dependencies=[":st2.pex@parametrize=py311"], + src="st2-py311.pex", + ), +) From 6db204492c3cf570dc37ee35acef5b66ed42f122 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 18:28:01 -0600 Subject: [PATCH 3/9] pants: Dry parametrization in packaging/BUILD.venv --- packaging/BUILD.venv | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv index bc0c804b76..d925744866 100644 --- a/packaging/BUILD.venv +++ b/packaging/BUILD.venv @@ -72,6 +72,15 @@ pex_binary( **_pex_py3("11"), ) + +def _venv_py3(minor: str): + return parametrize( + f"py3{minor}", + dependencies=[f":st2.pex@parametrize=py3{minor}"], + src=f"st2-py3{minor}.pex", # relative to this BUILD file + ) + + nfpm_content_file( name="st2_venv", description="Pex file that system packages can use to generate /opt/stackstorm/st2", @@ -79,24 +88,8 @@ nfpm_content_file( file_group="root", file_mode="rwxr-x---", dst="/opt/stackstorm/install/st2.pex", # or st2-py3*.pex - **parametrize( - "py38", - dependencies=[":st2.pex@parametrize=py38"], - src="st2-py38.pex", - ), - **parametrize( - "py39", - dependencies=[":st2.pex@parametrize=py39"], - src="st2-py39.pex", - ), - **parametrize( - "py310", - dependencies=[":st2.pex@parametrize=py310"], - src="st2-py310.pex", - ), - **parametrize( - "py311", - dependencies=[":st2.pex@parametrize=py311"], - src="st2-py311.pex", - ), + **_venv_py3("8"), + **_venv_py3("9"), + **_venv_py3("10"), + **_venv_py3("11"), ) From 0ece94f6a04317b19232eafbdfb47ecb53e99836 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 18:36:00 -0600 Subject: [PATCH 4/9] Packaging: Add self-install archives for each pack to /opt/stackstorm/install --- packaging/BUILD | 4 ++++ packaging/BUILD.packs | 25 +++++++++++++++++++++++++ pants-plugins/macros.py | 10 ++++++++++ 3 files changed, 39 insertions(+) create mode 100644 packaging/BUILD.packs diff --git a/packaging/BUILD b/packaging/BUILD index c27e803742..ba1a4d22ff 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -39,6 +39,8 @@ nfpm_deb_package( "./common:dirs", "./common:symlinks", ":st2_venv@parametrize=py38", + ":default_packs", + ":extra_packs", # TODO: Should we include these even though they won't be installed? ], scripts=dict( preinstall="deb/scripts/pre-install.sh", @@ -72,6 +74,8 @@ nfpm_rpm_package( "./common:dirs", "./common:symlinks", ":st2_venv@parametrize=py38", + ":default_packs", + ":extra_packs", # TODO: Should we include these even though they won't be installed? ], scripts=dict( preinstall="rpm/scripts/pre-install.sh", diff --git a/packaging/BUILD.packs b/packaging/BUILD.packs new file mode 100644 index 0000000000..93f100a85a --- /dev/null +++ b/packaging/BUILD.packs @@ -0,0 +1,25 @@ +# These are packs that should be installed with the default ST2 install. +_DEFAULT_PACKS = ( + "chatops", + "core", + "default", + "linux", + "packs", +) + +# These are packs that should NOT be installed with the default ST2 install. +_EXTRA_PACKS = ( + "debug", + "examples", + "hello_st2", +) + +# :archive_for_npfm targets created by st2_pack_archive() macro (see pants-plugins/macros.py) +target( + name="default_packs", + dependencies=[f"//contrib/{pack}:archive_for_nfpm" for pack in _DEFAULT_PACKS], +) +target( + name="extra_packs", + dependencies=[f"//contrib/{pack}:archive_for_nfpm" for pack in _EXTRA_PACKS], +) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index a8fd2021da..cd6c017d6d 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -170,6 +170,16 @@ def st2_pack_archive(**kwargs): output_path=f"packaging/packs/{pack_name}.tgz.run", ) + nfpm_content_file( # noqa: F821 + name="archive_for_nfpm", + dependencies=[":archive"], + src=f"packaging/packs/{pack_name}.tgz.run", + dst=f"/opt/stackstorm/install/packs/{pack_name}.tgz.run", + file_owner="root", + file_group=ST2_PACKS_GROUP, + file_mode="rwxr-x---", + ) + def st2_shell_sources_and_resources(**kwargs): """This creates a shell_sources and a resources target. From 6d3ef4c9d8023f5731d1aa53f942639bdb4231b2 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 27 Mar 2025 11:09:15 -0500 Subject: [PATCH 5/9] packaging: drop TODO - just include extra packs unless there is a reason not to --- packaging/BUILD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/BUILD b/packaging/BUILD index ba1a4d22ff..d356f838e4 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -40,7 +40,7 @@ nfpm_deb_package( "./common:symlinks", ":st2_venv@parametrize=py38", ":default_packs", - ":extra_packs", # TODO: Should we include these even though they won't be installed? + ":extra_packs", ], scripts=dict( preinstall="deb/scripts/pre-install.sh", @@ -75,7 +75,7 @@ nfpm_rpm_package( "./common:symlinks", ":st2_venv@parametrize=py38", ":default_packs", - ":extra_packs", # TODO: Should we include these even though they won't be installed? + ":extra_packs", ], scripts=dict( preinstall="rpm/scripts/pre-install.sh", From 850cc78c25c2a90fa3cc77dde159358cb95f8d93 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 7 Apr 2025 09:58:55 -0500 Subject: [PATCH 6/9] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e6af6007bc..4f26ffef30 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -79,7 +79,7 @@ Added to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 #6307 - #6311 #6314 #6315 #6317 #6319 #6312 #6320 #6321 #6323 #6324 #6325 #6326 #6327 + #6311 #6314 #6315 #6317 #6319 #6312 #6320 #6321 #6323 #6324 #6325 #6326 #6327 #6328 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 From 4446469d50df132caa76020400c21063f044e162 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 7 Apr 2025 23:47:30 -0500 Subject: [PATCH 7/9] pants-plugins/release: rule input/output must be hashable/immutable --- pants-plugins/release/packagecloud_rules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pants-plugins/release/packagecloud_rules.py b/pants-plugins/release/packagecloud_rules.py index bdda2399d8..46dce1ef67 100644 --- a/pants-plugins/release/packagecloud_rules.py +++ b/pants-plugins/release/packagecloud_rules.py @@ -74,7 +74,7 @@ } -@dataclass +@dataclass(frozen=True) class PackageCloudNextReleaseRequest: nfpm_arch: str distro_id: str @@ -83,7 +83,7 @@ class PackageCloudNextReleaseRequest: production: bool -@dataclass +@dataclass(frozen=True) class PackageCloudNextRelease: value: Optional[int] = None From 2cf9ea1ea3e25dc88c07de97bee30e39cd617515 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 7 Apr 2025 23:45:45 -0500 Subject: [PATCH 8/9] Packaging: include st2-py3*.pex in deb/rpm packages --- packaging/BUILD | 16 ++++++++++++++-- packaging/BUILD.venv | 4 +++- packaging/deb/scripts/post-install.sh | 22 +++++++++++++++++++++- packaging/rpm/scripts/post-install.sh | 22 +++++++++++++++++++++- pants.toml | 4 +++- 5 files changed, 62 insertions(+), 6 deletions(-) diff --git a/packaging/BUILD b/packaging/BUILD index d356f838e4..388f4f6185 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -22,6 +22,15 @@ _common_pkg_metadata = dict( _maintainer = "StackStorm Engineering " # TODO: update this +def _st2_venv_deps() -> list[str]: + """Return a list of deps required to build the st2 venv. + + The :st2_venv parametrizations depend on parametrizations of :st2.pex, + so, make sure to keep these 3 lists of parametrizations in sync. + """ + return [f":st2_venv@parametrize=py3{m}" for m in ("8", "9", "10", "11")] + + def _distro(distro_id: str, **kwargs): return parametrize( distro_id, @@ -38,7 +47,7 @@ nfpm_deb_package( "./common/systemd:generators", "./common:dirs", "./common:symlinks", - ":st2_venv@parametrize=py38", + *_st2_venv_deps(), ":default_packs", ":extra_packs", ], @@ -73,10 +82,13 @@ nfpm_rpm_package( "./common/systemd:generators", "./common:dirs", "./common:symlinks", - ":st2_venv@parametrize=py38", + *_st2_venv_deps(), ":default_packs", ":extra_packs", ], + ghost_contents=[ + "/opt/stackstorm/install/st2.pex", # a symlink (default in post-install; users may modify) + ], scripts=dict( preinstall="rpm/scripts/pre-install.sh", postinstall="rpm/scripts/post-install.sh", diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv index d925744866..f28c44db4e 100644 --- a/packaging/BUILD.venv +++ b/packaging/BUILD.venv @@ -23,6 +23,7 @@ python_sources() def _pex_py3(minor: str, constraint: str = ""): + """Parametrize :st2.pex for a given python minor version.""" if not constraint: constraint = f"CPython==3.{minor}.*" return parametrize( @@ -74,10 +75,12 @@ pex_binary( def _venv_py3(minor: str): + """Parametrize :st2_venv for a given python minor version.""" return parametrize( f"py3{minor}", dependencies=[f":st2.pex@parametrize=py3{minor}"], src=f"st2-py3{minor}.pex", # relative to this BUILD file + dst=f"/opt/stackstorm/install/st2-py3{minor}.pex", ) @@ -87,7 +90,6 @@ nfpm_content_file( file_owner="root", file_group="root", file_mode="rwxr-x---", - dst="/opt/stackstorm/install/st2.pex", # or st2-py3*.pex **_venv_py3("8"), **_venv_py3("9"), **_venv_py3("10"), diff --git a/packaging/deb/scripts/post-install.sh b/packaging/deb/scripts/post-install.sh index 27bb66a7e0..df4959e3f0 100644 --- a/packaging/deb/scripts/post-install.sh +++ b/packaging/deb/scripts/post-install.sh @@ -29,6 +29,9 @@ set -e # https://www.mankier.com/5/deb-triggers # https://stackoverflow.com/questions/15276535/dpkg-how-to-use-trigger +# The supported minor versions of python3 (python3.{minor}) in reverse order. +_ST2_PY3_MINOR="11 10 9 8" + # The default set of packs installed with st2. _ST2_PACKS=" chatops @@ -93,7 +96,24 @@ systemd_enable_and_restart() { } rebuild_st2_venv() { - /opt/stackstorm/install/st2.pex + _pex="/opt/stackstorm/install/st2.pex" + if [ ! -e "${_pex}" ]; then + # symlink does not exist or does not point to a valid file + # (the symlink target might not exist any more if upgrading + # to an st2 version that drops support for an older python) + rm -f "${_pex}" + for minor in ${_ST2_PY3_MINOR}; do + if [ -x "/usr/bin/python3.${minor}" ]; then + ln -s "st2-py3${minor}.pex" "${_pex}" + break + fi + done + if [ ! -e "${_pex}" ]; then + # symlink creation failed; the python dep is somehow missing + return 42 + fi + fi + "${_pex}" } extract_st2_pack() { diff --git a/packaging/rpm/scripts/post-install.sh b/packaging/rpm/scripts/post-install.sh index a7909c3f18..0736e0db61 100644 --- a/packaging/rpm/scripts/post-install.sh +++ b/packaging/rpm/scripts/post-install.sh @@ -7,6 +7,9 @@ set -e # * on upgrade: $1 > 1 # https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_syntax +# The supported minor versions of python3 (python3.{minor}) in reverse order. +_ST2_PY3_MINOR="11 10 9 8" + # The default set of packs installed with st2. _ST2_PACKS=" chatops @@ -31,7 +34,24 @@ st2workflowengine " rebuild_st2_venv() { - /opt/stackstorm/install/st2.pex + _pex="/opt/stackstorm/install/st2.pex" + if [ ! -e "${_pex}" ]; then + # symlink does not exist or does not point to a valid file + # (the symlink target might not exist any more if upgrading + # to an st2 version that drops support for an older python) + rm -f "${_pex}" + for minor in ${_ST2_PY3_MINOR}; do + if [ -x "/usr/bin/python3.${minor}" ]; then + ln -s "st2-py3${minor}.pex" "${_pex}" + break + fi + done + if [ ! -e "${_pex}" ]; then + # symlink creation failed; the python dep is somehow missing + return 42 + fi + fi + "${_pex}" } extract_st2_pack() { diff --git a/pants.toml b/pants.toml index 6c686a5748..cf771c2438 100644 --- a/pants.toml +++ b/pants.toml @@ -109,7 +109,9 @@ root_patterns = [ # DEFAULT has values that we can reuse/interpolate below [DEFAULT] # This is the range of python versions that we support. -# On update, make sure to also update parametrizations in packaging/BUILD*. +# On update, make sure to also update: +# - the parametrizations in packaging/BUILD and packaging/BUILD.venv +# - the list of python minor versions in packaging/*/scripts/post-install.sh st2_interpreter_constraints = "CPython>=3.8.1,<3.12" # This should match the pants interpreter_constraints: From 1ebd1995f69a197eaceca78cb2251fd623958f32 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 8 Apr 2025 12:31:39 -0500 Subject: [PATCH 9/9] Packaging: speed up el8 package build using zstd compression There was a multi-minute difference between el8 and el9. The compression algorithm is the only significant difference, so I timed building the same el8 package with different algorithms. - xz and lzma were the slowest at over 3 minutes. - gzip and zstd were the fastest at under 1 minute (at any level). The size differences were marginal, but zstd:best was the closest match in size to xz, so I used that to keep the package size as small as possible while speeding up the package build process. --- packaging/BUILD | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packaging/BUILD b/packaging/BUILD index 388f4f6185..d992274d57 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -101,7 +101,8 @@ nfpm_rpm_package( vendor="The StackStorm Project", packager=_maintainer, # group="System/Management", # was only useful for EL 5 and earlier + compression="zstd:best", # xz and lzma are ~3x slower than gzip or zstd **_common_pkg_metadata, - **_distro("el8", compression="xz"), - **_distro("el9", compression="zstd:default"), + **_distro("el8"), + **_distro("el9"), )