diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e6af6007bc..4f26ffef30 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -79,7 +79,7 @@ Added to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 #6307 - #6311 #6314 #6315 #6317 #6319 #6312 #6320 #6321 #6323 #6324 #6325 #6326 #6327 + #6311 #6314 #6315 #6317 #6319 #6312 #6320 #6321 #6323 #6324 #6325 #6326 #6327 #6328 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 diff --git a/packaging/BUILD b/packaging/BUILD index ad9af3e58c..d992274d57 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -22,6 +22,15 @@ _common_pkg_metadata = dict( _maintainer = "StackStorm Engineering " # TODO: update this +def _st2_venv_deps() -> list[str]: + """Return a list of deps required to build the st2 venv. + + The :st2_venv parametrizations depend on parametrizations of :st2.pex, + so, make sure to keep these 3 lists of parametrizations in sync. + """ + return [f":st2_venv@parametrize=py3{m}" for m in ("8", "9", "10", "11")] + + def _distro(distro_id: str, **kwargs): return parametrize( distro_id, @@ -38,6 +47,9 @@ nfpm_deb_package( "./common/systemd:generators", "./common:dirs", "./common:symlinks", + *_st2_venv_deps(), + ":default_packs", + ":extra_packs", ], scripts=dict( preinstall="deb/scripts/pre-install.sh", @@ -70,6 +82,12 @@ nfpm_rpm_package( "./common/systemd:generators", "./common:dirs", "./common:symlinks", + *_st2_venv_deps(), + ":default_packs", + ":extra_packs", + ], + ghost_contents=[ + "/opt/stackstorm/install/st2.pex", # a symlink (default in post-install; users may modify) ], scripts=dict( preinstall="rpm/scripts/pre-install.sh", @@ -83,7 +101,8 @@ nfpm_rpm_package( vendor="The StackStorm Project", packager=_maintainer, # group="System/Management", # was only useful for EL 5 and earlier + compression="zstd:best", # xz and lzma are ~3x slower than gzip or zstd **_common_pkg_metadata, - **_distro("el8", compression="xz"), - **_distro("el9", compression="zstd:default"), + **_distro("el8"), + **_distro("el9"), ) diff --git a/packaging/BUILD.packs b/packaging/BUILD.packs new file mode 100644 index 0000000000..93f100a85a --- /dev/null +++ b/packaging/BUILD.packs @@ -0,0 +1,25 @@ +# These are packs that should be installed with the default ST2 install. +_DEFAULT_PACKS = ( + "chatops", + "core", + "default", + "linux", + "packs", +) + +# These are packs that should NOT be installed with the default ST2 install. +_EXTRA_PACKS = ( + "debug", + "examples", + "hello_st2", +) + +# :archive_for_npfm targets created by st2_pack_archive() macro (see pants-plugins/macros.py) +target( + name="default_packs", + dependencies=[f"//contrib/{pack}:archive_for_nfpm" for pack in _DEFAULT_PACKS], +) +target( + name="extra_packs", + dependencies=[f"//contrib/{pack}:archive_for_nfpm" for pack in _EXTRA_PACKS], +) diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv index 2c665ec5f5..f28c44db4e 100644 --- a/packaging/BUILD.venv +++ b/packaging/BUILD.venv @@ -23,6 +23,7 @@ python_sources() def _pex_py3(minor: str, constraint: str = ""): + """Parametrize :st2.pex for a given python minor version.""" if not constraint: constraint = f"CPython==3.{minor}.*" return parametrize( @@ -71,3 +72,26 @@ pex_binary( **_pex_py3("10"), **_pex_py3("11"), ) + + +def _venv_py3(minor: str): + """Parametrize :st2_venv for a given python minor version.""" + return parametrize( + f"py3{minor}", + dependencies=[f":st2.pex@parametrize=py3{minor}"], + src=f"st2-py3{minor}.pex", # relative to this BUILD file + dst=f"/opt/stackstorm/install/st2-py3{minor}.pex", + ) + + +nfpm_content_file( + name="st2_venv", + description="Pex file that system packages can use to generate /opt/stackstorm/st2", + file_owner="root", + file_group="root", + file_mode="rwxr-x---", + **_venv_py3("8"), + **_venv_py3("9"), + **_venv_py3("10"), + **_venv_py3("11"), +) diff --git a/packaging/deb/scripts/post-install.sh b/packaging/deb/scripts/post-install.sh index 27bb66a7e0..df4959e3f0 100644 --- a/packaging/deb/scripts/post-install.sh +++ b/packaging/deb/scripts/post-install.sh @@ -29,6 +29,9 @@ set -e # https://www.mankier.com/5/deb-triggers # https://stackoverflow.com/questions/15276535/dpkg-how-to-use-trigger +# The supported minor versions of python3 (python3.{minor}) in reverse order. +_ST2_PY3_MINOR="11 10 9 8" + # The default set of packs installed with st2. _ST2_PACKS=" chatops @@ -93,7 +96,24 @@ systemd_enable_and_restart() { } rebuild_st2_venv() { - /opt/stackstorm/install/st2.pex + _pex="/opt/stackstorm/install/st2.pex" + if [ ! -e "${_pex}" ]; then + # symlink does not exist or does not point to a valid file + # (the symlink target might not exist any more if upgrading + # to an st2 version that drops support for an older python) + rm -f "${_pex}" + for minor in ${_ST2_PY3_MINOR}; do + if [ -x "/usr/bin/python3.${minor}" ]; then + ln -s "st2-py3${minor}.pex" "${_pex}" + break + fi + done + if [ ! -e "${_pex}" ]; then + # symlink creation failed; the python dep is somehow missing + return 42 + fi + fi + "${_pex}" } extract_st2_pack() { diff --git a/packaging/rpm/scripts/post-install.sh b/packaging/rpm/scripts/post-install.sh index a7909c3f18..0736e0db61 100644 --- a/packaging/rpm/scripts/post-install.sh +++ b/packaging/rpm/scripts/post-install.sh @@ -7,6 +7,9 @@ set -e # * on upgrade: $1 > 1 # https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_syntax +# The supported minor versions of python3 (python3.{minor}) in reverse order. +_ST2_PY3_MINOR="11 10 9 8" + # The default set of packs installed with st2. _ST2_PACKS=" chatops @@ -31,7 +34,24 @@ st2workflowengine " rebuild_st2_venv() { - /opt/stackstorm/install/st2.pex + _pex="/opt/stackstorm/install/st2.pex" + if [ ! -e "${_pex}" ]; then + # symlink does not exist or does not point to a valid file + # (the symlink target might not exist any more if upgrading + # to an st2 version that drops support for an older python) + rm -f "${_pex}" + for minor in ${_ST2_PY3_MINOR}; do + if [ -x "/usr/bin/python3.${minor}" ]; then + ln -s "st2-py3${minor}.pex" "${_pex}" + break + fi + done + if [ ! -e "${_pex}" ]; then + # symlink creation failed; the python dep is somehow missing + return 42 + fi + fi + "${_pex}" } extract_st2_pack() { diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index a8fd2021da..cd6c017d6d 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -170,6 +170,16 @@ def st2_pack_archive(**kwargs): output_path=f"packaging/packs/{pack_name}.tgz.run", ) + nfpm_content_file( # noqa: F821 + name="archive_for_nfpm", + dependencies=[":archive"], + src=f"packaging/packs/{pack_name}.tgz.run", + dst=f"/opt/stackstorm/install/packs/{pack_name}.tgz.run", + file_owner="root", + file_group=ST2_PACKS_GROUP, + file_mode="rwxr-x---", + ) + def st2_shell_sources_and_resources(**kwargs): """This creates a shell_sources and a resources target. diff --git a/pants-plugins/release/packagecloud_rules.py b/pants-plugins/release/packagecloud_rules.py index bdda2399d8..46dce1ef67 100644 --- a/pants-plugins/release/packagecloud_rules.py +++ b/pants-plugins/release/packagecloud_rules.py @@ -74,7 +74,7 @@ } -@dataclass +@dataclass(frozen=True) class PackageCloudNextReleaseRequest: nfpm_arch: str distro_id: str @@ -83,7 +83,7 @@ class PackageCloudNextReleaseRequest: production: bool -@dataclass +@dataclass(frozen=True) class PackageCloudNextRelease: value: Optional[int] = None diff --git a/pants.toml b/pants.toml index 6c686a5748..cf771c2438 100644 --- a/pants.toml +++ b/pants.toml @@ -109,7 +109,9 @@ root_patterns = [ # DEFAULT has values that we can reuse/interpolate below [DEFAULT] # This is the range of python versions that we support. -# On update, make sure to also update parametrizations in packaging/BUILD*. +# On update, make sure to also update: +# - the parametrizations in packaging/BUILD and packaging/BUILD.venv +# - the list of python minor versions in packaging/*/scripts/post-install.sh st2_interpreter_constraints = "CPython>=3.8.1,<3.12" # This should match the pants interpreter_constraints: