diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bc6b8dded1..e6eae0554c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -79,7 +79,7 @@ Added to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 #6307 - #6311 #6314 #6315 #6317 #6319 #6312 #6320 #6321 + #6311 #6314 #6315 #6317 #6319 #6312 #6320 #6321 #6323 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 diff --git a/lockfiles/pants-plugins.lock b/lockfiles/pants-plugins.lock index dc7f3bd479..138856518c 100644 --- a/lockfiles/pants-plugins.lock +++ b/lockfiles/pants-plugins.lock @@ -10,7 +10,8 @@ // ], // "generated_with_requirements": [ // "pantsbuild.pants.testutil==2.25.0", -// "pantsbuild.pants==2.25.0" +// "pantsbuild.pants==2.25.0", +// "requests" // ], // "manylinux": "manylinux2014", // "requirement_constraints": [], @@ -107,6 +108,97 @@ "requires_python": ">=3.8", "version": "25.3.0" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe", + "url": "https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651", + "url": "https://files.pythonhosted.org/packages/1c/ab/c9f1e32b7b1bf505bf26f0ef697775960db7932abeb7b516de930ba2705f/certifi-2025.1.31.tar.gz" + } + ], + "project_name": "certifi", + "requires_dists": [], + "requires_python": ">=3.6", + "version": "2025.1.31" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", + "url": "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3", + "url": "https://files.pythonhosted.org/packages/16/b0/572805e227f01586461c80e0fd25d65a2115599cc9dad142fee4b747c357/charset_normalizer-3.4.1.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146", + "url": "https://files.pythonhosted.org/packages/28/a3/a42e70d03cbdabc18997baf4f0227c73591a08041c149e710045c281f97b/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd", + "url": "https://files.pythonhosted.org/packages/37/ed/be39e5258e198655240db5e19e0b11379163ad7070962d6b0c87ed2c4d39/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8", + "url": "https://files.pythonhosted.org/packages/3d/7b/82865ba54c765560c8433f65e8acb9217cb839a9e32b42af4aa8e945870f/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77", + "url": "https://files.pythonhosted.org/packages/68/85/f4288e96039abdd5aeb5c546fa20a37b50da71b5cf01e75e87f16cd43304/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3", + "url": "https://files.pythonhosted.org/packages/71/64/d24ab1a997efb06402e3fc07317e94da358e2585165930d9d59ad45fcae2/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125", + "url": "https://files.pythonhosted.org/packages/72/80/41ef5d5a7935d2d3a773e3eaebf0a9350542f2cab4eac59a7a4741fbbbbe/charset_normalizer-3.4.1-cp311-cp311-macosx_10_9_universal2.whl" + }, + { + "algorithm": "sha256", + "hash": "28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1", + "url": "https://files.pythonhosted.org/packages/7a/28/0b9fefa7b8b080ec492110af6d88aa3dea91c464b17d53474b6e9ba5d2c5/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd", + "url": "https://files.pythonhosted.org/packages/85/e4/65699e8ab3014ecbe6f5c71d1a55d810fb716bbfd74f6283d5c2aa87febf/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_ppc64le.whl" + }, + { + "algorithm": "sha256", + "hash": "fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00", + "url": "https://files.pythonhosted.org/packages/88/83/489e9504711fa05d8dde1574996408026bdbdbd938f23be67deebb5eca92/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6", + "url": "https://files.pythonhosted.org/packages/b1/82/8e9fe624cc5374193de6860aba3ea8070f584c8565ee77c168ec13274bd2/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_s390x.whl" + }, + { + "algorithm": "sha256", + "hash": "eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12", + "url": "https://files.pythonhosted.org/packages/c6/c7/32da20821cf387b759ad24627a9aca289d2822de929b8a41b6241767b461/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + } + ], + "project_name": "charset-normalizer", + "requires_dists": [], + "requires_python": ">=3.7", + "version": "3.4.1" + }, { "artifacts": [ { @@ -146,6 +238,29 @@ "requires_python": null, "version": "0.16.3" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", + "url": "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", + "url": "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz" + } + ], + "project_name": "idna", + "requires_dists": [ + "flake8>=7.1.1; extra == \"all\"", + "mypy>=1.11.2; extra == \"all\"", + "pytest>=8.3.2; extra == \"all\"", + "ruff>=0.6.2; extra == \"all\"" + ], + "requires_python": ">=3.6", + "version": "3.10" + }, { "artifacts": [ { @@ -208,19 +323,19 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", - "url": "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" + "hash": "9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", + "url": "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", - "url": "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz" + "hash": "3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", + "url": "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz" } ], "project_name": "iniconfig", "requires_dists": [], - "requires_python": ">=3.7", - "version": "2.0.0" + "requires_python": ">=3.8", + "version": "2.1.0" }, { "artifacts": [ @@ -596,6 +711,31 @@ "requires_python": ">=3.8", "version": "6.0.2" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", + "url": "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + "url": "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz" + } + ], + "project_name": "requests", + "requires_dists": [ + "PySocks!=1.5.7,>=1.5.6; extra == \"socks\"", + "certifi>=2017.4.17", + "chardet<6,>=3.0.2; extra == \"use-chardet-on-py3\"", + "charset-normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<3,>=1.21.1" + ], + "requires_python": ">=3.8", + "version": "2.32.3" + }, { "artifacts": [ { @@ -942,6 +1082,30 @@ "requires_dists": [], "requires_python": ">=3.8", "version": "5.10.0" + }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", + "url": "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d", + "url": "https://files.pythonhosted.org/packages/aa/63/e53da845320b757bf29ef6a9062f5c669fe997973f966045cb019c3f4b66/urllib3-2.3.0.tar.gz" + } + ], + "project_name": "urllib3", + "requires_dists": [ + "brotli>=1.0.9; platform_python_implementation == \"CPython\" and extra == \"brotli\"", + "brotlicffi>=0.8.0; platform_python_implementation != \"CPython\" and extra == \"brotli\"", + "h2<5,>=4; extra == \"h2\"", + "pysocks!=1.5.7,<2.0,>=1.5.6; extra == \"socks\"", + "zstandard>=0.18.0; extra == \"zstd\"" + ], + "requires_python": ">=3.9", + "version": "2.3.0" } ], "platform_tag": null @@ -956,7 +1120,8 @@ "prefer_older_binary": false, "requirements": [ "pantsbuild.pants.testutil==2.25.0", - "pantsbuild.pants==2.25.0" + "pantsbuild.pants==2.25.0", + "requests" ], "requires_python": [ "==3.11.*" diff --git a/pants-plugins/release/BUILD b/pants-plugins/release/BUILD index 0eea8b1cf1..c485368c66 100644 --- a/pants-plugins/release/BUILD +++ b/pants-plugins/release/BUILD @@ -3,3 +3,7 @@ python_sources() python_tests( name="tests", ) + +python_requirements( + name="reqs", +) diff --git a/pants-plugins/release/packagecloud_rules.py b/pants-plugins/release/packagecloud_rules.py new file mode 100644 index 0000000000..bdda2399d8 --- /dev/null +++ b/pants-plugins/release/packagecloud_rules.py @@ -0,0 +1,155 @@ +# Copyright 2025 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional + +import requests +from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest +from requests.auth import HTTPBasicAuth + +from pants.engine.internals.selectors import Get +from pants.engine.rules import _uncacheable_rule, collect_rules + +ARCH_NAMES = { # {nfpm_arch: {pkg_type: packagecloud_arch}} + # The key comes from the 'arch' field of nfpm_*_package targets (GOARCH or GOARCH+GOARM). + # https://www.pantsbuild.org/stable/reference/targets/nfpm_deb_package#arch + # https://www.pantsbuild.org/stable/reference/targets/nfpm_rpm_package#arch + "amd64": { + "deb": "amd64", + "rpm": "x86_64", + } +} + +# This includes distros we do not support. +DISTROS_BY_PKG_TYPE = { # {pkg_type: {distro: {distro_id: distro_version}}} + "deb": { + "debian": { # no releases in packagecloud (so far) + "buster": "10", + "bullseye": "11", + "bookworm": "12", + "trixie": "13", + "forky": "14", + }, + "ubuntu": { # Only LTS releases + "trusty": "14.04", # the oldest with releases in packagecloud + "xenial": "16.04", + "bionic": "18.04", + "focal": "20.04", + "jammy": "22.04", + "noble": "24.04", + }, + }, + "rpm": { + "el": { # EL = Enterprise Linux (RHEL, Rocky, Alma, ...) + # 6 is the oldest with releases in packagecloud + f"el{v}": f"{v}" + for v in (6, 7, 8, 9) + }, + }, +} + +DISTRO_INFO = { + distro_id: { + "distro": distro, + "version": distro_version, + "pkg_type": pkg_type, + } + for pkg_type, distros in DISTROS_BY_PKG_TYPE.items() + for distro, distro_ids in distros.items() + for distro_id, distro_version in distro_ids.items() +} + + +@dataclass +class PackageCloudNextReleaseRequest: + nfpm_arch: str + distro_id: str + package_name: str + package_version: str + production: bool + + +@dataclass +class PackageCloudNextRelease: + value: Optional[int] = None + + +@_uncacheable_rule +async def packagecloud_get_next_release( + request: PackageCloudNextReleaseRequest, +) -> PackageCloudNextRelease: + env_vars: EnvironmentVars = await Get( + EnvironmentVars, EnvironmentVarsRequest(["PACKAGECLOUD_TOKEN"]) + ) + package_cloud_token = env_vars.get("PACKAGECLOUD_TOKEN") + if not package_cloud_token: + return PackageCloudNextRelease() + + client = requests.session() + client.auth = HTTPBasicAuth(package_cloud_token, "") + + def get(url_path: str) -> list[dict[str, Any]]: + response = client.get(f"https://packagecloud.io{url_path}") + response.raise_for_status() + ret: list[dict[str, Any]] = response.json() + next_url = response.links.get("next", {}).get("url") + while next_url: + response = client.get(f"https://packagecloud.io{next_url}") + response.raise_for_status() + ret.extend(response.json()) + next_url = response.links.get("next", {}).get("url") + return ret + + distro_id = request.distro_id + distro_info = DISTRO_INFO[distro_id] + pkg_is_unstable = "dev" in request.package_version + + # packagecloud url params: + org = "stackstorm" + repo = f"{'' if request.production else 'staging-'}{'unstable' if pkg_is_unstable else 'stable'}" + pkg_type = distro_info["pkg_type"] + distro = distro_info["distro"] + distro_version = distro_id if pkg_type == "deb" else distro_info["version"] + pkg_name = request.package_name + arch = ARCH_NAMES[request.nfpm_arch][pkg_type] + + # https://packagecloud.io/docs/api#resource_packages_method_index (api doc incorrectly drops /:package) + # /api/v1/repos/:user_id/:repo/packages/:type/:distro/:version/:package/:arch.json + index_url = f"/api/v1/repos/{org}/{repo}/packages/{pkg_type}/{distro}/{distro_version}/{pkg_name}/{arch}.json" + package_index: list[dict[str, Any]] = get(index_url) + if not package_index: + return PackageCloudNextRelease() + + versions_url: str = package_index[0]["versions_url"] + versions: list[dict[str, Any]] = get(versions_url) + releases = [ + version_info["release"] + for version_info in versions + if version_info["version"] == request.package_version + ] + if not releases: + return PackageCloudNextRelease() + + max_release = max(int(release) for release in releases) + next_release = max_release + 1 + return PackageCloudNextRelease(next_release) + + +def rules(): + return [ + *collect_rules(), + ] diff --git a/pants-plugins/release/requirements.txt b/pants-plugins/release/requirements.txt new file mode 100644 index 0000000000..f2293605cf --- /dev/null +++ b/pants-plugins/release/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/pants-plugins/release/rules.py b/pants-plugins/release/rules.py index 8ee338b57a..71f3f46b6e 100644 --- a/pants-plugins/release/rules.py +++ b/pants-plugins/release/rules.py @@ -23,7 +23,15 @@ import re from dataclasses import dataclass -from pants.backend.nfpm.fields.version import NfpmVersionField, NfpmVersionSchemaField +from pants.backend.nfpm.fields.all import ( + NfpmArchField, + NfpmPackageNameField, +) +from pants.backend.nfpm.fields.version import ( + NfpmVersionField, + NfpmVersionReleaseField, + NfpmVersionSchemaField, +) from pants.backend.nfpm.util_rules.inject_config import ( InjectedNfpmPackageFields, InjectNfpmPackageFieldsRequest, @@ -38,6 +46,12 @@ from pants.engine.rules import collect_rules, Get, MultiGet, rule, UnionRule from pants.util.frozendict import FrozenDict +from .packagecloud_rules import ( + PackageCloudNextReleaseRequest, + packagecloud_get_next_release, +) +from .packagecloud_rules import rules as packagecloud_rules + REQUIRED_KWARGS = ( "description", @@ -203,7 +217,8 @@ def is_applicable(cls, _: Target) -> bool: async def inject_package_fields( request: StackStormNfpmPackageFieldsRequest, ) -> InjectedNfpmPackageFields: - address = request.target.address + target = request.target + address = target.address version_file = "st2common/st2common/__init__.py" extracted_version = await Get( @@ -215,20 +230,35 @@ async def inject_package_fields( ) version: str = extracted_version.value - if version.endswith("dev") and version[-4] != "-": + is_dev = "dev" in version + if is_dev and "-dev" not in version: # nfpm parses this into version[-version_prerelease][+version_metadata] - # that dash is required to be a valid semver version. + # that dash is required to be a valid semver version (3.9dev => 3.9-dev). version = version.replace("dev", "-dev") + # this is specific to distro-version (EL8, EL9, Ubuntu Focal, Ubuntu Jammy, ...) + next_release = await packagecloud_get_next_release( + PackageCloudNextReleaseRequest( + nfpm_arch=target[NfpmArchField].value, + distro_id="", # TODO: add field for distro ID + package_name=target[NfpmPackageNameField].value, + package_version=version, + production=not is_dev, + ) + ) + release = 1 if next_release.value is None else next_release.value + fields: list[Field] = [ NfpmVersionSchemaField("semver", address=address), NfpmVersionField(version, address=address), + NfpmVersionReleaseField(release, address=address), ] return InjectedNfpmPackageFields(fields, address=address) def rules(): return [ + *packagecloud_rules(), *collect_rules(), UnionRule(SetupKwargsRequest, StackStormSetupKwargsRequest), UnionRule(InjectNfpmPackageFieldsRequest, StackStormNfpmPackageFieldsRequest), diff --git a/pants.toml b/pants.toml index 1a6ff6b16d..6c686a5748 100644 --- a/pants.toml +++ b/pants.toml @@ -30,8 +30,6 @@ backend_packages = [ # packaging "pants.backend.experimental.makeself", - - # packaging "pants.backend.experimental.nfpm", # internal plugins in pants-plugins/