diff --git a/.github/workflows/build_debian.yml b/.github/workflows/build_debian.yml index f172a6b..a34648c 100644 --- a/.github/workflows/build_debian.yml +++ b/.github/workflows/build_debian.yml @@ -2,6 +2,7 @@ name: Build Debian source package on: release: types: [published] + workflow_dispatch: jobs: check_version: runs-on: ubuntu-latest @@ -10,17 +11,22 @@ jobs: steps: - name: Checkout codebase uses: actions/checkout@v4 - - name: Extract version from release tag - id: version - run: | - VERSION=${GITHUB_REF#refs/tags/v} - echo "VERSION=${VERSION}" >> "$GITHUB_OUTPUT" - name: Extract version from changelog id: changelog_version run: | CHANGELOG_VERSION=$(dpkg-parsechangelog -S Version) echo "CHANGELOG_VERSION=${CHANGELOG_VERSION}" >> "$GITHUB_OUTPUT" + - name: Extract version from release tag + id: version + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "VERSION=${{ steps.changelog_version.outputs.CHANGELOG_VERSION }}" >> "$GITHUB_OUTPUT" + else + TAG_VERSION=${GITHUB_REF#refs/tags/v} + echo "VERSION=${TAG_VERSION}" >> "$GITHUB_OUTPUT" + fi - name: Validate Version consistency + if: ${{ github.event_name != 'workflow_dispatch' }} id: version_check env: VERSION: ${{ steps.version.outputs.VERSION }} @@ -32,6 +38,7 @@ jobs: fi echo "Version check passed: ${VERSION}" build_package: + if: ${{ github.event_name != 'workflow_dispatch' }} runs-on: ubuntu-latest needs: check_version steps: @@ -53,6 +60,7 @@ jobs: make sign-and-upload echo "upload completed successfully!" wait_for_source_builds: + if: ${{ github.event_name != 'workflow_dispatch' }} needs: - check_version - build_package @@ -80,7 +88,13 @@ jobs: if: always() run: rm -f /tmp/lp-creds.txt copy_to_other_distributions: - needs: wait_for_source_builds + needs: + - check_version + - wait_for_source_builds + if: | + always() && + needs.check_version.result == 'success' && + (needs.wait_for_source_builds.result == 'success' || needs.wait_for_source_builds.result == 'skipped') runs-on: ubuntu-latest steps: - name: Checkout codebase @@ -130,7 +144,7 @@ jobs: if: always() run: rm -f /tmp/lp-creds.txt block_release_step: - if: ${{ !github.event.release.prerelease }} + if: ${{ !github.event.release.prerelease && github.event_name != 'workflow_dispatch' }} name: Job to block publish of a release until it has been manually approved needs: wait_for_copy_builds runs-on: ubuntu-latest @@ -138,9 +152,17 @@ jobs: steps: - run: echo "Release approved — proceeding to promote to kolibri PPA." copy_package_from_proposed_to_ppa: - if: ${{ !github.event.release.prerelease }} + if: | + always() && + needs.wait_for_copy_builds.result == 'success' && + ( + (!github.event.release.prerelease && needs.block_release_step.result == 'success') || + (github.event_name == 'workflow_dispatch') + ) name: Promote packages from kolibri-proposed to kolibri - needs: block_release_step + needs: + - wait_for_copy_builds + - block_release_step runs-on: ubuntu-latest steps: - name: Checkout codebase diff --git a/README.rst b/README.rst index 011aa2d..c693866 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ To fetch and build a new version of this package, the following workflow is sugg You can optimize this workflow according to your own needs. -Changes can be built and released in ``kolibri-proposed`` by the `Learning Equality Launchpad team `__. +Changes can be built and released in ``kolibri-proposed`` by the `Learning Equality Launchpad team `__. Working in the repo ------------------- @@ -53,13 +53,82 @@ After this, pre-commit hooks will run automatically on ``git commit``. To run al Releasing --------- -Push new changes to ``kolibri-proposed`` and test them there. +Automated release workflow +~~~~~~~~~~~~~~~~~~~~~~~~~~ -To build packages for all current Ubuntu release series: +Publishing a GitHub release triggers the ``build_debian.yml`` workflow, which: -#. Install Launchpadlib: ``sudo apt install python-launchpadlib`` -#. Run ``ppa-copy-packages.py`` script to copy the builds for Xenial to all other currently active and supported Ubuntu releases on Launchpad. The script is run from command line with ``python2 ppa-copy-packages.py``. After this, you should be prompted to create an API key for your Launchpad account. -#. When a release in ``kolibri-proposed`` should be released as a stable release, use the binary copy function on Launchpad to copy builds from ``kolibri-proposed``. +#. Validates the release tag version against ``debian/changelog`` +#. Builds, signs, and uploads the source package to the ``kolibri-proposed`` PPA via ``dput`` +#. Waits for Launchpad to build the source package +#. Copies the built package to all supported Ubuntu series +#. Waits for all copy builds to complete +#. (Non-prerelease only) Requires manual approval via the ``release`` environment +#. Promotes packages from ``kolibri-proposed`` to ``kolibri`` PPA + +Launchpad credentials setup +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The workflow requires Launchpad API credentials stored as a GitHub Actions secret. + +To generate credentials: + +#. Install launchpadlib: ``pip install launchpadlib`` +#. Run the credentials helper script:: + + python3 scripts/create_lp_creds.py + +#. Approve the authorization request in your browser. This writes a credentials file (default: ``launchpad.credentials``). +#. Copy the full content of the credentials file. +#. In GitHub, go to the repository **Settings > Secrets and variables > Actions > New repository secret**. +#. Create a secret named ``LP_CREDENTIALS`` and paste the credentials file content. + +The workflow writes this secret to a temporary file at runtime and cleans it up after each job. + +Manual workflow dispatch +~~~~~~~~~~~~~~~~~~~~~~~~ + +The workflow supports a ``workflow_dispatch`` trigger for manual reruns. This is useful when a release workflow fails partway through — you can fix the issue and rerun the workflow without it breaking because earlier steps already succeeded. + +To trigger from the GitHub UI: + +#. Go to **Actions > Build Debian source package > Run workflow** +#. Click **Run workflow** + +To trigger from the command line:: + + gh workflow run build_debian.yml + +When triggered via ``workflow_dispatch``: + +- The ``build_package`` and ``wait_for_source_builds`` jobs are skipped (no release artifact to upload) +- The version is read from ``debian/changelog`` instead of the release tag +- The ``block_release_step`` manual approval gate is skipped +- All copy and promote steps run normally — they are idempotent and safely handle packages that were already copied in a previous run + +Launchpad copy script +~~~~~~~~~~~~~~~~~~~~~ + +The ``scripts/launchpad_copy.py`` script manages Launchpad PPA operations with three subcommands: + +``copy-to-series`` + Copies packages from the source Ubuntu series to all other supported series within the ``kolibri-proposed`` PPA:: + + python3 scripts/launchpad_copy.py copy-to-series + +``promote`` + Promotes all published packages from ``kolibri-proposed`` to the ``kolibri`` PPA:: + + python3 scripts/launchpad_copy.py promote + +``wait-for-builds`` + Polls Launchpad until all builds for a source package reach a terminal state:: + + python3 scripts/launchpad_copy.py wait-for-builds --package kolibri-server --version 1.0.0 + +All subcommands are idempotent — rerunning them after a partial success safely skips packages that were already copied or promoted. + +Additional flags: ``-v`` / ``-vv`` for verbosity, ``-q`` for quiet mode, ``--debug`` for HTTP-level debugging. Overview -------- diff --git a/scripts/launchpad_copy.py b/scripts/launchpad_copy.py index a191dfa..50673d7 100644 --- a/scripts/launchpad_copy.py +++ b/scripts/launchpad_copy.py @@ -247,13 +247,19 @@ def perform_queued_copies(self, ppa): log.info("") first = False log.info("Copying %s to %s", ", ".join(sorted(names)), target_series) - ppa.syncSources( - from_archive=ppa, - to_series=target_series, - to_pocket=pocket, - include_binaries=True, - source_names=sorted(names), - ) + try: + ppa.syncSources( + from_archive=ppa, + to_series=target_series, + to_pocket=pocket, + include_binaries=True, + source_names=sorted(names), + ) + except lre.BadRequest as e: + if "same version already published" in str(e): + log.info("Already copied to %s — skipping", target_series) + else: + raise def copy_to_series(self): """Copy packages from source series to all other supported Ubuntu series.""" @@ -410,12 +416,19 @@ def promote(self): ) copied_any = True except lre.BadRequest as e: - if "is obsolete and will not accept new uploads" in str(e): + msg = str(e) + if "is obsolete and will not accept new uploads" in msg: log.info( "Skip obsolete series for %s %s", pkg.source_package_name, pkg.source_package_version, ) + elif "same version already published" in msg: + log.info( + "Already published %s %s — skipping", + pkg.source_package_name, + pkg.source_package_version, + ) else: raise @@ -441,7 +454,6 @@ def build_parser(): ) parser.add_argument("-q", "--quiet", action="store_true", help="Suppress info output.") parser.add_argument("--debug", action="store_true", help="Enable HTTP debug output.") - subparsers = parser.add_subparsers(dest="command", required=True) subparsers.add_parser( diff --git a/tests/test_launchpad_copy.py b/tests/test_launchpad_copy.py index 742ef2e..cfa0c1e 100644 --- a/tests/test_launchpad_copy.py +++ b/tests/test_launchpad_copy.py @@ -215,6 +215,47 @@ def test_perform_queued_copies_skips_empty_queues(self): mock_ppa.syncSources.assert_not_called() + def test_perform_queued_copies_handles_already_synced(self): + """Idempotency: syncSources errors for already-copied packages are handled gracefully.""" + wrapper = LaunchpadWrapper() + wrapper.queue_copy("kolibri-server", "jammy", "noble", "Release") + + class MockBadRequest(Exception): + pass + + mock_ppa = MagicMock() + mock_ppa.syncSources.side_effect = MockBadRequest( + "kolibri-server 0.9.0 in noble (same version already published)" + ) + + with patch("launchpad_copy.lre") as mock_lre: + mock_lre.BadRequest = MockBadRequest + wrapper.perform_queued_copies(mock_ppa) + + # Should not raise — the error is handled gracefully + + def test_perform_queued_copies_logs_already_synced(self, caplog): + """Idempotency: logs a message when syncSources finds package already exists.""" + wrapper = LaunchpadWrapper() + wrapper.queue_copy("kolibri-server", "jammy", "noble", "Release") + + class MockBadRequest(Exception): + pass + + mock_ppa = MagicMock() + mock_ppa.syncSources.side_effect = MockBadRequest( + "kolibri-server 0.9.0 in noble (same version already published)" + ) + + with ( + patch("launchpad_copy.lre") as mock_lre, + caplog.at_level(logging.INFO, logger=log.name), + ): + mock_lre.BadRequest = MockBadRequest + wrapper.perform_queued_copies(mock_ppa) + + assert any("already" in r.message.lower() for r in caplog.records) + def test_get_usable_sources_filters_by_whitelist(self): wrapper = LaunchpadWrapper() mock_ppa = MagicMock() @@ -475,6 +516,85 @@ def test_skips_non_whitelisted_package(self): mock_dest_ppa.copyPackage.assert_not_called() assert result == 0 + def test_handles_already_published_package_gracefully(self): + """Idempotency: promote skips packages already copied to dest PPA.""" + wrapper = LaunchpadWrapper() + mock_source_ppa = MagicMock() + mock_dest_ppa = MagicMock() + + mock_pkg = MagicMock() + mock_pkg.source_package_name = "kolibri-server" + mock_pkg.source_package_version = "0.9.0" + mock_pkg.distro_series_link = "https://lp/ubuntu/jammy" + mock_pkg.pocket = "Release" + + mock_source_ppa.getPublishedSources.return_value = [mock_pkg] + + class MockBadRequest(Exception): + pass + + mock_dest_ppa.copyPackage.side_effect = MockBadRequest( + "kolibri-server 0.9.0 in jammy (same version already published in the target archive)" + ) + + with ( + patch.object( + type(wrapper), + "proposed_ppa", + new_callable=lambda: property(lambda self: mock_source_ppa), + ), + patch.object( + type(wrapper), + "release_ppa", + new_callable=lambda: property(lambda self: mock_dest_ppa), + ), + patch("launchpad_copy.lre") as mock_lre, + ): + mock_lre.BadRequest = MockBadRequest + result = wrapper.promote() + + assert result == 0 + + def test_already_published_logs_skip_message(self, caplog): + """Idempotency: promote logs that a package was already promoted.""" + wrapper = LaunchpadWrapper() + mock_source_ppa = MagicMock() + mock_dest_ppa = MagicMock() + + mock_pkg = MagicMock() + mock_pkg.source_package_name = "kolibri-server" + mock_pkg.source_package_version = "0.9.0" + mock_pkg.distro_series_link = "https://lp/ubuntu/jammy" + mock_pkg.pocket = "Release" + + mock_source_ppa.getPublishedSources.return_value = [mock_pkg] + + class MockBadRequest(Exception): + pass + + mock_dest_ppa.copyPackage.side_effect = MockBadRequest( + "kolibri-server 0.9.0 in jammy (same version already published in the target archive)" + ) + + with ( + patch.object( + type(wrapper), + "proposed_ppa", + new_callable=lambda: property(lambda self: mock_source_ppa), + ), + patch.object( + type(wrapper), + "release_ppa", + new_callable=lambda: property(lambda self: mock_dest_ppa), + ), + patch("launchpad_copy.lre") as mock_lre, + caplog.at_level(logging.INFO, logger=log.name), + ): + mock_lre.BadRequest = MockBadRequest + wrapper.promote() + + assert any("already published" in r.message.lower() and "kolibri-server" in r.message for r in caplog.records) + # --- wait-for-builds tests ---