[KF-7315] Update base to 24.04

main.py

@@ -31,3 +31,7 @@
 # client.summary_images()
 
 # client.update_image_tags("wip-test-update")
+
+
+
+

presets/kubeflow-repos.yaml

@@ -136,11 +136,6 @@
   - name: resource_dispatcher
     path: .
   url: https://github.com/canonical/resource-dispatcher
-- branch: main
-  charms:
-  - name: minio
-    path: .
-  url: https://github.com/canonical/minio-operator
 - branch: main
   charms:
   - name: mlflow_server

requirements.txt

@@ -4,4 +4,5 @@ PyYAML<=6.0.2
 python-hcl2==7.0.1
 prettytable==3.16.0
 envyaml==1.10.211231
-pandas==2.2.3
+pandas==2.2.3
+oyaml==1.0

scripts/kf7281-enable-tics/main.py

@@ -8,6 +8,8 @@
 import json
 import jinja2
 
+CURRENT_FOLDER = Path(__file__).parent
+
 def reformat_tox(filename: Path):
     """Import and export a tox file to get the formatting right.
 
@@ -62,7 +64,7 @@ def _single_repo_tics(repo_name: str, filename: Path):
         filename: Path, name of the Github Action file
     """
 
-    template = Path("./scripts/kf7281/tics-single-repo.yaml.j2")
+    template = CURRENT_FOLDER / "tics-single-repo.yaml.j2"
 
     env = jinja2.Environment()
     with open(template, "r") as fid:
@@ -85,7 +87,7 @@ def _multi_repo_tics(repo_name: str, charms: list[LocalCharmRepo], filename: Pat
         charms: list[kfcicli.charms.LocalCharmRepo], list of charms to be included in tiobe scanning
         filename: Path, name of the Github Action file
     """
-    template = Path("./scripts/kf7281/tics-multi-repo.yaml.j2")
+    template = CURRENT_FOLDER / "tics-single-repo.yaml.j2"
 
     env = jinja2.Environment()
     with open(template, "r") as fid:

scripts/kf7315-update-base-noble/charmcraft.yaml

@@ -0,0 +1,75 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+type: charm
+platforms:
+  ubuntu@24.04:amd64:
+
+# Files implicitly created by charmcraft without a part:
+# - dispatch (https://github.com/canonical/charmcraft/pull/1898)
+# - manifest.yaml
+#   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L259)
+# Files implicitly copied/"staged" by charmcraft without a part:
+# - actions.yaml, config.yaml, metadata.yaml
+#   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L290-L293
+#   https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L156-L157)
+parts:
+  # "python-deps" part name is arbitrary; use for consistency
+  # (but could become a magic constant in the future, similar to "poetry-deps"
+  # https://github.com/canonical/craft-parts/pull/901)
+  python-deps:
+    plugin: nil
+    override-build: |
+      # Use environment variable instead of `--break-system-packages` to avoid failing on older
+      # versions of pip that do not recognize `--break-system-packages`
+      # `--user` needed (in addition to `--break-system-packages`) for Ubuntu >=24.04
+      PIP_BREAK_SYSTEM_PACKAGES=true python3 -m pip install --user --upgrade pip==24.3.1  # renovate: charmcraft-pip-latest
+
+  # "charm-python" part name is arbitrary; use for consistency
+  # Avoid using "charm" part name since that has special meaning to charmcraft
+  charm-python:
+    # By default, the `python` plugin creates/stages these directories:
+    # - lib, src
+    #   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_python.py#L79-L81)
+    # - venv
+    #   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_python.py#L100
+    #   https://github.com/canonical/craft-parts/blob/afb0d652eb330b6aaad4f40fbd6e5357d358de47/craft_parts/plugins/base.py#L270)
+    plugin: python
+    source: .
+    after:
+      - python-deps
+    python-requirements: [requirements.txt]
+    build-packages:
+      - libffi-dev  # Needed to build Python dependencies with Rust from source
+      - libssl-dev  # Needed to build Python dependencies with Rust from source
+      - pkg-config  # Needed to build Python dependencies with Rust from source
+    override-build: |
+      # Workaround for https://github.com/canonical/charmcraft/issues/2068
+      # rustup used to install rustc and cargo, which are needed to build Python dependencies with Rust from source
+      if [[ "$CRAFT_PLATFORM" == ubuntu@20.04:* || "$CRAFT_PLATFORM" == ubuntu@22.04:* ]]
+      then
+        snap install rustup --classic
+      else
+        apt-get install rustup -y
+      fi
+
+      # If Ubuntu version < 24.04, rustup was installed from snap instead of from the Ubuntu
+      # archive—which means the rustup version could be updated at any time. Print rustup version
+      # to build log to make changes to the snap's rustup version easier to track
+      rustup --version
+
+      # rpds-py (Python package) >=0.19.0 requires rustc >=1.76, which is not available in the
+      # Ubuntu 22.04 archive. Install rustc and cargo using rustup instead of the Ubuntu archive
+      rustup set profile minimal
+      rustup default 1.83.0  # renovate: charmcraft-rust-latest
+
+      craftctl default
+      # Include requirements.txt in *.charm artifact for easier debugging
+      cp requirements.txt "$CRAFT_PART_INSTALL/requirements.txt"
+
+  # "files" part name is arbitrary; use for consistency
+  files:
+    plugin: dump
+    source: .
+    stage:
+      - LICENSE

scripts/kf7315-update-base-noble/main.py

@@ -0,0 +1,141 @@
+import os
+import shutil
+import subprocess
+
+import oyaml as yaml
+
+from kfcicli.main import *
+from kfcicli.utils import setup_logging
+import json
+
+logger = setup_logging(log_level="INFO", logger_name=__name__)
+
+with open("/home/deusebio/.kfcicli/credentials.json", "r") as fid:
+    credentials = GitCredentials(**json.loads(fid.read()))
+
+tmp_folder = "/home/deusebio/.kfcicli"
+
+filename=Path("./presets/kubeflow-repos.yaml")
+
+# This only contains the adminssion-webhook and katib operators
+filename=Path("./presets/test.main.yaml")
+
+client = KubeflowCI.read(
+    filename=filename,
+    base_path=Path(f"{tmp_folder}"),
+    credentials=credentials
+)
+
+CURRENT_FOLDER = Path(__file__).parent
+
+def update_deps(script: Path, path: Path) -> bool:
+
+    shutil.copy(script, path / script.name)
+
+    try:
+        subprocess.check_call(["/bin/bash", script.name], cwd=path)
+        return True
+    except subprocess.CalledProcessError:
+        return False
+    finally:
+        os.remove(path / script.name )
+
+def remove_python_step(steps: dict):
+    return [
+        step
+        for step in steps
+        if "setup-python" not in step.get("uses", "")
+    ]
+
+def add_python_step(steps: list[dict], python_version: str = "3.12"):
+    idx = [
+        ith for ith, step in enumerate(steps)
+        if "actions/checkout" in step.get("uses", "")
+    ]
+
+    python_step = {
+        "name": f"Set up Python {python_version}",
+        "uses": "actions/setup-python@v5.3.0",
+        "with": {"python-version": python_version}
+    }
+
+    if not idx:
+        return [python_step] + steps
+
+    return steps[:(idx[0]+1)] + [python_step] + steps[(idx[0]+1):]
+
+def refactor_ci(ci: dict):
+    from collections import OrderedDict
+    jobs =  OrderedDict()
+    for job_name, job in ci["jobs"].items():
+        if "steps" in job:
+            job["steps"] = remove_python_step(job["steps"])
+            jobs[job_name] = job
+        else:
+            jobs[job_name] = job
+    ci["jobs"] = jobs
+    return ci
+
+def update_base(repo: Client, charms: list[LocalCharmRepo], dry_run: bool):
+
+    for charm in charms:
+
+        charm_folder = ( repo.base_path / charm.tf_module ).parent
+
+        shutil.copy(CURRENT_FOLDER / "charmcraft.yaml", charm_folder / "charmcraft.yaml" )
+
+        if repo.is_dirty():
+            repo.update_branch(
+                commit_msg=f"updating charmcraft for charm {charm.name}",
+                directory=".",
+                push=not dry_run, force=True
+            )
+
+        success = update_deps(CURRENT_FOLDER / "update-deps.sh", charm_folder )
+
+        if not success:
+            logging.warning(f"Failing to update dependencies on charm {charm.name}")
+
+        if success and repo.is_dirty():
+            repo.update_branch(
+                commit_msg=f"updating deps for charm {charm.name}",
+                directory=".",
+                push=not dry_run, force=True
+            )
+
+    for ci_file in (repo.base_path / ".github" / "workflows").glob("*.yaml"):
+
+        logger.info(f"Updating file {ci_file}")
+        with open(ci_file, "r") as fid:
+            ci = yaml.safe_load(fid)
+
+        hash_1 = hash(json.dumps(ci))
+
+        new_ci = refactor_ci(ci)
+
+        hash_2 = hash(json.dumps(new_ci))
+
+        if hash_1 != hash_2:
+            logger.info(f"Changes detected in file {ci_file}. Overwriting...")
+            with open(ci_file, "w") as fid:
+                yaml.dump(new_ci, fid)
+
+            subprocess.check_output([
+                "sed", '-i', 's/^true:/on:/g', str(ci_file)
+            ])
+        else:
+            logger.info(f"No changes found in file {ci_file}")
+
+    if repo.is_dirty():
+        repo.update_branch(
+            commit_msg=f"Updating GitHub action file", directory=".",
+            push=not dry_run, force=True
+        )
+
+client.canon_run(
+    wrapper_func=update_base,
+    branch_name="kf-7315-update-base",
+    title="[KF-7315] Update bases to 24.04",
+    body="PR for updating bases to 24.04, and updating also python dependencies",
+    dry_run=False
+)

scripts/kf7315-update-base-noble/update-deps.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+PATH=$PATH:/home/deusebio/.pyenv/bin
+
+PYTHON_VERSION=$(python --version)
+
+# Check that Python used is at version 3.12
+[[ "$PYTHON_VERSION" =~ "Python 3.12."[0-9]+ ]] || exit 1
+
+echo "Creating env..."
+
+python -m venv my-env
+source my-env/bin/activate
+
+echo "Update envs..."
+
+pip install tox
+tox -e update-requirements
+
+echo "Cleanup"
+rm -rf my-env