diff --git a/main.py b/main.py index 248d5b5..e21950d 100644 --- a/main.py +++ b/main.py @@ -31,3 +31,7 @@ # client.summary_images() # client.update_image_tags("wip-test-update") + + + + diff --git a/presets/kubeflow-repos.yaml b/presets/kubeflow-repos.yaml index 9a23d73..c3f9c0a 100644 --- a/presets/kubeflow-repos.yaml +++ b/presets/kubeflow-repos.yaml @@ -136,11 +136,6 @@ - name: resource_dispatcher path: . url: https://github.com/canonical/resource-dispatcher -- branch: main - charms: - - name: minio - path: . - url: https://github.com/canonical/minio-operator - branch: main charms: - name: mlflow_server diff --git a/requirements.txt b/requirements.txt index d5c22ba..660f856 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ PyYAML<=6.0.2 python-hcl2==7.0.1 prettytable==3.16.0 envyaml==1.10.211231 -pandas==2.2.3 \ No newline at end of file +pandas==2.2.3 +oyaml==1.0 \ No newline at end of file diff --git a/scripts/kf7281-enable-tics/main.py b/scripts/kf7281-enable-tics/main.py index f3c4961..ba3fc6f 100644 --- a/scripts/kf7281-enable-tics/main.py +++ b/scripts/kf7281-enable-tics/main.py @@ -8,6 +8,8 @@ import json import jinja2 +CURRENT_FOLDER = Path(__file__).parent + def reformat_tox(filename: Path): """Import and export a tox file to get the formatting right. @@ -62,7 +64,7 @@ def _single_repo_tics(repo_name: str, filename: Path): filename: Path, name of the Github Action file """ - template = Path("./scripts/kf7281/tics-single-repo.yaml.j2") + template = CURRENT_FOLDER / "tics-single-repo.yaml.j2" env = jinja2.Environment() with open(template, "r") as fid: @@ -85,7 +87,7 @@ def _multi_repo_tics(repo_name: str, charms: list[LocalCharmRepo], filename: Pat charms: list[kfcicli.charms.LocalCharmRepo], list of charms to be included in tiobe scanning filename: Path, name of the Github Action file """ - template = Path("./scripts/kf7281/tics-multi-repo.yaml.j2") + template = CURRENT_FOLDER / "tics-single-repo.yaml.j2" env = jinja2.Environment() with open(template, "r") as fid: diff --git a/scripts/kf7315-update-base-noble/charmcraft.yaml b/scripts/kf7315-update-base-noble/charmcraft.yaml new file mode 100644 index 0000000..43bdada --- /dev/null +++ b/scripts/kf7315-update-base-noble/charmcraft.yaml @@ -0,0 +1,75 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +type: charm +platforms: + ubuntu@24.04:amd64: + +# Files implicitly created by charmcraft without a part: +# - dispatch (https://github.com/canonical/charmcraft/pull/1898) +# - manifest.yaml +# (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L259) +# Files implicitly copied/"staged" by charmcraft without a part: +# - actions.yaml, config.yaml, metadata.yaml +# (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L290-L293 +# https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L156-L157) +parts: + # "python-deps" part name is arbitrary; use for consistency + # (but could become a magic constant in the future, similar to "poetry-deps" + # https://github.com/canonical/craft-parts/pull/901) + python-deps: + plugin: nil + override-build: | + # Use environment variable instead of `--break-system-packages` to avoid failing on older + # versions of pip that do not recognize `--break-system-packages` + # `--user` needed (in addition to `--break-system-packages`) for Ubuntu >=24.04 + PIP_BREAK_SYSTEM_PACKAGES=true python3 -m pip install --user --upgrade pip==24.3.1 # renovate: charmcraft-pip-latest + + # "charm-python" part name is arbitrary; use for consistency + # Avoid using "charm" part name since that has special meaning to charmcraft + charm-python: + # By default, the `python` plugin creates/stages these directories: + # - lib, src + # (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_python.py#L79-L81) + # - venv + # (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_python.py#L100 + # https://github.com/canonical/craft-parts/blob/afb0d652eb330b6aaad4f40fbd6e5357d358de47/craft_parts/plugins/base.py#L270) + plugin: python + source: . + after: + - python-deps + python-requirements: [requirements.txt] + build-packages: + - libffi-dev # Needed to build Python dependencies with Rust from source + - libssl-dev # Needed to build Python dependencies with Rust from source + - pkg-config # Needed to build Python dependencies with Rust from source + override-build: | + # Workaround for https://github.com/canonical/charmcraft/issues/2068 + # rustup used to install rustc and cargo, which are needed to build Python dependencies with Rust from source + if [[ "$CRAFT_PLATFORM" == ubuntu@20.04:* || "$CRAFT_PLATFORM" == ubuntu@22.04:* ]] + then + snap install rustup --classic + else + apt-get install rustup -y + fi + + # If Ubuntu version < 24.04, rustup was installed from snap instead of from the Ubuntu + # archive—which means the rustup version could be updated at any time. Print rustup version + # to build log to make changes to the snap's rustup version easier to track + rustup --version + + # rpds-py (Python package) >=0.19.0 requires rustc >=1.76, which is not available in the + # Ubuntu 22.04 archive. Install rustc and cargo using rustup instead of the Ubuntu archive + rustup set profile minimal + rustup default 1.83.0 # renovate: charmcraft-rust-latest + + craftctl default + # Include requirements.txt in *.charm artifact for easier debugging + cp requirements.txt "$CRAFT_PART_INSTALL/requirements.txt" + + # "files" part name is arbitrary; use for consistency + files: + plugin: dump + source: . + stage: + - LICENSE \ No newline at end of file diff --git a/scripts/kf7315-update-base-noble/main.py b/scripts/kf7315-update-base-noble/main.py new file mode 100644 index 0000000..4fb003e --- /dev/null +++ b/scripts/kf7315-update-base-noble/main.py @@ -0,0 +1,141 @@ +import os +import shutil +import subprocess + +import oyaml as yaml + +from kfcicli.main import * +from kfcicli.utils import setup_logging +import json + +logger = setup_logging(log_level="INFO", logger_name=__name__) + +with open("/home/deusebio/.kfcicli/credentials.json", "r") as fid: + credentials = GitCredentials(**json.loads(fid.read())) + +tmp_folder = "/home/deusebio/.kfcicli" + +filename=Path("./presets/kubeflow-repos.yaml") + +# This only contains the adminssion-webhook and katib operators +filename=Path("./presets/test.main.yaml") + +client = KubeflowCI.read( + filename=filename, + base_path=Path(f"{tmp_folder}"), + credentials=credentials +) + +CURRENT_FOLDER = Path(__file__).parent + +def update_deps(script: Path, path: Path) -> bool: + + shutil.copy(script, path / script.name) + + try: + subprocess.check_call(["/bin/bash", script.name], cwd=path) + return True + except subprocess.CalledProcessError: + return False + finally: + os.remove(path / script.name ) + +def remove_python_step(steps: dict): + return [ + step + for step in steps + if "setup-python" not in step.get("uses", "") + ] + +def add_python_step(steps: list[dict], python_version: str = "3.12"): + idx = [ + ith for ith, step in enumerate(steps) + if "actions/checkout" in step.get("uses", "") + ] + + python_step = { + "name": f"Set up Python {python_version}", + "uses": "actions/setup-python@v5.3.0", + "with": {"python-version": python_version} + } + + if not idx: + return [python_step] + steps + + return steps[:(idx[0]+1)] + [python_step] + steps[(idx[0]+1):] + +def refactor_ci(ci: dict): + from collections import OrderedDict + jobs = OrderedDict() + for job_name, job in ci["jobs"].items(): + if "steps" in job: + job["steps"] = remove_python_step(job["steps"]) + jobs[job_name] = job + else: + jobs[job_name] = job + ci["jobs"] = jobs + return ci + +def update_base(repo: Client, charms: list[LocalCharmRepo], dry_run: bool): + + for charm in charms: + + charm_folder = ( repo.base_path / charm.tf_module ).parent + + shutil.copy(CURRENT_FOLDER / "charmcraft.yaml", charm_folder / "charmcraft.yaml" ) + + if repo.is_dirty(): + repo.update_branch( + commit_msg=f"updating charmcraft for charm {charm.name}", + directory=".", + push=not dry_run, force=True + ) + + success = update_deps(CURRENT_FOLDER / "update-deps.sh", charm_folder ) + + if not success: + logging.warning(f"Failing to update dependencies on charm {charm.name}") + + if success and repo.is_dirty(): + repo.update_branch( + commit_msg=f"updating deps for charm {charm.name}", + directory=".", + push=not dry_run, force=True + ) + + for ci_file in (repo.base_path / ".github" / "workflows").glob("*.yaml"): + + logger.info(f"Updating file {ci_file}") + with open(ci_file, "r") as fid: + ci = yaml.safe_load(fid) + + hash_1 = hash(json.dumps(ci)) + + new_ci = refactor_ci(ci) + + hash_2 = hash(json.dumps(new_ci)) + + if hash_1 != hash_2: + logger.info(f"Changes detected in file {ci_file}. Overwriting...") + with open(ci_file, "w") as fid: + yaml.dump(new_ci, fid) + + subprocess.check_output([ + "sed", '-i', 's/^true:/on:/g', str(ci_file) + ]) + else: + logger.info(f"No changes found in file {ci_file}") + + if repo.is_dirty(): + repo.update_branch( + commit_msg=f"Updating GitHub action file", directory=".", + push=not dry_run, force=True + ) + +client.canon_run( + wrapper_func=update_base, + branch_name="kf-7315-update-base", + title="[KF-7315] Update bases to 24.04", + body="PR for updating bases to 24.04, and updating also python dependencies", + dry_run=False +) diff --git a/scripts/kf7315-update-base-noble/update-deps.sh b/scripts/kf7315-update-base-noble/update-deps.sh new file mode 100644 index 0000000..541d34f --- /dev/null +++ b/scripts/kf7315-update-base-noble/update-deps.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +PATH=$PATH:/home/deusebio/.pyenv/bin + +PYTHON_VERSION=$(python --version) + +# Check that Python used is at version 3.12 +[[ "$PYTHON_VERSION" =~ "Python 3.12."[0-9]+ ]] || exit 1 + +echo "Creating env..." + +python -m venv my-env +source my-env/bin/activate + +echo "Update envs..." + +pip install tox +tox -e update-requirements + +echo "Cleanup" +rm -rf my-env \ No newline at end of file