diff --git a/.github/workflows/Publish.yaml b/.github/workflows/Publish.yaml index 927b3ef..53ac64f 100644 --- a/.github/workflows/Publish.yaml +++ b/.github/workflows/Publish.yaml @@ -9,8 +9,8 @@ env: jobs: build-binary: - # building on ubuntu-20.04 so we'll link to glibc 2.31 (bullseye+) - runs-on: ubuntu-24.04 + # building on ubuntu-22.04 so we'll link to glibc 2.35 (bookworm+) + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/QA.yaml b/.github/workflows/QA.yaml index 48ccee5..28dbf5b 100644 --- a/.github/workflows/QA.yaml +++ b/.github/workflows/QA.yaml @@ -8,7 +8,7 @@ on: jobs: check-qa: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 @@ -24,9 +24,6 @@ jobs: pip install -U pip pip install -e .[lint,scripts,test,check] - - name: Check black formatting - run: inv lint-black - - name: Check ruff run: inv lint-ruff diff --git a/.github/workflows/Tests.yaml b/.github/workflows/Tests.yaml index 20df6a3..18b7ef2 100644 --- a/.github/workflows/Tests.yaml +++ b/.github/workflows/Tests.yaml @@ -11,8 +11,8 @@ env: jobs: build-binary: - # building on ubuntu-20.04 so we'll link to glibc 2.31 (bullseye+) - runs-on: ubuntu-24.04 + # building on ubuntu-22.04 so we'll link to glibc 2.35 (bookworm+) + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 280cfa0..7637de5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,20 +2,16 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer -- repo: https://github.com/psf/black - rev: "24.2.0" - hooks: - - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.2 + rev: v0.15.5 hooks: - id: ruff - repo: https://github.com/RobertCraigie/pyright-python - rev: v1.1.352 + rev: v1.1.408 hooks: - id: pyright name: pyright (system) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf8662a..27c9316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.1.5] - 2026-03-11 ### Fixed - [workflows] Uploading to correct folder (#41) +- Support for latest GHCR images (OCI manifest) via docker-export 1.2 + +### Changed + +- Bumped most dependencies ## [1.1.4] - 2025-12-10 diff --git a/get-oci-sizes.py b/get-oci-sizes.py index e288472..3c203f2 100755 --- a/get-oci-sizes.py +++ b/get-oci-sizes.py @@ -25,7 +25,7 @@ """ import pathlib -import subprocess +import subprocess # noqa: S404 import sys import tarfile import tempfile diff --git a/pyproject.toml b/pyproject.toml index fd39bdf..f407203 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,47 +15,51 @@ license = {text = "GPL-3.0-or-later"} classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", ] dependencies = [ - "requests==2.31.0", + "requests==2.32.5", + "types-requests==2.32.4.20260107", "PyYAML==6.0.1", - "cli-ui==0.17.2", + "cli-ui==0.19.0", "humanfriendly==10.0", - "progressbar2==4.4.2", - "docker-export==1.1.0", - "typeguard==4.1.5", - "offspot-config==2.2.0", + "progressbar2==4.5.0", + "docker-export==1.2.0", + "typeguard==4.5.1", + "offspot-config==2.9.2", # "offspot-config@git+https://github.com/offspot/offspot-config#egg=main", "natsort==8.4.0", - "aria2p==0.12.0", + "aria2p==0.12.1", ] dynamic = ["version"] [project.optional-dependencies] scripts = [ - "invoke==2.2.0", + "invoke==2.2.1", + "types-invoke==2.0.0.10", ] lint = [ - "black==24.2.0", - "ruff==0.2.2", + "ruff==0.15.5", ] check = [ - "pyright==1.1.358", + "pyright==1.1.408", ] test = [ - "pytest==8.0.2", - "pytest-cov==4.1.0", - "coverage==7.4.3", + "pytest==9.0.2", + "pytest-cov==7.0.0", + "coverage==7.13.4", ] binary = [ "ordered-set==4.1.0", "zstandard==0.25.0", - "nuitka==2.8.9", - "requests==2.31.0", + "nuitka==4.0.3", + "requests==2.32.5", ] dev = [ - "pre-commit==3.6.0", + "pre-commit==4.5.1", "image-creator[scripts]", "image-creator[lint]", "image-creator[test]", @@ -81,7 +85,7 @@ exclude = [ ] [[tool.hatch.envs.default.matrix]] -python = ["3.11"] +python = ["3.11", "3.12", "3.13", "3.14"] [tool.hatch.envs.default] features = ["dev"] @@ -90,7 +94,7 @@ features = ["dev"] features = ["scripts", "test"] [[tool.hatch.envs.test.matrix]] -python = ["3.11"] +python = ["3.11", "3.12", "3.13", "3.14"] [tool.hatch.envs.test.scripts] run = "inv test --args '{args}'" @@ -115,6 +119,9 @@ fixall = "inv fixall --args '{args}'" [tool.hatch.envs.check] features = ["scripts", "check"] +[[tool.hatch.envs.check.matrix]] +python = ["3.11", "3.12", "3.13", "3.14"] + [tool.hatch.envs.check.scripts] pyright = "inv check-pyright --args '{args}'" all = "inv checkall --args '{args}'" @@ -123,22 +130,21 @@ all = "inv checkall --args '{args}'" features = ["scripts", "binary"] [[tool.hatch.envs.binary.matrix]] -python = ["3.11"] +python = ["3.13"] [tool.hatch.envs.binary.scripts] build = "inv binary --filename '{args}'" download-aria2 = "inv download-aria2c" -[tool.black] -line-length = 88 -target-version = ['py311'] - [tool.ruff] -target-version = "py311" line-length = 88 src = ["src"] +[tool.ruff.format] +preview = true + [tool.ruff.lint] +preview = true select = [ "A", # flake8-builtins # "ANN", # flake8-annotations @@ -210,7 +216,9 @@ ignore = [ # Ignore complexity "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", # naive UTC timezones - "DTZ003" + "DTZ003", + # allow method that dont use self + "PLR6301", ] unfixable = [ # Don't touch unused imports @@ -220,10 +228,6 @@ unfixable = [ [tool.ruff.lint.isort] known-first-party = ["image_creator"] -[tool.ruff.lint.flake8-bugbear] -# add exceptions to B008 for fastapi. -extend-immutable-calls = ["fastapi.Depends", "fastapi.Query"] - [tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "all" diff --git a/src/image_creator/__init__.py b/src/image_creator/__init__.py index c72e379..9b102be 100644 --- a/src/image_creator/__init__.py +++ b/src/image_creator/__init__.py @@ -1 +1 @@ -__version__ = "1.1.4" +__version__ = "1.1.5" diff --git a/src/image_creator/cache/manager.py b/src/image_creator/cache/manager.py index 91353f6..d8e8557 100644 --- a/src/image_creator/cache/manager.py +++ b/src/image_creator/cache/manager.py @@ -42,7 +42,8 @@ def path_for_image(image: OCIImage) -> pathlib.Path: if image.oci.digest: fname += f"@{image.oci.digest}" return ( - pathlib.Path("images") + pathlib + .Path("images") .joinpath(image.oci.registry) .joinpath(image.oci.repository) .joinpath(fname) @@ -74,7 +75,8 @@ def path_for_file(file: File) -> pathlib.Path: fname += f"#{file.url.fragment}" return ( - pathlib.Path("files") + pathlib + .Path("files") .joinpath(file.url.scheme) .joinpath(file.url.netloc) # excluding last part (basename) and first (leading slash) @@ -250,9 +252,7 @@ def matched_ident_version(entry: CacheEntry) -> idv | None: return if hasattr(policy, "filters"): - for ( - filter_ - ) in ( + for filter_ in ( policy.filters # pyright: ignore [reportAttributeAccessIssue, reportGeneralTypeIssues] ): filter_num = 0 @@ -277,32 +277,30 @@ def matched_ident_version(entry: CacheEntry) -> idv | None: continue if filter_.max_age_dt and entry.added_on < filter_.max_age_dt: - evictions.append( + evictions.append(( + entry, ( - entry, "Too old for filter max_age " - f"({format_duration(filter_.max_age_seconds)})", - ) - ) + f"({format_duration(filter_.max_age_seconds)})" + ), + )) continue if filter_.max_size and filter_size + entry.size > filter_.max_size: - evictions.append( + evictions.append(( + entry, ( - entry, "Would exceed filter max_size " - f"({format_size(filter_.max_size_bytes)})", - ) - ) + f"({format_size(filter_.max_size_bytes)})" + ), + )) continue if filter_.max_num and filter_num + 1 > filter_.max_num: - evictions.append( - ( - entry, - f"Would exceed filter max_num ({filter_.max_num}", - ) - ) + evictions.append(( + entry, + f"Would exceed filter max_num ({filter_.max_num}", + )) continue # if filter requested version numbers, keep a list of candidates @@ -329,17 +327,13 @@ def matched_ident_version(entry: CacheEntry) -> idv | None: # sort (naturally) by version (from filename) for obsolete in natsorted( item["entries"], - key=lambda item: matched_ident_version( - item - ).version, # pyright: ignore [reportOptionalMemberAccess] + key=lambda item: matched_ident_version(item).version, # pyright: ignore [reportOptionalMemberAccess] )[: -filter_.keep_identified_versions]: - evictions.append( - ( - obsolete, - "Version now obsolete (keeping " - f"only {filter_.keep_identified_versions})", - ) - ) + evictions.append(( + obsolete, + ("Version now obsolete (keeping " + f"only {filter_.keep_identified_versions})"), + )) total_versioned_entries = {} @@ -353,33 +347,26 @@ def matched_ident_version(entry: CacheEntry) -> idv | None: continue if policy.max_age_dt and entry.added_on < policy.max_age_dt: - evictions.append( - ( - entry, - f"Too old for {type(policy).__name__} max_age " - f"({format_duration(policy.max_age_seconds)})", - ) - ) + evictions.append(( + entry, + (f"Too old for {type(policy).__name__} max_age " + f"({format_duration(policy.max_age_seconds)})"), + )) continue if policy.max_size and total_size + entry.size > policy.max_size: - evictions.append( - ( - entry, - f"Would exceed {type(policy).__name__} max_size " - f"({format_size(policy.max_size_bytes)})", - ) - ) + evictions.append(( + entry, + (f"Would exceed {type(policy).__name__} max_size " + f"({format_size(policy.max_size_bytes)})"), + )) continue if policy.max_num and total_num + 1 > policy.max_num: - evictions.append( - ( - entry, - f"Would exceed {type(policy).__name__} max_num " - f"({policy.max_num})", - ) - ) + evictions.append(( + entry, + f"Would exceed {type(policy).__name__} max_num ({policy.max_num})", + )) continue # if filter requested version numbers, keep a list of candidates @@ -406,17 +393,13 @@ def matched_ident_version(entry: CacheEntry) -> idv | None: # sort (naturally) by version (from filename) for obsolete in natsorted( item["entries"], - key=lambda item: matched_ident_version( - item - ).version, # pyright: ignore [reportOptionalMemberAccess] + key=lambda item: matched_ident_version(item).version, # pyright: ignore [reportOptionalMemberAccess] )[: -policy.keep_identified_versions]: - evictions.append( - ( - obsolete, - "Version now obsolete (keeping " - f"only {policy.keep_identified_versions})", - ) - ) + evictions.append(( + obsolete, + ("Version now obsolete (keeping " + f"only {policy.keep_identified_versions})"), + )) return evictions @@ -463,7 +446,7 @@ def size(self) -> int: """total size of cache""" if not self.discovered: self.walk() - return sum([entry.size for entry in self.entries.values()]) + return sum(entry.size for entry in self.entries.values()) def get(self, item: Item) -> CacheEntry: """CacheEntry for Item""" diff --git a/src/image_creator/cache/policy.py b/src/image_creator/cache/policy.py index 5f3899f..5eb7292 100644 --- a/src/image_creator/cache/policy.py +++ b/src/image_creator/cache/policy.py @@ -231,11 +231,11 @@ def read_from(cls, text: str): payload = yaml_load(text, Loader=SafeLoader) # Subpolicies have subclasses for human-friendlyness - _sub_policy_map = {"oci_images": OCIImagePolicy, "files": FilesPolicy} + sub_policy_map = {"oci_images": OCIImagePolicy, "files": FilesPolicy} # build SubPolicies first (args of the main Policy) for name in cls.sub_names(): - sub_policy_cls = _sub_policy_map.get(name, SubPolicy) + sub_policy_cls = sub_policy_map.get(name, SubPolicy) # remove he key from payload ; we'll replace it with actual SubPolocy subload = payload.pop(name, None) diff --git a/src/image_creator/entrypoint.py b/src/image_creator/entrypoint.py index bfa17ab..e3f9784 100644 --- a/src/image_creator/entrypoint.py +++ b/src/image_creator/entrypoint.py @@ -80,8 +80,8 @@ def main(): kwargs = dict(parser.parse_args()._get_kwargs()) Global._debug = kwargs.get("debug", False) - # purposedly import after setting debug - from image_creator.creator import ImageCreator + # purposely import after setting debug + from image_creator.creator import ImageCreator # noqa: PLC0415 try: app = ImageCreator( diff --git a/src/image_creator/logger.py b/src/image_creator/logger.py index 6520185..4ef4b4f 100644 --- a/src/image_creator/logger.py +++ b/src/image_creator/logger.py @@ -18,7 +18,7 @@ warn = ui.UnicodeSequence(ui.brown, "⚠️ ", "[!]") # noqa: RUF001 -class Logger: +class Logger: # noqa: PLR0904 """Custom cli_ui-based logger providing ~unified UI for steps and tasks Operations are either Steps or tasks within a Step @@ -123,7 +123,7 @@ def mark_as(self, what: str): def clear(self): """clear in-task or in-step same-line hanging to prevent writing to previous""" - if self.currently in ("task",): + if self.currently in {"task"}: ui.info("") def start_step(self, step: str): @@ -146,7 +146,7 @@ def start_task(self, task: str): ui.message(" ", ui.bold, ui.blue, "=>", ui.reset, task, end=" ") ui.CONFIG["timestamp"] = False - def end_task(self, success: bool | None = None, message: str | None = None): + def end_task(self, *, success: bool | None = None, message: str | None = None): """End current task with custom success symbol and message""" tokens = [] if success is None else [ui.check if success else ui.cross] if message: diff --git a/src/image_creator/steps/__init__.py b/src/image_creator/steps/__init__.py index 037732f..e69de29 100644 --- a/src/image_creator/steps/__init__.py +++ b/src/image_creator/steps/__init__.py @@ -1,42 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from image_creator.constants import logger - - -class Step: - """StepInterface""" - - # name of step to be overriden - @property - def name(self) -> str: - return getattr(self, "_name", repr(self)) - return repr(self) - - def __repr__(self): - return self.__class__.__name__ - - def __str__(self): - return self.name - - def run(self, payload: dict[str, Any]) -> int: - """actual step implementation. 0 on success""" - raise NotImplementedError() - - def cleanup(self, payload: dict[str, Any]): - """clean resources reserved in run()""" - ... - - -class VirtualInitStep(Step): ... - - -class GivingFeedback(Step): - _name: str = "Giving creation feedback" - - def run(self, payload: dict[str, Any]) -> int: - payload["succeeded"] = True - logger.start_task("Image created successfuly") - logger.succeed_task(str(payload["options"].output_path)) - return 0 diff --git a/src/image_creator/steps/base.py b/src/image_creator/steps/base.py index b264e4f..f7dcc74 100644 --- a/src/image_creator/steps/base.py +++ b/src/image_creator/steps/base.py @@ -12,7 +12,7 @@ ) from image_creator.constants import logger -from image_creator.steps import Step +from image_creator.steps.step import Step class DownloadImage(Step): diff --git a/src/image_creator/steps/cache.py b/src/image_creator/steps/cache.py index 3475527..27edcb6 100644 --- a/src/image_creator/steps/cache.py +++ b/src/image_creator/steps/cache.py @@ -8,7 +8,7 @@ from image_creator.cache.manager import CacheManager from image_creator.cache.policy import MainPolicy from image_creator.constants import logger -from image_creator.steps import Step +from image_creator.steps.step import Step class CheckCache(Step): diff --git a/src/image_creator/steps/check_inputs.py b/src/image_creator/steps/check_inputs.py index 1b4ce9e..48a45eb 100644 --- a/src/image_creator/steps/check_inputs.py +++ b/src/image_creator/steps/check_inputs.py @@ -16,7 +16,7 @@ from image_creator.constants import Global, logger -from image_creator.steps import Step +from image_creator.steps.step import Step from image_creator.utils import requirements from image_creator.utils.download import read_text_from @@ -77,7 +77,7 @@ def run(self, payload: dict[str, Any]) -> int: "check_target_location", "check_target_nondestructive", ): - res = getattr(self, method).__call__(payload) + res = getattr(self, method)(payload) if res != 0: return res @@ -248,10 +248,10 @@ def run(self, payload: dict[str, Any]) -> int: if payload["cache"].candidates: nb_candidates = len(payload["cache"].candidates) size_candidates = sum( - [entry.size for entry in payload["cache"].candidates.values()] + entry.size for entry in payload["cache"].candidates.values() ) msgs.append(f"{nb_candidates} to add ({format_size(size_candidates)})") - logger.end_task(message=". ".join(msgs)) + logger.end_task(success=None, message=". ".join(msgs)) return 0 if all_valid else 4 diff --git a/src/image_creator/steps/contents.py b/src/image_creator/steps/contents.py index 4638425..516ac41 100644 --- a/src/image_creator/steps/contents.py +++ b/src/image_creator/steps/contents.py @@ -23,7 +23,7 @@ from image_creator.cache.manager import CacheManager from image_creator.constants import logger -from image_creator.steps import Step +from image_creator.steps.step import Step from image_creator.utils.aria2 import Download, Downloader, DownloadError, Feedback @@ -168,7 +168,7 @@ def finish(self): class FilesProcessor: """Downloads the File objects supplied using a ThreadPoolExecutor""" - def __init__( + def __init__( # noqa: PLR0917 self, files, cache: CacheManager, @@ -347,7 +347,7 @@ def on_completion(*, file: File, succeeded: bool, feedback: Feedback | None): ) bytes_total = sum( - [file.size if file.size else 0 for file in payload["config"].remote_files] + file.size if file.size else 0 for file in payload["config"].remote_files ) manager = FilesProcessor( diff --git a/src/image_creator/steps/image.py b/src/image_creator/steps/image.py index c159fa9..78a15bf 100644 --- a/src/image_creator/steps/image.py +++ b/src/image_creator/steps/image.py @@ -5,7 +5,7 @@ from offspot_config.utils.misc import format_size from image_creator.constants import logger -from image_creator.steps import Step +from image_creator.steps.step import Step from image_creator.utils.image import Image diff --git a/src/image_creator/steps/machine.py b/src/image_creator/steps/machine.py index 6c005ab..6132418 100644 --- a/src/image_creator/steps/machine.py +++ b/src/image_creator/steps/machine.py @@ -4,7 +4,6 @@ from image_creator.constants import logger from image_creator.logger import Status -from image_creator.steps import GivingFeedback, VirtualInitStep from image_creator.steps.base import DownloadImage from image_creator.steps.cache import ApplyCachePolicy, CheckCache, PrintingCache from image_creator.steps.check_inputs import ( @@ -28,6 +27,7 @@ ) from image_creator.steps.oci_images import DownloadingOCIImages from image_creator.steps.sizes import ComputeSizes +from image_creator.steps.step import GivingFeedback, VirtualInitStep class StepMachine: @@ -76,7 +76,7 @@ def remove_step(cls, step: str): cls.steps.remove(stepcls) def _get_step(self, index: int): - return self.steps[index].__call__() + return self.steps[index]() @property def step_num(self): diff --git a/src/image_creator/steps/oci_images.py b/src/image_creator/steps/oci_images.py index 6ba61bd..11bdb67 100644 --- a/src/image_creator/steps/oci_images.py +++ b/src/image_creator/steps/oci_images.py @@ -9,7 +9,7 @@ from offspot_config.utils.misc import copy_file, format_size, get_filesize, rmtree from image_creator.constants import Global, logger -from image_creator.steps import Step +from image_creator.steps.step import Step def download_image(image: OCIImage, dest: pathlib.Path, build_dir: pathlib.Path): diff --git a/src/image_creator/steps/sizes.py b/src/image_creator/steps/sizes.py index 7ab348c..d697e99 100644 --- a/src/image_creator/steps/sizes.py +++ b/src/image_creator/steps/sizes.py @@ -6,7 +6,7 @@ from offspot_config.utils.misc import format_size, get_freespace from image_creator.constants import logger -from image_creator.steps import Step +from image_creator.steps.step import Step def round_for_cluster(size: int) -> int: @@ -39,13 +39,13 @@ class ComputeSizes(Step): def run(self, payload: dict[str, Any]) -> int: tar_images_size = sum( - [image.filesize for image in payload["config"].all_images] + image.filesize for image in payload["config"].all_images ) expanded_images_size = sum( - [image.fullsize for image in payload["config"].all_images] + image.fullsize for image in payload["config"].all_images ) expanded_files_size = sum( - [file.fullsize for file in payload["config"].all_files] + file.fullsize for file in payload["config"].all_files ) raw_content_size = sum( @@ -110,21 +110,19 @@ def get_needs(self, payload: dict[str, Any], image_size: int) -> dict[str, int]: remote_compressed_files = [ file for file in payload["config"].remote_files if file.via != "direct" ] - needs["build_dir"] = sum([file.size for file in remote_compressed_files]) + needs["build_dir"] = sum(file.size for file in remote_compressed_files) # cache needs: # - what will be introduced to cache if payload["options"].cache_dir: needs["cache_dir"] = sum( - [entry.size for entry in payload["cache"].candidates.values()] + entry.size for entry in payload["cache"].candidates.values() ) # exclude from build-dir the size of those we'll have in cache needs["build_dir"] -= sum( - [ - file.size + file.size for file in remote_compressed_files if file in payload["cache"] or payload["cache"].has_candidate(file) - ] ) return needs diff --git a/src/image_creator/steps/step.py b/src/image_creator/steps/step.py new file mode 100644 index 0000000..037732f --- /dev/null +++ b/src/image_creator/steps/step.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import Any + +from image_creator.constants import logger + + +class Step: + """StepInterface""" + + # name of step to be overriden + @property + def name(self) -> str: + return getattr(self, "_name", repr(self)) + return repr(self) + + def __repr__(self): + return self.__class__.__name__ + + def __str__(self): + return self.name + + def run(self, payload: dict[str, Any]) -> int: + """actual step implementation. 0 on success""" + raise NotImplementedError() + + def cleanup(self, payload: dict[str, Any]): + """clean resources reserved in run()""" + ... + + +class VirtualInitStep(Step): ... + + +class GivingFeedback(Step): + _name: str = "Giving creation feedback" + + def run(self, payload: dict[str, Any]) -> int: + payload["succeeded"] = True + logger.start_task("Image created successfuly") + logger.succeed_task(str(payload["options"].output_path)) + return 0 diff --git a/src/image_creator/utils/aria2.py b/src/image_creator/utils/aria2.py index ffce389..4f42a5d 100644 --- a/src/image_creator/utils/aria2.py +++ b/src/image_creator/utils/aria2.py @@ -5,7 +5,7 @@ import shutil import signal import socket -import subprocess +import subprocess # noqa: S404 import tempfile import time import uuid @@ -14,7 +14,12 @@ from typing import NamedTuple, Self import aria2p -from requests.exceptions import ConnectionError, RequestException +from requests.exceptions import ( + ConnectionError as RequestsConnectionError, +) +from requests.exceptions import ( + RequestException, +) from image_creator import __version__ from image_creator.constants import logger @@ -205,7 +210,7 @@ def stop(self): time.sleep(1) -class Download(aria2p.Download): +class Download(aria2p.Download): # noqa: PLR0904 """aria2p Download subclass with additional accessors You would mostly be receiving them from Downloader.add()""" @@ -277,7 +282,9 @@ def is_processing_or_raises(self) -> bool: if succeeded is None: return True if succeeded is False: - raise DownloadError(*self.error) + if self.error: + raise DownloadError(*self.error) + raise DownloadError("Unknown error") return False @property @@ -395,7 +402,7 @@ def create_from( def update(self): try: super().update() - except (RequestException, ConnectionError) as exc: + except (RequestException, RequestsConnectionError) as exc: if not self.done: raise exc @@ -515,13 +522,13 @@ def is_dottorrent(self) -> bool: def followers_active(self): """Whether this has still active followers""" return any( - dl.status in ("active", "waiting", "paused") for dl in self.followed_by + dl.status in {"active", "waiting", "paused"} for dl in self.followed_by ) @property def active(self): """Whether this very download is active""" - return self.status in ("active", "waiting", "paused") + return self.status in {"active", "waiting", "paused"} @property def actual_files(self) -> list[aria2p.File]: @@ -544,7 +551,7 @@ def is_real_file(file: aria2p.File, from_dl: Download) -> bool: # WARN: this is our Kiwix convention. Could be anything. # maybe retrieve Content Type initially - if self.is_metalink and file.path.suffix in (".meta4", ".metalink"): + if self.is_metalink and file.path.suffix in {".meta4", ".metalink"}: return False if self.is_torrent and file.path.suffix == ".torrent": return False @@ -569,7 +576,6 @@ def post_process(self): return if not self.error: - # we only post-process the metadata dl (once all followers have completed) if self.is_torrent and not self.is_metadata: return @@ -694,7 +700,6 @@ class Downloader: # not for BT, not for ML if pieces hashes ("piece_length", "1M"), ("optimize_concurrent_downloads", "false"), - # ("rpc_save_upload_metadata", True), ("user_agent", f"offspot image-creator/{__version__}"), ("force_sequential", False), @@ -1030,7 +1035,7 @@ def _cb_on_download_stop(self, api: aria2p.API, gid: str): # noqa: ARG002 """aria2p.API callback for a stopped download""" if not self.is_ours(gid): return - logger.debug("Download #{gid} has been stopped") + logger.debug(f"Download #{gid} has been stopped") dl = self.get(gid) if dl.callback: diff --git a/src/image_creator/utils/image.py b/src/image_creator/utils/image.py index 2fee3fd..c2a3f6e 100644 --- a/src/image_creator/utils/image.py +++ b/src/image_creator/utils/image.py @@ -5,7 +5,7 @@ import os import pathlib import re -import subprocess +import subprocess # noqa: S404 import tempfile import time @@ -160,13 +160,17 @@ def get_device_sectors(dev_path: str) -> int: """number of sectors composing this device""" dev_name = get_loop_name(dev_path) - return int(pathlib.Path(f"/sys/block/{dev_name}/size").read_text()) + return int(pathlib.Path(f"/sys/block/{dev_name}/size").read_text(encoding="ASCII")) def get_thirdpart_start_sector(dev_path) -> int: """Start sector number of third partition of device""" dev_name = get_loop_name(dev_path) - return int(pathlib.Path(f"/sys/block/{dev_name}/{dev_name}p3/start").read_text()) + return int( + pathlib.Path(f"/sys/block/{dev_name}/{dev_name}p3/start").read_text( + encoding="ASCII" + ) + ) def check_third_partition_device(dev_path: str): @@ -230,7 +234,7 @@ def fsck_ext4(dev_path: str): # 16 - Usage or syntax error # 32 - E2fsck canceled by user request # 128 - Shared library error - if ps.returncode not in (0, 1, 2): + if ps.returncode not in {0, 1, 2}: raise OSError( f"Command {command!s} returned unexpected exit status {ps.returncode!s}." ) diff --git a/src/image_creator/utils/requirements.py b/src/image_creator/utils/requirements.py index 274a966..10b8017 100644 --- a/src/image_creator/utils/requirements.py +++ b/src/image_creator/utils/requirements.py @@ -2,7 +2,7 @@ import os import re -import subprocess +import subprocess # noqa: S404 help_text = """ Requirements @@ -41,7 +41,7 @@ def is_root() -> bool: def has_ext4_support() -> bool: """whether ext4 filesystem is enabled""" - with open("/proc/filesystems") as fh: + with open("/proc/filesystems", encoding="ASCII") as fh: for line in fh.readlines(): if re.match(r"\s*ext4\s?$", line.strip()): return True diff --git a/tasks.py b/tasks.py index d645edf..9427d2a 100644 --- a/tasks.py +++ b/tasks.py @@ -1,4 +1,4 @@ -# pyright: strict, reportUntypedFunctionDecorator=false +# pyright: strict, reportUntypedFunctionDecorator=false, reportUnknownMemberType=false import base64 import hashlib import os @@ -40,6 +40,7 @@ def report_cov(ctx: Context, *, html: bool = False): """report coverage""" ctx.run("coverage combine", warn=True, pty=use_pty) ctx.run("coverage report --show-missing", pty=use_pty) + ctx.run("coverage xml", pty=use_pty) if html: ctx.run("coverage html", pty=use_pty) @@ -57,13 +58,6 @@ def coverage(ctx: Context, args: str = "", *, html: bool = False): report_cov(ctx, html=html) -@task(optional=["args"], help={"args": "black additional arguments"}) -def lint_black(ctx: Context, args: str = "."): - args = args or "." # needed for hatch script - ctx.run("black --version", pty=use_pty) - ctx.run(f"black --check --diff {args}", pty=use_pty) - - @task(optional=["args"], help={"args": "ruff additional arguments"}) def lint_ruff(ctx: Context, args: str = "."): args = args or "." # needed for hatch script @@ -74,13 +68,12 @@ def lint_ruff(ctx: Context, args: str = "."): @task( optional=["args"], help={ - "args": "linting tools (black, ruff) additional arguments, typically a path", + "args": "linting tools (ruff) additional arguments, typically a path", }, ) def lintall(ctx: Context, args: str = "."): """Check linting""" args = args or "." # needed for hatch script - lint_black(ctx, args) lint_ruff(ctx, args) @@ -97,30 +90,22 @@ def checkall(ctx: Context, args: str = ""): check_pyright(ctx, args) -@task(optional=["args"], help={"args": "black additional arguments"}) -def fix_black(ctx: Context, args: str = "."): - """fix black formatting""" - args = args or "." # needed for hatch script - ctx.run(f"black {args}", pty=use_pty) - - @task(optional=["args"], help={"args": "ruff additional arguments"}) def fix_ruff(ctx: Context, args: str = "."): """fix all ruff rules""" args = args or "." # needed for hatch script - ctx.run(f"ruff --fix {args}", pty=use_pty) + ctx.run(f"ruff check --fix {args}", pty=use_pty) @task( optional=["args"], help={ - "args": "linting tools (black, ruff) additional arguments, typically a path", + "args": "linting tools (ruff) additional arguments, typically a path", }, ) def fixall(ctx: Context, args: str = "."): """Fix everything automatically""" args = args or "." # needed for hatch script - fix_black(ctx, args) fix_ruff(ctx, args) lintall(ctx, args) @@ -152,7 +137,7 @@ def download_aria2c(ctx: Context, *, force: bool = False): # noqa: ARG001 print("") computed_sum = base64.standard_b64encode(md5sum.digest()).decode("UTF-8").strip() if received_sum and received_sum != computed_sum: - print("Checksum mismatch! {received_sum=} - {computed_sum=}") + print(f"Checksum mismatch! {received_sum=} - {computed_sum=}") print("Removing.") aria2c_bin.unlink() aria2c_bin_zip.unlink() @@ -192,7 +177,7 @@ def binary(ctx: Context, filename: str = "", *, no_compress: bool = False): "-m", "nuitka", "--onefile", - "--python-flag=no_site,no_warnings,no_asserts,no_docstrings", + "--python-flag=no_site,no_asserts,no_docstrings", "--include-package=image_creator", "--include-data-files=aria2c=aria2c", "--show-modules",