diff --git a/app/cli.py b/app/cli.py index e46d4c1..eab1158 100644 --- a/app/cli.py +++ b/app/cli.py @@ -10,8 +10,8 @@ from app.utils.gitmastery import ( find_exercise_root, find_gitmastery_root, - read_gitmastery_config, read_exercise_config, + read_gitmastery_config, ) from app.utils.version import Version from app.version import __version__ @@ -45,6 +45,10 @@ def cli(ctx, verbose) -> None: ctx.obj[CliContextKey.GITMASTERY_EXERCISE_CONFIG] = exercise_root_config ctx.obj[CliContextKey.VERBOSE] = verbose + # We make the assumption that within a single command run, the "state of the world" + # is immutable, allowing us to cache things + ctx.obj[CliContextKey.WEB_CACHE] = {} + ctx.obj[CliContextKey.TAG_CACHE] = [] current_version = Version.parse_version_string(__version__) ctx.obj[CliContextKey.VERSION] = current_version diff --git a/app/commands/setup_folder.py b/app/commands/setup_folder.py index f7e0b91..9f80c07 100644 --- a/app/commands/setup_folder.py +++ b/app/commands/setup_folder.py @@ -6,6 +6,7 @@ from app.commands.check.git import git from app.commands.progress.constants import PROGRESS_LOCAL_FOLDER_NAME from app.utils.click import error, info, invoke_command, prompt +from app.utils.exercises import get_latest_release_exercise_version @click.command("setup") @@ -37,8 +38,22 @@ def setup() -> None: info("Setting up your local progress tracker...") os.makedirs(PROGRESS_LOCAL_FOLDER_NAME, exist_ok=True) with open(".gitmastery.json", "w") as gitmastery_file: + version_to_pin = get_latest_release_exercise_version() + if version_to_pin is None: + # For now, we just error out because we should never be in this bad state. + raise ValueError( + "Unexpected error occurred when fetching exercises due to missing exercises tag. Contact the Git-Mastery team." + ) + version_to_pin.pinned = True + info(f"Pinning your exercises to {version_to_pin}") gitmastery_file.write( - json.dumps({"progress_local": True, "progress_remote": False}) + json.dumps( + { + "progress_local": True, + "progress_remote": False, + "exercises_version": version_to_pin.to_version_string(), + } + ) ) with open("progress/progress.json", "a") as progress_file: diff --git a/app/gitmastery_config.py b/app/gitmastery_config.py index 3568701..997a28e 100644 --- a/app/gitmastery_config.py +++ b/app/gitmastery_config.py @@ -2,11 +2,14 @@ from dataclasses import dataclass from pathlib import Path +from app.utils.version import Version + @dataclass class GitMasteryConfig: progress_local: bool progress_remote: bool + exercises_version: Version path: Path cds: int diff --git a/app/utils/click.py b/app/utils/click.py index 857e61f..82b04e3 100644 --- a/app/utils/click.py +++ b/app/utils/click.py @@ -1,7 +1,7 @@ import logging import sys from enum import StrEnum -from typing import Any, Dict, NoReturn, Optional +from typing import Any, Dict, List, NoReturn, Optional import click @@ -16,6 +16,8 @@ class CliContextKey(StrEnum): GITMASTERY_EXERCISE_CONFIG = "GITMASTERY_EXERCISE_CONFIG" VERBOSE = "VERBOSE" VERSION = "VERSION" + WEB_CACHE = "WEB_CACHE" + TAG_CACHE = "TAG_CACHE" class ClickColor(StrEnum): @@ -109,6 +111,14 @@ def get_exercise_root_config() -> Optional[ExerciseConfig]: ) +def get_web_cache() -> Dict[str, str | bytes | Any]: + return click.get_current_context().obj.get(CliContextKey.WEB_CACHE, {}) + + +def get_tag_cache() -> List[str]: + return click.get_current_context().obj.get(CliContextKey.TAG_CACHE, []) + + def invoke_command(command: click.Command) -> None: ctx = click.get_current_context() ctx.invoke(command) diff --git a/app/utils/exercises.py b/app/utils/exercises.py new file mode 100644 index 0000000..a78efde --- /dev/null +++ b/app/utils/exercises.py @@ -0,0 +1,76 @@ +from typing import List, Optional + +from app.utils.click import get_tag_cache, get_verbose +from app.utils.command import run +from app.utils.version import Version + + +def get_exercises_tags() -> List[str]: + tag_cache = get_tag_cache() + if len(tag_cache) != 0: + if get_verbose(): + print("Fetching tags from cache") + # Use the in-memory, per command cache for tags to avoid re-querying + return tag_cache + + result = run( + [ + "git", + "ls-remote", + "--tags", + "--refs", + "https://github.com/git-mastery/exercises", + ] + ) + versions = [] + if result.is_success() and result.stdout: + lines = result.stdout.split("\n") + for line in lines: + tag_raw = line.split()[1] + if tag_raw.startswith("refs/tags/"): + tag = tag_raw[len("refs/tags/") :] + versions.append(tag) + tag_cache = versions + if get_verbose(): + print("Queried for tags to store in cache") + return versions + + +def get_all_exercise_tags() -> List[Version]: + tags = get_exercises_tags() + return list(sorted([Version.parse_version_string(t) for t in tags], reverse=True)) + + +def get_latest_release_exercise_version() -> Optional[Version]: + all_tags = get_all_exercise_tags() + if len(all_tags) == 0: + # Although this should not be happening, we will let the callsite handle this + return None + + # These should always ignore the development versions, just focus on the release + # versions + for tag in all_tags: + if tag.prerelease is None: + return tag + + return None + + +def get_latest_development_exercise_version() -> Optional[Version]: + all_tags = get_all_exercise_tags() + if len(all_tags) == 0: + # Although this should not be happening, we will let the callsite handle this + return None + + for tag in all_tags: + if tag.build is None: + return tag + return None + + +def get_latest_exercise_version_within_pin(pin_version: Version) -> Optional[Version]: + all_tags = get_all_exercise_tags() + for tag in all_tags: + if tag.within_pin(pin_version): + return tag + return None diff --git a/app/utils/git.py b/app/utils/git.py index 0ee281f..749fa96 100644 --- a/app/utils/git.py +++ b/app/utils/git.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import List, Optional from app.utils.command import run diff --git a/app/utils/gitmastery.py b/app/utils/gitmastery.py index c44caac..2e9e5c3 100644 --- a/app/utils/gitmastery.py +++ b/app/utils/gitmastery.py @@ -2,7 +2,6 @@ import os import sys import tempfile -import urllib.parse from pathlib import Path from typing import Any, Dict, Optional, Tuple, TypeVar, Union @@ -11,8 +10,20 @@ from app.exercise_config import ExerciseConfig from app.gitmastery_config import GitMasteryConfig -from app.utils.click import error, get_exercise_root_config, get_gitmastery_root_config +from app.utils.click import ( + error, + get_exercise_root_config, + get_gitmastery_root_config, + get_verbose, + get_web_cache, +) +from app.utils.exercises import ( + get_latest_development_exercise_version, + get_latest_exercise_version_within_pin, + get_latest_release_exercise_version, +) from app.utils.general import ensure_str +from app.utils.version import Version GITMASTERY_CONFIG_NAME = ".gitmastery.json" GITMASTERY_EXERCISE_CONFIG_NAME = ".gitmastery-exercise.json" @@ -21,6 +32,33 @@ ) +def _construct_gitmastery_exercises_url(filepath: str, version: Version) -> str: + if version.release: + latest_release = get_latest_release_exercise_version() + if latest_release is None: + raise ValueError("This should not happen. Contact the Git-Mastery team.") + ref = f"tags/v{latest_release.to_version_string()}" + elif version.development: + latest_development = get_latest_development_exercise_version() + if latest_development is None: + raise ValueError("This should not happen. Contact the Git-Mastery team.") + ref = f"tags/v{latest_development.to_version_string()}" + elif not version.pinned: + ref = f"tags/v{version.to_version_string()}" + else: + # If pinned, we need to basically search for all the available tags within the + # range + latest_within_pin = get_latest_exercise_version_within_pin(version) + if latest_within_pin is None: + raise ValueError("This should not happen. Contact the Git-Mastery team.") + ref = f"tags/v{latest_within_pin.to_version_string()}" + + url = ( + f"https://raw.githubusercontent.com/git-mastery/exercises/refs/{ref}/{filepath}" + ) + return url + + def _find_root(filename: str) -> Optional[Tuple[Path, int]]: current = Path.cwd() steps = 0 @@ -67,6 +105,9 @@ def read_gitmastery_config(gitmastery_config_path: Path, cds: int) -> GitMastery cds=cds, progress_local=raw_config.get("progress_local", False), progress_remote=raw_config.get("progress_remote", False), + exercises_version=Version.parse_version_string( + raw_config.get("exercises_version", "release") + ), ) @@ -120,32 +161,56 @@ def generate_cds_string(cds: int) -> str: def get_gitmastery_file_path(path: str): - return urllib.parse.urljoin(GITMASTERY_EXERCISES_BASE_URL, path) + config = get_gitmastery_root_config() + if config is None: + version = Version.RELEASE + else: + version = config.exercises_version + return _construct_gitmastery_exercises_url(path, version) def fetch_file_contents(url: str, is_binary: bool) -> str | bytes: + web_cache = get_web_cache() + if url in web_cache: + if get_verbose(): + print(f"Fetching {url} contents from WEB_CACHE") + return web_cache[url] response = requests.get(url) + if get_verbose(): + print(f"Querying {url} for contents") + if response.status_code == 200: if is_binary: - return response.content - return response.text + web_cache[url] = response.content + else: + web_cache[url] = response.text + return web_cache[url] else: error( f"Failed to fetch resource {click.style(url, bold=True, italic=True)}. Inform the Git-Mastery team." ) - return "" def fetch_file_contents_or_none( url: str, is_binary: bool ) -> Optional[Union[str, bytes]]: + web_cache = get_web_cache() + if url in web_cache: + if get_verbose(): + print(f"Fetching {url} contents from WEB_CACHE") + return web_cache[url] response = requests.get(url) + if get_verbose(): + print(f"Querying {url} for contents") + if response.status_code == 200: if is_binary: - return response.content - return response.text + web_cache[url] = response.content + else: + web_cache[url] = response.text + return web_cache[url] return None @@ -177,10 +242,14 @@ def get_variable_from_url( def exercise_exists(exercise: str, timeout: int = 5) -> bool: try: + exercise_url = get_gitmastery_file_path( + f"{exercise.replace('-', '_')}/.gitmastery-exercise.json" + ) + if get_verbose(): + print(exercise_url) + response = requests.head( - get_gitmastery_file_path( - f"{exercise.replace('-', '_')}/.gitmastery-exercise.json" - ), + exercise_url, allow_redirects=True, timeout=timeout, ) @@ -194,8 +263,14 @@ def hands_on_exists(hands_on: str, timeout: int = 5) -> bool: hands_on = hands_on[3:] try: + hands_on_url = get_gitmastery_file_path( + f"hands_on/{hands_on.replace('-', '_')}.py" + ) + if get_verbose(): + print(hands_on_url) + response = requests.head( - get_gitmastery_file_path(f"hands_on/{hands_on.replace('-', '_')}.py"), + hands_on_url, allow_redirects=True, timeout=timeout, ) diff --git a/app/utils/version.py b/app/utils/version.py index 629ab28..99fe30d 100644 --- a/app/utils/version.py +++ b/app/utils/version.py @@ -1,4 +1,7 @@ from dataclasses import dataclass +from typing import ClassVar, Optional, Self, Type + +import semver @dataclass @@ -6,16 +9,153 @@ class Version: major: int minor: int patch: int + prerelease: Optional[str] + build: Optional[str] + + pinned: bool + release: bool + development: bool + + RELEASE: ClassVar[Self] + DEVELOPMENT: ClassVar[Self] + + @classmethod + def parse_version_string(cls: Type[Self], version: str) -> Self: + if version == "release": + return cls.RELEASE + + if version == "development": + return cls.DEVELOPMENT + + if version.startswith("v"): + version = version[1:] + pinned = False + if version.startswith("^"): + pinned = True + version = version[1:] + semver_version = semver.Version.parse(version) + + if pinned and semver_version.build is not None: + raise ValueError( + "Version pinning is not supported for version strings with a build string" + ) + + return cls( + major=semver_version.major, + minor=semver_version.minor, + patch=semver_version.patch, + prerelease=semver_version.prerelease, + build=semver_version.build, + pinned=pinned, + release=False, + development=False, + ) + + def __lt__(self, other: Self) -> bool: + if self.release or self.development: + # The release or development versions will always never be less than + # anything else + # There is a caveat in the logic where the release may be < development, + # but this is unlikely to occur + return False + if other.release or other.development: + # If the other is release or development, whatever this is - assuming not + # release or development - will always be behind + return True + self_version = self.__to_semver_version__(self) + other_version = self.__to_semver_version__(other) + + # This comparison will also use the prerelease version + # Build is ignored + return self_version < other_version + + def __eq__(self, other: object) -> bool: + if not isinstance(other, type(self)): + return False + + if self.release: + return self.release == other.release + + if self.development: + return self.development == other.development - @staticmethod - def parse_version_string(version: str) -> "Version": - only_version = version[1:] - [major, minor, patch] = only_version.split(".") - return Version(int(major), int(minor), int(patch)) + self_version = self.__to_semver_version__(self) + other_version = self.__to_semver_version__(other) - def is_behind(self, other: "Version") -> bool: + # This comparison will also use the prerelease version + # Build is ignored + return self_version == other_version + + def is_behind(self, other: Self) -> bool: """Returns if the current version is behind the other version based on major and minor versions.""" return (other.major, other.minor) > (self.major, self.minor) + def to_version_string(self) -> str: + if self.release: + return "release" + + if self.development: + return "development" + + version = self.__to_semver_version__(self) + if self.pinned: + return "^" + str(version) + return str(version) + + def within_pin(self, pin_version: Self) -> bool: + if not pin_version.pinned: + raise ValueError("Version provided should be pinned") + + if pin_version.prerelease is None and self.prerelease is not None: + # Not looking to check against prerelease versions, so ignore a prerelease + # version + return False + + min_bound = f">={pin_version.major}.{pin_version.minor}.{pin_version.patch}" + max_bound = f"<{pin_version.major + 1}.0.0" + self_version = self.__to_semver_version__(self) + within_min_bound = self_version.match(min_bound) + within_max_bound = self_version.match(max_bound) + return within_min_bound and within_max_bound + + def __to_semver_version__(self, version: Self) -> semver.Version: + parts = ( + version.major, + version.minor, + version.patch, + version.prerelease, + version.build, + ) + v = semver.VersionInfo(*parts) + return v + def __repr__(self) -> str: + if self.release: + return "release" + if self.development: + return "development" + return f"v{self.major}.{self.minor}.{self.patch}" + + +Version.RELEASE = Version( + major=0, + minor=0, + patch=0, + prerelease="", + build="", + pinned=False, + release=True, + development=False, +) + +Version.DEVELOPMENT = Version( + major=0, + minor=0, + patch=0, + prerelease="", + build="", + pinned=False, + release=False, + development=True, +) diff --git a/requirements.txt b/requirements.txt index e19f1e7..9ff50b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,5 @@ pytermgui git-autograder==6.* pytz PyYAML +semver +pytest diff --git a/tests/utils/test_exercises.py b/tests/utils/test_exercises.py new file mode 100644 index 0000000..58699bc --- /dev/null +++ b/tests/utils/test_exercises.py @@ -0,0 +1,56 @@ +from typing import List +from unittest import mock + +import pytest +from app.utils.exercises import ( + get_latest_exercise_version_within_pin, + get_latest_release_exercise_version, +) +from app.utils.version import Version + + +def test_get_latest_release_exercise_version(): + exercises = [ + "0.0.1-beta.0", + "0.9.9", + "v1.0.0", + "1.1.1", + "12.1.1-beta.1", + "12.1.1-beta.4", + ] + with mock.patch( + "app.utils.version.get_all_exercise_tags", return_value=_get_versions(exercises) + ): + assert get_latest_release_exercise_version() == Version( + 0, 9, 9, None, None, False, False, False + ) + + +@pytest.mark.parametrize( + "pin,expected", + [ + ("^1.1.0", Version(1, 1, 1, None, None, False, False, False)), + ("^12.0.0", None), + ], +) +def test_get_latest_exercise_version_within_pin(pin, expected): + exercises = [ + "0.0.1-beta.0", + "0.9.9", + "v1.0.0", + "1.1.1", + "12.1.1-beta.1", + "12.1.1-beta.4", + ] + with mock.patch( + "app.utils.version.get_all_exercise_tags", + return_value=_get_versions(exercises), + ): + version = get_latest_exercise_version_within_pin( + Version.parse_version_string(pin) + ) + assert version == expected + + +def _get_versions(version_strings: List[str]) -> List[Version]: + return [Version.parse_version_string(v) for v in version_strings] diff --git a/tests/utils/test_version.py b/tests/utils/test_version.py new file mode 100644 index 0000000..695de3a --- /dev/null +++ b/tests/utils/test_version.py @@ -0,0 +1,152 @@ +from typing import List +from unittest import mock + +import pytest +from app.utils.version import ( + Version, +) + + +def test_parse_version_string_pinned_with_build(): + with pytest.raises( + ValueError, + match="Version pinning is not supported for version strings with a build string", + ): + Version.parse_version_string("^1.0.0-X+build") + + +@pytest.mark.parametrize( + "s, expected", + [ + ("v^1.0.0", Version(1, 0, 0, None, None, True, False, False)), + ("^1.0.0", Version(1, 0, 0, None, None, True, False, False)), + ("v^1.12.0-beta.1", Version(1, 12, 0, "beta.1", None, True, False, False)), + ("^1.12.0-beta.2", Version(1, 12, 0, "beta.2", None, True, False, False)), + ( + "1.12.0-beta.2+build.12", + Version(1, 12, 0, "beta.2", "build.12", False, False, False), + ), + ("1.12.0-beta.2", Version(1, 12, 0, "beta.2", None, False, False, False)), + ("1.12.0", Version(1, 12, 0, None, None, False, False, False)), + ("release", Version.RELEASE), + ("development", Version.DEVELOPMENT), + ], +) +def test_parse_version_string_regular(s, expected): + v = Version.parse_version_string(s) + assert v == expected + + +@pytest.mark.parametrize( + "left,right", + [ + ("release", "release"), + ("release", "1.0.0"), + ], +) +def test_version_lt_release(left, right): + assert not Version.parse_version_string(left) < Version.parse_version_string(right) + + +@pytest.mark.parametrize( + "left,right", + [ + ("development", "release"), + ("development", "1.0.0"), + ], +) +def test_version_lt_development(left, right): + assert not Version.parse_version_string(left) < Version.parse_version_string(right) + + +@pytest.mark.parametrize( + "left,right", + [ + ("v1.0.0", "v1.1.1"), + ("v1.0.0", "v1.0.1"), + ("v1.0.0", "v2.0.0"), + ("1.0.0", "release"), + ("1.0.0", "development"), + ("1.1.0-beta.0", "1.1.0-beta.1"), + ("1.1.0-beta.0+build", "1.1.0-beta.1+a"), + ], +) +def test_version_lt_regular(left, right): + assert Version.parse_version_string(left) < Version.parse_version_string(right) + + +def test_version_eq_different_type(): + assert not (Version.parse_version_string("1.0.0") == 1) + + +@pytest.mark.parametrize( + "left,right,expected", + [ + ("release", "release", True), + ("development", "development", True), + ("release", "development", False), + ("development", "release", False), + ("development", "v1.1.1", False), + ], +) +def test_version_lt_track(left, right, expected): + assert ( + Version.parse_version_string(left) == Version.parse_version_string(right) + ) == expected + + +def test_version_sort(): + version_strings = [ + "v1.0.0", + "12.1.1-beta.4", + "1.1.1", + "12.1.1-beta.1", + "0.9.9", + ] + + versions = [Version.parse_version_string(v) for v in version_strings] + versions.sort() + + expected_version_strings = [ + "0.9.9", + "v1.0.0", + "1.1.1", + "12.1.1-beta.1", + "12.1.1-beta.4", + ] + assert versions == [ + Version.parse_version_string(v) for v in expected_version_strings + ] + + +@pytest.mark.parametrize( + "version,expected", + [ + ("1.0.0", False), + ("1.11.2", True), + ("1.12.0", True), + ("1.11.2-beta.1", False), + ("2.0.0", False), + ], +) +def test_within_pin_without_prerelease(version, expected): + pin_version = Version.parse_version_string("^1.11.1") + version_v = Version.parse_version_string(version) + assert version_v.within_pin(pin_version) == expected + + +@pytest.mark.parametrize( + "version,expected", + [ + ("1.0.0", False), + ("1.11.2", True), + ("1.12.0", True), + ("1.11.2-beta.2", True), + ("1.11.2-beta.0", True), + ("2.0.0", False), + ], +) +def test_within_pin_with_prerelease(version, expected): + pin_version = Version.parse_version_string("^1.11.1-beta.1") + version_v = Version.parse_version_string(version) + assert version_v.within_pin(pin_version) == expected