From ec62be45cf06bdbf3ee78f103843c12dded9d27f Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 5 Mar 2025 15:57:29 -0300 Subject: [PATCH 01/16] enh: add testing --- .github/workflows/ci.yml | 44 ++++++++++++++++++++++++ junifer_data/tests/test_utils.py | 57 ++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 junifer_data/tests/test_utils.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bcff6c4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: Run Tests + +on: + - push + - pull_request + +jobs: + run-tests: + runs-on: ubuntu-latest + # Use custom CI image + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install tox tox-gh-actions + - name: Configure Git for DataLad + run: | + git config --global user.email "junifer-runner@github.com" + git config --global user.name "GitHub Runner" + - name: Test with tox + # https://github.com/actions/checkout/issues/1169#issuecomment-2291682583 + run: | + git config --system --add safe.directory $(pwd) + tox + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true + flags: junifer + if: success() && matrix.python-version == 3.12 \ No newline at end of file diff --git a/junifer_data/tests/test_utils.py b/junifer_data/tests/test_utils.py new file mode 100644 index 0000000..1672342 --- /dev/null +++ b/junifer_data/tests/test_utils.py @@ -0,0 +1,57 @@ +"""Tests for junifer_data.utils.""" + +from pathlib import Path + +import pytest + +from junifer_data import check_dataset + + +def test_check_dataset_main(tmp_path: Path) -> None: + """Test check_dataset with main tag. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest fixture that provides a temporary directory. + + """ + dataset = check_dataset(data_dir=tmp_path) + assert dataset.pathobj.name == "main" + + +def test_check_dataset_hexsha_errors(tmp_path: Path) -> None: + """Test check_dataset hexsha errors. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest fixture that provides a temporary directory. + + """ + with pytest.raises(ValueError, match="Cannot verify hexsha for main tag."): + check_dataset(data_dir=tmp_path, hexsha="wrong") + + with pytest.raises(ValueError, match="Commit verification failed."): + check_dataset(data_dir=tmp_path, tag="1", hexsha="wrong") + + # Now clone the dataset without checking + dataset = check_dataset(data_dir=tmp_path, tag="1") + + with pytest.raises(ValueError, match="Commit verification failed."): + check_dataset(data_dir=tmp_path, tag="1", hexsha="wrong") + + # Check with the right hexsha + dataset = check_dataset( + data_dir=tmp_path, + tag="1", + hexsha="e9aecf7b5a2fff82de00d265e02afde42a448647", + ) + + ds_path = tmp_path / "v1" + + with open(ds_path / "test.txt", "w") as f: + f.write("test") + + with pytest.raises(RuntimeError, match="dirty junifer-data"): + check_dataset(data_dir=tmp_path, tag="1") From 23925704c9066254f82e59d352bbe2055c6f7e2e Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 5 Mar 2025 16:00:58 -0300 Subject: [PATCH 02/16] Fix ruff --- junifer_data/tests/test_utils.py | 4 ++-- pyproject.toml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/junifer_data/tests/test_utils.py b/junifer_data/tests/test_utils.py index 1672342..704b4f0 100644 --- a/junifer_data/tests/test_utils.py +++ b/junifer_data/tests/test_utils.py @@ -36,13 +36,13 @@ def test_check_dataset_hexsha_errors(tmp_path: Path) -> None: check_dataset(data_dir=tmp_path, tag="1", hexsha="wrong") # Now clone the dataset without checking - dataset = check_dataset(data_dir=tmp_path, tag="1") + check_dataset(data_dir=tmp_path, tag="1") with pytest.raises(ValueError, match="Commit verification failed."): check_dataset(data_dir=tmp_path, tag="1", hexsha="wrong") # Check with the right hexsha - dataset = check_dataset( + check_dataset( data_dir=tmp_path, tag="1", hexsha="e9aecf7b5a2fff82de00d265e02afde42a448647", diff --git a/pyproject.toml b/pyproject.toml index 1c1e8c8..ab839b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,8 @@ ignore = [ "D107", # use specific rule codes when ignoring type issues "PGH003", + # ignore too complex expressions + "C901", ] [tool.ruff.lint.isort] From 9f5abe3b5a10d6f2fddf7039503845e08ad2276a Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 5 Mar 2025 16:02:09 -0300 Subject: [PATCH 03/16] fix ci --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bcff6c4..10b333a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,6 @@ jobs: - name: Test with tox # https://github.com/actions/checkout/issues/1169#issuecomment-2291682583 run: | - git config --system --add safe.directory $(pwd) tox - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 @@ -41,4 +40,4 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true flags: junifer - if: success() && matrix.python-version == 3.12 \ No newline at end of file + if: success() && matrix.python-version == 3.12 From 9360041b5f093b2ca5df582d5c201563f6d239d4 Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 5 Mar 2025 16:15:32 -0300 Subject: [PATCH 04/16] Update tox config + coverage --- codecov.yml | 34 ++++++++++++++++++++++++++++++++++ tox.ini | 12 +++++++++--- 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..4ddba72 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,34 @@ +codecov: + notify: {} + require_ci_to_pass: false + +comment: # this is a top-level key + layout: "reach, diff, flags, files" + behavior: default + require_changes: false # if true: only post the comment if coverage changes + require_base: no # [yes :: must have a base report to post] + +coverage: + status: +codecov: + notify: {} + require_ci_to_pass: false + +comment: true + +coverage: + status: + patch: + default: + if_ci_failed: failure + if_no_uploads: error + if_not_found: success + target: 95.0 + project: + default: false + library: + if_ci_failed: failure + if_no_uploads: error + if_not_found: success + target: 90.0 + \ No newline at end of file diff --git a/tox.ini b/tox.ini index 8794b35..104d7d1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,15 @@ [tox] requires = tox>=4 -env_list = - ruff, - changelog +env_list = ruff, changelog, coverage, py3{9,10,11,12,13} + +[gh-actions] +python = + 3.9: py39 + 3.10: py310 + 3.11: py311 + 3.12: coverage + 3.13: py313 [testenv:ruff] description = run ruff From 4b0d6cfd60cd311480c6fb4fdeea24b504d52436 Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 5 Mar 2025 16:16:36 -0300 Subject: [PATCH 05/16] Update tox.ini --- tox.ini | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tox.ini b/tox.ini index 104d7d1..ce7d83f 100644 --- a/tox.ini +++ b/tox.ini @@ -27,3 +27,34 @@ deps = towncrier commands = towncrier build --draft + +[testenv:coverage] +skip_install = false +deps = + pytest-cov +commands = + pytest --cov={envsitepackagesdir}/junifer_data --cov-report=xml --cov-report=term {envsitepackagesdir}/junifer_data + + +[coverage:paths] +source = + junifer_data + */site-packages/junifer_data + +[coverage:run] +branch = true +omit = + */setup.py + */_version.py + */tests/* +parallel = false + +[coverage:report] +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + # Type checking if statements should not be considered + if TYPE_CHECKING: + # Don't complain if non-runnable code isn't run: + if __name__ == .__main__.: +precision = 2 From 91a2aa49bac62b26c89175ccb1c8bf96a1abfae8 Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 5 Mar 2025 16:20:16 -0300 Subject: [PATCH 06/16] Fix tox.ini so it actually tests --- tox.ini | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tox.ini b/tox.ini index ce7d83f..d1bb46d 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,15 @@ python = 3.12: coverage 3.13: py313 +[testenv] +skip_install = false +passenv = + HOME +deps = + pytest +commands = + pytest + [testenv:ruff] description = run ruff skip_install = true From 57ab090f18d9c044878f19a3782599fd60e7b76b Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 5 Mar 2025 16:22:38 -0300 Subject: [PATCH 07/16] Add git-annex dep for github actions --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10b333a..afc2bf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: uses: actions/checkout@v4 with: submodules: true + - name: Set up git-annex + uses: jstritch/setup-git-annex@v1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -30,6 +32,7 @@ jobs: run: | git config --global user.email "junifer-runner@github.com" git config --global user.name "GitHub Runner" + - name: Test with tox # https://github.com/actions/checkout/issues/1169#issuecomment-2291682583 run: | From b841f3d74842a6f052dd27c076f2688652aec0a9 Mon Sep 17 00:00:00 2001 From: Fede Date: Wed, 5 Mar 2025 16:35:25 -0300 Subject: [PATCH 08/16] Add more tests for hexsha tags --- junifer_data/tests/test_utils.py | 40 +++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/junifer_data/tests/test_utils.py b/junifer_data/tests/test_utils.py index 704b4f0..c6d4c11 100644 --- a/junifer_data/tests/test_utils.py +++ b/junifer_data/tests/test_utils.py @@ -19,6 +19,22 @@ def test_check_dataset_main(tmp_path: Path) -> None: dataset = check_dataset(data_dir=tmp_path) assert dataset.pathobj.name == "main" + # Check-out again, should update + check_dataset(data_dir=tmp_path) + + +def test_check_dataset_tag_errors(tmp_path: Path) -> None: + """Test check_dataset hexsha errors. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest fixture that provides a temporary directory. + + """ + with pytest.raises(RuntimeError, match="Failed to checkout state"): + check_dataset(data_dir=tmp_path, tag="wrong") + def test_check_dataset_hexsha_errors(tmp_path: Path) -> None: """Test check_dataset hexsha errors. @@ -42,7 +58,7 @@ def test_check_dataset_hexsha_errors(tmp_path: Path) -> None: check_dataset(data_dir=tmp_path, tag="1", hexsha="wrong") # Check with the right hexsha - check_dataset( + dataset = check_dataset( data_dir=tmp_path, tag="1", hexsha="e9aecf7b5a2fff82de00d265e02afde42a448647", @@ -55,3 +71,25 @@ def test_check_dataset_hexsha_errors(tmp_path: Path) -> None: with pytest.raises(RuntimeError, match="dirty junifer-data"): check_dataset(data_dir=tmp_path, tag="1") + + # We will now update the tag + dataset.repo.add((ds_path / "test.txt").as_posix()) + dataset.repo.commit(msg="update") + + with pytest.raises(ValueError, match="Wrong commit checked out."): + check_dataset(data_dir=tmp_path, tag="1") + + # Update tag + dataset.repo.tag("v1", options=["-d"]) + dataset.repo.tag("v1") + + # Does not fail + check_dataset(data_dir=tmp_path, tag="1") + + # But does not have the right hexsha + with pytest.raises(ValueError, match="Commit verification failed."): + check_dataset( + data_dir=tmp_path, + tag="1", + hexsha="e9aecf7b5a2fff82de00d265e02afde42a448647", + ) \ No newline at end of file From cbe1f66f0d468a40f140bd4663073555bf3a2ffc Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Aug 2025 14:38:56 +0200 Subject: [PATCH 09/16] chore: improve tox.ini --- tox.ini | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index d1bb46d..c49a440 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,11 @@ [tox] requires = tox>=4 -env_list = ruff, changelog, coverage, py3{9,10,11,12,13} +env_list = + ruff, + changelog, + coverage, + py3{9,10,11,12,13} [gh-actions] python = @@ -44,7 +48,6 @@ deps = commands = pytest --cov={envsitepackagesdir}/junifer_data --cov-report=xml --cov-report=term {envsitepackagesdir}/junifer_data - [coverage:paths] source = junifer_data From 7ce378b9c7403766e01cefe63c10c7b73a9628fd Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Aug 2025 14:39:09 +0200 Subject: [PATCH 10/16] chore: remove unnecessary noqa --- junifer_data/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer_data/_utils.py b/junifer_data/_utils.py index f7ae929..c4d27d3 100644 --- a/junifer_data/_utils.py +++ b/junifer_data/_utils.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) -def check_dataset( # noqa: C901 +def check_dataset( data_dir: Union[str, Path, None] = None, tag: Optional[str] = None, hexsha: Optional[str] = None, From a7bb9d1525b9839077a0abdd60dce670485f3532 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Aug 2025 14:39:23 +0200 Subject: [PATCH 11/16] chore: improve tests --- junifer_data/tests/test_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/junifer_data/tests/test_utils.py b/junifer_data/tests/test_utils.py index c6d4c11..a570b93 100644 --- a/junifer_data/tests/test_utils.py +++ b/junifer_data/tests/test_utils.py @@ -48,9 +48,6 @@ def test_check_dataset_hexsha_errors(tmp_path: Path) -> None: with pytest.raises(ValueError, match="Cannot verify hexsha for main tag."): check_dataset(data_dir=tmp_path, hexsha="wrong") - with pytest.raises(ValueError, match="Commit verification failed."): - check_dataset(data_dir=tmp_path, tag="1", hexsha="wrong") - # Now clone the dataset without checking check_dataset(data_dir=tmp_path, tag="1") @@ -92,4 +89,4 @@ def test_check_dataset_hexsha_errors(tmp_path: Path) -> None: data_dir=tmp_path, tag="1", hexsha="e9aecf7b5a2fff82de00d265e02afde42a448647", - ) \ No newline at end of file + ) From e3ab97de2fb2f6e949cf9804ae0173d31ae96955 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Aug 2025 14:53:00 +0200 Subject: [PATCH 12/16] chore: update ci.yml --- .github/workflows/ci.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afc2bf7..d0c675b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,22 +5,25 @@ on: - pull_request jobs: - run-tests: + test: runs-on: ubuntu-latest - # Use custom CI image strategy: fail-fast: false matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: + - '3.9' + - '3.10' + - '3.11' + - '3.12' + - '3.13' steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: true + - uses: actions/checkout@v4 - name: Set up git-annex - uses: jstritch/setup-git-annex@v1 - - name: Set up Python ${{ matrix.python-version }} + run: | + apt-get update + apt-get install -y git-annex + - name: Install Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -36,6 +39,7 @@ jobs: - name: Test with tox # https://github.com/actions/checkout/issues/1169#issuecomment-2291682583 run: | + git config --system --add safe.directory $(pwd) tox - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 From 3ec3f080e4ef2d48174141a2f8e2d1c1052992d6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Aug 2025 14:53:55 +0200 Subject: [PATCH 13/16] chore: use sudo for git-annex install in ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0c675b..4dc724c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,8 +21,8 @@ jobs: - uses: actions/checkout@v4 - name: Set up git-annex run: | - apt-get update - apt-get install -y git-annex + sudo apt-get update + sudo apt-get install -y git-annex - name: Install Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From 82f32a07cd72e97ea2f18e55274f62e0adad76b4 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Aug 2025 14:55:19 +0200 Subject: [PATCH 14/16] chore: update tox run in ci.yml --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dc724c..a1bd972 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,9 +37,7 @@ jobs: git config --global user.name "GitHub Runner" - name: Test with tox - # https://github.com/actions/checkout/issues/1169#issuecomment-2291682583 run: | - git config --system --add safe.directory $(pwd) tox - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 From 5afa41104ba22675eb0aaedad7ea8fe9c7b02acd Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Aug 2025 15:03:17 +0200 Subject: [PATCH 15/16] docs: update changelog --- changelog.d/4.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4.added.md diff --git a/changelog.d/4.added.md b/changelog.d/4.added.md new file mode 100644 index 0000000..40fd7ac --- /dev/null +++ b/changelog.d/4.added.md @@ -0,0 +1 @@ +Add unit tests and CI workflow From 2ea601acb96f073bfcda2c9a7b624c41e5357708 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 20 Aug 2025 15:04:00 +0200 Subject: [PATCH 16/16] chore: update codecov.yml --- codecov.yml | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/codecov.yml b/codecov.yml index 4ddba72..92b8a3f 100644 --- a/codecov.yml +++ b/codecov.yml @@ -10,25 +10,5 @@ comment: # this is a top-level key coverage: status: -codecov: - notify: {} - require_ci_to_pass: false - -comment: true - -coverage: - status: - patch: - default: - if_ci_failed: failure - if_no_uploads: error - if_not_found: success - target: 95.0 - project: - default: false - library: - if_ci_failed: failure - if_no_uploads: error - if_not_found: success - target: 90.0 - \ No newline at end of file + project: off + patch: off