diff --git a/demo/demo_python.ipynb b/demo/demo_python.ipynb index 732078e..da3c0b1 100644 --- a/demo/demo_python.ipynb +++ b/demo/demo_python.ipynb @@ -17,7 +17,11 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [], "source": [ "%load_ext autoreload\n", @@ -33,7 +37,11 @@ { "cell_type": "code", "execution_count": 2, - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [], "source": [ "%%bash\n", @@ -52,96 +60,14 @@ { "cell_type": "code", "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Repo(state=State(config={'data': [{'encoding': None}], 'remote': None}, meta={}, data=Empty DataFrame\n", - "Columns: [sha1]\n", - "Index: []), dothm=, worktree=Worktree('/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1'))" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "repo1 = Repo.init(\"repo1\")\n", - "repo1\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== .hm directory ===\n", - "config.yml\n", - "data.tsv\n", - "meta.yml\n", - "README.md\n", - "\n", - "=== initial config.yml ===\n", - "# Edit this file only if your branch needs regex substitutions or a preset remote.\n", - "# For simple names, you can just run: hallmark add \"a{a}_i{i}.h5\"\n", - "data:\n", - " -\n", - " # fmt: \"{release}_{source}_{year}_{doy:03d}_{band}.uvfits\"\n", - " encoding:\n", - " # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)\n", - "remote:\n", - " # name: origin\n", - " # url: https://example.com/path/to/data/\n", - "\n", - "=== objects stored so far ===\n", - " 0\n" - ] + "metadata": { + "vscode": { + "languageId": "shellscript" } - ], - "source": [ - "%%bash\n", - "echo \"=== .hm directory ===\"\n", - "ls repo1/.hm/\n", - "echo \"\"\n", - "echo \"=== initial config.yml ===\"\n", - "cat repo1/.hm/config.yml\n", - "echo \"\"\n", - "echo \"=== objects stored so far ===\"\n", - "find repo1/.hm/objects -type f 2>/dev/null | wc -l\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Repository info\n", - "\n", - "The Python API exposes the same local hallmark paths directly on the `Repo` object.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dot-hallmark repo: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1/.hm\"\n", - "hallmark worktree: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1\"\n" - ] - } - ], + }, + "outputs": [], "source": [ - "print(f'dot-hallmark repo: \"{repo1.dothm.path}\"')\n", - "print(f'hallmark worktree: \"{repo1.worktree}\"')\n" + "repo1 = Repo.init(\"repo1\")\n" ] }, { @@ -153,14 +79,18 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": {}, + "execution_count": 4, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "~/astro_se_int/hallmark/demo/repo1 ~/astro_se_int/hallmark/demo\n", "Files in repo1/:\n", "a0_i0.h5\n", "a0_i120.h5\n", @@ -172,7 +102,7 @@ "a0.75_i30.h5\n", "a0.75_i60.h5\n", "a0.75_i90.h5\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + "~/astro_se_int/hallmark/demo\n" ] } ], @@ -194,8 +124,12 @@ }, { "cell_type": "code", - "execution_count": 7, - "metadata": {}, + "execution_count": 5, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [ { "data": { @@ -302,59 +236,70 @@ "9 a0_i90.h5 0.00 90.0" ] }, - "execution_count": 7, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "pf = repo1.add(\"a{a}_i{i}.h5\")\n", - "pf\n" + "paraframe = repo1.add(\"a{a}_i{i}.h5\")\n", + "paraframe" ] }, { - "cell_type": "code", - "execution_count": 8, + "cell_type": "markdown", "metadata": {}, + "source": [ + "This paraframe object will also be reflected in the data.tsv file." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== config.yml after add ===\n", - "data:\n", - "- fmt: a{a}_i{i}.h5\n", - " encoding: null\n", - "remote: null\n", - "\n", - "=== data.tsv after add ===\n", - "sha1\ta\ti\n", - "f656d419104e7af783c119186a4d81b15563310f\t0.75\t0\n", - "ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\t0.75\t120\n", - "890b2ea509575be9bc74ad515731a2562cd7406e\t0.75\t30\n", - "d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\t0.75\t60\n", - "ed2cfc6bb157a42729d3c6f45b228b432edec4be\t0.75\t90\n", - "18682593a012503b71935017dbc044665254a23c\t0\t0\n", - "1c057a4885fd04379ee1217375720665713c902f\t0\t120\n", - "829b87d845b367a9cc87df6ff510d1bdb7444aca\t0\t30\n", - "075a30e0a9c4dddd92447f463219d4c2a842883d\t0\t60\n", - "de40f4983d86342b03809bfcc09958f43f359b89\t0\t90\n" - ] + "data": { + "text/plain": [ + "{'branch': 'main',\n", + " 'staged': {'state': ['config.yml', 'data.tsv', 'meta.yml'],\n", + " 'added': ['a0.75_i0.h5',\n", + " 'a0.75_i120.h5',\n", + " 'a0.75_i30.h5',\n", + " 'a0.75_i60.h5',\n", + " 'a0.75_i90.h5',\n", + " 'a0_i0.h5',\n", + " 'a0_i120.h5',\n", + " 'a0_i30.h5',\n", + " 'a0_i60.h5',\n", + " 'a0_i90.h5'],\n", + " 'modified': [],\n", + " 'deleted': []},\n", + " 'worktree': {'modified': [], 'deleted': []},\n", + " 'untracked': []}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "%%bash\n", - "echo \"=== config.yml after add ===\"\n", - "cat repo1/.hm/config.yml\n", - "echo \"\"\n", - "echo \"=== data.tsv after add ===\"\n", - "cat repo1/.hm/data.tsv\n" + "repo1.status()" ] }, { "cell_type": "code", - "execution_count": 9, - "metadata": {}, + "execution_count": 7, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [ { "data": { @@ -362,7 +307,7 @@ "True" ] }, - "execution_count": 9, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -373,8 +318,39 @@ }, { "cell_type": "code", - "execution_count": 10, - "metadata": {}, + "execution_count": 8, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'branch': 'main',\n", + " 'staged': {'state': [], 'added': [], 'modified': [], 'deleted': []},\n", + " 'worktree': {'modified': [], 'deleted': []},\n", + " 'untracked': []}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.status()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [ { "name": "stdout", @@ -400,9 +376,31 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 10, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "commit d3da52e3a3077ce58995037957d74d1316fadde8\n", + "Author: Nayera Abdessalam \n", + "Date: Fri May 1 09:27:14 2026 -0700\n", + "\n", + " add main dataset\n", + "\n", + "commit ecb6fc7ffa7fd6b61d59a580d131c829caf942a8\n", + "Author: Nayera Abdessalam \n", + "Date: Fri May 1 09:27:14 2026 -0700\n", + "\n", + " Initial commit: local `.hm` repository\n" + ] + } + ], "source": [ "print(repo1.log())\n" ] @@ -411,19 +409,25 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Clean status\n" + "## 4. Demo for `repo.checkout('BRANCH')`\n", + "\n", + "We use a temporary branch so the main workflow stays intact.\n" ] }, { "cell_type": "code", "execution_count": 11, - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [ { "data": { "text/plain": [ - "{'branch': 'main',\n", - " 'staged': {'added': [], 'modified': [], 'deleted': []},\n", + "{'branch': 'experiment',\n", + " 'staged': {'state': [], 'added': [], 'modified': [], 'deleted': []},\n", " 'worktree': {'modified': [], 'deleted': []},\n", " 'untracked': []}" ] @@ -434,35 +438,38 @@ } ], "source": [ - "repo1.status()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Current add modes\n", + "# add(\".\") rebuilds the manifest from files that currently exist\n", "\n", - "- `repo.add(\"a{a}_i{i}.h5\")` parses files using the branch fmt and updates `config.yml`.\n", - "- `repo.add(\".\")` rebuilds `data.tsv` from files that currently exist using the fmt already set in `config.yml`.\n", - "- `repo.set_config(fmt=...)` is the explicit way to change branch fmt before `repo.add(\".\")`.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Demo: `repo.add(\".\")` removes deleted files from the manifest\n", + "repo1.checkout(\"experiment\")\n", + "\n", + "for path in Path(str(repo1.worktree)).glob(\"a*.h5\"):\n", + " path.unlink()\n", + "\n", + "b_files = [\n", + " f\"b{a}_i{i}.h5\"\n", + " for a in [0, 0.75]\n", + " for i in [0, 30, 60, 90, 120]\n", + " ]\n", + "\n", + "for name in b_files:\n", + " path = Path(str(repo1.worktree)) / name\n", + " path.write_text(f\"{name}\\n\", encoding=\"utf-8\")\n", + "\n", + "repo1.set_config(fmt=\"b{a}_i{i}.h5\")\n", + "repo1.add(\".\")\n", + "repo1.commit(\"add experiment dataset\")\n", "\n", - "We use a temporary branch so the main workflow stays intact.\n", - "On this branch we switch to `b*` files first so it is easy to distinguish from `main`.\n", - "Because the filename family changes, we update the branch fmt explicitly with `repo.set_config(...)` before using `repo.add(\".\")`.\n" + "repo1.status()\n" ] }, { "cell_type": "code", "execution_count": 12, - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [ { "data": { @@ -476,125 +483,90 @@ } ], "source": [ - "repo1.checkout(\"sync-demo\")\n" + "# Check back to main to see that the manifest has been correctly rebuilt\n", + "repo1.checkout(\"main\")" ] }, { "cell_type": "code", "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Files after switching sync-demo to b files:\n", - "b0_i0.h5\n", - "b0_i120.h5\n", - "b0_i30.h5\n", - "b0_i60.h5\n", - "b0_i90.h5\n", - "b0.75_i0.h5\n", - "b0.75_i120.h5\n", - "b0.75_i30.h5\n", - "b0.75_i60.h5\n", - "b0.75_i90.h5\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] + "metadata": { + "vscode": { + "languageId": "shellscript" } - ], - "source": [ - "%%bash\n", - "pushd repo1\n", - "rm a*.h5\n", - "for f in b{0,0.75}_i{0,30,60,90,120}.h5; do echo \"$f\" > \"$f\"; done\n", - "echo \"Files after switching sync-demo to b files:\"\n", - "ls *.h5\n", - "popd\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, + }, "outputs": [ { "data": { "text/plain": [ - "{'data': [{'fmt': 'b{a}_i{i}.h5', 'encoding': None}], 'remote': None}" + "{'current': 'main', 'names': ['experiment', 'main']}" ] }, - "execution_count": 14, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo1.set_config(fmt=\"b{a}_i{i}.h5\")\n", - "repo1.state.config\n" + "# Show available branches\n", + "repo1.branches()" ] }, { "cell_type": "code", - "execution_count": 15, - "metadata": {}, + "execution_count": 14, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "config.yml after repo.set_config:\n", - "data:\n", - "- fmt: b{a}_i{i}.h5\n", - " encoding: null\n", - "remote: null\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + "Files after checkout back to main:\n", + "repo1/a0_i0.h5\n", + "repo1/a0_i120.h5\n", + "repo1/a0_i30.h5\n", + "repo1/a0_i60.h5\n", + "repo1/a0_i90.h5\n", + "repo1/a0.75_i0.h5\n", + "repo1/a0.75_i120.h5\n", + "repo1/a0.75_i30.h5\n", + "repo1/a0.75_i60.h5\n", + "repo1/a0.75_i90.h5\n" ] } ], "source": [ "%%bash\n", - "pushd repo1\n", - "echo \"config.yml after repo.set_config:\"\n", - "cat .hm/config.yml\n", - "popd\n" + "echo \"Files after checkout back to main:\"\n", + "ls repo1/*.h5\n" ] }, { - "cell_type": "code", - "execution_count": 16, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Files after deleting some tracked b files:\n", - "b0_i0.h5\n", - "b0_i120.h5\n", - "b0_i30.h5\n", - "b0_i60.h5\n", - "b0_i90.h5\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], "source": [ - "%%bash\n", - "pushd repo1\n", - "rm b0.75_i{0,30,60,90,120}.h5\n", - "echo \"Files after deleting some tracked b files:\"\n", - "ls *.h5\n", - "popd\n" + "## 5. Filter Functionality\n" ] }, { - "cell_type": "code", - "execution_count": 17, + "cell_type": "markdown", "metadata": {}, + "source": [ + "We can also show that we can filter data from the ParaFrame Object" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [ { "data": { @@ -624,32 +596,32 @@ " \n", " \n", " \n", - " 0\n", - " b0_i0.h5\n", + " 5\n", + " a0_i0.h5\n", " 0.0\n", " 0.0\n", " \n", " \n", - " 1\n", - " b0_i120.h5\n", + " 6\n", + " a0_i120.h5\n", " 0.0\n", " 120.0\n", " \n", " \n", - " 2\n", - " b0_i30.h5\n", - " 0.0\n", + " 7\n", + " a0_i30.h5\n", + " 0.0\n", " 30.0\n", " \n", " \n", - " 3\n", - " b0_i60.h5\n", + " 8\n", + " a0_i60.h5\n", " 0.0\n", " 60.0\n", " \n", " \n", - " 4\n", - " b0_i90.h5\n", + " 9\n", + " a0_i90.h5\n", " 0.0\n", " 90.0\n", " \n", @@ -658,692 +630,32 @@ "" ], "text/plain": [ - " path a i\n", - "0 b0_i0.h5 0.0 0.0\n", - "1 b0_i120.h5 0.0 120.0\n", - "2 b0_i30.h5 0.0 30.0\n", - "3 b0_i60.h5 0.0 60.0\n", - "4 b0_i90.h5 0.0 90.0" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "repo1.add(\".\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Manifest after repo.add dot:\n", - "sha1\ta\ti\n", - "56676e3e5629923617f8c9e542bd1c2ccaf8fa3a\t0\t0\n", - "683e82f624b175c075ac3ea0cca56e94cd7422f8\t0\t120\n", - "cfdbae05e3afc29354d948a32263f7ac1ed5850b\t0\t30\n", - "3a947fb1280b5bca62615d6c7543b88080588e7a\t0\t60\n", - "1711d3b47eac6ae479b6f0f240f83f41b393ebb4\t0\t90\n" - ] - } - ], - "source": [ - "%%bash\n", - "echo \"Manifest after repo.add dot:\"\n", - "cat repo1/.hm/data.tsv\n" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'branch': 'sync-demo',\n", - " 'staged': {'added': ['b0_i0.h5',\n", - " 'b0_i120.h5',\n", - " 'b0_i30.h5',\n", - " 'b0_i60.h5',\n", - " 'b0_i90.h5'],\n", - " 'modified': [],\n", - " 'deleted': ['a0.75_i0.h5',\n", - " 'a0.75_i120.h5',\n", - " 'a0.75_i30.h5',\n", - " 'a0.75_i60.h5',\n", - " 'a0.75_i90.h5',\n", - " 'a0_i0.h5',\n", - " 'a0_i120.h5',\n", - " 'a0_i30.h5',\n", - " 'a0_i60.h5',\n", - " 'a0_i90.h5']},\n", - " 'worktree': {'modified': [], 'deleted': []},\n", - " 'untracked': []}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "repo1.status()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "repo1.commit(\"sync manifest with hallmark add dot\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "repo1.checkout(\"main\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Files after checkout back to main:\n", - "repo1/a0_i0.h5\n", - "repo1/a0_i120.h5\n", - "repo1/a0_i30.h5\n", - "repo1/a0_i60.h5\n", - "repo1/a0_i90.h5\n", - "repo1/a0.75_i0.h5\n", - "repo1/a0.75_i120.h5\n", - "repo1/a0.75_i30.h5\n", - "repo1/a0.75_i60.h5\n", - "repo1/a0.75_i90.h5\n" - ] - } - ], - "source": [ - "%%bash\n", - "echo \"Files after checkout back to main:\"\n", - "ls repo1/*.h5\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Checkout a new branch\n", - "\n", - "The new branch inherits the same fmt from `main`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "repo1.checkout(\"experiment\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Current .hm branch:\n", - "experiment\n", - "\n", - "config.yml on experiment:\n", - "data:\n", - "- fmt: a{a}_i{i}.h5\n", - " encoding: null\n", - "remote: null\n", - "\n", - "Files after checkout:\n", - "repo1/a0_i0.h5\n", - "repo1/a0_i120.h5\n", - "repo1/a0_i30.h5\n", - "repo1/a0_i60.h5\n", - "repo1/a0_i90.h5\n", - "repo1/a0.75_i0.h5\n", - "repo1/a0.75_i120.h5\n", - "repo1/a0.75_i30.h5\n", - "repo1/a0.75_i60.h5\n", - "repo1/a0.75_i90.h5\n" - ] - } - ], - "source": [ - "%%bash\n", - "echo \"Current .hm branch:\"\n", - "git -C repo1/.hm branch --show-current\n", - "echo \"\"\n", - "echo \"config.yml on experiment:\"\n", - "cat repo1/.hm/config.yml\n", - "echo \"\"\n", - "echo \"Files after checkout:\"\n", - "ls repo1/*.h5\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Replace the experiment branch dataset with new files that still match the same fmt\n" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Files after replacing the dataset on experiment:\n", - "a1_i105.h5\n", - "a1_i110.h5\n", - "a1_i15.h5\n", - "a1_i45.h5\n", - "a1_i75.h5\n", - "a1.5_i105.h5\n", - "a1.5_i110.h5\n", - "a1.5_i15.h5\n", - "a1.5_i45.h5\n", - "a1.5_i75.h5\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], - "source": [ - "%%bash\n", - "pushd repo1\n", - "rm a*.h5\n", - "for f in a{1,1.5}_i{15,45,75,105,110}.h5; do echo \"$f\" > \"$f\"; done\n", - "echo \"Files after replacing the dataset on experiment:\"\n", - "ls *.h5\n", - "popd\n" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
0a1.5_i105.h51.5105.0
1a1.5_i110.h51.5110.0
2a1.5_i15.h51.515.0
3a1.5_i45.h51.545.0
4a1.5_i75.h51.575.0
5a1_i105.h51.0105.0
6a1_i110.h51.0110.0
7a1_i15.h51.015.0
8a1_i45.h51.045.0
9a1_i75.h51.075.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "0 a1.5_i105.h5 1.5 105.0\n", - "1 a1.5_i110.h5 1.5 110.0\n", - "2 a1.5_i15.h5 1.5 15.0\n", - "3 a1.5_i45.h5 1.5 45.0\n", - "4 a1.5_i75.h5 1.5 75.0\n", - "5 a1_i105.h5 1.0 105.0\n", - "6 a1_i110.h5 1.0 110.0\n", - "7 a1_i15.h5 1.0 15.0\n", - "8 a1_i45.h5 1.0 45.0\n", - "9 a1_i75.h5 1.0 75.0" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "repo1.add(\".\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'branch': 'experiment',\n", - " 'staged': {'added': ['a1.5_i105.h5',\n", - " 'a1.5_i110.h5',\n", - " 'a1.5_i15.h5',\n", - " 'a1.5_i45.h5',\n", - " 'a1.5_i75.h5',\n", - " 'a1_i105.h5',\n", - " 'a1_i110.h5',\n", - " 'a1_i15.h5',\n", - " 'a1_i45.h5',\n", - " 'a1_i75.h5'],\n", - " 'modified': [],\n", - " 'deleted': ['a0.75_i0.h5',\n", - " 'a0.75_i120.h5',\n", - " 'a0.75_i30.h5',\n", - " 'a0.75_i60.h5',\n", - " 'a0.75_i90.h5',\n", - " 'a0_i0.h5',\n", - " 'a0_i120.h5',\n", - " 'a0_i30.h5',\n", - " 'a0_i60.h5',\n", - " 'a0_i90.h5']},\n", - " 'worktree': {'modified': [], 'deleted': []},\n", - " 'untracked': []}" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "repo1.status()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "experiment data.tsv before commit:\n", - "sha1\ta\ti\n", - "f0ca3ed468c8796645672d2ebb48d48ae49b6d3c\t1.5\t105\n", - "9d1801d46e071b98a87afcd1edc59fe9302b9a4b\t1.5\t110\n", - "ed34eda23a05ea4c3cd1bf41988c6e2f76cf41e1\t1.5\t15\n", - "b47e5f555daf30afea5b0f99cadad1e2d8c689ec\t1.5\t45\n", - "b2a40517fd5681088f88903165ae54af3c75cf25\t1.5\t75\n", - "635b4b44a00c0a2bde5a7f0e263bb98dcbdae6e2\t1\t105\n", - "24ef5885eff61f1a8fcd77c19329a2651eca0228\t1\t110\n", - "577954873e813a982f70867b49a374a230df3650\t1\t15\n", - "1580545be2a5d485338e3538c7b16709d5fa6e4a\t1\t45\n", - "e2406e4872637ef09f574731be5b5ba5e8aa5761\t1\t75\n" - ] - } - ], - "source": [ - "%%bash\n", - "echo \"experiment data.tsv before commit:\"\n", - "cat repo1/.hm/data.tsv\n" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "repo1.commit(\"replace experiment dataset\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Objects stored after experiment commit:\n", - " 25\n" - ] - } - ], - "source": [ - "%%bash\n", - "echo \"Objects stored after experiment commit:\"\n", - "find repo1/.hm/objects -type f | wc -l\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7. Checkout back to `main`\n" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "repo1.checkout(\"main\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Files after checkout main:\n", - "repo1/a0_i0.h5\n", - "repo1/a0_i120.h5\n", - "repo1/a0_i30.h5\n", - "repo1/a0_i60.h5\n", - "repo1/a0_i90.h5\n", - "repo1/a0.75_i0.h5\n", - "repo1/a0.75_i120.h5\n", - "repo1/a0.75_i30.h5\n", - "repo1/a0.75_i60.h5\n", - "repo1/a0.75_i90.h5\n", - "\n", - "main data.tsv:\n", - "sha1\ta\ti\n", - "f656d419104e7af783c119186a4d81b15563310f\t0.75\t0\n", - "ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\t0.75\t120\n", - "890b2ea509575be9bc74ad515731a2562cd7406e\t0.75\t30\n", - "d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\t0.75\t60\n", - "ed2cfc6bb157a42729d3c6f45b228b432edec4be\t0.75\t90\n", - "18682593a012503b71935017dbc044665254a23c\t0\t0\n", - "1c057a4885fd04379ee1217375720665713c902f\t0\t120\n", - "829b87d845b367a9cc87df6ff510d1bdb7444aca\t0\t30\n", - "075a30e0a9c4dddd92447f463219d4c2a842883d\t0\t60\n", - "de40f4983d86342b03809bfcc09958f43f359b89\t0\t90\n" - ] - } - ], - "source": [ - "%%bash\n", - "echo \"Files after checkout main:\"\n", - "ls repo1/*.h5\n", - "echo \"\"\n", - "echo \"main data.tsv:\"\n", - "cat repo1/.hm/data.tsv\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 8. Checkout back to `experiment` to verify\n" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" + " path a i\n", + "5 a0_i0.h5 0.0 0.0\n", + "6 a0_i120.h5 0.0 120.0\n", + "7 a0_i30.h5 0.0 30.0\n", + "8 a0_i60.h5 0.0 60.0\n", + "9 a0_i90.h5 0.0 90.0" ] }, - "execution_count": 33, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo1.checkout(\"experiment\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Files after checkout experiment:\n", - "repo1/a1_i105.h5\n", - "repo1/a1_i110.h5\n", - "repo1/a1_i15.h5\n", - "repo1/a1_i45.h5\n", - "repo1/a1_i75.h5\n", - "repo1/a1.5_i105.h5\n", - "repo1/a1.5_i110.h5\n", - "repo1/a1.5_i15.h5\n", - "repo1/a1.5_i45.h5\n", - "repo1/a1.5_i75.h5\n", - "\n", - "experiment data.tsv:\n", - "sha1\ta\ti\n", - "f0ca3ed468c8796645672d2ebb48d48ae49b6d3c\t1.5\t105\n", - "9d1801d46e071b98a87afcd1edc59fe9302b9a4b\t1.5\t110\n", - "ed34eda23a05ea4c3cd1bf41988c6e2f76cf41e1\t1.5\t15\n", - "b47e5f555daf30afea5b0f99cadad1e2d8c689ec\t1.5\t45\n", - "b2a40517fd5681088f88903165ae54af3c75cf25\t1.5\t75\n", - "635b4b44a00c0a2bde5a7f0e263bb98dcbdae6e2\t1\t105\n", - "24ef5885eff61f1a8fcd77c19329a2651eca0228\t1\t110\n", - "577954873e813a982f70867b49a374a230df3650\t1\t15\n", - "1580545be2a5d485338e3538c7b16709d5fa6e4a\t1\t45\n", - "e2406e4872637ef09f574731be5b5ba5e8aa5761\t1\t75\n" - ] - } - ], - "source": [ - "%%bash\n", - "echo \"Files after checkout experiment:\"\n", - "ls repo1/*.h5\n", - "echo \"\"\n", - "echo \"experiment data.tsv:\"\n", - "cat repo1/.hm/data.tsv\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 9. Uncommitted changes block checkout\n" + "# Single Parameter Filter\n", + "paraframe(a=0)" ] }, { "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Files after creating a2_i15.h5:\n", - "a1_i105.h5\n", - "a1_i110.h5\n", - "a1_i15.h5\n", - "a1_i45.h5\n", - "a1_i75.h5\n", - "a1.5_i105.h5\n", - "a1.5_i110.h5\n", - "a1.5_i15.h5\n", - "a1.5_i45.h5\n", - "a1.5_i75.h5\n", - "a2_i15.h5\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] + "execution_count": 16, + "metadata": { + "vscode": { + "languageId": "shellscript" } - ], - "source": [ - "%%bash\n", - "pushd repo1\n", - "echo \"a2_i15.h5\" > a2_i15.h5\n", - "echo \"Files after creating a2_i15.h5:\"\n", - "ls *.h5\n", - "popd\n" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, + }, "outputs": [ { "data": { @@ -1373,253 +685,293 @@ " \n", " \n", " \n", - " 0\n", - " a1.5_i105.h5\n", - " 1.5\n", - " 105.0\n", - " \n", - " \n", - " 1\n", - " a1.5_i110.h5\n", - " 1.5\n", - " 110.0\n", - " \n", - " \n", " 2\n", - " a1.5_i15.h5\n", - " 1.5\n", - " 15.0\n", - " \n", - " \n", - " 3\n", - " a1.5_i45.h5\n", - " 1.5\n", - " 45.0\n", - " \n", - " \n", - " 4\n", - " a1.5_i75.h5\n", - " 1.5\n", - " 75.0\n", + " a0.75_i30.h5\n", + " 0.75\n", + " 30.0\n", " \n", " \n", " 5\n", - " a1_i105.h5\n", - " 1.0\n", - " 105.0\n", + " a0_i0.h5\n", + " 0.00\n", + " 0.0\n", " \n", " \n", " 6\n", - " a1_i110.h5\n", - " 1.0\n", - " 110.0\n", + " a0_i120.h5\n", + " 0.00\n", + " 120.0\n", " \n", " \n", " 7\n", - " a1_i15.h5\n", - " 1.0\n", - " 15.0\n", + " a0_i30.h5\n", + " 0.00\n", + " 30.0\n", " \n", " \n", " 8\n", - " a1_i45.h5\n", - " 1.0\n", - " 45.0\n", + " a0_i60.h5\n", + " 0.00\n", + " 60.0\n", " \n", " \n", " 9\n", - " a1_i75.h5\n", - " 1.0\n", - " 75.0\n", - " \n", - " \n", - " 10\n", - " a2_i15.h5\n", - " 2.0\n", - " 15.0\n", + " a0_i90.h5\n", + " 0.00\n", + " 90.0\n", " \n", " \n", "\n", "" ], "text/plain": [ - " path a i\n", - "0 a1.5_i105.h5 1.5 105.0\n", - "1 a1.5_i110.h5 1.5 110.0\n", - "2 a1.5_i15.h5 1.5 15.0\n", - "3 a1.5_i45.h5 1.5 45.0\n", - "4 a1.5_i75.h5 1.5 75.0\n", - "5 a1_i105.h5 1.0 105.0\n", - "6 a1_i110.h5 1.0 110.0\n", - "7 a1_i15.h5 1.0 15.0\n", - "8 a1_i45.h5 1.0 45.0\n", - "9 a1_i75.h5 1.0 75.0\n", - "10 a2_i15.h5 2.0 15.0" + " path a i\n", + "2 a0.75_i30.h5 0.75 30.0\n", + "5 a0_i0.h5 0.00 0.0\n", + "6 a0_i120.h5 0.00 120.0\n", + "7 a0_i30.h5 0.00 30.0\n", + "8 a0_i60.h5 0.00 60.0\n", + "9 a0_i90.h5 0.00 90.0" ] }, - "execution_count": 36, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo1.add(\".\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Dirty status\n" + "# Or Filter\n", + "paraframe(a=0, i=30)" ] }, { "cell_type": "code", - "execution_count": 37, - "metadata": {}, + "execution_count": 17, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [ { "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathai
7a0_i30.h50.030.0
\n", + "
" + ], "text/plain": [ - "{'branch': 'experiment',\n", - " 'staged': {'added': ['a2_i15.h5'], 'modified': [], 'deleted': []},\n", - " 'worktree': {'modified': [], 'deleted': []},\n", - " 'untracked': []}" + " path a i\n", + "7 a0_i30.h5 0.0 30.0" ] }, - "execution_count": 37, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo1.status()\n" + "# And Filter\n", + "paraframe(a=0)(i=30)" ] }, { - "cell_type": "markdown", - "metadata": {}, + "cell_type": "code", + "execution_count": 18, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathai
2a0.75_i30.h50.7530.0
3a0.75_i60.h50.7560.0
4a0.75_i90.h50.7590.0
7a0_i30.h50.0030.0
8a0_i60.h50.0060.0
9a0_i90.h50.0090.0
\n", + "
" + ], + "text/plain": [ + " path a i\n", + "2 a0.75_i30.h5 0.75 30.0\n", + "3 a0.75_i60.h5 0.75 60.0\n", + "4 a0.75_i90.h5 0.75 90.0\n", + "7 a0_i30.h5 0.00 30.0\n", + "8 a0_i60.h5 0.00 60.0\n", + "9 a0_i90.h5 0.00 90.0" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "To fix this, commit the staged changes first or discard them.\n" + "# Masking Filter retains Pandas DataFrame functionality\n", + "paraframe[(30 <= paraframe.i) & (paraframe.i <= 100)]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 10. Clone a hallmark repository\n", - "\n", - "This clone demo matches the CLI notebook: it clones the hallmark repo, then downloads the configured remote data files. If you rerun the clone cell, the notebook catches `DestinationExistsError` and prints only Hallmark's fatal message.\n" + "## 6. Clone Functionality\n" ] }, { "cell_type": "code", - "execution_count": 38, - "metadata": {}, + "execution_count": 27, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully cloned to \"hallmark-demo-clone\"\n", - "Downloading remote data files...\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 8/8 [00:00<00:00, 31.32file/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully downloaded 8 files (4.3 MB)\n" + "100%|██████████| 8/8 [00:03<00:00, 2.26file/s]\n" ] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" + "ename": "DownloadError", + "evalue": "Failed to download 8 file(s):\n - Checksum mismatch for SR1_M87_2017_095_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_101_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_hi_hops_netcal_StokesI.uvfits.part\n - ... 3 more error(s)", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mDownloadError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[27]\u001b[39m\u001b[32m, line 5\u001b[39m\n\u001b[32m 2\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mhallmark\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01merror\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m DestinationExistsError\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m clone = Repo.clone(\u001b[33m'\u001b[39m\u001b[33mhttps://github.com/l6a/hallmark-demo-repo.hm.git\u001b[39m\u001b[33m'\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mhallmark-demo-clone\u001b[39m\u001b[33m\"\u001b[39m, show_progress=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 6\u001b[39m clone_result = {\n\u001b[32m 7\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mdot_hallmark_repo\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(clone.dothm.path),\n\u001b[32m 8\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mhallmark_worktree\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(clone.worktree),\n\u001b[32m 9\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mbranches\u001b[39m\u001b[33m\"\u001b[39m: clone.branches(),\n\u001b[32m 10\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mdata_rows\u001b[39m\u001b[33m\"\u001b[39m: \u001b[38;5;28mlen\u001b[39m(clone.state.data),\n\u001b[32m 11\u001b[39m }\n\u001b[32m 12\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m DestinationExistsError \u001b[38;5;28;01mas\u001b[39;00m exc:\n", + "\u001b[36mFile \u001b[39m\u001b[32m:23\u001b[39m, in \u001b[36mclone\u001b[39m\u001b[34m(cls, url, path, fetch_data, max_workers, show_progress)\u001b[39m\n", + "\u001b[31mDownloadError\u001b[39m: Failed to download 8 file(s):\n - Checksum mismatch for SR1_M87_2017_095_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_101_hi_hops_netcal_StokesI.uvfits.part\n - Checksum mismatch for SR1_M87_2017_096_hi_hops_netcal_StokesI.uvfits.part\n - ... 3 more error(s)" ] } ], "source": [ - "clone_url = \"https://github.com/l6a/hallmark-demo-repo.hm.git\"\n", - "clone_path = Path(\"hallmark-demo-clone\")\n", + "# Demo: clone a remote demo repository\n", + "from hallmark.error import DestinationExistsError\n", "\n", "try:\n", - " cloned_repo = Repo.clone(clone_url, clone_path)\n", - " print(f'Successfully cloned to \"{clone_path}\"')\n", - " print(\"Downloading remote data files...\")\n", - " results = download_remote_data(cloned_repo, clone_path, show_progress=True)\n", - " if results[\"failed\"] == 0:\n", - " mb_total = results[\"total_bytes\"] / (1024 * 1024)\n", - " print(f'Successfully downloaded {results[\"succeeded\"]} files ({mb_total:.1f} MB)')\n", - " else:\n", - " print(\n", - " \"Download completed with errors: \"\n", - " f'{results[\"succeeded\"]} succeeded, {results[\"failed\"]} failed'\n", - " )\n", - " for error in results[\"errors\"]:\n", - " print(f' - {error}')\n", + " clone = Repo.clone('https://github.com/l6a/hallmark-demo-repo.hm.git', \"hallmark-demo-clone\", show_progress=True)\n", + " clone_result = {\n", + " \"dot_hallmark_repo\": str(clone.dothm.path),\n", + " \"hallmark_worktree\": str(clone.worktree),\n", + " \"branches\": clone.branches(),\n", + " \"data_rows\": len(clone.state.data),\n", + " }\n", "except DestinationExistsError as exc:\n", - " print(exc)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dot-hallmark repo: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/hallmark-demo-clone/.hm\"\n", - "hallmark worktree: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/hallmark-demo-clone\"\n" - ] - } - ], - "source": [ - "cloned_repo = Repo(clone_path)\n", - "print(f'dot-hallmark repo: \"{cloned_repo.dothm.path}\"')\n", - "print(f'hallmark worktree: \"{cloned_repo.worktree}\"')\n" + " clone_result = str(exc)\n", + "\n", + "clone_result\n" ] }, { "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== cloned config.yml ===\n", - "data:\n", - " - fmt: '{release}_{source}_{year}_{doy:03d}_{band}_{pipeline}_{step}_{type}.uvfits'\n", - "remote:\n", - " name: origin\n", - " url: https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/EHTC_FirstM87Results_Apr2019/uvfits/" - ] + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" } - ], - "source": [ - "%%bash\n", - "echo \"=== cloned config.yml ===\"\n", - "cat hallmark-demo-clone/.hm/config.yml\n" - ] + }, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "hallmark", + "display_name": "Python (astro_se_int)", "language": "python", - "name": "python3" + "name": "astro_se_int" }, "language_info": { "codemirror_mode": { @@ -1631,7 +983,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.13.5" } }, "nbformat": 4, diff --git a/demo/hallmark-demo-clone/.hm b/demo/hallmark-demo-clone/.hm new file mode 160000 index 0000000..e425610 --- /dev/null +++ b/demo/hallmark-demo-clone/.hm @@ -0,0 +1 @@ +Subproject commit e425610625509efbcc1de996d1d084cd8756512e diff --git a/demo/repo1/.hm b/demo/repo1/.hm new file mode 160000 index 0000000..9484854 --- /dev/null +++ b/demo/repo1/.hm @@ -0,0 +1 @@ +Subproject commit 94848549a806d8f0da4d32a5b6258148aa88770b diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 03c3b7c..02c3b54 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -21,7 +21,7 @@ from git.exc import GitError from .downloader import DownloadError, download_remote_data -from .error import CloneError, DestinationExistsError +from .error import CloneError, DestinationExistsError, CheckoutError from . import Repo # from "__init__.py" @@ -96,6 +96,7 @@ def emit_section(title, entries, fg): emit_section( "Changes to be committed:", [ + ("state", staged["state"]), ("new file", staged["added"]), ("modified", staged["modified"]), ("deleted", staged["deleted"]), @@ -116,19 +117,26 @@ def emit_section(title, entries, fg): for path in untracked: click.echo(" " + click.style(path, fg="red")) - if not any([staged["added"], staged["modified"], staged["deleted"], + if not any([staged["state"], staged["added"], staged["modified"], staged["deleted"], worktree["modified"], worktree["deleted"], untracked]): click.echo("") click.echo("nothing to commit, working tree clean") @hallmark.command(short_help="Add files to hallmark index.") +@click.option( + "--regex", + "encoding", + is_flag=True, + default=False, + show_default=True, + help="Enable regex-based encoding rules from config.yml.") @click.argument("inputs", nargs=-1, required=True) @click.pass_obj -def add(repo, inputs): +def add(repo, encoding, inputs): """Add files to the hallmark index. - `hallmark add FORMAT` uses the branch format string workflow. + `hallmark add [--regex] FORMAT` uses the branch format string workflow. `hallmark add "."` rebuilds the manifest from current files that match the branch `fmt` in `config.yml`. Explicit path inputs such as shell-expanded `*` are not supported yet @@ -136,7 +144,7 @@ def add(repo, inputs): """ try: if len(inputs) == 1: - pf = repo.add(inputs[0]) + pf = repo.add(inputs[0], encoding) else: pf = repo.add_paths(list(inputs)) except (RuntimeError, ValueError, FileNotFoundError) as e: @@ -205,6 +213,17 @@ def log(repo): click.echo(history) +@hallmark.command(short_help="List hallmark branches.") +@click.pass_obj +def branch(repo): + """List local hallmark branches.""" + snapshot = repo.branches() + current = snapshot["current"] + for name in snapshot["names"]: + prefix = "*" if name == current else " " + click.echo(f"{prefix} {name}") + + @hallmark.command(short_help="Switch to another branch.") @click.argument("target_branch") @click.pass_obj @@ -219,7 +238,7 @@ def checkout(repo, target_branch): try: if repo.checkout(target_branch): click.echo(f'Switched to branch "{target_branch}".') - except (GitError, RuntimeError, ValueError, FileNotFoundError) as e: + except (GitError, RuntimeError, ValueError, FileNotFoundError, CheckoutError) as e: raise ClickException(str(e)) @@ -245,7 +264,7 @@ def clone(url, path, no_fetch_data, max_workers): Supports concurrent downloads for efficient retrieval of large datasets. """ try: - repo = Repo.clone(url, path) + repo = Repo.clone(url, path, fetch_data=False) click.echo(f'Successfully cloned to "{path}"') if not no_fetch_data: diff --git a/mod/hallmark/error.py b/mod/hallmark/error.py index 59aee4f..332768f 100644 --- a/mod/hallmark/error.py +++ b/mod/hallmark/error.py @@ -56,3 +56,6 @@ def from_git_command( class DestinationExistsError(CloneError): """Raised when clone destination already exists.""" + +class CheckoutError(HallmarkError): + """Raised when a hallmark checkout cannot proceed safely.""" diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 03356c7..f35278f 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -23,7 +23,7 @@ from typing import Dict, List, Optional, Tuple, Union from .dothm import Dothm -from .error import DestinationExistsError +from .error import CheckoutError, DestinationExistsError from .objects import Objects from .paraframe import ParaFrame from .repo_config import branch_encodings, branch_fmt, path_from_row, row_to_path, \ @@ -94,7 +94,15 @@ def init(cls, path: Union[Path, str]) -> "Repo": return cls(path) @classmethod - def clone(cls, url: str, path: Union[Path, str]) -> "Repo": + def clone( + cls, + url: str, + path: Union[Path, str], + *, + fetch_data: bool = True, + max_workers: int = 4, + show_progress: bool = False, + ) -> "Repo": clone_path = Path(path) if clone_path.exists(): raise DestinationExistsError( @@ -110,7 +118,31 @@ def clone(cls, url: str, path: Union[Path, str]) -> "Repo": if worktree_path: Worktree.init(worktree_path) - return cls(path) + repo = cls(path) + repo.download_result = None + + if fetch_data and worktree_path: + from .downloader import DownloadError, download_remote_data + + result = download_remote_data( + repo, + worktree_path, + max_workers=max_workers, + show_progress=show_progress, + ) + repo.download_result = result + if result["failed"]: + errors = result.get("errors", []) + details = "\n".join(f" - {error}" for error in errors[:5]) + remaining = result["failed"] - len(errors[:5]) + if remaining > 0: + details += f"\n - ... {remaining} more error(s)" + raise DownloadError( + f"Failed to download {result['failed']} file(s):\n" + f"{details}" + ) + + return repo @staticmethod def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str: @@ -147,6 +179,11 @@ def status(self) -> dict[str, object]: head_state = load_head_state(self) head_map = manifest_map(head_state) staged_map = manifest_map(self.state) + state_changes = sorted({ + diff.a_path or diff.b_path + for diff in self.dothm.index.diff("HEAD") + if diff.a_path or diff.b_path + }) staged_added = sorted(path for path in staged_map if path not in head_map) staged_deleted = sorted(path for path in head_map if path not in staged_map) @@ -180,6 +217,7 @@ def status(self) -> dict[str, object]: return { "branch": self.dothm.active_branch.name, "staged": { + "state": state_changes, "added": staged_added, "modified": staged_modified, "deleted": staged_deleted, @@ -215,6 +253,11 @@ def add(self, fstr: str, encoding: bool = False) -> ParaFrame: self.dothm.dump(self.state) return pf.drop(columns=["sha1"], errors="ignore") + try: + previous_fmt = branch_fmt(self) + except RuntimeError: + previous_fmt = None + set_branch_fmt(self, fstr) with chdir(self.worktree): @@ -232,7 +275,11 @@ def add(self, fstr: str, encoding: bool = False) -> ParaFrame: for path in pf["path"].astype(str) ] - self.state.update(manifest_frame_from_pf(pf, fstr)) + manifest = manifest_frame_from_pf(pf, fstr) + if previous_fmt is None or previous_fmt != fstr: + self.state.replace(manifest) + else: + self.state.update(manifest) self.dothm.dump(self.state) return pf.drop(columns=["sha1"], errors="ignore") @@ -253,31 +300,37 @@ def log(self) -> str: return "" return self.dothm.git.log() + def branches(self) -> dict[str, object]: + current = self.dothm.active_branch.name + names = sorted(head.name for head in self.dothm.heads) + return {"current": current, "names": names} + def checkout(self, target_branch: str) -> bool: if not isinstance(target_branch, str) or not target_branch.strip(): raise ValueError("branch name must be a non-empty string") if self.worktree is None: - raise RuntimeError("cannot checkout without a worktree") + raise CheckoutError("cannot checkout without a worktree") ensure_clean_tracked_files(self) existing = {head.name for head in self.dothm.heads} new_branch = target_branch not in existing current_tracked = tracked_paths(self) target_state = load_branch_data(self, target_branch) + target_fmt = target_state.config["data"][0]["fmt"] + target_tracked = { + row_to_path(row, target_fmt) + for _, row in target_state.data.iterrows() + } for _, row in target_state.data.iterrows(): - rel_path = row_to_path(row, target_state.config["data"][0]["fmt"]) - if rel_path not in current_tracked and (self.worktree / rel_path).exists(): - raise RuntimeError( - f'target tracked path "{rel_path}" already exists ' - "as an untracked file") - - # remove current tracked files from worktree - for _, row in self.state.data.iterrows(): - path = self.worktree / path_from_row(self, row) - if path.exists(): - path.unlink() + rel_path = row_to_path(row, target_fmt) + target_path = self.worktree / rel_path + if rel_path not in current_tracked and target_path.exists(): + if self.checksum(target_path) != row["sha1"]: + raise CheckoutError( + f'target tracked path "{rel_path}" already exists ' + "as an untracked file") # switch .hm branch if new_branch: @@ -288,10 +341,26 @@ def checkout(self, target_branch: str) -> bool: # reload state from the new branch self.state = self.dothm.load() + removed_paths = [] + for rel_path in sorted(current_tracked - target_tracked, reverse=True): + path = self.worktree / rel_path + if path.exists(): + path.unlink() + removed_paths.append(path) + # restore files from objects store via hardlinks for _, row in self.state.data.iterrows(): self.objects.restore(row["sha1"], self.worktree / path_from_row(self, row)) + for path in removed_paths: + parent = path.parent + while parent != self.worktree and parent.exists(): + try: + parent.rmdir() + except OSError: + break + parent = parent.parent + return True def add_worktree(self, target_branch: str) -> bool: diff --git a/mod/hallmark/repo_worktree.py b/mod/hallmark/repo_worktree.py index 6da32de..1122bc3 100644 --- a/mod/hallmark/repo_worktree.py +++ b/mod/hallmark/repo_worktree.py @@ -4,6 +4,7 @@ from pathlib import Path from .repo_config import branch_fmt, path_from_row +from .error import CheckoutError def effective_cwd(repo) -> Path: @@ -38,21 +39,21 @@ def tracked_paths(repo) -> set[Path]: def ensure_clean_tracked_files(repo) -> None: if repo.worktree is None: - raise RuntimeError("cannot checkout without a worktree") + raise CheckoutError("cannot checkout without a worktree") for _, row in repo.state.data.iterrows(): rel_path = path_from_row(repo, row) path = repo.worktree / rel_path if not path.exists(): - raise RuntimeError( + raise CheckoutError( f'tracked file "{rel_path}" is missing; commit or \ restore it before checkout') if repo.checksum(path) != row["sha1"]: - raise RuntimeError( + raise CheckoutError( f'tracked file "{rel_path}" has uncommitted changes; \ commit them before checkout') if repo.dothm.index.diff("HEAD"): - raise RuntimeError( + raise CheckoutError( "you have uncommitted hallmark state changes — " \ "commit them before checkout") diff --git a/test/test_cli.py b/test/test_cli.py index ab9495d..ffea834 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -19,6 +19,7 @@ from click.testing import CliRunner from git import Repo as GitRepo +from hallmark import ParaFrame from hallmark.cli import hallmark from hallmark.downloader import DownloadError @@ -123,6 +124,26 @@ def test_cli_add_dot_and_explicit_paths(): assert "explicit path add is not supported" in result.output +def test_cli_add_regex_flag(monkeypatch): + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(hallmark, ["init", "repo"]) + + with chdir("repo"): + called = {} + + def fake_add(self, fmt, encoding=False): + called["fmt"] = fmt + called["encoding"] = encoding + return ParaFrame([{"path": "am0.5_i30.h5"}]) + + monkeypatch.setattr("hallmark.cli.Repo.add", fake_add) + result = runner.invoke(hallmark, ["add", "--regex", "."]) + + assert result.exit_code == 0 + assert called == {"fmt": ".", "encoding": True} + + def test_cli_status(): runner = CliRunner() with runner.isolated_filesystem(): @@ -182,6 +203,27 @@ def test_cli_set_config_and_add_dot(): assert r"aspin: m([0-9]+(\.[0-9]+)?|\.[0-9]+)" in config +def test_cli_status_shows_staged_state_after_set_config(): + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(hallmark, ["init", "repo"]) + + with chdir("repo"): + result = runner.invoke(hallmark, ["set-config", "--fmt", "b{a}_i{i}.h5"]) + assert result.exit_code == 0 + + result = runner.invoke(hallmark, ["status"]) + + assert result.exit_code == 0 + assert "Changes to be committed:" in result.output + assert "state: config.yml" in result.output + assert "nothing to commit, working tree clean" not in result.output + + result = runner.invoke(hallmark, ["commit", "-m", "config only"]) + assert result.exit_code == 0 + assert "Committed staged state changes." in result.output + + def test_cli_set_config_rejects_malformed_encoding(): runner = CliRunner() with runner.isolated_filesystem(): @@ -218,6 +260,24 @@ def test_cli_log(): assert result.output.strip() == expected.strip() +def test_cli_branch_lists_local_branches_and_marks_current(): + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(hallmark, ["init", "repo"]) + + with chdir("repo"): + Path("a0_i0.h5").write_text("a0_i0.h5\n", encoding="utf-8") + runner.invoke(hallmark, ["add", "a{a}_i{i}.h5"]) + runner.invoke(hallmark, ["commit", "-m", "add first file"]) + runner.invoke(hallmark, ["checkout", "experiment"]) + + result = runner.invoke(hallmark, ["branch"]) + + assert result.exit_code == 0 + assert " main" in result.output + assert "* experiment" in result.output + + def test_clone_existing_destination_reports_plain_git_stderr(): runner = CliRunner() with runner.isolated_filesystem(): diff --git a/test/test_hallmark.py b/test/test_hallmark.py index 2968fa9..a9bd798 100644 --- a/test/test_hallmark.py +++ b/test/test_hallmark.py @@ -209,6 +209,26 @@ def test_repo_add_pattern_keeps_deleted_manifest_rows(tmp_path): ] +def test_repo_add_pattern_replaces_manifest_when_fmt_changes(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0.4_i30_w3.h5", "b0.4_i30_w3.h5"]) + + repo.add("a{a}_i{i}_w{w}.h5") + repo.commit("main a data") + repo.checkout("experiment") + repo.add("b{a}_i{i}_w{w}.h5") + + assert repo.state.config["data"] == [{"fmt": "b{a}_i{i}_w{w}.h5", "encoding": None}] + assert repo.state.data.to_dict(orient="records") == [ + { + "sha1": Repo.checksum(repo.worktree / "b0.4_i30_w3.h5"), + "a": "0.4", + "i": "30", + "w": "3", + } + ] + + def test_repo_add_paths_is_not_supported_yet(tmp_path): repo = Repo.init(tmp_path / "repo") _write_files(repo.worktree, ["a0_i0.h5", "a0_i30.h5", "b0_i45.h5"]) @@ -300,7 +320,12 @@ def test_repo_status_reports_staged_worktree_and_untracked_changes(tmp_path): snapshot = repo.status() assert snapshot["branch"] == "main" - assert snapshot["staged"] == {"added": [], "modified": [], "deleted": []} + assert snapshot["staged"] == { + "state": [], + "added": [], + "modified": [], + "deleted": [], + } assert snapshot["worktree"]["modified"] == ["a0_i0.h5"] assert snapshot["worktree"]["deleted"] == ["a0_i30.h5"] assert snapshot["untracked"] == ["extra.h5"] @@ -378,6 +403,33 @@ def test_checkout_leaves_untracked_files(tmp_path): for path in Path(str(repo.worktree)).glob("*.h5")) == ["a0_i0.h5"] +def test_checkout_rebuilds_worktree_for_branch_specific_nested_fmt(tmp_path): + repo = Repo.init(tmp_path / "repo") + (repo.worktree / "main").mkdir() + _write_files(repo.worktree, ["main/a0_i0.h5"]) + repo.add("main/a{a}_i{i}.h5") + repo.commit("main data") + + repo.checkout("experiment") + (repo.worktree / "exp" / "run1").mkdir(parents=True) + _write_files(repo.worktree, ["exp/run1/b0_i0.h5"]) + repo.add("exp/run{run}/b{a}_i{i}.h5") + repo.commit("experiment data") + + repo.checkout("main") + root = Path(str(repo.worktree)) + assert sorted(str(path.relative_to(root)) for path in root.rglob("*.h5")) == [ + "main/a0_i0.h5", + ] + assert not (repo.worktree / "exp").exists() + + repo.checkout("experiment") + assert sorted(str(path.relative_to(root)) for path in root.rglob("*.h5")) == [ + "exp/run1/b0_i0.h5", + ] + assert not (repo.worktree / "main").exists() + + def test_checkout_aborts_on_dirty_tracked_file(tmp_path): repo = Repo.init(tmp_path / "repo") _write_files(repo.worktree, ["a0_i0.h5"]) @@ -410,3 +462,69 @@ def test_checkout_aborts_on_untracked_path_conflict(tmp_path): with pytest.raises(RuntimeError, match='target tracked path "a1_i45.h5" already exists as an untracked file'): repo.checkout("experiment") + + +def test_checkout_allows_return_to_branch_when_target_files_already_match(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5", "b0_i0.h5"]) + repo.add("a{a}_i{i}.h5") + repo.commit("main data") + + repo.checkout("experiment") + repo.add("b{a}_i{i}.h5") + repo.commit("experiment data") + + repo.checkout("main") + + assert sorted(path.name for path in Path(str(repo.worktree)).glob("*.h5")) == [ + "a0_i0.h5", + ] + + +def test_repo_clone_downloads_remote_data_by_default(monkeypatch, tmp_path): + source = Repo.init(tmp_path / "source") + _write_files(source.worktree, ["a0_i0.h5"]) + source.add("a{a}_i{i}.h5") + source.set_config(remote_url="https://example.com/data/") + expected_sha1 = Repo.checksum(source.worktree / "a0_i0.h5") + source.commit("add source data") + + captured = {} + + def fake_download_file(url, destination, sha1, chunk_size=8192): + captured["url"] = url + captured["sha1"] = sha1 + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text("downloaded\n", encoding="utf-8") + return destination.stat().st_size + + monkeypatch.setattr("hallmark.downloader._download_file", fake_download_file) + + clone = Repo.clone(str(source.dothm.path), tmp_path / "clone") + + assert captured == { + "url": "https://example.com/data/a0_i0.h5", + "sha1": expected_sha1, + } + assert (clone.worktree / "a0_i0.h5").read_text(encoding="utf-8") == \ + "downloaded\n" + assert clone.download_result["succeeded"] == 1 + assert clone.download_result["failed"] == 0 + + +def test_repo_clone_can_skip_remote_data_download(monkeypatch, tmp_path): + source = Repo.init(tmp_path / "source") + _write_files(source.worktree, ["a0_i0.h5"]) + source.add("a{a}_i{i}.h5") + source.set_config(remote_url="https://example.com/data/") + source.commit("add source data") + + def fail_download(*args, **kwargs): + raise AssertionError("download should not be attempted") + + monkeypatch.setattr("hallmark.downloader._download_file", fail_download) + + clone = Repo.clone(str(source.dothm.path), tmp_path / "clone", fetch_data=False) + + assert not (clone.worktree / "a0_i0.h5").exists() + assert clone.download_result is None