diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5bbddd..16556b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9","3.10","3.11","3.12","3.13","3.14"] + python-version: ["3.9", "3.10","3.11","3.12","3.13","3.14"] steps: - uses: actions/checkout@v4 @@ -27,11 +27,11 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install ruff pytest + python -m pip install ruff pytest python -m pip install -e . - name: Lint with ruff run: | - ruff check --select E,F --exclude ./demos + ruff check --select E,F --exclude ./demo - name: Test with pytest run: | - pytest + pytest diff --git a/.gitignore b/.gitignore index 059e881..ca5850a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ !*.*py !*.*ml !*file +!demo/demo_cli.ipynb +!demo/demo_python.ipynb __pycache__/ *.egg-info/ diff --git a/demo/ParaFrame.ipynb b/demo/ParaFrame.ipynb deleted file mode 100644 index 5cd5fbf..0000000 --- a/demo/ParaFrame.ipynb +++ /dev/null @@ -1,1678 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# ParaFrame Demo\n", - "\n", - "This notebook demonstrates the current `hallmark` ParaFrame API.\n", - "\n", - "`ParaFrame` is a Pandas DataFrame subclass that discovers and parses files whose names follow a Python format string pattern. It extracts named parameters directly into DataFrame columns, making it easy to filter and select files across large parameter surveys.\n", - "\n", - "Key components:\n", - "- **`.hallmark.yaml`** — config file that lives alongside your data, declaring format strings and optional regex encodings for special filename conventions\n", - "- **`ParaFrame.parse()`** — discovers matching files and returns a DataFrame\n", - "- **`ParaFrame.__call__()`** — filters the DataFrame by column values" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "1", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:44.420836Z", - "iopub.status.busy": "2026-03-17T01:36:44.420675Z", - "iopub.status.idle": "2026-03-17T01:36:44.495945Z", - "shell.execute_reply": "2026-03-17T01:36:44.495136Z" - } - }, - "outputs": [], - "source": [ - "# auto reload modules when they change\n", - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, - "source": [ - "## 1. The `.hallmark.yaml` Config File\n", - "\n", - "Before calling `ParaFrame.parse()`, a `.hallmark.yaml` must exist in your data directory. It declares:\n", - "- `fmt`: the Python format string for your file naming convention\n", - "- `encoding` (optional): regex patterns for parameters that use special filename conventions\n", - "\n", - "Here is the `.hallmark.yaml` used in this demo:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:44.497846Z", - "iopub.status.busy": "2026-03-17T01:36:44.497728Z", - "iopub.status.idle": "2026-03-17T01:36:44.764964Z", - "shell.execute_reply": "2026-03-17T01:36:44.764361Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "data:\n", - "- fmt: '{mag:d}a{aspin}_w{win:d}.h5'\n", - " # path_to_fmt: m5/data\n", - " encoding:\n", - " aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)\n", - "\n", - "- fmt: a_{a:d}/b_{b:d}.txt\n", - " # path_to_fmt: data\n", - "\n", - "- fmt: a{aspin}/b_{b:d}.txt\n", - " # path_to_fmt: data\n", - " encoding:\n", - " aspin: ''\n" - ] - } - ], - "source": [ - "! cat data/hallmark.yaml # check what fmts are present" - ] - }, - { - "cell_type": "markdown", - "id": "4", - "metadata": {}, - "source": [ - "## 2. Simple File Discovery (no encoding)\n", - "\n", - "Files structured as `a_{a:d}/b_{b:d}.txt` — no special encoding, so `encoding=False` (the default)." - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "### Create sample data files" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "6", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:44.767668Z", - "iopub.status.busy": "2026-03-17T01:36:44.767500Z", - "iopub.status.idle": "2026-03-17T01:36:44.853271Z", - "shell.execute_reply": "2026-03-17T01:36:44.852483Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "b_10.txt\n", - "b_11.txt\n", - "b_12.txt\n", - "b_13.txt\n", - "\u001b[30m\u001b[47mb_14.txt\u001b[m\u001b[m\n" - ] - } - ], - "source": [ - "%%bash\n", - "# make some test files to parse\n", - "for a in {0..4}; do\n", - " mkdir -p \"data/a_$a\"\n", - " for b in {10..14}; do\n", - " touch \"data/a_$a/b_$b.txt\"\n", - " done\n", - "done\n", - "ls data/a_0/" - ] - }, - { - "cell_type": "markdown", - "id": "7", - "metadata": {}, - "source": [ - "### Parse with `repo_path`\n", - "\n", - "`repo_path` tells `ParaFrame` where to find `.hallmark.yaml` and where to search for files." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "8", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:44.855422Z", - "iopub.status.busy": "2026-03-17T01:36:44.855306Z", - "iopub.status.idle": "2026-03-17T01:36:45.283614Z", - "shell.execute_reply": "2026-03-17T01:36:45.283219Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pattern: data/a_{a:d}/b_{b:d}.txt\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
0a_0/b_10.txt0.010.0
1a_0/b_11.txt0.011.0
2a_0/b_12.txt0.012.0
3a_0/b_13.txt0.013.0
4a_0/b_14.txt0.014.0
5a_1/b_10.txt1.010.0
6a_1/b_11.txt1.011.0
7a_1/b_12.txt1.012.0
8a_1/b_13.txt1.013.0
9a_1/b_14.txt1.014.0
10a_2/b_10.txt2.010.0
11a_2/b_11.txt2.011.0
12a_2/b_12.txt2.012.0
13a_2/b_13.txt2.013.0
14a_2/b_14.txt2.014.0
15a_3/b_10.txt3.010.0
16a_3/b_11.txt3.011.0
17a_3/b_12.txt3.012.0
18a_3/b_13.txt3.013.0
19a_3/b_14.txt3.014.0
20a_4/b_10.txt4.010.0
21a_4/b_11.txt4.011.0
22a_4/b_12.txt4.012.0
23a_4/b_13.txt4.013.0
24a_4/b_14.txt4.014.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "0 a_0/b_10.txt 0.0 10.0\n", - "1 a_0/b_11.txt 0.0 11.0\n", - "2 a_0/b_12.txt 0.0 12.0\n", - "3 a_0/b_13.txt 0.0 13.0\n", - "4 a_0/b_14.txt 0.0 14.0\n", - "5 a_1/b_10.txt 1.0 10.0\n", - "6 a_1/b_11.txt 1.0 11.0\n", - "7 a_1/b_12.txt 1.0 12.0\n", - "8 a_1/b_13.txt 1.0 13.0\n", - "9 a_1/b_14.txt 1.0 14.0\n", - "10 a_2/b_10.txt 2.0 10.0\n", - "11 a_2/b_11.txt 2.0 11.0\n", - "12 a_2/b_12.txt 2.0 12.0\n", - "13 a_2/b_13.txt 2.0 13.0\n", - "14 a_2/b_14.txt 2.0 14.0\n", - "15 a_3/b_10.txt 3.0 10.0\n", - "16 a_3/b_11.txt 3.0 11.0\n", - "17 a_3/b_12.txt 3.0 12.0\n", - "18 a_3/b_13.txt 3.0 13.0\n", - "19 a_3/b_14.txt 3.0 14.0\n", - "20 a_4/b_10.txt 4.0 10.0\n", - "21 a_4/b_11.txt 4.0 11.0\n", - "22 a_4/b_12.txt 4.0 12.0\n", - "23 a_4/b_13.txt 4.0 13.0\n", - "24 a_4/b_14.txt 4.0 14.0" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from hallmark import ParaFrame\n", - "\n", - "# repo_path tells hallmark where to find .hallmark.yaml\n", - "pf = ParaFrame.parse(\"a_{a:d}/b_{b:d}.txt\", repo_path=\"data\")\n", - "pf" - ] - }, - { - "cell_type": "markdown", - "id": "9", - "metadata": {}, - "source": [ - "### Parse with `set_rel_yaml_path`\n", - "\n", - "Instead of passing `repo_path` every time, set it globally once." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "10", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.285073Z", - "iopub.status.busy": "2026-03-17T01:36:45.284949Z", - "iopub.status.idle": "2026-03-17T01:36:45.302480Z", - "shell.execute_reply": "2026-03-17T01:36:45.302088Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pattern: data/a_{a:d}/b_{b:d}.txt\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
0a_0/b_10.txt0.010.0
1a_0/b_11.txt0.011.0
2a_0/b_12.txt0.012.0
3a_0/b_13.txt0.013.0
4a_0/b_14.txt0.014.0
5a_1/b_10.txt1.010.0
6a_1/b_11.txt1.011.0
7a_1/b_12.txt1.012.0
8a_1/b_13.txt1.013.0
9a_1/b_14.txt1.014.0
10a_2/b_10.txt2.010.0
11a_2/b_11.txt2.011.0
12a_2/b_12.txt2.012.0
13a_2/b_13.txt2.013.0
14a_2/b_14.txt2.014.0
15a_3/b_10.txt3.010.0
16a_3/b_11.txt3.011.0
17a_3/b_12.txt3.012.0
18a_3/b_13.txt3.013.0
19a_3/b_14.txt3.014.0
20a_4/b_10.txt4.010.0
21a_4/b_11.txt4.011.0
22a_4/b_12.txt4.012.0
23a_4/b_13.txt4.013.0
24a_4/b_14.txt4.014.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "0 a_0/b_10.txt 0.0 10.0\n", - "1 a_0/b_11.txt 0.0 11.0\n", - "2 a_0/b_12.txt 0.0 12.0\n", - "3 a_0/b_13.txt 0.0 13.0\n", - "4 a_0/b_14.txt 0.0 14.0\n", - "5 a_1/b_10.txt 1.0 10.0\n", - "6 a_1/b_11.txt 1.0 11.0\n", - "7 a_1/b_12.txt 1.0 12.0\n", - "8 a_1/b_13.txt 1.0 13.0\n", - "9 a_1/b_14.txt 1.0 14.0\n", - "10 a_2/b_10.txt 2.0 10.0\n", - "11 a_2/b_11.txt 2.0 11.0\n", - "12 a_2/b_12.txt 2.0 12.0\n", - "13 a_2/b_13.txt 2.0 13.0\n", - "14 a_2/b_14.txt 2.0 14.0\n", - "15 a_3/b_10.txt 3.0 10.0\n", - "16 a_3/b_11.txt 3.0 11.0\n", - "17 a_3/b_12.txt 3.0 12.0\n", - "18 a_3/b_13.txt 3.0 13.0\n", - "19 a_3/b_14.txt 3.0 14.0\n", - "20 a_4/b_10.txt 4.0 10.0\n", - "21 a_4/b_11.txt 4.0 11.0\n", - "22 a_4/b_12.txt 4.0 12.0\n", - "23 a_4/b_13.txt 4.0 13.0\n", - "24 a_4/b_14.txt 4.0 14.0" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from hallmark import ParaFrame, set_rel_yaml_path\n", - "\n", - "# set once globally so you dont have to pass repo_path every time\n", - "set_rel_yaml_path(\"data\")\n", - "\n", - "pf = ParaFrame.parse(\"a_{a:d}/b_{b:d}.txt\")\n", - "pf" - ] - }, - { - "cell_type": "markdown", - "id": "11", - "metadata": {}, - "source": [ - "### Debug mode\n", - "\n", - "`debug=True` prints the resolved glob pattern and match count — useful when no files are found." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "12", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.303832Z", - "iopub.status.busy": "2026-03-17T01:36:45.303732Z", - "iopub.status.idle": "2026-03-17T01:36:45.316212Z", - "shell.execute_reply": "2026-03-17T01:36:45.315734Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pattern: data/a_{a:d}/b_{b:d}.txt\n", - "0 data/a_{a:d}/b_{b:d}.txt () {}\n", - "1 data/a_{a:s}/b_{b:d}.txt () {'a': '*'}\n", - "2 data/a_{a:s}/b_{b:s}.txt () {'a': '*', 'b': '*'}\n", - "Pattern: \"data/a_*/b_*.txt\"\n", - "25 matches, e.g., \"data/a_0/b_10.txt\"\n" - ] - } - ], - "source": [ - "# debug=True shows the resolved glob pattern and how many files matched\n", - "pf = ParaFrame.parse(\"a_{a:d}/b_{b:d}.txt\", repo_path=\"data\", debug=True)" - ] - }, - { - "cell_type": "markdown", - "id": "13", - "metadata": {}, - "source": [ - "## 3. Filtering\n", - "\n", - "Call a `ParaFrame` like a function to filter by column values:\n", - "- **scalar** → exact match\n", - "- **list or tuple** → OR match\n", - "- **chaining** → AND logic" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "14", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.317526Z", - "iopub.status.busy": "2026-03-17T01:36:45.317420Z", - "iopub.status.idle": "2026-03-17T01:36:45.327893Z", - "shell.execute_reply": "2026-03-17T01:36:45.327195Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
0a_0/b_10.txt0.010.0
1a_0/b_11.txt0.011.0
2a_0/b_12.txt0.012.0
3a_0/b_13.txt0.013.0
4a_0/b_14.txt0.014.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "0 a_0/b_10.txt 0.0 10.0\n", - "1 a_0/b_11.txt 0.0 11.0\n", - "2 a_0/b_12.txt 0.0 12.0\n", - "3 a_0/b_13.txt 0.0 13.0\n", - "4 a_0/b_14.txt 0.0 14.0" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Filter a == 0\n", - "pf(a=0)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "15", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.329283Z", - "iopub.status.busy": "2026-03-17T01:36:45.329178Z", - "iopub.status.idle": "2026-03-17T01:36:45.338757Z", - "shell.execute_reply": "2026-03-17T01:36:45.338404Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
0a_0/b_10.txt0.010.0
1a_0/b_11.txt0.011.0
2a_0/b_12.txt0.012.0
3a_0/b_13.txt0.013.0
4a_0/b_14.txt0.014.0
5a_1/b_10.txt1.010.0
6a_1/b_11.txt1.011.0
7a_1/b_12.txt1.012.0
8a_1/b_13.txt1.013.0
9a_1/b_14.txt1.014.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "0 a_0/b_10.txt 0.0 10.0\n", - "1 a_0/b_11.txt 0.0 11.0\n", - "2 a_0/b_12.txt 0.0 12.0\n", - "3 a_0/b_13.txt 0.0 13.0\n", - "4 a_0/b_14.txt 0.0 14.0\n", - "5 a_1/b_10.txt 1.0 10.0\n", - "6 a_1/b_11.txt 1.0 11.0\n", - "7 a_1/b_12.txt 1.0 12.0\n", - "8 a_1/b_13.txt 1.0 13.0\n", - "9 a_1/b_14.txt 1.0 14.0" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Filter a == 0 or a == 1\n", - "pf(a=[0, 1])" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "16", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.340105Z", - "iopub.status.busy": "2026-03-17T01:36:45.340030Z", - "iopub.status.idle": "2026-03-17T01:36:45.348202Z", - "shell.execute_reply": "2026-03-17T01:36:45.347785Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
0a_0/b_10.txt0.010.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "0 a_0/b_10.txt 0.0 10.0" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Filter a == 0 AND b == 10\n", - "pf(a=0)(b=10)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "17", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.349400Z", - "iopub.status.busy": "2026-03-17T01:36:45.349317Z", - "iopub.status.idle": "2026-03-17T01:36:45.358152Z", - "shell.execute_reply": "2026-03-17T01:36:45.357766Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
10a_2/b_10.txt2.010.0
11a_2/b_11.txt2.011.0
12a_2/b_12.txt2.012.0
13a_2/b_13.txt2.013.0
14a_2/b_14.txt2.014.0
15a_3/b_10.txt3.010.0
16a_3/b_11.txt3.011.0
17a_3/b_12.txt3.012.0
18a_3/b_13.txt3.013.0
19a_3/b_14.txt3.014.0
20a_4/b_10.txt4.010.0
21a_4/b_11.txt4.011.0
22a_4/b_12.txt4.012.0
23a_4/b_13.txt4.013.0
24a_4/b_14.txt4.014.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "10 a_2/b_10.txt 2.0 10.0\n", - "11 a_2/b_11.txt 2.0 11.0\n", - "12 a_2/b_12.txt 2.0 12.0\n", - "13 a_2/b_13.txt 2.0 13.0\n", - "14 a_2/b_14.txt 2.0 14.0\n", - "15 a_3/b_10.txt 3.0 10.0\n", - "16 a_3/b_11.txt 3.0 11.0\n", - "17 a_3/b_12.txt 3.0 12.0\n", - "18 a_3/b_13.txt 3.0 13.0\n", - "19 a_3/b_14.txt 3.0 14.0\n", - "20 a_4/b_10.txt 4.0 10.0\n", - "21 a_4/b_11.txt 4.0 11.0\n", - "22 a_4/b_12.txt 4.0 12.0\n", - "23 a_4/b_13.txt 4.0 13.0\n", - "24 a_4/b_14.txt 4.0 14.0" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# use pandas masking directly for more complex conditions\n", - "pf[(2 <= pf.a) & (pf.a <= 4)]" - ] - }, - { - "cell_type": "markdown", - "id": "18", - "metadata": {}, - "source": [ - "### Using filtered paths in a workflow" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "19", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.359332Z", - "iopub.status.busy": "2026-03-17T01:36:45.359244Z", - "iopub.status.idle": "2026-03-17T01:36:45.366724Z", - "shell.execute_reply": "2026-03-17T01:36:45.366336Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Processing \"a_0/b_10.txt\"...\n", - "Processing \"a_0/b_11.txt\"...\n", - "Processing \"a_0/b_12.txt\"...\n", - "Processing \"a_0/b_13.txt\"...\n", - "Processing \"a_0/b_14.txt\"...\n", - "Processing \"a_1/b_10.txt\"...\n", - "Processing \"a_2/b_10.txt\"...\n", - "Processing \"a_3/b_10.txt\"...\n", - "Processing \"a_4/b_10.txt\"...\n" - ] - } - ], - "source": [ - "# pf(a=0, b=10) means a==0 OR b==10\n", - "for p in pf(a=0, b=10).path:\n", - " print(f'Processing \"{p}\"...')" - ] - }, - { - "cell_type": "markdown", - "id": "20", - "metadata": {}, - "source": [ - "## 4. Encoding — Special Filename Conventions\n", - "\n", - "Some datasets use non-standard filename conventions. For example, black hole spin values are stored as `m0.94` to represent `-0.94` (the `m` prefix means \"minus\").\n", - "\n", - "The `.hallmark.yaml` handles this with the `encoding` key, which holds a regex pattern. When `encoding=True` is passed, the regex substitution is applied to the filename before parsing.\n", - "\n", - "Relevant YAML entry:\n", - "```yaml\n", - "- fmt: '{mag:d}a{aspin}_w{win:d}.h5'\n", - " encoding:\n", - " aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "21", - "metadata": {}, - "source": [ - "### Create sample spin data files" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "22", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.368197Z", - "iopub.status.busy": "2026-03-17T01:36:45.368111Z", - "iopub.status.idle": "2026-03-17T01:36:45.421524Z", - "shell.execute_reply": "2026-03-17T01:36:45.421043Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "data/1a0_w10.h5\n", - "data/1a0_w20.h5\n", - "data/1a0.5_w10.h5\n", - "data/1a0.5_w20.h5\n", - "data/1am0.5_w10.h5\n", - "data/1am0.5_w20.h5\n", - "data/1am0.94_w10.h5\n", - "data/1am0.94_w20.h5\n", - "data/2a0_w10.h5\n", - "data/2a0_w20.h5\n", - "data/2a0.5_w10.h5\n", - "data/2a0.5_w20.h5\n", - "data/2am0.5_w10.h5\n", - "data/2am0.5_w20.h5\n", - "data/2am0.94_w10.h5\n", - "data/2am0.94_w20.h5\n" - ] - } - ], - "source": [ - "%%bash\n", - "# spin values use m prefix to mean negative e.g. m0.94 = -0.94\n", - "for mag in 1 2; do\n", - " for aspin in 0 0.5 m0.5 m0.94; do\n", - " for win in 10 20; do\n", - " touch \"data/${mag}a${aspin}_w${win}.h5\"\n", - " done\n", - " done\n", - "done\n", - "ls data/*.h5" - ] - }, - { - "cell_type": "markdown", - "id": "23", - "metadata": {}, - "source": [ - "### Parse with `encoding=True`\n", - "\n", - "The `m` prefix is decoded by the regex: `m0.94` → `-0.94`, `m0.5` → `-0.5`." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "24", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.422874Z", - "iopub.status.busy": "2026-03-17T01:36:45.422783Z", - "iopub.status.idle": "2026-03-17T01:36:45.438474Z", - "shell.execute_reply": "2026-03-17T01:36:45.438096Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pattern: data/{mag:d}a{aspin}_w{win:d}.h5\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathmagaspinwin
01a0.5_w10.h51.00.5010.0
11a0.5_w20.h51.00.5020.0
21a0_w10.h51.00.0010.0
31a0_w20.h51.00.0020.0
41am0.5_w10.h51.0-0.5010.0
51am0.5_w20.h51.0-0.5020.0
61am0.94_w10.h51.0-0.9410.0
71am0.94_w20.h51.0-0.9420.0
82a0.5_w10.h52.00.5010.0
92a0.5_w20.h52.00.5020.0
102a0_w10.h52.00.0010.0
112a0_w20.h52.00.0020.0
122am0.5_w10.h52.0-0.5010.0
132am0.5_w20.h52.0-0.5020.0
142am0.94_w10.h52.0-0.9410.0
152am0.94_w20.h52.0-0.9420.0
\n", - "
" - ], - "text/plain": [ - " path mag aspin win\n", - "0 1a0.5_w10.h5 1.0 0.50 10.0\n", - "1 1a0.5_w20.h5 1.0 0.50 20.0\n", - "2 1a0_w10.h5 1.0 0.00 10.0\n", - "3 1a0_w20.h5 1.0 0.00 20.0\n", - "4 1am0.5_w10.h5 1.0 -0.50 10.0\n", - "5 1am0.5_w20.h5 1.0 -0.50 20.0\n", - "6 1am0.94_w10.h5 1.0 -0.94 10.0\n", - "7 1am0.94_w20.h5 1.0 -0.94 20.0\n", - "8 2a0.5_w10.h5 2.0 0.50 10.0\n", - "9 2a0.5_w20.h5 2.0 0.50 20.0\n", - "10 2a0_w10.h5 2.0 0.00 10.0\n", - "11 2a0_w20.h5 2.0 0.00 20.0\n", - "12 2am0.5_w10.h5 2.0 -0.50 10.0\n", - "13 2am0.5_w20.h5 2.0 -0.50 20.0\n", - "14 2am0.94_w10.h5 2.0 -0.94 10.0\n", - "15 2am0.94_w20.h5 2.0 -0.94 20.0" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# encoding=True applies the regex from .hallmark.yaml before parsing\n", - "# so m0.94 in the filename becomes -0.94 in the dataframe\n", - "pf_spin = ParaFrame.parse(\"{mag:d}a{aspin}_w{win:d}.h5\", repo_path=\"data\", encoding=True)\n", - "pf_spin" - ] - }, - { - "cell_type": "markdown", - "id": "25", - "metadata": {}, - "source": [ - "The `aspin` column contains decoded values (e.g. `-0.94`) even though the filenames use `m0.94`. Filtering uses the decoded values:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "26", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.439842Z", - "iopub.status.busy": "2026-03-17T01:36:45.439764Z", - "iopub.status.idle": "2026-03-17T01:36:45.451332Z", - "shell.execute_reply": "2026-03-17T01:36:45.451016Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathmagaspinwin
41am0.5_w10.h51.0-0.5010.0
51am0.5_w20.h51.0-0.5020.0
61am0.94_w10.h51.0-0.9410.0
71am0.94_w20.h51.0-0.9420.0
122am0.5_w10.h52.0-0.5010.0
132am0.5_w20.h52.0-0.5020.0
142am0.94_w10.h52.0-0.9410.0
152am0.94_w20.h52.0-0.9420.0
\n", - "
" - ], - "text/plain": [ - " path mag aspin win\n", - "4 1am0.5_w10.h5 1.0 -0.50 10.0\n", - "5 1am0.5_w20.h5 1.0 -0.50 20.0\n", - "6 1am0.94_w10.h5 1.0 -0.94 10.0\n", - "7 1am0.94_w20.h5 1.0 -0.94 20.0\n", - "12 2am0.5_w10.h5 2.0 -0.50 10.0\n", - "13 2am0.5_w20.h5 2.0 -0.50 20.0\n", - "14 2am0.94_w10.h5 2.0 -0.94 10.0\n", - "15 2am0.94_w20.h5 2.0 -0.94 20.0" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# filter using the decoded float values not the raw filename strings\n", - "pf_spin(aspin=[-0.5, -0.94])" - ] - }, - { - "cell_type": "markdown", - "id": "27", - "metadata": {}, - "source": [ - "### Encoding validation\n", - "\n", - "`ParaFrame` enforces consistency between your call and the YAML spec:\n", - "- If the YAML has a regex for a parameter, you **must** pass `encoding=True`\n", - "- If the YAML has no regex, you **must** use `encoding=False` (the default)\n", - "\n", - "This prevents silent mis-parses." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "28", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.452772Z", - "iopub.status.busy": "2026-03-17T01:36:45.452695Z", - "iopub.status.idle": "2026-03-17T01:36:45.461001Z", - "shell.execute_reply": "2026-03-17T01:36:45.460544Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: '{mag:d}a{aspin}_w{win:d}.h5' has a regex spec, \n", - " so you must use encoding=True\n" - ] - } - ], - "source": [ - "# hallmark raises an error if you forget encoding=True when the yaml has a regex\n", - "try:\n", - " ParaFrame.parse(\"{mag:d}a{aspin}_w{win:d}.h5\", repo_path=\"data\", encoding=False)\n", - "except ValueError as e:\n", - " print(e)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "hallmark", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/demo/demo_bash.ipynb b/demo/demo_bash.ipynb deleted file mode 100644 index 3364fc4..0000000 --- a/demo/demo_bash.ipynb +++ /dev/null @@ -1,281 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# `Hallmark` Command Line Demo\n", - "\n", - "This Jupyter notebook demonstrates how `hallmark` is used in a command line.\n", - "\n", - "Because all cells are shell command, it is much more natural to use a `bash` kernel.\n", - "To install a `bash` kernel to your Jupyter/python (virtual) environment, run\n", - "```bash\n", - "pip install bash_kernel\n", - "python -m bash_kernel.install\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "## Remove Old Demo Repositories\n", - "\n", - "First, let's remove all old demo repositories." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "2", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [], - "source": [ - "rm -rf repo*" - ] - }, - { - "cell_type": "markdown", - "id": "3", - "metadata": {}, - "source": [ - "## Initialize a Standard (Non-Bare) `Hallmark` Repository\n", - "\n", - "`Hallmark`'s commandline syntax is similar to `git`.\n", - "To create an empty (standard) `hallmark` repository, we run `hallmark init [REPO]`." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "4", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "bash: hallmark: command not found\n" - ] - }, - { - "ename": "", - "evalue": "127", - "output_type": "error", - "traceback": [] - } - ], - "source": [ - "hallmark init repo1" - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "Check the content of the hallmark repository.\n", - "It contains a \".hm\" directory, which is itself a git repository." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "6", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\"repo1\": No such file or directory (os error 2)\n", - "\"repo1/.hm\": No such file or directory (os error 2)\n" - ] - }, - { - "ename": "", - "evalue": "2", - "output_type": "error", - "traceback": [] - } - ], - "source": [ - "ls -a repo1 repo1/.hm" - ] - }, - { - "cell_type": "markdown", - "id": "7", - "metadata": {}, - "source": [ - "We can `cd` into the \".hm\" directory and then run `git`.\n", - "However, because `cd` is not a very good practice in Jupyter notebook,\n", - "we will use `pushd` and `popd` instead." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [], - "source": [ - "pushd repo1/.hm && git status && popd" - ] - }, - { - "cell_type": "markdown", - "id": "9", - "metadata": {}, - "source": [ - "Note, similar to `git init`, `hallmark init` does not commit your files." - ] - }, - { - "cell_type": "markdown", - "id": "10", - "metadata": {}, - "source": [ - "## Initialize a Bare Hallmark Repository\n", - "\n", - "Next, we will initialize a bare repository.\n", - "Unlikely `git`, all we need is to explicitly postfix \".hm\" to the repository name." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [], - "source": [ - "hallmark init repo2.hm" - ] - }, - { - "cell_type": "markdown", - "id": "12", - "metadata": {}, - "source": [ - "`repo2.hm` is itself a `git` repository." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [], - "source": [ - "ls -a repo2.hm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "14", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [], - "source": [ - "pushd repo2.hm && git status && popd" - ] - }, - { - "cell_type": "markdown", - "id": "15", - "metadata": {}, - "source": [ - "## Add Files According to Python Format String" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [], - "source": [ - "pushd repo1 && touch a{0,0.75,0.94}_i{0..180..30}.h5 && popd\n", - "ls repo1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "17", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [], - "source": [ - "pushd repo1 && hallmark add \"a{a}_i{i}.h5\" && popd" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [], - "source": [ - "pushd repo1 && hallmark commit -m \"Commit test\" && popd" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Bash", - "language": "bash", - "name": "bash" - }, - "language_info": { - "codemirror_mode": "shell", - "file_extension": ".sh", - "mimetype": "text/x-sh", - "name": "bash" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/demo/demo_cli.ipynb b/demo/demo_cli.ipynb new file mode 100644 index 0000000..152fc11 --- /dev/null +++ b/demo/demo_cli.ipynb @@ -0,0 +1,859 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Hallmark CLI Demo: init, info, add, commit, checkout, status, and clone\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "rm -rf repo1\n", + "rm -rf hallmark-demo-clone\n", + "mkdir repo1\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Initialize the hallmark repo\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "hallmark init repo1\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== .hm directory ===\n", + " config.yml\n", + " data.tsv\n", + " meta.yml\n", + "\u001b[33m󰂺 \u001b[1;4mREADME.md\u001b[0m\n", + "\n", + "=== initial config.yml ===\n", + "# Edit this file only if your branch needs regex substitutions or a preset remote.\n", + "# For simple names, you can just run: hallmark add \"a{a}_i{i}.h5\"\n", + "data:\n", + " -\n", + " # fmt: \"{release}_{source}_{year}_{doy:03d}_{band}.uvfits\"\n", + " encoding:\n", + " # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)\n", + "remote:\n", + " # name: origin\n", + " # url: https://example.com/path/to/data/\n", + "\n", + "=== objects stored so far ===\n", + " 0\n" + ] + } + ], + "source": [ + "echo \"=== .hm directory ===\"\n", + "ls repo1/.hm/\n", + "echo \"\"\n", + "echo \"=== initial config.yml ===\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"=== objects stored so far ===\"\n", + "find repo1/.hm/objects -type f 2>/dev/null | wc -l\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Repository info\n", + "\n", + "The CLI `info` command shows the local hallmark paths for the current repo.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "dot-hallmark repo: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1/.hm\"\n", + "hallmark worktree: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1\"\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "pushd repo1 && hallmark info && popd\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Create data files on `main`\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Files in repo1/:\n", + " a0.75_i0.h5\n", + " a0.75_i30.h5\n", + " a0.75_i60.h5\n", + " a0.75_i90.h5\n", + " a0.75_i120.h5\n", + " a0_i0.h5\n", + " a0_i30.h5\n", + " a0_i60.h5\n", + " a0_i90.h5\n", + " a0_i120.h5\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "pushd repo1\n", + "for f in a{0,0.75}_i{0,30,60,90,120}.h5; do echo \"$f\" > \"$f\"; done\n", + "echo \"Files in repo1/:\"\n", + "ls *.h5\n", + "popd\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Add files and commit on `main`\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Changes to be committed\n", + " a0.75_i0.h5\n", + "a0.75_i120.h5\n", + " a0.75_i30.h5\n", + " a0.75_i60.h5\n", + " a0.75_i90.h5\n", + " a0_i0.h5\n", + " a0_i120.h5\n", + " a0_i30.h5\n", + " a0_i60.h5\n", + " a0_i90.h5\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "pushd repo1 && hallmark add \"a{a}_i{i}.h5\" && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== config.yml after add ===\n", + "data:\n", + "- fmt: a{a}_i{i}.h5\n", + " encoding: null\n", + "remote: null\n", + "\n", + "=== data.tsv after add ===\n", + "sha1\ta\ti\n", + "f656d419104e7af783c119186a4d81b15563310f\t0.75\t0\n", + "ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\t0.75\t120\n", + "890b2ea509575be9bc74ad515731a2562cd7406e\t0.75\t30\n", + "d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\t0.75\t60\n", + "ed2cfc6bb157a42729d3c6f45b228b432edec4be\t0.75\t90\n", + "18682593a012503b71935017dbc044665254a23c\t0\t0\n", + "1c057a4885fd04379ee1217375720665713c902f\t0\t120\n", + "829b87d845b367a9cc87df6ff510d1bdb7444aca\t0\t30\n", + "075a30e0a9c4dddd92447f463219d4c2a842883d\t0\t60\n", + "de40f4983d86342b03809bfcc09958f43f359b89\t0\t90\n" + ] + } + ], + "source": [ + "echo \"=== config.yml after add ===\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"=== data.tsv after add ===\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Committed staged state changes.\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "pushd repo1 && hallmark commit -m \"add main dataset\" && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Objects stored after commit:\n", + " 10\n" + ] + } + ], + "source": [ + "echo \"Objects stored after commit:\"\n", + "find repo1/.hm/objects -type f | wc -l\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Commit log\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "commit 7c035e02716b9d3ee6ccecb9c0abb7769a9dd96a\n", + "Author: Ram Adithya \n", + "Date: Mon Apr 20 19:28:53 2026 -0700\n", + "\n", + " add main dataset\n", + "\n", + "commit d709e546105c75dad9b43b71f543151974d6d531\n", + "Author: Ram Adithya \n", + "Date: Mon Apr 20 19:28:50 2026 -0700\n", + "\n", + " Initial commit: local `.hm` repository\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "pushd repo1 && hallmark log && popd\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Clean status\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark status && popd\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Current add modes\n", + "\n", + "- `hallmark add \"a{a}_i{i}.h5\"` parses files using the branch fmt and updates `config.yml`.\n", + "- `hallmark add .` rebuilds `data.tsv` from files that currently exist using the fmt already set in `config.yml`.\n", + "- `hallmark set-config --fmt ...` is the explicit way to change branch fmt before `hallmark add .`.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Demo: `hallmark add .` removes deleted files from the manifest\n", + "\n", + "We use a temporary branch so the main workflow stays intact.\n", + "On this branch we switch to `b*` files first so it is easy to distinguish from `main`.\n", + "Because the filename family changes, we update the branch fmt explicitly with `hallmark set-config` before using `hallmark add .`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark checkout sync-demo && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1\n", + "rm a*.h5\n", + "for f in b{0,0.75}_i{0,30,60,90,120}.h5; do echo \"$f\" > \"$f\"; done\n", + "echo \"Files after switching sync-demo to b files:\"\n", + "ls *.h5\n", + "popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1\n", + "hallmark set-config --fmt \"b{a}_i{i}.h5\"\n", + "echo \"config.yml after hallmark set-config:\"\n", + "cat .hm/config.yml\n", + "popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1\n", + "rm b0.75_i{0,30,60,90,120}.h5\n", + "echo \"Files after deleting some tracked b files:\"\n", + "ls *.h5\n", + "popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark add . && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "echo \"Manifest after hallmark add dot:\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark status && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark commit -m \"sync manifest with hallmark add dot\" && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark checkout main && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "echo \"Files after checkout back to main:\"\n", + "ls repo1/*.h5\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Checkout a new branch\n", + "\n", + "The new branch inherits the same fmt from `main`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark checkout experiment && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "echo \"Current .hm branch:\"\n", + "git -C repo1/.hm branch --show-current\n", + "echo \"\"\n", + "echo \"config.yml on experiment:\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"Files after checkout:\"\n", + "ls repo1/*.h5\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Replace the experiment branch dataset with new files that still match the same fmt\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1\n", + "rm a*.h5\n", + "for f in a{1,1.5}_i{15,45,75,105,110}.h5; do echo \"$f\" > \"$f\"; done\n", + "echo \"Files after replacing the dataset on experiment:\"\n", + "ls *.h5\n", + "popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark add . && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark status && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "echo \"experiment data.tsv before commit:\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark commit -m \"replace experiment dataset\" && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "echo \"Objects stored after experiment commit:\"\n", + "find repo1/.hm/objects -type f | wc -l\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Checkout back to `main`\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark checkout main && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "echo \"Files after checkout main:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"main data.tsv:\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 8. Checkout back to `experiment` to verify\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark checkout experiment && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "echo \"Files after checkout experiment:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"experiment data.tsv:\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 9. Uncommitted changes block checkout\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1\n", + "echo \"a2_i15.h5\" > a2_i15.h5\n", + "echo \"Files after creating a2_i15.h5:\"\n", + "ls *.h5\n", + "popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark add . && popd\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dirty status\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark status && popd\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To fix this, commit the staged changes first or discard them.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 10. Clone a hallmark repository\n", + "\n", + "This clone demo uses the default CLI behavior, which clones the hallmark repo and then downloads the configured data files. If you rerun the clone cell, it will print Hallmark's fatal destination-exists message.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "hallmark clone \"https://github.com/l6a/hallmark-demo-repo.hm.git\" hallmark-demo-clone\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd hallmark-demo-clone\n", + "hallmark info\n", + "echo \"\"\n", + "echo \"=== cloned config.yml ===\"\n", + "cat .hm/config.yml\n", + "popd\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Bash", + "language": "bash", + "name": "bash" + }, + "language_info": { + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/demo/demo_python.ipynb b/demo/demo_python.ipynb index 05fb0fe..732078e 100644 --- a/demo/demo_python.ipynb +++ b/demo/demo_python.ipynb @@ -2,261 +2,199 @@ "cells": [ { "cell_type": "markdown", - "id": "0", "metadata": {}, "source": [ - "# `Hallmark` Demo\n", - "\n", - "This Jupyter notebook demonstrates how `hallmark` is used." - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "id": "1", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" + "# Hallmark Python Demo: init, info, add, commit, checkout, status, and clone\n" ] }, { "cell_type": "markdown", - "id": "2", "metadata": {}, "source": [ - "## Remove Old Demo Repositories\n", - "\n", - "First, let's remove all old demo repositories." + "## Setup\n" ] }, { "cell_type": "code", - "execution_count": 76, - "id": "3", + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "! rm -rf repo*" - ] - }, - { - "cell_type": "markdown", - "id": "4", - "metadata": {}, - "source": [ - "## Initialize a Standard (Non-Bare) `Hallmark` Repository\n", + "%load_ext autoreload\n", + "%autoreload 2\n", "\n", - "`Hallmark` is similar to `git`.\n", - "To create an empty (standard) `hallmark` repository, we call `hallmark.Repo.init(REPO)`." + "from pathlib import Path\n", + "\n", + "from hallmark import Repo\n", + "from hallmark.downloader import download_remote_data\n", + "from hallmark.error import DestinationExistsError\n" ] }, { "cell_type": "code", - "execution_count": 77, - "id": "5", + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "from hallmark import Repo\n", - "\n", - "repo1 = Repo.init(\"repo1\")" + "%%bash\n", + "rm -rf repo1\n", + "rm -rf hallmark-demo-clone\n", + "mkdir repo1\n" ] }, { "cell_type": "markdown", - "id": "6", "metadata": {}, "source": [ - "Check the content of the hallmark repository.\n", - "It contains a \".hm\" directory, which is itself a git repository." + "## 1. Initialize the hallmark repo\n" ] }, { "cell_type": "code", - "execution_count": 78, - "id": "7", + "execution_count": 3, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "repo1:\n", - ". .. .hm\n", - "\n", - "repo1/.hm:\n", - ". .. .git README.md\tconfig.yml data.tsv meta.yml\n" - ] + "data": { + "text/plain": [ + "Repo(state=State(config={'data': [{'encoding': None}], 'remote': None}, meta={}, data=Empty DataFrame\n", + "Columns: [sha1]\n", + "Index: []), dothm=, worktree=Worktree('/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1'))" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "! ls -a repo1 repo1/.hm" - ] - }, - { - "cell_type": "markdown", - "id": "8", - "metadata": {}, - "source": [ - "We can `cd` into the \".hm\" directory and then run `git`.\n", - "However, because `cd` is not a very good practice in Jupyter notebook,\n", - "we will use `pushd` and `popd` instead." + "repo1 = Repo.init(\"repo1\")\n", + "repo1\n" ] }, { "cell_type": "code", - "execution_count": 79, - "id": "9", + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/EHT/hallmark/demo/repo1/.hm ~/EHT/hallmark/demo\n", - "On branch master\n", - "Changes to be committed:\n", - " (use \"git restore --staged ...\" to unstage)\n", - "\t\u001b[32mnew file: config.yml\u001b[m\n", - "\t\u001b[32mnew file: data.tsv\u001b[m\n", - "\t\u001b[32mnew file: meta.yml\u001b[m\n", + "=== .hm directory ===\n", + "config.yml\n", + "data.tsv\n", + "meta.yml\n", + "README.md\n", "\n", - "~/EHT/hallmark/demo\n" + "=== initial config.yml ===\n", + "# Edit this file only if your branch needs regex substitutions or a preset remote.\n", + "# For simple names, you can just run: hallmark add \"a{a}_i{i}.h5\"\n", + "data:\n", + " -\n", + " # fmt: \"{release}_{source}_{year}_{doy:03d}_{band}.uvfits\"\n", + " encoding:\n", + " # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)\n", + "remote:\n", + " # name: origin\n", + " # url: https://example.com/path/to/data/\n", + "\n", + "=== objects stored so far ===\n", + " 0\n" ] } ], "source": [ - "! pushd repo1/.hm && git status && popd" - ] - }, - { - "cell_type": "markdown", - "id": "10", - "metadata": {}, - "source": [ - "Note, similar to `git init`, `hallmark.Repo.init()` does not commit your files." + "%%bash\n", + "echo \"=== .hm directory ===\"\n", + "ls repo1/.hm/\n", + "echo \"\"\n", + "echo \"=== initial config.yml ===\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"=== objects stored so far ===\"\n", + "find repo1/.hm/objects -type f 2>/dev/null | wc -l\n" ] }, { "cell_type": "markdown", - "id": "11", "metadata": {}, "source": [ - "## Initialize a Bare Hallmark Repository\n", + "### Repository info\n", "\n", - "Next, we will initialize a bare repository.\n", - "Unlikely `git`, all we need is to explicitly postfix \".hm\" to the repository name." - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "id": "12", - "metadata": {}, - "outputs": [], - "source": [ - "repo2 = Repo.init(\"repo2.hm\")" - ] - }, - { - "cell_type": "markdown", - "id": "13", - "metadata": {}, - "source": [ - "`repo2.hm` is itself a `git` repository." - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "id": "14", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - ". .. .git README.md\tconfig.yml data.tsv meta.yml\n" - ] - } - ], - "source": [ - "! ls -a repo2.hm" + "The Python API exposes the same local hallmark paths directly on the `Repo` object.\n" ] }, { "cell_type": "code", - "execution_count": 82, - "id": "15", + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/EHT/hallmark/demo/repo2.hm ~/EHT/hallmark/demo\n", - "On branch master\n", - "Changes to be committed:\n", - " (use \"git restore --staged ...\" to unstage)\n", - "\t\u001b[32mnew file: config.yml\u001b[m\n", - "\t\u001b[32mnew file: data.tsv\u001b[m\n", - "\t\u001b[32mnew file: meta.yml\u001b[m\n", - "\n", - "~/EHT/hallmark/demo\n" + "dot-hallmark repo: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1/.hm\"\n", + "hallmark worktree: \"/Users/ramadithyamuthukumarasamy/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1\"\n" ] } ], "source": [ - "! pushd repo2.hm && git status && popd" + "print(f'dot-hallmark repo: \"{repo1.dothm.path}\"')\n", + "print(f'hallmark worktree: \"{repo1.worktree}\"')\n" ] }, { "cell_type": "markdown", - "id": "16", "metadata": {}, "source": [ - "## Add Files According to Python Format String" + "## 2. Create data files on `main`\n" ] }, { "cell_type": "code", - "execution_count": 83, - "id": "17", + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/EHT/hallmark/demo/repo1 ~/EHT/hallmark/demo\n", - "~/EHT/hallmark/demo\n", - "a0.75_i0.h5 a0.75_i30.h5 a0.94_i120.h5 a0.94_i60.h5 a0_i150.h5 a0_i90.h5\n", - "a0.75_i120.h5 a0.75_i60.h5 a0.94_i150.h5 a0.94_i90.h5 a0_i180.h5\n", - "a0.75_i150.h5 a0.75_i90.h5 a0.94_i180.h5 a0_i0.h5\t a0_i30.h5\n", - "a0.75_i180.h5 a0.94_i0.h5 a0.94_i30.h5 a0_i120.h5\t a0_i60.h5\n" + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Files in repo1/:\n", + "a0_i0.h5\n", + "a0_i120.h5\n", + "a0_i30.h5\n", + "a0_i60.h5\n", + "a0_i90.h5\n", + "a0.75_i0.h5\n", + "a0.75_i120.h5\n", + "a0.75_i30.h5\n", + "a0.75_i60.h5\n", + "a0.75_i90.h5\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" ] } ], "source": [ - "! pushd repo1 && touch a{0,0.75,0.94}_i{0..180..30}.h5 && popd\n", - "! ls repo1" + "%%bash\n", + "pushd repo1\n", + "for f in a{0,0.75}_i{0,30,60,90,120}.h5; do echo \"$f\" > \"$f\"; done\n", + "echo \"Files in repo1/:\"\n", + "ls *.h5\n", + "popd\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Add files and commit on `main`\n" ] }, { "cell_type": "code", - "execution_count": 84, - "id": "18", + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -283,7 +221,6 @@ " path\n", " a\n", " i\n", - " sha1\n", " \n", " \n", " \n", @@ -292,190 +229,131 @@ " a0.75_i0.h5\n", " 0.75\n", " 0.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 1\n", " a0.75_i120.h5\n", " 0.75\n", " 120.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 2\n", - " a0.75_i150.h5\n", - " 0.75\n", - " 150.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - " \n", - " \n", - " 3\n", - " a0.75_i180.h5\n", - " 0.75\n", - " 180.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - " \n", - " \n", - " 4\n", " a0.75_i30.h5\n", " 0.75\n", " 30.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", - " 5\n", + " 3\n", " a0.75_i60.h5\n", " 0.75\n", " 60.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", - " 6\n", + " 4\n", " a0.75_i90.h5\n", " 0.75\n", " 90.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - " \n", - " \n", - " 7\n", - " a0.94_i0.h5\n", - " 0.94\n", - " 0.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - " \n", - " \n", - " 8\n", - " a0.94_i120.h5\n", - " 0.94\n", - " 120.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - " \n", - " \n", - " 9\n", - " a0.94_i150.h5\n", - " 0.94\n", - " 150.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - " \n", - " \n", - " 10\n", - " a0.94_i180.h5\n", - " 0.94\n", - " 180.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - " \n", - " \n", - " 11\n", - " a0.94_i30.h5\n", - " 0.94\n", - " 30.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - " \n", - " \n", - " 12\n", - " a0.94_i60.h5\n", - " 0.94\n", - " 60.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - " \n", - " \n", - " 13\n", - " a0.94_i90.h5\n", - " 0.94\n", - " 90.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", - " 14\n", + " 5\n", " a0_i0.h5\n", " 0.00\n", " 0.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", - " 15\n", + " 6\n", " a0_i120.h5\n", " 0.00\n", " 120.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - " \n", - " \n", - " 16\n", - " a0_i150.h5\n", - " 0.00\n", - " 150.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - " \n", - " \n", - " 17\n", - " a0_i180.h5\n", - " 0.00\n", - " 180.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", - " 18\n", + " 7\n", " a0_i30.h5\n", " 0.00\n", " 30.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", - " 19\n", + " 8\n", " a0_i60.h5\n", " 0.00\n", " 60.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", - " 20\n", + " 9\n", " a0_i90.h5\n", " 0.00\n", " 90.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", "\n", "" ], "text/plain": [ - " path a i sha1\n", - "0 a0.75_i0.h5 0.75 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "1 a0.75_i120.h5 0.75 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "2 a0.75_i150.h5 0.75 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "3 a0.75_i180.h5 0.75 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "4 a0.75_i30.h5 0.75 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "5 a0.75_i60.h5 0.75 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "6 a0.75_i90.h5 0.75 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "7 a0.94_i0.h5 0.94 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "8 a0.94_i120.h5 0.94 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "9 a0.94_i150.h5 0.94 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "10 a0.94_i180.h5 0.94 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "11 a0.94_i30.h5 0.94 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "12 a0.94_i60.h5 0.94 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "13 a0.94_i90.h5 0.94 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "14 a0_i0.h5 0.00 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "15 a0_i120.h5 0.00 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "16 a0_i150.h5 0.00 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "17 a0_i180.h5 0.00 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "18 a0_i30.h5 0.00 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "19 a0_i60.h5 0.00 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "20 a0_i90.h5 0.00 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709" + " path a i\n", + "0 a0.75_i0.h5 0.75 0.0\n", + "1 a0.75_i120.h5 0.75 120.0\n", + "2 a0.75_i30.h5 0.75 30.0\n", + "3 a0.75_i60.h5 0.75 60.0\n", + "4 a0.75_i90.h5 0.75 90.0\n", + "5 a0_i0.h5 0.00 0.0\n", + "6 a0_i120.h5 0.00 120.0\n", + "7 a0_i30.h5 0.00 30.0\n", + "8 a0_i60.h5 0.00 60.0\n", + "9 a0_i90.h5 0.00 90.0" ] }, - "execution_count": 84, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo1.add(\"a{a}_i{i}.h5\")" + "pf = repo1.add(\"a{a}_i{i}.h5\")\n", + "pf\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== config.yml after add ===\n", + "data:\n", + "- fmt: a{a}_i{i}.h5\n", + " encoding: null\n", + "remote: null\n", + "\n", + "=== data.tsv after add ===\n", + "sha1\ta\ti\n", + "f656d419104e7af783c119186a4d81b15563310f\t0.75\t0\n", + "ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\t0.75\t120\n", + "890b2ea509575be9bc74ad515731a2562cd7406e\t0.75\t30\n", + "d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\t0.75\t60\n", + "ed2cfc6bb157a42729d3c6f45b228b432edec4be\t0.75\t90\n", + "18682593a012503b71935017dbc044665254a23c\t0\t0\n", + "1c057a4885fd04379ee1217375720665713c902f\t0\t120\n", + "829b87d845b367a9cc87df6ff510d1bdb7444aca\t0\t30\n", + "075a30e0a9c4dddd92447f463219d4c2a842883d\t0\t60\n", + "de40f4983d86342b03809bfcc09958f43f359b89\t0\t90\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"=== config.yml after add ===\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"=== data.tsv after add ===\"\n", + "cat repo1/.hm/data.tsv\n" ] }, { "cell_type": "code", - "execution_count": 85, - "id": "19", + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -484,90 +362,994 @@ "True" ] }, - "execution_count": 85, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.commit(\"add main dataset\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Objects stored after commit:\n", + " 10\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"Objects stored after commit:\"\n", + "find repo1/.hm/objects -type f | wc -l\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Commit log\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(repo1.log())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Clean status\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'branch': 'main',\n", + " 'staged': {'added': [], 'modified': [], 'deleted': []},\n", + " 'worktree': {'modified': [], 'deleted': []},\n", + " 'untracked': []}" + ] + }, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo1.commit(\"Commit test\")" + "repo1.status()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Current add modes\n", + "\n", + "- `repo.add(\"a{a}_i{i}.h5\")` parses files using the branch fmt and updates `config.yml`.\n", + "- `repo.add(\".\")` rebuilds `data.tsv` from files that currently exist using the fmt already set in `config.yml`.\n", + "- `repo.set_config(fmt=...)` is the explicit way to change branch fmt before `repo.add(\".\")`.\n" ] }, { "cell_type": "markdown", - "id": "c7ca72f0", "metadata": {}, "source": [ - "## Use Case with Encoding" + "## 4. Demo: `repo.add(\".\")` removes deleted files from the manifest\n", + "\n", + "We use a temporary branch so the main workflow stays intact.\n", + "On this branch we switch to `b*` files first so it is easy to distinguish from `main`.\n", + "Because the filename family changes, we update the branch fmt explicitly with `repo.set_config(...)` before using `repo.add(\".\")`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.checkout(\"sync-demo\")\n" ] }, { "cell_type": "code", - "execution_count": 86, - "id": "b9326142", + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/EHT/hallmark/demo/repo1 ~/EHT/hallmark/demo\n", - "~/EHT/hallmark/demo\n" + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Files after switching sync-demo to b files:\n", + "b0_i0.h5\n", + "b0_i120.h5\n", + "b0_i30.h5\n", + "b0_i60.h5\n", + "b0_i90.h5\n", + "b0.75_i0.h5\n", + "b0.75_i120.h5\n", + "b0.75_i30.h5\n", + "b0.75_i60.h5\n", + "b0.75_i90.h5\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" ] } ], "source": [ "%%bash\n", "pushd repo1\n", - "\n", - "touch am0.75_i0.h5 \\\n", - " am0.75_i30.h5 \\\n", - " am0.75_i60.h5 \\\n", - " am0.94_i0.h5 \\\n", - " am0.94_i30.h5 \\\n", - " am0.94_i+30.h5 \\\n", - " am0.94_i60.h5\n", - "\n", - "popd" + "rm a*.h5\n", + "for f in b{0,0.75}_i{0,30,60,90,120}.h5; do echo \"$f\" > \"$f\"; done\n", + "echo \"Files after switching sync-demo to b files:\"\n", + "ls *.h5\n", + "popd\n" ] }, { "cell_type": "code", - "execution_count": 87, - "id": "8571f48d", + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': [{'fmt': 'b{a}_i{i}.h5', 'encoding': None}], 'remote': None}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "%%bash\n", - "cat > repo1/.hm/config.yml <<'EOF'\n", - "encodings:\n", - " - fmt: \"a{aspin}_i{i}.h5\"\n", - " encoding:\n", - " aspin: \"m([0-9]+(\\\\.[0-9]+)?)\"\n", - "EOF" + "repo1.set_config(fmt=\"b{a}_i{i}.h5\")\n", + "repo1.state.config\n" ] }, { "cell_type": "code", - "execution_count": 88, - "id": "36f58ff9", + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "config.yml after repo.set_config:\n", + "data:\n", + "- fmt: b{a}_i{i}.h5\n", + " encoding: null\n", + "remote: null\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], "source": [ - "repo1.state.config = repo1.dothm.load_yml(\"config\")" + "%%bash\n", + "pushd repo1\n", + "echo \"config.yml after repo.set_config:\"\n", + "cat .hm/config.yml\n", + "popd\n" ] }, { "cell_type": "code", - "execution_count": 89, - "id": "3c10efb2", + "execution_count": 16, "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "
\n", - "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathai
0b0_i0.h50.00.0
1b0_i120.h50.0120.0
2b0_i30.h50.030.0
3b0_i60.h50.060.0
4b0_i90.h50.090.0
\n", + "
" + ], + "text/plain": [ + " path a i\n", + "0 b0_i0.h5 0.0 0.0\n", + "1 b0_i120.h5 0.0 120.0\n", + "2 b0_i30.h5 0.0 30.0\n", + "3 b0_i60.h5 0.0 60.0\n", + "4 b0_i90.h5 0.0 90.0" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.add(\".\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Manifest after repo.add dot:\n", + "sha1\ta\ti\n", + "56676e3e5629923617f8c9e542bd1c2ccaf8fa3a\t0\t0\n", + "683e82f624b175c075ac3ea0cca56e94cd7422f8\t0\t120\n", + "cfdbae05e3afc29354d948a32263f7ac1ed5850b\t0\t30\n", + "3a947fb1280b5bca62615d6c7543b88080588e7a\t0\t60\n", + "1711d3b47eac6ae479b6f0f240f83f41b393ebb4\t0\t90\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"Manifest after repo.add dot:\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'branch': 'sync-demo',\n", + " 'staged': {'added': ['b0_i0.h5',\n", + " 'b0_i120.h5',\n", + " 'b0_i30.h5',\n", + " 'b0_i60.h5',\n", + " 'b0_i90.h5'],\n", + " 'modified': [],\n", + " 'deleted': ['a0.75_i0.h5',\n", + " 'a0.75_i120.h5',\n", + " 'a0.75_i30.h5',\n", + " 'a0.75_i60.h5',\n", + " 'a0.75_i90.h5',\n", + " 'a0_i0.h5',\n", + " 'a0_i120.h5',\n", + " 'a0_i30.h5',\n", + " 'a0_i60.h5',\n", + " 'a0_i90.h5']},\n", + " 'worktree': {'modified': [], 'deleted': []},\n", + " 'untracked': []}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.status()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.commit(\"sync manifest with hallmark add dot\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.checkout(\"main\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files after checkout back to main:\n", + "repo1/a0_i0.h5\n", + "repo1/a0_i120.h5\n", + "repo1/a0_i30.h5\n", + "repo1/a0_i60.h5\n", + "repo1/a0_i90.h5\n", + "repo1/a0.75_i0.h5\n", + "repo1/a0.75_i120.h5\n", + "repo1/a0.75_i30.h5\n", + "repo1/a0.75_i60.h5\n", + "repo1/a0.75_i90.h5\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"Files after checkout back to main:\"\n", + "ls repo1/*.h5\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Checkout a new branch\n", + "\n", + "The new branch inherits the same fmt from `main`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.checkout(\"experiment\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current .hm branch:\n", + "experiment\n", + "\n", + "config.yml on experiment:\n", + "data:\n", + "- fmt: a{a}_i{i}.h5\n", + " encoding: null\n", + "remote: null\n", + "\n", + "Files after checkout:\n", + "repo1/a0_i0.h5\n", + "repo1/a0_i120.h5\n", + "repo1/a0_i30.h5\n", + "repo1/a0_i60.h5\n", + "repo1/a0_i90.h5\n", + "repo1/a0.75_i0.h5\n", + "repo1/a0.75_i120.h5\n", + "repo1/a0.75_i30.h5\n", + "repo1/a0.75_i60.h5\n", + "repo1/a0.75_i90.h5\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"Current .hm branch:\"\n", + "git -C repo1/.hm branch --show-current\n", + "echo \"\"\n", + "echo \"config.yml on experiment:\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"Files after checkout:\"\n", + "ls repo1/*.h5\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Replace the experiment branch dataset with new files that still match the same fmt\n" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Files after replacing the dataset on experiment:\n", + "a1_i105.h5\n", + "a1_i110.h5\n", + "a1_i15.h5\n", + "a1_i45.h5\n", + "a1_i75.h5\n", + "a1.5_i105.h5\n", + "a1.5_i110.h5\n", + "a1.5_i15.h5\n", + "a1.5_i45.h5\n", + "a1.5_i75.h5\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "%%bash\n", + "pushd repo1\n", + "rm a*.h5\n", + "for f in a{1,1.5}_i{15,45,75,105,110}.h5; do echo \"$f\" > \"$f\"; done\n", + "echo \"Files after replacing the dataset on experiment:\"\n", + "ls *.h5\n", + "popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathai
0a1.5_i105.h51.5105.0
1a1.5_i110.h51.5110.0
2a1.5_i15.h51.515.0
3a1.5_i45.h51.545.0
4a1.5_i75.h51.575.0
5a1_i105.h51.0105.0
6a1_i110.h51.0110.0
7a1_i15.h51.015.0
8a1_i45.h51.045.0
9a1_i75.h51.075.0
\n", + "
" + ], + "text/plain": [ + " path a i\n", + "0 a1.5_i105.h5 1.5 105.0\n", + "1 a1.5_i110.h5 1.5 110.0\n", + "2 a1.5_i15.h5 1.5 15.0\n", + "3 a1.5_i45.h5 1.5 45.0\n", + "4 a1.5_i75.h5 1.5 75.0\n", + "5 a1_i105.h5 1.0 105.0\n", + "6 a1_i110.h5 1.0 110.0\n", + "7 a1_i15.h5 1.0 15.0\n", + "8 a1_i45.h5 1.0 45.0\n", + "9 a1_i75.h5 1.0 75.0" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.add(\".\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'branch': 'experiment',\n", + " 'staged': {'added': ['a1.5_i105.h5',\n", + " 'a1.5_i110.h5',\n", + " 'a1.5_i15.h5',\n", + " 'a1.5_i45.h5',\n", + " 'a1.5_i75.h5',\n", + " 'a1_i105.h5',\n", + " 'a1_i110.h5',\n", + " 'a1_i15.h5',\n", + " 'a1_i45.h5',\n", + " 'a1_i75.h5'],\n", + " 'modified': [],\n", + " 'deleted': ['a0.75_i0.h5',\n", + " 'a0.75_i120.h5',\n", + " 'a0.75_i30.h5',\n", + " 'a0.75_i60.h5',\n", + " 'a0.75_i90.h5',\n", + " 'a0_i0.h5',\n", + " 'a0_i120.h5',\n", + " 'a0_i30.h5',\n", + " 'a0_i60.h5',\n", + " 'a0_i90.h5']},\n", + " 'worktree': {'modified': [], 'deleted': []},\n", + " 'untracked': []}" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.status()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "experiment data.tsv before commit:\n", + "sha1\ta\ti\n", + "f0ca3ed468c8796645672d2ebb48d48ae49b6d3c\t1.5\t105\n", + "9d1801d46e071b98a87afcd1edc59fe9302b9a4b\t1.5\t110\n", + "ed34eda23a05ea4c3cd1bf41988c6e2f76cf41e1\t1.5\t15\n", + "b47e5f555daf30afea5b0f99cadad1e2d8c689ec\t1.5\t45\n", + "b2a40517fd5681088f88903165ae54af3c75cf25\t1.5\t75\n", + "635b4b44a00c0a2bde5a7f0e263bb98dcbdae6e2\t1\t105\n", + "24ef5885eff61f1a8fcd77c19329a2651eca0228\t1\t110\n", + "577954873e813a982f70867b49a374a230df3650\t1\t15\n", + "1580545be2a5d485338e3538c7b16709d5fa6e4a\t1\t45\n", + "e2406e4872637ef09f574731be5b5ba5e8aa5761\t1\t75\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"experiment data.tsv before commit:\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.commit(\"replace experiment dataset\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Objects stored after experiment commit:\n", + " 25\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"Objects stored after experiment commit:\"\n", + "find repo1/.hm/objects -type f | wc -l\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Checkout back to `main`\n" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.checkout(\"main\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files after checkout main:\n", + "repo1/a0_i0.h5\n", + "repo1/a0_i120.h5\n", + "repo1/a0_i30.h5\n", + "repo1/a0_i60.h5\n", + "repo1/a0_i90.h5\n", + "repo1/a0.75_i0.h5\n", + "repo1/a0.75_i120.h5\n", + "repo1/a0.75_i30.h5\n", + "repo1/a0.75_i60.h5\n", + "repo1/a0.75_i90.h5\n", + "\n", + "main data.tsv:\n", + "sha1\ta\ti\n", + "f656d419104e7af783c119186a4d81b15563310f\t0.75\t0\n", + "ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\t0.75\t120\n", + "890b2ea509575be9bc74ad515731a2562cd7406e\t0.75\t30\n", + "d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\t0.75\t60\n", + "ed2cfc6bb157a42729d3c6f45b228b432edec4be\t0.75\t90\n", + "18682593a012503b71935017dbc044665254a23c\t0\t0\n", + "1c057a4885fd04379ee1217375720665713c902f\t0\t120\n", + "829b87d845b367a9cc87df6ff510d1bdb7444aca\t0\t30\n", + "075a30e0a9c4dddd92447f463219d4c2a842883d\t0\t60\n", + "de40f4983d86342b03809bfcc09958f43f359b89\t0\t90\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"Files after checkout main:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"main data.tsv:\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 8. Checkout back to `experiment` to verify\n" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.checkout(\"experiment\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files after checkout experiment:\n", + "repo1/a1_i105.h5\n", + "repo1/a1_i110.h5\n", + "repo1/a1_i15.h5\n", + "repo1/a1_i45.h5\n", + "repo1/a1_i75.h5\n", + "repo1/a1.5_i105.h5\n", + "repo1/a1.5_i110.h5\n", + "repo1/a1.5_i15.h5\n", + "repo1/a1.5_i45.h5\n", + "repo1/a1.5_i75.h5\n", + "\n", + "experiment data.tsv:\n", + "sha1\ta\ti\n", + "f0ca3ed468c8796645672d2ebb48d48ae49b6d3c\t1.5\t105\n", + "9d1801d46e071b98a87afcd1edc59fe9302b9a4b\t1.5\t110\n", + "ed34eda23a05ea4c3cd1bf41988c6e2f76cf41e1\t1.5\t15\n", + "b47e5f555daf30afea5b0f99cadad1e2d8c689ec\t1.5\t45\n", + "b2a40517fd5681088f88903165ae54af3c75cf25\t1.5\t75\n", + "635b4b44a00c0a2bde5a7f0e263bb98dcbdae6e2\t1\t105\n", + "24ef5885eff61f1a8fcd77c19329a2651eca0228\t1\t110\n", + "577954873e813a982f70867b49a374a230df3650\t1\t15\n", + "1580545be2a5d485338e3538c7b16709d5fa6e4a\t1\t45\n", + "e2406e4872637ef09f574731be5b5ba5e8aa5761\t1\t75\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"Files after checkout experiment:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"experiment data.tsv:\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 9. Uncommitted changes block checkout\n" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Files after creating a2_i15.h5:\n", + "a1_i105.h5\n", + "a1_i110.h5\n", + "a1_i15.h5\n", + "a1_i45.h5\n", + "a1_i75.h5\n", + "a1.5_i105.h5\n", + "a1.5_i110.h5\n", + "a1.5_i15.h5\n", + "a1.5_i45.h5\n", + "a1.5_i75.h5\n", + "a2_i15.h5\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "%%bash\n", + "pushd repo1\n", + "echo \"a2_i15.h5\" > a2_i15.h5\n", + "echo \"Files after creating a2_i15.h5:\"\n", + "ls *.h5\n", + "popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "